├── .github └── workflows │ └── github-actions-demo.yml ├── .gitignore ├── LICENSE ├── README.md ├── docs └── Changelog.md ├── pydbclib ├── __init__.py ├── database.py ├── drivers.py ├── exceptions.py ├── record.py ├── sql.py └── utils.py ├── setup.py └── test.py /.github/workflows/github-actions-demo.yml: -------------------------------------------------------------------------------- 1 | name: GitHub Actions Demo 2 | run-name: ${{ github.actor }} is testing out GitHub Actions 🚀 3 | on: [push] 4 | jobs: 5 | Explore-GitHub-Actions: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." 9 | - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" 10 | - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." 11 | - name: Check out repository code 12 | uses: actions/checkout@v3 13 | - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." 14 | - run: echo "🖥️ The workflow is now ready to test your code on the runner." 15 | - name: List files in the repository 16 | run: | 17 | ls ${{ github.workspace }} 18 | - run: echo "🍏 This job's status is ${{ job.status }}." 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Distribution / packaging 2 | __pycache__/ 3 | .idea 4 | env/ 5 | bin/ 6 | build/ 7 | develop-eggs/ 8 | dist/ 9 | eggs/ 10 | lib/ 11 | lib64/ 12 | parts/ 13 | sdist/ 14 | var/ 15 | logs/ 16 | venv/ 17 | tmp/ 18 | *.egg-info/ 19 | .installed.cfg 20 | *.egg 21 | install.* 22 | *.exe 23 | *.py[cod] 24 | sftp-config.json 25 | .Python 26 | demo.py 27 | 28 | # Installer logs 29 | pip-log.txt 30 | pip-delete-this-directory.txt 31 | 32 | # Unit test / coverage reports 33 | .pytest_cache 34 | htmlcov/ 35 | .tox/ 36 | .coverage 37 | .cache 38 | test.db 39 | nosetests.xml 40 | coverage.xml 41 | 42 | *.tar 43 | *.gz 44 | *.zip 45 | 46 | # Django stuff: 47 | *.log 48 | *.pot 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # OS 54 | .DS_Store 55 | .DS_Store? 56 | ._* 57 | .Spotlight-V100 58 | .Trashes 59 | Icon? 60 | ehthumbs.db 61 | Thumbs.db 62 | MANIFEST 63 | 64 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pydbclib 2 | 3 | Pydbclib is Python Database Connectivity Lib, a general database operation toolkit for **Python 3.6+** 4 | 5 | ## Installation: 6 | ```shell script 7 | pip3 install pydbclib 8 | ``` 9 | 10 | ## Example: 11 | 12 | ```python 13 | from pydbclib import connect 14 | # 使用with上下文，可以自动提交，自动关闭连接 15 | with connect("sqlite:///:memory:") as db: 16 | db.execute('create table foo(a integer, b varchar(20))') 17 | # 统一使用':[name]'形式的SQL的占位符 18 | db.execute("insert into foo(a,b) values(:a,:b)", [{"a": 1, "b": "one"}]*4) 19 | print(db.read("select * from foo").get_one()) 20 | print(db.read("select * from foo").get_all()) 21 | print(db.read("select * from foo").to_df()) 22 | 23 | # 对表常用操作的封装 24 | table = db.get_table("foo") 25 | table.insert([{"a": 2, "b": "two"}]*2) # 插入两条记录 26 | table.find({"b": "two"}).get_all() # 查出b='two'的所有记录 27 | table.update({"a": 2, "b": "two"}, {"b": "2"}) # 将a=2 and b='two'的所有记录的b字段值更新为'2' 28 | table.find({"a": 2}).get_all() # 查出a=2的所有记录 29 | table.delete({"a": 2}) # 删除a=2的所有记录 30 | ``` 31 | 32 | #### 常用数据库连接示例 33 | Common Driver 34 | 35 | # 使用普通数据库驱动连接，driver参数指定驱动包名称 36 | # 例如pymysql包driver='pymysql',connect函数其余的参数和driver参数指定的包的创建连接参数一致 37 | # 连接mysql 38 | db = pydbclib.connect(user="user", password="password", database="test", driver="pymysql") 39 | # 连接oracle 40 | db = pydbclib.connect('user/password@local:1521/xe', driver="cx_Oracle") 41 | # 通过odbc方式连接 42 | db = pydbclib.connect('DSN=mysqldb;UID=user;PWD=password', driver="pyodbc") 43 | # 通过已有驱动连接方式连接 44 | import pymysql 45 | con = pymysql.connect(user="user", password="password", database="test") 46 | db = pydbclib.connect(driver=con) 47 | 48 | Sqlalchemy Driver 49 | 50 | # 使用Sqlalchemy包来连接数据库，drvier参数默认为'sqlalchemy' 51 | # 连接oracle 52 | db = pydbclib.connect("oracle://user:password@local:1521/xe") 53 | # 连接mysql 54 | db = pydbclib.connect("mysql+pyodbc://:@mysqldb") 55 | # 通过已有engine连接 56 | from sqlalchemy import create_engine 57 | engine = create_engine("mysql+pymysql://user:password@localhost:3306/test") 58 | db = pydbclib.connect(driver=engine) 59 | 60 | 61 | 62 | ### 详细使用文档 63 | 64 | https://blog.csdn.net/li_yatao/article/details/79685992 65 | -------------------------------------------------------------------------------- /docs/Changelog.md: -------------------------------------------------------------------------------- 1 | #### v2.0 2 | 1. 重构代码 3 | 4 | #### v1.2.3 5 | 1. init 6 | 7 | #### v0.0.1 8 | 1. 创建 9 | -------------------------------------------------------------------------------- /pydbclib/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Python Database Connectivity lib 4 | """ 5 | from pydbclib.database import Database 6 | from pydbclib.drivers import CommonDriver, SQLAlchemyDriver 7 | 8 | __author__ = "liyatao" 9 | __version__ = '2.2.3' 10 | 11 | 12 | def connect(*args, **kwargs): 13 | driver = kwargs.get("driver", "sqlalchemy") 14 | kwargs.update(driver=driver) 15 | if isinstance(driver, str): 16 | driver_class = {"sqlalchemy": SQLAlchemyDriver}.get(driver.lower(), CommonDriver) 17 | elif hasattr(driver, "cursor"): 18 | driver_class = CommonDriver 19 | else: 20 | driver_class = SQLAlchemyDriver 21 | return Database(driver_class(*args, **kwargs)) 22 | -------------------------------------------------------------------------------- /pydbclib/database.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | @time: 2020/3/26 11:30 上午 4 | @desc: 5 | """ 6 | from collections.abc import Iterator 7 | 8 | from pydbclib.exceptions import ParameterError 9 | from pydbclib.record import Records 10 | from pydbclib.utils import batch_dataset, get_records 11 | 12 | 13 | class Database(object): 14 | """ 15 | 数据库操作封装 16 | 方法： 17 | get_table 18 | execute 19 | bulk 20 | read 21 | read_one 22 | """ 23 | 24 | def __init__(self, driver): 25 | self.driver = driver 26 | 27 | def get_table(self, name): 28 | return Table(name, self) 29 | 30 | """ 31 | 数据库操作封装 32 | """ 33 | 34 | def execute(self, sql, args=None, autocommit=False): 35 | """ 36 | 执行sql语句： 37 | :param sql: sql语句 38 | :param args: sql语句参数 39 | :param autocommit: 执行完sql是否自动提交 40 | :return: ResultProxy 41 | 42 | Example: 43 | db.execute( 44 | "insert into foo(a,b) values(:a,:b)", 45 | {"a": 1, "b": "one"} 46 | ) 47 | 48 | 对条写入 49 | db.execute( 50 | "insert into foo(a,b) values(:a,:b)", 51 | [ 52 | {"a": 1, "b": "one"}, 53 | {"a": 2, "b": "two"} 54 | ] 55 | ) 56 | """ 57 | if args is None or isinstance(args, dict): 58 | res = self.driver.execute(sql, args) 59 | elif isinstance(args, (list, tuple)): 60 | res = self.driver.execute_many(sql, args) 61 | else: 62 | raise ParameterError("'params'参数类型无效") 63 | if autocommit: 64 | self.commit() 65 | return res 66 | 67 | def bulk(self, sql, args, batch_size=100000): 68 | """批量插入""" 69 | if isinstance(args, (list, tuple, Iterator)): 70 | rowcount = 0 71 | for batch in batch_dataset(args, batch_size): 72 | rowcount += self.driver.bulk(sql, batch) 73 | return rowcount 74 | else: 75 | raise ParameterError("'params'参数类型无效") 76 | 77 | def read(self, sql, args=None, as_dict=True, batch_size=10000): 78 | """ 79 | 查询返回所有表记录 80 | :param sql: sql语句 81 | :param args: sql语句参数 82 | :param as_dict: 返回记录是否转换成字典形式（True: [{"a": 1, "b": "one"}]， False: [(1, "one)]），默认为True 83 | :param batch_size: 每次查询返回的缓存的数量，大数据量可以适当提高大小 84 | :return: 生成器对象 85 | """ 86 | r = self.driver.execute(sql, args) 87 | if as_dict: 88 | # columns = [i[0].lower() for i in r.description] 89 | columns = r.get_columns() 90 | records = get_records(r, batch_size, columns) 91 | else: 92 | records = get_records(r, batch_size) 93 | return Records(records, as_dict) 94 | 95 | def read_one(self, sql, args=None, as_dict=True): 96 | """ 97 | 查询返回一条表记录 98 | :param sql: sql语句 99 | :param args: sql语句参数 100 | :param as_dict: 返回记录是否转换成字典形式（True: [{"a": 1, "b": "one"}]， False: [(1, "one)]），默认为True 101 | :return: to_dict=True {"a": 1, "b": "one"}, to_dict=False (1, "one") 102 | """ 103 | r = self.driver.execute(sql, args) 104 | record = r.fetchone() 105 | # Unbuffered Cursor needed 106 | r.fetchall() 107 | if as_dict: 108 | if record is None: 109 | return None 110 | else: 111 | # columns = [i[0].lower() for i in r.description] 112 | columns = r.get_columns() 113 | return dict(zip(columns, record)) 114 | else: 115 | return record 116 | 117 | def commit(self): 118 | self.driver.commit() 119 | 120 | def rollback(self): 121 | self.driver.rollback() 122 | 123 | def close(self): 124 | self.driver.close() 125 | 126 | def __enter__(self): 127 | return self 128 | 129 | def __exit__(self, exc_type, exc_val, exc_tb): 130 | try: 131 | if exc_type is None: 132 | self.commit() 133 | else: 134 | self.rollback() 135 | finally: 136 | self.close() 137 | 138 | 139 | def format_condition(condition): 140 | param = {} 141 | if isinstance(condition, dict): 142 | expressions = [] 143 | for i, k in enumerate(condition): 144 | param[f"c{i}"] = condition[k] 145 | expressions.append(f"{k}=:c{i}") 146 | condition = " and ".join(expressions) 147 | condition = f" where {condition}" if condition else "" 148 | return condition, param 149 | 150 | 151 | def format_update(update): 152 | param = {} 153 | if isinstance(update, dict): 154 | expressions = [] 155 | for i, k in enumerate(update): 156 | param[f"u{i}"] = update[k] 157 | expressions.append(f"{k}=:u{i}") 158 | update = ",".join(expressions) 159 | if not update: 160 | raise ParameterError("'update' 参数不能为空值") 161 | return update, param 162 | 163 | 164 | class Table(object): 165 | """ 166 | 数据库表操作封装 167 | 方法： 168 | get_columns 169 | insert 170 | bulk 171 | update 172 | delete 173 | find_one 174 | find 175 | """ 176 | 177 | def __init__(self, name, db): 178 | self.name = name 179 | self.db = db 180 | 181 | def get_columns(self): 182 | """获取表字段名称""" 183 | r = self.db.execute(f"select * from {self.name} where 1=0") 184 | r.fetchall() 185 | # return [i[0].lower() for i in r.description] 186 | return r.get_columns() 187 | 188 | def insert(self, records): 189 | """ 190 | 表中插入记录 191 | :param records: 要插入的记录数据，字典or字典列表 192 | """ 193 | if isinstance(records, dict): 194 | return self._insert_one(records) 195 | else: 196 | return self._insert_many(records) 197 | 198 | def bulk(self, records, batch_size=100000): 199 | if isinstance(records, (list, tuple, Iterator)): 200 | rowcount = 0 201 | for batch in batch_dataset(records, batch_size): 202 | rowcount += self._insert_many(batch) 203 | self.db.commit() 204 | return rowcount 205 | else: 206 | raise ParameterError("'params'参数类型无效") 207 | 208 | def update(self, condition, update): 209 | """ 210 | 表更新操作 211 | :param condition: 更新条件，字典类型或者sql条件表达式 212 | :param update: 要更新的字段，字典类型 213 | :return: 返回影响行数 214 | """ 215 | condition, p1 = format_condition(condition) 216 | update, p2 = format_update(update) 217 | p1.update(p2) 218 | return self.db.execute(f"update {self.name} set {update}{condition}", p1).rowcount 219 | 220 | def delete(self, condition): 221 | """ 222 | 删除表中记录 223 | :param condition: 删除条件，字典类型或者sql条件表达式 224 | :return: 返回影响行数 225 | """ 226 | condition, param = format_condition(condition) 227 | return self.db.execute(f"delete from {self.name}{condition}", param).rowcount 228 | 229 | def find_one(self, condition=None, fields=None): 230 | """ 231 | 按条件查询一条表记录 232 | :param condition: 查询条件，字典类型或者sql条件表达式 233 | :param fields: 指定返回的字段 234 | :return: 字典类型，如 {"a": 1, "b": "one"} 235 | """ 236 | if fields is None: 237 | fields = "*" 238 | else: 239 | fields = ','.join(fields) 240 | condition, param = format_condition(condition) 241 | return self.db.read_one(f"select {fields} from {self.name}{condition}", param) 242 | 243 | def find(self, condition=None, fields=None): 244 | """ 245 | 按条件查询所有符合条件的表记录 246 | :param condition: 查询条件，字典类型或者sql条件表达式 247 | :param fields: 指定返回的字段 248 | :return: 生成器类型 249 | """ 250 | if fields is None: 251 | fields = "*" 252 | else: 253 | fields = ','.join(fields) 254 | condition, param = format_condition(condition) 255 | return self.db.read(f"select {fields} from {self.name}{condition}", param) 256 | 257 | def _get_insert_sql(self, columns): 258 | return f"insert into {self.name} ({','.join(columns)})" \ 259 | f" values ({','.join([':%s' % i for i in columns])})" 260 | 261 | def _insert_one(self, record): 262 | """ 263 | 表中插入一条记录 264 | :param record: 要插入的记录数据，字典类型 265 | """ 266 | if isinstance(record, dict): 267 | columns = record.keys() 268 | return self.db.execute(self._get_insert_sql(columns), record).rowcount 269 | else: 270 | raise ParameterError("无效的参数") 271 | 272 | def _insert_many(self, records): 273 | """ 274 | 表中插入多条记录 275 | :param records: 要插入的记录数据，字典集合 276 | """ 277 | if not isinstance(records, (tuple, list)): 278 | raise ParameterError("records param must list or tuple") 279 | sample = records[0] 280 | if isinstance(sample, dict): 281 | columns = sample.keys() 282 | return self.db.execute(self._get_insert_sql(columns), records).rowcount 283 | else: 284 | raise ParameterError("无效的参数") 285 | -------------------------------------------------------------------------------- /pydbclib/drivers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | @time: 2020/3/18 2:36 下午 4 | @desc: 5 | """ 6 | import sys 7 | from abc import ABC, abstractmethod 8 | 9 | from log4py import Logger 10 | 11 | from pydbclib.sql import compilers 12 | from pydbclib.utils import get_suffix, get_dbapi_module 13 | 14 | 15 | class Driver(ABC): 16 | 17 | @property 18 | @abstractmethod 19 | def session(self): 20 | pass 21 | 22 | @abstractmethod 23 | def execute(self, sql, params=None, **kw): 24 | pass 25 | 26 | @abstractmethod 27 | def execute_many(self, sql, params=None, **kw): 28 | pass 29 | 30 | def bulk(self, sql, params): 31 | # return self.connection.execute(sql, params).rowcount 32 | r = self.execute_many(sql, params) 33 | self.commit() 34 | return r.rowcount 35 | 36 | @abstractmethod 37 | def rollback(self): 38 | pass 39 | 40 | @abstractmethod 41 | def commit(self): 42 | pass 43 | 44 | @abstractmethod 45 | def close(self): 46 | pass 47 | 48 | 49 | class ResultProxy(object): 50 | 51 | def __init__(self, context): 52 | self.context = context 53 | 54 | def __getattr__(self, item): 55 | """不存在的属性都代理到context中去找""" 56 | if item == "description": 57 | return self._get_description() 58 | else: 59 | return getattr(self.context, item) 60 | 61 | def _get_description(self): 62 | # 去除hive表名前缀 63 | # {'pokes.foo': 238, 'pokes.bar': 'val_238'} => {'foo': 238, 'bar': 'val_238'} 64 | if hasattr(self.context, "description"): 65 | description = self.context.description 66 | else: 67 | description = self.context._cursor_description() 68 | return [(get_suffix(r[0]), *r[1:]) for r in description] if description else description 69 | 70 | def get_columns(self): 71 | """获取查询结果的字段名称""" 72 | return [i[0].lower() for i in self.description] 73 | 74 | 75 | @Logger.class_logger() 76 | class CommonDriver(Driver): 77 | 78 | def __init__(self, *args, **kwargs): 79 | driver_param = kwargs.pop("driver") 80 | self._cursor = None 81 | if hasattr(driver_param, "cursor"): 82 | self.driver_name = get_dbapi_module(driver_param.__class__.__module__) 83 | self.dbapi = sys.modules[self.driver_name] 84 | self.con = driver_param 85 | else: 86 | __import__(driver_param) 87 | self.driver_name = driver_param 88 | self.dbapi = sys.modules[driver_param] 89 | self.con = self.dbapi.connect(*args, **kwargs) 90 | self.compiler = compilers[self.dbapi.paramstyle] 91 | 92 | @property 93 | def session(self): 94 | if not self._cursor: 95 | self._cursor = self.con.cursor() 96 | return self._cursor 97 | 98 | def execute(self, sql, params=None, **kw): 99 | sql, params = self.compiler(sql, params).process_one() 100 | params = params if params else [] 101 | self.logger.info("{}, {}".format(sql, params)) 102 | self.session.execute(sql, params, **kw) 103 | return ResultProxy(self._cursor) 104 | 105 | def execute_many(self, sql, params=None, **kw): 106 | sql, params = self.compiler(sql, params).process() 107 | params = params if params else [] 108 | self.logger.info("{}, {}".format(sql, params)) 109 | self.session.executemany(sql, params, **kw) 110 | return ResultProxy(self._cursor) 111 | 112 | def rollback(self): 113 | self.con.rollback() 114 | 115 | def commit(self): 116 | self.con.commit() 117 | 118 | def close(self): 119 | if self.session is not None: 120 | self.session.close() 121 | if self.con is not None: 122 | self.con.close() 123 | 124 | 125 | @Logger.class_logger() 126 | class SQLAlchemyDriver(Driver): 127 | 128 | def __init__(self, *args, **kwargs): 129 | self.driver_name = "sqlalchemy" 130 | driver_param = kwargs.pop("driver") 131 | self._session = None 132 | from sqlalchemy import engine, create_engine 133 | if isinstance(driver_param, engine.base.Engine): 134 | self.engine = driver_param 135 | else: 136 | self.engine = create_engine(*args, **kwargs) 137 | 138 | @property 139 | def session(self): 140 | if not self._session: 141 | from sqlalchemy.orm import sessionmaker 142 | self._session = sessionmaker(bind=self.engine)() 143 | return self._session 144 | 145 | def execute(self, sql, params=None, **kw): 146 | self.logger.info("{}, {}".format(sql, params)) 147 | r = self.session.execute(sql, params, **kw) 148 | return ResultProxy(r) 149 | 150 | def execute_many(self, sql, params=None, **kw): 151 | self.logger.info("{}, {}".format(sql, params)) 152 | return ResultProxy(self.session.execute(sql, params, **kw)) 153 | 154 | def rollback(self): 155 | self.session.rollback() 156 | 157 | def commit(self): 158 | self.session.commit() 159 | 160 | def close(self): 161 | if self.session is not None: 162 | self.session.close() 163 | -------------------------------------------------------------------------------- /pydbclib/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | @time: 2020/3/19 10:31 上午 4 | @desc: 5 | """ 6 | 7 | 8 | class SQLFormatError(Exception): 9 | pass 10 | 11 | 12 | class ParameterError(Exception): 13 | pass 14 | 15 | 16 | class ConnectError(Exception): 17 | pass 18 | 19 | 20 | class ExecuteError(Exception): 21 | pass 22 | -------------------------------------------------------------------------------- /pydbclib/record.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | @time: 2020/4/13 11:28 下午 4 | @desc: 5 | """ 6 | import itertools 7 | 8 | 9 | def to_df_iterator(records, batch_size): 10 | import pandas 11 | while 1: 12 | _records = records.get(batch_size) 13 | if _records: 14 | yield pandas.DataFrame.from_records(_records) 15 | else: 16 | return None 17 | 18 | 19 | class Records(object): 20 | 21 | def __init__(self, rows, as_dict): 22 | self._rows = rows 23 | self.as_dict = as_dict 24 | self._limit_num = None 25 | 26 | def __iter__(self): 27 | return self 28 | 29 | def next(self): 30 | return next(self._rows) 31 | 32 | __next__ = next 33 | 34 | def map(self, function): 35 | self._rows = (function(r) for r in self._rows) 36 | return self 37 | 38 | def filter(self, function): 39 | self._rows = (r for r in self._rows if function(r)) 40 | return self 41 | 42 | def rename(self, mapper): 43 | """ 44 | 字段重命名 45 | """ 46 | def function(record): 47 | if isinstance(record, dict): 48 | return {mapper.get(k, k): v for k, v in record.items()} 49 | else: 50 | return dict(zip(mapper, record)) 51 | return self.map(function) 52 | 53 | def limit(self, num): 54 | def rows_limited(rows, limit): 55 | for i, r in enumerate(rows): 56 | if i < limit: 57 | yield r 58 | else: 59 | return None 60 | self._rows = rows_limited(self._rows, num) 61 | return self 62 | 63 | def get_one(self): 64 | r = self.get(1) 65 | return r[0] if len(r) > 0 else None 66 | 67 | def get(self, num): 68 | return [i for i in itertools.islice(self._rows, num)] 69 | 70 | def get_all(self): 71 | return [r for r in self._rows] 72 | 73 | def to_df(self, batch_size=None): 74 | if batch_size is None: 75 | import pandas 76 | return pandas.DataFrame.from_records(self) 77 | else: 78 | return to_df_iterator(self, batch_size) 79 | 80 | def to_csv(self, file_path, sep=',', header=False, columns=None, batch_size=100000, **kwargs): 81 | """ 82 | 用于大数据量分批写入文件 83 | :param file_path: 文件路径 84 | :param sep: 分割符号，hive默认\001 85 | :param header: 是否写入表头 86 | :param columns: 按给定字段排序 87 | :param batch_size: 每批次写入文件行数 88 | """ 89 | mode = "w" 90 | for df in self.to_df(batch_size=batch_size): 91 | df.to_csv(file_path, sep=sep, index=False, header=header, columns=columns, mode=mode, **kwargs) 92 | mode = "a" 93 | header = False 94 | -------------------------------------------------------------------------------- /pydbclib/sql.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | sql 语句参数名称提取及参数形式替换 4 | 可以考虑sqlparse库解析 5 | import sqlparse 6 | sql2 = "select * from asr where uuid=%s" 7 | parsed = sqlparse.parse(sql) 8 | stmt = parsed[0] 9 | parsed = sqlparse.parse(sql1) 10 | stmt = parsed[0] 11 | for token in stmt.flatten(): 12 | token.ttype is sqlparse.tokens.Name.Placeholder 13 | """ 14 | import sqlparse 15 | 16 | from pydbclib.exceptions import SQLFormatError 17 | 18 | 19 | class Compiler(object): 20 | def __init__(self, sql, parameters): 21 | self.sql = sql 22 | self.parameters = parameters 23 | 24 | def process_one(self): 25 | return self.sql, self.parameters 26 | 27 | def process(self): 28 | return self.sql, self.parameters 29 | 30 | 31 | class QmarkCompiler(Compiler): 32 | place_holder = "?" 33 | 34 | def process_one(self): 35 | if not self.parameters: 36 | return self.sql, None 37 | elif isinstance(self.parameters, (list, tuple)): 38 | sql, keys = self.parse_sql() 39 | if len(set(keys)) == len(keys): 40 | return sql, self.parameters 41 | else: 42 | postions = self.to_postions(keys) 43 | return sql, [self.parameters[p] for p in postions] 44 | else: 45 | sql, keys = self.parse_sql() 46 | return sql, tuple(self.parameters[k] for k in keys) 47 | 48 | def process(self): 49 | if not self.parameters: 50 | return self.sql, None 51 | elif isinstance(self.parameters[0], (list, tuple)): 52 | sql, keys = self.parse_sql() 53 | if len(set(keys)) == len(keys): 54 | return sql, self.parameters 55 | else: 56 | postions = self.to_postions(keys) 57 | return sql, [tuple(parameter[p] for p in postions) for parameter in self.parameters] 58 | else: 59 | sql, keys = self.parse_sql() 60 | return sql, [tuple(parameter[k] for k in keys) for parameter in self.parameters] 61 | 62 | @staticmethod 63 | def to_postions(keys): 64 | postions = {} 65 | i = 0 66 | for k in keys: 67 | if k not in postions: 68 | postions[k] = i 69 | i += 1 70 | return [postions[k] for k in keys] 71 | 72 | def parse_sql(self): 73 | parsed = sqlparse.parse(self.sql) 74 | stmt = parsed[0] 75 | tokens = list(stmt.flatten()) 76 | keys = [] 77 | for token in tokens: 78 | if token.ttype is sqlparse.tokens.Name.Placeholder: 79 | if ":" in token.value: 80 | keys.append(token.value[1:]) 81 | token.value = self.place_holder 82 | else: 83 | raise SQLFormatError(f"无效的占位符{token.value}, 只支持使用':'开头的占位符") 84 | return "".join(t.value for t in tokens), keys 85 | 86 | 87 | class FormatCompiler(QmarkCompiler): 88 | place_holder = "%s" 89 | 90 | 91 | """ 92 | 'format'：表示使用 %s 93 | 'pyformat':表示使用 %(name)s 94 | 'qmark'：表示使用 ? 95 | 'numeric': 表示使用 :1 96 | 'named': 表示使用 :name 97 | """ 98 | compilers = { 99 | "named": Compiler, 100 | "qmark": QmarkCompiler, 101 | "format": FormatCompiler, 102 | "pyformat": FormatCompiler 103 | } 104 | 105 | 106 | if __name__ == "__main__": 107 | print(QmarkCompiler("insert into test(id, name, age) values(:a, ':a', :b)", [{"a": 12, "b": "lyt"}]).process()) 108 | print(QmarkCompiler("insert into test(id, name, age) values(:a, ':a', :b)", None).process()) 109 | -------------------------------------------------------------------------------- /pydbclib/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | @time: 2020/4/15 3:42 下午 4 | @desc: 5 | """ 6 | import os 7 | import sys 8 | 9 | 10 | def get_dbapi_module(module_name): 11 | if module_name in sys.modules and hasattr(sys.modules[module_name], "paramstyle"): 12 | return module_name 13 | elif '.' in module_name: 14 | return get_dbapi_module(os.path.splitext(module_name)[0]) 15 | else: 16 | raise ValueError("Unknown DBAPI") 17 | 18 | 19 | def to_camel_style(text): 20 | res = '' 21 | j = 0 22 | for i in text.lower().split('_'): 23 | if j == 0: 24 | res = i 25 | else: 26 | res = res + i[0].upper() + i[1:] 27 | j += 1 28 | return res 29 | 30 | 31 | def get_records(result, batch_size, columns=None): 32 | records = result.fetchmany(1000) 33 | while records: 34 | if columns: 35 | records = [dict(zip(columns, i)) for i in records] 36 | for record in records: 37 | yield record 38 | records = result.fetchmany(batch_size) 39 | 40 | 41 | def batch_dataset(dataset, batch_size): 42 | cache = [] 43 | for data in dataset: 44 | cache.append(data) 45 | if len(cache) >= batch_size: 46 | yield cache 47 | cache = [] 48 | if cache: 49 | yield cache 50 | 51 | 52 | def get_suffix(text): 53 | left, right = os.path.splitext(text) 54 | return right[1:] if right else left 55 | 56 | 57 | def demo_connect(): 58 | from . import connect 59 | db = connect("sqlite:///:memory:") 60 | db.execute('create table foo(a integer, b varchar(20))') 61 | record = {"a": 1, "b": "one"} 62 | db.execute("INSERT INTO foo(a,b) values(:a,:b)", [record] * 10) 63 | record = {"a": 2, "b": "two"} 64 | db.execute("INSERT INTO foo(a,b) values(:a,:b)", [record] * 10) 65 | return db 66 | 67 | 68 | if __name__ == '__main__': 69 | print(to_camel_style("hello_world")) 70 | print(get_suffix('ab')) 71 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import re 2 | import ast 3 | from setuptools import setup, find_packages 4 | 5 | _version_re = re.compile(r'__version__\s+=\s+(.*)') 6 | 7 | with open('pydbclib/__init__.py', 'rb') as f: 8 | rs = _version_re.search(f.read().decode('utf-8')).group(1) 9 | version = str(ast.literal_eval(rs)) 10 | 11 | setup( 12 | name='pydbclib', 13 | version=version, 14 | install_requires=['sqlalchemy>=1.1.14, <1.4.0', 'sqlparse', "log4py>=2.1"], 15 | description='Python Database Connectivity Lib', 16 | classifiers=[ 17 | 'Development Status :: 5 - Production/Stable', 18 | 'Intended Audience :: Developers', 19 | 'License :: OSI Approved :: Apache Software License', 20 | 'Programming Language :: Python', 21 | 'Programming Language :: Python :: 3.6', 22 | 'Programming Language :: Python :: 3.7', 23 | 'Programming Language :: Python :: 3.8', 24 | ], 25 | author='liyatao', 26 | url='https://github.com/taogeYT/pydbclib', 27 | author_email='li_yatao@outlook.com', 28 | license='Apache 2.0', 29 | packages=find_packages(), 30 | include_package_data=False, 31 | zip_safe=True, 32 | python_requires='>=3.6', 33 | ) 34 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | @time: 2020/3/26 11:42 上午 4 | @desc: 5 | """ 6 | import unittest 7 | import sqlite3 8 | from collections.abc import Iterator 9 | 10 | from sqlalchemy import create_engine 11 | 12 | from pydbclib import connect 13 | 14 | 15 | class TestConnect(unittest.TestCase): 16 | 17 | def test_common_driver(self): 18 | with connect(":memory:", driver="sqlite3") as db: 19 | db.execute("select 1") 20 | con = sqlite3.connect(":memory:") 21 | with connect(driver=con) as db: 22 | db.execute("select 1") 23 | 24 | def test_sqlalchemy_driver(self): 25 | with connect("sqlite:///:memory:") as db: 26 | db.execute("select 1") 27 | engine = create_engine("sqlite:///:memory:") 28 | with connect(driver=engine) as db: 29 | db.execute("select 1") 30 | 31 | 32 | class TestDataBase(unittest.TestCase): 33 | db = None 34 | record = {"a": 1, "b": "1"} 35 | 36 | @classmethod 37 | def setUpClass(cls): 38 | cls.db = connect("sqlite:///:memory:") 39 | 40 | @classmethod 41 | def tearDownClass(cls) -> None: 42 | cls.db.close() 43 | 44 | def setUp(self): 45 | self.db.execute("CREATE TABLE foo (a integer, b varchar(20))") 46 | 47 | def tearDown(self): 48 | self.db.rollback() 49 | self.db.execute('DROP TABLE foo') 50 | 51 | def test_execute(self): 52 | r = self.db.execute("insert into foo(a,b) values(:a,:b)", [self.record]*10) 53 | self.assertEqual(r.rowcount, 10) 54 | self.assertEqual(self.db.execute("update foo set a=2").rowcount, 10) 55 | 56 | def test_bulk(self): 57 | r = self.db.bulk("insert into foo(a,b) values(:a,:b)", (r for r in [self.record]*1000), batch_size=100) 58 | self.assertEqual(r, 1000) 59 | self.assertEqual(self.db.read("select * from foo").get_all(), [self.record]*1000) 60 | 61 | def test_read(self): 62 | r = self.db.read("select * from foo") 63 | self.assertEqual(r.get(1), []) 64 | self.assertIsInstance(r, Iterator) 65 | self.db.get_table("foo").insert([self.record]*10) 66 | r = self.db.read("select * from foo").map(lambda x: {**x, "c": 3}) 67 | self.assertEqual(r.get(1), [{**self.record, "c": 3}]) 68 | self.assertEqual(self.db.read("select * from foo").get_all(), [self.record]*10) 69 | self.assertEqual(self.db.read("select * from foo", as_dict=False).get(1), [(1, "1")]) 70 | 71 | def test_read_one(self): 72 | self.db.get_table("foo").insert([self.record] * 10) 73 | r = self.db.read_one("select * from foo") 74 | self.assertEqual(r, self.record) 75 | 76 | 77 | class TestTable(unittest.TestCase): 78 | db = None 79 | table = None 80 | record = {"a": 1, "b": "1"} 81 | 82 | @classmethod 83 | def setUpClass(cls): 84 | cls.db = connect(":memory:", driver="sqlite3") 85 | 86 | @classmethod 87 | def tearDownClass(cls) -> None: 88 | cls.db.close() 89 | 90 | def setUp(self): 91 | self.db.execute("CREATE TABLE foo (a integer, b varchar(20))") 92 | self.table = self.db.get_table("foo") 93 | 94 | def tearDown(self): 95 | self.db.rollback() 96 | self.db.execute('DROP TABLE foo') 97 | 98 | def test_get_columns(self): 99 | self.assertEqual(self.table.get_columns(), ["a", "b"]) 100 | 101 | def test_insert(self): 102 | self.assertEqual(self.table.insert(self.record), 1) 103 | self.assertEqual(self.table.insert([self.record]*10), 10) 104 | 105 | def test_bulk(self): 106 | self.assertEqual(self.table.bulk([self.record]*1000, batch_size=100), 1000) 107 | self.assertEqual(self.table.find().get_all(), [self.record]*1000) 108 | 109 | def test_update(self): 110 | self.table.insert([self.record]*10) 111 | self.assertEqual(self.table.update({"a": 1}, {"b": "2"}), 10) 112 | self.assertEqual(self.table.find({"a": 1}).get(10), [{"a": 1, "b": "2"}]*10) 113 | 114 | def test_delete(self): 115 | self.table.insert([self.record] * 10) 116 | self.assertEqual(self.table.delete({"a": 1}), 10) 117 | 118 | def test_find(self): 119 | self.assertEqual(self.table.find().get_one(), None) 120 | self.table.insert(self.record) 121 | r = self.table.find({"a": 1}).map(lambda x: {**x, "c": 3}) 122 | self.assertEqual(r.get_one(), {**self.record, "c": 3}) 123 | self.assertIsInstance(r, Iterator) 124 | self.assertEqual(self.table.find({"a": 2}).get(1), []) 125 | 126 | def test_find_one(self): 127 | self.assertEqual(self.table.find_one(), None) 128 | self.table.insert(self.record) 129 | self.assertEqual(self.table.find_one(), self.record) 130 | 131 | def test_to_df(self): 132 | self.assertTrue(self.table.find({"a": 1}).to_df().empty) 133 | self.table.insert([self.record]*10) 134 | df = self.table.find({"a": 1}).limit(1).to_df() 135 | self.assertEqual(df.loc[0, 'a'], self.record['a']) 136 | self.assertEqual(df.loc[0, 'b'], self.record['b']) 137 | 138 | 139 | if __name__ == '__main__': 140 | unittest.main() 141 | --------------------------------------------------------------------------------