├── .github └── workflows │ ├── publish.yml │ └── test.yml ├── .gitignore ├── LICENSE ├── README.md ├── db_to_sqlite ├── __init__.py └── cli.py ├── setup.py └── tests ├── __init__.py ├── conftest.py ├── shared.py ├── test_db_to_sqlite.py ├── test_docs.py ├── test_fixtures.py └── test_redact.py /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Python Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | test: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] 13 | services: 14 | postgres: 15 | image: postgres 16 | env: 17 | POSTGRES_PASSWORD: postgres 18 | options: >- 19 | --health-cmd pg_isready 20 | --health-interval 10s 21 | --health-timeout 5s 22 | --health-retries 5 23 | ports: 24 | - 5432:5432 25 | mysql: 26 | image: mysql 27 | env: 28 | MYSQL_ALLOW_EMPTY_PASSWORD: yes 29 | MYSQL_DATABASE: test_db_to_sqlite 30 | options: >- 31 | --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 32 | ports: 33 | - 3306:3306 34 | steps: 35 | - uses: actions/checkout@v3 36 | - name: Set up Python ${{ matrix.python-version }} 37 | uses: actions/setup-python@v4 38 | with: 39 | python-version: ${{ matrix.python-version }} 40 | - uses: actions/cache@v3 41 | name: Configure pip caching 42 | with: 43 | path: ~/.cache/pip 44 | key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} 45 | restore-keys: | 46 | ${{ runner.os }}-pip- 47 | 48 | - name: Install dependencies 49 | run: | 50 | pip install -e '.[test,test_postgresql,test_mysql]' 51 | - name: Run tests 52 | env: 53 | POSTGRESQL_TEST_DB_CONNECTION: postgresql://postgres:postgres@127.0.0.1:${{ job.services.postgres.ports['5432'] }}/test_db_to_sqlite 54 | MYSQL_TEST_DB_CONNECTION: mysql://root@127.0.0.1:${{ job.services.mysql.ports['3306'] }}/test_db_to_sqlite 55 | run: |- 56 | pytest -vv --timeout=3 --timeout_method=thread 57 | 58 | deploy: 59 | runs-on: ubuntu-latest 60 | needs: [test] 61 | steps: 62 | - uses: actions/checkout@v3 63 | - name: Set up Python 64 | uses: actions/setup-python@v4 65 | with: 66 | python-version: '3.11' 67 | - uses: actions/cache@v3 68 | name: Configure pip caching 69 | with: 70 | path: ~/.cache/pip 71 | key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} 72 | restore-keys: | 73 | ${{ runner.os }}-publish-pip- 74 | - name: Install SpatiaLite 75 | run: | 76 | sudo apt-get install libsqlite3-mod-spatialite 77 | - name: Install dependencies 78 | run: | 79 | pip install setuptools wheel twine 80 | - name: Publish 81 | env: 82 | TWINE_USERNAME: __token__ 83 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 84 | run: | 85 | python setup.py sdist bdist_wheel 86 | twine upload dist/* 87 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push] 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] 11 | services: 12 | postgres: 13 | image: postgres 14 | env: 15 | POSTGRES_PASSWORD: postgres 16 | options: >- 17 | --health-cmd pg_isready 18 | --health-interval 10s 19 | --health-timeout 5s 20 | --health-retries 5 21 | ports: 22 | - 5432:5432 23 | mysql: 24 | image: mysql 25 | env: 26 | MYSQL_ALLOW_EMPTY_PASSWORD: yes 27 | MYSQL_DATABASE: test_db_to_sqlite 28 | options: >- 29 | --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 30 | ports: 31 | - 3306:3306 32 | steps: 33 | - uses: actions/checkout@v3 34 | - name: Set up Python ${{ matrix.python-version }} 35 | uses: actions/setup-python@v4 36 | with: 37 | python-version: ${{ matrix.python-version }} 38 | - uses: actions/cache@v3 39 | name: Configure pip caching 40 | with: 41 | path: ~/.cache/pip 42 | key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} 43 | restore-keys: | 44 | ${{ runner.os }}-pip- 45 | 46 | - name: Install dependencies 47 | run: | 48 | pip install -e '.[test,test_postgresql,test_mysql]' 49 | - name: Run tests 50 | env: 51 | POSTGRESQL_TEST_DB_CONNECTION: postgresql://postgres:postgres@127.0.0.1:${{ job.services.postgres.ports['5432'] }}/test_db_to_sqlite 52 | MYSQL_TEST_DB_CONNECTION: mysql://root@127.0.0.1:${{ job.services.mysql.ports['3306'] }}/test_db_to_sqlite 53 | run: |- 54 | pytest -vv --timeout=3 --timeout_method=thread 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | *.egg-info 3 | __pycache__ 4 | dist 5 | build 6 | venv -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # db-to-sqlite 2 | 3 | [![PyPI](https://img.shields.io/pypi/v/db-to-sqlite.svg)](https://pypi.python.org/pypi/db-to-sqlite) 4 | [![Changelog](https://img.shields.io/github/v/release/simonw/db-to-sqlite?include_prereleases&label=changelog)](https://github.com/simonw/db-to-sqlite/releases) 5 | [![Tests](https://github.com/simonw/db-to-sqlite/workflows/Test/badge.svg)](https://github.com/simonw/db-to-sqlite/actions?query=workflow%3ATest) 6 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/db-to-sqlite/blob/main/LICENSE) 7 | 8 | CLI tool for exporting tables or queries from any SQL database to a SQLite file. 9 | 10 | ## Installation 11 | 12 | Install from PyPI like so: 13 | 14 | pip install db-to-sqlite 15 | 16 | If you want to use it with MySQL, you can install the extra dependency like this: 17 | 18 | pip install 'db-to-sqlite[mysql]' 19 | 20 | Installing the `mysqlclient` library on OS X can be tricky - I've found [this recipe](https://gist.github.com/simonw/90ac0afd204cd0d6d9c3135c3888d116) to work (run that before installing `db-to-sqlite`). 21 | 22 | For PostgreSQL, use this: 23 | 24 | pip install 'db-to-sqlite[postgresql]' 25 | 26 | ## Usage 27 | ``` 28 | Usage: db-to-sqlite [OPTIONS] CONNECTION PATH 29 | 30 | Load data from any database into SQLite. 31 | 32 | PATH is a path to the SQLite file to create, e.c. /tmp/my_database.db 33 | 34 | CONNECTION is a SQLAlchemy connection string, for example: 35 | 36 | postgresql://localhost/my_database 37 | postgresql://username:passwd@localhost/my_database 38 | 39 | mysql://root@localhost/my_database 40 | mysql://username:passwd@localhost/my_database 41 | 42 | More: https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls 43 | 44 | Options: 45 | --version Show the version and exit. 46 | --all Detect and copy all tables 47 | --table TEXT Specific tables to copy 48 | --skip TEXT When using --all skip these tables 49 | --redact TEXT... (table, column) pairs to redact with *** 50 | --sql TEXT Optional SQL query to run 51 | --output TEXT Table in which to save --sql query results 52 | --pk TEXT Optional column to use as a primary key 53 | --index-fks / --no-index-fks Should foreign keys have indexes? Default on 54 | -p, --progress Show progress bar 55 | --postgres-schema TEXT PostgreSQL schema to use 56 | --help Show this message and exit. 57 | ``` 58 | 59 | For example, to save the content of the `blog_entry` table from a PostgreSQL database to a local file called `blog.db` you could do this: 60 | 61 | db-to-sqlite "postgresql://localhost/myblog" blog.db \ 62 | --table=blog_entry 63 | 64 | You can specify `--table` more than once. 65 | 66 | You can also save the data from all of your tables, effectively creating a SQLite copy of your entire database. Any foreign key relationships will be detected and added to the SQLite database. For example: 67 | 68 | db-to-sqlite "postgresql://localhost/myblog" blog.db \ 69 | --all 70 | 71 | When running `--all` you can specify tables to skip using `--skip`: 72 | 73 | db-to-sqlite "postgresql://localhost/myblog" blog.db \ 74 | --all \ 75 | --skip=django_migrations 76 | 77 | If you want to save the results of a custom SQL query, do this: 78 | 79 | db-to-sqlite "postgresql://localhost/myblog" output.db \ 80 | --output=query_results \ 81 | --sql="select id, title, created from blog_entry" \ 82 | --pk=id 83 | 84 | The `--output` option specifies the table that should contain the results of the query. 85 | 86 | ## Using db-to-sqlite with PostgreSQL schemas 87 | 88 | If the tables you want to copy from your PostgreSQL database aren't in the default schema, you can specify an alternate one with the `--postgres-schema` option: 89 | 90 | db-to-sqlite "postgresql://localhost/myblog" blog.db \ 91 | --all \ 92 | --postgres-schema my_schema 93 | 94 | ## Using db-to-sqlite with MS SQL 95 | 96 | The best way to get the connection string needed for the MS SQL connections below is to use urllib from the Standard Library as below 97 | 98 | params = urllib.parse.quote_plus( 99 | "DRIVER={SQL Server Native Client 11.0};" 100 | "SERVER=localhost;" 101 | "DATABASE=my_database;" 102 | "Trusted_Connection=yes;" 103 | ) 104 | 105 | The above will resolve to 106 | 107 | DRIVER%3D%7BSQL+Server+Native+Client+11.0%7D%3B+SERVER%3Dlocalhost%3B+DATABASE%3Dmy_database%3B+Trusted_Connection%3Dyes 108 | 109 | You can then use the string above in the odbc_connect below 110 | 111 | mssql+pyodbc:///?odbc_connect=DRIVER%3D%7BSQL+Server+Native+Client+11.0%7D%3B+SERVER%3Dlocalhost%3B+DATABASE%3Dmy_database%3B+Trusted_Connection%3Dyes 112 | mssql+pyodbc:///?odbc_connect=DRIVER%3D%7BSQL+Server+Native+Client+11.0%7D%3B+SERVER%3Dlocalhost%3B+DATABASE%3Dmy_database%3B+UID%3Dusername%3B+PWD%3Dpasswd 113 | 114 | ## Using db-to-sqlite with Heroku Postgres 115 | 116 | If you run an application on [Heroku](https://www.heroku.com/) using their [Postgres database product](https://www.heroku.com/postgres), you can use the `heroku config` command to access a compatible connection string: 117 | 118 | $ heroku config --app myappname | grep HEROKU_POSTG 119 | HEROKU_POSTGRESQL_OLIVE_URL: postgres://username:password@ec2-xxx-xxx-xxx-x.compute-1.amazonaws.com:5432/dbname 120 | 121 | You can pass this to `db-to-sqlite` to create a local SQLite database with the data from your Heroku instance. 122 | 123 | You can even do this using a bash one-liner: 124 | 125 | $ db-to-sqlite $(heroku config --app myappname | grep HEROKU_POSTG | cut -d: -f 2-) \ 126 | /tmp/heroku.db --all -p 127 | 1/23: django_migrations 128 | ... 129 | 17/23: blog_blogmark 130 | [####################################] 100% 131 | ... 132 | 133 | ## Related projects 134 | 135 | * [Datasette](https://github.com/simonw/datasette): A tool for exploring and publishing data. Works great with SQLite files generated using `db-to-sqlite`. 136 | * [sqlite-utils](https://github.com/simonw/sqlite-utils): Python CLI utility and library for manipulating SQLite databases. 137 | * [csvs-to-sqlite](https://github.com/simonw/csvs-to-sqlite): Convert CSV files into a SQLite database. 138 | 139 | ## Development 140 | 141 | To set up this tool locally, first checkout the code. Then create a new virtual environment: 142 | 143 | cd db-to-sqlite 144 | python3 -m venv venv 145 | source venv/bin/activate 146 | 147 | Or if you are using `pipenv`: 148 | 149 | pipenv shell 150 | 151 | Now install the dependencies and test dependencies: 152 | 153 | pip install -e '.[test]' 154 | 155 | To run the tests: 156 | 157 | pytest 158 | 159 | This will skip tests against MySQL or PostgreSQL if you do not have their additional dependencies installed. 160 | 161 | You can install those extra dependencies like so: 162 | 163 | pip install -e '.[test_mysql,test_postgresql]' 164 | 165 | You can alternative use `pip install psycopg2-binary` if you cannot install the `psycopg2` dependency used by the `test_postgresql` extra. 166 | 167 | See [Running a MySQL server using Homebrew](https://til.simonwillison.net/homebrew/mysql-homebrew) for tips on running the tests against MySQL on macOS, including how to install the `mysqlclient` dependency. 168 | 169 | The PostgreSQL and MySQL tests default to expecting to run against servers on localhost. You can use environment variables to point them at different test database servers: 170 | 171 | - `MYSQL_TEST_DB_CONNECTION` - defaults to `mysql://root@localhost/test_db_to_sqlite` 172 | - `POSTGRESQL_TEST_DB_CONNECTION` - defaults to `postgresql://localhost/test_db_to_sqlite` 173 | 174 | The database you indicate in the environment variable - `test_db_to_sqlite` by default - will be deleted and recreated on every test run. 175 | -------------------------------------------------------------------------------- /db_to_sqlite/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simonw/db-to-sqlite/b7c95fcfe853ca5fc2236bfbf7e4a80ca9239969/db_to_sqlite/__init__.py -------------------------------------------------------------------------------- /db_to_sqlite/cli.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import click 4 | from sqlalchemy import create_engine, inspect, text 5 | from sqlite_utils import Database 6 | 7 | 8 | @click.command() 9 | @click.version_option() 10 | @click.argument("connection") 11 | @click.argument("path", type=click.Path(exists=False), required=True) 12 | @click.option("--all", help="Detect and copy all tables", is_flag=True) 13 | @click.option("--table", help="Specific tables to copy", multiple=True) 14 | @click.option("--skip", help="When using --all skip these tables", multiple=True) 15 | @click.option( 16 | "--redact", 17 | help="(table, column) pairs to redact with ***", 18 | nargs=2, 19 | type=str, 20 | multiple=True, 21 | ) 22 | @click.option("--sql", help="Optional SQL query to run") 23 | @click.option("--output", help="Table in which to save --sql query results") 24 | @click.option("--pk", help="Optional column to use as a primary key") 25 | @click.option( 26 | "--index-fks/--no-index-fks", 27 | default=True, 28 | help="Should foreign keys have indexes? Default on", 29 | ) 30 | @click.option("-p", "--progress", help="Show progress bar", is_flag=True) 31 | @click.option("--postgres-schema", help="PostgreSQL schema to use") 32 | def cli( 33 | connection, 34 | path, 35 | all, 36 | table, 37 | skip, 38 | redact, 39 | sql, 40 | output, 41 | pk, 42 | index_fks, 43 | progress, 44 | postgres_schema, 45 | ): 46 | """ 47 | Load data from any database into SQLite. 48 | 49 | PATH is a path to the SQLite file to create, e.c. /tmp/my_database.db 50 | 51 | CONNECTION is a SQLAlchemy connection string, for example: 52 | 53 | postgresql://localhost/my_database 54 | postgresql://username:passwd@localhost/my_database 55 | 56 | mysql://root@localhost/my_database 57 | mysql://username:passwd@localhost/my_database 58 | 59 | More: https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls 60 | """ 61 | if not all and not table and not sql: 62 | raise click.ClickException("--all OR --table OR --sql required") 63 | if skip and not all: 64 | raise click.ClickException("--skip can only be used with --all") 65 | redact_columns = {} 66 | for table_name, column_name in redact: 67 | redact_columns.setdefault(table_name, set()).add(column_name) 68 | db = Database(path) 69 | if postgres_schema: 70 | conn_args = {"options": "-csearch_path={}".format(postgres_schema)} 71 | else: 72 | conn_args = {} 73 | if connection.startswith("postgres://"): 74 | connection = connection.replace("postgres://", "postgresql://") 75 | db_conn = create_engine(connection, connect_args=conn_args).connect() 76 | inspector = inspect(db_conn) 77 | # Figure out which tables we are copying, if any 78 | tables = table 79 | if all: 80 | tables = inspector.get_table_names() 81 | if tables: 82 | foreign_keys_to_add = [] 83 | for i, table in enumerate(tables): 84 | if progress: 85 | click.echo("{}/{}: {}".format(i + 1, len(tables), table), err=True) 86 | if table in skip: 87 | if progress: 88 | click.echo(" ... skipping", err=True) 89 | continue 90 | pks = inspector.get_pk_constraint(table)["constrained_columns"] 91 | if len(pks) == 1: 92 | pks = pks[0] 93 | fks = inspector.get_foreign_keys(table) 94 | foreign_keys_to_add.extend( 95 | [ 96 | ( 97 | # table, column, other_table, other_column 98 | table, 99 | fk["constrained_columns"][0], 100 | fk["referred_table"], 101 | fk["referred_columns"][0], 102 | ) 103 | for fk in fks 104 | ] 105 | ) 106 | count = None 107 | table_quoted = db_conn.dialect.identifier_preparer.quote_identifier(table) 108 | if progress: 109 | count = db_conn.execute( 110 | text("select count(*) from {}".format(table_quoted)) 111 | ).fetchone()[0] 112 | results = db_conn.execute(text("select * from {}".format(table_quoted))) 113 | redact_these = redact_columns.get(table) or set() 114 | rows = (redacted_dict(r, redact_these) for r in results) 115 | # Make sure generator is not empty 116 | try: 117 | first = next(rows) 118 | except StopIteration: 119 | # This is an empty table - create an empty copy 120 | if not db[table].exists(): 121 | create_columns = {} 122 | for column in inspector.get_columns(table): 123 | try: 124 | column_type = column["type"].python_type 125 | except NotImplementedError: 126 | column_type = str 127 | create_columns[column["name"]] = column_type 128 | db[table].create(create_columns) 129 | else: 130 | rows = itertools.chain([first], rows) 131 | if progress: 132 | with click.progressbar(rows, length=count) as bar: 133 | db[table].insert_all(bar, pk=pks, replace=True) 134 | else: 135 | db[table].insert_all(rows, pk=pks, replace=True) 136 | foreign_keys_to_add_final = [] 137 | for table, column, other_table, other_column in foreign_keys_to_add: 138 | # Make sure both tables exist and are not skipped - they may not 139 | # exist if they were empty and hence .insert_all() didn't have a 140 | # reason to create them. 141 | if ( 142 | db[table].exists() 143 | and table not in skip 144 | and db[other_table].exists() 145 | and other_table not in skip 146 | # Also skip if this column is redacted 147 | and ((table, column) not in redact) 148 | ): 149 | foreign_keys_to_add_final.append( 150 | (table, column, other_table, other_column) 151 | ) 152 | if foreign_keys_to_add_final: 153 | # Add using .add_foreign_keys() to avoid running multiple VACUUMs 154 | if progress: 155 | click.echo( 156 | "\nAdding {} foreign key{}\n{}".format( 157 | len(foreign_keys_to_add_final), 158 | "s" if len(foreign_keys_to_add_final) != 1 else "", 159 | "\n".join( 160 | " {}.{} => {}.{}".format(*fk) 161 | for fk in foreign_keys_to_add_final 162 | ), 163 | ), 164 | err=True, 165 | ) 166 | db.add_foreign_keys(foreign_keys_to_add_final) 167 | if sql: 168 | if not output: 169 | raise click.ClickException("--sql must be accompanied by --output") 170 | results = db_conn.execute(text(sql)) 171 | rows = (dict(r._mapping) for r in results) 172 | db[output].insert_all(rows, pk=pk) 173 | if index_fks: 174 | db.index_foreign_keys() 175 | 176 | 177 | def detect_primary_key(db_conn, table): 178 | inspector = inspect(db_conn) 179 | pks = inspector.get_pk_constraint(table)["constrained_columns"] 180 | if len(pks) > 1: 181 | raise click.ClickException("Multiple primary keys not currently supported") 182 | return pks[0] if pks else None 183 | 184 | 185 | def redacted_dict(row, redact): 186 | d = dict(row._mapping) 187 | for key in redact: 188 | if key in d: 189 | d[key] = "***" 190 | return d 191 | 192 | 193 | if __name__ == "__main__": 194 | cli() 195 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os 3 | 4 | from setuptools import find_packages, setup 5 | 6 | VERSION = "1.5" 7 | 8 | 9 | def get_long_description(): 10 | with io.open( 11 | os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"), 12 | encoding="utf8", 13 | ) as fp: 14 | return fp.read() 15 | 16 | 17 | setup( 18 | name="db-to-sqlite", 19 | description="CLI tool for exporting tables or queries from any SQL database to a SQLite file", 20 | long_description=get_long_description(), 21 | long_description_content_type="text/markdown", 22 | author="Simon Willison", 23 | version=VERSION, 24 | license="Apache License, Version 2.0", 25 | packages=find_packages(), 26 | install_requires=["sqlalchemy", "sqlite-utils>=2.9.1", "click"], 27 | extras_require={ 28 | "test": ["pytest", "pytest-timeout"], 29 | "test_mysql": ["pytest", "mysqlclient"], 30 | "test_postgresql": ["pytest", "psycopg2"], 31 | "mysql": ["mysqlclient"], 32 | "postgresql": ["psycopg2"], 33 | }, 34 | python_requires=">=3.7", 35 | entry_points=""" 36 | [console_scripts] 37 | db-to-sqlite=db_to_sqlite.cli:cli 38 | """, 39 | url="https://datasette.io/tools/db-to-sqlite", 40 | project_urls={ 41 | "Documentation": "https://github.com/simonw/db-to-sqlite/blob/main/README.md", 42 | "Changelog": "https://github.com/simonw/db-to-sqlite/releases", 43 | "Source code": "https://github.com/simonw/db-to-sqlite", 44 | "Issues": "https://github.com/simonw/db-to-sqlite/issues", 45 | "CI": "https://travis-ci.com/simonw/db-to-sqlite", 46 | }, 47 | classifiers=[ 48 | "Intended Audience :: Developers", 49 | "Intended Audience :: Science/Research", 50 | "Intended Audience :: End Users/Desktop", 51 | "Topic :: Database", 52 | "License :: OSI Approved :: Apache Software License", 53 | "Programming Language :: Python :: 3.7", 54 | "Programming Language :: Python :: 3.8", 55 | "Programming Language :: Python :: 3.9", 56 | "Programming Language :: Python :: 3.10", 57 | "Programming Language :: Python :: 3.11", 58 | ], 59 | ) 60 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simonw/db-to-sqlite/b7c95fcfe853ca5fc2236bfbf7e4a80ca9239969/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from click.testing import CliRunner 3 | from sqlalchemy.engine.url import make_url 4 | 5 | from db_to_sqlite import cli 6 | 7 | from .shared import MYSQL_TEST_DB_CONNECTION, POSTGRESQL_TEST_DB_CONNECTION 8 | 9 | try: 10 | import MySQLdb 11 | except ImportError: 12 | MySQLdb = None 13 | try: 14 | import psycopg2 15 | except ImportError: 16 | psycopg2 = None 17 | 18 | MYSQL_SQL = """ 19 | CREATE TABLE IF NOT EXISTS categories ( 20 | id int not null auto_increment primary key, 21 | name varchar(32) not null 22 | ) ENGINE=InnoDB; 23 | 24 | CREATE TABLE IF NOT EXISTS vendors ( 25 | id int not null auto_increment primary key, 26 | name varchar(32) not null 27 | ) ENGINE=InnoDB; 28 | 29 | CREATE TABLE IF NOT EXISTS vendor_categories ( 30 | cat_id int not null, 31 | vendor_id int not null, 32 | PRIMARY KEY (cat_id, vendor_id) 33 | ) ENGINE=InnoDB; 34 | 35 | CREATE TABLE IF NOT EXISTS products ( 36 | id int not null auto_increment primary key, 37 | name varchar(32) not null, 38 | cat_id int not null, 39 | vendor_id int, 40 | price decimal(2,1), 41 | FOREIGN KEY fk_cat(cat_id) REFERENCES categories(id) 42 | ON UPDATE CASCADE ON DELETE CASCADE, 43 | FOREIGN KEY fk_vendor(vendor_id) REFERENCES vendors(id) 44 | ON UPDATE CASCADE ON DELETE CASCADE 45 | ) ENGINE=InnoDB; 46 | 47 | CREATE TABLE IF NOT EXISTS user ( 48 | id int not null auto_increment primary key, 49 | name varchar(32) not null 50 | ) ENGINE=InnoDB; 51 | 52 | CREATE TABLE IF NOT EXISTS empty_table ( 53 | id int not null auto_increment primary key, 54 | name varchar(32) not null, 55 | ip varchar(15) 56 | ) ENGINE=InnoDB; 57 | 58 | DELETE FROM products; 59 | DELETE FROM categories; 60 | DELETE FROM user; 61 | 62 | INSERT INTO categories (id, name) VALUES (1, "Junk"); 63 | INSERT INTO vendors (id, name) VALUES (1, "Acme Corp"); 64 | INSERT INTO products (id, name, cat_id, vendor_id) 65 | VALUES (1, "Bobcat Statue", 1, 1); 66 | INSERT INTO products (id, name, cat_id, vendor_id, price) 67 | VALUES (2, "Yoga Scarf", 1, null, 2.1); 68 | 69 | INSERT INTO vendor_categories (cat_id, vendor_id) 70 | VALUES (1, 1); 71 | 72 | INSERT INTO user (id, name) 73 | VALUES (1, 'Lila'); 74 | """ 75 | 76 | POSTGRESQL_SQL = """ 77 | CREATE TABLE IF NOT EXISTS categories ( 78 | id int not null primary key, 79 | name varchar(32) not null 80 | ); 81 | 82 | CREATE TABLE IF NOT EXISTS vendors ( 83 | id int not null primary key, 84 | name varchar(32) not null 85 | ); 86 | 87 | CREATE TABLE IF NOT EXISTS vendor_categories ( 88 | cat_id int not null, 89 | vendor_id int not null, 90 | PRIMARY KEY (cat_id, vendor_id) 91 | ); 92 | 93 | CREATE TABLE IF NOT EXISTS products ( 94 | id int not null primary key, 95 | name varchar(32) not null, 96 | cat_id int not null, 97 | vendor_id int, 98 | price numeric(2,1), 99 | FOREIGN KEY (cat_id) REFERENCES categories(id), 100 | FOREIGN KEY (vendor_id) REFERENCES vendors(id) 101 | ); 102 | 103 | CREATE TABLE IF NOT EXISTS "user" ( 104 | id int not null primary key, 105 | name varchar(32) not null 106 | ); 107 | 108 | CREATE TABLE IF NOT EXISTS empty_table ( 109 | id int not null primary key, 110 | name varchar(32) not null, 111 | ip inet 112 | ); 113 | 114 | DELETE FROM products; 115 | DELETE FROM categories; 116 | DELETE FROM vendors; 117 | DELETE FROM vendor_categories; 118 | DELETE FROM "user"; 119 | 120 | INSERT INTO categories (id, name) VALUES (1, 'Junk'); 121 | INSERT INTO vendors (id, name) VALUES (1, 'Acme Corp'); 122 | INSERT INTO products (id, name, cat_id, vendor_id) 123 | VALUES (1, 'Bobcat Statue', 1, 1); 124 | INSERT INTO products (id, name, cat_id, vendor_id, price) 125 | VALUES (2, 'Yoga Scarf', 1, null, 2.1); 126 | 127 | INSERT INTO vendor_categories (cat_id, vendor_id) 128 | VALUES (1, 1); 129 | 130 | INSERT INTO "user" (id, name) 131 | VALUES (1, 'Lila'); 132 | 133 | CREATE SCHEMA other_schema; 134 | 135 | CREATE TABLE IF NOT EXISTS other_schema.other_schema_categories ( 136 | id int not null primary key, 137 | name varchar(32) not null 138 | ); 139 | 140 | INSERT INTO other_schema.other_schema_categories (id, name) VALUES (1, 'Other junk'); 141 | 142 | """ 143 | 144 | 145 | @pytest.fixture(autouse=True, scope="session") 146 | def setup_mysql(): 147 | if MySQLdb is None: 148 | yield 149 | return 150 | bits = make_url(MYSQL_TEST_DB_CONNECTION) 151 | kwargs = { 152 | "passwd": bits.password or "", 153 | "host": bits.host, 154 | } 155 | if bits.username: 156 | kwargs["user"] = bits.username 157 | if bits.port: 158 | kwargs["port"] = int(bits.port) 159 | db = MySQLdb.connect(**kwargs) 160 | cursor = db.cursor() 161 | cursor.execute("CREATE DATABASE IF NOT EXISTS {};".format(bits.database)) 162 | cursor.execute("USE {};".format(bits.database)) 163 | cursor.execute(MYSQL_SQL) 164 | cursor.close() 165 | db.commit() 166 | db.close() 167 | yield 168 | 169 | 170 | @pytest.fixture(autouse=True, scope="session") 171 | def setup_postgresql(): 172 | if psycopg2 is None: 173 | yield 174 | return 175 | bits = make_url(POSTGRESQL_TEST_DB_CONNECTION) 176 | db = psycopg2.connect(user=bits.username, password=bits.password, host=bits.host) 177 | db.autocommit = True 178 | cursor = db.cursor() 179 | cursor.execute("SELECT datname FROM pg_database;") 180 | databases = [r[0] for r in cursor.fetchall()] 181 | if bits.database in databases: 182 | cursor.execute("DROP DATABASE {};".format(bits.database)) 183 | cursor.execute("CREATE DATABASE {};".format(bits.database)) 184 | cursor.close() 185 | db.commit() 186 | db.close() 187 | db = psycopg2.connect( 188 | user=bits.username, password=bits.password, host=bits.host, dbname=bits.database 189 | ) 190 | db.autocommit = True 191 | cursor = db.cursor() 192 | cursor.execute(POSTGRESQL_SQL) 193 | db.commit() 194 | db.close() 195 | yield 196 | 197 | 198 | @pytest.fixture 199 | def cli_runner(): 200 | def inner(args, **kwargs): 201 | return CliRunner().invoke(cli.cli, args, catch_exceptions=False, **kwargs) 202 | 203 | return inner 204 | -------------------------------------------------------------------------------- /tests/shared.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | try: 6 | import MySQLdb 7 | except ImportError: 8 | MySQLdb = None 9 | try: 10 | import psycopg2 11 | except ImportError: 12 | psycopg2 = None 13 | 14 | MYSQL_TEST_DB_CONNECTION = os.environ.get( 15 | "MYSQL_TEST_DB_CONNECTION", "mysql://root@localhost/test_db_to_sqlite" 16 | ) 17 | POSTGRESQL_TEST_DB_CONNECTION = os.environ.get( 18 | "POSTGRESQL_TEST_DB_CONNECTION", "postgresql://localhost/test_db_to_sqlite" 19 | ) 20 | 21 | 22 | def all_databases(fn): 23 | "Decorator which parameterizes test function for mysql and postgresql" 24 | return pytest.mark.parametrize( 25 | "connection", 26 | [ 27 | pytest.param( 28 | MYSQL_TEST_DB_CONNECTION, 29 | marks=pytest.mark.skipif( 30 | MySQLdb is None, reason="pip install mysqlclient" 31 | ), 32 | ), 33 | pytest.param( 34 | POSTGRESQL_TEST_DB_CONNECTION, 35 | marks=pytest.mark.skipif( 36 | psycopg2 is None, reason="pip install psycopg2" 37 | ), 38 | ), 39 | # Make sure it works with postgres:// connection strings too 40 | pytest.param( 41 | POSTGRESQL_TEST_DB_CONNECTION.replace("postgresql://", "postgres://"), 42 | marks=pytest.mark.skipif( 43 | psycopg2 is None, reason="pip install psycopg2" 44 | ), 45 | ), 46 | ], 47 | )(fn) 48 | -------------------------------------------------------------------------------- /tests/test_db_to_sqlite.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import sqlite_utils 3 | from sqlite_utils.db import ForeignKey 4 | 5 | from .shared import POSTGRESQL_TEST_DB_CONNECTION, all_databases, psycopg2 6 | 7 | 8 | @all_databases 9 | def test_db_to_sqlite(connection, tmpdir, cli_runner): 10 | db_path = str(tmpdir / "test.db") 11 | cli_runner([connection, db_path, "--all"]) 12 | db = sqlite_utils.Database(db_path) 13 | assert { 14 | "categories", 15 | "products", 16 | "vendors", 17 | "vendor_categories", 18 | "user", 19 | "empty_table", 20 | } == set(db.table_names()) 21 | assert [ 22 | {"id": 1, "name": "Bobcat Statue", "cat_id": 1, "vendor_id": 1, "price": None}, 23 | {"id": 2, "name": "Yoga Scarf", "cat_id": 1, "vendor_id": None, "price": 2.1}, 24 | ] == list(db["products"].rows) 25 | assert [{"id": 1, "name": "Junk"}] == list(db["categories"].rows) 26 | assert [{"cat_id": 1, "vendor_id": 1}] == list(db["vendor_categories"].rows) 27 | assert [{"id": 1, "name": "Lila"}] == list(db["user"].rows) 28 | assert ( 29 | db["empty_table"].schema 30 | == "CREATE TABLE [empty_table] (\n [id] INTEGER,\n [name] TEXT,\n [ip] TEXT\n)" 31 | ) 32 | # Check foreign keys 33 | assert [ 34 | ForeignKey( 35 | table="products", 36 | column="cat_id", 37 | other_table="categories", 38 | other_column="id", 39 | ), 40 | ForeignKey( 41 | table="products", 42 | column="vendor_id", 43 | other_table="vendors", 44 | other_column="id", 45 | ), 46 | ] == sorted(db["products"].foreign_keys) 47 | # Confirm vendor_categories has a compound primary key 48 | assert db["vendor_categories"].pks == ["cat_id", "vendor_id"] 49 | 50 | 51 | @all_databases 52 | def test_index_fks(connection, tmpdir, cli_runner): 53 | db_path = str(tmpdir / "test_with_fks.db") 54 | # With --no-index-fks should create no indexes 55 | cli_runner([connection, db_path, "--all", "--no-index-fks"]) 56 | db = sqlite_utils.Database(db_path) 57 | assert [] == db["products"].indexes 58 | # Without it (the default) it should create the indexes 59 | cli_runner([connection, db_path, "--all"]) 60 | db = sqlite_utils.Database(db_path) 61 | assert [["cat_id"], ["vendor_id"]] == [i.columns for i in db["products"].indexes] 62 | 63 | 64 | @all_databases 65 | def test_specific_tables(connection, tmpdir, cli_runner): 66 | db_path = str(tmpdir / "test_specific_tables.db") 67 | result = cli_runner( 68 | [connection, db_path, "--table", "categories", "--table", "products", "-p"] 69 | ) 70 | assert 0 == result.exit_code, result.output 71 | db = sqlite_utils.Database(db_path) 72 | assert {"categories", "products"} == set(db.table_names()) 73 | assert ( 74 | "1/2: categories\n\n2/2: products\n\n\nAdding 1 foreign key\n products.cat_id => categories.id\n" 75 | == result.output 76 | ) 77 | 78 | 79 | @all_databases 80 | def test_sql_query(connection, tmpdir, cli_runner): 81 | db_path = str(tmpdir / "test_sql.db") 82 | # Without --output it throws an error 83 | result = cli_runner( 84 | [connection, db_path, "--sql", "select name, cat_id from products"] 85 | ) 86 | assert 0 != result.exit_code 87 | assert "Error: --sql must be accompanied by --output" == result.output.strip() 88 | # With --output it does the right thing 89 | result = cli_runner( 90 | [ 91 | connection, 92 | db_path, 93 | "--sql", 94 | "select name, cat_id from products", 95 | "--output", 96 | "out", 97 | ] 98 | ) 99 | assert 0 == result.exit_code, result.output 100 | db = sqlite_utils.Database(db_path) 101 | assert {"out"} == set(db.table_names()) 102 | assert [ 103 | {"name": "Bobcat Statue", "cat_id": 1}, 104 | {"name": "Yoga Scarf", "cat_id": 1}, 105 | ] == list(db["out"].rows) 106 | 107 | 108 | @pytest.mark.skipif(psycopg2 is None, reason="pip install psycopg2") 109 | def test_postgres_schema(tmpdir, cli_runner): 110 | db_path = str(tmpdir / "test_sql.db") 111 | connection = POSTGRESQL_TEST_DB_CONNECTION 112 | result = cli_runner( 113 | [connection, db_path, "--all", "--postgres-schema", "other_schema"] 114 | ) 115 | assert result.exit_code == 0 116 | db = sqlite_utils.Database(db_path) 117 | assert db.tables[0].schema == ( 118 | "CREATE TABLE [other_schema_categories] (\n" 119 | " [id] INTEGER PRIMARY KEY,\n" 120 | " [name] TEXT\n" 121 | ")" 122 | ) 123 | -------------------------------------------------------------------------------- /tests/test_docs.py: -------------------------------------------------------------------------------- 1 | import difflib 2 | from pathlib import Path 3 | 4 | readme_path = Path(__file__).parent.parent / "README.md" 5 | 6 | 7 | def test_readme_contains_latest_help(cli_runner): 8 | result = cli_runner(["--help"], terminal_width=88) 9 | help_text = result.output 10 | # This should be in the README 11 | help_text = help_text.replace("Usage: cli ", "Usage: db-to-sqlite ") 12 | readme = readme_path.read_text() 13 | assert help_text in readme 14 | -------------------------------------------------------------------------------- /tests/test_fixtures.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sqlalchemy.engine.url import make_url 3 | 4 | from .shared import MYSQL_TEST_DB_CONNECTION, POSTGRESQL_TEST_DB_CONNECTION 5 | 6 | try: 7 | import MySQLdb 8 | except ImportError: 9 | MySQLdb = None 10 | try: 11 | import psycopg2 12 | except ImportError: 13 | psycopg2 = None 14 | 15 | 16 | @pytest.mark.skipif( 17 | MySQLdb is None, reason="MySQLdb module not available - pip install mysqlclient" 18 | ) 19 | def test_fixture_mysql(): 20 | bits = make_url(MYSQL_TEST_DB_CONNECTION) 21 | db = MySQLdb.connect( 22 | user=bits.username, passwd=bits.password or "", host=bits.host, db=bits.database 23 | ) 24 | cursor = db.cursor() 25 | cursor.execute("show tables") 26 | try: 27 | assert { 28 | ("categories",), 29 | ("vendors",), 30 | ("products",), 31 | ("vendor_categories",), 32 | ("empty_table",), 33 | ("user",), 34 | } == set(cursor.fetchall()) 35 | finally: 36 | db.close() 37 | 38 | 39 | @pytest.mark.skipif( 40 | psycopg2 is None, reason="psycopg2 module not available - pip install psycopg2" 41 | ) 42 | def test_fixture_postgresql(): 43 | bits = make_url(POSTGRESQL_TEST_DB_CONNECTION) 44 | db = psycopg2.connect( 45 | user=bits.username, password=bits.password, host=bits.host, dbname=bits.database 46 | ) 47 | db.autocommit = True 48 | cursor = db.cursor() 49 | cursor.execute( 50 | """ 51 | SELECT table_schema, table_name FROM information_schema.tables 52 | WHERE table_catalog = 'test_db_to_sqlite' 53 | AND table_type = 'BASE TABLE' 54 | AND table_schema NOT IN ('information_schema', 'pg_catalog') 55 | """ 56 | ) 57 | rows = cursor.fetchall() 58 | assert { 59 | ("public", "vendor_categories"), 60 | ("public", "categories"), 61 | ("public", "products"), 62 | ("public", "vendors"), 63 | ("public", "user"), 64 | ("public", "empty_table"), 65 | ("other_schema", "other_schema_categories"), 66 | } == set(rows) 67 | -------------------------------------------------------------------------------- /tests/test_redact.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import sqlite_utils 3 | from sqlite_utils.db import ForeignKey 4 | 5 | from .shared import all_databases 6 | 7 | 8 | @all_databases 9 | def test_redact(connection, tmpdir, cli_runner): 10 | db_path = str(tmpdir / "test_redact.db") 11 | result = cli_runner( 12 | [ 13 | connection, 14 | db_path, 15 | "--all", 16 | "--redact", 17 | "products", 18 | "name", 19 | "--redact", 20 | "products", 21 | "vendor_id", 22 | ] 23 | ) 24 | assert 0 == result.exit_code, (result.output, result.exception) 25 | db = sqlite_utils.Database(db_path) 26 | assert [ 27 | {"id": 1, "name": "***", "cat_id": 1, "vendor_id": "***", "price": None}, 28 | {"id": 2, "name": "***", "cat_id": 1, "vendor_id": "***", "price": 2.1}, 29 | ] == list(db["products"].rows) 30 | assert [ 31 | ForeignKey( 32 | table="products", 33 | column="cat_id", 34 | other_table="categories", 35 | other_column="id", 36 | ) 37 | ] == sorted(db["products"].foreign_keys) 38 | --------------------------------------------------------------------------------