├── .github ├── FUNDING.yml └── workflows │ ├── codeql-analysis.yml │ ├── documentation-links.yml │ ├── publish.yml │ ├── spellcheck.yml │ ├── test-coverage.yml │ ├── test-sqlite-support.yml │ └── test.yml ├── .gitignore ├── .gitpod.yml ├── .readthedocs.yaml ├── Justfile ├── LICENSE ├── MANIFEST.in ├── README.md ├── codecov.yml ├── docs ├── .gitignore ├── Makefile ├── _static │ └── js │ │ └── custom.js ├── _templates │ └── base.html ├── changelog.rst ├── cli-reference.rst ├── cli.rst ├── codespell-ignore-words.txt ├── conf.py ├── contributing.rst ├── index.rst ├── installation.rst ├── plugins.rst ├── python-api.rst ├── reference.rst └── tutorial.ipynb ├── mypy.ini ├── setup.cfg ├── setup.py ├── sqlite_utils ├── __init__.py ├── __main__.py ├── cli.py ├── db.py ├── hookspecs.py ├── plugins.py ├── py.typed ├── recipes.py └── utils.py └── tests ├── __init__.py ├── conftest.py ├── ext.c ├── sniff ├── example1.csv ├── example2.csv ├── example3.csv └── example4.csv ├── test_analyze.py ├── test_analyze_tables.py ├── test_attach.py ├── test_cli.py ├── test_cli_bulk.py ├── test_cli_convert.py ├── test_cli_insert.py ├── test_cli_memory.py ├── test_column_affinity.py ├── test_constructor.py ├── test_conversions.py ├── test_convert.py ├── test_create.py ├── test_create_view.py ├── test_default_value.py ├── test_delete.py ├── test_docs.py ├── test_duplicate.py ├── test_enable_counts.py ├── test_extract.py ├── test_extracts.py ├── test_fts.py ├── test_get.py ├── test_gis.py ├── test_hypothesis.py ├── test_insert_files.py ├── test_introspect.py ├── test_lookup.py ├── test_m2m.py ├── test_plugins.py ├── test_query.py ├── test_recipes.py ├── test_recreate.py ├── test_register_function.py ├── test_rows.py ├── test_rows_from_file.py ├── test_sniff.py ├── test_suggest_column_types.py ├── test_tracer.py ├── test_transform.py ├── test_update.py ├── test_upsert.py ├── test_utils.py └── test_wal.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [simonw] 2 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | schedule: 7 | - cron: '0 4 * * 5' 8 | 9 | jobs: 10 | analyze: 11 | name: Analyze 12 | runs-on: ubuntu-latest 13 | 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | # Override automatic language detection by changing the below list 18 | # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] 19 | language: ['python'] 20 | # Learn more... 21 | # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection 22 | 23 | steps: 24 | - name: Checkout repository 25 | uses: actions/checkout@v2 26 | with: 27 | # We must fetch at least the immediate parents so that if this is 28 | # a pull request then we can checkout the head. 29 | fetch-depth: 2 30 | 31 | # If this run was triggered by a pull request event, then checkout 32 | # the head of the pull request instead of the merge commit. 33 | - run: git checkout HEAD^2 34 | if: ${{ github.event_name == 'pull_request' }} 35 | 36 | # Initializes the CodeQL tools for scanning. 37 | - name: Initialize CodeQL 38 | uses: github/codeql-action/init@v1 39 | with: 40 | languages: ${{ matrix.language }} 41 | # If you wish to specify custom queries, you can do so here or in a config file. 42 | # By default, queries listed here will override any specified in a config file. 43 | # Prefix the list here with "+" to use these queries and those in the config file. 44 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 45 | 46 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 47 | # If this step fails, then you should remove it and run the build manually (see below) 48 | - name: Autobuild 49 | uses: github/codeql-action/autobuild@v1 50 | 51 | # ℹ️ Command-line programs to run using the OS shell. 52 | # 📚 https://git.io/JvXDl 53 | 54 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 55 | # and modify them (or add more) to build your code if your project 56 | # uses a compiled language 57 | 58 | #- run: | 59 | # make bootstrap 60 | # make release 61 | 62 | - name: Perform CodeQL Analysis 63 | uses: github/codeql-action/analyze@v1 64 | -------------------------------------------------------------------------------- /.github/workflows/documentation-links.yml: -------------------------------------------------------------------------------- 1 | name: Read the Docs Pull Request Preview 2 | on: 3 | pull_request_target: 4 | types: 5 | - opened 6 | 7 | permissions: 8 | pull-requests: write 9 | 10 | jobs: 11 | documentation-links: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: readthedocs/actions/preview@v1 15 | with: 16 | project-slug: "sqlite-utils" 17 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Python Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | test: 9 | runs-on: ${{ matrix.os }} 10 | strategy: 11 | matrix: 12 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 13 | os: [ubuntu-latest, windows-latest, macos-latest] 14 | steps: 15 | - uses: actions/checkout@v4 16 | - name: Set up Python ${{ matrix.python-version }} 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: ${{ matrix.python-version }} 20 | - uses: actions/cache@v4 21 | name: Configure pip caching 22 | with: 23 | path: ~/.cache/pip 24 | key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} 25 | restore-keys: | 26 | ${{ runner.os }}-pip- 27 | - name: Install dependencies 28 | run: | 29 | pip install -e '.[test]' 30 | - name: Run tests 31 | run: | 32 | pytest 33 | deploy: 34 | runs-on: ubuntu-latest 35 | needs: [test] 36 | steps: 37 | - uses: actions/checkout@v4 38 | - name: Set up Python 39 | uses: actions/setup-python@v5 40 | with: 41 | python-version: '3.13' 42 | - uses: actions/cache@v4 43 | name: Configure pip caching 44 | with: 45 | path: ~/.cache/pip 46 | key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} 47 | restore-keys: | 48 | ${{ runner.os }}-publish-pip- 49 | - name: Install dependencies 50 | run: | 51 | pip install setuptools wheel twine 52 | - name: Publish 53 | env: 54 | TWINE_USERNAME: __token__ 55 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 56 | run: | 57 | python setup.py sdist bdist_wheel 58 | twine upload dist/* 59 | -------------------------------------------------------------------------------- /.github/workflows/spellcheck.yml: -------------------------------------------------------------------------------- 1 | name: Check spelling in documentation 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | spellcheck: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | - name: Set up Python 11 | uses: actions/setup-python@v5 12 | with: 13 | python-version: "3.12" 14 | - uses: actions/cache@v4 15 | name: Configure pip caching 16 | with: 17 | path: ~/.cache/pip 18 | key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} 19 | restore-keys: | 20 | ${{ runner.os }}-pip- 21 | - name: Install dependencies 22 | run: | 23 | pip install -e '.[docs]' 24 | - name: Check spelling 25 | run: | 26 | codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt 27 | codespell sqlite_utils --ignore-words docs/codespell-ignore-words.txt 28 | -------------------------------------------------------------------------------- /.github/workflows/test-coverage.yml: -------------------------------------------------------------------------------- 1 | name: Calculate test coverage 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Check out repo 15 | uses: actions/checkout@v4 16 | - name: Set up Python 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: "3.11" 20 | - uses: actions/cache@v4 21 | name: Configure pip caching 22 | with: 23 | path: ~/.cache/pip 24 | key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} 25 | restore-keys: | 26 | ${{ runner.os }}-pip- 27 | - name: Install SpatiaLite 28 | run: sudo apt-get install libsqlite3-mod-spatialite 29 | - name: Install Python dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | python -m pip install -e .[test] 33 | python -m pip install pytest-cov 34 | - name: Run tests 35 | run: |- 36 | ls -lah 37 | pytest --cov=sqlite_utils --cov-report xml:coverage.xml --cov-report term 38 | ls -lah 39 | - name: Upload coverage report 40 | uses: codecov/codecov-action@v1 41 | with: 42 | token: ${{ secrets.CODECOV_TOKEN }} 43 | file: coverage.xml 44 | -------------------------------------------------------------------------------- /.github/workflows/test-sqlite-support.yml: -------------------------------------------------------------------------------- 1 | name: Test SQLite versions 2 | 3 | on: [push, pull_request] 4 | 5 | permissions: 6 | contents: read 7 | 8 | jobs: 9 | test: 10 | runs-on: ${{ matrix.platform }} 11 | continue-on-error: true 12 | strategy: 13 | matrix: 14 | platform: [ubuntu-latest] 15 | python-version: ["3.9"] 16 | sqlite-version: [ 17 | "3.46", 18 | "3.23.1", # 2018-04-10, before UPSERT 19 | ] 20 | steps: 21 | - uses: actions/checkout@v4 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v5 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | allow-prereleases: true 27 | cache: pip 28 | cache-dependency-path: setup.py 29 | - name: Set up SQLite ${{ matrix.sqlite-version }} 30 | uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6 31 | with: 32 | version: ${{ matrix.sqlite-version }} 33 | cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1" 34 | - run: python3 -c "import sqlite3; print(sqlite3.sqlite_version)" 35 | - name: Install dependencies 36 | run: | 37 | pip install -e '.[test]' 38 | pip freeze 39 | - name: Run tests 40 | run: | 41 | python -m pytest 42 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push, pull_request] 4 | 5 | env: 6 | FORCE_COLOR: 1 7 | 8 | jobs: 9 | test: 10 | runs-on: ${{ matrix.os }} 11 | strategy: 12 | matrix: 13 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 14 | numpy: [0, 1] 15 | os: [ubuntu-latest, macos-latest, windows-latest, macos-14] 16 | # Skip 3.9 on macos-14 - it only has 3.10+ 17 | exclude: 18 | - python-version: "3.9" 19 | os: macos-14 20 | steps: 21 | - uses: actions/checkout@v4 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v5 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | allow-prereleases: true 27 | - uses: actions/cache@v4 28 | name: Configure pip caching 29 | with: 30 | path: ~/.cache/pip 31 | key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} 32 | restore-keys: | 33 | ${{ runner.os }}-pip- 34 | - name: Install dependencies 35 | run: | 36 | pip install -e '.[test,mypy,flake8]' 37 | - name: Optionally install tui dependencies 38 | run: pip install -e '.[tui]' 39 | - name: Optionally install numpy 40 | if: matrix.numpy == 1 41 | run: pip install numpy 42 | - name: Install SpatiaLite 43 | if: matrix.os == 'ubuntu-latest' 44 | run: sudo apt-get install libsqlite3-mod-spatialite 45 | - name: On macOS with Python 3.10 test with sqlean.py 46 | if: matrix.os == 'macos-latest' && matrix.python-version == '3.10' 47 | run: pip install sqlean.py sqlite-dump 48 | - name: Build extension for --load-extension test 49 | if: matrix.os == 'ubuntu-latest' 50 | run: |- 51 | (cd tests && gcc ext.c -fPIC -shared -o ext.so && ls -lah) 52 | - name: Run tests 53 | run: | 54 | pytest -v 55 | - name: run mypy 56 | run: mypy sqlite_utils tests 57 | - name: run flake8 58 | run: flake8 59 | - name: Check formatting 60 | run: black . --check 61 | - name: Check if cog needs to be run 62 | run: | 63 | cog --check README.md docs/*.rst 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | build 3 | *.db 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | venv 8 | .eggs 9 | .pytest_cache 10 | *.egg-info 11 | .DS_Store 12 | .mypy_cache 13 | .coverage 14 | .schema 15 | .vscode 16 | .hypothesis 17 | Pipfile 18 | Pipfile.lock 19 | pyproject.toml 20 | tests/*.dylib 21 | tests/*.so 22 | tests/*.dll 23 | -------------------------------------------------------------------------------- /.gitpod.yml: -------------------------------------------------------------------------------- 1 | tasks: 2 | - init: pip install '.[test]' -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sphinx: 4 | configuration: docs/conf.py 5 | 6 | build: 7 | os: ubuntu-22.04 8 | tools: 9 | python: "3.11" 10 | 11 | python: 12 | install: 13 | - method: pip 14 | path: . 15 | extra_requirements: 16 | - docs 17 | 18 | formats: 19 | - pdf 20 | - epub 21 | -------------------------------------------------------------------------------- /Justfile: -------------------------------------------------------------------------------- 1 | # Run tests and linters 2 | @default: test lint 3 | 4 | # Setup project 5 | @init: 6 | pipenv run pip install -e '.[test,docs,mypy,flake8]' 7 | 8 | # Run pytest with supplied options 9 | @test *options: 10 | pipenv run pytest {{options}} 11 | 12 | # Run linters: black, flake8, mypy, cog 13 | @lint: 14 | pipenv run black . --check 15 | pipenv run flake8 16 | pipenv run mypy sqlite_utils tests 17 | pipenv run cog --check README.md docs/*.rst 18 | pipenv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt 19 | 20 | # Rebuild docs with cog 21 | @cog: 22 | pipenv run cog -r README.md docs/*.rst 23 | 24 | # Serve live docs on localhost:8000 25 | @docs: cog 26 | cd docs && pipenv run make livehtml 27 | 28 | # Apply Black 29 | @black: 30 | pipenv run black . 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | recursive-include docs *.rst 4 | recursive-include tests *.py 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sqlite-utils 2 | 3 | [![PyPI](https://img.shields.io/pypi/v/sqlite-utils.svg)](https://pypi.org/project/sqlite-utils/) 4 | [![Changelog](https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog)](https://sqlite-utils.datasette.io/en/stable/changelog.html) 5 | [![Python 3.x](https://img.shields.io/pypi/pyversions/sqlite-utils.svg?logo=python&logoColor=white)](https://pypi.org/project/sqlite-utils/) 6 | [![Tests](https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg)](https://github.com/simonw/sqlite-utils/actions?query=workflow%3ATest) 7 | [![Documentation Status](https://readthedocs.org/projects/sqlite-utils/badge/?version=stable)](http://sqlite-utils.datasette.io/en/stable/?badge=stable) 8 | [![codecov](https://codecov.io/gh/simonw/sqlite-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/simonw/sqlite-utils) 9 | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/sqlite-utils/blob/main/LICENSE) 10 | [![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://discord.gg/Ass7bCAMDw) 11 | 12 | Python CLI utility and library for manipulating SQLite databases. 13 | 14 | ## Some feature highlights 15 | 16 | - [Pipe JSON](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-json-data) (or [CSV or TSV](https://sqlite-utils.datasette.io/en/stable/cli.html#inserting-csv-or-tsv-data)) directly into a new SQLite database file, automatically creating a table with the appropriate schema 17 | - [Run in-memory SQL queries](https://sqlite-utils.datasette.io/en/stable/cli.html#querying-data-directly-using-an-in-memory-database), including joins, directly against data in CSV, TSV or JSON files and view the results 18 | - [Configure SQLite full-text search](https://sqlite-utils.datasette.io/en/stable/cli.html#configuring-full-text-search) against your database tables and run search queries against them, ordered by relevance 19 | - Run [transformations against your tables](https://sqlite-utils.datasette.io/en/stable/cli.html#transforming-tables) to make schema changes that SQLite `ALTER TABLE` does not directly support, such as changing the type of a column 20 | - [Extract columns](https://sqlite-utils.datasette.io/en/stable/cli.html#extracting-columns-into-a-separate-table) into separate tables to better normalize your existing data 21 | - [Install plugins](https://sqlite-utils.datasette.io/en/stable/plugins.html) to add custom SQL functions and additional features 22 | 23 | Read more on my blog, in this series of posts on [New features in sqlite-utils](https://simonwillison.net/series/sqlite-utils-features/) and other [entries tagged sqliteutils](https://simonwillison.net/tags/sqliteutils/). 24 | 25 | ## Installation 26 | 27 | pip install sqlite-utils 28 | 29 | Or if you use [Homebrew](https://brew.sh/) for macOS: 30 | 31 | brew install sqlite-utils 32 | 33 | ## Using as a CLI tool 34 | 35 | Now you can do things with the CLI utility like this: 36 | 37 | $ sqlite-utils memory dogs.csv "select * from t" 38 | [{"id": 1, "age": 4, "name": "Cleo"}, 39 | {"id": 2, "age": 2, "name": "Pancakes"}] 40 | 41 | $ sqlite-utils insert dogs.db dogs dogs.csv --csv 42 | [####################################] 100% 43 | 44 | $ sqlite-utils tables dogs.db --counts 45 | [{"table": "dogs", "count": 2}] 46 | 47 | $ sqlite-utils dogs.db "select id, name from dogs" 48 | [{"id": 1, "name": "Cleo"}, 49 | {"id": 2, "name": "Pancakes"}] 50 | 51 | $ sqlite-utils dogs.db "select * from dogs" --csv 52 | id,age,name 53 | 1,4,Cleo 54 | 2,2,Pancakes 55 | 56 | $ sqlite-utils dogs.db "select * from dogs" --table 57 | id age name 58 | ---- ----- -------- 59 | 1 4 Cleo 60 | 2 2 Pancakes 61 | 62 | You can import JSON data into a new database table like this: 63 | 64 | $ curl https://api.github.com/repos/simonw/sqlite-utils/releases \ 65 | | sqlite-utils insert releases.db releases - --pk id 66 | 67 | Or for data in a CSV file: 68 | 69 | $ sqlite-utils insert dogs.db dogs dogs.csv --csv 70 | 71 | `sqlite-utils memory` lets you import CSV or JSON data into an in-memory database and run SQL queries against it in a single command: 72 | 73 | $ cat dogs.csv | sqlite-utils memory - "select name, age from stdin" 74 | 75 | See the [full CLI documentation](https://sqlite-utils.datasette.io/en/stable/cli.html) for comprehensive coverage of many more commands. 76 | 77 | ## Using as a library 78 | 79 | You can also `import sqlite_utils` and use it as a Python library like this: 80 | 81 | ```python 82 | import sqlite_utils 83 | db = sqlite_utils.Database("demo_database.db") 84 | # This line creates a "dogs" table if one does not already exist: 85 | db["dogs"].insert_all([ 86 | {"id": 1, "age": 4, "name": "Cleo"}, 87 | {"id": 2, "age": 2, "name": "Pancakes"} 88 | ], pk="id") 89 | ``` 90 | 91 | Check out the [full library documentation](https://sqlite-utils.datasette.io/en/stable/python-api.html) for everything else you can do with the Python library. 92 | 93 | ## Related projects 94 | 95 | * [Datasette](https://datasette.io/): A tool for exploring and publishing data 96 | * [csvs-to-sqlite](https://github.com/simonw/csvs-to-sqlite): Convert CSV files into a SQLite database 97 | * [db-to-sqlite](https://github.com/simonw/db-to-sqlite): CLI tool for exporting a MySQL or PostgreSQL database as a SQLite file 98 | * [dogsheep](https://dogsheep.github.io/): A family of tools for personal analytics, built on top of `sqlite-utils` 99 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | informational: true 6 | patch: 7 | default: 8 | informational: true 9 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = sqlite-utils 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | 22 | livehtml: 23 | sphinx-autobuild -a -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0) --watch ../sqlite_utils 24 | -------------------------------------------------------------------------------- /docs/_static/js/custom.js: -------------------------------------------------------------------------------- 1 | jQuery(function ($) { 2 | // Show banner linking to /stable/ if this is a /latest/ page 3 | if (!/\/latest\//.test(location.pathname)) { 4 | return; 5 | } 6 | var stableUrl = location.pathname.replace("/latest/", "/stable/"); 7 | // Check it's not a 404 8 | fetch(stableUrl, { method: "HEAD" }).then((response) => { 9 | if (response.status == 200) { 10 | var warning = $( 11 | `
12 |

Note

13 |

14 | This documentation covers the development version of sqlite-utils.

15 |

See this page for the current stable release. 16 |

17 |
` 18 | ); 19 | warning.find("a").attr("href", stableUrl); 20 | $("article[role=main]").prepend(warning); 21 | } 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /docs/_templates/base.html: -------------------------------------------------------------------------------- 1 | {%- extends "!base.html" %} 2 | 3 | {% block site_meta %} 4 | {{ super() }} 5 | 6 | {% endblock %} 7 | 8 | {% block scripts %} 9 | {{ super() }} 10 | 15 | 42 | {% endblock %} 43 | -------------------------------------------------------------------------------- /docs/codespell-ignore-words.txt: -------------------------------------------------------------------------------- 1 | doub 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from subprocess import Popen, PIPE 5 | from beanbag_docutils.sphinx.ext.github import github_linkcode_resolve 6 | 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | # import os 21 | # import sys 22 | # sys.path.insert(0, os.path.abspath('.')) 23 | 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | # 29 | # needs_sphinx = '1.0' 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = [ 35 | "sphinx.ext.extlinks", 36 | "sphinx.ext.autodoc", 37 | "sphinx_copybutton", 38 | "sphinx.ext.linkcode", 39 | ] 40 | autodoc_member_order = "bysource" 41 | autodoc_typehints = "description" 42 | 43 | extlinks = { 44 | "issue": ("https://github.com/simonw/sqlite-utils/issues/%s", "#%s"), 45 | } 46 | 47 | 48 | def linkcode_resolve(domain, info): 49 | return github_linkcode_resolve( 50 | domain=domain, 51 | info=info, 52 | allowed_module_names=["sqlite_utils"], 53 | github_org_id="simonw", 54 | github_repo_id="sqlite-utils", 55 | branch="main", 56 | ) 57 | 58 | 59 | # Add any paths that contain templates here, relative to this directory. 60 | templates_path = ["_templates"] 61 | 62 | # The suffix(es) of source filenames. 63 | # You can specify multiple suffix as a list of string: 64 | # 65 | # source_suffix = ['.rst', '.md'] 66 | source_suffix = ".rst" 67 | 68 | # The master toctree document. 69 | master_doc = "index" 70 | 71 | # General information about the project. 72 | project = "sqlite-utils" 73 | copyright = "2018-2022, Simon Willison" 74 | author = "Simon Willison" 75 | 76 | # The version info for the project you're documenting, acts as replacement for 77 | # |version| and |release|, also used in various other places throughout the 78 | # built documents. 79 | # 80 | # The short X.Y version. 81 | pipe = Popen("git describe --tags --always", stdout=PIPE, shell=True) 82 | git_version = pipe.stdout.read().decode("utf8") 83 | 84 | if git_version: 85 | version = git_version.rsplit("-", 1)[0] 86 | release = git_version 87 | else: 88 | version = "" 89 | release = "" 90 | 91 | # The language for content autogenerated by Sphinx. Refer to documentation 92 | # for a list of supported languages. 93 | # 94 | # This is also used if you do content translation via gettext catalogs. 95 | # Usually you set "language" from the command line for these cases. 96 | language = "en" 97 | 98 | # List of patterns, relative to source directory, that match files and 99 | # directories to ignore when looking for source files. 100 | # This patterns also effect to html_static_path and html_extra_path 101 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 102 | 103 | # The name of the Pygments (syntax highlighting) style to use. 104 | pygments_style = "sphinx" 105 | 106 | # Only syntax highlight of code-block is used: 107 | highlight_language = "none" 108 | 109 | # If true, `todo` and `todoList` produce output, else they produce nothing. 110 | todo_include_todos = False 111 | 112 | 113 | # -- Options for HTML output ---------------------------------------------- 114 | 115 | # The theme to use for HTML and HTML Help pages. See the documentation for 116 | # a list of builtin themes. 117 | # 118 | html_theme = "furo" 119 | html_title = "sqlite-utils" 120 | 121 | # Theme options are theme-specific and customize the look and feel of a theme 122 | # further. For a list of options available for each theme, see the 123 | # documentation. 124 | # 125 | # html_theme_options = {} 126 | 127 | # Add any paths that contain custom static files (such as style sheets) here, 128 | # relative to this directory. They are copied after the builtin static files, 129 | # so a file named "default.css" will overwrite the builtin "default.css". 130 | html_static_path = ["_static"] 131 | 132 | html_js_files = ["js/custom.js"] 133 | 134 | # -- Options for HTMLHelp output ------------------------------------------ 135 | 136 | # Output file base name for HTML help builder. 137 | htmlhelp_basename = "sqlite-utils-doc" 138 | 139 | 140 | # -- Options for LaTeX output --------------------------------------------- 141 | 142 | latex_elements = { 143 | # The paper size ('letterpaper' or 'a4paper'). 144 | # 145 | # 'papersize': 'letterpaper', 146 | # The font size ('10pt', '11pt' or '12pt'). 147 | # 148 | # 'pointsize': '10pt', 149 | # Additional stuff for the LaTeX preamble. 150 | # 151 | # 'preamble': '', 152 | # Latex figure (float) alignment 153 | # 154 | # 'figure_align': 'htbp', 155 | } 156 | 157 | # Grouping the document tree into LaTeX files. List of tuples 158 | # (source start file, target name, title, 159 | # author, documentclass [howto, manual, or own class]). 160 | latex_documents = [ 161 | ( 162 | master_doc, 163 | "sqlite-utils.tex", 164 | "sqlite-utils documentation", 165 | "Simon Willison", 166 | "manual", 167 | ) 168 | ] 169 | 170 | 171 | # -- Options for manual page output --------------------------------------- 172 | 173 | # One entry per manual page. List of tuples 174 | # (source start file, name, description, authors, manual section). 175 | man_pages = [(master_doc, "sqlite-utils", "sqlite-utils documentation", [author], 1)] 176 | 177 | 178 | # -- Options for Texinfo output ------------------------------------------- 179 | 180 | # Grouping the document tree into Texinfo files. List of tuples 181 | # (source start file, target name, title, author, 182 | # dir menu entry, description, category) 183 | texinfo_documents = [ 184 | ( 185 | master_doc, 186 | "sqlite-utils", 187 | "sqlite-utils documentation", 188 | author, 189 | "sqlite-utils", 190 | "Python library for manipulating SQLite databases", 191 | "Miscellaneous", 192 | ) 193 | ] 194 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. _contributing: 2 | 3 | ============== 4 | Contributing 5 | ============== 6 | 7 | Development of ``sqlite-utils`` takes place in the `sqlite-utils GitHub repository `__. 8 | 9 | All improvements to the software should start with an issue. Read `How I build a feature `__ for a detailed description of the recommended process for building bug fixes or enhancements. 10 | 11 | .. _contributing_checkout: 12 | 13 | Obtaining the code 14 | ================== 15 | 16 | To work on this library locally, first checkout the code. Then create a new virtual environment:: 17 | 18 | git clone git@github.com:simonw/sqlite-utils 19 | cd sqlite-utils 20 | python3 -mvenv venv 21 | source venv/bin/activate 22 | 23 | Or if you are using ``pipenv``:: 24 | 25 | pipenv shell 26 | 27 | Within the virtual environment running ``sqlite-utils`` should run your locally editable version of the tool. You can use ``which sqlite-utils`` to confirm that you are running the version that lives in your virtual environment. 28 | 29 | .. _contributing_tests: 30 | 31 | Running the tests 32 | ================= 33 | 34 | To install the dependencies and test dependencies:: 35 | 36 | pip install -e '.[test]' 37 | 38 | To run the tests:: 39 | 40 | pytest 41 | 42 | .. _contributing_docs: 43 | 44 | Building the documentation 45 | ========================== 46 | 47 | To build the documentation, first install the documentation dependencies:: 48 | 49 | pip install -e '.[docs]' 50 | 51 | Then run ``make livehtml`` from the ``docs/`` directory to start a server on port 8000 that will serve the documentation and live-reload any time you make an edit to a ``.rst`` file:: 52 | 53 | cd docs 54 | make livehtml 55 | 56 | The `cog `__ tool is used to maintain portions of the documentation. You can run it like so:: 57 | 58 | cog -r docs/*.rst 59 | 60 | .. _contributing_linting: 61 | 62 | Linting and formatting 63 | ====================== 64 | 65 | ``sqlite-utils`` uses `Black `__ for code formatting, and `flake8 `__ and `mypy `__ for linting and type checking. 66 | 67 | Black is installed as part of ``pip install -e '.[test]'`` - you can then format your code by running it in the root of the project:: 68 | 69 | black . 70 | 71 | To install ``mypy`` and ``flake8`` run the following:: 72 | 73 | pip install -e '.[flake8,mypy]' 74 | 75 | Both commands can then be run in the root of the project like this:: 76 | 77 | flake8 78 | mypy sqlite_utils 79 | 80 | All three of these tools are run by our CI mechanism against every commit and pull request. 81 | 82 | .. _contributing_just: 83 | 84 | Using Just and pipenv 85 | ===================== 86 | 87 | If you install `Just `__ and `pipenv `__ you can use them to manage your local development environment. 88 | 89 | To create a virtual environment and install all development dependencies, run:: 90 | 91 | cd sqlite-utils 92 | just init 93 | 94 | To run all of the tests and linters:: 95 | 96 | just 97 | 98 | To run tests, or run a specific test module or test by name:: 99 | 100 | just test # All tests 101 | just test tests/test_cli_memory.py # Just this module 102 | just test -k test_memory_no_detect_types # Just this test 103 | 104 | To run just the linters:: 105 | 106 | just lint 107 | 108 | To apply Black to your code:: 109 | 110 | just black 111 | 112 | To update documentation using Cog:: 113 | 114 | just cog 115 | 116 | To run the live documentation server (this will run Cog first):: 117 | 118 | just docs 119 | 120 | And to list all available commands:: 121 | 122 | just -l 123 | 124 | .. _release_process: 125 | 126 | Release process 127 | =============== 128 | 129 | Releases are performed using tags. When a new release is published on GitHub, a `GitHub Actions workflow `__ will perform the following: 130 | 131 | * Run the unit tests against all supported Python versions. If the tests pass... 132 | * Build a wheel bundle of the underlying Python source code 133 | * Push that new wheel up to PyPI: https://pypi.org/project/sqlite-utils/ 134 | 135 | To deploy new releases you will need to have push access to the GitHub repository. 136 | 137 | ``sqlite-utils`` follows `Semantic Versioning `__:: 138 | 139 | major.minor.patch 140 | 141 | We increment ``major`` for backwards-incompatible releases. 142 | 143 | We increment ``minor`` for new features. 144 | 145 | We increment ``patch`` for bugfix releass. 146 | 147 | To release a new version, first create a commit that updates the version number in ``setup.py`` and the :ref:`the changelog ` with highlights of the new version. An example `commit can be seen here `__:: 148 | 149 | # Update changelog 150 | git commit -m " Release 3.29 151 | 152 | Refs #423, #458, #467, #469, #470, #471, #472, #475" -a 153 | git push 154 | 155 | Referencing the issues that are part of the release in the commit message ensures the name of the release shows up on those issue pages, e.g. `here `__. 156 | 157 | You can generate the list of issue references for a specific release by copying and pasting text from the release notes or GitHub changes-since-last-release view into this `Extract issue numbers from pasted text `__ tool. 158 | 159 | To create the tag for the release, create `a new release `__ on GitHub matching the new version number. You can convert the release notes to Markdown by copying and pasting the rendered HTML into this `Paste to Markdown tool `__. 160 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | sqlite-utils |version| 3 | ======================= 4 | 5 | |PyPI| |Changelog| |CI| |License| |discord| 6 | 7 | .. |PyPI| image:: https://img.shields.io/pypi/v/sqlite-utils.svg 8 | :target: https://pypi.org/project/sqlite-utils/ 9 | .. |Changelog| image:: https://img.shields.io/github/v/release/simonw/sqlite-utils?include_prereleases&label=changelog 10 | :target: https://sqlite-utils.datasette.io/en/stable/changelog.html 11 | .. |CI| image:: https://github.com/simonw/sqlite-utils/workflows/Test/badge.svg 12 | :target: https://github.com/simonw/sqlite-utils/actions 13 | .. |License| image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg 14 | :target: https://github.com/simonw/sqlite-utils/blob/main/LICENSE 15 | .. |discord| image:: https://img.shields.io/discord/823971286308356157?label=discord 16 | :target: https://discord.gg/Ass7bCAMDw 17 | 18 | *CLI tool and Python library for manipulating SQLite databases* 19 | 20 | This library and command-line utility helps create SQLite databases from an existing collection of data. 21 | 22 | Most of the functionality is available as either a Python API or through the ``sqlite-utils`` command-line tool. 23 | 24 | sqlite-utils is not intended to be a full ORM: the focus is utility helpers to make creating the initial database and populating it with data as productive as possible. 25 | 26 | It is designed as a useful complement to `Datasette `_. 27 | 28 | `Cleaning data with sqlite-utils and Datasette `_ provides a tutorial introduction (and accompanying ten minute video) about using this tool. 29 | 30 | Contents 31 | -------- 32 | 33 | .. toctree:: 34 | :maxdepth: 3 35 | 36 | installation 37 | cli 38 | python-api 39 | plugins 40 | reference 41 | cli-reference 42 | contributing 43 | changelog 44 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. _installation: 2 | 3 | ============== 4 | Installation 5 | ============== 6 | 7 | ``sqlite-utils`` is tested on Linux, macOS and Windows. 8 | 9 | .. _installation_homebrew: 10 | 11 | Using Homebrew 12 | ============== 13 | 14 | The :ref:`sqlite-utils command-line tool ` can be installed on macOS using Homebrew:: 15 | 16 | brew install sqlite-utils 17 | 18 | If you have it installed and want to upgrade to the most recent release, you can run:: 19 | 20 | brew upgrade sqlite-utils 21 | 22 | Then run ``sqlite-utils --version`` to confirm the installed version. 23 | 24 | .. _installation_pip: 25 | 26 | Using pip 27 | ========= 28 | 29 | The `sqlite-utils package `__ on PyPI includes both the :ref:`sqlite_utils Python library ` and the ``sqlite-utils`` command-line tool. You can install them using ``pip`` like so:: 30 | 31 | pip install sqlite-utils 32 | 33 | .. _installation_pipx: 34 | 35 | Using pipx 36 | ========== 37 | 38 | `pipx `__ is a tool for installing Python command-line applications in their own isolated environments. You can use ``pipx`` to install the ``sqlite-utils`` command-line tool like this:: 39 | 40 | pipx install sqlite-utils 41 | 42 | .. _installation_sqlite3_alternatives: 43 | 44 | Alternatives to sqlite3 45 | ======================= 46 | 47 | By default, ``sqlite-utils`` uses the ``sqlite3`` package bundled with the Python standard library. 48 | 49 | Depending on your operating system, this may come with some limitations. 50 | 51 | On some platforms the ability to load additional extensions (via ``conn.load_extension(...)`` or ``--load-extension=/path/to/extension``) may be disabled. 52 | 53 | You may also see the error ``sqlite3.OperationalError: table sqlite_master may not be modified`` when trying to alter an existing table. 54 | 55 | You can work around these limitations by installing either the `pysqlite3 `__ package or the `sqlean.py `__ package, both of which provide drop-in replacements for the standard library ``sqlite3`` module but with a recent version of SQLite and full support for loading extensions. 56 | 57 | To install ``sqlean.py`` (which has compiled binary wheels available for all major platforms) run the following: 58 | 59 | .. code-block:: bash 60 | 61 | sqlite-utils install sqlean.py 62 | 63 | ``pysqlite3`` and ``sqlean.py`` do not provide implementations of the ``.iterdump()`` method. To use that method (see :ref:`python_api_itedump`) or the ``sqlite-utils dump`` command you should also install the ``sqlite-dump`` package: 64 | 65 | .. code-block:: bash 66 | 67 | sqlite-utils install sqlite-dump 68 | 69 | .. _installation_completion: 70 | 71 | Setting up shell completion 72 | =========================== 73 | 74 | You can configure shell tab completion for the ``sqlite-utils`` command using these commands. 75 | 76 | For ``bash``: 77 | 78 | .. code-block:: bash 79 | 80 | eval "$(_SQLITE_UTILS_COMPLETE=bash_source sqlite-utils)" 81 | 82 | For ``zsh``: 83 | 84 | .. code-block:: zsh 85 | 86 | eval "$(_SQLITE_UTILS_COMPLETE=zsh_source sqlite-utils)" 87 | 88 | Add this code to ``~/.zshrc`` or ``~/.bashrc`` to automatically run it when you start a new shell. 89 | 90 | See `the Click documentation `__ for more details. -------------------------------------------------------------------------------- /docs/plugins.rst: -------------------------------------------------------------------------------- 1 | .. _plugins: 2 | 3 | ========= 4 | Plugins 5 | ========= 6 | 7 | ``sqlite-utils`` supports plugins, which can be used to add extra features to the software. 8 | 9 | Plugins can add new commands, for example ``sqlite-utils some-command ...`` 10 | 11 | Plugins can be installed using the ``sqlite-utils install`` command: 12 | 13 | .. code-block:: bash 14 | 15 | sqlite-utils install sqlite-utils-name-of-plugin 16 | 17 | You can see a JSON list of plugins that have been installed by running this: 18 | 19 | .. code-block:: bash 20 | 21 | sqlite-utils plugins 22 | 23 | Plugin hooks such as :ref:`plugins_hooks_prepare_connection` affect each instance of the ``Database`` class. You can opt-out of these plugins by creating that class instance like so: 24 | 25 | .. code-block:: python 26 | 27 | db = Database(memory=True, execute_plugins=False) 28 | 29 | .. _plugins_building: 30 | 31 | Building a plugin 32 | ----------------- 33 | 34 | Plugins are created in a directory named after the plugin. To create a "hello world" plugin, first create a ``hello-world`` directory: 35 | 36 | .. code-block:: bash 37 | 38 | mkdir hello-world 39 | cd hello-world 40 | 41 | In that folder create two files. The first is a ``pyproject.toml`` file describing the plugin: 42 | 43 | .. code-block:: toml 44 | 45 | [project] 46 | name = "sqlite-utils-hello-world" 47 | version = "0.1" 48 | 49 | [project.entry-points.sqlite_utils] 50 | hello_world = "sqlite_utils_hello_world" 51 | 52 | The ``[project.entry-points.sqlite_utils]`` section tells ``sqlite-utils`` which module to load when executing the plugin. 53 | 54 | Then create ``sqlite_utils_hello_world.py`` with the following content: 55 | 56 | .. code-block:: python 57 | 58 | import click 59 | import sqlite_utils 60 | 61 | @sqlite_utils.hookimpl 62 | def register_commands(cli): 63 | @cli.command() 64 | def hello_world(): 65 | "Say hello world" 66 | click.echo("Hello world!") 67 | 68 | Install the plugin in "editable" mode - so you can make changes to the code and have them picked up instantly by ``sqlite-utils`` - like this: 69 | 70 | .. code-block:: bash 71 | 72 | sqlite-utils install -e . 73 | 74 | Or pass the path to your plugin directory: 75 | 76 | .. code-block:: bash 77 | 78 | sqlite-utils install -e /dev/sqlite-utils-hello-world 79 | 80 | Now, running this should execute your new command: 81 | 82 | .. code-block:: bash 83 | 84 | sqlite-utils hello-world 85 | 86 | Your command will also be listed in the output of ``sqlite-utils --help``. 87 | 88 | See the `LLM plugin documentation `__ for tips on distributing your plugin. 89 | 90 | .. _plugins_hooks: 91 | 92 | Plugin hooks 93 | ------------ 94 | 95 | Plugin hooks allow ``sqlite-utils`` to be customized. 96 | 97 | .. _plugins_hooks_register_commands: 98 | 99 | register_commands(cli) 100 | ~~~~~~~~~~~~~~~~~~~~~~ 101 | 102 | This hook can be used to register additional commands with the ``sqlite-utils`` CLI. It is called with the ``cli`` object, which is a ``click.Group`` instance. 103 | 104 | Example implementation: 105 | 106 | .. code-block:: python 107 | 108 | import click 109 | import sqlite_utils 110 | 111 | @sqlite_utils.hookimpl 112 | def register_commands(cli): 113 | @cli.command() 114 | def hello_world(): 115 | "Say hello world" 116 | click.echo("Hello world!") 117 | 118 | New commands implemented by plugins can invoke existing commands using the `context.invoke `__ mechanism. 119 | 120 | As a special niche feature, if your plugin needs to import some files and then act against an in-memory database containing those files you can forward to the :ref:`sqlite-utils memory command ` and pass it ``return_db=True``: 121 | 122 | .. code-block:: python 123 | 124 | @cli.command() 125 | @click.pass_context 126 | @click.argument( 127 | "paths", 128 | type=click.Path(file_okay=True, dir_okay=False, allow_dash=True), 129 | required=False, 130 | nargs=-1, 131 | ) 132 | def show_schema_for_files(ctx, paths): 133 | from sqlite_utils.cli import memory 134 | db = ctx.invoke(memory, paths=paths, return_db=True) 135 | # Now do something with that database 136 | click.echo(db.schema) 137 | 138 | .. _plugins_hooks_prepare_connection: 139 | 140 | prepare_connection(conn) 141 | ~~~~~~~~~~~~~~~~~~~~~~~~ 142 | 143 | This hook is called when a new SQLite database connection is created. You can 144 | use it to `register custom SQL functions `_, 145 | aggregates and collations. For example: 146 | 147 | .. code-block:: python 148 | 149 | import sqlite_utils 150 | 151 | @sqlite_utils.hookimpl 152 | def prepare_connection(conn): 153 | conn.create_function( 154 | "hello", 1, lambda name: f"Hello, {name}!" 155 | ) 156 | 157 | This registers a SQL function called ``hello`` which takes a single 158 | argument and can be called like this: 159 | 160 | .. code-block:: sql 161 | 162 | select hello("world"); -- "Hello, world!" 163 | -------------------------------------------------------------------------------- /docs/reference.rst: -------------------------------------------------------------------------------- 1 | .. _reference: 2 | 3 | =============== 4 | API reference 5 | =============== 6 | 7 | .. contents:: :local: 8 | :class: this-will-duplicate-information-and-it-is-still-useful-here 9 | 10 | .. _reference_db_database: 11 | 12 | sqlite_utils.db.Database 13 | ======================== 14 | 15 | .. autoclass:: sqlite_utils.db.Database 16 | :members: 17 | :undoc-members: 18 | :special-members: __getitem__ 19 | :exclude-members: use_counts_table, execute_returning_dicts, resolve_foreign_keys 20 | 21 | .. _reference_db_queryable: 22 | 23 | sqlite_utils.db.Queryable 24 | ========================= 25 | 26 | :ref:`Table ` and :ref:`View ` are both subclasses of ``Queryable``, providing access to the following methods: 27 | 28 | .. autoclass:: sqlite_utils.db.Queryable 29 | :members: 30 | :undoc-members: 31 | :exclude-members: execute_count 32 | 33 | .. _reference_db_table: 34 | 35 | sqlite_utils.db.Table 36 | ===================== 37 | 38 | .. autoclass:: sqlite_utils.db.Table 39 | :members: 40 | :undoc-members: 41 | :show-inheritance: 42 | :exclude-members: guess_foreign_column, value_or_default, build_insert_queries_and_params, insert_chunk, add_missing_columns 43 | 44 | .. _reference_db_view: 45 | 46 | sqlite_utils.db.View 47 | ==================== 48 | 49 | .. autoclass:: sqlite_utils.db.View 50 | :members: 51 | :undoc-members: 52 | :show-inheritance: 53 | 54 | .. _reference_db_other: 55 | 56 | Other 57 | ===== 58 | 59 | .. _reference_db_other_column: 60 | 61 | sqlite_utils.db.Column 62 | ---------------------- 63 | 64 | .. autoclass:: sqlite_utils.db.Column 65 | 66 | .. _reference_db_other_column_details: 67 | 68 | sqlite_utils.db.ColumnDetails 69 | ----------------------------- 70 | 71 | .. autoclass:: sqlite_utils.db.ColumnDetails 72 | 73 | sqlite_utils.utils 74 | ================== 75 | 76 | .. _reference_utils_hash_record: 77 | 78 | sqlite_utils.utils.hash_record 79 | ------------------------------ 80 | 81 | .. autofunction:: sqlite_utils.utils.hash_record 82 | 83 | .. _reference_utils_rows_from_file: 84 | 85 | sqlite_utils.utils.rows_from_file 86 | --------------------------------- 87 | 88 | .. autofunction:: sqlite_utils.utils.rows_from_file 89 | 90 | .. _reference_utils_typetracker: 91 | 92 | sqlite_utils.utils.TypeTracker 93 | ------------------------------ 94 | 95 | .. autoclass:: sqlite_utils.utils.TypeTracker 96 | :members: wrap, types 97 | 98 | .. _reference_utils_chunks: 99 | 100 | sqlite_utils.utils.chunks 101 | ------------------------- 102 | 103 | .. autofunction:: sqlite_utils.utils.chunks 104 | 105 | .. _reference_utils_flatten: 106 | 107 | sqlite_utils.utils.flatten 108 | -------------------------- 109 | 110 | .. autofunction:: sqlite_utils.utils.flatten 111 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | 3 | [mypy-pysqlite3,sqlean,sqlite_dump] 4 | ignore_missing_imports = True -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 160 3 | # Black compatibility, E203 whitespace before ':': 4 | extend-ignore = E203 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import io 3 | import os 4 | 5 | VERSION = "4.0a0" 6 | 7 | 8 | def get_long_description(): 9 | with io.open( 10 | os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"), 11 | encoding="utf8", 12 | ) as fp: 13 | return fp.read() 14 | 15 | 16 | setup( 17 | name="sqlite-utils", 18 | description="CLI tool and Python library for manipulating SQLite databases", 19 | long_description=get_long_description(), 20 | long_description_content_type="text/markdown", 21 | author="Simon Willison", 22 | version=VERSION, 23 | license="Apache License, Version 2.0", 24 | packages=find_packages(exclude=["tests", "tests.*"]), 25 | package_data={"sqlite_utils": ["py.typed"]}, 26 | install_requires=[ 27 | "sqlite-fts4", 28 | "click", 29 | "click-default-group>=1.2.3", 30 | "tabulate", 31 | "python-dateutil", 32 | "pluggy", 33 | ], 34 | extras_require={ 35 | "test": ["pytest", "black>=24.1.1", "hypothesis", "cogapp"], 36 | "docs": [ 37 | "furo", 38 | "sphinx-autobuild", 39 | "codespell", 40 | "sphinx-copybutton", 41 | "beanbag-docutils>=2.0", 42 | "pygments-csv-lexer", 43 | ], 44 | "mypy": [ 45 | "mypy", 46 | "types-click", 47 | "types-tabulate", 48 | "types-python-dateutil", 49 | "types-pluggy", 50 | "data-science-types", 51 | ], 52 | "flake8": ["flake8"], 53 | }, 54 | entry_points=""" 55 | [console_scripts] 56 | sqlite-utils=sqlite_utils.cli:cli 57 | """, 58 | url="https://github.com/simonw/sqlite-utils", 59 | project_urls={ 60 | "Documentation": "https://sqlite-utils.datasette.io/en/stable/", 61 | "Changelog": "https://sqlite-utils.datasette.io/en/stable/changelog.html", 62 | "Source code": "https://github.com/simonw/sqlite-utils", 63 | "Issues": "https://github.com/simonw/sqlite-utils/issues", 64 | "CI": "https://github.com/simonw/sqlite-utils/actions", 65 | }, 66 | python_requires=">=3.9", 67 | classifiers=[ 68 | "Development Status :: 5 - Production/Stable", 69 | "Intended Audience :: Developers", 70 | "Intended Audience :: Science/Research", 71 | "Intended Audience :: End Users/Desktop", 72 | "Topic :: Database", 73 | "License :: OSI Approved :: Apache Software License", 74 | "Programming Language :: Python :: 3.9", 75 | "Programming Language :: Python :: 3.10", 76 | "Programming Language :: Python :: 3.11", 77 | "Programming Language :: Python :: 3.12", 78 | "Programming Language :: Python :: 3.13", 79 | ], 80 | # Needed to bundle py.typed so mypy can see it: 81 | zip_safe=False, 82 | ) 83 | -------------------------------------------------------------------------------- /sqlite_utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .utils import suggest_column_types 2 | from .hookspecs import hookimpl 3 | from .hookspecs import hookspec 4 | from .db import Database 5 | 6 | __all__ = ["Database", "suggest_column_types", "hookimpl", "hookspec"] 7 | -------------------------------------------------------------------------------- /sqlite_utils/__main__.py: -------------------------------------------------------------------------------- 1 | from .cli import cli 2 | 3 | if __name__ == "__main__": 4 | cli() 5 | -------------------------------------------------------------------------------- /sqlite_utils/hookspecs.py: -------------------------------------------------------------------------------- 1 | from pluggy import HookimplMarker 2 | from pluggy import HookspecMarker 3 | 4 | hookspec = HookspecMarker("sqlite_utils") 5 | hookimpl = HookimplMarker("sqlite_utils") 6 | 7 | 8 | @hookspec 9 | def register_commands(cli): 10 | """Register additional CLI commands, e.g. 'sqlite-utils mycommand ...'""" 11 | 12 | 13 | @hookspec 14 | def prepare_connection(conn): 15 | """Modify SQLite connection in some way e.g. register custom SQL functions""" 16 | -------------------------------------------------------------------------------- /sqlite_utils/plugins.py: -------------------------------------------------------------------------------- 1 | import pluggy 2 | import sys 3 | from . import hookspecs 4 | 5 | pm = pluggy.PluginManager("sqlite_utils") 6 | pm.add_hookspecs(hookspecs) 7 | 8 | if not getattr(sys, "_called_from_test", False): 9 | # Only load plugins if not running tests 10 | pm.load_setuptools_entrypoints("sqlite_utils") 11 | 12 | 13 | def get_plugins(): 14 | plugins = [] 15 | plugin_to_distinfo = dict(pm.list_plugin_distinfo()) 16 | for plugin in pm.get_plugins(): 17 | plugin_info = { 18 | "name": plugin.__name__, 19 | "hooks": [h.name for h in pm.get_hookcallers(plugin)], 20 | } 21 | distinfo = plugin_to_distinfo.get(plugin) 22 | if distinfo: 23 | plugin_info["version"] = distinfo.version 24 | plugin_info["name"] = distinfo.project_name 25 | plugins.append(plugin_info) 26 | return plugins 27 | -------------------------------------------------------------------------------- /sqlite_utils/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simonw/sqlite-utils/094b010fd870e9fe9f020d4df200d4de7f27209b/sqlite_utils/py.typed -------------------------------------------------------------------------------- /sqlite_utils/recipes.py: -------------------------------------------------------------------------------- 1 | from dateutil import parser 2 | import json 3 | 4 | IGNORE = object() 5 | SET_NULL = object() 6 | 7 | 8 | def parsedate(value, dayfirst=False, yearfirst=False, errors=None): 9 | """ 10 | Parse a date and convert it to ISO date format: yyyy-mm-dd 11 | \b 12 | - dayfirst=True: treat xx as the day in xx/yy/zz 13 | - yearfirst=True: treat xx as the year in xx/yy/zz 14 | - errors=r.IGNORE to ignore values that cannot be parsed 15 | - errors=r.SET_NULL to set values that cannot be parsed to null 16 | """ 17 | try: 18 | return ( 19 | parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst) 20 | .date() 21 | .isoformat() 22 | ) 23 | except parser.ParserError: 24 | if errors is IGNORE: 25 | return value 26 | elif errors is SET_NULL: 27 | return None 28 | else: 29 | raise 30 | 31 | 32 | def parsedatetime(value, dayfirst=False, yearfirst=False, errors=None): 33 | """ 34 | Parse a datetime and convert it to ISO datetime format: yyyy-mm-ddTHH:MM:SS 35 | \b 36 | - dayfirst=True: treat xx as the day in xx/yy/zz 37 | - yearfirst=True: treat xx as the year in xx/yy/zz 38 | - errors=r.IGNORE to ignore values that cannot be parsed 39 | - errors=r.SET_NULL to set values that cannot be parsed to null 40 | """ 41 | try: 42 | return parser.parse(value, dayfirst=dayfirst, yearfirst=yearfirst).isoformat() 43 | except parser.ParserError: 44 | if errors is IGNORE: 45 | return value 46 | elif errors is SET_NULL: 47 | return None 48 | else: 49 | raise 50 | 51 | 52 | def jsonsplit(value, delimiter=",", type=str): 53 | """ 54 | Convert a string like a,b,c into a JSON array ["a", "b", "c"] 55 | """ 56 | return json.dumps([type(s.strip()) for s in value.split(delimiter)]) 57 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/simonw/sqlite-utils/094b010fd870e9fe9f020d4df200d4de7f27209b/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils import Database 2 | from sqlite_utils.utils import sqlite3 3 | import pytest 4 | 5 | CREATE_TABLES = """ 6 | create table Gosh (c1 text, c2 text, c3 text); 7 | create table Gosh2 (c1 text, c2 text, c3 text); 8 | """ 9 | 10 | 11 | def pytest_configure(config): 12 | import sys 13 | 14 | sys._called_from_test = True 15 | 16 | 17 | @pytest.fixture 18 | def fresh_db(): 19 | return Database(memory=True) 20 | 21 | 22 | @pytest.fixture 23 | def existing_db(): 24 | database = Database(memory=True) 25 | database.executescript( 26 | """ 27 | CREATE TABLE foo (text TEXT); 28 | INSERT INTO foo (text) values ("one"); 29 | INSERT INTO foo (text) values ("two"); 30 | INSERT INTO foo (text) values ("three"); 31 | """ 32 | ) 33 | return database 34 | 35 | 36 | @pytest.fixture 37 | def db_path(tmpdir): 38 | path = str(tmpdir / "test.db") 39 | db = sqlite3.connect(path) 40 | db.executescript(CREATE_TABLES) 41 | return path 42 | -------------------------------------------------------------------------------- /tests/ext.c: -------------------------------------------------------------------------------- 1 | /* 2 | ** This file implements a SQLite extension with multiple entrypoints. 3 | ** 4 | ** The default entrypoint, sqlite3_ext_init, has a single function "a". 5 | ** The 1st alternate entrypoint, sqlite3_ext_b_init, has a single function "b". 6 | ** The 2nd alternate entrypoint, sqlite3_ext_c_init, has a single function "c". 7 | ** 8 | ** Compiling instructions: 9 | ** https://www.sqlite.org/loadext.html#compiling_a_loadable_extension 10 | ** 11 | */ 12 | 13 | #include "sqlite3ext.h" 14 | 15 | SQLITE_EXTENSION_INIT1 16 | 17 | // SQL function that returns back the value supplied during sqlite3_create_function() 18 | static void func(sqlite3_context *context, int argc, sqlite3_value **argv) { 19 | sqlite3_result_text(context, (char *) sqlite3_user_data(context), -1, SQLITE_STATIC); 20 | } 21 | 22 | 23 | // The default entrypoint, since it matches the "ext.dylib"/"ext.so" name 24 | #ifdef _WIN32 25 | __declspec(dllexport) 26 | #endif 27 | int sqlite3_ext_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { 28 | SQLITE_EXTENSION_INIT2(pApi); 29 | return sqlite3_create_function(db, "a", 0, 0, "a", func, 0, 0); 30 | } 31 | 32 | // Alternate entrypoint #1 33 | #ifdef _WIN32 34 | __declspec(dllexport) 35 | #endif 36 | int sqlite3_ext_b_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { 37 | SQLITE_EXTENSION_INIT2(pApi); 38 | return sqlite3_create_function(db, "b", 0, 0, "b", func, 0, 0); 39 | } 40 | 41 | // Alternate entrypoint #2 42 | #ifdef _WIN32 43 | __declspec(dllexport) 44 | #endif 45 | int sqlite3_ext_c_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { 46 | SQLITE_EXTENSION_INIT2(pApi); 47 | return sqlite3_create_function(db, "c", 0, 0, "c", func, 0, 0); 48 | } 49 | -------------------------------------------------------------------------------- /tests/sniff/example1.csv: -------------------------------------------------------------------------------- 1 | id,species,name,age 2 | 1,dog,Cleo,5 3 | 2,dog,Pancakes,4 4 | 3,cat,Mozie,8 5 | 4,spider,"Daisy, the tarantula",6 6 | -------------------------------------------------------------------------------- /tests/sniff/example2.csv: -------------------------------------------------------------------------------- 1 | id;species;name;age 2 | 1;dog;Cleo;5 3 | 2;dog;Pancakes;4 4 | 3;cat;Mozie;8 5 | 4;spider;"Daisy, the tarantula";6 6 | -------------------------------------------------------------------------------- /tests/sniff/example3.csv: -------------------------------------------------------------------------------- 1 | id,species,name,age 2 | 1,dog,Cleo,5 3 | 2,dog,Pancakes,4 4 | 3,cat,Mozie,8 5 | 4,spider,'Daisy, the tarantula',6 6 | -------------------------------------------------------------------------------- /tests/sniff/example4.csv: -------------------------------------------------------------------------------- 1 | id species name age 2 | 1 dog Cleo 5 3 | 2 dog Pancakes 4 4 | 3 cat Mozie 8 5 | 4 spider 'Daisy, the tarantula' 6 6 | -------------------------------------------------------------------------------- /tests/test_analyze.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture 5 | def db(fresh_db): 6 | fresh_db["one_index"].insert({"id": 1, "name": "Cleo"}, pk="id") 7 | fresh_db["one_index"].create_index(["name"]) 8 | fresh_db["two_indexes"].insert({"id": 1, "name": "Cleo", "species": "dog"}, pk="id") 9 | fresh_db["two_indexes"].create_index(["name"]) 10 | fresh_db["two_indexes"].create_index(["species"]) 11 | return fresh_db 12 | 13 | 14 | def test_analyze_whole_database(db): 15 | assert set(db.table_names()) == {"one_index", "two_indexes"} 16 | db.analyze() 17 | assert set(db.table_names()).issuperset( 18 | {"one_index", "two_indexes", "sqlite_stat1"} 19 | ) 20 | assert list(db["sqlite_stat1"].rows) == [ 21 | {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, 22 | {"tbl": "two_indexes", "idx": "idx_two_indexes_name", "stat": "1 1"}, 23 | {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"}, 24 | ] 25 | 26 | 27 | @pytest.mark.parametrize("method", ("db_method_with_name", "table_method")) 28 | def test_analyze_one_table(db, method): 29 | assert set(db.table_names()).issuperset({"one_index", "two_indexes"}) 30 | if method == "db_method_with_name": 31 | db.analyze("one_index") 32 | elif method == "table_method": 33 | db["one_index"].analyze() 34 | 35 | assert set(db.table_names()).issuperset( 36 | {"one_index", "two_indexes", "sqlite_stat1"} 37 | ) 38 | assert list(db["sqlite_stat1"].rows) == [ 39 | {"tbl": "one_index", "idx": "idx_one_index_name", "stat": "1 1"} 40 | ] 41 | 42 | 43 | def test_analyze_index_by_name(db): 44 | assert set(db.table_names()) == {"one_index", "two_indexes"} 45 | db.analyze("idx_two_indexes_species") 46 | assert set(db.table_names()).issuperset( 47 | {"one_index", "two_indexes", "sqlite_stat1"} 48 | ) 49 | assert list(db["sqlite_stat1"].rows) == [ 50 | {"tbl": "two_indexes", "idx": "idx_two_indexes_species", "stat": "1 1"}, 51 | ] 52 | -------------------------------------------------------------------------------- /tests/test_analyze_tables.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils.db import Database, ColumnDetails 2 | from sqlite_utils import cli 3 | from click.testing import CliRunner 4 | import pytest 5 | import sqlite3 6 | 7 | 8 | @pytest.fixture 9 | def db_to_analyze(fresh_db): 10 | stuff = fresh_db["stuff"] 11 | stuff.insert_all( 12 | [ 13 | {"id": 1, "owner": "Terryterryterry", "size": 5}, 14 | {"id": 2, "owner": "Joan", "size": 4}, 15 | {"id": 3, "owner": "Kumar", "size": 5}, 16 | {"id": 4, "owner": "Anne", "size": 5}, 17 | {"id": 5, "owner": "Terryterryterry", "size": 5}, 18 | {"id": 6, "owner": "Joan", "size": 4}, 19 | {"id": 7, "owner": "Kumar", "size": 5}, 20 | {"id": 8, "owner": "Joan", "size": 4}, 21 | ], 22 | pk="id", 23 | ) 24 | return fresh_db 25 | 26 | 27 | @pytest.fixture 28 | def big_db_to_analyze_path(tmpdir): 29 | path = str(tmpdir / "test.db") 30 | db = Database(path) 31 | categories = { 32 | "A": 40, 33 | "B": 30, 34 | "C": 20, 35 | "D": 10, 36 | } 37 | to_insert = [] 38 | for category, count in categories.items(): 39 | for _ in range(count): 40 | to_insert.append( 41 | { 42 | "category": category, 43 | "all_null": None, 44 | } 45 | ) 46 | db["stuff"].insert_all(to_insert) 47 | return path 48 | 49 | 50 | @pytest.mark.parametrize( 51 | "column,extra_kwargs,expected", 52 | [ 53 | ( 54 | "id", 55 | {}, 56 | ColumnDetails( 57 | table="stuff", 58 | column="id", 59 | total_rows=8, 60 | num_null=0, 61 | num_blank=0, 62 | num_distinct=8, 63 | most_common=None, 64 | least_common=None, 65 | ), 66 | ), 67 | ( 68 | "owner", 69 | {}, 70 | ColumnDetails( 71 | table="stuff", 72 | column="owner", 73 | total_rows=8, 74 | num_null=0, 75 | num_blank=0, 76 | num_distinct=4, 77 | most_common=[("Joan", 3), ("Kumar", 2)], 78 | least_common=[("Anne", 1), ("Terry...", 2)], 79 | ), 80 | ), 81 | ( 82 | "size", 83 | {}, 84 | ColumnDetails( 85 | table="stuff", 86 | column="size", 87 | total_rows=8, 88 | num_null=0, 89 | num_blank=0, 90 | num_distinct=2, 91 | most_common=[(5, 5), (4, 3)], 92 | least_common=None, 93 | ), 94 | ), 95 | ( 96 | "owner", 97 | {"most_common": False}, 98 | ColumnDetails( 99 | table="stuff", 100 | column="owner", 101 | total_rows=8, 102 | num_null=0, 103 | num_blank=0, 104 | num_distinct=4, 105 | most_common=None, 106 | least_common=[("Anne", 1), ("Terry...", 2)], 107 | ), 108 | ), 109 | ( 110 | "owner", 111 | {"least_common": False}, 112 | ColumnDetails( 113 | table="stuff", 114 | column="owner", 115 | total_rows=8, 116 | num_null=0, 117 | num_blank=0, 118 | num_distinct=4, 119 | most_common=[("Joan", 3), ("Kumar", 2)], 120 | least_common=None, 121 | ), 122 | ), 123 | ], 124 | ) 125 | def test_analyze_column(db_to_analyze, column, extra_kwargs, expected): 126 | assert ( 127 | db_to_analyze["stuff"].analyze_column( 128 | column, common_limit=2, value_truncate=5, **extra_kwargs 129 | ) 130 | == expected 131 | ) 132 | 133 | 134 | @pytest.fixture 135 | def db_to_analyze_path(db_to_analyze, tmpdir): 136 | path = str(tmpdir / "test.db") 137 | db = sqlite3.connect(path) 138 | sql = "\n".join(db_to_analyze.iterdump()) 139 | db.executescript(sql) 140 | return path 141 | 142 | 143 | def test_analyze_table(db_to_analyze_path): 144 | result = CliRunner().invoke(cli.cli, ["analyze-tables", db_to_analyze_path]) 145 | assert ( 146 | result.output.strip() 147 | == ( 148 | """ 149 | stuff.id: (1/3) 150 | 151 | Total rows: 8 152 | Null rows: 0 153 | Blank rows: 0 154 | 155 | Distinct values: 8 156 | 157 | stuff.owner: (2/3) 158 | 159 | Total rows: 8 160 | Null rows: 0 161 | Blank rows: 0 162 | 163 | Distinct values: 4 164 | 165 | Most common: 166 | 3: Joan 167 | 2: Terryterryterry 168 | 2: Kumar 169 | 1: Anne 170 | 171 | stuff.size: (3/3) 172 | 173 | Total rows: 8 174 | Null rows: 0 175 | Blank rows: 0 176 | 177 | Distinct values: 2 178 | 179 | Most common: 180 | 5: 5 181 | 3: 4""" 182 | ).strip() 183 | ) 184 | 185 | 186 | def test_analyze_table_save(db_to_analyze_path): 187 | result = CliRunner().invoke( 188 | cli.cli, ["analyze-tables", db_to_analyze_path, "--save"] 189 | ) 190 | assert result.exit_code == 0 191 | rows = list(Database(db_to_analyze_path)["_analyze_tables_"].rows) 192 | assert rows == [ 193 | { 194 | "table": "stuff", 195 | "column": "id", 196 | "total_rows": 8, 197 | "num_null": 0, 198 | "num_blank": 0, 199 | "num_distinct": 8, 200 | "most_common": None, 201 | "least_common": None, 202 | }, 203 | { 204 | "table": "stuff", 205 | "column": "owner", 206 | "total_rows": 8, 207 | "num_null": 0, 208 | "num_blank": 0, 209 | "num_distinct": 4, 210 | "most_common": '[["Joan", 3], ["Terryterryterry", 2], ["Kumar", 2], ["Anne", 1]]', 211 | "least_common": None, 212 | }, 213 | { 214 | "table": "stuff", 215 | "column": "size", 216 | "total_rows": 8, 217 | "num_null": 0, 218 | "num_blank": 0, 219 | "num_distinct": 2, 220 | "most_common": "[[5, 5], [4, 3]]", 221 | "least_common": None, 222 | }, 223 | ] 224 | 225 | 226 | @pytest.mark.parametrize( 227 | "no_most,no_least", 228 | ( 229 | (False, False), 230 | (True, False), 231 | (False, True), 232 | (True, True), 233 | ), 234 | ) 235 | def test_analyze_table_save_no_most_no_least_options( 236 | no_most, no_least, big_db_to_analyze_path 237 | ): 238 | args = [ 239 | "analyze-tables", 240 | big_db_to_analyze_path, 241 | "--save", 242 | "--common-limit", 243 | "2", 244 | "--column", 245 | "category", 246 | ] 247 | if no_most: 248 | args.append("--no-most") 249 | if no_least: 250 | args.append("--no-least") 251 | result = CliRunner().invoke(cli.cli, args) 252 | assert result.exit_code == 0 253 | rows = list(Database(big_db_to_analyze_path)["_analyze_tables_"].rows) 254 | expected = { 255 | "table": "stuff", 256 | "column": "category", 257 | "total_rows": 100, 258 | "num_null": 0, 259 | "num_blank": 0, 260 | "num_distinct": 4, 261 | "most_common": None, 262 | "least_common": None, 263 | } 264 | if not no_most: 265 | expected["most_common"] = '[["A", 40], ["B", 30]]' 266 | if not no_least: 267 | expected["least_common"] = '[["D", 10], ["C", 20]]' 268 | 269 | assert rows == [expected] 270 | 271 | 272 | def test_analyze_table_column_all_nulls(big_db_to_analyze_path): 273 | result = CliRunner().invoke( 274 | cli.cli, 275 | ["analyze-tables", big_db_to_analyze_path, "stuff", "--column", "all_null"], 276 | ) 277 | assert result.exit_code == 0 278 | assert result.output == ( 279 | "stuff.all_null: (1/1)\n\n Total rows: 100\n" 280 | " Null rows: 100\n" 281 | " Blank rows: 0\n" 282 | "\n" 283 | " Distinct values: 0\n\n" 284 | ) 285 | 286 | 287 | @pytest.mark.parametrize( 288 | "args,expected_error", 289 | ( 290 | (["-c", "bad_column"], "These columns were not found: bad_column\n"), 291 | (["one", "-c", "age"], "These columns were not found: age\n"), 292 | (["two", "-c", "age"], None), 293 | ( 294 | ["one", "-c", "age", "--column", "bad"], 295 | "These columns were not found: age, bad\n", 296 | ), 297 | ), 298 | ) 299 | def test_analyze_table_validate_columns(tmpdir, args, expected_error): 300 | path = str(tmpdir / "test_validate_columns.db") 301 | db = Database(path) 302 | db["one"].insert( 303 | { 304 | "id": 1, 305 | "name": "one", 306 | } 307 | ) 308 | db["two"].insert( 309 | { 310 | "id": 1, 311 | "age": 5, 312 | } 313 | ) 314 | result = CliRunner().invoke( 315 | cli.cli, 316 | ["analyze-tables", path] + args, 317 | catch_exceptions=False, 318 | ) 319 | assert result.exit_code == (1 if expected_error else 0) 320 | if expected_error: 321 | assert expected_error in result.output 322 | -------------------------------------------------------------------------------- /tests/test_attach.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils import Database 2 | 3 | 4 | def test_attach(tmpdir): 5 | foo_path = str(tmpdir / "foo.db") 6 | bar_path = str(tmpdir / "bar.db") 7 | db = Database(foo_path) 8 | with db.conn: 9 | db["foo"].insert({"id": 1, "text": "foo"}) 10 | db2 = Database(bar_path) 11 | with db2.conn: 12 | db2["bar"].insert({"id": 1, "text": "bar"}) 13 | db.attach("bar", bar_path) 14 | assert db.execute( 15 | "select * from foo union all select * from bar.bar" 16 | ).fetchall() == [(1, "foo"), (1, "bar")] 17 | -------------------------------------------------------------------------------- /tests/test_cli_bulk.py: -------------------------------------------------------------------------------- 1 | from click.testing import CliRunner 2 | from sqlite_utils import cli, Database 3 | import pathlib 4 | import pytest 5 | import subprocess 6 | import sys 7 | import time 8 | 9 | 10 | @pytest.fixture 11 | def test_db_and_path(tmpdir): 12 | db_path = str(pathlib.Path(tmpdir) / "data.db") 13 | db = Database(db_path) 14 | db["example"].insert_all( 15 | [ 16 | {"id": 1, "name": "One"}, 17 | {"id": 2, "name": "Two"}, 18 | ], 19 | pk="id", 20 | ) 21 | return db, db_path 22 | 23 | 24 | def test_cli_bulk(test_db_and_path): 25 | db, db_path = test_db_and_path 26 | result = CliRunner().invoke( 27 | cli.cli, 28 | [ 29 | "bulk", 30 | db_path, 31 | "insert into example (id, name) values (:id, myupper(:name))", 32 | "-", 33 | "--nl", 34 | "--functions", 35 | "myupper = lambda s: s.upper()", 36 | ], 37 | input='{"id": 3, "name": "Three"}\n{"id": 4, "name": "Four"}\n', 38 | ) 39 | assert result.exit_code == 0, result.output 40 | assert [ 41 | {"id": 1, "name": "One"}, 42 | {"id": 2, "name": "Two"}, 43 | {"id": 3, "name": "THREE"}, 44 | {"id": 4, "name": "FOUR"}, 45 | ] == list(db["example"].rows) 46 | 47 | 48 | def test_cli_bulk_batch_size(test_db_and_path): 49 | db, db_path = test_db_and_path 50 | proc = subprocess.Popen( 51 | [ 52 | sys.executable, 53 | "-m", 54 | "sqlite_utils", 55 | "bulk", 56 | db_path, 57 | "insert into example (id, name) values (:id, :name)", 58 | "-", 59 | "--nl", 60 | "--batch-size", 61 | "2", 62 | ], 63 | stdin=subprocess.PIPE, 64 | stdout=sys.stdout, 65 | ) 66 | # Writing one record should not commit 67 | proc.stdin.write(b'{"id": 3, "name": "Three"}\n\n') 68 | proc.stdin.flush() 69 | time.sleep(1) 70 | assert db["example"].count == 2 71 | 72 | # Writing another should trigger a commit: 73 | proc.stdin.write(b'{"id": 4, "name": "Four"}\n\n') 74 | proc.stdin.flush() 75 | time.sleep(1) 76 | assert db["example"].count == 4 77 | 78 | proc.stdin.close() 79 | proc.wait() 80 | assert proc.returncode == 0 81 | 82 | 83 | def test_cli_bulk_error(test_db_and_path): 84 | _, db_path = test_db_and_path 85 | result = CliRunner().invoke( 86 | cli.cli, 87 | [ 88 | "bulk", 89 | db_path, 90 | "insert into example (id, name) value (:id, :name)", 91 | "-", 92 | "--nl", 93 | ], 94 | input='{"id": 3, "name": "Three"}', 95 | ) 96 | assert result.exit_code == 1 97 | assert result.output == 'Error: near "value": syntax error\n' 98 | -------------------------------------------------------------------------------- /tests/test_cli_memory.py: -------------------------------------------------------------------------------- 1 | import click 2 | import json 3 | import pytest 4 | from click.testing import CliRunner 5 | 6 | from sqlite_utils import Database, cli 7 | 8 | 9 | def test_memory_basic(): 10 | result = CliRunner().invoke(cli.cli, ["memory", "select 1 + 1"]) 11 | assert result.exit_code == 0 12 | assert result.output.strip() == '[{"1 + 1": 2}]' 13 | 14 | 15 | @pytest.mark.parametrize("sql_from", ("test", "t", "t1")) 16 | @pytest.mark.parametrize("use_stdin", (True, False)) 17 | def test_memory_csv(tmpdir, sql_from, use_stdin): 18 | content = "id,name\n1,Cleo\n2,Bants" 19 | input = None 20 | if use_stdin: 21 | input = content 22 | csv_path = "-" 23 | if sql_from == "test": 24 | sql_from = "stdin" 25 | else: 26 | csv_path = str(tmpdir / "test.csv") 27 | with open(csv_path, "w") as fp: 28 | fp.write(content) 29 | result = CliRunner().invoke( 30 | cli.cli, 31 | ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], 32 | input=input, 33 | ) 34 | assert result.exit_code == 0 35 | assert ( 36 | result.output.strip() == '{"id": 1, "name": "Cleo"}\n{"id": 2, "name": "Bants"}' 37 | ) 38 | 39 | 40 | @pytest.mark.parametrize("use_stdin", (True, False)) 41 | def test_memory_tsv(tmpdir, use_stdin): 42 | data = "id\tname\n1\tCleo\n2\tBants" 43 | if use_stdin: 44 | input = data 45 | path = "stdin:tsv" 46 | sql_from = "stdin" 47 | else: 48 | input = None 49 | path = str(tmpdir / "chickens.tsv") 50 | with open(path, "w") as fp: 51 | fp.write(data) 52 | path = path + ":tsv" 53 | sql_from = "chickens" 54 | result = CliRunner().invoke( 55 | cli.cli, 56 | ["memory", path, "select * from {}".format(sql_from)], 57 | input=input, 58 | ) 59 | assert result.exit_code == 0, result.output 60 | assert json.loads(result.output.strip()) == [ 61 | {"id": 1, "name": "Cleo"}, 62 | {"id": 2, "name": "Bants"}, 63 | ] 64 | 65 | 66 | @pytest.mark.parametrize("use_stdin", (True, False)) 67 | def test_memory_json(tmpdir, use_stdin): 68 | data = '[{"name": "Bants"}, {"name": "Dori", "age": 1, "nested": {"nest": 1}}]' 69 | if use_stdin: 70 | input = data 71 | path = "stdin:json" 72 | sql_from = "stdin" 73 | else: 74 | input = None 75 | path = str(tmpdir / "chickens.json") 76 | with open(path, "w") as fp: 77 | fp.write(data) 78 | path = path + ":json" 79 | sql_from = "chickens" 80 | result = CliRunner().invoke( 81 | cli.cli, 82 | ["memory", path, "select * from {}".format(sql_from)], 83 | input=input, 84 | ) 85 | assert result.exit_code == 0, result.output 86 | assert json.loads(result.output.strip()) == [ 87 | {"name": "Bants", "age": None, "nested": None}, 88 | {"name": "Dori", "age": 1, "nested": '{"nest": 1}'}, 89 | ] 90 | 91 | 92 | @pytest.mark.parametrize("use_stdin", (True, False)) 93 | def test_memory_json_nl(tmpdir, use_stdin): 94 | data = '{"name": "Bants"}\n\n{"name": "Dori"}' 95 | if use_stdin: 96 | input = data 97 | path = "stdin:nl" 98 | sql_from = "stdin" 99 | else: 100 | input = None 101 | path = str(tmpdir / "chickens.json") 102 | with open(path, "w") as fp: 103 | fp.write(data) 104 | path = path + ":nl" 105 | sql_from = "chickens" 106 | result = CliRunner().invoke( 107 | cli.cli, 108 | ["memory", path, "select * from {}".format(sql_from)], 109 | input=input, 110 | ) 111 | assert result.exit_code == 0, result.output 112 | assert json.loads(result.output.strip()) == [ 113 | {"name": "Bants"}, 114 | {"name": "Dori"}, 115 | ] 116 | 117 | 118 | @pytest.mark.parametrize("use_stdin", (True, False)) 119 | def test_memory_csv_encoding(tmpdir, use_stdin): 120 | latin1_csv = ( 121 | b"date,name,latitude,longitude\n" b"2020-03-04,S\xe3o Paulo,-23.561,-46.645\n" 122 | ) 123 | input = None 124 | if use_stdin: 125 | input = latin1_csv 126 | csv_path = "-" 127 | sql_from = "stdin" 128 | else: 129 | csv_path = str(tmpdir / "test.csv") 130 | with open(csv_path, "wb") as fp: 131 | fp.write(latin1_csv) 132 | sql_from = "test" 133 | # Without --encoding should error: 134 | assert ( 135 | CliRunner() 136 | .invoke( 137 | cli.cli, 138 | ["memory", csv_path, "select * from {}".format(sql_from), "--nl"], 139 | input=input, 140 | ) 141 | .exit_code 142 | == 1 143 | ) 144 | # With --encoding should work: 145 | result = CliRunner().invoke( 146 | cli.cli, 147 | ["memory", "-", "select * from stdin", "--encoding", "latin-1", "--nl"], 148 | input=latin1_csv, 149 | ) 150 | assert result.exit_code == 0, result.output 151 | assert json.loads(result.output.strip()) == { 152 | "date": "2020-03-04", 153 | "name": "São Paulo", 154 | "latitude": -23.561, 155 | "longitude": -46.645, 156 | } 157 | 158 | 159 | @pytest.mark.parametrize("extra_args", ([], ["select 1"])) 160 | def test_memory_dump(extra_args): 161 | result = CliRunner().invoke( 162 | cli.cli, 163 | ["memory", "-"] + extra_args + ["--dump"], 164 | input="id,name\n1,Cleo\n2,Bants", 165 | ) 166 | assert result.exit_code == 0 167 | expected = ( 168 | "BEGIN TRANSACTION;\n" 169 | 'CREATE TABLE IF NOT EXISTS "stdin" (\n' 170 | " [id] INTEGER,\n" 171 | " [name] TEXT\n" 172 | ");\n" 173 | "INSERT INTO \"stdin\" VALUES(1,'Cleo');\n" 174 | "INSERT INTO \"stdin\" VALUES(2,'Bants');\n" 175 | "CREATE VIEW t1 AS select * from [stdin];\n" 176 | "CREATE VIEW t AS select * from [stdin];\n" 177 | "COMMIT;" 178 | ) 179 | # Using sqlite-dump it won't have IF NOT EXISTS 180 | expected_alternative = expected.replace("IF NOT EXISTS ", "") 181 | assert result.output.strip() in (expected, expected_alternative) 182 | 183 | 184 | @pytest.mark.parametrize("extra_args", ([], ["select 1"])) 185 | def test_memory_schema(extra_args): 186 | result = CliRunner().invoke( 187 | cli.cli, 188 | ["memory", "-"] + extra_args + ["--schema"], 189 | input="id,name\n1,Cleo\n2,Bants", 190 | ) 191 | assert result.exit_code == 0 192 | assert result.output.strip() == ( 193 | 'CREATE TABLE "stdin" (\n' 194 | " [id] INTEGER,\n" 195 | " [name] TEXT\n" 196 | ");\n" 197 | "CREATE VIEW t1 AS select * from [stdin];\n" 198 | "CREATE VIEW t AS select * from [stdin];" 199 | ) 200 | 201 | 202 | @pytest.mark.parametrize("extra_args", ([], ["select 1"])) 203 | def test_memory_save(tmpdir, extra_args): 204 | save_to = str(tmpdir / "save.db") 205 | result = CliRunner().invoke( 206 | cli.cli, 207 | ["memory", "-"] + extra_args + ["--save", save_to], 208 | input="id,name\n1,Cleo\n2,Bants", 209 | ) 210 | assert result.exit_code == 0 211 | db = Database(save_to) 212 | assert list(db["stdin"].rows) == [ 213 | {"id": 1, "name": "Cleo"}, 214 | {"id": 2, "name": "Bants"}, 215 | ] 216 | 217 | 218 | @pytest.mark.parametrize("option", ("-n", "--no-detect-types")) 219 | def test_memory_no_detect_types(option): 220 | result = CliRunner().invoke( 221 | cli.cli, 222 | ["memory", "-", "select * from stdin"] + [option], 223 | input="id,name,weight\n1,Cleo,45.5\n2,Bants,3.5", 224 | ) 225 | assert result.exit_code == 0, result.output 226 | assert json.loads(result.output.strip()) == [ 227 | {"id": "1", "name": "Cleo", "weight": "45.5"}, 228 | {"id": "2", "name": "Bants", "weight": "3.5"}, 229 | ] 230 | 231 | 232 | def test_memory_flatten(): 233 | result = CliRunner().invoke( 234 | cli.cli, 235 | ["memory", "-", "select * from stdin", "--flatten"], 236 | input=json.dumps( 237 | { 238 | "httpRequest": { 239 | "latency": "0.112114537s", 240 | "requestMethod": "GET", 241 | }, 242 | "insertId": "6111722f000b5b4c4d4071e2", 243 | } 244 | ), 245 | ) 246 | assert result.exit_code == 0, result.output 247 | assert json.loads(result.output.strip()) == [ 248 | { 249 | "httpRequest_latency": "0.112114537s", 250 | "httpRequest_requestMethod": "GET", 251 | "insertId": "6111722f000b5b4c4d4071e2", 252 | } 253 | ] 254 | 255 | 256 | def test_memory_analyze(): 257 | result = CliRunner().invoke( 258 | cli.cli, 259 | ["memory", "-", "--analyze"], 260 | input="id,name\n1,Cleo\n2,Bants", 261 | ) 262 | assert result.exit_code == 0 263 | assert result.output == ( 264 | "stdin.id: (1/2)\n\n" 265 | " Total rows: 2\n" 266 | " Null rows: 0\n" 267 | " Blank rows: 0\n\n" 268 | " Distinct values: 2\n\n" 269 | "stdin.name: (2/2)\n\n" 270 | " Total rows: 2\n" 271 | " Null rows: 0\n" 272 | " Blank rows: 0\n\n" 273 | " Distinct values: 2\n\n" 274 | ) 275 | 276 | 277 | def test_memory_two_files_with_same_stem(tmpdir): 278 | (tmpdir / "one").mkdir() 279 | (tmpdir / "two").mkdir() 280 | one = tmpdir / "one" / "data.csv" 281 | two = tmpdir / "two" / "data.csv" 282 | one.write_text("id,name\n1,Cleo\n2,Bants", encoding="utf-8") 283 | two.write_text("id,name\n3,Blue\n4,Lila", encoding="utf-8") 284 | result = CliRunner().invoke(cli.cli, ["memory", str(one), str(two), "", "--schema"]) 285 | assert result.exit_code == 0 286 | assert result.output == ( 287 | 'CREATE TABLE "data" (\n' 288 | " [id] INTEGER,\n" 289 | " [name] TEXT\n" 290 | ");\n" 291 | "CREATE VIEW t1 AS select * from [data];\n" 292 | "CREATE VIEW t AS select * from [data];\n" 293 | 'CREATE TABLE "data_2" (\n' 294 | " [id] INTEGER,\n" 295 | " [name] TEXT\n" 296 | ");\n" 297 | "CREATE VIEW t2 AS select * from [data_2];\n" 298 | ) 299 | 300 | 301 | def test_memory_functions(): 302 | result = CliRunner().invoke( 303 | cli.cli, 304 | ["memory", "select hello()", "--functions", "hello = lambda: 'Hello'"], 305 | ) 306 | assert result.exit_code == 0 307 | assert result.output.strip() == '[{"hello()": "Hello"}]' 308 | 309 | 310 | def test_memory_return_db(tmpdir): 311 | # https://github.com/simonw/sqlite-utils/issues/643 312 | from sqlite_utils.cli import cli 313 | 314 | path = str(tmpdir / "dogs.csv") 315 | open(path, "w").write("id,name\n1,Cleo") 316 | 317 | with click.Context(cli) as ctx: 318 | db = ctx.invoke(cli.commands["memory"], paths=(path,), return_db=True) 319 | 320 | assert db.table_names() == ["dogs"] 321 | -------------------------------------------------------------------------------- /tests/test_column_affinity.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sqlite_utils.utils import column_affinity 3 | 4 | EXAMPLES = [ 5 | # Examples from https://www.sqlite.org/datatype3.html#affinity_name_examples 6 | ("INT", int), 7 | ("INTEGER", int), 8 | ("TINYINT", int), 9 | ("SMALLINT", int), 10 | ("MEDIUMINT", int), 11 | ("BIGINT", int), 12 | ("UNSIGNED BIG INT", int), 13 | ("INT2", int), 14 | ("INT8", int), 15 | ("CHARACTER(20)", str), 16 | ("VARCHAR(255)", str), 17 | ("VARYING CHARACTER(255)", str), 18 | ("NCHAR(55)", str), 19 | ("NATIVE CHARACTER(70)", str), 20 | ("NVARCHAR(100)", str), 21 | ("TEXT", str), 22 | ("CLOB", str), 23 | ("BLOB", bytes), 24 | ("REAL", float), 25 | ("DOUBLE", float), 26 | ("DOUBLE PRECISION", float), 27 | ("FLOAT", float), 28 | # Numeric, treated as float: 29 | ("NUMERIC", float), 30 | ("DECIMAL(10,5)", float), 31 | ("BOOLEAN", float), 32 | ("DATE", float), 33 | ("DATETIME", float), 34 | ] 35 | 36 | 37 | @pytest.mark.parametrize("column_def,expected_type", EXAMPLES) 38 | def test_column_affinity(column_def, expected_type): 39 | assert expected_type is column_affinity(column_def) 40 | 41 | 42 | @pytest.mark.parametrize("column_def,expected_type", EXAMPLES) 43 | def test_columns_dict(fresh_db, column_def, expected_type): 44 | fresh_db.execute("create table foo (col {})".format(column_def)) 45 | assert {"col": expected_type} == fresh_db["foo"].columns_dict 46 | -------------------------------------------------------------------------------- /tests/test_constructor.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils import Database 2 | from sqlite_utils.utils import sqlite3 3 | import pytest 4 | 5 | 6 | def test_recursive_triggers(): 7 | db = Database(memory=True) 8 | assert db.execute("PRAGMA recursive_triggers").fetchone()[0] 9 | 10 | 11 | def test_recursive_triggers_off(): 12 | db = Database(memory=True, recursive_triggers=False) 13 | assert not db.execute("PRAGMA recursive_triggers").fetchone()[0] 14 | 15 | 16 | def test_memory_name(): 17 | db1 = Database(memory_name="shared") 18 | db2 = Database(memory_name="shared") 19 | db1["dogs"].insert({"name": "Cleo"}) 20 | assert list(db2["dogs"].rows) == [{"name": "Cleo"}] 21 | 22 | 23 | def test_sqlite_version(): 24 | db = Database(memory=True) 25 | version = db.sqlite_version 26 | assert isinstance(version, tuple) 27 | as_string = ".".join(map(str, version)) 28 | actual = next(db.query("select sqlite_version() as v"))["v"] 29 | assert actual == as_string 30 | 31 | 32 | @pytest.mark.parametrize("memory", [True, False]) 33 | def test_database_close(tmpdir, memory): 34 | if memory: 35 | db = Database(memory=True) 36 | else: 37 | db = Database(str(tmpdir / "test.db")) 38 | assert db.execute("select 1 + 1").fetchone()[0] == 2 39 | db.close() 40 | with pytest.raises(sqlite3.ProgrammingError): 41 | db.execute("select 1 + 1") 42 | -------------------------------------------------------------------------------- /tests/test_conversions.py: -------------------------------------------------------------------------------- 1 | def test_insert_conversion(fresh_db): 2 | table = fresh_db["table"] 3 | table.insert({"foo": "bar"}, conversions={"foo": "upper(?)"}) 4 | assert [{"foo": "BAR"}] == list(table.rows) 5 | 6 | 7 | def test_insert_all_conversion(fresh_db): 8 | table = fresh_db["table"] 9 | table.insert_all([{"foo": "bar"}], conversions={"foo": "upper(?)"}) 10 | assert [{"foo": "BAR"}] == list(table.rows) 11 | 12 | 13 | def test_upsert_conversion(fresh_db): 14 | table = fresh_db["table"] 15 | table.upsert({"id": 1, "foo": "bar"}, pk="id", conversions={"foo": "upper(?)"}) 16 | assert [{"id": 1, "foo": "BAR"}] == list(table.rows) 17 | table.upsert( 18 | {"id": 1, "bar": "baz"}, pk="id", conversions={"bar": "upper(?)"}, alter=True 19 | ) 20 | assert [{"id": 1, "foo": "BAR", "bar": "BAZ"}] == list(table.rows) 21 | 22 | 23 | def test_upsert_all_conversion(fresh_db): 24 | table = fresh_db["table"] 25 | table.upsert_all( 26 | [{"id": 1, "foo": "bar"}], pk="id", conversions={"foo": "upper(?)"} 27 | ) 28 | assert [{"id": 1, "foo": "BAR"}] == list(table.rows) 29 | 30 | 31 | def test_update_conversion(fresh_db): 32 | table = fresh_db["table"] 33 | table.insert({"id": 5, "foo": "bar"}, pk="id") 34 | table.update(5, {"foo": "baz"}, conversions={"foo": "upper(?)"}) 35 | assert [{"id": 5, "foo": "BAZ"}] == list(table.rows) 36 | 37 | 38 | def test_table_constructor_conversion(fresh_db): 39 | table = fresh_db.table("table", conversions={"bar": "upper(?)"}) 40 | table.insert({"bar": "baz"}) 41 | assert [{"bar": "BAZ"}] == list(table.rows) 42 | -------------------------------------------------------------------------------- /tests/test_convert.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils.db import BadMultiValues 2 | import pytest 3 | 4 | 5 | @pytest.mark.parametrize( 6 | "columns,fn,expected", 7 | ( 8 | ( 9 | "title", 10 | lambda value: value.upper(), 11 | {"title": "MIXED CASE", "abstract": "Abstract"}, 12 | ), 13 | ( 14 | ["title", "abstract"], 15 | lambda value: value.upper(), 16 | {"title": "MIXED CASE", "abstract": "ABSTRACT"}, 17 | ), 18 | ( 19 | "title", 20 | lambda value: {"upper": value.upper(), "lower": value.lower()}, 21 | { 22 | "title": '{"upper": "MIXED CASE", "lower": "mixed case"}', 23 | "abstract": "Abstract", 24 | }, 25 | ), 26 | ), 27 | ) 28 | def test_convert(fresh_db, columns, fn, expected): 29 | table = fresh_db["table"] 30 | table.insert({"title": "Mixed Case", "abstract": "Abstract"}) 31 | table.convert(columns, fn) 32 | assert list(table.rows) == [expected] 33 | 34 | 35 | @pytest.mark.parametrize( 36 | "where,where_args", (("id > 1", None), ("id > :id", {"id": 1}), ("id > ?", [1])) 37 | ) 38 | def test_convert_where(fresh_db, where, where_args): 39 | table = fresh_db["table"] 40 | table.insert_all( 41 | [ 42 | {"id": 1, "title": "One"}, 43 | {"id": 2, "title": "Two"}, 44 | ], 45 | pk="id", 46 | ) 47 | table.convert( 48 | "title", lambda value: value.upper(), where=where, where_args=where_args 49 | ) 50 | assert list(table.rows) == [{"id": 1, "title": "One"}, {"id": 2, "title": "TWO"}] 51 | 52 | 53 | def test_convert_skip_false(fresh_db): 54 | table = fresh_db["table"] 55 | table.insert_all([{"x": 0}, {"x": 1}]) 56 | assert table.get(1)["x"] == 0 57 | assert table.get(2)["x"] == 1 58 | table.convert("x", lambda x: x + 1, skip_false=False) 59 | assert table.get(1)["x"] == 1 60 | assert table.get(2)["x"] == 2 61 | 62 | 63 | @pytest.mark.parametrize( 64 | "drop,expected", 65 | ( 66 | (False, {"title": "Mixed Case", "other": "MIXED CASE"}), 67 | (True, {"other": "MIXED CASE"}), 68 | ), 69 | ) 70 | def test_convert_output(fresh_db, drop, expected): 71 | table = fresh_db["table"] 72 | table.insert({"title": "Mixed Case"}) 73 | table.convert("title", lambda v: v.upper(), output="other", drop=drop) 74 | assert list(table.rows) == [expected] 75 | 76 | 77 | def test_convert_output_multiple_column_error(fresh_db): 78 | table = fresh_db["table"] 79 | with pytest.raises(AssertionError) as excinfo: 80 | table.convert(["title", "other"], lambda v: v, output="out") 81 | assert "output= can only be used with a single column" in str(excinfo.value) 82 | 83 | 84 | @pytest.mark.parametrize( 85 | "type,expected", 86 | ( 87 | (int, {"other": 123}), 88 | (float, {"other": 123.0}), 89 | ), 90 | ) 91 | def test_convert_output_type(fresh_db, type, expected): 92 | table = fresh_db["table"] 93 | table.insert({"number": "123"}) 94 | table.convert("number", lambda v: v, output="other", output_type=type, drop=True) 95 | assert list(table.rows) == [expected] 96 | 97 | 98 | def test_convert_multi(fresh_db): 99 | table = fresh_db["table"] 100 | table.insert({"title": "Mixed Case"}) 101 | table.convert( 102 | "title", 103 | lambda v: { 104 | "upper": v.upper(), 105 | "lower": v.lower(), 106 | "both": { 107 | "upper": v.upper(), 108 | "lower": v.lower(), 109 | }, 110 | }, 111 | multi=True, 112 | ) 113 | assert list(table.rows) == [ 114 | { 115 | "title": "Mixed Case", 116 | "upper": "MIXED CASE", 117 | "lower": "mixed case", 118 | "both": '{"upper": "MIXED CASE", "lower": "mixed case"}', 119 | } 120 | ] 121 | 122 | 123 | def test_convert_multi_where(fresh_db): 124 | table = fresh_db["table"] 125 | table.insert_all( 126 | [ 127 | {"id": 1, "title": "One"}, 128 | {"id": 2, "title": "Two"}, 129 | ], 130 | pk="id", 131 | ) 132 | table.convert( 133 | "title", 134 | lambda v: {"upper": v.upper(), "lower": v.lower()}, 135 | multi=True, 136 | where="id > ?", 137 | where_args=[1], 138 | ) 139 | assert list(table.rows) == [ 140 | {"id": 1, "lower": None, "title": "One", "upper": None}, 141 | {"id": 2, "lower": "two", "title": "Two", "upper": "TWO"}, 142 | ] 143 | 144 | 145 | def test_convert_multi_exception(fresh_db): 146 | table = fresh_db["table"] 147 | table.insert({"title": "Mixed Case"}) 148 | with pytest.raises(BadMultiValues): 149 | table.convert("title", lambda v: v.upper(), multi=True) 150 | 151 | 152 | def test_convert_repeated(fresh_db): 153 | table = fresh_db["table"] 154 | col = "num" 155 | table.insert({col: 1}) 156 | table.convert(col, lambda x: x * 2) 157 | table.convert(col, lambda _x: 0) 158 | assert table.get(1) == {col: 0} 159 | -------------------------------------------------------------------------------- /tests/test_create_view.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sqlite_utils.utils import OperationalError 3 | 4 | 5 | def test_create_view(fresh_db): 6 | fresh_db.create_view("bar", "select 1 + 1") 7 | rows = fresh_db.execute("select * from bar").fetchall() 8 | assert [(2,)] == rows 9 | 10 | 11 | def test_create_view_error(fresh_db): 12 | fresh_db.create_view("bar", "select 1 + 1") 13 | with pytest.raises(OperationalError): 14 | fresh_db.create_view("bar", "select 1 + 2") 15 | 16 | 17 | def test_create_view_only_arrow_one_param(fresh_db): 18 | with pytest.raises(AssertionError): 19 | fresh_db.create_view("bar", "select 1 + 2", ignore=True, replace=True) 20 | 21 | 22 | def test_create_view_ignore(fresh_db): 23 | fresh_db.create_view("bar", "select 1 + 1").create_view( 24 | "bar", "select 1 + 2", ignore=True 25 | ) 26 | rows = fresh_db.execute("select * from bar").fetchall() 27 | assert [(2,)] == rows 28 | 29 | 30 | def test_create_view_replace(fresh_db): 31 | fresh_db.create_view("bar", "select 1 + 1").create_view( 32 | "bar", "select 1 + 2", replace=True 33 | ) 34 | rows = fresh_db.execute("select * from bar").fetchall() 35 | assert [(3,)] == rows 36 | 37 | 38 | def test_create_view_replace_with_same_does_nothing(fresh_db): 39 | fresh_db.create_view("bar", "select 1 + 1") 40 | initial_version = fresh_db.execute("PRAGMA schema_version").fetchone()[0] 41 | fresh_db.create_view("bar", "select 1 + 1", replace=True) 42 | after_version = fresh_db.execute("PRAGMA schema_version").fetchone()[0] 43 | assert after_version == initial_version 44 | -------------------------------------------------------------------------------- /tests/test_default_value.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | EXAMPLES = [ 5 | ("TEXT DEFAULT 'foo'", "'foo'", "'foo'"), 6 | ("TEXT DEFAULT 'foo)'", "'foo)'", "'foo)'"), 7 | ("INTEGER DEFAULT '1'", "'1'", "'1'"), 8 | ("INTEGER DEFAULT 1", "1", "'1'"), 9 | ("INTEGER DEFAULT (1)", "1", "'1'"), 10 | # Expressions 11 | ( 12 | "TEXT DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW'))", 13 | "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", 14 | "(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW'))", 15 | ), 16 | # Special values 17 | ("TEXT DEFAULT CURRENT_TIME", "CURRENT_TIME", "CURRENT_TIME"), 18 | ("TEXT DEFAULT CURRENT_DATE", "CURRENT_DATE", "CURRENT_DATE"), 19 | ("TEXT DEFAULT CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP"), 20 | ("TEXT DEFAULT current_timestamp", "current_timestamp", "current_timestamp"), 21 | ("TEXT DEFAULT (CURRENT_TIMESTAMP)", "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP"), 22 | # Strings 23 | ("TEXT DEFAULT 'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'", "'CURRENT_TIMESTAMP'"), 24 | ('TEXT DEFAULT "CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"', '"CURRENT_TIMESTAMP"'), 25 | ] 26 | 27 | 28 | @pytest.mark.parametrize("column_def,initial_value,expected_value", EXAMPLES) 29 | def test_quote_default_value(fresh_db, column_def, initial_value, expected_value): 30 | fresh_db.execute("create table foo (col {})".format(column_def)) 31 | assert initial_value == fresh_db["foo"].columns[0].default_value 32 | assert expected_value == fresh_db.quote_default_value( 33 | fresh_db["foo"].columns[0].default_value 34 | ) 35 | -------------------------------------------------------------------------------- /tests/test_delete.py: -------------------------------------------------------------------------------- 1 | def test_delete_rowid_table(fresh_db): 2 | table = fresh_db["table"] 3 | table.insert({"foo": 1}).last_pk 4 | rowid = table.insert({"foo": 2}).last_pk 5 | table.delete(rowid) 6 | assert [{"foo": 1}] == list(table.rows) 7 | 8 | 9 | def test_delete_pk_table(fresh_db): 10 | table = fresh_db["table"] 11 | table.insert({"id": 1}, pk="id") 12 | table.insert({"id": 2}, pk="id") 13 | table.delete(1) 14 | assert [{"id": 2}] == list(table.rows) 15 | 16 | 17 | def test_delete_where(fresh_db): 18 | table = fresh_db["table"] 19 | for i in range(1, 11): 20 | table.insert({"id": i}, pk="id") 21 | assert table.count == 10 22 | table.delete_where("id > ?", [5]) 23 | assert table.count == 5 24 | 25 | 26 | def test_delete_where_all(fresh_db): 27 | table = fresh_db["table"] 28 | for i in range(1, 11): 29 | table.insert({"id": i}, pk="id") 30 | assert table.count == 10 31 | table.delete_where() 32 | assert table.count == 0 33 | 34 | 35 | def test_delete_where_analyze(fresh_db): 36 | table = fresh_db["table"] 37 | table.insert_all(({"id": i, "i": i} for i in range(10)), pk="id") 38 | table.create_index(["i"], analyze=True) 39 | assert "sqlite_stat1" in fresh_db.table_names() 40 | assert list(fresh_db["sqlite_stat1"].rows) == [ 41 | {"tbl": "table", "idx": "idx_table_i", "stat": "10 1"} 42 | ] 43 | table.delete_where("id > ?", [5], analyze=True) 44 | assert list(fresh_db["sqlite_stat1"].rows) == [ 45 | {"tbl": "table", "idx": "idx_table_i", "stat": "6 1"} 46 | ] 47 | -------------------------------------------------------------------------------- /tests/test_docs.py: -------------------------------------------------------------------------------- 1 | from click.testing import CliRunner 2 | from sqlite_utils import cli, recipes 3 | from pathlib import Path 4 | import pytest 5 | import re 6 | 7 | docs_path = Path(__file__).parent.parent / "docs" 8 | commands_re = re.compile(r"(?:\$ | )sqlite-utils (\S+)") 9 | recipes_re = re.compile(r"r\.(\w+)\(") 10 | 11 | 12 | @pytest.fixture(scope="session") 13 | def documented_commands(): 14 | rst = "" 15 | for doc in ("cli.rst", "plugins.rst"): 16 | rst += (docs_path / doc).read_text() 17 | return { 18 | command 19 | for command in commands_re.findall(rst) 20 | if "." not in command and ":" not in command 21 | } 22 | 23 | 24 | @pytest.fixture(scope="session") 25 | def documented_recipes(): 26 | rst = (docs_path / "cli.rst").read_text() 27 | return set(recipes_re.findall(rst)) 28 | 29 | 30 | @pytest.mark.parametrize("command", cli.cli.commands.keys()) 31 | def test_commands_are_documented(documented_commands, command): 32 | assert command in documented_commands 33 | 34 | 35 | @pytest.mark.parametrize("command", cli.cli.commands.values()) 36 | def test_commands_have_help(command): 37 | assert command.help, "{} is missing its help".format(command) 38 | 39 | 40 | def test_convert_help(): 41 | result = CliRunner().invoke(cli.cli, ["convert", "--help"]) 42 | assert result.exit_code == 0 43 | for expected in ( 44 | "r.jsonsplit(value, ", 45 | "r.parsedate(value, ", 46 | "r.parsedatetime(value, ", 47 | ): 48 | assert expected in result.output 49 | 50 | 51 | @pytest.mark.parametrize( 52 | "recipe", 53 | [ 54 | n 55 | for n in dir(recipes) 56 | if not n.startswith("_") 57 | and n not in ("json", "parser") 58 | and callable(getattr(recipes, n)) 59 | ], 60 | ) 61 | def test_recipes_are_documented(documented_recipes, recipe): 62 | assert recipe in documented_recipes 63 | -------------------------------------------------------------------------------- /tests/test_duplicate.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils.db import NoTable 2 | import datetime 3 | import pytest 4 | 5 | 6 | def test_duplicate(fresh_db): 7 | # Create table using native Sqlite statement: 8 | fresh_db.execute( 9 | """CREATE TABLE [table1] ( 10 | [text_col] TEXT, 11 | [real_col] REAL, 12 | [int_col] INTEGER, 13 | [bool_col] INTEGER, 14 | [datetime_col] TEXT)""" 15 | ) 16 | # Insert one row of mock data: 17 | dt = datetime.datetime.now() 18 | data = { 19 | "text_col": "Cleo", 20 | "real_col": 3.14, 21 | "int_col": -255, 22 | "bool_col": True, 23 | "datetime_col": str(dt), 24 | } 25 | table1 = fresh_db["table1"] 26 | row_id = table1.insert(data).last_rowid 27 | # Duplicate table: 28 | table2 = table1.duplicate("table2") 29 | # Ensure data integrity: 30 | assert data == table2.get(row_id) 31 | # Ensure schema integrity: 32 | assert [ 33 | {"name": "text_col", "type": "TEXT"}, 34 | {"name": "real_col", "type": "REAL"}, 35 | {"name": "int_col", "type": "INT"}, 36 | {"name": "bool_col", "type": "INT"}, 37 | {"name": "datetime_col", "type": "TEXT"}, 38 | ] == [{"name": col.name, "type": col.type} for col in table2.columns] 39 | 40 | 41 | def test_duplicate_fails_if_table_does_not_exist(fresh_db): 42 | with pytest.raises(NoTable): 43 | fresh_db["not_a_table"].duplicate("duplicated") 44 | -------------------------------------------------------------------------------- /tests/test_enable_counts.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils import Database 2 | from sqlite_utils import cli 3 | from click.testing import CliRunner 4 | import pytest 5 | 6 | 7 | def test_enable_counts_specific_table(fresh_db): 8 | foo = fresh_db["foo"] 9 | assert fresh_db.table_names() == [] 10 | for i in range(10): 11 | foo.insert({"name": "item {}".format(i)}) 12 | assert fresh_db.table_names() == ["foo"] 13 | assert foo.count == 10 14 | # Now enable counts 15 | foo.enable_counts() 16 | assert foo.triggers_dict == { 17 | "foo_counts_insert": ( 18 | "CREATE TRIGGER [foo_counts_insert] AFTER INSERT ON [foo]\n" 19 | "BEGIN\n" 20 | " INSERT OR REPLACE INTO [_counts]\n" 21 | " VALUES (\n 'foo',\n" 22 | " COALESCE(\n" 23 | " (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n" 24 | " 0\n" 25 | " ) + 1\n" 26 | " );\n" 27 | "END" 28 | ), 29 | "foo_counts_delete": ( 30 | "CREATE TRIGGER [foo_counts_delete] AFTER DELETE ON [foo]\n" 31 | "BEGIN\n" 32 | " INSERT OR REPLACE INTO [_counts]\n" 33 | " VALUES (\n" 34 | " 'foo',\n" 35 | " COALESCE(\n" 36 | " (SELECT count FROM [_counts] WHERE [table] = 'foo'),\n" 37 | " 0\n" 38 | " ) - 1\n" 39 | " );\n" 40 | "END" 41 | ), 42 | } 43 | assert fresh_db.table_names() == ["foo", "_counts"] 44 | assert list(fresh_db["_counts"].rows) == [{"count": 10, "table": "foo"}] 45 | # Add some items to test the triggers 46 | for i in range(5): 47 | foo.insert({"name": "item {}".format(10 + i)}) 48 | assert foo.count == 15 49 | assert list(fresh_db["_counts"].rows) == [{"count": 15, "table": "foo"}] 50 | # Delete some items 51 | foo.delete_where("rowid < 7") 52 | assert foo.count == 9 53 | assert list(fresh_db["_counts"].rows) == [{"count": 9, "table": "foo"}] 54 | foo.delete_where() 55 | assert foo.count == 0 56 | assert list(fresh_db["_counts"].rows) == [{"count": 0, "table": "foo"}] 57 | 58 | 59 | def test_enable_counts_all_tables(fresh_db): 60 | foo = fresh_db["foo"] 61 | bar = fresh_db["bar"] 62 | foo.insert({"name": "Cleo"}) 63 | bar.insert({"name": "Cleo"}) 64 | foo.enable_fts(["name"]) 65 | fresh_db.enable_counts() 66 | assert set(fresh_db.table_names()) == { 67 | "foo", 68 | "bar", 69 | "foo_fts", 70 | "foo_fts_data", 71 | "foo_fts_idx", 72 | "foo_fts_docsize", 73 | "foo_fts_config", 74 | "_counts", 75 | } 76 | assert list(fresh_db["_counts"].rows) == [ 77 | {"count": 1, "table": "foo"}, 78 | {"count": 1, "table": "bar"}, 79 | {"count": 3, "table": "foo_fts_data"}, 80 | {"count": 1, "table": "foo_fts_idx"}, 81 | {"count": 1, "table": "foo_fts_docsize"}, 82 | {"count": 1, "table": "foo_fts_config"}, 83 | ] 84 | 85 | 86 | @pytest.fixture 87 | def counts_db_path(tmpdir): 88 | path = str(tmpdir / "test.db") 89 | db = Database(path) 90 | db["foo"].insert({"name": "bar"}) 91 | db["bar"].insert({"name": "bar"}) 92 | db["bar"].insert({"name": "bar"}) 93 | db["baz"].insert({"name": "bar"}) 94 | return path 95 | 96 | 97 | @pytest.mark.parametrize( 98 | "extra_args,expected_triggers", 99 | [ 100 | ( 101 | [], 102 | [ 103 | "foo_counts_insert", 104 | "foo_counts_delete", 105 | "bar_counts_insert", 106 | "bar_counts_delete", 107 | "baz_counts_insert", 108 | "baz_counts_delete", 109 | ], 110 | ), 111 | ( 112 | ["bar"], 113 | [ 114 | "bar_counts_insert", 115 | "bar_counts_delete", 116 | ], 117 | ), 118 | ], 119 | ) 120 | def test_cli_enable_counts(counts_db_path, extra_args, expected_triggers): 121 | db = Database(counts_db_path) 122 | assert list(db.triggers_dict.keys()) == [] 123 | result = CliRunner().invoke(cli.cli, ["enable-counts", counts_db_path] + extra_args) 124 | assert result.exit_code == 0 125 | assert list(db.triggers_dict.keys()) == expected_triggers 126 | 127 | 128 | def test_uses_counts_after_enable_counts(counts_db_path): 129 | db = Database(counts_db_path) 130 | logged = [] 131 | with db.tracer(lambda sql, parameters: logged.append((sql, parameters))): 132 | assert db.table("foo").count == 1 133 | assert logged == [ 134 | ("select name from sqlite_master where type = 'view'", None), 135 | ("select count(*) from [foo]", []), 136 | ] 137 | logged.clear() 138 | assert not db.use_counts_table 139 | db.enable_counts() 140 | assert db.use_counts_table 141 | assert db.table("foo").count == 1 142 | assert logged == [ 143 | ( 144 | "CREATE TABLE IF NOT EXISTS [_counts](\n [table] TEXT PRIMARY KEY,\n count INTEGER DEFAULT 0\n);", 145 | None, 146 | ), 147 | ("select name from sqlite_master where type = 'table'", None), 148 | ("select name from sqlite_master where type = 'view'", None), 149 | ("select name from sqlite_master where type = 'view'", None), 150 | ("select name from sqlite_master where type = 'view'", None), 151 | ("select name from sqlite_master where type = 'view'", None), 152 | ("select sql from sqlite_master where name = ?", ("foo",)), 153 | ("SELECT quote(:value)", {"value": "foo"}), 154 | ("select sql from sqlite_master where name = ?", ("bar",)), 155 | ("SELECT quote(:value)", {"value": "bar"}), 156 | ("select sql from sqlite_master where name = ?", ("baz",)), 157 | ("SELECT quote(:value)", {"value": "baz"}), 158 | ("select sql from sqlite_master where name = ?", ("_counts",)), 159 | ("select name from sqlite_master where type = 'view'", None), 160 | ("select [table], count from _counts where [table] in (?)", ["foo"]), 161 | ] 162 | 163 | 164 | def test_reset_counts(counts_db_path): 165 | db = Database(counts_db_path) 166 | db["foo"].enable_counts() 167 | db["bar"].enable_counts() 168 | assert db.cached_counts() == {"foo": 1, "bar": 2} 169 | # Corrupt the value 170 | db["_counts"].update("foo", {"count": 3}) 171 | assert db.cached_counts() == {"foo": 3, "bar": 2} 172 | assert db["foo"].count == 3 173 | # Reset them 174 | db.reset_counts() 175 | assert db.cached_counts() == {"foo": 1, "bar": 2} 176 | assert db["foo"].count == 1 177 | 178 | 179 | def test_reset_counts_cli(counts_db_path): 180 | db = Database(counts_db_path) 181 | db["foo"].enable_counts() 182 | db["bar"].enable_counts() 183 | assert db.cached_counts() == {"foo": 1, "bar": 2} 184 | db["_counts"].update("foo", {"count": 3}) 185 | result = CliRunner().invoke(cli.cli, ["reset-counts", counts_db_path]) 186 | assert result.exit_code == 0 187 | assert db.cached_counts() == {"foo": 1, "bar": 2} 188 | -------------------------------------------------------------------------------- /tests/test_extract.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils.db import InvalidColumns 2 | import itertools 3 | import pytest 4 | 5 | 6 | @pytest.mark.parametrize("table", [None, "Species"]) 7 | @pytest.mark.parametrize("fk_column", [None, "species"]) 8 | def test_extract_single_column(fresh_db, table, fk_column): 9 | expected_table = table or "species" 10 | expected_fk = fk_column or "{}_id".format(expected_table) 11 | iter_species = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) 12 | fresh_db["tree"].insert_all( 13 | ( 14 | { 15 | "id": i, 16 | "name": "Tree {}".format(i), 17 | "species": next(iter_species), 18 | "end": 1, 19 | } 20 | for i in range(1, 1001) 21 | ), 22 | pk="id", 23 | ) 24 | fresh_db["tree"].extract("species", table=table, fk_column=fk_column) 25 | assert fresh_db["tree"].schema == ( 26 | 'CREATE TABLE "tree" (\n' 27 | " [id] INTEGER PRIMARY KEY,\n" 28 | " [name] TEXT,\n" 29 | " [{}] INTEGER REFERENCES [{}]([id]),\n".format(expected_fk, expected_table) 30 | + " [end] INTEGER\n" 31 | + ")" 32 | ) 33 | assert fresh_db[expected_table].schema == ( 34 | "CREATE TABLE [{}] (\n".format(expected_table) 35 | + " [id] INTEGER PRIMARY KEY,\n" 36 | " [species] TEXT\n" 37 | ")" 38 | ) 39 | assert list(fresh_db[expected_table].rows) == [ 40 | {"id": 1, "species": "Palm"}, 41 | {"id": 2, "species": "Spruce"}, 42 | {"id": 3, "species": "Mangrove"}, 43 | {"id": 4, "species": "Oak"}, 44 | ] 45 | assert list(itertools.islice(fresh_db["tree"].rows, 0, 4)) == [ 46 | {"id": 1, "name": "Tree 1", expected_fk: 1, "end": 1}, 47 | {"id": 2, "name": "Tree 2", expected_fk: 2, "end": 1}, 48 | {"id": 3, "name": "Tree 3", expected_fk: 3, "end": 1}, 49 | {"id": 4, "name": "Tree 4", expected_fk: 4, "end": 1}, 50 | ] 51 | 52 | 53 | def test_extract_multiple_columns_with_rename(fresh_db): 54 | iter_common = itertools.cycle(["Palm", "Spruce", "Mangrove", "Oak"]) 55 | iter_latin = itertools.cycle(["Arecaceae", "Picea", "Rhizophora", "Quercus"]) 56 | fresh_db["tree"].insert_all( 57 | ( 58 | { 59 | "id": i, 60 | "name": "Tree {}".format(i), 61 | "common_name": next(iter_common), 62 | "latin_name": next(iter_latin), 63 | } 64 | for i in range(1, 1001) 65 | ), 66 | pk="id", 67 | ) 68 | 69 | fresh_db["tree"].extract( 70 | ["common_name", "latin_name"], rename={"common_name": "name"} 71 | ) 72 | assert fresh_db["tree"].schema == ( 73 | 'CREATE TABLE "tree" (\n' 74 | " [id] INTEGER PRIMARY KEY,\n" 75 | " [name] TEXT,\n" 76 | " [common_name_latin_name_id] INTEGER REFERENCES [common_name_latin_name]([id])\n" 77 | ")" 78 | ) 79 | assert fresh_db["common_name_latin_name"].schema == ( 80 | "CREATE TABLE [common_name_latin_name] (\n" 81 | " [id] INTEGER PRIMARY KEY,\n" 82 | " [name] TEXT,\n" 83 | " [latin_name] TEXT\n" 84 | ")" 85 | ) 86 | assert list(fresh_db["common_name_latin_name"].rows) == [ 87 | {"name": "Palm", "id": 1, "latin_name": "Arecaceae"}, 88 | {"name": "Spruce", "id": 2, "latin_name": "Picea"}, 89 | {"name": "Mangrove", "id": 3, "latin_name": "Rhizophora"}, 90 | {"name": "Oak", "id": 4, "latin_name": "Quercus"}, 91 | ] 92 | assert list(itertools.islice(fresh_db["tree"].rows, 0, 4)) == [ 93 | {"id": 1, "name": "Tree 1", "common_name_latin_name_id": 1}, 94 | {"id": 2, "name": "Tree 2", "common_name_latin_name_id": 2}, 95 | {"id": 3, "name": "Tree 3", "common_name_latin_name_id": 3}, 96 | {"id": 4, "name": "Tree 4", "common_name_latin_name_id": 4}, 97 | ] 98 | 99 | 100 | def test_extract_invalid_columns(fresh_db): 101 | fresh_db["tree"].insert( 102 | { 103 | "id": 1, 104 | "name": "Tree 1", 105 | "common_name": "Palm", 106 | "latin_name": "Arecaceae", 107 | }, 108 | pk="id", 109 | ) 110 | with pytest.raises(InvalidColumns): 111 | fresh_db["tree"].extract(["bad_column"]) 112 | 113 | 114 | def test_extract_rowid_table(fresh_db): 115 | fresh_db["tree"].insert( 116 | { 117 | "name": "Tree 1", 118 | "common_name": "Palm", 119 | "latin_name": "Arecaceae", 120 | } 121 | ) 122 | fresh_db["tree"].extract(["common_name", "latin_name"]) 123 | assert fresh_db["tree"].schema == ( 124 | 'CREATE TABLE "tree" (\n' 125 | " [name] TEXT,\n" 126 | " [common_name_latin_name_id] INTEGER REFERENCES [common_name_latin_name]([id])\n" 127 | ")" 128 | ) 129 | assert ( 130 | fresh_db.execute( 131 | """ 132 | select 133 | tree.name, 134 | common_name_latin_name.common_name, 135 | common_name_latin_name.latin_name 136 | from tree 137 | join common_name_latin_name 138 | on tree.common_name_latin_name_id = common_name_latin_name.id 139 | """ 140 | ).fetchall() 141 | == [("Tree 1", "Palm", "Arecaceae")] 142 | ) 143 | 144 | 145 | def test_reuse_lookup_table(fresh_db): 146 | fresh_db["species"].insert({"id": 1, "name": "Wolf"}, pk="id") 147 | fresh_db["sightings"].insert({"id": 10, "species": "Wolf"}, pk="id") 148 | fresh_db["individuals"].insert( 149 | {"id": 10, "name": "Terriana", "species": "Fox"}, pk="id" 150 | ) 151 | fresh_db["sightings"].extract("species", rename={"species": "name"}) 152 | fresh_db["individuals"].extract("species", rename={"species": "name"}) 153 | assert fresh_db["sightings"].schema == ( 154 | 'CREATE TABLE "sightings" (\n' 155 | " [id] INTEGER PRIMARY KEY,\n" 156 | " [species_id] INTEGER REFERENCES [species]([id])\n" 157 | ")" 158 | ) 159 | assert fresh_db["individuals"].schema == ( 160 | 'CREATE TABLE "individuals" (\n' 161 | " [id] INTEGER PRIMARY KEY,\n" 162 | " [name] TEXT,\n" 163 | " [species_id] INTEGER REFERENCES [species]([id])\n" 164 | ")" 165 | ) 166 | assert list(fresh_db["species"].rows) == [ 167 | {"id": 1, "name": "Wolf"}, 168 | {"id": 2, "name": "Fox"}, 169 | ] 170 | 171 | 172 | def test_extract_error_on_incompatible_existing_lookup_table(fresh_db): 173 | fresh_db["species"].insert({"id": 1}) 174 | fresh_db["tree"].insert({"name": "Tree 1", "common_name": "Palm"}) 175 | with pytest.raises(InvalidColumns): 176 | fresh_db["tree"].extract("common_name", table="species") 177 | 178 | # Try again with incompatible existing column type 179 | fresh_db["species2"].insert({"id": 1, "common_name": 3.5}) 180 | with pytest.raises(InvalidColumns): 181 | fresh_db["tree"].extract("common_name", table="species2") 182 | 183 | 184 | def test_extract_works_with_null_values(fresh_db): 185 | fresh_db["listens"].insert_all( 186 | [ 187 | {"id": 1, "track_title": "foo", "album_title": "bar"}, 188 | {"id": 2, "track_title": "baz", "album_title": None}, 189 | ], 190 | pk="id", 191 | ) 192 | fresh_db["listens"].extract( 193 | columns=["album_title"], table="albums", fk_column="album_id" 194 | ) 195 | assert list(fresh_db["listens"].rows) == [ 196 | {"id": 1, "track_title": "foo", "album_id": 1}, 197 | {"id": 2, "track_title": "baz", "album_id": 2}, 198 | ] 199 | assert list(fresh_db["albums"].rows) == [ 200 | {"id": 1, "album_title": "bar"}, 201 | {"id": 2, "album_title": None}, 202 | ] 203 | -------------------------------------------------------------------------------- /tests/test_extracts.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils.db import Index 2 | import pytest 3 | 4 | 5 | @pytest.mark.parametrize( 6 | "kwargs,expected_table", 7 | [ 8 | (dict(extracts={"species_id": "Species"}), "Species"), 9 | (dict(extracts=["species_id"]), "species_id"), 10 | (dict(extracts=("species_id",)), "species_id"), 11 | ], 12 | ) 13 | @pytest.mark.parametrize("use_table_factory", [True, False]) 14 | def test_extracts(fresh_db, kwargs, expected_table, use_table_factory): 15 | table_kwargs = {} 16 | insert_kwargs = {} 17 | if use_table_factory: 18 | table_kwargs = kwargs 19 | else: 20 | insert_kwargs = kwargs 21 | trees = fresh_db.table("Trees", **table_kwargs) 22 | trees.insert_all( 23 | [ 24 | {"id": 1, "species_id": "Oak"}, 25 | {"id": 2, "species_id": "Oak"}, 26 | {"id": 3, "species_id": "Palm"}, 27 | ], 28 | **insert_kwargs 29 | ) 30 | # Should now have two tables: Trees and Species 31 | assert {expected_table, "Trees"} == set(fresh_db.table_names()) 32 | assert ( 33 | "CREATE TABLE [{}] (\n [id] INTEGER PRIMARY KEY,\n [value] TEXT\n)".format( 34 | expected_table 35 | ) 36 | == fresh_db[expected_table].schema 37 | ) 38 | assert ( 39 | "CREATE TABLE [Trees] (\n [id] INTEGER,\n [species_id] INTEGER REFERENCES [{}]([id])\n)".format( 40 | expected_table 41 | ) 42 | == fresh_db["Trees"].schema 43 | ) 44 | # Should have a foreign key reference 45 | assert len(fresh_db["Trees"].foreign_keys) == 1 46 | fk = fresh_db["Trees"].foreign_keys[0] 47 | assert fk.table == "Trees" 48 | assert fk.column == "species_id" 49 | 50 | # Should have unique index on Species 51 | assert [ 52 | Index( 53 | seq=0, 54 | name="idx_{}_value".format(expected_table), 55 | unique=1, 56 | origin="c", 57 | partial=0, 58 | columns=["value"], 59 | ) 60 | ] == fresh_db[expected_table].indexes 61 | # Finally, check the rows 62 | assert [{"id": 1, "value": "Oak"}, {"id": 2, "value": "Palm"}] == list( 63 | fresh_db[expected_table].rows 64 | ) 65 | assert [ 66 | {"id": 1, "species_id": 1}, 67 | {"id": 2, "species_id": 1}, 68 | {"id": 3, "species_id": 2}, 69 | ] == list(fresh_db["Trees"].rows) 70 | -------------------------------------------------------------------------------- /tests/test_get.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sqlite_utils.db import NotFoundError 3 | 4 | 5 | def test_get_rowid(fresh_db): 6 | dogs = fresh_db["dogs"] 7 | cleo = {"name": "Cleo", "age": 4} 8 | row_id = dogs.insert(cleo).last_rowid 9 | assert cleo == dogs.get(row_id) 10 | 11 | 12 | def test_get_primary_key(fresh_db): 13 | dogs = fresh_db["dogs"] 14 | cleo = {"name": "Cleo", "age": 4, "id": 5} 15 | last_pk = dogs.insert(cleo, pk="id").last_pk 16 | assert 5 == last_pk 17 | assert cleo == dogs.get(5) 18 | 19 | 20 | @pytest.mark.parametrize( 21 | "argument,expected_msg", 22 | [(100, None), (None, None), ((1, 2), "Need 1 primary key value"), ("2", None)], 23 | ) 24 | def test_get_not_found(argument, expected_msg, fresh_db): 25 | fresh_db["dogs"].insert( 26 | {"id": 1, "name": "Cleo", "age": 4, "is_good": True}, pk="id" 27 | ) 28 | with pytest.raises(NotFoundError) as excinfo: 29 | fresh_db["dogs"].get(argument) 30 | if expected_msg is not None: 31 | assert expected_msg == excinfo.value.args[0] 32 | -------------------------------------------------------------------------------- /tests/test_gis.py: -------------------------------------------------------------------------------- 1 | import json 2 | import pytest 3 | 4 | from click.testing import CliRunner 5 | from sqlite_utils.cli import cli 6 | from sqlite_utils.db import Database 7 | from sqlite_utils.utils import find_spatialite, sqlite3 8 | 9 | try: 10 | import sqlean 11 | except ImportError: 12 | sqlean = None 13 | 14 | 15 | pytestmark = [ 16 | pytest.mark.skipif( 17 | not find_spatialite(), reason="Could not find SpatiaLite extension" 18 | ), 19 | pytest.mark.skipif( 20 | not hasattr(sqlite3.Connection, "enable_load_extension"), 21 | reason="sqlite3.Connection missing enable_load_extension", 22 | ), 23 | pytest.mark.skipif( 24 | sqlean is not None, reason="sqlean.py is not compatible with SpatiaLite" 25 | ), 26 | ] 27 | 28 | 29 | # python API tests 30 | def test_find_spatialite(): 31 | spatialite = find_spatialite() 32 | assert spatialite is None or isinstance(spatialite, str) 33 | 34 | 35 | def test_init_spatialite(): 36 | db = Database(memory=True) 37 | spatialite = find_spatialite() 38 | db.init_spatialite(spatialite) 39 | assert "spatial_ref_sys" in db.table_names() 40 | 41 | 42 | def test_add_geometry_column(): 43 | db = Database(memory=True) 44 | spatialite = find_spatialite() 45 | db.init_spatialite(spatialite) 46 | 47 | # create a table first 48 | table = db.create_table("locations", {"id": str, "properties": str}) 49 | table.add_geometry_column( 50 | column_name="geometry", 51 | geometry_type="Point", 52 | srid=4326, 53 | coord_dimension=2, 54 | ) 55 | 56 | assert db["geometry_columns"].get(["locations", "geometry"]) == { 57 | "f_table_name": "locations", 58 | "f_geometry_column": "geometry", 59 | "geometry_type": 1, # point 60 | "coord_dimension": 2, 61 | "srid": 4326, 62 | "spatial_index_enabled": 0, 63 | } 64 | 65 | 66 | def test_create_spatial_index(): 67 | db = Database(memory=True) 68 | spatialite = find_spatialite() 69 | assert db.init_spatialite(spatialite) 70 | 71 | # create a table, add a geometry column with default values 72 | table = db.create_table("locations", {"id": str, "properties": str}) 73 | assert table.add_geometry_column("geometry", "Point") 74 | 75 | # index it 76 | assert table.create_spatial_index("geometry") 77 | 78 | assert "idx_locations_geometry" in db.table_names() 79 | 80 | 81 | def test_double_create_spatial_index(): 82 | db = Database(memory=True) 83 | spatialite = find_spatialite() 84 | db.init_spatialite(spatialite) 85 | 86 | # create a table, add a geometry column with default values 87 | table = db.create_table("locations", {"id": str, "properties": str}) 88 | table.add_geometry_column("geometry", "Point") 89 | 90 | # index it, return True 91 | assert table.create_spatial_index("geometry") 92 | 93 | assert "idx_locations_geometry" in db.table_names() 94 | 95 | # call it again, return False 96 | assert not table.create_spatial_index("geometry") 97 | 98 | 99 | # cli tests 100 | @pytest.mark.parametrize("use_spatialite_shortcut", [True, False]) 101 | def test_query_load_extension(use_spatialite_shortcut): 102 | # Without --load-extension: 103 | result = CliRunner().invoke(cli, [":memory:", "select spatialite_version()"]) 104 | assert result.exit_code == 1 105 | assert "no such function: spatialite_version" in result.output 106 | # With --load-extension: 107 | if use_spatialite_shortcut: 108 | load_extension = "spatialite" 109 | else: 110 | load_extension = find_spatialite() 111 | result = CliRunner().invoke( 112 | cli, 113 | [ 114 | ":memory:", 115 | "select spatialite_version()", 116 | "--load-extension={}".format(load_extension), 117 | ], 118 | ) 119 | assert result.exit_code == 0, result.stdout 120 | assert ["spatialite_version()"] == list(json.loads(result.output)[0].keys()) 121 | 122 | 123 | def test_cli_create_spatialite(tmpdir): 124 | # sqlite-utils create test.db --init-spatialite 125 | db_path = tmpdir / "created.db" 126 | result = CliRunner().invoke( 127 | cli, ["create-database", str(db_path), "--init-spatialite"] 128 | ) 129 | 130 | assert result.exit_code == 0 131 | assert db_path.exists() 132 | assert db_path.read_binary()[:16] == b"SQLite format 3\x00" 133 | 134 | db = Database(str(db_path)) 135 | assert "spatial_ref_sys" in db.table_names() 136 | 137 | 138 | def test_cli_add_geometry_column(tmpdir): 139 | # create a rowid table with one column 140 | db_path = tmpdir / "spatial.db" 141 | db = Database(str(db_path)) 142 | db.init_spatialite() 143 | 144 | table = db["locations"].create({"name": str}) 145 | 146 | result = CliRunner().invoke( 147 | cli, 148 | [ 149 | "add-geometry-column", 150 | str(db_path), 151 | table.name, 152 | "geometry", 153 | "--type", 154 | "POINT", 155 | ], 156 | ) 157 | 158 | assert result.exit_code == 0 159 | 160 | assert db["geometry_columns"].get(["locations", "geometry"]) == { 161 | "f_table_name": "locations", 162 | "f_geometry_column": "geometry", 163 | "geometry_type": 1, # point 164 | "coord_dimension": 2, 165 | "srid": 4326, 166 | "spatial_index_enabled": 0, 167 | } 168 | 169 | 170 | def test_cli_add_geometry_column_options(tmpdir): 171 | # create a rowid table with one column 172 | db_path = tmpdir / "spatial.db" 173 | db = Database(str(db_path)) 174 | db.init_spatialite() 175 | table = db["locations"].create({"name": str}) 176 | 177 | result = CliRunner().invoke( 178 | cli, 179 | [ 180 | "add-geometry-column", 181 | str(db_path), 182 | table.name, 183 | "geometry", 184 | "-t", 185 | "POLYGON", 186 | "--srid", 187 | "3857", # https://epsg.io/3857 188 | "--not-null", 189 | ], 190 | ) 191 | 192 | assert result.exit_code == 0 193 | 194 | assert db["geometry_columns"].get(["locations", "geometry"]) == { 195 | "f_table_name": "locations", 196 | "f_geometry_column": "geometry", 197 | "geometry_type": 3, # polygon 198 | "coord_dimension": 2, 199 | "srid": 3857, 200 | "spatial_index_enabled": 0, 201 | } 202 | 203 | column = table.columns[1] 204 | assert column.notnull 205 | 206 | 207 | def test_cli_add_geometry_column_invalid_type(tmpdir): 208 | # create a rowid table with one column 209 | db_path = tmpdir / "spatial.db" 210 | db = Database(str(db_path)) 211 | db.init_spatialite() 212 | 213 | table = db["locations"].create({"name": str}) 214 | 215 | result = CliRunner().invoke( 216 | cli, 217 | [ 218 | "add-geometry-column", 219 | str(db_path), 220 | table.name, 221 | "geometry", 222 | "--type", 223 | "NOT-A-TYPE", 224 | ], 225 | ) 226 | 227 | assert 2 == result.exit_code 228 | 229 | 230 | def test_cli_create_spatial_index(tmpdir): 231 | # create a rowid table with one column 232 | db_path = tmpdir / "spatial.db" 233 | db = Database(str(db_path)) 234 | db.init_spatialite() 235 | 236 | table = db["locations"].create({"name": str}) 237 | table.add_geometry_column("geometry", "POINT") 238 | 239 | result = CliRunner().invoke( 240 | cli, ["create-spatial-index", str(db_path), table.name, "geometry"] 241 | ) 242 | 243 | assert result.exit_code == 0 244 | 245 | assert "idx_locations_geometry" in db.table_names() 246 | -------------------------------------------------------------------------------- /tests/test_hypothesis.py: -------------------------------------------------------------------------------- 1 | from hypothesis import given 2 | import hypothesis.strategies as st 3 | import sqlite_utils 4 | 5 | 6 | # SQLite integers are -(2^63) to 2^63 - 1 7 | @given(st.integers(-9223372036854775808, 9223372036854775807)) 8 | def test_roundtrip_integers(integer): 9 | db = sqlite_utils.Database(memory=True) 10 | row = { 11 | "integer": integer, 12 | } 13 | db["test"].insert(row) 14 | assert list(db["test"].rows) == [row] 15 | 16 | 17 | @given(st.text()) 18 | def test_roundtrip_text(text): 19 | db = sqlite_utils.Database(memory=True) 20 | row = { 21 | "text": text, 22 | } 23 | db["test"].insert(row) 24 | assert list(db["test"].rows) == [row] 25 | 26 | 27 | @given(st.binary(max_size=1024 * 1024)) 28 | def test_roundtrip_binary(binary): 29 | db = sqlite_utils.Database(memory=True) 30 | row = { 31 | "binary": binary, 32 | } 33 | db["test"].insert(row) 34 | assert list(db["test"].rows) == [row] 35 | 36 | 37 | @given(st.floats(allow_nan=False)) 38 | def test_roundtrip_floats(floats): 39 | db = sqlite_utils.Database(memory=True) 40 | row = { 41 | "floats": floats, 42 | } 43 | db["test"].insert(row) 44 | assert list(db["test"].rows) == [row] 45 | -------------------------------------------------------------------------------- /tests/test_insert_files.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils import cli, Database 2 | from click.testing import CliRunner 3 | import os 4 | import pathlib 5 | import pytest 6 | import sys 7 | 8 | 9 | @pytest.mark.parametrize("silent", (False, True)) 10 | @pytest.mark.parametrize( 11 | "pk_args,expected_pks", 12 | ( 13 | (["--pk", "path"], ["path"]), 14 | (["--pk", "path", "--pk", "name"], ["path", "name"]), 15 | ), 16 | ) 17 | def test_insert_files(silent, pk_args, expected_pks): 18 | runner = CliRunner() 19 | with runner.isolated_filesystem(): 20 | tmpdir = pathlib.Path(".") 21 | db_path = str(tmpdir / "files.db") 22 | (tmpdir / "one.txt").write_text("This is file one", "utf-8") 23 | (tmpdir / "two.txt").write_text("Two is shorter", "utf-8") 24 | (tmpdir / "nested").mkdir() 25 | (tmpdir / "nested" / "three.zz.txt").write_text("Three is nested", "utf-8") 26 | coltypes = ( 27 | "name", 28 | "path", 29 | "fullpath", 30 | "sha256", 31 | "md5", 32 | "mode", 33 | "content", 34 | "content_text", 35 | "mtime", 36 | "ctime", 37 | "mtime_int", 38 | "ctime_int", 39 | "mtime_iso", 40 | "ctime_iso", 41 | "size", 42 | "suffix", 43 | "stem", 44 | ) 45 | cols = [] 46 | for coltype in coltypes: 47 | cols += ["-c", "{}:{}".format(coltype, coltype)] 48 | result = runner.invoke( 49 | cli.cli, 50 | ["insert-files", db_path, "files", str(tmpdir)] 51 | + cols 52 | + pk_args 53 | + (["--silent"] if silent else []), 54 | catch_exceptions=False, 55 | ) 56 | assert result.exit_code == 0, result.stdout 57 | db = Database(db_path) 58 | rows_by_path = {r["path"]: r for r in db["files"].rows} 59 | one, two, three = ( 60 | rows_by_path["one.txt"], 61 | rows_by_path["two.txt"], 62 | rows_by_path[os.path.join("nested", "three.zz.txt")], 63 | ) 64 | assert { 65 | "content": b"This is file one", 66 | "content_text": "This is file one", 67 | "md5": "556dfb57fce9ca301f914e2273adf354", 68 | "name": "one.txt", 69 | "path": "one.txt", 70 | "sha256": "e34138f26b5f7368f298b4e736fea0aad87ddec69fbd04dc183b20f4d844bad5", 71 | "size": 16, 72 | "stem": "one", 73 | "suffix": ".txt", 74 | }.items() <= one.items() 75 | assert { 76 | "content": b"Two is shorter", 77 | "content_text": "Two is shorter", 78 | "md5": "f86f067b083af1911043eb215e74ac70", 79 | "name": "two.txt", 80 | "path": "two.txt", 81 | "sha256": "9368988ed16d4a2da0af9db9b686d385b942cb3ffd4e013f43aed2ec041183d9", 82 | "size": 14, 83 | "stem": "two", 84 | "suffix": ".txt", 85 | }.items() <= two.items() 86 | assert { 87 | "content": b"Three is nested", 88 | "content_text": "Three is nested", 89 | "md5": "12580f341781f5a5b589164d3cd39523", 90 | "name": "three.zz.txt", 91 | "path": os.path.join("nested", "three.zz.txt"), 92 | "sha256": "6dd45aaaaa6b9f96af19363a92c8fca5d34791d3c35c44eb19468a6a862cc8cd", 93 | "size": 15, 94 | "stem": "three.zz", 95 | "suffix": ".txt", 96 | }.items() <= three.items() 97 | # Assert the other int/str/float columns exist and are of the right types 98 | expected_types = { 99 | "ctime": float, 100 | "ctime_int": int, 101 | "ctime_iso": str, 102 | "mtime": float, 103 | "mtime_int": int, 104 | "mtime_iso": str, 105 | "mode": int, 106 | "fullpath": str, 107 | "content": bytes, 108 | "content_text": str, 109 | "stem": str, 110 | "suffix": str, 111 | } 112 | for colname, expected_type in expected_types.items(): 113 | for row in (one, two, three): 114 | assert isinstance(row[colname], expected_type) 115 | assert set(db["files"].pks) == set(expected_pks) 116 | 117 | 118 | @pytest.mark.parametrize( 119 | "use_text,encoding,input,expected", 120 | ( 121 | (False, None, "hello world", b"hello world"), 122 | (True, None, "hello world", "hello world"), 123 | (False, None, b"S\xe3o Paulo", b"S\xe3o Paulo"), 124 | (True, "latin-1", b"S\xe3o Paulo", "S\xe3o Paulo"), 125 | ), 126 | ) 127 | def test_insert_files_stdin(use_text, encoding, input, expected): 128 | runner = CliRunner() 129 | with runner.isolated_filesystem(): 130 | tmpdir = pathlib.Path(".") 131 | db_path = str(tmpdir / "files.db") 132 | args = ["insert-files", db_path, "files", "-", "--name", "stdin-name"] 133 | if use_text: 134 | args += ["--text"] 135 | if encoding is not None: 136 | args += ["--encoding", encoding] 137 | result = runner.invoke( 138 | cli.cli, 139 | args, 140 | catch_exceptions=False, 141 | input=input, 142 | ) 143 | assert result.exit_code == 0, result.stdout 144 | db = Database(db_path) 145 | row = list(db["files"].rows)[0] 146 | key = "content" 147 | if use_text: 148 | key = "content_text" 149 | assert {"path": "stdin-name", key: expected}.items() <= row.items() 150 | 151 | 152 | @pytest.mark.skipif( 153 | sys.platform.startswith("win"), 154 | reason="Windows has a different way of handling default encodings", 155 | ) 156 | def test_insert_files_bad_text_encoding_error(): 157 | runner = CliRunner() 158 | with runner.isolated_filesystem(): 159 | tmpdir = pathlib.Path(".") 160 | latin = tmpdir / "latin.txt" 161 | latin.write_bytes(b"S\xe3o Paulo") 162 | db_path = str(tmpdir / "files.db") 163 | result = runner.invoke( 164 | cli.cli, 165 | ["insert-files", db_path, "files", str(latin), "--text"], 166 | catch_exceptions=False, 167 | ) 168 | assert result.exit_code == 1, result.output 169 | assert result.output.strip().startswith( 170 | "Error: Could not read file '{}' as text".format(str(latin.resolve())) 171 | ) 172 | -------------------------------------------------------------------------------- /tests/test_introspect.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils.db import Index, View, Database, XIndex, XIndexColumn 2 | import pytest 3 | 4 | 5 | def test_table_names(existing_db): 6 | assert ["foo"] == existing_db.table_names() 7 | 8 | 9 | def test_view_names(fresh_db): 10 | fresh_db.create_view("foo_view", "select 1") 11 | assert ["foo_view"] == fresh_db.view_names() 12 | 13 | 14 | def test_table_names_fts4(existing_db): 15 | existing_db["woo"].insert({"title": "Hello"}).enable_fts( 16 | ["title"], fts_version="FTS4" 17 | ) 18 | existing_db["woo2"].insert({"title": "Hello"}).enable_fts( 19 | ["title"], fts_version="FTS5" 20 | ) 21 | assert ["woo_fts"] == existing_db.table_names(fts4=True) 22 | assert ["woo2_fts"] == existing_db.table_names(fts5=True) 23 | 24 | 25 | def test_detect_fts(existing_db): 26 | existing_db["woo"].insert({"title": "Hello"}).enable_fts( 27 | ["title"], fts_version="FTS4" 28 | ) 29 | existing_db["woo2"].insert({"title": "Hello"}).enable_fts( 30 | ["title"], fts_version="FTS5" 31 | ) 32 | assert "woo_fts" == existing_db["woo"].detect_fts() 33 | assert "woo_fts" == existing_db["woo_fts"].detect_fts() 34 | assert "woo2_fts" == existing_db["woo2"].detect_fts() 35 | assert "woo2_fts" == existing_db["woo2_fts"].detect_fts() 36 | assert existing_db["foo"].detect_fts() is None 37 | 38 | 39 | @pytest.mark.parametrize("reverse_order", (True, False)) 40 | def test_detect_fts_similar_tables(fresh_db, reverse_order): 41 | # https://github.com/simonw/sqlite-utils/issues/434 42 | table1, table2 = ("demo", "demo2") 43 | if reverse_order: 44 | table1, table2 = table2, table1 45 | 46 | fresh_db[table1].insert({"title": "Hello"}).enable_fts( 47 | ["title"], fts_version="FTS4" 48 | ) 49 | fresh_db[table2].insert({"title": "Hello"}).enable_fts( 50 | ["title"], fts_version="FTS4" 51 | ) 52 | assert fresh_db[table1].detect_fts() == "{}_fts".format(table1) 53 | assert fresh_db[table2].detect_fts() == "{}_fts".format(table2) 54 | 55 | 56 | def test_tables(existing_db): 57 | assert len(existing_db.tables) == 1 58 | assert existing_db.tables[0].name == "foo" 59 | 60 | 61 | def test_views(fresh_db): 62 | fresh_db.create_view("foo_view", "select 1") 63 | assert len(fresh_db.views) == 1 64 | view = fresh_db.views[0] 65 | assert isinstance(view, View) 66 | assert view.name == "foo_view" 67 | assert repr(view) == "" 68 | assert view.columns_dict == {"1": str} 69 | 70 | 71 | def test_count(existing_db): 72 | assert existing_db["foo"].count == 3 73 | assert existing_db["foo"].count_where() == 3 74 | assert existing_db["foo"].execute_count() == 3 75 | 76 | 77 | def test_count_where(existing_db): 78 | assert existing_db["foo"].count_where("text != ?", ["two"]) == 2 79 | assert existing_db["foo"].count_where("text != :t", {"t": "two"}) == 2 80 | 81 | 82 | def test_columns(existing_db): 83 | table = existing_db["foo"] 84 | assert [{"name": "text", "type": "TEXT"}] == [ 85 | {"name": col.name, "type": col.type} for col in table.columns 86 | ] 87 | 88 | 89 | def test_table_schema(existing_db): 90 | assert existing_db["foo"].schema == "CREATE TABLE foo (text TEXT)" 91 | 92 | 93 | def test_database_schema(existing_db): 94 | assert existing_db.schema == "CREATE TABLE foo (text TEXT);" 95 | 96 | 97 | def test_table_repr(fresh_db): 98 | table = fresh_db["dogs"].insert({"name": "Cleo", "age": 4}) 99 | assert "" == repr(table) 100 | assert "
" == repr(fresh_db["cats"]) 101 | 102 | 103 | def test_indexes(fresh_db): 104 | fresh_db.executescript( 105 | """ 106 | create table Gosh (c1 text, c2 text, c3 text); 107 | create index Gosh_c1 on Gosh(c1); 108 | create index Gosh_c2c3 on Gosh(c2, c3); 109 | """ 110 | ) 111 | assert [ 112 | Index( 113 | seq=0, 114 | name="Gosh_c2c3", 115 | unique=0, 116 | origin="c", 117 | partial=0, 118 | columns=["c2", "c3"], 119 | ), 120 | Index(seq=1, name="Gosh_c1", unique=0, origin="c", partial=0, columns=["c1"]), 121 | ] == fresh_db["Gosh"].indexes 122 | 123 | 124 | def test_xindexes(fresh_db): 125 | fresh_db.executescript( 126 | """ 127 | create table Gosh (c1 text, c2 text, c3 text); 128 | create index Gosh_c1 on Gosh(c1); 129 | create index Gosh_c2c3 on Gosh(c2, c3 desc); 130 | """ 131 | ) 132 | assert fresh_db["Gosh"].xindexes == [ 133 | XIndex( 134 | name="Gosh_c2c3", 135 | columns=[ 136 | XIndexColumn(seqno=0, cid=1, name="c2", desc=0, coll="BINARY", key=1), 137 | XIndexColumn(seqno=1, cid=2, name="c3", desc=1, coll="BINARY", key=1), 138 | XIndexColumn(seqno=2, cid=-1, name=None, desc=0, coll="BINARY", key=0), 139 | ], 140 | ), 141 | XIndex( 142 | name="Gosh_c1", 143 | columns=[ 144 | XIndexColumn(seqno=0, cid=0, name="c1", desc=0, coll="BINARY", key=1), 145 | XIndexColumn(seqno=1, cid=-1, name=None, desc=0, coll="BINARY", key=0), 146 | ], 147 | ), 148 | ] 149 | 150 | 151 | @pytest.mark.parametrize( 152 | "column,expected_table_guess", 153 | ( 154 | ("author", "authors"), 155 | ("author_id", "authors"), 156 | ("authors", "authors"), 157 | ("genre", "genre"), 158 | ("genre_id", "genre"), 159 | ), 160 | ) 161 | def test_guess_foreign_table(fresh_db, column, expected_table_guess): 162 | fresh_db.create_table("authors", {"name": str}) 163 | fresh_db.create_table("genre", {"name": str}) 164 | assert expected_table_guess == fresh_db["books"].guess_foreign_table(column) 165 | 166 | 167 | @pytest.mark.parametrize( 168 | "pk,expected", ((None, ["rowid"]), ("id", ["id"]), (["id", "id2"], ["id", "id2"])) 169 | ) 170 | def test_pks(fresh_db, pk, expected): 171 | fresh_db["foo"].insert_all([{"id": 1, "id2": 2}], pk=pk) 172 | assert expected == fresh_db["foo"].pks 173 | 174 | 175 | def test_triggers_and_triggers_dict(fresh_db): 176 | assert [] == fresh_db.triggers 177 | authors = fresh_db["authors"] 178 | authors.insert_all( 179 | [ 180 | {"name": "Frank Herbert", "famous_works": "Dune"}, 181 | {"name": "Neal Stephenson", "famous_works": "Cryptonomicon"}, 182 | ] 183 | ) 184 | fresh_db["other"].insert({"foo": "bar"}) 185 | assert authors.triggers == [] 186 | assert authors.triggers_dict == {} 187 | assert fresh_db["other"].triggers == [] 188 | assert fresh_db.triggers_dict == {} 189 | authors.enable_fts( 190 | ["name", "famous_works"], fts_version="FTS4", create_triggers=True 191 | ) 192 | expected_triggers = { 193 | ("authors_ai", "authors"), 194 | ("authors_ad", "authors"), 195 | ("authors_au", "authors"), 196 | } 197 | assert expected_triggers == {(t.name, t.table) for t in fresh_db.triggers} 198 | assert expected_triggers == { 199 | (t.name, t.table) for t in fresh_db["authors"].triggers 200 | } 201 | expected_triggers = { 202 | "authors_ai": ( 203 | "CREATE TRIGGER [authors_ai] AFTER INSERT ON [authors] BEGIN\n" 204 | " INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\n" 205 | "END" 206 | ), 207 | "authors_ad": ( 208 | "CREATE TRIGGER [authors_ad] AFTER DELETE ON [authors] BEGIN\n" 209 | " INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n" 210 | "END" 211 | ), 212 | "authors_au": ( 213 | "CREATE TRIGGER [authors_au] AFTER UPDATE ON [authors] BEGIN\n" 214 | " INSERT INTO [authors_fts] ([authors_fts], rowid, [name], [famous_works]) VALUES('delete', old.rowid, old.[name], old.[famous_works]);\n" 215 | " INSERT INTO [authors_fts] (rowid, [name], [famous_works]) VALUES (new.rowid, new.[name], new.[famous_works]);\nEND" 216 | ), 217 | } 218 | assert authors.triggers_dict == expected_triggers 219 | assert fresh_db["other"].triggers == [] 220 | assert fresh_db["other"].triggers_dict == {} 221 | assert fresh_db.triggers_dict == expected_triggers 222 | 223 | 224 | def test_has_counts_triggers(fresh_db): 225 | authors = fresh_db["authors"] 226 | authors.insert({"name": "Frank Herbert"}) 227 | assert not authors.has_counts_triggers 228 | authors.enable_counts() 229 | assert authors.has_counts_triggers 230 | 231 | 232 | @pytest.mark.parametrize( 233 | "sql,expected_name,expected_using", 234 | [ 235 | ( 236 | """ 237 | CREATE VIRTUAL TABLE foo USING FTS5(name) 238 | """, 239 | "foo", 240 | "FTS5", 241 | ), 242 | ( 243 | """ 244 | CREATE VIRTUAL TABLE "foo" USING FTS4(name) 245 | """, 246 | "foo", 247 | "FTS4", 248 | ), 249 | ( 250 | """ 251 | CREATE VIRTUAL TABLE IF NOT EXISTS `foo` USING FTS4(name) 252 | """, 253 | "foo", 254 | "FTS4", 255 | ), 256 | ( 257 | """ 258 | CREATE VIRTUAL TABLE IF NOT EXISTS `foo` USING fts5(name) 259 | """, 260 | "foo", 261 | "FTS5", 262 | ), 263 | ( 264 | """ 265 | CREATE TABLE IF NOT EXISTS `foo` (id integer primary key) 266 | """, 267 | "foo", 268 | None, 269 | ), 270 | ], 271 | ) 272 | def test_virtual_table_using(fresh_db, sql, expected_name, expected_using): 273 | fresh_db.execute(sql) 274 | assert fresh_db[expected_name].virtual_table_using == expected_using 275 | 276 | 277 | def test_use_rowid(fresh_db): 278 | fresh_db["rowid_table"].insert({"name": "Cleo"}) 279 | fresh_db["regular_table"].insert({"id": 1, "name": "Cleo"}, pk="id") 280 | assert fresh_db["rowid_table"].use_rowid 281 | assert not fresh_db["regular_table"].use_rowid 282 | 283 | 284 | @pytest.mark.skipif( 285 | not Database(memory=True).supports_strict, 286 | reason="Needs SQLite version that supports strict", 287 | ) 288 | @pytest.mark.parametrize( 289 | "create_table,expected_strict", 290 | ( 291 | ("create table t (id integer) strict", True), 292 | ("create table t (id integer) STRICT", True), 293 | ("create table t (id integer primary key) StriCt, WITHOUT ROWID", True), 294 | ("create table t (id integer primary key) WITHOUT ROWID", False), 295 | ("create table t (id integer)", False), 296 | ), 297 | ) 298 | def test_table_strict(fresh_db, create_table, expected_strict): 299 | fresh_db.execute(create_table) 300 | table = fresh_db["t"] 301 | assert table.strict == expected_strict 302 | 303 | 304 | @pytest.mark.parametrize( 305 | "value", 306 | ( 307 | 1, 308 | 1.3, 309 | "foo", 310 | True, 311 | b"binary", 312 | ), 313 | ) 314 | def test_table_default_values(fresh_db, value): 315 | fresh_db["default_values"].insert( 316 | {"nodefault": 1, "value": value}, defaults={"value": value} 317 | ) 318 | default_values = fresh_db["default_values"].default_values 319 | assert default_values == {"value": value} 320 | -------------------------------------------------------------------------------- /tests/test_lookup.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils.db import Index 2 | import pytest 3 | 4 | 5 | def test_lookup_new_table(fresh_db): 6 | species = fresh_db["species"] 7 | palm_id = species.lookup({"name": "Palm"}) 8 | oak_id = species.lookup({"name": "Oak"}) 9 | cherry_id = species.lookup({"name": "Cherry"}) 10 | assert palm_id == species.lookup({"name": "Palm"}) 11 | assert oak_id == species.lookup({"name": "Oak"}) 12 | assert cherry_id == species.lookup({"name": "Cherry"}) 13 | assert palm_id != oak_id != cherry_id 14 | # Ensure the correct indexes were created 15 | assert [ 16 | Index( 17 | seq=0, 18 | name="idx_species_name", 19 | unique=1, 20 | origin="c", 21 | partial=0, 22 | columns=["name"], 23 | ) 24 | ] == species.indexes 25 | 26 | 27 | def test_lookup_new_table_compound_key(fresh_db): 28 | species = fresh_db["species"] 29 | palm_id = species.lookup({"name": "Palm", "type": "Tree"}) 30 | oak_id = species.lookup({"name": "Oak", "type": "Tree"}) 31 | assert palm_id == species.lookup({"name": "Palm", "type": "Tree"}) 32 | assert oak_id == species.lookup({"name": "Oak", "type": "Tree"}) 33 | assert [ 34 | Index( 35 | seq=0, 36 | name="idx_species_name_type", 37 | unique=1, 38 | origin="c", 39 | partial=0, 40 | columns=["name", "type"], 41 | ) 42 | ] == species.indexes 43 | 44 | 45 | def test_lookup_adds_unique_constraint_to_existing_table(fresh_db): 46 | species = fresh_db.table("species", pk="id") 47 | palm_id = species.insert({"name": "Palm"}).last_pk 48 | species.insert({"name": "Oak"}) 49 | assert [] == species.indexes 50 | assert palm_id == species.lookup({"name": "Palm"}) 51 | assert [ 52 | Index( 53 | seq=0, 54 | name="idx_species_name", 55 | unique=1, 56 | origin="c", 57 | partial=0, 58 | columns=["name"], 59 | ) 60 | ] == species.indexes 61 | 62 | 63 | def test_lookup_fails_if_constraint_cannot_be_added(fresh_db): 64 | species = fresh_db.table("species", pk="id") 65 | species.insert_all([{"id": 1, "name": "Palm"}, {"id": 2, "name": "Palm"}]) 66 | # This will fail because the name column is not unique 67 | with pytest.raises(Exception, match="UNIQUE constraint failed"): 68 | species.lookup({"name": "Palm"}) 69 | 70 | 71 | def test_lookup_with_extra_values(fresh_db): 72 | species = fresh_db["species"] 73 | id = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2020-01-01"}) 74 | assert species.get(id) == { 75 | "id": 1, 76 | "name": "Palm", 77 | "type": "Tree", 78 | "first_seen": "2020-01-01", 79 | } 80 | # A subsequent lookup() should ignore the second dictionary 81 | id2 = species.lookup({"name": "Palm", "type": "Tree"}, {"first_seen": "2021-02-02"}) 82 | assert id2 == id 83 | assert species.get(id2) == { 84 | "id": 1, 85 | "name": "Palm", 86 | "type": "Tree", 87 | "first_seen": "2020-01-01", 88 | } 89 | 90 | 91 | def test_lookup_with_extra_insert_parameters(fresh_db): 92 | other_table = fresh_db["other_table"] 93 | other_table.insert({"id": 1, "name": "Name"}, pk="id") 94 | species = fresh_db["species"] 95 | id = species.lookup( 96 | {"name": "Palm", "type": "Tree"}, 97 | { 98 | "first_seen": "2020-01-01", 99 | "make_not_null": 1, 100 | "fk_to_other": 1, 101 | "default_is_dog": "cat", 102 | "extract_this": "This is extracted", 103 | "convert_to_upper": "upper", 104 | "make_this_integer": "2", 105 | "this_at_front": 1, 106 | }, 107 | pk="renamed_id", 108 | foreign_keys=(("fk_to_other", "other_table", "id"),), 109 | column_order=("this_at_front",), 110 | not_null={"make_not_null"}, 111 | defaults={"default_is_dog": "dog"}, 112 | extracts=["extract_this"], 113 | conversions={"convert_to_upper": "upper(?)"}, 114 | columns={"make_this_integer": int}, 115 | ) 116 | assert species.schema == ( 117 | "CREATE TABLE [species] (\n" 118 | " [renamed_id] INTEGER PRIMARY KEY,\n" 119 | " [this_at_front] INTEGER,\n" 120 | " [name] TEXT,\n" 121 | " [type] TEXT,\n" 122 | " [first_seen] TEXT,\n" 123 | " [make_not_null] INTEGER NOT NULL,\n" 124 | " [fk_to_other] INTEGER REFERENCES [other_table]([id]),\n" 125 | " [default_is_dog] TEXT DEFAULT 'dog',\n" 126 | " [extract_this] INTEGER REFERENCES [extract_this]([id]),\n" 127 | " [convert_to_upper] TEXT,\n" 128 | " [make_this_integer] INTEGER\n" 129 | ")" 130 | ) 131 | assert species.get(id) == { 132 | "renamed_id": id, 133 | "this_at_front": 1, 134 | "name": "Palm", 135 | "type": "Tree", 136 | "first_seen": "2020-01-01", 137 | "make_not_null": 1, 138 | "fk_to_other": 1, 139 | "default_is_dog": "cat", 140 | "extract_this": 1, 141 | "convert_to_upper": "UPPER", 142 | "make_this_integer": 2, 143 | } 144 | assert species.indexes == [ 145 | Index( 146 | seq=0, 147 | name="idx_species_name_type", 148 | unique=1, 149 | origin="c", 150 | partial=0, 151 | columns=["name", "type"], 152 | ) 153 | ] 154 | 155 | 156 | @pytest.mark.parametrize("strict", (False, True)) 157 | def test_lookup_new_table_strict(fresh_db, strict): 158 | fresh_db["species"].lookup({"name": "Palm"}, strict=strict) 159 | assert fresh_db["species"].strict == strict or not fresh_db.supports_strict 160 | -------------------------------------------------------------------------------- /tests/test_m2m.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils.db import ForeignKey, NoObviousTable 2 | import pytest 3 | 4 | 5 | def test_insert_m2m_single(fresh_db): 6 | dogs = fresh_db["dogs"] 7 | dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( 8 | "humans", {"id": 1, "name": "Natalie D"}, pk="id" 9 | ) 10 | assert {"dogs_humans", "humans", "dogs"} == set(fresh_db.table_names()) 11 | humans = fresh_db["humans"] 12 | dogs_humans = fresh_db["dogs_humans"] 13 | assert [{"id": 1, "name": "Natalie D"}] == list(humans.rows) 14 | assert [{"humans_id": 1, "dogs_id": 1}] == list(dogs_humans.rows) 15 | 16 | 17 | def test_insert_m2m_alter(fresh_db): 18 | dogs = fresh_db["dogs"] 19 | dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( 20 | "humans", {"id": 1, "name": "Natalie D"}, pk="id" 21 | ) 22 | dogs.update(1).m2m( 23 | "humans", {"id": 2, "name": "Simon W", "nerd": True}, pk="id", alter=True 24 | ) 25 | assert list(fresh_db["humans"].rows) == [ 26 | {"id": 1, "name": "Natalie D", "nerd": None}, 27 | {"id": 2, "name": "Simon W", "nerd": 1}, 28 | ] 29 | assert list(fresh_db["dogs_humans"].rows) == [ 30 | {"humans_id": 1, "dogs_id": 1}, 31 | {"humans_id": 2, "dogs_id": 1}, 32 | ] 33 | 34 | 35 | def test_insert_m2m_list(fresh_db): 36 | dogs = fresh_db["dogs"] 37 | dogs.insert({"id": 1, "name": "Cleo"}, pk="id").m2m( 38 | "humans", 39 | [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}], 40 | pk="id", 41 | ) 42 | assert {"dogs", "humans", "dogs_humans"} == set(fresh_db.table_names()) 43 | humans = fresh_db["humans"] 44 | dogs_humans = fresh_db["dogs_humans"] 45 | assert [{"humans_id": 1, "dogs_id": 1}, {"humans_id": 2, "dogs_id": 1}] == list( 46 | dogs_humans.rows 47 | ) 48 | assert [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}] == list( 49 | humans.rows 50 | ) 51 | assert [ 52 | ForeignKey( 53 | table="dogs_humans", column="dogs_id", other_table="dogs", other_column="id" 54 | ), 55 | ForeignKey( 56 | table="dogs_humans", 57 | column="humans_id", 58 | other_table="humans", 59 | other_column="id", 60 | ), 61 | ] == dogs_humans.foreign_keys 62 | 63 | 64 | def test_insert_m2m_iterable(fresh_db): 65 | iterable_records = ({"id": 1, "name": "Phineas"}, {"id": 2, "name": "Ferb"}) 66 | 67 | def iterable(): 68 | for record in iterable_records: 69 | yield record 70 | 71 | platypuses = fresh_db["platypuses"] 72 | platypuses.insert({"id": 1, "name": "Perry"}, pk="id").m2m( 73 | "humans", 74 | iterable(), 75 | pk="id", 76 | ) 77 | 78 | assert {"platypuses", "humans", "humans_platypuses"} == set(fresh_db.table_names()) 79 | humans = fresh_db["humans"] 80 | humans_platypuses = fresh_db["humans_platypuses"] 81 | assert [ 82 | {"humans_id": 1, "platypuses_id": 1}, 83 | {"humans_id": 2, "platypuses_id": 1}, 84 | ] == list(humans_platypuses.rows) 85 | assert [{"id": 1, "name": "Phineas"}, {"id": 2, "name": "Ferb"}] == list( 86 | humans.rows 87 | ) 88 | assert [ 89 | ForeignKey( 90 | table="humans_platypuses", 91 | column="platypuses_id", 92 | other_table="platypuses", 93 | other_column="id", 94 | ), 95 | ForeignKey( 96 | table="humans_platypuses", 97 | column="humans_id", 98 | other_table="humans", 99 | other_column="id", 100 | ), 101 | ] == humans_platypuses.foreign_keys 102 | 103 | 104 | def test_m2m_with_table_objects(fresh_db): 105 | dogs = fresh_db.table("dogs", pk="id") 106 | humans = fresh_db.table("humans", pk="id") 107 | dogs.insert({"id": 1, "name": "Cleo"}).m2m( 108 | humans, [{"id": 1, "name": "Natalie D"}, {"id": 2, "name": "Simon W"}] 109 | ) 110 | expected_tables = {"dogs", "humans", "dogs_humans"} 111 | assert expected_tables == set(fresh_db.table_names()) 112 | assert dogs.count == 1 113 | assert humans.count == 2 114 | assert fresh_db["dogs_humans"].count == 2 115 | 116 | 117 | def test_m2m_lookup(fresh_db): 118 | people = fresh_db.table("people", pk="id") 119 | people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) 120 | people_tags = fresh_db["people_tags"] 121 | tags = fresh_db["tags"] 122 | assert people_tags.exists() 123 | assert tags.exists() 124 | assert [ 125 | ForeignKey( 126 | table="people_tags", 127 | column="people_id", 128 | other_table="people", 129 | other_column="id", 130 | ), 131 | ForeignKey( 132 | table="people_tags", column="tags_id", other_table="tags", other_column="id" 133 | ), 134 | ] == people_tags.foreign_keys 135 | assert [{"people_id": 1, "tags_id": 1}] == list(people_tags.rows) 136 | assert [{"id": 1, "name": "Wahyu"}] == list(people.rows) 137 | assert [{"id": 1, "tag": "Coworker"}] == list(tags.rows) 138 | 139 | 140 | def test_m2m_requires_either_records_or_lookup(fresh_db): 141 | people = fresh_db.table("people", pk="id").insert({"name": "Wahyu"}) 142 | with pytest.raises(AssertionError): 143 | people.m2m("tags") 144 | with pytest.raises(AssertionError): 145 | people.m2m("tags", {"tag": "hello"}, lookup={"foo": "bar"}) 146 | 147 | 148 | def test_m2m_explicit_table_name_argument(fresh_db): 149 | people = fresh_db.table("people", pk="id") 150 | people.insert({"name": "Wahyu"}).m2m( 151 | "tags", lookup={"tag": "Coworker"}, m2m_table="tagged" 152 | ) 153 | assert fresh_db["tags"].exists 154 | assert fresh_db["tagged"].exists 155 | assert not fresh_db["people_tags"].exists() 156 | 157 | 158 | def test_m2m_table_candidates(fresh_db): 159 | fresh_db.create_table("one", {"id": int, "name": str}, pk="id") 160 | fresh_db.create_table("two", {"id": int, "name": str}, pk="id") 161 | fresh_db.create_table("three", {"id": int, "name": str}, pk="id") 162 | # No candidates at first 163 | assert [] == fresh_db.m2m_table_candidates("one", "two") 164 | # Create a candidate 165 | fresh_db.create_table( 166 | "one_m2m_two", {"one_id": int, "two_id": int}, foreign_keys=["one_id", "two_id"] 167 | ) 168 | assert ["one_m2m_two"] == fresh_db.m2m_table_candidates("one", "two") 169 | # Add another table and there should be two candidates 170 | fresh_db.create_table( 171 | "one_m2m_two_and_three", 172 | {"one_id": int, "two_id": int, "three_id": int}, 173 | foreign_keys=["one_id", "two_id", "three_id"], 174 | ) 175 | assert {"one_m2m_two", "one_m2m_two_and_three"} == set( 176 | fresh_db.m2m_table_candidates("one", "two") 177 | ) 178 | 179 | 180 | def test_uses_existing_m2m_table_if_exists(fresh_db): 181 | # Code should look for an existing table with fks to both tables 182 | # and use that if it exists. 183 | people = fresh_db.create_table("people", {"id": int, "name": str}, pk="id") 184 | fresh_db["tags"].lookup({"tag": "Coworker"}) 185 | fresh_db.create_table( 186 | "tagged", 187 | {"people_id": int, "tags_id": int}, 188 | foreign_keys=["people_id", "tags_id"], 189 | ) 190 | people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) 191 | assert fresh_db["tags"].exists() 192 | assert fresh_db["tagged"].exists() 193 | assert not fresh_db["people_tags"].exists() 194 | assert not fresh_db["tags_people"].exists() 195 | assert [{"people_id": 1, "tags_id": 1}] == list(fresh_db["tagged"].rows) 196 | 197 | 198 | def test_requires_explicit_m2m_table_if_multiple_options(fresh_db): 199 | # If the code scans for m2m tables and finds more than one candidate 200 | # it should require that the m2m_table=x argument is used 201 | people = fresh_db.create_table("people", {"id": int, "name": str}, pk="id") 202 | fresh_db["tags"].lookup({"tag": "Coworker"}) 203 | fresh_db.create_table( 204 | "tagged", 205 | {"people_id": int, "tags_id": int}, 206 | foreign_keys=["people_id", "tags_id"], 207 | ) 208 | fresh_db.create_table( 209 | "tagged2", 210 | {"people_id": int, "tags_id": int}, 211 | foreign_keys=["people_id", "tags_id"], 212 | ) 213 | with pytest.raises(NoObviousTable): 214 | people.insert({"name": "Wahyu"}).m2m("tags", lookup={"tag": "Coworker"}) 215 | -------------------------------------------------------------------------------- /tests/test_plugins.py: -------------------------------------------------------------------------------- 1 | from click.testing import CliRunner 2 | import click 3 | import importlib 4 | import pytest 5 | from sqlite_utils import cli, Database, hookimpl, plugins 6 | 7 | 8 | def _supports_pragma_function_list(): 9 | db = Database(memory=True) 10 | try: 11 | db.execute("select * from pragma_function_list()") 12 | except Exception: 13 | return False 14 | return True 15 | 16 | 17 | def test_register_commands(): 18 | importlib.reload(cli) 19 | assert plugins.get_plugins() == [] 20 | 21 | class HelloWorldPlugin: 22 | __name__ = "HelloWorldPlugin" 23 | 24 | @hookimpl 25 | def register_commands(self, cli): 26 | @cli.command(name="hello-world") 27 | def hello_world(): 28 | "Print hello world" 29 | click.echo("Hello world!") 30 | 31 | try: 32 | plugins.pm.register(HelloWorldPlugin(), name="HelloWorldPlugin") 33 | importlib.reload(cli) 34 | 35 | assert plugins.get_plugins() == [ 36 | {"name": "HelloWorldPlugin", "hooks": ["register_commands"]} 37 | ] 38 | 39 | runner = CliRunner() 40 | result = runner.invoke(cli.cli, ["hello-world"]) 41 | assert result.exit_code == 0 42 | assert result.output == "Hello world!\n" 43 | 44 | finally: 45 | plugins.pm.unregister(name="HelloWorldPlugin") 46 | importlib.reload(cli) 47 | assert plugins.get_plugins() == [] 48 | 49 | 50 | @pytest.mark.skipif( 51 | not _supports_pragma_function_list(), 52 | reason="Needs SQLite version that supports pragma_function_list()", 53 | ) 54 | def test_prepare_connection(): 55 | importlib.reload(cli) 56 | assert plugins.get_plugins() == [] 57 | 58 | class HelloFunctionPlugin: 59 | __name__ = "HelloFunctionPlugin" 60 | 61 | @hookimpl 62 | def prepare_connection(self, conn): 63 | conn.create_function("hello", 1, lambda name: f"Hello, {name}!") 64 | 65 | db = Database(memory=True) 66 | 67 | def _functions(db): 68 | return [ 69 | row[0] 70 | for row in db.execute( 71 | "select distinct name from pragma_function_list() order by 1" 72 | ).fetchall() 73 | ] 74 | 75 | assert "hello" not in _functions(db) 76 | 77 | try: 78 | plugins.pm.register(HelloFunctionPlugin(), name="HelloFunctionPlugin") 79 | 80 | assert plugins.get_plugins() == [ 81 | {"name": "HelloFunctionPlugin", "hooks": ["prepare_connection"]} 82 | ] 83 | 84 | db = Database(memory=True) 85 | assert "hello" in _functions(db) 86 | result = db.execute('select hello("world")').fetchone()[0] 87 | assert result == "Hello, world!" 88 | 89 | # Test execute_plugins=False 90 | db2 = Database(memory=True, execute_plugins=False) 91 | assert "hello" not in _functions(db2) 92 | 93 | finally: 94 | plugins.pm.unregister(name="HelloFunctionPlugin") 95 | assert plugins.get_plugins() == [] 96 | -------------------------------------------------------------------------------- /tests/test_query.py: -------------------------------------------------------------------------------- 1 | import types 2 | 3 | 4 | def test_query(fresh_db): 5 | fresh_db["dogs"].insert_all([{"name": "Cleo"}, {"name": "Pancakes"}]) 6 | results = fresh_db.query("select * from dogs order by name desc") 7 | assert isinstance(results, types.GeneratorType) 8 | assert list(results) == [{"name": "Pancakes"}, {"name": "Cleo"}] 9 | 10 | 11 | def test_execute_returning_dicts(fresh_db): 12 | # Like db.query() but returns a list, included for backwards compatibility 13 | # see https://github.com/simonw/sqlite-utils/issues/290 14 | fresh_db["test"].insert({"id": 1, "bar": 2}, pk="id") 15 | assert fresh_db.execute_returning_dicts("select * from test") == [ 16 | {"id": 1, "bar": 2} 17 | ] 18 | -------------------------------------------------------------------------------- /tests/test_recipes.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils import recipes 2 | from sqlite_utils.utils import sqlite3 3 | import json 4 | import pytest 5 | 6 | 7 | @pytest.fixture 8 | def dates_db(fresh_db): 9 | fresh_db["example"].insert_all( 10 | [ 11 | {"id": 1, "dt": "5th October 2019 12:04"}, 12 | {"id": 2, "dt": "6th October 2019 00:05:06"}, 13 | {"id": 3, "dt": ""}, 14 | {"id": 4, "dt": None}, 15 | ], 16 | pk="id", 17 | ) 18 | return fresh_db 19 | 20 | 21 | def test_parsedate(dates_db): 22 | dates_db["example"].convert("dt", recipes.parsedate) 23 | assert list(dates_db["example"].rows) == [ 24 | {"id": 1, "dt": "2019-10-05"}, 25 | {"id": 2, "dt": "2019-10-06"}, 26 | {"id": 3, "dt": ""}, 27 | {"id": 4, "dt": None}, 28 | ] 29 | 30 | 31 | def test_parsedatetime(dates_db): 32 | dates_db["example"].convert("dt", recipes.parsedatetime) 33 | assert list(dates_db["example"].rows) == [ 34 | {"id": 1, "dt": "2019-10-05T12:04:00"}, 35 | {"id": 2, "dt": "2019-10-06T00:05:06"}, 36 | {"id": 3, "dt": ""}, 37 | {"id": 4, "dt": None}, 38 | ] 39 | 40 | 41 | @pytest.mark.parametrize( 42 | "recipe,kwargs,expected", 43 | ( 44 | ("parsedate", {}, "2005-03-04"), 45 | ("parsedate", {"dayfirst": True}, "2005-04-03"), 46 | ("parsedatetime", {}, "2005-03-04T00:00:00"), 47 | ("parsedatetime", {"dayfirst": True}, "2005-04-03T00:00:00"), 48 | ), 49 | ) 50 | def test_dayfirst_yearfirst(fresh_db, recipe, kwargs, expected): 51 | fresh_db["example"].insert_all( 52 | [ 53 | {"id": 1, "dt": "03/04/05"}, 54 | ], 55 | pk="id", 56 | ) 57 | fresh_db["example"].convert( 58 | "dt", lambda value: getattr(recipes, recipe)(value, **kwargs) 59 | ) 60 | assert list(fresh_db["example"].rows) == [ 61 | {"id": 1, "dt": expected}, 62 | ] 63 | 64 | 65 | @pytest.mark.parametrize("fn", ("parsedate", "parsedatetime")) 66 | @pytest.mark.parametrize("errors", (None, recipes.SET_NULL, recipes.IGNORE)) 67 | def test_dateparse_errors(fresh_db, fn, errors): 68 | fresh_db["example"].insert_all( 69 | [ 70 | {"id": 1, "dt": "invalid"}, 71 | ], 72 | pk="id", 73 | ) 74 | if errors is None: 75 | # Should raise an error 76 | with pytest.raises(sqlite3.OperationalError): 77 | fresh_db["example"].convert("dt", lambda value: getattr(recipes, fn)(value)) 78 | else: 79 | fresh_db["example"].convert( 80 | "dt", lambda value: getattr(recipes, fn)(value, errors=errors) 81 | ) 82 | rows = list(fresh_db["example"].rows) 83 | expected = [{"id": 1, "dt": None if errors is recipes.SET_NULL else "invalid"}] 84 | assert rows == expected 85 | 86 | 87 | @pytest.mark.parametrize("delimiter", [None, ";", "-"]) 88 | def test_jsonsplit(fresh_db, delimiter): 89 | fresh_db["example"].insert_all( 90 | [ 91 | {"id": 1, "tags": (delimiter or ",").join(["foo", "bar"])}, 92 | {"id": 2, "tags": (delimiter or ",").join(["bar", "baz"])}, 93 | ], 94 | pk="id", 95 | ) 96 | if delimiter is not None: 97 | 98 | def fn(value): 99 | return recipes.jsonsplit(value, delimiter=delimiter) 100 | 101 | else: 102 | fn = recipes.jsonsplit 103 | 104 | fresh_db["example"].convert("tags", fn) 105 | assert list(fresh_db["example"].rows) == [ 106 | {"id": 1, "tags": '["foo", "bar"]'}, 107 | {"id": 2, "tags": '["bar", "baz"]'}, 108 | ] 109 | 110 | 111 | @pytest.mark.parametrize( 112 | "type,expected", 113 | ( 114 | (None, ["1", "2", "3"]), 115 | (float, [1.0, 2.0, 3.0]), 116 | (int, [1, 2, 3]), 117 | ), 118 | ) 119 | def test_jsonsplit_type(fresh_db, type, expected): 120 | fresh_db["example"].insert_all( 121 | [ 122 | {"id": 1, "records": "1,2,3"}, 123 | ], 124 | pk="id", 125 | ) 126 | if type is not None: 127 | 128 | def fn(value): 129 | return recipes.jsonsplit(value, type=type) 130 | 131 | else: 132 | fn = recipes.jsonsplit 133 | 134 | fresh_db["example"].convert("records", fn) 135 | assert json.loads(fresh_db["example"].get(1)["records"]) == expected 136 | -------------------------------------------------------------------------------- /tests/test_recreate.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils import Database 2 | import sqlite3 3 | import pathlib 4 | import pytest 5 | 6 | 7 | def test_recreate_ignored_for_in_memory(): 8 | # None of these should raise an exception: 9 | Database(memory=True, recreate=False) 10 | Database(memory=True, recreate=True) 11 | Database(":memory:", recreate=False) 12 | Database(":memory:", recreate=True) 13 | 14 | 15 | def test_recreate_not_allowed_for_connection(): 16 | conn = sqlite3.connect(":memory:") 17 | with pytest.raises(AssertionError): 18 | Database(conn, recreate=True) 19 | 20 | 21 | @pytest.mark.parametrize( 22 | "use_path,create_file_first", 23 | [(True, True), (True, False), (False, True), (False, False)], 24 | ) 25 | def test_recreate(tmp_path, use_path, create_file_first): 26 | filepath = str(tmp_path / "data.db") 27 | if use_path: 28 | filepath = pathlib.Path(filepath) 29 | if create_file_first: 30 | db = Database(filepath) 31 | db["t1"].insert({"foo": "bar"}) 32 | assert ["t1"] == db.table_names() 33 | db.close() 34 | Database(filepath, recreate=True)["t2"].insert({"foo": "bar"}) 35 | assert ["t2"] == Database(filepath).table_names() 36 | -------------------------------------------------------------------------------- /tests/test_register_function.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | import pytest 3 | import sys 4 | from unittest.mock import MagicMock, call 5 | from sqlite_utils.utils import sqlite3 6 | 7 | 8 | def test_register_function(fresh_db): 9 | @fresh_db.register_function 10 | def reverse_string(s): 11 | return "".join(reversed(list(s))) 12 | 13 | result = fresh_db.execute('select reverse_string("hello")').fetchone()[0] 14 | assert result == "olleh" 15 | 16 | 17 | def test_register_function_custom_name(fresh_db): 18 | @fresh_db.register_function(name="revstr") 19 | def reverse_string(s): 20 | return "".join(reversed(list(s))) 21 | 22 | result = fresh_db.execute('select revstr("hello")').fetchone()[0] 23 | assert result == "olleh" 24 | 25 | 26 | def test_register_function_multiple_arguments(fresh_db): 27 | @fresh_db.register_function 28 | def a_times_b_plus_c(a, b, c): 29 | return a * b + c 30 | 31 | result = fresh_db.execute("select a_times_b_plus_c(2, 3, 4)").fetchone()[0] 32 | assert result == 10 33 | 34 | 35 | def test_register_function_deterministic(fresh_db): 36 | @fresh_db.register_function(deterministic=True) 37 | def to_lower(s): 38 | return s.lower() 39 | 40 | result = fresh_db.execute("select to_lower('BOB')").fetchone()[0] 41 | assert result == "bob" 42 | 43 | 44 | def test_register_function_deterministic_tries_again_if_exception_raised(fresh_db): 45 | fresh_db.conn = MagicMock() 46 | fresh_db.conn.create_function = MagicMock() 47 | 48 | @fresh_db.register_function(deterministic=True) 49 | def to_lower_2(s): 50 | return s.lower() 51 | 52 | fresh_db.conn.create_function.assert_called_with( 53 | "to_lower_2", 1, to_lower_2, deterministic=True 54 | ) 55 | 56 | first = True 57 | 58 | def side_effect(*args, **kwargs): 59 | # Raise exception only first time this is called 60 | nonlocal first 61 | if first: 62 | first = False 63 | raise sqlite3.NotSupportedError() 64 | 65 | # But if sqlite3.NotSupportedError is raised, it tries again 66 | fresh_db.conn.create_function.reset_mock() 67 | fresh_db.conn.create_function.side_effect = side_effect 68 | 69 | @fresh_db.register_function(deterministic=True) 70 | def to_lower_3(s): 71 | return s.lower() 72 | 73 | # Should have been called once with deterministic=True and once without 74 | assert fresh_db.conn.create_function.call_args_list == [ 75 | call("to_lower_3", 1, to_lower_3, deterministic=True), 76 | call("to_lower_3", 1, to_lower_3), 77 | ] 78 | 79 | 80 | def test_register_function_replace(fresh_db): 81 | @fresh_db.register_function() 82 | def one(): 83 | return "one" 84 | 85 | assert "one" == fresh_db.execute("select one()").fetchone()[0] 86 | 87 | # This will silently fail to replaec the function 88 | @fresh_db.register_function() 89 | def one(): # noqa 90 | return "two" 91 | 92 | assert "one" == fresh_db.execute("select one()").fetchone()[0] 93 | 94 | # This will replace it 95 | @fresh_db.register_function(replace=True) 96 | def one(): # noqa 97 | return "two" 98 | 99 | assert "two" == fresh_db.execute("select one()").fetchone()[0] 100 | -------------------------------------------------------------------------------- /tests/test_rows.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def test_rows(existing_db): 5 | assert [{"text": "one"}, {"text": "two"}, {"text": "three"}] == list( 6 | existing_db["foo"].rows 7 | ) 8 | 9 | 10 | @pytest.mark.parametrize( 11 | "where,where_args,expected_ids", 12 | [ 13 | ("name = ?", ["Pancakes"], {2}), 14 | ("age > ?", [3], {1}), 15 | ("age > :age", {"age": 3}, {1}), 16 | ("name is not null", [], {1, 2}), 17 | ("is_good = ?", [True], {1, 2}), 18 | ], 19 | ) 20 | def test_rows_where(where, where_args, expected_ids, fresh_db): 21 | table = fresh_db["dogs"] 22 | table.insert_all( 23 | [ 24 | {"id": 1, "name": "Cleo", "age": 4, "is_good": True}, 25 | {"id": 2, "name": "Pancakes", "age": 3, "is_good": True}, 26 | ], 27 | pk="id", 28 | ) 29 | assert expected_ids == { 30 | r["id"] for r in table.rows_where(where, where_args, select="id") 31 | } 32 | 33 | 34 | @pytest.mark.parametrize( 35 | "where,order_by,expected_ids", 36 | [ 37 | (None, None, [1, 2, 3]), 38 | (None, "id desc", [3, 2, 1]), 39 | (None, "age", [3, 2, 1]), 40 | ("id > 1", "age", [3, 2]), 41 | ], 42 | ) 43 | def test_rows_where_order_by(where, order_by, expected_ids, fresh_db): 44 | table = fresh_db["dogs"] 45 | table.insert_all( 46 | [ 47 | {"id": 1, "name": "Cleo", "age": 4}, 48 | {"id": 2, "name": "Pancakes", "age": 3}, 49 | {"id": 3, "name": "Bailey", "age": 2}, 50 | ], 51 | pk="id", 52 | ) 53 | assert expected_ids == [r["id"] for r in table.rows_where(where, order_by=order_by)] 54 | 55 | 56 | @pytest.mark.parametrize( 57 | "offset,limit,expected", 58 | [ 59 | (None, 3, [1, 2, 3]), 60 | (0, 3, [1, 2, 3]), 61 | (3, 3, [4, 5, 6]), 62 | ], 63 | ) 64 | def test_rows_where_offset_limit(fresh_db, offset, limit, expected): 65 | table = fresh_db["rows"] 66 | table.insert_all([{"id": id} for id in range(1, 101)], pk="id") 67 | assert table.count == 100 68 | assert expected == [ 69 | r["id"] for r in table.rows_where(offset=offset, limit=limit, order_by="id") 70 | ] 71 | 72 | 73 | def test_pks_and_rows_where_rowid(fresh_db): 74 | table = fresh_db["rowid_table"] 75 | table.insert_all({"number": i + 10} for i in range(3)) 76 | pks_and_rows = list(table.pks_and_rows_where()) 77 | assert pks_and_rows == [ 78 | (1, {"rowid": 1, "number": 10}), 79 | (2, {"rowid": 2, "number": 11}), 80 | (3, {"rowid": 3, "number": 12}), 81 | ] 82 | 83 | 84 | def test_pks_and_rows_where_simple_pk(fresh_db): 85 | table = fresh_db["simple_pk_table"] 86 | table.insert_all(({"id": i + 10} for i in range(3)), pk="id") 87 | pks_and_rows = list(table.pks_and_rows_where()) 88 | assert pks_and_rows == [ 89 | (10, {"id": 10}), 90 | (11, {"id": 11}), 91 | (12, {"id": 12}), 92 | ] 93 | 94 | 95 | def test_pks_and_rows_where_compound_pk(fresh_db): 96 | table = fresh_db["compound_pk_table"] 97 | table.insert_all( 98 | ({"type": "number", "number": i, "plusone": i + 1} for i in range(3)), 99 | pk=("type", "number"), 100 | ) 101 | pks_and_rows = list(table.pks_and_rows_where()) 102 | assert pks_and_rows == [ 103 | (("number", 0), {"type": "number", "number": 0, "plusone": 1}), 104 | (("number", 1), {"type": "number", "number": 1, "plusone": 2}), 105 | (("number", 2), {"type": "number", "number": 2, "plusone": 3}), 106 | ] 107 | -------------------------------------------------------------------------------- /tests/test_rows_from_file.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils.utils import rows_from_file, Format, RowError 2 | from io import BytesIO, StringIO 3 | import pytest 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "input,expected_format", 8 | ( 9 | (b"id,name\n1,Cleo", Format.CSV), 10 | (b"id\tname\n1\tCleo", Format.TSV), 11 | (b'[{"id": "1", "name": "Cleo"}]', Format.JSON), 12 | ), 13 | ) 14 | def test_rows_from_file_detect_format(input, expected_format): 15 | rows, format = rows_from_file(BytesIO(input)) 16 | assert format == expected_format 17 | rows_list = list(rows) 18 | assert rows_list == [{"id": "1", "name": "Cleo"}] 19 | 20 | 21 | @pytest.mark.parametrize( 22 | "ignore_extras,extras_key,expected", 23 | ( 24 | (True, None, [{"id": "1", "name": "Cleo"}]), 25 | (False, "_rest", [{"id": "1", "name": "Cleo", "_rest": ["oops"]}]), 26 | # expected of None means expect an error: 27 | (False, False, None), 28 | ), 29 | ) 30 | def test_rows_from_file_extra_fields_strategies(ignore_extras, extras_key, expected): 31 | try: 32 | rows, format = rows_from_file( 33 | BytesIO(b"id,name\r\n1,Cleo,oops"), 34 | format=Format.CSV, 35 | ignore_extras=ignore_extras, 36 | extras_key=extras_key, 37 | ) 38 | list_rows = list(rows) 39 | except RowError: 40 | if expected is None: 41 | # This is fine, 42 | return 43 | else: 44 | # We did not expect an error 45 | raise 46 | assert list_rows == expected 47 | 48 | 49 | def test_rows_from_file_error_on_string_io(): 50 | with pytest.raises(TypeError) as ex: 51 | rows_from_file(StringIO("id,name\r\n1,Cleo")) 52 | assert ex.value.args == ( 53 | "rows_from_file() requires a file-like object that supports peek(), such as io.BytesIO", 54 | ) 55 | -------------------------------------------------------------------------------- /tests/test_sniff.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils import cli, Database 2 | from click.testing import CliRunner 3 | import pathlib 4 | import pytest 5 | 6 | sniff_dir = pathlib.Path(__file__).parent / "sniff" 7 | 8 | 9 | @pytest.mark.parametrize("filepath", sniff_dir.glob("example*")) 10 | def test_sniff(tmpdir, filepath): 11 | db_path = str(tmpdir / "test.db") 12 | runner = CliRunner() 13 | result = runner.invoke( 14 | cli.cli, 15 | ["insert", db_path, "creatures", str(filepath), "--sniff"], 16 | catch_exceptions=False, 17 | ) 18 | assert result.exit_code == 0, result.stdout 19 | db = Database(db_path) 20 | assert list(db["creatures"].rows) == [ 21 | {"id": "1", "species": "dog", "name": "Cleo", "age": "5"}, 22 | {"id": "2", "species": "dog", "name": "Pancakes", "age": "4"}, 23 | {"id": "3", "species": "cat", "name": "Mozie", "age": "8"}, 24 | {"id": "4", "species": "spider", "name": "Daisy, the tarantula", "age": "6"}, 25 | ] 26 | -------------------------------------------------------------------------------- /tests/test_suggest_column_types.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from collections import OrderedDict 3 | from sqlite_utils.utils import suggest_column_types 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "records,types", 8 | [ 9 | ([{"a": 1}], {"a": int}), 10 | ([{"a": 1}, {"a": None}], {"a": int}), 11 | ([{"a": "baz"}], {"a": str}), 12 | ([{"a": "baz"}, {"a": None}], {"a": str}), 13 | ([{"a": 1.2}], {"a": float}), 14 | ([{"a": 1.2}, {"a": None}], {"a": float}), 15 | ([{"a": [1]}], {"a": str}), 16 | ([{"a": [1]}, {"a": None}], {"a": str}), 17 | ([{"a": (1,)}], {"a": str}), 18 | ([{"a": {"b": 1}}], {"a": str}), 19 | ([{"a": {"b": 1}}, {"a": None}], {"a": str}), 20 | ([{"a": OrderedDict({"b": 1})}], {"a": str}), 21 | ([{"a": 1}, {"a": 1.1}], {"a": float}), 22 | ([{"a": b"b"}], {"a": bytes}), 23 | ([{"a": b"b"}, {"a": None}], {"a": bytes}), 24 | ([{"a": "a", "b": None}], {"a": str, "b": str}), 25 | ], 26 | ) 27 | def test_suggest_column_types(records, types): 28 | assert types == suggest_column_types(records) 29 | -------------------------------------------------------------------------------- /tests/test_tracer.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils import Database 2 | 3 | 4 | def test_tracer(): 5 | collected = [] 6 | db = Database( 7 | memory=True, tracer=lambda sql, params: collected.append((sql, params)) 8 | ) 9 | dogs = db.table("dogs") 10 | dogs.insert({"name": "Cleopaws"}) 11 | dogs.enable_fts(["name"]) 12 | dogs.search("Cleopaws") 13 | assert collected == [ 14 | ("PRAGMA recursive_triggers=on;", None), 15 | ("select name from sqlite_master where type = 'view'", None), 16 | ("select name from sqlite_master where type = 'table'", None), 17 | ("select name from sqlite_master where type = 'view'", None), 18 | ("select name from sqlite_master where type = 'view'", None), 19 | ("select name from sqlite_master where type = 'table'", None), 20 | ("select name from sqlite_master where type = 'view'", None), 21 | ("CREATE TABLE [dogs] (\n [name] TEXT\n);\n ", None), 22 | ("select name from sqlite_master where type = 'view'", None), 23 | ("INSERT INTO [dogs] ([name]) VALUES (?)", ["Cleopaws"]), 24 | ( 25 | "CREATE VIRTUAL TABLE [dogs_fts] USING FTS5 (\n [name],\n content=[dogs]\n)", 26 | None, 27 | ), 28 | ( 29 | "INSERT INTO [dogs_fts] (rowid, [name])\n SELECT rowid, [name] FROM [dogs];", 30 | None, 31 | ), 32 | ] 33 | 34 | 35 | def test_with_tracer(): 36 | collected = [] 37 | 38 | def tracer(sql, params): 39 | return collected.append((sql, params)) 40 | 41 | db = Database(memory=True) 42 | 43 | dogs = db.table("dogs") 44 | 45 | dogs.insert({"name": "Cleopaws"}) 46 | dogs.enable_fts(["name"]) 47 | 48 | assert len(collected) == 0 49 | 50 | with db.tracer(tracer): 51 | list(dogs.search("Cleopaws")) 52 | 53 | assert len(collected) == 5 54 | assert collected == [ 55 | ( 56 | "SELECT name FROM sqlite_master\n" 57 | " WHERE rootpage = 0\n" 58 | " AND (\n" 59 | " sql LIKE :like\n" 60 | " OR sql LIKE :like2\n" 61 | " OR (\n" 62 | " tbl_name = :table\n" 63 | " AND sql LIKE '%VIRTUAL TABLE%USING FTS%'\n" 64 | " )\n" 65 | " )", 66 | { 67 | "like": "%VIRTUAL TABLE%USING FTS%content=[dogs]%", 68 | "like2": '%VIRTUAL TABLE%USING FTS%content="dogs"%', 69 | "table": "dogs", 70 | }, 71 | ), 72 | ("select name from sqlite_master where type = 'view'", None), 73 | ("select name from sqlite_master where type = 'view'", None), 74 | ("select sql from sqlite_master where name = ?", ("dogs_fts",)), 75 | ( 76 | "with original as (\n" 77 | " select\n" 78 | " rowid,\n" 79 | " *\n" 80 | " from [dogs]\n" 81 | ")\n" 82 | "select\n" 83 | " [original].*\n" 84 | "from\n" 85 | " [original]\n" 86 | " join [dogs_fts] on [original].rowid = [dogs_fts].rowid\n" 87 | "where\n" 88 | " [dogs_fts] match :query\n" 89 | "order by\n" 90 | " [dogs_fts].rank", 91 | {"query": "Cleopaws"}, 92 | ), 93 | ] 94 | 95 | # Outside the with block collected should not be appended to 96 | dogs.insert({"name": "Cleopaws"}) 97 | assert len(collected) == 5 98 | -------------------------------------------------------------------------------- /tests/test_update.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import json 3 | 4 | import pytest 5 | 6 | from sqlite_utils.db import NotFoundError 7 | 8 | 9 | def test_update_rowid_table(fresh_db): 10 | table = fresh_db["table"] 11 | rowid = table.insert({"foo": "bar"}).last_pk 12 | table.update(rowid, {"foo": "baz"}) 13 | assert [{"foo": "baz"}] == list(table.rows) 14 | 15 | 16 | def test_update_pk_table(fresh_db): 17 | table = fresh_db["table"] 18 | pk = table.insert({"foo": "bar", "id": 5}, pk="id").last_pk 19 | assert 5 == pk 20 | table.update(pk, {"foo": "baz"}) 21 | assert [{"id": 5, "foo": "baz"}] == list(table.rows) 22 | 23 | 24 | def test_update_compound_pk_table(fresh_db): 25 | table = fresh_db["table"] 26 | pk = table.insert({"id1": 5, "id2": 3, "v": 1}, pk=("id1", "id2")).last_pk 27 | assert (5, 3) == pk 28 | table.update(pk, {"v": 2}) 29 | assert [{"id1": 5, "id2": 3, "v": 2}] == list(table.rows) 30 | 31 | 32 | @pytest.mark.parametrize( 33 | "pk,update_pk", 34 | ( 35 | (None, 2), 36 | (None, None), 37 | ("id1", None), 38 | ("id1", 4), 39 | (("id1", "id2"), None), 40 | (("id1", "id2"), 4), 41 | (("id1", "id2"), (4, 5)), 42 | ), 43 | ) 44 | def test_update_invalid_pk(fresh_db, pk, update_pk): 45 | table = fresh_db["table"] 46 | table.insert({"id1": 5, "id2": 3, "v": 1}, pk=pk).last_pk 47 | with pytest.raises(NotFoundError): 48 | table.update(update_pk, {"v": 2}) 49 | 50 | 51 | def test_update_alter(fresh_db): 52 | table = fresh_db["table"] 53 | rowid = table.insert({"foo": "bar"}).last_pk 54 | table.update(rowid, {"new_col": 1.2}, alter=True) 55 | assert [{"foo": "bar", "new_col": 1.2}] == list(table.rows) 56 | # Let's try adding three cols at once 57 | table.update( 58 | rowid, 59 | {"str_col": "str", "bytes_col": b"\xa0 has bytes", "int_col": -10}, 60 | alter=True, 61 | ) 62 | assert [ 63 | { 64 | "foo": "bar", 65 | "new_col": 1.2, 66 | "str_col": "str", 67 | "bytes_col": b"\xa0 has bytes", 68 | "int_col": -10, 69 | } 70 | ] == list(table.rows) 71 | 72 | 73 | def test_update_alter_with_invalid_column_characters(fresh_db): 74 | table = fresh_db["table"] 75 | rowid = table.insert({"foo": "bar"}).last_pk 76 | with pytest.raises(AssertionError): 77 | table.update(rowid, {"new_col[abc]": 1.2}, alter=True) 78 | 79 | 80 | def test_update_with_no_values_sets_last_pk(fresh_db): 81 | table = fresh_db.table("dogs", pk="id") 82 | table.insert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Pancakes"}]) 83 | table.update(1) 84 | assert table.last_pk == 1 85 | table.update(2) 86 | assert table.last_pk == 2 87 | with pytest.raises(NotFoundError): 88 | table.update(3) 89 | 90 | 91 | @pytest.mark.parametrize( 92 | "data_structure", 93 | ( 94 | ["list with one item"], 95 | ["list with", "two items"], 96 | {"dictionary": "simple"}, 97 | {"dictionary": {"nested": "complex"}}, 98 | collections.OrderedDict( 99 | [ 100 | ("key1", {"nested": "complex"}), 101 | ("key2", "foo"), 102 | ] 103 | ), 104 | [{"list": "of"}, {"two": "dicts"}], 105 | ), 106 | ) 107 | def test_update_dictionaries_and_lists_as_json(fresh_db, data_structure): 108 | fresh_db["test"].insert({"id": 1, "data": ""}, pk="id") 109 | fresh_db["test"].update(1, {"data": data_structure}) 110 | row = fresh_db.execute("select id, data from test").fetchone() 111 | assert row[0] == 1 112 | assert data_structure == json.loads(row[1]) 113 | -------------------------------------------------------------------------------- /tests/test_upsert.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils.db import PrimaryKeyRequired 2 | from sqlite_utils import Database 3 | import pytest 4 | 5 | 6 | @pytest.mark.parametrize("use_old_upsert", (False, True)) 7 | def test_upsert(use_old_upsert): 8 | db = Database(memory=True, use_old_upsert=use_old_upsert) 9 | table = db["table"] 10 | table.insert({"id": 1, "name": "Cleo"}, pk="id") 11 | table.upsert({"id": 1, "age": 5}, pk="id", alter=True) 12 | assert list(table.rows) == [{"id": 1, "name": "Cleo", "age": 5}] 13 | assert table.last_pk == 1 14 | 15 | 16 | def test_upsert_all(fresh_db): 17 | table = fresh_db["table"] 18 | table.upsert_all([{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Nixie"}], pk="id") 19 | table.upsert_all([{"id": 1, "age": 5}, {"id": 2, "age": 5}], pk="id", alter=True) 20 | assert list(table.rows) == [ 21 | {"id": 1, "name": "Cleo", "age": 5}, 22 | {"id": 2, "name": "Nixie", "age": 5}, 23 | ] 24 | assert table.last_pk is None 25 | 26 | 27 | def test_upsert_all_single_column(fresh_db): 28 | table = fresh_db["table"] 29 | table.upsert_all([{"name": "Cleo"}], pk="name") 30 | assert list(table.rows) == [{"name": "Cleo"}] 31 | assert table.pks == ["name"] 32 | 33 | 34 | def test_upsert_all_not_null(fresh_db): 35 | # https://github.com/simonw/sqlite-utils/issues/538 36 | fresh_db["comments"].upsert_all( 37 | [{"id": 1, "name": "Cleo"}], 38 | pk="id", 39 | not_null=["name"], 40 | ) 41 | assert list(fresh_db["comments"].rows) == [{"id": 1, "name": "Cleo"}] 42 | 43 | 44 | def test_upsert_error_if_no_pk(fresh_db): 45 | table = fresh_db["table"] 46 | with pytest.raises(PrimaryKeyRequired): 47 | table.upsert_all([{"id": 1, "name": "Cleo"}]) 48 | with pytest.raises(PrimaryKeyRequired): 49 | table.upsert({"id": 1, "name": "Cleo"}) 50 | 51 | 52 | def test_upsert_with_hash_id(fresh_db): 53 | table = fresh_db["table"] 54 | table.upsert({"foo": "bar"}, hash_id="pk") 55 | assert [{"pk": "a5e744d0164540d33b1d7ea616c28f2fa97e754a", "foo": "bar"}] == list( 56 | table.rows 57 | ) 58 | assert "a5e744d0164540d33b1d7ea616c28f2fa97e754a" == table.last_pk 59 | 60 | 61 | @pytest.mark.parametrize("hash_id", (None, "custom_id")) 62 | def test_upsert_with_hash_id_columns(fresh_db, hash_id): 63 | table = fresh_db["table"] 64 | table.upsert({"a": 1, "b": 2, "c": 3}, hash_id=hash_id, hash_id_columns=("a", "b")) 65 | assert list(table.rows) == [ 66 | { 67 | hash_id or "id": "4acc71e0547112eb432f0a36fb1924c4a738cb49", 68 | "a": 1, 69 | "b": 2, 70 | "c": 3, 71 | } 72 | ] 73 | assert table.last_pk == "4acc71e0547112eb432f0a36fb1924c4a738cb49" 74 | table.upsert({"a": 1, "b": 2, "c": 4}, hash_id=hash_id, hash_id_columns=("a", "b")) 75 | assert list(table.rows) == [ 76 | { 77 | hash_id or "id": "4acc71e0547112eb432f0a36fb1924c4a738cb49", 78 | "a": 1, 79 | "b": 2, 80 | "c": 4, 81 | } 82 | ] 83 | 84 | 85 | def test_upsert_compound_primary_key(fresh_db): 86 | table = fresh_db["table"] 87 | table.upsert_all( 88 | [ 89 | {"species": "dog", "id": 1, "name": "Cleo", "age": 4}, 90 | {"species": "cat", "id": 1, "name": "Catbag"}, 91 | ], 92 | pk=("species", "id"), 93 | ) 94 | assert table.last_pk is None 95 | table.upsert({"species": "dog", "id": 1, "age": 5}, pk=("species", "id")) 96 | assert ("dog", 1) == table.last_pk 97 | assert [ 98 | {"species": "dog", "id": 1, "name": "Cleo", "age": 5}, 99 | {"species": "cat", "id": 1, "name": "Catbag", "age": None}, 100 | ] == list(table.rows) 101 | # .upsert_all() with a single item should set .last_pk 102 | table.upsert_all([{"species": "cat", "id": 1, "age": 5}], pk=("species", "id")) 103 | assert ("cat", 1) == table.last_pk 104 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from sqlite_utils import utils 2 | import csv 3 | import io 4 | import pytest 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "input,expected,should_be_is", 9 | [ 10 | ({}, None, True), 11 | ({"foo": "bar"}, None, True), 12 | ( 13 | {"content": {"$base64": True, "encoded": "aGVsbG8="}}, 14 | {"content": b"hello"}, 15 | False, 16 | ), 17 | ], 18 | ) 19 | def test_decode_base64_values(input, expected, should_be_is): 20 | actual = utils.decode_base64_values(input) 21 | if should_be_is: 22 | assert actual is input 23 | else: 24 | assert actual == expected 25 | 26 | 27 | @pytest.mark.parametrize( 28 | "size,expected", 29 | ( 30 | (1, [["a"], ["b"], ["c"], ["d"]]), 31 | (2, [["a", "b"], ["c", "d"]]), 32 | (3, [["a", "b", "c"], ["d"]]), 33 | (4, [["a", "b", "c", "d"]]), 34 | ), 35 | ) 36 | def test_chunks(size, expected): 37 | input = ["a", "b", "c", "d"] 38 | chunks = list(map(list, utils.chunks(input, size))) 39 | assert chunks == expected 40 | 41 | 42 | def test_hash_record(): 43 | expected = "d383e7c0ba88f5ffcdd09be660de164b3847401a" 44 | assert utils.hash_record({"name": "Cleo", "twitter": "CleoPaws"}) == expected 45 | assert ( 46 | utils.hash_record( 47 | {"name": "Cleo", "twitter": "CleoPaws", "age": 7}, keys=("name", "twitter") 48 | ) 49 | == expected 50 | ) 51 | assert ( 52 | utils.hash_record({"name": "Cleo", "twitter": "CleoPaws", "age": 7}) != expected 53 | ) 54 | 55 | 56 | def test_maximize_csv_field_size_limit(): 57 | # Reset to default in case other tests have changed it 58 | csv.field_size_limit(utils.ORIGINAL_CSV_FIELD_SIZE_LIMIT) 59 | long_value = "a" * 131073 60 | long_csv = "id,text\n1,{}".format(long_value) 61 | fp = io.BytesIO(long_csv.encode("utf-8")) 62 | # Using rows_from_file should error 63 | with pytest.raises(csv.Error): 64 | rows, _ = utils.rows_from_file(fp, utils.Format.CSV) 65 | list(rows) 66 | # But if we call maximize_csv_field_size_limit() first it should be OK: 67 | utils.maximize_csv_field_size_limit() 68 | fp2 = io.BytesIO(long_csv.encode("utf-8")) 69 | rows2, _ = utils.rows_from_file(fp2, utils.Format.CSV) 70 | rows_list2 = list(rows2) 71 | assert len(rows_list2) == 1 72 | assert rows_list2[0]["id"] == "1" 73 | assert rows_list2[0]["text"] == long_value 74 | 75 | 76 | @pytest.mark.parametrize( 77 | "input,expected", 78 | ( 79 | ({"foo": {"bar": 1}}, {"foo_bar": 1}), 80 | ({"foo": {"bar": [1, 2, {"baz": 3}]}}, {"foo_bar": [1, 2, {"baz": 3}]}), 81 | ({"foo": {"bar": 1, "baz": {"three": 3}}}, {"foo_bar": 1, "foo_baz_three": 3}), 82 | ), 83 | ) 84 | def test_flatten(input, expected): 85 | assert utils.flatten(input) == expected 86 | -------------------------------------------------------------------------------- /tests/test_wal.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from sqlite_utils import Database 3 | 4 | 5 | @pytest.fixture 6 | def db_path_tmpdir(tmpdir): 7 | path = tmpdir / "test.db" 8 | db = Database(str(path)) 9 | return db, path, tmpdir 10 | 11 | 12 | def test_enable_disable_wal(db_path_tmpdir): 13 | db, path, tmpdir = db_path_tmpdir 14 | assert len(tmpdir.listdir()) == 1 15 | assert "delete" == db.journal_mode 16 | assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()] 17 | db.enable_wal() 18 | assert "wal" == db.journal_mode 19 | db["test"].insert({"foo": "bar"}) 20 | assert "test.db-wal" in [f.basename for f in tmpdir.listdir()] 21 | db.disable_wal() 22 | assert "delete" == db.journal_mode 23 | assert "test.db-wal" not in [f.basename for f in tmpdir.listdir()] 24 | --------------------------------------------------------------------------------