├── .git-blame-ignore-revs ├── .github └── workflows │ ├── lint.yaml │ └── test.yaml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── MIGRATING.md ├── README.md ├── environment.yaml ├── examples ├── example.md ├── folio_demo.py ├── output_13_1.png └── output_14_1.png ├── pdm.lock ├── precious.toml ├── pyproject.toml ├── src └── ldlite │ ├── __init__.py │ ├── _camelcase.py │ ├── _csv.py │ ├── _jsonx.py │ ├── _query.py │ ├── _request.py │ ├── _select.py │ ├── _sqlx.py │ ├── _xlsx.py │ └── py.typed ├── srs.md └── tests ├── __init__.py ├── conftest.py ├── test___init__.py ├── test_cases ├── base.py ├── drop_tables_cases.py ├── query_cases.py ├── to_csv_cases.py └── to_csv_samples │ ├── basic.csv │ ├── datatypes.csv │ ├── escaped_chars.csv │ └── sorting.csv ├── test_duckdb.py ├── test_postgres.py └── test_sqlite.py /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | ### Commits to bring in typing, linting, and formatting ### 2 | 48d70aea77b99b23ff93c62d5d0af22e2c1e4ef1 3 | d699f3575113b42e6943c6a2c0e9d739052ec321 4 | b48aad82200a8d457e59ccb9492ed75bb7431e85 5 | ### 6 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema: https://raw.githubusercontent.com/SchemaStore/schemastore/refs/heads/master/src/schemas/json/github-workflow.json 2 | --- 3 | name: Lint 4 | 5 | on: 6 | workflow_dispatch: 7 | pull_request: 8 | types: 9 | - opened 10 | - review_requested 11 | - ready_for_review 12 | - auto_merge_enabled 13 | 14 | jobs: 15 | lint: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | 20 | # install dependencies 21 | - uses: pdm-project/setup-pdm@v4 22 | with: 23 | cache: true 24 | - run: pdm install -G:all 25 | - uses: jaxxstorm/action-install-gh-release@v1.10.0 26 | with: 27 | repo: houseabsolute/precious 28 | tag: v0.9.0 29 | cache: enable 30 | 31 | # lint 32 | - run: pdm run precious lint --all 33 | - run: | 34 | set -eoux pipefail 35 | 36 | pdm run precious tidy --all 37 | git diff --exit-code 38 | if: success() || failure() 39 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema: https://raw.githubusercontent.com/SchemaStore/schemastore/refs/heads/master/src/schemas/json/github-workflow.json 2 | --- 3 | name: Test 4 | 5 | on: 6 | workflow_dispatch: 7 | pull_request: 8 | types: 9 | - opened 10 | - review_requested 11 | - ready_for_review 12 | - auto_merge_enabled 13 | 14 | jobs: 15 | test: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | 20 | # install dependencies 21 | - uses: pdm-project/setup-pdm@v4 22 | with: 23 | cache: true 24 | - run: pdm install -G test 25 | 26 | # test 27 | - run: pdm run test 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | !precious.toml 2 | ldlite.db 3 | ldlite.db.wal 4 | 5 | .coverage 6 | __pycache__ 7 | dist 8 | ldlite.egg-info 9 | tmp 10 | .idea 11 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Pull requests are not currently accepted for this project. 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MIGRATING.md: -------------------------------------------------------------------------------- 1 | # Migrating to a newer version of LDLite 2 | 3 | LDLite follows the [Semantic Versioning](https://semver.org/). 4 | For the most part, it should be safe to upgrade LDLite to the latest MINOR and PATCH versions. 5 | This guide is intended to be for MAJOR version updates. 6 | Please consult the documentation for your package manager of choice to understand how to receive minor updates automatically. 7 | 8 | To check your existing ldlite version use 9 | ``` 10 | python -m pip freeze | grep ldlite 11 | ``` 12 | or the equivalent command for your package manager. 13 | If you'd like support or assistance upgrading please feel free to reach out to ldlite-support@fivecolleges.edu or the #ldlite channel in Slack. 14 | 15 | ## Latest Major Release 16 | 17 | ### 1.0.0 - The Sunflower Ready Release 18 | 19 | The Sunflower release of FOLIO is bringing some necessary security changes that impact how integrations connect to the API. 20 | 1. Refresh Token Rotation was introduced in the Poppy release and will be the only authentication method as of Sunflower. 21 | 1. Eureka is a new platform for Auth and Routing using open source technologies, replacing Okapi which is proprietary to FOLIO. 22 | 23 | ##### Steps to upgrade from 0.0.36 or below 24 | 25 | Please upgrade to 0.1.0 first. 26 | You can consult the [tags on the ldlite repository](https://github.com/library-data-platform/ldlite/tags) to see what issues you might encounter. 27 | 28 | Upgrade from 0.0.36 to 0.1.0 by running 29 | ``` 30 | python -m pip install --upgrade 'ldlite==0.1.0' 31 | ``` 32 | or the equivalent command in your package manager of choice. 33 | 34 | ##### Steps to upgrade from 0.1.0 35 | 36 | First, update all of the places you're connecting to FOLIO to use Refresh Token Rotation auth 37 | ``` 38 | # change any of these calls 39 | ld.connect_okapi(url="...", tenant="...", user="...", password="...") 40 | ld.connect_okapi(url="...", tenant="...", user="...", password="...", legacy_auth=True) 41 | ld.connect_okapi_token(url="...", tenant="...", token="...") 42 | 43 | # to this 44 | ld.connect_okapi(url="...", tenant="...", user="...", password="...", legacy_auth=False) 45 | ``` 46 | Verify that ldlite continues to function normally. 47 | 48 | Once you have made and verified these changes you're ready to upgrade to 1.0.0 by running 49 | ``` 50 | python -m pip install --upgrade 'ldlite==1.0.0' 51 | ``` 52 | or the equivalent command in your package manager of choice. 53 | 54 | After upgrading, change the places you're connecting to FOLIO to the non-Okapi specific method 55 | ``` 56 | # change this call 57 | ld.connect_okapi(url="...", tenant="...", user="...", password="...", legacy_auth=False) 58 | 59 | # to this 60 | ld.connect_folio(url="...", tenant="...", user="...", password="...") 61 | ``` 62 | Verify that ldlite continues to function normally. 63 | 64 | You're now ready for Sunflower and Eureka! Please note, the url you use to connect to Eureka will change from the one you are using for Okapi. 65 | After upgrading FOLIO you can find the Eureka URL in the same location as the Okapi URL: 66 | > Settings > Software versions > Services > On url 67 | 68 | ## Previous Major Releases 69 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | LDLite 2 | ====== 3 | 4 | Copyright (C) 2021-2022 The Open Library Foundation. This software is 5 | distributed under the terms of the Apache License, Version 2.0. See 6 | the file 7 | [LICENSE](https://github.com/library-data-platform/ldlite/blob/master/LICENSE) 8 | for more information. 9 | 10 | LDLite is a lightweight, open source reporting tool for FOLIO 11 | services. It is part of the Library Data Platform project and 12 | provides basic LDP functions without requiring the server to be 13 | installed. 14 | 15 | To install LDLite or upgrade to the latest version: 16 | 17 | ```bash 18 | $ python -m pip install --upgrade ldlite 19 | ``` 20 | 21 | (On some systems it might be `python3` rather than `python`.) 22 | Check out the [migration guide](./MIGRATING.md) for more information about major version upgrades. 23 | 24 | > [!Warning] 25 | > The legacy /auth/login endpoint with a non expiring token is going to be removed in the Sunflower release. 26 | > As of 1.0.0 this library defaults to using the newer /authn/login-with-expiry. 27 | > Because of these changes the connect_okapi_token method will no longer function and will be removed as part of the 2.0.0 release. 28 | > For more information on these changes see: https://folio-org.atlassian.net/wiki/spaces/FOLIJET/pages/1396980/Refresh+Token+Rotation+RTR 29 | 30 | To extract and transform data: 31 | 32 | ```python 33 | $ python 34 | >> > import ldlite 35 | >> > ld = ldlite.LDLite() 36 | >> > ld.connect_folio(url='https://folio-etesting-snapshot-kong.ci.folio.org', 37 | tenant='diku', 38 | user='diku_admin', 39 | password='admin') 40 | >> > db = ld.connect_db() 41 | >> > _ = ld.query(table='g', path='/groups', query='cql.allRecords=1 sortby id') 42 | ldlite: querying: / groups 43 | ldlite: created 44 | tables: g, g__t, g__tcatalog 45 | >> > ld.select(table='g__t') 46 | ``` 47 | 48 | ``` 49 | __id | id | desc | expiration_offset_in_days | group 50 | ------+--------------------------------------+-----------------------+---------------------------+----------- 51 | 1 | 3684a786-6671-4268-8ed0-9db82ebca60b | Staff Member | 730 | staff 52 | 2 | 503a81cd-6c26-400f-b620-14c08943697c | Faculty Member | 365 | faculty 53 | 3 | ad0bc554-d5bc-463c-85d1-5562127ae91b | Graduate Student | | graduate 54 | 4 | bdc2b6d4-5ceb-4a12-ab46-249b9a68473e | Undergraduate Student | | undergrad 55 | (4 rows) 56 | ``` 57 | 58 | ```python 59 | >> > _ = ld.query(table='u', path='/users', query='cql.allRecords=1 sortby id') 60 | ldlite: querying: / users 61 | ldlite: created 62 | tables: u, u__t, u__t__departments, u__t__personal__addresses, u__t__proxy_for, u__tcatalog 63 | >> > cur = db.cursor() 64 | >> > _ = cur.execute(""" 65 | CREATE TABLE user_groups AS 66 | SELECT u__t.id, u__t.username, g__t.group 67 | FROM u__t 68 | JOIN g__t ON u__t.patron_group = g__t.id; 69 | """) 70 | >> > ld.export_excel(table='user_groups', filename='groups.xlsx') 71 | ``` 72 | 73 | Features 74 | -------- 75 | 76 | * Queries FOLIO modules and transforms JSON data into tables for 77 | easier reporting 78 | * Full SQL query support and export to CSV or Excel 79 | * Compatible with DBeaver database tool 80 | * Compatible with DuckDB and PostgreSQL database systems 81 | * PostgreSQL support enables: 82 | * Sharing the data in a multiuser database 83 | * Access to the data using more database tools 84 | * Storing the data in an existing LDP database if available 85 | * Runs on Windows, macOS, and Linux. 86 | 87 | More examples 88 | ------------- 89 | 90 | * [An example running in Jupyter 91 | Notebook](https://github.com/library-data-platform/ldlite/blob/main/examples/example.md) 92 | 93 | * [Loading sample data from FOLIO demo 94 | sites](https://github.com/library-data-platform/ldlite/blob/main/examples/folio_demo.py) 95 | 96 | * [Using LDLite with SRS MARC data](https://github.com/library-data-platform/ldlite/blob/main/srs.md) 97 | 98 | LDLite resources 99 | ---------------- 100 | 101 | * [LDLite API documentation](https://library-data-platform.github.io/ldlite/ldlite.html) 102 | 103 | * The LDP project runs a Slack workspace which is a good place to ask 104 | questions or to share your work. It also serves as a community space 105 | for working together on library data problems. To request an invitation, 106 | use the [Contact page](https://librarydataplatform.org/contact/) 107 | on the LDP website. 108 | 109 | * Report bugs at [Issues](https://github.com/library-data-platform/ldlite/issues) 110 | 111 | Other resources 112 | --------------- 113 | 114 | * [FOLIO API documentation](https://dev.folio.org/reference/api/) 115 | 116 | * [Python learning resources](https://www.python.org/about/gettingstarted/) 117 | -------------------------------------------------------------------------------- /environment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: ldlite 3 | channels: 4 | - conda-forge 5 | - nodefaults 6 | dependencies: 7 | - python>=3.7,<3.10 8 | - pdm==2.24.1 9 | - postgresql==17.5 10 | - precious==0.9.0 11 | -------------------------------------------------------------------------------- /examples/example.md: -------------------------------------------------------------------------------- 1 | ```python 2 | # First we import the "ldlite" package which allows us to use LDLite functions. 3 | 4 | import ldlite 5 | ``` 6 | 7 | ```python 8 | # We also have to initialize LDLite. This will create a variable "ld" which we 9 | # can use to call LDLite functions. 10 | 11 | ld = ldlite.LDLite() 12 | ``` 13 | 14 | ```python 15 | # LDLite must be configured to connect to a FOLIO instance and to a database 16 | # that will be used for reporting. To configure the FOLIO connection, we use 17 | # the function "ld.connect_folio()". 18 | 19 | ld.connect_folio(url='https://folio-etesting-snapshot-kong.ci.folio.org', 20 | tenant='diku', 21 | user='diku_admin', 22 | password='admin') 23 | ``` 24 | 25 | ```python 26 | # Next we call "ld.connect_db()" to create a database and connect it to DBLite. 27 | # The database will be stored in a file called "ldlite.db". 28 | 29 | db = ld.connect_db(filename='ldlite.db') 30 | ``` 31 | 32 | ```python 33 | # The function "ld.query()" is used to send CQL queries to FOLIO. In this case 34 | # we will query patron groups and store the result in a new table named "g". 35 | # In addition to "g", this will create other tables having names beginning with 36 | # "g__t" where JSON data will be transformed to tables. 37 | 38 | _ = ld.query(table='g', path='/groups', query='cql.allRecords=1 sortby id') 39 | ``` 40 | 41 | ldlite: querying: /groups 42 | ldlite: created tables: g, g__t, g__tcatalog 43 | 44 | ```python 45 | # We can use "ld.select()" to show a quick view of a table, in this case table 46 | # "g". 47 | 48 | ld.select(table='g', limit=10) 49 | ``` 50 | 51 | __id | jsonb 52 | ------+--------------------------------------------------------- 53 | 1 | { 54 | | "group": "staff", 55 | | "desc": "Staff Member", 56 | | "id": "3684a786-6671-4268-8ed0-9db82ebca60b", 57 | | "expirationOffsetInDays": 730, 58 | | "metadata": { 59 | | "createdDate": "2021-09-20T01:53:31.055+00:00", 60 | | "updatedDate": "2021-09-20T01:53:31.055+00:00" 61 | | } 62 | | } 63 | 2 | { 64 | | "group": "faculty", 65 | | "desc": "Faculty Member", 66 | | "id": "503a81cd-6c26-400f-b620-14c08943697c", 67 | | "expirationOffsetInDays": 365, 68 | | "metadata": { 69 | | "createdDate": "2021-09-20T01:53:31.084+00:00", 70 | | "updatedDate": "2021-09-20T01:53:31.084+00:00" 71 | | } 72 | | } 73 | 3 | { 74 | | "group": "graduate", 75 | | "desc": "Graduate Student", 76 | | "id": "ad0bc554-d5bc-463c-85d1-5562127ae91b", 77 | | "metadata": { 78 | | "createdDate": "2021-09-20T01:53:31.108+00:00", 79 | | "updatedDate": "2021-09-20T01:53:31.108+00:00" 80 | | } 81 | | } 82 | 4 | { 83 | | "group": "undergrad", 84 | | "desc": "Undergraduate Student", 85 | | "id": "bdc2b6d4-5ceb-4a12-ab46-249b9a68473e", 86 | | "metadata": { 87 | | "createdDate": "2021-09-20T01:53:31.123+00:00", 88 | | "updatedDate": "2021-09-20T01:53:31.123+00:00" 89 | | } 90 | | } 91 | (4 rows) 92 | 93 | ```python 94 | # When ld.query() created table "g", it also created another table 95 | # "g__t" with JSON fields extracted into columns. 96 | 97 | ld.select(table='g__t', limit=10) 98 | ``` 99 | 100 | __id | id | desc | expiration_offset_in_days | group 101 | ------+--------------------------------------+-----------------------+---------------------------+----------- 102 | 1 | 3684a786-6671-4268-8ed0-9db82ebca60b | Staff Member | 730 | staff 103 | 2 | 503a81cd-6c26-400f-b620-14c08943697c | Faculty Member | 365 | faculty 104 | 3 | ad0bc554-d5bc-463c-85d1-5562127ae91b | Graduate Student | | graduate 105 | 4 | bdc2b6d4-5ceb-4a12-ab46-249b9a68473e | Undergraduate Student | | undergrad 106 | (4 rows) 107 | 108 | ```python 109 | # We will also query user data and store the result in table "u" etc. 110 | 111 | _ = ld.query(table='u', path='/users', query='cql.allRecords=1 sortby id') 112 | ``` 113 | 114 | ldlite: querying: /users 115 | ldlite: created tables: u, u__t, u__t__departments, u__t__personal__addresses, u__t__proxy_for, u__tcatalog 116 | 117 | ```python 118 | # We can now join table "u" to table "g" and generate a list of user names and 119 | # their associated group. The result will be stored in a new table 120 | # "user_groups". 121 | 122 | cur = db.cursor() 123 | 124 | cur.execute(""" 125 | CREATE TABLE user_groups AS 126 | SELECT u__t.id, u__t.username, g__t.group 127 | FROM u__t 128 | JOIN g__t ON u__t.patron_group = g__t.id; 129 | """) 130 | 131 | ld.select(table='user_groups', limit=10) 132 | ``` 133 | 134 | id | username | group 135 | --------------------------------------+----------+----------- 136 | 00bc2807-4d5b-4a27-a2b5-b7b1ba431cc4 | sallie | faculty 137 | 011dc219-6b7f-4d93-ae7f-f512ed651493 | elmer | staff 138 | 01b9d72b-9aab-4efd-97a4-d03c1667bf0d | rick1 | graduate 139 | 0414af69-f89c-40f2-bea9-a9b5d0a179d4 | diana | faculty 140 | 046353cf-3963-482c-9792-32ade0a33afa | jorge | faculty 141 | 04e1cda1-a049-463b-97af-98c59a8fd806 | nicola | undergrad 142 | 066795ce-4938-48f2-9411-f3f922b51e1c | arlo | graduate 143 | 07066a1f-1fb7-4793-bbca-7cd8d1ea90ab | vergie | faculty 144 | 08522da4-668a-4450-a769-3abfae5678ad | johan | staff 145 | 0a246f61-d85f-42b6-8dcc-48d25a46690b | maxine | staff 146 | (10 rows) 147 | 148 | ```python 149 | # The "user_groups" table can also be exported to a CSV or Excel file. 150 | 151 | ld.export_csv(table='user_groups', filename='user_groups.csv') 152 | 153 | ld.export_excel(table='user_groups', filename='user_groups.xlsx') 154 | ``` 155 | 156 | ```python 157 | # We can look at the distribution of groups. 158 | 159 | df = cur.execute(""" 160 | SELECT coalesce("group", 'unknown') AS user_group, 161 | count(*) AS count 162 | FROM user_groups 163 | GROUP BY user_group 164 | ORDER BY count DESC; 165 | """).fetchdf() 166 | 167 | print(df) 168 | ``` 169 | 170 | user_group count 171 | 0 staff 105 172 | 1 undergrad 88 173 | 2 faculty 71 174 | 3 graduate 62 175 | 176 | ```python 177 | # We can plot the distribution with a bar graph. 178 | 179 | import matplotlib 180 | 181 | _ = df.plot(kind='bar', x='user_group') 182 | ``` 183 | 184 | ![png](output_13_1.png) 185 | 186 | ```python 187 | # Or a pie chart. 188 | 189 | _ = df.plot(kind='pie', x='user_group', y='count') 190 | ``` 191 | 192 | ![png](output_14_1.png) 193 | -------------------------------------------------------------------------------- /examples/folio_demo.py: -------------------------------------------------------------------------------- 1 | # This script uses LDLite to extract sample data from the FOLIO demo sites. 2 | from __future__ import annotations 3 | 4 | import sys 5 | 6 | import ldlite 7 | 8 | # Demo sites 9 | okapi_snapshot = "https://folio-snapshot-okapi.dev.folio.org" 10 | eureka_snapshot = "https://folio-etesting-snapshot-kong.ci.folio.org" 11 | 12 | ############################################################################### 13 | # Select a demo site here: 14 | selected_site = okapi_snapshot 15 | ############################################################################### 16 | # Note that these demo sites are unavailable at certain times in the evening 17 | # (Eastern time) or if a bug is introduced and makes one of them unresponsive. 18 | # For information about the status of the demo sites, please see the 19 | # hosted-reference-envs channel in the FOLIO Slack organization. For general 20 | # information about FOLIO demo sites, see the "Reference Environments" section 21 | # of the FOLIO Wiki at: 22 | # https://folio-org.atlassian.net/wiki/spaces/FOLIJET/pages/513704182/Reference+environments 23 | ############################################################################### 24 | 25 | ld = ldlite.LDLite() 26 | if selected_site == okapi_snapshot: 27 | ld.connect_okapi( 28 | url=selected_site, 29 | tenant="diku", 30 | user="diku_admin", 31 | password="admin", 32 | ) 33 | else: 34 | ld.connect_folio( 35 | url=selected_site, 36 | tenant="diku", 37 | user="diku_admin", 38 | password="admin", 39 | ) 40 | 41 | ld.connect_db(filename="ldlite.db") 42 | # For PostgreSQL, use connect_db_postgresql() instead of connect_db(): 43 | # ld.connect_db_postgresql(dsn='dbname=ldlite host=localhost user=ldlite') 44 | 45 | allrec = "cql.allRecords=1 sortby id" 46 | 47 | queries: list[tuple[str, ...] | tuple[str, str, object, int]] = [ 48 | ("folio_agreements.entitlement", "/erm/entitlements", allrec), 49 | ("folio_agreements.erm_resource", "/erm/resource", allrec), 50 | ("folio_agreements.org", "/erm/org", allrec), 51 | ("folio_agreements.refdata_value", "/erm/refdata", allrec), 52 | ("folio_agreements.usage_data_provider", "/usage-data-providers", allrec), 53 | ("folio_audit.circulation_logs", "/audit-data/circulation/logs", allrec), 54 | ("folio_circulation.audit_loan", "/loan-storage/loan-history", allrec), 55 | ( 56 | "folio_circulation.cancellation_reason", 57 | "/cancellation-reason-storage/cancellation-reasons", 58 | allrec, 59 | ), 60 | ("folio_circulation.check_in", "/check-in-storage/check-ins", allrec), 61 | ( 62 | "folio_circulation.fixed_due_date_schedule", 63 | "/fixed-due-date-schedule-storage/fixed-due-date-schedules", 64 | allrec, 65 | ), 66 | ("folio_circulation.loan", "/loan-storage/loans", allrec), 67 | ("folio_circulation.loan_policy", "/loan-policy-storage/loan-policies", allrec), 68 | ( 69 | "folio_circulation.patron_action_session", 70 | "/patron-action-session-storage/patron-action-sessions", 71 | allrec, 72 | ), 73 | ( 74 | "folio_circulation.patron_notice_policy", 75 | "/patron-notice-policy-storage/patron-notice-policies", 76 | allrec, 77 | ), 78 | ("folio_circulation.request", "/request-storage/requests", allrec), 79 | ( 80 | "folio_circulation.request_policy", 81 | "/request-policy-storage/request-policies", 82 | allrec, 83 | ), 84 | ( 85 | "folio_circulation.scheduled_notice", 86 | "/scheduled-notice-storage/scheduled-notices", 87 | allrec, 88 | ), 89 | ("folio_circulation.staff_slips", "/staff-slips-storage/staff-slips", allrec), 90 | ( 91 | "folio_circulation.user_request_preference", 92 | "/request-preference-storage/request-preference", 93 | allrec, 94 | ), 95 | ("folio_configuration.config_data", "/configurations/entries", allrec), 96 | ( 97 | "folio_courses.coursereserves_copyrightstates", 98 | "/coursereserves/copyrightstatuses", 99 | allrec, 100 | ), 101 | ( 102 | "folio_courses.coursereserves_courselistings", 103 | "/coursereserves/courselistings", 104 | allrec, 105 | ), 106 | ("folio_courses.coursereserves_courses", "/coursereserves/courses", allrec), 107 | ("folio_courses.coursereserves_coursetypes", "/coursereserves/coursetypes", allrec), 108 | ("folio_courses.coursereserves_departments", "/coursereserves/departments", allrec), 109 | ( 110 | "folio_courses.coursereserves_processingstates", 111 | "/coursereserves/processingstatuses", 112 | allrec, 113 | ), 114 | ("folio_courses.coursereserves_reserves", "/coursereserves/reserves", allrec), 115 | ("folio_courses.coursereserves_roles", "/coursereserves/roles", allrec), 116 | ("folio_courses.coursereserves_terms", "/coursereserves/terms", allrec), 117 | ("folio_erm_usage.counter_reports", "/counter-reports", allrec), 118 | ("folio_feesfines.accounts", "/accounts", allrec), 119 | ("folio_feesfines.comments", "/comments", allrec), 120 | ("folio_feesfines.feefineactions", "/feefineactions", allrec), 121 | ("folio_feesfines.feefines", "/feefines", allrec), 122 | ("folio_feesfines.lost_item_fee_policy", "/lost-item-fees-policies", allrec), 123 | ("folio_feesfines.manualblocks", "/manualblocks", allrec), 124 | ("folio_feesfines.overdue_fine_policy", "/overdue-fines-policies", allrec), 125 | ("folio_feesfines.owners", "/owners", allrec), 126 | ("folio_feesfines.payments", "/payments", allrec), 127 | ("folio_feesfines.refunds", "/refunds", allrec), 128 | ("folio_feesfines.transfer_criteria", "/transfer-criterias", allrec), 129 | ("folio_feesfines.transfers", "/transfers", allrec), 130 | ("folio_feesfines.waives", "/waives", allrec), 131 | ("folio_finance.budget", "/finance-storage/budgets", allrec), 132 | ("folio_finance.expense_class", "/finance-storage/expense-classes", allrec), 133 | ("folio_finance.fiscal_year", "/finance-storage/fiscal-years", allrec), 134 | ("folio_finance.fund", "/finance-storage/funds", allrec), 135 | ("folio_finance.fund_type", "/finance-storage/fund-types", allrec), 136 | ( 137 | "folio_finance.group_fund_fiscal_year", 138 | "/finance-storage/group-fund-fiscal-years", 139 | allrec, 140 | ), 141 | ("folio_finance.groups", "/finance-storage/groups", allrec), 142 | ("folio_finance.ledger", "/finance-storage/ledgers", allrec), 143 | ("folio_finance.transaction", "/finance-storage/transactions", allrec), 144 | ("folio_inventory.alternative_title_type", "/alternative-title-types", allrec), 145 | ("folio_inventory.call_number_type", "/call-number-types", allrec), 146 | ("folio_inventory.classification_type", "/classification-types", allrec), 147 | ("folio_inventory.contributor_name_type", "/contributor-name-types", allrec), 148 | ("folio_inventory.contributor_type", "/contributor-types", allrec), 149 | ( 150 | "folio_inventory.electronic_access_relationship", 151 | "/electronic-access-relationships", 152 | allrec, 153 | ), 154 | ("folio_inventory.holdings_note_type", "/holdings-note-types", allrec), 155 | ("folio_inventory.holdings_record", "/holdings-storage/holdings", allrec), 156 | ("folio_inventory.holdings_records_source", "/holdings-sources", allrec), 157 | ("folio_inventory.holdings_type", "/holdings-types", allrec), 158 | ("folio_inventory.identifier_type", "/identifier-types", allrec), 159 | ("folio_inventory.ill_policy", "/ill-policies", allrec), 160 | ("folio_inventory.instance", "/instance-storage/instances", allrec), 161 | ("folio_inventory.instance_format", "/instance-formats", allrec), 162 | ("folio_inventory.instance_note_type", "/instance-note-types", allrec), 163 | ( 164 | "folio_inventory.instance_relationship", 165 | "/instance-storage/instance-relationships", 166 | allrec, 167 | ), 168 | ( 169 | "folio_inventory.instance_relationship_type", 170 | "/instance-relationship-types", 171 | allrec, 172 | ), 173 | ("folio_inventory.instance_status", "/instance-statuses", allrec), 174 | ("folio_inventory.instance_type", "/instance-types", allrec), 175 | ("folio_inventory.item", "/item-storage/items", allrec), 176 | ("folio_inventory.item_damaged_status", "/item-damaged-statuses", allrec), 177 | ("folio_inventory.item_note_type", "/item-note-types", allrec), 178 | ("folio_inventory.loan_type", "/loan-types", allrec), 179 | ("folio_inventory.location", "/locations", allrec), 180 | ("folio_inventory.loccampus", "/location-units/campuses", allrec), 181 | ("folio_inventory.locinstitution", "/location-units/institutions", allrec), 182 | ("folio_inventory.loclibrary", "/location-units/libraries", allrec), 183 | ("folio_inventory.material_type", "/material-types", allrec), 184 | ("folio_inventory.mode_of_issuance", "/modes-of-issuance", allrec), 185 | ("folio_inventory.nature_of_content_term", "/nature-of-content-terms", allrec), 186 | ("folio_inventory.service_point", "/service-points", allrec), 187 | ("folio_inventory.service_point_user", "/service-points-users", allrec), 188 | ("folio_inventory.statistical_code", "/statistical-codes", allrec), 189 | ("folio_inventory.statistical_code_type", "/statistical-code-types", allrec), 190 | ("folio_invoice.invoice_lines", "/invoice-storage/invoice-lines", allrec), 191 | ("folio_invoice.invoices", "/invoice-storage/invoices", allrec), 192 | ("folio_invoice.voucher_lines", "/voucher-storage/voucher-lines", allrec), 193 | ("folio_invoice.vouchers", "/voucher-storage/vouchers", allrec), 194 | ("folio_licenses.license", "/licenses/licenses", allrec), 195 | ("folio_notes.note_data", "/notes", allrec), 196 | ("folio_orders.acquisitions_unit", "/acquisitions-units-storage/units", allrec), 197 | ( 198 | "folio_orders.acquisitions_unit_membership", 199 | "/acquisitions-units-storage/memberships", 200 | allrec, 201 | ), 202 | ("folio_orders.alert", "/orders-storage/alerts", allrec), 203 | ( 204 | "folio_orders.order_invoice_relationship", 205 | "/orders-storage/order-invoice-relns", 206 | allrec, 207 | ), 208 | ("folio_orders.order_templates", "/orders-storage/order-templates", allrec), 209 | ("folio_orders.pieces", "/orders-storage/pieces", allrec), 210 | ("folio_orders.po_line", "/orders-storage/po-lines", allrec), 211 | ("folio_orders.purchase_order", "/orders-storage/purchase-orders", allrec), 212 | ("folio_orders.reporting_code", "/orders-storage/reporting-codes", allrec), 213 | ("folio_organizations.addresses", "/organizations-storage/addresses", allrec), 214 | ("folio_organizations.categories", "/organizations-storage/categories", allrec), 215 | ("folio_organizations.contacts", "/organizations-storage/contacts", allrec), 216 | ("folio_organizations.emails", "/organizations-storage/emails", allrec), 217 | ("folio_organizations.interfaces", "/organizations-storage/interfaces", allrec), 218 | ( 219 | "folio_organizations.organizations", 220 | "/organizations-storage/organizations", 221 | allrec, 222 | ), 223 | ( 224 | "folio_organizations.phone_numbers", 225 | "/organizations-storage/phone-numbers", 226 | allrec, 227 | ), 228 | ("folio_organizations.urls", "/organizations-storage/urls", allrec), 229 | ("folio_source_record.records", "/source-storage/records", {}, 2), 230 | ("folio_users.addresstype", "/addresstypes", allrec), 231 | ("folio_users.departments", "/departments", allrec), 232 | ("folio_users.groups", "/groups", allrec), 233 | ("folio_users.proxyfor", "/proxiesfor", allrec), 234 | ("folio_users.users", "/users", allrec), 235 | ] 236 | 237 | tables: list[str] = [] 238 | for q in queries: 239 | try: 240 | if len(q) == 4: 241 | tables += ld.query( 242 | table=q[0], 243 | path=q[1], 244 | query=str(q[2]), 245 | json_depth=int(q[3]), 246 | ) 247 | else: 248 | tables += ld.query(table=q[0], path=q[1], query=str(q[2])) 249 | except (ValueError, RuntimeError) as e: 250 | print( 251 | 'folio_demo.py: error processing "' + str(q[1]) + '": ' + str(e), 252 | file=sys.stderr, 253 | ) 254 | print() 255 | print("Tables:") 256 | for t in tables: 257 | print(t) 258 | print("(" + str(len(tables)) + " tables)") 259 | -------------------------------------------------------------------------------- /examples/output_13_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/library-data-platform/ldlite/f052dfb7a7a77c390f69a509f96a7b9f96c871f0/examples/output_13_1.png -------------------------------------------------------------------------------- /examples/output_14_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/library-data-platform/ldlite/f052dfb7a7a77c390f69a509f96a7b9f96c871f0/examples/output_14_1.png -------------------------------------------------------------------------------- /pdm.lock: -------------------------------------------------------------------------------- 1 | # This file is @generated by PDM. 2 | # It is not intended for manual editing. 3 | 4 | [metadata] 5 | groups = ["default", "lint", "test", "types"] 6 | strategy = ["inherit_metadata"] 7 | lock_version = "4.5.0" 8 | content_hash = "sha256:24895d65ebf0ce3756d6df7313a3ddea2154a88391e9beab293f2305f99ff605" 9 | 10 | [[metadata.targets]] 11 | requires_python = ">=3.7,<3.10" 12 | 13 | [[package]] 14 | name = "certifi" 15 | version = "2025.4.26" 16 | requires_python = ">=3.6" 17 | summary = "Python package for providing Mozilla's CA Bundle." 18 | groups = ["default"] 19 | files = [ 20 | {file = "certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3"}, 21 | {file = "certifi-2025.4.26.tar.gz", hash = "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6"}, 22 | ] 23 | 24 | [[package]] 25 | name = "charset-normalizer" 26 | version = "3.4.2" 27 | requires_python = ">=3.7" 28 | summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." 29 | groups = ["default"] 30 | files = [ 31 | {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184"}, 32 | {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa"}, 33 | {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344"}, 34 | {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da"}, 35 | {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02"}, 36 | {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d"}, 37 | {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4"}, 38 | {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f"}, 39 | {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64"}, 40 | {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f"}, 41 | {file = "charset_normalizer-3.4.2-cp37-cp37m-win32.whl", hash = "sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58"}, 42 | {file = "charset_normalizer-3.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2"}, 43 | {file = "charset_normalizer-3.4.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb"}, 44 | {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a"}, 45 | {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45"}, 46 | {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5"}, 47 | {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1"}, 48 | {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027"}, 49 | {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b"}, 50 | {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455"}, 51 | {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01"}, 52 | {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58"}, 53 | {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681"}, 54 | {file = "charset_normalizer-3.4.2-cp38-cp38-win32.whl", hash = "sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7"}, 55 | {file = "charset_normalizer-3.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a"}, 56 | {file = "charset_normalizer-3.4.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4"}, 57 | {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7"}, 58 | {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836"}, 59 | {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597"}, 60 | {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7"}, 61 | {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f"}, 62 | {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba"}, 63 | {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12"}, 64 | {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518"}, 65 | {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5"}, 66 | {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3"}, 67 | {file = "charset_normalizer-3.4.2-cp39-cp39-win32.whl", hash = "sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471"}, 68 | {file = "charset_normalizer-3.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e"}, 69 | {file = "charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0"}, 70 | {file = "charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63"}, 71 | ] 72 | 73 | [[package]] 74 | name = "colorama" 75 | version = "0.4.6" 76 | requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" 77 | summary = "Cross-platform colored terminal text." 78 | groups = ["default", "test"] 79 | marker = "platform_system == \"Windows\" or sys_platform == \"win32\"" 80 | files = [ 81 | {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, 82 | {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, 83 | ] 84 | 85 | [[package]] 86 | name = "coverage" 87 | version = "7.2.7" 88 | requires_python = ">=3.7" 89 | summary = "Code coverage measurement for Python" 90 | groups = ["test"] 91 | files = [ 92 | {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, 93 | {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, 94 | {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, 95 | {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, 96 | {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, 97 | {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, 98 | {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, 99 | {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, 100 | {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, 101 | {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, 102 | {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, 103 | {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, 104 | {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, 105 | {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, 106 | {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, 107 | {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, 108 | {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, 109 | {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, 110 | {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, 111 | {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, 112 | {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, 113 | {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, 114 | {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, 115 | {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, 116 | {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, 117 | {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, 118 | {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, 119 | {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, 120 | {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, 121 | {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, 122 | {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, 123 | ] 124 | 125 | [[package]] 126 | name = "decopatch" 127 | version = "1.4.10" 128 | summary = "Create decorators easily in python." 129 | groups = ["test"] 130 | dependencies = [ 131 | "enum34; python_version < \"3.4\"", 132 | "funcsigs; python_version < \"3.3\"", 133 | "makefun>=1.5.0", 134 | ] 135 | files = [ 136 | {file = "decopatch-1.4.10-py2.py3-none-any.whl", hash = "sha256:e151f7f93de2b1b3fd3f3272dcc7cefd1a69f68ec1c2d8e288ecd9deb36dc5f7"}, 137 | {file = "decopatch-1.4.10.tar.gz", hash = "sha256:957f49c93f4150182c23f8fb51d13bb3213e0f17a79e09c8cca7057598b55720"}, 138 | ] 139 | 140 | [[package]] 141 | name = "duckdb" 142 | version = "0.6.1" 143 | summary = "DuckDB embedded database" 144 | groups = ["default"] 145 | dependencies = [ 146 | "numpy>=1.14", 147 | ] 148 | files = [ 149 | {file = "duckdb-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8442e074de6e1969c3d2b24363a5a6d7f866d5ac3f4e358e357495b389eff6c1"}, 150 | {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a6bf2ae7bec803352dade14561cb0b461b2422e70f75d9f09b36ba2dad2613b"}, 151 | {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5054792f22733f89d9cbbced2bafd8772d72d0fe77f159310221cefcf981c680"}, 152 | {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:21cc503dffc2c68bb825e4eb3098e82f40e910b3d09e1b3b7f090d39ad53fbea"}, 153 | {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54b3da77ad893e99c073087ff7f75a8c98154ac5139d317149f12b74367211db"}, 154 | {file = "duckdb-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:f1d709aa6a26172a3eab804b57763d5cdc1a4b785ac1fc2b09568578e52032ee"}, 155 | {file = "duckdb-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f4edcaa471d791393e37f63e3c7c728fa6324e3ac7e768b9dc2ea49065cd37cc"}, 156 | {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d218c2dd3bda51fb79e622b7b2266183ac9493834b55010aa01273fa5b7a7105"}, 157 | {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c7155cb93ab432eca44b651256c359281d26d927ff43badaf1d2276dd770832"}, 158 | {file = "duckdb-0.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0925778200090d3d5d8b6bb42b4d05d24db1e8912484ba3b7e7b7f8569f17dcb"}, 159 | {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b544dd04bb851d08bc68b317a7683cec6091547ae75555d075f8c8a7edb626e"}, 160 | {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c37d5a0391cf3a3a66e63215968ffb78e6b84f659529fa4bd10478f6203071"}, 161 | {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ce376966260eb5c351fcc6af627a979dbbcae3efeb2e70f85b23aa45a21e289d"}, 162 | {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73c974b09dd08dff5e8bdedba11c7d0aa0fc46ca93954ee7d19e1e18c9883ac1"}, 163 | {file = "duckdb-0.6.1-cp38-cp38-win32.whl", hash = "sha256:bfe39ed3a03e8b1ed764f58f513b37b24afe110d245803a41655d16d391ad9f1"}, 164 | {file = "duckdb-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:afa97d982dbe6b125631a17e222142e79bee88f7a13fc4cee92d09285e31ec83"}, 165 | {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c35ff4b1117096ef72d101524df0079da36c3735d52fcf1d907ccffa63bd6202"}, 166 | {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c54910fbb6de0f21d562e18a5c91540c19876db61b862fc9ffc8e31be8b3f03"}, 167 | {file = "duckdb-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99a7172563a3ae67d867572ce27cf3962f58e76f491cb7f602f08c2af39213b3"}, 168 | {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7363ffe857d00216b659116647fbf1e925cb3895699015d4a4e50b746de13041"}, 169 | {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06c1cef25f896b2284ba048108f645c72fab5c54aa5a6f62f95663f44ff8a79b"}, 170 | {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e92dd6aad7e8c29d002947376b6f5ce28cae29eb3b6b58a64a46cdbfc5cb7943"}, 171 | {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b280b2d8a01ecd4fe2feab041df70233c534fafbe33a38565b52c1e017529c7"}, 172 | {file = "duckdb-0.6.1-cp39-cp39-win32.whl", hash = "sha256:d9212d76e90b8469743924a4d22bef845be310d0d193d54ae17d9ef1f753cfa7"}, 173 | {file = "duckdb-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:00b7be8f67ec1a8edaa8844f521267baa1a795f4c482bfad56c72c26e1862ab2"}, 174 | {file = "duckdb-0.6.1.tar.gz", hash = "sha256:6d26e9f1afcb924a6057785e506810d48332d4764ddc4a5b414d0f2bf0cacfb4"}, 175 | ] 176 | 177 | [[package]] 178 | name = "exceptiongroup" 179 | version = "1.3.0" 180 | requires_python = ">=3.7" 181 | summary = "Backport of PEP 654 (exception groups)" 182 | groups = ["test"] 183 | marker = "python_version < \"3.11\"" 184 | dependencies = [ 185 | "typing-extensions>=4.6.0; python_version < \"3.13\"", 186 | ] 187 | files = [ 188 | {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, 189 | {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, 190 | ] 191 | 192 | [[package]] 193 | name = "idna" 194 | version = "3.10" 195 | requires_python = ">=3.6" 196 | summary = "Internationalized Domain Names in Applications (IDNA)" 197 | groups = ["default"] 198 | files = [ 199 | {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, 200 | {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, 201 | ] 202 | 203 | [[package]] 204 | name = "importlib-metadata" 205 | version = "6.7.0" 206 | requires_python = ">=3.7" 207 | summary = "Read metadata from Python packages" 208 | groups = ["test"] 209 | marker = "python_version < \"3.8\"" 210 | dependencies = [ 211 | "typing-extensions>=3.6.4; python_version < \"3.8\"", 212 | "zipp>=0.5", 213 | ] 214 | files = [ 215 | {file = "importlib_metadata-6.7.0-py3-none-any.whl", hash = "sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5"}, 216 | {file = "importlib_metadata-6.7.0.tar.gz", hash = "sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4"}, 217 | ] 218 | 219 | [[package]] 220 | name = "iniconfig" 221 | version = "2.0.0" 222 | requires_python = ">=3.7" 223 | summary = "brain-dead simple config-ini parsing" 224 | groups = ["test"] 225 | files = [ 226 | {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, 227 | {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, 228 | ] 229 | 230 | [[package]] 231 | name = "makefun" 232 | version = "1.16.0" 233 | summary = "Small library to dynamically create python functions." 234 | groups = ["test"] 235 | dependencies = [ 236 | "funcsigs; python_version < \"3.3\"", 237 | ] 238 | files = [ 239 | {file = "makefun-1.16.0-py2.py3-none-any.whl", hash = "sha256:43baa4c3e7ae2b17de9ceac20b669e9a67ceeadff31581007cca20a07bbe42c4"}, 240 | {file = "makefun-1.16.0.tar.gz", hash = "sha256:e14601831570bff1f6d7e68828bcd30d2f5856f24bad5de0ccb22921ceebc947"}, 241 | ] 242 | 243 | [[package]] 244 | name = "mypy" 245 | version = "1.4.1" 246 | requires_python = ">=3.7" 247 | summary = "Optional static typing for Python" 248 | groups = ["lint"] 249 | dependencies = [ 250 | "mypy-extensions>=1.0.0", 251 | "tomli>=1.1.0; python_version < \"3.11\"", 252 | "typed-ast<2,>=1.4.0; python_version < \"3.8\"", 253 | "typing-extensions>=4.1.0", 254 | ] 255 | files = [ 256 | {file = "mypy-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:470c969bb3f9a9efcedbadcd19a74ffb34a25f8e6b0e02dae7c0e71f8372f97b"}, 257 | {file = "mypy-1.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5952d2d18b79f7dc25e62e014fe5a23eb1a3d2bc66318df8988a01b1a037c5b"}, 258 | {file = "mypy-1.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:190b6bab0302cec4e9e6767d3eb66085aef2a1cc98fe04936d8a42ed2ba77bb7"}, 259 | {file = "mypy-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9d40652cc4fe33871ad3338581dca3297ff5f2213d0df345bcfbde5162abf0c9"}, 260 | {file = "mypy-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:01fd2e9f85622d981fd9063bfaef1aed6e336eaacca00892cd2d82801ab7c042"}, 261 | {file = "mypy-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2460a58faeea905aeb1b9b36f5065f2dc9a9c6e4c992a6499a2360c6c74ceca3"}, 262 | {file = "mypy-1.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2746d69a8196698146a3dbe29104f9eb6a2a4d8a27878d92169a6c0b74435b6"}, 263 | {file = "mypy-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ae704dcfaa180ff7c4cfbad23e74321a2b774f92ca77fd94ce1049175a21c97f"}, 264 | {file = "mypy-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:43d24f6437925ce50139a310a64b2ab048cb2d3694c84c71c3f2a1626d8101dc"}, 265 | {file = "mypy-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c482e1246726616088532b5e964e39765b6d1520791348e6c9dc3af25b233828"}, 266 | {file = "mypy-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43b592511672017f5b1a483527fd2684347fdffc041c9ef53428c8dc530f79a3"}, 267 | {file = "mypy-1.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:34a9239d5b3502c17f07fd7c0b2ae6b7dd7d7f6af35fbb5072c6208e76295816"}, 268 | {file = "mypy-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5703097c4936bbb9e9bce41478c8d08edd2865e177dc4c52be759f81ee4dd26c"}, 269 | {file = "mypy-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e02d700ec8d9b1859790c0475df4e4092c7bf3272a4fd2c9f33d87fac4427b8f"}, 270 | {file = "mypy-1.4.1-py3-none-any.whl", hash = "sha256:45d32cec14e7b97af848bddd97d85ea4f0db4d5a149ed9676caa4eb2f7402bb4"}, 271 | {file = "mypy-1.4.1.tar.gz", hash = "sha256:9bbcd9ab8ea1f2e1c8031c21445b511442cc45c89951e49bbf852cbb70755b1b"}, 272 | ] 273 | 274 | [[package]] 275 | name = "mypy-extensions" 276 | version = "1.0.0" 277 | requires_python = ">=3.5" 278 | summary = "Type system extensions for programs checked with the mypy type checker." 279 | groups = ["lint"] 280 | files = [ 281 | {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, 282 | {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, 283 | ] 284 | 285 | [[package]] 286 | name = "numpy" 287 | version = "1.21.6" 288 | requires_python = ">=3.7,<3.11" 289 | summary = "NumPy is the fundamental package for array computing with Python." 290 | groups = ["default"] 291 | files = [ 292 | {file = "numpy-1.21.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6aaf96c7f8cebc220cdfc03f1d5a31952f027dda050e5a703a0d1c396075e3e7"}, 293 | {file = "numpy-1.21.6-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:67c261d6c0a9981820c3a149d255a76918278a6b03b6a036800359aba1256d46"}, 294 | {file = "numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a6be4cb0ef3b8c9250c19cc122267263093eee7edd4e3fa75395dfda8c17a8e2"}, 295 | {file = "numpy-1.21.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c4068a8c44014b2d55f3c3f574c376b2494ca9cc73d2f1bd692382b6dffe3db"}, 296 | {file = "numpy-1.21.6-cp37-cp37m-win32.whl", hash = "sha256:7c7e5fa88d9ff656e067876e4736379cc962d185d5cd808014a8a928d529ef4e"}, 297 | {file = "numpy-1.21.6-cp37-cp37m-win_amd64.whl", hash = "sha256:bcb238c9c96c00d3085b264e5c1a1207672577b93fa666c3b14a45240b14123a"}, 298 | {file = "numpy-1.21.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:82691fda7c3f77c90e62da69ae60b5ac08e87e775b09813559f8901a88266552"}, 299 | {file = "numpy-1.21.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:643843bcc1c50526b3a71cd2ee561cf0d8773f062c8cbaf9ffac9fdf573f83ab"}, 300 | {file = "numpy-1.21.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:357768c2e4451ac241465157a3e929b265dfac85d9214074985b1786244f2ef3"}, 301 | {file = "numpy-1.21.6-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9f411b2c3f3d76bba0865b35a425157c5dcf54937f82bbeb3d3c180789dd66a6"}, 302 | {file = "numpy-1.21.6-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4aa48afdce4660b0076a00d80afa54e8a97cd49f457d68a4342d188a09451c1a"}, 303 | {file = "numpy-1.21.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a96eef20f639e6a97d23e57dd0c1b1069a7b4fd7027482a4c5c451cd7732f4"}, 304 | {file = "numpy-1.21.6-cp38-cp38-win32.whl", hash = "sha256:5c3c8def4230e1b959671eb959083661b4a0d2e9af93ee339c7dada6759a9470"}, 305 | {file = "numpy-1.21.6-cp38-cp38-win_amd64.whl", hash = "sha256:bf2ec4b75d0e9356edea834d1de42b31fe11f726a81dfb2c2112bc1eaa508fcf"}, 306 | {file = "numpy-1.21.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4391bd07606be175aafd267ef9bea87cf1b8210c787666ce82073b05f202add1"}, 307 | {file = "numpy-1.21.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:67f21981ba2f9d7ba9ade60c9e8cbaa8cf8e9ae51673934480e45cf55e953673"}, 308 | {file = "numpy-1.21.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee5ec40fdd06d62fe5d4084bef4fd50fd4bb6bfd2bf519365f569dc470163ab0"}, 309 | {file = "numpy-1.21.6-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1dbe1c91269f880e364526649a52eff93ac30035507ae980d2fed33aaee633ac"}, 310 | {file = "numpy-1.21.6-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d9caa9d5e682102453d96a0ee10c7241b72859b01a941a397fd965f23b3e016b"}, 311 | {file = "numpy-1.21.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58459d3bad03343ac4b1b42ed14d571b8743dc80ccbf27444f266729df1d6f5b"}, 312 | {file = "numpy-1.21.6-cp39-cp39-win32.whl", hash = "sha256:7f5ae4f304257569ef3b948810816bc87c9146e8c446053539947eedeaa32786"}, 313 | {file = "numpy-1.21.6-cp39-cp39-win_amd64.whl", hash = "sha256:e31f0bb5928b793169b87e3d1e070f2342b22d5245c755e2b81caa29756246c3"}, 314 | {file = "numpy-1.21.6-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:dd1c8f6bd65d07d3810b90d02eba7997e32abbdf1277a481d698969e921a3be0"}, 315 | {file = "numpy-1.21.6.zip", hash = "sha256:ecb55251139706669fdec2ff073c98ef8e9a84473e51e716211b41aa0f18e656"}, 316 | ] 317 | 318 | [[package]] 319 | name = "packaging" 320 | version = "24.0" 321 | requires_python = ">=3.7" 322 | summary = "Core utilities for Python packages" 323 | groups = ["test"] 324 | files = [ 325 | {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, 326 | {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, 327 | ] 328 | 329 | [[package]] 330 | name = "pandas" 331 | version = "1.1.5" 332 | requires_python = ">=3.6.1" 333 | summary = "Powerful data structures for data analysis, time series, and statistics" 334 | groups = ["default"] 335 | dependencies = [ 336 | "numpy>=1.15.4", 337 | "python-dateutil>=2.7.3", 338 | "pytz>=2017.2", 339 | ] 340 | files = [ 341 | {file = "pandas-1.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:26fa92d3ac743a149a31b21d6f4337b0594b6302ea5575b37af9ca9611e8981a"}, 342 | {file = "pandas-1.1.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c16d59c15d946111d2716856dd5479221c9e4f2f5c7bc2d617f39d870031e086"}, 343 | {file = "pandas-1.1.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:3be7a7a0ca71a2640e81d9276f526bca63505850add10206d0da2e8a0a325dae"}, 344 | {file = "pandas-1.1.5-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:573fba5b05bf2c69271a32e52399c8de599e4a15ab7cec47d3b9c904125ab788"}, 345 | {file = "pandas-1.1.5-cp37-cp37m-win32.whl", hash = "sha256:21b5a2b033380adbdd36b3116faaf9a4663e375325831dac1b519a44f9e439bb"}, 346 | {file = "pandas-1.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:24c7f8d4aee71bfa6401faeba367dd654f696a77151a8a28bc2013f7ced4af98"}, 347 | {file = "pandas-1.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2860a97cbb25444ffc0088b457da0a79dc79f9c601238a3e0644312fcc14bf11"}, 348 | {file = "pandas-1.1.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:5008374ebb990dad9ed48b0f5d0038124c73748f5384cc8c46904dace27082d9"}, 349 | {file = "pandas-1.1.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2c2f7c670ea4e60318e4b7e474d56447cf0c7d83b3c2a5405a0dbb2600b9c48e"}, 350 | {file = "pandas-1.1.5-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:0a643bae4283a37732ddfcecab3f62dd082996021b980f580903f4e8e01b3c5b"}, 351 | {file = "pandas-1.1.5-cp38-cp38-win32.whl", hash = "sha256:5447ea7af4005b0daf695a316a423b96374c9c73ffbd4533209c5ddc369e644b"}, 352 | {file = "pandas-1.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:4c62e94d5d49db116bef1bd5c2486723a292d79409fc9abd51adf9e05329101d"}, 353 | {file = "pandas-1.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:731568be71fba1e13cae212c362f3d2ca8932e83cb1b85e3f1b4dd77d019254a"}, 354 | {file = "pandas-1.1.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:c61c043aafb69329d0f961b19faa30b1dab709dd34c9388143fc55680059e55a"}, 355 | {file = "pandas-1.1.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:2b1c6cd28a0dfda75c7b5957363333f01d370936e4c6276b7b8e696dd500582a"}, 356 | {file = "pandas-1.1.5-cp39-cp39-win32.whl", hash = "sha256:c94ff2780a1fd89f190390130d6d36173ca59fcfb3fe0ff596f9a56518191ccb"}, 357 | {file = "pandas-1.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:edda9bacc3843dfbeebaf7a701763e68e741b08fccb889c003b0a52f0ee95782"}, 358 | {file = "pandas-1.1.5.tar.gz", hash = "sha256:f10fc41ee3c75a474d3bdf68d396f10782d013d7f67db99c0efbfd0acb99701b"}, 359 | ] 360 | 361 | [[package]] 362 | name = "pluggy" 363 | version = "1.2.0" 364 | requires_python = ">=3.7" 365 | summary = "plugin and hook calling mechanisms for python" 366 | groups = ["test"] 367 | dependencies = [ 368 | "importlib-metadata>=0.12; python_version < \"3.8\"", 369 | ] 370 | files = [ 371 | {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, 372 | {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, 373 | ] 374 | 375 | [[package]] 376 | name = "pre-commit-hooks" 377 | version = "4.4.0" 378 | requires_python = ">=3.7" 379 | summary = "Some out-of-the-box hooks for pre-commit." 380 | groups = ["lint"] 381 | dependencies = [ 382 | "ruamel-yaml>=0.15", 383 | "tomli>=1.1.0; python_version < \"3.11\"", 384 | ] 385 | files = [ 386 | {file = "pre_commit_hooks-4.4.0-py2.py3-none-any.whl", hash = "sha256:fc8837335476221ccccda3d176ed6ae29fe58753ce7e8b7863f5d0f987328fc6"}, 387 | {file = "pre_commit_hooks-4.4.0.tar.gz", hash = "sha256:7011eed8e1a25cde94693da009cba76392194cecc2f3f06c51a44ea6ad6c2af9"}, 388 | ] 389 | 390 | [[package]] 391 | name = "psycopg2" 392 | version = "2.9.5" 393 | requires_python = ">=3.6" 394 | summary = "psycopg2 - Python-PostgreSQL Database Adapter" 395 | groups = ["default"] 396 | files = [ 397 | {file = "psycopg2-2.9.5-cp37-cp37m-win32.whl", hash = "sha256:922cc5f0b98a5f2b1ff481f5551b95cd04580fd6f0c72d9b22e6c0145a4840e0"}, 398 | {file = "psycopg2-2.9.5-cp37-cp37m-win_amd64.whl", hash = "sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a"}, 399 | {file = "psycopg2-2.9.5-cp38-cp38-win32.whl", hash = "sha256:f5b6320dbc3cf6cfb9f25308286f9f7ab464e65cfb105b64cc9c52831748ced2"}, 400 | {file = "psycopg2-2.9.5-cp38-cp38-win_amd64.whl", hash = "sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e"}, 401 | {file = "psycopg2-2.9.5-cp39-cp39-win32.whl", hash = "sha256:322fd5fca0b1113677089d4ebd5222c964b1760e361f151cbb2706c4912112c5"}, 402 | {file = "psycopg2-2.9.5-cp39-cp39-win_amd64.whl", hash = "sha256:190d51e8c1b25a47484e52a79638a8182451d6f6dff99f26ad9bd81e5359a0fa"}, 403 | {file = "psycopg2-2.9.5.tar.gz", hash = "sha256:a5246d2e683a972e2187a8714b5c2cf8156c064629f9a9b1a873c1730d9e245a"}, 404 | ] 405 | 406 | [[package]] 407 | name = "pytest" 408 | version = "7.4.4" 409 | requires_python = ">=3.7" 410 | summary = "pytest: simple powerful testing with Python" 411 | groups = ["test"] 412 | dependencies = [ 413 | "colorama; sys_platform == \"win32\"", 414 | "exceptiongroup>=1.0.0rc8; python_version < \"3.11\"", 415 | "importlib-metadata>=0.12; python_version < \"3.8\"", 416 | "iniconfig", 417 | "packaging", 418 | "pluggy<2.0,>=0.12", 419 | "tomli>=1.0.0; python_version < \"3.11\"", 420 | ] 421 | files = [ 422 | {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, 423 | {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, 424 | ] 425 | 426 | [[package]] 427 | name = "pytest-cases" 428 | version = "3.8.6" 429 | summary = "Separate test code from test cases in pytest." 430 | groups = ["test"] 431 | dependencies = [ 432 | "decopatch", 433 | "funcsigs; python_version < \"3.3\"", 434 | "functools32; python_version < \"3.2\"", 435 | "makefun>=1.15.1", 436 | "packaging", 437 | ] 438 | files = [ 439 | {file = "pytest_cases-3.8.6-py2.py3-none-any.whl", hash = "sha256:564c722492ea7e7ec3ac433fd14070180e65866f49fa35bfe938c0d5d9afba67"}, 440 | {file = "pytest_cases-3.8.6.tar.gz", hash = "sha256:5c24e0ab0cb6f8e802a469b7965906a333d3babb874586ebc56f7e2cbe1a7c44"}, 441 | ] 442 | 443 | [[package]] 444 | name = "python-dateutil" 445 | version = "2.9.0.post0" 446 | requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" 447 | summary = "Extensions to the standard Python datetime module" 448 | groups = ["default"] 449 | dependencies = [ 450 | "six>=1.5", 451 | ] 452 | files = [ 453 | {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, 454 | {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, 455 | ] 456 | 457 | [[package]] 458 | name = "pytz" 459 | version = "2025.2" 460 | summary = "World timezone definitions, modern and historical" 461 | groups = ["default"] 462 | files = [ 463 | {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, 464 | {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, 465 | ] 466 | 467 | [[package]] 468 | name = "requests" 469 | version = "2.28.2" 470 | requires_python = ">=3.7, <4" 471 | summary = "Python HTTP for Humans." 472 | groups = ["default"] 473 | dependencies = [ 474 | "certifi>=2017.4.17", 475 | "charset-normalizer<4,>=2", 476 | "idna<4,>=2.5", 477 | "urllib3<1.27,>=1.21.1", 478 | ] 479 | files = [ 480 | {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, 481 | {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, 482 | ] 483 | 484 | [[package]] 485 | name = "ruamel-yaml" 486 | version = "0.18.12" 487 | requires_python = ">=3.7" 488 | summary = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" 489 | groups = ["lint"] 490 | dependencies = [ 491 | "ruamel-yaml-clib>=0.2.7; platform_python_implementation == \"CPython\" and python_version < \"3.14\"", 492 | ] 493 | files = [ 494 | {file = "ruamel.yaml-0.18.12-py3-none-any.whl", hash = "sha256:790ba4c48b6a6e6b12b532a7308779eb12d2aaab3a80fdb8389216f28ea2b287"}, 495 | {file = "ruamel.yaml-0.18.12.tar.gz", hash = "sha256:5a38fd5ce39d223bebb9e3a6779e86b9427a03fb0bf9f270060f8b149cffe5e2"}, 496 | ] 497 | 498 | [[package]] 499 | name = "ruamel-yaml-clib" 500 | version = "0.2.8" 501 | requires_python = ">=3.6" 502 | summary = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" 503 | groups = ["lint"] 504 | marker = "platform_python_implementation == \"CPython\" and python_version < \"3.14\"" 505 | files = [ 506 | {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, 507 | {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, 508 | {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:77159f5d5b5c14f7c34073862a6b7d34944075d9f93e681638f6d753606c6ce6"}, 509 | {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, 510 | {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7"}, 511 | {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3"}, 512 | {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b"}, 513 | {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, 514 | {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, 515 | {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, 516 | {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:305889baa4043a09e5b76f8e2a51d4ffba44259f6b4c72dec8ca56207d9c6fe1"}, 517 | {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, 518 | {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28"}, 519 | {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d"}, 520 | {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win32.whl", hash = "sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe"}, 521 | {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, 522 | {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, 523 | {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, 524 | {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:a1a45e0bb052edf6a1d3a93baef85319733a888363938e1fc9924cb00c8df24c"}, 525 | {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, 526 | {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b"}, 527 | {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880"}, 528 | {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win32.whl", hash = "sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5"}, 529 | {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15"}, 530 | {file = "ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512"}, 531 | ] 532 | 533 | [[package]] 534 | name = "ruff" 535 | version = "0.11.12" 536 | requires_python = ">=3.7" 537 | summary = "An extremely fast Python linter and code formatter, written in Rust." 538 | groups = ["lint"] 539 | files = [ 540 | {file = "ruff-0.11.12-py3-none-linux_armv6l.whl", hash = "sha256:c7680aa2f0d4c4f43353d1e72123955c7a2159b8646cd43402de6d4a3a25d7cc"}, 541 | {file = "ruff-0.11.12-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:2cad64843da9f134565c20bcc430642de897b8ea02e2e79e6e02a76b8dcad7c3"}, 542 | {file = "ruff-0.11.12-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9b6886b524a1c659cee1758140138455d3c029783d1b9e643f3624a5ee0cb0aa"}, 543 | {file = "ruff-0.11.12-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cc3a3690aad6e86c1958d3ec3c38c4594b6ecec75c1f531e84160bd827b2012"}, 544 | {file = "ruff-0.11.12-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f97fdbc2549f456c65b3b0048560d44ddd540db1f27c778a938371424b49fe4a"}, 545 | {file = "ruff-0.11.12-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74adf84960236961090e2d1348c1a67d940fd12e811a33fb3d107df61eef8fc7"}, 546 | {file = "ruff-0.11.12-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:b56697e5b8bcf1d61293ccfe63873aba08fdbcbbba839fc046ec5926bdb25a3a"}, 547 | {file = "ruff-0.11.12-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d47afa45e7b0eaf5e5969c6b39cbd108be83910b5c74626247e366fd7a36a13"}, 548 | {file = "ruff-0.11.12-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bf9603fe1bf949de8b09a2da896f05c01ed7a187f4a386cdba6760e7f61be"}, 549 | {file = "ruff-0.11.12-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08033320e979df3b20dba567c62f69c45e01df708b0f9c83912d7abd3e0801cd"}, 550 | {file = "ruff-0.11.12-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:929b7706584f5bfd61d67d5070f399057d07c70585fa8c4491d78ada452d3bef"}, 551 | {file = "ruff-0.11.12-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7de4a73205dc5756b8e09ee3ed67c38312dce1aa28972b93150f5751199981b5"}, 552 | {file = "ruff-0.11.12-py3-none-musllinux_1_2_i686.whl", hash = "sha256:2635c2a90ac1b8ca9e93b70af59dfd1dd2026a40e2d6eebaa3efb0465dd9cf02"}, 553 | {file = "ruff-0.11.12-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:d05d6a78a89166f03f03a198ecc9d18779076ad0eec476819467acb401028c0c"}, 554 | {file = "ruff-0.11.12-py3-none-win32.whl", hash = "sha256:f5a07f49767c4be4772d161bfc049c1f242db0cfe1bd976e0f0886732a4765d6"}, 555 | {file = "ruff-0.11.12-py3-none-win_amd64.whl", hash = "sha256:5a4d9f8030d8c3a45df201d7fb3ed38d0219bccd7955268e863ee4a115fa0832"}, 556 | {file = "ruff-0.11.12-py3-none-win_arm64.whl", hash = "sha256:65194e37853158d368e333ba282217941029a28ea90913c67e558c611d04daa5"}, 557 | {file = "ruff-0.11.12.tar.gz", hash = "sha256:43cf7f69c7d7c7d7513b9d59c5d8cafd704e05944f978614aa9faff6ac202603"}, 558 | ] 559 | 560 | [[package]] 561 | name = "six" 562 | version = "1.17.0" 563 | requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" 564 | summary = "Python 2 and 3 compatibility utilities" 565 | groups = ["default"] 566 | files = [ 567 | {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, 568 | {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, 569 | ] 570 | 571 | [[package]] 572 | name = "tomli" 573 | version = "2.0.1" 574 | requires_python = ">=3.7" 575 | summary = "A lil' TOML parser" 576 | groups = ["lint", "test"] 577 | marker = "python_version < \"3.11\"" 578 | files = [ 579 | {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, 580 | {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, 581 | ] 582 | 583 | [[package]] 584 | name = "tqdm" 585 | version = "4.64.1" 586 | requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" 587 | summary = "Fast, Extensible Progress Meter" 588 | groups = ["default"] 589 | dependencies = [ 590 | "colorama; platform_system == \"Windows\"", 591 | "importlib-resources; python_version < \"3.7\"", 592 | ] 593 | files = [ 594 | {file = "tqdm-4.64.1-py2.py3-none-any.whl", hash = "sha256:6fee160d6ffcd1b1c68c65f14c829c22832bc401726335ce92c52d395944a6a1"}, 595 | {file = "tqdm-4.64.1.tar.gz", hash = "sha256:5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4"}, 596 | ] 597 | 598 | [[package]] 599 | name = "typed-ast" 600 | version = "1.5.5" 601 | requires_python = ">=3.6" 602 | summary = "a fork of Python 2 and 3 ast modules with type comment support" 603 | groups = ["lint"] 604 | marker = "python_version < \"3.8\"" 605 | files = [ 606 | {file = "typed_ast-1.5.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2188bc33d85951ea4ddad55d2b35598b2709d122c11c75cffd529fbc9965508e"}, 607 | {file = "typed_ast-1.5.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0635900d16ae133cab3b26c607586131269f88266954eb04ec31535c9a12ef1e"}, 608 | {file = "typed_ast-1.5.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57bfc3cf35a0f2fdf0a88a3044aafaec1d2f24d8ae8cd87c4f58d615fb5b6311"}, 609 | {file = "typed_ast-1.5.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:fe58ef6a764de7b4b36edfc8592641f56e69b7163bba9f9c8089838ee596bfb2"}, 610 | {file = "typed_ast-1.5.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d09d930c2d1d621f717bb217bf1fe2584616febb5138d9b3e8cdd26506c3f6d4"}, 611 | {file = "typed_ast-1.5.5-cp37-cp37m-win_amd64.whl", hash = "sha256:d40c10326893ecab8a80a53039164a224984339b2c32a6baf55ecbd5b1df6431"}, 612 | {file = "typed_ast-1.5.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fd946abf3c31fb50eee07451a6aedbfff912fcd13cf357363f5b4e834cc5e71a"}, 613 | {file = "typed_ast-1.5.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ed4a1a42df8a3dfb6b40c3d2de109e935949f2f66b19703eafade03173f8f437"}, 614 | {file = "typed_ast-1.5.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:045f9930a1550d9352464e5149710d56a2aed23a2ffe78946478f7b5416f1ede"}, 615 | {file = "typed_ast-1.5.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:381eed9c95484ceef5ced626355fdc0765ab51d8553fec08661dce654a935db4"}, 616 | {file = "typed_ast-1.5.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:bfd39a41c0ef6f31684daff53befddae608f9daf6957140228a08e51f312d7e6"}, 617 | {file = "typed_ast-1.5.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8c524eb3024edcc04e288db9541fe1f438f82d281e591c548903d5b77ad1ddd4"}, 618 | {file = "typed_ast-1.5.5-cp38-cp38-win_amd64.whl", hash = "sha256:7f58fabdde8dcbe764cef5e1a7fcb440f2463c1bbbec1cf2a86ca7bc1f95184b"}, 619 | {file = "typed_ast-1.5.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:042eb665ff6bf020dd2243307d11ed626306b82812aba21836096d229fdc6a10"}, 620 | {file = "typed_ast-1.5.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:622e4a006472b05cf6ef7f9f2636edc51bda670b7bbffa18d26b255269d3d814"}, 621 | {file = "typed_ast-1.5.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1efebbbf4604ad1283e963e8915daa240cb4bf5067053cf2f0baadc4d4fb51b8"}, 622 | {file = "typed_ast-1.5.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0aefdd66f1784c58f65b502b6cf8b121544680456d1cebbd300c2c813899274"}, 623 | {file = "typed_ast-1.5.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:48074261a842acf825af1968cd912f6f21357316080ebaca5f19abbb11690c8a"}, 624 | {file = "typed_ast-1.5.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:429ae404f69dc94b9361bb62291885894b7c6fb4640d561179548c849f8492ba"}, 625 | {file = "typed_ast-1.5.5-cp39-cp39-win_amd64.whl", hash = "sha256:335f22ccb244da2b5c296e6f96b06ee9bed46526db0de38d2f0e5a6597b81155"}, 626 | {file = "typed_ast-1.5.5.tar.gz", hash = "sha256:94282f7a354f36ef5dbce0ef3467ebf6a258e370ab33d5b40c249fa996e590dd"}, 627 | ] 628 | 629 | [[package]] 630 | name = "types-psycopg2" 631 | version = "2.9.21.20" 632 | requires_python = ">=3.7" 633 | summary = "Typing stubs for psycopg2" 634 | groups = ["types"] 635 | files = [ 636 | {file = "types-psycopg2-2.9.21.20.tar.gz", hash = "sha256:73baea689575bf5bb1b915b783fb0524044c6242928aeef1ae5a9e32f0780d3d"}, 637 | {file = "types_psycopg2-2.9.21.20-py3-none-any.whl", hash = "sha256:5b1e2e1d9478f8a298ea7038f8ea988e0ccc1f0af39f84636d57ef0da6f29e95"}, 638 | ] 639 | 640 | [[package]] 641 | name = "types-requests" 642 | version = "2.31.0.6" 643 | requires_python = ">=3.7" 644 | summary = "Typing stubs for requests" 645 | groups = ["types"] 646 | dependencies = [ 647 | "types-urllib3", 648 | ] 649 | files = [ 650 | {file = "types-requests-2.31.0.6.tar.gz", hash = "sha256:cd74ce3b53c461f1228a9b783929ac73a666658f223e28ed29753771477b3bd0"}, 651 | {file = "types_requests-2.31.0.6-py3-none-any.whl", hash = "sha256:a2db9cb228a81da8348b49ad6db3f5519452dd20a9c1e1a868c83c5fe88fd1a9"}, 652 | ] 653 | 654 | [[package]] 655 | name = "types-tqdm" 656 | version = "4.66.0.5" 657 | requires_python = ">=3.7" 658 | summary = "Typing stubs for tqdm" 659 | groups = ["types"] 660 | files = [ 661 | {file = "types-tqdm-4.66.0.5.tar.gz", hash = "sha256:74bd7e469238c28816300f72a9b713d02036f6b557734616430adb7b7e74112c"}, 662 | {file = "types_tqdm-4.66.0.5-py3-none-any.whl", hash = "sha256:d2c38085bec440e8ad1e94e8619f7cb3d1dd0a7ee06a863ccd0610a5945046ef"}, 663 | ] 664 | 665 | [[package]] 666 | name = "types-urllib3" 667 | version = "1.26.25.14" 668 | summary = "Typing stubs for urllib3" 669 | groups = ["types"] 670 | files = [ 671 | {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, 672 | {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, 673 | ] 674 | 675 | [[package]] 676 | name = "typing-extensions" 677 | version = "4.7.1" 678 | requires_python = ">=3.7" 679 | summary = "Backported and Experimental Type Hints for Python 3.7+" 680 | groups = ["lint", "test"] 681 | files = [ 682 | {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, 683 | {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, 684 | ] 685 | 686 | [[package]] 687 | name = "urllib3" 688 | version = "1.26.20" 689 | requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" 690 | summary = "HTTP library with thread-safe connection pooling, file post, and more." 691 | groups = ["default"] 692 | files = [ 693 | {file = "urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e"}, 694 | {file = "urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32"}, 695 | ] 696 | 697 | [[package]] 698 | name = "xlsxwriter" 699 | version = "3.0.6" 700 | requires_python = ">=3.6" 701 | summary = "A Python module for creating Excel XLSX files." 702 | groups = ["default"] 703 | files = [ 704 | {file = "XlsxWriter-3.0.6-py3-none-any.whl", hash = "sha256:56eae8ae587536734009aa819845c3e3c865462399823085b0baabbb081a929c"}, 705 | {file = "XlsxWriter-3.0.6.tar.gz", hash = "sha256:2f9e5ea13343fe85486e349d4e5fdf746bb69dc7bc1dedfa9b5fae2bb48c0795"}, 706 | ] 707 | 708 | [[package]] 709 | name = "zipp" 710 | version = "3.15.0" 711 | requires_python = ">=3.7" 712 | summary = "Backport of pathlib-compatible object wrapper for zip files" 713 | groups = ["test"] 714 | marker = "python_version < \"3.8\"" 715 | files = [ 716 | {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, 717 | {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, 718 | ] 719 | -------------------------------------------------------------------------------- /precious.toml: -------------------------------------------------------------------------------- 1 | [commands."common.EOF"] 2 | type = "tidy" 3 | include = ["*"] 4 | exclude = ["*.png"] 5 | cmd = ["end-of-file-fixer"] 6 | ok_exit_codes = [0, 1] 7 | [commands."common.whitespace"] 8 | type = "tidy" 9 | include = ["*"] 10 | exclude = ["*.png"] 11 | cmd = ["trailing-whitespace-fixer", "--markdown-linebreak-ext=md"] 12 | ok_exit_codes = [0, 1] 13 | [commands."common.large-files"] 14 | type = "lint" 15 | include = ["*"] 16 | cmd = ["check-added-large-files"] 17 | ok_exit_codes = [0] 18 | [commands."common.case"] 19 | type = "lint" 20 | include = ["*"] 21 | cmd = ["check-case-conflict"] 22 | ok_exit_codes = [0] 23 | 24 | [commands."ruff.lint"] 25 | type = "both" 26 | include = ["*.py"] 27 | cmd = ["ruff", "check", "--quiet"] 28 | tidy_flags = ["--fix-only"] 29 | ok_exit_codes = [0] 30 | [commands."ruff.format"] 31 | type = "both" 32 | include = ["*.py"] 33 | cmd = ["ruff", "format", "--quiet"] 34 | lint_flags = ["--diff"] 35 | ok_exit_codes = [0] 36 | [commands.mypy] 37 | type = "lint" 38 | include = ["*.py"] 39 | invoke = "once" 40 | path_args = "none" 41 | cmd = ["mypy", "src/ldlite/", "tests/"] 42 | ok_exit_codes = [0] 43 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["pdm-backend"] 3 | build-backend = "pdm.backend" 4 | 5 | [project] 6 | name = "ldlite" 7 | version = "1.0.0" 8 | description = "Lightweight analytics tool for FOLIO services" 9 | authors = [ 10 | {name = "Katherine Bargar", email = "kbargar@fivecolleges.edu"}, 11 | {name = "Nassib Nassar", email = "nassib@indexdata.com"}, 12 | ] 13 | dependencies = [ 14 | "duckdb==0.6.1", 15 | "pandas<=1.5.2", 16 | "psycopg2==2.9.5", 17 | "requests==2.28.2", 18 | "tqdm==4.64.1", 19 | "XlsxWriter==3.0.6", 20 | ] 21 | requires-python = ">=3.7,<3.10" 22 | readme = "README.md" 23 | license = {text = "Apache-2.0"} 24 | classifiers = [ 25 | "License :: OSI Approved :: Apache Software License", 26 | "Operating System :: OS Independent", 27 | "Programming Language :: Python :: 3", 28 | ] 29 | 30 | [project.urls] 31 | Homepage = "https://github.com/library-data-platform/ldlite" 32 | "Bug Tracker" = "https://github.com/library-data-platform/ldlite/issues" 33 | 34 | [tool.pytest.ini_options] 35 | pythonpath = "src" 36 | addopts = [ 37 | "--import-mode=importlib", 38 | ] 39 | 40 | [tool.mypy] 41 | python_version = "3.9" 42 | strict = true 43 | [[tool.mypy.overrides]] 44 | module = ["xlsxwriter.*"] 45 | ignore_missing_imports = true 46 | 47 | [tool.ruff] 48 | target-version = "py39" 49 | [tool.ruff.lint] 50 | select = ["ALL"] 51 | fixable = ["ALL"] 52 | ignore = ["FBT", "D105", "FIX002", "PLR2004", "TD002", "TD003", "COM812"] 53 | pydocstyle.convention = "google" 54 | [tool.ruff.lint.per-file-ignores] 55 | "examples/*" = ["D", "INP001", "T201", "S106", "ERA001", "PERF203"] 56 | "tests/*" = ["D", "S", "INP001", "N813"] 57 | "src/ldlite/{_csv.py,_jsonx.py,_select.py,_xlsx.py}" = ["S608"] 58 | "src/ldlite/__init__.py" = ["T201"] 59 | [tool.ruff.lint.flake8-annotations] 60 | mypy-init-return = true 61 | 62 | [tool.coverage.run] 63 | branch = true 64 | include = ["src/**"] 65 | omit = ["src/ldlite/_xlsx.py"] 66 | 67 | [tool.pdm] 68 | distribution = true 69 | package-dir = "src" 70 | 71 | [tool.pdm.scripts] 72 | test.composite = ["rm -f .coverage", "python -m coverage run -m pytest -vv {args}", "python -m coverage report"] 73 | 74 | [dependency-groups] 75 | lint = [ 76 | "mypy>=1.4.1", 77 | "ruff>=0.11.9", 78 | "pre-commit-hooks>=4.4.0", 79 | ] 80 | test = [ 81 | "pytest>=7.4.4", 82 | "pytest-cases>=3.8.6", 83 | "coverage>=7.2.7", 84 | ] 85 | types = [ 86 | "types-requests>=2.31.0.6", 87 | "types-psycopg2>=2.9.21.20", 88 | "types-tqdm>=4.66.0.5", 89 | ] 90 | -------------------------------------------------------------------------------- /src/ldlite/__init__.py: -------------------------------------------------------------------------------- 1 | """LDLite is a lightweight reporting tool for FOLIO services. 2 | 3 | It is part of the Library Data Platform project and provides basic LDP 4 | functions without requiring the platform to be installed. 5 | 6 | LDLite functions include retrieving data from a FOLIO instance, transforming 7 | the data, and storing the data in a reporting database for further querying. 8 | 9 | To install LDLite or upgrade to the latest version: 10 | 11 | python -m pip install --upgrade ldlite 12 | 13 | Example: 14 | # Import and initialize LDLite. 15 | import ldlite 16 | ld = ldlite.LDLite() 17 | 18 | # Connect to a database. 19 | db = ld.connect_db(filename='ldlite.db') 20 | 21 | # Connect to FOLIO. 22 | ld.connect_folio(url='https://folio-etesting-snapshot-kong.ci.folio.org', 23 | tenant='diku', 24 | user='diku_admin', 25 | password='admin') 26 | 27 | # Send a CQL query and store the results in table "g", "g_j", etc. 28 | ld.query(table='g', path='/groups', query='cql.allRecords=1 sortby id') 29 | 30 | # Print the result tables. 31 | ld.select(table='g') 32 | ld.select(table='g_j') 33 | # etc. 34 | 35 | """ 36 | 37 | from __future__ import annotations 38 | 39 | import json 40 | import sqlite3 41 | import sys 42 | from typing import TYPE_CHECKING, NoReturn, cast 43 | 44 | import duckdb 45 | import psycopg2 46 | import requests 47 | from tqdm import tqdm 48 | 49 | from ._csv import to_csv 50 | from ._jsonx import Attr, drop_json_tables, transform_json 51 | from ._query import query_dict 52 | from ._request import request_get 53 | from ._select import select 54 | from ._sqlx import DBType, as_postgres, autocommit, encode_sql_str, json_type, sqlid 55 | from ._xlsx import to_xlsx 56 | 57 | if TYPE_CHECKING: 58 | from _typeshed import dbapi 59 | 60 | 61 | class LDLite: 62 | """LDLite contains the primary functionality for reporting.""" 63 | 64 | def __init__(self) -> None: 65 | """Creates an instance of LDLite. 66 | 67 | Example: 68 | import ldlite 69 | 70 | ld = ldlite.LDLite() 71 | 72 | """ 73 | self.page_size = 1000 74 | self._verbose = False 75 | self._quiet = False 76 | self.dbtype: DBType = DBType.UNDEFINED 77 | self.db: dbapi.DBAPIConnection | None = None 78 | self.login_token: str | None = None 79 | self.legacy_auth = True 80 | self.okapi_url: str | None = None 81 | self.okapi_tenant: str | None = None 82 | self.okapi_user: str | None = None 83 | self.okapi_password: str | None = None 84 | self._okapi_timeout = 60 85 | self._okapi_max_retries = 2 86 | 87 | def _set_page_size(self, page_size: int) -> None: 88 | self.page_size = page_size 89 | 90 | def connect_db(self, filename: str | None = None) -> None: 91 | """Connects to an embedded database for storing data. 92 | 93 | The optional *filename* designates a local file containing the 94 | database or where the database will be created if it does not exist. 95 | If *filename* is not specified, the database will be stored in memory 96 | and will not be persisted to disk. 97 | 98 | This method returns a connection to the database which can be used to 99 | submit SQL queries. 100 | 101 | Example: 102 | db = ld.connect_db(filename='ldlite.db') 103 | 104 | """ 105 | self._connect_db_duckdb(filename) 106 | 107 | def _connect_db_duckdb( 108 | self, 109 | filename: str | None = None, 110 | ) -> duckdb.DuckDBPyConnection: 111 | """Connects to an embedded DuckDB database for storing data. 112 | 113 | The optional *filename* designates a local file containing the DuckDB 114 | database or where the database will be created if it does not exist. 115 | If *filename* is not specified, the database will be stored in memory 116 | and will not be persisted to disk. 117 | 118 | This method returns a connection to the database which can be used to 119 | submit SQL queries. 120 | 121 | Example: 122 | db = ld.connect_db_duckdb(filename='ldlite.db') 123 | 124 | """ 125 | self.dbtype = DBType.DUCKDB 126 | fn = filename if filename is not None else ":memory:" 127 | db = duckdb.connect(database=fn) 128 | self.db = cast("dbapi.DBAPIConnection", duckdb.connect(database=fn)) 129 | return db 130 | 131 | def connect_db_postgresql(self, dsn: str) -> psycopg2.extensions.connection: 132 | """Connects to a PostgreSQL database for storing data. 133 | 134 | The data source name is specified by *dsn*. This method returns a 135 | connection to the database which can be used to submit SQL queries. 136 | The returned connection defaults to autocommit mode. 137 | 138 | Example: 139 | db = ld.connect_db_postgresql(dsn='dbname=ld host=localhost user=ldlite') 140 | 141 | """ 142 | self.dbtype = DBType.POSTGRES 143 | db = psycopg2.connect(dsn) 144 | self.db = cast("dbapi.DBAPIConnection", db) 145 | autocommit(self.db, self.dbtype, True) 146 | return db 147 | 148 | def experimental_connect_db_sqlite( 149 | self, 150 | filename: str | None = None, 151 | ) -> sqlite3.Connection: 152 | """Connects to an embedded SQLite database for storing data. 153 | 154 | This method is experimental and may not be supported in future versions. 155 | 156 | The optional *filename* designates a local file containing the SQLite 157 | database or where the database will be created if it does not exist. 158 | If *filename* is not specified, the database will be stored in memory 159 | and will not be persisted to disk. 160 | 161 | This method returns a connection to the database which can be used to 162 | submit SQL queries. 163 | 164 | Example: 165 | db = ld.connect_db_sqlite(filename='ldlite.db') 166 | 167 | """ 168 | self.dbtype = DBType.SQLITE 169 | fn = filename if filename is not None else ":memory:" 170 | self.db = sqlite3.connect(fn) 171 | autocommit(self.db, self.dbtype, True) 172 | return self.db 173 | 174 | def _login(self) -> None: 175 | if self._verbose: 176 | print("ldlite: logging in to folio", file=sys.stderr) 177 | hdr = { 178 | "X-Okapi-Tenant": str(self.okapi_tenant), 179 | "Content-Type": "application/json", 180 | } 181 | data = {"username": self.okapi_user, "password": self.okapi_password} 182 | 183 | if self.okapi_url is None: 184 | msg = "okapi_url is required" 185 | raise ValueError(msg) 186 | authn = ( 187 | self.okapi_url 188 | + "/authn/" 189 | + ("login" if self.legacy_auth else "login-with-expiry") 190 | ) 191 | resp = requests.post( 192 | authn, 193 | headers=hdr, 194 | data=json.dumps(data), 195 | timeout=self._okapi_timeout, 196 | ) 197 | if resp.status_code != 201: 198 | raise RuntimeError("HTTP response status code: " + str(resp.status_code)) 199 | 200 | if self.legacy_auth and "x-okapi-token" in resp.headers: 201 | self.login_token = resp.headers["x-okapi-token"] 202 | elif not self.legacy_auth and "folioAccessToken" in resp.cookies: 203 | self.login_token = resp.cookies["folioAccessToken"] 204 | else: 205 | msg = "authentication service did not return a login token" 206 | raise RuntimeError(msg) 207 | 208 | def _check_okapi(self) -> None: 209 | if self.login_token is None: 210 | msg = "connection to folio not configured: use connect_folio()" 211 | raise RuntimeError(msg) 212 | 213 | def _check_db(self) -> None: 214 | if self.db is None: 215 | msg = "no database connection: use connect_db() or connect_db_postgresql()" 216 | raise RuntimeError(msg) 217 | 218 | def connect_folio(self, url: str, tenant: str, user: str, password: str) -> None: 219 | """Connects to a FOLIO instance with a user name and password. 220 | 221 | The *url*, *tenant*, *user*, and *password* settings are FOLIO-specific 222 | connection parameters. 223 | 224 | Example: 225 | ld.connect_folio(url='https://folio-etesting-snapshot-kong.ci.folio.org', 226 | tenant='diku', 227 | user='diku_admin', 228 | password='admin') 229 | 230 | """ 231 | self.connect_okapi(url, tenant, user, password) 232 | 233 | def connect_okapi( 234 | self, 235 | url: str, 236 | tenant: str, 237 | user: str, 238 | password: str, 239 | legacy_auth: bool = False, 240 | ) -> None: # pragma: nocover 241 | """Deprecated; use connect_folio(). This will be removed after Sunflower.""" 242 | if not url.startswith("https://"): 243 | msg = 'url must begin with "https://"' 244 | raise ValueError(msg) 245 | self.okapi_url = url.rstrip("/") 246 | self.okapi_tenant = tenant 247 | self.okapi_user = user 248 | self.okapi_password = password 249 | self.legacy_auth = legacy_auth 250 | self._login() 251 | 252 | def connect_okapi_token( 253 | self, 254 | url: str, 255 | tenant: str, 256 | token: str, 257 | ) -> None: # pragma: nocover 258 | """Deprecated; use connect_folio(). This will be removed after Sunflower.""" 259 | self.okapi_url = url.rstrip("/") 260 | self.okapi_tenant = tenant 261 | self.login_token = token 262 | 263 | def drop_tables(self, table: str) -> None: 264 | """Drops a specified table and any accompanying tables. 265 | 266 | A table called *table*_jtable is used to retrieve the names of the 267 | tables created by JSON transformation. 268 | 269 | Example: 270 | ld.drop_tables('g') 271 | 272 | """ 273 | if self.db is None: 274 | self._check_db() 275 | return 276 | autocommit(self.db, self.dbtype, True) 277 | schema_table = table.strip().split(".") 278 | if len(schema_table) < 1 or len(schema_table) > 2: 279 | raise ValueError("invalid table name: " + table) 280 | self._check_db() 281 | cur = self.db.cursor() 282 | try: 283 | cur.execute("DROP TABLE IF EXISTS " + sqlid(table)) 284 | except (RuntimeError, psycopg2.Error): 285 | pass 286 | finally: 287 | cur.close() 288 | drop_json_tables(self.db, table) 289 | 290 | def set_folio_max_retries(self, max_retries: int) -> None: 291 | """Sets the maximum number of retries for FOLIO requests. 292 | 293 | This method changes the configured maximum number of retries which is 294 | initially set to 2. The *max_retries* parameter is the new value. 295 | 296 | Note that a request is only retried if a timeout occurs. 297 | 298 | Example: 299 | ld.set_folio_max_retries(5) 300 | 301 | """ 302 | self.set_okapi_max_retries(max_retries) 303 | 304 | def set_okapi_max_retries(self, max_retries: int) -> None: 305 | """Deprecated; use set_folio_max_retries(). Will be removed after Sunflower.""" 306 | self._okapi_max_retries = max_retries 307 | 308 | def set_folio_timeout(self, timeout: int) -> None: 309 | """Sets the timeout for connections to FOLIO. 310 | 311 | This method changes the configured timeout which is initially set to 60 312 | seconds. The *timeout* parameter is the new timeout in seconds. 313 | 314 | Example: 315 | ld.set_folio_timeout(300) 316 | 317 | """ 318 | self.set_okapi_timeout(timeout) 319 | 320 | def set_okapi_timeout(self, timeout: int) -> None: 321 | """Deprecated; use set_folio_timeout(). This will be removed after Sunflower.""" 322 | self._okapi_timeout = timeout 323 | 324 | def query( # noqa: C901, PLR0912, PLR0913, PLR0915 325 | self, 326 | table: str, 327 | path: str, 328 | query: str | None = None, 329 | json_depth: int = 3, 330 | limit: int | None = None, 331 | transform: bool | None = None, 332 | ) -> list[str]: 333 | """Submits a query to a FOLIO module, and transforms and stores the result. 334 | 335 | The retrieved result is stored in *table* within the reporting 336 | database. the *table* name may include a schema name; 337 | however, if the database is SQLite, which does not support 338 | schemas, the schema name will be added to the table name as a 339 | prefix. 340 | 341 | The *path* parameter is the request path. 342 | 343 | If *query* is a string, it is assumed to be a CQL or similar 344 | query and is encoded as query=*query*. If *query* is a 345 | dictionary, it is interpreted as a set of query parameters. 346 | Each value of the dictionary must be either a string or a list 347 | of strings. If a string, it is encoded as key=value. If a 348 | list of strings, it is encoded as key=value1&key=value2&... 349 | 350 | By default JSON data are transformed into one or more tables 351 | that are created in addition to *table*. New tables overwrite 352 | any existing tables having the same name. If *json_depth* is 353 | specified within the range 0 < *json_depth* < 5, this 354 | determines how far into nested JSON data the transformation 355 | will descend. (The default is 3.) If *json_depth* is 356 | specified as 0, JSON data are not transformed. 357 | 358 | If *limit* is specified, then only up to *limit* records are 359 | retrieved. 360 | 361 | The *transform* parameter is no longer supported and will be 362 | removed in the future. Instead, specify *json_depth* as 0 to 363 | disable JSON transformation. 364 | 365 | This method returns a list of newly created tables, or raises 366 | ValueError or RuntimeError. 367 | 368 | Example: 369 | ld.query(table='g', path='/groups', query='cql.allRecords=1 sortby id') 370 | 371 | """ 372 | if transform is not None: 373 | msg = ( 374 | "transform is no longer supported: " 375 | "use json_depth=0 to disable JSON transformation" 376 | ) 377 | raise ValueError(msg) 378 | schema_table = table.split(".") 379 | if len(schema_table) != 1 and len(schema_table) != 2: 380 | raise ValueError("invalid table name: " + table) 381 | if json_depth is None or json_depth < 0 or json_depth > 4: 382 | raise ValueError("invalid value for json_depth: " + str(json_depth)) 383 | self._check_okapi() 384 | if self.db is None: 385 | self._check_db() 386 | return [] 387 | if len(schema_table) == 2 and self.dbtype == DBType.SQLITE: 388 | table = schema_table[0] + "_" + schema_table[1] 389 | schema_table = [table] 390 | if not self._quiet: 391 | print("ldlite: querying: " + path, file=sys.stderr) 392 | querycopy = query_dict(query) 393 | drop_json_tables(self.db, table) 394 | autocommit(self.db, self.dbtype, False) 395 | try: 396 | cur = self.db.cursor() 397 | try: 398 | if len(schema_table) == 2: 399 | cur.execute("CREATE SCHEMA IF NOT EXISTS " + sqlid(schema_table[0])) 400 | cur.execute("DROP TABLE IF EXISTS " + sqlid(table)) 401 | cur.execute( 402 | "CREATE TABLE " 403 | + sqlid(table) 404 | + "(__id integer, jsonb " 405 | + json_type(self.dbtype) 406 | + ")", 407 | ) 408 | finally: 409 | cur.close() 410 | self.db.commit() 411 | # First get total number of records 412 | hdr = { 413 | "X-Okapi-Tenant": str(self.okapi_tenant), 414 | "X-Okapi-Token": str(self.login_token), 415 | } 416 | querycopy["offset"] = "0" 417 | querycopy["limit"] = "1" 418 | resp = request_get( 419 | str(self.okapi_url) + path, 420 | params=querycopy, 421 | headers=hdr, 422 | timeout=self._okapi_timeout, 423 | max_retries=self._okapi_max_retries, 424 | ) 425 | if resp.status_code == 401: 426 | # Retry 427 | # Warning! There is now an edge case with expiring tokens. 428 | # If a request has already been retried some number of times 429 | # then it would be retried for the full _okapi_max_retries value again. 430 | # This will be cleaned up in future releases after tests are added 431 | # to allow for bigger internal changes. 432 | self._login() 433 | hdr = { 434 | "X-Okapi-Tenant": str(self.okapi_tenant), 435 | "X-Okapi-Token": str(self.login_token), 436 | } 437 | resp = request_get( 438 | str(self.okapi_url) + path, 439 | params=querycopy, 440 | headers=hdr, 441 | timeout=self._okapi_timeout, 442 | max_retries=self._okapi_max_retries, 443 | ) 444 | if resp.status_code != 200: 445 | raise RuntimeError( 446 | "HTTP response status code: " + str(resp.status_code), 447 | ) 448 | try: 449 | j = resp.json() 450 | except (json.JSONDecodeError, ValueError) as e: 451 | raise RuntimeError("received server response: " + resp.text) from e 452 | total_records = j.get("totalRecords", -1) 453 | total = total_records if total_records is not None else 0 454 | if self._verbose: 455 | print("ldlite: estimated row count: " + str(total), file=sys.stderr) 456 | # Read result pages 457 | count = 0 458 | page = 0 459 | pbar = None 460 | pbartotal = 0 461 | if not self._quiet: 462 | if total == -1: 463 | pbar = tqdm( 464 | desc="reading", 465 | leave=False, 466 | mininterval=3, 467 | smoothing=0, 468 | colour="#A9A9A9", 469 | bar_format="{desc} {elapsed} {bar}{postfix}", 470 | ) 471 | else: 472 | pbar = tqdm( 473 | desc="reading", 474 | total=total, 475 | leave=False, 476 | mininterval=3, 477 | smoothing=0, 478 | colour="#A9A9A9", 479 | bar_format="{desc} {bar}{postfix}", 480 | ) 481 | cur = self.db.cursor() 482 | try: 483 | while True: 484 | offset = page * self.page_size 485 | lim = self.page_size 486 | querycopy["offset"] = str(offset) 487 | querycopy["limit"] = str(lim) 488 | resp = request_get( 489 | str(self.okapi_url) + path, 490 | params=querycopy, 491 | headers=hdr, 492 | timeout=self._okapi_timeout, 493 | max_retries=self._okapi_max_retries, 494 | ) 495 | if resp.status_code == 401: 496 | # See warning above for retries 497 | self._login() 498 | hdr = { 499 | "X-Okapi-Tenant": str(self.okapi_tenant), 500 | "X-Okapi-Token": str(self.login_token), 501 | } 502 | resp = request_get( 503 | str(self.okapi_url) + path, 504 | params=querycopy, 505 | headers=hdr, 506 | timeout=self._okapi_timeout, 507 | max_retries=self._okapi_max_retries, 508 | ) 509 | if resp.status_code != 200: 510 | raise RuntimeError( 511 | "HTTP response status code: " + str(resp.status_code), 512 | ) 513 | try: 514 | j = resp.json() 515 | except (json.JSONDecodeError, ValueError) as e: 516 | raise RuntimeError( 517 | "received server response: " + resp.text, 518 | ) from e 519 | data = next(iter(j.values())) if isinstance(j, dict) else j 520 | lendata = len(data) 521 | if lendata == 0: 522 | break 523 | for d in data: 524 | cur.execute( 525 | "INSERT INTO " 526 | + sqlid(table) 527 | + " VALUES(" 528 | + str(count + 1) 529 | + "," 530 | + encode_sql_str(self.dbtype, json.dumps(d, indent=4)) 531 | + ")", 532 | ) 533 | count += 1 534 | if pbar is not None: 535 | if pbartotal + 1 > total: 536 | pbartotal = total 537 | pbar.update(total - pbartotal) 538 | else: 539 | pbartotal += 1 540 | pbar.update(1) 541 | if limit is not None and count == limit: 542 | break 543 | if limit is not None and count == limit: 544 | break 545 | page += 1 546 | finally: 547 | cur.close() 548 | if pbar is not None: 549 | pbar.close() 550 | self.db.commit() 551 | newtables = [table] 552 | newattrs = {} 553 | if json_depth > 0: 554 | jsontables, jsonattrs = transform_json( 555 | self.db, 556 | self.dbtype, 557 | table, 558 | count, 559 | self._quiet, 560 | json_depth, 561 | ) 562 | newtables += jsontables 563 | newattrs = jsonattrs 564 | for t in newattrs: 565 | newattrs[t]["__id"] = Attr("__id", "bigint") 566 | newattrs[table] = {"__id": Attr("__id", "bigint")} 567 | finally: 568 | autocommit(self.db, self.dbtype, True) 569 | # Create indexes (for postgres) 570 | # This broke in the 0.34.0 release but is probably better broken 571 | if False: 572 | index_total = sum(map(len, newattrs.values())) 573 | if not self._quiet: 574 | pbar = tqdm( 575 | desc="indexing", 576 | total=index_total, 577 | leave=False, 578 | mininterval=3, 579 | smoothing=0, 580 | colour="#A9A9A9", 581 | bar_format="{desc} {bar}{postfix}", 582 | ) 583 | pbartotal = 0 584 | for t, attrs in newattrs.items(): 585 | for attr in attrs.values(): 586 | cur = self.db.cursor() 587 | try: 588 | cur.execute( 589 | "CREATE INDEX ON " 590 | + sqlid(t) 591 | + " (" 592 | + sqlid(attr.name) 593 | + ")", 594 | ) 595 | except (RuntimeError, psycopg2.Error): 596 | pass 597 | finally: 598 | cur.close() 599 | if pbar is not None: 600 | pbartotal += 1 601 | pbar.update(1) 602 | if pbar is not None: 603 | pbar.close() 604 | # Return table names 605 | if not self._quiet: 606 | print("ldlite: created tables: " + ", ".join(newtables), file=sys.stderr) 607 | return newtables 608 | 609 | def quiet(self, enable: bool) -> None: 610 | """Configures suppression of progress messages. 611 | 612 | If *enable* is True, progress messages are suppressed; if False, they 613 | are not suppressed. 614 | 615 | Example: 616 | ld.quiet(True) 617 | 618 | """ 619 | if enable and self._verbose: 620 | msg = '"verbose" and "quiet" modes cannot both be enabled' 621 | raise ValueError(msg) 622 | self._quiet = enable 623 | 624 | def select( 625 | self, 626 | table: str, 627 | columns: list[str] | None = None, 628 | limit: int | None = None, 629 | ) -> None: 630 | """Prints rows of a table in the reporting database. 631 | 632 | By default all rows and columns of *table* are printed to standard 633 | output. If *columns* is specified, then only the named columns are 634 | printed. If *limit* is specified, then only up to *limit* rows are 635 | printed. 636 | 637 | Examples: 638 | ld.select(table='loans', limit=10) 639 | ld.select(table='loans', columns=['id', 'item_id', 'loan_date']) 640 | 641 | """ 642 | if self.db is None: 643 | self._check_db() 644 | return 645 | 646 | f = sys.stdout 647 | if self._verbose: 648 | print("ldlite: reading from table: " + table, file=sys.stderr) 649 | autocommit(self.db, self.dbtype, False) 650 | try: 651 | select(self.db, self.dbtype, table, columns, limit, f) 652 | if (pgdb := as_postgres(self.db, self.dbtype)) is not None: 653 | pgdb.rollback() 654 | finally: 655 | autocommit(self.db, self.dbtype, True) 656 | 657 | def export_csv(self, filename: str, table: str, header: bool = True) -> None: 658 | """Export a table in the reporting database to a CSV file. 659 | 660 | All rows of *table* are exported to *filename*, or *filename*.csv if 661 | *filename* does not have an extension. 662 | 663 | If *header* is True (the default), the CSV file will begin with a 664 | header line containing the column names. 665 | 666 | Example: 667 | ld.to_csv(table='g', filename='g.csv') 668 | 669 | """ 670 | if self.db is None: 671 | self._check_db() 672 | return 673 | 674 | autocommit(self.db, self.dbtype, False) 675 | try: 676 | to_csv(self.db, self.dbtype, table, filename, header) 677 | if (pgdb := as_postgres(self.db, self.dbtype)) is not None: 678 | pgdb.rollback() 679 | finally: 680 | autocommit(self.db, self.dbtype, True) 681 | 682 | def to_csv(self) -> NoReturn: # pragma: nocover 683 | """Deprecated; use export_csv().""" 684 | msg = "to_csv() is no longer supported: use export_csv()" 685 | raise ValueError(msg) 686 | 687 | def export_excel( 688 | self, 689 | filename: str, 690 | table: str, 691 | header: bool = True, 692 | ) -> None: # pragma: nocover 693 | """Export a table in the reporting database to an Excel file. 694 | 695 | All rows of *table* are exported to *filename*, or *filename*.xlsx if 696 | *filename* does not have an extension. 697 | 698 | If *header* is True (the default), the worksheet will begin with a row 699 | containing the column names. 700 | 701 | Example: 702 | ld.export_excel(table='g', filename='g') 703 | 704 | """ 705 | if self.db is None: 706 | self._check_db() 707 | return 708 | 709 | autocommit(self.db, self.dbtype, False) 710 | try: 711 | to_xlsx(self.db, self.dbtype, table, filename, header) 712 | if (pgdb := as_postgres(self.db, self.dbtype)) is not None: 713 | pgdb.rollback() 714 | finally: 715 | autocommit(self.db, self.dbtype, True) 716 | 717 | def to_xlsx(self) -> NoReturn: # pragma: nocover 718 | """Deprecated; use export_excel().""" 719 | msg = "to_xlsx() is no longer supported: use export_excel()" 720 | raise ValueError(msg) 721 | 722 | def verbose(self, enable: bool) -> None: 723 | """Configures verbose output. 724 | 725 | If *enable* is True, verbose output is enabled; if False, it is 726 | disabled. 727 | 728 | Example: 729 | ld.verbose(True) 730 | 731 | """ 732 | if enable and self._quiet: 733 | msg = '"verbose" and "quiet" modes cannot both be enabled' 734 | raise ValueError(msg) 735 | self._verbose = enable 736 | 737 | 738 | if __name__ == "__main__": 739 | pass 740 | -------------------------------------------------------------------------------- /src/ldlite/_camelcase.py: -------------------------------------------------------------------------------- 1 | def _decode_triple(c1: str, c2: str, c3: str) -> str: # noqa: C901 2 | """Decodes a sliding window of character triples. 3 | 4 | Examines a sequence of three characters c1, c2, and c3; decodes c2; and 5 | returns the decoded output. 6 | """ 7 | b = "" 8 | c1u = c1.isupper() 9 | c2u = c2.isupper() 10 | c3u = c3.isupper() 11 | write = chr(0) 12 | write_break = False 13 | # First check triples that include zeros 14 | if c2 == chr(0): 15 | return b 16 | if c1 == chr(0) and c2 != chr(0): 17 | write = c2.lower() 18 | elif c1 != chr(0) and c2 != chr(0) and c3 == chr(0): 19 | if not c1u and c2u: 20 | write_break = True 21 | write = c2.lower() 22 | elif c1u and c2u: 23 | write = c2.lower() 24 | elif (not c1u and not c2u) or (c1u and not c2u): 25 | write = c2 26 | # Check triples having no zeros 27 | elif ( 28 | (not c1u and not c2u and not c3u) 29 | or (c1u and not c2u and not c3u) 30 | or (not c1u and not c2u and c3u) 31 | or (c1u and not c2u and c3u) 32 | ): 33 | write = c2 34 | elif ( 35 | (not c1u and c2u and not c3u) 36 | or (c1u and c2u and not c3u) 37 | or (not c1u and c2u and c3u) 38 | ): 39 | write_break = True 40 | write = c2.lower() 41 | elif c1u and c2u and c3u: 42 | write = c2.lower() 43 | else: 44 | # All cases should have been checked by this point 45 | raise ValueError('unexpected state: ("' + c1 + '", "' + c2 + '", "' + c3 + '")') 46 | # Write decoded characters 47 | if write_break: 48 | b += "_" 49 | b += write 50 | return b 51 | 52 | 53 | def decode_camel_case(s: str) -> str: 54 | """Parses camel case string into lowercase words separated by underscores. 55 | 56 | A sequence of uppercase letters is interpreted as a word, except that the 57 | last uppercase letter of a sequence is considered the start of a new word 58 | if it is followed by a lowercase letter. 59 | """ 60 | b = "" 61 | # c1, c2, and c3 are a sliding window of character triples 62 | c1 = chr(0) 63 | _ = c1 64 | c2 = chr(0) 65 | c3 = chr(0) 66 | for c in s: 67 | c1 = c2 68 | c2 = c3 69 | c3 = c 70 | b += _decode_triple(c1, c2, c3) 71 | # Last character 72 | c1 = c2 73 | c2 = c3 74 | c3 = chr(0) 75 | b += _decode_triple(c1, c2, c3) 76 | return b 77 | -------------------------------------------------------------------------------- /src/ldlite/_csv.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from pathlib import Path 4 | from typing import TYPE_CHECKING 5 | 6 | from ._sqlx import DBType, as_sqlite, server_cursor, sqlid 7 | 8 | if TYPE_CHECKING: 9 | from _typeshed import dbapi 10 | 11 | 12 | def _escape_csv(field: str) -> str: 13 | b = "" 14 | for f in field: 15 | if f == '"': 16 | b += '""' 17 | else: 18 | b += f 19 | return b 20 | 21 | 22 | def to_csv( # noqa: PLR0912 23 | db: dbapi.DBAPIConnection, 24 | dbtype: DBType, 25 | table: str, 26 | filename: str, 27 | header: bool, 28 | ) -> None: 29 | # Read attributes 30 | attrs: list[tuple[str, dbapi.DBAPITypeCode]] = [("__id", "NUMBER")] 31 | 32 | if sql3db := as_sqlite(db, dbtype): 33 | sql3cur = sql3db.cursor() 34 | try: 35 | sql3cur.execute("PRAGMA table_info(" + sqlid(table) + ")") 36 | t_attrs = [(a[1], a[2]) for a in sql3cur.fetchall()[1:]] 37 | attrs.extend(sorted(t_attrs, key=lambda a: a[0])) 38 | finally: 39 | sql3cur.close() 40 | 41 | else: 42 | cur = server_cursor(db, dbtype) 43 | try: 44 | cur.execute("SELECT * FROM " + sqlid(table) + " LIMIT 1") 45 | cur.fetchall() 46 | if cur.description is not None: 47 | t_attrs = [(a[0], a[1]) for a in cur.description[1:]] 48 | attrs.extend(sorted(t_attrs, key=lambda a: a[0])) 49 | finally: 50 | cur.close() 51 | 52 | # Write data 53 | cur = server_cursor(db, dbtype) 54 | try: 55 | cols = ",".join([sqlid(a[0]) for a in attrs]) 56 | cur.execute( 57 | "SELECT " 58 | + cols 59 | + " FROM " 60 | + sqlid(table) 61 | + " ORDER BY " 62 | + ",".join([str(i + 2) for i in range(len(attrs[1:]))]), 63 | ) 64 | fn = Path(filename if "." in filename else filename + ".csv") 65 | with fn.open("w") as f: 66 | if header: 67 | print(",".join(['"' + a[0] + '"' for a in attrs]), file=f) 68 | while True: 69 | row = cur.fetchone() 70 | if row is None: 71 | break 72 | s = "" 73 | for i, data in enumerate(row): 74 | d = "" if data is None else data 75 | if i != 0: 76 | s += "," 77 | if attrs[i][1] in [ 78 | "NUMBER", 79 | "bigint", 80 | "numeric", 81 | 20, 82 | 1700, 83 | ]: 84 | s += str(d).rstrip("0").rstrip(".") 85 | elif attrs[i][1] in [ 86 | "boolean", 87 | "bool", 88 | 16, 89 | ]: 90 | s += str(bool(d)) 91 | 92 | else: 93 | s += '"' + _escape_csv(str(d)) + '"' 94 | print(s, file=f) 95 | finally: 96 | cur.close() 97 | -------------------------------------------------------------------------------- /src/ldlite/_jsonx.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | import sqlite3 5 | import uuid 6 | from typing import TYPE_CHECKING, Literal, Union 7 | 8 | import duckdb 9 | import psycopg2 10 | from tqdm import tqdm 11 | 12 | from ._camelcase import decode_camel_case 13 | from ._sqlx import ( 14 | DBType, 15 | cast_to_varchar, 16 | encode_sql, 17 | server_cursor, 18 | sqlid, 19 | varchar_type, 20 | ) 21 | 22 | if TYPE_CHECKING: 23 | from _typeshed import dbapi 24 | 25 | JsonValue = Union[float, int, str, bool, "Json", list["JsonValue"], None] 26 | Json = dict[str, JsonValue] 27 | 28 | 29 | def _is_uuid(val: str) -> bool: 30 | try: 31 | uuid.UUID(val) 32 | except ValueError: 33 | return False 34 | 35 | return True 36 | 37 | 38 | class Attr: 39 | def __init__( 40 | self, 41 | name: str, 42 | datatype: Literal["varchar", "integer", "numeric", "boolean", "uuid", "bigint"], 43 | order: None | Literal[1, 2, 3] = None, 44 | data: JsonValue = None, 45 | ): 46 | self.name = name 47 | self.datatype: Literal[ 48 | "varchar", 49 | "integer", 50 | "numeric", 51 | "boolean", 52 | "uuid", 53 | "bigint", 54 | ] = datatype 55 | self.order: None | Literal[1, 2, 3] = order 56 | self.data = data 57 | 58 | def sqlattr(self, dbtype: DBType) -> str: 59 | return ( 60 | sqlid(self.name) 61 | + " " 62 | + (varchar_type(dbtype) if self.datatype == "varchar" else self.datatype) 63 | ) 64 | 65 | def __repr__(self) -> str: 66 | if self.data is None: 67 | return ( 68 | "(name=" 69 | + self.name 70 | + ", datatype=" 71 | + self.datatype 72 | + ", order=" 73 | + str(self.order) 74 | + ")" 75 | ) 76 | return ( 77 | "(name=" 78 | + self.name 79 | + ", datatype=" 80 | + self.datatype 81 | + ", order=" 82 | + str(self.order) 83 | + ", data=" 84 | + str(self.data) 85 | + ")" 86 | ) 87 | 88 | 89 | def _old_jtable(table: str) -> str: 90 | return table + "_jtable" 91 | 92 | 93 | def _tcatalog(table: str) -> str: 94 | return table + "__tcatalog" 95 | 96 | 97 | # noinspection DuplicatedCode 98 | def _old_drop_json_tables(db: dbapi.DBAPIConnection, table: str) -> None: 99 | jtable_sql = sqlid(_old_jtable(table)) 100 | cur = db.cursor() 101 | try: 102 | cur.execute("SELECT table_name FROM " + jtable_sql) 103 | rows = list(cur.fetchall()) 104 | for row in rows: 105 | t = row[0] 106 | cur2 = db.cursor() 107 | try: 108 | cur2.execute("DROP TABLE " + sqlid(t)) 109 | except (psycopg2.Error, duckdb.CatalogException, sqlite3.OperationalError): 110 | continue 111 | finally: 112 | cur2.close() 113 | except ( 114 | psycopg2.Error, 115 | sqlite3.OperationalError, 116 | duckdb.CatalogException, 117 | ): 118 | pass 119 | finally: 120 | cur.close() 121 | cur = db.cursor() 122 | try: 123 | cur.execute("DROP TABLE " + jtable_sql) 124 | except ( 125 | psycopg2.Error, 126 | duckdb.CatalogException, 127 | sqlite3.OperationalError, 128 | ): 129 | pass 130 | finally: 131 | cur.close() 132 | 133 | 134 | # noinspection DuplicatedCode 135 | def drop_json_tables(db: dbapi.DBAPIConnection, table: str) -> None: 136 | tcatalog_sql = sqlid(_tcatalog(table)) 137 | cur = db.cursor() 138 | try: 139 | cur.execute("SELECT table_name FROM " + tcatalog_sql) 140 | rows = list(cur.fetchall()) 141 | for row in rows: 142 | t = row[0] 143 | cur2 = db.cursor() 144 | try: 145 | cur2.execute("DROP TABLE " + sqlid(t)) 146 | except (psycopg2.Error, duckdb.CatalogException, sqlite3.OperationalError): 147 | continue 148 | finally: 149 | cur2.close() 150 | except ( 151 | psycopg2.Error, 152 | duckdb.CatalogException, 153 | sqlite3.OperationalError, 154 | ): 155 | pass 156 | finally: 157 | cur.close() 158 | cur = db.cursor() 159 | try: 160 | cur.execute("DROP TABLE " + tcatalog_sql) 161 | except ( 162 | psycopg2.Error, 163 | duckdb.CatalogException, 164 | sqlite3.OperationalError, 165 | ): 166 | pass 167 | finally: 168 | cur.close() 169 | _old_drop_json_tables(db, table) 170 | 171 | 172 | def _table_name(parents: list[tuple[int, str]]) -> str: 173 | j = len(parents) 174 | while j > 0 and parents[j - 1][0] == 0: 175 | j -= 1 176 | table = "" 177 | i = 0 178 | while i < j: 179 | if i != 0: 180 | table += "__" 181 | table += parents[i][1] 182 | i += 1 183 | return table 184 | 185 | 186 | def _compile_array_attrs( # noqa: PLR0913 187 | dbtype: DBType, 188 | parents: list[tuple[int, str]], 189 | prefix: str, 190 | jarray: list[JsonValue], 191 | newattrs: dict[str, dict[str, Attr]], 192 | depth: int, 193 | arrayattr: str, 194 | max_depth: int, 195 | quasikey: dict[str, Attr], 196 | ) -> None: 197 | if depth > max_depth: 198 | return 199 | table = _table_name(parents) 200 | qkey = {} 201 | for k, a in quasikey.items(): 202 | qkey[k] = Attr(a.name, a.datatype, order=1) 203 | if table not in newattrs: 204 | newattrs[table] = {} 205 | for k, a in quasikey.items(): 206 | newattrs[table][k] = Attr(a.name, a.datatype, order=1) 207 | j_ord = Attr(prefix + "o", "integer", order=2) 208 | qkey[prefix + "o"] = j_ord 209 | newattrs[table][prefix + "o"] = j_ord 210 | for v in jarray: 211 | if isinstance(v, dict): 212 | _compile_attrs(dbtype, parents, prefix, v, newattrs, depth, max_depth, qkey) 213 | elif isinstance(v, list): 214 | # TODO: ??? 215 | continue 216 | elif isinstance(v, (float, int)): 217 | newattrs[table][arrayattr] = Attr( 218 | decode_camel_case(arrayattr), 219 | "numeric", 220 | order=3, 221 | ) 222 | else: 223 | newattrs[table][arrayattr] = Attr( 224 | decode_camel_case(arrayattr), 225 | "varchar", 226 | order=3, 227 | ) 228 | 229 | 230 | def _compile_attrs( # noqa: C901, PLR0912, PLR0913 231 | dbtype: DBType, 232 | parents: list[tuple[int, str]], 233 | prefix: str, 234 | jdict: Json, 235 | newattrs: dict[str, dict[str, Attr]], 236 | depth: int, 237 | max_depth: int, 238 | quasikey: dict[str, Attr], 239 | ) -> None: 240 | if depth > max_depth: 241 | return 242 | table = _table_name(parents) 243 | qkey = {} 244 | for k, a in quasikey.items(): 245 | qkey[k] = Attr(a.name, a.datatype, order=1) 246 | arrays: list[tuple[str, list[JsonValue], str]] = [] 247 | objects: list[tuple[str, Json, str]] = [] 248 | for k, v in jdict.items(): 249 | if k is None or v is None: 250 | continue 251 | attr = prefix + k 252 | if isinstance(v, dict): 253 | if depth == max_depth: 254 | newattrs[table][attr] = Attr( 255 | decode_camel_case(attr), 256 | "varchar", 257 | order=3, 258 | ) 259 | else: 260 | objects.append((attr, v, k)) 261 | elif isinstance(v, list): 262 | arrays.append((attr, v, k)) 263 | elif isinstance(v, bool): 264 | a = Attr(decode_camel_case(attr), "boolean", order=3) 265 | qkey[attr] = a 266 | newattrs[table][attr] = a 267 | elif isinstance(v, (float, int)): 268 | a = Attr(decode_camel_case(attr), "numeric", order=3) 269 | qkey[attr] = a 270 | newattrs[table][attr] = a 271 | elif dbtype == DBType.POSTGRES and _is_uuid(v): 272 | a = Attr(decode_camel_case(attr), "uuid", order=3) 273 | qkey[attr] = a 274 | newattrs[table][attr] = a 275 | else: 276 | a = Attr(decode_camel_case(attr), "varchar", order=3) 277 | qkey[attr] = a 278 | newattrs[table][attr] = a 279 | for b in objects: 280 | p = [(0, decode_camel_case(b[2]))] 281 | _compile_attrs( 282 | dbtype, 283 | parents + p, 284 | decode_camel_case(b[0]) + "__", 285 | b[1], 286 | newattrs, 287 | depth + 1, 288 | max_depth, 289 | qkey, 290 | ) 291 | for y in arrays: 292 | p = [(1, decode_camel_case(y[2]))] 293 | _compile_array_attrs( 294 | dbtype, 295 | parents + p, 296 | decode_camel_case(y[0]) + "__", 297 | y[1], 298 | newattrs, 299 | depth + 1, 300 | y[0], 301 | max_depth, 302 | qkey, 303 | ) 304 | 305 | 306 | def _transform_array_data( # noqa: PLR0913 307 | dbtype: DBType, 308 | prefix: str, 309 | cur: dbapi.DBAPICursor, 310 | parents: list[tuple[int, str]], 311 | jarray: list[JsonValue], 312 | newattrs: dict[str, dict[str, Attr]], 313 | depth: int, 314 | row_ids: dict[str, int], 315 | arrayattr: str, 316 | max_depth: int, 317 | quasikey: dict[str, Attr], 318 | ) -> None: 319 | if depth > max_depth: 320 | return 321 | table = _table_name(parents) 322 | for i, v in enumerate(jarray): 323 | if v is None: 324 | continue 325 | if isinstance(v, dict): 326 | qkey = {k: quasikey[k] for k in quasikey} 327 | qkey[prefix + "o"] = Attr(prefix + "o", "integer", data=i + 1) 328 | _transform_data( 329 | dbtype, 330 | prefix, 331 | cur, 332 | parents, 333 | v, 334 | newattrs, 335 | depth, 336 | row_ids, 337 | max_depth, 338 | qkey, 339 | ) 340 | continue 341 | if isinstance(v, list): 342 | # TODO: ??? 343 | continue 344 | a = newattrs[table][arrayattr] 345 | a.data = v 346 | value = v 347 | q = "INSERT INTO " + sqlid(table) + "(__id" 348 | q += ( 349 | "" 350 | if len(quasikey) == 0 351 | else "," + ",".join([sqlid(kv[1].name) for kv in quasikey.items()]) 352 | ) 353 | q += "," + prefix + "o," + sqlid(a.name) 354 | q += ")VALUES(" + str(row_ids[table]) 355 | q += ( 356 | "" 357 | if len(quasikey) == 0 358 | else "," 359 | + ",".join([encode_sql(dbtype, kv[1].data) for kv in quasikey.items()]) 360 | ) 361 | q += "," + str(i + 1) + "," + encode_sql(dbtype, value) + ")" 362 | try: 363 | cur.execute(q) 364 | except (RuntimeError, psycopg2.Error) as e: 365 | raise RuntimeError("error executing SQL: " + q) from e 366 | row_ids[table] += 1 367 | 368 | 369 | def _compile_data( # noqa: C901, PLR0912, PLR0913 370 | dbtype: DBType, 371 | prefix: str, 372 | cur: dbapi.DBAPICursor, 373 | parents: list[tuple[int, str]], 374 | jdict: Json, 375 | newattrs: dict[str, dict[str, Attr]], 376 | depth: int, 377 | row_ids: dict[str, int], 378 | max_depth: int, 379 | quasikey: dict[str, Attr], 380 | ) -> None | list[tuple[str, JsonValue]]: 381 | if depth > max_depth: 382 | return None 383 | table = _table_name(parents) 384 | qkey = {k: quasikey[k] for k in quasikey} 385 | row: list[tuple[str, JsonValue]] = [] 386 | arrays = [] 387 | objects = [] 388 | for k, v in jdict.items(): 389 | if k is None: 390 | continue 391 | attr = prefix + k 392 | if isinstance(v, dict) and depth < max_depth: 393 | objects.append((attr, v, k)) 394 | elif isinstance(v, list): 395 | arrays.append((attr, v, k)) 396 | if attr not in newattrs[table]: 397 | continue 398 | aa = newattrs[table][attr] 399 | a = Attr(aa.name, aa.datatype, data=v) 400 | if a.datatype in {"bigint", "float", "boolean"}: 401 | qkey[attr] = a 402 | row.append((a.name, v)) 403 | else: 404 | qkey[attr] = a 405 | if isinstance(v, dict): 406 | row.append((a.name, json.dumps(v, indent=4))) 407 | else: 408 | row.append((a.name, v)) 409 | for b in objects: 410 | p = [(0, decode_camel_case(b[2]))] 411 | r = _compile_data( 412 | dbtype, 413 | decode_camel_case(b[0]) + "__", 414 | cur, 415 | parents + p, 416 | b[1], 417 | newattrs, 418 | depth + 1, 419 | row_ids, 420 | max_depth, 421 | qkey, 422 | ) 423 | if r is not None: 424 | row += r 425 | for y in arrays: 426 | p = [(1, decode_camel_case(y[2]))] 427 | _transform_array_data( 428 | dbtype, 429 | decode_camel_case(y[0]) + "__", 430 | cur, 431 | parents + p, 432 | y[1], 433 | newattrs, 434 | depth + 1, 435 | row_ids, 436 | y[0], 437 | max_depth, 438 | qkey, 439 | ) 440 | return row 441 | 442 | 443 | def _transform_data( # noqa: PLR0913 444 | dbtype: DBType, 445 | prefix: str, 446 | cur: dbapi.DBAPICursor, 447 | parents: list[tuple[int, str]], 448 | jdict: Json, 449 | newattrs: dict[str, dict[str, Attr]], 450 | depth: int, 451 | row_ids: dict[str, int], 452 | max_depth: int, 453 | quasikey: dict[str, Attr], 454 | ) -> None: 455 | if depth > max_depth: 456 | return 457 | table = _table_name(parents) 458 | row = [(a.name, a.data) for a in quasikey.values()] 459 | r = _compile_data( 460 | dbtype, 461 | prefix, 462 | cur, 463 | parents, 464 | jdict, 465 | newattrs, 466 | depth, 467 | row_ids, 468 | max_depth, 469 | quasikey, 470 | ) 471 | if r is not None: 472 | row += r 473 | q = "INSERT INTO " + sqlid(table) + "(__id," 474 | q += ",".join([sqlid(kv[0]) for kv in row]) 475 | q += ")VALUES(" + str(row_ids[table]) + "," 476 | q += ",".join([encode_sql(dbtype, kv[1]) for kv in row]) 477 | q += ")" 478 | try: 479 | cur.execute(q) 480 | except (RuntimeError, psycopg2.Error) as e: 481 | raise RuntimeError("error executing SQL: " + q) from e 482 | row_ids[table] += 1 483 | 484 | 485 | def transform_json( # noqa: C901, PLR0912, PLR0913, PLR0915 486 | db: dbapi.DBAPIConnection, 487 | dbtype: DBType, 488 | table: str, 489 | total: int, 490 | quiet: bool, 491 | max_depth: int, 492 | ) -> tuple[list[str], dict[str, dict[str, Attr]]]: 493 | # Scan all fields for JSON data 494 | # First get a list of the string attributes 495 | str_attrs: list[str] = [] 496 | cur = db.cursor() 497 | try: 498 | cur.execute("SELECT * FROM " + sqlid(table) + " LIMIT 1") 499 | if cur.description is not None: 500 | str_attrs.extend([a[0] for a in cur.description]) 501 | finally: 502 | cur.close() 503 | # Scan data for JSON objects 504 | if len(str_attrs) == 0: 505 | return [], {} 506 | json_attrs: list[str] = [] 507 | json_attrs_set: set[str] = set() 508 | newattrs: dict[str, dict[str, Attr]] = {} 509 | cur = server_cursor(db, dbtype) 510 | try: 511 | cur.execute( 512 | "SELECT " 513 | + ",".join([cast_to_varchar(sqlid(a), dbtype) for a in str_attrs]) 514 | + " FROM " 515 | + sqlid(table), 516 | ) 517 | pbar = None 518 | pbartotal = 0 519 | if not quiet: 520 | pbar = tqdm( 521 | desc="scanning", 522 | total=total, 523 | leave=False, 524 | mininterval=3, 525 | smoothing=0, 526 | colour="#A9A9A9", 527 | bar_format="{desc} {bar}{postfix}", 528 | ) 529 | while True: 530 | row = cur.fetchone() 531 | if row is None: 532 | break 533 | for i, data in enumerate(row): 534 | if data is None: 535 | continue 536 | ds = data.strip() 537 | if len(ds) == 0 or ds[0] != "{": 538 | continue 539 | try: 540 | jdict = json.loads(ds) 541 | except ValueError: 542 | continue 543 | attr = str_attrs[i] 544 | if attr not in json_attrs_set: 545 | json_attrs.append(attr) 546 | json_attrs_set.add(attr) 547 | attr_index = json_attrs.index(attr) 548 | table_j = ( 549 | table + "__t" 550 | if attr_index == 0 551 | else table + "__t" + str(attr_index + 1) 552 | ) 553 | if table_j not in newattrs: 554 | newattrs[table_j] = {} 555 | _compile_attrs( 556 | dbtype, 557 | [(1, table_j)], 558 | "", 559 | jdict, 560 | newattrs, 561 | 1, 562 | max_depth, 563 | {}, 564 | ) 565 | if pbar is not None: 566 | pbartotal += 1 567 | pbar.update(1) 568 | if pbar is not None: 569 | pbar.close() 570 | finally: 571 | cur.close() 572 | # Create table schemas 573 | cur = db.cursor() 574 | try: 575 | for t, attrs in newattrs.items(): 576 | cur.execute("DROP TABLE IF EXISTS " + sqlid(t)) 577 | cur.execute("CREATE TABLE " + sqlid(t) + "(__id bigint)") 578 | for a in attrs.values(): 579 | if a.order == 1: 580 | cur.execute( 581 | "ALTER TABLE " + sqlid(t) + " ADD COLUMN " + a.sqlattr(dbtype), 582 | ) 583 | for a in attrs.values(): 584 | if a.order == 2: 585 | cur.execute( 586 | "ALTER TABLE " + sqlid(t) + " ADD COLUMN " + a.sqlattr(dbtype), 587 | ) 588 | for a in attrs.values(): 589 | if a.order == 3: 590 | cur.execute( 591 | "ALTER TABLE " + sqlid(t) + " ADD COLUMN " + a.sqlattr(dbtype), 592 | ) 593 | finally: 594 | cur.close() 595 | db.commit() 596 | # Set all row IDs to 1 597 | row_ids = {} 598 | for t in newattrs: 599 | row_ids[t] = 1 600 | # Run transformation 601 | # Select only JSON columns 602 | if len(json_attrs) == 0: 603 | return [], {} 604 | cur = server_cursor(db, dbtype) 605 | try: 606 | cur.execute( 607 | "SELECT " 608 | + ",".join([cast_to_varchar(sqlid(a), dbtype) for a in json_attrs]) 609 | + " FROM " 610 | + sqlid(table), 611 | ) 612 | pbar = None 613 | pbartotal = 0 614 | if not quiet: 615 | pbar = tqdm( 616 | desc="transforming", 617 | total=total, 618 | leave=False, 619 | mininterval=3, 620 | smoothing=0, 621 | colour="#A9A9A9", 622 | bar_format="{desc} {bar}{postfix}", 623 | ) 624 | cur2 = db.cursor() 625 | while True: 626 | row = cur.fetchone() 627 | if row is None: 628 | break 629 | for i, data in enumerate(row): 630 | if data is None: 631 | continue 632 | d = data.strip() 633 | if len(d) == 0 or d[0] != "{": 634 | continue 635 | try: 636 | jdict = json.loads(d) 637 | except ValueError: 638 | continue 639 | table_j = table + "__t" if i == 0 else table + "__t" + str(i + 1) 640 | _transform_data( 641 | dbtype, 642 | "", 643 | cur2, 644 | [(1, table_j)], 645 | jdict, 646 | newattrs, 647 | 1, 648 | row_ids, 649 | max_depth, 650 | {}, 651 | ) 652 | if pbar is not None: 653 | pbartotal += 1 654 | pbar.update(1) 655 | if pbar is not None: 656 | pbar.close() 657 | except ( 658 | RuntimeError, 659 | psycopg2.Error, 660 | sqlite3.OperationalError, 661 | duckdb.CatalogException, 662 | ) as e: 663 | raise RuntimeError("running JSON transform: " + str(e)) from e 664 | finally: 665 | cur.close() 666 | db.commit() 667 | tcatalog = _tcatalog(table) 668 | cur = db.cursor() 669 | try: 670 | cur.execute( 671 | "CREATE TABLE " 672 | + sqlid(tcatalog) 673 | + "(table_name " 674 | + varchar_type(dbtype) 675 | + " NOT NULL)", 676 | ) 677 | for t in newattrs: 678 | cur.execute( 679 | "INSERT INTO " 680 | + sqlid(tcatalog) 681 | + " VALUES(" 682 | + encode_sql(dbtype, t) 683 | + ")", 684 | ) 685 | except ( 686 | RuntimeError, 687 | psycopg2.Error, 688 | sqlite3.OperationalError, 689 | duckdb.CatalogException, 690 | ) as e: 691 | raise RuntimeError("writing table catalog for JSON transform: " + str(e)) from e 692 | finally: 693 | cur.close() 694 | db.commit() 695 | return sorted([*list(newattrs.keys()), tcatalog]), newattrs 696 | -------------------------------------------------------------------------------- /src/ldlite/_query.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import copy 4 | 5 | 6 | def query_dict(query: None | str | dict[str, str]) -> dict[str, str]: 7 | if isinstance(query, str): 8 | return {"query": query} 9 | if isinstance(query, dict): 10 | return copy.deepcopy(query) 11 | 12 | return {} 13 | -------------------------------------------------------------------------------- /src/ldlite/_request.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | 4 | def request_get( 5 | url: str, 6 | params: dict[str, str], 7 | headers: dict[str, str], 8 | timeout: int, 9 | max_retries: int, 10 | ) -> requests.Response: 11 | r = 0 12 | while r < max_retries: 13 | try: 14 | return requests.get(url, params=params, headers=headers, timeout=timeout) 15 | except requests.exceptions.Timeout: 16 | pass 17 | r += 1 18 | return requests.get(url, params=params, headers=headers, timeout=timeout) 19 | -------------------------------------------------------------------------------- /src/ldlite/_select.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sys 4 | from typing import TYPE_CHECKING, TextIO 5 | 6 | from ._sqlx import DBType, server_cursor, sqlid 7 | 8 | if TYPE_CHECKING: 9 | from collections.abc import Sequence 10 | 11 | from _typeshed import dbapi 12 | 13 | 14 | def _format_attr(attr: tuple[str, dbapi.DBAPITypeCode], width: int) -> str: 15 | s = "" 16 | a = attr[0] 17 | len_a = len(a) 18 | shift_left = 1 if (len_a % 2 == 1 and width % 2 == 0 and len_a < width) else 0 19 | start = int(width / 2) - int(len_a / 2) - shift_left 20 | for _ in range(start): 21 | s += " " 22 | s += a 23 | for _ in range(width - start - len_a): 24 | s += " " 25 | return s 26 | 27 | 28 | def _maxlen(lines: list[str]) -> int: 29 | m = 0 30 | for s in lines: 31 | m = max(m, len(s)) 32 | return m 33 | 34 | 35 | def _rstrip_lines(lines: list[str]) -> list[str]: 36 | return [s.rstrip() for s in lines] 37 | 38 | 39 | def _format_value(value: list[str], dtype: dbapi.DBAPITypeCode) -> list[str]: 40 | if len(value) > 1: 41 | return value 42 | if dtype in {"bool", 16}: 43 | return ["t"] if value[0] == "True" else ["f"] 44 | return value 45 | 46 | 47 | def _format_row( 48 | row: Sequence[str], 49 | attrs: list[tuple[str, dbapi.DBAPITypeCode]], 50 | width: list[int], 51 | ) -> str: 52 | s = "" 53 | # Count number of lines 54 | rowlines = [] 55 | maxlen = [] 56 | maxlines = 1 57 | for i, data in enumerate(row): 58 | lines = [""] 59 | if data is not None: 60 | lines = _format_value(_rstrip_lines(str(data).splitlines()), attrs[i][1]) 61 | maxlen.append(_maxlen(lines)) 62 | rowlines.append(lines) 63 | lines_len = len(lines) 64 | maxlines = max(maxlines, lines_len) 65 | # Write lines 66 | for i in range(maxlines): 67 | for j, lines in enumerate(rowlines): 68 | lines_i = lines[i] if i < len(lines) else "" 69 | s += " " if j == 0 else "| " 70 | if attrs[j][1] == "NUMBER" or attrs[j][1] == 20 or attrs[j][1] == 23: 71 | start = width[j] - len(lines_i) 72 | else: 73 | start = 0 74 | for _ in range(start): 75 | s += " " 76 | s += lines_i 77 | for _ in range(width[j] - start - len(lines_i)): 78 | s += " " 79 | s += " " 80 | s += "\n" 81 | return s 82 | 83 | 84 | def select( # noqa: C901, PLR0912, PLR0913, PLR0915 85 | db: dbapi.DBAPIConnection, 86 | dbtype: DBType, 87 | table: str, 88 | columns: list[str] | None, 89 | limit: int | None, 90 | file: TextIO = sys.stdout, 91 | ) -> None: 92 | if columns is None or columns == []: 93 | colspec = "*" 94 | else: 95 | colspec = ",".join([sqlid(c) for c in columns]) 96 | # Get attributes 97 | attrs: list[tuple[str, dbapi.DBAPITypeCode]] = [] 98 | width: list[int] = [] 99 | cur = db.cursor() 100 | try: 101 | cur.execute("SELECT " + colspec + " FROM " + sqlid(table) + " LIMIT 1") 102 | if cur.description is not None: 103 | for a in cur.description: 104 | attrs.append((a[0], a[1])) 105 | width.append(len(a[0])) 106 | finally: 107 | cur.close() 108 | # Scan 109 | cur = server_cursor(db, dbtype) 110 | try: 111 | cur.execute( 112 | "SELECT " 113 | + ",".join([sqlid(a[0]) for a in attrs]) 114 | + " FROM " 115 | + sqlid(table), 116 | ) 117 | while True: 118 | row = cur.fetchone() 119 | if row is None: 120 | break 121 | for i, v in enumerate(row): 122 | lines = [""] 123 | if v is not None: 124 | lines = str(v).splitlines() 125 | for _, ln in enumerate(lines): 126 | width[i] = max(width[i], len(ln.rstrip())) 127 | finally: 128 | cur.close() 129 | cur = server_cursor(db, dbtype) 130 | try: 131 | q = "SELECT " + ",".join([sqlid(a[0]) for a in attrs]) + " FROM " + sqlid(table) 132 | if limit is not None: 133 | q += " LIMIT " + str(limit) 134 | cur.execute(q) 135 | # Attribute names 136 | s = "" 137 | for i, v in enumerate(attrs): 138 | s += " " if i == 0 else "| " 139 | s += _format_attr(v, width[i]) 140 | s += " " 141 | print(s, file=file) 142 | # Header bar 143 | s = "" 144 | for i in range(len(attrs)): 145 | s += "" if i == 0 else "+" 146 | s += "-" 147 | for _ in range(width[i]): 148 | s += "-" 149 | s += "-" 150 | print(s, file=file) 151 | # Data rows 152 | row_i = 0 153 | while True: 154 | row = cur.fetchone() 155 | if row is None: 156 | break 157 | s = _format_row(row, attrs, width) 158 | print(s, end="", file=file) 159 | row_i += 1 160 | print( 161 | "(" + str(row_i) + " " + ("row" if row_i == 1 else "rows") + ")", 162 | file=file, 163 | ) 164 | print(file=file) 165 | finally: 166 | cur.close() 167 | -------------------------------------------------------------------------------- /src/ldlite/_sqlx.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import secrets 4 | from enum import Enum 5 | from typing import TYPE_CHECKING, cast 6 | 7 | if TYPE_CHECKING: 8 | import sqlite3 9 | 10 | import duckdb 11 | import psycopg2 12 | from _typeshed import dbapi 13 | 14 | from ._jsonx import JsonValue 15 | 16 | 17 | class DBType(Enum): 18 | UNDEFINED = 0 19 | DUCKDB = 1 20 | POSTGRES = 2 21 | SQLITE = 4 22 | 23 | 24 | def as_duckdb( 25 | db: dbapi.DBAPIConnection, 26 | dbtype: DBType, 27 | ) -> duckdb.DuckDBPyConnection | None: 28 | if dbtype != DBType.DUCKDB: 29 | return None 30 | 31 | return cast("duckdb.DuckDBPyConnection", db) 32 | 33 | 34 | def as_postgres( 35 | db: dbapi.DBAPIConnection, 36 | dbtype: DBType, 37 | ) -> psycopg2.extensions.connection | None: 38 | if dbtype != DBType.POSTGRES: 39 | return None 40 | 41 | return cast("psycopg2.extensions.connection", db) 42 | 43 | 44 | def as_sqlite( 45 | db: dbapi.DBAPIConnection, 46 | dbtype: DBType, 47 | ) -> sqlite3.Connection | None: 48 | if dbtype != DBType.SQLITE: 49 | return None 50 | 51 | return cast("sqlite3.Connection", db) 52 | 53 | 54 | def strip_schema(table: str) -> str: 55 | st = table.split(".") 56 | if len(st) == 1: 57 | return table 58 | if len(st) == 2: 59 | return st[1] 60 | raise ValueError("invalid table name: " + table) 61 | 62 | 63 | def autocommit(db: dbapi.DBAPIConnection, dbtype: DBType, enable: bool) -> None: 64 | if (pgdb := as_postgres(db, dbtype)) is not None: 65 | pgdb.rollback() 66 | pgdb.set_session(autocommit=enable) 67 | 68 | if (sql3db := as_sqlite(db, dbtype)) is not None: 69 | sql3db.rollback() 70 | if enable: 71 | sql3db.isolation_level = None 72 | else: 73 | sql3db.isolation_level = "DEFERRED" 74 | 75 | 76 | def server_cursor(db: dbapi.DBAPIConnection, dbtype: DBType) -> dbapi.DBAPICursor: 77 | if (pgdb := as_postgres(db, dbtype)) is not None: 78 | return cast( 79 | "dbapi.DBAPICursor", 80 | pgdb.cursor(name=("ldlite" + secrets.token_hex(4))), 81 | ) 82 | return db.cursor() 83 | 84 | 85 | def sqlid(ident: str) -> str: 86 | sp = ident.split(".") 87 | if len(sp) == 1: 88 | return '"' + ident + '"' 89 | return ".".join(['"' + s + '"' for s in sp]) 90 | 91 | 92 | def cast_to_varchar(ident: str, dbtype: DBType) -> str: 93 | if dbtype == DBType.SQLITE: 94 | return "CAST(" + ident + " as TEXT)" 95 | return ident + "::varchar" 96 | 97 | 98 | def varchar_type(dbtype: DBType) -> str: 99 | if dbtype == DBType.POSTGRES or DBType.SQLITE: 100 | return "text" 101 | return "varchar" 102 | 103 | 104 | def json_type(dbtype: DBType) -> str: 105 | if dbtype == DBType.POSTGRES: 106 | return "jsonb" 107 | if dbtype == DBType.SQLITE: 108 | return "text" 109 | return "varchar" 110 | 111 | 112 | def encode_sql_str(dbtype: DBType, s: str) -> str: # noqa: C901, PLR0912 113 | b = "E'" if dbtype == DBType.POSTGRES else "'" 114 | if dbtype in (DBType.SQLITE, DBType.DUCKDB): 115 | for c in s: 116 | if c == "'": 117 | b += "''" 118 | else: 119 | b += c 120 | if dbtype == DBType.POSTGRES: 121 | for c in s: 122 | if c == "'": 123 | b += "''" 124 | elif c == "\\": 125 | b += "\\\\" 126 | elif c == "\n": 127 | b += "\\n" 128 | elif c == "\r": 129 | b += "\\r" 130 | elif c == "\t": 131 | b += "\\t" 132 | elif c == "\f": 133 | b += "\\f" 134 | elif c == "\b": 135 | b += "\\b" 136 | else: 137 | b += c 138 | b += "'" 139 | return b 140 | 141 | 142 | def encode_sql(dbtype: DBType, data: JsonValue) -> str: 143 | if data is None: 144 | return "NULL" 145 | if isinstance(data, str): 146 | return encode_sql_str(dbtype, data) 147 | if isinstance(data, int): 148 | return str(data) 149 | if isinstance(data, bool): 150 | return "TRUE" if data else "FALSE" 151 | return encode_sql_str(dbtype, str(data)) 152 | -------------------------------------------------------------------------------- /src/ldlite/_xlsx.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | import xlsxwriter 6 | 7 | from ._sqlx import DBType, server_cursor, sqlid 8 | 9 | if TYPE_CHECKING: 10 | from _typeshed import dbapi 11 | 12 | 13 | def to_xlsx( # noqa: C901, PLR0912, PLR0915 14 | db: dbapi.DBAPIConnection, 15 | dbtype: DBType, 16 | table: str, 17 | filename: str, 18 | header: bool, 19 | ) -> None: 20 | # Read attributes 21 | attrs: list[tuple[str, dbapi.DBAPITypeCode]] = [] 22 | width: list[int] = [] 23 | cur = db.cursor() 24 | try: 25 | cur.execute("SELECT * FROM " + sqlid(table) + " LIMIT 1") 26 | if cur.description is not None: 27 | for a in cur.description: 28 | attrs.append((a[0], a[1])) 29 | width.append(len(a[0])) 30 | finally: 31 | cur.close() 32 | cols = ",".join([sqlid(a[0]) for a in attrs]) 33 | query = ( 34 | "SELECT " 35 | + cols 36 | + " FROM " 37 | + sqlid(table) 38 | + " ORDER BY " 39 | + ",".join([str(i + 1) for i in range(len(attrs))]) 40 | ) 41 | # Scan 42 | cur = server_cursor(db, dbtype) 43 | try: 44 | cur.execute(query) 45 | while True: 46 | row = cur.fetchone() 47 | if row is None: 48 | break 49 | for i, data in enumerate(row): 50 | lines = [""] 51 | if data is not None: 52 | lines = str(data).splitlines() 53 | for _, ln in enumerate(lines): 54 | width[i] = max(width[i], len(ln)) 55 | finally: 56 | cur.close() 57 | # Write data 58 | cur = server_cursor(db, dbtype) 59 | try: 60 | cur.execute(query) 61 | fn = filename if "." in filename else filename + ".xlsx" 62 | workbook = xlsxwriter.Workbook(fn, {"constant_memory": True}) 63 | try: 64 | worksheet = workbook.add_worksheet() 65 | for i, w in enumerate(width): 66 | worksheet.set_column(i, i, w + 2) 67 | if header: 68 | worksheet.freeze_panes(1, 0) 69 | for i, attr in enumerate(attrs): 70 | fmt = workbook.add_format() 71 | fmt.set_bold() 72 | fmt.set_align("center") 73 | worksheet.write(0, i, attr[0], fmt) 74 | row_i = 1 if header else 0 75 | datafmt = workbook.add_format() 76 | datafmt.set_align("top") 77 | while True: 78 | row = cur.fetchone() 79 | if row is None: 80 | break 81 | for i, data in enumerate(row): 82 | worksheet.write(row_i, i, data, datafmt) 83 | row_i += 1 84 | finally: 85 | workbook.close() 86 | finally: 87 | cur.close() 88 | -------------------------------------------------------------------------------- /src/ldlite/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/library-data-platform/ldlite/f052dfb7a7a77c390f69a509f96a7b9f96c871f0/src/ldlite/py.typed -------------------------------------------------------------------------------- /srs.md: -------------------------------------------------------------------------------- 1 | Using LDLite with SRS MARC data 2 | =============================== 3 | 4 | This page summarizes experimental use of LDLite to report on MARC data 5 | retrieved from Source Record Storage (SRS). The sequence of steps 6 | below covers querying SRS data and using 7 | [ldpmarc](https://github.com/library-data-platform/ldpmarc) to 8 | transform the data to tabular format for easier querying via SQL. The 9 | suggested process assumes PostgreSQL is being used for the reporting 10 | database. 11 | 12 | The following requires ldpmarc v1.6.0-beta5 or later. 13 | 14 | 15 | Querying and retrieving SRS data 16 | -------------------------------- 17 | 18 | We will assume that LDLite has been initialized and configured: 19 | 20 | ```python 21 | ld = ldlite.LDLite() 22 | # etc. 23 | ``` 24 | 25 | A basic query that will retrieve SRS data is: 26 | 27 | ```python 28 | ld.query(table='folio_source_record.records', path='/source-storage/records', json_depth=2, limit=1000) 29 | ``` 30 | 31 | In this example, the *limit* parameter has been used to reduce the 32 | number of records retrieved. Alternatively, SRS allows filtering by 33 | `snapshotId`, `recordType`, or `state`, for example: 34 | 35 | ```python 36 | ld.query(table='folio_source_record.records', path='/source-storage/records', query={'state': 'OLD'}, json_depth=2) 37 | ``` 38 | 39 | If the total number of SRS records is small, it may be possible to 40 | retrieve all of the records; but this could be prohibitively slow. 41 | 42 | The *json_depth* parameter should be set to 2, which will cause the 43 | transformation to stop when it reaches the MARC JSON record and to 44 | write the JSON object as a whole. 45 | 46 | After this query has completed, `folio_source_record.records__t` 47 | should contain the MARC JSON data and metadata. 48 | 49 | 50 | Adjustments to work with ldpmarc 51 | -------------------------------- 52 | 53 | The ldpmarc tool expects certain tables and columns to be present. We 54 | can create them from `folio_source_record.records__t`: 55 | 56 | ```sql 57 | DROP TABLE IF EXISTS folio_source_record.records_lb; 58 | 59 | CREATE TABLE folio_source_record.records_lb AS 60 | SELECT __id, 61 | id::uuid, 62 | state, 63 | matched_id::uuid, 64 | external_ids_holder__instance_id::uuid AS external_id, 65 | external_ids_holder__instance_hrid AS external_hrid 66 | FROM folio_source_record.records__t; 67 | 68 | CREATE INDEX ON folio_source_record.records_lb (__id); 69 | 70 | CREATE INDEX ON folio_source_record.records_lb (id); 71 | 72 | CREATE INDEX ON folio_source_record.records_lb (state); 73 | 74 | CREATE INDEX ON folio_source_record.records_lb (matched_id); 75 | 76 | CREATE INDEX ON folio_source_record.records_lb (external_id); 77 | 78 | CREATE INDEX ON folio_source_record.records_lb (external_hrid); 79 | 80 | DROP TABLE IF EXISTS folio_source_record.marc_records_lb; 81 | 82 | CREATE TABLE folio_source_record.marc_records_lb AS 83 | SELECT __id, 84 | id::uuid, 85 | parsed_record__content AS content 86 | FROM folio_source_record.records__t; 87 | 88 | CREATE INDEX ON folio_source_record.marc_records_lb (__id); 89 | 90 | CREATE INDEX ON folio_source_record.marc_records_lb (id); 91 | 92 | CREATE INDEX ON folio_source_record.marc_records_lb (content); 93 | ``` 94 | 95 | 96 | Running ldpmarc 97 | --------------- 98 | 99 | The data now should be compatible with ldpmarc. Before continuing, 100 | see the [ldpmarc readme 101 | file](https://github.com/library-data-platform/ldpmarc/blob/main/README.md) 102 | for installation and usage documentation, but note that LDP1 and 103 | Metadb are not required for this process. 104 | 105 | When we run ldpmarc with the `-M` option (below), it will look for 106 | database connection parameters in a configuration file called 107 | `metadb.conf` located within a Metadb data directory. If the data 108 | directory is called, for example, `data/`, then `data/metadb.conf` 109 | should contain settings in the form: 110 | 111 | ```ini 112 | [main] 113 | host = 114 | port = 5432 115 | database = 116 | systemuser = 117 | systemuser_password = 118 | sslmode = 119 | ``` 120 | 121 | Then to run ldpmarc: 122 | 123 | ```bash 124 | ldpmarc -D data -M -f 125 | ``` 126 | 127 | This should create a new table `folio_source_record.marctab` 128 | containing the MARC records in a form such as: 129 | 130 | ``` 131 | srs_id | line | matched_id | instance_hrid | instance_id | field | ind1 | ind2 | ord | sf | content 132 | --------------------------------------+------+--------------------------------------+---------------+--------------------------------------+-------+------+------+-----+----+------------------------------------------ 133 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 1 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 000 | | | 1 | | 00457nca a2200181 c 4500 134 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 2 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 001 | | | 1 | | 354326643 135 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 3 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 003 | | | 1 | | DE-101 136 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 4 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 005 | | | 1 | | 20171202045622.0 137 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 5 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 007 | | | 1 | | q| 138 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 6 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 008 | | | 1 | | 050609s1980 ||||||||r|||||||||||||| 139 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 7 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 016 | 7 | | 1 | 2 | DE-101 140 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 8 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 016 | 7 | | 1 | a | 354326643 141 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 9 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 016 | 7 | | 2 | 2 | DE-101c 142 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 10 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 016 | 7 | | 2 | a | 596503000 143 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 11 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 035 | | | 1 | a | (DE-599)DNB354326643 144 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 12 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 035 | | | 2 | a | (OCoLC)724812418 145 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 13 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 040 | | | 1 | a | 9999 146 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 14 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 040 | | | 1 | b | ger 147 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 15 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 040 | | | 1 | c | DE-101 148 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 16 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 040 | | | 1 | d | 9999 149 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 17 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 100 | 1 | | 1 | a | Mason, Benjamin 150 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 18 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 100 | 1 | | 1 | e | Komponist 151 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 19 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 100 | 1 | | 1 | 4 | cmp 152 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 20 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 245 | 1 | 0 | 1 | a | Bread and water 153 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 21 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 773 | 0 | 8 | 1 | w | (DE-101)354326635 154 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 22 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 999 | f | f | 1 | i | fef9f415-1b35-3e30-89cc-17857a611338 155 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | 23 | 14ea8ed4-672b-11eb-8681-aed9fae510e9 | | fef9f415-1b35-3e30-89cc-17857a611338 | 999 | f | f | 1 | s | 14ea8ed4-672b-11eb-8681-aed9fae510e9 156 | ``` 157 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/library-data-platform/ldlite/f052dfb7a7a77c390f69a509f96a7b9f96c871f0/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def pytest_addoption(parser: pytest.Parser) -> None: 5 | parser.addoption("--pg-host", action="store") 6 | -------------------------------------------------------------------------------- /tests/test___init__.py: -------------------------------------------------------------------------------- 1 | from dataclasses import astuple, dataclass 2 | 3 | import pytest 4 | from pytest_cases import parametrize_with_cases 5 | from requests import exceptions 6 | 7 | 8 | def test_ok() -> None: 9 | from ldlite import LDLite as uut 10 | 11 | ld = uut() 12 | ld.connect_folio( 13 | url="https://folio-etesting-snapshot-kong.ci.folio.org", 14 | tenant="diku", 15 | user="diku_admin", 16 | password="admin", 17 | ) 18 | ld.connect_db() 19 | ld.query(table="g", path="/groups", query="cql.allRecords=1 sortby id") 20 | ld.select(table="g__t") 21 | 22 | 23 | def test_no_connect_folio() -> None: 24 | from ldlite import LDLite as uut 25 | 26 | ld = uut() 27 | ld.connect_db() 28 | with pytest.raises(RuntimeError): 29 | ld.query(table="g", path="/groups", query="cql.allRecords=1 sortby id") 30 | 31 | 32 | def test_no_connect_db() -> None: 33 | from ldlite import LDLite as uut 34 | 35 | ld = uut() 36 | ld.connect_folio( 37 | url="https://folio-etesting-snapshot-kong.ci.folio.org", 38 | tenant="diku", 39 | user="diku_admin", 40 | password="admin", 41 | ) 42 | with pytest.raises(RuntimeError): 43 | ld.query(table="g", path="/groups", query="cql.allRecords=1 sortby id") 44 | 45 | 46 | @dataclass(frozen=True) 47 | class FolioConnectionCase: 48 | expected: type[Exception] 49 | url: str = "https://folio-etesting-snapshot-kong.ci.folio.org" 50 | tenant: str = "diku" 51 | user: str = "diku_admin" 52 | password: str = "admin" 53 | 54 | 55 | class FolioConnectionCases: 56 | def case_url(self) -> FolioConnectionCase: 57 | return FolioConnectionCase( 58 | expected=exceptions.RequestException, 59 | url="https://not.folio.fivecolleges.edu", 60 | ) 61 | 62 | def case_tenant(self) -> FolioConnectionCase: 63 | return FolioConnectionCase( 64 | expected=RuntimeError, 65 | tenant="not a tenant", 66 | ) 67 | 68 | def case_user(self) -> FolioConnectionCase: 69 | return FolioConnectionCase( 70 | expected=RuntimeError, 71 | user="not a user", 72 | ) 73 | 74 | def case_password(self) -> FolioConnectionCase: 75 | return FolioConnectionCase( 76 | expected=RuntimeError, 77 | password="", 78 | ) 79 | 80 | 81 | @parametrize_with_cases("tc", cases=FolioConnectionCases) 82 | def test_bad_folio_connection(tc: FolioConnectionCase) -> None: 83 | from ldlite import LDLite as uut 84 | 85 | ld = uut() 86 | with pytest.raises(tc.expected): 87 | ld.connect_folio(*astuple(tc)[1:]) 88 | -------------------------------------------------------------------------------- /tests/test_cases/base.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from functools import cached_property 3 | from typing import TYPE_CHECKING, Any 4 | from unittest.mock import MagicMock 5 | from uuid import uuid4 6 | 7 | if TYPE_CHECKING: 8 | import ldlite 9 | 10 | 11 | @dataclass(frozen=True) 12 | class TestCase: 13 | values: dict[str, list[dict[str, Any]]] 14 | 15 | @cached_property 16 | def db(self) -> str: 17 | db = "db" + str(uuid4()).split("-")[0] 18 | print(db) # noqa: T201 19 | return db 20 | 21 | def patch_request_get( 22 | self, 23 | ld: "ldlite.LDLite", 24 | _request_get_mock: MagicMock, 25 | ) -> None: 26 | # _check_okapi() hack 27 | ld.login_token = "token" 28 | ld.okapi_url = "url" 29 | # leave tqdm out of it 30 | ld.quiet(enable=True) 31 | 32 | side_effects = [] 33 | for values in self.values.values(): 34 | total_mock = MagicMock() 35 | total_mock.status_code = 200 36 | total_mock.json.return_value = {} 37 | 38 | value_mocks = [] 39 | for v in values: 40 | value_mock = MagicMock() 41 | value_mock.status_code = 200 42 | value_mock.json.return_value = v 43 | value_mocks.append(value_mock) 44 | 45 | end_mock = MagicMock() 46 | end_mock.status_code = 200 47 | end_mock.json.return_value = {"empty": []} 48 | 49 | side_effects.extend([total_mock, *value_mocks, end_mock]) 50 | 51 | _request_get_mock.side_effect = side_effects 52 | -------------------------------------------------------------------------------- /tests/test_cases/drop_tables_cases.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from .base import TestCase 4 | 5 | 6 | @dataclass(frozen=True) 7 | class DropTablesCase(TestCase): 8 | drop: str 9 | expected_tables: list[str] 10 | 11 | 12 | class DropTablesCases: 13 | def case_one_table(self) -> DropTablesCase: 14 | return DropTablesCase( 15 | drop="prefix", 16 | values={"prefix": [{"purchaseOrders": [{"id": "1"}]}]}, 17 | expected_tables=[], 18 | ) 19 | 20 | def case_two_tables(self) -> DropTablesCase: 21 | return DropTablesCase( 22 | drop="prefix", 23 | values={ 24 | "prefix": [ 25 | { 26 | "purchaseOrders": [ 27 | { 28 | "id": "1", 29 | "subObjects": [{"id": "2"}, {"id": "3"}], 30 | }, 31 | ], 32 | }, 33 | ], 34 | }, 35 | expected_tables=[], 36 | ) 37 | 38 | def case_separate_table(self) -> DropTablesCase: 39 | return DropTablesCase( 40 | drop="prefix", 41 | values={ 42 | "prefix": [{"purchaseOrders": [{"id": "1"}]}], 43 | "notdropped": [{"purchaseOrders": [{"id": "1"}]}], 44 | }, 45 | expected_tables=[ 46 | "notdropped", 47 | "notdropped__t", 48 | "notdropped__tcatalog", 49 | ], 50 | ) 51 | -------------------------------------------------------------------------------- /tests/test_cases/query_cases.py: -------------------------------------------------------------------------------- 1 | import json 2 | from dataclasses import dataclass 3 | from typing import Any 4 | 5 | from pytest_cases import parametrize 6 | 7 | from .base import TestCase 8 | 9 | 10 | @dataclass(frozen=True) 11 | class QueryCase(TestCase): 12 | json_depth: int 13 | expected_tables: list[str] 14 | expected_values: dict[str, tuple[list[str], list[tuple[Any, ...]]]] 15 | 16 | 17 | class QueryTestCases: 18 | @parametrize(json_depth=range(1, 2)) 19 | def case_one_table(self, json_depth: int) -> QueryCase: 20 | return QueryCase( 21 | json_depth=json_depth, 22 | values={ 23 | "prefix": [ 24 | { 25 | "purchaseOrders": [ 26 | { 27 | "id": "b096504a-3d54-4664-9bf5-1b872466fd66", 28 | "value": "value", 29 | }, 30 | ], 31 | }, 32 | ], 33 | }, 34 | expected_tables=["prefix", "prefix__t", "prefix__tcatalog"], 35 | expected_values={ 36 | "prefix__t": ( 37 | ["id", "value"], 38 | [("b096504a-3d54-4664-9bf5-1b872466fd66", "value")], 39 | ), 40 | "prefix__tcatalog": (["table_name"], [("prefix__t",)]), 41 | }, 42 | ) 43 | 44 | @parametrize(json_depth=range(2, 3)) 45 | def case_two_tables(self, json_depth: int) -> QueryCase: 46 | return QueryCase( 47 | json_depth=json_depth, 48 | values={ 49 | "prefix": [ 50 | { 51 | "purchaseOrders": [ 52 | { 53 | "id": "b096504a-3d54-4664-9bf5-1b872466fd66", 54 | "value": "value", 55 | "subObjects": [ 56 | { 57 | "id": "2b94c631-fca9-4892-a730-03ee529ffe2a", 58 | "value": "sub-value-1", 59 | }, 60 | { 61 | "id": "f5bda109-a719-4f72-b797-b9c22f45e4e1", 62 | "value": "sub-value-2", 63 | }, 64 | ], 65 | }, 66 | ], 67 | }, 68 | ], 69 | }, 70 | expected_tables=[ 71 | "prefix", 72 | "prefix__t", 73 | "prefix__t__sub_objects", 74 | "prefix__tcatalog", 75 | ], 76 | expected_values={ 77 | "prefix__t": ( 78 | ["id", "value"], 79 | [("b096504a-3d54-4664-9bf5-1b872466fd66", "value")], 80 | ), 81 | "prefix__t__sub_objects": ( 82 | ["id", "sub_objects__id", "sub_objects__value"], 83 | [ 84 | ( 85 | "b096504a-3d54-4664-9bf5-1b872466fd66", 86 | "2b94c631-fca9-4892-a730-03ee529ffe2a", 87 | "sub-value-1", 88 | ), 89 | ( 90 | "b096504a-3d54-4664-9bf5-1b872466fd66", 91 | "f5bda109-a719-4f72-b797-b9c22f45e4e1", 92 | "sub-value-2", 93 | ), 94 | ], 95 | ), 96 | "prefix__tcatalog": ( 97 | ["table_name"], 98 | [("prefix__t",), ("prefix__t__sub_objects",)], 99 | ), 100 | }, 101 | ) 102 | 103 | @parametrize(json_depth=range(1)) 104 | def case_table_no_expansion(self, json_depth: int) -> QueryCase: 105 | return QueryCase( 106 | json_depth=json_depth, 107 | values={ 108 | "prefix": [ 109 | { 110 | "purchaseOrders": [ 111 | { 112 | "id": "b096504a-3d54-4664-9bf5-1b872466fd66", 113 | "value": "value", 114 | "subObjects": [ 115 | { 116 | "id": "2b94c631-fca9-4892-a730-03ee529ffe2a", 117 | "value": "sub-value", 118 | }, 119 | ], 120 | }, 121 | ], 122 | }, 123 | ], 124 | }, 125 | expected_tables=["prefix"], 126 | expected_values={}, 127 | ) 128 | 129 | def case_table_underexpansion(self) -> QueryCase: 130 | return QueryCase( 131 | json_depth=2, 132 | values={ 133 | "prefix": [ 134 | { 135 | "purchaseOrders": [ 136 | { 137 | "id": "b096504a-3d54-4664-9bf5-1b872466fd66", 138 | "subObjects": [ 139 | { 140 | "id": "2b94c631-fca9-4892-a730-03ee529ffe2a", 141 | "value": "sub-value", 142 | "subSubObjects": [ 143 | { 144 | "id": ( 145 | "2b94c631-fca9-4892-a730-03ee529ffe2a" 146 | ), 147 | "value": "sub-sub-value", 148 | }, 149 | ], 150 | }, 151 | ], 152 | }, 153 | ], 154 | }, 155 | ], 156 | }, 157 | expected_tables=[ 158 | "prefix", 159 | "prefix__t", 160 | "prefix__t__sub_objects", 161 | "prefix__tcatalog", 162 | ], 163 | expected_values={ 164 | "prefix__t__sub_objects": ( 165 | ["*"], 166 | [ 167 | ( 168 | 1, 169 | "b096504a-3d54-4664-9bf5-1b872466fd66", 170 | 1, 171 | "2b94c631-fca9-4892-a730-03ee529ffe2a", 172 | "sub-value", 173 | ), 174 | ], 175 | ), 176 | "prefix__tcatalog": ( 177 | ["table_name"], 178 | [("prefix__t",), ("prefix__t__sub_objects",)], 179 | ), 180 | }, 181 | ) 182 | 183 | @parametrize(json_depth=range(3, 4)) 184 | def case_three_tables(self, json_depth: int) -> QueryCase: 185 | return QueryCase( 186 | json_depth=json_depth, 187 | values={ 188 | "prefix": [ 189 | { 190 | "purchaseOrders": [ 191 | { 192 | "id": "b096504a-3d54-4664-9bf5-1b872466fd66", 193 | "value": "value", 194 | "subObjects": [ 195 | { 196 | "id": "2b94c631-fca9-4892-a730-03ee529ffe2a", 197 | "value": "sub-value", 198 | "subSubObjects": [ 199 | { 200 | "id": ( 201 | "2b94c631-fca9-4892-a730-03ee529ffe2a" 202 | ), 203 | "value": "sub-sub-value", 204 | }, 205 | ], 206 | }, 207 | ], 208 | }, 209 | ], 210 | }, 211 | ], 212 | }, 213 | expected_tables=[ 214 | "prefix", 215 | "prefix__t", 216 | "prefix__t__sub_objects", 217 | "prefix__t__sub_objects__sub_sub_objects", 218 | "prefix__tcatalog", 219 | ], 220 | expected_values={ 221 | "prefix__t__sub_objects__sub_sub_objects": ( 222 | [ 223 | "id", 224 | "sub_objects__id", 225 | "sub_objects__sub_sub_objects__id", 226 | "sub_objects__sub_sub_objects__value", 227 | ], 228 | [ 229 | ( 230 | "b096504a-3d54-4664-9bf5-1b872466fd66", 231 | "2b94c631-fca9-4892-a730-03ee529ffe2a", 232 | "2b94c631-fca9-4892-a730-03ee529ffe2a", 233 | "sub-sub-value", 234 | ), 235 | ], 236 | ), 237 | "prefix__tcatalog": ( 238 | ["table_name"], 239 | [ 240 | ("prefix__t",), 241 | ("prefix__t__sub_objects",), 242 | ("prefix__t__sub_objects__sub_sub_objects",), 243 | ], 244 | ), 245 | }, 246 | ) 247 | 248 | def case_nested_object(self) -> QueryCase: 249 | return QueryCase( 250 | json_depth=2, 251 | values={ 252 | "prefix": [ 253 | { 254 | "purchaseOrders": [ 255 | { 256 | "id": "b096504a-3d54-4664-9bf5-1b872466fd66", 257 | "value": "value", 258 | "subObject": { 259 | "id": "2b94c631-fca9-4892-a730-03ee529ffe2a", 260 | "value": "sub-value", 261 | }, 262 | }, 263 | ], 264 | }, 265 | ], 266 | }, 267 | expected_tables=["prefix", "prefix__t", "prefix__tcatalog"], 268 | expected_values={ 269 | "prefix__t": ( 270 | ["id", "value", "sub_object__id", "sub_object__value"], 271 | [ 272 | ( 273 | "b096504a-3d54-4664-9bf5-1b872466fd66", 274 | "value", 275 | "2b94c631-fca9-4892-a730-03ee529ffe2a", 276 | "sub-value", 277 | ), 278 | ], 279 | ), 280 | "prefix__tcatalog": ( 281 | ["table_name"], 282 | [("prefix__t",)], 283 | ), 284 | }, 285 | ) 286 | 287 | def case_doubly_nested_object(self) -> QueryCase: 288 | return QueryCase( 289 | json_depth=3, 290 | values={ 291 | "prefix": [ 292 | { 293 | "purchaseOrders": [ 294 | { 295 | "id": "b096504a-3d54-4664-9bf5-1b872466fd66", 296 | "value": "value", 297 | "subObject": { 298 | "id": "2b94c631-fca9-4892-a730-03ee529ffe2a", 299 | "value": "sub-value", 300 | "subSubObject": { 301 | "id": "2b94c631-fca9-4892-a730-03ee529ffe2a", 302 | "value": "sub-sub-value", 303 | }, 304 | }, 305 | }, 306 | ], 307 | }, 308 | ], 309 | }, 310 | expected_tables=["prefix", "prefix__t", "prefix__tcatalog"], 311 | expected_values={ 312 | "prefix__t": ( 313 | [ 314 | "id", 315 | "value", 316 | "sub_object__id", 317 | "sub_object__sub_sub_object__id", 318 | "sub_object__sub_sub_object__value", 319 | ], 320 | [ 321 | ( 322 | "b096504a-3d54-4664-9bf5-1b872466fd66", 323 | "value", 324 | "2b94c631-fca9-4892-a730-03ee529ffe2a", 325 | "2b94c631-fca9-4892-a730-03ee529ffe2a", 326 | "sub-sub-value", 327 | ), 328 | ], 329 | ), 330 | "prefix__tcatalog": ( 331 | ["table_name"], 332 | [("prefix__t",)], 333 | ), 334 | }, 335 | ) 336 | 337 | def case_nested_object_underexpansion(self) -> QueryCase: 338 | return QueryCase( 339 | json_depth=1, 340 | values={ 341 | "prefix": [ 342 | { 343 | "purchaseOrders": [ 344 | { 345 | "id": "b096504a-3d54-4664-9bf5-1b872466fd66", 346 | "value": "value", 347 | "subObject": { 348 | "id": "2b94c631-fca9-4892-a730-03ee529ffe2a", 349 | "value": "sub-value", 350 | }, 351 | }, 352 | ], 353 | }, 354 | ], 355 | }, 356 | expected_tables=["prefix", "prefix__t", "prefix__tcatalog"], 357 | expected_values={ 358 | "prefix__t": ( 359 | ["id", "value", "sub_object"], 360 | [ 361 | ( 362 | "b096504a-3d54-4664-9bf5-1b872466fd66", 363 | "value", 364 | json.dumps( 365 | { 366 | "id": "2b94c631-fca9-4892-a730-03ee529ffe2a", 367 | "value": "sub-value", 368 | }, 369 | indent=4, 370 | ), 371 | ), 372 | ], 373 | ), 374 | "prefix__tcatalog": ( 375 | ["table_name"], 376 | [("prefix__t",)], 377 | ), 378 | }, 379 | ) 380 | 381 | def case_id_generation(self) -> QueryCase: 382 | return QueryCase( 383 | json_depth=4, 384 | values={ 385 | "prefix": [ 386 | { 387 | "purchaseOrders": [ 388 | { 389 | "id": "b096504a-3d54-4664-9bf5-1b872466fd66", 390 | "subObjects": [ 391 | { 392 | "id": "2b94c631-fca9-4892-a730-03ee529ffe2a", 393 | "subSubObjects": [ 394 | { 395 | "id": ( 396 | "2b94c631-fca9-4892-a730-03ee529ffe2a" 397 | ), 398 | }, 399 | { 400 | "id": ( 401 | "8516a913-8bf7-55a4-ab71-417aba9171c9" 402 | ), 403 | }, 404 | ], 405 | }, 406 | { 407 | "id": "b5d8cdc4-9441-487c-90cf-0c7ec97728eb", 408 | "subSubObjects": [ 409 | { 410 | "id": ( 411 | "13a24cc8-a15c-4158-abbd-4abf25c8815a" 412 | ), 413 | }, 414 | { 415 | "id": ( 416 | "37344879-09ce-4cd8-976f-bf1a57c0cfa6" 417 | ), 418 | }, 419 | ], 420 | }, 421 | ], 422 | }, 423 | ], 424 | }, 425 | ], 426 | }, 427 | expected_tables=[ 428 | "prefix", 429 | "prefix__t", 430 | "prefix__t__sub_objects", 431 | "prefix__t__sub_objects__sub_sub_objects", 432 | "prefix__tcatalog", 433 | ], 434 | expected_values={ 435 | "prefix__t__sub_objects": ( 436 | ["__id", "id", "sub_objects__o", "sub_objects__id"], 437 | [ 438 | ( 439 | 1, 440 | "b096504a-3d54-4664-9bf5-1b872466fd66", 441 | 1, 442 | "2b94c631-fca9-4892-a730-03ee529ffe2a", 443 | ), 444 | ( 445 | 2, 446 | "b096504a-3d54-4664-9bf5-1b872466fd66", 447 | 2, 448 | "b5d8cdc4-9441-487c-90cf-0c7ec97728eb", 449 | ), 450 | ], 451 | ), 452 | "prefix__t__sub_objects__sub_sub_objects": ( 453 | ["__id", "sub_objects__o", "sub_objects__sub_sub_objects__o"], 454 | [ 455 | (1, 1, 1), 456 | (2, 1, 2), 457 | (3, 2, 1), 458 | (4, 2, 2), 459 | ], 460 | ), 461 | }, 462 | ) 463 | -------------------------------------------------------------------------------- /tests/test_cases/to_csv_cases.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from pathlib import Path 3 | 4 | from .base import TestCase 5 | 6 | _SAMPLE_PATH = Path() / "tests" / "test_cases" / "to_csv_samples" 7 | 8 | 9 | @dataclass(frozen=True) 10 | class ToCsvCase(TestCase): 11 | expected_csvs: list[tuple[str, Path]] 12 | 13 | 14 | class ToCsvCases: 15 | def case_basic(self) -> ToCsvCase: 16 | return ToCsvCase( 17 | values={"prefix": [{"purchaseOrders": [{"val": "value"}]}]}, 18 | expected_csvs=[("prefix__t", _SAMPLE_PATH / "basic.csv")], 19 | ) 20 | 21 | def case_datatypes(self) -> ToCsvCase: 22 | return ToCsvCase( 23 | values={ 24 | "prefix": [ 25 | { 26 | "purchaseOrders": [ 27 | { 28 | "string": "string", 29 | "integer": 1, 30 | "numeric": 1.1, 31 | "boolean": True, 32 | "uuid": "6a31a12a-9570-405c-af20-6abf2992859c", 33 | }, 34 | ], 35 | }, 36 | ], 37 | }, 38 | expected_csvs=[("prefix__t", _SAMPLE_PATH / "datatypes.csv")], 39 | ) 40 | 41 | def case_escaped_chars(self) -> ToCsvCase: 42 | return ToCsvCase( 43 | values={ 44 | "prefix": [ 45 | { 46 | "purchaseOrders": [ 47 | { 48 | "comma": "Double, double toil and trouble", 49 | "doubleQuote": 'Cry "Havoc!" a horse', 50 | "newLine": """To be 51 | or not 52 | to be""", 53 | "singleQuote": "Cry 'Havoc!' a horse", 54 | }, 55 | { 56 | "comma": "Z", 57 | "doubleQuote": "Z", 58 | "newLine": "Z", 59 | "singleQuote": "Z", 60 | }, 61 | ], 62 | }, 63 | ], 64 | }, 65 | expected_csvs=[("prefix__t", _SAMPLE_PATH / "escaped_chars.csv")], 66 | ) 67 | 68 | def case_sorting(self) -> ToCsvCase: 69 | return ToCsvCase( 70 | values={ 71 | "prefix": [ 72 | { 73 | "purchaseOrders": [ 74 | { 75 | "C": "YY", 76 | "B": "XX", 77 | "A": "ZZ", 78 | }, 79 | { 80 | "C": "Y", 81 | "B": "XX", 82 | "A": "ZZ", 83 | }, 84 | { 85 | "C": "Y", 86 | "B": "X", 87 | "A": "Z", 88 | }, 89 | ], 90 | }, 91 | ], 92 | }, 93 | expected_csvs=[("prefix__t", _SAMPLE_PATH / "sorting.csv")], 94 | ) 95 | -------------------------------------------------------------------------------- /tests/test_cases/to_csv_samples/basic.csv: -------------------------------------------------------------------------------- 1 | "__id","val" 2 | 1,"value" 3 | -------------------------------------------------------------------------------- /tests/test_cases/to_csv_samples/datatypes.csv: -------------------------------------------------------------------------------- 1 | "__id","boolean","integer","numeric","string","uuid" 2 | 1,True,1,1.1,"string","6a31a12a-9570-405c-af20-6abf2992859c" 3 | -------------------------------------------------------------------------------- /tests/test_cases/to_csv_samples/escaped_chars.csv: -------------------------------------------------------------------------------- 1 | "__id","comma","double_quote","new_line","single_quote" 2 | 1,"Double, double toil and trouble","Cry ""Havoc!"" a horse","To be 3 | or not 4 | to be","Cry 'Havoc!' a horse" 5 | 2,"Z","Z","Z","Z" 6 | -------------------------------------------------------------------------------- /tests/test_cases/to_csv_samples/sorting.csv: -------------------------------------------------------------------------------- 1 | "__id","a","b","c" 2 | 3,"Z","X","Y" 3 | 2,"ZZ","XX","Y" 4 | 1,"ZZ","XX","YY" 5 | -------------------------------------------------------------------------------- /tests/test_duckdb.py: -------------------------------------------------------------------------------- 1 | from difflib import unified_diff 2 | from pathlib import Path 3 | from unittest import mock 4 | from unittest.mock import MagicMock 5 | 6 | import duckdb 7 | import pytest 8 | from pytest_cases import parametrize_with_cases 9 | 10 | from .test_cases import drop_tables_cases as dtc 11 | from .test_cases import query_cases as qc 12 | from .test_cases import to_csv_cases as csvc 13 | 14 | 15 | @mock.patch("ldlite.request_get") 16 | @parametrize_with_cases("tc", cases=dtc.DropTablesCases) 17 | def test_drop_tables(request_get_mock: MagicMock, tc: dtc.DropTablesCase) -> None: 18 | from ldlite import LDLite as uut 19 | 20 | ld = uut() 21 | tc.patch_request_get(ld, request_get_mock) 22 | dsn = f":memory:{tc.db}" 23 | ld.connect_db(dsn) 24 | 25 | for prefix in tc.values: 26 | ld.query(table=prefix, path="/patched") 27 | ld.drop_tables(tc.drop) 28 | 29 | with duckdb.connect(dsn) as res: 30 | res.execute("SHOW TABLES;") 31 | assert sorted([r[0] for r in res.fetchall()]) == sorted(tc.expected_tables) 32 | 33 | 34 | @mock.patch("ldlite.request_get") 35 | @parametrize_with_cases("tc", cases=qc.QueryTestCases) 36 | def test_query(request_get_mock: MagicMock, tc: qc.QueryCase) -> None: 37 | from ldlite import LDLite as uut 38 | 39 | ld = uut() 40 | tc.patch_request_get(ld, request_get_mock) 41 | dsn = f":memory:{tc.db}" 42 | ld.connect_db(dsn) 43 | 44 | for prefix in tc.values: 45 | ld.query(table=prefix, path="/patched", json_depth=tc.json_depth) 46 | 47 | with duckdb.connect(dsn) as res: 48 | res.execute("SHOW TABLES;") 49 | assert sorted([r[0] for r in res.fetchall()]) == sorted(tc.expected_tables) 50 | 51 | for table, (cols, values) in tc.expected_values.items(): 52 | with duckdb.connect(dsn) as res: 53 | res.execute(f"SELECT {','.join(cols)} FROM {table};") 54 | for v in values: 55 | assert res.fetchone() == v 56 | 57 | assert res.fetchone() is None 58 | 59 | 60 | @mock.patch("ldlite.request_get") 61 | @parametrize_with_cases("tc", cases=csvc.ToCsvCases) 62 | def test_to_csv(request_get_mock: MagicMock, tc: csvc.ToCsvCase, tmpdir: str) -> None: 63 | from ldlite import LDLite as uut 64 | 65 | ld = uut() 66 | tc.patch_request_get(ld, request_get_mock) 67 | ld.connect_db(f":memory:{tc.db}") 68 | 69 | for prefix in tc.values: 70 | ld.query(table=prefix, path="/patched") 71 | 72 | for table, expected in tc.expected_csvs: 73 | actual = (Path(tmpdir) / table).with_suffix(".csv") 74 | 75 | ld.export_csv(str(actual), table) 76 | 77 | with expected.open("r") as f: 78 | expected_lines = f.readlines() 79 | with actual.open("r") as f: 80 | actual_lines = f.readlines() 81 | 82 | diff = list(unified_diff(expected_lines, actual_lines)) 83 | if len(diff) > 0: 84 | pytest.fail("".join(diff)) 85 | -------------------------------------------------------------------------------- /tests/test_postgres.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import contextlib 4 | from difflib import unified_diff 5 | from pathlib import Path 6 | from typing import TYPE_CHECKING, Callable 7 | from unittest import mock 8 | from unittest.mock import MagicMock 9 | 10 | import psycopg2 11 | import pytest 12 | from pytest_cases import parametrize_with_cases 13 | 14 | from .test_cases import drop_tables_cases as dtc 15 | from .test_cases import query_cases as qc 16 | from .test_cases import to_csv_cases as csvc 17 | 18 | if TYPE_CHECKING: 19 | from unittest.mock import MagicMock 20 | 21 | 22 | @pytest.fixture(scope="session") 23 | def pg_dsn(pytestconfig: pytest.Config) -> None | Callable[[str], str]: 24 | host = pytestconfig.getoption("pg_host") 25 | if host is None: 26 | return None 27 | 28 | def setup(db: str) -> str: 29 | base_dsn = f"host={host} user=ldlite password=ldlite" 30 | with contextlib.closing(psycopg2.connect(base_dsn)) as base_conn: 31 | base_conn.autocommit = True 32 | with base_conn.cursor() as curr: 33 | curr.execute(f"CREATE DATABASE {db};") 34 | 35 | return base_dsn + f" dbname={db}" 36 | 37 | return setup 38 | 39 | 40 | @mock.patch("ldlite.request_get") 41 | @parametrize_with_cases("tc", cases=dtc.DropTablesCases) 42 | def test_drop_tables( 43 | request_get_mock: MagicMock, 44 | pg_dsn: None | Callable[[str], str], 45 | tc: dtc.DropTablesCase, 46 | ) -> None: 47 | if pg_dsn is None: 48 | pytest.skip("Specify the pg host using --pg-host to run") 49 | 50 | from ldlite import LDLite as uut 51 | 52 | ld = uut() 53 | tc.patch_request_get(ld, request_get_mock) 54 | dsn = pg_dsn(tc.db) 55 | ld.connect_db_postgresql(dsn) 56 | 57 | for prefix in tc.values: 58 | ld.query(table=prefix, path="/patched") 59 | ld.drop_tables(tc.drop) 60 | 61 | with psycopg2.connect(dsn) as conn, conn.cursor() as res: 62 | res.execute( 63 | """ 64 | SELECT table_name 65 | FROM information_schema.tables 66 | WHERE table_schema='public' 67 | """, 68 | ) 69 | assert sorted([r[0] for r in res.fetchall()]) == sorted(tc.expected_tables) 70 | 71 | 72 | @mock.patch("ldlite.request_get") 73 | @parametrize_with_cases("tc", cases=qc.QueryTestCases) 74 | def test_query( 75 | request_get_mock: MagicMock, 76 | pg_dsn: None | Callable[[str], str], 77 | tc: qc.QueryCase, 78 | ) -> None: 79 | if pg_dsn is None: 80 | pytest.skip("Specify the pg host using --pg-host to run") 81 | 82 | from ldlite import LDLite as uut 83 | 84 | ld = uut() 85 | tc.patch_request_get(ld, request_get_mock) 86 | dsn = pg_dsn(tc.db) 87 | ld.connect_db_postgresql(dsn) 88 | 89 | for prefix in tc.values: 90 | ld.query(table=prefix, path="/patched", json_depth=tc.json_depth) 91 | 92 | with psycopg2.connect(dsn) as conn: 93 | with conn.cursor() as res: 94 | res.execute( 95 | """ 96 | SELECT table_name 97 | FROM information_schema.tables 98 | WHERE table_schema='public' 99 | """, 100 | ) 101 | assert sorted([r[0] for r in res.fetchall()]) == sorted(tc.expected_tables) 102 | 103 | for table, (cols, values) in tc.expected_values.items(): 104 | with conn.cursor() as res: 105 | res.execute(f"SELECT {','.join(cols)} FROM {table};") 106 | for v in values: 107 | assert res.fetchone() == v 108 | 109 | assert res.fetchone() is None 110 | 111 | 112 | @mock.patch("ldlite.request_get") 113 | @parametrize_with_cases("tc", cases=csvc.ToCsvCases) 114 | def test_to_csv( 115 | request_get_mock: MagicMock, 116 | pg_dsn: None | Callable[[str], str], 117 | tc: csvc.ToCsvCase, 118 | tmpdir: str, 119 | ) -> None: 120 | if pg_dsn is None: 121 | pytest.skip("Specify the pg host using --pg-host to run") 122 | 123 | from ldlite import LDLite as uut 124 | 125 | ld = uut() 126 | tc.patch_request_get(ld, request_get_mock) 127 | ld.connect_db_postgresql(dsn=pg_dsn(tc.db)) 128 | 129 | for prefix in tc.values: 130 | ld.query(table=prefix, path="/patched") 131 | 132 | for table, expected in tc.expected_csvs: 133 | actual = (Path(tmpdir) / table).with_suffix(".csv") 134 | 135 | ld.export_csv(str(actual), table) 136 | 137 | with expected.open("r") as f: 138 | expected_lines = f.readlines() 139 | with actual.open("r") as f: 140 | actual_lines = f.readlines() 141 | 142 | diff = list(unified_diff(expected_lines, actual_lines)) 143 | if len(diff) > 0: 144 | pytest.fail("".join(diff)) 145 | -------------------------------------------------------------------------------- /tests/test_sqlite.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import sqlite3 3 | from difflib import unified_diff 4 | from pathlib import Path 5 | from unittest import mock 6 | from unittest.mock import MagicMock 7 | 8 | import pytest 9 | from pytest_cases import parametrize_with_cases 10 | 11 | from .test_cases import drop_tables_cases as dtc 12 | from .test_cases import query_cases as qc 13 | from .test_cases import to_csv_cases as csvc 14 | 15 | 16 | @mock.patch("ldlite.request_get") 17 | @parametrize_with_cases("tc", cases=dtc.DropTablesCases) 18 | def test_drop_tables(request_get_mock: MagicMock, tc: dtc.DropTablesCase) -> None: 19 | from ldlite import LDLite as uut 20 | 21 | ld = uut() 22 | tc.patch_request_get(ld, request_get_mock) 23 | dsn = f"file:{tc.db}?mode=memory&cache=shared" 24 | ld.experimental_connect_db_sqlite(dsn) 25 | 26 | for prefix in tc.values: 27 | ld.query(table=prefix, path="/patched") 28 | ld.drop_tables(tc.drop) 29 | 30 | with sqlite3.connect(dsn) as conn, contextlib.closing(conn.cursor()) as res: 31 | res.execute("SELECT name FROM sqlite_master WHERE type='table';") 32 | assert sorted([r[0] for r in res.fetchall()]) == sorted(tc.expected_tables) 33 | 34 | 35 | @mock.patch("ldlite.request_get") 36 | @parametrize_with_cases("tc", cases=qc.QueryTestCases) 37 | def test_query(request_get_mock: MagicMock, tc: qc.QueryCase) -> None: 38 | from ldlite import LDLite as uut 39 | 40 | ld = uut() 41 | tc.patch_request_get(ld, request_get_mock) 42 | dsn = f"file:{tc.db}?mode=memory&cache=shared" 43 | ld.experimental_connect_db_sqlite(dsn) 44 | 45 | for prefix in tc.values: 46 | ld.query(table=prefix, path="/patched", json_depth=tc.json_depth) 47 | 48 | with sqlite3.connect(dsn) as conn: 49 | with contextlib.closing(conn.cursor()) as res: 50 | res.execute("SELECT name FROM sqlite_master WHERE type='table';") 51 | assert sorted([r[0] for r in res.fetchall()]) == sorted(tc.expected_tables) 52 | 53 | for table, (cols, values) in tc.expected_values.items(): 54 | with contextlib.closing(conn.cursor()) as res: 55 | res.execute(f"SELECT {','.join(cols)} FROM {table};") 56 | for v in values: 57 | assert res.fetchone() == v 58 | 59 | assert res.fetchone() is None 60 | 61 | 62 | @mock.patch("ldlite.request_get") 63 | @parametrize_with_cases("tc", cases=csvc.ToCsvCases) 64 | def test_to_csv(request_get_mock: MagicMock, tc: csvc.ToCsvCase, tmpdir: str) -> None: 65 | from ldlite import LDLite as uut 66 | 67 | ld = uut() 68 | tc.patch_request_get(ld, request_get_mock) 69 | ld.experimental_connect_db_sqlite(f"file:{tc.db}?mode=memory&cache=shared") 70 | 71 | for prefix in tc.values: 72 | ld.query(table=prefix, path="/patched") 73 | 74 | for table, expected in tc.expected_csvs: 75 | actual = (Path(tmpdir) / table).with_suffix(".csv") 76 | 77 | ld.export_csv(str(actual), table) 78 | 79 | with expected.open("r") as f: 80 | expected_lines = f.readlines() 81 | with actual.open("r") as f: 82 | actual_lines = f.readlines() 83 | 84 | diff = list(unified_diff(expected_lines, actual_lines)) 85 | if len(diff) > 0: 86 | pytest.fail("".join(diff)) 87 | --------------------------------------------------------------------------------