├── .github
└── workflows
│ ├── publish.yml
│ └── test.yml
├── .gitignore
├── LICENSE
├── README.md
├── datasette_upload_csvs
├── __init__.py
└── templates
│ ├── upload_csv.html
│ └── upload_csv_done.html
├── pytest.ini
├── setup.py
└── tests
├── conftest.py
└── test_datasette_upload_csvs.py
/.github/workflows/publish.yml:
--------------------------------------------------------------------------------
1 | name: Publish Python Package
2 |
3 | on:
4 | release:
5 | types: [created]
6 |
7 | jobs:
8 | test:
9 | runs-on: ubuntu-latest
10 | strategy:
11 | matrix:
12 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
13 | steps:
14 | - uses: actions/checkout@v4
15 | - name: Set up Python ${{ matrix.python-version }}
16 | uses: actions/setup-python@v5
17 | with:
18 | python-version: ${{ matrix.python-version }}
19 | cache: pip
20 | cache-dependency-path: '**/setup.py'
21 | - name: Install dependencies
22 | run: |
23 | pip install -e '.[test]'
24 | - name: Run tests
25 | run: |
26 | pytest
27 | deploy:
28 | runs-on: ubuntu-latest
29 | environment: release
30 | permissions:
31 | id-token: write
32 | needs: [test]
33 | steps:
34 | - uses: actions/checkout@v4
35 | - name: Set up Python
36 | uses: actions/setup-python@v5
37 | with:
38 | python-version: '3.12'
39 | cache: pip
40 | cache-dependency-path: '**/setup.py'
41 | - name: Install dependencies
42 | run: |
43 | pip install setuptools wheel build
44 | - name: Build
45 | run: |
46 | python -m build
47 | - name: Publish
48 | uses: pypa/gh-action-pypi-publish@release/v1
49 |
--------------------------------------------------------------------------------
/.github/workflows/test.yml:
--------------------------------------------------------------------------------
1 | name: Test
2 |
3 | on: [push]
4 |
5 | jobs:
6 | test:
7 | runs-on: ubuntu-latest
8 | strategy:
9 | fail-fast: false
10 | matrix:
11 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
12 | datasette-version: ["<1.0", ">=1.0a13"]
13 | steps:
14 | - uses: actions/checkout@v4
15 | - name: Set up Python ${{ matrix.python-version }}
16 | uses: actions/setup-python@v5
17 | with:
18 | python-version: ${{ matrix.python-version }}
19 | cache: pip
20 | cache-dependency-path: setup.py
21 | - name: Install dependencies
22 | run: |
23 | pip install -e '.[test]'
24 | pip install "datasette${{ matrix.datasette-version }}"
25 | - name: Run tests
26 | timeout-minutes: 1
27 | run: |
28 | pytest
29 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .venv
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 | venv
6 | venv-1
7 | .eggs
8 | .pytest_cache
9 | *.egg-info
10 | .DS_Store
11 | .vscode
12 | *.db
13 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # datasette-upload-csvs
2 |
3 | [](https://pypi.org/project/datasette-upload-csvs/)
4 | [](https://github.com/simonw/datasette-upload-csvs/releases)
5 | [](https://github.com/simonw/datasette-upload-csvs/actions?query=workflow%3ATest)
6 | [](https://github.com/simonw/datasette-upload-csvs/blob/main/LICENSE)
7 |
8 | Datasette plugin for uploading CSV files and converting them to database tables
9 |
10 | ## Installation
11 |
12 | datasette install datasette-upload-csvs
13 |
14 | ## Usage
15 |
16 | The plugin adds an interface at `/-/upload-csvs` for uploading a CSV file and using it to create a new database table.
17 |
18 | By default only [the root actor](https://datasette.readthedocs.io/en/stable/authentication.html#using-the-root-actor) can access the page - so you'll need to run Datasette with the `--root` option and click on the link shown in the terminal to sign in and access the page.
19 |
20 | The `upload-csvs` permission governs access. You can use permission plugins such as [datasette-permissions-sql](https://github.com/simonw/datasette-permissions-sql) to grant additional access to the write interface.
21 |
--------------------------------------------------------------------------------
/datasette_upload_csvs/__init__.py:
--------------------------------------------------------------------------------
1 | import asyncio
2 | from datasette import hookimpl
3 | from datasette.utils.asgi import Response, Forbidden
4 | from charset_normalizer import detect
5 | from starlette.requests import Request
6 | from urllib.parse import quote_plus
7 | import csv as csv_std
8 | import codecs
9 | import datetime
10 | import io
11 | import os
12 | import sqlite_utils
13 | from sqlite_utils.utils import TypeTracker
14 | import uuid
15 |
16 |
17 | @hookimpl
18 | def permission_allowed(actor, action):
19 | if action == "upload-csvs" and actor and actor.get("id") == "root":
20 | return True
21 |
22 |
23 | @hookimpl
24 | def register_routes():
25 | return [
26 | (r"^/-/upload-csvs$", upload_csvs),
27 | (r"^/-/upload-csv$", lambda: Response.redirect("/-/upload-csvs")),
28 | ]
29 |
30 |
31 | @hookimpl
32 | def menu_links(datasette, actor):
33 | async def inner():
34 | if await datasette.permission_allowed(
35 | actor, "upload-csvs", default=False
36 | ) and any(
37 | db.is_mutable and db.name not in ("_memory", "_internal")
38 | for db in datasette.databases.values()
39 | ):
40 | return [
41 | {"href": datasette.urls.path("/-/upload-csvs"), "label": "Upload CSVs"},
42 | ]
43 |
44 | return inner
45 |
46 |
47 | @hookimpl
48 | def database_actions(datasette, actor, database):
49 | async def inner():
50 | db = datasette.get_database(database)
51 | if (
52 | await datasette.permission_allowed(actor, "upload-csvs", default=False)
53 | and db.is_mutable
54 | and db.name not in ("_memory", "_internal")
55 | ):
56 | return [
57 | {
58 | "href": datasette.urls.path(
59 | "/-/upload-csvs?database={}".format(quote_plus(db.name))
60 | ),
61 | "label": "Upload CSV",
62 | "description": "Create a new table by uploading a CSV file",
63 | }
64 | ]
65 |
66 | return inner
67 |
68 |
69 | async def upload_csvs(scope, receive, datasette, request):
70 | if not await datasette.permission_allowed(
71 | request.actor, "upload-csvs", default=False
72 | ):
73 | raise Forbidden("Permission denied for upload-csvs")
74 |
75 | num_bytes_to_detect_with = 2048 * 1024
76 | # ?_num_bytes= can over-ride this, used by the tests
77 | if request.args.get("_num_bytes_to_detect_with"):
78 | num_bytes_to_detect_with = int(request.args["_num_bytes_to_detect_with"])
79 |
80 | # For the moment just use the first database that's not immutable
81 | dbs = [
82 | db
83 | for db in datasette.databases.values()
84 | if db.is_mutable and db.name not in ("_internal", "_memory")
85 | ]
86 | if not dbs:
87 | raise Forbidden("No mutable databases available")
88 |
89 | default_db = dbs[0]
90 |
91 | # We need the ds_request to pass to render_template for CSRF tokens
92 | ds_request = request
93 |
94 | # We use the Starlette request object to handle file uploads
95 | starlette_request = Request(scope, receive)
96 | if starlette_request.method != "POST":
97 | selected_db = ds_request.args.get("database")
98 | databases = []
99 | # If there are multiple databases let them choose
100 | if len(dbs) > 1:
101 | databases = [
102 | {"name": db.name, "selected": db.name == selected_db} for db in dbs
103 | ]
104 | return Response.html(
105 | await datasette.render_template(
106 | "upload_csv.html",
107 | {"databases": databases, "selected_name": selected_db},
108 | request=ds_request,
109 | )
110 | )
111 |
112 | formdata = await starlette_request.form()
113 | database_name = formdata.get("database") or default_db.name
114 | db = datasette.get_database(database_name)
115 | csv = formdata["csv"]
116 | # csv.file is a SpooledTemporaryFile. csv.filename is the filename
117 | table_name = formdata.get("table")
118 | if not table_name:
119 | table_name = csv.filename
120 | if table_name.endswith(".csv"):
121 | table_name = table_name[:-4]
122 |
123 | # If the table already exists, add a suffix
124 | suffix = 2
125 | base_table_name = table_name
126 | while await db.table_exists(table_name):
127 | table_name = "{}_{}".format(base_table_name, suffix)
128 | suffix += 1
129 |
130 | total_size = get_temporary_file_size(csv.file)
131 | task_id = str(uuid.uuid4())
132 |
133 | # Use the first 2MB to detect the character encoding
134 | first_bytes = csv.file.read(num_bytes_to_detect_with)
135 | csv.file.seek(0)
136 | encoding = detect(first_bytes)["encoding"]
137 |
138 | # latin-1 is a superset of ascii, and less likely to hit errors
139 | # https://github.com/simonw/datasette-upload-csvs/issues/25
140 | if encoding == "ascii":
141 | encoding = "latin-1"
142 |
143 | def insert_initial_record(conn):
144 | database = sqlite_utils.Database(conn)
145 | with conn:
146 | database["_csv_progress_"].insert(
147 | {
148 | "id": task_id,
149 | "table_name": table_name,
150 | "bytes_todo": total_size,
151 | "bytes_done": 0,
152 | "rows_done": 0,
153 | "started": str(datetime.datetime.utcnow()),
154 | "completed": None,
155 | "error": None,
156 | },
157 | pk="id",
158 | alter=True,
159 | )
160 |
161 | await db.execute_write_fn(insert_initial_record)
162 |
163 | def make_insert_batch(batch):
164 | def inner(conn):
165 | db = sqlite_utils.Database(conn)
166 | with conn:
167 | db[table_name].insert_all(batch, alter=True)
168 |
169 | return inner
170 |
171 | # We run a parser in a separate async task, writing and yielding every 100 rows
172 | async def parse_csv():
173 | i = 0
174 | tracker = TypeTracker()
175 | try:
176 | reader = csv_std.reader(codecs.iterdecode(csv.file, encoding))
177 | headers = next(reader)
178 |
179 | docs = tracker.wrap(dict(zip(headers, row)) for row in reader)
180 |
181 | batch = []
182 | for doc in docs:
183 | batch.append(doc)
184 | i += 1
185 | if i % 10 == 0:
186 | await db.execute_write(
187 | "update _csv_progress_ set rows_done = ?, bytes_done = ? where id = ?",
188 | (i, csv.file.tell(), task_id),
189 | )
190 | if i % 100 == 0:
191 | await db.execute_write_fn(make_insert_batch(batch))
192 | batch = []
193 | # And yield to the event loop
194 | await asyncio.sleep(0)
195 |
196 | if batch:
197 | await db.execute_write_fn(make_insert_batch(batch))
198 |
199 | # Mark progress as complete
200 | def mark_complete(conn):
201 | nonlocal i
202 | database = sqlite_utils.Database(conn)
203 | with conn:
204 | database["_csv_progress_"].update(
205 | task_id,
206 | {
207 | "rows_done": i,
208 | "bytes_done": total_size,
209 | "completed": str(datetime.datetime.utcnow()),
210 | },
211 | )
212 |
213 | await db.execute_write_fn(mark_complete)
214 |
215 | # Transform columns to detected types
216 | def transform_columns(conn):
217 | database = sqlite_utils.Database(conn)
218 | with conn:
219 | database[table_name].transform(types=tracker.types)
220 |
221 | await db.execute_write_fn(transform_columns)
222 |
223 | except Exception as error:
224 | await db.execute_write(
225 | "update _csv_progress_ set error = ? where id = ?",
226 | (str(error), task_id),
227 | )
228 |
229 | # Run that as a task
230 | asyncio.create_task(parse_csv())
231 |
232 | if formdata.get("xhr"):
233 | return Response.json(
234 | {
235 | "url": datasette.urls.table(db.name, table_name),
236 | "database_path": quote_plus(db.name),
237 | "task_id": task_id,
238 | "bytes_todo": total_size,
239 | }
240 | )
241 |
242 | return Response.html(
243 | await datasette.render_template(
244 | "upload_csv_done.html",
245 | {
246 | "database": db.name,
247 | "table": table_name,
248 | "table_url": datasette.urls.table(db.name, table_name),
249 | },
250 | )
251 | )
252 |
253 |
254 | def get_temporary_file_size(file):
255 | if isinstance(file._file, (io.BytesIO, io.StringIO)):
256 | return len(file._file.getvalue())
257 | try:
258 | return os.fstat(file._file.fileno()).st_size
259 | except Exception:
260 | raise
261 |
--------------------------------------------------------------------------------
/datasette_upload_csvs/templates/upload_csv.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 |
3 | {% block title %}Upload CSV{% if selected_name %} to "{{ selected_name }}"{% endif %}{% endblock %}
4 |
5 | {% block extra_head %}
6 | {{ super() }}
7 |
46 | {% endblock %}
47 |
48 | {% block content %}
49 | Upload CSV
50 |
51 |
76 |
77 |
78 |
220 | {% endblock %}
221 |
--------------------------------------------------------------------------------
/datasette_upload_csvs/templates/upload_csv_done.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 |
3 | {% block title %}Upload in progress{% endblock %}
4 |
5 | {% block content %}
6 | Upload in progress
7 |
8 | Importing rows into
9 | {{ table }}
10 |
11 | {% endblock %}
12 |
--------------------------------------------------------------------------------
/pytest.ini:
--------------------------------------------------------------------------------
1 | [pytest]
2 | asyncio_mode = strict
3 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup
2 | import os
3 |
4 | VERSION = "0.9.1"
5 |
6 |
7 | def get_long_description():
8 | with open(
9 | os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"),
10 | encoding="utf8",
11 | ) as fp:
12 | return fp.read()
13 |
14 |
15 | setup(
16 | name="datasette-upload-csvs",
17 | description="Datasette plugin for uploading CSV files and converting them to database tables",
18 | long_description=get_long_description(),
19 | long_description_content_type="text/markdown",
20 | author="Simon Willison",
21 | url="https://datasette.io/plugins/datasette-upload-csvs",
22 | project_urls={
23 | "Issues": "https://github.com/simonw/datasette-upload-csvs/issues",
24 | "CI": "https://github.com/simonw/datasette-upload-csvs/actions",
25 | "Changelog": "https://github.com/simonw/datasette-upload-csvs/releases",
26 | },
27 | license="Apache License, Version 2.0",
28 | version=VERSION,
29 | packages=["datasette_upload_csvs"],
30 | entry_points={"datasette": ["upload_csvs = datasette_upload_csvs"]},
31 | python_requires=">=3.7",
32 | install_requires=[
33 | "datasette>=0.61",
34 | "asgi-csrf>=0.7",
35 | "starlette",
36 | "aiofiles",
37 | "python-multipart",
38 | "charset-normalizer",
39 | "sqlite-utils",
40 | ],
41 | extras_require={
42 | "test": ["pytest", "pytest-asyncio", "asgiref", "httpx", "asgi-lifespan"]
43 | },
44 | package_data={"datasette_upload_csvs": ["templates/*.html"]},
45 | )
46 |
--------------------------------------------------------------------------------
/tests/conftest.py:
--------------------------------------------------------------------------------
1 | import datasette
2 |
3 |
4 | def pytest_report_header():
5 | return "Datasette: {}".format(datasette.__version__)
6 |
--------------------------------------------------------------------------------
/tests/test_datasette_upload_csvs.py:
--------------------------------------------------------------------------------
1 | from datasette.app import Datasette
2 | from datasette.utils import tilde_encode
3 | import asyncio
4 | from asgi_lifespan import LifespanManager
5 | import json
6 | from unittest.mock import ANY
7 | import pathlib
8 | import pytest
9 | import httpx
10 | import sqlite_utils
11 |
12 |
13 | @pytest.mark.asyncio
14 | async def test_lifespan():
15 | ds = Datasette([], memory=True)
16 | app = ds.app()
17 | async with LifespanManager(app):
18 | async with httpx.AsyncClient(app=app) as client:
19 | response = await client.get("http://localhost/")
20 | assert response.status_code == 200
21 |
22 |
23 | @pytest.mark.asyncio
24 | async def test_redirect():
25 | datasette = Datasette([], memory=True)
26 | async with httpx.AsyncClient(app=datasette.app()) as client:
27 | response = await client.get("http://localhost/-/upload-csv")
28 | assert response.status_code == 302
29 | assert response.headers["location"] == "/-/upload-csvs"
30 |
31 |
32 | @pytest.mark.asyncio
33 | @pytest.mark.parametrize("auth", [True, False])
34 | @pytest.mark.parametrize("has_database", [True, False])
35 | async def test_menu(tmpdir, auth, has_database):
36 | path = str(tmpdir / "data.db")
37 | db = sqlite_utils.Database(path)
38 | db.vacuum()
39 | ds = Datasette([path] if has_database else [], memory=True)
40 | app = ds.app()
41 | async with LifespanManager(app):
42 | async with httpx.AsyncClient(app=app) as client:
43 | cookies = {}
44 | if auth:
45 | cookies = {"ds_actor": ds.sign({"a": {"id": "root"}}, "actor")}
46 | response = await client.get("http://localhost/", cookies=cookies)
47 | assert response.status_code == 200
48 | should_allow = False
49 | if auth and has_database:
50 | assert "/-/upload-csvs" in response.text
51 | else:
52 | assert "/-/upload-csvs" not in response.text
53 | assert (
54 | (
55 | await client.get("http://localhost/-/upload-csvs", cookies=cookies)
56 | ).status_code
57 | == 200
58 | if should_allow
59 | else 403
60 | )
61 |
62 |
63 | SIMPLE = b"name,age\nCleo,5\nPancakes,4"
64 | SIMPLE_EXPECTED = [{"name": "Cleo", "age": 5}, {"name": "Pancakes", "age": 4}]
65 | NOT_UTF8 = (
66 | b"IncidentNumber,DateTimeOfCall,CalYear,FinYear,TypeOfIncident,PumpCount,PumpHoursTotal,HourlyNotionalCost(\xa3),IncidentNotionalCost(\xa3)\r\n"
67 | b"139091,01/01/2009 03:01,2009,2008/09,Special Service,1,2,2.55,5.10\r\n"
68 | b"275091,01/01/2009 08:51,2009,2008/09,Special Service,1,1,2.55,2.55"
69 | )
70 | NOT_UTF8_EXPECTED = [
71 | {
72 | "IncidentNumber": 139091,
73 | "DateTimeOfCall": "01/01/2009 03:01",
74 | "CalYear": 2009,
75 | "FinYear": "2008/09",
76 | "TypeOfIncident": "Special Service",
77 | "PumpCount": 1,
78 | "PumpHoursTotal": 2,
79 | "HourlyNotionalCost(£)": 2.55,
80 | "IncidentNotionalCost(£)": 5.10,
81 | },
82 | {
83 | "IncidentNumber": 275091,
84 | "DateTimeOfCall": "01/01/2009 08:51",
85 | "CalYear": 2009,
86 | "FinYear": "2008/09",
87 | "TypeOfIncident": "Special Service",
88 | "PumpCount": 1,
89 | "PumpHoursTotal": 1,
90 | "HourlyNotionalCost(£)": 2.55,
91 | "IncidentNotionalCost(£)": 2.55,
92 | },
93 | ]
94 | LATIN1_AFTER_FIRST_2KB = ("just_one_column\n" + "aabbcc\n" * 1048 + "a.b.é").encode(
95 | "latin-1"
96 | )
97 |
98 |
99 | @pytest.mark.asyncio
100 | @pytest.mark.parametrize(
101 | "filename,content,expected_table,expected_rows",
102 | (
103 | ("dogs.csv", SIMPLE, "dogs", SIMPLE_EXPECTED),
104 | (
105 | "weird ~ filename here.csv.csv",
106 | SIMPLE,
107 | "weird ~ filename here.csv",
108 | SIMPLE_EXPECTED,
109 | ),
110 | ("not-utf8.csv", NOT_UTF8, "not-utf8", NOT_UTF8_EXPECTED),
111 | ("latin1-after-x.csv", "LATIN1_AFTER_FIRST_2KB", "latin1-after-x", ANY),
112 | # This table already exists
113 | ("already_exists.csv", SIMPLE, "already_exists_2", SIMPLE_EXPECTED),
114 | ),
115 | )
116 | @pytest.mark.parametrize("use_xhr", (True, False))
117 | @pytest.mark.parametrize("database", ("data", "data2"))
118 | async def test_upload(
119 | tmpdir, filename, content, expected_table, expected_rows, use_xhr, database
120 | ):
121 | expected_url = "/{}/{}".format(database, tilde_encode(expected_table))
122 | path = str(tmpdir / "data.db")
123 | path2 = str(tmpdir / "data2.db")
124 | dbs_by_name = {}
125 | for p in (path, path2):
126 | db = sqlite_utils.Database(p)
127 | dbs_by_name[pathlib.Path(p).stem] = db
128 | db.vacuum()
129 | db.enable_wal()
130 | db["already_exists"].insert({"id": 1})
131 | binary_content = content
132 | # Trick to avoid a 12MB string being part of the pytest rendered test name:
133 | if content == "LATIN1_AFTER_FIRST_2KB":
134 | binary_content = LATIN1_AFTER_FIRST_2KB
135 |
136 | db["hello"].insert({"hello": "world"})
137 |
138 | datasette = Datasette([path, path2])
139 |
140 | cookies = {"ds_actor": datasette.sign({"a": {"id": "root"}}, "actor")}
141 |
142 | # First test the upload page exists
143 | async with httpx.AsyncClient(app=datasette.app()) as client:
144 | response = await client.get("http://localhost/-/upload-csvs", cookies=cookies)
145 | assert response.status_code == 200
146 | assert (
147 | '