You can export this data to a Jupyter or Observable notebook by copying and pasting the following:
22 | 23 |Make sure you have Pandas. Import it in a cell like this:
26 | 27 |28 | import pandas 29 |30 | 31 | If this shows an error you can run
%pip install pandas in a notebook cell to install it.
32 |
33 | Now paste the following into a cell to load the {{ count }} row{% if count != 1 %}s{% endif %} into a DataFrame called df:
36 | df = pandas.read_json(
37 | {{ json.dumps(json_url) }}
38 | )
39 |
40 |
41 | Run df in a new cell to see the table.
You can export all {% if total_count %}{{ "{:,}".format(total_count) }} {% endif %}rows using a single streaming CSV export like this:
45 |{{ pandas_stream_code }}
46 | {% endif %}
47 | Import the data into a variable called rows like this:
rows = d3.json(
52 | {{ json.dumps(json_url) }}
53 | )
54 |
55 | {% if allow_csv_stream and csv_stream_url and has_next %}
56 | You can export all {% if total_count %}{{ "{:,}".format(total_count) }} {% endif %}rows using a single streaming CSV export like this:
57 |rows = d3.csv(
58 | {{ json.dumps(csv_stream_url) }},
59 | d3.autoType
60 | )
61 | {% endif %}
62 | {% else %}
63 |
64 | {% endif %}
65 |
66 |
115 |
116 | {% endblock %}
117 |
--------------------------------------------------------------------------------
/datasette_export_notebook/utils.py:
--------------------------------------------------------------------------------
1 | def detect_types(rows):
2 | """
3 | Returns {colname: string_type} for these rows, where
4 | string_type is one of "int", "str" or "float"
5 | """
6 | columns = {}
7 | for row in rows:
8 | for column, value in dict(row).items():
9 | if value is not None:
10 | columns.setdefault(column, set()).add(type(value).__name__)
11 |
12 | # Only suggest type if column had just that type (or that + null)
13 | suggestions = {}
14 | for column, types in columns.items():
15 | if len(types) == 1:
16 | suggestions[column] = list(types)[0]
17 |
18 | return suggestions
19 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup
2 | import os
3 |
4 | VERSION = "1.0.1"
5 |
6 |
7 | def get_long_description():
8 | with open(
9 | os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"),
10 | encoding="utf8",
11 | ) as fp:
12 | return fp.read()
13 |
14 |
15 | setup(
16 | name="datasette-export-notebook",
17 | description="Datasette plugin providing instructions for exporting data to Jupyter or Observable",
18 | long_description=get_long_description(),
19 | long_description_content_type="text/markdown",
20 | author="Simon Willison",
21 | url="https://github.com/simonw/datasette-export-notebook",
22 | project_urls={
23 | "Issues": "https://github.com/simonw/datasette-export-notebook/issues",
24 | "CI": "https://github.com/simonw/datasette-export-notebook/actions",
25 | "Changelog": "https://github.com/simonw/datasette-export-notebook/releases",
26 | },
27 | license="Apache License, Version 2.0",
28 | version=VERSION,
29 | packages=["datasette_export_notebook"],
30 | entry_points={"datasette": ["export_notebook = datasette_export_notebook"]},
31 | install_requires=["datasette"],
32 | extras_require={"test": ["pytest", "pytest-asyncio", "sqlite-utils"]},
33 | tests_require=["datasette-export-notebook[test]"],
34 | package_data={"datasette_export_notebook": ["templates/*.html"]},
35 | python_requires=">=3.6",
36 | )
37 |
--------------------------------------------------------------------------------
/tests/test_export_notebook.py:
--------------------------------------------------------------------------------
1 | from datasette.app import Datasette
2 | import pytest
3 | import sqlite_utils
4 | import sqlite3
5 | import textwrap
6 | from datasette_export_notebook.utils import detect_types
7 |
8 |
9 | @pytest.fixture
10 | def db_path(tmpdir):
11 | db_path = str(tmpdir / "db.db")
12 | db = sqlite_utils.Database(db_path)
13 | db["blah"].insert_all({"id": i} for i in range(80))
14 | db["big"].insert_all({"id": i} for i in range(800))
15 | return db_path
16 |
17 |
18 | @pytest.mark.asyncio
19 | @pytest.mark.parametrize(
20 | "path,expected_json_url,expected_csv_url",
21 | [
22 | ("/db/blah.Notebook", "http://localhost/db/blah.json?_shape=array", ""),
23 | (
24 | "/db/big.Notebook",
25 | "http://localhost/db/big.json?_shape=array",
26 | "http://localhost/db/big.csv?_stream=on",
27 | ),
28 | ],
29 | )
30 | async def test_export_notebook(
31 | db_path,
32 | path,
33 | expected_json_url,
34 | expected_csv_url,
35 | ):
36 | datasette = Datasette([db_path], cors=True)
37 | response = await datasette.client.get(path)
38 | assert 200 == response.status_code
39 | assert "--cors" not in response.text
40 | assert (
41 | """
42 | df = pandas.read_json(
43 | "{}"
44 | )
45 | """.strip().format(
46 | expected_json_url
47 | )
48 | in response.text
49 | )
50 | assert (
51 | """
52 | rows = d3.json(
53 | "{}"
54 | )""".strip().format(
55 | expected_json_url
56 | )
57 | in response.text
58 | )
59 | if not expected_csv_url:
60 | assert ".csv" not in response.text
61 | else:
62 | assert (
63 | "df = pandas.read_csv(\n "{}"".format(expected_csv_url)
64 | in response.text
65 | )
66 | assert "rows = d3.csv(\n "{}"".format(expected_csv_url) in response.text
67 |
68 |
69 | @pytest.mark.asyncio
70 | async def test_export_notebook_pandas_stream_with_types(db_path):
71 | datasette = Datasette([db_path], cors=True)
72 | response = await datasette.client.get("/db/big.Notebook")
73 | expected = (
74 | 'df = pandas.read_csv(\n'
75 | " "http://localhost/db/big.csv?_stream=on", dtype={\n"
76 | " "rowid": int,\n"
77 | " "id": int,\n"
78 | "})"
79 | )
80 | assert expected in response.text
81 |
82 |
83 | @pytest.mark.asyncio
84 | async def test_notebook_no_cors(db_path):
85 | datasette = Datasette([db_path])
86 | response = await datasette.client.get("/db/big.Notebook")
87 | assert (
88 | "Export to Observable is only available if Datasette is running with the"
89 | in response.text
90 | )
91 |
92 |
93 | @pytest.mark.asyncio
94 | async def test_notebook_no_csv(db_path):
95 | datasette = Datasette([db_path], settings={"allow_csv_stream": False})
96 | response = await datasette.client.get("/db/big.Notebook")
97 | assert ".csv" not in response.text
98 |
99 |
100 | @pytest.mark.parametrize(
101 | "rows,expected_types",
102 | [
103 | ([{"id": 1}], {"id": "int"}),
104 | ([{"id": None}], {}),
105 | ([{"id": 1, "name": "bob"}], {"id": "int", "name": "str"}),
106 | ([{"f": 3.5}, {"f": None}], {"f": "float"}),
107 | ],
108 | )
109 | def test_detect_types(rows, expected_types):
110 | db = sqlite_utils.Database(memory=True)
111 | db["t"].insert_all(rows)
112 | db.conn.row_factory = sqlite3.Row
113 | rows = db.conn.execute("select * from t").fetchall()
114 | column_types = detect_types(rows)
115 | assert column_types == expected_types
116 |
--------------------------------------------------------------------------------