├── .gitattributes ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .pre-commit-config.yaml ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.rst ├── examples └── Scotrail.ipynb ├── ibis_datasette ├── __init__.py ├── _version.py └── core.py ├── setup.cfg ├── setup.py ├── tests └── test_core.py └── versioneer.py /.gitattributes: -------------------------------------------------------------------------------- 1 | ibis_datasette/_version.py export-subst 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Build and Test 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | paths-ignore: 9 | - "README.rst" 10 | release: 11 | types: [published] 12 | 13 | jobs: 14 | lint: 15 | name: Lint and flake code 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | 21 | - name: Install Python 22 | uses: actions/setup-python@v2 23 | with: 24 | python-version: "3.10" 25 | 26 | - name: Install Test Dependencies 27 | run: | 28 | pip install pytest datasette 29 | 30 | - name: Build 31 | run: | 32 | pip install -e . 33 | 34 | - name: Run black & flake8 35 | uses: pre-commit/action@v2.0.0 36 | 37 | - name: Run tests 38 | run: py.test -v tests 39 | 40 | build_release: 41 | name: Build Source Distribution 42 | runs-on: ubuntu-latest 43 | if: github.event_name == 'release' && github.event.action == 'published' 44 | 45 | steps: 46 | - uses: actions/checkout@v2 47 | 48 | - name: Install Python 49 | uses: actions/setup-python@v2 50 | with: 51 | python-version: "3.10" 52 | 53 | - name: Install build deps 54 | run: pip install wheel 55 | 56 | - name: Build source distribution 57 | run: python setup.py sdist 58 | 59 | - name: Build wheel 60 | run: python setup.py bdist_wheel --universal 61 | 62 | - name: Upload artifacts 63 | uses: actions/upload-artifact@v2 64 | with: 65 | path: dist/ 66 | 67 | upload_pypi: 68 | needs: [build_release] 69 | runs-on: ubuntu-latest 70 | if: github.event_name == 'release' && github.event.action == 'published' 71 | steps: 72 | - uses: actions/download-artifact@v2 73 | with: 74 | name: artifact 75 | path: dist 76 | 77 | - uses: pypa/gh-action-pypi-publish@master 78 | with: 79 | user: __token__ 80 | password: ${{ secrets.PYPI_API_TOKEN }} 81 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info 3 | *.pem 4 | build/ 5 | dist/ 6 | docs/build/ 7 | .cache/ 8 | .coverage 9 | .pytest_cache/ 10 | .ipynb_checkpoints/ 11 | .eggs/ 12 | .DS_Store 13 | htmlcov/ 14 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: 22.3.0 4 | hooks: 5 | - id: black 6 | language_version: python3 7 | args: ["--target-version=py38"] 8 | 9 | - repo: https://gitlab.com/pycqa/flake8 10 | rev: 4.0.1 11 | hooks: 12 | - id: flake8 13 | language_version: python3 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # This dockerfile is for mybinder.org support for running examples. 2 | # It's *not* required for using ibis-datasette, and should mostly 3 | # be ignored. 4 | FROM python:3.10-slim 5 | 6 | ARG NB_USER=jovyan 7 | ARG NB_UID=1000 8 | ENV USER ${NB_USER} 9 | ENV NB_UID ${NB_UID} 10 | ENV HOME /home/${NB_USER} 11 | 12 | RUN adduser --disabled-password \ 13 | --gecos "Default user" \ 14 | --uid ${NB_UID} \ 15 | ${NB_USER} 16 | 17 | RUN apt-get update && \ 18 | apt-get install -y ffmpeg git && \ 19 | apt-get clean && \ 20 | rm -rf /var/lib/apt/lists/* 21 | 22 | # Install dependencies 23 | RUN pip install --no-cache --upgrade pip \ 24 | && pip install --no-cache \ 25 | notebook \ 26 | jupyterlab \ 27 | sqlglot \ 28 | pydub \ 29 | ipywidgets \ 30 | git+https://github.com/ibis-project/ibis.git \ 31 | git+https://github.com/jcrist/ibis-datasette.git \ 32 | && find /usr/local/lib/python3.10/site-packages/ -follow -type f -name '*.a' -delete \ 33 | && find /usr/local/lib/python3.10/site-packages/ -follow -type f -name '*.pyc' -delete \ 34 | && find /usr/local/lib/python3.10/site-packages/ -follow -type f -name '*.js.map' -delete 35 | 36 | COPY --chown=${NB_UID} examples ${HOME} 37 | 38 | USER ${USER} 39 | WORKDIR ${HOME} 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022, Jim Crist-Harif 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include ibis_datasette/*.py 2 | include setup.py 3 | include versioneer.py 4 | include README.rst 5 | include LICENSE 6 | include MANIFEST.in 7 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ibis-datasette 2 | ============== 3 | 4 | |github| |pypi| |binder| 5 | 6 | ``ibis-datasette`` provides a datasette_ backend for ibis_. This lets you query 7 | any ``datasette`` server using a familiar dataframe-like API (rather than SQL). 8 | 9 | 10 | Installation 11 | ------------ 12 | 13 | ``ibis-datasette`` is available on pypi_: 14 | 15 | .. code-block:: 16 | 17 | $ pip install ibis-datasette 18 | 19 | 20 | Usage 21 | ----- 22 | 23 | Once installed, you can connect to any ``datasette`` server using the 24 | ``ibis.datasette.connect`` function. This takes the `full URL to a database`_ 25 | For example, to connect to the `legislators database`_. 26 | 27 | .. code-block:: python 28 | 29 | In [1]: import ibis 30 | 31 | In [2]: con = ibis.datasette.connect("https://congress-legislators.datasettes.com/legislators") 32 | 33 | 34 | Once connected, you can interact with tables using ``ibis`` just as you would a 35 | local ``sqlite`` database: 36 | 37 | .. code-block:: python 38 | 39 | In [3]: ibis.options.interactive = True # enable interactive mode 40 | 41 | In [4]: con.list_tables() 42 | Out[4]: 43 | ['executive_terms', 44 | 'executives', 45 | 'legislator_terms', 46 | 'legislators', 47 | 'offices', 48 | 'social_media'] 49 | 50 | In [5]: t = con.tables.legislators # access the `legislators` table 51 | 52 | In [6]: t.name_first.topk(5) # top 5 first names for legislators 53 | Out[6]: 54 | name_first count 55 | 0 John 1273 56 | 1 William 1024 57 | 2 James 721 58 | 3 Thomas 457 59 | 4 Charles 442 60 | 61 | 62 | LICENSE 63 | ------- 64 | 65 | New BSD. See the `License File`_. 66 | 67 | .. |github| image:: https://github.com/jcrist/ibis-datasette/actions/workflows/ci.yml/badge.svg 68 | :target: https://github.com/jcrist/ibis-datasette/actions/workflows/ci.yml 69 | .. |pypi| image:: https://img.shields.io/pypi/v/ibis-datasette.svg 70 | :target: https://pypi.org/project/ibis-datasette/ 71 | .. |binder| image:: https://mybinder.org/badge_logo.svg 72 | :target: https://gke.mybinder.org/v2/gh/jcrist/ibis-datasette/main?urlpath=/tree/ 73 | 74 | .. _pypi: https://pypi.org/project/ibis-datasette/ 75 | .. _ibis: https://ibis-project.org/ 76 | .. _datasette: https://datasette.io/ 77 | .. _full URL to a database: https://docs.datasette.io/en/stable/pages.html#database 78 | .. _legislators database: https://congress-legislators.datasettes.com/legislators 79 | .. _License File: https://github.com/jcrist/ibis-datasette/blob/main/LICENSE 80 | -------------------------------------------------------------------------------- /examples/Scotrail.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "id": "37beaf2b", 6 | "metadata": {}, 7 | "source": [ 8 | "# ScotRail Analysis with Ibis\n", 9 | "\n", 10 | "This notebook uses [ibis-datasette](https://github.com/jcrist/ibis-datasette) to analyze data from the [ScotRail](https://scotrail.datasette.io/) [datasette](datasette.io/).\n", 11 | "\n", 12 | "This datasette is _super_ fun to play around with. It's composed of ~2400 different audioclips (and transcriptions) from Scottish train operator ScotRail's automated station announcements.\n", 13 | "\n", 14 | "If you haven't seen it, I encourage you to read Simon Willison's [excellent blogpost](https://simonwillison.net/2022/Aug/21/scotrail/) on putting this datasette together, and some interesting queries to try (we'll be replicating one of these below).\n", 15 | "\n", 16 | "While you can use the [datasette UI](https://scotrail.datasette.io/) directly, I wanted to use [ibis](https://ibis-project.org) and the full power of Python to explore and build some interesting things.\n", 17 | "\n", 18 | "---\n", 19 | "\n", 20 | "Like most notebooks, first we start with some imports and initialization.\n", 21 | "\n", 22 | "Here we:\n", 23 | "\n", 24 | "- Import `ibis` and its `_` helper (more on this later)\n", 25 | "- Enable ibis's interactive mode\n", 26 | "- We also tweak pandas' display options to better render wide columns. This makes the transcriptions below easier to read." 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": null, 32 | "id": "8c7b53cb", 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "import ibis\n", 37 | "from ibis import _\n", 38 | "\n", 39 | "ibis.options.interactive = True\n", 40 | "\n", 41 | "import pandas as pd\n", 42 | "pd.set_option('max_colwidth', 400)" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "id": "bf8266fa", 48 | "metadata": {}, 49 | "source": [ 50 | "Next we need to connect to the datasette. This is done by passing the full URL to `ibis.datasette.connect`:" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "id": "45919b94", 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "con = ibis.datasette.connect(\"https://scotrail.datasette.io/scotrail\")" 61 | ] 62 | }, 63 | { 64 | "cell_type": "markdown", 65 | "id": "1d4c5ef8", 66 | "metadata": {}, 67 | "source": [ 68 | "Once connected, we can start poking around.\n", 69 | "\n", 70 | "The first thing I usually do when exploring a new datasette is examine the tables and schemas:" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "id": "b34a3d48", 77 | "metadata": {}, 78 | "outputs": [], 79 | "source": [ 80 | "con.list_tables()" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": null, 86 | "id": "9cc005a3", 87 | "metadata": {}, 88 | "outputs": [], 89 | "source": [ 90 | "con.tables.announcements.schema()" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "id": "539332a2", 97 | "metadata": {}, 98 | "outputs": [], 99 | "source": [ 100 | " con.tables.announcements.head()" 101 | ] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "id": "3071232f", 106 | "metadata": {}, 107 | "source": [ 108 | "The main table is `announcments`, the most interesting columns of which are:\n", 109 | "\n", 110 | "- `Transcription`: a full transcription of the audio clip\n", 111 | "- `Category`: a category that the audio clip belongs to\n", 112 | "- `mp3`: a link to the audio clip, hosted on GitHub\n", 113 | "\n", 114 | "Since we're going to be accessing this table a lot below, lets save it to a shorter local variable name:" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": null, 120 | "id": "1dc1cdeb", 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "t = con.tables.announcements" 125 | ] 126 | }, 127 | { 128 | "cell_type": "markdown", 129 | "id": "edb2fa9e", 130 | "metadata": {}, 131 | "source": [ 132 | "To get a better sense of the scale of data we're working with, lets take a closer look at the `Category` column.\n", 133 | "\n", 134 | "I want to know how many categories there are, and how the audio clips are distributed across these categories.\n", 135 | "\n", 136 | "To do this, we can use:\n", 137 | "\n", 138 | "- `.group_by(\"Category\")` to split the data into separate groups by `Category`\n", 139 | "- `.count()` to then count how many rows are in each category.\n", 140 | "- `.sort_by(ibis.desc(\"count\"))` to then sort the rows by `count`, descending." 141 | ] 142 | }, 143 | { 144 | "cell_type": "code", 145 | "execution_count": null, 146 | "id": "cb46cf07", 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "category_counts = (\n", 151 | " t.group_by(\"Category\")\n", 152 | " .count()\n", 153 | " .sort_by(ibis.desc(\"count\"))\n", 154 | ")\n", 155 | "\n", 156 | "category_counts" 157 | ] 158 | }, 159 | { 160 | "cell_type": "markdown", 161 | "id": "fe5d094e", 162 | "metadata": {}, 163 | "source": [ 164 | "Here we can see there are 23 categories, with 90% of the audio clips falling into the first 6. A few categories to highlight:\n", 165 | "\n", 166 | "- `Destination` is a ScotRail stop\n", 167 | "- `Reason` is a reason for a cancellation. These are fun to look through.\n", 168 | "- `Passenger information` is a bit of miscellaneous. (\"The train is ready to leave\" for example)\n", 169 | "- `Number` and `Time` are just clips of saying numbers and times\n", 170 | "- `Train operating company` is the name of a train operating company\n", 171 | "- `Apology` is the start of an apology for a service disruption (\"I am sorry to announce that the\" for example)\n", 172 | "\n", 173 | "The `Reason` category is the most fun to look through. There are all sorts of reasons a train might be cancelled, from \"Sheep on the railway\" to \"A wartime bomb near the railway\".\n", 174 | "\n", 175 | "---\n", 176 | "\n", 177 | "One reoccuring reason is theft (err, \"attempted theft\") of various things. Lets find all reasons involving \"theft\". \n", 178 | "\n", 179 | "This can be done by using `.filter()` to filter rows based on a predicate. Here we need two predicates:\n", 180 | "\n", 181 | "- `_.Category == \"Reason\"` selects all rows that have a category of \"Reason\"\n", 182 | "- `_.Transcription.contains(\"theft\")` selects all rows with a transcription containing the string \"theft\"" 183 | ] 184 | }, 185 | { 186 | "cell_type": "code", 187 | "execution_count": null, 188 | "id": "f8c5c5e0", 189 | "metadata": {}, 190 | "outputs": [], 191 | "source": [ 192 | "thefts = t.filter((_.Category == \"Reason\") & _.Transcription.contains(\"theft\"))\n", 193 | "\n", 194 | "thefts" 195 | ] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "id": "f7939faa", 200 | "metadata": {}, 201 | "source": [ 202 | "All of these rows also include a link to an `mp3` file containing that clip. To play a clip in a jupyter notebook, we can make use of `IPython.display.Audio`. For example, lets play the first clip from above:" 203 | ] 204 | }, 205 | { 206 | "cell_type": "code", 207 | "execution_count": null, 208 | "id": "db197d99", 209 | "metadata": {}, 210 | "outputs": [], 211 | "source": [ 212 | "from IPython.display import Audio\n", 213 | "\n", 214 | "mp3_url = thefts.limit(1).execute().mp3.iloc[0]\n", 215 | "\n", 216 | "Audio(mp3_url)" 217 | ] 218 | }, 219 | { 220 | "cell_type": "markdown", 221 | "id": "4f77f8c4", 222 | "metadata": {}, 223 | "source": [ 224 | "## Generating a Random Apology\n", 225 | "\n", 226 | "In [his blogpost](https://simonwillison.net/2022/Aug/21/scotrail/) Simon wrote up a SQL query for generating a Random apology by combining a few random rows from different categories above. It generates surprisingly coherent sentences, you can see the datasette version [here](https://scotrail.datasette.io/scotrail/random_apology).\n", 227 | "\n", 228 | "If you're interested you can click `show` at the top to see the full SQL query - it's readable, but a bit long.\n", 229 | "\n", 230 | "I wanted to reproduce the same query using `ibis`. Since `ibis` is just a Python library, you can make use of things like functions to abstract away some of the repetitiveness in the SQL query above.\n", 231 | "\n", 232 | "Here's what I came up with:" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": null, 238 | "id": "08f9030b", 239 | "metadata": {}, 240 | "outputs": [], 241 | "source": [ 242 | "def random(category):\n", 243 | " \"\"\"Select a random row from a given category\"\"\"\n", 244 | " return (\n", 245 | " t.filter(_.Category == category)\n", 246 | " .sort_by(ibis.random())\n", 247 | " .select(\"Transcription\", \"mp3\")\n", 248 | " .limit(1)\n", 249 | " )\n", 250 | "\n", 251 | "def phrase(text):\n", 252 | " \"\"\"Select a row with a specific transcription\"\"\"\n", 253 | " return (\n", 254 | " t.filter(_.Transcription == text)\n", 255 | " .select(\"Transcription\", \"mp3\")\n", 256 | " .limit(1)\n", 257 | " )\n", 258 | "\n", 259 | "query = ibis.union(\n", 260 | " random(\"Apology\"),\n", 261 | " random(\"Train operating company\"),\n", 262 | " random(\"Destination\"),\n", 263 | " phrase(\"has been cancelled\"),\n", 264 | " phrase(\"due to\"),\n", 265 | " random(\"Reason\"),\n", 266 | ")" 267 | ] 268 | }, 269 | { 270 | "cell_type": "markdown", 271 | "id": "a43351da", 272 | "metadata": {}, 273 | "source": [ 274 | "Since the query selects random rows, if you run the cell below multiple times, you should see different results every time:" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": null, 280 | "id": "b907d967", 281 | "metadata": {}, 282 | "outputs": [], 283 | "source": [ 284 | "query.execute()" 285 | ] 286 | }, 287 | { 288 | "cell_type": "markdown", 289 | "id": "3f193d7f", 290 | "metadata": {}, 291 | "source": [ 292 | "If we wanted to do all computation in the backend, we could use `group_concat` ([docs](https://www.sqlite.org/lang_aggfunc.html#group_concat)) to then concatenate the Transcription rows together, returning a single string:" 293 | ] 294 | }, 295 | { 296 | "cell_type": "code", 297 | "execution_count": null, 298 | "id": "4f58082b", 299 | "metadata": {}, 300 | "outputs": [], 301 | "source": [ 302 | "random_apology = query.Transcription.group_concat(\" \")\n", 303 | "\n", 304 | "random_apology" 305 | ] 306 | }, 307 | { 308 | "cell_type": "markdown", 309 | "id": "470d2e2c", 310 | "metadata": {}, 311 | "source": [ 312 | "Note that the full query above is translated to SQL and executed on the `datasette` server, no computation is happening locally.\n", 313 | "\n", 314 | "If you want to see the generated SQL, you can use the `ibis.show_sql` function:" 315 | ] 316 | }, 317 | { 318 | "cell_type": "code", 319 | "execution_count": null, 320 | "id": "7485b4d7", 321 | "metadata": {}, 322 | "outputs": [], 323 | "source": [ 324 | "ibis.show_sql(random_apology)" 325 | ] 326 | }, 327 | { 328 | "cell_type": "markdown", 329 | "id": "c62d8676", 330 | "metadata": {}, 331 | "source": [ 332 | "However, we're only using `ibis` to push the bulk of the computation to the backend. We don't need to handle _everything_ in SQL, only enough to reduce the size of the results to something reasonable to return from the `datasette` server.\n", 333 | "\n", 334 | "We also have access to the full Python ecosystem to process results. This lets us do some things that wouldn't be possible in SQL alone, like concatenating `mp3` files :)." 335 | ] 336 | }, 337 | { 338 | "cell_type": "markdown", 339 | "id": "82eaa616", 340 | "metadata": {}, 341 | "source": [ 342 | "## A \"Random Apology\" Button" 343 | ] 344 | }, 345 | { 346 | "cell_type": "markdown", 347 | "id": "51bd234d", 348 | "metadata": {}, 349 | "source": [ 350 | "The [ipywidgets](https://ipywidgets.readthedocs.io) library provides support for building simple UIs in Python, with the rendering handled by the notebook. This is nice for me, as I am _not_ a web engineer - I'm a novice at best at javascript/html. However, I do know how to write Python.\n", 351 | "\n", 352 | "Below we hack together a quick UI with `ipywidgets` to make a button for generating a random apology, complete with a merged `mp3` file so you can listen to your work. You don't really need to understand this code, it has nothing to do with `ibis` or `ibis-datasette` itself.\n", 353 | "\n", 354 | "Clicking the button will pull generate a new random apology, download and merge the mp3 files, and display both the apology sentence and merged mp3." 355 | ] 356 | }, 357 | { 358 | "cell_type": "code", 359 | "execution_count": null, 360 | "id": "8e405257", 361 | "metadata": {}, 362 | "outputs": [], 363 | "source": [ 364 | "import tempfile\n", 365 | "import os\n", 366 | "import pydub\n", 367 | "import httpx\n", 368 | "import ipywidgets\n", 369 | "from IPython.display import Audio, display\n", 370 | "\n", 371 | "output = ipywidgets.Output()\n", 372 | "button = ipywidgets.Button(description='Random Apology', icon=\"repeat\")\n", 373 | "UI = ipywidgets.VBox([button, output])\n", 374 | "\n", 375 | "\n", 376 | "def concatenate_mp3s(urls: list[str]) -> bytes:\n", 377 | " with httpx.Client(follow_redirects=True) as client, tempfile.TemporaryDirectory() as tempdir:\n", 378 | " output = None\n", 379 | " for i, url in enumerate(urls):\n", 380 | " path = os.path.join(tempdir, f\"part{i}.mp3\")\n", 381 | " with open(path, \"wb\") as f:\n", 382 | " resp = client.get(url)\n", 383 | " resp.raise_for_status()\n", 384 | " f.write(resp.content)\n", 385 | " part = pydub.AudioSegment.from_mp3(path)\n", 386 | " if output is None:\n", 387 | " output = part\n", 388 | " else:\n", 389 | " output = output + part\n", 390 | " out_path = os.path.join(tempdir, \"output.mp3\")\n", 391 | " output.export(out_path, format=\"mp3\")\n", 392 | " with open(out_path, \"rb\") as f:\n", 393 | " return f.read()\n", 394 | "\n", 395 | "\n", 396 | "@button.on_click\n", 397 | "def on_click(*args):\n", 398 | " output.clear_output()\n", 399 | " result = query.execute()\n", 400 | " msg = \" \".join(result.Transcription)\n", 401 | " mp3 = concatenate_mp3s(result.mp3)\n", 402 | " with output:\n", 403 | " print(msg)\n", 404 | " display(Audio(mp3))\n", 405 | "\n", 406 | " \n", 407 | "UI" 408 | ] 409 | }, 410 | { 411 | "cell_type": "markdown", 412 | "id": "4d833b56", 413 | "metadata": {}, 414 | "source": [ 415 | "## Review\n", 416 | "\n", 417 | "`datasette` makes it easier to publish accessible open data on the web, with a UI exposed for writing SQL queries. However, not everyone is extremely SQL literate (myself included). `ibis` and `ibis-datasette` let Python programmers access this same data resource, but through a familiar dataframe-like interface.\n", 418 | "\n", 419 | "For more information on `ibis`, see the [official documentation](https://ibis-project.org)." 420 | ] 421 | } 422 | ], 423 | "metadata": { 424 | "kernelspec": { 425 | "display_name": "Python 3 (ipykernel)", 426 | "language": "python", 427 | "name": "python3" 428 | }, 429 | "language_info": { 430 | "codemirror_mode": { 431 | "name": "ipython", 432 | "version": 3 433 | }, 434 | "file_extension": ".py", 435 | "mimetype": "text/x-python", 436 | "name": "python", 437 | "nbconvert_exporter": "python", 438 | "pygments_lexer": "ipython3", 439 | "version": "3.10.5" 440 | } 441 | }, 442 | "nbformat": 4, 443 | "nbformat_minor": 5 444 | } 445 | -------------------------------------------------------------------------------- /ibis_datasette/__init__.py: -------------------------------------------------------------------------------- 1 | from .core import Backend 2 | 3 | from . import _version 4 | 5 | __version__ = _version.get_versions()["version"] 6 | -------------------------------------------------------------------------------- /ibis_datasette/_version.py: -------------------------------------------------------------------------------- 1 | # This file helps to compute a version number in source trees obtained from 2 | # git-archive tarball (such as those provided by githubs download-from-tag 3 | # feature). Distribution tarballs (built by setup.py sdist) and build 4 | # directories (produced by setup.py build) will contain a much shorter file 5 | # that just contains the computed version number. 6 | 7 | # This file is released into the public domain. Generated by 8 | # versioneer-0.23 (https://github.com/python-versioneer/python-versioneer) 9 | 10 | """Git implementation of _version.py.""" 11 | 12 | import errno 13 | import os 14 | import re 15 | import subprocess 16 | import sys 17 | from typing import Callable, Dict 18 | import functools 19 | 20 | 21 | def get_keywords(): 22 | """Get the keywords needed to look up the version information.""" 23 | # these strings will be replaced by git during git-archive. 24 | # setup.py/versioneer.py will grep for the variable names, so they must 25 | # each be defined on a line of their own. _version.py will just call 26 | # get_keywords(). 27 | git_refnames = " (HEAD -> main)" 28 | git_full = "4389a02269807b0e6550c23e83107aac276d03ee" 29 | git_date = "2022-08-24 20:01:18 -0500" 30 | keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} 31 | return keywords 32 | 33 | 34 | class VersioneerConfig: 35 | """Container for Versioneer configuration parameters.""" 36 | 37 | 38 | def get_config(): 39 | """Create, populate and return the VersioneerConfig() object.""" 40 | # these strings are filled in when 'setup.py versioneer' creates 41 | # _version.py 42 | cfg = VersioneerConfig() 43 | cfg.VCS = "git" 44 | cfg.style = "pep440" 45 | cfg.tag_prefix = "" 46 | cfg.parentdir_prefix = "ibis-datasette-" 47 | cfg.versionfile_source = "ibis_datasette/_version.py" 48 | cfg.verbose = False 49 | return cfg 50 | 51 | 52 | class NotThisMethod(Exception): 53 | """Exception raised if a method is not valid for the current scenario.""" 54 | 55 | 56 | LONG_VERSION_PY: Dict[str, str] = {} 57 | HANDLERS: Dict[str, Dict[str, Callable]] = {} 58 | 59 | 60 | def register_vcs_handler(vcs, method): # decorator 61 | """Create decorator to mark a method as the handler of a VCS.""" 62 | 63 | def decorate(f): 64 | """Store f in HANDLERS[vcs][method].""" 65 | if vcs not in HANDLERS: 66 | HANDLERS[vcs] = {} 67 | HANDLERS[vcs][method] = f 68 | return f 69 | 70 | return decorate 71 | 72 | 73 | def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): 74 | """Call the given command(s).""" 75 | assert isinstance(commands, list) 76 | process = None 77 | 78 | popen_kwargs = {} 79 | if sys.platform == "win32": 80 | # This hides the console window if pythonw.exe is used 81 | startupinfo = subprocess.STARTUPINFO() 82 | startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW 83 | popen_kwargs["startupinfo"] = startupinfo 84 | 85 | for command in commands: 86 | try: 87 | dispcmd = str([command] + args) 88 | # remember shell=False, so use git.cmd on windows, not just git 89 | process = subprocess.Popen( 90 | [command] + args, 91 | cwd=cwd, 92 | env=env, 93 | stdout=subprocess.PIPE, 94 | stderr=(subprocess.PIPE if hide_stderr else None), 95 | **popen_kwargs, 96 | ) 97 | break 98 | except OSError: 99 | e = sys.exc_info()[1] 100 | if e.errno == errno.ENOENT: 101 | continue 102 | if verbose: 103 | print("unable to run %s" % dispcmd) 104 | print(e) 105 | return None, None 106 | else: 107 | if verbose: 108 | print("unable to find command, tried %s" % (commands,)) 109 | return None, None 110 | stdout = process.communicate()[0].strip().decode() 111 | if process.returncode != 0: 112 | if verbose: 113 | print("unable to run %s (error)" % dispcmd) 114 | print("stdout was %s" % stdout) 115 | return None, process.returncode 116 | return stdout, process.returncode 117 | 118 | 119 | def versions_from_parentdir(parentdir_prefix, root, verbose): 120 | """Try to determine the version from the parent directory name. 121 | 122 | Source tarballs conventionally unpack into a directory that includes both 123 | the project name and a version string. We will also support searching up 124 | two directory levels for an appropriately named parent directory 125 | """ 126 | rootdirs = [] 127 | 128 | for _ in range(3): 129 | dirname = os.path.basename(root) 130 | if dirname.startswith(parentdir_prefix): 131 | return { 132 | "version": dirname[len(parentdir_prefix) :], 133 | "full-revisionid": None, 134 | "dirty": False, 135 | "error": None, 136 | "date": None, 137 | } 138 | rootdirs.append(root) 139 | root = os.path.dirname(root) # up a level 140 | 141 | if verbose: 142 | print( 143 | "Tried directories %s but none started with prefix %s" 144 | % (str(rootdirs), parentdir_prefix) 145 | ) 146 | raise NotThisMethod("rootdir doesn't start with parentdir_prefix") 147 | 148 | 149 | @register_vcs_handler("git", "get_keywords") 150 | def git_get_keywords(versionfile_abs): 151 | """Extract version information from the given file.""" 152 | # the code embedded in _version.py can just fetch the value of these 153 | # keywords. When used from setup.py, we don't want to import _version.py, 154 | # so we do it with a regexp instead. This function is not used from 155 | # _version.py. 156 | keywords = {} 157 | try: 158 | with open(versionfile_abs, "r") as fobj: 159 | for line in fobj: 160 | if line.strip().startswith("git_refnames ="): 161 | mo = re.search(r'=\s*"(.*)"', line) 162 | if mo: 163 | keywords["refnames"] = mo.group(1) 164 | if line.strip().startswith("git_full ="): 165 | mo = re.search(r'=\s*"(.*)"', line) 166 | if mo: 167 | keywords["full"] = mo.group(1) 168 | if line.strip().startswith("git_date ="): 169 | mo = re.search(r'=\s*"(.*)"', line) 170 | if mo: 171 | keywords["date"] = mo.group(1) 172 | except OSError: 173 | pass 174 | return keywords 175 | 176 | 177 | @register_vcs_handler("git", "keywords") 178 | def git_versions_from_keywords(keywords, tag_prefix, verbose): 179 | """Get version information from git keywords.""" 180 | if "refnames" not in keywords: 181 | raise NotThisMethod("Short version file found") 182 | date = keywords.get("date") 183 | if date is not None: 184 | # Use only the last line. Previous lines may contain GPG signature 185 | # information. 186 | date = date.splitlines()[-1] 187 | 188 | # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant 189 | # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 190 | # -like" string, which we must then edit to make compliant), because 191 | # it's been around since git-1.5.3, and it's too difficult to 192 | # discover which version we're using, or to work around using an 193 | # older one. 194 | date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) 195 | refnames = keywords["refnames"].strip() 196 | if refnames.startswith("$Format"): 197 | if verbose: 198 | print("keywords are unexpanded, not using") 199 | raise NotThisMethod("unexpanded keywords, not a git-archive tarball") 200 | refs = {r.strip() for r in refnames.strip("()").split(",")} 201 | # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of 202 | # just "foo-1.0". If we see a "tag: " prefix, prefer those. 203 | TAG = "tag: " 204 | tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} 205 | if not tags: 206 | # Either we're using git < 1.8.3, or there really are no tags. We use 207 | # a heuristic: assume all version tags have a digit. The old git %d 208 | # expansion behaves like git log --decorate=short and strips out the 209 | # refs/heads/ and refs/tags/ prefixes that would let us distinguish 210 | # between branches and tags. By ignoring refnames without digits, we 211 | # filter out many common branch names like "release" and 212 | # "stabilization", as well as "HEAD" and "master". 213 | tags = {r for r in refs if re.search(r"\d", r)} 214 | if verbose: 215 | print("discarding '%s', no digits" % ",".join(refs - tags)) 216 | if verbose: 217 | print("likely tags: %s" % ",".join(sorted(tags))) 218 | for ref in sorted(tags): 219 | # sorting will prefer e.g. "2.0" over "2.0rc1" 220 | if ref.startswith(tag_prefix): 221 | r = ref[len(tag_prefix) :] 222 | # Filter out refs that exactly match prefix or that don't start 223 | # with a number once the prefix is stripped (mostly a concern 224 | # when prefix is '') 225 | if not re.match(r"\d", r): 226 | continue 227 | if verbose: 228 | print("picking %s" % r) 229 | return { 230 | "version": r, 231 | "full-revisionid": keywords["full"].strip(), 232 | "dirty": False, 233 | "error": None, 234 | "date": date, 235 | } 236 | # no suitable tags, so version is "0+unknown", but full hex is still there 237 | if verbose: 238 | print("no suitable tags, using unknown + full revision id") 239 | return { 240 | "version": "0+unknown", 241 | "full-revisionid": keywords["full"].strip(), 242 | "dirty": False, 243 | "error": "no suitable tags", 244 | "date": None, 245 | } 246 | 247 | 248 | @register_vcs_handler("git", "pieces_from_vcs") 249 | def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): 250 | """Get version from 'git describe' in the root of the source tree. 251 | 252 | This only gets called if the git-archive 'subst' keywords were *not* 253 | expanded, and _version.py hasn't already been rewritten with a short 254 | version string, meaning we're inside a checked out source tree. 255 | """ 256 | GITS = ["git"] 257 | if sys.platform == "win32": 258 | GITS = ["git.cmd", "git.exe"] 259 | 260 | # GIT_DIR can interfere with correct operation of Versioneer. 261 | # It may be intended to be passed to the Versioneer-versioned project, 262 | # but that should not change where we get our version from. 263 | env = os.environ.copy() 264 | env.pop("GIT_DIR", None) 265 | runner = functools.partial(runner, env=env) 266 | 267 | _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) 268 | if rc != 0: 269 | if verbose: 270 | print("Directory %s not under git control" % root) 271 | raise NotThisMethod("'git rev-parse --git-dir' returned error") 272 | 273 | # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] 274 | # if there isn't one, this yields HEX[-dirty] (no NUM) 275 | describe_out, rc = runner( 276 | GITS, 277 | [ 278 | "describe", 279 | "--tags", 280 | "--dirty", 281 | "--always", 282 | "--long", 283 | "--match", 284 | f"{tag_prefix}[[:digit:]]*", 285 | ], 286 | cwd=root, 287 | ) 288 | # --long was added in git-1.5.5 289 | if describe_out is None: 290 | raise NotThisMethod("'git describe' failed") 291 | describe_out = describe_out.strip() 292 | full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) 293 | if full_out is None: 294 | raise NotThisMethod("'git rev-parse' failed") 295 | full_out = full_out.strip() 296 | 297 | pieces = {} 298 | pieces["long"] = full_out 299 | pieces["short"] = full_out[:7] # maybe improved later 300 | pieces["error"] = None 301 | 302 | branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) 303 | # --abbrev-ref was added in git-1.6.3 304 | if rc != 0 or branch_name is None: 305 | raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") 306 | branch_name = branch_name.strip() 307 | 308 | if branch_name == "HEAD": 309 | # If we aren't exactly on a branch, pick a branch which represents 310 | # the current commit. If all else fails, we are on a branchless 311 | # commit. 312 | branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) 313 | # --contains was added in git-1.5.4 314 | if rc != 0 or branches is None: 315 | raise NotThisMethod("'git branch --contains' returned error") 316 | branches = branches.split("\n") 317 | 318 | # Remove the first line if we're running detached 319 | if "(" in branches[0]: 320 | branches.pop(0) 321 | 322 | # Strip off the leading "* " from the list of branches. 323 | branches = [branch[2:] for branch in branches] 324 | if "master" in branches: 325 | branch_name = "master" 326 | elif not branches: 327 | branch_name = None 328 | else: 329 | # Pick the first branch that is returned. Good or bad. 330 | branch_name = branches[0] 331 | 332 | pieces["branch"] = branch_name 333 | 334 | # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] 335 | # TAG might have hyphens. 336 | git_describe = describe_out 337 | 338 | # look for -dirty suffix 339 | dirty = git_describe.endswith("-dirty") 340 | pieces["dirty"] = dirty 341 | if dirty: 342 | git_describe = git_describe[: git_describe.rindex("-dirty")] 343 | 344 | # now we have TAG-NUM-gHEX or HEX 345 | 346 | if "-" in git_describe: 347 | # TAG-NUM-gHEX 348 | mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) 349 | if not mo: 350 | # unparsable. Maybe git-describe is misbehaving? 351 | pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out 352 | return pieces 353 | 354 | # tag 355 | full_tag = mo.group(1) 356 | if not full_tag.startswith(tag_prefix): 357 | if verbose: 358 | fmt = "tag '%s' doesn't start with prefix '%s'" 359 | print(fmt % (full_tag, tag_prefix)) 360 | pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( 361 | full_tag, 362 | tag_prefix, 363 | ) 364 | return pieces 365 | pieces["closest-tag"] = full_tag[len(tag_prefix) :] 366 | 367 | # distance: number of commits since tag 368 | pieces["distance"] = int(mo.group(2)) 369 | 370 | # commit: short hex revision ID 371 | pieces["short"] = mo.group(3) 372 | 373 | else: 374 | # HEX: no tags 375 | pieces["closest-tag"] = None 376 | out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) 377 | pieces["distance"] = len(out.split()) # total number of commits 378 | 379 | # commit date: see ISO-8601 comment in git_versions_from_keywords() 380 | date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() 381 | # Use only the last line. Previous lines may contain GPG signature 382 | # information. 383 | date = date.splitlines()[-1] 384 | pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) 385 | 386 | return pieces 387 | 388 | 389 | def plus_or_dot(pieces): 390 | """Return a + if we don't already have one, else return a .""" 391 | if "+" in pieces.get("closest-tag", ""): 392 | return "." 393 | return "+" 394 | 395 | 396 | def render_pep440(pieces): 397 | """Build up version string, with post-release "local version identifier". 398 | 399 | Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you 400 | get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty 401 | 402 | Exceptions: 403 | 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] 404 | """ 405 | if pieces["closest-tag"]: 406 | rendered = pieces["closest-tag"] 407 | if pieces["distance"] or pieces["dirty"]: 408 | rendered += plus_or_dot(pieces) 409 | rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) 410 | if pieces["dirty"]: 411 | rendered += ".dirty" 412 | else: 413 | # exception #1 414 | rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) 415 | if pieces["dirty"]: 416 | rendered += ".dirty" 417 | return rendered 418 | 419 | 420 | def render_pep440_branch(pieces): 421 | """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . 422 | 423 | The ".dev0" means not master branch. Note that .dev0 sorts backwards 424 | (a feature branch will appear "older" than the master branch). 425 | 426 | Exceptions: 427 | 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] 428 | """ 429 | if pieces["closest-tag"]: 430 | rendered = pieces["closest-tag"] 431 | if pieces["distance"] or pieces["dirty"]: 432 | if pieces["branch"] != "master": 433 | rendered += ".dev0" 434 | rendered += plus_or_dot(pieces) 435 | rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) 436 | if pieces["dirty"]: 437 | rendered += ".dirty" 438 | else: 439 | # exception #1 440 | rendered = "0" 441 | if pieces["branch"] != "master": 442 | rendered += ".dev0" 443 | rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) 444 | if pieces["dirty"]: 445 | rendered += ".dirty" 446 | return rendered 447 | 448 | 449 | def pep440_split_post(ver): 450 | """Split pep440 version string at the post-release segment. 451 | 452 | Returns the release segments before the post-release and the 453 | post-release version number (or -1 if no post-release segment is present). 454 | """ 455 | vc = str.split(ver, ".post") 456 | return vc[0], int(vc[1] or 0) if len(vc) == 2 else None 457 | 458 | 459 | def render_pep440_pre(pieces): 460 | """TAG[.postN.devDISTANCE] -- No -dirty. 461 | 462 | Exceptions: 463 | 1: no tags. 0.post0.devDISTANCE 464 | """ 465 | if pieces["closest-tag"]: 466 | if pieces["distance"]: 467 | # update the post release segment 468 | tag_version, post_version = pep440_split_post(pieces["closest-tag"]) 469 | rendered = tag_version 470 | if post_version is not None: 471 | rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) 472 | else: 473 | rendered += ".post0.dev%d" % (pieces["distance"]) 474 | else: 475 | # no commits, use the tag as the version 476 | rendered = pieces["closest-tag"] 477 | else: 478 | # exception #1 479 | rendered = "0.post0.dev%d" % pieces["distance"] 480 | return rendered 481 | 482 | 483 | def render_pep440_post(pieces): 484 | """TAG[.postDISTANCE[.dev0]+gHEX] . 485 | 486 | The ".dev0" means dirty. Note that .dev0 sorts backwards 487 | (a dirty tree will appear "older" than the corresponding clean one), 488 | but you shouldn't be releasing software with -dirty anyways. 489 | 490 | Exceptions: 491 | 1: no tags. 0.postDISTANCE[.dev0] 492 | """ 493 | if pieces["closest-tag"]: 494 | rendered = pieces["closest-tag"] 495 | if pieces["distance"] or pieces["dirty"]: 496 | rendered += ".post%d" % pieces["distance"] 497 | if pieces["dirty"]: 498 | rendered += ".dev0" 499 | rendered += plus_or_dot(pieces) 500 | rendered += "g%s" % pieces["short"] 501 | else: 502 | # exception #1 503 | rendered = "0.post%d" % pieces["distance"] 504 | if pieces["dirty"]: 505 | rendered += ".dev0" 506 | rendered += "+g%s" % pieces["short"] 507 | return rendered 508 | 509 | 510 | def render_pep440_post_branch(pieces): 511 | """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . 512 | 513 | The ".dev0" means not master branch. 514 | 515 | Exceptions: 516 | 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] 517 | """ 518 | if pieces["closest-tag"]: 519 | rendered = pieces["closest-tag"] 520 | if pieces["distance"] or pieces["dirty"]: 521 | rendered += ".post%d" % pieces["distance"] 522 | if pieces["branch"] != "master": 523 | rendered += ".dev0" 524 | rendered += plus_or_dot(pieces) 525 | rendered += "g%s" % pieces["short"] 526 | if pieces["dirty"]: 527 | rendered += ".dirty" 528 | else: 529 | # exception #1 530 | rendered = "0.post%d" % pieces["distance"] 531 | if pieces["branch"] != "master": 532 | rendered += ".dev0" 533 | rendered += "+g%s" % pieces["short"] 534 | if pieces["dirty"]: 535 | rendered += ".dirty" 536 | return rendered 537 | 538 | 539 | def render_pep440_old(pieces): 540 | """TAG[.postDISTANCE[.dev0]] . 541 | 542 | The ".dev0" means dirty. 543 | 544 | Exceptions: 545 | 1: no tags. 0.postDISTANCE[.dev0] 546 | """ 547 | if pieces["closest-tag"]: 548 | rendered = pieces["closest-tag"] 549 | if pieces["distance"] or pieces["dirty"]: 550 | rendered += ".post%d" % pieces["distance"] 551 | if pieces["dirty"]: 552 | rendered += ".dev0" 553 | else: 554 | # exception #1 555 | rendered = "0.post%d" % pieces["distance"] 556 | if pieces["dirty"]: 557 | rendered += ".dev0" 558 | return rendered 559 | 560 | 561 | def render_git_describe(pieces): 562 | """TAG[-DISTANCE-gHEX][-dirty]. 563 | 564 | Like 'git describe --tags --dirty --always'. 565 | 566 | Exceptions: 567 | 1: no tags. HEX[-dirty] (note: no 'g' prefix) 568 | """ 569 | if pieces["closest-tag"]: 570 | rendered = pieces["closest-tag"] 571 | if pieces["distance"]: 572 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 573 | else: 574 | # exception #1 575 | rendered = pieces["short"] 576 | if pieces["dirty"]: 577 | rendered += "-dirty" 578 | return rendered 579 | 580 | 581 | def render_git_describe_long(pieces): 582 | """TAG-DISTANCE-gHEX[-dirty]. 583 | 584 | Like 'git describe --tags --dirty --always -long'. 585 | The distance/hash is unconditional. 586 | 587 | Exceptions: 588 | 1: no tags. HEX[-dirty] (note: no 'g' prefix) 589 | """ 590 | if pieces["closest-tag"]: 591 | rendered = pieces["closest-tag"] 592 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 593 | else: 594 | # exception #1 595 | rendered = pieces["short"] 596 | if pieces["dirty"]: 597 | rendered += "-dirty" 598 | return rendered 599 | 600 | 601 | def render(pieces, style): 602 | """Render the given version pieces into the requested style.""" 603 | if pieces["error"]: 604 | return { 605 | "version": "unknown", 606 | "full-revisionid": pieces.get("long"), 607 | "dirty": None, 608 | "error": pieces["error"], 609 | "date": None, 610 | } 611 | 612 | if not style or style == "default": 613 | style = "pep440" # the default 614 | 615 | if style == "pep440": 616 | rendered = render_pep440(pieces) 617 | elif style == "pep440-branch": 618 | rendered = render_pep440_branch(pieces) 619 | elif style == "pep440-pre": 620 | rendered = render_pep440_pre(pieces) 621 | elif style == "pep440-post": 622 | rendered = render_pep440_post(pieces) 623 | elif style == "pep440-post-branch": 624 | rendered = render_pep440_post_branch(pieces) 625 | elif style == "pep440-old": 626 | rendered = render_pep440_old(pieces) 627 | elif style == "git-describe": 628 | rendered = render_git_describe(pieces) 629 | elif style == "git-describe-long": 630 | rendered = render_git_describe_long(pieces) 631 | else: 632 | raise ValueError("unknown style '%s'" % style) 633 | 634 | return { 635 | "version": rendered, 636 | "full-revisionid": pieces["long"], 637 | "dirty": pieces["dirty"], 638 | "error": None, 639 | "date": pieces.get("date"), 640 | } 641 | 642 | 643 | def get_versions(): 644 | """Get version information or return default if unable to do so.""" 645 | # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have 646 | # __file__, we can work backwards from there to the root. Some 647 | # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which 648 | # case we can only use expanded keywords. 649 | 650 | cfg = get_config() 651 | verbose = cfg.verbose 652 | 653 | try: 654 | return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) 655 | except NotThisMethod: 656 | pass 657 | 658 | try: 659 | root = os.path.realpath(__file__) 660 | # versionfile_source is the relative path from the top of the source 661 | # tree (where the .git directory might live) to this file. Invert 662 | # this to find the root from __file__. 663 | for _ in cfg.versionfile_source.split("/"): 664 | root = os.path.dirname(root) 665 | except NameError: 666 | return { 667 | "version": "0+unknown", 668 | "full-revisionid": None, 669 | "dirty": None, 670 | "error": "unable to find root of source tree", 671 | "date": None, 672 | } 673 | 674 | try: 675 | pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) 676 | return render(pieces, cfg.style) 677 | except NotThisMethod: 678 | pass 679 | 680 | try: 681 | if cfg.parentdir_prefix: 682 | return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) 683 | except NotThisMethod: 684 | pass 685 | 686 | return { 687 | "version": "0+unknown", 688 | "full-revisionid": None, 689 | "dirty": None, 690 | "error": "unable to compute version", 691 | "date": None, 692 | } 693 | -------------------------------------------------------------------------------- /ibis_datasette/core.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import urllib.parse 3 | import contextvars 4 | import contextlib 5 | from urllib.parse import urlencode 6 | 7 | import httpx 8 | import sqlalchemy as sa 9 | import sqlalchemy.engine.reflection 10 | import sqlalchemy.dialects.sqlite.base 11 | from ibis.backends.base.sql.alchemy import BaseAlchemyBackend 12 | from ibis.backends.sqlite.compiler import SQLiteCompiler 13 | 14 | 15 | _cacheable = contextvars.ContextVar("cacheable", default=False) 16 | 17 | 18 | @contextlib.contextmanager 19 | def cacheable(): 20 | """A contextmanager for marking a request as cacheable""" 21 | t = _cacheable.set(True) 22 | try: 23 | yield 24 | finally: 25 | _cacheable.reset(t) 26 | 27 | 28 | class _Client: 29 | def __init__(self, client, base_url): 30 | self._client = client 31 | self._base_url = base_url 32 | 33 | def _get(self, suffix): 34 | url = self._base_url + suffix 35 | resp = self._client.get(url) 36 | try: 37 | resp.raise_for_status() 38 | except httpx.HTTPStatusError as exc: 39 | if exc.response.status_code == 404: 40 | raise ValueError(f"{url!r} is not a valid datasette URL") from None 41 | elif exc.response.status_code == 400: 42 | json = exc.response.json() 43 | raise ValueError(json["error"]) from None 44 | raise 45 | return resp 46 | 47 | @functools.lru_cache(32) 48 | def _cached_get(self, url): 49 | return self._get(url) 50 | 51 | def get(self, url): 52 | if _cacheable.get(): 53 | return self._cached_get(url) 54 | return self._get(url) 55 | 56 | 57 | class Cursor: 58 | def __init__(self, conn): 59 | self._conn = conn 60 | self._rows = None 61 | self._description = None 62 | self._next = None 63 | 64 | @property 65 | def rowcount(self): 66 | return -1 67 | 68 | @property 69 | def description(self): 70 | return self._description 71 | 72 | def close(self): 73 | pass 74 | 75 | def _do_query(self, query_string): 76 | self._description = None 77 | self._next = None 78 | self._rows = None 79 | 80 | json = self._conn._get(query_string).json() 81 | 82 | self._description = [ 83 | (col, None, None, None, None, None, None) for col in json["columns"] 84 | ] 85 | self._next = json.get("next", None) 86 | self._rows = iter(json.get("rows", [])) 87 | 88 | def _next_row(self): 89 | if self._rows is not None: 90 | try: 91 | return next(self._rows) 92 | except StopIteration: 93 | self._rows = None 94 | 95 | if self._next is None: 96 | return None 97 | self._do_query(f"?_next={self._next}") 98 | if self._rows is not None: 99 | try: 100 | return next(self._rows) 101 | except StopIteration: 102 | self._rows = None 103 | 104 | def execute(self, statement, parameters=None): 105 | # Skip pragmas 106 | if statement.lstrip().startswith("PRAGMA"): 107 | raise NotImplementedError("PRAGMA operations aren't supported") 108 | 109 | query = {} 110 | 111 | if parameters: 112 | if isinstance(parameters, tuple): 113 | raise NotImplementedError( 114 | "qmark (?) style parametrized queries are not supported" 115 | ) 116 | else: 117 | # XXX: Traverse in reverse order to ensure replacements don't 118 | # ever conflict (if we replace param_2 before param_20, we can 119 | # end up accidentally overwriting param_20). All of this is bad 120 | # and hacky. 121 | for k, v in sorted(parameters.items(), reverse=True): 122 | if isinstance(v, bool): 123 | v = int(v) 124 | if isinstance(v, (int, float)): 125 | # XXX: datasette doesn't handle numeric parameters. This 126 | # looks far worse than it actually is - the 127 | # parameter names are auto-generated by sqlalchemy, and 128 | # are unlikely to collide with any embedded strings. 129 | # Further, since we're only swapping in numeric values 130 | # SQL injection attacks shouldn't be possible. Still, 131 | # this is gross - it'd be good to improve it. 132 | statement = statement.replace(f":{k}", str(v)) 133 | else: 134 | query[k] = v 135 | 136 | query["sql"] = statement 137 | self._do_query(f"?{urlencode(query)}") 138 | 139 | def executemany(self, statement, parameters=None): 140 | raise NotImplementedError( 141 | "executemany shouldn't be used for SELECT statements (which are the " 142 | "only thing datasette supports). This operation is not implemented." 143 | ) 144 | 145 | def fetchone(self): 146 | return self._next_row() 147 | 148 | def fetchmany(self, size=None): 149 | out = [] 150 | if size: 151 | for _ in range(size): 152 | if row := self._next_row(): 153 | out.append(row) 154 | return out 155 | return self.fetchall() 156 | 157 | def fetchall(self): 158 | out = [] 159 | while row := self._next_row(): 160 | out.append(row) 161 | return out 162 | 163 | 164 | class Connection: 165 | def __init__(self, client, base_url): 166 | self._client = _Client(client, base_url) 167 | 168 | def _get(self, suffix): 169 | return self._client.get(suffix) 170 | 171 | def __enter__(self): 172 | return self 173 | 174 | def __exit__(self, *args, **kwargs): 175 | self.close() 176 | 177 | def cursor(self): 178 | return Cursor(self) 179 | 180 | def commit(self): 181 | pass 182 | 183 | def close(self): 184 | pass 185 | 186 | 187 | class DBAPI: 188 | apilevel = "2.0" 189 | threadsafety = 2 190 | paramstyle = "named" 191 | sqlite_version_info = (3, 39, 0) 192 | 193 | Error = RuntimeError 194 | 195 | 196 | class IbisDatasetteDialect(sa.dialects.sqlite.base.SQLiteDialect): 197 | name = "datasette" 198 | driver = "ibis_datasette" 199 | supports_statement_cache = True 200 | 201 | @functools.cached_property 202 | def httpx_client(self): 203 | return httpx.Client(follow_redirects=True) 204 | 205 | def connect(self, *args, **kwargs): 206 | return Connection(self.httpx_client, kwargs["url"]) 207 | 208 | @classmethod 209 | def get_pool_class(cls, url): 210 | return sa.pool.SingletonThreadPool 211 | 212 | @staticmethod 213 | def dbapi(): 214 | return DBAPI() 215 | 216 | def get_isolation_level(self, dbapi_conn): 217 | return "SERIALIZABLE" 218 | 219 | @sa.engine.reflection.cache 220 | def has_table(self, connection, table_name, schema=None, **kw): 221 | self._ensure_has_table_connection(connection) 222 | 223 | info = self._get_table_pragma( 224 | connection, "table_xinfo", table_name, schema=schema 225 | ) 226 | return bool(info) 227 | 228 | @sa.engine.reflection.cache 229 | def _get_table_sql(self, connection, table_name, schema=None, **kw): 230 | qtable = self.identifier_preparer.quote_identifier(table_name) 231 | s = ( 232 | f"SELECT sql FROM sqlite_master WHERE name = {qtable} " 233 | f"AND type in ('table', 'view')" 234 | ) 235 | with cacheable(): 236 | value = connection.exec_driver_sql(s).scalar() 237 | if value is None and not self._is_sys_table(table_name): 238 | raise sa.exc.NoSuchTableError(table_name) 239 | return value 240 | 241 | def _get_table_pragma(self, connection, pragma, table_name, schema=None): 242 | qtable = self.identifier_preparer.quote_identifier(table_name) 243 | s = f"SELECT * FROM pragma_{pragma}({qtable})" 244 | with cacheable(): 245 | return connection.exec_driver_sql(s).fetchall() 246 | 247 | def _get_server_version_info(self, connection): 248 | # XXX: We can't know the sqlite version on the remote, assume it's 249 | # recent enough to use table_xinfo 250 | return (3, 39, 0) 251 | 252 | def do_rollback(self, connection): 253 | pass # no-op 254 | 255 | def do_begin(self, connection): 256 | pass # no-op 257 | 258 | 259 | sa.dialects.registry.register( 260 | "ibisdatasette", "ibis_datasette.core", "IbisDatasetteDialect" 261 | ) 262 | 263 | 264 | class Backend(BaseAlchemyBackend): 265 | name = "sqlite" 266 | compiler = SQLiteCompiler 267 | 268 | def __getstate__(self): 269 | r = super().__getstate__() 270 | r.update( 271 | dict( 272 | compiler=self.compiler, 273 | database_name=self.database_name, 274 | _con=None, # clear connection on copy() 275 | _meta=None, 276 | ) 277 | ) 278 | return r 279 | 280 | def do_connect(self, url): 281 | parsed = urllib.parse.urlparse(url) 282 | if not parsed.path: 283 | raise ValueError( 284 | f"`connect` expects a datasette URL including the path for a " 285 | f"specific database, got {url!r}" 286 | ) 287 | if not url.endswith(".json"): 288 | url += ".json" 289 | 290 | query = urllib.parse.urlencode({"url": url}) 291 | engine = sa.create_engine(url=f"ibisdatasette://?{query}") 292 | 293 | with engine.dialect.connect(url=url) as con: 294 | resp = con._get("") 295 | json = resp.json() 296 | 297 | if not json.get("allow_execute_sql", False): 298 | raise ValueError( 299 | "This datasette instance disallows custom SQL queries; " 300 | "ibis-datasette cannot query it." 301 | ) 302 | 303 | self.database_name = "main" 304 | super().do_connect(engine) 305 | self._meta = sa.MetaData(bind=self.con) 306 | 307 | def _get_sqla_table(self, name, schema=None, autoload=True): 308 | return sa.Table( 309 | name, 310 | self.meta, 311 | schema=schema or self.current_database, 312 | autoload=autoload, 313 | ) 314 | 315 | def list_tables(self, like=None, database=None): 316 | """List the tables in the database. 317 | 318 | Parameters 319 | ---------- 320 | like 321 | A pattern to use for listing tables. 322 | 323 | """ 324 | with cacheable(): 325 | return super().list_tables(like=like, database=database) 326 | 327 | def table(self, name): 328 | """Create a table expression from a table in the SQLite database. 329 | 330 | Parameters 331 | ---------- 332 | name 333 | Table name 334 | 335 | Returns 336 | ------- 337 | Table 338 | Table expression 339 | """ 340 | alch_table = self._get_sqla_table(name) 341 | node = self.table_class(source=self, sqla_table=alch_table) 342 | return self.table_expr_class(node) 343 | 344 | def _table_from_schema(self, name, schema, database=None): 345 | columns = self._columns_from_schema(name, schema) 346 | return sa.Table(name, self.meta, schema=database, *columns) 347 | 348 | @property 349 | def _current_schema(self): 350 | return self.current_database 351 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = 3 | __init__.py, 4 | _version.py, 5 | versioneer.py 6 | ignore = 7 | E721, # Comparing types instead of isinstance 8 | E741, # Ambiguous variable names 9 | W503, # line break before binary operator 10 | W504, # line break after binary operator 11 | max-line-length = 100 12 | 13 | [versioneer] 14 | VCS = git 15 | style = pep440 16 | versionfile_source = ibis_datasette/_version.py 17 | versionfile_build = ibis_datasette/_version.py 18 | tag_prefix = 19 | parentdir_prefix = ibis-datasette- 20 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import versioneer 3 | 4 | from setuptools import setup 5 | 6 | 7 | install_requires = ["httpx", "ibis-framework", "sqlalchemy"] 8 | 9 | 10 | setup( 11 | name="ibis-datasette", 12 | version=versioneer.get_version(), 13 | cmdclass=versioneer.get_cmdclass(), 14 | description="An ibis backend for querying datasette", 15 | long_description=( 16 | open("README.rst", encoding="utf-8").read() 17 | if os.path.exists("README.rst") 18 | else "" 19 | ), 20 | maintainer="Jim Crist-Harif", 21 | maintainer_email="jcristharif@gmail.com", 22 | url="https://github.com/jcrist/ibis-datasette", 23 | project_urls={ 24 | "Source": "https://github.com/jcrist/ibis-datasette/", 25 | "Issue Tracker": "https://github.com/jcrist/ibis-datasette/issues", 26 | }, 27 | keywords="ibis datasette pandas sqlite", 28 | classifiers=[ 29 | "License :: OSI Approved :: BSD License", 30 | "Programming Language :: Python :: 3.8", 31 | "Programming Language :: Python :: 3.9", 32 | "Programming Language :: Python :: 3.10", 33 | ], 34 | license="BSD", 35 | packages=["ibis_datasette"], 36 | python_requires=">=3.8", 37 | install_requires=install_requires, 38 | entry_points={"ibis.backends": ["datasette = ibis_datasette"]}, 39 | zip_safe=False, 40 | ) 41 | -------------------------------------------------------------------------------- /tests/test_core.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import operator 3 | import random 4 | import sqlite3 5 | import string 6 | import subprocess 7 | import time 8 | 9 | import pytest 10 | import ibis 11 | from ibis import _ 12 | 13 | 14 | def randstr(): 15 | length = random.randint(4, 30) 16 | return "".join(random.choices(string.ascii_letters, k=length)) 17 | 18 | 19 | NAMES = [ 20 | "alice", 21 | "brad", 22 | "caroline", 23 | "doug", 24 | "emily", 25 | "frank", 26 | "gwen", 27 | "harold", 28 | "isabel", 29 | "john", 30 | "katrina", 31 | "lou", 32 | "manny", 33 | "nora", 34 | "oren", 35 | "patty", 36 | ] 37 | 38 | 39 | def randname(): 40 | return random.choice(NAMES) 41 | 42 | 43 | @pytest.fixture(scope="session") 44 | def database(tmp_path_factory): 45 | path = str(tmp_path_factory.mktemp("databases") / "test.db") 46 | con = sqlite3.connect(path) 47 | con.execute("CREATE TABLE table1 (col1 text, col2 text, col3 int, col4 real)") 48 | con.execute("CREATE TABLE table2 (x text, y int)") 49 | 50 | with con: 51 | con.executemany( 52 | "INSERT INTO table1 VALUES (?, ?, ?, ?)", 53 | [ 54 | (randstr(), randname(), random.randint(0, 100), random.random()) 55 | for _ in range(5000) 56 | ], 57 | ) 58 | 59 | con.executemany( 60 | "INSERT INTO table2 VALUES (?, ?)", 61 | [(randname(), random.randint(0, 20)) for _ in range(3000)], 62 | ) 63 | 64 | con.close() 65 | return path 66 | 67 | 68 | @pytest.fixture(scope="session") 69 | def datasette(database): 70 | ds_proc = subprocess.Popen( 71 | ["datasette", "serve", "-p", "8041", database], 72 | stdout=subprocess.PIPE, 73 | stderr=subprocess.STDOUT, 74 | ) 75 | # Give the server time to start 76 | time.sleep(1.5) 77 | # Check it started successfully 78 | assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8") 79 | yield "http://127.0.0.1:8041" 80 | # Shut it down at the end of the pytest session 81 | ds_proc.terminate() 82 | 83 | 84 | @pytest.fixture 85 | def url(datasette): 86 | return datasette + "/test" 87 | 88 | 89 | def test_connect_errors_not_database_url(datasette): 90 | with pytest.raises(ValueError, match="`connect` expects"): 91 | ibis.datasette.connect(datasette) 92 | 93 | 94 | def test_connect_errors_invalid_url(url): 95 | with pytest.raises(ValueError, match="is not a valid datasette URL"): 96 | ibis.datasette.connect(url + "/missing") 97 | 98 | 99 | def test_list_tables(url): 100 | con = ibis.datasette.connect(url) 101 | tables = sorted(con.list_tables()) 102 | assert tables == ["table1", "table2"] 103 | 104 | 105 | def test_access_table(url): 106 | con = ibis.datasette.connect(url) 107 | t1 = con.tables.table1 108 | assert t1.columns == ["col1", "col2", "col3", "col4"] 109 | schema = ibis.schema( 110 | [ 111 | ("col1", "string"), 112 | ("col2", "string"), 113 | ("col3", "int32"), 114 | ("col4", "float64"), 115 | ] 116 | ) 117 | assert t1.schema() == schema 118 | 119 | 120 | def test_table_does_not_exist(url): 121 | con = ibis.datasette.connect(url) 122 | with pytest.raises(AttributeError): 123 | con.tables.missing 124 | 125 | 126 | def test_table_query(url): 127 | con = ibis.datasette.connect(url) 128 | t1 = con.tables.table1 129 | query = t1.group_by(t1.col2).col3.mean() 130 | out = query.execute() 131 | assert len(out) == len(NAMES) 132 | 133 | 134 | def test_table_limit(url): 135 | con = ibis.datasette.connect(url) 136 | t1 = con.tables.table1 137 | out = t1.limit(123).execute() 138 | assert len(out) == 123 139 | 140 | 141 | @pytest.mark.parametrize("bound", [200, 200.5]) 142 | def test_numeric_query_parameters(url, bound): 143 | con = ibis.datasette.connect(url) 144 | t1 = con.tables.table1 145 | query = t1.group_by("col2").count().filter(lambda _: _["count"] > bound) 146 | out = query.execute() 147 | assert len(out) # out is empty if bound interpreted as string 148 | 149 | 150 | def test_many_query_parameters(url): 151 | con = ibis.datasette.connect(url) 152 | query = con.tables.table1.filter( 153 | functools.reduce(operator.or_, (_.col3 == x for x in range(30))) 154 | ).col3.nunique() 155 | out = query.execute() 156 | assert out == 30 157 | 158 | 159 | def test_string_query_parameters(url): 160 | con = ibis.datasette.connect(url) 161 | t1 = con.tables.table1 162 | assert t1.filter(t1.col2 == "alice").count().execute() 163 | 164 | 165 | def test_api_error_raised(url): 166 | con = ibis.datasette.connect(url) 167 | with pytest.raises(ValueError, match="missing"): 168 | con.raw_sql("SELECT * FROM missing") 169 | -------------------------------------------------------------------------------- /versioneer.py: -------------------------------------------------------------------------------- 1 | # Version: 0.23 2 | 3 | """The Versioneer - like a rocketeer, but for versions. 4 | 5 | The Versioneer 6 | ============== 7 | 8 | * like a rocketeer, but for versions! 9 | * https://github.com/python-versioneer/python-versioneer 10 | * Brian Warner 11 | * License: Public Domain (CC0-1.0) 12 | * Compatible with: Python 3.7, 3.8, 3.9, 3.10 and pypy3 13 | * [![Latest Version][pypi-image]][pypi-url] 14 | * [![Build Status][travis-image]][travis-url] 15 | 16 | This is a tool for managing a recorded version number in distutils/setuptools-based 17 | python projects. The goal is to remove the tedious and error-prone "update 18 | the embedded version string" step from your release process. Making a new 19 | release should be as easy as recording a new tag in your version-control 20 | system, and maybe making new tarballs. 21 | 22 | 23 | ## Quick Install 24 | 25 | * `pip install versioneer` to somewhere in your $PATH 26 | * add a `[versioneer]` section to your setup.cfg (see [Install](INSTALL.md)) 27 | * run `versioneer install` in your source tree, commit the results 28 | * Verify version information with `python setup.py version` 29 | 30 | ## Version Identifiers 31 | 32 | Source trees come from a variety of places: 33 | 34 | * a version-control system checkout (mostly used by developers) 35 | * a nightly tarball, produced by build automation 36 | * a snapshot tarball, produced by a web-based VCS browser, like github's 37 | "tarball from tag" feature 38 | * a release tarball, produced by "setup.py sdist", distributed through PyPI 39 | 40 | Within each source tree, the version identifier (either a string or a number, 41 | this tool is format-agnostic) can come from a variety of places: 42 | 43 | * ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows 44 | about recent "tags" and an absolute revision-id 45 | * the name of the directory into which the tarball was unpacked 46 | * an expanded VCS keyword ($Id$, etc) 47 | * a `_version.py` created by some earlier build step 48 | 49 | For released software, the version identifier is closely related to a VCS 50 | tag. Some projects use tag names that include more than just the version 51 | string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool 52 | needs to strip the tag prefix to extract the version identifier. For 53 | unreleased software (between tags), the version identifier should provide 54 | enough information to help developers recreate the same tree, while also 55 | giving them an idea of roughly how old the tree is (after version 1.2, before 56 | version 1.3). Many VCS systems can report a description that captures this, 57 | for example `git describe --tags --dirty --always` reports things like 58 | "0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the 59 | 0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has 60 | uncommitted changes). 61 | 62 | The version identifier is used for multiple purposes: 63 | 64 | * to allow the module to self-identify its version: `myproject.__version__` 65 | * to choose a name and prefix for a 'setup.py sdist' tarball 66 | 67 | ## Theory of Operation 68 | 69 | Versioneer works by adding a special `_version.py` file into your source 70 | tree, where your `__init__.py` can import it. This `_version.py` knows how to 71 | dynamically ask the VCS tool for version information at import time. 72 | 73 | `_version.py` also contains `$Revision$` markers, and the installation 74 | process marks `_version.py` to have this marker rewritten with a tag name 75 | during the `git archive` command. As a result, generated tarballs will 76 | contain enough information to get the proper version. 77 | 78 | To allow `setup.py` to compute a version too, a `versioneer.py` is added to 79 | the top level of your source tree, next to `setup.py` and the `setup.cfg` 80 | that configures it. This overrides several distutils/setuptools commands to 81 | compute the version when invoked, and changes `setup.py build` and `setup.py 82 | sdist` to replace `_version.py` with a small static file that contains just 83 | the generated version data. 84 | 85 | ## Installation 86 | 87 | See [INSTALL.md](./INSTALL.md) for detailed installation instructions. 88 | 89 | ## Version-String Flavors 90 | 91 | Code which uses Versioneer can learn about its version string at runtime by 92 | importing `_version` from your main `__init__.py` file and running the 93 | `get_versions()` function. From the "outside" (e.g. in `setup.py`), you can 94 | import the top-level `versioneer.py` and run `get_versions()`. 95 | 96 | Both functions return a dictionary with different flavors of version 97 | information: 98 | 99 | * `['version']`: A condensed version string, rendered using the selected 100 | style. This is the most commonly used value for the project's version 101 | string. The default "pep440" style yields strings like `0.11`, 102 | `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section 103 | below for alternative styles. 104 | 105 | * `['full-revisionid']`: detailed revision identifier. For Git, this is the 106 | full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". 107 | 108 | * `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the 109 | commit date in ISO 8601 format. This will be None if the date is not 110 | available. 111 | 112 | * `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that 113 | this is only accurate if run in a VCS checkout, otherwise it is likely to 114 | be False or None 115 | 116 | * `['error']`: if the version string could not be computed, this will be set 117 | to a string describing the problem, otherwise it will be None. It may be 118 | useful to throw an exception in setup.py if this is set, to avoid e.g. 119 | creating tarballs with a version string of "unknown". 120 | 121 | Some variants are more useful than others. Including `full-revisionid` in a 122 | bug report should allow developers to reconstruct the exact code being tested 123 | (or indicate the presence of local changes that should be shared with the 124 | developers). `version` is suitable for display in an "about" box or a CLI 125 | `--version` output: it can be easily compared against release notes and lists 126 | of bugs fixed in various releases. 127 | 128 | The installer adds the following text to your `__init__.py` to place a basic 129 | version in `YOURPROJECT.__version__`: 130 | 131 | from ._version import get_versions 132 | __version__ = get_versions()['version'] 133 | del get_versions 134 | 135 | ## Styles 136 | 137 | The setup.cfg `style=` configuration controls how the VCS information is 138 | rendered into a version string. 139 | 140 | The default style, "pep440", produces a PEP440-compliant string, equal to the 141 | un-prefixed tag name for actual releases, and containing an additional "local 142 | version" section with more detail for in-between builds. For Git, this is 143 | TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags 144 | --dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the 145 | tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and 146 | that this commit is two revisions ("+2") beyond the "0.11" tag. For released 147 | software (exactly equal to a known tag), the identifier will only contain the 148 | stripped tag, e.g. "0.11". 149 | 150 | Other styles are available. See [details.md](details.md) in the Versioneer 151 | source tree for descriptions. 152 | 153 | ## Debugging 154 | 155 | Versioneer tries to avoid fatal errors: if something goes wrong, it will tend 156 | to return a version of "0+unknown". To investigate the problem, run `setup.py 157 | version`, which will run the version-lookup code in a verbose mode, and will 158 | display the full contents of `get_versions()` (including the `error` string, 159 | which may help identify what went wrong). 160 | 161 | ## Known Limitations 162 | 163 | Some situations are known to cause problems for Versioneer. This details the 164 | most significant ones. More can be found on Github 165 | [issues page](https://github.com/python-versioneer/python-versioneer/issues). 166 | 167 | ### Subprojects 168 | 169 | Versioneer has limited support for source trees in which `setup.py` is not in 170 | the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are 171 | two common reasons why `setup.py` might not be in the root: 172 | 173 | * Source trees which contain multiple subprojects, such as 174 | [Buildbot](https://github.com/buildbot/buildbot), which contains both 175 | "master" and "slave" subprojects, each with their own `setup.py`, 176 | `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI 177 | distributions (and upload multiple independently-installable tarballs). 178 | * Source trees whose main purpose is to contain a C library, but which also 179 | provide bindings to Python (and perhaps other languages) in subdirectories. 180 | 181 | Versioneer will look for `.git` in parent directories, and most operations 182 | should get the right version string. However `pip` and `setuptools` have bugs 183 | and implementation details which frequently cause `pip install .` from a 184 | subproject directory to fail to find a correct version string (so it usually 185 | defaults to `0+unknown`). 186 | 187 | `pip install --editable .` should work correctly. `setup.py install` might 188 | work too. 189 | 190 | Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in 191 | some later version. 192 | 193 | [Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking 194 | this issue. The discussion in 195 | [PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the 196 | issue from the Versioneer side in more detail. 197 | [pip PR#3176](https://github.com/pypa/pip/pull/3176) and 198 | [pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve 199 | pip to let Versioneer work correctly. 200 | 201 | Versioneer-0.16 and earlier only looked for a `.git` directory next to the 202 | `setup.cfg`, so subprojects were completely unsupported with those releases. 203 | 204 | ### Editable installs with setuptools <= 18.5 205 | 206 | `setup.py develop` and `pip install --editable .` allow you to install a 207 | project into a virtualenv once, then continue editing the source code (and 208 | test) without re-installing after every change. 209 | 210 | "Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a 211 | convenient way to specify executable scripts that should be installed along 212 | with the python package. 213 | 214 | These both work as expected when using modern setuptools. When using 215 | setuptools-18.5 or earlier, however, certain operations will cause 216 | `pkg_resources.DistributionNotFound` errors when running the entrypoint 217 | script, which must be resolved by re-installing the package. This happens 218 | when the install happens with one version, then the egg_info data is 219 | regenerated while a different version is checked out. Many setup.py commands 220 | cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into 221 | a different virtualenv), so this can be surprising. 222 | 223 | [Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes 224 | this one, but upgrading to a newer version of setuptools should probably 225 | resolve it. 226 | 227 | 228 | ## Updating Versioneer 229 | 230 | To upgrade your project to a new release of Versioneer, do the following: 231 | 232 | * install the new Versioneer (`pip install -U versioneer` or equivalent) 233 | * edit `setup.cfg`, if necessary, to include any new configuration settings 234 | indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. 235 | * re-run `versioneer install` in your source tree, to replace 236 | `SRC/_version.py` 237 | * commit any changed files 238 | 239 | ## Future Directions 240 | 241 | This tool is designed to make it easily extended to other version-control 242 | systems: all VCS-specific components are in separate directories like 243 | src/git/ . The top-level `versioneer.py` script is assembled from these 244 | components by running make-versioneer.py . In the future, make-versioneer.py 245 | will take a VCS name as an argument, and will construct a version of 246 | `versioneer.py` that is specific to the given VCS. It might also take the 247 | configuration arguments that are currently provided manually during 248 | installation by editing setup.py . Alternatively, it might go the other 249 | direction and include code from all supported VCS systems, reducing the 250 | number of intermediate scripts. 251 | 252 | ## Similar projects 253 | 254 | * [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time 255 | dependency 256 | * [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of 257 | versioneer 258 | * [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based setuptools 259 | plugin 260 | 261 | ## License 262 | 263 | To make Versioneer easier to embed, all its code is dedicated to the public 264 | domain. The `_version.py` that it creates is also in the public domain. 265 | Specifically, both are released under the Creative Commons "Public Domain 266 | Dedication" license (CC0-1.0), as described in 267 | https://creativecommons.org/publicdomain/zero/1.0/ . 268 | 269 | [pypi-image]: https://img.shields.io/pypi/v/versioneer.svg 270 | [pypi-url]: https://pypi.python.org/pypi/versioneer/ 271 | [travis-image]: 272 | https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg 273 | [travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer 274 | 275 | """ 276 | # pylint:disable=invalid-name,import-outside-toplevel,missing-function-docstring 277 | # pylint:disable=missing-class-docstring,too-many-branches,too-many-statements 278 | # pylint:disable=raise-missing-from,too-many-lines,too-many-locals,import-error 279 | # pylint:disable=too-few-public-methods,redefined-outer-name,consider-using-with 280 | # pylint:disable=attribute-defined-outside-init,too-many-arguments 281 | 282 | import configparser 283 | import errno 284 | import json 285 | import os 286 | import re 287 | import subprocess 288 | import sys 289 | from typing import Callable, Dict 290 | import functools 291 | 292 | 293 | class VersioneerConfig: 294 | """Container for Versioneer configuration parameters.""" 295 | 296 | 297 | def get_root(): 298 | """Get the project root directory. 299 | 300 | We require that all commands are run from the project root, i.e. the 301 | directory that contains setup.py, setup.cfg, and versioneer.py . 302 | """ 303 | root = os.path.realpath(os.path.abspath(os.getcwd())) 304 | setup_py = os.path.join(root, "setup.py") 305 | versioneer_py = os.path.join(root, "versioneer.py") 306 | if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): 307 | # allow 'python path/to/setup.py COMMAND' 308 | root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) 309 | setup_py = os.path.join(root, "setup.py") 310 | versioneer_py = os.path.join(root, "versioneer.py") 311 | if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): 312 | err = ( 313 | "Versioneer was unable to run the project root directory. " 314 | "Versioneer requires setup.py to be executed from " 315 | "its immediate directory (like 'python setup.py COMMAND'), " 316 | "or in a way that lets it use sys.argv[0] to find the root " 317 | "(like 'python path/to/setup.py COMMAND')." 318 | ) 319 | raise VersioneerBadRootError(err) 320 | try: 321 | # Certain runtime workflows (setup.py install/develop in a setuptools 322 | # tree) execute all dependencies in a single python process, so 323 | # "versioneer" may be imported multiple times, and python's shared 324 | # module-import table will cache the first one. So we can't use 325 | # os.path.dirname(__file__), as that will find whichever 326 | # versioneer.py was first imported, even in later projects. 327 | my_path = os.path.realpath(os.path.abspath(__file__)) 328 | me_dir = os.path.normcase(os.path.splitext(my_path)[0]) 329 | vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) 330 | if me_dir != vsr_dir: 331 | print( 332 | "Warning: build in %s is using versioneer.py from %s" 333 | % (os.path.dirname(my_path), versioneer_py) 334 | ) 335 | except NameError: 336 | pass 337 | return root 338 | 339 | 340 | def get_config_from_root(root): 341 | """Read the project setup.cfg file to determine Versioneer config.""" 342 | # This might raise OSError (if setup.cfg is missing), or 343 | # configparser.NoSectionError (if it lacks a [versioneer] section), or 344 | # configparser.NoOptionError (if it lacks "VCS="). See the docstring at 345 | # the top of versioneer.py for instructions on writing your setup.cfg . 346 | setup_cfg = os.path.join(root, "setup.cfg") 347 | parser = configparser.ConfigParser() 348 | with open(setup_cfg, "r") as cfg_file: 349 | parser.read_file(cfg_file) 350 | VCS = parser.get("versioneer", "VCS") # mandatory 351 | 352 | # Dict-like interface for non-mandatory entries 353 | section = parser["versioneer"] 354 | 355 | cfg = VersioneerConfig() 356 | cfg.VCS = VCS 357 | cfg.style = section.get("style", "") 358 | cfg.versionfile_source = section.get("versionfile_source") 359 | cfg.versionfile_build = section.get("versionfile_build") 360 | cfg.tag_prefix = section.get("tag_prefix") 361 | if cfg.tag_prefix in ("''", '""', None): 362 | cfg.tag_prefix = "" 363 | cfg.parentdir_prefix = section.get("parentdir_prefix") 364 | cfg.verbose = section.get("verbose") 365 | return cfg 366 | 367 | 368 | class NotThisMethod(Exception): 369 | """Exception raised if a method is not valid for the current scenario.""" 370 | 371 | 372 | # these dictionaries contain VCS-specific tools 373 | LONG_VERSION_PY: Dict[str, str] = {} 374 | HANDLERS: Dict[str, Dict[str, Callable]] = {} 375 | 376 | 377 | def register_vcs_handler(vcs, method): # decorator 378 | """Create decorator to mark a method as the handler of a VCS.""" 379 | 380 | def decorate(f): 381 | """Store f in HANDLERS[vcs][method].""" 382 | HANDLERS.setdefault(vcs, {})[method] = f 383 | return f 384 | 385 | return decorate 386 | 387 | 388 | def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): 389 | """Call the given command(s).""" 390 | assert isinstance(commands, list) 391 | process = None 392 | 393 | popen_kwargs = {} 394 | if sys.platform == "win32": 395 | # This hides the console window if pythonw.exe is used 396 | startupinfo = subprocess.STARTUPINFO() 397 | startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW 398 | popen_kwargs["startupinfo"] = startupinfo 399 | 400 | for command in commands: 401 | try: 402 | dispcmd = str([command] + args) 403 | # remember shell=False, so use git.cmd on windows, not just git 404 | process = subprocess.Popen( 405 | [command] + args, 406 | cwd=cwd, 407 | env=env, 408 | stdout=subprocess.PIPE, 409 | stderr=(subprocess.PIPE if hide_stderr else None), 410 | **popen_kwargs, 411 | ) 412 | break 413 | except OSError: 414 | e = sys.exc_info()[1] 415 | if e.errno == errno.ENOENT: 416 | continue 417 | if verbose: 418 | print("unable to run %s" % dispcmd) 419 | print(e) 420 | return None, None 421 | else: 422 | if verbose: 423 | print("unable to find command, tried %s" % (commands,)) 424 | return None, None 425 | stdout = process.communicate()[0].strip().decode() 426 | if process.returncode != 0: 427 | if verbose: 428 | print("unable to run %s (error)" % dispcmd) 429 | print("stdout was %s" % stdout) 430 | return None, process.returncode 431 | return stdout, process.returncode 432 | 433 | 434 | LONG_VERSION_PY[ 435 | "git" 436 | ] = r''' 437 | # This file helps to compute a version number in source trees obtained from 438 | # git-archive tarball (such as those provided by githubs download-from-tag 439 | # feature). Distribution tarballs (built by setup.py sdist) and build 440 | # directories (produced by setup.py build) will contain a much shorter file 441 | # that just contains the computed version number. 442 | 443 | # This file is released into the public domain. Generated by 444 | # versioneer-0.23 (https://github.com/python-versioneer/python-versioneer) 445 | 446 | """Git implementation of _version.py.""" 447 | 448 | import errno 449 | import os 450 | import re 451 | import subprocess 452 | import sys 453 | from typing import Callable, Dict 454 | import functools 455 | 456 | 457 | def get_keywords(): 458 | """Get the keywords needed to look up the version information.""" 459 | # these strings will be replaced by git during git-archive. 460 | # setup.py/versioneer.py will grep for the variable names, so they must 461 | # each be defined on a line of their own. _version.py will just call 462 | # get_keywords(). 463 | git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" 464 | git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" 465 | git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" 466 | keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} 467 | return keywords 468 | 469 | 470 | class VersioneerConfig: 471 | """Container for Versioneer configuration parameters.""" 472 | 473 | 474 | def get_config(): 475 | """Create, populate and return the VersioneerConfig() object.""" 476 | # these strings are filled in when 'setup.py versioneer' creates 477 | # _version.py 478 | cfg = VersioneerConfig() 479 | cfg.VCS = "git" 480 | cfg.style = "%(STYLE)s" 481 | cfg.tag_prefix = "%(TAG_PREFIX)s" 482 | cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" 483 | cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" 484 | cfg.verbose = False 485 | return cfg 486 | 487 | 488 | class NotThisMethod(Exception): 489 | """Exception raised if a method is not valid for the current scenario.""" 490 | 491 | 492 | LONG_VERSION_PY: Dict[str, str] = {} 493 | HANDLERS: Dict[str, Dict[str, Callable]] = {} 494 | 495 | 496 | def register_vcs_handler(vcs, method): # decorator 497 | """Create decorator to mark a method as the handler of a VCS.""" 498 | def decorate(f): 499 | """Store f in HANDLERS[vcs][method].""" 500 | if vcs not in HANDLERS: 501 | HANDLERS[vcs] = {} 502 | HANDLERS[vcs][method] = f 503 | return f 504 | return decorate 505 | 506 | 507 | def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, 508 | env=None): 509 | """Call the given command(s).""" 510 | assert isinstance(commands, list) 511 | process = None 512 | 513 | popen_kwargs = {} 514 | if sys.platform == "win32": 515 | # This hides the console window if pythonw.exe is used 516 | startupinfo = subprocess.STARTUPINFO() 517 | startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW 518 | popen_kwargs["startupinfo"] = startupinfo 519 | 520 | for command in commands: 521 | try: 522 | dispcmd = str([command] + args) 523 | # remember shell=False, so use git.cmd on windows, not just git 524 | process = subprocess.Popen([command] + args, cwd=cwd, env=env, 525 | stdout=subprocess.PIPE, 526 | stderr=(subprocess.PIPE if hide_stderr 527 | else None), **popen_kwargs) 528 | break 529 | except OSError: 530 | e = sys.exc_info()[1] 531 | if e.errno == errno.ENOENT: 532 | continue 533 | if verbose: 534 | print("unable to run %%s" %% dispcmd) 535 | print(e) 536 | return None, None 537 | else: 538 | if verbose: 539 | print("unable to find command, tried %%s" %% (commands,)) 540 | return None, None 541 | stdout = process.communicate()[0].strip().decode() 542 | if process.returncode != 0: 543 | if verbose: 544 | print("unable to run %%s (error)" %% dispcmd) 545 | print("stdout was %%s" %% stdout) 546 | return None, process.returncode 547 | return stdout, process.returncode 548 | 549 | 550 | def versions_from_parentdir(parentdir_prefix, root, verbose): 551 | """Try to determine the version from the parent directory name. 552 | 553 | Source tarballs conventionally unpack into a directory that includes both 554 | the project name and a version string. We will also support searching up 555 | two directory levels for an appropriately named parent directory 556 | """ 557 | rootdirs = [] 558 | 559 | for _ in range(3): 560 | dirname = os.path.basename(root) 561 | if dirname.startswith(parentdir_prefix): 562 | return {"version": dirname[len(parentdir_prefix):], 563 | "full-revisionid": None, 564 | "dirty": False, "error": None, "date": None} 565 | rootdirs.append(root) 566 | root = os.path.dirname(root) # up a level 567 | 568 | if verbose: 569 | print("Tried directories %%s but none started with prefix %%s" %% 570 | (str(rootdirs), parentdir_prefix)) 571 | raise NotThisMethod("rootdir doesn't start with parentdir_prefix") 572 | 573 | 574 | @register_vcs_handler("git", "get_keywords") 575 | def git_get_keywords(versionfile_abs): 576 | """Extract version information from the given file.""" 577 | # the code embedded in _version.py can just fetch the value of these 578 | # keywords. When used from setup.py, we don't want to import _version.py, 579 | # so we do it with a regexp instead. This function is not used from 580 | # _version.py. 581 | keywords = {} 582 | try: 583 | with open(versionfile_abs, "r") as fobj: 584 | for line in fobj: 585 | if line.strip().startswith("git_refnames ="): 586 | mo = re.search(r'=\s*"(.*)"', line) 587 | if mo: 588 | keywords["refnames"] = mo.group(1) 589 | if line.strip().startswith("git_full ="): 590 | mo = re.search(r'=\s*"(.*)"', line) 591 | if mo: 592 | keywords["full"] = mo.group(1) 593 | if line.strip().startswith("git_date ="): 594 | mo = re.search(r'=\s*"(.*)"', line) 595 | if mo: 596 | keywords["date"] = mo.group(1) 597 | except OSError: 598 | pass 599 | return keywords 600 | 601 | 602 | @register_vcs_handler("git", "keywords") 603 | def git_versions_from_keywords(keywords, tag_prefix, verbose): 604 | """Get version information from git keywords.""" 605 | if "refnames" not in keywords: 606 | raise NotThisMethod("Short version file found") 607 | date = keywords.get("date") 608 | if date is not None: 609 | # Use only the last line. Previous lines may contain GPG signature 610 | # information. 611 | date = date.splitlines()[-1] 612 | 613 | # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant 614 | # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 615 | # -like" string, which we must then edit to make compliant), because 616 | # it's been around since git-1.5.3, and it's too difficult to 617 | # discover which version we're using, or to work around using an 618 | # older one. 619 | date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) 620 | refnames = keywords["refnames"].strip() 621 | if refnames.startswith("$Format"): 622 | if verbose: 623 | print("keywords are unexpanded, not using") 624 | raise NotThisMethod("unexpanded keywords, not a git-archive tarball") 625 | refs = {r.strip() for r in refnames.strip("()").split(",")} 626 | # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of 627 | # just "foo-1.0". If we see a "tag: " prefix, prefer those. 628 | TAG = "tag: " 629 | tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} 630 | if not tags: 631 | # Either we're using git < 1.8.3, or there really are no tags. We use 632 | # a heuristic: assume all version tags have a digit. The old git %%d 633 | # expansion behaves like git log --decorate=short and strips out the 634 | # refs/heads/ and refs/tags/ prefixes that would let us distinguish 635 | # between branches and tags. By ignoring refnames without digits, we 636 | # filter out many common branch names like "release" and 637 | # "stabilization", as well as "HEAD" and "master". 638 | tags = {r for r in refs if re.search(r'\d', r)} 639 | if verbose: 640 | print("discarding '%%s', no digits" %% ",".join(refs - tags)) 641 | if verbose: 642 | print("likely tags: %%s" %% ",".join(sorted(tags))) 643 | for ref in sorted(tags): 644 | # sorting will prefer e.g. "2.0" over "2.0rc1" 645 | if ref.startswith(tag_prefix): 646 | r = ref[len(tag_prefix):] 647 | # Filter out refs that exactly match prefix or that don't start 648 | # with a number once the prefix is stripped (mostly a concern 649 | # when prefix is '') 650 | if not re.match(r'\d', r): 651 | continue 652 | if verbose: 653 | print("picking %%s" %% r) 654 | return {"version": r, 655 | "full-revisionid": keywords["full"].strip(), 656 | "dirty": False, "error": None, 657 | "date": date} 658 | # no suitable tags, so version is "0+unknown", but full hex is still there 659 | if verbose: 660 | print("no suitable tags, using unknown + full revision id") 661 | return {"version": "0+unknown", 662 | "full-revisionid": keywords["full"].strip(), 663 | "dirty": False, "error": "no suitable tags", "date": None} 664 | 665 | 666 | @register_vcs_handler("git", "pieces_from_vcs") 667 | def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): 668 | """Get version from 'git describe' in the root of the source tree. 669 | 670 | This only gets called if the git-archive 'subst' keywords were *not* 671 | expanded, and _version.py hasn't already been rewritten with a short 672 | version string, meaning we're inside a checked out source tree. 673 | """ 674 | GITS = ["git"] 675 | if sys.platform == "win32": 676 | GITS = ["git.cmd", "git.exe"] 677 | 678 | # GIT_DIR can interfere with correct operation of Versioneer. 679 | # It may be intended to be passed to the Versioneer-versioned project, 680 | # but that should not change where we get our version from. 681 | env = os.environ.copy() 682 | env.pop("GIT_DIR", None) 683 | runner = functools.partial(runner, env=env) 684 | 685 | _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, 686 | hide_stderr=True) 687 | if rc != 0: 688 | if verbose: 689 | print("Directory %%s not under git control" %% root) 690 | raise NotThisMethod("'git rev-parse --git-dir' returned error") 691 | 692 | # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] 693 | # if there isn't one, this yields HEX[-dirty] (no NUM) 694 | describe_out, rc = runner(GITS, [ 695 | "describe", "--tags", "--dirty", "--always", "--long", 696 | "--match", f"{tag_prefix}[[:digit:]]*" 697 | ], cwd=root) 698 | # --long was added in git-1.5.5 699 | if describe_out is None: 700 | raise NotThisMethod("'git describe' failed") 701 | describe_out = describe_out.strip() 702 | full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) 703 | if full_out is None: 704 | raise NotThisMethod("'git rev-parse' failed") 705 | full_out = full_out.strip() 706 | 707 | pieces = {} 708 | pieces["long"] = full_out 709 | pieces["short"] = full_out[:7] # maybe improved later 710 | pieces["error"] = None 711 | 712 | branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], 713 | cwd=root) 714 | # --abbrev-ref was added in git-1.6.3 715 | if rc != 0 or branch_name is None: 716 | raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") 717 | branch_name = branch_name.strip() 718 | 719 | if branch_name == "HEAD": 720 | # If we aren't exactly on a branch, pick a branch which represents 721 | # the current commit. If all else fails, we are on a branchless 722 | # commit. 723 | branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) 724 | # --contains was added in git-1.5.4 725 | if rc != 0 or branches is None: 726 | raise NotThisMethod("'git branch --contains' returned error") 727 | branches = branches.split("\n") 728 | 729 | # Remove the first line if we're running detached 730 | if "(" in branches[0]: 731 | branches.pop(0) 732 | 733 | # Strip off the leading "* " from the list of branches. 734 | branches = [branch[2:] for branch in branches] 735 | if "master" in branches: 736 | branch_name = "master" 737 | elif not branches: 738 | branch_name = None 739 | else: 740 | # Pick the first branch that is returned. Good or bad. 741 | branch_name = branches[0] 742 | 743 | pieces["branch"] = branch_name 744 | 745 | # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] 746 | # TAG might have hyphens. 747 | git_describe = describe_out 748 | 749 | # look for -dirty suffix 750 | dirty = git_describe.endswith("-dirty") 751 | pieces["dirty"] = dirty 752 | if dirty: 753 | git_describe = git_describe[:git_describe.rindex("-dirty")] 754 | 755 | # now we have TAG-NUM-gHEX or HEX 756 | 757 | if "-" in git_describe: 758 | # TAG-NUM-gHEX 759 | mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) 760 | if not mo: 761 | # unparsable. Maybe git-describe is misbehaving? 762 | pieces["error"] = ("unable to parse git-describe output: '%%s'" 763 | %% describe_out) 764 | return pieces 765 | 766 | # tag 767 | full_tag = mo.group(1) 768 | if not full_tag.startswith(tag_prefix): 769 | if verbose: 770 | fmt = "tag '%%s' doesn't start with prefix '%%s'" 771 | print(fmt %% (full_tag, tag_prefix)) 772 | pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" 773 | %% (full_tag, tag_prefix)) 774 | return pieces 775 | pieces["closest-tag"] = full_tag[len(tag_prefix):] 776 | 777 | # distance: number of commits since tag 778 | pieces["distance"] = int(mo.group(2)) 779 | 780 | # commit: short hex revision ID 781 | pieces["short"] = mo.group(3) 782 | 783 | else: 784 | # HEX: no tags 785 | pieces["closest-tag"] = None 786 | out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) 787 | pieces["distance"] = len(out.split()) # total number of commits 788 | 789 | # commit date: see ISO-8601 comment in git_versions_from_keywords() 790 | date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], cwd=root)[0].strip() 791 | # Use only the last line. Previous lines may contain GPG signature 792 | # information. 793 | date = date.splitlines()[-1] 794 | pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) 795 | 796 | return pieces 797 | 798 | 799 | def plus_or_dot(pieces): 800 | """Return a + if we don't already have one, else return a .""" 801 | if "+" in pieces.get("closest-tag", ""): 802 | return "." 803 | return "+" 804 | 805 | 806 | def render_pep440(pieces): 807 | """Build up version string, with post-release "local version identifier". 808 | 809 | Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you 810 | get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty 811 | 812 | Exceptions: 813 | 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] 814 | """ 815 | if pieces["closest-tag"]: 816 | rendered = pieces["closest-tag"] 817 | if pieces["distance"] or pieces["dirty"]: 818 | rendered += plus_or_dot(pieces) 819 | rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) 820 | if pieces["dirty"]: 821 | rendered += ".dirty" 822 | else: 823 | # exception #1 824 | rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], 825 | pieces["short"]) 826 | if pieces["dirty"]: 827 | rendered += ".dirty" 828 | return rendered 829 | 830 | 831 | def render_pep440_branch(pieces): 832 | """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . 833 | 834 | The ".dev0" means not master branch. Note that .dev0 sorts backwards 835 | (a feature branch will appear "older" than the master branch). 836 | 837 | Exceptions: 838 | 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] 839 | """ 840 | if pieces["closest-tag"]: 841 | rendered = pieces["closest-tag"] 842 | if pieces["distance"] or pieces["dirty"]: 843 | if pieces["branch"] != "master": 844 | rendered += ".dev0" 845 | rendered += plus_or_dot(pieces) 846 | rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) 847 | if pieces["dirty"]: 848 | rendered += ".dirty" 849 | else: 850 | # exception #1 851 | rendered = "0" 852 | if pieces["branch"] != "master": 853 | rendered += ".dev0" 854 | rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], 855 | pieces["short"]) 856 | if pieces["dirty"]: 857 | rendered += ".dirty" 858 | return rendered 859 | 860 | 861 | def pep440_split_post(ver): 862 | """Split pep440 version string at the post-release segment. 863 | 864 | Returns the release segments before the post-release and the 865 | post-release version number (or -1 if no post-release segment is present). 866 | """ 867 | vc = str.split(ver, ".post") 868 | return vc[0], int(vc[1] or 0) if len(vc) == 2 else None 869 | 870 | 871 | def render_pep440_pre(pieces): 872 | """TAG[.postN.devDISTANCE] -- No -dirty. 873 | 874 | Exceptions: 875 | 1: no tags. 0.post0.devDISTANCE 876 | """ 877 | if pieces["closest-tag"]: 878 | if pieces["distance"]: 879 | # update the post release segment 880 | tag_version, post_version = pep440_split_post(pieces["closest-tag"]) 881 | rendered = tag_version 882 | if post_version is not None: 883 | rendered += ".post%%d.dev%%d" %% (post_version + 1, pieces["distance"]) 884 | else: 885 | rendered += ".post0.dev%%d" %% (pieces["distance"]) 886 | else: 887 | # no commits, use the tag as the version 888 | rendered = pieces["closest-tag"] 889 | else: 890 | # exception #1 891 | rendered = "0.post0.dev%%d" %% pieces["distance"] 892 | return rendered 893 | 894 | 895 | def render_pep440_post(pieces): 896 | """TAG[.postDISTANCE[.dev0]+gHEX] . 897 | 898 | The ".dev0" means dirty. Note that .dev0 sorts backwards 899 | (a dirty tree will appear "older" than the corresponding clean one), 900 | but you shouldn't be releasing software with -dirty anyways. 901 | 902 | Exceptions: 903 | 1: no tags. 0.postDISTANCE[.dev0] 904 | """ 905 | if pieces["closest-tag"]: 906 | rendered = pieces["closest-tag"] 907 | if pieces["distance"] or pieces["dirty"]: 908 | rendered += ".post%%d" %% pieces["distance"] 909 | if pieces["dirty"]: 910 | rendered += ".dev0" 911 | rendered += plus_or_dot(pieces) 912 | rendered += "g%%s" %% pieces["short"] 913 | else: 914 | # exception #1 915 | rendered = "0.post%%d" %% pieces["distance"] 916 | if pieces["dirty"]: 917 | rendered += ".dev0" 918 | rendered += "+g%%s" %% pieces["short"] 919 | return rendered 920 | 921 | 922 | def render_pep440_post_branch(pieces): 923 | """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . 924 | 925 | The ".dev0" means not master branch. 926 | 927 | Exceptions: 928 | 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] 929 | """ 930 | if pieces["closest-tag"]: 931 | rendered = pieces["closest-tag"] 932 | if pieces["distance"] or pieces["dirty"]: 933 | rendered += ".post%%d" %% pieces["distance"] 934 | if pieces["branch"] != "master": 935 | rendered += ".dev0" 936 | rendered += plus_or_dot(pieces) 937 | rendered += "g%%s" %% pieces["short"] 938 | if pieces["dirty"]: 939 | rendered += ".dirty" 940 | else: 941 | # exception #1 942 | rendered = "0.post%%d" %% pieces["distance"] 943 | if pieces["branch"] != "master": 944 | rendered += ".dev0" 945 | rendered += "+g%%s" %% pieces["short"] 946 | if pieces["dirty"]: 947 | rendered += ".dirty" 948 | return rendered 949 | 950 | 951 | def render_pep440_old(pieces): 952 | """TAG[.postDISTANCE[.dev0]] . 953 | 954 | The ".dev0" means dirty. 955 | 956 | Exceptions: 957 | 1: no tags. 0.postDISTANCE[.dev0] 958 | """ 959 | if pieces["closest-tag"]: 960 | rendered = pieces["closest-tag"] 961 | if pieces["distance"] or pieces["dirty"]: 962 | rendered += ".post%%d" %% pieces["distance"] 963 | if pieces["dirty"]: 964 | rendered += ".dev0" 965 | else: 966 | # exception #1 967 | rendered = "0.post%%d" %% pieces["distance"] 968 | if pieces["dirty"]: 969 | rendered += ".dev0" 970 | return rendered 971 | 972 | 973 | def render_git_describe(pieces): 974 | """TAG[-DISTANCE-gHEX][-dirty]. 975 | 976 | Like 'git describe --tags --dirty --always'. 977 | 978 | Exceptions: 979 | 1: no tags. HEX[-dirty] (note: no 'g' prefix) 980 | """ 981 | if pieces["closest-tag"]: 982 | rendered = pieces["closest-tag"] 983 | if pieces["distance"]: 984 | rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) 985 | else: 986 | # exception #1 987 | rendered = pieces["short"] 988 | if pieces["dirty"]: 989 | rendered += "-dirty" 990 | return rendered 991 | 992 | 993 | def render_git_describe_long(pieces): 994 | """TAG-DISTANCE-gHEX[-dirty]. 995 | 996 | Like 'git describe --tags --dirty --always -long'. 997 | The distance/hash is unconditional. 998 | 999 | Exceptions: 1000 | 1: no tags. HEX[-dirty] (note: no 'g' prefix) 1001 | """ 1002 | if pieces["closest-tag"]: 1003 | rendered = pieces["closest-tag"] 1004 | rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) 1005 | else: 1006 | # exception #1 1007 | rendered = pieces["short"] 1008 | if pieces["dirty"]: 1009 | rendered += "-dirty" 1010 | return rendered 1011 | 1012 | 1013 | def render(pieces, style): 1014 | """Render the given version pieces into the requested style.""" 1015 | if pieces["error"]: 1016 | return {"version": "unknown", 1017 | "full-revisionid": pieces.get("long"), 1018 | "dirty": None, 1019 | "error": pieces["error"], 1020 | "date": None} 1021 | 1022 | if not style or style == "default": 1023 | style = "pep440" # the default 1024 | 1025 | if style == "pep440": 1026 | rendered = render_pep440(pieces) 1027 | elif style == "pep440-branch": 1028 | rendered = render_pep440_branch(pieces) 1029 | elif style == "pep440-pre": 1030 | rendered = render_pep440_pre(pieces) 1031 | elif style == "pep440-post": 1032 | rendered = render_pep440_post(pieces) 1033 | elif style == "pep440-post-branch": 1034 | rendered = render_pep440_post_branch(pieces) 1035 | elif style == "pep440-old": 1036 | rendered = render_pep440_old(pieces) 1037 | elif style == "git-describe": 1038 | rendered = render_git_describe(pieces) 1039 | elif style == "git-describe-long": 1040 | rendered = render_git_describe_long(pieces) 1041 | else: 1042 | raise ValueError("unknown style '%%s'" %% style) 1043 | 1044 | return {"version": rendered, "full-revisionid": pieces["long"], 1045 | "dirty": pieces["dirty"], "error": None, 1046 | "date": pieces.get("date")} 1047 | 1048 | 1049 | def get_versions(): 1050 | """Get version information or return default if unable to do so.""" 1051 | # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have 1052 | # __file__, we can work backwards from there to the root. Some 1053 | # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which 1054 | # case we can only use expanded keywords. 1055 | 1056 | cfg = get_config() 1057 | verbose = cfg.verbose 1058 | 1059 | try: 1060 | return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, 1061 | verbose) 1062 | except NotThisMethod: 1063 | pass 1064 | 1065 | try: 1066 | root = os.path.realpath(__file__) 1067 | # versionfile_source is the relative path from the top of the source 1068 | # tree (where the .git directory might live) to this file. Invert 1069 | # this to find the root from __file__. 1070 | for _ in cfg.versionfile_source.split('/'): 1071 | root = os.path.dirname(root) 1072 | except NameError: 1073 | return {"version": "0+unknown", "full-revisionid": None, 1074 | "dirty": None, 1075 | "error": "unable to find root of source tree", 1076 | "date": None} 1077 | 1078 | try: 1079 | pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) 1080 | return render(pieces, cfg.style) 1081 | except NotThisMethod: 1082 | pass 1083 | 1084 | try: 1085 | if cfg.parentdir_prefix: 1086 | return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) 1087 | except NotThisMethod: 1088 | pass 1089 | 1090 | return {"version": "0+unknown", "full-revisionid": None, 1091 | "dirty": None, 1092 | "error": "unable to compute version", "date": None} 1093 | ''' 1094 | 1095 | 1096 | @register_vcs_handler("git", "get_keywords") 1097 | def git_get_keywords(versionfile_abs): 1098 | """Extract version information from the given file.""" 1099 | # the code embedded in _version.py can just fetch the value of these 1100 | # keywords. When used from setup.py, we don't want to import _version.py, 1101 | # so we do it with a regexp instead. This function is not used from 1102 | # _version.py. 1103 | keywords = {} 1104 | try: 1105 | with open(versionfile_abs, "r") as fobj: 1106 | for line in fobj: 1107 | if line.strip().startswith("git_refnames ="): 1108 | mo = re.search(r'=\s*"(.*)"', line) 1109 | if mo: 1110 | keywords["refnames"] = mo.group(1) 1111 | if line.strip().startswith("git_full ="): 1112 | mo = re.search(r'=\s*"(.*)"', line) 1113 | if mo: 1114 | keywords["full"] = mo.group(1) 1115 | if line.strip().startswith("git_date ="): 1116 | mo = re.search(r'=\s*"(.*)"', line) 1117 | if mo: 1118 | keywords["date"] = mo.group(1) 1119 | except OSError: 1120 | pass 1121 | return keywords 1122 | 1123 | 1124 | @register_vcs_handler("git", "keywords") 1125 | def git_versions_from_keywords(keywords, tag_prefix, verbose): 1126 | """Get version information from git keywords.""" 1127 | if "refnames" not in keywords: 1128 | raise NotThisMethod("Short version file found") 1129 | date = keywords.get("date") 1130 | if date is not None: 1131 | # Use only the last line. Previous lines may contain GPG signature 1132 | # information. 1133 | date = date.splitlines()[-1] 1134 | 1135 | # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant 1136 | # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 1137 | # -like" string, which we must then edit to make compliant), because 1138 | # it's been around since git-1.5.3, and it's too difficult to 1139 | # discover which version we're using, or to work around using an 1140 | # older one. 1141 | date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) 1142 | refnames = keywords["refnames"].strip() 1143 | if refnames.startswith("$Format"): 1144 | if verbose: 1145 | print("keywords are unexpanded, not using") 1146 | raise NotThisMethod("unexpanded keywords, not a git-archive tarball") 1147 | refs = {r.strip() for r in refnames.strip("()").split(",")} 1148 | # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of 1149 | # just "foo-1.0". If we see a "tag: " prefix, prefer those. 1150 | TAG = "tag: " 1151 | tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} 1152 | if not tags: 1153 | # Either we're using git < 1.8.3, or there really are no tags. We use 1154 | # a heuristic: assume all version tags have a digit. The old git %d 1155 | # expansion behaves like git log --decorate=short and strips out the 1156 | # refs/heads/ and refs/tags/ prefixes that would let us distinguish 1157 | # between branches and tags. By ignoring refnames without digits, we 1158 | # filter out many common branch names like "release" and 1159 | # "stabilization", as well as "HEAD" and "master". 1160 | tags = {r for r in refs if re.search(r"\d", r)} 1161 | if verbose: 1162 | print("discarding '%s', no digits" % ",".join(refs - tags)) 1163 | if verbose: 1164 | print("likely tags: %s" % ",".join(sorted(tags))) 1165 | for ref in sorted(tags): 1166 | # sorting will prefer e.g. "2.0" over "2.0rc1" 1167 | if ref.startswith(tag_prefix): 1168 | r = ref[len(tag_prefix) :] 1169 | # Filter out refs that exactly match prefix or that don't start 1170 | # with a number once the prefix is stripped (mostly a concern 1171 | # when prefix is '') 1172 | if not re.match(r"\d", r): 1173 | continue 1174 | if verbose: 1175 | print("picking %s" % r) 1176 | return { 1177 | "version": r, 1178 | "full-revisionid": keywords["full"].strip(), 1179 | "dirty": False, 1180 | "error": None, 1181 | "date": date, 1182 | } 1183 | # no suitable tags, so version is "0+unknown", but full hex is still there 1184 | if verbose: 1185 | print("no suitable tags, using unknown + full revision id") 1186 | return { 1187 | "version": "0+unknown", 1188 | "full-revisionid": keywords["full"].strip(), 1189 | "dirty": False, 1190 | "error": "no suitable tags", 1191 | "date": None, 1192 | } 1193 | 1194 | 1195 | @register_vcs_handler("git", "pieces_from_vcs") 1196 | def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): 1197 | """Get version from 'git describe' in the root of the source tree. 1198 | 1199 | This only gets called if the git-archive 'subst' keywords were *not* 1200 | expanded, and _version.py hasn't already been rewritten with a short 1201 | version string, meaning we're inside a checked out source tree. 1202 | """ 1203 | GITS = ["git"] 1204 | if sys.platform == "win32": 1205 | GITS = ["git.cmd", "git.exe"] 1206 | 1207 | # GIT_DIR can interfere with correct operation of Versioneer. 1208 | # It may be intended to be passed to the Versioneer-versioned project, 1209 | # but that should not change where we get our version from. 1210 | env = os.environ.copy() 1211 | env.pop("GIT_DIR", None) 1212 | runner = functools.partial(runner, env=env) 1213 | 1214 | _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) 1215 | if rc != 0: 1216 | if verbose: 1217 | print("Directory %s not under git control" % root) 1218 | raise NotThisMethod("'git rev-parse --git-dir' returned error") 1219 | 1220 | # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] 1221 | # if there isn't one, this yields HEX[-dirty] (no NUM) 1222 | describe_out, rc = runner( 1223 | GITS, 1224 | [ 1225 | "describe", 1226 | "--tags", 1227 | "--dirty", 1228 | "--always", 1229 | "--long", 1230 | "--match", 1231 | f"{tag_prefix}[[:digit:]]*", 1232 | ], 1233 | cwd=root, 1234 | ) 1235 | # --long was added in git-1.5.5 1236 | if describe_out is None: 1237 | raise NotThisMethod("'git describe' failed") 1238 | describe_out = describe_out.strip() 1239 | full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) 1240 | if full_out is None: 1241 | raise NotThisMethod("'git rev-parse' failed") 1242 | full_out = full_out.strip() 1243 | 1244 | pieces = {} 1245 | pieces["long"] = full_out 1246 | pieces["short"] = full_out[:7] # maybe improved later 1247 | pieces["error"] = None 1248 | 1249 | branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) 1250 | # --abbrev-ref was added in git-1.6.3 1251 | if rc != 0 or branch_name is None: 1252 | raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") 1253 | branch_name = branch_name.strip() 1254 | 1255 | if branch_name == "HEAD": 1256 | # If we aren't exactly on a branch, pick a branch which represents 1257 | # the current commit. If all else fails, we are on a branchless 1258 | # commit. 1259 | branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) 1260 | # --contains was added in git-1.5.4 1261 | if rc != 0 or branches is None: 1262 | raise NotThisMethod("'git branch --contains' returned error") 1263 | branches = branches.split("\n") 1264 | 1265 | # Remove the first line if we're running detached 1266 | if "(" in branches[0]: 1267 | branches.pop(0) 1268 | 1269 | # Strip off the leading "* " from the list of branches. 1270 | branches = [branch[2:] for branch in branches] 1271 | if "master" in branches: 1272 | branch_name = "master" 1273 | elif not branches: 1274 | branch_name = None 1275 | else: 1276 | # Pick the first branch that is returned. Good or bad. 1277 | branch_name = branches[0] 1278 | 1279 | pieces["branch"] = branch_name 1280 | 1281 | # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] 1282 | # TAG might have hyphens. 1283 | git_describe = describe_out 1284 | 1285 | # look for -dirty suffix 1286 | dirty = git_describe.endswith("-dirty") 1287 | pieces["dirty"] = dirty 1288 | if dirty: 1289 | git_describe = git_describe[: git_describe.rindex("-dirty")] 1290 | 1291 | # now we have TAG-NUM-gHEX or HEX 1292 | 1293 | if "-" in git_describe: 1294 | # TAG-NUM-gHEX 1295 | mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) 1296 | if not mo: 1297 | # unparsable. Maybe git-describe is misbehaving? 1298 | pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out 1299 | return pieces 1300 | 1301 | # tag 1302 | full_tag = mo.group(1) 1303 | if not full_tag.startswith(tag_prefix): 1304 | if verbose: 1305 | fmt = "tag '%s' doesn't start with prefix '%s'" 1306 | print(fmt % (full_tag, tag_prefix)) 1307 | pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( 1308 | full_tag, 1309 | tag_prefix, 1310 | ) 1311 | return pieces 1312 | pieces["closest-tag"] = full_tag[len(tag_prefix) :] 1313 | 1314 | # distance: number of commits since tag 1315 | pieces["distance"] = int(mo.group(2)) 1316 | 1317 | # commit: short hex revision ID 1318 | pieces["short"] = mo.group(3) 1319 | 1320 | else: 1321 | # HEX: no tags 1322 | pieces["closest-tag"] = None 1323 | out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) 1324 | pieces["distance"] = len(out.split()) # total number of commits 1325 | 1326 | # commit date: see ISO-8601 comment in git_versions_from_keywords() 1327 | date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() 1328 | # Use only the last line. Previous lines may contain GPG signature 1329 | # information. 1330 | date = date.splitlines()[-1] 1331 | pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) 1332 | 1333 | return pieces 1334 | 1335 | 1336 | def do_vcs_install(versionfile_source, ipy): 1337 | """Git-specific installation logic for Versioneer. 1338 | 1339 | For Git, this means creating/changing .gitattributes to mark _version.py 1340 | for export-subst keyword substitution. 1341 | """ 1342 | GITS = ["git"] 1343 | if sys.platform == "win32": 1344 | GITS = ["git.cmd", "git.exe"] 1345 | files = [versionfile_source] 1346 | if ipy: 1347 | files.append(ipy) 1348 | try: 1349 | my_path = __file__ 1350 | if my_path.endswith(".pyc") or my_path.endswith(".pyo"): 1351 | my_path = os.path.splitext(my_path)[0] + ".py" 1352 | versioneer_file = os.path.relpath(my_path) 1353 | except NameError: 1354 | versioneer_file = "versioneer.py" 1355 | files.append(versioneer_file) 1356 | present = False 1357 | try: 1358 | with open(".gitattributes", "r") as fobj: 1359 | for line in fobj: 1360 | if line.strip().startswith(versionfile_source): 1361 | if "export-subst" in line.strip().split()[1:]: 1362 | present = True 1363 | break 1364 | except OSError: 1365 | pass 1366 | if not present: 1367 | with open(".gitattributes", "a+") as fobj: 1368 | fobj.write(f"{versionfile_source} export-subst\n") 1369 | files.append(".gitattributes") 1370 | run_command(GITS, ["add", "--"] + files) 1371 | 1372 | 1373 | def versions_from_parentdir(parentdir_prefix, root, verbose): 1374 | """Try to determine the version from the parent directory name. 1375 | 1376 | Source tarballs conventionally unpack into a directory that includes both 1377 | the project name and a version string. We will also support searching up 1378 | two directory levels for an appropriately named parent directory 1379 | """ 1380 | rootdirs = [] 1381 | 1382 | for _ in range(3): 1383 | dirname = os.path.basename(root) 1384 | if dirname.startswith(parentdir_prefix): 1385 | return { 1386 | "version": dirname[len(parentdir_prefix) :], 1387 | "full-revisionid": None, 1388 | "dirty": False, 1389 | "error": None, 1390 | "date": None, 1391 | } 1392 | rootdirs.append(root) 1393 | root = os.path.dirname(root) # up a level 1394 | 1395 | if verbose: 1396 | print( 1397 | "Tried directories %s but none started with prefix %s" 1398 | % (str(rootdirs), parentdir_prefix) 1399 | ) 1400 | raise NotThisMethod("rootdir doesn't start with parentdir_prefix") 1401 | 1402 | 1403 | SHORT_VERSION_PY = """ 1404 | # This file was generated by 'versioneer.py' (0.23) from 1405 | # revision-control system data, or from the parent directory name of an 1406 | # unpacked source archive. Distribution tarballs contain a pre-generated copy 1407 | # of this file. 1408 | 1409 | import json 1410 | 1411 | version_json = ''' 1412 | %s 1413 | ''' # END VERSION_JSON 1414 | 1415 | 1416 | def get_versions(): 1417 | return json.loads(version_json) 1418 | """ 1419 | 1420 | 1421 | def versions_from_file(filename): 1422 | """Try to determine the version from _version.py if present.""" 1423 | try: 1424 | with open(filename) as f: 1425 | contents = f.read() 1426 | except OSError: 1427 | raise NotThisMethod("unable to read _version.py") 1428 | mo = re.search( 1429 | r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S 1430 | ) 1431 | if not mo: 1432 | mo = re.search( 1433 | r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S 1434 | ) 1435 | if not mo: 1436 | raise NotThisMethod("no version_json in _version.py") 1437 | return json.loads(mo.group(1)) 1438 | 1439 | 1440 | def write_to_version_file(filename, versions): 1441 | """Write the given version number to the given _version.py file.""" 1442 | os.unlink(filename) 1443 | contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) 1444 | with open(filename, "w") as f: 1445 | f.write(SHORT_VERSION_PY % contents) 1446 | 1447 | print("set %s to '%s'" % (filename, versions["version"])) 1448 | 1449 | 1450 | def plus_or_dot(pieces): 1451 | """Return a + if we don't already have one, else return a .""" 1452 | if "+" in pieces.get("closest-tag", ""): 1453 | return "." 1454 | return "+" 1455 | 1456 | 1457 | def render_pep440(pieces): 1458 | """Build up version string, with post-release "local version identifier". 1459 | 1460 | Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you 1461 | get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty 1462 | 1463 | Exceptions: 1464 | 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] 1465 | """ 1466 | if pieces["closest-tag"]: 1467 | rendered = pieces["closest-tag"] 1468 | if pieces["distance"] or pieces["dirty"]: 1469 | rendered += plus_or_dot(pieces) 1470 | rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) 1471 | if pieces["dirty"]: 1472 | rendered += ".dirty" 1473 | else: 1474 | # exception #1 1475 | rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) 1476 | if pieces["dirty"]: 1477 | rendered += ".dirty" 1478 | return rendered 1479 | 1480 | 1481 | def render_pep440_branch(pieces): 1482 | """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . 1483 | 1484 | The ".dev0" means not master branch. Note that .dev0 sorts backwards 1485 | (a feature branch will appear "older" than the master branch). 1486 | 1487 | Exceptions: 1488 | 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] 1489 | """ 1490 | if pieces["closest-tag"]: 1491 | rendered = pieces["closest-tag"] 1492 | if pieces["distance"] or pieces["dirty"]: 1493 | if pieces["branch"] != "master": 1494 | rendered += ".dev0" 1495 | rendered += plus_or_dot(pieces) 1496 | rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) 1497 | if pieces["dirty"]: 1498 | rendered += ".dirty" 1499 | else: 1500 | # exception #1 1501 | rendered = "0" 1502 | if pieces["branch"] != "master": 1503 | rendered += ".dev0" 1504 | rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) 1505 | if pieces["dirty"]: 1506 | rendered += ".dirty" 1507 | return rendered 1508 | 1509 | 1510 | def pep440_split_post(ver): 1511 | """Split pep440 version string at the post-release segment. 1512 | 1513 | Returns the release segments before the post-release and the 1514 | post-release version number (or -1 if no post-release segment is present). 1515 | """ 1516 | vc = str.split(ver, ".post") 1517 | return vc[0], int(vc[1] or 0) if len(vc) == 2 else None 1518 | 1519 | 1520 | def render_pep440_pre(pieces): 1521 | """TAG[.postN.devDISTANCE] -- No -dirty. 1522 | 1523 | Exceptions: 1524 | 1: no tags. 0.post0.devDISTANCE 1525 | """ 1526 | if pieces["closest-tag"]: 1527 | if pieces["distance"]: 1528 | # update the post release segment 1529 | tag_version, post_version = pep440_split_post(pieces["closest-tag"]) 1530 | rendered = tag_version 1531 | if post_version is not None: 1532 | rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) 1533 | else: 1534 | rendered += ".post0.dev%d" % (pieces["distance"]) 1535 | else: 1536 | # no commits, use the tag as the version 1537 | rendered = pieces["closest-tag"] 1538 | else: 1539 | # exception #1 1540 | rendered = "0.post0.dev%d" % pieces["distance"] 1541 | return rendered 1542 | 1543 | 1544 | def render_pep440_post(pieces): 1545 | """TAG[.postDISTANCE[.dev0]+gHEX] . 1546 | 1547 | The ".dev0" means dirty. Note that .dev0 sorts backwards 1548 | (a dirty tree will appear "older" than the corresponding clean one), 1549 | but you shouldn't be releasing software with -dirty anyways. 1550 | 1551 | Exceptions: 1552 | 1: no tags. 0.postDISTANCE[.dev0] 1553 | """ 1554 | if pieces["closest-tag"]: 1555 | rendered = pieces["closest-tag"] 1556 | if pieces["distance"] or pieces["dirty"]: 1557 | rendered += ".post%d" % pieces["distance"] 1558 | if pieces["dirty"]: 1559 | rendered += ".dev0" 1560 | rendered += plus_or_dot(pieces) 1561 | rendered += "g%s" % pieces["short"] 1562 | else: 1563 | # exception #1 1564 | rendered = "0.post%d" % pieces["distance"] 1565 | if pieces["dirty"]: 1566 | rendered += ".dev0" 1567 | rendered += "+g%s" % pieces["short"] 1568 | return rendered 1569 | 1570 | 1571 | def render_pep440_post_branch(pieces): 1572 | """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . 1573 | 1574 | The ".dev0" means not master branch. 1575 | 1576 | Exceptions: 1577 | 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] 1578 | """ 1579 | if pieces["closest-tag"]: 1580 | rendered = pieces["closest-tag"] 1581 | if pieces["distance"] or pieces["dirty"]: 1582 | rendered += ".post%d" % pieces["distance"] 1583 | if pieces["branch"] != "master": 1584 | rendered += ".dev0" 1585 | rendered += plus_or_dot(pieces) 1586 | rendered += "g%s" % pieces["short"] 1587 | if pieces["dirty"]: 1588 | rendered += ".dirty" 1589 | else: 1590 | # exception #1 1591 | rendered = "0.post%d" % pieces["distance"] 1592 | if pieces["branch"] != "master": 1593 | rendered += ".dev0" 1594 | rendered += "+g%s" % pieces["short"] 1595 | if pieces["dirty"]: 1596 | rendered += ".dirty" 1597 | return rendered 1598 | 1599 | 1600 | def render_pep440_old(pieces): 1601 | """TAG[.postDISTANCE[.dev0]] . 1602 | 1603 | The ".dev0" means dirty. 1604 | 1605 | Exceptions: 1606 | 1: no tags. 0.postDISTANCE[.dev0] 1607 | """ 1608 | if pieces["closest-tag"]: 1609 | rendered = pieces["closest-tag"] 1610 | if pieces["distance"] or pieces["dirty"]: 1611 | rendered += ".post%d" % pieces["distance"] 1612 | if pieces["dirty"]: 1613 | rendered += ".dev0" 1614 | else: 1615 | # exception #1 1616 | rendered = "0.post%d" % pieces["distance"] 1617 | if pieces["dirty"]: 1618 | rendered += ".dev0" 1619 | return rendered 1620 | 1621 | 1622 | def render_git_describe(pieces): 1623 | """TAG[-DISTANCE-gHEX][-dirty]. 1624 | 1625 | Like 'git describe --tags --dirty --always'. 1626 | 1627 | Exceptions: 1628 | 1: no tags. HEX[-dirty] (note: no 'g' prefix) 1629 | """ 1630 | if pieces["closest-tag"]: 1631 | rendered = pieces["closest-tag"] 1632 | if pieces["distance"]: 1633 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 1634 | else: 1635 | # exception #1 1636 | rendered = pieces["short"] 1637 | if pieces["dirty"]: 1638 | rendered += "-dirty" 1639 | return rendered 1640 | 1641 | 1642 | def render_git_describe_long(pieces): 1643 | """TAG-DISTANCE-gHEX[-dirty]. 1644 | 1645 | Like 'git describe --tags --dirty --always -long'. 1646 | The distance/hash is unconditional. 1647 | 1648 | Exceptions: 1649 | 1: no tags. HEX[-dirty] (note: no 'g' prefix) 1650 | """ 1651 | if pieces["closest-tag"]: 1652 | rendered = pieces["closest-tag"] 1653 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 1654 | else: 1655 | # exception #1 1656 | rendered = pieces["short"] 1657 | if pieces["dirty"]: 1658 | rendered += "-dirty" 1659 | return rendered 1660 | 1661 | 1662 | def render(pieces, style): 1663 | """Render the given version pieces into the requested style.""" 1664 | if pieces["error"]: 1665 | return { 1666 | "version": "unknown", 1667 | "full-revisionid": pieces.get("long"), 1668 | "dirty": None, 1669 | "error": pieces["error"], 1670 | "date": None, 1671 | } 1672 | 1673 | if not style or style == "default": 1674 | style = "pep440" # the default 1675 | 1676 | if style == "pep440": 1677 | rendered = render_pep440(pieces) 1678 | elif style == "pep440-branch": 1679 | rendered = render_pep440_branch(pieces) 1680 | elif style == "pep440-pre": 1681 | rendered = render_pep440_pre(pieces) 1682 | elif style == "pep440-post": 1683 | rendered = render_pep440_post(pieces) 1684 | elif style == "pep440-post-branch": 1685 | rendered = render_pep440_post_branch(pieces) 1686 | elif style == "pep440-old": 1687 | rendered = render_pep440_old(pieces) 1688 | elif style == "git-describe": 1689 | rendered = render_git_describe(pieces) 1690 | elif style == "git-describe-long": 1691 | rendered = render_git_describe_long(pieces) 1692 | else: 1693 | raise ValueError("unknown style '%s'" % style) 1694 | 1695 | return { 1696 | "version": rendered, 1697 | "full-revisionid": pieces["long"], 1698 | "dirty": pieces["dirty"], 1699 | "error": None, 1700 | "date": pieces.get("date"), 1701 | } 1702 | 1703 | 1704 | class VersioneerBadRootError(Exception): 1705 | """The project root directory is unknown or missing key files.""" 1706 | 1707 | 1708 | def get_versions(verbose=False): 1709 | """Get the project version from whatever source is available. 1710 | 1711 | Returns dict with two keys: 'version' and 'full'. 1712 | """ 1713 | if "versioneer" in sys.modules: 1714 | # see the discussion in cmdclass.py:get_cmdclass() 1715 | del sys.modules["versioneer"] 1716 | 1717 | root = get_root() 1718 | cfg = get_config_from_root(root) 1719 | 1720 | assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" 1721 | handlers = HANDLERS.get(cfg.VCS) 1722 | assert handlers, "unrecognized VCS '%s'" % cfg.VCS 1723 | verbose = verbose or cfg.verbose 1724 | assert ( 1725 | cfg.versionfile_source is not None 1726 | ), "please set versioneer.versionfile_source" 1727 | assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" 1728 | 1729 | versionfile_abs = os.path.join(root, cfg.versionfile_source) 1730 | 1731 | # extract version from first of: _version.py, VCS command (e.g. 'git 1732 | # describe'), parentdir. This is meant to work for developers using a 1733 | # source checkout, for users of a tarball created by 'setup.py sdist', 1734 | # and for users of a tarball/zipball created by 'git archive' or github's 1735 | # download-from-tag feature or the equivalent in other VCSes. 1736 | 1737 | get_keywords_f = handlers.get("get_keywords") 1738 | from_keywords_f = handlers.get("keywords") 1739 | if get_keywords_f and from_keywords_f: 1740 | try: 1741 | keywords = get_keywords_f(versionfile_abs) 1742 | ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) 1743 | if verbose: 1744 | print("got version from expanded keyword %s" % ver) 1745 | return ver 1746 | except NotThisMethod: 1747 | pass 1748 | 1749 | try: 1750 | ver = versions_from_file(versionfile_abs) 1751 | if verbose: 1752 | print("got version from file %s %s" % (versionfile_abs, ver)) 1753 | return ver 1754 | except NotThisMethod: 1755 | pass 1756 | 1757 | from_vcs_f = handlers.get("pieces_from_vcs") 1758 | if from_vcs_f: 1759 | try: 1760 | pieces = from_vcs_f(cfg.tag_prefix, root, verbose) 1761 | ver = render(pieces, cfg.style) 1762 | if verbose: 1763 | print("got version from VCS %s" % ver) 1764 | return ver 1765 | except NotThisMethod: 1766 | pass 1767 | 1768 | try: 1769 | if cfg.parentdir_prefix: 1770 | ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) 1771 | if verbose: 1772 | print("got version from parentdir %s" % ver) 1773 | return ver 1774 | except NotThisMethod: 1775 | pass 1776 | 1777 | if verbose: 1778 | print("unable to compute version") 1779 | 1780 | return { 1781 | "version": "0+unknown", 1782 | "full-revisionid": None, 1783 | "dirty": None, 1784 | "error": "unable to compute version", 1785 | "date": None, 1786 | } 1787 | 1788 | 1789 | def get_version(): 1790 | """Get the short version string for this project.""" 1791 | return get_versions()["version"] 1792 | 1793 | 1794 | def get_cmdclass(cmdclass=None): 1795 | """Get the custom setuptools subclasses used by Versioneer. 1796 | 1797 | If the package uses a different cmdclass (e.g. one from numpy), it 1798 | should be provide as an argument. 1799 | """ 1800 | if "versioneer" in sys.modules: 1801 | del sys.modules["versioneer"] 1802 | # this fixes the "python setup.py develop" case (also 'install' and 1803 | # 'easy_install .'), in which subdependencies of the main project are 1804 | # built (using setup.py bdist_egg) in the same python process. Assume 1805 | # a main project A and a dependency B, which use different versions 1806 | # of Versioneer. A's setup.py imports A's Versioneer, leaving it in 1807 | # sys.modules by the time B's setup.py is executed, causing B to run 1808 | # with the wrong versioneer. Setuptools wraps the sub-dep builds in a 1809 | # sandbox that restores sys.modules to it's pre-build state, so the 1810 | # parent is protected against the child's "import versioneer". By 1811 | # removing ourselves from sys.modules here, before the child build 1812 | # happens, we protect the child from the parent's versioneer too. 1813 | # Also see https://github.com/python-versioneer/python-versioneer/issues/52 1814 | 1815 | cmds = {} if cmdclass is None else cmdclass.copy() 1816 | 1817 | # we add "version" to setuptools 1818 | from setuptools import Command 1819 | 1820 | class cmd_version(Command): 1821 | description = "report generated version string" 1822 | user_options = [] 1823 | boolean_options = [] 1824 | 1825 | def initialize_options(self): 1826 | pass 1827 | 1828 | def finalize_options(self): 1829 | pass 1830 | 1831 | def run(self): 1832 | vers = get_versions(verbose=True) 1833 | print("Version: %s" % vers["version"]) 1834 | print(" full-revisionid: %s" % vers.get("full-revisionid")) 1835 | print(" dirty: %s" % vers.get("dirty")) 1836 | print(" date: %s" % vers.get("date")) 1837 | if vers["error"]: 1838 | print(" error: %s" % vers["error"]) 1839 | 1840 | cmds["version"] = cmd_version 1841 | 1842 | # we override "build_py" in setuptools 1843 | # 1844 | # most invocation pathways end up running build_py: 1845 | # distutils/build -> build_py 1846 | # distutils/install -> distutils/build ->.. 1847 | # setuptools/bdist_wheel -> distutils/install ->.. 1848 | # setuptools/bdist_egg -> distutils/install_lib -> build_py 1849 | # setuptools/install -> bdist_egg ->.. 1850 | # setuptools/develop -> ? 1851 | # pip install: 1852 | # copies source tree to a tempdir before running egg_info/etc 1853 | # if .git isn't copied too, 'git describe' will fail 1854 | # then does setup.py bdist_wheel, or sometimes setup.py install 1855 | # setup.py egg_info -> ? 1856 | 1857 | # pip install -e . and setuptool/editable_wheel will invoke build_py 1858 | # but the build_py command is not expected to copy any files. 1859 | 1860 | # we override different "build_py" commands for both environments 1861 | if "build_py" in cmds: 1862 | _build_py = cmds["build_py"] 1863 | else: 1864 | from setuptools.command.build_py import build_py as _build_py 1865 | 1866 | class cmd_build_py(_build_py): 1867 | def run(self): 1868 | root = get_root() 1869 | cfg = get_config_from_root(root) 1870 | versions = get_versions() 1871 | _build_py.run(self) 1872 | if getattr(self, "editable_mode", False): 1873 | # During editable installs `.py` and data files are 1874 | # not copied to build_lib 1875 | return 1876 | # now locate _version.py in the new build/ directory and replace 1877 | # it with an updated value 1878 | if cfg.versionfile_build: 1879 | target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) 1880 | print("UPDATING %s" % target_versionfile) 1881 | write_to_version_file(target_versionfile, versions) 1882 | 1883 | cmds["build_py"] = cmd_build_py 1884 | 1885 | if "build_ext" in cmds: 1886 | _build_ext = cmds["build_ext"] 1887 | else: 1888 | from setuptools.command.build_ext import build_ext as _build_ext 1889 | 1890 | class cmd_build_ext(_build_ext): 1891 | def run(self): 1892 | root = get_root() 1893 | cfg = get_config_from_root(root) 1894 | versions = get_versions() 1895 | _build_ext.run(self) 1896 | if self.inplace: 1897 | # build_ext --inplace will only build extensions in 1898 | # build/lib<..> dir with no _version.py to write to. 1899 | # As in place builds will already have a _version.py 1900 | # in the module dir, we do not need to write one. 1901 | return 1902 | # now locate _version.py in the new build/ directory and replace 1903 | # it with an updated value 1904 | target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) 1905 | if not os.path.exists(target_versionfile): 1906 | print( 1907 | f"Warning: {target_versionfile} does not exist, skipping " 1908 | "version update. This can happen if you are running build_ext " 1909 | "without first running build_py." 1910 | ) 1911 | return 1912 | print("UPDATING %s" % target_versionfile) 1913 | write_to_version_file(target_versionfile, versions) 1914 | 1915 | cmds["build_ext"] = cmd_build_ext 1916 | 1917 | if "cx_Freeze" in sys.modules: # cx_freeze enabled? 1918 | from cx_Freeze.dist import build_exe as _build_exe 1919 | 1920 | # nczeczulin reports that py2exe won't like the pep440-style string 1921 | # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. 1922 | # setup(console=[{ 1923 | # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION 1924 | # "product_version": versioneer.get_version(), 1925 | # ... 1926 | 1927 | class cmd_build_exe(_build_exe): 1928 | def run(self): 1929 | root = get_root() 1930 | cfg = get_config_from_root(root) 1931 | versions = get_versions() 1932 | target_versionfile = cfg.versionfile_source 1933 | print("UPDATING %s" % target_versionfile) 1934 | write_to_version_file(target_versionfile, versions) 1935 | 1936 | _build_exe.run(self) 1937 | os.unlink(target_versionfile) 1938 | with open(cfg.versionfile_source, "w") as f: 1939 | LONG = LONG_VERSION_PY[cfg.VCS] 1940 | f.write( 1941 | LONG 1942 | % { 1943 | "DOLLAR": "$", 1944 | "STYLE": cfg.style, 1945 | "TAG_PREFIX": cfg.tag_prefix, 1946 | "PARENTDIR_PREFIX": cfg.parentdir_prefix, 1947 | "VERSIONFILE_SOURCE": cfg.versionfile_source, 1948 | } 1949 | ) 1950 | 1951 | cmds["build_exe"] = cmd_build_exe 1952 | del cmds["build_py"] 1953 | 1954 | if "py2exe" in sys.modules: # py2exe enabled? 1955 | from py2exe.distutils_buildexe import py2exe as _py2exe 1956 | 1957 | class cmd_py2exe(_py2exe): 1958 | def run(self): 1959 | root = get_root() 1960 | cfg = get_config_from_root(root) 1961 | versions = get_versions() 1962 | target_versionfile = cfg.versionfile_source 1963 | print("UPDATING %s" % target_versionfile) 1964 | write_to_version_file(target_versionfile, versions) 1965 | 1966 | _py2exe.run(self) 1967 | os.unlink(target_versionfile) 1968 | with open(cfg.versionfile_source, "w") as f: 1969 | LONG = LONG_VERSION_PY[cfg.VCS] 1970 | f.write( 1971 | LONG 1972 | % { 1973 | "DOLLAR": "$", 1974 | "STYLE": cfg.style, 1975 | "TAG_PREFIX": cfg.tag_prefix, 1976 | "PARENTDIR_PREFIX": cfg.parentdir_prefix, 1977 | "VERSIONFILE_SOURCE": cfg.versionfile_source, 1978 | } 1979 | ) 1980 | 1981 | cmds["py2exe"] = cmd_py2exe 1982 | 1983 | # sdist farms its file list building out to egg_info 1984 | if "egg_info" in cmds: 1985 | _sdist = cmds["egg_info"] 1986 | else: 1987 | from setuptools.command.egg_info import egg_info as _egg_info 1988 | 1989 | class cmd_egg_info(_egg_info): 1990 | def find_sources(self): 1991 | # egg_info.find_sources builds the manifest list and writes it 1992 | # in one shot 1993 | super().find_sources() 1994 | 1995 | # Modify the filelist and normalize it 1996 | root = get_root() 1997 | cfg = get_config_from_root(root) 1998 | self.filelist.append("versioneer.py") 1999 | if cfg.versionfile_source: 2000 | # There are rare cases where versionfile_source might not be 2001 | # included by default, so we must be explicit 2002 | self.filelist.append(cfg.versionfile_source) 2003 | self.filelist.sort() 2004 | self.filelist.remove_duplicates() 2005 | 2006 | # The write method is hidden in the manifest_maker instance that 2007 | # generated the filelist and was thrown away 2008 | # We will instead replicate their final normalization (to unicode, 2009 | # and POSIX-style paths) 2010 | from setuptools import unicode_utils 2011 | 2012 | normalized = [ 2013 | unicode_utils.filesys_decode(f).replace(os.sep, "/") 2014 | for f in self.filelist.files 2015 | ] 2016 | 2017 | manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") 2018 | with open(manifest_filename, "w") as fobj: 2019 | fobj.write("\n".join(normalized)) 2020 | 2021 | cmds["egg_info"] = cmd_egg_info 2022 | 2023 | # we override different "sdist" commands for both environments 2024 | if "sdist" in cmds: 2025 | _sdist = cmds["sdist"] 2026 | else: 2027 | from setuptools.command.sdist import sdist as _sdist 2028 | 2029 | class cmd_sdist(_sdist): 2030 | def run(self): 2031 | versions = get_versions() 2032 | self._versioneer_generated_versions = versions 2033 | # unless we update this, the command will keep using the old 2034 | # version 2035 | self.distribution.metadata.version = versions["version"] 2036 | return _sdist.run(self) 2037 | 2038 | def make_release_tree(self, base_dir, files): 2039 | root = get_root() 2040 | cfg = get_config_from_root(root) 2041 | _sdist.make_release_tree(self, base_dir, files) 2042 | # now locate _version.py in the new base_dir directory 2043 | # (remembering that it may be a hardlink) and replace it with an 2044 | # updated value 2045 | target_versionfile = os.path.join(base_dir, cfg.versionfile_source) 2046 | print("UPDATING %s" % target_versionfile) 2047 | write_to_version_file( 2048 | target_versionfile, self._versioneer_generated_versions 2049 | ) 2050 | 2051 | cmds["sdist"] = cmd_sdist 2052 | 2053 | return cmds 2054 | 2055 | 2056 | CONFIG_ERROR = """ 2057 | setup.cfg is missing the necessary Versioneer configuration. You need 2058 | a section like: 2059 | 2060 | [versioneer] 2061 | VCS = git 2062 | style = pep440 2063 | versionfile_source = src/myproject/_version.py 2064 | versionfile_build = myproject/_version.py 2065 | tag_prefix = 2066 | parentdir_prefix = myproject- 2067 | 2068 | You will also need to edit your setup.py to use the results: 2069 | 2070 | import versioneer 2071 | setup(version=versioneer.get_version(), 2072 | cmdclass=versioneer.get_cmdclass(), ...) 2073 | 2074 | Please read the docstring in ./versioneer.py for configuration instructions, 2075 | edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. 2076 | """ 2077 | 2078 | SAMPLE_CONFIG = """ 2079 | # See the docstring in versioneer.py for instructions. Note that you must 2080 | # re-run 'versioneer.py setup' after changing this section, and commit the 2081 | # resulting files. 2082 | 2083 | [versioneer] 2084 | #VCS = git 2085 | #style = pep440 2086 | #versionfile_source = 2087 | #versionfile_build = 2088 | #tag_prefix = 2089 | #parentdir_prefix = 2090 | 2091 | """ 2092 | 2093 | OLD_SNIPPET = """ 2094 | from ._version import get_versions 2095 | __version__ = get_versions()['version'] 2096 | del get_versions 2097 | """ 2098 | 2099 | INIT_PY_SNIPPET = """ 2100 | from . import {0} 2101 | __version__ = {0}.get_versions()['version'] 2102 | """ 2103 | 2104 | 2105 | def do_setup(): 2106 | """Do main VCS-independent setup function for installing Versioneer.""" 2107 | root = get_root() 2108 | try: 2109 | cfg = get_config_from_root(root) 2110 | except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: 2111 | if isinstance(e, (OSError, configparser.NoSectionError)): 2112 | print("Adding sample versioneer config to setup.cfg", file=sys.stderr) 2113 | with open(os.path.join(root, "setup.cfg"), "a") as f: 2114 | f.write(SAMPLE_CONFIG) 2115 | print(CONFIG_ERROR, file=sys.stderr) 2116 | return 1 2117 | 2118 | print(" creating %s" % cfg.versionfile_source) 2119 | with open(cfg.versionfile_source, "w") as f: 2120 | LONG = LONG_VERSION_PY[cfg.VCS] 2121 | f.write( 2122 | LONG 2123 | % { 2124 | "DOLLAR": "$", 2125 | "STYLE": cfg.style, 2126 | "TAG_PREFIX": cfg.tag_prefix, 2127 | "PARENTDIR_PREFIX": cfg.parentdir_prefix, 2128 | "VERSIONFILE_SOURCE": cfg.versionfile_source, 2129 | } 2130 | ) 2131 | 2132 | ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") 2133 | if os.path.exists(ipy): 2134 | try: 2135 | with open(ipy, "r") as f: 2136 | old = f.read() 2137 | except OSError: 2138 | old = "" 2139 | module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] 2140 | snippet = INIT_PY_SNIPPET.format(module) 2141 | if OLD_SNIPPET in old: 2142 | print(" replacing boilerplate in %s" % ipy) 2143 | with open(ipy, "w") as f: 2144 | f.write(old.replace(OLD_SNIPPET, snippet)) 2145 | elif snippet not in old: 2146 | print(" appending to %s" % ipy) 2147 | with open(ipy, "a") as f: 2148 | f.write(snippet) 2149 | else: 2150 | print(" %s unmodified" % ipy) 2151 | else: 2152 | print(" %s doesn't exist, ok" % ipy) 2153 | ipy = None 2154 | 2155 | # Make VCS-specific changes. For git, this means creating/changing 2156 | # .gitattributes to mark _version.py for export-subst keyword 2157 | # substitution. 2158 | do_vcs_install(cfg.versionfile_source, ipy) 2159 | return 0 2160 | 2161 | 2162 | def scan_setup_py(): 2163 | """Validate the contents of setup.py against Versioneer's expectations.""" 2164 | found = set() 2165 | setters = False 2166 | errors = 0 2167 | with open("setup.py", "r") as f: 2168 | for line in f.readlines(): 2169 | if "import versioneer" in line: 2170 | found.add("import") 2171 | if "versioneer.get_cmdclass()" in line: 2172 | found.add("cmdclass") 2173 | if "versioneer.get_version()" in line: 2174 | found.add("get_version") 2175 | if "versioneer.VCS" in line: 2176 | setters = True 2177 | if "versioneer.versionfile_source" in line: 2178 | setters = True 2179 | if len(found) != 3: 2180 | print("") 2181 | print("Your setup.py appears to be missing some important items") 2182 | print("(but I might be wrong). Please make sure it has something") 2183 | print("roughly like the following:") 2184 | print("") 2185 | print(" import versioneer") 2186 | print(" setup( version=versioneer.get_version(),") 2187 | print(" cmdclass=versioneer.get_cmdclass(), ...)") 2188 | print("") 2189 | errors += 1 2190 | if setters: 2191 | print("You should remove lines like 'versioneer.VCS = ' and") 2192 | print("'versioneer.versionfile_source = ' . This configuration") 2193 | print("now lives in setup.cfg, and should be removed from setup.py") 2194 | print("") 2195 | errors += 1 2196 | return errors 2197 | 2198 | 2199 | if __name__ == "__main__": 2200 | cmd = sys.argv[1] 2201 | if cmd == "setup": 2202 | errors = do_setup() 2203 | errors += scan_setup_py() 2204 | if errors: 2205 | sys.exit(1) 2206 | --------------------------------------------------------------------------------