├── .gitignore ├── LICENSE ├── README.rst ├── color_scheme_3d ├── here_aqua_gray.png └── visualising_color_schemes_3d.ipynb ├── environment.yml ├── freemium ├── freemium.ipynb ├── stops_berlin.geojson └── utils.py ├── mapping ├── geodesic_polylines.ipynb ├── here_maps_api_explorer_no_creds.ipynb └── images │ ├── 12.png │ ├── 13.png │ ├── 14.png │ ├── 15.png │ └── 9.png ├── meetups └── meetup_analysis.ipynb ├── pixelplanet ├── README.rst ├── basemap_ipywidgets.ipynb └── environment.yml ├── planecrashinfo ├── README.rst ├── build_database.py ├── build_geolocations.py ├── data │ ├── data.csv │ └── geolocs.json ├── environment.yml ├── images │ └── header.png ├── planecrashinfo.ipynb └── planecrashinfo_light.py └── spark └── scala_for_pythoneers.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Notebook Collection 2 | =================== 3 | 4 | This is an experimental emerging collection of Jupyter notebooks... 5 | 6 | (This is not working at the moment!) You should be able to run them inside a local browser using a notebook packaging/hosting service named http://MyBinder.org. Just click on this image: 7 | 8 | .. image:: http://mybinder.org/badge.svg 9 | :target: http://mybinder.org:/repo/deeplook/notebooks 10 | -------------------------------------------------------------------------------- /color_scheme_3d/here_aqua_gray.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplook/notebooks/4511c4de9c4aee35098a90e44380e51c508f60d4/color_scheme_3d/here_aqua_gray.png -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: deeplook_notebooks 2 | channels: 3 | - defaults 4 | dependencies: 5 | - conda-forge::backports.shutil_get_terminal_size=1.0.0=py35_0 6 | - conda-forge::basemap=1.0.8.dev0=np111py35_2 7 | - conda-forge::ca-certificates=2016.9.26=0 8 | - conda-forge::certifi=2016.9.26=py35_0 9 | - conda-forge::cycler=0.10.0=py35_0 10 | - conda-forge::decorator=4.0.10=py35_0 11 | - conda-forge::entrypoints=0.2.2=py35_0 12 | - conda-forge::freetype=2.6.3=1 13 | - conda-forge::geopy=1.11.0=py35_0 14 | - conda-forge::geos=3.4.2=2 15 | - conda-forge::ipykernel=4.5.0=py35_0 16 | - conda-forge::ipython=5.1.0=py35_1 17 | - conda-forge::ipython_genutils=0.1.0=py35_0 18 | - conda-forge::ipywidgets=5.2.2=py35_2 19 | - conda-forge::jinja2=2.8=py35_1 20 | - conda-forge::jsonschema=2.5.1=py35_0 21 | - conda-forge::jupyter_client=4.4.0=py35_0 22 | - conda-forge::jupyter_console=5.0.0=py35_0 23 | - conda-forge::jupyter_core=4.2.0=py35_0 24 | - conda-forge::libpng=1.6.26=0 25 | - conda-forge::libsodium=1.0.10=0 26 | - conda-forge::markupsafe=0.23=py35_0 27 | - conda-forge::matplotlib=1.5.3=np111py35_2 28 | - conda-forge::mistune=0.7.3=py35_0 29 | - conda-forge::nbconvert=4.2.0=py35_0 30 | - conda-forge::nbformat=4.1.0=py35_0 31 | - conda-forge::ncurses=5.9=9 32 | - conda-forge::notebook=4.2.3=py35_0 33 | - conda-forge::openssl=1.0.2h=2 34 | - conda-forge::pandas=0.19.1=np111py35_0 35 | - conda-forge::pexpect=4.2.1=py35_0 36 | - conda-forge::pickleshare=0.7.3=py35_0 37 | - conda-forge::pip=9.0.1=py35_0 38 | - conda-forge::prompt_toolkit=1.0.9=py35_0 39 | - conda-forge::ptyprocess=0.5.1=py35_0 40 | - conda-forge::pygments=2.1.3=py35_1 41 | - conda-forge::pyparsing=2.1.10=py35_0 42 | - conda-forge::pyproj=1.9.5.1=py35_0 43 | - conda-forge::pyqt=4.11.4=py35_1 44 | - conda-forge::pyshp=1.2.10=py35_0 45 | - conda-forge::python=3.5.2=2 46 | - conda-forge::python-dateutil=2.5.3=py35_0 47 | - conda-forge::pytz=2016.7=py35_0 48 | - conda-forge::pyzmq=16.0.1=py35_0 49 | - conda-forge::qtconsole=4.2.1=py35_1 50 | - conda-forge::readline=6.2=0 51 | - conda-forge::setuptools=28.8.0=py35_0 52 | - conda-forge::simplegeneric=0.8.1=py35_0 53 | - conda-forge::sip=4.18=py35_0 54 | - conda-forge::six=1.10.0=py35_0 55 | - conda-forge::sqlite=3.13.0=1 56 | - conda-forge::terminado=0.6=py35_0 57 | - conda-forge::tk=8.5.19=0 58 | - conda-forge::tornado=4.4.2=py35_0 59 | - conda-forge::traitlets=4.3.1=py35_0 60 | - conda-forge::wcwidth=0.1.7=py35_0 61 | - conda-forge::wheel=0.29.0=py35_0 62 | - conda-forge::widgetsnbextension=1.2.6=py35_3 63 | - conda-forge::xz=5.2.2=0 64 | - conda-forge::zeromq=4.1.5=0 65 | - conda-forge::zlib=1.2.8=3 66 | - jupyter=1.0.0=py35_3 67 | - mkl=11.3.3=0 68 | - numpy=1.11.2=py35_0 69 | - qt=4.8.7=4 70 | - pip: 71 | - backports.shutil-get-terminal-size==1.0.0 72 | - ipython-genutils==0.1.0 73 | - jupyter-client==4.4.0 74 | - jupyter-console==5.0.0 75 | - jupyter-core==4.2.0 76 | - prompt-toolkit==1.0.9 77 | -------------------------------------------------------------------------------- /freemium/freemium.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Tour of Free(mium) HERE APIs\n", 8 | "\n", 9 | "A short \"teaser\" presentation rushing through a small subset of many free APIs made by [HERE Technologies](https://here.com) under the [Freemium plan](https://engage.here.com/freemium). This notebook shows simple examples mostly for geocoding, places, maps and routing. They are designed for rapid consumption during a meetup talk. To that end, some code snippets longer than a few lines are imported from a module named `utils.py`. Third-party modules are imported in the respective sections below as needed. (See `utils.py` for a rough requirements list.)\n", 10 | "\n", 11 | "**Goal:** Showing enough examples to *wet you appetite* for more, not delivering a polished \"paper\" or \"package\".\n", 12 | "\n", 13 | "**N.B.:** This notebook is saved intentionally without cells executed as some of those would contain the HERE credentials used." 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "## Freemium Plan\n", 21 | "\n", 22 | "- started in August 2018\n", 23 | "- large number of APIs included\n", 24 | "- 250,000 API calls/month\n", 25 | "- 5,000 monthly users on iOS/Android\n", 26 | "- no credit card needed\n", 27 | "- https://engage.here.com/freemium\n", 28 | "- https://developer.here.com/documentation" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "## Setup\n", 36 | "\n", 37 | "Credentials are imported from a `here_credentials.py` module if existing (via `utils.py`) defined as `app_id` and `app_code`, or from environment variables (`HEREMAPS_APP_ID`, `HEREMAPS_APP_CODE`)." 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "execution_count": null, 43 | "metadata": {}, 44 | "outputs": [], 45 | "source": [ 46 | "import random\n", 47 | "import urllib" 48 | ] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "execution_count": null, 53 | "metadata": {}, 54 | "outputs": [], 55 | "source": [ 56 | "import utils" 57 | ] 58 | }, 59 | { 60 | "cell_type": "code", 61 | "execution_count": null, 62 | "metadata": {}, 63 | "outputs": [], 64 | "source": [ 65 | "app_id = utils.app_id\n", 66 | "app_code = utils.app_code\n", 67 | "\n", 68 | "berlin_lat_lon = [52.5, 13.4]\n", 69 | "\n", 70 | "here_berlin_addr = 'Invalidenstr. 116, 10115 Berlin, Germany'" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "## Geocoding\n", 78 | "\n", 79 | "- documentation: https://developer.here.com/documentation#geocoder\n", 80 | "- raw REST\n", 81 | "- geopy plugin\n", 82 | "- geocoding\n", 83 | "- reverse geocoding" 84 | ] 85 | }, 86 | { 87 | "cell_type": "markdown", 88 | "metadata": {}, 89 | "source": [ 90 | "### Raw REST" 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": {}, 97 | "outputs": [], 98 | "source": [ 99 | "import requests" 100 | ] 101 | }, 102 | { 103 | "cell_type": "code", 104 | "execution_count": null, 105 | "metadata": {}, 106 | "outputs": [], 107 | "source": [ 108 | "here_berlin_addr" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "searchtext = urllib.parse.quote(here_berlin_addr)\n", 118 | "searchtext" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": null, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "url = (\n", 128 | " 'https://geocoder.api.here.com/6.2/geocode.json'\n", 129 | " f'?searchtext={searchtext}&app_id={app_id}&app_code={app_code}'\n", 130 | ")\n", 131 | "utils.mask_app_id(url)" 132 | ] 133 | }, 134 | { 135 | "cell_type": "code", 136 | "execution_count": null, 137 | "metadata": {}, 138 | "outputs": [], 139 | "source": [ 140 | "obj = requests.get(url).json()\n", 141 | "obj" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": null, 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "loc = obj['Response']['View'][0]['Result'][0]['Location']['DisplayPosition']\n", 151 | "loc['Latitude'], loc['Longitude']" 152 | ] 153 | }, 154 | { 155 | "cell_type": "markdown", 156 | "metadata": {}, 157 | "source": [ 158 | "### Geopy Plugin\n", 159 | "\n", 160 | "- \"geocoders united\"\n", 161 | "- HERE plugin in Geopy 1.15.0, https://github.com/geopy/geopy/releases/tag/1.15.0\n", 162 | "- ``pip install geopy>=1.15.0``\n", 163 | "- only most essential parts of geocoder API covered!" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": null, 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "from geopy.geocoders import Here" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": null, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [ 181 | "geocoder = Here(app_id, app_code)" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "here_berlin_addr" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": null, 196 | "metadata": {}, 197 | "outputs": [], 198 | "source": [ 199 | "loc = geocoder.geocode(here_berlin_addr)\n", 200 | "loc" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "metadata": {}, 207 | "outputs": [], 208 | "source": [ 209 | "loc.latitude, loc.longitude" 210 | ] 211 | }, 212 | { 213 | "cell_type": "code", 214 | "execution_count": null, 215 | "metadata": {}, 216 | "outputs": [], 217 | "source": [ 218 | "loc.raw" 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "execution_count": null, 224 | "metadata": {}, 225 | "outputs": [], 226 | "source": [ 227 | "here_berlin_lat_lon = loc.latitude, loc.longitude\n", 228 | "here_berlin_lat_lon" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "metadata": { 235 | "scrolled": true 236 | }, 237 | "outputs": [], 238 | "source": [ 239 | "loc = geocoder.reverse('{}, {}'.format(*here_berlin_lat_lon))\n", 240 | "loc" 241 | ] 242 | }, 243 | { 244 | "cell_type": "code", 245 | "execution_count": null, 246 | "metadata": {}, 247 | "outputs": [], 248 | "source": [ 249 | "loc.latitude, loc.longitude" 250 | ] 251 | }, 252 | { 253 | "cell_type": "markdown", 254 | "metadata": {}, 255 | "source": [ 256 | "## Places\n", 257 | "\n", 258 | "- https://developer.here.com/documentation#places\n", 259 | "- receive places within some neighbourhood" 260 | ] 261 | }, 262 | { 263 | "cell_type": "code", 264 | "execution_count": null, 265 | "metadata": {}, 266 | "outputs": [], 267 | "source": [ 268 | "searchtext = 'Cafe'\n", 269 | "lat, lon = here_berlin_lat_lon\n", 270 | "url = (\n", 271 | " 'https://places.api.here.com/places/v1/autosuggest'\n", 272 | " f'?q={searchtext}&at={lat},{lon}'\n", 273 | " f'&app_id={app_id}&app_code={app_code}'\n", 274 | ")\n", 275 | "utils.mask_app_id(url)" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "execution_count": null, 281 | "metadata": { 282 | "scrolled": true 283 | }, 284 | "outputs": [], 285 | "source": [ 286 | "obj = requests.get(url).json()\n", 287 | "obj" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": null, 293 | "metadata": {}, 294 | "outputs": [], 295 | "source": [ 296 | "for p in [res for res in obj['results'] if res['type']=='urn:nlp-types:place']:\n", 297 | " print('{!r:23} {:4d} m {}'.format(p['position'], p['distance'], p['title']))" 298 | ] 299 | }, 300 | { 301 | "cell_type": "markdown", 302 | "metadata": {}, 303 | "source": [ 304 | "## Maps\n", 305 | "\n", 306 | "- https://developer.here.com/documentation#map_tile\n", 307 | "- get single maptiles\n", 308 | "- use different maptiles\n", 309 | "- build entire maps\n", 310 | "- use folium, ipyleaflet & geopandas\n", 311 | "- revisit geocoding with shapes\n", 312 | "- draw these shapes" 313 | ] 314 | }, 315 | { 316 | "cell_type": "markdown", 317 | "metadata": {}, 318 | "source": [ 319 | "### Single Map Tiles" 320 | ] 321 | }, 322 | { 323 | "cell_type": "code", 324 | "execution_count": null, 325 | "metadata": {}, 326 | "outputs": [], 327 | "source": [ 328 | "from IPython.display import Image" 329 | ] 330 | }, 331 | { 332 | "cell_type": "code", 333 | "execution_count": null, 334 | "metadata": {}, 335 | "outputs": [], 336 | "source": [ 337 | "(lat, lon), zoom = berlin_lat_lon, 10\n", 338 | "xtile, ytile = utils.deg2tile(lat, lon, zoom)\n", 339 | "xtile, ytile" 340 | ] 341 | }, 342 | { 343 | "cell_type": "code", 344 | "execution_count": null, 345 | "metadata": {}, 346 | "outputs": [], 347 | "source": [ 348 | "# %load -s deg2tile utils\n", 349 | "def deg2tile(lat_deg, lon_deg, zoom):\n", 350 | " lat_rad = radians(lat_deg)\n", 351 | " n = 2.0 ** zoom\n", 352 | " xtile = int((lon_deg + 180.0) / 360.0 * n)\n", 353 | " ytile = int((1.0 - log(tan(lat_rad) + (1 / cos(lat_rad))) / pi) / 2.0 * n)\n", 354 | " return (xtile, ytile)\n", 355 | "\n", 356 | "\n", 357 | "# not used here\n" 358 | ] 359 | }, 360 | { 361 | "cell_type": "code", 362 | "execution_count": null, 363 | "metadata": { 364 | "scrolled": true 365 | }, 366 | "outputs": [], 367 | "source": [ 368 | "tiles_url = utils.build_here_tiles_url(\n", 369 | " maptype='base',\n", 370 | " tiletype='maptile',\n", 371 | " scheme='normal.day',\n", 372 | " x=xtile,\n", 373 | " y=ytile,\n", 374 | " z=zoom)" 375 | ] 376 | }, 377 | { 378 | "cell_type": "code", 379 | "execution_count": null, 380 | "metadata": {}, 381 | "outputs": [], 382 | "source": [ 383 | "utils.mask_app_id(tiles_url)" 384 | ] 385 | }, 386 | { 387 | "cell_type": "code", 388 | "execution_count": null, 389 | "metadata": {}, 390 | "outputs": [], 391 | "source": [ 392 | "img = Image(url=tiles_url)\n", 393 | "img" 394 | ] 395 | }, 396 | { 397 | "cell_type": "code", 398 | "execution_count": null, 399 | "metadata": {}, 400 | "outputs": [], 401 | "source": [ 402 | "# %load -s build_here_tiles_url utils\n", 403 | "def build_here_tiles_url(**kwdict):\n", 404 | " \"\"\"\n", 405 | " Return a HERE map tiles URL, based on default values that can be \n", 406 | " overwritten by kwdict...\n", 407 | " \n", 408 | " To be used for map building services like leaflet, folium, and \n", 409 | " geopandas (with additional fields inside a dict)...\n", 410 | " \"\"\"\n", 411 | " params = dict(\n", 412 | " app_id = app_id,\n", 413 | " app_code = app_code,\n", 414 | " maptype = 'traffic',\n", 415 | " tiletype = 'traffictile',\n", 416 | " scheme = 'normal.day',\n", 417 | " tilesize = '256',\n", 418 | " tileformat = 'png8',\n", 419 | " lg = 'eng',\n", 420 | " x = '{x}',\n", 421 | " y = '{y}',\n", 422 | " z = '{z}',\n", 423 | " server = random.choice('1234')\n", 424 | " )\n", 425 | " params.update(kwdict)\n", 426 | " url = (\n", 427 | " 'https://{server}.{maptype}.maps.api.here.com'\n", 428 | " '/maptile/2.1/{tiletype}/newest/{scheme}/{z}/{x}/{y}/{tilesize}/{tileformat}'\n", 429 | " '?lg={lg}&app_id={app_id}&app_code={app_code}'\n", 430 | " ).format(**params)\n", 431 | " return url\n" 432 | ] 433 | }, 434 | { 435 | "cell_type": "markdown", 436 | "metadata": {}, 437 | "source": [ 438 | "### Full Maps" 439 | ] 440 | }, 441 | { 442 | "cell_type": "code", 443 | "execution_count": null, 444 | "metadata": {}, 445 | "outputs": [], 446 | "source": [ 447 | "import folium" 448 | ] 449 | }, 450 | { 451 | "cell_type": "code", 452 | "execution_count": null, 453 | "metadata": {}, 454 | "outputs": [], 455 | "source": [ 456 | "folium.Map(location=berlin_lat_lon, zoom_start=10, tiles='Stamen Terrain')" 457 | ] 458 | }, 459 | { 460 | "cell_type": "code", 461 | "execution_count": null, 462 | "metadata": {}, 463 | "outputs": [], 464 | "source": [ 465 | "m = folium.Map(location=berlin_lat_lon, zoom_start=10)\n", 466 | "folium.GeoJson('stops_berlin.geojson', name='BVG Stops').add_to(m)\n", 467 | "folium.LayerControl().add_to(m)\n", 468 | "m" 469 | ] 470 | }, 471 | { 472 | "cell_type": "markdown", 473 | "metadata": {}, 474 | "source": [ 475 | "### Now HERE" 476 | ] 477 | }, 478 | { 479 | "cell_type": "code", 480 | "execution_count": null, 481 | "metadata": {}, 482 | "outputs": [], 483 | "source": [ 484 | "tiles_url = utils.build_here_tiles_url()" 485 | ] 486 | }, 487 | { 488 | "cell_type": "code", 489 | "execution_count": null, 490 | "metadata": {}, 491 | "outputs": [], 492 | "source": [ 493 | "utils.mask_app_id(tiles_url)" 494 | ] 495 | }, 496 | { 497 | "cell_type": "code", 498 | "execution_count": null, 499 | "metadata": {}, 500 | "outputs": [], 501 | "source": [ 502 | "folium.Map(\n", 503 | " location=berlin_lat_lon, \n", 504 | " zoom_start=10, \n", 505 | " tiles=tiles_url, \n", 506 | " attr='HERE.com')" 507 | ] 508 | }, 509 | { 510 | "cell_type": "markdown", 511 | "metadata": {}, 512 | "source": [ 513 | "### Geocoding Revisited\n", 514 | "\n", 515 | "- more GIS-savvy\n", 516 | "- (a litlle) more geo-spatial smarts" 517 | ] 518 | }, 519 | { 520 | "cell_type": "code", 521 | "execution_count": null, 522 | "metadata": {}, 523 | "outputs": [], 524 | "source": [ 525 | "%matplotlib inline" 526 | ] 527 | }, 528 | { 529 | "cell_type": "code", 530 | "execution_count": null, 531 | "metadata": {}, 532 | "outputs": [], 533 | "source": [ 534 | "import geopandas\n", 535 | "import shapely\n", 536 | "import shapely.wkt\n", 537 | "from geopy.geocoders import Here" 538 | ] 539 | }, 540 | { 541 | "cell_type": "code", 542 | "execution_count": null, 543 | "metadata": {}, 544 | "outputs": [], 545 | "source": [ 546 | "geocoder = Here(app_id, app_code)" 547 | ] 548 | }, 549 | { 550 | "cell_type": "code", 551 | "execution_count": null, 552 | "metadata": {}, 553 | "outputs": [], 554 | "source": [ 555 | "here_berlin_addr" 556 | ] 557 | }, 558 | { 559 | "cell_type": "code", 560 | "execution_count": null, 561 | "metadata": { 562 | "scrolled": true 563 | }, 564 | "outputs": [], 565 | "source": [ 566 | "loc = geocoder.geocode(\n", 567 | " here_berlin_addr, \n", 568 | " additional_data='IncludeShapeLevel,postalCode') # <- get shapes!\n", 569 | "loc.raw" 570 | ] 571 | }, 572 | { 573 | "cell_type": "code", 574 | "execution_count": null, 575 | "metadata": {}, 576 | "outputs": [], 577 | "source": [ 578 | "wkt_shape = loc.raw['Location']['Shape']['Value']" 579 | ] 580 | }, 581 | { 582 | "cell_type": "code", 583 | "execution_count": null, 584 | "metadata": {}, 585 | "outputs": [], 586 | "source": [ 587 | "shape = shapely.wkt.loads(wkt_shape)\n", 588 | "shape" 589 | ] 590 | }, 591 | { 592 | "cell_type": "code", 593 | "execution_count": null, 594 | "metadata": {}, 595 | "outputs": [], 596 | "source": [ 597 | "type(shape)" 598 | ] 599 | }, 600 | { 601 | "cell_type": "code", 602 | "execution_count": null, 603 | "metadata": {}, 604 | "outputs": [], 605 | "source": [ 606 | "here_berlin_point = shapely.geometry.Point(*reversed(here_berlin_lat_lon))\n", 607 | "here_berlin_point" 608 | ] 609 | }, 610 | { 611 | "cell_type": "code", 612 | "execution_count": null, 613 | "metadata": {}, 614 | "outputs": [], 615 | "source": [ 616 | "shape.contains(here_berlin_point)" 617 | ] 618 | }, 619 | { 620 | "cell_type": "code", 621 | "execution_count": null, 622 | "metadata": {}, 623 | "outputs": [], 624 | "source": [ 625 | "shape.contains(shapely.geometry.Point(0, 0))" 626 | ] 627 | }, 628 | { 629 | "cell_type": "code", 630 | "execution_count": null, 631 | "metadata": { 632 | "scrolled": false 633 | }, 634 | "outputs": [], 635 | "source": [ 636 | "data = [\n", 637 | " ['10115 Berlin', shape], \n", 638 | " ['HERE HQ', here_berlin_point]\n", 639 | "]\n", 640 | "df = geopandas.GeoDataFrame(data=data, columns=['object', 'geometry'])\n", 641 | "df" 642 | ] 643 | }, 644 | { 645 | "cell_type": "code", 646 | "execution_count": null, 647 | "metadata": {}, 648 | "outputs": [], 649 | "source": [ 650 | "url = utils.build_here_tiles_url(x='tileX', y='tileY', z='tileZ')" 651 | ] 652 | }, 653 | { 654 | "cell_type": "code", 655 | "execution_count": null, 656 | "metadata": {}, 657 | "outputs": [], 658 | "source": [ 659 | "utils.mask_app_id(url)" 660 | ] 661 | }, 662 | { 663 | "cell_type": "code", 664 | "execution_count": null, 665 | "metadata": {}, 666 | "outputs": [], 667 | "source": [ 668 | "df.crs = {'init': 'epsg:4326'} # dataframe is WGS84\n", 669 | "ax = df.plot(figsize=(10, 10), alpha=0.5, edgecolor='k')\n", 670 | "utils.add_basemap(ax, zoom=15, url=url)" 671 | ] 672 | }, 673 | { 674 | "cell_type": "code", 675 | "execution_count": null, 676 | "metadata": {}, 677 | "outputs": [], 678 | "source": [ 679 | "# %load -s add_basemap utils\n", 680 | "def add_basemap(ax, zoom, url='http://tile.stamen.com/terrain/tileZ/tileX/tileY.png'):\n", 681 | " # Special thanks to Prof. Martin Christen at FHNW.ch in Basel for\n", 682 | " # his GIS-Hack to make the output scales show proper lat/lon values!\n", 683 | " xmin, xmax, ymin, ymax = ax.axis()\n", 684 | " basemap, extent = ctx.bounds2img(xmin, ymin, xmax, ymax, zoom=zoom, ll=True, url=url)\n", 685 | " \n", 686 | " # calculate extent from WebMercator to WGS84\n", 687 | " xmin84, ymin84 = Mercator2WGS84(extent[0], extent[2])\n", 688 | " xmax84, ymax84 = Mercator2WGS84(extent[1], extent[3])\n", 689 | " extentwgs84 = (xmin84, xmax84, ymin84, ymax84)\n", 690 | " \n", 691 | " ax.imshow(basemap, extent=extentwgs84, interpolation='bilinear')\n", 692 | " # restore original x/y limits\n", 693 | " ax.axis((xmin, xmax, ymin, ymax))\n" 694 | ] 695 | }, 696 | { 697 | "cell_type": "markdown", 698 | "metadata": {}, 699 | "source": [ 700 | "## Routing\n", 701 | "\n", 702 | "- https://developer.here.com/documentation#routing\n", 703 | "- routes\n", 704 | "- modes\n", 705 | "- maneuvers\n", 706 | "- ipyleaflet preferred over folium" 707 | ] 708 | }, 709 | { 710 | "cell_type": "code", 711 | "execution_count": null, 712 | "metadata": {}, 713 | "outputs": [], 714 | "source": [ 715 | "from ipyleaflet import Map, Marker, CircleMarker, Polyline, basemap_to_tiles\n", 716 | "from ipywidgets import HTML" 717 | ] 718 | }, 719 | { 720 | "cell_type": "code", 721 | "execution_count": null, 722 | "metadata": {}, 723 | "outputs": [], 724 | "source": [ 725 | "here_berlin_addr" 726 | ] 727 | }, 728 | { 729 | "cell_type": "code", 730 | "execution_count": null, 731 | "metadata": {}, 732 | "outputs": [], 733 | "source": [ 734 | "here_berlin_lat_lon" 735 | ] 736 | }, 737 | { 738 | "cell_type": "code", 739 | "execution_count": null, 740 | "metadata": {}, 741 | "outputs": [], 742 | "source": [ 743 | "dt_oper_berlin_addr = 'Bismarkstr. 35, 10627 Berlin, Germany'" 744 | ] 745 | }, 746 | { 747 | "cell_type": "code", 748 | "execution_count": null, 749 | "metadata": {}, 750 | "outputs": [], 751 | "source": [ 752 | "loc = geocoder.geocode(dt_oper_berlin_addr)\n", 753 | "dt_oper_berlin_lat_lon = loc.latitude, loc.longitude\n", 754 | "dt_oper_berlin_lat_lon" 755 | ] 756 | }, 757 | { 758 | "cell_type": "code", 759 | "execution_count": null, 760 | "metadata": {}, 761 | "outputs": [], 762 | "source": [ 763 | "route = utils.get_route_positions(\n", 764 | " here_berlin_lat_lon, \n", 765 | " dt_oper_berlin_lat_lon,\n", 766 | " mode='fastest;car;traffic:disabled',\n", 767 | " language='en')" 768 | ] 769 | }, 770 | { 771 | "cell_type": "code", 772 | "execution_count": null, 773 | "metadata": {}, 774 | "outputs": [], 775 | "source": [ 776 | "route" 777 | ] 778 | }, 779 | { 780 | "cell_type": "code", 781 | "execution_count": null, 782 | "metadata": { 783 | "scrolled": false 784 | }, 785 | "outputs": [], 786 | "source": [ 787 | "center = utils.mid_point(\n", 788 | " here_berlin_lat_lon, \n", 789 | " dt_oper_berlin_lat_lon)\n", 790 | "here_basemap = utils.build_here_basemap()\n", 791 | "layers = [basemap_to_tiles(here_basemap)]\n", 792 | "m = Map(center=center, layers=layers, zoom=13)\n", 793 | "m" 794 | ] 795 | }, 796 | { 797 | "cell_type": "code", 798 | "execution_count": null, 799 | "metadata": {}, 800 | "outputs": [], 801 | "source": [ 802 | "route[0]['shape'][:4]" 803 | ] 804 | }, 805 | { 806 | "cell_type": "code", 807 | "execution_count": null, 808 | "metadata": {}, 809 | "outputs": [], 810 | "source": [ 811 | "path = list(utils.chunks(route[0]['shape'], 2))\n", 812 | "path[:2]" 813 | ] 814 | }, 815 | { 816 | "cell_type": "code", 817 | "execution_count": null, 818 | "metadata": {}, 819 | "outputs": [], 820 | "source": [ 821 | "sum(map(lambda pq: utils.geo_distance(*pq), list(utils.pairwise(path))))" 822 | ] 823 | }, 824 | { 825 | "cell_type": "code", 826 | "execution_count": null, 827 | "metadata": {}, 828 | "outputs": [], 829 | "source": [ 830 | "m += Polyline(locations=path, color='red', fill=False)" 831 | ] 832 | }, 833 | { 834 | "cell_type": "code", 835 | "execution_count": null, 836 | "metadata": {}, 837 | "outputs": [], 838 | "source": [ 839 | "for man in route[0]['leg'][0]['maneuver']:\n", 840 | " lat = man['position']['latitude']\n", 841 | " lon = man['position']['longitude']\n", 842 | " desc = man['instruction']\n", 843 | " marker = Marker(location=(lat, lon), draggable=False)\n", 844 | " marker.popup = HTML(value=desc)\n", 845 | " m += marker" 846 | ] 847 | }, 848 | { 849 | "cell_type": "code", 850 | "execution_count": null, 851 | "metadata": {}, 852 | "outputs": [], 853 | "source": [ 854 | "for lat, lon in path:\n", 855 | " m += CircleMarker(location=(lat, lon), radius=3, color='blue')" 856 | ] 857 | }, 858 | { 859 | "cell_type": "code", 860 | "execution_count": null, 861 | "metadata": {}, 862 | "outputs": [], 863 | "source": [ 864 | "reverse_route = utils.get_route_positions(\n", 865 | " dt_oper_berlin_lat_lon,\n", 866 | " here_berlin_lat_lon,\n", 867 | " mode='shortest;pedestrian',\n", 868 | " language='en')" 869 | ] 870 | }, 871 | { 872 | "cell_type": "code", 873 | "execution_count": null, 874 | "metadata": {}, 875 | "outputs": [], 876 | "source": [ 877 | "utils.add_route_to_map(reverse_route, m)" 878 | ] 879 | }, 880 | { 881 | "cell_type": "code", 882 | "execution_count": null, 883 | "metadata": {}, 884 | "outputs": [], 885 | "source": [ 886 | "path = list(utils.chunks(reverse_route[0]['shape'], 2))\n", 887 | "sum(map(lambda pq: utils.geo_distance(*pq), list(utils.pairwise(path))))" 888 | ] 889 | }, 890 | { 891 | "cell_type": "code", 892 | "execution_count": null, 893 | "metadata": {}, 894 | "outputs": [], 895 | "source": [ 896 | "# %load -s add_route_to_map utils.py\n", 897 | "def add_route_to_map(route, some_map, color='blue'):\n", 898 | " \"\"\"\n", 899 | " Add a route from the HERE REST API to the given map.\n", 900 | " \n", 901 | " This includes markers for all points where a maneuver is needed, like 'turn left'.\n", 902 | " And it includes a path with lat/lons from start to end and little circle markers\n", 903 | " around them.\n", 904 | " \"\"\"\n", 905 | " path_positions = list(chunks(route[0]['shape'], 2))\n", 906 | " maneuvers = {\n", 907 | " (man['position']['latitude'], man['position']['longitude']): man['instruction']\n", 908 | " for man in route[0]['leg'][0]['maneuver']}\n", 909 | "\n", 910 | " polyline = Polyline(\n", 911 | " locations=path_positions,\n", 912 | " color=color,\n", 913 | " fill=False\n", 914 | " )\n", 915 | " some_map += polyline\n", 916 | " \n", 917 | " for lat, lon in path_positions:\n", 918 | " if (lat, lon) in maneuvers:\n", 919 | " some_map += CircleMarker(location=(lat, lon), radius=2)\n", 920 | " \n", 921 | " marker = Marker(location=(lat, lon), draggable=False)\n", 922 | " message1 = HTML()\n", 923 | " message1.value = maneuvers[(lat, lon)]\n", 924 | " marker.popup = message1\n", 925 | " some_map += marker\n", 926 | " else:\n", 927 | " some_map += CircleMarker(location=(lat, lon), radius=3)\n" 928 | ] 929 | }, 930 | { 931 | "cell_type": "markdown", 932 | "metadata": {}, 933 | "source": [ 934 | "### Isolines" 935 | ] 936 | }, 937 | { 938 | "cell_type": "code", 939 | "execution_count": null, 940 | "metadata": {}, 941 | "outputs": [], 942 | "source": [ 943 | "import requests\n", 944 | "import ipywidgets as widgets" 945 | ] 946 | }, 947 | { 948 | "cell_type": "code", 949 | "execution_count": null, 950 | "metadata": {}, 951 | "outputs": [], 952 | "source": [ 953 | "lat, lon = here_berlin_lat_lon\n", 954 | "url = (\n", 955 | " 'https://isoline.route.api.here.com'\n", 956 | " '/routing/7.2/calculateisoline.json'\n", 957 | " f'?app_id={app_id}&app_code={app_code}' \n", 958 | " f'&start=geo!{lat},{lon}'\n", 959 | " '&mode=fastest;car;traffic:disabled'\n", 960 | " '&range=300,600' # seconds/meters\n", 961 | " '&rangetype=time' # time/distance\n", 962 | " #'&departure=now' # 2013-07-04T17:00:00+02\n", 963 | " #'&resolution=20' # meters\n", 964 | ")\n", 965 | "obj = requests.get(url).json()" 966 | ] 967 | }, 968 | { 969 | "cell_type": "code", 970 | "execution_count": null, 971 | "metadata": {}, 972 | "outputs": [], 973 | "source": [ 974 | "obj" 975 | ] 976 | }, 977 | { 978 | "cell_type": "code", 979 | "execution_count": null, 980 | "metadata": {}, 981 | "outputs": [], 982 | "source": [ 983 | "here_basemap = utils.build_here_basemap()\n", 984 | "layers = [basemap_to_tiles(here_basemap)]\n", 985 | "m = Map(center=(lat, lon), layers=layers, zoom=12)\n", 986 | "m" 987 | ] 988 | }, 989 | { 990 | "cell_type": "code", 991 | "execution_count": null, 992 | "metadata": {}, 993 | "outputs": [], 994 | "source": [ 995 | "m += Marker(location=(lat, lon))" 996 | ] 997 | }, 998 | { 999 | "cell_type": "code", 1000 | "execution_count": null, 1001 | "metadata": {}, 1002 | "outputs": [], 1003 | "source": [ 1004 | "for isoline in obj['response']['isoline']:\n", 1005 | " shape = isoline['component'][0]['shape']\n", 1006 | " path = [tuple(map(float, pos.split(','))) for pos in shape]\n", 1007 | " m += Polyline(locations=path, color='red', weight=2, fill=True)" 1008 | ] 1009 | }, 1010 | { 1011 | "cell_type": "markdown", 1012 | "metadata": {}, 1013 | "source": [ 1014 | "#### More interactively" 1015 | ] 1016 | }, 1017 | { 1018 | "cell_type": "code", 1019 | "execution_count": null, 1020 | "metadata": {}, 1021 | "outputs": [], 1022 | "source": [ 1023 | "here_basemap = utils.build_here_basemap()\n", 1024 | "layers = [basemap_to_tiles(here_basemap)]\n", 1025 | "m = Map(center=(lat, lon), layers=layers, zoom=13)\n", 1026 | "m" 1027 | ] 1028 | }, 1029 | { 1030 | "cell_type": "code", 1031 | "execution_count": null, 1032 | "metadata": {}, 1033 | "outputs": [], 1034 | "source": [ 1035 | "lat, lon = here_berlin_lat_lon" 1036 | ] 1037 | }, 1038 | { 1039 | "cell_type": "code", 1040 | "execution_count": null, 1041 | "metadata": {}, 1042 | "outputs": [], 1043 | "source": [ 1044 | "dist_iso = utils.Isoline(m, \n", 1045 | " lat=lat, lon=lon, \n", 1046 | " app_id=app_id, app_code=app_code)" 1047 | ] 1048 | }, 1049 | { 1050 | "cell_type": "code", 1051 | "execution_count": null, 1052 | "metadata": {}, 1053 | "outputs": [], 1054 | "source": [ 1055 | "# can't get this working directly on dist_iso with __call__ :(\n", 1056 | "def dist_iso_func(meters=1000):\n", 1057 | " dist_iso(meters=meters)" 1058 | ] 1059 | }, 1060 | { 1061 | "cell_type": "code", 1062 | "execution_count": null, 1063 | "metadata": {}, 1064 | "outputs": [], 1065 | "source": [ 1066 | "widgets.interact(dist_iso_func, meters=(1000, 2000, 200))" 1067 | ] 1068 | }, 1069 | { 1070 | "cell_type": "code", 1071 | "execution_count": null, 1072 | "metadata": {}, 1073 | "outputs": [], 1074 | "source": [ 1075 | "# %load -s Isoline utils\n", 1076 | "class Isoline(object):\n", 1077 | " def __init__(self, the_map, **kwdict):\n", 1078 | " self.the_map = the_map\n", 1079 | " self.isoline = None\n", 1080 | " self.url = (\n", 1081 | " 'https://isoline.route.api.here.com'\n", 1082 | " '/routing/7.2/calculateisoline.json'\n", 1083 | " '?app_id={app_id}&app_code={app_code}' \n", 1084 | " '&start=geo!{lat},{lon}'\n", 1085 | " '&mode=fastest;car;traffic:disabled'\n", 1086 | " '&range={{meters}}' # seconds/meters\n", 1087 | " '&rangetype=distance' # time/distance\n", 1088 | " #'&departure=now' # 2013-07-04T17:00:00+02\n", 1089 | " #'&resolution=20' # meters\n", 1090 | " ).format(**kwdict)\n", 1091 | " self.cache = {}\n", 1092 | "\n", 1093 | " def __call__(self, meters=1000):\n", 1094 | " if meters not in self.cache:\n", 1095 | " print('loading', meters)\n", 1096 | " url = self.url.format(meters=meters)\n", 1097 | " obj = requests.get(url).json()\n", 1098 | " self.cache[meters] = obj\n", 1099 | " obj = self.cache[meters]\n", 1100 | " isoline = obj['response']['isoline'][0]\n", 1101 | " shape = isoline['component'][0]['shape']\n", 1102 | " path = [tuple(map(float, pos.split(','))) for pos in shape]\n", 1103 | " if self.isoline:\n", 1104 | " self.the_map -= self.isoline\n", 1105 | " self.isoline = Polyline(locations=path, color='red', weight=2, fill=True)\n", 1106 | " self.the_map += self.isoline\n" 1107 | ] 1108 | }, 1109 | { 1110 | "cell_type": "markdown", 1111 | "metadata": {}, 1112 | "source": [ 1113 | "## More to come... (in another meetup ;)\n", 1114 | "\n", 1115 | "- dynamic map content (based on traitlets)\n", 1116 | "- streaming data\n", 1117 | "- ZeroMQ integration\n", 1118 | "- sneak preview below" 1119 | ] 1120 | }, 1121 | { 1122 | "cell_type": "code", 1123 | "execution_count": null, 1124 | "metadata": {}, 1125 | "outputs": [], 1126 | "source": [ 1127 | "here_basemap = utils.build_here_basemap()\n", 1128 | "layers = [basemap_to_tiles(here_basemap)]\n", 1129 | "m = Map(center=berlin_lat_lon, layers=layers, zoom=13)\n", 1130 | "m" 1131 | ] 1132 | }, 1133 | { 1134 | "cell_type": "code", 1135 | "execution_count": null, 1136 | "metadata": {}, 1137 | "outputs": [], 1138 | "source": [ 1139 | "marker = Marker(location=berlin_lat_lon)\n", 1140 | "marker.location" 1141 | ] 1142 | }, 1143 | { 1144 | "cell_type": "code", 1145 | "execution_count": null, 1146 | "metadata": {}, 1147 | "outputs": [], 1148 | "source": [ 1149 | "m += marker" 1150 | ] 1151 | }, 1152 | { 1153 | "cell_type": "code", 1154 | "execution_count": null, 1155 | "metadata": {}, 1156 | "outputs": [], 1157 | "source": [ 1158 | "m -= marker" 1159 | ] 1160 | }, 1161 | { 1162 | "cell_type": "code", 1163 | "execution_count": null, 1164 | "metadata": {}, 1165 | "outputs": [], 1166 | "source": [ 1167 | "m += marker" 1168 | ] 1169 | }, 1170 | { 1171 | "cell_type": "code", 1172 | "execution_count": null, 1173 | "metadata": {}, 1174 | "outputs": [], 1175 | "source": [ 1176 | "marker.location = [52.49, 13.39]" 1177 | ] 1178 | }, 1179 | { 1180 | "cell_type": "code", 1181 | "execution_count": null, 1182 | "metadata": {}, 1183 | "outputs": [], 1184 | "source": [ 1185 | "loc = marker.location\n", 1186 | "for i in range(5000):\n", 1187 | " d_lat = (random.random() - 0.5) / 100\n", 1188 | " d_lon = (random.random() - 0.5) / 100\n", 1189 | " marker.location = [loc[0] + d_lat, loc[1] + d_lon]" 1190 | ] 1191 | }, 1192 | { 1193 | "cell_type": "markdown", 1194 | "metadata": {}, 1195 | "source": [ 1196 | "## Take-Aways\n", 1197 | "\n", 1198 | "- HERE Freemium rocks!\n", 1199 | "- Jupyter rocks!\n", 1200 | "- Jupyter Lab rocks! (not shown here ;)\n", 1201 | "- GeoPandas rocks! (not much shown here ;)\n", 1202 | "- Ipyleaflet rocks!\n", 1203 | "- Ipywidgets rock!\n", 1204 | "\n", 1205 | "Q: With all of this in your hands, what will **you** rock?!" 1206 | ] 1207 | } 1208 | ], 1209 | "metadata": { 1210 | "kernelspec": { 1211 | "display_name": "Python 3", 1212 | "language": "python", 1213 | "name": "python3" 1214 | }, 1215 | "language_info": { 1216 | "codemirror_mode": { 1217 | "name": "ipython", 1218 | "version": 3 1219 | }, 1220 | "file_extension": ".py", 1221 | "mimetype": "text/x-python", 1222 | "name": "python", 1223 | "nbconvert_exporter": "python", 1224 | "pygments_lexer": "ipython3", 1225 | "version": "3.6.5" 1226 | } 1227 | }, 1228 | "nbformat": 4, 1229 | "nbformat_minor": 2 1230 | } 1231 | -------------------------------------------------------------------------------- /freemium/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utilities, ... 3 | 4 | Requirements (not strictly tested from scratch, sorry!) 5 | 6 | conda install -y rasterio # pulls in many other things 7 | conda install -c conda-forge ipyleaflet 8 | # jupyter labextension install jupyter-leaflet # for jupyterlab 9 | conda install -c conda-forge ipywidgets 10 | pip install requests 11 | pip install folium 12 | pip install pillow 13 | pip install mercantile 14 | pip install contextily 15 | pip install geographiclib 16 | pip install geopy>=1.15.0 17 | 18 | It is recommended to start with an Anaconda distribution. 19 | """ 20 | 21 | import re 22 | import os 23 | import sys 24 | import random 25 | from itertools import tee 26 | 27 | import requests 28 | import contextily as ctx 29 | from geographiclib.geodesic import Geodesic 30 | from geopy.geocoders import Here 31 | from geopy.distance import geodesic 32 | from ipyleaflet import Marker, CircleMarker, Polyline 33 | from ipywidgets import HTML 34 | from pyproj import Proj, transform 35 | 36 | 37 | app_id = os.getenv('HEREMAPS_APP_ID') 38 | app_code = os.getenv('HEREMAPS_APP_CODE') 39 | if not app_id or not app_code: 40 | try: 41 | from here_credentials import app_id, app_code 42 | except ImportError: 43 | raise ValueError('Cannot find value for APP_ID and/or APP_CODE...') 44 | 45 | geocoder = Here(app_id=app_id, app_code=app_code) 46 | 47 | 48 | def mask_app_id(text): 49 | "Mask out credentials in given string for presentations." 50 | 51 | masked = re.sub('app_id=[\-\w]+', 'app_id=******', text) 52 | masked = re.sub('app_code=[\-\w]+', 'app_code=******', masked) 53 | return masked 54 | 55 | 56 | # Conversion between lat/lon in degrees (and zoom) to x/y/zoom as used in tile sets, 57 | # from http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Python 58 | 59 | from math import radians, degrees, log, cos, tan, pi, atan, sinh 60 | 61 | def deg2tile(lat_deg, lon_deg, zoom): 62 | lat_rad = radians(lat_deg) 63 | n = 2.0 ** zoom 64 | xtile = int((lon_deg + 180.0) / 360.0 * n) 65 | ytile = int((1.0 - log(tan(lat_rad) + (1 / cos(lat_rad))) / pi) / 2.0 * n) 66 | return (xtile, ytile) 67 | 68 | 69 | # not used here 70 | def tile2deg(xtile, ytile, zoom): 71 | n = 2.0 ** zoom 72 | lon_deg = xtile / n * 360.0 - 180.0 73 | lat_rad = atan(sinh(pi * (1 - 2 * ytile / n))) 74 | lat_deg = degrees(lat_rad) 75 | return (lat_deg, lon_deg) 76 | 77 | 78 | def pairwise(iterable): 79 | "s -> (s0,s1), (s1,s2), (s2, s3), ..." 80 | a, b = tee(iterable) 81 | next(b, None) 82 | return zip(a, b) 83 | 84 | 85 | def chunks(seq, n): 86 | "Yield successive n-sized chunks from l." 87 | # for item in zip(*(iter(seq),) * n): yield item 88 | for i in range(0, len(seq), n): 89 | yield seq[i:i + n] 90 | 91 | 92 | # not used here 93 | def latlon_for_address(address): 94 | "Return a lat/lon tuple for the given address by geocoding it." 95 | res = geocoder.geocode(address) 96 | return res.latitude, res.longitude 97 | 98 | 99 | def build_here_tiles_url(**kwdict): 100 | """ 101 | Return a HERE map tiles URL, based on default values that can be 102 | overwritten by kwdict... 103 | 104 | To be used for map building services like leaflet, folium, and 105 | geopandas (with additional fields inside a dict)... 106 | """ 107 | params = dict( 108 | app_id = app_id, 109 | app_code = app_code, 110 | maptype = 'traffic', 111 | tiletype = 'traffictile', 112 | scheme = 'normal.day', 113 | tilesize = '256', 114 | tileformat = 'png8', 115 | lg = 'eng', 116 | x = '{x}', 117 | y = '{y}', 118 | z = '{z}', 119 | server = random.choice('1234') 120 | ) 121 | params.update(kwdict) 122 | url = ( 123 | 'https://{server}.{maptype}.maps.api.here.com' 124 | '/maptile/2.1/{tiletype}/newest/{scheme}/{z}/{x}/{y}/{tilesize}/{tileformat}' 125 | '?lg={lg}&app_id={app_id}&app_code={app_code}' 126 | ).format(**params) 127 | return url 128 | 129 | 130 | def build_here_basemap(**kwdict): 131 | """ 132 | Return a dict HERE map tiles URL, based on default values that can be 133 | overwritten with params in kwdict... 134 | 135 | To be used for ipyleaflet. 136 | """ 137 | params = dict( 138 | url = build_here_tiles_url(**kwdict), 139 | min_zoom = 1, 140 | max_zoom = 18, 141 | attribution = 'Tiles © HERE.com', 142 | name = 'HERE' 143 | ) 144 | params.update(kwdict) 145 | return params 146 | 147 | 148 | def get_route_positions(start, end, **kwargs): 149 | """ 150 | Get routing data. 151 | """ 152 | lat0, lon0 = start 153 | lat1, lon1 = end 154 | params = dict( 155 | language='en', 156 | mode='fastest;car;traffic:disabled' 157 | ) 158 | params.update(kwargs) 159 | url = ( 160 | f'https://route.cit.api.here.com' 161 | f'/routing/7.2/calculateroute.json' 162 | f'?app_id={app_id}&app_code={app_code}' 163 | # f'&waypoint0=street!{addr}' 164 | f'&waypoint0=geo!{lat0},{lon0}' 165 | f'&waypoint1=geo!{lat1},{lon1}' 166 | # f'&language={language}' 167 | # f'&mode=fastest;car;traffic:disabled' 168 | f'&metricsystem=metric' 169 | f'&jsonattributes=41' # ? 170 | # f'maneuverattributes=po,ti,pt,ac,di,fj,ix' # ? 171 | f'&routeattributes=sh,gr' 172 | f'&instructionFormat=text' # or html 173 | # f'&mode=fastest;publicTransport&combineChange=true&departure=now' 174 | ) 175 | for key in params: 176 | val = params[key] 177 | url += f'&{key}={val}' 178 | obj = requests.get(url).json() 179 | return obj['response']['route'] 180 | 181 | leg = obj['response']['route'][0]['leg'] 182 | res = [] 183 | for man in leg[0]['maneuver']: 184 | pos = man['position'] 185 | lat, lon = pos['latitude'], pos['longitude'] 186 | inst = man['instruction'] 187 | res.append(dict(lat=lat, lon=lon, maneuver=inst)) 188 | return res 189 | 190 | 191 | def add_route_to_map(route, some_map, color='blue'): 192 | """ 193 | Add a route from the HERE REST API to the given map. 194 | 195 | This includes markers for all points where a maneuver is needed, like 'turn left'. 196 | And it includes a path with lat/lons from start to end and little circle markers 197 | around them. 198 | """ 199 | path_positions = list(chunks(route[0]['shape'], 2)) 200 | maneuvers = { 201 | (man['position']['latitude'], man['position']['longitude']): man['instruction'] 202 | for man in route[0]['leg'][0]['maneuver']} 203 | 204 | polyline = Polyline( 205 | locations=path_positions, 206 | color=color, 207 | fill=False 208 | ) 209 | some_map += polyline 210 | 211 | for lat, lon in path_positions: 212 | if (lat, lon) in maneuvers: 213 | some_map += CircleMarker(location=(lat, lon), radius=2) 214 | 215 | marker = Marker(location=(lat, lon), draggable=False) 216 | message1 = HTML() 217 | message1.value = maneuvers[(lat, lon)] 218 | marker.popup = message1 219 | some_map += marker 220 | else: 221 | some_map += CircleMarker(location=(lat, lon), radius=3) 222 | 223 | 224 | def geo_distance(p, q): 225 | "Return the geodesic distance from point p to q (both lat/lon pairs) in meters." 226 | 227 | (lat0, lon0), (lat1, lon1) = p, q 228 | g = Geodesic.WGS84.Inverse(lat0, lon0, lat1, lon1) 229 | return g['s12'] 230 | 231 | 232 | def mid_point(loc1, loc2): 233 | """ 234 | Calculate point in the geodesic middle between two given points. 235 | """ 236 | geod = Geodesic.WGS84 237 | inv_line = geod.InverseLine(*(loc1 + loc2)) 238 | distance_m = geod.Inverse(*(loc1 + loc2))["s12"] 239 | loc = inv_line.Position(distance_m / 2, Geodesic.STANDARD | Geodesic.LONG_UNROLL) 240 | lat, lon = loc['lat2'], loc['lon2'] 241 | return lat, lon 242 | 243 | 244 | class Isoline(object): 245 | def __init__(self, the_map, **kwdict): 246 | self.the_map = the_map 247 | self.isoline = None 248 | self.url = ( 249 | 'https://isoline.route.api.here.com' 250 | '/routing/7.2/calculateisoline.json' 251 | '?app_id={app_id}&app_code={app_code}' 252 | '&start=geo!{lat},{lon}' 253 | '&mode=fastest;car;traffic:disabled' 254 | '&range={{meters}}' # seconds/meters 255 | '&rangetype=distance' # time/distance 256 | #'&departure=now' # 2013-07-04T17:00:00+02 257 | #'&resolution=20' # meters 258 | ).format(**kwdict) 259 | self.cache = {} 260 | 261 | def __call__(self, meters=1000): 262 | if meters not in self.cache: 263 | print('loading', meters) 264 | url = self.url.format(meters=meters) 265 | obj = requests.get(url).json() 266 | self.cache[meters] = obj 267 | obj = self.cache[meters] 268 | isoline = obj['response']['isoline'][0] 269 | shape = isoline['component'][0]['shape'] 270 | path = [tuple(map(float, pos.split(','))) for pos in shape] 271 | if self.isoline: 272 | self.the_map -= self.isoline 273 | self.isoline = Polyline(locations=path, color='red', weight=2, fill=True) 274 | self.the_map += self.isoline 275 | 276 | 277 | def Mercator2WGS84(x, y): 278 | return transform(Proj(init='epsg:3857'), Proj(init='epsg:4326'), x, y) 279 | 280 | def add_basemap(ax, zoom, url='http://tile.stamen.com/terrain/tileZ/tileX/tileY.png'): 281 | # Special thanks to Prof. Martin Christen at FHNW.ch in Basel for 282 | # his GIS-Hack to make the output scales show proper lat/lon values! 283 | xmin, xmax, ymin, ymax = ax.axis() 284 | basemap, extent = ctx.bounds2img(xmin, ymin, xmax, ymax, zoom=zoom, ll=True, url=url) 285 | 286 | # calculate extent from WebMercator to WGS84 287 | xmin84, ymin84 = Mercator2WGS84(extent[0], extent[2]) 288 | xmax84, ymax84 = Mercator2WGS84(extent[1], extent[3]) 289 | extentwgs84 = (xmin84, xmax84, ymin84, ymax84) 290 | 291 | ax.imshow(basemap, extent=extentwgs84, interpolation='bilinear') 292 | # restore original x/y limits 293 | ax.axis((xmin, xmax, ymin, ymax)) -------------------------------------------------------------------------------- /mapping/geodesic_polylines.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Using truly geodesic polylines with Folium\n", 8 | "\n", 9 | "This notebook shows how long straight lines are unusable on map projections like the very popular Mercator one, also used in [Folium](https://github.com/python-visualization/folium). And it implements a more detailed version with intermediate points in order to get a more realistic visualization of long lines on such map projections. These might look unusual (except for pilots), but they are much closer to the truth.\n", 10 | "\n", 11 | "You will need to pip-install `folium` and `geographiclib` for this notebook to work as expected. If you look at this inside a GitHub repo you won't see the maps. Then try on the notebook viewer, [there you will](http://nbviewer.jupyter.org/github/deeplook/notebooks/blob/master/mapping/geodesic_polylines.ipynb). " 12 | ] 13 | }, 14 | { 15 | "cell_type": "code", 16 | "execution_count": 1, 17 | "metadata": { 18 | "collapsed": true 19 | }, 20 | "outputs": [], 21 | "source": [ 22 | "%matplotlib inline\n", 23 | "\n", 24 | "import math\n", 25 | "\n", 26 | "import folium" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 2, 32 | "metadata": { 33 | "collapsed": true 34 | }, 35 | "outputs": [], 36 | "source": [ 37 | "la = 34.05351, -118.24531\n", 38 | "nyc = 40.71453, -74.00713\n", 39 | "berlin = 52.516071, 13.37698\n", 40 | "potsdam = 52.39962, 13.04784\n", 41 | "singapore = 1.29017, 103.852\n", 42 | "sydney = -33.86971, 151.20711" 43 | ] 44 | }, 45 | { 46 | "cell_type": "markdown", 47 | "metadata": {}, 48 | "source": [ 49 | "## Straight lines" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 3, 55 | "metadata": { 56 | "collapsed": true 57 | }, 58 | "outputs": [], 59 | "source": [ 60 | "map = folium.Map()" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 4, 66 | "metadata": {}, 67 | "outputs": [ 68 | { 69 | "data": { 70 | "text/plain": [ 71 | "" 72 | ] 73 | }, 74 | "execution_count": 4, 75 | "metadata": {}, 76 | "output_type": "execute_result" 77 | } 78 | ], 79 | "source": [ 80 | "folium.Marker(location=la, popup=\"Los Angeles\").add_to(map)\n", 81 | "folium.Marker(location=nyc, popup=\"New York\").add_to(map)\n", 82 | "folium.Marker(location=berlin, popup=\"Berlin\").add_to(map)\n", 83 | "folium.Marker(location=singapore, popup=\"Singapore\").add_to(map)\n", 84 | "folium.Marker(location=sydney, popup=\"Sydney\").add_to(map)\n", 85 | "\n", 86 | "folium.PolyLine([la, nyc, berlin, singapore, sydney],\n", 87 | " weight=2,\n", 88 | " color=\"red\"\n", 89 | ").add_to(map)" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": 5, 95 | "metadata": {}, 96 | "outputs": [ 97 | { 98 | "data": { 99 | "text/html": [ 100 | "
" 101 | ], 102 | "text/plain": [ 103 | "" 104 | ] 105 | }, 106 | "execution_count": 5, 107 | "metadata": {}, 108 | "output_type": "execute_result" 109 | } 110 | ], 111 | "source": [ 112 | "map" 113 | ] 114 | }, 115 | { 116 | "cell_type": "markdown", 117 | "metadata": {}, 118 | "source": [ 119 | "## Geodesic lines" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": 6, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "from geographiclib.geodesic import Geodesic" 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 7, 134 | "metadata": {}, 135 | "outputs": [], 136 | "source": [ 137 | "def intermediatePoints(start, end, min_length_km=2000, segment_length_km=1000):\n", 138 | " geod = Geodesic.WGS84\n", 139 | " if geod.Inverse(*(start + end))[\"s12\"] / 1000.0 < min_length_km:\n", 140 | " yield start\n", 141 | " yield end\n", 142 | " else:\n", 143 | " inv_line = geod.InverseLine(*(start + end))\n", 144 | " segment_length_m = 1000 * segment_length_km\n", 145 | " n = int(math.ceil(inv_line.s13 / segment_length_m))\n", 146 | " for i in range(n + 1):\n", 147 | " s = min(segment_length_m * i, inv_line.s13)\n", 148 | " g = inv_line.Position(s, Geodesic.STANDARD | Geodesic.LONG_UNROLL)\n", 149 | " yield g[\"lat2\"], g[\"lon2\"]" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 8, 155 | "metadata": {}, 156 | "outputs": [ 157 | { 158 | "name": "stdout", 159 | "output_type": "stream", 160 | "text": [ 161 | "52.516071000000004 13.37698\n", 162 | "53.98060697563462 28.186710757145015\n", 163 | "53.54352761856573 43.36110532382015\n", 164 | "51.269764128107006 57.605966972807366\n", 165 | "47.45591983586865 70.09894541405228\n", 166 | "42.47711273544207 80.67820310705217\n", 167 | "36.66746005578033 89.58694414588376\n", 168 | "30.28155147707397 97.19250070851366\n", 169 | "23.50236612011881 103.84466410408318\n", 170 | "16.461311189816005 109.8356359866446\n", 171 | "9.256358051557065 115.40317997219232\n", 172 | "1.9655955488759838 120.74640030861372\n", 173 | "-5.342834815344675 126.04322573894493\n", 174 | "-12.602793633961847 131.4671688473819\n", 175 | "-19.742610059885617 137.20381613244894\n", 176 | "-26.67641934166494 143.46796206741342\n", 177 | "-33.292748740783914 150.52093336626385\n", 178 | "-33.869710000000005 151.20710999999997\n" 179 | ] 180 | } 181 | ], 182 | "source": [ 183 | "for lat, lon in intermediatePoints(berlin, sydney):\n", 184 | " print(lat, lon)" 185 | ] 186 | }, 187 | { 188 | "cell_type": "code", 189 | "execution_count": 9, 190 | "metadata": {}, 191 | "outputs": [ 192 | { 193 | "name": "stdout", 194 | "output_type": "stream", 195 | "text": [ 196 | "52.516071 13.37698\n", 197 | "52.39962 13.04784\n" 198 | ] 199 | } 200 | ], 201 | "source": [ 202 | "for lat, lon in intermediatePoints(berlin, potsdam):\n", 203 | " print(lat, lon)" 204 | ] 205 | }, 206 | { 207 | "cell_type": "markdown", 208 | "metadata": {}, 209 | "source": [ 210 | "## Mapping geodesic lines" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": 10, 216 | "metadata": {}, 217 | "outputs": [], 218 | "source": [ 219 | "from itertools import tee\n", 220 | "\n", 221 | "def pairwise(iterable):\n", 222 | " \"s -> (s0,s1), (s1,s2), (s2, s3), ...\"\n", 223 | " a, b = tee(iterable)\n", 224 | " next(b, None)\n", 225 | " return zip(a, b)" 226 | ] 227 | }, 228 | { 229 | "cell_type": "code", 230 | "execution_count": 11, 231 | "metadata": { 232 | "scrolled": false 233 | }, 234 | "outputs": [ 235 | { 236 | "data": { 237 | "text/html": [ 238 | "
" 239 | ], 240 | "text/plain": [ 241 | "" 242 | ] 243 | }, 244 | "execution_count": 11, 245 | "metadata": {}, 246 | "output_type": "execute_result" 247 | } 248 | ], 249 | "source": [ 250 | "map = folium.Map()\n", 251 | "\n", 252 | "folium.Marker(location=la, popup=\"Los Angeles\").add_to(map)\n", 253 | "folium.Marker(location=nyc, popup=\"New York\").add_to(map)\n", 254 | "folium.Marker(location=berlin, popup=\"Berlin\").add_to(map)\n", 255 | "folium.Marker(location=singapore, popup=\"Singapore\").add_to(map)\n", 256 | "folium.Marker(location=sydney, popup=\"Sydney\").add_to(map)\n", 257 | "\n", 258 | "for start, end in pairwise([la, nyc, berlin, singapore, sydney]):\n", 259 | " folium.PolyLine(list(intermediatePoints(start, end)), weight=2).add_to(map)\n", 260 | "\n", 261 | "for start, end in pairwise([la, nyc, berlin, singapore, sydney]):\n", 262 | " folium.PolyLine([start, end], weight=2, color=\"red\").add_to(map)\n", 263 | "\n", 264 | "map" 265 | ] 266 | }, 267 | { 268 | "cell_type": "markdown", 269 | "metadata": {}, 270 | "source": [ 271 | "## Building a geodesic PolyLine subclass" 272 | ] 273 | }, 274 | { 275 | "cell_type": "code", 276 | "execution_count": 12, 277 | "metadata": {}, 278 | "outputs": [], 279 | "source": [ 280 | "# not used anymore, but still nice ;-)\n", 281 | "def extract_dict(aDict, keys):\n", 282 | " \"Return a sub-dict of ``aDict`` by keys and remove those from ``aDict``.\"\n", 283 | " return {k: aDict.pop(k) for k in keys}" 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": 13, 289 | "metadata": {}, 290 | "outputs": [], 291 | "source": [ 292 | "class GeodesicPolyLine(folium.PolyLine):\n", 293 | " \"\"\"\n", 294 | " A geodesic version of a PolyLine inserting intermediate points when needed. \n", 295 | " \n", 296 | " This will calculate intermediate points with some segment length whenever\n", 297 | " the geodesic length between two adjacent locations is above some maximal\n", 298 | " value.\n", 299 | " \"\"\"\n", 300 | " def __init__(self, locations, min_length_km=2000, segment_length_km=1000, **kwargs):\n", 301 | " kwargs1 = dict(min_length_km=min_length_km, segment_length_km=segment_length_km)\n", 302 | " geodesic_locs = [intermediatePoints(start, end, **kwargs1) for start, end in pairwise(locations)]\n", 303 | " super().__init__(geodesic_locs, **kwargs)" 304 | ] 305 | }, 306 | { 307 | "cell_type": "code", 308 | "execution_count": 14, 309 | "metadata": {}, 310 | "outputs": [ 311 | { 312 | "data": { 313 | "text/html": [ 314 | "
" 315 | ], 316 | "text/plain": [ 317 | "" 318 | ] 319 | }, 320 | "execution_count": 14, 321 | "metadata": {}, 322 | "output_type": "execute_result" 323 | } 324 | ], 325 | "source": [ 326 | "map = folium.Map()\n", 327 | "folium.Marker(location=la, popup=\"Los Angeles\").add_to(map)\n", 328 | "folium.Marker(location=berlin, popup=\"Berlin\").add_to(map)\n", 329 | "folium.Marker(location=sydney, popup=\"Sydney\").add_to(map)\n", 330 | "GeodesicPolyLine([la, berlin, sydney]).add_to(map)\n", 331 | "map" 332 | ] 333 | }, 334 | { 335 | "cell_type": "markdown", 336 | "metadata": {}, 337 | "source": [ 338 | "Test using longer segment lengths:" 339 | ] 340 | }, 341 | { 342 | "cell_type": "code", 343 | "execution_count": 15, 344 | "metadata": { 345 | "scrolled": false 346 | }, 347 | "outputs": [ 348 | { 349 | "data": { 350 | "text/html": [ 351 | "
" 352 | ], 353 | "text/plain": [ 354 | "" 355 | ] 356 | }, 357 | "execution_count": 15, 358 | "metadata": {}, 359 | "output_type": "execute_result" 360 | } 361 | ], 362 | "source": [ 363 | "map = folium.Map()\n", 364 | "folium.Marker(location=la, popup=\"Los Angeles\").add_to(map)\n", 365 | "folium.Marker(location=berlin, popup=\"Berlin\").add_to(map)\n", 366 | "folium.Marker(location=sydney, popup=\"Sydney\").add_to(map)\n", 367 | "GeodesicPolyLine([la, berlin, sydney], segment_length_km=2000).add_to(map)\n", 368 | "map" 369 | ] 370 | }, 371 | { 372 | "cell_type": "markdown", 373 | "metadata": {}, 374 | "source": [ 375 | "Test using longer minimum length:" 376 | ] 377 | }, 378 | { 379 | "cell_type": "code", 380 | "execution_count": 16, 381 | "metadata": { 382 | "scrolled": false 383 | }, 384 | "outputs": [ 385 | { 386 | "data": { 387 | "text/html": [ 388 | "
" 389 | ], 390 | "text/plain": [ 391 | "" 392 | ] 393 | }, 394 | "execution_count": 16, 395 | "metadata": {}, 396 | "output_type": "execute_result" 397 | } 398 | ], 399 | "source": [ 400 | "map = folium.Map()\n", 401 | "folium.Marker(location=la, popup=\"Los Angeles\").add_to(map)\n", 402 | "folium.Marker(location=berlin, popup=\"Berlin\").add_to(map)\n", 403 | "folium.Marker(location=sydney, popup=\"Sydney\").add_to(map)\n", 404 | "GeodesicPolyLine([la, berlin, sydney], min_length_km=10000).add_to(map)\n", 405 | "map" 406 | ] 407 | }, 408 | { 409 | "cell_type": "markdown", 410 | "metadata": {}, 411 | "source": [ 412 | "Let's check the distances (in km):" 413 | ] 414 | }, 415 | { 416 | "cell_type": "code", 417 | "execution_count": 17, 418 | "metadata": {}, 419 | "outputs": [ 420 | { 421 | "data": { 422 | "text/plain": [ 423 | "9331.178262315017" 424 | ] 425 | }, 426 | "execution_count": 17, 427 | "metadata": {}, 428 | "output_type": "execute_result" 429 | } 430 | ], 431 | "source": [ 432 | "Geodesic.WGS84.Inverse(*(la + berlin))[\"s12\"] / 1000" 433 | ] 434 | }, 435 | { 436 | "cell_type": "code", 437 | "execution_count": 18, 438 | "metadata": {}, 439 | "outputs": [ 440 | { 441 | "data": { 442 | "text/plain": [ 443 | "16090.2938772803" 444 | ] 445 | }, 446 | "execution_count": 18, 447 | "metadata": {}, 448 | "output_type": "execute_result" 449 | } 450 | ], 451 | "source": [ 452 | "Geodesic.WGS84.Inverse(*(berlin + sydney))[\"s12\"] / 1000" 453 | ] 454 | } 455 | ], 456 | "metadata": { 457 | "anaconda-cloud": {}, 458 | "kernelspec": { 459 | "display_name": "Python [default]", 460 | "language": "python", 461 | "name": "python3" 462 | }, 463 | "language_info": { 464 | "codemirror_mode": { 465 | "name": "ipython", 466 | "version": 3 467 | }, 468 | "file_extension": ".py", 469 | "mimetype": "text/x-python", 470 | "name": "python", 471 | "nbconvert_exporter": "python", 472 | "pygments_lexer": "ipython3", 473 | "version": "3.5.2" 474 | } 475 | }, 476 | "nbformat": 4, 477 | "nbformat_minor": 2 478 | } 479 | -------------------------------------------------------------------------------- /mapping/here_maps_api_explorer_no_creds.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# HERE Map Tiles Rest API Explorer" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": { 13 | "collapsed": true 14 | }, 15 | "source": [ 16 | "This notebook is intended to show how to access map tiles from [HERE Technologies](https://here.com) using their [RESTful Map Tile API](https://developer.here.com/documentation/map-tile/topics/quick-start.html) and build a variety of maps using [Folium](https://github.com/python-visualization/folium), a high-level library for creating web maps with [Leaflet](https://leafletjs.com).\n", 17 | "\n", 18 | "Folium supports already a number of map tiles providers, including OpenStreetMap or Mapbox (the latter needing an API_KEY), but not yet those made by HERE. To see such HERE map tiles and maps inside this [Jupyter](http://jupyter.org) notebook you will need to [register for a free HERE account](https://developer.here.com). You can chose a 90-day free trial or a free public \"BASIC\" plan with [standard features](https://developer.here.com/plans#standard_features) (see [more about plans](https://developer.here.com/plans?basicPlanOverlay)). And then you can obtain an APP_CODE and APP_ID for accessing HERE maps.\n", 19 | "\n", 20 | "Running this notebook also requires to have pip-installed the external packages named `requests`, `folium`, and `ipywidgets` (Python 3 is recommended, but 2 should also work). The APP_CODE and APP_ID values are assumed to be set as environment variables named `c` and `HEREMAPS_APP_ID`, respectively. These will be included in the HTML maps created with Folium/Leaflet, so be careful where you share your results!\n", 21 | "\n", 22 | "**N.B.** When writing this notebook it appeared that you can get HERE map tiles even with fake APP_CODE and APP_ID like `foo` and `bar` (as stored inside some cells in this notebook), but it is unpredictable to say which ones you will get or not. This is likely a way for HERE to show *something* even without credentials, while making the result not really useful, instead of limiting zoom levels or using watermarks (which might be a costly thing when serving many map tiles). Anyway, you can have a free account, see above.\n", 23 | "\n", 24 | "You will see the output best if you run the notebook locally with proper HERE credentials. But if you read it [on the Jupyter Notebook viewer](http://nbviewer.jupyter.org/github/deeplook/notebooks/blob/master/mapping/here_maps_api_explorer_no_creds.ipynb) or [on GitHub](https://github.com/deeplook/notebooks/blob/master/mapping/here_maps_api_explorer_no_creds.ipynb), or if you don't have real HERE credentials you might actually see no output for some cells. In these cases there is a comment to explain and a static screenshot is provided, too.\n", 25 | "\n", 26 | "Finally, it is assumed that you have a little experience with Jupyter notebooks, so not all steps are explained in detail. That was long intro, so now let's start playing a bit with map tiles and build maps! " 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "## Testing your environment" 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "First, run a little test to see if you are ready to go. Make sure you address any import or assertion error appearing in the next cell (by pip-installing the missing packages or adding the missing environment variables). Then move on!" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 1, 46 | "metadata": {}, 47 | "outputs": [ 48 | { 49 | "name": "stdout", 50 | "output_type": "stream", 51 | "text": [ 52 | "You are ready to go!\n" 53 | ] 54 | } 55 | ], 56 | "source": [ 57 | "import os\n", 58 | "msg = \"Error: Environment variable {} not found\"\n", 59 | "for varname in [\"HEREMAPS_APP_ID\", \"HEREMAPS_APP_CODE\"]:\n", 60 | " assert os.getenv(varname), msg.format(varname)\n", 61 | "\n", 62 | "import folium\n", 63 | "import requests\n", 64 | "import ipywidgets\n", 65 | "\n", 66 | "print(\"You are ready to go!\")" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "## Getting single HERE map tiles\n", 74 | "\n", 75 | "Web maps are composed of individual so-called map tiles, traditionally small precomputed bitmaps (although the industry is moving into using so-called map vector tiles, which is not a topic in this notebook, though). Now, lets see how to get a single map tile using the HERE REST API and your credentials. The URL contains quite a few parameters describing which tile exactly to download, but more about that later:" 76 | ] 77 | }, 78 | { 79 | "cell_type": "code", 80 | "execution_count": 2, 81 | "metadata": { 82 | "scrolled": true 83 | }, 84 | "outputs": [ 85 | { 86 | "data": { 87 | "text/html": [ 88 | "" 89 | ], 90 | "text/plain": [ 91 | "" 92 | ] 93 | }, 94 | "execution_count": 2, 95 | "metadata": {}, 96 | "output_type": "execute_result" 97 | } 98 | ], 99 | "source": [ 100 | "import os\n", 101 | "from IPython.display import Image\n", 102 | "\n", 103 | "# load HERE API credentials, get your own from http://developer.here.com\n", 104 | "app_id, app_code = map(os.getenv, [\"HEREMAPS_APP_ID\", \"HEREMAPS_APP_CODE\"])\n", 105 | "\n", 106 | "# then get and display a specific map tile\n", 107 | "url = \"https://1.base.maps.api.here.com/maptile/2.1/maptile/newest/normal.day/13/4400/2686/256/png8?lg=eng&app_id=%s&app_code=%s\" % (app_id, app_code)\n", 108 | "Image(url=url)" 109 | ] 110 | }, 111 | { 112 | "cell_type": "markdown", 113 | "metadata": {}, 114 | "source": [ 115 | "### Static version\n", 116 | "\n", 117 | "If you don't see the tile image in the section above it's likely because you read this notebook on GitHub or on the Jupyter NoteBook Viewer. Subsequent sections will include static versrions included inside this notebook. " 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "### Inspecting HTTP response header\n", 125 | "\n", 126 | "Have a look at the HTTP response headers, too! Here it's all fine." 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": 3, 132 | "metadata": {}, 133 | "outputs": [ 134 | { 135 | "name": "stdout", 136 | "output_type": "stream", 137 | "text": [ 138 | "200\n" 139 | ] 140 | }, 141 | { 142 | "data": { 143 | "text/plain": [ 144 | "{'Access-Control-Allow-Origin': '*',\n", 145 | " 'Cache-Control': 'public, max-age=63327',\n", 146 | " 'Connection': 'keep-alive',\n", 147 | " 'Content-Length': '31354',\n", 148 | " 'Content-Type': 'image/png',\n", 149 | " 'Date': 'Sun, 20 Aug 2017 14:13:36 GMT',\n", 150 | " 'ETag': 'afd6f70912',\n", 151 | " 'Last-Modified': 'Fri, 21 Jul 2017 00:00:00 GMT',\n", 152 | " 'Server': 'Apache',\n", 153 | " 'X-NLP-IRT': 'D=72183',\n", 154 | " 'X-Served-By': 'i-0cf8d7c9e85afbe31.eu-west-1a'}" 155 | ] 156 | }, 157 | "execution_count": 3, 158 | "metadata": {}, 159 | "output_type": "execute_result" 160 | } 161 | ], 162 | "source": [ 163 | "import requests\n", 164 | "\n", 165 | "resp = requests.get(url)\n", 166 | "print(resp.status_code)\n", 167 | "dict(resp.headers.items())" 168 | ] 169 | }, 170 | { 171 | "cell_type": "markdown", 172 | "metadata": {}, 173 | "source": [ 174 | "## Getting specific map tiles\n", 175 | "\n", 176 | "Normally you want to get a map tile that contains a specific geographic position at some zoom level. Depending on this zoom level a map of the entire world is composed of a varying number of tiles with x- and y-positions on a rectangular grid. You can use two simple functions to convert between these two domains, geographic coordinates on a sphere and the respective tile positions on a rectangular grid (including zoom level):" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": 4, 182 | "metadata": { 183 | "collapsed": true 184 | }, 185 | "outputs": [], 186 | "source": [ 187 | "# Conversion between lat/lon in degrees (and zoom) to x/y/zoom as used in tile sets,\n", 188 | "# from http://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Python\n", 189 | "\n", 190 | "from math import radians, degrees, log, cos, tan, pi, atan, sinh\n", 191 | "\n", 192 | "def deg2tile(lat_deg, lon_deg, zoom):\n", 193 | " lat_rad = radians(lat_deg)\n", 194 | " n = 2.0 ** zoom\n", 195 | " xtile = int((lon_deg + 180.0) / 360.0 * n)\n", 196 | " ytile = int((1.0 - log(tan(lat_rad) + (1 / cos(lat_rad))) / pi) / 2.0 * n)\n", 197 | " return (xtile, ytile)\n", 198 | "\n", 199 | "def tile2deg(xtile, ytile, zoom):\n", 200 | " n = 2.0 ** zoom\n", 201 | " lon_deg = xtile / n * 360.0 - 180.0\n", 202 | " lat_rad = atan(sinh(pi * (1 - 2 * ytile / n)))\n", 203 | " lat_deg = degrees(lat_rad)\n", 204 | " return (lat_deg, lon_deg)" 205 | ] 206 | }, 207 | { 208 | "cell_type": "markdown", 209 | "metadata": {}, 210 | "source": [ 211 | "Let's make a round-trip test with a the latitude and longitude of a geographic point somewhere in the middle of Berlin, Germany (taken from the respective [Wikipedia page](https://en.wikipedia.org/wiki/Berlin)):" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 5, 217 | "metadata": {}, 218 | "outputs": [ 219 | { 220 | "data": { 221 | "text/plain": [ 222 | "(8802, 5373)" 223 | ] 224 | }, 225 | "execution_count": 5, 226 | "metadata": {}, 227 | "output_type": "execute_result" 228 | } 229 | ], 230 | "source": [ 231 | "deg2tile(52.518611, 13.408333, 14)" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": 6, 237 | "metadata": {}, 238 | "outputs": [ 239 | { 240 | "data": { 241 | "text/plain": [ 242 | "(52.522905940278065, 13.4033203125)" 243 | ] 244 | }, 245 | "execution_count": 6, 246 | "metadata": {}, 247 | "output_type": "execute_result" 248 | } 249 | ], 250 | "source": [ 251 | "tile2deg(8802, 5373, 14)" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "execution_count": 7, 257 | "metadata": {}, 258 | "outputs": [ 259 | { 260 | "data": { 261 | "text/plain": [ 262 | "(8802, 5373)" 263 | ] 264 | }, 265 | "execution_count": 7, 266 | "metadata": {}, 267 | "output_type": "execute_result" 268 | } 269 | ], 270 | "source": [ 271 | "deg2tile(52.522905940278065, 13.4033203125, 14)" 272 | ] 273 | }, 274 | { 275 | "cell_type": "markdown", 276 | "metadata": {}, 277 | "source": [ 278 | "And now let's get the map tile containing this geographic point (52.518611, 13.408333) somewhere inside, at level zoom 14: " 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": 8, 284 | "metadata": { 285 | "scrolled": true 286 | }, 287 | "outputs": [ 288 | { 289 | "data": { 290 | "text/html": [ 291 | "" 292 | ], 293 | "text/plain": [ 294 | "" 295 | ] 296 | }, 297 | "execution_count": 8, 298 | "metadata": {}, 299 | "output_type": "execute_result" 300 | } 301 | ], 302 | "source": [ 303 | "url = \"https://1.base.maps.api.here.com/maptile/2.1/maptile/newest/normal.day/14/8802/5373/256/png8?lg=eng&app_id=%s&app_code=%s\" % (app_id, app_code)\n", 304 | "Image(url=url)" 305 | ] 306 | }, 307 | { 308 | "cell_type": "markdown", 309 | "metadata": {}, 310 | "source": [ 311 | "## Adding an interactive interface\n", 312 | "\n", 313 | "You can make this easier to explore by using a few little widgets as an interactive interface for some of these parameters. Here we take only latitude, longitude, zoom level and only two scheme names (the [API documentation](https://developer.here.com/documentation/map-tile/topics/resource-base-basetile.html) lists many more!)." 314 | ] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "execution_count": 9, 319 | "metadata": {}, 320 | "outputs": [ 321 | { 322 | "data": { 323 | "application/vnd.jupyter.widget-view+json": { 324 | "model_id": "af3d4d0c410c429ab7f7248bb04c05dc" 325 | } 326 | }, 327 | "metadata": {}, 328 | "output_type": "display_data" 329 | } 330 | ], 331 | "source": [ 332 | "from ipywidgets import interact\n", 333 | "\n", 334 | "schemes = [\"normal.day\", \"normal.night\"]\n", 335 | "\n", 336 | "@interact(lat=(-90., 90.), lon=(-180., 180.), zoom=(0, 18), scheme=schemes, show_url=False)\n", 337 | "def get_here_maptile(lat=52.518611, lon=13.408333, zoom=11, scheme=\"normal.day\", show_url=False):\n", 338 | " x, y = deg2tile(lat, lon, zoom)\n", 339 | " params = dict(x=x, y=y, zoom=zoom, scheme=scheme, app_id=app_id, app_code=app_code)\n", 340 | " url = \"https://1.base.maps.api.here.com/maptile/2.1/maptile/newest/{scheme}/{zoom}/{x}/{y}/256/png8?lg=eng&app_id={app_id}&app_code={app_code}\".format(**params)\n", 341 | " if show_url:\n", 342 | " print(url)\n", 343 | " return Image(url=url)" 344 | ] 345 | }, 346 | { 347 | "cell_type": "markdown", 348 | "metadata": {}, 349 | "source": [ 350 | "### Static version\n", 351 | "\n", 352 | "If you don't see the widgets and/or image output in the section above it's likely because you read this notebook on GitHub or on the Jupyter NoteBook Viewer and/or you don't have valid HERE credentials. In this case, this is a static sample image of what you see when running the notebook locally with valid HERE credentials: \n", 353 | "\n", 354 | "" 355 | ] 356 | }, 357 | { 358 | "cell_type": "markdown", 359 | "metadata": {}, 360 | "source": [ 361 | "## Building entire maps\n", 362 | "\n", 363 | "Of course, the whole purpose of map tiles is to build entire maps from a bunch of them, like those you know from [HERE](https://wego.here.com) or [OpenStreetMap](http://www.openstreetmap.org). All the magic of map tile selection and assembly happens in the background if you use the right tool, like Leaflet and the excellent Folium interface to it. Effectively, using Folium you can do it in one line of code (the default map tiles are set to `tiles=\"OpenStreetMap\"`, but you can change that):" 364 | ] 365 | }, 366 | { 367 | "cell_type": "code", 368 | "execution_count": 10, 369 | "metadata": { 370 | "scrolled": true 371 | }, 372 | "outputs": [ 373 | { 374 | "data": { 375 | "text/html": [ 376 | "
" 377 | ], 378 | "text/plain": [ 379 | "" 380 | ] 381 | }, 382 | "execution_count": 10, 383 | "metadata": {}, 384 | "output_type": "execute_result" 385 | } 386 | ], 387 | "source": [ 388 | "import folium; folium.Map(location=(52.518611, 13.408333), zoom_start=14)" 389 | ] 390 | }, 391 | { 392 | "cell_type": "markdown", 393 | "metadata": {}, 394 | "source": [ 395 | "Any such map output can be saved as a self-contained HTML file (without the map tiles, of course) and easily passed on to somebody else. (This other person might have to use her own credentials, if needed, though.):" 396 | ] 397 | }, 398 | { 399 | "cell_type": "code", 400 | "execution_count": 11, 401 | "metadata": { 402 | "collapsed": true 403 | }, 404 | "outputs": [], 405 | "source": [ 406 | "import folium\n", 407 | "m = folium.Map(location=(52.518611, 13.408333), zoom_start=14)\n", 408 | "m.save(\"mymap.html\")" 409 | ] 410 | }, 411 | { 412 | "cell_type": "markdown", 413 | "metadata": {}, 414 | "source": [ 415 | "## Making it interactive" 416 | ] 417 | }, 418 | { 419 | "cell_type": "markdown", 420 | "metadata": {}, 421 | "source": [ 422 | "One nice feature of Folium is that you can chose between a number of predefined tile sets by using the `tiles` parameter for `folium.Map`, so we can easily switch between those in a more interactive way, too. In addition, the following snippet lets you also change latitude, longitude and zoom level interactively:" 423 | ] 424 | }, 425 | { 426 | "cell_type": "code", 427 | "execution_count": 12, 428 | "metadata": {}, 429 | "outputs": [ 430 | { 431 | "data": { 432 | "application/vnd.jupyter.widget-view+json": { 433 | "model_id": "0251bbe7936740c59778594f1dfa7a16" 434 | } 435 | }, 436 | "metadata": {}, 437 | "output_type": "display_data" 438 | } 439 | ], 440 | "source": [ 441 | "import folium\n", 442 | "from ipywidgets import interact\n", 443 | "\n", 444 | "# Cloudmade Mapbox needs an API key, Mapbox Control Room is limited to a few levels\n", 445 | "tiles = [name.strip() for name in \"\"\"\n", 446 | " OpenStreetMap\n", 447 | " Mapbox Bright\n", 448 | " Mapbox Control Room\n", 449 | " Stamen Terrain\n", 450 | " Stamen Toner\n", 451 | " Stamen Watercolor\n", 452 | " CartoDB positron\n", 453 | " CartoDB dark_matter\"\"\".strip().split('\\n')]\n", 454 | "\n", 455 | "@interact(lat=(-90., 90.), lon=(-180., 180.), tiles=tiles, zoom=(1, 18))\n", 456 | "def create_map(lat=52.518611, lon=13.408333, tiles=\"Stamen Toner\", zoom=10):\n", 457 | " return folium.Map(location=(lat, lon), tiles=tiles, zoom_start=zoom)" 458 | ] 459 | }, 460 | { 461 | "cell_type": "markdown", 462 | "metadata": {}, 463 | "source": [ 464 | "### Static version\n", 465 | "\n", 466 | "If you don't see the widgets and/or map output in the section above it's likely because you read this notebook on GitHub or on the Jupyter NoteBook Viewer. In this case, this is a static sample image of what you see when running the notebook locally with valid HERE credentials: \n", 467 | "\n", 468 | "![alt text](images/12.png)\n", 469 | "\n", 470 | "And one thing missing here is, of course: when you interact with the map itself by zooming in or out or panning around you would like to see the widget values change accordingly. But that would need some JavaScript magic. Your contribution is welcome!" 471 | ] 472 | }, 473 | { 474 | "cell_type": "markdown", 475 | "metadata": {}, 476 | "source": [ 477 | "## Building HERE Maps\n", 478 | "\n", 479 | "But Folium goes one step further and lets you use an arbitrary source of map tiles, simply by providing a URL for the `tiles` parameter in `folium.Map()` that needs to contain placeholders for tile numbers and the zoom level. This means you can use HERE map tiles, even without built-in support for them in Folium." 480 | ] 481 | }, 482 | { 483 | "cell_type": "code", 484 | "execution_count": 13, 485 | "metadata": {}, 486 | "outputs": [ 487 | { 488 | "data": { 489 | "text/html": [ 490 | "
" 491 | ], 492 | "text/plain": [ 493 | "" 494 | ] 495 | }, 496 | "execution_count": 13, 497 | "metadata": {}, 498 | "output_type": "execute_result" 499 | } 500 | ], 501 | "source": [ 502 | "import os\n", 503 | "import folium\n", 504 | "\n", 505 | "c = map(os.getenv, [\"HEREMAPS_APP_ID\", \"HEREMAPS_APP_CODE\"])\n", 506 | "tiles=\"https://1.base.maps.api.here.com/maptile/2.1/maptile/newest/normal.day/{z}/{x}/{y}/256/png8?lg=eng&app_id=%s&app_code=%s\" % (app_id, app_code)\n", 507 | "folium.Map(location=(52.518611, 13.408333), tiles=tiles, zoom_start=10, attr=\"HERE.com\")" 508 | ] 509 | }, 510 | { 511 | "cell_type": "markdown", 512 | "metadata": {}, 513 | "source": [ 514 | "### Static version\n", 515 | "\n", 516 | "If you don't see the widgets and/or map output in the section above it's likely because you read this notebook on GitHub or on the Jupyter Notebook Viewer and/or you don't have valid HERE credentials. In this case, this is a static sample image of what you see when running the notebook locally with valid HERE credentials: \n", 517 | "\n", 518 | "![alt text](images/13.png)" 519 | ] 520 | }, 521 | { 522 | "cell_type": "markdown", 523 | "metadata": {}, 524 | "source": [ 525 | "## Adding more interactive features\n", 526 | "\n", 527 | "Now let's build a more feature-rich interactive interface to explore the API. This time there are more parameters to play with, using preselected values for some of them. Note that not all combinations are meaningful. Where they are not you simply don't see a map result below. There could be some validation, but... there isn't. This time we use a class, but that's not strictly needed." 528 | ] 529 | }, 530 | { 531 | "cell_type": "code", 532 | "execution_count": 14, 533 | "metadata": {}, 534 | "outputs": [ 535 | { 536 | "data": { 537 | "application/vnd.jupyter.widget-view+json": { 538 | "model_id": "8edcdbff86d9419e81774381599c5828" 539 | } 540 | }, 541 | "metadata": {}, 542 | "output_type": "display_data" 543 | }, 544 | { 545 | "data": { 546 | "text/plain": [ 547 | ".>" 548 | ] 549 | }, 550 | "execution_count": 14, 551 | "metadata": {}, 552 | "output_type": "execute_result" 553 | } 554 | ], 555 | "source": [ 556 | "import os\n", 557 | "import datetime\n", 558 | "\n", 559 | "app_id, app_code = map(os.getenv, [\"HEREMAPS_APP_ID\", \"HEREMAPS_APP_CODE\"])\n", 560 | "\n", 561 | "tile_formats = \"png8 png jpeg\".split()\n", 562 | "map_types = \"base aerial pano traffic\".split()\n", 563 | "tile_types = \"\"\"\n", 564 | " maptile traffictile flowbasetile trucktile rcdistonlytile\n", 565 | " mapnopttile blinetile alabeltile\n", 566 | "\"\"\".strip().split()\n", 567 | "languages_3 = \"eng ger fre ita gre rus ara hin chi ---\".split()\n", 568 | "\n", 569 | "schemes = \"\"\"\n", 570 | " normal.day normal.day.grey normal.day.transit normal.night\n", 571 | " normal.day.mobile\n", 572 | " pedestrian.day pedestrian.day.mobile\n", 573 | " carnav.day.grey\n", 574 | " normal.traffic.day\n", 575 | " reduced.night\n", 576 | " satellite.day\n", 577 | " hybrid.day\n", 578 | " wrong\n", 579 | "\"\"\".strip().split()\n", 580 | "\n", 581 | "\n", 582 | "class HereMap(object):\n", 583 | " def draw(self, map_type, tile_type, scheme, tile_format, language,\n", 584 | " lat, lon, zoom=10, show_url=False):\n", 585 | " \"Draw a HERE map. Default values are given as parameter keyword values.\"\n", 586 | " \n", 587 | " # raise error for not allowed combinations\n", 588 | " if scheme == \"wrong\":\n", 589 | " raise Exception(\"Combination not allowed\")\n", 590 | " \n", 591 | " # set parameter defaults for map tiles to request\n", 592 | " number = '1' # (1-4)\n", 593 | " map_version = \"newest\" # or some hex hash value\n", 594 | " server_env = \"\" # production, or \"cit.\" for \"customer integration test\"\n", 595 | " tile_size = \"256\"\n", 596 | "\n", 597 | " # build map tiles URL\n", 598 | " server = \"https://{number}.{map_type}.maps.{server_env}api.here.com\"\n", 599 | " path = \"/maptile/2.1/{tile_type}/{map_version}/{scheme}/{{z}}/{{x}}/{{y}}/{tile_size}/{tile_format}\"\n", 600 | " query = \"?app_id={app_id}&app_code={app_code}\"\n", 601 | " if language != \"---\":\n", 602 | " query += \"&lg={language}\"\n", 603 | " params = dict(number=number, map_type=map_type, tile_type=tile_type,\n", 604 | " scheme=scheme, tile_format=tile_format, tile_size=tile_size,\n", 605 | " map_version=map_version, server_env=server_env,\n", 606 | " app_id=app_id, app_code=app_code,\n", 607 | " language=language\n", 608 | " )\n", 609 | " tiles_url = (server + path + query).format(**params)\n", 610 | "\n", 611 | " # set map attribution text\n", 612 | " year = datetime.datetime.now().year\n", 613 | " attr = 'Data by HERE.com, %s' % year\n", 614 | "\n", 615 | " if show_url:\n", 616 | " print(tiles_url)\n", 617 | "\n", 618 | " # build and return a map\n", 619 | " return folium.Map(\n", 620 | " location=(lat, lon),\n", 621 | " tiles=tiles_url,\n", 622 | " zoom_start=zoom,\n", 623 | " attr=attr\n", 624 | " )\n", 625 | "\n", 626 | "hmap = HereMap()\n", 627 | "interact(hmap.draw, map_type=map_types, tile_type=tile_types, scheme=schemes,\n", 628 | " tile_format=tile_formats, language=languages_3, lat=52.5159, lon=13.3777, zoom=(1, 18), show_url=False)" 629 | ] 630 | }, 631 | { 632 | "cell_type": "markdown", 633 | "metadata": {}, 634 | "source": [ 635 | "### Static version\n", 636 | "\n", 637 | "If you don't see the widgets and/or map output in the section above it's likely because you read this notebook on GitHub or on the Jupyter NoteBook Viewer and/or you don't have valid HERE credentials. In this case, this is a static sample image of what you see when running the notebook locally with valid HERE credentials: \n", 638 | "\n", 639 | "![alt text](images/14.png)" 640 | ] 641 | }, 642 | { 643 | "cell_type": "markdown", 644 | "metadata": {}, 645 | "source": [ 646 | "## Building a map dashboard\n", 647 | "\n", 648 | "Once you have explored the map parameter space long enough, you might arrive at a few settings that you would probably need more regularly. This is the moment when you can prepare such queries into something like a very simple dashboard-like overview as shown below. This one shows a few maps displaying traffic conditions, information for trucks, public transit and a normal and hybrid satellite view for a few selected cities." 649 | ] 650 | }, 651 | { 652 | "cell_type": "code", 653 | "execution_count": 15, 654 | "metadata": {}, 655 | "outputs": [ 656 | { 657 | "data": { 658 | "application/vnd.jupyter.widget-view+json": { 659 | "model_id": "438628cdd1304beca4891fade406eeed" 660 | } 661 | }, 662 | "metadata": {}, 663 | "output_type": "display_data" 664 | } 665 | ], 666 | "source": [ 667 | "import folium\n", 668 | "from ipywidgets import interact\n", 669 | "\n", 670 | "cities = [\"Berlin\", \"Paris\", \"Chicago\", \"Singapore\"]\n", 671 | "examples = [\"Traffic\", \"Truck info\", \"Transit\", \"Regular\", \"Satellite\"]\n", 672 | "@interact(city=cities, example=examples)\n", 673 | "def show_canned_examples(city, example):\n", 674 | " attr = \"HERE.com\"\n", 675 | " latlon_for_city = {\n", 676 | " \"Berlin\": (52.518611, 13.408333), \n", 677 | " \"Paris\": (48.8567, 2.3508), \n", 678 | " \"Chicago\": (41.88416, -87.63243),\n", 679 | " \"Singapore\": (1.283333, 103.833333)\n", 680 | " }\n", 681 | " zoom = 14\n", 682 | " queries = {\n", 683 | " \"Traffic\":\n", 684 | " \"https://1.traffic.maps.api.here.com/maptile/2.1/traffictile/newest/normal.traffic.day/{z}/{x}/{y}/256/png8?lg=eng&app_id=%s&app_code=%s\" % (app_id, app_code),\n", 685 | " \"Regular\":\n", 686 | " \"https://1.base.maps.api.here.com/maptile/2.1/maptile/newest/normal.day/{z}/{x}/{y}/256/png8?lg=eng&app_id=%s&app_code=%s\" % (app_id, app_code),\n", 687 | " \"Truck info\":\n", 688 | " \"https://1.base.maps.api.here.com/maptile/2.1/trucktile/newest/normal.day.grey/{z}/{x}/{y}/256/png8?lg=eng&app_id=%s&app_code=%s\" % (app_id, app_code),\n", 689 | " \"Transit\":\n", 690 | " \"https://1.base.maps.api.here.com/maptile/2.1/maptile/newest/normal.day.transit/{z}/{x}/{y}/256/png8?lg=eng&app_id=%s&app_code=%s\" % (app_id, app_code),\n", 691 | " \"Satellite\":\n", 692 | " \"https://1.aerial.maps.api.here.com/maptile/2.1/maptile/newest/hybrid.day/{z}/{x}/{y}/256/png8?lg=eng&app_id=%s&app_code=%s\" % (app_id, app_code),\n", 693 | " }\n", 694 | " return folium.Map(location=latlon_for_city[city], tiles=queries[example],attr=attr, zoom_start=zoom)" 695 | ] 696 | }, 697 | { 698 | "cell_type": "markdown", 699 | "metadata": {}, 700 | "source": [ 701 | "### Static version\n", 702 | "\n", 703 | "Again, if you don't see the widgets and/or map output above this paragraph it's likely because you read this notebook on GitHub or on the Jupyter NoteBook Viewer and/or you don't have valid HERE credentials. In this case, this is a static sample image of what you see when running the notebook locally with valid HERE credentials: \n", 704 | "\n", 705 | "![alt text](images/15.png)" 706 | ] 707 | }, 708 | { 709 | "cell_type": "markdown", 710 | "metadata": {}, 711 | "source": [ 712 | "## Conclusion\n", 713 | "\n", 714 | "This concludes our little exploration with map tiles and maps. This notebook has shown how to use Folium (and Leaflet) to grab individual map tiles from a map service provider not yet directly supported by Folium, [HERE.com](https://here.com), explore them interactively using the RESTful API inside a Jupyter notebook, and generate full-fledged maps. It has shown how surprisingly simple this is with the powerful tools and great content available today. And there are lots of other interesting APIs, e.g. about venue maps, see the [HERE API documentation](https://developer.here.com/documentation).\n", 715 | "\n", 716 | "There are many more features of HERE maps, Folium and Leaflet, that have not been touched in this notebook, especially about using additional data layers. This might be addressed in other notebooks using Folium and HERE's emerging [Open Location Platform](https://here.com/en/innovation/here-open-location-platform). So please stay tuned!" 717 | ] 718 | } 719 | ], 720 | "metadata": { 721 | "anaconda-cloud": {}, 722 | "kernelspec": { 723 | "display_name": "Python 3", 724 | "language": "python", 725 | "name": "python3" 726 | }, 727 | "language_info": { 728 | "codemirror_mode": { 729 | "name": "ipython", 730 | "version": 3 731 | }, 732 | "file_extension": ".py", 733 | "mimetype": "text/x-python", 734 | "name": "python", 735 | "nbconvert_exporter": "python", 736 | "pygments_lexer": "ipython3", 737 | "version": "3.6.1" 738 | } 739 | }, 740 | "nbformat": 4, 741 | "nbformat_minor": 2 742 | } 743 | -------------------------------------------------------------------------------- /mapping/images/12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplook/notebooks/4511c4de9c4aee35098a90e44380e51c508f60d4/mapping/images/12.png -------------------------------------------------------------------------------- /mapping/images/13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplook/notebooks/4511c4de9c4aee35098a90e44380e51c508f60d4/mapping/images/13.png -------------------------------------------------------------------------------- /mapping/images/14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplook/notebooks/4511c4de9c4aee35098a90e44380e51c508f60d4/mapping/images/14.png -------------------------------------------------------------------------------- /mapping/images/15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplook/notebooks/4511c4de9c4aee35098a90e44380e51c508f60d4/mapping/images/15.png -------------------------------------------------------------------------------- /mapping/images/9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplook/notebooks/4511c4de9c4aee35098a90e44380e51c508f60d4/mapping/images/9.png -------------------------------------------------------------------------------- /pixelplanet/README.rst: -------------------------------------------------------------------------------- 1 | Pixel Planet 2 | ============ 3 | 4 | This is an exploration of using IPython Widgets with a Maptplotlib Basemap... 5 | -------------------------------------------------------------------------------- /pixelplanet/environment.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - appnope=0.1.0=py35_0 3 | - backports=1.0=py35_0 4 | - basemap=1.0.7=np110py35_0 5 | - cycler=0.10.0=py35_0 6 | - decorator=4.0.10=py35_0 7 | - entrypoints=0.2.2=py35_0 8 | - freetype=2.5.5=1 9 | - geos=3.3.3=0 10 | - get_terminal_size=1.0.0=py35_0 11 | - ipykernel=4.3.1=py35_0 12 | - ipython=4.2.0=py35_1 13 | - ipython_genutils=0.1.0=py35_0 14 | - ipywidgets=4.1.1=py35_0 15 | - jinja2=2.8=py35_1 16 | - jsonschema=2.5.1=py35_0 17 | - jupyter=1.0.0=py35_3 18 | - jupyter_client=4.3.0=py35_0 19 | - jupyter_console=4.1.1=py35_0 20 | - jupyter_core=4.1.0=py35_0 21 | - libpng=1.6.22=0 22 | - markupsafe=0.23=py35_2 23 | - matplotlib=1.5.1=np110py35_0 24 | - mistune=0.7.2=py35_1 25 | - mkl=11.3.3=0 26 | - nbconvert=4.2.0=py35_0 27 | - nbformat=4.0.1=py35_0 28 | - notebook=4.2.1=py35_0 29 | - numpy=1.10.4=py35_2 30 | - openssl=1.0.2h=1 31 | - path.py=8.2.1=py35_0 32 | - pexpect=4.0.1=py35_0 33 | - pickleshare=0.7.2=py35_0 34 | - pip=8.1.2=py35_0 35 | - ptyprocess=0.5.1=py35_0 36 | - pygments=2.1.3=py35_0 37 | - pyparsing=2.1.4=py35_0 38 | - pyqt=4.11.4=py35_3 39 | - python=3.5.1=5 40 | - python-dateutil=2.5.3=py35_0 41 | - python.app=1.2=py35_4 42 | - pytz=2016.4=py35_0 43 | - pyzmq=15.2.0=py35_1 44 | - qt=4.8.7=3 45 | - qtconsole=4.2.1=py35_0 46 | - readline=6.2=2 47 | - setuptools=23.0.0=py35_0 48 | - simplegeneric=0.8.1=py35_1 49 | - sip=4.16.9=py35_0 50 | - six=1.10.0=py35_0 51 | - sqlite=3.13.0=0 52 | - terminado=0.6=py35_0 53 | - tk=8.5.18=0 54 | - tornado=4.3=py35_1 55 | - traitlets=4.2.1=py35_0 56 | - wheel=0.29.0=py35_0 57 | - xz=5.2.2=0 58 | - zlib=1.2.8=3 59 | - pip: 60 | - backports.shutil-get-terminal-size==1.0.0 61 | - ipython-genutils==0.1.0 62 | - jupyter-client==4.3.0 63 | - jupyter-console==4.1.1 64 | - jupyter-core==4.1.0 65 | 66 | -------------------------------------------------------------------------------- /planecrashinfo/README.rst: -------------------------------------------------------------------------------- 1 | Plane Crash Info 2 | ================ 3 | 4 | .. image:: images/header.png 5 | :width: 100 % 6 | :align: center 7 | 8 | Introduction 9 | ------------ 10 | 11 | This is an exploration of data found at http://planecrashinfo.com, a site that collects a lot of data about plane crashes back to ca. 1920. While it does a great job at collecting that data it presents only few plots trying to help with interpreting it. This notebook attempts to provide the means for easily accessing the data and exploring it using a more analytic/interactive/visual approach. To do so, standard tools from a Python-based data science stack are used, like Jupyter_, Pandas_, and Matplotlib_/Basemap_. 12 | 13 | .. _Jupyter: http://jupyter.org 14 | .. _Pandas: http://pandas.pydata.org 15 | .. _Matplotlib: http://www.matplotlib.org 16 | .. _Basemap: http://matplotlib.org/basemap/ 17 | 18 | **N.B.** This is work in progress… If you have any suggestion, please feel free to raise an issue or provide a pull request. 19 | 20 | 21 | Installation 22 | ------------ 23 | 24 | Using Python 3 is recommended, because this is the future! ;-) And Conda is the recommended packaging system, since Pip cannot install Basemap. To install all dependencies in a fresh virtual environment (named ``pci`` short for ``planecrashinfo`` here), it is recommended to follow this procedure in your root environment: 25 | 26 | .. code-block:: console 27 | 28 | # create virtual environment 29 | conda create -c conda-forge -n pci basemap pandas jupyter geopy 30 | 31 | # activate environment 32 | source activate pci 33 | 34 | If you want to save a file describing this entire environment in order to recreate it later, you can also export the environment (once it's created) and recreate it from scratch as shown here: 35 | 36 | .. code-block:: console 37 | 38 | # export environment 39 | conda-env export -n pci -f environment.yml 40 | 41 | # recreate environment from exported file 42 | conda-env create -f environment.yml 43 | 44 | You can also try to use the ``environment.yml`` file contained in this folder. 45 | 46 | Preparation 47 | ----------- 48 | 49 | For the notebook to execute correctly it needs two data files, both included in this repo, ``data.csv`` and ``geolocs.json``. These can also be created by running the following commands, respectively: 50 | 51 | .. code-block:: console 52 | 53 | python build_database.py 54 | python build_geolocations.py 55 | 56 | This might take quite a while, ca. 1-2 hours, and create a few additional CSV files, which are not needed for the notebook, but can be nice to play with. 57 | 58 | 59 | Exploration 60 | ----------- 61 | 62 | To get started exploring the data, then just run the Jupyter notebook as follows: 63 | 64 | .. code-block:: console 65 | 66 | jupyter-notebook planecrashinfo.ipynb 67 | 68 | 69 | Uninstallation 70 | -------------- 71 | 72 | To deactivate and/or remove the environment again you simply do the following: 73 | 74 | .. code-block:: console 75 | 76 | # deactivate environment 77 | source deactivate pci 78 | 79 | # remove environment 80 | conda-env remove -n pci 81 | 82 | -------------------------------------------------------------------------------- /planecrashinfo/build_database.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | """ 4 | Get data from planecrashinfo.com and save it into a local CSV 'database'. 5 | """ 6 | 7 | import os 8 | import re 9 | import sys 10 | import datetime 11 | 12 | import numpy as np 13 | import pandas as pd 14 | 15 | 16 | def get_accident_years(): 17 | """ 18 | Get years with data from `planecrashinfo.com/database.htm`. 19 | 20 | Returns a list of integers, from 1920 to the current year, 21 | e.g. [1920, 1921, ..., 2016]. 22 | """ 23 | url = 'http://www.planecrashinfo.com/database.htm' 24 | dfs = pd.read_html(url, index_col=0) 25 | years = dfs[1].values.flatten() 26 | years = list(map(int, years[~np.isnan(years)])) 27 | return years 28 | 29 | 30 | def get_accidents_per_year(year): 31 | """ 32 | Get #crashes for given year like `planecrashinfo.com/2000/2000.htm`. 33 | 34 | Returns an integer representing the number of accidents for the given 35 | year (easy since there is no paging used planecrashinfo.com). 36 | """ 37 | url = 'http://www.planecrashinfo.com/%d/%d.htm' % (year, year) 38 | dfs = pd.read_html(url) 39 | return len(dfs[0]) - 1 40 | 41 | 42 | # not used yet 43 | def get_single_accident_html(year, num): 44 | """ 45 | Get data for some accident in some year like `planecrashinfo.com/2000/2000-14.htm`. 46 | """ 47 | url = 'http://www.planecrashinfo.com/%d/%d-%d.htm' % (year, year, num) 48 | html = requests.get(url).text 49 | return html 50 | 51 | 52 | def get_single_accident(year, num): 53 | """ 54 | Get data for some accident in some year like `planecrashinfo.com/2000/2000-14.htm`. 55 | """ 56 | url = 'http://www.planecrashinfo.com/%d/%d-%d.htm' % (year, year, num) 57 | dfs = pd.read_html(url, index_col=0, header=0) 58 | df = dfs[0] 59 | return df 60 | 61 | 62 | # database handling 63 | 64 | def get_database(years=[], verbose=True): 65 | "Get entire database, as one dataframe with all desired years." 66 | 67 | dfs = [] 68 | for y in years or get_accident_years(): 69 | for acc_num in range(1, get_accidents_per_year(y) + 1): 70 | # if acc_num > 4: 71 | # break 72 | if verbose: 73 | print(y, acc_num) 74 | dfs.append(get_single_accident(y, acc_num)) 75 | df_total = pd.DataFrame([df.T.iloc[0] for df in dfs]) 76 | return df_total 77 | 78 | 79 | def download_separate_years(years=[], verbose=True): 80 | "Scrape and store all data from `planecrashinfo.com` as single CSV files." 81 | 82 | # Best practice to avoid downloading identical files many times... 83 | 84 | for y in years: 85 | dest = 'data/%d_original.csv' % y 86 | if os.path.exists(dest): 87 | continue 88 | if verbose: 89 | print(y) 90 | try: 91 | df = get_database(years=[y], verbose=verbose) 92 | except TimeoutError: 93 | df = get_database(years=[y], verbose=verbose) 94 | df.to_csv(dest) 95 | 96 | 97 | def build_one_database(years=[], verbose=True): 98 | "Build one database from separately downloaded files." 99 | 100 | dfs = [] 101 | for y in years: # 1920 missing 102 | path = 'data/%d_original.csv' % y 103 | df = pd.read_csv(path) 104 | if verbose: 105 | print(y) 106 | dfs.append(df) 107 | return pd.concat(dfs) 108 | 109 | 110 | if __name__ == '__main__': 111 | download_separate_years(years=range(1921, 2017)) 112 | # sys.exit(0) 113 | 114 | # df = get_database(years=range(1921, 2017), verbose=True) 115 | # path = 'data/1921-2016_original.csv' 116 | # df.to_csv(path) 117 | 118 | # build full database from single yearly files 119 | df_all = build_one_database(range(1921, 2017)) 120 | path = 'data/data.csv' 121 | df_all.to_csv(path) 122 | -------------------------------------------------------------------------------- /planecrashinfo/build_geolocations.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | """ 4 | Build a JSON file with geo-locations from plane crash places in the DB. 5 | """ 6 | 7 | import os 8 | import time 9 | import json 10 | import random 11 | 12 | import geopy 13 | from geopy.geocoders import Nominatim 14 | import pandas as pd 15 | 16 | from planecrashinfo_light import clean_database 17 | 18 | 19 | geolocator = Nominatim() 20 | 21 | 22 | def location_to_latlon(place): 23 | """ 24 | Get lat/lon dict or None for some place. 25 | """ 26 | try: 27 | # Can raise geopy.exc.GeocoderServiceError, 28 | # so be gentle and give it some time to breathe. 29 | location = geolocator.geocode(place) 30 | time.sleep(random.random()) 31 | except: # geopy.exc.GeocoderTimedOut: 32 | print('* ' + place, None) 33 | return None 34 | if location is None: 35 | print('* ' + place, None) 36 | return None 37 | print(place, (location.latitude, location.longitude)) 38 | return {'lat': location.latitude, 'lon': location.longitude} 39 | 40 | 41 | def get_geolocations(df, previous=None): 42 | """ 43 | Return a dict with lat/lon for origins and destinations of one df." 44 | 45 | { 46 | 'Bergen': {'lat': 60.3943532, 'lon': 5.325551}, 47 | 'Cairo': {'lat': 30.0488185, 'lon': 31.2436663}, 48 | 'Moroni Hahaya': None, 49 | ... 50 | } 51 | """ 52 | res = previous or {} 53 | for name in ['Origin', 'Destination']: 54 | s = df.groupby(name).size().sort_values(ascending=True) 55 | for loc, count in s.items(): 56 | # ignore unspecific place 57 | if loc in ('Sightseeing', 'Training', 'Test flight', 'Military exercises', 'aerial survelliance'): 58 | print('* Ignoring: %s' % loc) 59 | continue 60 | # ignore known place 61 | if res.get(loc, None) != None: 62 | print('* Already found: %s' % loc) 63 | else: 64 | latlon = location_to_latlon(loc) 65 | res[loc] = latlon 66 | return res 67 | 68 | 69 | def build(): 70 | """ 71 | Build a file with gelocations for places in the database. 72 | """ 73 | geolocs = {} 74 | geolocs_path = 'data/geolocs.json' 75 | if os.path.exists(geolocs_path): 76 | geolocs = json.load(open()) 77 | print('Starting with %d' % len(geolocs)) 78 | 79 | for y in range(1921, 2017): 80 | path = 'data/%d_original.csv' % y 81 | print('Loading %s' % path) 82 | df = pd.read_csv(path) 83 | df = clean_database(df) 84 | geolocs.update(get_geolocations(df, geolocs)) 85 | json.dump(geolocs, open(path, 'w'), indent=4) 86 | print('Saved %d to %s\n' % (len(geolocs), geolocs_path)) 87 | 88 | 89 | if __name__ == '__main__': 90 | build() 91 | -------------------------------------------------------------------------------- /planecrashinfo/environment.yml: -------------------------------------------------------------------------------- 1 | name: pci 2 | channels: !!python/tuple 3 | - defaults 4 | dependencies: 5 | - appnope=0.1.0=py35_0 6 | - conda-forge::backports.shutil_get_terminal_size=1.0.0=py35_0 7 | - conda-forge::basemap=1.0.8.dev0=np111py35_2 8 | - conda-forge::ca-certificates=2016.9.26=0 9 | - conda-forge::certifi=2016.9.26=py35_0 10 | - conda-forge::cycler=0.10.0=py35_0 11 | - conda-forge::decorator=4.0.10=py35_0 12 | - conda-forge::entrypoints=0.2.2=py35_0 13 | - conda-forge::freetype=2.6.3=1 14 | - conda-forge::geopy=1.11.0=py35_0 15 | - conda-forge::geos=3.4.2=2 16 | - conda-forge::ipykernel=4.5.0=py35_0 17 | - conda-forge::ipython=5.1.0=py35_1 18 | - conda-forge::ipython_genutils=0.1.0=py35_0 19 | - conda-forge::ipywidgets=5.2.2=py35_2 20 | - conda-forge::jinja2=2.8=py35_1 21 | - conda-forge::jsonschema=2.5.1=py35_0 22 | - conda-forge::jupyter_client=4.4.0=py35_0 23 | - conda-forge::jupyter_console=5.0.0=py35_0 24 | - conda-forge::jupyter_core=4.2.0=py35_0 25 | - conda-forge::libpng=1.6.26=0 26 | - conda-forge::libsodium=1.0.10=0 27 | - conda-forge::markupsafe=0.23=py35_0 28 | - conda-forge::matplotlib=1.5.3=np111py35_2 29 | - conda-forge::mistune=0.7.3=py35_0 30 | - conda-forge::nbconvert=4.2.0=py35_0 31 | - conda-forge::nbformat=4.1.0=py35_0 32 | - conda-forge::ncurses=5.9=9 33 | - conda-forge::notebook=4.2.3=py35_0 34 | - conda-forge::openssl=1.0.2h=2 35 | - conda-forge::pandas=0.19.1=np111py35_0 36 | - conda-forge::pexpect=4.2.1=py35_0 37 | - conda-forge::pickleshare=0.7.3=py35_0 38 | - conda-forge::pip=9.0.1=py35_0 39 | - conda-forge::prompt_toolkit=1.0.9=py35_0 40 | - conda-forge::ptyprocess=0.5.1=py35_0 41 | - conda-forge::pygments=2.1.3=py35_1 42 | - conda-forge::pyparsing=2.1.10=py35_0 43 | - conda-forge::pyproj=1.9.5.1=py35_0 44 | - conda-forge::pyqt=4.11.4=py35_1 45 | - conda-forge::pyshp=1.2.10=py35_0 46 | - conda-forge::python=3.5.2=2 47 | - conda-forge::python-dateutil=2.5.3=py35_0 48 | - conda-forge::pytz=2016.7=py35_0 49 | - conda-forge::pyzmq=16.0.1=py35_0 50 | - conda-forge::qtconsole=4.2.1=py35_1 51 | - conda-forge::readline=6.2=0 52 | - conda-forge::setuptools=28.8.0=py35_0 53 | - conda-forge::simplegeneric=0.8.1=py35_0 54 | - conda-forge::sip=4.18=py35_0 55 | - conda-forge::six=1.10.0=py35_0 56 | - conda-forge::sqlite=3.13.0=1 57 | - conda-forge::terminado=0.6=py35_0 58 | - conda-forge::tk=8.5.19=0 59 | - conda-forge::tornado=4.4.2=py35_0 60 | - conda-forge::traitlets=4.3.1=py35_0 61 | - conda-forge::wcwidth=0.1.7=py35_0 62 | - conda-forge::wheel=0.29.0=py35_0 63 | - conda-forge::widgetsnbextension=1.2.6=py35_3 64 | - conda-forge::xz=5.2.2=0 65 | - conda-forge::zeromq=4.1.5=0 66 | - conda-forge::zlib=1.2.8=3 67 | - jupyter=1.0.0=py35_3 68 | - mkl=11.3.3=0 69 | - numpy=1.11.2=py35_0 70 | - python.app=1.2=py35_4 71 | - qt=4.8.7=4 72 | - pip: 73 | - backports.shutil-get-terminal-size==1.0.0 74 | - ipython-genutils==0.1.0 75 | - jupyter-client==4.4.0 76 | - jupyter-console==5.0.0 77 | - jupyter-core==4.2.0 78 | - prompt-toolkit==1.0.9 79 | -------------------------------------------------------------------------------- /planecrashinfo/images/header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deeplook/notebooks/4511c4de9c4aee35098a90e44380e51c508f60d4/planecrashinfo/images/header.png -------------------------------------------------------------------------------- /planecrashinfo/planecrashinfo_light.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | """ 4 | Initial code for working with data from PlaneCrashInfo.com. 5 | 6 | More to come... 7 | """ 8 | 9 | import re 10 | 11 | import numpy as np 12 | import pandas as pd 13 | 14 | 15 | # database cleaning 16 | 17 | US_STATES = [ 18 | ('AL', 'Alabama'), 19 | ('AK', 'Alaska'), 20 | ('AZ', 'Arizona'), 21 | ('AR', 'Arkansas'), 22 | ('CA', 'California'), 23 | ('CO', 'Colorado'), 24 | ('CT', 'Connecticut'), 25 | ('DE', 'Delaware'), 26 | ('DC', 'District Of Columbia'), 27 | ('FL', 'Florida'), 28 | ('GA', 'Georgia'), 29 | ('HI', 'Hawaii'), 30 | ('ID', 'Idaho'), 31 | ('IL', 'Illinois'), 32 | ('IN', 'Indiana'), 33 | ('IA', 'Iowa'), 34 | ('KS', 'Kansas'), 35 | ('KY', 'Kentucky'), 36 | ('LA', 'Louisiana'), 37 | ('ME', 'Maine'), 38 | ('MD', 'Maryland'), 39 | ('MA', 'Massachusetts'), 40 | ('MI', 'Michigan'), 41 | ('MN', 'Minnesota'), 42 | ('MS', 'Mississippi'), 43 | ('MO', 'Missouri'), 44 | ('MT', 'Montana'), 45 | ('NE', 'Nebraska'), 46 | ('NV', 'Nevada'), 47 | ('NH', 'New Hampshire'), 48 | ('NJ', 'New Jersey'), 49 | ('NM', 'New Mexico'), 50 | ('NY', 'New York'), 51 | ('NC', 'North Carolina'), 52 | ('ND', 'North Dakota'), 53 | ('OH', 'Ohio'), 54 | ('OK', 'Oklahoma'), 55 | ('OR', 'Oregon'), 56 | ('PA', 'Pennsylvania'), 57 | ('RI', 'Rhode Island'), 58 | ('SC', 'South Carolina'), 59 | ('SD', 'South Dakota'), 60 | ('TN', 'Tennessee'), 61 | ('TX', 'Texas'), 62 | ('UT', 'Utah'), 63 | ('VT', 'Vermont'), 64 | ('VA', 'Virginia'), 65 | ('WA', 'Washington'), 66 | ('WV', 'West Virginia'), 67 | ('WI', 'Wisconsin'), 68 | ('WY', 'Wyoming') 69 | ] 70 | 71 | US_STATES_FLAT = [entry[0] for entry in US_STATES] + [entry[1] for entry in US_STATES] 72 | 73 | def country_of_loc(location): 74 | """ 75 | Get country of given location. 76 | 77 | E.g. 'St. Moritz, Switzerland' -> 'Switzerland' or 78 | 'Jackson, Mississippi' -> 'USA' 79 | """ 80 | if type(location) != str: 81 | return '?' 82 | country = location.split(',')[-1].strip() 83 | if country in US_STATES_FLAT: 84 | country = 'USA' 85 | return country 86 | 87 | 88 | def int_or_nan(x): 89 | try: 90 | x = int(x) 91 | except ValueError: 92 | x = np.nan 93 | return x 94 | 95 | 96 | def split_fatalities(entry): 97 | """ 98 | Split 'Fatalities' entry into dictionary. 99 | 100 | '?' are converted to np.nan. Counts should be ints, not floats... 101 | 102 | E.g. '15 (passengers:13 crew:2)' 103 | -> {'total':15, 'passengers':13, 'crew':2}. 104 | """ 105 | names = 'total passengers crew'.split() 106 | counts = re.findall('(\?|\d+)', entry) 107 | counts = [int_or_nan(c) for c in counts] 108 | return dict(list(zip(names, counts))) 109 | 110 | 111 | def clean_database(df): 112 | "Return copy of database after cleaning." 113 | 114 | dfc = df.copy() 115 | 116 | # drop useless columns 117 | dfc = dfc.drop(['Unnamed: 0', 'Unnamed: 0.1', 'Registration:', 'Flight #:', 'cn / ln:'], axis=1) 118 | 119 | # remove trailing ':' in index/column names 120 | dfc = dfc.rename(columns={cn: cn[:-1] for cn in list(dfc.columns)}) 121 | 122 | # replace all '?' values with NaN 123 | dfc.replace(to_replace='?', value=np.nan, inplace=True) 124 | 125 | # make datetimeindex from date/time fields 126 | # dfc = dfc.set_index(dfc['Date']) 127 | dfc = dfc.set_index(pd.DatetimeIndex(dfc['Date'])) 128 | dfc.sort_index(inplace=True) 129 | dfc = dfc.drop(['Date'], axis=1) 130 | 131 | # split route field into origin/destination 132 | values = dfc.Route.values 133 | dfc['Origin'] = [v.split(' - ')[0] if type(v) != float else np.nan for v in values] 134 | dfc['Destination'] = [v.split(' - ')[-1] if type(v) != float else np.nan for v in values] 135 | 136 | # split fatalities field 137 | dfc['Fatalities total'] = dfc['Fatalities'].apply(lambda x: split_fatalities(x)['total']) 138 | 139 | # make Ground field numeric (or nan) 140 | dfc['Ground'] = dfc['Ground'].apply(pd.to_numeric) 141 | 142 | # add country field extracted from accident location 143 | dfc['Location Country'] = dfc['Location'].apply(lambda x: country_of_loc(x)) 144 | 145 | return dfc 146 | --------------------------------------------------------------------------------