├── .dockerignore ├── .github ├── FUNDING.yml └── workflows │ └── main.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── brightsky ├── __init__.py ├── __main__.py ├── cli.py ├── db.py ├── enhancements.py ├── export.py ├── parsers.py ├── polling.py ├── query.py ├── settings.py ├── tasks.py ├── utils.py ├── web │ ├── __init__.py │ ├── app.py │ ├── intro.md │ ├── models.py │ └── params.py └── worker.py ├── docker-compose.yml ├── docs ├── CNAME ├── apple-touch-icon.png ├── brightsky.yml ├── demo │ ├── alerts │ │ ├── cells.json │ │ └── index.html │ ├── cities.json │ ├── img │ │ ├── arrow_down.svg │ │ └── arrow_right.svg │ ├── index.html │ └── radar │ │ ├── index.html │ │ ├── js-colormaps.js │ │ ├── pause.svg │ │ └── play.svg ├── docs │ ├── elements-styles.min.css │ ├── elements-web-components.min.js │ └── index.html ├── favicon-16x16.png ├── favicon-32x32.png ├── favicon.svg ├── img │ ├── architecture.svg │ ├── bmbf.svg │ ├── book.svg │ ├── coffee.svg │ ├── dwd.svg │ ├── eye.svg │ ├── github.svg │ ├── heart.svg │ ├── okfde.svg │ └── pf.svg └── index.html ├── migrations ├── 0001_migrations.sql ├── 0002_parsed_files.sql ├── 0003_weather.sql ├── 0004_station_name.sql ├── 0005_additional_weather_params.sql ├── 0006_wmo_station_ids.sql ├── 0007_synop.sql ├── 0008_current_weather.sql ├── 0009_sources_date_range.sql ├── 0010_weather_index_performance.sql ├── 0011_weather_condition.sql ├── 0012_remove_legacy_recent_records.sql ├── 0013_precipitation_probability.sql ├── 0014_solar.sql ├── 0015_radar.sql ├── 0016_alerts.sql ├── 0017_fix_ll_to_earth.sql └── 0018_alerts_status.sql ├── pyproject.toml ├── requirements-dev.in ├── requirements-dev.txt ├── requirements.txt ├── scripts ├── benchmark.py ├── benchmark_compression.py ├── benchmark_radar.py ├── equal_responses.py └── radar_coordinates.py └── tests ├── __init__.py ├── conftest.py ├── data ├── 10minutenwerte_SOLAR_01766_akt.zip ├── DE1200_RV2305081330.tar.bz2 ├── Meta_Daten_zehn_min_sd_01766.zip ├── Z_CAP_C_EDZW_LATEST_PVW_STATUS_PREMIUMDWD_COMMUNEUNION_MUL.zip ├── alert_cells.json ├── dwd_opendata_index.html ├── observations_current.csv ├── observations_recent_FF_akt.zip └── station_list.html ├── test_db.py ├── test_export.py ├── test_parsers.py ├── test_polling.py ├── test_settings.py ├── test_tasks.py ├── test_utils.py ├── test_web.py └── utils.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .gitignore -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: jdemaeyer 2 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Integration 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | tags: [ 'v*' ] 7 | pull_request: 8 | branches: [ master ] 9 | 10 | jobs: 11 | test: 12 | name: Run Test Suite 13 | runs-on: ubuntu-latest 14 | services: 15 | postgres: 16 | image: postgres:16-alpine 17 | env: 18 | POSTGRES_PASSWORD: pgpass 19 | options: >- 20 | --health-cmd pg_isready 21 | --health-interval 10s 22 | --health-timeout 5s 23 | --health-retries 5 24 | ports: 25 | - 5432:5432 26 | steps: 27 | - uses: actions/checkout@v4 28 | - name: Set up Python 3.13 29 | uses: actions/setup-python@v5 30 | with: 31 | python-version: '3.13' 32 | - name: Run tests 33 | env: 34 | BRIGHTSKY_DATABASE_URL: postgres://postgres:pgpass@localhost/brightsky_test 35 | run: | 36 | python -m pip install -r requirements-dev.txt 37 | ruff check . 38 | pytest 39 | 40 | push-to-pypi: 41 | name: Push Python Package to PyPI 42 | if: startsWith(github.ref, 'refs/tags/v') 43 | needs: test 44 | runs-on: ubuntu-latest 45 | permissions: 46 | id-token: write 47 | steps: 48 | - uses: actions/checkout@v4 49 | - name: Set up Python 3.13 50 | uses: actions/setup-python@v5 51 | with: 52 | python-version: '3.13' 53 | - name: Build Package 54 | run: | 55 | python -m pip install --upgrade pip 56 | python -m pip install --upgrade --user build 57 | python -m build 58 | - name: Push Package to PyPI 59 | uses: pypa/gh-action-pypi-publish@release/v1 60 | 61 | push-to-dockerhub: 62 | name: Push Docker Image to Docker Hub 63 | if: startsWith(github.ref, 'refs/tags/v') 64 | needs: test 65 | runs-on: ubuntu-latest 66 | steps: 67 | - uses: actions/checkout@v4 68 | - name: Set up QEMU 69 | uses: docker/setup-qemu-action@v3 70 | with: 71 | platforms: 'arm64' 72 | - name: Set up Docker Buildx 73 | uses: docker/setup-buildx-action@v3 74 | - name: Login to Docker Hub 75 | uses: docker/login-action@v3 76 | with: 77 | username: ${{ secrets.DOCKER_USERNAME }} 78 | password: ${{ secrets.DOCKER_ACCESSTOKEN }} 79 | - name: Build Image and Push to Docker Hub 80 | uses: docker/build-push-action@v5 81 | with: 82 | push: true 83 | tags: jdemaeyer/brightsky:${{ github.ref_name }},jdemaeyer/brightsky:latest 84 | platforms: linux/amd64,linux/arm64 85 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | .data 3 | .env 4 | .tox 5 | .venv 6 | build 7 | dist 8 | *.egg-info 9 | *.pyc 10 | docker-compose.override.yml 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.13-slim 2 | 3 | WORKDIR /app 4 | 5 | ENV PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 6 | 7 | COPY requirements.txt . 8 | 9 | RUN pip install -r requirements.txt 10 | 11 | COPY migrations migrations 12 | COPY brightsky brightsky 13 | COPY pyproject.toml . 14 | COPY README.md . 15 | 16 | RUN pip install . 17 | 18 | ENTRYPOINT ["python", "-m", "brightsky"] 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Jakob de Maeyer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Bright Sky 4 | 5 | [![API Status](https://img.shields.io/website?down_message=offline&label=api&up_message=online&url=https%3A%2F%2Fapi.brightsky.dev%2F)](https://api.brightsky.dev/) 6 | [![Docs Status](https://img.shields.io/website?down_message=offline&label=docs&up_message=online&url=https%3A%2F%2Fbrightsky.dev%2Fdocs%2F)](https://brightsky.dev/docs/) 7 | [![Build Status](https://img.shields.io/github/actions/workflow/status/jdemaeyer/brightsky/main.yml)](https://github.com/jdemaeyer/brightsky/actions) 8 | [![PyPI Release](https://img.shields.io/pypi/v/brightsky)](https://pypi.org/project/brightsky/) 9 | [![Docker Hub Release](https://img.shields.io/docker/v/jdemaeyer/brightsky/latest?label=docker)](https://hub.docker.com/r/jdemaeyer/brightsky) 10 | 11 | ### JSON API for DWD's open weather data. 12 | 13 | The DWD ([Deutscher Wetterdienst](https://www.dwd.de/)), as Germany's 14 | meteorological service, publishes a myriad of meteorological observations and 15 | calculations as part of their [Open Data 16 | program](https://www.dwd.de/DE/leistungen/opendata/opendata.html). 17 | 18 | [**Bright Sky**](https://brightsky.dev/) is an open-source project aiming to 19 | make some of the more popular data — in particular weather observations from 20 | the DWD station network and weather forecasts from the MOSMIX model — available 21 | in a free, simple JSON API. 22 | 23 | 24 | ### Looking for something specific? 25 | 26 | #### I just want to retrieve some weather data 27 | 28 | You can use the free [public Bright Sky instance](https://brightsky.dev/)! 29 | 30 | #### I want to run my own instance of Bright Sky 31 | 32 | Check out the [infrastructure 33 | repo](https://github.com/jdemaeyer/brightsky-infrastructure/)! 34 | 35 | #### I want to parse DWD weather files from the command line or in Python 36 | 37 | The parsing core for Bright Sky is maintained in a separate package named 38 | [`dwdparse`](https://github.com/jdemaeyer/dwdparse), which has no dependencies 39 | outside the standard library. If you find that's not quite serving your needs, 40 | check out [`wetterdienst`](https://github.com/earthobservations/wetterdienst). 41 | 42 | #### I want to contribute to Bright Sky's source code 43 | 44 | Read on. :) 45 | 46 | 47 | ### On Bright Sky's versioning 48 | 49 | Starting from version 2.0, where we extracted the parsing core into a [separate 50 | package](https://github.com/jdemaeyer/dwdparse), Bright Sky is **no longer 51 | intended to be used as a Python library**, but only as the service available at 52 | [`brightsky.dev`](https://brightsky.dev/). 53 | 54 | Consequentially, we adjust our version numbers from the _perspective of that 55 | service and its users_ – i.e., we will increase the major version number only 56 | when we introduce backwards-incompatible (or otherwise very major) changes to 57 | the actual JSON API interface, e.g. by changing URLs or parameters. This means 58 | that **increases of the minor version number may introduce 59 | backwards-incompatible changes to the internals of the `brightsky` package, 60 | including the database structure**. If you use `brightsky` as a Python library, 61 | please version-pin to a minor version, e.g. by putting `brightsky==2.0.*` in 62 | your `requirements.txt`. 63 | 64 | 65 | ## Quickstart 66 | 67 | ### Running a full-fledged API instance 68 | 69 | _Note: These instructions are aimed at running a Bright Sky instance for 70 | development and testing. Check out our [infrastructure 71 | repository](https://github.com/jdemaeyer/brightsky-infrastructure/) if you want 72 | to set up a production-level API instance._ 73 | 74 | Just run `docker-compose up` and you should be good to go. This will set up a 75 | PostgreSQL database (with persistent storage in `.data`), run a Redis server, 76 | and start the Bright Sky worker and webserver. The worker periodically polls 77 | the DWD Open Data Server for updates, parses them, and stores them in the 78 | database. The webserver will be listening to API requests on port 5000. 79 | 80 | 81 | ## Architecture 82 | 83 | ![Bright Sky's Architecture](docs/img/architecture.svg) 84 | 85 | Bright Sky is a rather simple project consisting of four components: 86 | 87 | * The `brightsky` worker, which leverages the logic contained in the 88 | `brightsky` Python package to retrieve weather records from the DWD server, 89 | parse them, and store them in a database. It will periodically poll the DWD 90 | servers for new data. 91 | 92 | * The `brightsky` webserver (API), which serves as gate to our database and 93 | processes all queries for weather records coming from the outside world. 94 | 95 | * A PostgreSQL database consisting of two relevant tables: 96 | 97 | * `sources` contains information on the locations for which we hold weather 98 | records, and 99 | * `weather` contains the history of actual meteorological measurements (or 100 | forecasts) for these locations. 101 | 102 | The database structure can be set up by running the `migrate` command, which 103 | will simply apply all `.sql` files found in the `migrations` folder. 104 | 105 | * A Redis server, which is used as the backend of the worker's task queue. 106 | 107 | Most of the tasks performed by the worker and webserver can also be performed 108 | independently. Run `docker-compose run --rm brightsky` to get a list of 109 | available commands. 110 | 111 | 112 | ## Hacking 113 | 114 | Constantly rebuilding the `brightsky` container while working on the code can 115 | become cumbersome, and the default setting of parsing records dating all the 116 | way back to 2010 will make your development database unnecessarily large. You 117 | can set up a more lightweight development environment as follows: 118 | 119 | 1. Create a virtual environment and install our dependencies: 120 | `python -m virtualenv .venv && source .venv/bin/activate && pip install -r 121 | requirements.txt && pip install -e .` 122 | 123 | 2. Start a PostgreSQL container: 124 | `docker-compose run --rm -p 5432:5432 postgres` 125 | 126 | 3. Start a Redis container: 127 | `docker-compose run --rm -p 6379:6379 redis` 128 | 129 | 4. Point `brightsky` to your containers, and configure a tighter date 130 | threshold for parsing DWD data, by adding the following `.env` file: 131 | ``` 132 | BRIGHTSKY_DATABASE_URL=postgres://postgres:pgpass@localhost 133 | BRIGHTSKY_BENCHMARK_DATABASE_URL=postgres://postgres:pgpass@localhost/benchmark 134 | BRIGHTSKY_REDIS_URL=redis://localhost 135 | BRIGHTSKY_MIN_DATE=2020-01-01 136 | ``` 137 | 138 | You should now be able to directly run `brightsky` commands via `python -m 139 | brightsky`, and changes to the source code should be effective immediately. 140 | 141 | 142 | ### Tests 143 | 144 | Large parts of our test suite run against a real Postgres database. By default, 145 | these tests will be skipped. To enable them, make sure the 146 | `BRIGHTSKY_DATABASE_URL` environment variable is set when calling `pytest`, 147 | e.g. via: 148 | ``` 149 | BRIGHTSKY_DATABASE_URL=postgres://postgres:pgpass@localhost/brightsky_test pytest 150 | ``` 151 | 152 | Beware that adding this environment variable to your `.env` file will not work 153 | as that file is not read by `pytest`. The database will be **dropped and 154 | recreated** on every test run, so don't use your normal Bright Sky database. ;) 155 | 156 | 157 | ## Acknowledgements 158 | 159 | Bright Sky's development is boosted by the priceless guidance and support of 160 | the [Open Knowledge Foundation](https://www.okfn.de/)'s [Prototype 161 | Fund](https://prototypefund.de/) program, and is generously funded by Germany's 162 | [Federal Ministry of Education and Research](https://www.bmbf.de/). Obvious as 163 | it may be, it should be mentioned that none of this would be possible without 164 | the painstaking, never-ending effort of the [Deutscher 165 | Wetterdienst](https://www.dwd.de/). 166 | 167 | Prototype Fund     168 | Open Knowledge Foundation Germany     169 | Bundesministerium für Bildung und Forschung     170 | Deutscher Wetterdienst 171 | -------------------------------------------------------------------------------- /brightsky/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '2.2.6' 2 | -------------------------------------------------------------------------------- /brightsky/__main__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from brightsky import __version__ 4 | from brightsky.cli import cli 5 | from brightsky.utils import configure_logging, load_dotenv 6 | 7 | 8 | def _getenv_float(key): 9 | x = os.getenv(key) 10 | return float(x) if x else None 11 | 12 | 13 | if __name__ == '__main__': 14 | load_dotenv() 15 | configure_logging() 16 | if os.getenv('SENTRY_DSN'): 17 | import sentry_sdk 18 | sentry_sdk.init( 19 | dsn=os.getenv('SENTRY_DSN'), 20 | release=__version__, 21 | traces_sample_rate=_getenv_float('SENTRY_TRACES_SAMPLE_RATE'), 22 | profiles_sample_rate=_getenv_float('SENTRY_PROFILES_SAMPLE_RATE'), 23 | ) 24 | cli(prog_name='python -m brightsky') 25 | -------------------------------------------------------------------------------- /brightsky/cli.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | import click 5 | import uvicorn 6 | from fastapi.testclient import TestClient 7 | from huey.consumer_options import ConsumerConfig 8 | 9 | from brightsky import db, tasks 10 | from brightsky.utils import parse_date 11 | from brightsky.web import app 12 | from brightsky.worker import huey 13 | 14 | 15 | def dump_records(it): 16 | for record in it: 17 | print(json.dumps(record, default=str)) 18 | 19 | 20 | def migrate_callback(ctx, param, value): 21 | if value: 22 | db.migrate() 23 | 24 | 25 | def parse_date_arg(ctx, param, value): 26 | if not value: 27 | return 28 | return parse_date(value) 29 | 30 | 31 | @click.group() 32 | @click.option( 33 | '--migrate', help='Migrate database before running command.', 34 | is_flag=True, is_eager=True, expose_value=False, callback=migrate_callback) 35 | def cli(): 36 | pass 37 | 38 | 39 | @cli.command() 40 | def migrate(): 41 | """Apply all pending database migrations.""" 42 | db.migrate() 43 | 44 | 45 | @cli.command() 46 | @click.argument( 47 | 'targets', 48 | required=True, 49 | nargs=-1, 50 | metavar='TARGET [TARGET ...]', 51 | ) 52 | def parse(targets): 53 | for target in targets: 54 | tasks.parse(target) 55 | 56 | 57 | @cli.command() 58 | @click.option( 59 | '--enqueue/--no-enqueue', default=False, 60 | help='Enqueue updated files for processing by the worker') 61 | def poll(enqueue): 62 | """Detect updated files on DWD Open Data Server.""" 63 | files = tasks.poll(enqueue=enqueue) 64 | if not enqueue: 65 | dump_records(files) 66 | 67 | 68 | @cli.command() 69 | def clean(): 70 | """Clean expired forecast and observations from database.""" 71 | tasks.clean() 72 | 73 | 74 | @cli.command() 75 | @click.option('--workers', default=3, type=int, help='Number of threads') 76 | def work(workers): 77 | """Start brightsky worker.""" 78 | huey.flush() 79 | config = ConsumerConfig(worker_type='thread', workers=workers) 80 | config.validate() 81 | consumer = huey.create_consumer(**config.values) 82 | consumer.run() 83 | 84 | 85 | @cli.command() 86 | @click.option('--bind', default='127.0.0.1:5000', help='Bind address') 87 | @click.option( 88 | '--reload/--no-reload', default=False, 89 | help='Reload server on source code changes') 90 | @click.option('--workers', default=1, type=int, help='Number of workers') 91 | def serve(bind, reload, workers): 92 | """Start brightsky API webserver.""" 93 | host, port = bind.rsplit(':', 1) 94 | uvicorn.run( 95 | 'brightsky.web:app', 96 | host=host, 97 | port=int(port), 98 | reload=reload, 99 | workers=workers, 100 | ) 101 | 102 | 103 | @cli.command(context_settings={'ignore_unknown_options': True}) 104 | @click.argument('endpoint') 105 | @click.argument('parameters', nargs=-1, type=click.UNPROCESSED) 106 | def query(endpoint, parameters): 107 | """Query API and print JSON response. 108 | 109 | Parameters must be supplied as --name value or --name=value. See 110 | https://brightsky.dev/docs/ for the available endpoints and arguments. 111 | 112 | \b 113 | Examples: 114 | python -m brightsky query weather --lat 52 --lon 7.6 --date 2018-08-13 115 | python -m brightsky query current_weather --lat=52 --lon=7.6 116 | """ 117 | for route in app.routes: 118 | if route.path == f'/{endpoint}': 119 | break 120 | else: 121 | raise click.UsageError(f"Unknown endpoint '{endpoint}'") 122 | logging.getLogger().setLevel(logging.WARNING) 123 | with TestClient(app) as client: 124 | resp = client.get(f'/{endpoint}', params=_parse_params(parameters)) 125 | print(json.dumps(resp.json())) 126 | 127 | 128 | def _parse_params(parameters): 129 | # I'm sure there's a function in click or argparse somewhere that does this 130 | # but I can't find it 131 | usage = "Supply API parameters as --name value or --name=value" 132 | params = {} 133 | param_name = None 134 | for param in parameters: 135 | if param_name is None: 136 | if not param.startswith('--'): 137 | raise click.UsageError(usage) 138 | param = param[2:] 139 | if '=' in param: 140 | name, value = param.split('=', 1) 141 | params[name] = value 142 | else: 143 | param_name = param 144 | else: 145 | params[param_name] = param 146 | param_name = None 147 | if param_name is not None: 148 | raise click.UsageError(usage) 149 | return params 150 | -------------------------------------------------------------------------------- /brightsky/db.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import logging 3 | import os 4 | import re 5 | from contextlib import contextmanager, suppress 6 | 7 | import psycopg2 8 | from psycopg2.extras import DictCursor 9 | from psycopg2.pool import ThreadedConnectionPool 10 | 11 | from brightsky.settings import settings 12 | 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | @contextmanager 18 | def get_connection(): 19 | if not hasattr(get_connection, '_pool'): 20 | maxconn = 1 if 'gunicorn' in os.getenv('SERVER_SOFTWARE', '') else 100 21 | get_connection._pool = ThreadedConnectionPool( 22 | 1, maxconn, settings.DATABASE_URL, cursor_factory=DictCursor) 23 | pool = get_connection._pool 24 | conn = pool.getconn() 25 | try: 26 | with conn: 27 | yield conn 28 | except psycopg2.InterfaceError: 29 | logger.warning('Discarding dead connection pool') 30 | pool.closeall() 31 | del get_connection._pool 32 | raise 33 | finally: 34 | if not pool.closed: 35 | pool.putconn(conn) 36 | 37 | 38 | def fetch(*args, **kwargs): 39 | for retry in range(5): 40 | with suppress(psycopg2.InterfaceError): 41 | with get_connection() as conn: 42 | with conn.cursor() as cur: 43 | cur.execute(*args, **kwargs) 44 | return cur.fetchall() 45 | 46 | 47 | def migrate(): 48 | logger.info("Migrating database") 49 | with get_connection() as conn: 50 | with conn.cursor() as cur: 51 | try: 52 | cur.execute('SELECT MAX(id) FROM migrations;') 53 | except psycopg2.errors.UndefinedTable: 54 | conn.rollback() 55 | latest_migration = 0 56 | else: 57 | latest_migration = cur.fetchone()[0] 58 | 59 | migration_paths = [ 60 | f for f in sorted(glob.glob('migrations/*.sql')) 61 | if (m := re.match(r'(\d+)_', os.path.basename(f))) 62 | and int(m.group(1)) > latest_migration 63 | ] 64 | 65 | for path in migration_paths: 66 | logger.info("Applying %s", path) 67 | match = re.match(r'(\d+)_?(.*)\.sql', os.path.basename(path)) 68 | migration_id = int(match.group(1)) 69 | migration_name = match.group(2) 70 | with open(path) as f: 71 | cur.execute(f.read()) 72 | cur.execute( 73 | 'INSERT INTO migrations (id, name) VALUES (%s, %s);', 74 | (migration_id, migration_name)) 75 | conn.commit() 76 | if migration_paths: 77 | logger.info("Applied %d migrations", len(migration_paths)) 78 | else: 79 | logger.info("No new migrations") 80 | -------------------------------------------------------------------------------- /brightsky/enhancements.py: -------------------------------------------------------------------------------- 1 | from dwdparse.units import convert_record 2 | 3 | from brightsky.settings import settings 4 | from brightsky.utils import daytime 5 | 6 | 7 | def enhance(result, timezone=None, units='si'): 8 | if 'sources' in result: 9 | enhance_sources(result['sources'], timezone=timezone) 10 | if 'weather' in result: 11 | source_map = {s['id']: s for s in result['sources']} 12 | enhance_records( 13 | result['weather'], 14 | source_map, 15 | timezone=timezone, 16 | units=units, 17 | ) 18 | if 'radar' in result: 19 | enhance_radar(result['radar'], timezone=timezone) 20 | if 'alerts' in result: 21 | enhance_alerts(result['alerts'], timezone=timezone) 22 | 23 | 24 | def enhance_records(records, source_map, timezone=None, units='si'): 25 | if not isinstance(records, list): 26 | records = [records] 27 | for record in records: 28 | record['icon'] = get_icon(record, source_map) 29 | if units != 'si': 30 | convert_record(record, units) 31 | process_timestamp(record, 'timestamp', timezone) 32 | 33 | 34 | def enhance_sources(sources, timezone=None): 35 | for source in sources: 36 | process_timestamp(source, 'first_record', timezone) 37 | process_timestamp(source, 'last_record', timezone) 38 | 39 | 40 | def enhance_radar(radar, timezone=None): 41 | for record in radar: 42 | process_timestamp(record, 'timestamp', timezone) 43 | 44 | 45 | def enhance_alerts(alerts, timezone=None): 46 | for alert in alerts: 47 | for key in ['effective', 'onset', 'expires']: 48 | process_timestamp(alert, key, timezone) 49 | 50 | 51 | def process_timestamp(o, key, timezone): 52 | if not timezone: 53 | return 54 | elif not o[key]: 55 | return 56 | o[key] = o[key].astimezone(timezone) 57 | 58 | 59 | def get_icon(record, source_map): 60 | if record['condition'] in ( 61 | 'fog', 'sleet', 'snow', 'hail', 'thunderstorm'): 62 | return record['condition'] 63 | try: 64 | precipitation = record['precipitation'] 65 | except KeyError: 66 | precipitation = record['precipitation_10'] 67 | try: 68 | wind_speed = record['wind_speed'] 69 | except KeyError: 70 | wind_speed = record['wind_speed_10'] 71 | # Don't show 'rain' icon for little precipitation, and do show 'rain' 72 | # icon when condition is None but there is significant precipitation 73 | is_rainy = ( 74 | record['condition'] == 'rain' and precipitation is None) or ( 75 | (precipitation or 0) > settings.ICON_RAIN_THRESHOLD) 76 | if is_rainy: 77 | return 'rain' 78 | elif (wind_speed or 0) > settings.ICON_WIND_THRESHOLD: 79 | return 'wind' 80 | elif (record['cloud_cover'] or 0) >= settings.ICON_CLOUDY_THRESHOLD: 81 | return 'cloudy' 82 | source = source_map[record['source_id']] 83 | daytime_str = daytime(source['lat'], source['lon'], record['timestamp']) 84 | if (record['cloud_cover'] or 0) >= settings.ICON_PARTLY_CLOUDY_THRESHOLD: 85 | return f'partly-cloudy-{daytime_str}' 86 | return f'clear-{daytime_str}' 87 | -------------------------------------------------------------------------------- /brightsky/parsers.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import datetime 3 | import re 4 | 5 | import dwdparse.parsers 6 | import numpy as np 7 | from dateutil.tz import tzutc 8 | from isal import isal_zlib as zlib 9 | 10 | from brightsky.db import fetch 11 | from brightsky.export import ( 12 | AlertExporter, 13 | DBExporter, 14 | RADOLANExporter, 15 | SYNOPExporter, 16 | ) 17 | from brightsky.settings import settings 18 | 19 | 20 | class BrightSkyMixin: 21 | 22 | PRIORITY = 10 23 | exporter = DBExporter 24 | 25 | def skip_path(self, path): 26 | return False 27 | 28 | 29 | class ObservationsBrightSkyMixin(BrightSkyMixin): 30 | 31 | def skip_path(self, path): 32 | if (m := re.search(r'_(\d{8})_(\d{8})_hist\.zip$', str(path))): 33 | end_date = datetime.datetime.strptime( 34 | m.group(2), 35 | '%Y%m%d', 36 | ).replace(tzinfo=tzutc()) 37 | if end_date < settings.MIN_DATE: 38 | return True 39 | if settings.MAX_DATE: 40 | start_date = datetime.datetime.strptime( 41 | m.group(1), '%Y%m%d').replace(tzinfo=tzutc()) 42 | if start_date > settings.MAX_DATE: 43 | return True 44 | return False 45 | 46 | def skip_timestamp(self, timestamp): 47 | if timestamp < settings.MIN_DATE: 48 | return True 49 | elif settings.MAX_DATE and timestamp > settings.MAX_DATE: 50 | return True 51 | return False 52 | 53 | 54 | class MOSMIXParser(BrightSkyMixin, dwdparse.parsers.MOSMIXParser): 55 | 56 | PRIORITY = 20 57 | 58 | 59 | class SYNOPParser(BrightSkyMixin, dwdparse.parsers.SYNOPParser): 60 | 61 | PRIORITY = 30 62 | exporter = SYNOPExporter 63 | 64 | 65 | class CurrentObservationsParser( 66 | BrightSkyMixin, 67 | dwdparse.parsers.CurrentObservationsParser, 68 | ): 69 | 70 | PRIORITY = 30 71 | 72 | def skip_path(self, path): 73 | return path.endswith(tuple( 74 | f'{station:_<5}-BEOB.csv' 75 | for station in settings.IGNORED_CURRENT_OBSERVATIONS_STATIONS 76 | )) 77 | 78 | def parse(self, path, lat=None, lon=None, height=None, station_name=None): 79 | if any(x is None for x in (lat, lon, height, station_name)): 80 | with open(path) as f: 81 | reader = csv.DictReader(f, delimiter=';') 82 | wmo_station_id = next(reader)[self.DATE_COLUMN].rstrip('_') 83 | lat, lon, height, station_name = self._load_location( 84 | wmo_station_id, 85 | ) 86 | return super().parse( 87 | path, 88 | lat=lat, 89 | lon=lon, 90 | height=height, 91 | station_name=station_name, 92 | ) 93 | 94 | def _load_location(self, wmo_station_id): 95 | rows = fetch( 96 | """ 97 | SELECT lat, lon, height, station_name 98 | FROM sources 99 | WHERE wmo_station_id = %s 100 | ORDER BY observation_type DESC, id DESC 101 | LIMIT 1 102 | """, 103 | (wmo_station_id,), 104 | ) 105 | if not rows: 106 | raise ValueError(f'Cannot find location for WMO {wmo_station_id}') 107 | return rows[0] 108 | 109 | 110 | class CloudCoverObservationsParser( 111 | ObservationsBrightSkyMixin, 112 | dwdparse.parsers.CloudCoverObservationsParser, 113 | ): 114 | pass 115 | 116 | 117 | class DewPointObservationsParser( 118 | ObservationsBrightSkyMixin, 119 | dwdparse.parsers.DewPointObservationsParser, 120 | ): 121 | pass 122 | 123 | 124 | class TemperatureObservationsParser( 125 | ObservationsBrightSkyMixin, 126 | dwdparse.parsers.TemperatureObservationsParser, 127 | ): 128 | pass 129 | 130 | 131 | class PrecipitationObservationsParser( 132 | ObservationsBrightSkyMixin, 133 | dwdparse.parsers.PrecipitationObservationsParser, 134 | ): 135 | pass 136 | 137 | 138 | class SolarRadiationObservationsParser( 139 | ObservationsBrightSkyMixin, 140 | dwdparse.parsers.SolarRadiationObservationsParser, 141 | ): 142 | 143 | def skip_timestamp(self, timestamp): 144 | # We aggregate solar radiation from ten-minute data, where the values 145 | # correspond to radiation for the NEXT ten minutes, i.e. the value 146 | # tagged 14:30 contains the solar radiation between 14:30 - 14:40 (I 147 | # have not found a place where this is officially documented, but this 148 | # interpretation makes the values align with the hourly data from the 149 | # 'current' sources). 150 | # This makes solar radiation the only parameter where the 'recent' 151 | # sources produce a data point for today, which is otherwise only 152 | # served by the 'current' sources. To avoid excessive fill-up when 153 | # querying today's weather, we ignore this last data point (but will 154 | # pick it up on the next day). 155 | if timestamp.date() == datetime.date.today(): 156 | return True 157 | return super().skip_timestamp(timestamp) 158 | 159 | 160 | class VisibilityObservationsParser( 161 | ObservationsBrightSkyMixin, 162 | dwdparse.parsers.VisibilityObservationsParser, 163 | ): 164 | pass 165 | 166 | 167 | class WindObservationsParser( 168 | ObservationsBrightSkyMixin, 169 | dwdparse.parsers.WindObservationsParser, 170 | ): 171 | pass 172 | 173 | 174 | class WindGustsObservationsParser( 175 | ObservationsBrightSkyMixin, 176 | dwdparse.parsers.WindGustsObservationsParser, 177 | ): 178 | pass 179 | 180 | 181 | class SunshineObservationsParser( 182 | ObservationsBrightSkyMixin, 183 | dwdparse.parsers.SunshineObservationsParser, 184 | ): 185 | pass 186 | 187 | 188 | class PressureObservationsParser( 189 | ObservationsBrightSkyMixin, 190 | dwdparse.parsers.PressureObservationsParser, 191 | ): 192 | pass 193 | 194 | 195 | class RADOLANParser(BrightSkyMixin, dwdparse.parsers.RADOLANParser): 196 | 197 | PRIORITY = 30 198 | exporter = RADOLANExporter 199 | 200 | def process_raw_data(self, raw): 201 | # XXX: Unlike with the other weather parameters, because of it's large 202 | # size, we're storing the radar data in a half-raw state and 203 | # performing some final processing during runtime. This brings 204 | # down the response time for retrieving one full radar scan 205 | # (single timestamp, 1200x1100 pixels) from 1.5 seconds to about 206 | # 1 ms, mainly because of the reduced data transfer when fetching 207 | # the scan from the database. An important caveat of this is that 208 | # we are replacing `None` with `0`! 209 | data = np.array(raw, dtype='i2') 210 | data[data > 4095] = 0 211 | data = np.flipud(data.reshape((1200, 1100))) 212 | return zlib.compress(np.ascontiguousarray(data)) 213 | 214 | 215 | class CAPParser(BrightSkyMixin, dwdparse.parsers.CAPParser): 216 | 217 | PRIORITY = 40 218 | exporter = AlertExporter 219 | 220 | 221 | def get_parser(filename): 222 | parsers = { 223 | r'DE1200_RV': RADOLANParser, 224 | r'MOSMIX_(S|L)_LATEST(_240)?\.kmz$': MOSMIXParser, 225 | r'Z_CAP_C_EDZW_LATEST_.*_COMMUNEUNION_MUL\.zip': CAPParser, 226 | r'Z__C_EDZW_\d+_.*\.json\.bz2$': SYNOPParser, 227 | r'\w{5}-BEOB\.csv$': CurrentObservationsParser, 228 | 'stundenwerte_FF_': WindObservationsParser, 229 | 'stundenwerte_N_': CloudCoverObservationsParser, 230 | 'stundenwerte_P0_': PressureObservationsParser, 231 | 'stundenwerte_RR_': PrecipitationObservationsParser, 232 | 'stundenwerte_SD_': SunshineObservationsParser, 233 | 'stundenwerte_TD_': DewPointObservationsParser, 234 | 'stundenwerte_TU_': TemperatureObservationsParser, 235 | 'stundenwerte_VV_': VisibilityObservationsParser, 236 | '10minutenwerte_extrema_wind_': WindGustsObservationsParser, 237 | '10minutenwerte_SOLAR_': SolarRadiationObservationsParser, 238 | } 239 | for pattern, parser in parsers.items(): 240 | if re.match(pattern, filename): 241 | return parser 242 | -------------------------------------------------------------------------------- /brightsky/polling.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import re 4 | 5 | import dateutil.parser 6 | import requests 7 | from dateutil.tz import tzutc 8 | from parsel import Selector 9 | 10 | from brightsky.db import fetch 11 | from brightsky.parsers import get_parser 12 | 13 | 14 | class DWDPoller: 15 | 16 | urls = [ 17 | 'https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_L/' 18 | 'all_stations/kml/', 19 | 'https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_S/' 20 | 'all_stations/kml/', 21 | 'https://opendata.dwd.de/weather/alerts/cap/COMMUNEUNION_DWD_STAT/', 22 | 'https://opendata.dwd.de/weather/radar/composite/rv/', 23 | 'https://opendata.dwd.de/weather/weather_reports/synoptic/germany/' 24 | 'json/', 25 | 'https://opendata.dwd.de/weather/weather_reports/poi/', 26 | ] + [ 27 | 'https://opendata.dwd.de/climate_environment/CDC/observations_germany/' 28 | f'climate/hourly/{subfolder}/' 29 | for subfolder in [ 30 | 'air_temperature', 'cloudiness', 'dew_point', 'visibility', 31 | 'precipitation', 'pressure', 'sun', 'wind'] 32 | ] + [ 33 | 'https://opendata.dwd.de/climate_environment/CDC/observations_germany/' 34 | f'climate/10_minutes/{param}/{subfolder}/' 35 | for param in ['extreme_wind', 'solar'] 36 | for subfolder in ['recent', 'historical'] 37 | ] 38 | 39 | @property 40 | def logger(self): 41 | if not hasattr(self, '_logger'): 42 | self._logger = logging.getLogger(self.__class__.__name__) 43 | return self._logger 44 | 45 | def poll(self): 46 | self.logger.info("Polling for updated files") 47 | parsed_files = { 48 | row['url']: row 49 | for row in fetch('SELECT * FROM parsed_files') 50 | } 51 | for url in self.urls: 52 | for file_info in self.poll_url(url): 53 | if not self.matches_known_fingerprint(parsed_files, file_info): 54 | yield file_info 55 | 56 | def poll_url(self, url): 57 | self.logger.debug("Loading %s", url) 58 | resp = requests.get(url) 59 | resp.raise_for_status() 60 | return self.parse(url, resp.text) 61 | 62 | def parse(self, url, resp_text): 63 | sel = Selector(resp_text) 64 | directories = [] 65 | files = [] 66 | for anchor_sel in sel.css('a'): 67 | link = anchor_sel.css('::attr(href)').extract_first() 68 | if link.startswith('.'): 69 | continue 70 | link_url = f'{url}{link}' 71 | if link.endswith('/'): 72 | directories.append(link_url) 73 | else: 74 | fingerprint = anchor_sel.xpath( 75 | './following-sibling::text()[1]').extract_first() 76 | match = re.match( 77 | r'\s*(\d+-\w+-\d+ \d+:\d+(:\d+)?)\s+(\d+)', fingerprint) 78 | last_modified = dateutil.parser.parse( 79 | match.group(1)).replace(tzinfo=tzutc()) 80 | file_size = int(match.group(3)) 81 | parser_cls = get_parser(link) 82 | if parser_cls and not parser_cls().skip_path(link): 83 | files.append({ 84 | 'url': link_url, 85 | 'parser': parser_cls.__name__, 86 | 'last_modified': last_modified, 87 | 'file_size': file_size, 88 | }) 89 | self.logger.debug( 90 | "Found %d directories and %d files at %s", 91 | len(directories), len(files), url) 92 | yield from files 93 | for dir_url in directories: 94 | yield from self.poll_url(dir_url) 95 | 96 | def matches_known_fingerprint(self, parsed_files, file_info): 97 | parsed_info = parsed_files.get(file_info['url']) 98 | if not parsed_info: 99 | return False 100 | last_modified_diff = abs( 101 | file_info['last_modified'] - parsed_info['last_modified']) 102 | # The downloaded file timestamp will sometimes be off from the index 103 | # page timestamp by one second, presumably because it is delivered from 104 | # a different server than the one that delivered the index page. If the 105 | # file was modified at 59 seconds past the minute this can lead to a 106 | # timestamp that is off by one minute (as seconds are not part of the 107 | # timestamp). To not re-process these files over and over we allow a 108 | # one minute time difference from the known fingerprint. 109 | return ( 110 | file_info['file_size'] == parsed_info['file_size'] and 111 | last_modified_diff <= datetime.timedelta(minutes=1)) 112 | -------------------------------------------------------------------------------- /brightsky/settings.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | from multiprocessing import cpu_count 4 | 5 | from dateutil.tz import tzutc 6 | 7 | from brightsky.utils import load_dotenv 8 | 9 | 10 | CORS_ALLOW_ALL_ORIGINS = False 11 | CORS_ALLOW_ALL_HEADERS = False 12 | CORS_ALLOWED_ORIGINS = [] 13 | CORS_ALLOWED_HEADERS = [] 14 | DATABASE_CONNECTION_POOL_SIZE = cpu_count() 15 | DATABASE_URL = 'postgres://localhost' 16 | ICON_CLOUDY_THRESHOLD = 80 17 | ICON_PARTLY_CLOUDY_THRESHOLD = 25 18 | ICON_RAIN_THRESHOLD = 0.5 19 | ICON_WIND_THRESHOLD = 10.8 20 | IGNORED_CURRENT_OBSERVATIONS_STATIONS = ['K386'] 21 | KEEP_DOWNLOADS = False 22 | MIN_DATE = datetime.datetime(2010, 1, 1, tzinfo=tzutc()) 23 | MAX_DATE = None 24 | POLLING_CRONTAB_MINUTE = '*' 25 | REDIS_URL = 'redis://localhost' 26 | SERVER_URL = 'http://localhost:5000' 27 | WARN_CELLS_URL = ( 28 | 'https://maps.dwd.de/geoserver/wfs' 29 | '?SERVICE=WFS&VERSION=2.0.0&REQUEST=GetFeature' 30 | '&TYPENAMES=Warngebiete_Gemeinden&OUTPUTFORMAT=json' 31 | ) 32 | 33 | 34 | def _make_bool(bool_str): 35 | return bool_str == '1' 36 | 37 | 38 | def _make_date(date_str): 39 | return datetime.datetime.fromisoformat(date_str).replace(tzinfo=tzutc()) 40 | 41 | 42 | def _make_list(list_str, separator=','): 43 | if not list_str: 44 | return [] 45 | return list_str.split(separator) 46 | 47 | 48 | _SETTING_PARSERS = { 49 | 'MAX_DATE': _make_date, 50 | 51 | bool: _make_bool, 52 | datetime.datetime: _make_date, 53 | float: float, 54 | int: int, 55 | list: _make_list, 56 | } 57 | 58 | 59 | class Settings(dict): 60 | """A dictionary that makes its keys available as attributes""" 61 | 62 | def __init__(self, *args, **kwargs): 63 | super().__init__(*args, **kwargs) 64 | self.loaded = False 65 | 66 | def load(self): 67 | load_dotenv() 68 | for k, v in globals().items(): 69 | if k.isupper() and not k.startswith('_'): 70 | self[k] = v 71 | for k, v in os.environ.items(): 72 | if k.startswith('BRIGHTSKY_') and k.isupper(): 73 | setting_name = k.split('_', 1)[1] 74 | setting_type = type(self.get(setting_name)) 75 | setting_parser = _SETTING_PARSERS.get( 76 | setting_name, _SETTING_PARSERS.get(setting_type)) 77 | if setting_parser: 78 | v = setting_parser(v) 79 | self[setting_name] = v 80 | 81 | def __getattr__(self, name): 82 | if not self.loaded: 83 | self.load() 84 | self.loaded = True 85 | return self[name] 86 | 87 | 88 | settings = Settings() 89 | -------------------------------------------------------------------------------- /brightsky/tasks.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import tempfile 4 | 5 | from brightsky.db import get_connection 6 | from brightsky.parsers import get_parser 7 | from brightsky.polling import DWDPoller 8 | from brightsky.utils import download 9 | from brightsky.worker import huey, process 10 | 11 | 12 | logger = logging.getLogger('brightsky') 13 | 14 | 15 | def parse(url): 16 | parser = get_parser(os.path.basename(url))() 17 | with tempfile.TemporaryDirectory() as tmpdir: 18 | path, fingerprint = download(url, tmpdir) 19 | extra = { 20 | kwarg: download(extra_url, tmpdir)[0] 21 | for kwarg, extra_url in parser.get_extra_urls(path).items() 22 | } 23 | exporter = parser.exporter() 24 | exporter.export(parser.parse(path, **extra), fingerprint=fingerprint) 25 | 26 | 27 | def poll(enqueue=False): 28 | updated_files = DWDPoller().poll() 29 | if enqueue: 30 | if (expired_locks := huey.expire_locks(1800)): 31 | logger.warning( 32 | 'Removed expired locks: %s', ', '.join(expired_locks)) 33 | pending_urls = [ 34 | t.args[0] for t in huey.pending() if t.name == 'process'] 35 | enqueued = 0 36 | for updated_file in updated_files: 37 | url = updated_file['url'] 38 | if url in pending_urls: 39 | logger.debug('Skipping "%s": already queued', url) 40 | continue 41 | elif huey.is_locked(url): 42 | logger.debug('Skipping "%s": already running', url) 43 | continue 44 | logger.debug('Enqueueing "%s"', url) 45 | parser_cls = get_parser(os.path.basename(url)) 46 | process(url, priority=parser_cls.PRIORITY) 47 | enqueued += 1 48 | queue_size = len([t for t in huey.pending() if t.name == 'process']) 49 | logger.info( 50 | 'Enqueued %d updated files for processing. Queue size: %d', 51 | enqueued, 52 | queue_size, 53 | ) 54 | return updated_files 55 | 56 | 57 | def clean(): 58 | expiry_intervals = { 59 | 'weather': { 60 | 'forecast': '3 hours', 61 | 'current': '48 hours', 62 | }, 63 | 'synop': { 64 | 'synop': '30 hours', 65 | }, 66 | } 67 | radar_expiry_interval = '6 hours' 68 | parsed_files_expiry_intervals = { 69 | '%/Z__C_EDZW_%': '1 week', 70 | '%/DE1200_RV%': '1 week', 71 | } 72 | with get_connection() as conn: 73 | with conn.cursor() as cur: 74 | logger.info( 75 | 'Deleting expired weather records: %s', expiry_intervals) 76 | for table, table_expires in expiry_intervals.items(): 77 | for observation_type, interval in table_expires.items(): 78 | cur.execute( 79 | f""" 80 | DELETE FROM {table} WHERE 81 | source_id IN ( 82 | SELECT id FROM sources 83 | WHERE observation_type = %s) AND 84 | timestamp < current_timestamp - %s::interval; 85 | """, 86 | (observation_type, interval), 87 | ) 88 | conn.commit() 89 | if cur.rowcount: 90 | logger.info( 91 | 'Deleted %d outdated %s weather records from %s', 92 | cur.rowcount, observation_type, table) 93 | cur.execute( 94 | f""" 95 | UPDATE sources SET 96 | first_record = record_range.first_record, 97 | last_record = record_range.last_record 98 | FROM ( 99 | SELECT 100 | source_id, 101 | MIN(timestamp) AS first_record, 102 | MAX(timestamp) AS last_record 103 | FROM {table} 104 | GROUP BY source_id 105 | ) AS record_range 106 | WHERE sources.id = record_range.source_id; 107 | """) 108 | conn.commit() 109 | logger.info('Deleting expired radar records') 110 | cur.execute( 111 | """ 112 | DELETE FROM radar WHERE 113 | timestamp < current_timestamp - %s::interval; 114 | """, 115 | (radar_expiry_interval,), 116 | ) 117 | conn.commit() 118 | if cur.rowcount: 119 | logger.info('Deleted %d outdated radar records', cur.rowcount) 120 | logger.info( 121 | 'Deleting expired parsed files: %s', 122 | parsed_files_expiry_intervals) 123 | for filename, interval in parsed_files_expiry_intervals.items(): 124 | cur.execute( 125 | """ 126 | DELETE FROM parsed_files WHERE 127 | url LIKE %s AND 128 | parsed_at < current_timestamp - %s::interval; 129 | """, 130 | (filename, interval)) 131 | conn.commit() 132 | if cur.rowcount: 133 | logger.info( 134 | 'Deleted %d outdated parsed files for pattern "%s"', 135 | cur.rowcount, filename) 136 | -------------------------------------------------------------------------------- /brightsky/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from contextlib import suppress 4 | from functools import lru_cache 5 | 6 | import coloredlogs 7 | import dateutil.parser 8 | from astral import Observer 9 | from astral.sun import daylight 10 | 11 | import requests 12 | 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | USER_AGENT = 'Bright Sky / https://brightsky.dev/' 18 | 19 | 20 | def configure_logging(): 21 | log_fmt = '%(asctime)s %(name)s %(levelname)s %(message)s' 22 | coloredlogs.install(level=logging.DEBUG, fmt=log_fmt) 23 | # Disable some third-party noise 24 | logging.getLogger('huey').setLevel(logging.INFO) 25 | logging.getLogger('urllib3').setLevel(logging.WARNING) 26 | 27 | 28 | def load_dotenv(path='.env'): 29 | if not int(os.getenv('BRIGHTSKY_LOAD_DOTENV', 1)): 30 | return 31 | with suppress(FileNotFoundError): 32 | with open(path) as f: 33 | for line in f: 34 | if line.strip() and not line.strip().startswith('#'): 35 | key, val = line.strip().split('=', 1) 36 | os.environ.setdefault(key, val) 37 | 38 | 39 | def download(url, directory): 40 | """ 41 | Download a resource from `url` into `directory`, returning its path and 42 | fingerprint. 43 | """ 44 | resp = requests.get(url, headers={'User-Agent': USER_AGENT}) 45 | resp.raise_for_status() 46 | filename = os.path.basename(url) 47 | path = os.path.join(directory, filename) 48 | with open(path, 'wb') as f: 49 | f.write(resp.content) 50 | fingerprint = { 51 | 'url': url, 52 | 'last_modified': dateutil.parser.parse(resp.headers['Last-Modified']), 53 | 'file_size': int(resp.headers['Content-Length']), 54 | } 55 | return path, fingerprint 56 | 57 | 58 | def parse_date(date_str): 59 | try: 60 | return dateutil.parser.isoparse(date_str) 61 | except ValueError as e: 62 | # Auto-correct common error of not encoding '+' as '%2b' in URL 63 | handled_errors = [ 64 | 'Inconsistent use of colon separator', 65 | 'Unused components in ISO string', 66 | ] 67 | if e.args and e.args[0] in handled_errors and date_str.count(' ') == 1: 68 | return parse_date(date_str.replace(' ', '+')) 69 | raise e from None 70 | 71 | 72 | @lru_cache 73 | def sunrise_sunset(lat, lon, date): 74 | return daylight(Observer(lat, lon), date) 75 | 76 | 77 | def daytime(lat, lon, timestamp): 78 | try: 79 | sunrise, sunset = sunrise_sunset(lat, lon, timestamp.date()) 80 | except ValueError as e: 81 | return 'day' if 'above' in e.args[0] else 'night' 82 | if sunset < sunrise: 83 | return 'night' if sunset <= timestamp <= sunrise else 'day' 84 | return 'day' if sunrise <= timestamp <= sunset else 'night' 85 | -------------------------------------------------------------------------------- /brightsky/web/__init__.py: -------------------------------------------------------------------------------- 1 | from .app import app, make_app 2 | 3 | 4 | __all__ = [ 5 | 'app', 6 | 'make_app', 7 | ] 8 | -------------------------------------------------------------------------------- /brightsky/web/intro.md: -------------------------------------------------------------------------------- 1 | Bright Sky is a free and open-source weather API. It aims to provide an easy-to-use gateway to weather data that the [DWD](https://www.dwd.de/) – Germany's meteorological service – publishes on their [open data server](https://opendata.dwd.de/). 2 | 3 | The public instance at `https://api.brightsky.dev/` is free-to-use for all purposes, **no API key required**! Please note that the [DWD's Terms of Use](https://www.dwd.de/EN/service/copyright/copyright_artikel.html) apply to all data you retrieve through the API. 4 | 5 | > This documentation is generated from an OpenAPI specification. The current version is available from https://api.brightsky.dev/openapi.json. 6 | 7 | 8 | ## Quickstart 9 | 10 | * Check out [`/current_weather`](operations/getCurrentWeather) if you want to know what the weather's like _right now_. 11 | * Check out [`/weather`](operations/getWeather) for hourly weather observations and forecasts. 12 | * Check out [`/radar`](operations/getRadar) if you're looking for a high-resolution rain radar. 13 | * Check out [`/alerts`](operations/getAlerts) if you're interested in weather alerts. 14 | 15 | ... or keep reading below for some background information. 16 | 17 | 18 | ## Good to Know 19 | 20 | * **Geographical coverage**: due to its nature as German meteorological service, the observations published by the DWD have a strong focus on Germany. The _forecasts_ cover the whole world, albeit at a much lower density outside of Germany. 21 | * **Historical coverage**: Bright Sky serves historical data going back to January 1st, 2010. If you need data that goes further back, check out our [infrastructure repository](https://github.com/jdemaeyer/brightsky-infrastructure) to easily set up your own instance of Bright Sky! 22 | * **Source IDs**: Bright Sky's _source IDs_ are a technical artifact and – unlike the [DWD station IDs](https://www.dwd.de/DE/leistungen/klimadatendeutschland/stationsliste.html) and [WMO station IDs](https://opendata.dwd.de/climate_environment/CDC/help/stations_list_CLIMAT_data.txt) – have no meaning in the real world. When making requests to Bright Sky, try to avoid them and supply lat/lon or station IDs instead. 23 | 24 | 25 | ## Useful Links 26 | 27 | * [Bright Sky source code and issue tracking](https://github.com/jdemaeyer/brightsky/) 28 | * [Bright Sky infrastructure configuration](https://github.com/jdemaeyer/brightsky-infrastructure/) 29 | * [DWD Open Data landing page](https://www.dwd.de/EN/ourservices/opendata/opendata.html) 30 | * [Additional explanation files for DWD Open Data](https://www.dwd.de/DE/leistungen/opendata/hilfe.html?nn=495490&lsbId=627548), including: 31 | * [List of main observation stations](https://www.dwd.de/DE/leistungen/opendata/help/stationen/ha_messnetz.xls?__blob=publicationFile&v=1) 32 | * [List of additional observation stations](https://www.dwd.de/DE/leistungen/opendata/help/stationen/na_messnetz.xlsx?__blob=publicationFile&v=10) 33 | * [List of MOSMIX stations](https://www.dwd.de/DE/leistungen/met_verfahren_mosmix/mosmix_stationskatalog.cfg?view=nasPublication&nn=495490) 34 | * [List of meteorological parameters](https://www.dwd.de/DE/leistungen/opendata/help/schluessel_datenformate/kml/mosmix_elemente_pdf.pdf?__blob=publicationFile&v=2) 35 | * [DWD Open Data FAQ (German)](https://www.dwd.de/DE/leistungen/opendata/faqs_opendata.html) 36 | * [DWD Copyright information](https://www.dwd.de/EN/service/copyright/copyright_artikel.html) 37 | 38 | 39 | ## Data Sources 40 | 41 | All data available through Bright Sky is taken or derived from data on the [DWD open data server](https://opendata.dwd.de/): 42 | 43 | * **Current weather / SYNOP**: 44 | * https://opendata.dwd.de/weather/weather_reports/synoptic/germany/json/ 45 | * **Hourly weather**: 46 | * Historical: https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/ 47 | * Current day: https://opendata.dwd.de/weather/weather_reports/poi/ 48 | * Forecasts: https://opendata.dwd.de/weather/local_forecasts/mos/ 49 | * **Radar**: 50 | * https://opendata.dwd.de/weather/radar/composite/rv/ 51 | * **Alerts**: 52 | * https://opendata.dwd.de/weather/alerts/cap/COMMUNEUNION_DWD_STAT/ 53 | -------------------------------------------------------------------------------- /brightsky/worker.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import pathlib 3 | import resource 4 | import threading 5 | import time 6 | 7 | from dwdparse.stations import StationIDConverter, load_stations 8 | from dwdparse.utils import fetch 9 | from huey import crontab, PriorityRedisHuey 10 | from huey.api import TaskLock as TaskLock_ 11 | from huey.exceptions import TaskLockedException 12 | 13 | from brightsky import tasks 14 | from brightsky.settings import settings 15 | 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | class ExpiringLocksHuey(PriorityRedisHuey): 21 | 22 | def lock_task(self, lock_name): 23 | return TaskLock(self, lock_name) 24 | 25 | def expire_locks(self, timeout): 26 | expired = set() 27 | threshold = time.time() - timeout 28 | for key in list(self._locks): 29 | value = self.get(key, peek=True) 30 | if value and float(value) < threshold: 31 | self.delete(key) 32 | expired.add(key) 33 | return expired 34 | 35 | def is_locked(self, lock_name): 36 | return TaskLock(self, lock_name).is_locked() 37 | 38 | 39 | class TaskLock(TaskLock_): 40 | 41 | def __enter__(self): 42 | if not self._huey.put_if_empty(self._key, str(time.time())): 43 | raise TaskLockedException('unable to set lock: %s' % self._name) 44 | 45 | def is_locked(self): 46 | return self._huey.storage.has_data_for_key(self._key) 47 | 48 | 49 | huey = ExpiringLocksHuey( 50 | 'brightsky', 51 | results=False, 52 | url=settings.REDIS_URL, 53 | ) 54 | 55 | 56 | @huey.periodic_task(crontab(minute='42', hour='3'), priority=40) 57 | @huey.on_startup() 58 | def update_stations(): 59 | path = pathlib.Path('.cache', 'stations.html') 60 | with update_stations._lock: 61 | if time.time() - update_stations._last_update < 60: 62 | return 63 | # On startup, skip download and load from cache if available 64 | if not update_stations._last_update and path.is_file(): 65 | load_stations(path=path) 66 | else: 67 | try: 68 | path.parent.mkdir(parents=True, exist_ok=True) 69 | with open(path, 'wb') as f: 70 | f.write(fetch(StationIDConverter.STATION_LIST_URL)) 71 | except OSError: 72 | # Probably missing permissions, load without caching 73 | load_stations() 74 | else: 75 | load_stations(path=path) 76 | update_stations._last_update = time.time() 77 | update_stations._lock = threading.Lock() # noqa: E305 78 | update_stations._last_update = 0 79 | 80 | 81 | @huey.task() 82 | def process(url): 83 | with huey.lock_task(url): 84 | tasks.parse(url) 85 | 86 | 87 | @huey.periodic_task( 88 | crontab(minute=settings.POLLING_CRONTAB_MINUTE), priority=50) 89 | def poll(): 90 | tasks.poll(enqueue=True) 91 | 92 | 93 | @huey.periodic_task(crontab(minute='23'), priority=0) 94 | def clean(): 95 | tasks.clean() 96 | 97 | 98 | @huey.periodic_task(crontab(), priority=100) 99 | def log_health(): 100 | max_mem = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024) 101 | logger.info(f"Maximum memory usage: {max_mem:,} MiB") 102 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | x-brightsky: 2 | &brightsky 3 | build: . 4 | depends_on: 5 | - postgres 6 | - redis 7 | environment: 8 | BRIGHTSKY_DATABASE_URL: postgres://postgres:pgpass@postgres 9 | BRIGHTSKY_REDIS_URL: redis://redis 10 | 11 | services: 12 | postgres: 13 | image: postgres:16-alpine 14 | shm_size: 512mb 15 | environment: 16 | POSTGRES_PASSWORD: pgpass 17 | volumes: 18 | - .data:/var/lib/postgresql/data 19 | restart: unless-stopped 20 | redis: 21 | image: redis:5-alpine 22 | restart: unless-stopped 23 | worker: 24 | <<: *brightsky 25 | command: --migrate work 26 | restart: unless-stopped 27 | web: 28 | <<: *brightsky 29 | command: serve --bind 0.0.0.0:5000 30 | restart: unless-stopped 31 | ports: 32 | - 5000:5000 33 | brightsky: 34 | <<: *brightsky 35 | scale: 0 36 | -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | brightsky.dev -------------------------------------------------------------------------------- /docs/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdemaeyer/brightsky/79b17f026dfdbc29309261d4970c76c8bef06b5d/docs/apple-touch-icon.png -------------------------------------------------------------------------------- /docs/brightsky.yml: -------------------------------------------------------------------------------- 1 | THIS SPEC NOW LIVES AT https://api.brightsky.dev/openapi.json 2 | -------------------------------------------------------------------------------- /docs/demo/alerts/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Bright Sky – Alerts demo 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 119 | 120 | 121 |
122 |
123 |
139 | 140 | 141 | 142 | 143 |

Live alerts:

144 | 191 |
192 |
193 | 256 | 257 | 258 | -------------------------------------------------------------------------------- /docs/demo/img/arrow_down.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 9 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /docs/demo/img/arrow_right.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 7 | 8 | 9 | 43 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /docs/demo/radar/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Bright Sky – Radar demo 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 78 | 79 | 80 | 81 | 241 | 242 | 243 | -------------------------------------------------------------------------------- /docs/demo/radar/js-colormaps.js: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | Add contents of data.js here: 4 | */ 5 | 6 | const data = {"turbo": {"interpolate": true, "colors": [[0.19, 0.0718, 0.2322], [0.1948, 0.0834, 0.2615], [0.1996, 0.095, 0.2902], [0.2042, 0.1065, 0.3184], [0.2086, 0.118, 0.3461], [0.2129, 0.1295, 0.3731], [0.2171, 0.1409, 0.3996], [0.2211, 0.1522, 0.4256], [0.225, 0.1635, 0.451], [0.2288, 0.1748, 0.4758], [0.2324, 0.186, 0.5], [0.2358, 0.1972, 0.5237], [0.2392, 0.2083, 0.5469], [0.2423, 0.2194, 0.5694], [0.2454, 0.2304, 0.5914], [0.2483, 0.2414, 0.6129], [0.2511, 0.2524, 0.6337], [0.2537, 0.2633, 0.6541], [0.2562, 0.2741, 0.6738], [0.2585, 0.2849, 0.693], [0.2607, 0.2957, 0.7116], [0.2628, 0.3064, 0.7297], [0.2647, 0.3171, 0.7472], [0.2665, 0.3277, 0.7641], [0.2682, 0.3382, 0.7805], [0.2697, 0.3488, 0.7963], [0.271, 0.3593, 0.8116], [0.2723, 0.3697, 0.8262], [0.2733, 0.3801, 0.8404], [0.2743, 0.3904, 0.8539], [0.2751, 0.4007, 0.8669], [0.2758, 0.411, 0.8794], [0.2763, 0.4212, 0.8912], [0.2767, 0.4313, 0.9025], [0.2769, 0.4414, 0.9133], [0.277, 0.4515, 0.9235], [0.277, 0.4615, 0.9331], [0.2768, 0.4715, 0.9421], [0.2765, 0.4814, 0.9506], [0.276, 0.4913, 0.9586], [0.2754, 0.5012, 0.9659], [0.2747, 0.5109, 0.9728], [0.2738, 0.5207, 0.979], [0.2727, 0.5304, 0.9846], [0.2711, 0.5402, 0.9893], [0.2688, 0.55, 0.993], [0.2659, 0.5598, 0.9958], [0.2625, 0.5697, 0.9977], [0.2586, 0.5796, 0.9988], [0.2542, 0.5895, 0.999], [0.2495, 0.5994, 0.9984], [0.2443, 0.6094, 0.997], [0.2387, 0.6193, 0.9948], [0.2329, 0.6292, 0.992], [0.2268, 0.6391, 0.9885], [0.2204, 0.649, 0.9844], [0.2138, 0.6589, 0.9796], [0.2071, 0.6687, 0.9742], [0.2002, 0.6784, 0.9683], [0.1933, 0.6881, 0.9619], [0.1862, 0.6978, 0.955], [0.1792, 0.7073, 0.9476], [0.1722, 0.7168, 0.9398], [0.1653, 0.7262, 0.9316], [0.1584, 0.7355, 0.923], [0.1517, 0.7447, 0.9142], [0.1452, 0.7538, 0.905], [0.1389, 0.7628, 0.8955], [0.1328, 0.7716, 0.8858], [0.127, 0.7804, 0.8759], [0.1215, 0.789, 0.8658], [0.1164, 0.7974, 0.8556], [0.1117, 0.8057, 0.8452], [0.1074, 0.8138, 0.8348], [0.1036, 0.8218, 0.8244], [0.1003, 0.8296, 0.8139], [0.0975, 0.8371, 0.8034], [0.0953, 0.8446, 0.793], [0.0938, 0.8518, 0.7826], [0.0929, 0.8588, 0.7724], [0.0927, 0.8655, 0.7623], [0.0932, 0.8721, 0.7524], [0.0945, 0.8784, 0.7426], [0.0966, 0.8845, 0.7332], [0.0996, 0.8904, 0.7239], [0.1034, 0.896, 0.715], [0.1082, 0.9014, 0.706], [0.1137, 0.9067, 0.6965], [0.1201, 0.9119, 0.6866], [0.1273, 0.917, 0.6763], [0.1353, 0.922, 0.6656], [0.1439, 0.9268, 0.6545], [0.1532, 0.9315, 0.6431], [0.1632, 0.9361, 0.6314], [0.1738, 0.9405, 0.6194], [0.1849, 0.9448, 0.6071], [0.1966, 0.949, 0.5947], [0.2088, 0.953, 0.582], [0.2214, 0.9569, 0.5691], [0.2345, 0.9606, 0.5561], [0.248, 0.9642, 0.543], [0.2618, 0.9676, 0.5298], [0.276, 0.9709, 0.5165], [0.2904, 0.974, 0.5032], [0.3051, 0.977, 0.4899], [0.3201, 0.9797, 0.4765], [0.3352, 0.9823, 0.4632], [0.3504, 0.9848, 0.45], [0.3658, 0.987, 0.4369], [0.3813, 0.9891, 0.4239], [0.3968, 0.991, 0.411], [0.4123, 0.9927, 0.3983], [0.4278, 0.9942, 0.3858], [0.4432, 0.9955, 0.3734], [0.4585, 0.9966, 0.3614], [0.4738, 0.9976, 0.3496], [0.4888, 0.9983, 0.3382], [0.5036, 0.9988, 0.327], [0.5182, 0.9991, 0.3162], [0.5326, 0.9992, 0.3058], [0.5466, 0.9991, 0.2958], [0.5603, 0.9987, 0.2862], [0.5736, 0.9982, 0.2771], [0.5865, 0.9974, 0.2685], [0.5989, 0.9964, 0.2604], [0.6109, 0.9951, 0.2528], [0.6223, 0.9937, 0.2458], [0.6332, 0.992, 0.2394], [0.6436, 0.99, 0.2336], [0.6539, 0.9878, 0.2284], [0.6643, 0.9852, 0.2237], [0.6746, 0.9825, 0.2196], [0.6849, 0.9794, 0.216], [0.6952, 0.9761, 0.2129], [0.7055, 0.9726, 0.2103], [0.7158, 0.9688, 0.2082], [0.726, 0.9647, 0.2064], [0.7361, 0.9604, 0.205], [0.7462, 0.9559, 0.2041], [0.7562, 0.9512, 0.2034], [0.7661, 0.9463, 0.2031], [0.7759, 0.9411, 0.2031], [0.7856, 0.9358, 0.2034], [0.7952, 0.9302, 0.2039], [0.8047, 0.9245, 0.2046], [0.8141, 0.9186, 0.2055], [0.8233, 0.9125, 0.2066], [0.8324, 0.9063, 0.2079], [0.8413, 0.8999, 0.2093], [0.8501, 0.8933, 0.2107], [0.8587, 0.8866, 0.2123], [0.8671, 0.8797, 0.2139], [0.8753, 0.8727, 0.2156], [0.8833, 0.8655, 0.2172], [0.8911, 0.8583, 0.2188], [0.8987, 0.8509, 0.2204], [0.906, 0.8434, 0.2219], [0.9132, 0.8358, 0.2233], [0.92, 0.8281, 0.2246], [0.9267, 0.8202, 0.2257], [0.933, 0.8124, 0.2267], [0.9391, 0.8044, 0.2274], [0.9449, 0.7963, 0.228], [0.9504, 0.7882, 0.2283], [0.9556, 0.78, 0.2284], [0.9605, 0.7718, 0.2281], [0.9651, 0.7635, 0.2275], [0.9693, 0.7552, 0.2266], [0.9732, 0.7468, 0.2254], [0.9768, 0.7384, 0.2237], [0.98, 0.73, 0.2216], [0.9829, 0.7214, 0.2192], [0.9855, 0.7125, 0.2165], [0.9878, 0.7033, 0.2136], [0.9899, 0.6938, 0.2104], [0.9916, 0.6841, 0.2071], [0.9931, 0.6741, 0.2035], [0.9944, 0.6639, 0.1997], [0.9954, 0.6534, 0.1958], [0.9961, 0.6428, 0.1916], [0.9965, 0.6319, 0.1874], [0.9968, 0.6209, 0.183], [0.9967, 0.6098, 0.1784], [0.9964, 0.5985, 0.1738], [0.9959, 0.587, 0.169], [0.9952, 0.5755, 0.1641], [0.9942, 0.5639, 0.1592], [0.993, 0.5521, 0.1542], [0.9915, 0.5404, 0.1491], [0.9899, 0.5285, 0.144], [0.988, 0.5167, 0.1388], [0.9859, 0.5048, 0.1337], [0.9836, 0.4929, 0.1285], [0.9811, 0.481, 0.1233], [0.9784, 0.4692, 0.1182], [0.9754, 0.4574, 0.113], [0.9723, 0.4456, 0.108], [0.969, 0.434, 0.1029], [0.9656, 0.4224, 0.098], [0.9619, 0.4109, 0.0931], [0.958, 0.3996, 0.0883], [0.954, 0.3884, 0.0836], [0.9498, 0.3773, 0.079], [0.9454, 0.3664, 0.0746], [0.9408, 0.3557, 0.0703], [0.9361, 0.3451, 0.0662], [0.9312, 0.3348, 0.0622], [0.9262, 0.3247, 0.0584], [0.921, 0.3149, 0.0548], [0.9157, 0.3053, 0.0513], [0.9102, 0.296, 0.0481], [0.9046, 0.287, 0.0452], [0.8989, 0.2782, 0.0424], [0.893, 0.2698, 0.0399], [0.8869, 0.2615, 0.0375], [0.8807, 0.2533, 0.0352], [0.8742, 0.2453, 0.033], [0.8676, 0.2373, 0.0308], [0.8608, 0.2294, 0.0288], [0.8538, 0.2217, 0.0268], [0.8466, 0.2141, 0.0249], [0.8393, 0.2065, 0.023], [0.8317, 0.1991, 0.0213], [0.824, 0.1918, 0.0197], [0.8161, 0.1846, 0.0181], [0.808, 0.1775, 0.0166], [0.7997, 0.1706, 0.0152], [0.7912, 0.1637, 0.0139], [0.7826, 0.1569, 0.0126], [0.7738, 0.1503, 0.0115], [0.7648, 0.1437, 0.0104], [0.7556, 0.1373, 0.0094], [0.7462, 0.131, 0.0085], [0.7366, 0.1248, 0.0077], [0.7269, 0.1187, 0.007], [0.7169, 0.1127, 0.0063], [0.7068, 0.1068, 0.0057], [0.6965, 0.101, 0.0052], [0.686, 0.0954, 0.0048], [0.6754, 0.0898, 0.0045], [0.6645, 0.0844, 0.0042], [0.6534, 0.079, 0.0041], [0.6422, 0.0738, 0.004], [0.6308, 0.0687, 0.004], [0.6192, 0.0637, 0.0041], [0.6075, 0.0588, 0.0043], [0.5955, 0.054, 0.0045], [0.5834, 0.0493, 0.0049], [0.571, 0.0447, 0.0053], [0.5585, 0.0403, 0.0058], [0.5458, 0.0359, 0.0064], [0.533, 0.0317, 0.007], [0.5199, 0.0276, 0.0078], [0.5066, 0.0235, 0.0086], [0.4932, 0.0196, 0.0096], [0.4796, 0.0158, 0.0106]]}}; 7 | 8 | const turbo = partial('turbo'); 9 | const turbo_r = partial('turbo_r'); 10 | 11 | 12 | /* 13 | Define auxiliary functions for evaluating colormaps 14 | */ 15 | 16 | function evaluate_cmap(x, name, reverse) { 17 | /** 18 | * Evaluate colormap `name` at some value `x`. 19 | * @param {number} x - The value (between 0 and 1) at which to evaluate the colormap. 20 | * @param {string} name - The name of the colormap (see matplotlib documentation). 21 | * @reverse {boolean} reverse - Whether or not to reverse the colormap. 22 | * @return {list} - A 3-tuple (R, G, B) containing the color assigned to `x`. 23 | */ 24 | 25 | // Ensure that the value of `x` is valid (i.e., 0 <= x <= 1) 26 | if (!(0 <= x <= 1)) { 27 | alert('Illegal value for x! Must be in [0, 1].') 28 | } 29 | 30 | // Ensure that `name` is a valid colormap 31 | if (!(name in data)) { 32 | alert('Colormap ' + name + 'does not exist!'); 33 | } 34 | 35 | // We can get the reverse colormap by evaluating colormap(1-x) 36 | if (reverse === true) { 37 | x = 1 - x; 38 | } 39 | 40 | // Get the colors and whether or not we need to interpolate 41 | let colors = data[name]['colors']; 42 | let interpolate = data[name]['interpolate']; 43 | 44 | if (interpolate === true) { 45 | return interpolated(x, colors); 46 | } else { 47 | return qualitative(x, colors); 48 | } 49 | } 50 | 51 | function interpolated(x, colors) { 52 | let lo = Math.floor(x * (colors.length - 1)); 53 | let hi = Math.ceil(x * (colors.length - 1)); 54 | let r = Math.round((colors[lo][0] + colors[hi][0]) / 2 * 255); 55 | let g = Math.round((colors[lo][1] + colors[hi][1]) / 2 * 255); 56 | let b = Math.round((colors[lo][2] + colors[hi][2]) / 2 * 255); 57 | return [r, g, b]; 58 | } 59 | 60 | function qualitative(x, colors) { 61 | let idx = 0; 62 | while (x > (idx + 1) / (colors.length - 0) ) { idx++; } 63 | let r = Math.round(colors[idx][0] * 255); 64 | let g = Math.round(colors[idx][1] * 255); 65 | let b = Math.round(colors[idx][2] * 255); 66 | return [r, g, b]; 67 | } 68 | 69 | function partial(name) { 70 | if (name.endsWith('_r')) { 71 | return function(x) { return evaluate_cmap(x, name.substring(0, name.length - 2), true) }; 72 | } else { 73 | return function(x) { return evaluate_cmap(x, name, false) }; 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /docs/demo/radar/pause.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 17 | 19 | 38 | 45 | 46 | -------------------------------------------------------------------------------- /docs/demo/radar/play.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 17 | 19 | 37 | 44 | 45 | -------------------------------------------------------------------------------- /docs/docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Bright Sky API Documentation 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 34 | 35 | 36 |
37 | 46 |
47 | 48 | 56 | 57 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /docs/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdemaeyer/brightsky/79b17f026dfdbc29309261d4970c76c8bef06b5d/docs/favicon-16x16.png -------------------------------------------------------------------------------- /docs/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdemaeyer/brightsky/79b17f026dfdbc29309261d4970c76c8bef06b5d/docs/favicon-32x32.png -------------------------------------------------------------------------------- /docs/favicon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 20 | 40 | 42 | 45 | 49 | 53 | 54 | 64 | 65 | 69 | 78 | 87 | 96 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /docs/img/book.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/img/coffee.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 37 | 40 | 44 | 48 | 52 | 56 | 57 | 59 | 60 | 62 | 63 | 65 | 66 | 68 | 69 | 71 | 72 | 74 | 75 | 77 | 78 | 80 | 81 | 83 | 84 | 86 | 87 | 89 | 90 | 92 | 93 | 95 | 96 | 98 | 99 | 101 | 102 | 103 | -------------------------------------------------------------------------------- /docs/img/eye.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/img/github.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /docs/img/heart.svg: -------------------------------------------------------------------------------- 1 | 2 | 20 | 22 | 41 | 44 | 45 | -------------------------------------------------------------------------------- /docs/img/pf.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 9 | 10 | 11 | 12 | 14 | 16 | 19 | 21 | 24 | 26 | 28 | 31 | 34 | 35 | 36 | 37 | 39 | 41 | 44 | 45 | 46 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /migrations/0001_migrations.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE migrations ( 2 | id integer PRIMARY KEY, 3 | name varchar(255), 4 | applied timestamptz DEFAULT current_timestamp 5 | ); 6 | -------------------------------------------------------------------------------- /migrations/0002_parsed_files.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE parsed_files ( 2 | url text PRIMARY KEY, 3 | last_modified timestamptz NOT NULL, 4 | file_size integer NOT NULL, 5 | parsed_at timestamptz NOT NULL 6 | ); 7 | -------------------------------------------------------------------------------- /migrations/0003_weather.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTENSION IF NOT EXISTS cube WITH SCHEMA public; 2 | CREATE EXTENSION IF NOT EXISTS earthdistance WITH SCHEMA public; 3 | ALTER FUNCTION public.ll_to_earth SET search_path = public; 4 | 5 | CREATE TYPE observation_type AS ENUM ('historical', 'recent', 'current', 'forecast'); 6 | 7 | CREATE TABLE sources ( 8 | id serial PRIMARY KEY, 9 | station_id varchar(5) NOT NULL, 10 | observation_type observation_type NOT NULL, 11 | lat real NOT NULL, 12 | lon real NOT NULL, 13 | height real NOT NULL, 14 | 15 | CONSTRAINT weather_source_key UNIQUE (station_id, observation_type, lat, lon, height) 16 | ); 17 | 18 | CREATE INDEX weather_source_location ON sources USING gist(ll_to_earth(lat, lon)); 19 | 20 | CREATE TABLE weather ( 21 | timestamp timestamptz NOT NULL, 22 | source_id int NOT NULL REFERENCES sources(id) ON DELETE CASCADE, 23 | precipitation real CHECK (precipitation >= 0), 24 | pressure_msl integer CHECK (pressure_msl > 0), 25 | sunshine smallint CHECK (sunshine BETWEEN 0 and 3600), 26 | temperature real CHECK (temperature > 0), 27 | wind_direction smallint CHECK (wind_direction BETWEEN 0 AND 360), 28 | wind_speed real CHECK (wind_speed >= 0), 29 | 30 | CONSTRAINT weather_key UNIQUE (timestamp, source_id) 31 | ); 32 | -------------------------------------------------------------------------------- /migrations/0004_station_name.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE sources ADD COLUMN station_name varchar(255) NOT NULL DEFAULT ''; 2 | ALTER TABLE sources ALTER COLUMN station_name DROP DEFAULT; 3 | 4 | -- Force full re-parse 5 | DELETE FROM parsed_files; 6 | -------------------------------------------------------------------------------- /migrations/0005_additional_weather_params.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE weather 2 | ADD COLUMN cloud_cover smallint CHECK (cloud_cover BETWEEN 0 and 100), 3 | ADD COLUMN dew_point real CHECK (dew_point > 0), 4 | ADD COLUMN relative_humidity smallint CHECK (relative_humidity BETWEEN 0 and 100), 5 | ADD COLUMN visibility int CHECK (visibility >= 0), 6 | ADD COLUMN wind_gust_direction smallint CHECK (wind_gust_direction BETWEEN 0 AND 360), 7 | ADD COLUMN wind_gust_speed real CHECK (wind_gust_speed >= 0); 8 | 9 | -- relative_humidity is parsed from air temperature files 10 | DELETE FROM parsed_files WHERE url ILIKE '%stundenwerte_TU_%' 11 | -------------------------------------------------------------------------------- /migrations/0006_wmo_station_ids.sql: -------------------------------------------------------------------------------- 1 | -- There are a bunch of stations only differing in their DWD station ID but not 2 | -- their observation_type or location. We'll have to decide how to deal with 3 | -- these during parsing. 4 | DELETE 5 | FROM sources s 6 | USING sources s2 7 | WHERE 8 | s.observation_type = s2.observation_type AND 9 | s.lat = s2.lat AND 10 | s.lon = s2.lon AND 11 | s.height = s2.height AND 12 | s.id > s2.id; 13 | 14 | ALTER TABLE sources 15 | RENAME COLUMN station_id TO dwd_station_id; 16 | ALTER TABLE sources 17 | ALTER COLUMN dwd_station_id DROP NOT NULL, 18 | ADD COLUMN wmo_station_id varchar(5), 19 | DROP CONSTRAINT weather_source_key, 20 | ADD CONSTRAINT weather_source_key UNIQUE (observation_type, lat, lon, height); 21 | -------------------------------------------------------------------------------- /migrations/0007_synop.sql: -------------------------------------------------------------------------------- 1 | ALTER TYPE observation_type ADD VALUE 'synop' BEFORE 'forecast'; 2 | 3 | CREATE TABLE synop ( 4 | timestamp timestamptz NOT NULL, 5 | source_id int NOT NULL REFERENCES sources(id) ON DELETE CASCADE, 6 | 7 | cloud_cover smallint CHECK (cloud_cover BETWEEN 0 and 100), 8 | dew_point real CHECK (dew_point > 0), 9 | precipitation_10 real CHECK (precipitation_10 >= 0), 10 | precipitation_30 real CHECK (precipitation_30 >= 0), 11 | precipitation_60 real CHECK (precipitation_60 >= 0), 12 | pressure_msl integer CHECK (pressure_msl > 0), 13 | relative_humidity smallint CHECK (relative_humidity BETWEEN 0 and 100), 14 | sunshine_10 smallint CHECK (sunshine_10 BETWEEN 0 and 600), 15 | sunshine_30 smallint CHECK (sunshine_30 BETWEEN 0 and 1800), 16 | sunshine_60 smallint CHECK (sunshine_60 BETWEEN 0 and 3600), 17 | temperature real CHECK (temperature > 0), 18 | visibility int CHECK (visibility >= 0), 19 | wind_direction_10 smallint CHECK (wind_direction_10 BETWEEN 0 AND 360), 20 | wind_direction_30 smallint CHECK (wind_direction_30 BETWEEN 0 AND 360), 21 | wind_direction_60 smallint CHECK (wind_direction_60 BETWEEN 0 AND 360), 22 | wind_speed_10 real CHECK (wind_speed_10 >= 0), 23 | wind_speed_30 real CHECK (wind_speed_30 >= 0), 24 | wind_speed_60 real CHECK (wind_speed_60 >= 0), 25 | wind_gust_direction_10 smallint CHECK (wind_gust_direction_10 BETWEEN 0 AND 360), 26 | wind_gust_direction_30 smallint CHECK (wind_gust_direction_30 BETWEEN 0 AND 360), 27 | wind_gust_direction_60 smallint CHECK (wind_gust_direction_60 BETWEEN 0 AND 360), 28 | wind_gust_speed_10 real CHECK (wind_gust_speed_10 >= 0), 29 | wind_gust_speed_30 real CHECK (wind_gust_speed_30 >= 0), 30 | wind_gust_speed_60 real CHECK (wind_gust_speed_60 >= 0), 31 | 32 | CONSTRAINT synop_key UNIQUE (timestamp, source_id) 33 | ); 34 | -------------------------------------------------------------------------------- /migrations/0008_current_weather.sql: -------------------------------------------------------------------------------- 1 | CREATE FUNCTION last_agg ( anyelement, anyelement ) 2 | RETURNS anyelement LANGUAGE SQL IMMUTABLE STRICT AS $$ 3 | SELECT $2; 4 | $$; 5 | 6 | CREATE AGGREGATE LAST ( anyelement ) ( 7 | sfunc = last_agg, 8 | stype = anyelement 9 | ); 10 | 11 | CREATE MATERIALIZED VIEW current_weather AS 12 | WITH last_timestamp AS ( 13 | SELECT 14 | source_id, 15 | MAX(timestamp) AS last_timestamp 16 | FROM synop 17 | GROUP BY source_id 18 | ) 19 | SELECT 20 | last_timestamp.source_id, 21 | last_timestamp.last_timestamp AS timestamp, 22 | latest.cloud_cover, 23 | latest.dew_point, 24 | latest.precipitation_10, 25 | last_half_hour.precipitation_30, 26 | last_hour.precipitation_60, 27 | latest.pressure_msl, 28 | latest.relative_humidity, 29 | latest.visibility, 30 | latest.wind_direction_10, 31 | last_half_hour.wind_direction_30, 32 | last_hour.wind_direction_60, 33 | latest.wind_speed_10, 34 | last_half_hour.wind_speed_30, 35 | last_hour.wind_speed_60, 36 | latest.wind_gust_direction_10, 37 | last_half_hour.wind_gust_direction_30, 38 | last_hour.wind_gust_direction_60, 39 | latest.wind_gust_speed_10, 40 | last_half_hour.wind_gust_speed_30, 41 | last_hour.wind_gust_speed_60, 42 | sunshine.sunshine_30, 43 | sunshine.sunshine_60, 44 | latest.temperature 45 | FROM last_timestamp 46 | JOIN ( 47 | SELECT 48 | source_id, 49 | LAST(cloud_cover ORDER BY timestamp) AS cloud_cover, 50 | LAST(dew_point ORDER BY timestamp) AS dew_point, 51 | LAST(precipitation_10 ORDER BY timestamp) AS precipitation_10, 52 | LAST(pressure_msl ORDER BY timestamp) AS pressure_msl, 53 | LAST(relative_humidity ORDER BY timestamp) AS relative_humidity, 54 | LAST(visibility ORDER BY timestamp) AS visibility, 55 | LAST(wind_direction_10 ORDER BY timestamp) AS wind_direction_10, 56 | LAST(wind_speed_10 ORDER BY timestamp) AS wind_speed_10, 57 | LAST(wind_gust_direction_10 ORDER BY timestamp) AS wind_gust_direction_10, 58 | LAST(wind_gust_speed_10 ORDER BY timestamp) AS wind_gust_speed_10, 59 | LAST(temperature ORDER BY timestamp) AS temperature 60 | FROM synop s 61 | WHERE timestamp >= now() - '90 minutes'::interval 62 | GROUP BY source_id 63 | ) latest ON last_timestamp.source_id = latest.source_id 64 | LEFT JOIN ( 65 | SELECT 66 | synop.source_id, 67 | round(AVG(precipitation_10) * 6 * 100) / 100 AS precipitation_60, 68 | round(AVG(wind_speed_10) * 10) / 10 AS wind_speed_60, 69 | (round(atan2d(AVG(sind(wind_direction_10)), AVG(cosd(wind_direction_10))))::int + 360) % 360 AS wind_direction_60, 70 | MAX(wind_gust_speed_10) AS wind_gust_speed_60, 71 | LAST(wind_gust_direction_10 ORDER BY wind_gust_speed_10) AS wind_gust_direction_60 72 | FROM synop 73 | JOIN last_timestamp ON synop.source_id = last_timestamp.source_id 74 | WHERE timestamp > last_timestamp - '60 minutes'::interval 75 | GROUP BY synop.source_id 76 | ) last_hour ON latest.source_id = last_hour.source_id 77 | LEFT JOIN ( 78 | SELECT 79 | synop.source_id, 80 | round(AVG(precipitation_10) * 3 * 100) / 100 AS precipitation_30, 81 | round(AVG(wind_speed_10) * 10) / 10 AS wind_speed_30, 82 | (round(atan2d(AVG(sind(wind_direction_10)), AVG(cosd(wind_direction_10))))::int + 360) % 360 AS wind_direction_30, 83 | MAX(wind_gust_speed_10) AS wind_gust_speed_30, 84 | LAST(wind_gust_direction_10 ORDER BY wind_gust_speed_10) AS wind_gust_direction_30 85 | FROM synop 86 | JOIN last_timestamp ON synop.source_id = last_timestamp.source_id 87 | WHERE timestamp > last_timestamp - '30 minutes'::interval 88 | GROUP BY synop.source_id 89 | ) last_half_hour ON latest.source_id = last_half_hour.source_id 90 | LEFT JOIN ( 91 | SELECT 92 | s30_latest.source_id, 93 | CASE 94 | WHEN s30_latest.timestamp > s60.timestamp THEN s30_latest.sunshine_30 95 | ELSE s60.sunshine_60 - s30_latest.sunshine_30 96 | END AS sunshine_30, 97 | CASE 98 | WHEN s30_latest.timestamp > s60.timestamp THEN s30_latest.sunshine_30 + s60.sunshine_60 - s30_previous.sunshine_30 99 | ELSE s60.sunshine_60 100 | END AS sunshine_60 101 | FROM ( 102 | SELECT DISTINCT ON (source_id) source_id, timestamp, sunshine_30 103 | FROM synop 104 | WHERE sunshine_30 IS NOT NULL 105 | ORDER BY source_id, timestamp DESC 106 | ) s30_latest 107 | JOIN ( 108 | SELECT source_id, timestamp, sunshine_30 109 | FROM synop 110 | ) s30_previous ON 111 | s30_latest.source_id = s30_previous.source_id AND 112 | s30_previous.timestamp = s30_latest.timestamp - '1 hour'::interval 113 | JOIN ( 114 | SELECT DISTINCT ON (source_id) source_id, timestamp, sunshine_60 115 | FROM synop 116 | WHERE sunshine_60 IS NOT NULL 117 | ORDER BY source_id, timestamp DESC 118 | ) s60 ON 119 | s30_previous.source_id = s60.source_id AND 120 | s60.timestamp > s30_previous.timestamp 121 | ) sunshine ON latest.source_id = sunshine.source_id 122 | ORDER BY latest.source_id; 123 | 124 | CREATE UNIQUE INDEX current_weather_key ON current_weather (source_id); 125 | -------------------------------------------------------------------------------- /migrations/0009_sources_date_range.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE sources 2 | ADD COLUMN first_record timestamptz, 3 | ADD COLUMN last_record timestamptz; 4 | 5 | UPDATE sources SET 6 | first_record = record_range.first_record, 7 | last_record = record_range.last_record 8 | FROM ( 9 | SELECT 10 | source_id, 11 | MIN(timestamp) AS first_record, 12 | MAX(timestamp) AS last_record 13 | FROM weather 14 | GROUP BY source_id 15 | UNION 16 | SELECT 17 | source_id, 18 | MIN(timestamp) AS first_record, 19 | MAX(timestamp) AS last_record 20 | FROM synop 21 | GROUP BY source_id 22 | ) AS record_range 23 | WHERE sources.id = record_range.source_id; 24 | -------------------------------------------------------------------------------- /migrations/0010_weather_index_performance.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE weather 2 | DROP CONSTRAINT weather_key, 3 | ADD CONSTRAINT weather_key UNIQUE (source_id, timestamp); 4 | -------------------------------------------------------------------------------- /migrations/0011_weather_condition.sql: -------------------------------------------------------------------------------- 1 | CREATE TYPE weather_condition AS ENUM ( 2 | 'dry', 3 | 'fog', 4 | 'rain', 5 | 'sleet', 6 | 'snow', 7 | 'hail', 8 | 'thunderstorm' 9 | ); 10 | 11 | ALTER TABLE weather ADD COLUMN condition weather_condition; 12 | ALTER TABLE synop ADD COLUMN condition weather_condition; 13 | 14 | -- Re-parse all precipitation records 15 | DELETE FROM parsed_files WHERE url ILIKE '%stundenwerte_RR_%'; 16 | 17 | -- Same query as before with condition field added 18 | DROP MATERIALIZED VIEW current_weather; 19 | CREATE MATERIALIZED VIEW current_weather AS 20 | WITH last_timestamp AS ( 21 | SELECT 22 | source_id, 23 | MAX(timestamp) AS last_timestamp 24 | FROM synop 25 | GROUP BY source_id 26 | ) 27 | SELECT 28 | last_timestamp.source_id, 29 | last_timestamp.last_timestamp AS timestamp, 30 | latest.cloud_cover, 31 | latest.condition, 32 | latest.dew_point, 33 | latest.precipitation_10, 34 | last_half_hour.precipitation_30, 35 | last_hour.precipitation_60, 36 | latest.pressure_msl, 37 | latest.relative_humidity, 38 | latest.visibility, 39 | latest.wind_direction_10, 40 | last_half_hour.wind_direction_30, 41 | last_hour.wind_direction_60, 42 | latest.wind_speed_10, 43 | last_half_hour.wind_speed_30, 44 | last_hour.wind_speed_60, 45 | latest.wind_gust_direction_10, 46 | last_half_hour.wind_gust_direction_30, 47 | last_hour.wind_gust_direction_60, 48 | latest.wind_gust_speed_10, 49 | last_half_hour.wind_gust_speed_30, 50 | last_hour.wind_gust_speed_60, 51 | sunshine.sunshine_30, 52 | sunshine.sunshine_60, 53 | latest.temperature 54 | FROM last_timestamp 55 | JOIN ( 56 | SELECT 57 | source_id, 58 | LAST(cloud_cover ORDER BY timestamp) AS cloud_cover, 59 | LAST(condition ORDER BY timestamp) AS condition, 60 | LAST(dew_point ORDER BY timestamp) AS dew_point, 61 | LAST(precipitation_10 ORDER BY timestamp) AS precipitation_10, 62 | LAST(pressure_msl ORDER BY timestamp) AS pressure_msl, 63 | LAST(relative_humidity ORDER BY timestamp) AS relative_humidity, 64 | LAST(visibility ORDER BY timestamp) AS visibility, 65 | LAST(wind_direction_10 ORDER BY timestamp) AS wind_direction_10, 66 | LAST(wind_speed_10 ORDER BY timestamp) AS wind_speed_10, 67 | LAST(wind_gust_direction_10 ORDER BY timestamp) AS wind_gust_direction_10, 68 | LAST(wind_gust_speed_10 ORDER BY timestamp) AS wind_gust_speed_10, 69 | LAST(temperature ORDER BY timestamp) AS temperature 70 | FROM synop s 71 | WHERE timestamp >= now() - '90 minutes'::interval 72 | GROUP BY source_id 73 | ) latest ON last_timestamp.source_id = latest.source_id 74 | LEFT JOIN ( 75 | SELECT 76 | synop.source_id, 77 | round(AVG(precipitation_10) * 6 * 100) / 100 AS precipitation_60, 78 | round(AVG(wind_speed_10) * 10) / 10 AS wind_speed_60, 79 | (round(atan2d(AVG(sind(wind_direction_10)), AVG(cosd(wind_direction_10))))::int + 360) % 360 AS wind_direction_60, 80 | MAX(wind_gust_speed_10) AS wind_gust_speed_60, 81 | LAST(wind_gust_direction_10 ORDER BY wind_gust_speed_10) AS wind_gust_direction_60 82 | FROM synop 83 | JOIN last_timestamp ON synop.source_id = last_timestamp.source_id 84 | WHERE timestamp > last_timestamp - '60 minutes'::interval 85 | GROUP BY synop.source_id 86 | ) last_hour ON latest.source_id = last_hour.source_id 87 | LEFT JOIN ( 88 | SELECT 89 | synop.source_id, 90 | round(AVG(precipitation_10) * 3 * 100) / 100 AS precipitation_30, 91 | round(AVG(wind_speed_10) * 10) / 10 AS wind_speed_30, 92 | (round(atan2d(AVG(sind(wind_direction_10)), AVG(cosd(wind_direction_10))))::int + 360) % 360 AS wind_direction_30, 93 | MAX(wind_gust_speed_10) AS wind_gust_speed_30, 94 | LAST(wind_gust_direction_10 ORDER BY wind_gust_speed_10) AS wind_gust_direction_30 95 | FROM synop 96 | JOIN last_timestamp ON synop.source_id = last_timestamp.source_id 97 | WHERE timestamp > last_timestamp - '30 minutes'::interval 98 | GROUP BY synop.source_id 99 | ) last_half_hour ON latest.source_id = last_half_hour.source_id 100 | LEFT JOIN ( 101 | SELECT 102 | s30_latest.source_id, 103 | CASE 104 | WHEN s30_latest.timestamp > s60.timestamp THEN s30_latest.sunshine_30 105 | ELSE s60.sunshine_60 - s30_latest.sunshine_30 106 | END AS sunshine_30, 107 | CASE 108 | WHEN s30_latest.timestamp > s60.timestamp THEN s30_latest.sunshine_30 + s60.sunshine_60 - s30_previous.sunshine_30 109 | ELSE s60.sunshine_60 110 | END AS sunshine_60 111 | FROM ( 112 | SELECT DISTINCT ON (source_id) source_id, timestamp, sunshine_30 113 | FROM synop 114 | WHERE sunshine_30 IS NOT NULL 115 | ORDER BY source_id, timestamp DESC 116 | ) s30_latest 117 | JOIN ( 118 | SELECT source_id, timestamp, sunshine_30 119 | FROM synop 120 | ) s30_previous ON 121 | s30_latest.source_id = s30_previous.source_id AND 122 | s30_previous.timestamp = s30_latest.timestamp - '1 hour'::interval 123 | JOIN ( 124 | SELECT DISTINCT ON (source_id) source_id, timestamp, sunshine_60 125 | FROM synop 126 | WHERE sunshine_60 IS NOT NULL 127 | ORDER BY source_id, timestamp DESC 128 | ) s60 ON 129 | s30_previous.source_id = s60.source_id AND 130 | s60.timestamp > s30_previous.timestamp 131 | ) sunshine ON latest.source_id = sunshine.source_id 132 | ORDER BY latest.source_id; 133 | 134 | CREATE UNIQUE INDEX current_weather_key ON current_weather (source_id); 135 | -------------------------------------------------------------------------------- /migrations/0012_remove_legacy_recent_records.sql: -------------------------------------------------------------------------------- 1 | -- This will delete all recent weather records by cascade 2 | DELETE FROM sources WHERE observation_type = 'recent'; 3 | -------------------------------------------------------------------------------- /migrations/0013_precipitation_probability.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE weather 2 | ADD COLUMN precipitation_probability smallint CHECK (precipitation_probability BETWEEN 0 and 100), 3 | ADD COLUMN precipitation_probability_6h smallint CHECK (precipitation_probability_6h BETWEEN 0 and 100); 4 | -------------------------------------------------------------------------------- /migrations/0014_solar.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE weather 2 | ADD COLUMN solar integer CHECK (solar >= 0); 3 | 4 | ALTER TABLE synop 5 | ADD COLUMN solar_10 integer CHECK (solar_10 >= 0), 6 | ADD COLUMN solar_30 integer CHECK (solar_10 >= 0), 7 | ADD COLUMN solar_60 integer CHECK (solar_10 >= 0); 8 | 9 | -- Same query as before with solar_XX fields added 10 | DROP MATERIALIZED VIEW current_weather; 11 | CREATE MATERIALIZED VIEW current_weather AS 12 | WITH last_timestamp AS ( 13 | SELECT 14 | source_id, 15 | MAX(timestamp) AS last_timestamp 16 | FROM synop 17 | GROUP BY source_id 18 | ) 19 | SELECT 20 | last_timestamp.source_id, 21 | last_timestamp.last_timestamp AS timestamp, 22 | latest.cloud_cover, 23 | latest.condition, 24 | latest.dew_point, 25 | latest.solar_10, 26 | last_half_hour.solar_30, 27 | last_hour.solar_60, 28 | latest.precipitation_10, 29 | last_half_hour.precipitation_30, 30 | last_hour.precipitation_60, 31 | latest.pressure_msl, 32 | latest.relative_humidity, 33 | latest.visibility, 34 | latest.wind_direction_10, 35 | last_half_hour.wind_direction_30, 36 | last_hour.wind_direction_60, 37 | latest.wind_speed_10, 38 | last_half_hour.wind_speed_30, 39 | last_hour.wind_speed_60, 40 | latest.wind_gust_direction_10, 41 | last_half_hour.wind_gust_direction_30, 42 | last_hour.wind_gust_direction_60, 43 | latest.wind_gust_speed_10, 44 | last_half_hour.wind_gust_speed_30, 45 | last_hour.wind_gust_speed_60, 46 | sunshine.sunshine_30, 47 | sunshine.sunshine_60, 48 | latest.temperature 49 | FROM last_timestamp 50 | JOIN ( 51 | SELECT 52 | source_id, 53 | LAST(cloud_cover ORDER BY timestamp) AS cloud_cover, 54 | LAST(condition ORDER BY timestamp) AS condition, 55 | LAST(dew_point ORDER BY timestamp) AS dew_point, 56 | LAST(solar_10 ORDER BY timestamp) AS solar_10, 57 | LAST(precipitation_10 ORDER BY timestamp) AS precipitation_10, 58 | LAST(pressure_msl ORDER BY timestamp) AS pressure_msl, 59 | LAST(relative_humidity ORDER BY timestamp) AS relative_humidity, 60 | LAST(visibility ORDER BY timestamp) AS visibility, 61 | LAST(wind_direction_10 ORDER BY timestamp) AS wind_direction_10, 62 | LAST(wind_speed_10 ORDER BY timestamp) AS wind_speed_10, 63 | LAST(wind_gust_direction_10 ORDER BY timestamp) AS wind_gust_direction_10, 64 | LAST(wind_gust_speed_10 ORDER BY timestamp) AS wind_gust_speed_10, 65 | LAST(temperature ORDER BY timestamp) AS temperature 66 | FROM synop s 67 | WHERE timestamp >= now() - '90 minutes'::interval 68 | GROUP BY source_id 69 | ) latest ON last_timestamp.source_id = latest.source_id 70 | LEFT JOIN ( 71 | SELECT 72 | synop.source_id, 73 | round(AVG(solar_10) * 6)::int AS solar_60, 74 | round(AVG(precipitation_10) * 6 * 100) / 100 AS precipitation_60, 75 | round(AVG(wind_speed_10) * 10) / 10 AS wind_speed_60, 76 | (round(atan2d(AVG(sind(wind_direction_10)), AVG(cosd(wind_direction_10))))::int + 360) % 360 AS wind_direction_60, 77 | MAX(wind_gust_speed_10) AS wind_gust_speed_60, 78 | LAST(wind_gust_direction_10 ORDER BY wind_gust_speed_10) AS wind_gust_direction_60 79 | FROM synop 80 | JOIN last_timestamp ON synop.source_id = last_timestamp.source_id 81 | WHERE timestamp > last_timestamp - '60 minutes'::interval 82 | GROUP BY synop.source_id 83 | ) last_hour ON latest.source_id = last_hour.source_id 84 | LEFT JOIN ( 85 | SELECT 86 | synop.source_id, 87 | round(AVG(solar_10) * 3)::int AS solar_30, 88 | round(AVG(precipitation_10) * 3 * 100) / 100 AS precipitation_30, 89 | round(AVG(wind_speed_10) * 10) / 10 AS wind_speed_30, 90 | (round(atan2d(AVG(sind(wind_direction_10)), AVG(cosd(wind_direction_10))))::int + 360) % 360 AS wind_direction_30, 91 | MAX(wind_gust_speed_10) AS wind_gust_speed_30, 92 | LAST(wind_gust_direction_10 ORDER BY wind_gust_speed_10) AS wind_gust_direction_30 93 | FROM synop 94 | JOIN last_timestamp ON synop.source_id = last_timestamp.source_id 95 | WHERE timestamp > last_timestamp - '30 minutes'::interval 96 | GROUP BY synop.source_id 97 | ) last_half_hour ON latest.source_id = last_half_hour.source_id 98 | LEFT JOIN ( 99 | SELECT 100 | s30_latest.source_id, 101 | CASE 102 | WHEN s30_latest.timestamp > s60.timestamp THEN s30_latest.sunshine_30 103 | ELSE s60.sunshine_60 - s30_latest.sunshine_30 104 | END AS sunshine_30, 105 | CASE 106 | WHEN s30_latest.timestamp > s60.timestamp THEN s30_latest.sunshine_30 + s60.sunshine_60 - s30_previous.sunshine_30 107 | ELSE s60.sunshine_60 108 | END AS sunshine_60 109 | FROM ( 110 | SELECT DISTINCT ON (source_id) source_id, timestamp, sunshine_30 111 | FROM synop 112 | WHERE sunshine_30 IS NOT NULL 113 | ORDER BY source_id, timestamp DESC 114 | ) s30_latest 115 | JOIN ( 116 | SELECT source_id, timestamp, sunshine_30 117 | FROM synop 118 | ) s30_previous ON 119 | s30_latest.source_id = s30_previous.source_id AND 120 | s30_previous.timestamp = s30_latest.timestamp - '1 hour'::interval 121 | JOIN ( 122 | SELECT DISTINCT ON (source_id) source_id, timestamp, sunshine_60 123 | FROM synop 124 | WHERE sunshine_60 IS NOT NULL 125 | ORDER BY source_id, timestamp DESC 126 | ) s60 ON 127 | s30_previous.source_id = s60.source_id AND 128 | s60.timestamp > s30_previous.timestamp 129 | ) sunshine ON latest.source_id = sunshine.source_id 130 | ORDER BY latest.source_id; 131 | 132 | CREATE UNIQUE INDEX current_weather_key ON current_weather (source_id); 133 | -------------------------------------------------------------------------------- /migrations/0015_radar.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE radar ( 2 | timestamp timestamptz NOT NULL, 3 | 4 | source varchar(255) NOT NULL, 5 | precipitation_5 bytea NOT NULL, 6 | 7 | CONSTRAINT radar_key UNIQUE (timestamp) 8 | ); 9 | -------------------------------------------------------------------------------- /migrations/0016_alerts.sql: -------------------------------------------------------------------------------- 1 | CREATE TYPE alert_category AS ENUM ('met', 'health'); 2 | CREATE TYPE alert_response_type AS ENUM ('prepare', 'allclear', 'none', 'monitor'); 3 | CREATE TYPE alert_urgency AS ENUM ('immediate', 'future'); 4 | CREATE TYPE alert_severity AS ENUM ('minor', 'moderate', 'severe', 'extreme'); 5 | CREATE TYPE alert_certainty AS ENUM ('observed', 'likely'); 6 | 7 | CREATE TABLE alerts ( 8 | id serial PRIMARY KEY, 9 | alert_id varchar(255) NOT NULL, 10 | effective timestamptz NOT NULL, 11 | onset timestamptz NOT NULL, 12 | expires timestamptz, 13 | category alert_category, 14 | response_type alert_response_type, 15 | urgency alert_urgency, 16 | severity alert_severity, 17 | certainty alert_certainty, 18 | event_code smallint CHECK (event_code BETWEEN 11 AND 248), 19 | event_en text, 20 | event_de text, 21 | headline_en text NOT NULL, 22 | headline_de text NOT NULL, 23 | description_en text NOT NULL, 24 | description_de text NOT NULL, 25 | instruction_en text, 26 | instruction_de text, 27 | 28 | CONSTRAINT alerts_key UNIQUE (alert_id) 29 | ); 30 | 31 | CREATE TABLE alert_cells ( 32 | alert_id int NOT NULL REFERENCES alerts(id) ON DELETE CASCADE, 33 | warn_cell_id int NOT NULL, 34 | 35 | CONSTRAINT alert_cells_key UNIQUE (warn_cell_id, alert_id) 36 | ); 37 | -------------------------------------------------------------------------------- /migrations/0017_fix_ll_to_earth.sql: -------------------------------------------------------------------------------- 1 | -- via https://github.com/diogob/activerecord-postgres-earthdistance/issues/30#issuecomment-1657757447 2 | -- Fixed issue seems to have had no impact on performance but produced spurious 3 | -- 'type "earth" does not exist' error messages 4 | ALTER FUNCTION ll_to_earth SET search_path = public; 5 | -------------------------------------------------------------------------------- /migrations/0018_alerts_status.sql: -------------------------------------------------------------------------------- 1 | CREATE TYPE alert_status AS ENUM ('actual', 'test'); 2 | 3 | ALTER TABLE alerts ADD COLUMN status alert_status NOT NULL DEFAULT 'actual'; 4 | ALTER TABLE alerts ALTER COLUMN status DROP DEFAULT; 5 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "brightsky" 7 | dynamic = ["version"] 8 | requires-python = ">= 3.8" 9 | dependencies = [ 10 | "astral", 11 | "asyncpg", 12 | "click", 13 | "coloredlogs", 14 | "dwdparse[lean]", 15 | "falcon>=3", 16 | "fastapi", 17 | "gunicorn", 18 | "httpx", 19 | "huey", 20 | "isal", 21 | "numpy", 22 | "orjson", 23 | "parsel", 24 | "psycopg2-binary", 25 | "pyproj", 26 | "python-dateutil", 27 | "redis", 28 | "requests", 29 | "sentry-sdk", 30 | "shapely", 31 | "uvicorn", 32 | ] 33 | authors = [ 34 | {name = "Jakob de Maeyer", email = "jakob@brightsky.dev"}, 35 | ] 36 | description = "JSON API for DWD's open weather data." 37 | readme = "README.md" 38 | license = {text = "MIT"} 39 | classifiers = [ 40 | "Programming Language :: Python :: 3", 41 | "License :: OSI Approved :: MIT License", 42 | "Operating System :: OS Independent", 43 | ] 44 | 45 | [project.urls] 46 | Homepage = "https://brightsky.dev/" 47 | Documentation = "https://brightsky.dev/docs/" 48 | Source = "https://github.com/jdemaeyer/brightsky/" 49 | Tracker = "https://github.com/jdemaeyer/brightsky/issues/" 50 | 51 | [tool.setuptools.dynamic] 52 | version = {attr = "brightsky.__version__"} 53 | 54 | [tool.setuptools.packages.find] 55 | include = ["brightsky*"] 56 | 57 | [tool.pytest.ini_options] 58 | required_plugins = "pytest-env" 59 | 60 | [tool.pytest_env] 61 | BRIGHTSKY_LOAD_DOTENV = 0 62 | 63 | [tool.ruff] 64 | line-length = 79 65 | 66 | [tool.ruff.lint] 67 | select = ["E", "F"] 68 | 69 | [tool.ruff.lint.isort] 70 | lines-after-imports = 2 71 | -------------------------------------------------------------------------------- /requirements-dev.in: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | freezegun 3 | pytest 4 | pytest-env 5 | ruff 6 | uv 7 | watchfiles 8 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | # This file was autogenerated by uv via the following command: 2 | # uv pip compile -o requirements-dev.txt requirements-dev.in 3 | annotated-types==0.7.0 4 | # via 5 | # -r requirements.txt 6 | # pydantic 7 | anyio==4.9.0 8 | # via 9 | # -r requirements.txt 10 | # httpx 11 | # starlette 12 | # watchfiles 13 | astral==3.2 14 | # via -r requirements.txt 15 | asyncpg==0.30.0 16 | # via -r requirements.txt 17 | certifi==2025.4.26 18 | # via 19 | # -r requirements.txt 20 | # httpcore 21 | # httpx 22 | # pyproj 23 | # requests 24 | # sentry-sdk 25 | charset-normalizer==3.4.1 26 | # via 27 | # -r requirements.txt 28 | # requests 29 | click==8.1.8 30 | # via 31 | # -r requirements.txt 32 | # uvicorn 33 | coloredlogs==15.0.1 34 | # via -r requirements.txt 35 | cssselect==1.3.0 36 | # via 37 | # -r requirements.txt 38 | # parsel 39 | dwdparse==0.9.15 40 | # via -r requirements.txt 41 | falcon==4.0.2 42 | # via -r requirements.txt 43 | fastapi==0.115.12 44 | # via -r requirements.txt 45 | freezegun==1.5.1 46 | # via -r requirements-dev.in 47 | gunicorn==23.0.0 48 | # via -r requirements.txt 49 | h11==0.16.0 50 | # via 51 | # -r requirements.txt 52 | # httpcore 53 | # uvicorn 54 | httpcore==1.0.9 55 | # via 56 | # -r requirements.txt 57 | # httpx 58 | httpx==0.28.1 59 | # via -r requirements.txt 60 | huey==2.5.3 61 | # via -r requirements.txt 62 | humanfriendly==10.0 63 | # via 64 | # -r requirements.txt 65 | # coloredlogs 66 | idna==3.10 67 | # via 68 | # -r requirements.txt 69 | # anyio 70 | # httpx 71 | # requests 72 | ijson==3.3.0 73 | # via -r requirements.txt 74 | iniconfig==2.1.0 75 | # via pytest 76 | isal==1.7.2 77 | # via -r requirements.txt 78 | jmespath==1.0.1 79 | # via 80 | # -r requirements.txt 81 | # parsel 82 | lxml==5.4.0 83 | # via 84 | # -r requirements.txt 85 | # parsel 86 | numpy==2.2.5 87 | # via 88 | # -r requirements.txt 89 | # shapely 90 | orjson==3.10.18 91 | # via -r requirements.txt 92 | packaging==25.0 93 | # via 94 | # -r requirements.txt 95 | # gunicorn 96 | # parsel 97 | # pytest 98 | parsel==1.10.0 99 | # via -r requirements.txt 100 | pluggy==1.5.0 101 | # via pytest 102 | psycopg2-binary==2.9.10 103 | # via -r requirements.txt 104 | pydantic==2.11.4 105 | # via 106 | # -r requirements.txt 107 | # fastapi 108 | pydantic-core==2.33.2 109 | # via 110 | # -r requirements.txt 111 | # pydantic 112 | pyproj==3.7.1 113 | # via -r requirements.txt 114 | pytest==8.3.5 115 | # via 116 | # -r requirements-dev.in 117 | # pytest-env 118 | pytest-env==1.1.5 119 | # via -r requirements-dev.in 120 | python-dateutil==2.9.0.post0 121 | # via 122 | # -r requirements.txt 123 | # freezegun 124 | redis==5.2.1 125 | # via -r requirements.txt 126 | requests==2.32.3 127 | # via -r requirements.txt 128 | ruff==0.11.7 129 | # via -r requirements-dev.in 130 | sentry-sdk==2.27.0 131 | # via -r requirements.txt 132 | shapely==2.1.0 133 | # via -r requirements.txt 134 | six==1.17.0 135 | # via 136 | # -r requirements.txt 137 | # python-dateutil 138 | sniffio==1.3.1 139 | # via 140 | # -r requirements.txt 141 | # anyio 142 | starlette==0.46.2 143 | # via 144 | # -r requirements.txt 145 | # fastapi 146 | typing-extensions==4.13.2 147 | # via 148 | # -r requirements.txt 149 | # fastapi 150 | # pydantic 151 | # pydantic-core 152 | # typing-inspection 153 | typing-inspection==0.4.0 154 | # via 155 | # -r requirements.txt 156 | # pydantic 157 | urllib3==2.4.0 158 | # via 159 | # -r requirements.txt 160 | # requests 161 | # sentry-sdk 162 | uv==0.7.1 163 | # via -r requirements-dev.in 164 | uvicorn==0.34.2 165 | # via -r requirements.txt 166 | w3lib==2.3.1 167 | # via 168 | # -r requirements.txt 169 | # parsel 170 | watchfiles==1.0.5 171 | # via -r requirements-dev.in 172 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # This file was autogenerated by uv via the following command: 2 | # uv pip compile -o requirements.txt pyproject.toml 3 | annotated-types==0.7.0 4 | # via pydantic 5 | anyio==4.9.0 6 | # via 7 | # httpx 8 | # starlette 9 | astral==3.2 10 | # via brightsky (pyproject.toml) 11 | asyncpg==0.30.0 12 | # via brightsky (pyproject.toml) 13 | certifi==2025.4.26 14 | # via 15 | # httpcore 16 | # httpx 17 | # pyproj 18 | # requests 19 | # sentry-sdk 20 | charset-normalizer==3.4.1 21 | # via requests 22 | click==8.1.8 23 | # via 24 | # brightsky (pyproject.toml) 25 | # uvicorn 26 | coloredlogs==15.0.1 27 | # via brightsky (pyproject.toml) 28 | cssselect==1.3.0 29 | # via parsel 30 | dwdparse==0.9.15 31 | # via brightsky (pyproject.toml) 32 | falcon==4.0.2 33 | # via brightsky (pyproject.toml) 34 | fastapi==0.115.12 35 | # via brightsky (pyproject.toml) 36 | gunicorn==23.0.0 37 | # via brightsky (pyproject.toml) 38 | h11==0.16.0 39 | # via 40 | # httpcore 41 | # uvicorn 42 | httpcore==1.0.9 43 | # via httpx 44 | httpx==0.28.1 45 | # via brightsky (pyproject.toml) 46 | huey==2.5.3 47 | # via brightsky (pyproject.toml) 48 | humanfriendly==10.0 49 | # via coloredlogs 50 | idna==3.10 51 | # via 52 | # anyio 53 | # httpx 54 | # requests 55 | ijson==3.3.0 56 | # via dwdparse 57 | isal==1.7.2 58 | # via brightsky (pyproject.toml) 59 | jmespath==1.0.1 60 | # via parsel 61 | lxml==5.4.0 62 | # via parsel 63 | numpy==2.2.5 64 | # via 65 | # brightsky (pyproject.toml) 66 | # shapely 67 | orjson==3.10.18 68 | # via brightsky (pyproject.toml) 69 | packaging==25.0 70 | # via 71 | # gunicorn 72 | # parsel 73 | parsel==1.10.0 74 | # via brightsky (pyproject.toml) 75 | psycopg2-binary==2.9.10 76 | # via brightsky (pyproject.toml) 77 | pydantic==2.11.4 78 | # via fastapi 79 | pydantic-core==2.33.2 80 | # via pydantic 81 | pyproj==3.7.1 82 | # via brightsky (pyproject.toml) 83 | python-dateutil==2.9.0.post0 84 | # via brightsky (pyproject.toml) 85 | redis==5.2.1 86 | # via brightsky (pyproject.toml) 87 | requests==2.32.3 88 | # via brightsky (pyproject.toml) 89 | sentry-sdk==2.27.0 90 | # via brightsky (pyproject.toml) 91 | shapely==2.1.0 92 | # via brightsky (pyproject.toml) 93 | six==1.17.0 94 | # via python-dateutil 95 | sniffio==1.3.1 96 | # via anyio 97 | starlette==0.46.2 98 | # via fastapi 99 | typing-extensions==4.13.2 100 | # via 101 | # fastapi 102 | # pydantic 103 | # pydantic-core 104 | # typing-inspection 105 | typing-inspection==0.4.0 106 | # via pydantic 107 | urllib3==2.4.0 108 | # via 109 | # requests 110 | # sentry-sdk 111 | uvicorn==0.34.2 112 | # via brightsky (pyproject.toml) 113 | w3lib==2.3.1 114 | # via parsel 115 | -------------------------------------------------------------------------------- /scripts/benchmark.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import datetime 4 | import logging 5 | import random 6 | import re 7 | import time 8 | from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait 9 | from contextlib import contextmanager 10 | from functools import lru_cache 11 | from multiprocessing import cpu_count 12 | 13 | import click 14 | import psycopg2 15 | import requests 16 | from dateutil.tz import tzutc 17 | from falcon.testing import TestClient 18 | 19 | from brightsky import db, tasks 20 | from brightsky.settings import settings 21 | from brightsky.utils import configure_logging 22 | from brightsky.web import app 23 | 24 | 25 | logger = logging.getLogger('benchmark') 26 | 27 | 28 | SERVER = 'http://localhost:8000' 29 | 30 | 31 | @contextmanager 32 | def _time(description, precision=0, unit='s'): 33 | start = time.time() 34 | yield 35 | delta = round(time.time() - start, precision) 36 | if unit == 'h': 37 | delta_str = str(datetime.timedelta(seconds=delta)) 38 | else: 39 | delta_str = '{:{}.{}f} s'.format(delta, precision+4, precision) 40 | if not description.rstrip().endswith(':'): 41 | description += ':' 42 | click.echo(f'{description} {delta_str}') 43 | 44 | 45 | @lru_cache 46 | def get_client(): 47 | return TestClient(app) 48 | 49 | 50 | @click.group() 51 | def cli(): 52 | try: 53 | settings['DATABASE_URL'] = settings.BENCHMARK_DATABASE_URL 54 | except AttributeError: 55 | raise click.ClickException( 56 | 'Please set the BRIGHTSKY_BENCHMARK_DATABASE_URL environment ' 57 | 'variable') 58 | # This gives us roughly 100 days of weather records in total: 59 | # 89 from recent observations, 1 from current observations, 10 from MOSMIX 60 | settings['MIN_DATE'] = datetime.datetime(2020, 1, 1, tzinfo=tzutc()) 61 | settings['MAX_DATE'] = datetime.datetime(2020, 3, 30, tzinfo=tzutc()) 62 | 63 | 64 | @cli.command(help='Recreate and populate benchmark database') 65 | def build(): 66 | logger.info('Dropping and recreating database') 67 | db_url_base, db_name = settings.DATABASE_URL.rsplit('/', 1) 68 | with psycopg2.connect(db_url_base + '/postgres') as conn: 69 | with conn.cursor() as cur: 70 | conn.set_isolation_level(0) 71 | cur.execute('DROP DATABASE IF EXISTS %s' % (db_name,)) 72 | cur.execute('CREATE DATABASE %s' % (db_name,)) 73 | db.migrate() 74 | file_infos = tasks.poll() 75 | # Make sure we finish parsing MOSMIX before going ahead as current 76 | # observations depend on it 77 | tasks.parse(url=next(file_infos)['url'], export=True) 78 | with ThreadPoolExecutor(max_workers=2*cpu_count()+1) as executor: 79 | with _time('Database creation time', unit='h'): 80 | futures = [ 81 | executor.submit(tasks.parse, url=file_info['url'], export=True) 82 | for file_info in file_infos] 83 | finished, pending = wait(futures, return_when=FIRST_EXCEPTION) 84 | for f in pending: 85 | f.cancel() 86 | for f in finished: 87 | # Re-raise any occured exceptions 88 | if exc := f.exception(): 89 | raise exc 90 | 91 | 92 | @cli.command(help='Calculate database size') 93 | def db_size(): 94 | m = re.search(r'/(\w+)$', settings.DATABASE_URL) 95 | db_name = m.group(1) if m else 'postgres' 96 | with db.get_connection() as conn: 97 | with conn.cursor() as cur: 98 | cur.execute( 99 | 'SELECT pg_database_size(%s)', (db_name,)) 100 | db_size = cur.fetchone() 101 | table_sizes = {} 102 | for table in ['weather', 'synop', 'sources']: 103 | cur.execute('SELECT pg_total_relation_size(%s)', (table,)) 104 | table_sizes[table] = cur.fetchone()[0] 105 | click.echo('Total database size:\n%6d MB' % (db_size[0] / 1024 / 1024)) 106 | click.echo( 107 | 'Table sizes:\n' + '\n'.join( 108 | '%6d MB %s' % (size / 1024 / 1024, table) 109 | for table, size in table_sizes.items())) 110 | 111 | 112 | @cli.command(help='Re-parse MOSMIX data') 113 | def mosmix_parse(): 114 | MOSMIX_URL = ( 115 | 'https://opendata.dwd.de/weather/local_forecasts/mos/MOSMIX_S/' 116 | 'all_stations/kml/MOSMIX_S_LATEST_240.kmz') 117 | with _time('MOSMIX Re-parse', unit='h'): 118 | tasks.parse(url=MOSMIX_URL, export=True) 119 | 120 | 121 | def _query_sequential(path, kwargs_list, **base_kwargs): 122 | for kwargs in kwargs_list: 123 | requests.get( 124 | f'{SERVER}{path}', 125 | params={**base_kwargs, **kwargs}, 126 | ) 127 | 128 | 129 | def _query_parallel(path, kwargs_list, **base_kwargs): 130 | with ThreadPoolExecutor(max_workers=2*cpu_count()+1) as executor: 131 | for kwargs in kwargs_list: 132 | executor.submit( 133 | requests.get, 134 | f'{SERVER}{path}', 135 | params={**base_kwargs, **kwargs}, 136 | ) 137 | 138 | 139 | @cli.command('query', help='Query records from database') 140 | def query_(): 141 | with _time('\nTotal', precision=2): 142 | _query() 143 | 144 | 145 | def _query(): 146 | # Generate 50 random locations within Germany's bounding box. Locations 147 | # and sources will be the same across different runs since we hard-code the 148 | # PRNG seed. 149 | random.seed(1) 150 | location_kwargs = [ 151 | { 152 | 'lat': 47.30 + (i % 30) * (54.98 - 47.30) / 29, 153 | 'lon': 5.99 + (i // 30) * (15.02 - 5.99) / 29, 154 | } 155 | for i in range(900)] 156 | with db.get_connection() as conn: 157 | with conn.cursor() as cur: 158 | cur.execute( 159 | """ 160 | SELECT dwd_station_id, id 161 | FROM sources 162 | WHERE observation_type = %s 163 | """, 164 | ('historical',)) 165 | rows = random.choices(cur.fetchall(), k=100) 166 | station_kwargs = [ 167 | {'dwd_station_id': row['dwd_station_id']} for row in rows] 168 | source_kwargs = [{'source_id': row['id']} for row in rows] 169 | cur.execute( 170 | """ 171 | SELECT MAX(last_record) 172 | FROM sources 173 | WHERE observation_type = 'current' 174 | """) 175 | today = cur.fetchone()['max'].date().isoformat() 176 | date = '2022-08-13' 177 | last_date = '2022-08-20' 178 | 179 | def _test_with_kwargs(kwargs_list): 180 | with _time(' 900 one-day queries, sequential', precision=2): 181 | _query_sequential('/weather', kwargs_list, date=date) 182 | with _time(' 900 one-day queries, parallel: ', precision=2): 183 | _query_parallel('/weather', kwargs_list, date=date) 184 | with _time(' 900 one-week queries, sequential', precision=2): 185 | _query_sequential( 186 | '/weather', kwargs_list, date=date, last_date=last_date) 187 | with _time(' 900 one-week queries, parallel: ', precision=2): 188 | _query_parallel( 189 | '/weather', kwargs_list, date=date, last_date=last_date) 190 | 191 | click.echo('Sources by lat/lon:') 192 | with _time(' 900 queries, sequential: ', precision=2): 193 | _query_sequential('/sources', location_kwargs) 194 | with _time(' 900 queries, parallel: ', precision=2): 195 | _query_parallel('/sources', location_kwargs) 196 | click.echo('\nSources by station:') 197 | with _time(' 900 queries, sequential: ', precision=2): 198 | _query_sequential('/sources', station_kwargs) 199 | with _time(' 900 queries, parallel: ', precision=2): 200 | _query_parallel('/sources', station_kwargs) 201 | click.echo('\nSources by source:') 202 | with _time(' 900 queries, sequential: ', precision=2): 203 | _query_sequential('/sources', source_kwargs) 204 | with _time(' 900 queries, parallel: ', precision=2): 205 | _query_parallel('/sources', source_kwargs) 206 | 207 | click.echo('\nWeather by lat/lon:') 208 | _test_with_kwargs(location_kwargs) 209 | click.echo('\nWeather by lat/lon, today:') 210 | with _time(' 900 one-day queries, sequential', precision=2): 211 | _query_sequential('/weather', location_kwargs, date=today) 212 | with _time(' 900 one-day queries, parallel: ', precision=2): 213 | _query_parallel('/weather', location_kwargs, date=today) 214 | click.echo('\nWeather by station:') 215 | _test_with_kwargs(station_kwargs) 216 | click.echo('\nWeather by source:') 217 | _test_with_kwargs(source_kwargs) 218 | 219 | click.echo('\nCurrent weather by lat/lon:') 220 | with _time(' 900 queries, sequential: ', precision=2): 221 | _query_sequential('/current_weather', location_kwargs) 222 | with _time(' 900 queries, parallel: ', precision=2): 223 | _query_parallel('/current_weather', location_kwargs) 224 | click.echo('\nCurrent weather by station:') 225 | with _time(' 900 queries, sequential: ', precision=2): 226 | _query_sequential('/current_weather', station_kwargs) 227 | with _time(' 900 queries, parallel: ', precision=2): 228 | _query_parallel('/current_weather', station_kwargs) 229 | 230 | 231 | if __name__ == '__main__': 232 | configure_logging() 233 | cli() 234 | -------------------------------------------------------------------------------- /scripts/benchmark_compression.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import timeit 3 | 4 | 5 | URL = 'http://localhost:8000/radar' 6 | BBOXES = [ 7 | None, 8 | '200,200,800,800', 9 | '400,400,600,600', 10 | ] 11 | 12 | 13 | def p(s, back=0): 14 | sys.stdout.write('\b' * back) 15 | sys.stdout.write(s) 16 | sys.stdout.flush() 17 | 18 | 19 | def benchmark(): 20 | for fmt in ['compressed', 'bytes', 'plain']: 21 | p(f'{fmt:10s} ') 22 | for bbox in BBOXES: 23 | time = timeit.timeit( 24 | 'requests.get(url, params=params).raise_for_status()', 25 | setup='import requests', 26 | number=20, 27 | globals={ 28 | 'url': URL, 29 | 'params': { 30 | 'format': fmt, 31 | 'bbox': bbox, 32 | }, 33 | }, 34 | ) 35 | time = int(round(time / 20 * 1000)) 36 | p(f'{time:4d} ', 3) 37 | print('') 38 | 39 | 40 | if __name__ == '__main__': 41 | benchmark() 42 | -------------------------------------------------------------------------------- /scripts/benchmark_radar.py: -------------------------------------------------------------------------------- 1 | import array 2 | import os 3 | import time 4 | import zlib 5 | from contextlib import contextmanager 6 | 7 | import psycopg2 8 | 9 | from brightsky.db import fetch, get_connection 10 | from brightsky.utils import configure_logging, load_dotenv 11 | from brightsky.settings import settings 12 | from brightsky.tasks import parse 13 | 14 | 15 | @contextmanager 16 | def _time(description): 17 | start = time.time() 18 | yield 19 | delta = int(round((time.time() - start) / 50, 3) * 1000) 20 | description += ':' 21 | print(f'{description:15s} {delta:5d} ms') 22 | 23 | 24 | def setup(): 25 | with get_connection() as conn: 26 | with conn.cursor() as cur: 27 | cur.execute('drop table if exists radar;') 28 | cur.execute( 29 | """ 30 | CREATE TABLE radar ( 31 | timestamp timestamptz NOT NULL, 32 | 33 | source varchar(255) NOT NULL, 34 | precipitation_5 smallint[1200][1100] CHECK ( 35 | array_dims(precipitation_5) = '[1:1200][1:1100]' AND 36 | 0 <= ALL(precipitation_5) 37 | ), 38 | precipitation_5_raw bytea NOT NULL, 39 | 40 | CONSTRAINT radar_key UNIQUE (timestamp) 41 | ); 42 | """, 43 | ) 44 | conn.commit() 45 | 46 | 47 | def reset(): 48 | with get_connection() as conn: 49 | with conn.cursor() as cur: 50 | cur.execute('delete from radar') 51 | conn.commit() 52 | _vacuum() 53 | parse('https://opendata.dwd.de/weather/radar/composite/rv/DE1200_RV2305021315.tar.bz2') # noqa 54 | parse('https://opendata.dwd.de/weather/radar/composite/rv/DE1200_RV2305030700.tar.bz2') # noqa 55 | 56 | 57 | def _vacuum(): 58 | conn = psycopg2.connect(settings.DATABASE_URL) 59 | conn.autocommit = True 60 | cur = conn.cursor() 61 | cur.execute('VACUUM ANALYZE radar') 62 | conn.commit() 63 | conn.close() 64 | 65 | 66 | def print_size(): 67 | size = fetch( 68 | "select pg_size_pretty(pg_total_relation_size('radar'));" 69 | )[0][0] 70 | print("DB size: ", size) 71 | 72 | 73 | def time_full_array(): 74 | with _time('Array full'): 75 | fetch('select timestamp, source, precipitation_5 from radar') 76 | 77 | 78 | def time_clip_array(): 79 | with _time('Array clipped'): 80 | fetch( 81 | """ 82 | select timestamp, source, precipitation_5[400:600][400:600] 83 | from radar 84 | """ 85 | ) 86 | 87 | 88 | def _make_array(buf): 89 | a = array.array('H') 90 | a.frombytes(buf) 91 | return a 92 | 93 | 94 | def time_full_bytes(): 95 | with _time('Bytes full'): 96 | rows = fetch( 97 | 'select timestamp, source, precipitation_5_raw from radar', 98 | ) 99 | for row in rows: 100 | data = _make_array(zlib.decompress(row['precipitation_5_raw'])) 101 | # data = [ 102 | # x if x < 4096 else None 103 | # for x in data 104 | # ] 105 | precip_5 = [ # noqa 106 | data[i*1100:(i+1)*1100].tolist() 107 | for i in reversed(range(1200)) 108 | ] 109 | 110 | 111 | def time_clip_bytes(): 112 | with _time('Bytes clipped'): 113 | rows = fetch( 114 | 'select timestamp, source, precipitation_5_raw from radar', 115 | ) 116 | for row in rows: 117 | # data = array.array('H') 118 | raw = zlib.decompress(row['precipitation_5_raw']) 119 | precip_5 = [ # noqa 120 | _make_array(raw[i*2200+800:i*2200+1200]).tolist() 121 | for i in range(400, 600) 122 | ] 123 | # data = [ 124 | # x if x < 4096 else None 125 | # for x in data 126 | # ] 127 | # precip_5 = [ 128 | # data[i*1100+400:i*1100+600] 129 | # for i in range(400, 600) 130 | # ] 131 | 132 | 133 | def main(): 134 | setup() 135 | for level in range(10): 136 | print('') 137 | os.environ['ZLIB_COMPRESSION_LEVEL'] = str(level) 138 | reset() 139 | print('\nCOMPRESSION LEVEL', level) 140 | print_size() 141 | time_full_array() 142 | time_clip_array() 143 | time_full_bytes() 144 | time_clip_bytes() 145 | 146 | 147 | if __name__ == '__main__': 148 | configure_logging() 149 | load_dotenv() 150 | main() 151 | -------------------------------------------------------------------------------- /scripts/radar_coordinates.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | import os.path 5 | import re 6 | import subprocess 7 | import tempfile 8 | 9 | proj_str = '+proj=stere +lat_0=90 +lat_ts=60 +lon_0=10 +a=6378137 +b=6356752.3142451802 +no_defs +x_0=543196.83521776402 +y_0=3622588.8619310018 -f "%.10g"' # noqa 10 | 11 | HEIGHT = 1200 12 | WIDTH = 1100 13 | 14 | 15 | def main(): 16 | with tempfile.TemporaryDirectory() as tmpdir: 17 | in_path = os.path.join(tmpdir, 'in') 18 | with open(in_path, 'w') as f: 19 | for y in range(HEIGHT): 20 | for x in range(WIDTH): 21 | f.write(f'{x*1000} {-y*1000}\n') 22 | p = subprocess.run( 23 | f'invproj {proj_str} {in_path}', 24 | shell=True, 25 | cwd=tmpdir, 26 | capture_output=True, 27 | text=True, 28 | ) 29 | coords = [] 30 | for line in p.stdout.splitlines(): 31 | xy = [float(x) for x in re.split(r'\s+', line.strip())] 32 | assert len(xy) == 2 33 | coords.append(xy) 34 | coords = [ 35 | coords[row*WIDTH:(row+1)*WIDTH] 36 | for row in range(HEIGHT) 37 | ] 38 | with open('radar_coordinates.json', 'w') as f: 39 | # It'd be nicer if we could keep the proj output as strings and 40 | # convince json to output strings without quotes (so they will be 41 | # read as float), but that's not easily available through custom 42 | # encoders... 43 | json.dump(coords, f) 44 | 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdemaeyer/brightsky/79b17f026dfdbc29309261d4970c76c8bef06b5d/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | from dataclasses import dataclass 4 | from pathlib import Path 5 | from typing import Any 6 | from urllib.parse import urlparse 7 | 8 | import psycopg2 9 | import pytest 10 | from fastapi.testclient import TestClient 11 | from psycopg2.extras import execute_values 12 | 13 | from brightsky.db import get_connection, migrate 14 | 15 | 16 | @pytest.fixture(scope='session') 17 | def data_dir(): 18 | return Path(os.path.dirname(__file__)) / 'data' 19 | 20 | 21 | @pytest.fixture(scope='session') 22 | def _database(): 23 | if not os.getenv('BRIGHTSKY_DATABASE_URL'): 24 | pytest.skip('See README for running database-based tests.') 25 | url = urlparse(os.getenv('BRIGHTSKY_DATABASE_URL')) 26 | postgres_url = f'postgres://{url.netloc}' 27 | db_name = url.path.lstrip('/') 28 | assert db_name 29 | conn = psycopg2.connect(postgres_url) 30 | conn.autocommit = True 31 | with conn.cursor() as cur: 32 | cur.execute(f'DROP DATABASE IF EXISTS {db_name}') 33 | cur.execute(f'CREATE DATABASE {db_name}') 34 | conn.close() 35 | migrate() 36 | yield 37 | if hasattr(get_connection, '_pool'): 38 | get_connection._pool.closeall() 39 | conn = psycopg2.connect(postgres_url) 40 | conn.autocommit = True 41 | conn.autocommit = True 42 | with conn.cursor() as cur: 43 | cur.execute(f'DROP DATABASE {db_name}') 44 | conn.close() 45 | 46 | 47 | @dataclass 48 | class TestConnection: 49 | """Wrapper for database connection with some rough convenience functions""" 50 | 51 | conn: Any 52 | 53 | def insert(self, table, rows): 54 | with self.cursor() as cur: 55 | fields = tuple(rows[0]) 56 | field_placeholders = [f'%({field})s' for field in fields] 57 | execute_values( 58 | cur, 59 | f"INSERT INTO {table} ({', '.join(fields)}) VALUES %s", 60 | rows, 61 | template=f"({', '.join(field_placeholders)})") 62 | self.conn.commit() 63 | 64 | def fetch(self, sql): 65 | with self.conn.cursor() as cur: 66 | cur.execute(sql) 67 | rows = cur.fetchall() 68 | self.conn.commit() 69 | return rows 70 | 71 | def table(self, name): 72 | return self.fetch(f'SELECT * FROM {name}') 73 | 74 | def __getattr__(self, name): 75 | return getattr(self.conn, name) 76 | 77 | 78 | @pytest.fixture 79 | def db(_database): 80 | with get_connection() as conn: 81 | yield TestConnection(conn) 82 | with conn.cursor() as cur: 83 | cur.execute(""" 84 | DELETE FROM parsed_files; 85 | DELETE FROM synop; 86 | DELETE FROM weather; 87 | DELETE FROM sources; 88 | REFRESH MATERIALIZED VIEW current_weather; 89 | """) 90 | 91 | 92 | @pytest.fixture 93 | def api(db): 94 | from brightsky.web import app 95 | with TestClient(app) as client: 96 | yield client 97 | 98 | 99 | def pytest_configure(config): 100 | # Dirty mock so we don't download the station list on every test run 101 | from dwdparse.stations import _converter 102 | # Must contain all stations that we use in test data 103 | _converter.dwd_to_wmo = { 104 | 'XXX': '01028', 105 | 'YYY': '01049', 106 | '01766': '10315', 107 | '04911': '10788', 108 | '05484': 'M031', 109 | } 110 | _converter.wmo_to_dwd = dict( 111 | reversed(x) for x in _converter.dwd_to_wmo.items()) 112 | _converter.last_update = time.time() 113 | -------------------------------------------------------------------------------- /tests/data/10minutenwerte_SOLAR_01766_akt.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdemaeyer/brightsky/79b17f026dfdbc29309261d4970c76c8bef06b5d/tests/data/10minutenwerte_SOLAR_01766_akt.zip -------------------------------------------------------------------------------- /tests/data/DE1200_RV2305081330.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdemaeyer/brightsky/79b17f026dfdbc29309261d4970c76c8bef06b5d/tests/data/DE1200_RV2305081330.tar.bz2 -------------------------------------------------------------------------------- /tests/data/Meta_Daten_zehn_min_sd_01766.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdemaeyer/brightsky/79b17f026dfdbc29309261d4970c76c8bef06b5d/tests/data/Meta_Daten_zehn_min_sd_01766.zip -------------------------------------------------------------------------------- /tests/data/Z_CAP_C_EDZW_LATEST_PVW_STATUS_PREMIUMDWD_COMMUNEUNION_MUL.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdemaeyer/brightsky/79b17f026dfdbc29309261d4970c76c8bef06b5d/tests/data/Z_CAP_C_EDZW_LATEST_PVW_STATUS_PREMIUMDWD_COMMUNEUNION_MUL.zip -------------------------------------------------------------------------------- /tests/data/dwd_opendata_index.html: -------------------------------------------------------------------------------- 1 | 2 | Index of /climate_environment/CDC/observations_germany/climate/hourly/wind/recent/ 3 | 4 |

Index of /climate_environment/CDC/observations_germany/climate/hourly/wind/recent/

../
 5 | BESCHREIBUNG_obsgermany_climate_hourly_wind_rec..> 11-Jan-2020 12:38               68097
 6 | DESCRIPTION_obsgermany_climate_hourly_wind_rece..> 11-Jan-2020 12:38               67800
 7 | FF_Stundenwerte_Beschreibung_Stationen.txt         29-Mar-2020 08:55              104645
 8 | stundenwerte_FF_00011_akt.zip                      29-Mar-2020 08:55               70523
 9 | stundenwerte_FF_00090_akt.zip                      29-Mar-2020 08:56               71408
10 | stundenwerte_P0_00096_akt.zip                      29-Mar-2020 08:57               47355
11 | stundenwerte_RR_00102_akt.zip                      29-Mar-2020 08:58               74372
12 | stundenwerte_SD_00125_akt.zip                      29-Mar-2020 08:59               69633
13 | stundenwerte_TD_00161_akt.zip                      29-Mar-2020 09:02               70167
14 | stundenwerte_TU_00161_akt.zip                      29-Mar-2020 09:00               70165
15 | stundenwerte_VV_00161_akt.zip                      29-Mar-2020 09:03               70168
16 | stundenwerte_N_00161_akt.zip                      29-Mar-2020 09:01               70166
17 | MOSMIX_L_LATEST.kmz         29-Mar-2020 09:57            91636300
18 | MOSMIX_S_LATEST_240.kmz                            29-Mar-2020 10:21            38067304
19 | K611_-BEOB.csv                                     06-Apr-2020 10:38                7343
20 | 10minutenwerte_extrema_wind_01766_now.zip          08-Jun-2020 09:12                 701
21 | 10minutenwerte_extrema_wind_01766_akt.zip          08-Jun-2020 04:27              519692
22 | 10minutenwerte_extrema_wind_01766_20100101_2019..> 09-Apr-2020 09:16             3661729
23 | 10minutenwerte_SOLAR_01766_akt.zip          12-Apr-2023 00:50              367557
24 |
25 | 26 | -------------------------------------------------------------------------------- /tests/data/observations_current.csv: -------------------------------------------------------------------------------- 1 | surface observations;Parameter description;cloud_cover_total;daily_mean_of_temperature_previous_day;depth_of_new_snow;dew_point_temperature_at_2_meter_above_ground;diffuse_solar_radiation_last_hour;direct_solar_radiation_last_24_hours;direct_solar_radiation_last_hour;dry_bulb_temperature_at_2_meter_above_ground;evaporation/evapotranspiration_last_24_hours;global_radiation_last_hour;global_radiation_past_24_hours;height_of_base_of_lowest_cloud_above_station;horizontal_visibility;maximum_of_10_minutes_mean_of_wind_speed_for_previous_day;maximum_of_temperature_for_previous_day;maximum_temperature_last_12_hours_2_meters_above_ground;maximum_wind_speed_as_10_minutes_mean_during_last_hour;maximum_wind_speed_during_last_6_hours;maximum_wind_speed_for_previous_day;maximum_wind_speed_last_hour;mean_wind_direction_during_last_10 min_at_10_meters_above_ground;mean_wind_speed_during last_10_min_at_10_meters_above_ground;minimum_of_temperature_at_5_cm_above_ground_for_previous_day;minimum_of_temperature_for_previous_day;minimum_temperature_last_12_hours_2_meters_above_ground;minimum_temperature_last_12_hours_5_cm_above_ground;past_weather_1;past_weather_2;precipitation_amount_last_24_hours;precipitation_amount_last_3_hours;precipitation_amount_last_6_hours;precipitation_amount_last_hour;precipitation_last_12_hours;present_weather;pressure_reduced_to_mean_sea_level;relative_humidity;sea/water_temperature;temperature_at_5_cm_above_ground;total_snow_depth;total_time_of_sunshine_during_last_hour;total_time_of_sunshine_past_day 2 | 01049;Unit;%;Grad C;cm;Grad C;W/m2;W/m2;W/m2;Grad C;mm;W/m2;W/m2;m;km;km/h;Grad C;Grad C;km/h;km/h;km/h;km/h;Grad;km/h;Grad C;Grad C;Grad C;Grad C;CODE_TABLE;CODE_TABLE;mm;mm;mm;mm;mm;CODE_TABLE;hPa;%;Grad C;Grad C;cm;min;h 3 | Datum;Uhrzeit (UTC);Wolkenbedeckung;mittlere Temperatur (vergangener Tag, 2m);Neuschneehoehe;Taupunkttemperatur (2m);Diffuse Strahlung (letzte Stunde);Direkte Strahlung (vergangene 24 Stunden);Direkte Strahlung (letzte Stunde);Temperatur (2m);Evaporation (vergangene 24 Stunden);Globalstrahlung (letzte Stunde);Globalstrahlung (vergangene 24 Stunden);Wolkenuntergrenze;Sichtweite;Maximalwind (vergangener Tag);Maximumtemperatur (vergangener Tag, 2m);Maximumtemperatur (letzte 12 Stunden, 2m);Maximalwind (letzte Stunde);Windboen (letzte 6 Stunden);Windboen (vergangener Tag);Windboen (letzte Stunde);Windrichtung;Windgeschwindigkeit;Minimumtemperatur (vergangener Tag, 5cm);Minimumtemperatur (vergangener Tag, 2m);Minimumtemperatur (letzte 12 Stunden, 2m);Minimumtemperatur (letzte 12 Stunden, 5cm);vergangenes Wetter 1;vergangenes Wetter 2;Niederschlag (letzte 24 Stunden);Niederschlag (letzte 3 Stunden);Niederschlag (letzte 6 Stunden);Niederschlag (letzte Stunde);Niederschlag (letzte 12 Stunden);aktuelles Wetter;Druck (auf Meereshoehe);Relative Feuchte;Wassertemperatur;Temperatur (5cm);Schneehoehe;Sonnenscheindauer (letzte Stunde);Sonnenscheindauer (vergangener Tag) 4 | 06.04.20;08:00;---;---;---;-10;---;---;---;-3,1;---;---;---;---;---;---;---;---;16;---;---;21;140;14;---;---;---;---;---;---;---;---;---;0;---;---;1023,1;59;---;---;---;---;--- 5 | 06.04.20;07:00;---;---;---;-10,2;---;---;---;-4,5;---;---;---;---;---;---;---;---;17;---;---;22;140;14;---;---;---;---;---;---;---;---;---;0;---;---;1023,6;64;---;---;---;---;--- 6 | 06.04.20;06:00;---;---;---;-10,4;---;---;---;-5,8;---;---;---;---;---;15;---;-2,1;---;18;---;19;150;14;---;---;-11,4;---;---;---;0,7;---;---;0;0,1;---;1023,9;70;---;---;---;---;--- 7 | 06.04.20;05:00;---;---;---;-9,6;---;---;---;-6,8;---;---;---;---;---;---;---;---;7;---;---;12;120;7;---;---;---;---;---;---;---;---;---;0;---;---;1024;80;---;---;---;---;--- 8 | 06.04.20;04:00;---;---;---;-11,3;---;---;---;-9,6;---;---;---;---;---;---;---;---;6;---;---;10;130;7;---;---;---;---;---;---;---;---;---;0;---;---;1024,1;87;---;---;---;---;--- 9 | 06.04.20;03:00;---;---;---;-11,6;---;---;---;-10,3;---;---;---;---;---;---;---;---;---;---;---;13;180;4;---;---;---;---;---;---;---;---;---;0;---;---;1024;90;---;---;---;---;--- 10 | 06.04.20;02:00;---;---;---;-10,5;---;---;---;-9,3;---;---;---;---;---;---;---;---;11;---;---;14;170;7;---;---;---;---;---;---;---;---;---;0;---;---;1023,4;91;---;---;---;---;--- 11 | 06.04.20;01:00;---;---;---;-10,8;---;---;---;-9,1;---;---;---;---;---;---;---;---;13;---;---;15;180;4;---;---;---;---;---;---;---;---;---;0;---;---;1023,2;87;---;---;---;---;--- 12 | 06.04.20;00:00;---;---;---;-10,9;---;---;---;-9,3;---;---;---;---;---;16;---;---;---;22;---;15;110;11;---;---;---;---;---;---;---;---;0,1;0;---;---;1023,1;88;---;---;---;---;--- 13 | 05.04.20;23:00;---;---;---;-9,6;---;---;---;-8,1;---;---;---;---;---;---;---;---;13;---;---;15;70;7;---;---;---;---;---;---;---;---;---;0;---;---;1022,6;89;---;---;---;---;--- 14 | 05.04.20;22:00;---;---;---;-8,7;---;---;---;-7,6;---;---;---;---;---;---;---;---;14;---;---;17;140;11;---;---;---;---;---;---;---;---;---;0;---;---;1022;92;---;---;---;---;--- 15 | 05.04.20;21:00;---;---;---;-8,1;---;---;---;-6,4;---;---;---;---;---;---;---;---;---;---;---;17;130;14;---;---;---;---;---;---;---;---;---;0;---;---;1021,6;88;---;---;---;---;--- 16 | 05.04.20;20:00;---;---;---;-5,3;---;---;---;-3,8;---;---;---;---;---;---;---;---;10;---;---;14;170;7;---;---;---;---;---;---;---;---;---;0,1;---;---;1020,8;89;---;---;---;---;--- 17 | 05.04.20;19:00;---;---;---;-4,3;---;---;---;-2,7;---;---;---;---;---;---;---;---;16;---;---;20;230;11;---;---;---;---;---;---;---;---;---;0;---;---;1020,1;89;---;---;---;---;--- 18 | 05.04.20;18:00;---;---;---;-3,5;---;---;---;-2,1;---;---;---;---;---;26;---;0,8;---;36;---;27;220;18;---;---;-9,2;---;---;---;---;---;---;---;0,6;---;1019,6;90;---;---;---;---;--- 19 | 05.04.20;17:00;---;---;---;-3,1;---;---;---;-2,2;---;---;---;---;---;---;---;---;19;---;---;26;230;14;---;---;---;---;---;---;---;---;---;0,6;---;---;1019,1;94;---;---;---;---;--- 20 | 05.04.20;16:00;---;---;---;-4,3;---;---;---;-1,7;---;---;---;---;---;---;---;---;26;---;---;35;250;18;---;---;---;---;---;---;---;---;---;0;---;---;1019;82;---;---;---;---;--- 21 | 05.04.20;15:00;---;---;---;-6,1;---;---;---;-1;---;---;---;---;---;---;---;---;---;---;---;33;270;22;---;---;---;---;---;---;---;---;---;0;---;---;1018,3;68;---;---;---;---;--- 22 | 05.04.20;14:00;---;---;---;-7,1;---;---;---;0;---;---;---;---;---;---;---;---;17;---;---;25;240;11;---;---;---;---;---;---;---;---;---;0;---;---;1017,9;59;---;---;---;---;--- 23 | 05.04.20;13:00;---;---;---;-9,4;---;---;---;-0,8;---;---;---;---;---;---;---;---;13;---;---;20;190;11;---;---;---;---;---;---;---;---;---;0;---;---;1017,5;52;---;---;---;---;--- 24 | 05.04.20;12:00;---;---;---;-10,7;---;---;---;-1,2;---;---;---;---;---;23;---;---;---;---;---;29;168;12;---;---;---;---;---;---;---;---;0;0;---;---;1017,2;48;---;---;---;---;--- 25 | 05.04.20;11:00;---;---;---;-12,3;---;---;---;-0,8;---;---;---;---;---;---;---;---;21;---;---;28;210;18;---;---;---;---;---;---;---;---;---;0;---;---;1017;41;---;---;---;---;--- 26 | 05.04.20;10:00;---;---;---;-10,8;---;---;---;-0,7;---;---;---;---;---;---;---;---;16;---;---;21;210;14;---;---;---;---;---;---;---;---;---;0;---;---;1016,6;46;---;---;---;---;--- 27 | 05.04.20;09:00;---;---;---;-6,8;---;---;---;-0,2;---;---;---;---;---;---;---;---;---;---;---;11;80;4;---;---;---;---;---;---;---;---;---;0;---;---;1016,1;61;---;---;---;---;--- 28 | 05.04.20;08:00;---;---;---;-7,4;---;---;---;-3,2;---;---;---;---;---;---;---;---;8;---;---;11;70;7;---;---;---;---;---;---;---;---;---;0;---;---;1015,5;73;---;---;---;---;--- 29 | -------------------------------------------------------------------------------- /tests/data/observations_recent_FF_akt.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jdemaeyer/brightsky/79b17f026dfdbc29309261d4970c76c8bef06b5d/tests/data/observations_recent_FF_akt.zip -------------------------------------------------------------------------------- /tests/data/station_list.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Stationslexikon -- Klimadaten Deutschland -- 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 |

Stationslexikon

StationsnameStations_IDKennungStations-
kennung		BreiteLängeStations-
höhe		FlussgebietBundeslandBeginnEnde
Aachen3SO0220550.78276.0941202803100NW01.01.195131.03.2011
Aachen3MN1050150.78276.0941202803100NW01.04.195001.04.2011
Aachen3TU0220550.78276.0941202803100NW01.04.195031.03.2011
Ahaus7374EB0160052.08146.94146690250NW01.03.200608.06.2020
Göttingen1691SY1044451.50029.9507167544280NI01.10.195009.06.2020
Münster/Osnabrück1766SO0115152.13447.696948603180NW01.10.198908.06.2020
Münster/Osnabrück1766SY1031552.13447.696948603180NW01.10.198909.06.2020
Zehdenick5745SY1028352.96613.32751461050BB01.06.200409.06.2020
Zehdenick5745SYF26352.96613.32751461050BB01.06.200409.06.2020
Zehdenick5745TU0332652.96613.32751461050BB01.01.198108.06.2020
Zehren5746RR4126751.19913.405108412690SN01.01.198130.04.2020
34 | 35 |
generiert: 10.06.2020 -- Deutscher Wetterdienst --
36 | 37 | 38 | -------------------------------------------------------------------------------- /tests/test_db.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def test_migrate(db): 5 | assert len(db.table('migrations')) == len(os.listdir('migrations')) 6 | -------------------------------------------------------------------------------- /tests/test_export.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from dateutil.tz import tzutc 3 | 4 | import pytest 5 | 6 | from brightsky.export import DBExporter, SYNOPExporter 7 | 8 | 9 | SOURCES = [ 10 | { 11 | 'observation_type': 'historical', 12 | 'lat': 10.1, 13 | 'lon': 20.2, 14 | 'height': 30.3, 15 | 'wmo_station_id': '10001', 16 | 'dwd_station_id': 'XYZ', 17 | 'station_name': 'Münster', 18 | }, 19 | { 20 | 'observation_type': 'historical', 21 | 'lat': 40.4, 22 | 'lon': 50.5, 23 | 'height': 60.6, 24 | 'wmo_station_id': '10002', 25 | 'dwd_station_id': None, 26 | 'station_name': 'Aurich', 27 | }, 28 | { 29 | 'observation_type': 'historical', 30 | 'lat': 60.6, 31 | 'lon': 70.7, 32 | 'height': 80.8, 33 | 'wmo_station_id': '10003', 34 | 'dwd_station_id': None, 35 | 'station_name': 'Göttingen', 36 | }, 37 | ] 38 | RECORDS = [ 39 | { 40 | 'timestamp': datetime.datetime(2020, 8, 18, 18, tzinfo=tzutc()), 41 | 'temperature': 291.25, 42 | 'precipitation': 0.3, 43 | }, 44 | { 45 | 'timestamp': datetime.datetime(2020, 8, 18, 19, tzinfo=tzutc()), 46 | 'temperature': 290.25, 47 | 'precipitation': 0.2, 48 | }, 49 | { 50 | 'timestamp': datetime.datetime(2020, 8, 18, 20, tzinfo=tzutc()), 51 | 'temperature': 289.25, 52 | 'precipitation': 0.1, 53 | }, 54 | ] 55 | FINGERPRINT = { 56 | 'url': 'https://example.com/source.zip', 57 | 'last_modified': datetime.datetime(2020, 8, 19, 12, 34, tzinfo=tzutc()), 58 | 'file_size': 12345 59 | } 60 | 61 | 62 | @pytest.fixture 63 | def exporter(): 64 | exporter = DBExporter() 65 | exporter.export( 66 | [ 67 | {**SOURCES[0], **RECORDS[0]}, 68 | {**SOURCES[1], **RECORDS[1]}, 69 | ], 70 | fingerprint=FINGERPRINT) 71 | return exporter 72 | 73 | 74 | def _query_sources(db): 75 | return db.fetch("SELECT * FROM sources ORDER BY id") 76 | 77 | 78 | def _query_records(db, table='weather'): 79 | return db.fetch( 80 | f""" 81 | SELECT * FROM {table} 82 | JOIN sources ON {table}.source_id = sources.id 83 | ORDER BY sources.id, timestamp 84 | """) 85 | 86 | 87 | def test_db_exporter_creates_new_sources(db, exporter): 88 | db_sources = _query_sources(db) 89 | assert len(db_sources) == 2 90 | for source, row in zip(SOURCES[:2], db_sources): 91 | for k, v in source.items(): 92 | assert row[k] == v 93 | 94 | 95 | def test_db_exporter_reuses_existing_sources(db, exporter): 96 | exporter.export([{**SOURCES[0], **RECORDS[2]}]) 97 | db_sources = _query_sources(db) 98 | assert len(db_sources) == len(SOURCES[:2]) 99 | # Exports with only known sources should also not increase the sources_id 100 | # sequence 101 | exporter.export([{**SOURCES[2], **RECORDS[2]}]) 102 | db_sources = _query_sources(db) 103 | assert db_sources[2]['id'] == db_sources[0]['id'] + 2 104 | 105 | 106 | def test_db_exporter_creates_new_records(db, exporter): 107 | db_records = _query_records(db) 108 | for record, source, row in zip(RECORDS[:2], SOURCES[:2], db_records): 109 | for k, v in source.items(): 110 | assert row[k] == v 111 | for k, v in record.items(): 112 | assert row[k] == v 113 | 114 | 115 | def test_db_exporter_updates_existing_records(db, exporter): 116 | record = RECORDS[0].copy() 117 | record['precipitation'] = 10. 118 | record['cloud_cover'] = 50 119 | exporter.export([{**SOURCES[0], **record}]) 120 | db_records = _query_records(db) 121 | for k, v in record.items(): 122 | assert db_records[0][k] == v 123 | 124 | 125 | def test_db_exporter_updates_parsed_files(db, exporter): 126 | parsed_files = db.fetch("SELECT * FROM parsed_files") 127 | assert len(parsed_files) == 1 128 | for k, v in FINGERPRINT.items(): 129 | assert parsed_files[0][k] == v 130 | 131 | 132 | def test_db_exporter_updates_source_first_last_record(db, exporter): 133 | db_sources = _query_sources(db) 134 | assert db_sources[0]['first_record'] == RECORDS[0]['timestamp'] 135 | assert db_sources[0]['last_record'] == RECORDS[0]['timestamp'] 136 | exporter.export([{**SOURCES[0], **RECORDS[2]}]) 137 | db_sources = _query_sources(db) 138 | assert db_sources[0]['first_record'] == RECORDS[0]['timestamp'] 139 | assert db_sources[0]['last_record'] == RECORDS[2]['timestamp'] 140 | 141 | 142 | def test_synop_exporter(db): 143 | exporter = SYNOPExporter() 144 | assert len(_query_records(db, table='current_weather')) == 0 145 | # Exporter needs to merge separate records for the same source and time 146 | record = RECORDS[0].copy() 147 | record['timestamp'] = datetime.datetime.now(datetime.UTC).replace( 148 | minute=0, second=0, microsecond=0, tzinfo=tzutc()) 149 | extra_record = { 150 | 'timestamp': record['timestamp'], 151 | 'pressure_msl': 101010, 152 | } 153 | previous_record = RECORDS[1].copy() 154 | previous_record['timestamp'] = ( 155 | record['timestamp'] - datetime.timedelta(minutes=30)) 156 | exporter.export([ 157 | {**SOURCES[0], **record}, 158 | {**SOURCES[0], **extra_record}, 159 | {**SOURCES[0], **previous_record}, 160 | ]) 161 | # Merges records for the same source and timestamp 162 | synop_records = _query_records(db, table='synop') 163 | assert len(synop_records) == 2 164 | assert synop_records[-1]['timestamp'] == record['timestamp'] 165 | assert synop_records[-1]['temperature'] == record['temperature'] 166 | assert synop_records[-1]['pressure_msl'] == extra_record['pressure_msl'] 167 | # Updates current_weather 168 | # XXX: This test may be flaky as the concurrent refresh may not have 169 | # finished yet. Can we somehow wait until the lock is released? 170 | current_weather_records = _query_records(db, table='current_weather') 171 | assert len(current_weather_records) == 1 172 | -------------------------------------------------------------------------------- /tests/test_parsers.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | import numpy as np 4 | from dateutil.tz import tzutc 5 | from freezegun import freeze_time 6 | from isal import isal_zlib as zlib 7 | 8 | from brightsky.parsers import ( 9 | CloudCoverObservationsParser, 10 | CurrentObservationsParser, 11 | DewPointObservationsParser, 12 | get_parser, 13 | MOSMIXParser, 14 | PrecipitationObservationsParser, 15 | PressureObservationsParser, 16 | RADOLANParser, 17 | SolarRadiationObservationsParser, 18 | SunshineObservationsParser, 19 | SYNOPParser, 20 | TemperatureObservationsParser, 21 | VisibilityObservationsParser, 22 | WindGustsObservationsParser, 23 | WindObservationsParser, 24 | ) 25 | 26 | from .utils import settings 27 | 28 | 29 | def test_current_observation_parser_loads_station_data_from_db(db, data_dir): 30 | source = { 31 | 'observation_type': 'forecast', 32 | 'lat': 52.12, 33 | 'lon': 7.62, 34 | 'height': 5., 35 | 'wmo_station_id': '01049', 36 | 'station_name': 'Münster', 37 | } 38 | db.insert('sources', [source]) 39 | p = CurrentObservationsParser() 40 | record = next(p.parse(data_dir / 'observations_current.csv')) 41 | for field in ('lat', 'lon', 'height', 'station_name'): 42 | assert record[field] == source[field] 43 | 44 | 45 | def test_observations_parser_skips_file_if_out_of_range(data_dir): 46 | p = PressureObservationsParser() 47 | path = data_dir / 'observations_19950901_20150817_hist.zip' 48 | assert not p.skip_path(path) 49 | with settings( 50 | MIN_DATE=datetime.datetime(2016, 1, 1, tzinfo=tzutc()), 51 | ): 52 | assert p.skip_path(path) 53 | with settings( 54 | MAX_DATE=datetime.datetime(1995, 1, 1, tzinfo=tzutc()), 55 | ): 56 | assert p.skip_path(path) 57 | 58 | 59 | def test_observations_parser_skips_rows_if_before_cutoff(data_dir): 60 | p = WindObservationsParser() 61 | path = data_dir / 'observations_recent_FF_akt.zip' 62 | with settings( 63 | MIN_DATE=datetime.datetime(2019, 1, 1, tzinfo=tzutc()), 64 | ): 65 | records = list(p.parse(path)) 66 | assert len(records) == 5 67 | assert records[0]['timestamp'] == datetime.datetime( 68 | 2019, 4, 20, 21, tzinfo=tzutc()) 69 | with settings( 70 | MAX_DATE=datetime.datetime(2019, 1, 1, tzinfo=tzutc()), 71 | ): 72 | records = list(p.parse(path)) 73 | assert len(records) == 5 74 | assert records[-1]['timestamp'] == datetime.datetime( 75 | 2018, 9, 15, 4, tzinfo=tzutc()) 76 | 77 | 78 | def test_radolan_parser(data_dir): 79 | p = RADOLANParser() 80 | records = list(p.parse(data_dir / 'DE1200_RV2305081330.tar.bz2')) 81 | assert len(records) == 1 82 | data = np.frombuffer( 83 | zlib.decompress(records[0]['precipitation_5']), 84 | dtype='i2', 85 | ) 86 | assert len(data) == 1200 * 1100 87 | assert np.sum(data) == 564030 88 | assert len(np.where(data < 4096)[0]) == len(data) 89 | assert data.reshape((1200, 1100))[1117:1122, 334:339].tolist() == [ 90 | [3, 5, 2, 1, 3], 91 | [2, 3, 3, 0, 0], 92 | [3, 4, 1, 0, 3], 93 | [0, 8, 0, 0, 0], 94 | [0, 0, 0, 0, 0], 95 | ] 96 | 97 | 98 | @freeze_time('2023-06-11') 99 | def test_solar_radiation_parser_skips_today(data_dir): 100 | p = SolarRadiationObservationsParser() 101 | records = list(p.parse( 102 | data_dir / '10minutenwerte_SOLAR_01766_akt.zip', 103 | meta_path=data_dir / 'Meta_Daten_zehn_min_sd_01766.zip' 104 | )) 105 | assert records[-1]['timestamp'] == datetime.datetime( 106 | 2023, 6, 10, 23, tzinfo=tzutc()) 107 | 108 | 109 | def test_get_parser(): 110 | synop_with_timestamp = ( 111 | 'Z__C_EDZW_20200617114802_bda01,synop_bufr_GER_999999_999999__MW_617' 112 | '.json.bz2') 113 | synop_latest = ( 114 | 'Z__C_EDZW_latest_bda01,synop_bufr_GER_999999_999999__MW_XXX.json.bz2') 115 | expected = { 116 | '10minutenwerte_extrema_wind_00427_akt.zip': ( 117 | WindGustsObservationsParser), 118 | '10minutenwerte_SOLAR_01766_now.zip': SolarRadiationObservationsParser, 119 | 'stundenwerte_FF_00011_akt.zip': WindObservationsParser, 120 | 'stundenwerte_FF_00090_akt.zip': WindObservationsParser, 121 | 'stundenwerte_N_01766_akt.zip': CloudCoverObservationsParser, 122 | 'stundenwerte_P0_00096_akt.zip': PressureObservationsParser, 123 | 'stundenwerte_RR_00102_akt.zip': PrecipitationObservationsParser, 124 | 'stundenwerte_SD_00125_akt.zip': SunshineObservationsParser, 125 | 'stundenwerte_TD_01766.zip': DewPointObservationsParser, 126 | 'stundenwerte_TU_00161_akt.zip': TemperatureObservationsParser, 127 | 'stundenwerte_VV_00161_akt.zip': VisibilityObservationsParser, 128 | 'MOSMIX_S_LATEST_240.kmz': MOSMIXParser, 129 | 'DE1200_RV2305081330.tar.bz2': RADOLANParser, 130 | 'K611_-BEOB.csv': CurrentObservationsParser, 131 | synop_with_timestamp: SYNOPParser, 132 | synop_latest: None, 133 | } 134 | for filename, expected_parser in expected.items(): 135 | assert get_parser(filename) is expected_parser 136 | -------------------------------------------------------------------------------- /tests/test_polling.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from unittest.mock import patch 3 | 4 | from dateutil.tz import tzutc 5 | 6 | from brightsky.polling import DWDPoller 7 | 8 | 9 | def test_dwdpoller_parse(data_dir): 10 | with open(data_dir / 'dwd_opendata_index.html') as f: 11 | resp_text = f.read() 12 | expected = { 13 | '/dir/stundenwerte_FF_00011_akt.zip': ( 14 | 'WindObservationsParser', '2020-03-29 08:55', 70523), 15 | '/dir/stundenwerte_FF_00090_akt.zip': ( 16 | 'WindObservationsParser', '2020-03-29 08:56', 71408), 17 | '/dir/stundenwerte_P0_00096_akt.zip': ( 18 | 'PressureObservationsParser', '2020-03-29 08:57', 47355), 19 | '/dir/stundenwerte_RR_00102_akt.zip': ( 20 | 'PrecipitationObservationsParser', '2020-03-29 08:58', 74372), 21 | '/dir/stundenwerte_SD_00125_akt.zip': ( 22 | 'SunshineObservationsParser', '2020-03-29 08:59', 69633), 23 | '/dir/stundenwerte_TD_00163_akt.zip': ( 24 | 'DewPointObservationsParser', '2020-03-29 09:02', 70167), 25 | '/dir/stundenwerte_TU_00161_akt.zip': ( 26 | 'TemperatureObservationsParser', '2020-03-29 09:00', 70165), 27 | '/dir/stundenwerte_VV_00164_akt.zip': ( 28 | 'VisibilityObservationsParser', '2020-03-29 09:03', 70168), 29 | '/dir/stundenwerte_N_00162_akt.zip': ( 30 | 'CloudCoverObservationsParser', '2020-03-29 09:01', 70166), 31 | '/dir/MOSMIX_L_LATEST.kmz': ( 32 | 'MOSMIXParser', '2020-03-29 09:57', 91636300), 33 | '/dir/MOSMIX_S_LATEST_240.kmz': ( 34 | 'MOSMIXParser', '2020-03-29 10:21', 38067304), 35 | '/dir/K611_-BEOB.csv': ( 36 | 'CurrentObservationsParser', '2020-04-06 10:38', 7343), 37 | '/dir/10minutenwerte_extrema_wind_01766_now.zip': ( 38 | 'WindGustsObservationsParser', '2020-06-08 09:12', 701), 39 | '/dir/10minutenwerte_extrema_wind_01766_akt.zip': ( 40 | 'WindGustsObservationsParser', '2020-06-08 04:27', 519692), 41 | '/dir/10minutenwerte_extrema_wind_01766_20100101_20191231_hist.zip': ( 42 | 'WindGustsObservationsParser', '2020-04-09 09:16', 3661729), 43 | '/dir/10minutenwerte_SOLAR_01766_akt.zip': ( 44 | 'SolarRadiationObservationsParser', '2023-04-12 00:50', 367557), 45 | } 46 | assert list(DWDPoller().parse('/dir/', resp_text)) == [ 47 | { 48 | 'url': k, 49 | 'parser': v[0], 50 | 'last_modified': datetime.datetime.strptime( 51 | v[1], '%Y-%m-%d %H:%M').replace(tzinfo=tzutc()), 52 | 'file_size': v[2], 53 | } 54 | for k, v in expected.items()] 55 | 56 | 57 | def test_dwdpoller_poll_ignores_parsed_files(db, data_dir): 58 | poller = DWDPoller() 59 | poller.urls = ['http://example.com/'] 60 | url = 'http://example.com/stundenwerte_FF_00011_akt.zip' 61 | with open(data_dir / 'dwd_opendata_index.html') as f: 62 | resp_text = f.read() 63 | config = {'get.return_value.text': resp_text} 64 | with patch('brightsky.polling.requests', **config): 65 | urls = [info['url'] for info in poller.poll()] 66 | with db.cursor() as cur: 67 | cur.execute( 68 | """ 69 | INSERT INTO parsed_files ( 70 | url, last_modified, file_size, parsed_at) 71 | VALUES (%s, %s, %s, current_timestamp) 72 | """, 73 | (url, '2020-03-29 08:55', 70523)) 74 | db.commit() 75 | new_urls = [info['url'] for info in poller.poll()] 76 | assert url in urls 77 | assert url not in new_urls 78 | assert len(new_urls) == len(urls) - 1 79 | -------------------------------------------------------------------------------- /tests/test_settings.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from dateutil.tz import tzutc 4 | 5 | from brightsky.settings import Settings 6 | 7 | from .utils import environ 8 | 9 | 10 | def test_settings_loads_environment(): 11 | with environ(BRIGHTSKY_TEST='value'): 12 | assert Settings().TEST == 'value' 13 | 14 | 15 | def test_settings_parses_environment_bool(): 16 | assert isinstance(Settings().KEEP_DOWNLOADS, bool) 17 | with environ(BRIGHTSKY_KEEP_DOWNLOADS='0'): 18 | assert Settings().KEEP_DOWNLOADS is False 19 | with environ(BRIGHTSKY_KEEP_DOWNLOADS='1'): 20 | assert Settings().KEEP_DOWNLOADS is True 21 | 22 | 23 | def test_settings_parses_environment_date(): 24 | expected = datetime.datetime(2000, 1, 2, tzinfo=tzutc()) 25 | assert isinstance(Settings().MIN_DATE, datetime.datetime) 26 | with environ(BRIGHTSKY_MIN_DATE='2000-01-02'): 27 | assert Settings().MIN_DATE == expected 28 | assert Settings().MAX_DATE is None 29 | with environ(BRIGHTSKY_MAX_DATE='2000-01-02'): 30 | assert Settings().MAX_DATE == expected 31 | 32 | 33 | def test_settings_parses_environment_float(): 34 | assert isinstance(Settings().ICON_RAIN_THRESHOLD, float) 35 | with environ(BRIGHTSKY_ICON_RAIN_THRESHOLD='0'): 36 | assert Settings().ICON_RAIN_THRESHOLD == float('0') 37 | with environ(BRIGHTSKY_ICON_RAIN_THRESHOLD='1.5'): 38 | assert Settings().ICON_RAIN_THRESHOLD == float('1.5') 39 | 40 | 41 | def test_settings_parses_environment_list(): 42 | assert isinstance(Settings().CORS_ALLOWED_ORIGINS, list) 43 | with environ(BRIGHTSKY_CORS_ALLOWED_ORIGINS=''): 44 | assert Settings().CORS_ALLOWED_ORIGINS == [] 45 | with environ(BRIGHTSKY_CORS_ALLOWED_ORIGINS='a'): 46 | assert Settings().CORS_ALLOWED_ORIGINS == ['a'] 47 | with environ(BRIGHTSKY_CORS_ALLOWED_ORIGINS='a,b,c'): 48 | assert Settings().CORS_ALLOWED_ORIGINS == ['a', 'b', 'c'] 49 | -------------------------------------------------------------------------------- /tests/test_tasks.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from dateutil.tz import tzutc 4 | 5 | from brightsky.export import DBExporter, SYNOPExporter 6 | from brightsky.tasks import clean 7 | 8 | 9 | def test_clean_deletes_expired_parsed_files(db): 10 | now = datetime.datetime.now(datetime.UTC).replace(tzinfo=tzutc()) 11 | fingerprints = [ 12 | { 13 | 'url': 'https://example.com/Z__C_EDZW_very_old.json', 14 | 'last_modified': now, 15 | 'file_size': 1234, 16 | 'parsed_at': now - datetime.timedelta(days=14), 17 | }, 18 | { 19 | 'url': 'https://example.com/Z__C_EDZW_recent.json', 20 | 'last_modified': now, 21 | 'file_size': 1234, 22 | 'parsed_at': now - datetime.timedelta(days=1), 23 | }, 24 | ] 25 | db.insert('parsed_files', fingerprints) 26 | assert len(db.table('parsed_files')) == 2 27 | clean() 28 | rows = db.table('parsed_files') 29 | assert len(rows) == 1 30 | assert rows[0]['url'].endswith('_recent.json') 31 | 32 | 33 | PLACE = { 34 | 'lat': 10, 35 | 'lon': 20, 36 | 'height': 30, 37 | 'dwd_station_id': '01766', 38 | 'wmo_station_id': '10315', 39 | 'station_name': 'Münster', 40 | } 41 | 42 | 43 | def test_clean_deletes_expired_forecast_current_synop_records(db): 44 | now = datetime.datetime.now(datetime.UTC).replace( 45 | minute=0, second=0, microsecond=0, tzinfo=tzutc()) 46 | records = [ 47 | { 48 | 'observation_type': 'forecast', 49 | 'timestamp': now, 50 | **PLACE, 51 | 'temperature': 10., 52 | }, 53 | { 54 | 'observation_type': 'forecast', 55 | 'timestamp': now - datetime.timedelta(hours=6), 56 | **PLACE, 57 | 'temperature': 20., 58 | }, 59 | { 60 | 'observation_type': 'current', 61 | 'timestamp': now, 62 | **PLACE, 63 | 'temperature': 30., 64 | }, 65 | { 66 | 'observation_type': 'current', 67 | 'timestamp': now - datetime.timedelta(hours=6), 68 | **PLACE, 69 | 'temperature': 40., 70 | }, 71 | { 72 | 'observation_type': 'current', 73 | 'timestamp': now - datetime.timedelta(days=3), 74 | **PLACE, 75 | 'temperature': 50., 76 | }, 77 | ] 78 | synop_records = [ 79 | { 80 | 'observation_type': 'synop', 81 | 'timestamp': now, 82 | **PLACE, 83 | 'temperature': 60., 84 | }, 85 | { 86 | 'observation_type': 'synop', 87 | 'timestamp': now - datetime.timedelta(hours=6), 88 | **PLACE, 89 | 'temperature': 70., 90 | }, 91 | { 92 | 'observation_type': 'synop', 93 | 'timestamp': now - datetime.timedelta(days=3), 94 | **PLACE, 95 | 'temperature': 80., 96 | }, 97 | ] 98 | DBExporter().export(records) 99 | SYNOPExporter().export(synop_records) 100 | assert len(db.table('weather')) == 5 101 | assert len(db.table('synop')) == 3 102 | clean() 103 | assert len(db.table('weather')) == 3 104 | assert len(db.table('synop')) == 2 105 | rows = db.fetch('SELECT temperature FROM weather ORDER BY temperature') 106 | assert [r['temperature'] for r in rows] == [10., 30., 40.] 107 | rows = db.fetch('SELECT temperature FROM synop ORDER BY temperature') 108 | assert [r['temperature'] for r in rows] == [60., 70.] 109 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from dateutil.tz import tzoffset, tzutc 3 | 4 | from brightsky.utils import daytime, parse_date, sunrise_sunset 5 | 6 | 7 | def test_parse_date(): 8 | assert parse_date('2020-08-18') == datetime.datetime(2020, 8, 18, 0, 0) 9 | assert parse_date('2020-08-18 12:34') == datetime.datetime( 10 | 2020, 8, 18, 12, 34) 11 | assert parse_date('2020-08-18T12:34:56+02:00') == datetime.datetime( 12 | 2020, 8, 18, 12, 34, 56, tzinfo=tzoffset(None, 7200)) 13 | assert parse_date('2020-08-18T12:34:56 02:00') == datetime.datetime( 14 | 2020, 8, 18, 12, 34, 56, tzinfo=tzoffset(None, 7200)) 15 | 16 | 17 | def test_sunrise_sunset(): 18 | sunrise, sunset = sunrise_sunset(52, 7.6, datetime.date(2020, 8, 18)) 19 | assert sunrise < sunset 20 | assert sunrise.utcoffset().total_seconds() == 0 21 | assert sunset.utcoffset().total_seconds() == 0 22 | 23 | 24 | def test_daytime(): 25 | midnight_0 = datetime.datetime(2023, 7, 10, 0, 0, tzinfo=tzutc()) 26 | noon_0 = datetime.datetime(2023, 7, 10, 12, 0, tzinfo=tzutc()) 27 | midnight_10 = datetime.datetime(2023, 7, 9, 14, 0, tzinfo=tzutc()) 28 | noon_10 = datetime.datetime(2023, 7, 10, 2, 0, tzinfo=tzutc()) 29 | # Muenster 30 | assert daytime(52, 7.6, midnight_0) == 'night' 31 | assert daytime(52, 7.6, noon_0) == 'day' 32 | # Sydney 33 | assert daytime(-33.8, 151, midnight_10) == 'night' 34 | assert daytime(-33.8, 151, noon_10) == 'day' 35 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from contextlib import contextmanager 3 | from functools import partial 4 | 5 | from brightsky.settings import settings as bs_settings 6 | 7 | 8 | def is_subset(subset_dict, full_dict): 9 | return subset_dict == {k: full_dict[k] for k in subset_dict} 10 | 11 | 12 | @contextmanager 13 | def dict_override(d, **overrides): 14 | original = {k: d[k] for k, v in overrides.items() if k in d} 15 | d.update(overrides) 16 | try: 17 | yield 18 | finally: 19 | for k in overrides: 20 | del d[k] 21 | d.update(original) 22 | 23 | 24 | settings = partial(dict_override, bs_settings) 25 | environ = partial(dict_override, os.environ) 26 | --------------------------------------------------------------------------------