├── data └── .keep ├── Dockerfile ├── grib-download.timer ├── grib-download.service ├── app ├── bounds.json ├── bounds.py ├── cron.py └── convert.py ├── .gitignore ├── LICENSE ├── README.md └── harmonie_bounds.geojson /data/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mennotammens/grib-tools:latest 2 | 3 | COPY app /app 4 | WORKDIR /app 5 | 6 | CMD ["/app/cron.py"] 7 | -------------------------------------------------------------------------------- /grib-download.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Harmonie GRIB download and convert service timer 3 | 4 | [Timer] 5 | OnActiveSec=1min 6 | OnUnitActiveSec=5min 7 | 8 | [Install] 9 | WantedBy=timers.target 10 | -------------------------------------------------------------------------------- /grib-download.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Harmonie GRIB download and convert service 3 | After=network.target syslog.target 4 | 5 | [Service] 6 | # see man systemd.service 7 | Type=oneshot 8 | ExecStart=/usr/bin/docker run --rm --name %n -e KNMI_API_KEY=1234567890abcdef --mount type=bind,source=/srv/data/harmonie/,target=/data dzeuros/grib-download:latest 9 | StandardOutput=journal 10 | -------------------------------------------------------------------------------- /app/bounds.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"abbr": "ijmg", "name": "IJsselmeer", "bounds": [[4.107, 52.082], [5.994, 53.416]]}, 3 | {"abbr": "wad", "name": "Waddenzee", "bounds": [[4.218, 52.795], [7.548, 53.876]]}, 4 | {"abbr": "zld", "name": "Zeeland", "bounds": [[2.886, 51.047], [4.773, 52.197]]}, 5 | {"abbr": "nl", "name": "Nederland", "bounds": [[2.886, 50.725], [7.326, 53.876]]}, 6 | {"abbr": "nz", "name": "Noordzee", "bounds": [[0.37, 50.725], [4.81, 53.876]]}, 7 | {"abbr": "db", "name": "Duitse Bocht", "bounds": [[4.292, 52.979], [9.657, 55.601]]}, 8 | {"abbr": "dov", "name": "Dover", "bounds": [[0.0, 49.322], [3.33, 51.392]]} 9 | ] 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # harmonie-grib specifics 2 | data/* 3 | tmp/* 4 | ./ggrib 5 | *.geojson 6 | *.bz2 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | env/ 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *,cover 53 | .hypothesis/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | 62 | # Sphinx documentation 63 | docs/_build/ 64 | 65 | # PyBuilder 66 | target/ 67 | 68 | #Ipython Notebook 69 | .ipynb_checkpoints 70 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Menno Tammens 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /app/bounds.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | 5 | from convert import AREAS 6 | 7 | 8 | def geojson_feature(feature_type, coordinates, properties=None, **kwargs): 9 | properties = properties or {} 10 | properties.update(kwargs) 11 | return { 12 | 'type': 'Feature', 13 | 'properties': properties, 14 | 'geometry': { 15 | 'type': feature_type, 16 | 'coordinates': coordinates 17 | } 18 | } 19 | 20 | 21 | def geojson_bounds(bounds, properties=None, **kwargs): 22 | '''Convert a tuple of (sw, ne)-corners to a geojson polygon''' 23 | 24 | sw_lng, sw_lat = bounds[0] 25 | ne_lng, ne_lat = bounds[1] 26 | return geojson_feature('Polygon', [ 27 | [ 28 | bounds[0], 29 | (sw_lng, ne_lat), 30 | bounds[1], 31 | (ne_lng, sw_lat) 32 | ] 33 | ], properties, **kwargs) 34 | 35 | if __name__ == '__main__': 36 | all_bounds = [] 37 | for area in AREAS: 38 | bounds = geojson_bounds(area['bounds'], name=area['name']) 39 | all_bounds.append(bounds) 40 | 41 | with open('{}.geojson'.format(area['abbr']), 'w') as out: 42 | json.dump(bounds, out, indent=2) 43 | 44 | with open('harmonie_bounds.geojson', 'w') as out: 45 | json.dump({ 46 | 'type': 'FeatureCollection', 47 | 'features': all_bounds 48 | }, out, indent=2) 49 | 50 | 51 | print('Wrote bounds to .geojson files') 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Conversie GRIB-files HARMONIE-model 2 | 3 | Het KNMI stelt in het [KNMI Data Platform](https://dataplatform.knmi.nl/catalog/index.html) GRIB-files van het Harmonie-model vrij beschikbaar als [Open Data](http://creativecommons.org/publicdomain/mark/1.0/deed.nl). Het HARMONIE-AROME Cy40-model levert data tot 48 uur vooruit met een hoge resolutie (2,5×2,5km) en in tijdstappen van 1 uur. 4 | De GRIB-files van het KNMI zijn nog niet direct in GRIB-viewers als [XyGrib](https://opengribs.org/en/xygrib) te openen omdat het formaat een beetje afwijkt van wat de GRIB-viewers verwachten. Ik heb daarom de GRIB-files geconverteerd in een formaat dat wel te gebruiken is in XyGrib en diverse andere GRIB-viewers 5 | 6 | Voor meer informatie en discussie, zie dit topic op Zeilersforum: https://zeilersforum.nl/index.php/forum-125/meteo/575942-hoge-resolutie-grib-files 7 | 8 | ## Usage 9 | - From Docker Hub: 10 | 11 | ``` 12 | mkdir -p data 13 | docker run --rm --name grib-download -e KNMI_API_KEY=1234567890abcdef -e METEOGRAM_LAT=51.2345 -e METEOGRAM_LON=6.1234 --mount type=bind,source=$(pwd)/data,target=/data --user $(id -u):$(id -g) mennotammens/harmonie-grib:latest 14 | ``` 15 | 16 | - From GitHub: 17 | 18 | ``` 19 | git clone git@github.com:MennoTammens/harmonie-grib.git 20 | cd harmonie-grib 21 | docker build -t harmonie-grib:latest . 22 | docker run --rm --name grib-download -e KNMI_API_KEY=1234567890abcdef -e METEOGRAM_LAT=51.2345 -e METEOGRAM_LON=6.1234 --mount type=bind,source=$(pwd)/data,target=/data --user $(id -u):$(id -g) harmonie-grib:latest 23 | ``` 24 | -------------------------------------------------------------------------------- /app/cron.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import json 5 | from urllib.request import urlopen, Request 6 | from datetime import datetime, timedelta 7 | import tarfile 8 | import tempfile 9 | from pathlib import Path 10 | import convert 11 | 12 | 13 | DATA_DIR = Path('/data') 14 | TMP_DIR = DATA_DIR / 'tmp' 15 | 16 | API_URL = "https://api.dataplatform.knmi.nl/open-data" 17 | API_KEY = os.getenv('KNMI_API_KEY') 18 | DATASET_NAME = "harmonie_arome_cy43_p1" 19 | DATASET_VERSION = "1.0" 20 | 21 | 22 | def file_list(): 23 | yesterday = (datetime.now() - timedelta(hours=8)).strftime("%Y%m%d%H") 24 | req = Request( 25 | f"{API_URL}/datasets/{DATASET_NAME}/versions/{DATASET_VERSION}/files?startAfterFilename=HARM43_V1_P1_{yesterday}.tar", 26 | headers={"Authorization": API_KEY} 27 | ) 28 | with urlopen(req) as list_files_response: 29 | files = json.load(list_files_response).get("files") 30 | 31 | return files 32 | 33 | 34 | def get_file(filename, tmpdirname): 35 | req = Request( 36 | f"{API_URL}/datasets/{DATASET_NAME}/versions/{DATASET_VERSION}/files/{filename}/url", 37 | headers={"Authorization": API_KEY} 38 | ) 39 | with urlopen(req) as get_file_response: 40 | url = json.load(get_file_response).get("temporaryDownloadUrl") 41 | 42 | print(f'downloading {filename}') 43 | with urlopen(url) as remote: 44 | with tarfile.open(fileobj=remote, mode='r|*') as tar: 45 | for tarinfo in tar: 46 | print(tarinfo.name, flush=True) 47 | tar.extract(tarinfo, tmpdirname) 48 | print('downloaded '+filename, flush=True) 49 | 50 | 51 | def cron(): 52 | latest = file_list()[-1] 53 | filename = latest["filename"] 54 | run_time = filename[-14:-4] 55 | run_time_date = datetime.strptime(run_time, '%Y%m%d%H').strftime('%Y-%m-%d_%H') 56 | 57 | if (DATA_DIR / run_time_date).exists() or (TMP_DIR / f'HA40_N25_{run_time}00_00000_GB').exists(): 58 | print(f"Skipping download, {filename} already downloaded") 59 | else: 60 | with tempfile.TemporaryDirectory() as tmpdirname: 61 | get_file(filename, tmpdirname) 62 | convert.convert(tmpdirname) 63 | print('DONE') 64 | 65 | 66 | if __name__ == "__main__": 67 | cron() 68 | -------------------------------------------------------------------------------- /harmonie_bounds.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "geometry": { 6 | "type": "Polygon", 7 | "coordinates": [ 8 | [ 9 | [ 10 | 4.107, 11 | 52.082 12 | ], 13 | [ 14 | 4.107, 15 | 53.416 16 | ], 17 | [ 18 | 5.994, 19 | 53.416 20 | ], 21 | [ 22 | 5.994, 23 | 52.082 24 | ] 25 | ] 26 | ] 27 | }, 28 | "type": "Feature", 29 | "properties": { 30 | "name": "IJsselmeer" 31 | } 32 | }, 33 | { 34 | "geometry": { 35 | "type": "Polygon", 36 | "coordinates": [ 37 | [ 38 | [ 39 | 4.218, 40 | 52.795 41 | ], 42 | [ 43 | 4.218, 44 | 53.876 45 | ], 46 | [ 47 | 7.548, 48 | 53.876 49 | ], 50 | [ 51 | 7.548, 52 | 52.795 53 | ] 54 | ] 55 | ] 56 | }, 57 | "type": "Feature", 58 | "properties": { 59 | "name": "Waddenzee" 60 | } 61 | }, 62 | { 63 | "geometry": { 64 | "type": "Polygon", 65 | "coordinates": [ 66 | [ 67 | [ 68 | 2.886, 69 | 51.047 70 | ], 71 | [ 72 | 2.886, 73 | 52.197 74 | ], 75 | [ 76 | 4.773, 77 | 52.197 78 | ], 79 | [ 80 | 4.773, 81 | 51.047 82 | ] 83 | ] 84 | ] 85 | }, 86 | "type": "Feature", 87 | "properties": { 88 | "name": "Zeeland" 89 | } 90 | }, 91 | { 92 | "geometry": { 93 | "type": "Polygon", 94 | "coordinates": [ 95 | [ 96 | [ 97 | 2.886, 98 | 50.725 99 | ], 100 | [ 101 | 2.886, 102 | 53.876 103 | ], 104 | [ 105 | 7.326, 106 | 53.876 107 | ], 108 | [ 109 | 7.326, 110 | 50.725 111 | ] 112 | ] 113 | ] 114 | }, 115 | "type": "Feature", 116 | "properties": { 117 | "name": "Nederland" 118 | } 119 | }, 120 | { 121 | "geometry": { 122 | "type": "Polygon", 123 | "coordinates": [ 124 | [ 125 | [ 126 | 0.37, 127 | 50.725 128 | ], 129 | [ 130 | 0.37, 131 | 53.876 132 | ], 133 | [ 134 | 4.81, 135 | 53.876 136 | ], 137 | [ 138 | 4.81, 139 | 50.725 140 | ] 141 | ] 142 | ] 143 | }, 144 | "type": "Feature", 145 | "properties": { 146 | "name": "Noordzee" 147 | } 148 | }, 149 | { 150 | "geometry": { 151 | "type": "Polygon", 152 | "coordinates": [ 153 | [ 154 | [ 155 | 4.292, 156 | 52.979 157 | ], 158 | [ 159 | 4.292, 160 | 55.601 161 | ], 162 | [ 163 | 9.657, 164 | 55.601 165 | ], 166 | [ 167 | 9.657, 168 | 52.979 169 | ] 170 | ] 171 | ] 172 | }, 173 | "type": "Feature", 174 | "properties": { 175 | "name": "Duitse Bocht" 176 | } 177 | }, 178 | { 179 | "geometry": { 180 | "type": "Polygon", 181 | "coordinates": [ 182 | [ 183 | [ 184 | 0.0, 185 | 49.322 186 | ], 187 | [ 188 | 0.0, 189 | 51.392 190 | ], 191 | [ 192 | 3.33, 193 | 51.392 194 | ], 195 | [ 196 | 3.33, 197 | 49.322 198 | ] 199 | ] 200 | ] 201 | }, 202 | "type": "Feature", 203 | "properties": { 204 | "name": "Dover" 205 | } 206 | } 207 | ] 208 | } -------------------------------------------------------------------------------- /app/convert.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import shutil 5 | import subprocess 6 | import sys 7 | from datetime import datetime 8 | import json 9 | from pathlib import Path 10 | 11 | from numpy import sqrt 12 | 13 | import pygrib 14 | 15 | DATA_DIR = Path('/data') 16 | TMP_DIR = DATA_DIR / 'tmp' 17 | 18 | # bounds to create sliced versions for: 19 | AREAS = json.load(open('bounds.json')) 20 | 21 | # discover ggrib location 22 | GGRIB_BIN = None 23 | for location in ('/usr/local/bin/ggrib', './ggrib'): 24 | if os.path.isfile(location) and os.access(location, os.X_OK): 25 | GGRIB_BIN = location 26 | 27 | 28 | def convert(tmpdirname): 29 | tmp_dir = Path(tmpdirname) 30 | files = sorted(tmp_dir.glob('**/*_GB')) 31 | 32 | if len(files) != 49: 33 | print(f'Verkeerd aantal GRIB-bestanden in {tmp_dir}, exiting') 34 | #sys.exit(1) 35 | 36 | tmp_grib = tmp_dir / 'temp.grb' 37 | tmp_grib_wind = tmp_dir / 'temp_wind.grb' 38 | 39 | with tmp_grib.open('wb') as gribout, tmp_grib_wind.open('wb') as gribout_wind: 40 | def writeGribMessage(message, wind=False): 41 | message['generatingProcessIdentifier'] = 96 42 | message['centre'] = 'kwbc' 43 | message['subCentre'] = 0 44 | message['table2Version'] = 1 45 | 46 | gribout.write(message.tostring()) 47 | if wind: 48 | gribout_wind.write(message.tostring()) 49 | 50 | for filename in files: 51 | with pygrib.open(str(filename)) as grbs: 52 | # Mean sea level pressure 53 | msg_mslp = grbs.select(indicatorOfParameter=1, typeOfLevel='heightAboveSea')[0] 54 | msg_mslp.indicatorOfParameter = 2 55 | msg_mslp.indicatorOfTypeOfLevel = 'sfc' 56 | msg_mslp.typeOfLevel = 'meanSea' 57 | writeGribMessage(msg_mslp) 58 | 59 | # Relative humidity 60 | msg_rh = grbs.select(indicatorOfParameter=52, level=2)[0] 61 | msg_rh.values = msg_rh.values * 100 62 | writeGribMessage(msg_rh) 63 | 64 | # Temperature 2m 65 | msg_t = grbs.select(indicatorOfParameter=11, level=2)[0] 66 | writeGribMessage(msg_t) 67 | 68 | # U-wind 69 | msg_u = grbs.select(indicatorOfParameter=33, level=10)[0] 70 | writeGribMessage(msg_u, wind=True) 71 | 72 | # V-wind 73 | msg_v = grbs.select(indicatorOfParameter=34, level=10)[0] 74 | writeGribMessage(msg_v, wind=True) 75 | 76 | # Precipication Intensity 77 | msg_ip = grbs.select(indicatorOfParameter=181, level=0, stepType='instant')[0] 78 | msg_ip.indicatorOfParameter = 61 79 | msg_ip.typeOfLevel = 'surface' 80 | #msg_ip.level = 0 81 | msg_ip.values = msg_ip.values * 3600 # mm/s => mm/h 82 | if msg_ip['P2'] > 0: 83 | msg_ip['P1'] = msg_ip['P2'] - 1 84 | writeGribMessage(msg_ip) 85 | 86 | # Wind gusts 87 | msg_ug = grbs.select(indicatorOfParameter=162, level=10)[0] 88 | msg_vg = grbs.select(indicatorOfParameter=163, level=10)[0] 89 | msg_ug.values = sqrt(msg_ug.values ** 2 + msg_vg.values ** 2) 90 | msg_ug.indicatorOfParameter = 180 91 | msg_ug.typeOfLevel = 'surface' 92 | msg_ug.level = 0 93 | if msg_ug['P2'] > 0: 94 | msg_ug['P1'] = msg_ug['P2'] - 1 95 | writeGribMessage(msg_ug, wind=True) 96 | 97 | #os.remove(filename) 98 | 99 | 100 | run_time = str(files[0])[-21:-11] 101 | run_time_date = datetime.strptime(run_time, '%Y%m%d%H').strftime('%Y-%m-%d_%H') 102 | print(run_time_date) 103 | 104 | dst_dir = DATA_DIR / f'{run_time_date}' 105 | dst_dir.mkdir(exist_ok=True) 106 | 107 | filename_fmt = f'harmonie_xy_{run_time_date}_{{}}.grb' 108 | 109 | def bounded_slice(src, dst_dir, name, bounds): 110 | dst = dst_dir / filename_fmt.format(name) 111 | print(f'Writing {name} to {dst}') 112 | 113 | cmd = [GGRIB_BIN, src, dst] 114 | cmd.extend(map(str, bounds[0] + bounds[1])) 115 | subprocess.call(cmd) 116 | 117 | if GGRIB_BIN is None: 118 | print('ggrib binary not found, please install by typing `make ggrib`') 119 | else: 120 | for area in AREAS: 121 | bounded_slice(tmp_grib, dst_dir, area['abbr'], area['bounds']) 122 | bounded_slice(tmp_grib_wind, dst_dir, area['abbr'] + '_wind', area['bounds']) 123 | 124 | shutil.move(tmp_grib, dst_dir / f'harmonie_xy_{run_time_date}.grb') 125 | shutil.move(tmp_grib_wind, dst_dir / filename_fmt.format('wind')) 126 | 127 | (DATA_DIR / 'new').symlink_to(dst_dir.relative_to(DATA_DIR)) 128 | (DATA_DIR / 'new').rename(DATA_DIR / 'latest') 129 | 130 | # delete old files 131 | for dir in DATA_DIR.glob('*'): 132 | if dir not in (dst_dir, DATA_DIR / 'latest'): 133 | print(f'removing {dir}') 134 | shutil.rmtree(dir) 135 | 136 | if __name__ == "__main__": 137 | convert(TMP_DIR) 138 | --------------------------------------------------------------------------------