├── .gitignore ├── assimilate ├── app │ ├── data │ │ ├── __init__.py │ │ ├── json.py │ │ ├── hdf5.py │ │ └── jsonapi.py │ └── models │ │ ├── __init__.py │ │ ├── spline.py │ │ ├── combinators.py │ │ └── gp3d.py ├── requirements.txt └── Dockerfile ├── diffusion ├── app │ ├── data │ │ ├── __init__.py │ │ ├── json.py │ │ ├── hdf5.py │ │ └── jsonapi.py │ ├── util.py │ ├── train_guidance.py │ ├── train_vae.py │ ├── train_diffusion.py │ ├── train_conditioned_diffusion.py │ ├── train_vprediction.py │ ├── generate.py │ ├── guidance_test.py │ ├── movie.py │ ├── movie_cfg.py │ ├── generate_guided.py │ └── generate_guided_cfg.py ├── .containerignore ├── .gitignore ├── run ├── pyproject.toml ├── Dockerfile └── CRUSH.md ├── iturhfprop ├── .gitignore ├── cpanfile ├── app │ └── templates │ │ ├── footer.html.ep │ │ ├── worldmap.ep │ │ ├── p2p.ep │ │ ├── header.html.ep │ │ └── planner_table.html.ep └── Dockerfile ├── api ├── app │ ├── prestart.sh │ └── static │ │ └── index.html ├── requirements.txt └── Dockerfile ├── storm ├── requirements.txt ├── Dockerfile └── app │ └── main.py ├── utils └── migrate │ ├── cpanfile │ ├── Dockerfile │ └── app │ └── migrate.pl ├── etc ├── postgres.env.in ├── db.env.in └── ports.env ├── www ├── favicon.ico ├── static │ ├── kc2g.xcf │ ├── kc2g-16.png │ ├── kc2g-32.png │ ├── kc2g-64.png │ ├── giro-logo.png │ ├── kc2g-128.png │ ├── noaa-logo.jpg │ ├── osi-logo.png │ ├── wwrof-logo.jpg │ ├── serialize.js │ ├── promise.min.js │ └── style.css ├── footer.html ├── fof2 │ └── index.html ├── index.html ├── nav-header.html ├── stations │ └── index.html ├── ptp │ └── index.html └── acknowledgments │ └── index.html ├── iri2020 ├── pyproject.toml ├── src │ ├── iridreg_old.for │ ├── meson.build │ ├── iri_opt.F90 │ ├── reference │ │ ├── irisubg.pyf │ │ ├── irisubg.for │ │ ├── iriweb.for │ │ └── iriwebg.pyf │ ├── test.f90 │ ├── iri_ts.F90 │ ├── irimap.F90 │ ├── irtam_driver.F90 │ └── iri2020_driver.F90 ├── Dockerfile ├── .gitattributes ├── .gitignore ├── data │ └── igrf │ │ ├── dgrf1960.dat │ │ ├── dgrf1985.dat │ │ ├── dgrf1990.dat │ │ ├── dgrf1955.dat │ │ ├── dgrf1965.dat │ │ ├── dgrf1970.dat │ │ ├── dgrf1975.dat │ │ ├── dgrf1980.dat │ │ ├── dgrf1995.dat │ │ ├── dgrf1945.dat │ │ ├── dgrf1950.dat │ │ ├── igrf2020s.dat │ │ ├── dgrf2000.dat │ │ ├── igrf2020.dat │ │ ├── dgrf2015.dat │ │ ├── dgrf2005.dat │ │ └── dgrf2010.dat ├── LICENSE └── README.md ├── pred ├── requirements.txt ├── app │ ├── fit_cs.sh │ ├── cs.py │ ├── fit_cs.py │ ├── pred.py │ └── pred_opt.py └── Dockerfile ├── essn ├── requirements.txt └── Dockerfile ├── raytrace ├── requirements.txt ├── app │ ├── raytrace │ │ ├── constants.py │ │ └── __init__.py │ └── iono.py └── Dockerfile ├── .gitmodules ├── holdout-eval ├── requirements.txt └── Dockerfile ├── irimap ├── requirements.txt ├── Dockerfile └── irimap.py ├── loader ├── noaa-loader │ ├── listen │ └── process ├── cpanfile ├── lib │ └── Data │ │ └── SAO.pm ├── Dockerfile ├── giro-loader │ └── process ├── aus-loader │ ├── process │ └── urls └── app │ ├── debug.pl │ └── load.pl ├── ipe ├── requirements.txt └── Dockerfile ├── systemd ├── fetch-chain.timer ├── prop-cosmic.timer ├── backfill-irtam.timer ├── giro-loader.timer ├── prop-storm.timer ├── fetch-chain.service ├── prop-podman-pod.service ├── prop-storm.service ├── prop-memcached.service ├── prop-history.service ├── prop-renderer.service ├── prop-raytrace.service ├── prop-pred.service ├── prop-cosmic.service ├── prop-assimilate.service ├── irimap.service ├── prop-postgres.service ├── noaa-loader.service ├── backfill-irtam.service ├── prop-essn.service ├── prop-holdout-eval.service ├── giro-loader.service ├── prop-ipe.service ├── prop-scheduler.service ├── prop-api.service ├── prop-worker-gcs.service ├── prop-worker-highcpu.service └── prop-worker-default.service ├── .gitattributes ├── cosmic ├── requirements.txt ├── Dockerfile └── backfill_modip_mp.py ├── history ├── cpanfile └── Dockerfile └── scheduler ├── cpanfile ├── Dockerfile └── app └── lib ├── Task ├── Pred.pm ├── Assimilate.pm ├── BandQuality.pm ├── HoldoutEvaluate.pm ├── IRIMap.pm ├── eSSN.pm ├── IPE.pm └── Render.pm └── Minion └── Statsd.pm /.gitignore: -------------------------------------------------------------------------------- 1 | pred/*.txt 2 | -------------------------------------------------------------------------------- /assimilate/app/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assimilate/app/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /diffusion/app/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /iturhfprop/.gitignore: -------------------------------------------------------------------------------- 1 | antennas 2 | -------------------------------------------------------------------------------- /api/app/prestart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | true 3 | -------------------------------------------------------------------------------- /storm/requirements.txt: -------------------------------------------------------------------------------- 1 | scipy==1.10.0 2 | -------------------------------------------------------------------------------- /utils/migrate/cpanfile: -------------------------------------------------------------------------------- 1 | requires 'DBD::Pg'; 2 | requires 'DBI'; 3 | -------------------------------------------------------------------------------- /diffusion/.containerignore: -------------------------------------------------------------------------------- 1 | checkpoints 2 | logs 3 | out 4 | tc_cache 5 | -------------------------------------------------------------------------------- /etc/postgres.env.in: -------------------------------------------------------------------------------- 1 | POSTGRES_USER=prop 2 | POSTGRES_PASSWORD=%PASSWORD% 3 | -------------------------------------------------------------------------------- /www/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arodland/prop/HEAD/www/favicon.ico -------------------------------------------------------------------------------- /www/static/kc2g.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arodland/prop/HEAD/www/static/kc2g.xcf -------------------------------------------------------------------------------- /etc/db.env.in: -------------------------------------------------------------------------------- 1 | DB_HOST=pg 2 | DB_NAME=prop 3 | DB_USER=prop 4 | DB_PASSWORD=%PASSWORD% 5 | -------------------------------------------------------------------------------- /iri2020/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel", "numpy"] 3 | -------------------------------------------------------------------------------- /pred/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg[binary]==3.1.6 2 | Flask==2.2.5 3 | pyyaml 4 | pybind11 5 | -------------------------------------------------------------------------------- /www/static/kc2g-16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arodland/prop/HEAD/www/static/kc2g-16.png -------------------------------------------------------------------------------- /www/static/kc2g-32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arodland/prop/HEAD/www/static/kc2g-32.png -------------------------------------------------------------------------------- /www/static/kc2g-64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arodland/prop/HEAD/www/static/kc2g-64.png -------------------------------------------------------------------------------- /www/static/giro-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arodland/prop/HEAD/www/static/giro-logo.png -------------------------------------------------------------------------------- /www/static/kc2g-128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arodland/prop/HEAD/www/static/kc2g-128.png -------------------------------------------------------------------------------- /www/static/noaa-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arodland/prop/HEAD/www/static/noaa-logo.jpg -------------------------------------------------------------------------------- /www/static/osi-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arodland/prop/HEAD/www/static/osi-logo.png -------------------------------------------------------------------------------- /iri2020/src/iridreg_old.for: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arodland/prop/HEAD/iri2020/src/iridreg_old.for -------------------------------------------------------------------------------- /www/static/wwrof-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arodland/prop/HEAD/www/static/wwrof-logo.jpg -------------------------------------------------------------------------------- /diffusion/.gitignore: -------------------------------------------------------------------------------- 1 | /checkpoints 2 | /logs 3 | /out 4 | /tc_cache 5 | .crush 6 | app/taesd-iono-finetuned 7 | -------------------------------------------------------------------------------- /essn/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg[binary]==3.1.6 2 | scipy==1.10.0 3 | Flask==2.2.5 4 | uwsgi==2.0.22 5 | ppigrf==2.0.0 6 | -------------------------------------------------------------------------------- /raytrace/requirements.txt: -------------------------------------------------------------------------------- 1 | h5py==3.7.0 2 | hdf5plugin==4.1.1 3 | cython 4 | Flask==2.2.5 5 | numba==0.56.4 6 | uwsgi==2.0.22 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "iturhfprop/ITU-R-HF"] 2 | path = iturhfprop/ITU-R-HF 3 | url = https://github.com/arodland/ITU-R-HF 4 | -------------------------------------------------------------------------------- /assimilate/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg[binary]==3.1.6 2 | h5py==3.7.0 3 | hdf5plugin==4.1.1 4 | Flask==2.2.5 5 | wquantiles==0.6 6 | -------------------------------------------------------------------------------- /holdout-eval/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg[binary]==3.1.6 2 | Flask==2.2.5 3 | uwsgi==2.0.22 4 | h5py==3.7.0 5 | hdf5plugin==4.1.1 6 | -------------------------------------------------------------------------------- /irimap/requirements.txt: -------------------------------------------------------------------------------- 1 | wmm2020==1.0.0 2 | psycopg[binary]==3.1.6 3 | h5py==3.7.0 4 | hdf5plugin==4.1.1 5 | Flask==2.2.5 6 | uwsgi==2.0.22 7 | -------------------------------------------------------------------------------- /loader/noaa-loader/listen: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | /usr/bin/inoticoming --foreground --initialsearch /noaa-data/in --suffix .zip /noaa-loader/process \; 3 | -------------------------------------------------------------------------------- /ipe/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==2.2.5 2 | psycopg[binary]==3.1.6 3 | uwsgi==2.0.22 4 | scipy==1.10.0 5 | h5py==3.7.0 6 | hdf5plugin==4.1.1 7 | netCDF4==1.6.3 8 | -------------------------------------------------------------------------------- /pred/app/fit_cs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | count="$1" 3 | for x in `seq 1 "$count"` ; do 4 | echo "$x" 1>&2 5 | python /app/fit_cs.py 6 | sleep 0.1 7 | done 8 | 9 | -------------------------------------------------------------------------------- /systemd/fetch-chain.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=fetch CHAIN data 3 | 4 | [Timer] 5 | OnCalendar=*:21:00 UTC 6 | Persistent=true 7 | 8 | [Install] 9 | WantedBy=kc2gprop.target 10 | -------------------------------------------------------------------------------- /systemd/prop-cosmic.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=fetch COSMIC data 3 | 4 | [Timer] 5 | OnCalendar=6:47:00 UTC 6 | Persistent=true 7 | 8 | [Install] 9 | WantedBy=kc2gprop.target 10 | -------------------------------------------------------------------------------- /systemd/backfill-irtam.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Backfill IRTAM 3 | 4 | [Timer] 5 | OnCalendar=*-*-* *:3/15:00 6 | Persistent=true 7 | 8 | [Install] 9 | WantedBy=kc2gprop.target 10 | -------------------------------------------------------------------------------- /systemd/giro-loader.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=giro-loader every 5m 3 | 4 | [Timer] 5 | OnCalendar=*-*-* *:3/5:00 6 | Persistent=true 7 | 8 | [Install] 9 | WantedBy=kc2gprop.target 10 | -------------------------------------------------------------------------------- /assimilate/app/data/json.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import json 3 | 4 | def get_data(url): 5 | with urllib.request.urlopen(url) as res: 6 | return json.loads(res.read().decode()) 7 | 8 | -------------------------------------------------------------------------------- /diffusion/app/data/json.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import json 3 | 4 | def get_data(url): 5 | with urllib.request.urlopen(url) as res: 6 | return json.loads(res.read().decode()) 7 | 8 | -------------------------------------------------------------------------------- /loader/cpanfile: -------------------------------------------------------------------------------- 1 | requires 'DBD::Pg'; 2 | requires 'DBI'; 3 | requires 'Fortran::Format'; 4 | requires 'Moo'; 5 | requires 'Net::Statsd::Client'; 6 | requires 'Time::Piece'; 7 | requires 'XML::Twig'; 8 | -------------------------------------------------------------------------------- /systemd/prop-storm.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=prop-storm every 3 hours 3 | 4 | [Timer] 5 | OnCalendar=0/3:07:00 UTC 6 | Persistent=true 7 | 8 | [Install] 9 | WantedBy=kc2gprop.target 10 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.pl linguist-language=Perl 2 | *.pm linguist-language=Perl 3 | *.t linguist-language=Perl 4 | *.f90 linguist-language=Fortran 5 | *.F90 linguist-language=Fortran 6 | *.for linguist-language=Fortran 7 | -------------------------------------------------------------------------------- /etc/ports.env: -------------------------------------------------------------------------------- 1 | SCHEDULER_PORT=5500 2 | API_PORT=5501 3 | HISTORY_PORT=5502 4 | ESSN_PORT=5503 5 | IRIMAP_PORT=5504 6 | PRED_PORT=5505 7 | ASSIMILATE_PORT=5506 8 | RENDERER_PORT=5507 9 | RAYTRACE_PORT=5508 10 | -------------------------------------------------------------------------------- /cosmic/requirements.txt: -------------------------------------------------------------------------------- 1 | psycopg[binary]==3.1.6 2 | netCDF4==1.6.3 3 | h5py==3.7.0 4 | hdf5plugin==4.1.1 5 | scipy==1.10.0 6 | ppigrf==2.0.0 7 | PyGeodesy==23.4.23 8 | requests==2.31.0 9 | retry-requests==2.0.0 10 | -------------------------------------------------------------------------------- /history/cpanfile: -------------------------------------------------------------------------------- 1 | requires 'DBI'; 2 | requires 'DBD::Pg'; 3 | requires 'JSON::MaybeXS'; 4 | requires 'DateTime'; 5 | requires 'DateTime::Format::Strptime'; 6 | requires 'Web::Simple'; 7 | requires 'Plack'; 8 | requires 'Starman'; 9 | -------------------------------------------------------------------------------- /storm/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | RUN apt-get update && apt-get install -y dumb-init 3 | 4 | COPY requirements.txt /tmp/ 5 | RUN pip install -r /tmp/requirements.txt 6 | 7 | COPY ./app /app 8 | 9 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 10 | CMD python3 /app/main.py 11 | -------------------------------------------------------------------------------- /assimilate/app/data/hdf5.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import io 3 | 4 | import h5py 5 | 6 | def get_data(url): 7 | with urllib.request.urlopen(url) as res: 8 | content = res.read() 9 | bio = io.BytesIO(content) 10 | h5 = h5py.File(bio, 'r') 11 | return h5 12 | -------------------------------------------------------------------------------- /diffusion/app/data/hdf5.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import io 3 | 4 | import h5py 5 | 6 | def get_data(url): 7 | with urllib.request.urlopen(url) as res: 8 | content = res.read() 9 | bio = io.BytesIO(content) 10 | h5 = h5py.File(bio, 'r') 11 | return h5 12 | -------------------------------------------------------------------------------- /systemd/fetch-chain.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Fetch CHAIN data 3 | [Service] 4 | Type=oneshot 5 | User=prop 6 | Group=prop 7 | ExecStart=/usr/bin/wget -q -P /home/prop/iri-index/chain -N https://chain-new.chain-project.net/echaim_downloads/apf107.dat https://chain-new.chain-project.net/echaim_downloads/ig_rz.dat 8 | -------------------------------------------------------------------------------- /loader/lib/Data/SAO.pm: -------------------------------------------------------------------------------- 1 | package Data::SAO; 2 | 3 | use Data::SAO4; 4 | use Data::SAOXML; 5 | 6 | sub new { 7 | my ($class, %args) = @_; 8 | if ($args{filename} =~ /\.xml$/i) { 9 | return Data::SAOXML->new(%args); 10 | } else { 11 | return Data::SAO4->new(%args); 12 | } 13 | } 14 | 15 | 1; 16 | -------------------------------------------------------------------------------- /scheduler/cpanfile: -------------------------------------------------------------------------------- 1 | requires 'Mojolicious::Lite'; 2 | requires 'Minion' => '10.28'; 3 | requires 'Minion::Notifier'; 4 | requires 'Net::Statsd::Client'; 5 | requires 'Mojo::Pg'; 6 | requires 'Path::Tiny'; 7 | requires 'WWW::Google::Cloud::Auth::ServiceAccount'; 8 | requires 'LWP::Protocol::https'; 9 | requires 'DateTime'; 10 | -------------------------------------------------------------------------------- /www/footer.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | -------------------------------------------------------------------------------- /utils/migrate/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM perl:5 AS perl-cpm 2 | RUN cpanm --notest App::cpm 3 | 4 | FROM perl-cpm AS final 5 | # RUN apt-get update && apt-get install dumb-init 6 | 7 | COPY cpanfile . 8 | RUN cpm install -g 9 | 10 | COPY ./app /app 11 | COPY ./lib /perl5lib 12 | 13 | # ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 14 | CMD /app/migrate.pl 15 | -------------------------------------------------------------------------------- /history/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM perl:5 AS perl-cpm 2 | RUN cpanm --notest App::cpm 3 | 4 | FROM perl-cpm AS final 5 | RUN apt-get update && apt-get install dumb-init 6 | 7 | COPY cpanfile . 8 | RUN cpm install -g 9 | 10 | COPY ./app /app 11 | 12 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 13 | CMD starman --listen ":$HISTORY_PORT" --workers 12 /app/main.pl 14 | -------------------------------------------------------------------------------- /iri2020/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:bullseye-slim AS build 2 | RUN apt-get update && apt-get install -y gfortran build-essential meson 3 | COPY src /src 4 | WORKDIR /build 5 | RUN meson /src 6 | RUN ninja 7 | COPY data/ /build/iri2020/data/ 8 | 9 | FROM debian:bullseye-slim AS iri2020 10 | RUN apt-get update && apt-get install -y gfortran 11 | COPY --from=build /build /build 12 | -------------------------------------------------------------------------------- /iturhfprop/cpanfile: -------------------------------------------------------------------------------- 1 | requires 'Mojolicious::Lite'; 2 | requires 'Mojo::IOLoop::ReadWriteFork'; 3 | requires 'Ham::Locator'; 4 | requires 'EV'; 5 | requires 'Net::DNS::Native'; 6 | requires 'Future::AsyncAwait'; 7 | requires 'FU::Validate'; 8 | requires 'Mojolicious::Plugin::AccessLog'; 9 | requires 'FindBin'; 10 | requires 'PDL'; 11 | requires 'PDL::IO::HDF5' => '0.762'; 12 | -------------------------------------------------------------------------------- /systemd/prop-podman-pod.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Podman Pod for Prop 3 | After=network-online.target 4 | PartOf=kc2gprop.target 5 | [Service] 6 | Delegate=yes 7 | ExecStartPre=-podman pod rm -f prop 8 | ExecStart=podman pod create --name prop -p 127.0.0.1:5500-5520:5500-5520 9 | ExecStop=podman pod rm -f prop 10 | Type=oneshot 11 | RemainAfterExit=yes 12 | [Install] 13 | WantedBy=kc2gprop.target 14 | -------------------------------------------------------------------------------- /iturhfprop/app/templates/footer.html.ep: -------------------------------------------------------------------------------- 1 |
2 | 5 |
6 | 7 | -------------------------------------------------------------------------------- /loader/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM perl:5 AS perl-cpm 2 | RUN cpanm --notest App::cpm 3 | 4 | FROM perl-cpm AS final 5 | RUN apt-get update && apt-get install inoticoming dumb-init lftp 6 | 7 | COPY cpanfile . 8 | RUN cpm install -g 9 | 10 | COPY ./app /app 11 | COPY ./noaa-loader /noaa-loader 12 | COPY ./giro-loader /giro-loader 13 | COPY ./aus-loader /aus-loader 14 | COPY ./lib /perl5lib 15 | 16 | ENTRYPOINT ["/usr/bin/dumb-init", "--"] 17 | -------------------------------------------------------------------------------- /cosmic/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM iri2020 AS cosmic 2 | RUN apt-get update && apt-get install -y python3 python3-pip git dumb-init libnetcdf-dev 3 | WORKDIR /src 4 | COPY requirements.txt ./ 5 | RUN pip3 install -r requirements.txt 6 | COPY cosmic.py ./ 7 | COPY backfill_hmf2.py ./ 8 | COPY backfill_irtam.py ./ 9 | COPY backfill_modip_mp.py ./ 10 | 11 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 12 | CMD python3 -u /src/cosmic.py 13 | 14 | -------------------------------------------------------------------------------- /api/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==2.3.2 2 | flask-marshmallow==0.15.0 3 | Flask-SQLAlchemy==3.1.1 4 | kiwisolver==1.0.1 5 | maidenhead==1.5.0 6 | MarkupSafe==2.1.1 7 | marshmallow==3.19.0 8 | marshmallow-sqlalchemy==0.29.0 9 | numpy==2.2.6 10 | pandas[performance]==2.2.3 11 | psycopg[binary]==3.1.12 12 | pyarrow==20.0.0 13 | pymemcache==3.4.0 14 | pyparsing==2.3.0 15 | python-dateutil==2.8.2 16 | pytz==2022.7 17 | six==1.16.0 18 | SQLAlchemy==2.0.22 19 | uwsgi==2.0.29 20 | -------------------------------------------------------------------------------- /scheduler/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM perl:5.38.2 AS perl-cpm 2 | RUN wget -O /usr/local/bin/cpm https://raw.githubusercontent.com/skaji/cpm/0.997015/cpm && chmod +x /usr/local/bin/cpm 3 | 4 | FROM perl-cpm AS final 5 | RUN apt-get update && apt-get install dumb-init 6 | 7 | COPY cpanfile . 8 | RUN cpm install -g 9 | 10 | COPY ./app /app 11 | 12 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 13 | CMD perl -I/app/lib /app/main.pl daemon -l "http://*:$SCHEDULER_PORT" 14 | -------------------------------------------------------------------------------- /diffusion/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | exec podman -r run --rm -it --user 1000 --device=nvidia.com/gpu=all \ 3 | --mount type=bind,src="$PWD"/logs,dst=/app/lightning_logs \ 4 | --mount type=bind,src="$PWD"/checkpoints,dst=/app/checkpoints \ 5 | --mount type=bind,src="$PWD"/out,dst=/app/out \ 6 | --mount type=bind,src="$PWD"/tc_cache,dst=/tc_cache \ 7 | --shm-size=0 \ 8 | --net=host \ 9 | -e TORCHINDUCTOR_CACHE_DIR=/tc_cache \ 10 | prop-diffusion "$@" 11 | -------------------------------------------------------------------------------- /raytrace/app/raytrace/constants.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from math import radians 3 | 4 | r_earth_km = 6371 5 | r_earth_m = 1000 * r_earth_km 6 | maxhop = 0.62784 7 | min_phi = radians(10) 8 | max_phi = radians(89) 9 | m9_1 = 1.05 10 | m9_2 = 0.989 11 | min_takeoff = radians(1) 12 | max_takeoff = radians(30) 13 | sex_1 = 1.0 14 | sex_2 = 0.965 15 | sex_3 = 2.0 16 | sex_4 = -0.5 17 | ls_1 = 0.028 18 | ls_2 = 2.0 19 | 20 | lof_distance_base = 1. 21 | lof_threshold = 100. 22 | -------------------------------------------------------------------------------- /scheduler/app/lib/Task/Pred.pm: -------------------------------------------------------------------------------- 1 | package Task::Pred; 2 | use Mojo::Base 'Mojolicious::Plugin'; 3 | 4 | sub register { 5 | my ($self, $app) = @_; 6 | 7 | $app->minion->add_task(pred => sub { 8 | my ($job, %args) = @_; 9 | my $res = Mojo::UserAgent->new->inactivity_timeout(300)->post("http://localhost:$ENV{PRED_PORT}/generate" => 10 | form => \%args 11 | )->result; 12 | $res->is_success or die $res->error; 13 | }); 14 | } 15 | 16 | 1; 17 | -------------------------------------------------------------------------------- /scheduler/app/lib/Task/Assimilate.pm: -------------------------------------------------------------------------------- 1 | package Task::Assimilate; 2 | use Mojo::Base 'Mojolicious::Plugin'; 3 | 4 | sub register { 5 | my ($self, $app) = @_; 6 | 7 | $app->minion->add_task(assimilate => sub { 8 | my ($job, %args) = @_; 9 | my $res = Mojo::UserAgent->new->inactivity_timeout(300)->post("http://localhost:$ENV{ASSIMILATE_PORT}/generate", => 10 | form => \%args, 11 | )->result; 12 | $res->is_success or die $res->error; 13 | }); 14 | } 15 | 16 | 1; 17 | -------------------------------------------------------------------------------- /scheduler/app/lib/Task/BandQuality.pm: -------------------------------------------------------------------------------- 1 | package Task::BandQuality; 2 | use Mojo::Base 'Mojolicious::Plugin'; 3 | 4 | sub register { 5 | my ($self, $app) = @_; 6 | 7 | $app->minion->add_task(band_quality => sub { 8 | my ($job, %args) = @_; 9 | my $res = Mojo::UserAgent->new->inactivity_timeout(300)->post("http://localhost:$ENV{BANDQUALITY_PORT}/generate", => 10 | form => \%args, 11 | )->result; 12 | $res->is_success or die $res->error; 13 | }); 14 | } 15 | 16 | 1; 17 | -------------------------------------------------------------------------------- /scheduler/app/lib/Task/HoldoutEvaluate.pm: -------------------------------------------------------------------------------- 1 | package Task::HoldoutEvaluate; 2 | use Mojo::Base 'Mojolicious::Plugin'; 3 | 4 | sub register { 5 | my ($self, $app) = @_; 6 | 7 | $app->minion->add_task(holdout_evaluate => sub { 8 | my ($job, %args) = @_; 9 | my $res = Mojo::UserAgent->new->inactivity_timeout(300)->post("http://localhost:$ENV{HEVAL_PORT}/eval", => 10 | form => \%args, 11 | )->result; 12 | $res->is_success or die $res->error; 13 | }); 14 | } 15 | 16 | 1; 17 | -------------------------------------------------------------------------------- /scheduler/app/lib/Task/IRIMap.pm: -------------------------------------------------------------------------------- 1 | package Task::IRIMap; 2 | use Mojo::Base 'Mojolicious::Plugin'; 3 | use Mojo::UserAgent; 4 | 5 | sub register { 6 | my ($self, $app) = @_; 7 | 8 | $app->minion->add_task(irimap => sub { 9 | my ($job, %args) = @_; 10 | my $res = Mojo::UserAgent->new->inactivity_timeout(300)->post("http://localhost:$ENV{IRIMAP_PORT}/generate" => 11 | form => \%args 12 | )->result; 13 | $res->is_success or die $res->error; 14 | }); 15 | } 16 | 17 | 1; 18 | -------------------------------------------------------------------------------- /systemd/prop-storm.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Kp modelling 3 | StartLimitBurst=5 4 | StartLimitIntervalSec=600 5 | 6 | [Service] 7 | Type=oneshot 8 | Delegate=yes 9 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 10 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pod --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --mount type=bind,src=/home/prop/iri-index,dst=/iri-index prop-storm 11 | Restart=on-failure 12 | RestartSec=60 13 | -------------------------------------------------------------------------------- /assimilate/app/models/spline.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.interpolate import RectBivariateSpline 3 | 4 | class Spline: 5 | def __init__(self, table): 6 | self.table = table 7 | lat = np.linspace(-90, 90, 181) 8 | lon = np.linspace(-180, 180, 361) 9 | 10 | self.spline = RectBivariateSpline(lat, lon, self.table) 11 | 12 | def predict(self, latitude, longitude): 13 | return self.spline(latitude.flatten(), longitude.flatten(), grid=False).reshape(longitude.shape) 14 | -------------------------------------------------------------------------------- /loader/giro-loader/process: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | shopt -s nullglob 4 | 5 | cd /giro-data/in 6 | 7 | lftp -c 'open -u kc2g --env-password ftp://giro.uml.edu ; set ftp:sync-mode false ; mirror --Remove-source-files --parallel=4 . .' 8 | 9 | find . -type f | while read x ; do 10 | MONTH="$(date +%Y-%m)" 11 | mkdir -p "/giro-data/loaded/$MONTH" "/giro-data/load-error/$MONTH" 12 | perl -I/perl5lib /app/load.pl "$x" giro && mv "$x" "/giro-data/loaded/$MONTH" || mv "$x" "/giro-data/load-error/$MONTH" 13 | done 14 | -------------------------------------------------------------------------------- /diffusion/app/util.py: -------------------------------------------------------------------------------- 1 | def summarize_tensor(x): 2 | return f"\033[34m{str(tuple(x.shape)).ljust(24)}\033[0m (\033[31mmin {x.min().item():+.4f}\033[0m / \033[32mmean {x.mean().item():+.4f}\033[0m / \033[33mmax {x.max().item():+.4f}\033[0m)" 3 | 4 | 5 | def scale_to_diffusion(x): 6 | """Scale images from [0, 1] to [-1, 1] for diffusion models and VAE.""" 7 | return x * 2.0 - 1.0 8 | 9 | 10 | def scale_from_diffusion(x): 11 | """Scale images from [-1, 1] back to [0, 1] for visualization.""" 12 | return (x + 1.0) / 2.0 13 | -------------------------------------------------------------------------------- /www/fof2/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | FoF2 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /essn/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM iri2020 AS essn 2 | RUN apt-get update && apt-get install -y python3 python3-pip git dumb-init 3 | WORKDIR /src 4 | COPY requirements.txt ./ 5 | RUN pip3 install -r requirements.txt 6 | COPY essn.py ./ 7 | 8 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 9 | CMD uwsgi --http :${ESSN_PORT} --master --need-app --single-interpreter --enable-threads --max-requests 100 --http-timeout 300 --wsgi essn:app --processes 2 --enable-metrics --carbon-use-metrics --carbon-name-resolve --carbon ${STATSD_HOST}:2003 --carbon-id prop-essn --die-on-term 10 | -------------------------------------------------------------------------------- /ipe/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM iri2020 AS ipe 2 | RUN apt-get update && apt-get install -y python3 python3-pip git dumb-init libhdf5-dev libnetcdf-dev 3 | WORKDIR /src 4 | COPY requirements.txt ./ 5 | RUN pip3 install -r requirements.txt 6 | COPY ipe.py ./ 7 | 8 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 9 | CMD uwsgi --http :${IPE_PORT} --master --need-app --single-interpreter --enable-threads --max-requests 100 --http-timeout 300 --wsgi ipe:app --processes 8 --enable-metrics --carbon-use-metrics --carbon-name-resolve --carbon ${STATSD_HOST}:2003 --carbon-id prop-ipe --die-on-term 10 | -------------------------------------------------------------------------------- /loader/noaa-loader/process: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | shopt -s nullglob 4 | 5 | cd /noaa-data/in 6 | 7 | for x in *.zip ; do 8 | pushd /noaa-data/unzipped > /dev/null 9 | unzip "../in/$x" && rm "../in/$x" || mv "../in/$x" /noaa-data/unzip-fail 10 | popd > /dev/null 11 | done 12 | 13 | cd /noaa-data/unzipped 14 | 15 | for x in *.SAO ; do 16 | MONTH="$(date +%Y-%m)" 17 | mkdir -p "/noaa-data/loaded/$MONTH" "/noaa-data/load-error/$MONTH" 18 | perl -I/perl5lib /app/load.pl "$x" noaa && mv "$x" "/noaa-data/loaded/$MONTH" || mv "$x" "/noaa-data/load-error/$MONTH" 19 | done 20 | -------------------------------------------------------------------------------- /holdout-eval/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM iri2020 AS essn 2 | RUN apt-get update && apt-get install -y python3 python3-dateutil python3-pip python3-scipy python3-numpy git dumb-init 3 | WORKDIR /src 4 | COPY requirements.txt ./ 5 | RUN pip3 install -r requirements.txt 6 | COPY heval.py ./ 7 | 8 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 9 | CMD uwsgi --http :${HEVAL_PORT} --master --need-app --single-interpreter --enable-threads --max-requests 100 --http-timeout 300 --wsgi heval:app --processes 2 --enable-metrics --carbon-use-metrics --carbon-name-resolve --carbon ${STATSD_HOST}:2003 --carbon-id prop-holdout-eval --die-on-term 10 | -------------------------------------------------------------------------------- /iri2020/.gitattributes: -------------------------------------------------------------------------------- 1 | src/* linguist-vendored 2 | iri_opt.F90 linguist-vendored=false 3 | iri_ts.F90 linguist-vendored=false 4 | irimap.F90 linguist-vendored=false 5 | 6 | .gitattributes text eol=lf 7 | .gitignore text eol=lf 8 | Makefile text eol=lf 9 | *.yml text eol=lf 10 | LICENSE text eol=lf 11 | *.ipynb text eol=lf 12 | *.txt text eol=lf 13 | *.py text eol=lf 14 | *.sh text eol=lf 15 | *.c text eol=lf 16 | *.cpp text eol=lf 17 | *.f text eol=lf 18 | *.for text eol=lf 19 | *.f90 text eol=lf 20 | *.md text eol=lf 21 | *.rst text eol=lf 22 | *.csv text eol=lf 23 | *.m text eol=lf 24 | *.grc text eol=lf 25 | *.pas text eol=lf 26 | -------------------------------------------------------------------------------- /www/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | MUF(3000km) 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /loader/aus-loader/process: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | shopt -s nullglob 4 | 5 | cd /aus-data/in 6 | 7 | wget $(perl -I/perl5lib /aus-loader/urls) 8 | 9 | for x in *.zip ; do 10 | pushd /aus-data/unzipped > /dev/null 11 | unzip "../in/$x" && rm "../in/$x" || mv "../in/$x" /aus-data/unzip-fail 12 | popd > /dev/null 13 | done 14 | 15 | cd /aus-data/unzipped 16 | 17 | for x in *.SAO *.sao; do 18 | MONTH="$(date +%Y-%m)" 19 | mkdir -p "/aus-data/loaded/$MONTH" "/aus-data/load-error/$MONTH" 20 | perl -I/perl5lib /app/load.pl "$x" aus-sws && mv "$x" "/aus-data/loaded/$MONTH" || mv "$x" "/aus-data/load-error/$MONTH" 21 | done 22 | -------------------------------------------------------------------------------- /systemd/prop-memcached.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Memcached 3 | BindsTo=prop-podman-pod.service 4 | After=prop-podman-pod.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald memcached memcached -o modern -m 2048 -I 32m 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /irimap/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM iri2020 AS irimap 2 | RUN apt-get update && apt-get install -y python3 python3-dateutil python3-xarray python3-sklearn python3-pip python3-sympy python3-pybind11 cython3 git dumb-init cmake 3 | WORKDIR /src 4 | COPY requirements.txt ./ 5 | RUN pip3 install -r requirements.txt 6 | # wmm2020 does a build-on-first-use thing... 7 | RUN python3 -c 'import wmm2020' 8 | COPY irimap.py ./ 9 | 10 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 11 | CMD uwsgi --http :${IRIMAP_PORT} --master --need-app --single-interpreter --max-requests 100 --http-timeout 300 --wsgi irimap:app --processes 16 --enable-metrics --carbon-use-metrics --carbon-name-resolve --carbon ${STATSD_HOST}:2003 --carbon-id irimap --die-on-term 12 | -------------------------------------------------------------------------------- /systemd/prop-history.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop History API 3 | BindsTo=prop-podman-pod.service prop-postgres.service 4 | After=prop-podman-pod.service prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env prop-history 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /scheduler/app/lib/Task/eSSN.pm: -------------------------------------------------------------------------------- 1 | package Task::eSSN; 2 | use Mojo::Base 'Mojolicious::Plugin'; 3 | use Mojo::UserAgent; 4 | 5 | sub register { 6 | my ($self, $app) = @_; 7 | 8 | $app->minion->add_task(essn => sub { 9 | my ($job, %args) = @_; 10 | my $run_id = $args{run_id} || $job->id; 11 | 12 | my $res = Mojo::UserAgent->new->inactivity_timeout(300)->post("http://localhost:$ENV{ESSN_PORT}/generate" => 13 | form => { 14 | series => $args{series}, 15 | run_id => $run_id, 16 | holdouts => $args{holdouts}, 17 | ($args{v2} ? (v2 => 1) : ()), 18 | ($args{ts} ? (ts => $args{ts}) : ()), 19 | }, 20 | )->result; 21 | $res->is_success or die $res->error; 22 | }); 23 | } 24 | 25 | 1; 26 | -------------------------------------------------------------------------------- /systemd/prop-renderer.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Renderer 3 | BindsTo=prop-podman-pod.service prop-api.service 4 | After=prop-podman-pod.service prop-api.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/output,dst=/output prop-renderer 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /systemd/prop-raytrace.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Raytracer 3 | BindsTo=prop-api.service prop-assimilate.service prop-podman-pod.service 4 | After=prop-api.service prop-podman-pod.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env prop-raytrace 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12 2 | RUN apt-get update && apt-get install -y dumb-init 3 | ENV STATIC_INDEX 1 4 | 5 | COPY requirements.txt /tmp/ 6 | RUN pip install -r /tmp/requirements.txt 7 | RUN uwsgi --build-plugin https://github.com/KLab/uwsgi-cheaper-spare2 8 | 9 | COPY ./app /app 10 | RUN mv /cheaper_spare2_plugin.so /app 11 | 12 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 13 | WORKDIR /app 14 | CMD uwsgi --http :${API_PORT} --master --need-app --single-interpreter --enable-threads --max-requests 100 --http-timeout 300 --wsgi main:app --processes 128 --need-plugin cheaper_spare2 --cheaper-algo spare2 --cheaper 8 --cheaper-step 8 --cheaper-initial 8 --enable-metrics --carbon-use-metrics --carbon-name-resolve --carbon ${STATSD_HOST}:2003 --carbon-id prop-api --die-on-term 15 | -------------------------------------------------------------------------------- /raytrace/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | RUN apt-get update && apt-get install -y dumb-init python3-dev libhdf5-dev 3 | 4 | COPY requirements.txt /tmp/ 5 | RUN pip3 install -r /tmp/requirements.txt 6 | RUN uwsgi --build-plugin https://github.com/KLab/uwsgi-cheaper-spare2 7 | 8 | COPY ./app /app 9 | RUN mv /cheaper_spare2_plugin.so /app 10 | 11 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 12 | 13 | WORKDIR /app 14 | CMD uwsgi --http :${RAYTRACE_PORT} --master --need-app --single-interpreter --max-requests 100 --http-timeout 300 --wsgi main:app --processes 64 --need-plugin cheaper_spare2 --cheaper-algo spare2 --cheaper 8 --cheaper-step 8 --cheaper-initial 8 --enable-metrics --carbon-use-metrics --carbon-name-resolve --carbon ${STATSD_HOST}:2003 --carbon-id prop-raytrace --die-on-term 15 | -------------------------------------------------------------------------------- /systemd/prop-pred.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Prediction API 3 | BindsTo=prop-history.service 4 | After=prop-history.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/iri-index/predicted,dst=/build/iri2020/data/index prop-pred 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /pred/app/cs.py: -------------------------------------------------------------------------------- 1 | def cs_to_stdev(cs, adj100=False): 2 | if cs == -1: 3 | cs = 55 4 | if cs == 100 and adj100: 5 | cs = 85 6 | return 0.185 - 0.00173 * cs 7 | 8 | def stdev_to_cs(sd): 9 | cs = (0.185 - sd) / 0.00173 10 | 11 | if cs < 0: 12 | cs = 0 13 | if cs > 100: 14 | cs = 100 15 | 16 | return round(cs) 17 | 18 | def cs_to_stdev_new(cs, coeff, adj100=True): 19 | if cs == -1: 20 | cs = coeff['replace_-1'] 21 | if cs == 100 and adj100: 22 | cs = coeff['replace_100'] 23 | return coeff['int'] * (1. - coeff['sl'] * cs / 100) 24 | 25 | def stdev_to_cs_new(sd, coeff): 26 | cs = 100.* (1 - sd / coeff['int']) / coeff['sl'] 27 | if cs < 0: 28 | cs = 0 29 | elif cs > 100: 30 | cs = 100 31 | return cs 32 | -------------------------------------------------------------------------------- /systemd/prop-cosmic.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Fetch COSMIC data 3 | BindsTo=prop-postgres.service 4 | After=prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Type=oneshot 8 | Delegate=yes 9 | ExecStartPre=/usr/bin/rm -f %t/%N-pid %t/%N-cid 10 | ExecStart=/usr/bin/podman run --rm --replace --pod=prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/iri-index/predicted,dst=/build/iri2020/data/index prop-cosmic 11 | TimeoutStartSec=1800 12 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 13 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /systemd/prop-assimilate.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Assimilation Engine 3 | BindsTo=prop-api.service prop-podman-pod.service prop-postgres.service 4 | After=prop-api.service prop-podman-pod.service prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env prop-assimilate 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /diffusion/pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "pred4d" 3 | version = "0.0.1" 4 | 5 | dependencies = [ 6 | "psycopg[binary]==3.2.9", 7 | "flask==2.2.5", 8 | "pyyaml", 9 | "pybind11", 10 | "pandas", 11 | "scikit-optimize", 12 | "scikit-learn", 13 | "uwsgi", 14 | "matplotlib", 15 | "h5py", 16 | "hdf5plugin", 17 | "numpy", 18 | "torch==2.*", 19 | "torchvision", 20 | "lightning==2.*", 21 | "tensorboard", 22 | "datasets>=2.15.0", 23 | "huggingface_hub", 24 | "diffusers", 25 | "jsonargparse", 26 | "accelerate", 27 | ] 28 | 29 | [tool.uv.sources] 30 | torch = { index = "pytorch-cu128" } 31 | torchvision = { index = "pytorch-cu128" } 32 | 33 | [[tool.uv.index]] 34 | name = "pytorch-cu128" 35 | url = "https://download.pytorch.org/whl/cu128" 36 | explicit = true 37 | -------------------------------------------------------------------------------- /systemd/irimap.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=IRI Map Generator 3 | BindsTo=prop-podman-pod.service prop-postgres.service 4 | After=prop-podman-pod.service prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/iri-index/predicted,dst=/build/iri2020/data/index irimap 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /systemd/prop-postgres.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Postgres 3 | BindsTo=prop-podman-pod.service 4 | After=prop-podman-pod.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | StateDirectory=prop-postgres 8 | Delegate=yes 9 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 10 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --env-file /etc/kc2gprop/postgres.env --mount type=bind,src=/var/lib/prop-postgres,dst=/var/lib/postgresql/data --mount type=bind,src=/storage/backup,dst=/backup postgres:16 postgres 11 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 12 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 13 | KillMode=mixed 14 | Restart=on-failure 15 | [Install] 16 | WantedBy=kc2gprop.target 17 | -------------------------------------------------------------------------------- /systemd/noaa-loader.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=NOAA Data Loader 3 | BindsTo=prop-podman-pod.service prop-postgres.service 4 | After=prop-podman-pod.service prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1002 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/noaa-iono/noaa-data,dst=/noaa-data prop-loader /noaa-loader/listen 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /assimilate/app/models/combinators.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | class LogSpace: 4 | def __init__(self, base_model): 5 | self.base_model = base_model 6 | 7 | def train(self, df, t): 8 | self.base_model.train(df, np.log(t)) 9 | 10 | def predict(self, latitude, longitude): 11 | return np.exp(self.base_model.predict(latitude, longitude)) 12 | 13 | class Product: 14 | def __init__(self, model1, model2): 15 | self.model1 = model1 16 | self.model2 = model2 17 | 18 | def train(self, df, t1, t2): 19 | self.model1.train(df, t1) 20 | self.model2.train(df, t2) 21 | 22 | def predict(self, latitude, longitude): 23 | model1_pred = self.model1.predict(latitude, longitude) 24 | model2_pred = self.model2.predict(latitude, longitude) 25 | return model1_pred * model2_pred 26 | -------------------------------------------------------------------------------- /systemd/backfill-irtam.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Update COSMIC with IRTAM data 3 | BindsTo=prop-postgres.service 4 | After=prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Type=oneshot 8 | Delegate=yes 9 | ExecStartPre=/usr/bin/rm -f %t/%N-pid %t/%N-cid 10 | ExecStart=/usr/bin/podman run --rm --pod=prop --name %N --conmon-pidfile /%t/%N.pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/iri-index/predicted,dst=/build/iri2020/data/index --mount type=bind,src=/home/prop/irtam,dst=/irtam prop-cosmic python3 -u /src/backfill_irtam.py 11 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 12 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 13 | [Install] 14 | WantedBy=kc2gprop.target 15 | -------------------------------------------------------------------------------- /systemd/prop-essn.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop eSSN API 3 | BindsTo=prop-history.service prop-podman-pod.service prop-postgres.service 4 | After=prop-history.service prop-podman-pod.service prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/iri-index/predicted,dst=/build/iri2020/data/index prop-essn 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /systemd/prop-holdout-eval.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Holdout Evaluator 3 | BindsTo=prop-api.service prop-podman-pod.service prop-postgres.service 4 | After=prop-api.service prop-podman-pod.service prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/iri-index/chain,dst=/build/iri2020/data/index prop-holdout-eval 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /systemd/giro-loader.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=GIRO Data Loader 3 | BindsTo=prop-podman-pod.service prop-postgres.service 4 | After=prop-podman-pod.service prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Type=oneshot 8 | Delegate=yes 9 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 10 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env --env-file /etc/kc2gprop/giro.env --mount type=bind,src=/home/prop/giro-data,dst=/giro-data prop-loader /giro-loader/process 11 | TimeoutStartSec=600 12 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 13 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 14 | KillMode=mixed 15 | [Install] 16 | WantedBy=kc2gprop.target 17 | -------------------------------------------------------------------------------- /systemd/prop-ipe.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop WAM-IPE map 3 | BindsTo=prop-podman-pod.service prop-postgres.service 4 | After=prop-podman-pod.service prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/iri-index/predicted,dst=/build/iri2020/data/index --mount type=bind,src=/home/prop/ipe-data,dst=/ipe-data prop-ipe 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /systemd/prop-scheduler.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Scheduler 3 | BindsTo=prop-podman-pod.service prop-postgres.service 4 | After=prop-podman-pod.service prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/mojo.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/output,dst=/output --mount type=bind,src=/storage/prop-archive,dst=/archive prop-scheduler 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /iturhfprop/app/templates/worldmap.ep: -------------------------------------------------------------------------------- 1 | PathName "KC2G Online HF Circuit Prediction: Area Map" 2 | Path.L_tx.lat <%= $txlat %> 3 | Path.L_tx.lng <%= $txlon %> 4 | TXAntFilePath <%== $txant %> 5 | TXGOS <%= $txgos %> 6 | RXAntFilePath <%== $rxant %> 7 | RXGOS <%= $rxgos %> 8 | Path.year <%= $year %> 9 | Path.month <%= $month %> 10 | Path.hour <%= $hour %> 11 | Path.SSN <%= $ssn %> 12 | Path.frequency <%= $freq %> 13 | Path.txpower <%= int($txpow) %> 14 | Path.BW <%= $bw %> 15 | Path.SNRr <%= $snrr %> 16 | Path.SNRXXp 90 17 | Path.ManMadeNoise <%== $rxnoise %> 18 | Path.Modulation "ANALOG" 19 | Path.SorL "SHORTPATH" 20 | RptFileFormat <%== $reports %> 21 | LL.lat <%= $min_lat %> 22 | LL.lng -180.0 23 | LR.lat <%= $min_lat %> 24 | LR.lng 180.0 25 | UL.lat <%= $max_lat %> 26 | UL.lng -180.0 27 | UR.lat <%= $max_lat %> 28 | UR.lng 180.0 29 | latinc <%= $degree_step %> 30 | lnginc <%= $degree_step %> 31 | DataFilePath <%== $data_dir %> 32 | -------------------------------------------------------------------------------- /loader/aus-loader/urls: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use warnings; 4 | use Time::Piece (); 5 | 6 | my $yesterday = (Time::Piece::gmtime() - 86400)->truncate(to => 'day'); 7 | my $year = $yesterday->truncate(to => 'year'); 8 | my $yday = ($yesterday - $year)->days + 1; 9 | $year = $year->year; 10 | my $year2 = ($year % 100); 11 | 12 | my @ionosondes = ( 13 | [qw(bri5f BR52P)], 14 | [qw(cas6a CW46O)], 15 | [qw(cbr5f CB53N)], 16 | [qw(cck5f CS31K)], 17 | [qw(dwn5d DW41K)], 18 | [qw(hbt5f HO54K)], 19 | [qw(lea6a LM42J)], 20 | [qw(mawcd MW26P)], 21 | [qw(nlk5d NI63_)], 22 | [qw(nue5f ND61R)], 23 | [qw(per6a PE43K)], 24 | [qw(tvl6a TV51R)], 25 | ); 26 | 27 | for my $sonde (@ionosondes) { 28 | my ($dir, $code) = @$sonde; 29 | my $url = "https://downloads.sws.bom.gov.au/wdc/wdc_ion_auto/${dir}/sao/${year2}/${code}_${year}${yday}.zip"; 30 | print "$url\n"; 31 | } 32 | 33 | -------------------------------------------------------------------------------- /assimilate/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM continuumio/miniconda3 2 | 3 | RUN apt-get update && apt-get -y install unzip build-essential dumb-init 4 | 5 | WORKDIR / 6 | RUN conda create -y -q -n my_cartopy_env -c conda-forge python=3.10 statsmodels pandas cython xarray sympy networkx uwsgi george 7 | 8 | ENV PATH /opt/conda/envs/my_cartopy_env/bin:$PATH 9 | 10 | RUN echo "conda activate my_cartopy_env" >> ~/.bashrc 11 | 12 | COPY requirements.txt ./ 13 | RUN pip install -r requirements.txt 14 | 15 | WORKDIR / 16 | 17 | COPY app /app 18 | 19 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 20 | WORKDIR /app 21 | ENV OPENBLAS_NUM_THREADS=4 22 | ENV OMP_NUM_THREADS=4 23 | 24 | CMD uwsgi --http :${ASSIMILATE_PORT} --master --need-app --single-interpreter --max-requests 100 --http-timeout 300 --wsgi assimilate:app --processes 8 --enable-metrics --carbon-use-metrics --carbon-name-resolve --carbon ${STATSD_HOST}:2003 --carbon-id prop-assimilate --die-on-term 25 | -------------------------------------------------------------------------------- /diffusion/app/train_guidance.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lightning as L 3 | from lightning.pytorch.callbacks import ModelCheckpoint 4 | from lightning.pytorch.loggers import TensorBoardLogger 5 | from models import GuidanceModel, IRIData 6 | 7 | if __name__ == "__main__": 8 | if len(sys.argv) > 1: 9 | model = GuidanceModel.load_from_checkpoint(sys.argv[1]) 10 | else: 11 | model = GuidanceModel() 12 | 13 | data = IRIData("combined", train_batch=128, add_noise=0.2) 14 | checkpoint_callback = ModelCheckpoint(dirpath="checkpoints", filename='guidance-' + 15 | '-{v_num}-{epoch}-{val_loss:.2g}', save_top_k=1, monitor="val_loss", mode="min") 16 | trainer = L.Trainer(max_epochs=350, 17 | precision="bf16-mixed", callbacks=[checkpoint_callback], 18 | logger=TensorBoardLogger("lightning_logs", name="guidance")) 19 | trainer.fit(model, data) 20 | -------------------------------------------------------------------------------- /www/static/serialize.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Serialize all form data into a SearchParams string 3 | * (c) 2020 Chris Ferdinandi, MIT License, https://gomakethings.com 4 | * @param {Node} form The form to serialize 5 | * @return {String} The serialized form data 6 | */ 7 | var serialize = function (form) { 8 | var arr = []; 9 | Array.prototype.slice.call(form.elements).forEach(function (field) { 10 | if (!field.name || field.disabled || ['file', 'reset', 'submit', 'button'].indexOf(field.type) > -1) return; 11 | if (field.type === 'select-multiple') { 12 | Array.prototype.slice.call(field.options).forEach(function (option) { 13 | if (!option.selected) return; 14 | arr.push(encodeURIComponent(field.name) + '=' + encodeURIComponent(option.value)); 15 | }); 16 | return; 17 | } 18 | if (['checkbox', 'radio'].indexOf(field.type) >-1 && !field.checked) return; 19 | arr.push(encodeURIComponent(field.name) + '=' + encodeURIComponent(field.value)); 20 | }); 21 | return arr.join('&'); 22 | }; 23 | -------------------------------------------------------------------------------- /systemd/prop-api.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Current API 3 | BindsTo=prop-podman-pod.service prop-postgres.service prop-memcached.service 4 | After=prop-podman-pod.service prop-postgres.service prop-memcached.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | RuntimeDirectory=prop-api-profiles 9 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 10 | ExecStartPre=/usr/bin/chown prop:prop /run/prop-api-profiles 11 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/run/prop-api-profiles,dst=/profiles -e WORKERS_PER_CORE=0.25 -e SQLALCHEMY_WARN_20=1 prop-api 12 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 13 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 14 | KillMode=mixed 15 | Restart=on-failure 16 | [Install] 17 | WantedBy=kc2gprop.target 18 | -------------------------------------------------------------------------------- /diffusion/app/train_vae.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lightning as L 3 | from lightning.pytorch.callbacks import ModelCheckpoint 4 | from lightning.pytorch.loggers import TensorBoardLogger 5 | from models import VAEModel, IRIData 6 | 7 | if __name__ == "__main__": 8 | if len(sys.argv) > 1: 9 | model = VAEModel.load_from_checkpoint(sys.argv[1]) 10 | else: 11 | model = VAEModel() 12 | 13 | checkpoint_callback = ModelCheckpoint(dirpath="checkpoints", filename='vae-' + 14 | '{epoch}-{val_loss:.2g}', save_top_k=1, monitor="val_loss", mode="min") 15 | data = IRIData("combined", train_batch=64, add_noise=0.005) 16 | trainer = L.Trainer(max_epochs=50, 17 | log_every_n_steps=10, 18 | # accumulate_grad_batches=4, 19 | precision="bf16-mixed", callbacks=[checkpoint_callback], 20 | logger=TensorBoardLogger("lightning_logs", name="vae")) 21 | trainer.fit(model, data) 22 | -------------------------------------------------------------------------------- /iturhfprop/app/templates/p2p.ep: -------------------------------------------------------------------------------- 1 | PathName "KC2G Online HF Circuit Prediction: Point-to-Point" 2 | Path.L_tx.lat <%= $txlat %> 3 | Path.L_tx.lng <%= $txlon %> 4 | TXAntFilePath <%== $txant %> 5 | TXGOS <%= $txgos %> 6 | Path.L_rx.lat <%= $rxlat %> 7 | Path.L_rx.lng <%= $rxlon %> 8 | RXAntFilePath <%== $rxant %> 9 | RXGOS <%= $rxgos %> 10 | Path.year <%= $year %> 11 | Path.month <%= $month %> 12 | Path.hour 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24 13 | Path.SSN <%= $ssn %> 14 | Path.frequency <%= join(",", @$freqs) %> 15 | Path.txpower <%= int($txpow) %> 16 | Path.BW <%= $bw %> 17 | Path.SNRr <%= $snrr %> 18 | Path.SNRXXp 90 19 | Path.ManMadeNoise <%== $rxnoise %> 20 | Path.Modulation "ANALOG" 21 | Path.SorL "SHORTPATH" 22 | RptFileFormat "RPT_BMUFD | RPT_PR | RPT_GRW | RPT_NOISESOURCES | RPT_BCR" 23 | LL.lat <%= $rxlat %> 24 | LL.lng <%= $rxlon %> 25 | LR.lat <%= $rxlat %> 26 | LR.lng <%= $rxlon %> 27 | UL.lat <%= $rxlat %> 28 | UL.lng <%= $rxlon %> 29 | UR.lat <%= $rxlat %> 30 | UR.lng <%= $rxlon %> 31 | DataFilePath <%== $data_dir %> 32 | -------------------------------------------------------------------------------- /iturhfprop/app/templates/header.html.ep: -------------------------------------------------------------------------------- 1 |
2 | 3 | 30 | 31 |
32 | -------------------------------------------------------------------------------- /systemd/prop-worker-gcs.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Worker (GCS) 3 | BindsTo=prop-podman-pod.service prop-postgres.service 4 | After=prop-podman-pod.service prop-postgres.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/mojo.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/output,dst=/output --mount type=bind,src=/storage/prop-archive,dst=/archive --mount type=bind,src=/etc/kc2gprop/archiver.json,dst=/archiver.json prop-scheduler perl -I/app/lib /app/main.pl minion worker -m production -I 60 -q gcs_upload -j 8 -D 1 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /scheduler/app/lib/Task/IPE.pm: -------------------------------------------------------------------------------- 1 | package Task::IPE; 2 | use Mojo::Base 'Mojolicious::Plugin'; 3 | use Mojo::UserAgent; 4 | 5 | sub register { 6 | my ($self, $app) = @_; 7 | 8 | $app->minion->add_task(ipe => sub { 9 | my ($job, %args) = @_; 10 | my $ts = $args{target}; 11 | $ts -= ($ts % 300); 12 | my $key = "ipe-$ts"; 13 | return $job->retry({ delay => (10 + int rand 10) }) unless my $guard = $app->minion->guard($key, 300); 14 | my $res = Mojo::UserAgent->new->inactivity_timeout(300)->post("http://localhost:$ENV{IPE_PORT}/generate" => 15 | form => \%args 16 | )->result; 17 | $res->is_success or die $res->error; 18 | }); 19 | 20 | $app->minion->add_task(copy_ipe => sub { 21 | my ($job, %args) = @_; 22 | my $db = $app->pg->db; 23 | my $res = $db->query( 24 | 'INSERT INTO ipemap (time, run_id, dataset) SELECT time, ?, dataset FROM ipemap WHERE run_id=? AND time=to_timestamp(?)', 25 | $args{run_id}, $args{from_run_id}, $args{target} 26 | ); 27 | }); 28 | } 29 | 30 | 1; 31 | -------------------------------------------------------------------------------- /iri2020/.gitignore: -------------------------------------------------------------------------------- 1 | *.m~ 2 | fort.6 3 | .mypy_cache/ 4 | .pytest_cache/ 5 | baseIRI2016 6 | testiri2016 7 | bin/ 8 | 9 | # Byte-compiled / optimized / DLL files 10 | __pycache__/ 11 | *.py[cod] 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | env/ 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *,cover 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | 61 | # Sphinx documentation 62 | docs/_build/ 63 | 64 | # PyBuilder 65 | target/ 66 | -------------------------------------------------------------------------------- /pred/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM continuumio/miniconda3 2 | 3 | RUN apt-get update && apt-get -y install unzip build-essential dumb-init libgfortran5 4 | 5 | WORKDIR / 6 | 7 | RUN conda create -y -q --insecure -n my_cartopy_env -c conda-forge python=3.10 statsmodels pandas cython xarray sympy networkx scikit-learn scikit-optimize uwsgi george 8 | 9 | ENV PATH /opt/conda/envs/my_cartopy_env/bin:$PATH 10 | ENV PYTHON_EGG_CACHE=/tmp/egg-cache 11 | 12 | COPY --from=iri2020 /build/ /build/ 13 | 14 | RUN echo "conda activate my_cartopy_env" >> ~/.bashrc 15 | 16 | RUN pip install -U pip 17 | COPY requirements.txt ./ 18 | RUN pip install -r requirements.txt 19 | 20 | COPY app /app 21 | 22 | ENV OPENBLAS_NUM_THREADS=4 23 | ENV OMP_NUM_THREADS=4 24 | 25 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 26 | WORKDIR /app 27 | CMD uwsgi --http :${PRED_PORT} --master --need-app --single-interpreter --enable-threads --max-requests 100 --http-timeout 300 --wsgi server:app --processes 6 --enable-metrics --carbon-use-metrics --carbon-name-resolve --carbon ${STATSD_HOST}:2003 --carbon-id prop-pred --die-on-term 28 | -------------------------------------------------------------------------------- /systemd/prop-worker-highcpu.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Worker (high cpu) 3 | BindsTo=prop-podman-pod.service prop-postgres.service prop-assimilate.service prop-pred.service 4 | After=prop-podman-pod.service prop-postgres.service prop-assimilate.service prop-pred.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/mojo.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/output,dst=/output --mount type=bind,src=/storage/prop-archive,dst=/archive prop-scheduler perl -I/app/lib /app/main.pl minion worker -m production -I 60 -q assimilate -q pred -j 6 -D 1 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /systemd/prop-worker-default.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Prop Worker (default) 3 | BindsTo=prop-podman-pod.service prop-postgres.service prop-essn.service irimap.service prop-renderer.service 4 | After=prop-podman-pod.service prop-postgres.service prop-essn.service irimap.service prop-renderer.service 5 | PartOf=kc2gprop.target 6 | [Service] 7 | Delegate=yes 8 | ExecStartPre=/usr/bin/rm -f /%t/%N-pid /%t/%N-cid 9 | ExecStart=/usr/bin/podman run --rm --pod prop --name %N --conmon-pidfile /%t/%N-pid --cidfile /%t/%N-cid --cgroups=no-conmon --attach stdin --log-driver journald --user 1001 --env-file /etc/kc2gprop/db.env --env-file /etc/kc2gprop/mojo.env --env-file /etc/kc2gprop/ports.env --mount type=bind,src=/home/prop/output,dst=/output --mount type=bind,src=/storage/prop-archive,dst=/archive prop-scheduler perl -I/app/lib /app/main.pl minion worker -m production -I 60 -q default -j 16 -D 1 10 | ExecStop=/usr/bin/podman stop --ignore --cidfile %t/%N-cid -t 10 11 | ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/%N-cid 12 | KillMode=mixed 13 | Restart=on-failure 14 | [Install] 15 | WantedBy=kc2gprop.target 16 | -------------------------------------------------------------------------------- /loader/app/debug.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use warnings; 4 | 5 | use Data::SAO; 6 | use Data::Dumper; 7 | 8 | my $sao = Data::SAO->new(filename => $ARGV[0]); 9 | 10 | print "Station code: ", $sao->station_code, "\n"; 11 | print "Name: ", $sao->name, "\n"; 12 | 13 | my $geophys = $sao->geophysical_constants; 14 | 15 | print "Lat: $geophys->{latitude} Lon: $geophys->{longitude}\n"; 16 | print "Timestamp: ", $sao->timestamp->{date}, " ", $sao->timestamp->{time}, "\n"; 17 | print "Confidence: ", (0+$sao->confidence), "\n"; 18 | 19 | my %map = ( 20 | fof2 => 'foF2', 21 | fof1 => 'foF1', 22 | mufd => 'MUF(D)', 23 | md => 'M(D)', 24 | foes => 'foEs', 25 | foe => 'foE', 26 | hf2 => q{h'F2}, 27 | he => q{h'E}, 28 | hme => 'zmE', 29 | hmf2 => 'zmF2', 30 | hmf1 => 'zmF1', 31 | yf2 => 'yF2', 32 | yf1 => 'yF1', 33 | tec => 'TEC', 34 | scalef2 => 'scaleF2', 35 | fbes => 'fbEs', 36 | ); 37 | 38 | my $characteristics = $sao->scaled_characteristics; 39 | 40 | for my $key (sort keys %map) { 41 | if (defined(my $val = $characteristics->{$map{$key}})) { 42 | print "$key: $val "; 43 | } 44 | } 45 | print "\n\n"; 46 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf1960.dat: -------------------------------------------------------------------------------- 1 | dgrf1960 2 | 10 6371.2 1960.0 3 | -30421 4 | -2169 5 | 5791 6 | -1555 7 | 3002 8 | -1967 9 | 1590 10 | 206 11 | 1302 12 | -1992 13 | -414 14 | 1289 15 | 224 16 | 878 17 | -130 18 | 957 19 | 800 20 | 135 21 | 504 22 | -278 23 | -394 24 | 3 25 | 269 26 | -255 27 | -222 28 | 362 29 | 16 30 | 242 31 | 125 32 | -26 33 | -117 34 | -156 35 | -114 36 | -63 37 | 81 38 | 46 39 | 58 40 | -10 41 | 1 42 | 99 43 | -237 44 | 60 45 | -1 46 | -20 47 | -2 48 | -11 49 | -113 50 | -17 51 | 67 52 | -56 53 | -55 54 | 5 55 | -28 56 | 15 57 | -6 58 | -32 59 | 7 60 | -7 61 | 23 62 | 17 63 | -18 64 | 8 65 | -17 66 | 15 67 | 6 68 | 11 69 | -4 70 | -14 71 | -11 72 | 7 73 | 2 74 | -18 75 | 10 76 | 4 77 | -5 78 | 23 79 | 10 80 | 1 81 | 8 82 | -20 83 | 4 84 | 6 85 | -18 86 | 0 87 | 12 88 | -9 89 | 2 90 | 1 91 | 0 92 | 4 93 | -3 94 | -1 95 | 9 96 | -2 97 | 8 98 | 3 99 | 0 100 | -1 101 | 5 102 | 1 103 | -3 104 | 4 105 | 4 106 | 1 107 | 0 108 | 0 109 | -1 110 | 2 111 | 4 112 | -5 113 | 6 114 | 1 115 | 1 116 | -1 117 | -1 118 | 6 119 | 2 120 | 0 121 | 0 122 | -7 123 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf1985.dat: -------------------------------------------------------------------------------- 1 | dgrf1985 2 | 10 6371.2 1985.0 3 | -29873 4 | -1905 5 | 5500 6 | -2072 7 | 3044 8 | -2197 9 | 1687 10 | -306 11 | 1296 12 | -2208 13 | -310 14 | 1247 15 | 284 16 | 829 17 | -297 18 | 936 19 | 780 20 | 232 21 | 361 22 | -249 23 | -424 24 | 69 25 | 170 26 | -297 27 | -214 28 | 355 29 | 47 30 | 253 31 | 150 32 | -93 33 | -154 34 | -164 35 | -75 36 | -46 37 | 95 38 | 53 39 | 65 40 | -16 41 | 51 42 | 88 43 | -185 44 | 69 45 | 4 46 | -48 47 | 16 48 | -1 49 | -102 50 | 21 51 | 74 52 | -62 53 | -83 54 | 3 55 | -27 56 | 24 57 | -2 58 | -6 59 | 20 60 | 4 61 | 17 62 | 10 63 | -23 64 | 0 65 | -7 66 | 21 67 | 6 68 | 8 69 | 0 70 | -19 71 | -11 72 | 5 73 | -9 74 | -23 75 | 4 76 | 11 77 | 4 78 | 14 79 | 4 80 | -15 81 | -4 82 | -11 83 | 5 84 | 10 85 | -21 86 | 1 87 | 15 88 | -12 89 | 9 90 | 9 91 | -6 92 | -3 93 | -6 94 | -1 95 | 9 96 | 7 97 | 9 98 | 1 99 | -7 100 | -5 101 | 2 102 | -4 103 | -4 104 | 1 105 | 3 106 | 0 107 | -5 108 | 3 109 | -2 110 | 6 111 | 5 112 | -4 113 | 3 114 | 0 115 | 1 116 | -1 117 | 2 118 | 4 119 | 3 120 | 0 121 | 0 122 | -6 123 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf1990.dat: -------------------------------------------------------------------------------- 1 | dgrf1990 2 | 10 6371.2 1990.0 3 | -29775 4 | -1848 5 | 5406 6 | -2131 7 | 3059 8 | -2279 9 | 1686 10 | -373 11 | 1314 12 | -2239 13 | -284 14 | 1248 15 | 293 16 | 802 17 | -352 18 | 939 19 | 780 20 | 247 21 | 325 22 | -240 23 | -423 24 | 84 25 | 141 26 | -299 27 | -214 28 | 353 29 | 46 30 | 245 31 | 154 32 | -109 33 | -153 34 | -165 35 | -69 36 | -36 37 | 97 38 | 61 39 | 65 40 | -16 41 | 59 42 | 82 43 | -178 44 | 69 45 | 3 46 | -52 47 | 18 48 | 1 49 | -96 50 | 24 51 | 77 52 | -64 53 | -80 54 | 2 55 | -26 56 | 26 57 | 0 58 | -1 59 | 21 60 | 5 61 | 17 62 | 9 63 | -23 64 | 0 65 | -4 66 | 23 67 | 5 68 | 10 69 | -1 70 | -19 71 | -10 72 | 6 73 | -12 74 | -22 75 | 3 76 | 12 77 | 4 78 | 12 79 | 2 80 | -16 81 | -6 82 | -10 83 | 4 84 | 9 85 | -20 86 | 1 87 | 15 88 | -12 89 | 11 90 | 9 91 | -7 92 | -4 93 | -7 94 | -2 95 | 9 96 | 7 97 | 8 98 | 1 99 | -7 100 | -6 101 | 2 102 | -3 103 | -4 104 | 2 105 | 2 106 | 1 107 | -5 108 | 3 109 | -2 110 | 6 111 | 4 112 | -4 113 | 3 114 | 0 115 | 1 116 | -2 117 | 3 118 | 3 119 | 3 120 | -1 121 | 0 122 | -6 123 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf1955.dat: -------------------------------------------------------------------------------- 1 | dgrf1955 2 | 10 6371.2 1955.0 3 | -30500 4 | -2215 5 | 5820 6 | -1440 7 | 3003 8 | -1898 9 | 1581 10 | 291 11 | 1302 12 | -1944 13 | -462 14 | 1288 15 | 216 16 | 882 17 | -83 18 | 958 19 | 796 20 | 133 21 | 510 22 | -274 23 | -397 24 | -23 25 | 290 26 | -230 27 | -229 28 | 360 29 | 15 30 | 230 31 | 110 32 | -23 33 | -98 34 | -152 35 | -121 36 | -69 37 | 78 38 | 47 39 | 57 40 | -9 41 | 3 42 | 96 43 | -247 44 | 48 45 | -8 46 | -16 47 | 7 48 | -12 49 | -107 50 | -24 51 | 65 52 | -56 53 | -50 54 | 2 55 | -24 56 | 10 57 | -4 58 | -32 59 | 8 60 | -11 61 | 28 62 | 9 63 | -20 64 | 18 65 | -18 66 | 11 67 | 9 68 | 10 69 | -6 70 | -15 71 | -14 72 | 5 73 | 6 74 | -23 75 | 10 76 | 3 77 | -7 78 | 23 79 | 6 80 | -4 81 | 9 82 | -13 83 | 4 84 | 9 85 | -11 86 | -4 87 | 12 88 | -5 89 | 7 90 | 2 91 | 6 92 | 4 93 | -2 94 | 1 95 | 10 96 | 2 97 | 7 98 | 2 99 | -6 100 | 5 101 | 5 102 | -3 103 | -5 104 | -4 105 | -1 106 | 0 107 | 2 108 | -8 109 | -3 110 | -2 111 | 7 112 | -4 113 | 4 114 | 1 115 | -2 116 | -3 117 | 6 118 | 7 119 | -2 120 | -1 121 | 0 122 | -3 123 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf1965.dat: -------------------------------------------------------------------------------- 1 | dgrf1965 2 | 10 6371.2 1965.0 3 | -30334 4 | -2119 5 | 5776 6 | -1662 7 | 2997 8 | -2016 9 | 1594 10 | 114 11 | 1297 12 | -2038 13 | -404 14 | 1292 15 | 240 16 | 856 17 | -165 18 | 957 19 | 804 20 | 148 21 | 479 22 | -269 23 | -390 24 | 13 25 | 252 26 | -269 27 | -219 28 | 358 29 | 19 30 | 254 31 | 128 32 | -31 33 | -126 34 | -157 35 | -97 36 | -62 37 | 81 38 | 45 39 | 61 40 | -11 41 | 8 42 | 100 43 | -228 44 | 68 45 | 4 46 | -32 47 | 1 48 | -8 49 | -111 50 | -7 51 | 75 52 | -57 53 | -61 54 | 4 55 | -27 56 | 13 57 | -2 58 | -26 59 | 6 60 | -6 61 | 26 62 | 13 63 | -23 64 | 1 65 | -12 66 | 13 67 | 5 68 | 7 69 | -4 70 | -12 71 | -14 72 | 9 73 | 0 74 | -16 75 | 8 76 | 4 77 | -1 78 | 24 79 | 11 80 | -3 81 | 4 82 | -17 83 | 8 84 | 10 85 | -22 86 | 2 87 | 15 88 | -13 89 | 7 90 | 10 91 | -4 92 | -1 93 | -5 94 | -1 95 | 10 96 | 5 97 | 10 98 | 1 99 | -4 100 | -2 101 | 1 102 | -2 103 | -3 104 | 2 105 | 2 106 | 1 107 | -5 108 | 2 109 | -2 110 | 6 111 | 4 112 | -4 113 | 4 114 | 0 115 | 0 116 | -2 117 | 2 118 | 3 119 | 2 120 | 0 121 | 0 122 | -6 123 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf1970.dat: -------------------------------------------------------------------------------- 1 | dgrf1970 2 | 10 6371.2 1970.0 3 | -30220 4 | -2068 5 | 5737 6 | -1781 7 | 3000 8 | -2047 9 | 1611 10 | 25 11 | 1287 12 | -2091 13 | -366 14 | 1278 15 | 251 16 | 838 17 | -196 18 | 952 19 | 800 20 | 167 21 | 461 22 | -266 23 | -395 24 | 26 25 | 234 26 | -279 27 | -216 28 | 359 29 | 26 30 | 262 31 | 139 32 | -42 33 | -139 34 | -160 35 | -91 36 | -56 37 | 83 38 | 43 39 | 64 40 | -12 41 | 15 42 | 100 43 | -212 44 | 72 45 | 2 46 | -37 47 | 3 48 | -6 49 | -112 50 | 1 51 | 72 52 | -57 53 | -70 54 | 1 55 | -27 56 | 14 57 | -4 58 | -22 59 | 8 60 | -2 61 | 23 62 | 13 63 | -23 64 | -2 65 | -11 66 | 14 67 | 6 68 | 7 69 | -2 70 | -15 71 | -13 72 | 6 73 | -3 74 | -17 75 | 5 76 | 6 77 | 0 78 | 21 79 | 11 80 | -6 81 | 3 82 | -16 83 | 8 84 | 10 85 | -21 86 | 2 87 | 16 88 | -12 89 | 6 90 | 10 91 | -4 92 | -1 93 | -5 94 | 0 95 | 10 96 | 3 97 | 11 98 | 1 99 | -2 100 | -1 101 | 1 102 | -3 103 | -3 104 | 1 105 | 2 106 | 1 107 | -5 108 | 3 109 | -1 110 | 4 111 | 6 112 | -4 113 | 4 114 | 0 115 | 1 116 | -1 117 | 0 118 | 3 119 | 3 120 | 1 121 | -1 122 | -4 123 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf1975.dat: -------------------------------------------------------------------------------- 1 | dgrf1975 2 | 10 6371.2 1975.0 3 | -30100 4 | -2013 5 | 5675 6 | -1902 7 | 3010 8 | -2067 9 | 1632 10 | -68 11 | 1276 12 | -2144 13 | -333 14 | 1260 15 | 262 16 | 830 17 | -223 18 | 946 19 | 791 20 | 191 21 | 438 22 | -265 23 | -405 24 | 39 25 | 216 26 | -288 27 | -218 28 | 356 29 | 31 30 | 264 31 | 148 32 | -59 33 | -152 34 | -159 35 | -83 36 | -49 37 | 88 38 | 45 39 | 66 40 | -13 41 | 28 42 | 99 43 | -198 44 | 75 45 | 1 46 | -41 47 | 6 48 | -4 49 | -111 50 | 11 51 | 71 52 | -56 53 | -77 54 | 1 55 | -26 56 | 16 57 | -5 58 | -14 59 | 10 60 | 0 61 | 22 62 | 12 63 | -23 64 | -5 65 | -12 66 | 14 67 | 6 68 | 6 69 | -1 70 | -16 71 | -12 72 | 4 73 | -8 74 | -19 75 | 4 76 | 6 77 | 0 78 | 18 79 | 10 80 | -10 81 | 1 82 | -17 83 | 7 84 | 10 85 | -21 86 | 2 87 | 16 88 | -12 89 | 7 90 | 10 91 | -4 92 | -1 93 | -5 94 | -1 95 | 10 96 | 4 97 | 11 98 | 1 99 | -3 100 | -2 101 | 1 102 | -3 103 | -3 104 | 1 105 | 2 106 | 1 107 | -5 108 | 3 109 | -2 110 | 4 111 | 5 112 | -4 113 | 4 114 | -1 115 | 1 116 | -1 117 | 0 118 | 3 119 | 3 120 | 1 121 | -1 122 | -5 123 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf1980.dat: -------------------------------------------------------------------------------- 1 | dgrf1980 2 | 10 6371.2 1980.0 3 | -29992 4 | -1956 5 | 5604 6 | -1997 7 | 3027 8 | -2129 9 | 1663 10 | -200 11 | 1281 12 | -2180 13 | -336 14 | 1251 15 | 271 16 | 833 17 | -252 18 | 938 19 | 782 20 | 212 21 | 398 22 | -257 23 | -419 24 | 53 25 | 199 26 | -297 27 | -218 28 | 357 29 | 46 30 | 261 31 | 150 32 | -74 33 | -151 34 | -162 35 | -78 36 | -48 37 | 92 38 | 48 39 | 66 40 | -15 41 | 42 42 | 93 43 | -192 44 | 71 45 | 4 46 | -43 47 | 14 48 | -2 49 | -108 50 | 17 51 | 72 52 | -59 53 | -82 54 | 2 55 | -27 56 | 21 57 | -5 58 | -12 59 | 16 60 | 1 61 | 18 62 | 11 63 | -23 64 | -2 65 | -10 66 | 18 67 | 6 68 | 7 69 | 0 70 | -18 71 | -11 72 | 4 73 | -7 74 | -22 75 | 4 76 | 9 77 | 3 78 | 16 79 | 6 80 | -13 81 | -1 82 | -15 83 | 5 84 | 10 85 | -21 86 | 1 87 | 16 88 | -12 89 | 9 90 | 9 91 | -5 92 | -3 93 | -6 94 | -1 95 | 9 96 | 7 97 | 10 98 | 2 99 | -6 100 | -5 101 | 2 102 | -4 103 | -4 104 | 1 105 | 2 106 | 0 107 | -5 108 | 3 109 | -2 110 | 6 111 | 5 112 | -4 113 | 3 114 | 0 115 | 1 116 | -1 117 | 2 118 | 4 119 | 3 120 | 0 121 | 0 122 | -6 123 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf1995.dat: -------------------------------------------------------------------------------- 1 | dgrf1995 2 | 10 6371.2 1995.0 3 | -29692 4 | -1784 5 | 5306 6 | -2200 7 | 3070 8 | -2366 9 | 1681 10 | -413 11 | 1335 12 | -2267 13 | -262 14 | 1249 15 | 302 16 | 759 17 | -427 18 | 940 19 | 780 20 | 262 21 | 290 22 | -236 23 | -418 24 | 97 25 | 122 26 | -306 27 | -214 28 | 352 29 | 46 30 | 235 31 | 165 32 | -118 33 | -143 34 | -166 35 | -55 36 | -17 37 | 107 38 | 68 39 | 67 40 | -17 41 | 68 42 | 72 43 | -170 44 | 67 45 | -1 46 | -58 47 | 19 48 | 1 49 | -93 50 | 36 51 | 77 52 | -72 53 | -69 54 | 1 55 | -25 56 | 28 57 | 4 58 | 5 59 | 24 60 | 4 61 | 17 62 | 8 63 | -24 64 | -2 65 | -6 66 | 25 67 | 6 68 | 11 69 | -6 70 | -21 71 | -9 72 | 8 73 | -14 74 | -23 75 | 9 76 | 15 77 | 6 78 | 11 79 | -5 80 | -16 81 | -7 82 | -4 83 | 4 84 | 9 85 | -20 86 | 3 87 | 15 88 | -10 89 | 12 90 | 8 91 | -6 92 | -8 93 | -8 94 | -1 95 | 8 96 | 10 97 | 5 98 | -2 99 | -8 100 | -8 101 | 3 102 | -3 103 | -6 104 | 1 105 | 2 106 | 0 107 | -4 108 | 4 109 | -1 110 | 5 111 | 4 112 | -5 113 | 2 114 | -1 115 | 2 116 | -2 117 | 5 118 | 1 119 | 1 120 | -2 121 | 0 122 | -7 123 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf1945.dat: -------------------------------------------------------------------------------- 1 | dgrf1945 2 | 10 6371.2 1945.0 3 | -30594 4 | -2285 5 | 5810 6 | -1244 7 | 2990 8 | -1702 9 | 1578 10 | 477 11 | 1282 12 | -1834 13 | -499 14 | 1255 15 | 186 16 | 913 17 | -11 18 | 944 19 | 776 20 | 144 21 | 544 22 | -276 23 | -421 24 | -55 25 | 304 26 | -178 27 | -253 28 | 346 29 | -12 30 | 194 31 | 95 32 | -20 33 | -67 34 | -142 35 | -119 36 | -82 37 | 82 38 | 59 39 | 57 40 | 6 41 | 6 42 | 100 43 | -246 44 | 16 45 | -25 46 | -9 47 | 21 48 | -16 49 | -104 50 | -39 51 | 70 52 | -40 53 | -45 54 | 0 55 | -18 56 | 0 57 | 2 58 | -29 59 | 6 60 | -10 61 | 28 62 | 15 63 | -17 64 | 29 65 | -22 66 | 13 67 | 7 68 | 12 69 | -8 70 | -21 71 | -5 72 | -12 73 | 9 74 | -7 75 | 7 76 | 2 77 | -10 78 | 18 79 | 7 80 | 3 81 | 2 82 | -11 83 | 5 84 | -21 85 | -27 86 | 1 87 | 17 88 | -11 89 | 29 90 | 3 91 | -9 92 | 16 93 | 4 94 | -3 95 | 9 96 | -4 97 | 6 98 | -3 99 | 1 100 | -4 101 | 8 102 | -3 103 | 11 104 | 5 105 | 1 106 | 1 107 | 2 108 | -20 109 | -5 110 | -1 111 | -1 112 | -6 113 | 8 114 | 6 115 | -1 116 | -4 117 | -3 118 | -2 119 | 5 120 | 0 121 | -2 122 | -2 123 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf1950.dat: -------------------------------------------------------------------------------- 1 | dgrf1950 2 | 10 6371.2 1950.0 3 | -30554 4 | -2250 5 | 5815 6 | -1341 7 | 2998 8 | -1810 9 | 1576 10 | 381 11 | 1297 12 | -1889 13 | -476 14 | 1274 15 | 206 16 | 896 17 | -46 18 | 954 19 | 792 20 | 136 21 | 528 22 | -278 23 | -408 24 | -37 25 | 303 26 | -210 27 | -240 28 | 349 29 | 3 30 | 211 31 | 103 32 | -20 33 | -87 34 | -147 35 | -122 36 | -76 37 | 80 38 | 54 39 | 57 40 | -1 41 | 4 42 | 99 43 | -247 44 | 33 45 | -16 46 | -12 47 | 12 48 | -12 49 | -105 50 | -30 51 | 65 52 | -55 53 | -35 54 | 2 55 | -17 56 | 1 57 | 0 58 | -40 59 | 10 60 | -7 61 | 36 62 | 5 63 | -18 64 | 19 65 | -16 66 | 22 67 | 15 68 | 5 69 | -4 70 | -22 71 | -1 72 | 0 73 | 11 74 | -21 75 | 15 76 | -8 77 | -13 78 | 17 79 | 5 80 | -4 81 | -1 82 | -17 83 | 3 84 | -7 85 | -24 86 | -1 87 | 19 88 | -25 89 | 12 90 | 10 91 | 2 92 | 5 93 | 2 94 | -5 95 | 8 96 | -2 97 | 8 98 | 3 99 | -11 100 | 8 101 | -7 102 | -8 103 | 4 104 | 13 105 | -1 106 | -2 107 | 13 108 | -10 109 | -4 110 | 2 111 | 4 112 | -3 113 | 12 114 | 6 115 | 3 116 | -3 117 | 2 118 | 6 119 | 10 120 | 11 121 | 3 122 | 8 123 | -------------------------------------------------------------------------------- /assimilate/app/models/gp3d.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import george 3 | from george.kernels import Matern52Kernel, ExpSquaredKernel 4 | 5 | def sph_to_xyz(lat, lon): 6 | lon = lon * np.pi / 180. 7 | lat = lat * np.pi / 180. 8 | x = np.cos(lat) * np.cos(lon) 9 | y = np.cos(lat) * np.sin(lon) 10 | z = np.sin(lat) 11 | 12 | return x, y, z 13 | 14 | class GP3D: 15 | def __init__(self): 16 | pass 17 | 18 | def train(self, df, t, stdev): 19 | x, y, z = sph_to_xyz(df['station.latitude'].values, df['station.longitude'].values) 20 | 21 | kernel = 0.0809**2 * Matern52Kernel(0.0648, ndim=3) + 0.0845**2 * ExpSquaredKernel(0.481, ndim=3) 22 | self.gp = george.GP(kernel) 23 | self.gp.compute(np.column_stack((x, y, z)), stdev + 1e-3) 24 | self.z = t 25 | 26 | def predict(self, latitude, longitude): 27 | x, y, z = sph_to_xyz(latitude, longitude) 28 | xyz = np.column_stack((x.flatten(), y.flatten(), z.flatten())) 29 | pred, var = self.gp.predict(self.z, xyz, return_cov=False, return_var=True) 30 | self.stdev = np.sqrt(var.reshape(x.shape)) 31 | return pred.reshape(x.shape) 32 | -------------------------------------------------------------------------------- /iri2020/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Michael Hirsch, Ph.D. 4 | Copyright (c) 2017 Ronald Ilma 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /iri2020/src/meson.build: -------------------------------------------------------------------------------- 1 | project('iri2020', 'fortran', default_options : ['default_library=static']) 2 | 3 | add_project_arguments('-O3', '-ffast-math', '-march=znver2', language: 'fortran') 4 | 5 | iri2020= library('iri2020', 'irisub.for', 'irifun.for', 'iritec.for', 'iridreg.for', 'igrf.for', 'cira.for', 'iriflip.for', 'rocdrift.for', 'irirtam.for', 6 | fortran_args: '-w') 7 | 8 | driver = executable('iri2020_driver', 'iri2020_driver.F90', 9 | link_with: iri2020, 10 | fortran_args: '-DBIN_DIR='+ '"' + meson.build_root() + '"') 11 | 12 | map = executable('irimap', 'irimap.F90', 13 | link_with: iri2020, 14 | fortran_args: '-DBIN_DIR='+ '"' + meson.build_root() + '"') 15 | 16 | ts = executable('iri_ts', 'iri_ts.F90', 17 | link_with: iri2020, 18 | fortran_args: '-DBIN_DIR='+ '"' + meson.build_root() + '"') 19 | 20 | opt = executable('iri_opt', 'iri_opt.F90', 21 | link_with: iri2020, 22 | fortran_args: '-DBIN_DIR='+ '"' + meson.build_root() + '"') 23 | 24 | irtam = executable('irtam_driver', 'irtam_driver.F90', 25 | link_with: iri2020, 26 | fortran_args: '-DBIN_DIR='+ '"' + meson.build_root() + '"') 27 | 28 | testexe = executable('testiri2020', 'test.f90', 29 | link_with: iri2020) 30 | 31 | test('IRI2016', testexe) 32 | -------------------------------------------------------------------------------- /diffusion/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:12.8.0-cudnn-devel-ubuntu24.04 2 | 3 | RUN apt-get update && apt-get -y install unzip build-essential cmake dumb-init libgfortran5 gfortran curl git python3 python3-dev 4 | 5 | RUN curl -LsSf https://astral.sh/uv/install.sh | sh 6 | RUN /root/.local/bin/uv tool install 'git+https://github.com/bluss/sysconfigpatcher' 7 | RUN /root/.local/bin/uv venv --python 3.13 /app/.venv 8 | RUN /root/.local/bin/sysconfigpatcher /root/.local/share/uv/python/cpython-* 9 | RUN chmod go+rx /root && chmod -R go+rx /root/.local 10 | 11 | ENV PATH="/app/.venv/bin:$PATH" 12 | COPY pyproject.toml /app/pyproject.toml 13 | 14 | WORKDIR /app 15 | RUN /root/.local/bin/uv sync 16 | 17 | COPY --from=iri2020 /build/ /build/ 18 | 19 | ENV HOME=/tmp 20 | 21 | USER 1000 22 | RUN hf download --repo-type dataset arodland/IRI-iono-maps 23 | 24 | COPY app /app 25 | 26 | # ENV OPENBLAS_NUM_THREADS=4 27 | # ENV OMP_NUM_THREADS=4 28 | 29 | ENTRYPOINT ["/usr/bin/dumb-init", "--"] 30 | WORKDIR /app 31 | CMD uwsgi --http :${DIFFUSION_PORT} --master --lazy --single-interpreter --http-timeout 600 --wsgi server:app --processes 4 --enable-metrics --carbon-use-metrics --carbon-name-resolve --carbon ${STATSD_HOST}:2003 --carbon-id prop-pred4d --die-on-term 32 | -------------------------------------------------------------------------------- /diffusion/CRUSH.md: -------------------------------------------------------------------------------- 1 | # CRUSH.md - Codebase Guidelines for AI Agents 2 | 3 | ## Project Setup 4 | - Uses Pixi for dependency management (pixi.toml) 5 | - Runs in containerized environment via `./run` script 6 | - PyTorch + Lightning for deep learning, Diffusers for diffusion models 7 | 8 | ## Commands 9 | ```bash 10 | # Build the container 11 | podman -r build -t prop-diffusion . 12 | 13 | # Run training scripts 14 | ./run python app/train_diffusion.py 15 | ./run python app/train_guidance.py 16 | ./run python app/train_vae.py 17 | 18 | # Run generation/inference 19 | ./run python app/generate.py --checkpoint path/to/checkpoint.ckpt 20 | ./run python app/generate_guided.py 21 | ``` 22 | 23 | ## Code Style 24 | - **Imports**: stdlib → third-party → local (torch, lightning as L, diffusers) 25 | - **Type hints**: Use for CLI args (Path, int), minimal elsewhere 26 | - **Naming**: snake_case functions/vars, PascalCase classes 27 | - **No docstrings** except CLI main functions (one-line descriptions) 28 | - **Device handling**: `device = torch.device("cuda" if torch.cuda.is_available() else "cpu")` 29 | - **Context managers**: Use `torch.no_grad()` or `torch.inference_mode()` for inference 30 | - **Error handling**: Minimal - rely on framework defaults 31 | - **File organization**: Separate model definitions (models.py) from training scripts 32 | -------------------------------------------------------------------------------- /diffusion/app/train_diffusion.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lightning as L 3 | from lightning.pytorch.callbacks import ModelCheckpoint, StochasticWeightAveraging 4 | from lightning.pytorch.loggers import TensorBoardLogger 5 | from models import DiffusionModel, IRIData 6 | import diffusers 7 | 8 | 9 | if __name__ == "__main__": 10 | if len(sys.argv) > 1: 11 | model = DiffusionModel.load_from_checkpoint(sys.argv[1]) 12 | # Don't load VAE from checkpoint 13 | model.vae = diffusers.models.AutoencoderTiny.from_pretrained("./taesd-iono-finetuned") 14 | else: 15 | model = DiffusionModel() 16 | 17 | checkpoint_callback = ModelCheckpoint(dirpath="checkpoints", filename='vdiffusion-' + 18 | '{epoch}-{val_loss:.2g}', save_top_k=1, monitor="val_loss", mode="min") 19 | data = IRIData("combined", train_batch=64, add_noise=0.001) 20 | trainer = L.Trainer(max_epochs=100, 21 | log_every_n_steps=10, 22 | # accumulate_grad_batches=4, 23 | precision="bf16-mixed", 24 | callbacks=[ 25 | checkpoint_callback, 26 | ModelCheckpoint(dirpath="checkpoints", filename='vdiffusion-averaged'), 27 | StochasticWeightAveraging(swa_lrs=1e-5), 28 | ], 29 | logger=TensorBoardLogger("lightning_logs", name="vdiffusion")) 30 | trainer.fit(model, data) 31 | -------------------------------------------------------------------------------- /diffusion/app/train_conditioned_diffusion.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lightning as L 3 | from lightning.pytorch.callbacks import ModelCheckpoint, StochasticWeightAveraging 4 | from lightning.pytorch.loggers import TensorBoardLogger 5 | from models import ConditionedDiffusionModel, IRIData 6 | import diffusers 7 | 8 | 9 | if __name__ == "__main__": 10 | if len(sys.argv) > 1: 11 | model = ConditionedDiffusionModel.load_from_checkpoint(sys.argv[1]) 12 | # Don't load VAE from checkpoint 13 | model.vae = diffusers.models.AutoencoderTiny.from_pretrained("./taesd-iono-finetuned") 14 | else: 15 | model = ConditionedDiffusionModel(pred_type='epsilon') 16 | 17 | checkpoint_callback = ModelCheckpoint(dirpath="checkpoints", filename='cdiffusion-' + 18 | '{epoch}-{val_loss:.2g}', save_top_k=1, monitor="val_loss", mode="min") 19 | data = IRIData("combined", train_batch=64, add_noise=0.001) 20 | trainer = L.Trainer(max_epochs=100, 21 | log_every_n_steps=50, 22 | # accumulate_grad_batches=4, 23 | precision="bf16-mixed", 24 | callbacks=[ 25 | checkpoint_callback, 26 | ModelCheckpoint(dirpath="checkpoints", filename='cdiffusion-averaged'), 27 | StochasticWeightAveraging(swa_lrs=1e-5), 28 | ], 29 | logger=TensorBoardLogger("lightning_logs", name="cdiffusion")) 30 | trainer.fit(model, data) 31 | -------------------------------------------------------------------------------- /diffusion/app/train_vprediction.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import lightning as L 3 | from lightning.pytorch.callbacks import ModelCheckpoint, StochasticWeightAveraging 4 | from lightning.pytorch.loggers import TensorBoardLogger 5 | from models import ConditionedDiffusionModel, IRIData 6 | import diffusers 7 | 8 | 9 | if __name__ == "__main__": 10 | if len(sys.argv) > 1: 11 | model = ConditionedDiffusionModel.load_from_checkpoint(sys.argv[1]) 12 | # Don't load VAE from checkpoint 13 | model.vae = diffusers.models.AutoencoderTiny.from_pretrained("./taesd-iono-finetuned") 14 | else: 15 | model = ConditionedDiffusionModel(pred_type='v_prediction') 16 | 17 | checkpoint_callback = ModelCheckpoint(dirpath="checkpoints", filename='vprediction-' + 18 | '{epoch}-{val_loss:.2g}', save_top_k=1, monitor="val_loss", mode="min") 19 | data = IRIData("combined", train_batch=64, add_noise=0.000) 20 | trainer = L.Trainer(max_epochs=100, 21 | log_every_n_steps=50, 22 | # accumulate_grad_batches=4, 23 | precision="bf16-mixed", 24 | callbacks=[ 25 | checkpoint_callback, 26 | ModelCheckpoint(dirpath="checkpoints", filename='vprediction-averaged'), 27 | StochasticWeightAveraging(swa_lrs=1e-5), 28 | ], 29 | logger=TensorBoardLogger("lightning_logs", name="vprediction")) 30 | trainer.fit(model, data) 31 | -------------------------------------------------------------------------------- /iturhfprop/app/templates/planner_table.html.ep: -------------------------------------------------------------------------------- 1 | <% 2 | if ($start_hour eq 'CURRENT') { 3 | $start_hour = $run_info->{hour}; 4 | } elsif ($start_hour = 'ZERO_LOCAL') { 5 | $start_hour = (0-$tz_offset) % 24; 6 | } else { 7 | $start_hour = 0; 8 | } 9 | %> 10 | 11 | 12 | 13 | % for my $hh (0 .. 23) { 14 | % my $hour = ($start_hour + $hh) % 24; 15 | % if ($hour == $run_info->{hour}) { 16 | 17 | % } else { 18 | 19 | % } 20 | % } 21 | 22 | 23 | 24 | % for my $hh (0 .. 23) { 25 | % my $hour = ($start_hour + $hh + $tz_offset) % 24; 26 | 27 | % } 28 | 29 | 30 | % my $i = 0; 31 | % for my $row (@$table) { 32 | % my $band = $bands->[$i++]; 33 | 34 | 35 | % for my $hh (0 .. 23) { 36 | % my $hour = ($start_hour + $hh) % 24; 37 | % my $col = $row->[$hour]; 38 | 45 | % } 46 | 47 | 48 | % } 49 |
<%= sprintf("%02d", $hour) %>N
<%= $band->[0] %>{bcr}, $col->{pop}, 30+$col->{pr}) %>"> 41 | % if(!$col->{blank}) { 42 | <%= $col->{smeter} %> 43 | % } 44 | <%= $row->[0]{noise_s} %>
50 | -------------------------------------------------------------------------------- /diffusion/app/generate.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torchvision as tv 4 | import diffusers 5 | import jsonargparse 6 | from pathlib import Path 7 | from models import DiffusionModel 8 | 9 | def main( 10 | checkpoint: Path = Path("checkpoint.ckpt"), 11 | num_timesteps: int = 1000, 12 | num_samples: int = 1, 13 | seed: int = 0, 14 | channels: int = 3, 15 | ): 16 | """Generates images from a trained diffusion model.""" 17 | 18 | torch.manual_seed(seed) 19 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 20 | with device: 21 | model = DiffusionModel() 22 | checkpoint = torch.load(checkpoint, map_location=device, weights_only=True) 23 | model.load_state_dict(checkpoint["state_dict"]) 24 | model.eval() 25 | 26 | scheduler = diffusers.schedulers.DDIMScheduler() 27 | pipe = diffusers.DDIMPipeline(model.model, scheduler) 28 | pipe = pipe.to(device=device) 29 | 30 | with torch.inference_mode(): 31 | (pil_images, ) = pipe( 32 | batch_size=num_samples, 33 | num_inference_steps=num_timesteps, 34 | output_type="pil", 35 | return_dict=False 36 | ) 37 | images = torch.stack([tv.transforms.functional.to_tensor(pil_image.crop((0, 0, 361, 181))) 38 | for pil_image in pil_images]) 39 | image_grid = tv.utils.make_grid(images, nrow=math.ceil(math.sqrt(num_samples))) 40 | 41 | filename = "out/generated.png" 42 | tv.utils.save_image(image_grid, filename) 43 | print(f"Generated images saved to {filename}") 44 | 45 | 46 | if __name__ == "__main__": 47 | jsonargparse.CLI(main) 48 | -------------------------------------------------------------------------------- /utils/migrate/app/migrate.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use warnings; 4 | 5 | use DBI; 6 | 7 | my $giro_db = DBI->connect( 8 | "dbi:Pg:dbname=giro;host=$ENV{DB_HOST}", 9 | $ENV{DB_USER}, 10 | $ENV{DB_PASSWORD}, 11 | { RaiseError => 1 }, 12 | ); 13 | 14 | my $prop_db = DBI->connect( 15 | "dbi:Pg:dbname=prop;host=$ENV{DB_HOST}", 16 | $ENV{DB_USER}, 17 | $ENV{DB_PASSWORD}, 18 | { RaiseError => 1 }, 19 | ); 20 | 21 | my $sth = $giro_db->prepare("select * from station order by id asc"); 22 | $sth->execute; 23 | 24 | while (my $station = $sth->fetchrow_hashref) { 25 | $prop_db->do("insert into station (id, name, code, longitude, latitude, giro, use_for_essn) values (?, ?, ?, ?, ?, ?, ?)", 26 | undef, 27 | $station->{id}, $station->{name}, $station->{code}, $station->{longitude}, $station->{latitude}, 28 | (defined($station->{active}) ? 0 : 1), 29 | 1, 30 | ); 31 | } 32 | 33 | $sth = $giro_db->prepare("select * from measurement order by time asc"); 34 | $sth->execute; 35 | 36 | my $n = 0; 37 | 38 | $prop_db->begin_work; 39 | while (my $meas = $sth->fetchrow_hashref) { 40 | delete $meas->{id}; 41 | my $alt = delete $meas->{altitude}; 42 | my $source = defined($alt) ? 'giro' : 'noaa'; 43 | 44 | my @keys = keys %$meas; 45 | my @vals = @$meas{@keys}; 46 | for my $val (@vals) { 47 | if (defined($val) && $val eq "") { 48 | $val = undef; 49 | } 50 | } 51 | 52 | my $sql = "insert into measurement(source, " . join(',', @keys) . ") values (?," . join(',', ('?')x@keys) . ")"; 53 | $prop_db->do($sql, undef, $source, @vals); 54 | if ((++$n) % 10000 == 0) { 55 | print "$n\n"; 56 | $prop_db->commit; 57 | $prop_db->begin_work; 58 | } 59 | } 60 | 61 | $prop_db->commit; 62 | -------------------------------------------------------------------------------- /iri2020/data/igrf/igrf2020s.dat: -------------------------------------------------------------------------------- 1 | igrf2020s 2 | 8 6371.2 2025.0 3 | 5.7 4 | 7.4 5 | -25.9 6 | -11 7 | -7 8 | -30.2 9 | -2.1 10 | -22.4 11 | 2.2 12 | -5.9 13 | 6 14 | 3.1 15 | -1.1 16 | -12 17 | 0.5 18 | -1.2 19 | -1.6 20 | -0.1 21 | -5.9 22 | 6.5 23 | 5.2 24 | 3.6 25 | -5.1 26 | -5 27 | -0.3 28 | 0.5 29 | 0 30 | -0.6 31 | 2.5 32 | 0.2 33 | -0.6 34 | 1.3 35 | 3 36 | 0.9 37 | 0.3 38 | -0.5 39 | -0.3 40 | 0 41 | 0.4 42 | -1.6 43 | 1.3 44 | -1.3 45 | -1.4 46 | 0.8 47 | 0 48 | 0 49 | 0.9 50 | 1 51 | -0.1 52 | -0.2 53 | 0.6 54 | 0 55 | 0.6 56 | 0.7 57 | -0.8 58 | 0.1 59 | -0.2 60 | -0.5 61 | -1.1 62 | -0.8 63 | 0.1 64 | 0.8 65 | 0.3 66 | 0 67 | 0.1 68 | -0.2 69 | -0.1 70 | 0.6 71 | 0.4 72 | -0.2 73 | -0.1 74 | 0.5 75 | 0.4 76 | -0.3 77 | 0.3 78 | -0.4 79 | -0.1 80 | 0.5 81 | 0.4 82 | 0 83 | 0 84 | 0 85 | 0 86 | 0 87 | 0 88 | 0 89 | 0 90 | 0 91 | 0 92 | 0 93 | 0 94 | 0 95 | 0 96 | 0 97 | 0 98 | 0 99 | 0 100 | 0 101 | 0 102 | 0 103 | 0 104 | 0 105 | 0 106 | 0 107 | 0 108 | 0 109 | 0 110 | 0 111 | 0 112 | 0 113 | 0 114 | 0 115 | 0 116 | 0 117 | 0 118 | 0 119 | 0 120 | 0 121 | 0 122 | 0 123 | 0 124 | 0 125 | 0 126 | 0 127 | 0 128 | 0 129 | 0 130 | 0 131 | 0 132 | 0 133 | 0 134 | 0 135 | 0 136 | 0 137 | 0 138 | 0 139 | 0 140 | 0 141 | 0 142 | 0 143 | 0 144 | 0 145 | 0 146 | 0 147 | 0 148 | 0 149 | 0 150 | 0 151 | 0 152 | 0 153 | 0 154 | 0 155 | 0 156 | 0 157 | 0 158 | 0 159 | 0 160 | 0 161 | 0 162 | 0 163 | 0 164 | 0 165 | 0 166 | 0 167 | 0 168 | 0 169 | 0 170 | 0 171 | 0 172 | 0 173 | 0 174 | 0 175 | 0 176 | 0 177 | 0 178 | 0 179 | 0 180 | 0 181 | 0 182 | 0 183 | 0 184 | 0 185 | 0 186 | 0 187 | 0 188 | 0 189 | 0 190 | 0 191 | 0 192 | 0 193 | 0 194 | 0 195 | 0 196 | 0 197 | 0 198 | -------------------------------------------------------------------------------- /iturhfprop/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM perl:5.42.0 AS perl-base 2 | RUN apt-get update && apt-get install -y build-essential libhdf5-dev 3 | 4 | FROM perl-base AS perl-cpm 5 | RUN wget -O /usr/local/bin/cpm https://raw.githubusercontent.com/skaji/cpm/0.997015/cpm && chmod +x /usr/local/bin/cpm 6 | 7 | FROM perl-base as build-iturhfprop 8 | 9 | COPY ITU-R-HF /src 10 | 11 | WORKDIR /src/P372/Linux 12 | RUN make OPTIMIZE='-O3 -flto -march=native' 13 | # RUN make OPTIMIZE='-O1 -g' 14 | 15 | WORKDIR /src/P533/Linux 16 | RUN make OPTIMIZE='-O3 -flto -march=native' 17 | # RUN make OPTIMIZE='-O1 -g' 18 | 19 | WORKDIR /src/ITURHFProp/Linux 20 | RUN make OPTIMIZE='-O3 -flto -march=native' 21 | # RUN make OPTIMIZE='-O1 -g' 22 | 23 | FROM perl-cpm AS perl-deps 24 | RUN MAKEFLAGS=-j24 cpm install -g PDL 25 | RUN cpm install -g MAUKE/EV-4.34_002.tar.gz # Stable EV is broken on perl 5.42 26 | COPY cpanfile . 27 | RUN cpm install -g 28 | 29 | FROM perl-base AS final 30 | RUN apt-get update && apt-get install dumb-init 31 | 32 | COPY --from=build-iturhfprop /src/ITURHFProp/Linux/ITURHFProp /usr/local/bin 33 | COPY --from=build-iturhfprop /src/P372/Linux/libp372.so /usr/local/lib 34 | COPY --from=build-iturhfprop /src/P533/Linux/libp533.so /usr/local/lib 35 | COPY ITU-R-HF/ITURHFProp/Data /opt/iturhfprop/data 36 | COPY antennas /opt/antennas 37 | RUN ldconfig 38 | 39 | COPY --from=perl-deps /usr/local/lib/perl5 /usr/local/lib/perl5 40 | COPY --from=perl-deps /usr/local/bin/* /usr/local/bin 41 | COPY ./app /app 42 | 43 | ENTRYPOINT ["/usr/bin/dumb-init", "--verbose", "--"] 44 | ENV MOJO_REVERSE_PROXY=1 45 | ENV PATH_PREFIX=/hfprop/ 46 | CMD perl -I/app/lib /app/main.pl prefork -m production -w 2 -s 16 -l "http://*:$ITURHFPROP_PORT" 47 | # CMD perl -I/app/lib /app/main.pl prefork -w 2 -s 16 -l "http://*:$ITURHFPROP_PORT" 48 | # CMD perl -I/app/lib /app/main.pl daemon -l "http://*:$ITURHFPROP_PORT" 49 | -------------------------------------------------------------------------------- /iri2020/src/iri_opt.F90: -------------------------------------------------------------------------------- 1 | program basictest 2 | use, intrinsic:: iso_fortran_env, only: stderr=>error_unit, stdout=>output_unit 3 | implicit none 4 | 5 | logical :: jf(50) 6 | integer, parameter :: jmag = 0 7 | integer :: iyyyy, mmdd, Nalt 8 | real :: glat, glon, dhour 9 | integer :: ymdhms(6) 10 | real:: alt_km_range(3) 11 | real:: TECtotal, TECtop, TECbot 12 | real:: sfi, ssn, ig 13 | 14 | #ifndef BIN_DIR 15 | #define BIN_DIR '.' 16 | #endif 17 | character(*), parameter :: datadir = BIN_DIR // '/iri2020/data' 18 | character(256) :: datadir1 19 | common /folders/ datadir1 20 | 21 | real :: oarr(100), outf(20,1000) 22 | real, allocatable :: altkm(:) 23 | character(80) :: argv 24 | integer :: i 25 | 26 | jf = .true. 27 | jf(4:6) = .false. 28 | jf(21:23) = .false. 29 | jf(28:30) = .false. 30 | jf(33:36) = .false. 31 | jf(39) = .false. 32 | jf(40) = .false. 33 | jf(47) = .false. 34 | 35 | datadir1 = datadir 36 | call read_ig_rz 37 | call readapf107 38 | 39 | do 40 | read(*,*, end=42) glat, glon, ssn, ymdhms(1), ymdhms(2), ymdhms(3), & 41 | ymdhms(4), ymdhms(5), ymdhms(6) 42 | 43 | iyyyy = ymdhms(1) 44 | mmdd = ymdhms(2) * 100 + ymdhms(3) 45 | dhour = ymdhms(4) + ymdhms(5) / 60. + ymdhms(6) / 3600. 46 | 47 | if(ssn.gt.-99.) then 48 | sfi = 63.75+ssn*(0.728+ssn*0.000089) 49 | ig=(-0.0031*ssn+1.5332)*ssn-11.5634 50 | jf(17) = .false. 51 | jf(25) = .false. 52 | jf(27) = .false. 53 | jf(32) = .false. 54 | else 55 | jf(17) = .true. 56 | jf(25) = .true. 57 | jf(27) = .true. 58 | jf(32) = .true. 59 | endif 60 | 61 | OARR(33)=ssn 62 | OARR(41)=sfi 63 | OARR(46)=sfi 64 | OARR(39)=ig 65 | 66 | call IRI_SUB(JF,JMAG,glat,glon,IYYYY,MMDD,DHOUR+25., & 67 | 0, 0, 1, & 68 | OUTF,OARR, datadir) 69 | 70 | write(stdout, '(8ES16.8)') glat, glon, oarr(1), oarr(91), oarr(36), oarr(36) * oarr(91), oarr(2), oarr(92) 71 | enddo 72 | 42 continue 73 | end program 74 | -------------------------------------------------------------------------------- /pred/app/fit_cs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import datetime as dt 3 | import time 4 | import copy 5 | import urllib.request, json 6 | import pandas as pd 7 | import numpy as np 8 | from sklearn.model_selection import train_test_split 9 | import george 10 | from kernel import kernel 11 | from cs import cs_to_stdev, stdev_to_cs 12 | 13 | from pandas.io.json import json_normalize 14 | 15 | def get_data(url=os.getenv("HISTORY_URI")): 16 | with urllib.request.urlopen(url) as res: 17 | data = json.loads(res.read().decode()) 18 | 19 | return data 20 | 21 | data = get_data() 22 | 23 | out = [] 24 | 25 | for station in data: 26 | x, y, cslist, sigma = [], [], [], [] 27 | 28 | for pt in station['history']: 29 | tm = (pd.to_datetime(pt[0]) - pd.Timestamp("1970-01-01")) // pd.Timedelta("1s") 30 | cs = pt[1] 31 | if cs < 10 and cs != -1: 32 | continue 33 | 34 | sd = cs_to_stdev(cs, adj100=True) 35 | fof2, mufd, hmf2 = pt[2:5] 36 | 37 | x.append(tm / 86400.) 38 | y.append(np.log(mufd)) 39 | cslist.append(cs) 40 | sigma.append(sd + 1e-3) 41 | 42 | if len(x) < 40: 43 | continue 44 | 45 | x = np.array(x) 46 | y = np.array(y) 47 | ymean = np.mean(y) 48 | y -= ymean 49 | cslist = np.array(cslist) 50 | sigma = np.array(sigma) 51 | 52 | x_train, x_test, y_train, y_test, cs_train, cs_test, sigma_train, sigma_test = train_test_split( 53 | x, y, cslist, sigma, test_size=0.25) 54 | 55 | gp = george.GP(kernel) 56 | gp.compute(x_train, sigma_train) 57 | 58 | pred, sd = gp.predict(y_train, x_test, return_var=True) 59 | sd = sd**0.5 60 | 61 | for i in range(len(pred)): 62 | print("%d\t%d\t%d\t%f\t%f\t%f" % ( 63 | station['id'], 64 | x_test[i], 65 | cs_test[i], 66 | pred[i], 67 | y_test[i], 68 | sd[i], 69 | ) 70 | ) 71 | -------------------------------------------------------------------------------- /iri2020/src/reference/irisubg.pyf: -------------------------------------------------------------------------------- 1 | ! -*- f90 -*- 2 | ! Note: the context of this file is case sensitive. 3 | 4 | python module irisub ! in 5 | interface ! in :irisub 6 | subroutine irisubg(jf,jmag,alati,along,iyyyy,mmdd,dhour,heibeg,heiend,heistp,dirdata,outf,oarr) ! in :irisub:irisubg.for 7 | logical dimension(50),intent(in) :: jf 8 | integer intent(in) :: jmag 9 | real intent(in) :: alati 10 | real intent(in) :: along 11 | integer intent(in) :: iyyyy 12 | integer intent(in) :: mmdd 13 | real intent(in) :: dhour 14 | real intent(in) :: heibeg 15 | real intent(in) :: heiend 16 | real intent(in) :: heistp 17 | character*256 intent(in) :: dirdata 18 | real dimension(30,1000),intent(out) :: outf 19 | real dimension(100),intent(out) :: oarr 20 | character*256 :: dirdata1 21 | common /folders/ dirdata1 22 | end subroutine irisubg 23 | subroutine irisubgl(jf,jmag,iyyyy,mmdd,dhour,coordl,lenl,dirdata,outf1,oarr1) ! in :irisub:irisubg.for 24 | logical dimension(50),intent(in) :: jf 25 | integer intent(in) :: jmag 26 | integer intent(in) :: iyyyy 27 | integer intent(in) :: mmdd 28 | real intent(in) :: dhour 29 | real dimension(lenl,3),intent(in) :: coordl 30 | integer, optional,intent(hide),depend(coordl) :: lenl=shape(coordl,0) 31 | character*256 intent(in) :: dirdata 32 | real dimension(30,lenl),intent(out),depend(lenl) :: outf1 33 | real dimension(100,lenl),intent(out),depend(lenl) :: oarr1 34 | character*256 :: dirdata1 35 | common /folders/ dirdata1 36 | end subroutine irisubgl 37 | end interface 38 | end python module irisub 39 | 40 | ! This file was auto-generated with f2py (version:2). 41 | ! See http://cens.ioc.ee/projects/f2py2e/ 42 | -------------------------------------------------------------------------------- /iri2020/src/test.f90: -------------------------------------------------------------------------------- 1 | program basictest 2 | use, intrinsic:: iso_fortran_env, only: stderr=>error_unit 3 | implicit none 4 | 5 | logical, parameter :: jf(50) = .true. 6 | integer, parameter :: jmag = 1, iyyyy=1980, mmdd=0321, dhour=12, Nalt = 21 7 | real, parameter :: glat=0., glon=0. 8 | real,parameter :: alt_km_range(3) = [100., 500., 20.] 9 | character(*), parameter :: datadir='../iri2020/data' 10 | 11 | real :: oarr(100), outf(20,1000), altkm(Nalt) 12 | integer :: i 13 | 14 | altkm(1) = alt_km_range(1) 15 | do i = 2,Nalt 16 | altkm(i) = altkm(i-1) + alt_km_range(3) 17 | enddo 18 | 19 | 20 | call IRI_SUB(JF,JMAG,glat,glon,IYYYY,MMDD,DHOUR+25., & 21 | alt_km_range(1), alt_km_range(2), alt_km_range(3), & 22 | OUTF,OARR, datadir) 23 | 24 | print '(A,ES10.3,A,F5.1,A)','NmF2 ',oarr(1),' [m^-3] hmF2 ',oarr(2),' [km] ' 25 | print '(A,F10.3,A,I3,A,F10.3)','F10.7 ',oarr(41), ' Ap ',int(oarr(51)),' B0 ',oarr(10) 26 | 27 | print *,'Altitude (km) Ne (m^-3)' 28 | do i = 1,Nalt 29 | print '(F10.3, ES15.7)',altkm(i), outf(1,i) 30 | enddo 31 | 32 | 33 | if (outf(1,i-1) < 0) then 34 | write(stderr,*) 'output length short' 35 | stop 1 36 | endif 37 | if (outf(1,i) > 0) then 38 | write(stderr,*) 'output length long' 39 | stop 1 40 | endif 41 | 42 | end program 43 | 44 | 45 | ! logical, parameter :: jf(50) = .true. 46 | ! integer, parameter :: jmag = 1, iyyyy=1980,mmdd=0321,dhour=12 47 | ! real, parameter :: glat=0., glon=0. 48 | ! real,parameter :: altkm(*) = [130., 140., 150.] 49 | 50 | ! ** IRI parameters are being calculated *** 51 | !Ne: IRI-2001 for Topside 52 | !Ne, foF2: CCIR model is used. 53 | !Ne: B0,B1 Bil-2000 54 | !Ne, foF1: probability function used. 55 | !Ne, D: IRI1990 56 | !Ne, foF2: storm model included 57 | !Ion Com.: DS-95 & DY-85 58 | !Te: Aeros/AE/ISIS model 59 | !Auroral boundary model on 60 | !Ne, foE: storm model on 61 | ! 62 | ! NmF2 1.362E+12 [m^-3] hmF2 327.5 [km] 63 | ! F10.7 162.400 Ap 18 B0 88.054 64 | ! Ne 1.05585536E+09 815260608. 1.58453235E+09 65 | 66 | -------------------------------------------------------------------------------- /raytrace/app/iono.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from numba import jit 3 | import h5py 4 | import urllib.request 5 | import io 6 | import math 7 | from datetime import datetime, timezone 8 | 9 | @jit('f8[:](f8[:,:],f8[:],f8[:])',nopython=True,error_model='numpy') 10 | def bilerp(table, latitude, longitude): 11 | lat = latitude * 180 / np.pi + 90 12 | lon = longitude * 180 / np.pi + 180 13 | out = np.zeros_like(lat) 14 | 15 | for i in range(lat.size): 16 | latc = math.ceil(lat[i]) 17 | lonf = math.floor(lon[i]) 18 | lonc = math.ceil(lon[i]) 19 | latf, latc = int(np.floor(lat[i])), int(np.ceil(lat[i])) 20 | latp = lat[i] - latf 21 | lonf, lonc = int(np.floor(lon[i])), int(np.ceil(lon[i])) 22 | lonp = lon[i] - lonf 23 | 24 | top = lonp * table[latc, lonc] + (1.-lonp) * table[latc, lonf] 25 | bot = lonp * table[latf, lonc] + (1.-lonp) * table[latf, lonf] 26 | 27 | out[i] = latp * top + (1.-latp) * bot 28 | 29 | return out 30 | 31 | class Spline: 32 | def __init__(self, table): 33 | self.table = np.array(table) 34 | 35 | def predict(self, latitude, longitude): 36 | return bilerp(self.table, latitude.flatten(), longitude.flatten()).reshape(longitude.shape) 37 | 38 | class Iono: 39 | def __init__(self, url): 40 | with urllib.request.urlopen(url) as res: 41 | content = res.read() 42 | bio = io.BytesIO(content) 43 | self.h5 = h5py.File(bio, 'r') 44 | self.hmf2 = self.spline('/maps/hmf2') 45 | self.fof2 = self.spline('/maps/fof2') 46 | self.mufd = self.spline('/maps/mufd') 47 | self.foe = self.spline('/maps/foe') 48 | if '/maps/gyf' in self.h5: 49 | self.gyf = self.spline('/maps/gyf') 50 | self.ts = datetime.fromtimestamp(self.h5['/ts'][()], timezone.utc) 51 | if '/essn/ssn' in self.h5: 52 | self.ssn = self.h5['/essn/ssn'][()] 53 | 54 | def spline(self, ds): 55 | return Spline(self.h5[ds]) 56 | 57 | -------------------------------------------------------------------------------- /scheduler/app/lib/Minion/Statsd.pm: -------------------------------------------------------------------------------- 1 | package Minion::Statsd; 2 | use Mojo::Base 'Mojolicious::Plugin'; 3 | use Net::Statsd::Client; 4 | 5 | my $statsd = Net::Statsd::Client->new(host => $ENV{STATSD_HOST}); 6 | 7 | sub register { 8 | my ($self, $app) = @_; 9 | 10 | $app->minion_notifier->on(job => sub { 11 | my ($notifier, $job_id, $event) = @_; 12 | return unless $event eq 'finished' or $event eq 'failed'; 13 | my $job = $notifier->minion->job($job_id); 14 | if (!$job) { 15 | warn "Got notification for job $job_id that can't be fetched"; 16 | return; 17 | } 18 | my $task = $job->task; 19 | my %info = %{ $job->info }; 20 | my $queue = $info{queue}; 21 | 22 | if ($event eq 'failed') { 23 | if ($info{attempts} > 0) { 24 | $event = "failed_retryable"; 25 | } 26 | } 27 | 28 | $statsd->increment("minion.job.state.$event"); 29 | $statsd->increment("minion.job.task.$task"); 30 | $statsd->increment("minion.job.task.$task.state.$event"); 31 | $statsd->increment("minion.job.queue.$queue"); 32 | $statsd->increment("minion.job.queue.$queue.state.$event"); 33 | 34 | if ($event eq 'finished') { 35 | my $wait_ms = 1000 * ($info{started} - $info{delayed}); 36 | my $run_ms = 1000 * ($info{finished} - $info{started}); 37 | my $total_ms = 1000 * ($info{finished} - $info{created}); 38 | 39 | $statsd->timing_ms("minion.job.wait_time", $wait_ms); 40 | $statsd->timing_ms("minion.job.run_time", $run_ms); 41 | $statsd->timing_ms("minion.job.total_time", $total_ms); 42 | 43 | $statsd->timing_ms("minion.job.task.$task.wait_time", $wait_ms); 44 | $statsd->timing_ms("minion.job.task.$task.run_time", $run_ms); 45 | $statsd->timing_ms("minion.job.task.$task.total_time", $total_ms); 46 | 47 | $statsd->timing_ms("minion.job.queue.$queue.wait_time", $wait_ms); 48 | $statsd->timing_ms("minion.job.queue.$queue.run_time", $run_ms); 49 | $statsd->timing_ms("minion.job.queue.$queue.total_time", $total_ms); 50 | } 51 | }); 52 | } 53 | 54 | 1; 55 | -------------------------------------------------------------------------------- /iri2020/src/iri_ts.F90: -------------------------------------------------------------------------------- 1 | program basictest 2 | use, intrinsic:: iso_fortran_env, only: stderr=>error_unit, stdout=>output_unit 3 | implicit none 4 | 5 | logical :: jf(50) 6 | integer, parameter :: jmag = 0 7 | integer :: iyyyy, mmdd, Nalt, yyyymod, mmddmod 8 | real :: glat, glon, dhour, hourmod 9 | integer :: ymdhms(6) 10 | real:: alt_km_range(3) 11 | real:: TECtotal, TECtop, TECbot 12 | 13 | #ifndef BIN_DIR 14 | #define BIN_DIR '.' 15 | #endif 16 | character(*), parameter :: datadir = BIN_DIR // '/iri2020/data' 17 | character(256) :: datadir1 18 | common /folders/ datadir1 19 | 20 | real :: oarr(100), outf(20,1000) 21 | real, allocatable :: altkm(:) 22 | character(500) :: line 23 | 24 | integer :: i, io, noff 25 | integer :: offs(121) 26 | 27 | jf = .true. 28 | jf(4:6) = .false. 29 | jf(21:23) = .false. 30 | jf(28:30) = .false. 31 | jf(33:35) = .false. 32 | jf(39) = .false. 33 | jf(40) = .false. 34 | jf(47) = .false. 35 | 36 | do 37 | 38 | read(*,*, end=42) ymdhms(1), ymdhms(2), ymdhms(3), ymdhms(4), ymdhms(5), ymdhms(6), & 39 | glat, glon 40 | 41 | read(*, '(A)', end=42) line 42 | 43 | do i=1,120 44 | read(line, *, iostat=io) offs(1:i) 45 | if (io==-1) exit 46 | enddo 47 | offs(i) = 120 48 | noff = i 49 | 50 | iyyyy = ymdhms(1) 51 | mmdd = ymdhms(2) * 100 + ymdhms(3) 52 | dhour = ymdhms(4) + ymdhms(5) / 60. + ymdhms(6) / 3600. 53 | 54 | datadir1 = datadir 55 | call read_ig_rz 56 | call readapf107 57 | 58 | do i=1,noff 59 | hourmod = dhour + offs(i)/60. 60 | mmddmod = mmdd 61 | yyyymod = iyyyy 62 | 63 | if (hourmod.gt.24.) then 64 | hourmod = hourmod - 24. 65 | mmddmod = mmddmod + 1 66 | if (mmddmod.gt.1231) then 67 | mmddmod = 101 68 | yyyymod = yyyymod + 1 69 | endif 70 | endif 71 | 72 | call IRI_SUB(JF,JMAG,glat,glon,yyyymod,mmddmod,hourmod+25., & 73 | 0, 0, 1, & 74 | OUTF,OARR, datadir) 75 | 76 | write(stdout, '(3ES16.8)', advance="no") oarr(91), oarr(36) * oarr(91), oarr(2) 77 | enddo 78 | write(stdout,*) "" 79 | 80 | enddo 81 | 42 continue 82 | end program 83 | 84 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf2000.dat: -------------------------------------------------------------------------------- 1 | dgrf2000 2 | 13 6371.2 2000.0 3 | -29619.4 4 | -1728.2 5 | 5186.1 6 | -2267.7 7 | 3068.4 8 | -2481.6 9 | 1670.9 10 | -458.0 11 | 1339.6 12 | -2288.0 13 | -227.6 14 | 1252.1 15 | 293.4 16 | 714.5 17 | -491.1 18 | 932.3 19 | 786.8 20 | 272.6 21 | 250.0 22 | -231.9 23 | -403.0 24 | 119.8 25 | 111.3 26 | -303.8 27 | -218.8 28 | 351.4 29 | 43.8 30 | 222.3 31 | 171.9 32 | -130.4 33 | -133.1 34 | -168.6 35 | -39.3 36 | -12.9 37 | 106.3 38 | 72.3 39 | 68.2 40 | -17.4 41 | 74.2 42 | 63.7 43 | -160.9 44 | 65.1 45 | -5.9 46 | -61.2 47 | 16.9 48 | 0.7 49 | -90.4 50 | 43.8 51 | 79.0 52 | -74.0 53 | -64.6 54 | 0.0 55 | -24.2 56 | 33.3 57 | 6.2 58 | 9.1 59 | 24.0 60 | 6.9 61 | 14.8 62 | 7.3 63 | -25.4 64 | -1.2 65 | -5.8 66 | 24.4 67 | 6.6 68 | 11.9 69 | -9.2 70 | -21.5 71 | -7.9 72 | 8.5 73 | -16.6 74 | -21.5 75 | 9.1 76 | 15.5 77 | 7.0 78 | 8.9 79 | -7.9 80 | -14.9 81 | -7.0 82 | -2.1 83 | 5.0 84 | 9.4 85 | -19.7 86 | 3.0 87 | 13.4 88 | -8.4 89 | 12.5 90 | 6.3 91 | -6.2 92 | -8.9 93 | -8.4 94 | -1.5 95 | 8.4 96 | 9.3 97 | 3.8 98 | -4.3 99 | -8.2 100 | -8.2 101 | 4.8 102 | -2.6 103 | -6.0 104 | 1.7 105 | 1.7 106 | 0.0 107 | -3.1 108 | 4.0 109 | -0.5 110 | 4.9 111 | 3.7 112 | -5.9 113 | 1.0 114 | -1.2 115 | 2.0 116 | -2.9 117 | 4.2 118 | 0.2 119 | 0.3 120 | -2.2 121 | -1.1 122 | -7.4 123 | 2.7 124 | -1.7 125 | 0.1 126 | -1.9 127 | 1.3 128 | 1.5 129 | -0.9 130 | -0.1 131 | -2.6 132 | 0.1 133 | 0.9 134 | -0.7 135 | -0.7 136 | 0.7 137 | -2.8 138 | 1.7 139 | -0.9 140 | 0.1 141 | -1.2 142 | 1.2 143 | -1.9 144 | 4.0 145 | -0.9 146 | -2.2 147 | -0.3 148 | -0.4 149 | 0.2 150 | 0.3 151 | 0.9 152 | 2.5 153 | -0.2 154 | -2.6 155 | 0.9 156 | 0.7 157 | -0.5 158 | 0.3 159 | 0.3 160 | 0.0 161 | -0.3 162 | 0.0 163 | -0.4 164 | 0.3 165 | -0.1 166 | -0.9 167 | -0.2 168 | -0.4 169 | -0.4 170 | 0.8 171 | -0.2 172 | -0.9 173 | -0.9 174 | 0.3 175 | 0.2 176 | 0.1 177 | 1.8 178 | -0.4 179 | -0.4 180 | 1.3 181 | -1.0 182 | -0.4 183 | -0.1 184 | 0.7 185 | 0.7 186 | -0.4 187 | 0.3 188 | 0.3 189 | 0.6 190 | -0.1 191 | 0.3 192 | 0.4 193 | -0.2 194 | 0.0 195 | -0.5 196 | 0.1 197 | -0.9 198 | -------------------------------------------------------------------------------- /iri2020/data/igrf/igrf2020.dat: -------------------------------------------------------------------------------- 1 | igrf2020 2 | 13 6371.2 2020.0 3 | -29404.8 4 | -1450.9 5 | 4652.5 6 | -2499.6 7 | 2982.0 8 | -2991.6 9 | 1677.0 10 | -734.6 11 | 1363.2 12 | -2381.2 13 | -82.1 14 | 1236.2 15 | 241.9 16 | 525.7 17 | -543.4 18 | 903.0 19 | 809.5 20 | 281.9 21 | 86.3 22 | -158.4 23 | -309.4 24 | 199.7 25 | 48.0 26 | -349.7 27 | -234.3 28 | 363.2 29 | 47.7 30 | 187.8 31 | 208.3 32 | -140.7 33 | -121.2 34 | -151.2 35 | 32.3 36 | 13.5 37 | 98.9 38 | 66.0 39 | 65.5 40 | -19.1 41 | 72.9 42 | 25.1 43 | -121.5 44 | 52.8 45 | -36.2 46 | -64.5 47 | 13.5 48 | 8.9 49 | -64.7 50 | 68.1 51 | 80.6 52 | -76.7 53 | -51.5 54 | -8.2 55 | -16.9 56 | 56.5 57 | 2.2 58 | 15.8 59 | 23.5 60 | 6.4 61 | -2.2 62 | -7.2 63 | -27.2 64 | 9.8 65 | -1.8 66 | 23.7 67 | 9.7 68 | 8.4 69 | -17.6 70 | -15.3 71 | -0.5 72 | 12.8 73 | -21.1 74 | -11.7 75 | 15.3 76 | 14.9 77 | 13.7 78 | 3.6 79 | -16.5 80 | -6.9 81 | -0.3 82 | 2.8 83 | 5.0 84 | 8.4 85 | -23.4 86 | 2.9 87 | 11.0 88 | -1.5 89 | 9.8 90 | -1.1 91 | -5.1 92 | -13.2 93 | -6.3 94 | 1.1 95 | 7.8 96 | 8.8 97 | 0.4 98 | -9.3 99 | -1.4 100 | -11.9 101 | 9.6 102 | -1.9 103 | -6.2 104 | 3.4 105 | -0.1 106 | -0.2 107 | 1.7 108 | 3.6 109 | -0.9 110 | 4.8 111 | 0.7 112 | -8.6 113 | -0.9 114 | -0.1 115 | 1.9 116 | -4.3 117 | 1.4 118 | -3.4 119 | -2.4 120 | -0.1 121 | -3.8 122 | -8.8 123 | 3.0 124 | -1.4 125 | 0.0 126 | -2.5 127 | 2.5 128 | 2.3 129 | -0.6 130 | -0.9 131 | -0.4 132 | 0.3 133 | 0.6 134 | -0.7 135 | -0.2 136 | -0.1 137 | -1.7 138 | 1.4 139 | -1.6 140 | -0.6 141 | -3.0 142 | 0.2 143 | -2.0 144 | 3.1 145 | -2.6 146 | -2.0 147 | -0.1 148 | -1.2 149 | 0.5 150 | 0.5 151 | 1.3 152 | 1.4 153 | -1.2 154 | -1.8 155 | 0.7 156 | 0.1 157 | 0.3 158 | 0.8 159 | 0.5 160 | -0.2 161 | -0.3 162 | 0.6 163 | -0.5 164 | 0.2 165 | 0.1 166 | -0.9 167 | -1.1 168 | 0.0 169 | -0.3 170 | 0.5 171 | 0.1 172 | -0.9 173 | -0.9 174 | 0.5 175 | 0.6 176 | 0.7 177 | 1.4 178 | -0.3 179 | -0.4 180 | 0.8 181 | -1.3 182 | 0.0 183 | -0.1 184 | 0.8 185 | 0.3 186 | 0.0 187 | -0.1 188 | 0.4 189 | 0.5 190 | 0.1 191 | 0.5 192 | 0.5 193 | -0.4 194 | -0.5 195 | -0.4 196 | -0.4 197 | -0.6 198 | -------------------------------------------------------------------------------- /assimilate/app/data/jsonapi.py: -------------------------------------------------------------------------------- 1 | import os 2 | import urllib.request 3 | import json 4 | import numpy as np 5 | import pandas as pd 6 | from pandas import json_normalize 7 | 8 | known_metrics = ['fof2', 'md', 'mufd', 'foes', 'foe', 'hmf2', 'tec'] 9 | 10 | def get_data(url=os.getenv("METRICS_URI"), default_confidence=62): 11 | data = None 12 | 13 | if url.startswith(("http:", "https:")): 14 | with urllib.request.urlopen(url) as res: 15 | data = json.loads(res.read().decode()) 16 | else: 17 | with open(url) as f: 18 | data = json.load(f) 19 | 20 | df = json_normalize(data) 21 | 22 | for field in ['cs', 'station.longitude', 'station.latitude']: 23 | df[field] = df[field].apply(pd.to_numeric) 24 | 25 | for metric in known_metrics: 26 | if metric in df: 27 | df[metric] = df[metric].apply(pd.to_numeric) 28 | 29 | df.cs = df.cs.apply(pd.to_numeric) 30 | df.loc[df.cs == -1, 'cs'] = default_confidence 31 | df.loc[df.cs > 100., 'cs'] = default_confidence 32 | df.cs = df.cs / 100. 33 | 34 | df['md'] = df['mufd'] / df['fof2'] 35 | 36 | # Maybe this should be an add-in-quadrature, but the way we're using it is kind of tricky 37 | # and I need to really think out the error-propagation. Max feels right. 38 | if 'stdev_mufd' in df and 'stdev_fof2' in df: 39 | df['stdev_md'] = np.maximum(df['stdev_mufd'], df['stdev_fof2']) 40 | 41 | df.time = pd.to_datetime(df.time) 42 | 43 | df.loc[df['station.longitude'] > 180, 'station.longitude'] = df['station.longitude'] - 360 44 | df.sort_values(by=['station.longitude'], inplace=True) 45 | 46 | return df 47 | 48 | def filter(df, max_age=None, required_metrics=[], min_confidence=None): 49 | df = df.drop(df[df['station.use_for_maps'] == False].index) 50 | if max_age is not None: 51 | df = df.drop(df[df.time < max_age].index) 52 | 53 | if min_confidence is not None: 54 | df = df.drop(df[df.cs < min_confidence].index) 55 | 56 | df = df.dropna(subset=required_metrics) 57 | 58 | for metric in required_metrics: 59 | df = df.drop(df[df[metric] == 0.].index) 60 | 61 | return df 62 | -------------------------------------------------------------------------------- /diffusion/app/data/jsonapi.py: -------------------------------------------------------------------------------- 1 | import os 2 | import urllib.request 3 | import json 4 | import numpy as np 5 | import pandas as pd 6 | from pandas import json_normalize 7 | 8 | known_metrics = ['fof2', 'md', 'mufd', 'foes', 'foe', 'hmf2', 'tec'] 9 | 10 | def get_data(url=os.getenv("METRICS_URI"), default_confidence=62): 11 | data = None 12 | 13 | if url.startswith(("http:", "https:")): 14 | with urllib.request.urlopen(url) as res: 15 | data = json.loads(res.read().decode()) 16 | else: 17 | with open(url) as f: 18 | data = json.load(f) 19 | 20 | df = json_normalize(data) 21 | 22 | for field in ['cs', 'station.longitude', 'station.latitude']: 23 | df[field] = df[field].apply(pd.to_numeric) 24 | 25 | for metric in known_metrics: 26 | if metric in df: 27 | df[metric] = df[metric].apply(pd.to_numeric) 28 | 29 | df.cs = df.cs.apply(pd.to_numeric) 30 | df.loc[df.cs == -1, 'cs'] = default_confidence 31 | df.loc[df.cs > 100., 'cs'] = default_confidence 32 | df.cs = df.cs / 100. 33 | 34 | df['md'] = df['mufd'] / df['fof2'] 35 | 36 | # Maybe this should be an add-in-quadrature, but the way we're using it is kind of tricky 37 | # and I need to really think out the error-propagation. Max feels right. 38 | if 'stdev_mufd' in df and 'stdev_fof2' in df: 39 | df['stdev_md'] = np.maximum(df['stdev_mufd'], df['stdev_fof2']) 40 | 41 | df.time = pd.to_datetime(df.time) 42 | 43 | df.loc[df['station.longitude'] > 180, 'station.longitude'] = df['station.longitude'] - 360 44 | df.sort_values(by=['station.longitude'], inplace=True) 45 | 46 | return df 47 | 48 | def filter(df, max_age=None, required_metrics=[], min_confidence=None): 49 | df = df.drop(df[df['station.use_for_maps'] == False].index) 50 | if max_age is not None: 51 | df = df.drop(df[df.time < max_age].index) 52 | 53 | if min_confidence is not None: 54 | df = df.drop(df[df.cs < min_confidence].index) 55 | 56 | df = df.dropna(subset=required_metrics) 57 | 58 | for metric in required_metrics: 59 | df = df.drop(df[df[metric] == 0.].index) 60 | 61 | return df 62 | -------------------------------------------------------------------------------- /iri2020/src/irimap.F90: -------------------------------------------------------------------------------- 1 | program basictest 2 | use, intrinsic:: iso_fortran_env, only: stderr=>error_unit, stdout=>output_unit 3 | implicit none 4 | 5 | logical :: jf(50) 6 | integer, parameter :: jmag = 0 7 | integer :: iyyyy, mmdd, Nalt 8 | real :: glat, glon, dhour 9 | integer :: ymdhms(6) 10 | real:: alt_km_range(3) 11 | real:: TECtotal, TECtop, TECbot 12 | 13 | #ifndef BIN_DIR 14 | #define BIN_DIR '.' 15 | #endif 16 | character(*), parameter :: datadir = BIN_DIR // '/iri2020/data' 17 | character(256) :: datadir1 18 | common /folders/ datadir1 19 | 20 | real :: oarr(100), outf(20,1000) 21 | real, allocatable :: altkm(:) 22 | character(80) :: argv 23 | integer :: i 24 | real :: sfi, ssn, ig 25 | 26 | jf = .true. 27 | jf(4:6) = .false. 28 | jf(21:23) = .false. 29 | jf(28:30) = .false. 30 | jf(33:36) = .false. 31 | jf(39) = .false. 32 | jf(40) = .false. 33 | jf(47) = .false. 34 | 35 | ! --- command line input 36 | if (command_argument_count() < 6) then 37 | write(stderr,*) 'need input parameters: year month day hour minute second' 38 | stop 1 39 | endif 40 | 41 | do i=1,6 42 | call get_command_argument(i,argv) 43 | read(argv,*) ymdhms(i) 44 | enddo 45 | 46 | call get_command_argument(7, argv) 47 | read(argv,*) ssn 48 | 49 | if(ssn.gt.-99.) then 50 | sfi = 63.75+ssn*(0.728+ssn*0.000089) 51 | ig=(-0.0031*ssn+1.5332)*ssn-11.5634 52 | jf(17) = .false. 53 | jf(25) = .false. 54 | jf(27) = .false. 55 | jf(32) = .false. 56 | else 57 | jf(17) = .true. 58 | jf(25) = .true. 59 | jf(27) = .true. 60 | jf(32) = .true. 61 | endif 62 | 63 | iyyyy = ymdhms(1) 64 | mmdd = ymdhms(2) * 100 + ymdhms(3) 65 | dhour = ymdhms(4) + ymdhms(5) / 60. + ymdhms(6) / 3600. 66 | 67 | datadir1 = datadir 68 | call read_ig_rz 69 | call readapf107 70 | 71 | do glat=-90,90 72 | do glon=-180,180 73 | 74 | OARR(33)=ssn 75 | OARR(41)=sfi 76 | OARR(46)=sfi 77 | OARR(39)=ig 78 | 79 | call IRI_SUB(JF,JMAG,glat,glon,IYYYY,MMDD,DHOUR+25., & 80 | 0, 0, 1, & 81 | OUTF,OARR, datadir) 82 | 83 | ! lat, lon, nmf2, fof2, m(3000), muf(3000), hmf2, foe 84 | write(stdout, '(8ES16.8)') glat, glon, oarr(1), oarr(91), oarr(36), oarr(36) * oarr(91), oarr(2), oarr(92) 85 | enddo 86 | enddo 87 | 88 | end program 89 | 90 | -------------------------------------------------------------------------------- /www/nav-header.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 51 | 52 |
53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /diffusion/app/guidance_test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import math 3 | import datetime 4 | import lightning as L 5 | import torch 6 | import torch.nn.functional as F 7 | from lightning.pytorch.callbacks import ModelCheckpoint 8 | from models import GuidanceModel, IRIData 9 | 10 | if __name__ == "__main__": 11 | model = GuidanceModel.load_from_checkpoint(sys.argv[1]) 12 | 13 | ds = IRIData("combined", test_batch=1) 14 | ds.setup() 15 | data = ds.test_dataloader() 16 | 17 | with torch.no_grad(): 18 | for i, sample in enumerate(data): 19 | images = sample["images"].to(device=model.device) 20 | pred = model.model(images) 21 | target = sample["target"].to(device=model.device) 22 | loss = F.mse_loss(pred, target) 23 | 24 | pred_toy = torch.atan2(pred[:, 1], pred[:, 2]) / (2 * math.pi) 25 | target_toy = torch.atan2(target[:, 1], target[:, 2]) / (2 * math.pi) 26 | pred_tod = torch.atan2(pred[:, 3], pred[:, 4]) / (2 * math.pi) 27 | target_tod = torch.atan2(target[:, 3], target[:, 4]) / (2 * math.pi) 28 | pred_year = int(round((pred[:, 0] * 50. + 2000 - pred_toy).item())) 29 | target_year = int(round((target[:, 0] * 50. + 2000 - target_toy).item())) 30 | 31 | pred_base = datetime.datetime(year=pred_year, month=1, day=1) 32 | target_base = datetime.datetime(year=target_year, month=1, day=1) 33 | pred_date_str = (pred_base + datetime.timedelta(days=pred_toy.item() * 365)).strftime("%Y-%m-%d") 34 | target_date_str = (target_base + datetime.timedelta(days=target_toy.item() * 365)).strftime("%Y-%m-%d") 35 | pred_tod_str = (pred_base + datetime.timedelta(hours=pred_tod.item() * 24)).strftime("%H:%M") 36 | target_tod_str = (target_base + datetime.timedelta(hours=target_tod.item() * 24)).strftime("%H:%M") 37 | 38 | pred_ssn = pred[:, 4] * 100 + 100 39 | target_ssn = target[:, 4] * 100 + 100 40 | 41 | print(f"Predicted: {pred_date_str} {pred_tod_str} SSN {pred_ssn.item():.2f}, " 42 | f"Target: {target_date_str} {target_tod_str} SSN {target_ssn.item():.2f}, " 43 | f"Loss: {loss.item():.4f}") 44 | if i >= 10: 45 | break 46 | -------------------------------------------------------------------------------- /iri2020/src/irtam_driver.F90: -------------------------------------------------------------------------------- 1 | program basictest 2 | use, intrinsic:: iso_fortran_env, only: stderr=>error_unit, stdout=>output_unit 3 | implicit none 4 | 5 | logical :: jf(50) 6 | integer, parameter :: jmag = 0 7 | real :: glat, glon, dhour 8 | integer :: qtime(6) 9 | integer :: tov(5) 10 | integer :: tovdate, tovhhmm, otovdate, otovhhmm 11 | integer :: qdate 12 | integer :: iyyyy, mmdd 13 | real :: tovhour, irtamhour 14 | real :: ff2(1064), fh2(1064), fb0(1064), fb1(1064) 15 | real :: modip 16 | real :: fof2rt, hmf2rt 17 | 18 | real, external :: FOUT1 19 | 20 | #ifndef BIN_DIR 21 | #define BIN_DIR '.' 22 | #endif 23 | character(*), parameter :: datadir = BIN_DIR // '/iri2020/data' 24 | character(256) :: datadir1 25 | common /folders/ datadir1 26 | 27 | real :: oarr(100), outf(20,1000) 28 | 29 | jf = .true. 30 | jf(4:6) = .false. 31 | jf(21:23) = .false. 32 | jf(28:30) = .false. 33 | jf(33:36) = .false. 34 | jf(39) = .false. 35 | jf(40) = .false. 36 | jf(47) = .false. 37 | 38 | datadir1 = datadir 39 | call read_ig_rz 40 | call readapf107 41 | 42 | otovdate = -1 43 | otovhhmm = -1 44 | 45 | do 46 | read(*,*, end=42) glat, glon, & 47 | tov(1), tov(2), tov(3), tov(4), tov(5), & 48 | qtime(1), qtime(2), qtime(3), qtime(4), qtime(5), qtime(6) 49 | 50 | tovdate = tov(1)*10000 + tov(2)*100 + tov(3) 51 | tovhhmm = tov(4)*100 + tov(5) 52 | tovhour = tov(4) + tov(5) / 60. 53 | 54 | iyyyy = qtime(1) 55 | mmdd = qtime(2) * 100 + qtime(3) 56 | qdate = iyyyy*10000 + mmdd 57 | 58 | dhour = qtime(4) + qtime(5) / 60. + qtime(6) / 3600. 59 | if(qdate.gt.tovdate) then 60 | irtamhour = dhour + 24. 61 | else 62 | irtamhour = dhour 63 | endif 64 | 65 | if ((tovdate.ne.otovdate).or.(tovhhmm.ne.otovhhmm)) then 66 | call READIRTAMCOF(0, tovdate, tovhhmm, 1064, ff2) 67 | call READIRTAMCOF(1, tovdate, tovhhmm, 1064, fh2) 68 | endif 69 | 70 | otovdate = tovdate 71 | otovhhmm = tovhhmm 72 | 73 | call IRI_SUB(JF,JMAG,glat,glon,IYYYY,MMDD,DHOUR+25., & 74 | 0, 0, 1, & 75 | OUTF, OARR, datadir) 76 | 77 | modip = OARR(27) 78 | 79 | fof2rt = FOUT1(modip, glat, glon, irtamhour, tovhour, ff2) 80 | hmf2rt = FOUT1(modip, glat, glon, irtamhour, tovhour, fh2) 81 | 82 | write(stdout, '(2ES16.8)') fof2rt, hmf2rt 83 | enddo 84 | 42 continue 85 | end program 86 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf2015.dat: -------------------------------------------------------------------------------- 1 | dgrf2015 2 | 13 6371.2 2015.0 3 | -29441.46 4 | -1501.77 5 | 4795.99 6 | -2445.88 7 | 3012.2 8 | -2845.41 9 | 1676.35 10 | -642.17 11 | 1350.33 12 | -2352.26 13 | -115.29 14 | 1225.85 15 | 245.04 16 | 581.69 17 | -538.7 18 | 907.42 19 | 813.68 20 | 283.54 21 | 120.49 22 | -188.43 23 | -334.85 24 | 180.95 25 | 70.38 26 | -329.23 27 | -232.91 28 | 360.14 29 | 46.98 30 | 192.35 31 | 196.98 32 | -140.94 33 | -119.14 34 | -157.4 35 | 15.98 36 | 4.3 37 | 100.12 38 | 69.55 39 | 67.57 40 | -20.61 41 | 72.79 42 | 33.3 43 | -129.85 44 | 58.74 45 | -28.93 46 | -66.64 47 | 13.14 48 | 7.35 49 | -70.85 50 | 62.41 51 | 81.29 52 | -75.99 53 | -54.27 54 | -6.79 55 | -19.53 56 | 51.82 57 | 5.59 58 | 15.07 59 | 24.45 60 | 9.32 61 | 3.27 62 | -2.88 63 | -27.5 64 | 6.61 65 | -2.32 66 | 23.98 67 | 8.89 68 | 10.04 69 | -16.78 70 | -18.26 71 | -3.16 72 | 13.18 73 | -20.56 74 | -14.6 75 | 13.33 76 | 16.16 77 | 11.76 78 | 5.69 79 | -15.98 80 | -9.1 81 | -2.02 82 | 2.26 83 | 5.33 84 | 8.83 85 | -21.77 86 | 3.02 87 | 10.76 88 | -3.22 89 | 11.74 90 | 0.67 91 | -6.74 92 | -13.2 93 | -6.88 94 | -0.1 95 | 7.79 96 | 8.68 97 | 1.04 98 | -9.06 99 | -3.89 100 | -10.54 101 | 8.44 102 | -2.01 103 | -6.26 104 | 3.28 105 | 0.17 106 | -0.4 107 | 0.55 108 | 4.55 109 | -0.55 110 | 4.4 111 | 1.7 112 | -7.92 113 | -0.67 114 | -0.61 115 | 2.13 116 | -4.16 117 | 2.33 118 | -2.85 119 | -1.8 120 | -1.12 121 | -3.59 122 | -8.72 123 | 3 124 | -1.4 125 | 0 126 | -2.3 127 | 2.11 128 | 2.08 129 | -0.6 130 | -0.79 131 | -1.05 132 | 0.58 133 | 0.76 134 | -0.7 135 | -0.2 136 | 0.14 137 | -2.12 138 | 1.7 139 | -1.44 140 | -0.22 141 | -2.57 142 | 0.44 143 | -2.01 144 | 3.49 145 | -2.34 146 | -2.09 147 | -0.16 148 | -1.08 149 | 0.46 150 | 0.37 151 | 1.23 152 | 1.75 153 | -0.89 154 | -2.19 155 | 0.85 156 | 0.27 157 | 0.1 158 | 0.72 159 | 0.54 160 | -0.09 161 | -0.37 162 | 0.29 163 | -0.43 164 | 0.23 165 | 0.22 166 | -0.89 167 | -0.94 168 | -0.16 169 | -0.03 170 | 0.72 171 | -0.02 172 | -0.92 173 | -0.88 174 | 0.42 175 | 0.49 176 | 0.63 177 | 1.56 178 | -0.42 179 | -0.5 180 | 0.96 181 | -1.24 182 | -0.19 183 | -0.1 184 | 0.81 185 | 0.42 186 | -0.13 187 | -0.04 188 | 0.38 189 | 0.48 190 | 0.08 191 | 0.48 192 | 0.46 193 | -0.3 194 | -0.35 195 | -0.43 196 | -0.36 197 | -0.71 198 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf2005.dat: -------------------------------------------------------------------------------- 1 | dgrf2005 2 | 13 6371.2 2005.0 3 | -29554.63 4 | -1669.05 5 | 5077.99 6 | -2337.24 7 | 3047.69 8 | -2594.50 9 | 1657.76 10 | -515.43 11 | 1336.30 12 | -2305.83 13 | -198.86 14 | 1246.39 15 | 269.72 16 | 672.51 17 | -524.72 18 | 920.55 19 | 797.96 20 | 282.07 21 | 210.65 22 | -225.23 23 | -379.86 24 | 145.15 25 | 100.00 26 | -305.36 27 | -227.00 28 | 354.41 29 | 42.72 30 | 208.95 31 | 180.25 32 | -136.54 33 | -123.45 34 | -168.05 35 | -19.57 36 | -13.55 37 | 103.85 38 | 73.60 39 | 69.56 40 | -20.33 41 | 76.74 42 | 54.75 43 | -151.34 44 | 63.63 45 | -14.58 46 | -63.53 47 | 14.58 48 | 0.24 49 | -86.36 50 | 50.94 51 | 79.88 52 | -74.46 53 | -61.14 54 | -1.65 55 | -22.57 56 | 38.73 57 | 6.82 58 | 12.30 59 | 25.35 60 | 9.37 61 | 10.93 62 | 5.42 63 | -26.32 64 | 1.94 65 | -4.64 66 | 24.80 67 | 7.62 68 | 11.20 69 | -11.73 70 | -20.88 71 | -6.88 72 | 9.83 73 | -18.11 74 | -19.71 75 | 10.17 76 | 16.22 77 | 9.36 78 | 7.61 79 | -11.25 80 | -12.76 81 | -4.87 82 | -0.06 83 | 5.58 84 | 9.76 85 | -20.11 86 | 3.58 87 | 12.69 88 | -6.94 89 | 12.67 90 | 5.01 91 | -6.72 92 | -10.76 93 | -8.16 94 | -1.25 95 | 8.10 96 | 8.76 97 | 2.92 98 | -6.66 99 | -7.73 100 | -9.22 101 | 6.01 102 | -2.17 103 | -6.12 104 | 2.19 105 | 1.42 106 | 0.10 107 | -2.35 108 | 4.46 109 | -0.15 110 | 4.76 111 | 3.06 112 | -6.58 113 | 0.29 114 | -1.01 115 | 2.06 116 | -3.47 117 | 3.77 118 | -0.86 119 | -0.21 120 | -2.31 121 | -2.09 122 | -7.93 123 | 2.95 124 | -1.60 125 | 0.26 126 | -1.88 127 | 1.44 128 | 1.44 129 | -0.77 130 | -0.31 131 | -2.27 132 | 0.29 133 | 0.90 134 | -0.79 135 | -0.58 136 | 0.53 137 | -2.69 138 | 1.80 139 | -1.08 140 | 0.16 141 | -1.58 142 | 0.96 143 | -1.90 144 | 3.99 145 | -1.39 146 | -2.15 147 | -0.29 148 | -0.55 149 | 0.21 150 | 0.23 151 | 0.89 152 | 2.38 153 | -0.38 154 | -2.63 155 | 0.96 156 | 0.61 157 | -0.30 158 | 0.40 159 | 0.46 160 | 0.01 161 | -0.35 162 | 0.02 163 | -0.36 164 | 0.28 165 | 0.08 166 | -0.87 167 | -0.49 168 | -0.34 169 | -0.08 170 | 0.88 171 | -0.16 172 | -0.88 173 | -0.76 174 | 0.30 175 | 0.33 176 | 0.28 177 | 1.72 178 | -0.43 179 | -0.54 180 | 1.18 181 | -1.07 182 | -0.37 183 | -0.04 184 | 0.75 185 | 0.63 186 | -0.26 187 | 0.21 188 | 0.35 189 | 0.53 190 | -0.05 191 | 0.38 192 | 0.41 193 | -0.22 194 | -0.10 195 | -0.57 196 | -0.18 197 | -0.82 198 | -------------------------------------------------------------------------------- /iri2020/data/igrf/dgrf2010.dat: -------------------------------------------------------------------------------- 1 | dgrf2010 2 | 13 6371.2 2010.0 3 | -29496.57 4 | -1586.42 5 | 4944.26 6 | -2396.06 7 | 3026.34 8 | -2708.54 9 | 1668.17 10 | -575.73 11 | 1339.85 12 | -2326.54 13 | -160.40 14 | 1232.10 15 | 251.75 16 | 633.73 17 | -537.03 18 | 912.66 19 | 808.97 20 | 286.48 21 | 166.58 22 | -211.03 23 | -356.83 24 | 164.46 25 | 89.40 26 | -309.72 27 | -230.87 28 | 357.29 29 | 44.58 30 | 200.26 31 | 189.01 32 | -141.05 33 | -118.06 34 | -163.17 35 | -0.01 36 | -8.03 37 | 101.04 38 | 72.78 39 | 68.69 40 | -20.90 41 | 75.92 42 | 44.18 43 | -141.40 44 | 61.54 45 | -22.83 46 | -66.26 47 | 13.10 48 | 3.02 49 | -78.09 50 | 55.40 51 | 80.44 52 | -75.00 53 | -57.80 54 | -4.55 55 | -21.20 56 | 45.24 57 | 6.54 58 | 14.00 59 | 24.96 60 | 10.46 61 | 7.03 62 | 1.64 63 | -27.61 64 | 4.92 65 | -3.28 66 | 24.41 67 | 8.21 68 | 10.84 69 | -14.50 70 | -20.03 71 | -5.59 72 | 11.83 73 | -19.34 74 | -17.41 75 | 11.61 76 | 16.71 77 | 10.85 78 | 6.96 79 | -14.05 80 | -10.74 81 | -3.54 82 | 1.64 83 | 5.50 84 | 9.45 85 | -20.54 86 | 3.45 87 | 11.51 88 | -5.27 89 | 12.75 90 | 3.13 91 | -7.14 92 | -12.38 93 | -7.42 94 | -0.76 95 | 7.97 96 | 8.43 97 | 2.14 98 | -8.42 99 | -6.08 100 | -10.08 101 | 7.01 102 | -1.94 103 | -6.24 104 | 2.73 105 | 0.89 106 | -0.10 107 | -1.07 108 | 4.71 109 | -0.16 110 | 4.44 111 | 2.45 112 | -7.22 113 | -0.33 114 | -0.96 115 | 2.13 116 | -3.95 117 | 3.09 118 | -1.99 119 | -1.03 120 | -1.97 121 | -2.80 122 | -8.31 123 | 3.05 124 | -1.48 125 | 0.13 126 | -2.03 127 | 1.67 128 | 1.65 129 | -0.66 130 | -0.51 131 | -1.76 132 | 0.54 133 | 0.85 134 | -0.79 135 | -0.39 136 | 0.37 137 | -2.51 138 | 1.79 139 | -1.27 140 | 0.12 141 | -2.11 142 | 0.75 143 | -1.94 144 | 3.75 145 | -1.86 146 | -2.12 147 | -0.21 148 | -0.87 149 | 0.30 150 | 0.27 151 | 1.04 152 | 2.13 153 | -0.63 154 | -2.49 155 | 0.95 156 | 0.49 157 | -0.11 158 | 0.59 159 | 0.52 160 | 0.00 161 | -0.39 162 | 0.13 163 | -0.37 164 | 0.27 165 | 0.21 166 | -0.86 167 | -0.77 168 | -0.23 169 | 0.04 170 | 0.87 171 | -0.09 172 | -0.89 173 | -0.87 174 | 0.31 175 | 0.30 176 | 0.42 177 | 1.66 178 | -0.45 179 | -0.59 180 | 1.08 181 | -1.14 182 | -0.31 183 | -0.07 184 | 0.78 185 | 0.54 186 | -0.18 187 | 0.10 188 | 0.38 189 | 0.49 190 | 0.02 191 | 0.44 192 | 0.42 193 | -0.25 194 | -0.26 195 | -0.53 196 | -0.26 197 | -0.79 198 | -------------------------------------------------------------------------------- /scheduler/app/lib/Task/Render.pm: -------------------------------------------------------------------------------- 1 | package Task::Render; 2 | use v5.24; 3 | use Mojo::Base 'Mojolicious::Plugin'; 4 | use Path::Tiny; 5 | use DateTime; 6 | 7 | 8 | our %LOCATIONS = ( 9 | "us-east" => { lat => 37.5, lon => -77.5 }, 10 | "us-central" => { lat => 39, lon => -102 }, 11 | "us-west" => { lat => 41, lon => -121.5 }, 12 | "eu-central" => { lat => 50, lon => 13.5 }, 13 | ); 14 | 15 | sub register { 16 | my ($self, $app) = @_; 17 | 18 | $app->minion->add_task(rendersvg => sub { 19 | my ($job, %args) = @_; 20 | my $res = Mojo::UserAgent->new->inactivity_timeout(300)->post("http://localhost:$ENV{RENDERER_PORT}/rendersvg", => 21 | form => \%args, 22 | )->result; 23 | $res->is_success or die $res->error . "\n" . $res->body; 24 | }); 25 | 26 | $app->minion->add_task(rendermuf => sub { 27 | my ($job, $location, %args) = @_; 28 | my $res = Mojo::UserAgent->new->inactivity_timeout(300)->post("http://localhost:$ENV{RENDERER_PORT}/moflof.svg", => 29 | form => { $LOCATIONS{$location}->%*, %args }, 30 | )->result; 31 | $res->is_success or die $res->error . "\n" . $res->body; 32 | }); 33 | 34 | $app->minion->add_task(renderhtml => sub { 35 | my ($job, %args) = @_; 36 | 37 | if (delete $args{fallback_banner}) { 38 | my $dir = path('/output')->child($args{run_id})->mkdir; 39 | my $banner_file = $dir->child('banner.html'); 40 | my $last_data_ts = DateTime->from_epoch(epoch => $args{last_data})->strftime('%Y-%m-%d %H:%M'); 41 | $banner_file->spew(< 43 |

Data is stale

44 |

45 | No data has been received from GIRO for more than 3 hours. Last data was at $last_data_ts UTC. 46 | Maps are likely to be of low quality. 47 |

48 | 49 | EOHTML 50 | $banner_file->chmod(0644); 51 | } 52 | my $res = Mojo::UserAgent->new->inactivity_timeout(300)->post("http://localhost:$ENV{RENDERER_PORT}/renderhtml", => 53 | form => \%args, 54 | )->result; 55 | $res->is_success or die $res->error . "\n" . $res->body; 56 | 57 | }); 58 | 59 | $app->minion->add_task(finish_run => sub { 60 | my ($job, %args) = @_; 61 | $app->pg->db->query('update runs set state=?, ended=to_timestamp(?) where id=?', 62 | 'finished', time(), $args{run_id}, 63 | ); 64 | }); 65 | 66 | } 67 | 68 | 1; 69 | -------------------------------------------------------------------------------- /pred/app/pred.py: -------------------------------------------------------------------------------- 1 | import os 2 | import urllib.request, json 3 | import pandas as pd 4 | import numpy as np 5 | import george 6 | from kernel import kernel 7 | from cs import cs_to_stdev, stdev_to_cs 8 | 9 | from pandas.io.json import json_normalize 10 | 11 | def get_data(url=os.getenv("HISTORY_URI")): 12 | with urllib.request.urlopen(url) as res: 13 | data = json.loads(res.read().decode()) 14 | 15 | return data 16 | 17 | data = get_data() 18 | s = data[0] 19 | 20 | x, y_fof2, y_mufd, y_hmf2, sigma = [], [], [], [], [] 21 | first_tm = None 22 | last_tm = None 23 | 24 | for pt in s['history']: 25 | tm = (pd.to_datetime(pt[0]) - pd.Timestamp("1970-01-01")) // pd.Timedelta("1s") 26 | cs = pt[1] 27 | if cs < 10 and cs != -1: 28 | continue 29 | 30 | sd = cs_to_stdev(cs, adj100=True) 31 | fof2, mufd, hmf2 = pt[2:5] 32 | 33 | # print("%d\t%f\t%f\t%f\t%f\t%d" % (tm, fof2, mufd, hmf2, sd, cs)) 34 | x.append(tm / 86400.) 35 | y_fof2.append(np.log(fof2)) 36 | y_mufd.append(np.log(mufd)) 37 | y_hmf2.append(np.log(hmf2)) 38 | sigma.append(sd) 39 | if first_tm is None: 40 | first_tm = tm 41 | last_tm = tm 42 | 43 | print("") 44 | 45 | x = np.array(x) 46 | y_fof2 = np.array(y_fof2) 47 | mean_fof2 = np.mean(y_fof2) 48 | y_fof2 -= mean_fof2 49 | y_mufd = np.array(y_mufd) 50 | mean_mufd = np.mean(y_mufd) 51 | y_mufd -= mean_mufd 52 | y_hmf2 = np.array(y_hmf2) 53 | mean_hmf2 = np.mean(y_hmf2) 54 | y_hmf2 -= mean_hmf2 55 | sigma = np.array(sigma) 56 | 57 | gp = george.GP(kernel) 58 | gp.compute(x, sigma + 1e-3) 59 | 60 | tm = np.array(range(last_tm - (86400 * 2), last_tm + (86400 * 7) + 1, 300)) 61 | 62 | pred_fof2, sd_fof2 = gp.predict(y_fof2, tm / 86400., return_var=True) 63 | pred_fof2 += mean_fof2 64 | sd_fof2 = np.sqrt(sd_fof2) 65 | pred_mufd, sd_mufd = gp.predict(y_mufd, tm / 86400., return_var=True) 66 | pred_mufd += mean_mufd 67 | sd_mufd = np.sqrt(sd_mufd) 68 | pred_hmf2, sd_hmf2 = gp.predict(y_hmf2, tm / 86400., return_var=True) 69 | pred_hmf2 += mean_hmf2 70 | sd_hmf2 = np.sqrt(sd_hmf2) 71 | 72 | for i in range(len(tm)): 73 | print("%d\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%f\t%d" % ( 74 | tm[i], 75 | np.exp(pred_fof2[i]), np.exp(pred_fof2[i]-sd_fof2[i]), np.exp(pred_fof2[i]+sd_fof2[i]), 76 | np.exp(pred_mufd[i]), np.exp(pred_mufd[i]-sd_mufd[i]), np.exp(pred_mufd[i]+sd_mufd[i]), 77 | np.exp(pred_hmf2[i]), np.exp(pred_hmf2[i]-sd_hmf2[i]), np.exp(pred_hmf2[i]+sd_hmf2[i]), 78 | sd_mufd[i], stdev_to_cs(sd_mufd[i]) 79 | )) 80 | 81 | print("") 82 | #print("# learned kernel = %s" % gp.kernel_) 83 | -------------------------------------------------------------------------------- /iri2020/src/iri2020_driver.F90: -------------------------------------------------------------------------------- 1 | program basictest 2 | use, intrinsic:: iso_fortran_env, only: stderr=>error_unit, stdout=>output_unit 3 | implicit none 4 | 5 | logical :: jf(50) 6 | integer, parameter :: jmag = 0 7 | integer :: iyyyy, mmdd, Nalt 8 | real :: glat, glon, dhour 9 | integer :: ymdhms(6) 10 | real:: alt_km_range(3) 11 | real:: TECtotal, TECtop, TECbot 12 | 13 | #ifndef BIN_DIR 14 | #define BIN_DIR '.' 15 | #endif 16 | character(*), parameter :: datadir = BIN_DIR // '/iri2020/data' 17 | character(256) :: datadir1 18 | common /folders/ datadir1 19 | 20 | 21 | real :: oarr(100), outf(20,1000) 22 | real, allocatable :: altkm(:) 23 | character(80) :: argv 24 | integer :: i 25 | 26 | jf = .true. 27 | jf(4:6) = .false. 28 | jf(21:23) = .false. 29 | jf(26) = .false. 30 | jf(28:30) = .false. 31 | jf(33:35) = .false. 32 | jf(39) = .false. 33 | jf(47) = .false. 34 | 35 | ! --- command line input 36 | if (command_argument_count() < 11) then 37 | write(stderr,*) 'need input parameters: year month day hour minute second glat glon min_alt_km max_alt_km step_alt_km' 38 | stop 1 39 | endif 40 | 41 | do i=1,6 42 | call get_command_argument(i,argv) 43 | read(argv,*) ymdhms(i) 44 | enddo 45 | 46 | call get_command_argument(7, argv) 47 | read(argv,*) glat 48 | 49 | call get_command_argument(8, argv) 50 | read(argv,*) glon 51 | 52 | do i = 1,3 53 | call get_command_argument(8+i, argv) 54 | read(argv,*) alt_km_range(i) 55 | enddo 56 | 57 | ! --- parse 58 | Nalt = int((alt_km_range(2) - alt_km_range(1)) / alt_km_range(3)) + 1 59 | allocate(altkm(Nalt)) 60 | 61 | 62 | altkm(1) = alt_km_range(1) 63 | do i = 2,Nalt 64 | altkm(i) = altkm(i-1) + alt_km_range(3) 65 | enddo 66 | 67 | iyyyy = ymdhms(1) 68 | mmdd = ymdhms(2) * 100 + ymdhms(3) 69 | dhour = ymdhms(4) + ymdhms(5) / 60. + ymdhms(6) / 3600. 70 | 71 | datadir1 = datadir 72 | call read_ig_rz 73 | call readapf107 74 | 75 | call IRI_SUB(JF,JMAG,glat,glon,IYYYY,MMDD,DHOUR+25., & 76 | alt_km_range(1), alt_km_range(2), alt_km_range(3), & 77 | OUTF,OARR, datadir) 78 | 79 | ! --- for TEC 80 | call iri_tec(alt_km_range(1), alt_km_range(2), 2,& 81 | TECtotal, TECtop, TECbot) 82 | oarr(37) = TECtotal ! tec-units (10^16 m^-2) 83 | oarr(38) = TECtop ! % of total 84 | 85 | !print '(A,ES10.3,A,F5.1,A)','NmF2 ',oarr(1),' [m^-3] hmF2 ',oarr(2),' [km] ' 86 | !print '(A,F10.3,A,I3,A,F10.3)','F10.7 ',oarr(41), ' Ap ',int(oarr(51)),' B0 ',oarr(10) 87 | 88 | !print *,'Altitude Ne O2+' 89 | do i = 1,Nalt 90 | write(stdout, '(F10.3, 11ES16.8)') altkm(i), outf(:11,i) 91 | enddo 92 | 93 | 94 | write(stdout, '(/,100ES16.8)') oarr 95 | 96 | end program 97 | 98 | -------------------------------------------------------------------------------- /iri2020/src/reference/irisubg.for: -------------------------------------------------------------------------------- 1 | 2 | C 3 | C > f2py irisubg.for -m irisub -h irisubg.pyf 4 | C > f2py -c irisubg.pyf irisubg.for irisub.for irifun.for iritec.for iridreg.for cira.for igrf.for iriflip.for 5 | C 6 | 7 | subroutine irisubg(jf,jmag,alati,along,iyyyy,mmdd,dhour, 8 | & heibeg,heiend,heistp,dirdata,outf,oarr) 9 | 10 | real,intent(out) :: outf(30,1000),oarr(100) 11 | 12 | logical jf(50) 13 | integer jmag,iyyyy,mmdd 14 | real alati,along,dhour,heibeg,heiend,heistp 15 | character*256 dirdata,dirdata1 16 | 17 | 18 | Cf2py intent(in) jf,jmag,iyyyy,mmdd,alati,along,dhour 19 | Cf2py intent(in) heibeg,heiend,heistp,dirdata 20 | 21 | 22 | common /folders/ dirdata1 23 | dirdata1 = trim(dirdata) 24 | 25 | call read_ig_rz 26 | call readapf107 27 | 28 | call iri_sub(jf,jmag,alati,along,iyyyy,mmdd,dhour, 29 | & heibeg,heiend,heistp,outf,oarr) 30 | 31 | end subroutine irisubg 32 | 33 | 34 | 35 | subroutine irisubgl(jf,jmag,iyyyy,mmdd,dhour, 36 | & coordl,lenl,dirdata,outf1,oarr1) 37 | 38 | real, intent(in) :: coordl(lenl,3) 39 | real, intent(out) :: outf1(30,lenl),oarr1(100,lenl) 40 | 41 | logical jf(50) 42 | integer jmag,iyyyy,mmdd 43 | real alati,along,dhour,heibeg,heiend,heistp 44 | integer lenl,i,j 45 | 46 | character*256 dirdata,dirdata1 47 | 48 | real outf(30,1000),oarr(100) 49 | 50 | Cf2py intent(in) jf,jmag,iyyyy,mmdd,dhour,dirdata 51 | Cf2py integer intent(hide),depend(coordl) :: lenl=shape(coordl,0) 52 | 53 | common /folders/ dirdata1 54 | dirdata1 = trim(dirdata) 55 | 56 | call read_ig_rz 57 | call readapf107 58 | 59 | do i=1,lenl 60 | 61 | along = real(coordl(i,1),kind(along)) 62 | alati = real(coordl(i,3),kind(alati)) 63 | 64 | heibeg = real(coordl(i,2),kind(heibeg)) 65 | heiend = heibeg + 1.0 66 | heistp = 1.0 67 | 68 | call iri_sub(jf,jag,alati,along,iyyyy,mmdd,dhour, 69 | & heibeg,heiend,heistp,outf,oarr) 70 | 71 | do j=1,30 72 | outf1(j,i) = outf(j,1) 73 | end do 74 | 75 | do j=1,100 76 | oarr1(j,i) = oarr(j) 77 | end do 78 | 79 | end do 80 | 81 | end subroutine irisubgl 82 | -------------------------------------------------------------------------------- /diffusion/app/movie.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torchvision as tv 4 | import diffusers 5 | import jsonargparse 6 | from pathlib import Path 7 | from models import DiffusionModel, GuidanceModel 8 | from tqdm import tqdm 9 | import torch.nn.functional as F 10 | import contextlib 11 | from util import scale_from_diffusion 12 | 13 | def main( 14 | diffuser_checkpoint: Path = Path("diffuser_checkpoint.ckpt"), 15 | num_timesteps: int = 100, 16 | num_samples: int = 9, 17 | seed: int = 0, 18 | year: float = 2023.0, 19 | day: float = 123.0, 20 | ssn: float = 100.0, 21 | guidance_scale: float = 15.0, 22 | ): 23 | """Generates images from a trained diffusion model.""" 24 | 25 | dm = DiffusionModel.load_from_checkpoint(diffuser_checkpoint).to(device="cuda") 26 | dm.eval() 27 | 28 | scheduler = diffusers.schedulers.DDPMScheduler(rescale_betas_zero_snr=True) 29 | scheduler.set_timesteps(num_timesteps, device=dm.device) 30 | 31 | for qhour in range(96): 32 | torch.manual_seed(seed) 33 | hour = float(qhour) / 4.0 34 | target = torch.tensor([ 35 | (year - 2000. + day / 365.) / 50., 36 | math.sin(day * 2 * math.pi / 365), 37 | math.cos(day * 2 * math.pi / 365), 38 | math.sin(hour * 2 * math.pi / 24), 39 | math.cos(hour * 2 * math.pi / 24), 40 | ssn / 100.0 - 1.0, 41 | ]) 42 | target = target.to(dm.device) 43 | encoded_target = dm.param_encoder(target).unsqueeze(0).repeat(num_samples, 1) 44 | 45 | x = torch.randn((num_samples, 4, 24, 48), device=dm.device) 46 | 47 | for i, t in tqdm(enumerate(scheduler.timesteps)): 48 | torch.compiler.cudagraph_mark_step_begin() 49 | model_input = scheduler.scale_model_input(x, t) 50 | with torch.no_grad(): 51 | noise_pred_guided = dm.model(model_input, t, class_labels=encoded_target).sample 52 | noise_pred_unguided = dm.model(model_input, t, class_labels=null_target).sample 53 | noise_pred = noise_pred_unguided + guidance_scale * (noise_pred_guided - noise_pred_unguided) 54 | x = x.detach().requires_grad_() 55 | x = scheduler.step(noise_pred, t, x).prev_sample 56 | 57 | outs = scale_from_diffusion(dm.vae.decode(dm.scale_latents(x)).sample) 58 | outs = outs[..., :181, :361] 59 | 60 | image_grid = tv.utils.make_grid(outs, nrow=math.ceil(math.sqrt(num_samples))) 61 | 62 | filename = f"out/generated-{qhour:02d}.png" 63 | tv.utils.save_image(image_grid, filename) 64 | print(f"Generated images saved to {filename}. Final loss: {guidance_loss.mean().item()}") 65 | 66 | ensemble = torch.quantile(outs, 0.5, dim=0) 67 | tv.utils.save_image(ensemble, f"out/ensemble-{qhour:02d}.png") 68 | 69 | if __name__ == "__main__": 70 | jsonargparse.CLI(main) 71 | -------------------------------------------------------------------------------- /cosmic/backfill_modip_mp.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import io 4 | import datetime 5 | import numpy as np 6 | import ppigrf 7 | import multiprocessing 8 | import queue 9 | 10 | import psycopg 11 | import psycopg.rows 12 | 13 | def calc_angles(row): 14 | east, north, up = ppigrf.igrf(lat=float(row['latitude']), lon=float(row['longitude']), h=1, date=row['time']) 15 | dip_angle = np.degrees(np.arctan2(-up, np.hypot(east, north)))[0,0] 16 | modip = np.degrees(np.arctan2(np.radians(dip_angle), np.sqrt(np.cos(np.radians(float(row['latitude'])))))) 17 | return row['id'], dip_angle, modip 18 | 19 | def consume_queue(rows, results): 20 | while True: 21 | try: 22 | row = rows.get() 23 | angles = calc_angles(row) 24 | results.put(angles) 25 | rows.task_done() 26 | except ValueError as e: 27 | return 28 | 29 | NPROC=12 30 | 31 | if __name__ == '__main__': 32 | multiprocessing.set_start_method('spawn') 33 | rows = multiprocessing.JoinableQueue(NPROC*2) 34 | results = multiprocessing.Queue() 35 | 36 | processes = [] 37 | for i in range(NPROC): 38 | process = multiprocessing.Process(target=consume_queue, args=(rows, results)) 39 | processes.append(process) 40 | process.start() 41 | 42 | dsn = "dbname='%s' user='%s' host='%s' password='%s'" % (os.getenv("DB_NAME"), os.getenv("DB_USER"), os.getenv("DB_HOST"), os.getenv("DB_PASSWORD")) 43 | con = psycopg.connect(dsn) 44 | 45 | query = "select id, time, latitude, longitude from cosmic_eval where dip_angle is null order by time - hours_ahead * interval '1 hour' asc limit 100000" 46 | 47 | i = 0 48 | 49 | with con.cursor() as cur: 50 | cur.row_factory = psycopg.rows.dict_row 51 | cur.execute(query) 52 | for row in cur: 53 | rows.put(row) 54 | while True: 55 | try: 56 | rowid, dip_angle, modip = results.get_nowait() 57 | except queue.Empty as e: 58 | break 59 | 60 | 61 | con.execute("update cosmic_eval set dip_angle=%s, modip=%s where id=%s" % (dip_angle, modip, rowid)) 62 | i += 1 63 | if i % 1000 == 0: 64 | print("updated %d rows" % i) 65 | con.commit() 66 | 67 | rows.close() 68 | rows.join() 69 | 70 | while True: 71 | try: 72 | rowid, dip_angle, modip = results.get_nowait() 73 | except queue.Empty as e: 74 | break 75 | 76 | 77 | con.execute("update cosmic_eval set dip_angle=%s, modip=%s where id=%s" % (dip_angle, modip, rowid)) 78 | i += 1 79 | if i % 1000 == 0: 80 | print("updated %d rows" % i) 81 | con.commit() 82 | 83 | print("updated %d rows" % i) 84 | con.commit() 85 | 86 | for p in processes: 87 | p.kill() 88 | -------------------------------------------------------------------------------- /www/stations/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | GIRO Station Data 16 | 17 | 18 | 19 | 20 | 21 |
22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 |
StationTimeMUFDM(D)TECfoF2hmF2foEConfidenceLongitudeLatitude
39 |
40 |

41 | 42 | 43 | 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /api/app/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | GIRO Station Data 18 | 19 | 20 | 21 | 22 |
23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 |
StationTimeMUFDM(D)TECfoF2hmF2foEConfidenceLongitudeLatitudeSource
41 |
42 |

43 | 44 | 45 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /diffusion/app/movie_cfg.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torchvision as tv 4 | import diffusers 5 | import jsonargparse 6 | from pathlib import Path 7 | from models import ConditionedDiffusionModel 8 | from tqdm import tqdm 9 | import torch.nn.functional as F 10 | import contextlib 11 | from util import scale_from_diffusion 12 | 13 | def main( 14 | diffuser_checkpoint: Path = Path("diffuser_checkpoint.ckpt"), 15 | num_timesteps: int = 100, 16 | num_samples: int = 9, 17 | seed: int = 0, 18 | year: float = 2023.0, 19 | day: float = 123.0, 20 | ssn: float = 100.0, 21 | guidance_scale: float = 15.0, 22 | ): 23 | """Generates images from a trained diffusion model.""" 24 | 25 | dm = ConditionedDiffusionModel.load_from_checkpoint(diffuser_checkpoint).to(device="cuda") 26 | dm.eval() 27 | 28 | scheduler = diffusers.schedulers.DDPMScheduler( 29 | rescale_betas_zero_snr=True, prediction_type=dm.inference_scheduler.config.prediction_type) 30 | scheduler.set_timesteps(num_timesteps, device=dm.device) 31 | 32 | for qhour in range(96): 33 | torch.manual_seed(seed) 34 | hour = float(qhour) / 4.0 35 | target = torch.tensor([ 36 | (year - 2000. + day / 365.) / 50., 37 | math.sin(day * 2 * math.pi / 365), 38 | math.cos(day * 2 * math.pi / 365), 39 | math.sin(hour * 2 * math.pi / 24), 40 | math.cos(hour * 2 * math.pi / 24), 41 | ssn / 100.0 - 1.0, 42 | ]) 43 | target = target.to(dm.device) 44 | encoded_target = dm.param_encoder(target).unsqueeze(0).repeat(num_samples, 1) 45 | null_target = torch.zeros_like(encoded_target) 46 | 47 | x = torch.randn((num_samples, 4, 24, 48), device=dm.device) 48 | 49 | for i, t in tqdm(enumerate(scheduler.timesteps)): 50 | torch.compiler.cudagraph_mark_step_begin() 51 | model_input = scheduler.scale_model_input(x, t) 52 | with torch.no_grad(): 53 | noise_pred_guided = dm.model(model_input, t, class_labels=encoded_target).sample 54 | noise_pred_unguided = dm.model(model_input, t, class_labels=null_target).sample 55 | noise_pred = noise_pred_unguided + guidance_scale * (noise_pred_guided - noise_pred_unguided) 56 | x = x.detach().requires_grad_() 57 | x = scheduler.step(noise_pred, t, x).prev_sample 58 | 59 | outs = scale_from_diffusion(dm.vae.decode(dm.scale_latents(x)).sample) 60 | outs = outs[..., :181, :361].flip((2,)) 61 | 62 | image_grid = tv.utils.make_grid(outs, nrow=math.ceil(math.sqrt(num_samples))) 63 | 64 | filename = f"out/generated-{qhour:02d}.png" 65 | tv.utils.save_image(image_grid, filename) 66 | 67 | ensemble = torch.quantile(outs, 0.5, dim=0) 68 | tv.utils.save_image(ensemble, f"out/ensemble-{qhour:02d}.png") 69 | 70 | stdev = torch.std(outs, dim=0) 71 | stdev = (stdev * 100.).clip(0, 1) 72 | tv.utils.save_image(stdev, f"out/stdev-{qhour:02d}.png") 73 | 74 | if __name__ == "__main__": 75 | jsonargparse.CLI(main) 76 | -------------------------------------------------------------------------------- /diffusion/app/generate_guided.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torchvision as tv 4 | import diffusers 5 | import jsonargparse 6 | from pathlib import Path 7 | from models import DiffusionModel, GuidanceModel 8 | from tqdm import tqdm 9 | import torch.nn.functional as F 10 | import contextlib 11 | from util import scale_from_diffusion 12 | 13 | def main( 14 | diffuser_checkpoint: Path = Path("diffuser_checkpoint.ckpt"), 15 | guidance_checkpoint: Path = Path("guidance_checkpoint.ckpt"), 16 | num_timesteps: int = 20, 17 | num_samples: int = 9, 18 | seed: int = 0, 19 | year: float = 2023.0, 20 | day: float = 123.0, 21 | hour: float = 12.0, 22 | ssn: float = 100.0, 23 | guidance_scale: float = 4.0, 24 | ): 25 | """Generates images from a trained diffusion model.""" 26 | 27 | torch.manual_seed(seed) 28 | dm = DiffusionModel.load_from_checkpoint(diffuser_checkpoint).to(device="cuda") 29 | gm = GuidanceModel.load_from_checkpoint(guidance_checkpoint).to(device="cuda") 30 | dm.eval() 31 | gm.eval() 32 | 33 | scheduler = diffusers.schedulers.DDIMScheduler(rescale_betas_zero_snr=True) 34 | scheduler.set_timesteps(num_timesteps, device=dm.device) 35 | 36 | target = torch.tensor([ 37 | (year - 2000. + day / 365.) / 50., 38 | math.sin(day * 2 * math.pi / 365), 39 | math.cos(day * 2 * math.pi / 365), 40 | math.sin(hour * 2 * math.pi / 24), 41 | math.cos(hour * 2 * math.pi / 24), 42 | ssn / 100.0 - 1.0, 43 | ]) 44 | target = target.to(dm.device).unsqueeze(0).repeat(num_samples, 1) 45 | 46 | x = torch.randn((num_samples, 4, 24, 48), device=dm.device) 47 | 48 | for i, t in tqdm(enumerate(scheduler.timesteps)): 49 | print("") 50 | torch.compiler.cudagraph_mark_step_begin() 51 | model_input = scheduler.scale_model_input(x, t) 52 | with torch.no_grad(): 53 | noise_pred = dm.model(model_input, t).sample 54 | x = x.detach().requires_grad_() 55 | x0 = scheduler.step(noise_pred, t, x).pred_original_sample 56 | 57 | x0_decoded = scale_from_diffusion(dm.vae.decode(dm.scale_latents(x0)).sample) 58 | 59 | # tv.utils.save_image(scale_from_diffusion(x0[0, :, :, :]), f"out/step_{i:03d}.png") 60 | 61 | guidance_out = gm.model(x0_decoded[..., :184, :368]) 62 | print(guidance_out) 63 | guidance_loss = F.mse_loss(guidance_out, target) 64 | print(guidance_loss) 65 | guidance_grad = -torch.autograd.grad(guidance_loss, x)[0] 66 | x = x.detach() + guidance_grad * guidance_scale 67 | x = scheduler.step(noise_pred, t, x).prev_sample 68 | 69 | outs = scale_from_diffusion(dm.vae.decode(dm.scale_latents(x)).sample) 70 | 71 | image_grid = tv.utils.make_grid(outs, nrow=math.ceil(math.sqrt(num_samples))) 72 | 73 | filename = "out/generated.png" 74 | tv.utils.save_image(image_grid, filename) 75 | print(f"Generated images saved to {filename}") 76 | 77 | ensemble = torch.quantile(outs, 0.5, dim=0) 78 | tv.utils.save_image(ensemble, "out/ensemble.png") 79 | 80 | if __name__ == "__main__": 81 | jsonargparse.CLI(main) 82 | -------------------------------------------------------------------------------- /www/static/promise.min.js: -------------------------------------------------------------------------------- 1 | !function(e){("object"!=typeof exports||"undefined"==typeof module)&&"function"==typeof define&&define.amd?define(e):e()}(function(){"use strict";function e(n){var t=this.constructor;return this.then(function(e){return t.resolve(n()).then(function(){return e})},function(e){return t.resolve(n()).then(function(){return t.reject(e)})})}var n=setTimeout;function a(e){return e&&"undefined"!=typeof e.length}function o(){}function i(e){if(!(this instanceof i))throw new TypeError("Promises must be constructed via new");if("function"!=typeof e)throw new TypeError("not a function");this._state=0,this._handled=!1,this._value=undefined,this._deferreds=[],s(e,this)}function r(o,r){for(;3===o._state;)o=o._value;0!==o._state?(o._handled=!0,i._immediateFn(function(){var e=1===o._state?r.onFulfilled:r.onRejected;if(null!==e){var n;try{n=e(o._value)}catch(t){return void u(r.promise,t)}f(r.promise,n)}else(1===o._state?f:u)(r.promise,o._value)})):o._deferreds.push(r)}function f(e,n){try{if(n===e)throw new TypeError("A promise cannot be resolved with itself.");if(n&&("object"==typeof n||"function"==typeof n)){var t=n.then;if(n instanceof i)return e._state=3,e._value=n,void c(e);if("function"==typeof t)return void s(function o(e,n){return function(){e.apply(n,arguments)}}(t,n),e)}e._state=1,e._value=n,c(e)}catch(r){u(e,r)}}function u(e,n){e._state=2,e._value=n,c(e)}function c(e){2===e._state&&0===e._deferreds.length&&i._immediateFn(function(){e._handled||i._unhandledRejectionFn(e._value)});for(var n=0,t=e._deferreds.length;n 5 | 6 | 7 | 24h Point-to-Point MOF/LOF 8 | 9 | 10 | 23 | 24 | 25 | 26 |
27 |
28 | 29 | 30 | 31 | 32 | 33 | 38 | 39 | 40 | 41 |
42 |
43 | 44 |
45 |
46 | 110 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /diffusion/app/generate_guided_cfg.py: -------------------------------------------------------------------------------- 1 | import math 2 | import torch 3 | import torchvision as tv 4 | import diffusers 5 | import jsonargparse 6 | from pathlib import Path 7 | from models import ConditionedDiffusionModel, GuidanceModel 8 | from tqdm import tqdm 9 | import torch.nn.functional as F 10 | import contextlib 11 | from util import scale_from_diffusion 12 | 13 | def main( 14 | diffuser_checkpoint: Path = Path("diffuser_checkpoint.ckpt"), 15 | guidance_checkpoint: Path = None, 16 | num_timesteps: int = 20, 17 | num_samples: int = 9, 18 | seed: int = 0, 19 | year: float = 2023.0, 20 | day: float = 123.0, 21 | hour: float = 12.0, 22 | ssn: float = 100.0, 23 | guidance_scale: float = 4.0, 24 | ): 25 | """Generates images from a trained diffusion model.""" 26 | 27 | torch.manual_seed(seed) 28 | dm = ConditionedDiffusionModel.load_from_checkpoint(diffuser_checkpoint).to(device="cuda") 29 | dm.eval() 30 | 31 | if guidance_checkpoint is None: 32 | check_against_guidance = False 33 | else: 34 | check_against_guidance = True 35 | gm = GuidanceModel.load_from_checkpoint(guidance_checkpoint).to(device="cuda") 36 | gm.eval() 37 | 38 | scheduler = diffusers.schedulers.DDPMScheduler( 39 | rescale_betas_zero_snr=True, prediction_type=dm.inference_scheduler.config.prediction_type) 40 | scheduler.set_timesteps(num_timesteps, device=dm.device) 41 | 42 | target = torch.tensor([ 43 | (year - 2000. + day / 365.) / 50., 44 | math.sin(day * 2 * math.pi / 365), 45 | math.cos(day * 2 * math.pi / 365), 46 | math.sin(hour * 2 * math.pi / 24), 47 | math.cos(hour * 2 * math.pi / 24), 48 | ssn / 100.0 - 1.0, 49 | ]) 50 | target = target.to(dm.device) 51 | encoded_target = dm.param_encoder(target) 52 | target = target.unsqueeze(0).repeat(num_samples, 1) 53 | encoded_target = encoded_target.unsqueeze(0).repeat(num_samples, 1) 54 | null_target = torch.zeros_like(encoded_target) 55 | 56 | x = torch.randn((num_samples, 4, 24, 48), device=dm.device) 57 | 58 | for i, t in tqdm(enumerate(scheduler.timesteps)): 59 | print("") 60 | torch.compiler.cudagraph_mark_step_begin() 61 | model_input = scheduler.scale_model_input(x, t) 62 | with torch.no_grad(): 63 | noise_pred_guided = dm.model(model_input, t, class_labels=encoded_target).sample 64 | noise_pred_unguided = dm.model(model_input, t, class_labels=null_target).sample 65 | noise_pred = noise_pred_unguided + guidance_scale * (noise_pred_guided - noise_pred_unguided) 66 | 67 | if check_against_guidance: 68 | x = x.detach().requires_grad_() 69 | x0 = scheduler.step(noise_pred, t, x).pred_original_sample 70 | x0_decoded = scale_from_diffusion(dm.vae.decode(dm.scale_latents(x0)).sample) 71 | 72 | # tv.utils.save_image(scale_from_diffusion(x0[0, :, :, :]), f"out/step_{i:03d}.png") 73 | 74 | guidance_out = gm.model(x0_decoded[..., :184, :368]) 75 | print(guidance_out) 76 | guidance_loss = F.mse_loss(guidance_out, target) 77 | print(guidance_loss) 78 | x = scheduler.step(noise_pred, t, x).prev_sample 79 | 80 | outs = scale_from_diffusion(dm.vae.decode(dm.scale_latents(x)).sample) 81 | 82 | image_grid = tv.utils.make_grid(outs, nrow=math.ceil(math.sqrt(num_samples))) 83 | 84 | filename = "out/generated.png" 85 | tv.utils.save_image(image_grid, filename) 86 | print(f"Generated images saved to {filename}") 87 | 88 | ensemble = torch.quantile(outs, 0.5, dim=0) 89 | tv.utils.save_image(ensemble, "out/ensemble.png") 90 | 91 | if __name__ == "__main__": 92 | jsonargparse.CLI(main) 93 | -------------------------------------------------------------------------------- /iri2020/src/reference/iriweb.for: -------------------------------------------------------------------------------- 1 | subroutine iri_web(jmag,jf,alati,along,iyyyy,mmdd,iut,dhour, 2 | & height,h_tec_max,ivar,vbeg,vend,vstp,a,b) 3 | c----------------------------------------------------------------------- 4 | c changes: 5 | c 11/16/99 jf(30) instead of jf(17) 6 | c 10/31/08 outf, a, b (100 -> 500) 7 | c 8 | c----------------------------------------------------------------------- 9 | c input: jmag,alati,along,iyyyy,mmdd,dhour see IRI_SUB 10 | c height height in km 11 | c h_tec_max =0 no TEC otherwise upper boundary for integral 12 | c iut =1 for UT =0 for LT 13 | c ivar =1 altitude 14 | c =2,3 latitude,longitude 15 | c =4,5,6 year,month,day 16 | c =7 day of year 17 | c =8 hour (UT or LT) 18 | c vbeg,vend,vstp variable range (begin,end,step) 19 | c output: a similar to outf in IRI_SUB 20 | c b similar to oarr in IRI_SUB 21 | c 22 | c numstp number of steps; maximal 1000 23 | c----------------------------------------------------------------------- 24 | dimension outf(20,1000),oar(100),oarr(100),a(20,1000) 25 | dimension xvar(8),b(100,1000) 26 | logical jf(50) 27 | 28 | nummax=1000 29 | numstp=int((vend-vbeg)/vstp)+1 30 | if(numstp.gt.nummax) numstp=nummax 31 | 32 | do 6249 i=1,100 33 | 6249 oar(i)=b(i,1) 34 | 35 | if(ivar.eq.1) then 36 | do 1249 i=1,100 37 | 1249 oarr(i)=oar(i) 38 | xhour=dhour+iut*25. 39 | call IRI_SUB(JF,JMAG,ALATI,ALONG,IYYYY,MMDD,XHOUR, 40 | & VBEG,VEND,VSTP,a,OARR) 41 | if(h_tec_max.gt.50.) then 42 | call iri_tec (50.,h_tec_max,2,tec,tect,tecb) 43 | oarr(37)=tec 44 | oarr(38)=tect 45 | endif 46 | do 1111 i=1,100 47 | 1111 b(i,1)=oarr(i) 48 | return 49 | endif 50 | 51 | if(height.le.0.0) height=100 52 | xvar(2)=alati 53 | xvar(3)=along 54 | xvar(4)=iyyyy 55 | xvar(5)=mmdd/100 56 | xvar(6)=mmdd-xvar(5)*100 57 | xvar(7)=abs(mmdd*1.) 58 | xvar(8)=dhour 59 | 60 | xvar(ivar)=vbeg 61 | 62 | alati=xvar(2) 63 | along=xvar(3) 64 | iyyyy=int(xvar(4)) 65 | if(ivar.eq.7) then 66 | mmdd=-int(vbeg) 67 | else 68 | mmdd=int(xvar(5)*100+xvar(6)) 69 | endif 70 | dhour=xvar(8)+iut*25. 71 | 72 | do 1 i=1,numstp 73 | do 1349 iii=1,100 74 | 1349 oarr(iii)=b(iii,i) 75 | call IRI_SUB(JF,JMAG,ALATI,ALONG,IYYYY,MMDD,DHOUR, 76 | & height,height,1.,OUTF,OARR) 77 | if(h_tec_max.gt.50.) then 78 | call iri_tec (50.,h_tec_max,2,tec,tect,tecb) 79 | oarr(37)=tec 80 | oarr(38)=tect 81 | endif 82 | do 2 ii=1,20 83 | 2 a(ii,i)=outf(ii,1) 84 | do 2222 ii=1,100 85 | 2222 b(ii,i)=oarr(ii) 86 | xvar(ivar)=xvar(ivar)+vstp 87 | 88 | alati=xvar(2) 89 | along=xvar(3) 90 | iyyyy=int(xvar(4)) 91 | if(ivar.eq.7) then 92 | mmdd=-xvar(7) 93 | else 94 | mmdd=int(xvar(5)*100+xvar(6)) 95 | endif 96 | dhour=xvar(8)+iut*25. 97 | 1 continue 98 | 99 | return 100 | end 101 | 102 | -------------------------------------------------------------------------------- /iri2020/README.md: -------------------------------------------------------------------------------- 1 | [![image](https://zenodo.org/badge/DOI/10.5281/zenodo.240895.svg)](https://doi.org/10.5281/zenodo.240895) 2 | [![Build Status](https://travis-ci.org/scivision/IRI2016.svg?branch=master)](https://travis-ci.org/scivision/IRI2016) 3 | [![image](https://coveralls.io/repos/github/scivision/IRI2016/badge.svg?branch=master)](https://coveralls.io/github/scivision/IRI2016?branch=master) 4 | [![Build status](https://ci.appveyor.com/api/projects/status/euvvim6aus3dagwq?svg=true)](https://ci.appveyor.com/project/scivision/pyiri2016) 5 | [![PyPi version](https://img.shields.io/pypi/pyversions/iri2016.svg)](https://pypi.python.org/pypi/iri2016) 6 | [![PyPi Download stats](http://pepy.tech/badge/iri2016)](http://pepy.tech/project/iri2016) 7 | 8 | 9 | # IRI2016 ionosphere model from Python and Matlab 10 | 11 | ![image](./figures/iri2DExample02.gif) 12 | 13 | Python and [Matlab](#matlab) interfaces to the International Reference Ionosphere (IRI) 2016 model. 14 | A Fortran compiler and CMake or 15 | [Meson](https://github.com/mesonbuild/meson/) 16 | is required to build the IRI2016 code. 17 | 18 | ## Python 19 | 20 | 1. Install IRI2016 command-line driver program 21 | ```sh 22 | pip install -e .[tests] 23 | ``` 24 | 2. run selftest to ensure install was completed: 25 | ```sh 26 | pytest 27 | ``` 28 | 3. try example script e.g. [AltitudeProfile.py](./AltitudeProfile.py) 29 | 30 | ## Matlab 31 | 32 | 1. compile Fortran code via [matlab/setup_iri2016.m](./matlab/setup_iri2016.m) 33 | 2. drive the simulation via a seamless command line interface, example: [matlab/RunIRI2016.m](./matlab/RunIRI2016.m) 34 | 35 | 36 | 37 | ## Compiler 38 | 39 | Any Fortran compiler will do. 40 | IRI2016 has been tested with compilers including: 41 | 42 | * Gfortran 4.8, 5, 6, 7, 8 43 | * Intel `ifort` 44 | * PGI `pgf90` 45 | * Flang `flang` 46 | 47 | If you don't already have a Fortran compiler, install Gfortran by: 48 | 49 | * Linux: `apt install gfortran` 50 | * Mac: `brew install gcc` 51 | * [Windows](https://www.scivision.co/windows-gcc-gfortran-cmake-make-install/) 52 | 53 | 54 | ## Usage 55 | 56 | * Height-profile: plot density and temperatures vs [altitude](./AltitudeProfile.py) 57 | ![image](./figures/iri1DExample01.png) 58 | * Latitudinal profile: plot densities and height at the peak of F2, F2, and E regions vs [geographic latitude](./LatitudeProfile.py) 59 | ![image](./figures/iri1DExample02.png) 60 | * GMT profile: plot densities and height at the peak of F2, F2, and E regions vs universal [time](./TimeProfile.py) 61 | ![image](./figures/iri1DExample08.png) 62 | * Height vs GMT: plot Ne, Te, and Ti as a function of height and universal [time](./examples/example01.py) 63 | ![image](./figures/iri2DExample01.png) 64 | * Latitude vs Longitude: plot of foF2 a function of geographic latitude and [longitude](./examples/example02.py) 65 | ![image](./figures/iri2DExample02.png) 66 | 67 | ### Matlab / GNU Octave 68 | IRI2016 is readily accessible from Matlab and GNU Octave. 69 | 70 | 1. From Matlab, verify everything is working by from the `iri2016/` directory: 71 | ```matlab 72 | runtests('tests') 73 | ``` 74 | 2. Use [iri2016.m](./matlab/iri2016.m) function to access IRI2016 quantities. See [RunIRI2016.m](./matlab/RunIRI2016.m) for simple example use / plots. 75 | 76 | ![Matlab IRI2016 plot](./figures/matlab.png) 77 | 78 | ## Data files 79 | 80 | `iri2016/iri2016/data/index/{apf107,ig_rz}.dat` are 81 | [regularly updated](http://irimodel.org/indices/). 82 | Currently we don't auto-update those. 83 | 84 | ## Direct compilation 85 | 86 | These commands are not normally needed unless you want to work with the Fortran code more directly. 87 | 88 | ### Fortran compile 89 | 90 | ```sh 91 | cd build 92 | 93 | meson ../src 94 | 95 | ninja 96 | 97 | meson test 98 | ``` 99 | 100 | ## Notes 101 | 102 | * [2016 presentation](https://doi.org/10.5281/zenodo.1493021) 103 | -------------------------------------------------------------------------------- /storm/app/main.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import scipy.io 3 | import urllib.request 4 | import io 5 | import os 6 | import shutil 7 | 8 | def kp_to_ap(kp): 9 | idx = int((kp + 0.1) * 3) 10 | return [0,2,3,4,5,6,7,9,12,15,18,22,27,32,39,48,56,67,80,94,111,132,154,179,207,236,300,400][idx] 11 | 12 | with open('/iri-index/predicted/apf107.tmp', 'w') as out: 13 | nlines = 0 14 | with open('/iri-index/chain/apf107.dat') as fh: 15 | for line in fh: 16 | print(line, file=out, end='') 17 | nlines += 1 18 | 19 | forecast = {} 20 | ql = {} 21 | 22 | with urllib.request.urlopen("https://spaceweather.gfz-potsdam.de/fileadmin/ruggero/Kp_forecast/forecast_figures/KP_FORECAST_CURRENT.mat") as res: 23 | content = res.read() 24 | bio = io.BytesIO(content) 25 | mat = scipy.io.loadmat(bio) 26 | 27 | for time, kp in zip(mat['t'], mat['kp'][0]): 28 | dt = datetime.datetime.strptime(time, '%Y-%m-%d %H:%M:%S') 29 | forecast[dt] = kp_to_ap(kp) 30 | 31 | with urllib.request.urlopen("http://www-app3.gfz-potsdam.de/kp_index/Kp_ap_nowcast.txt") as res: 32 | # alt: ftp://ftp.gfz-potsdam.de/pub/home/obs/Kp_ap_Ap_SN_F107/Kp_ap_nowcast.txt 33 | for line in res: 34 | line = line.decode('utf-8') 35 | if line.startswith("#"): 36 | continue 37 | fields = line.split() 38 | dt = datetime.datetime( 39 | year = int(fields[0]), 40 | month = int(fields[1]), 41 | day = int(fields[2]), 42 | hour = int(float(fields[3])), 43 | minute = 0, 44 | second = 0, 45 | ) 46 | ap = int(fields[8]) 47 | if ap == -1: 48 | continue 49 | 50 | ql[dt] = ap 51 | 52 | keys_sorted = sorted(forecast.keys()) 53 | fc_first = keys_sorted[0] 54 | fc_last = keys_sorted[-1] 55 | fc_first_day = datetime.date(year=fc_first.year, month=fc_first.month, day=fc_first.day) 56 | fc_last_day = datetime.date(year=fc_last.year, month=fc_last.month, day=fc_last.day) 57 | 58 | keys_sorted = sorted(ql.keys()) 59 | ql_first = keys_sorted[0] 60 | ql_last = keys_sorted[-1] 61 | ql_first_day = datetime.date(year=ql_first.year, month=ql_first.month, day=ql_first.day) 62 | ql_last_day = datetime.date(year=ql_last.year, month=ql_last.month, day=ql_last.day) 63 | 64 | day = datetime.date(1958,1,1) + datetime.timedelta(days=nlines) 65 | 66 | while day <= fc_last_day: 67 | ap = [0,0,0,0,0,0,0,0] 68 | ap_day = 0 69 | 70 | if day >= ql_first_day: 71 | for idx in range(8): 72 | dt = datetime.datetime(year=day.year, month=day.month, day=day.day, hour=idx*3) 73 | if dt < ql_first: 74 | ap[idx] = ql[ql_first] 75 | elif dt <= ql_last: 76 | ap[idx] = ql[dt] 77 | elif dt < fc_first: 78 | ap[idx] = forecast[fc_first] 79 | elif dt <= fc_last: 80 | ap[idx] = forecast[dt] 81 | else: 82 | ap[idx] = forecast[fc_last] 83 | 84 | ap_day = round(sum(ap) / len(ap)) 85 | ap = [round(a) for a in ap] 86 | 87 | print('{:3d}{:3d}{:3d}{:3d}{:3d}{:3d}{:3d}{:3d}{:3d}{:3d}{:3d}{:3d}{:3d}{:5.1f}{:5.1f}{:5.1f}'.format( 88 | day.year % 100, 89 | day.month, 90 | day.day, 91 | *ap, 92 | ap_day, 93 | -11, 94 | 0, 95 | 0, 96 | 0, 97 | ), file=out 98 | ) 99 | day += datetime.timedelta(days=1) 100 | 101 | os.rename('/iri-index/predicted/apf107.tmp', '/iri-index/predicted/apf107.dat') 102 | shutil.copy('/iri-index/chain/ig_rz.dat', '/iri-index/predicted/ig_rz.tmp') 103 | os.rename('/iri-index/predicted/ig_rz.tmp', '/iri-index/predicted/ig_rz.dat') 104 | -------------------------------------------------------------------------------- /iri2020/src/reference/iriwebg.pyf: -------------------------------------------------------------------------------- 1 | ! -*- f90 -*- 2 | ! Note: the context of this file is case sensitive. 3 | 4 | python module iriweb ! in 5 | interface ! in :iriweb 6 | subroutine iriwebg(injmag,injf,inalati,inalong,iniyyyy,inmmdd,iniut,indhour,inheight,inh_tec_max,inivar,invbeg,invend,invstp,inaddinp,dirdata,outa,outb) ! in :iriweb:iriwebg.for 7 | real*8 intent(in) :: injmag 8 | real*8 dimension(50),intent(in) :: injf 9 | real*8 intent(in) :: inalati 10 | real*8 intent(in) :: inalong 11 | real*8 intent(in) :: iniyyyy 12 | real*8 intent(in) :: inmmdd 13 | real*8 intent(in) :: iniut 14 | real*8 intent(in) :: indhour 15 | real*8 intent(in) :: inheight 16 | real*8 intent(in) :: inh_tec_max 17 | real*8 intent(in) :: inivar 18 | real*8 intent(in) :: invbeg 19 | real*8 intent(in) :: invend 20 | real*8 intent(in) :: invstp 21 | real*8 dimension(12),intent(in) :: inaddinp 22 | character*256 intent(in) :: dirdata 23 | real*8 dimension(30,1000),intent(out) :: outa 24 | real*8 dimension(100,1000),intent(out) :: outb 25 | character*256 :: dirdata1 26 | common /folders/ dirdata1 27 | end subroutine iriwebg 28 | subroutine irisubg(jf,jmag,alati,along,iyyyy,mmdd,dhour,heibeg,heiend,heistp,dirdata,outf,oarr) ! in :iriweb:iriwebg.for 29 | logical dimension(50),intent(in) :: jf 30 | integer intent(in) :: jmag 31 | real intent(in) :: alati 32 | real intent(in) :: along 33 | integer intent(in) :: iyyyy 34 | integer intent(in) :: mmdd 35 | real intent(in) :: dhour 36 | real intent(in) :: heibeg 37 | real intent(in) :: heiend 38 | real intent(in) :: heistp 39 | character*256 intent(in) :: dirdata 40 | real dimension(30,1000),intent(out) :: outf 41 | real dimension(100),intent(out) :: oarr 42 | character*256 :: dirdata1 43 | common /folders/ dirdata1 44 | end subroutine irisubg 45 | subroutine irisubgl(jf,jmag,iyyyy,mmdd,dhour,coordl,lenl,dirdata,outf1,oarr1) ! in :iriweb:iriwebg.for 46 | logical dimension(50),intent(in) :: jf 47 | integer intent(in) :: jmag 48 | integer intent(in) :: iyyyy 49 | integer intent(in) :: mmdd 50 | real intent(in) :: dhour 51 | real dimension(lenl,3),intent(in) :: coordl 52 | integer, optional,intent(hide),depend(coordl) :: lenl=shape(coordl,0) 53 | character*256 intent(in) :: dirdata 54 | real dimension(30,lenl),intent(out),depend(lenl) :: outf1 55 | real dimension(100,lenl),intent(out),depend(lenl) :: oarr1 56 | character*256 :: dirdata1 57 | common /folders/ dirdata1 58 | end subroutine irisubgl 59 | subroutine firisubl(yyyy,ddd,uhour,coordl,lenl,dirdata,edens1,ierr1) ! in :iriweb:iriwebg.for 60 | integer intent(in) :: yyyy 61 | integer intent(in) :: ddd 62 | real intent(in) :: uhour 63 | real dimension(lenl,3),intent(in) :: coordl 64 | integer, optional,intent(hide),depend(coordl) :: lenl=shape(coordl,0) 65 | character*256 intent(in) :: dirdata 66 | real dimension(lenl),intent(out),depend(lenl) :: edens1 67 | real dimension(lenl),intent(out),depend(lenl) :: ierr1 68 | character*256 :: dirdata1 69 | common /folders/ dirdata1 70 | end subroutine firisubl 71 | subroutine initialize ! in :iriweb:iriwebg.for 72 | real :: dtr 73 | real :: pi 74 | real :: humr 75 | real :: dumr 76 | common /const/ dtr,pi 77 | common /const1/ humr,dumr 78 | end subroutine initialize 79 | end interface 80 | end python module iriweb 81 | 82 | ! This file was auto-generated with f2py (version:2). 83 | ! See http://cens.ioc.ee/projects/f2py2e/ 84 | -------------------------------------------------------------------------------- /raytrace/app/raytrace/__init__.py: -------------------------------------------------------------------------------- 1 | from . import constants 2 | from . import geodesy 3 | import numpy as np 4 | 5 | r0 = constants.r_earth_km 6 | 7 | def mof_lof(iono, from_lat, from_lon, to_lat, to_lon, longpath=False, h_min_flag=True): 8 | one_from = np.ndim(from_lat) == 0 and np.ndim(from_lon) == 0 9 | 10 | dist, bearing = geodesy.distance_bearing(from_lat, from_lon, to_lat, to_lon) 11 | 12 | if longpath: 13 | dist = 2 * np.pi - dist 14 | bearing = (bearing + np.pi) % (2 * np.pi) 15 | 16 | khop = np.floor(dist / constants.maxhop).astype(int) + 1 17 | half_hop = dist / (2 * khop) 18 | max_khop = np.max(khop) 19 | 20 | h_avg = np.zeros_like(to_lat) 21 | h_min = np.full_like(to_lat, 10000.) 22 | 23 | for hop in range(1, max_khop+1): 24 | idx = hop <= khop 25 | hop_frac = 2*hop - 1 26 | cp_lat, cp_lon = geodesy.destination( 27 | (from_lat if one_from else from_lat[idx]), 28 | (from_lon if one_from else from_lon[idx]), 29 | bearing[idx], 30 | half_hop[idx] * hop_frac, 31 | ) 32 | cp_lat = (cp_lat + np.pi / 2) % np.pi - np.pi / 2 33 | cp_lon = (cp_lon + np.pi) % (2 * np.pi) - np.pi 34 | h_min[idx] = np.fmin(h_min[idx], iono.hmf2.predict(cp_lat, cp_lon)) 35 | h_avg[idx] += iono.hmf2.predict(cp_lat, cp_lon) 36 | 37 | h_avg /= khop 38 | 39 | h_calc = h_min if h_min_flag else h_avg 40 | 41 | calc_maxhop = 2 * np.arccos(r0 / (h_calc + r0)) 42 | khop = np.floor(dist / calc_maxhop).astype(int) + 1 43 | half_hop = dist / (2 * khop) 44 | max_khop = np.max(khop) 45 | 46 | phi = np.arctan2(np.sin(half_hop), (1.0 + h_calc/r0 - np.cos(half_hop))) 47 | phi = np.clip(phi, constants.min_phi, constants.max_phi) 48 | 49 | curv_corr = .9679 + 1.046 * np.clip(half_hop, .0785, .2354) 50 | 51 | m9 = curv_corr * np.power(constants.m9_2, khop) / np.cos(phi) 52 | 53 | takeoff = np.arctan2(np.cos(half_hop) - (r0/(h_calc+r0)), np.sin(half_hop)) 54 | takeoff_true = takeoff 55 | takeoff = np.clip(takeoff, constants.min_takeoff, constants.max_takeoff) 56 | 57 | cmof = np.full_like(to_lat, 10000.) 58 | 59 | rms_gyf = np.zeros_like(to_lat) 60 | sum_I = np.zeros_like(to_lat) 61 | 62 | for hop in range(1, max_khop+1): 63 | idx = hop <= khop 64 | hop_frac = 2*hop - 1 65 | cp_lat, cp_lon = geodesy.destination( 66 | (from_lat if one_from else from_lat[idx]), 67 | (from_lon if one_from else from_lon[idx]), 68 | bearing[idx], 69 | half_hop[idx] * hop_frac, 70 | ) 71 | cp_lon = (cp_lon + np.pi) % (2 * np.pi) - np.pi 72 | cp_lat = (cp_lat + np.pi / 2) % np.pi - np.pi / 2 73 | cmof[idx] = np.fmin(cmof[idx], iono.fof2.predict(cp_lat, cp_lon)) 74 | 75 | sum_I[idx] += np.clip(np.exp(0.8445 * iono.foe.predict(cp_lat, cp_lon) - 2.937) - 0.4, 0, None) 76 | rms_gyf[idx] += np.power(iono.gyf.predict(cp_lat, cp_lon), 2) 77 | 78 | rms_gyf = np.sqrt(rms_gyf / khop) 79 | 80 | ### TODO: TEP 81 | 82 | cmof *= m9 83 | cmof = np.clip(cmof, 1.0, 50.0) 84 | 85 | pathlen = 2 * khop * (h_calc + r0 * (1 - np.cos(half_hop))) / np.sin(half_hop + takeoff_true) 86 | g_loss = np.clip(20 * np.log10(pathlen / constants.lof_distance_base), 0.0, None) 87 | 88 | # https://apps.dtic.mil/dtic/tr/fulltext/u2/a269557.pdf 89 | 90 | loss_tgt = constants.lof_threshold - g_loss 91 | loss_sec = loss_tgt * np.cos(phi) 92 | 93 | clof = np.full_like(to_lat, 2.0) 94 | clof = np.power(np.clip(677.2 * sum_I / loss_sec - 10.2, 0.0, None), 1 / 1.98) - rms_gyf 95 | # Empirical correction for the fact that the above calculation is missing the 20*log10(f) loss 96 | clof += 0.0591 - 4.344 * sum_I * (np.exp(-0.0775 * loss_sec) + 0.00366 * np.sqrt(loss_sec)) 97 | clof = np.clip(clof, 1.0, None) 98 | 99 | return { 100 | 'mof': cmof, 101 | 'lof': clof, 102 | 'm9': m9, 103 | 'takeoff': np.degrees(takeoff), 104 | 'phi': phi, 105 | 'khop': khop.astype(float), 106 | 'half_hop': half_hop, 107 | 'pathlen': pathlen, 108 | 'distance': dist, 109 | 'bearing': bearing, 110 | 'g_loss': g_loss, 111 | 'rms_gyf': rms_gyf, 112 | 'sum_I': 0.0 + sum_I.astype(float), 113 | } 114 | -------------------------------------------------------------------------------- /www/acknowledgments/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | MUF(3000km) 9 | 52 | 53 | 54 | 55 | 56 |
57 |
58 | 59 |

GIRO

60 |

Part of the ionosonde data that feeds this site is provided by the Global Ionospheric Radio Observatory who have graciously offered free access for that purpose. Thanks are given to all of the ionosonde stations participating in that network.

61 |
62 |
63 | 64 |

NOAA NCEI

65 |

Part of the ionosonde data that feeds this site is provided by the NOAA National Centers for Environmental Information, a department of the United States government. Thanks are given to all of the ionosonde stations participating in that network.

66 |
67 |
68 | 69 |

WWROF

70 |

The World Wide Radio Operators Foundation is a foundation created by amateur radio operators in 2009 to advance the state of the art in radio operating. They cover the ongoing cost of server colocation for the site, and in March 2022, they funded a $10,000 grant to purchase a new server to run the numerical models and host the prop.kc2g.com website.

71 |
72 |
73 | 74 |

Open Source

75 |

76 | This site began as a fork of Matt Smith AF7TI's "giroapp" and "giroviz", for which we are grateful, and all of the code for the site is provided under open-source licenses. 77 |

78 |

79 | In addition, the site would never have been possible without dozens of other open-source projects, including Python, Perl, NumPy, SciPy, pandas, matplotlib, cartopy, george, Flask, IRI-2020, IGRF-13, WMM2020, the space-physics packages iri2016, igrf, and wmm2020, Mojolicious, Minion, PostgreSQL, Podman, and many more.

80 |
81 |
82 | 83 | 84 | -------------------------------------------------------------------------------- /loader/app/load.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use warnings; 4 | 5 | use Data::SAO; 6 | use Data::Dumper; 7 | use DBI; 8 | use Time::HiRes 'time'; 9 | use Net::Statsd::Client; 10 | 11 | my $dbh = DBI->connect( 12 | "dbi:Pg:dbname=$ENV{DB_NAME};host=$ENV{DB_HOST}", 13 | $ENV{DB_USER}, 14 | $ENV{DB_PASSWORD}, 15 | { 16 | RaiseError => 1, 17 | } 18 | ); 19 | 20 | my $statsd = Net::Statsd::Client->new( 21 | host => $ENV{STATSD_HOST}, 22 | ); 23 | 24 | print "Input file: ", $ARGV[0], "\n"; 25 | 26 | my $sao = Data::SAO->new(filename => $ARGV[0]); 27 | my $source = $ARGV[1] || 'noaa'; 28 | 29 | my $code = $sao->station_code; 30 | my $name = $sao->name || $code; 31 | my $geophys = $sao->geophysical_constants; 32 | my ($lat, $lon) = ($geophys->{latitude}, $geophys->{longitude}); 33 | 34 | sub debug { 35 | print "[$code] ", @_, "\n"; 36 | } 37 | 38 | my ($station_id) = $dbh->selectrow_array("SELECT id FROM station WHERE code=?", undef, $code); 39 | if (!defined $station_id) { 40 | ($station_id) = $dbh->selectrow_array("INSERT INTO station (name, code, longitude, latitude, giro, use_for_essn) VALUES (?, ?, ?, ?, FALSE, FALSE) RETURNING id", undef, $name, $code, $lon, $lat); 41 | debug "Created station ID $station_id"; 42 | } else { 43 | debug "Found station ID $station_id"; 44 | } 45 | 46 | my $ts = $sao->timestamp; 47 | my $time = $ts->{date} . " " . $ts->{time}; 48 | debug "Timestamp: $time"; 49 | 50 | my ($measurement_id) = $dbh->selectrow_array("SELECT id FROM measurement WHERE station_id=? AND time=?", undef, $station_id, $time); 51 | if ($measurement_id) { 52 | debug "Already loaded"; 53 | exit 0; 54 | } 55 | 56 | if ($ts->{epoch} > time() + 3600) { 57 | debug "Timestamp is in the future, something isn't right."; 58 | exit 1; 59 | } 60 | 61 | my @cols = ('station_id', 'time', 'cs', 'source'); 62 | my @vals = ($station_id, $time, 0+$sao->confidence, $source); 63 | my @placeholders = ('?', '?', '?', '?'); 64 | 65 | my %map = ( 66 | fof2 => 'foF2', 67 | fof1 => 'foF1', 68 | mufd => 'MUF(D)', 69 | md => 'M(D)', 70 | foes => 'foEs', 71 | foe => 'foE', 72 | hf2 => q{h'F2}, 73 | he => q{h'E}, 74 | hme => 'zmE', 75 | hmf2 => 'zmF2', 76 | hmf1 => 'zmF1', 77 | yf2 => 'yF2', 78 | yf1 => 'yF1', 79 | tec => 'TEC', 80 | scalef2 => 'scaleF2', 81 | fbes => 'fbEs', 82 | ); 83 | 84 | my $characteristics = $sao->scaled_characteristics; 85 | 86 | # These stations are sending M(D) mislabeled as MUF(D) 87 | if (($code =~ /^(?:MM168|SD266|KB548|MG560|TK356)$/) && 88 | defined $characteristics->{'MUF(D)'} && $characteristics->{'MUF(D)'} <= 4.2 && 89 | !defined $characteristics->{'M(D)'}) { 90 | $characteristics->{'M(D)'} = delete $characteristics->{'MUF(D)'}; 91 | } 92 | 93 | delete $characteristics->{'foF2'} if defined $characteristics->{'foF2'} and $characteristics->{'foF2'} == 0; 94 | delete $characteristics->{'zmF2'} if defined $characteristics->{'zmF2'} and $characteristics->{'zmF2'} == 0; 95 | 96 | # Compute mufd from fof2 and md or fof2 and hmf2, if available 97 | # (pred can only work with stations that have fof2 + hmf2 + mufd, so it's worth trying to fill in) 98 | if (defined $characteristics->{'foF2'} && defined $characteristics->{'M(D)'} && !defined $characteristics->{'MUF(D)'}) { 99 | $characteristics->{'MUF(D)'} = $characteristics->{'foF2'} * $characteristics->{'M(D)'}; 100 | } elsif (defined $characteristics->{'foF2'} && defined $characteristics->{'zmF2'} && !defined $characteristics->{'MUF(D)'}) { 101 | $characteristics->{'M(D)'} = 1 / cos(atan2(sin(1500/6371), 1 + ($characteristics->{'zmF2'}+37)/6371 - cos(1500/6371))); 102 | $characteristics->{'MUF(D)'} = $characteristics->{'foF2'} * $characteristics->{'M(D)'}; 103 | } 104 | 105 | my $chars = 0; 106 | for my $key (sort keys %map) { 107 | if (defined(my $val = $characteristics->{$map{$key}})) { 108 | next if $val == 0; 109 | push @cols, $key; 110 | push @vals, $val; 111 | push @placeholders, '?'; 112 | $chars++; 113 | } 114 | } 115 | 116 | if ($chars == 0) { 117 | debug "Nothing to load."; 118 | exit 0; 119 | } 120 | 121 | my $sql = "INSERT INTO measurement (". join(", ", @cols) .") VALUES (". join(", ", @placeholders) .")"; 122 | 123 | $dbh->do($sql, undef, @vals); 124 | 125 | my $latency = sprintf "%.2f", (time() - $ts->{epoch}); 126 | 127 | my $format = $sao->file_format; 128 | 129 | $statsd->increment('prop.loader.loaded.total'); 130 | $statsd->increment("prop.loader.loaded.station.$code"); 131 | $statsd->increment("prop.loader.loaded.source.$source"); 132 | $statsd->increment("prop.loader.loaded.format.$format"); 133 | $statsd->increment("prop.loader.loaded.source.$source.format.$format"); 134 | $statsd->timing_ms("prop.loader.latency.overall", $latency * 1000); 135 | $statsd->timing_ms("prop.loader.latency.source.$source", $latency * 1000); 136 | $statsd->timing_ms("prop.loader.latency.station.$code", $latency * 1000); 137 | 138 | debug "Latency: $latency"; 139 | --------------------------------------------------------------------------------