├── .gitignore ├── MANIFEST.in ├── README.md ├── docs ├── Makefile ├── make.bat └── source │ ├── conf.py │ ├── index.rst │ ├── ml4convection.figures.rst │ ├── ml4convection.io.rst │ ├── ml4convection.machine_learning.rst │ ├── ml4convection.plotting.rst │ ├── ml4convection.rst │ ├── ml4convection.scripts.rst │ ├── ml4convection.utils.rst │ ├── modules.rst │ └── setup.rst ├── ml4convection ├── dead_code │ ├── create_radar_mask.py │ ├── pixelwise_evaluation.py │ ├── pixelwise_evaluation_test.py │ ├── plot_pixelwise_evaluation.py │ ├── read_twb_radar_file.f90 │ ├── read_twb_satellite_file.f90 │ ├── run_pixelwise_evaluation.py │ ├── run_spatial_evaluation.py │ ├── spatial_evaluation.py │ └── spatial_evaluation_test.py ├── figures │ ├── __init__.py │ ├── make_climo_mask_figure.py │ ├── make_data_overview_figure.py │ ├── make_data_overview_figure_lag_times.py │ ├── make_eval_figure_comparison.py │ ├── make_eval_figure_gridded.py │ ├── make_eval_figure_gridded_comparison.py │ ├── make_eval_figure_ungridded.py │ ├── make_filter_comparison_figures.py │ ├── make_fourier_figure.py │ ├── make_fourier_figure_targets_only.py │ ├── make_fourier_recomp_figure.py │ ├── make_fss_csi_figure.py │ ├── make_neigh_eval_schematic.py │ ├── make_permutation_figure.py │ ├── make_prediction_comparison_figure.py │ ├── make_predictor_prediction_figure.py │ ├── make_response_function_figure.py │ ├── make_response_function_figure_1d.py │ ├── make_satellite_qc_figure.py │ ├── make_wavelet_figure.py │ ├── make_wavelet_figure_tiled.py │ ├── plot_neigh_eval_comparison.py │ └── plot_wavelet_bands_full_grid.py ├── io │ ├── Channel_B08_Standary.LUT │ ├── Channel_B09_Standary.LUT │ ├── Channel_B10_Standary.LUT │ ├── Channel_B11_Standary.LUT │ ├── Channel_B13_Standary.LUT │ ├── Channel_B14_Standary.LUT │ ├── Channel_B16_Standary.LUT │ ├── __init__.py │ ├── border_io.py │ ├── borders.nc │ ├── climatology_io.py │ ├── example_io.py │ ├── example_io_test.py │ ├── new_satellite_io.py │ ├── new_satellite_io_test.py │ ├── prediction_io.py │ ├── prediction_io_test.py │ ├── radar_io.py │ ├── radar_io_test.py │ ├── satellite_io.py │ ├── satellite_io_test.py │ ├── twb_radar_io.py │ ├── twb_radar_io_test.py │ ├── twb_satellite_io.py │ └── twb_satellite_io_test.py ├── loss_function_paper │ ├── make_neigh_eval_schematic.py │ ├── plot_hyperparam_grids_lf_exp4to6.py │ ├── plot_hyperparam_grids_lf_exp_2hours.py │ ├── rank_learning_curves_2hours.py │ ├── rank_learning_curves_2hours_simpler.py │ ├── rank_learning_curves_exp1.py │ ├── rank_learning_curves_exp1and2.py │ ├── rank_learning_curves_exp2.py │ ├── rank_learning_curves_exp4.py │ ├── rank_learning_curves_exp4to6.py │ ├── rank_learning_curves_exp4to6_simpler.py │ └── rank_learning_curves_exp5.py ├── machine_learning │ ├── __init__.py │ ├── chiu_architecture.py │ ├── coord_conv.py │ ├── coord_conv_test.py │ ├── custom_losses.py │ ├── custom_losses_test.py │ ├── custom_metrics.py │ ├── custom_metrics_test.py │ ├── fourier_metrics.py │ ├── fourier_metrics_test.py │ ├── gradcam.py │ ├── neural_net.py │ ├── neural_net_test.py │ ├── permutation.py │ ├── permutation_test.py │ ├── saliency.py │ ├── standalone_utils.py │ ├── standalone_utils_test.py │ ├── u_net_architecture.py │ ├── u_net_pp_architecture.py │ ├── wavelet_metrics.py │ └── wavelet_metrics_test.py ├── plotting │ ├── __init__.py │ ├── cam_plotting.py │ ├── evaluation_plotting.py │ ├── evaluation_plotting_test.py │ ├── new_satellite_plotting.py │ ├── new_satellite_plotting_test.py │ ├── plotting_utils.py │ ├── prediction_plotting.py │ ├── saliency_plotting.py │ ├── satellite_plotting.py │ └── uq_evaluation_plotting.py ├── scripts │ ├── __init__.py │ ├── apply_neural_net.py │ ├── apply_persistence_model.py │ ├── average_predictions_over_time.py │ ├── composite_saliency_maps.py │ ├── compute_advanced_learning_curves.py │ ├── compute_advanced_scores_gridded.py │ ├── compute_advanced_scores_ungridded.py │ ├── compute_basic_learning_curves.py │ ├── compute_basic_scores_gridded.py │ ├── compute_basic_scores_ungridded.py │ ├── compute_crps.py │ ├── compute_spread_vs_skill.py │ ├── concat_figures.py │ ├── count_radar_observations.py │ ├── count_satellite_qc_diffs.py │ ├── create_predictors.py │ ├── create_radar_mask.py │ ├── create_targets.py │ ├── delete_basic_scores.py │ ├── delete_prediction_files.py │ ├── find_event_frequencies.py │ ├── find_extreme_examples.py │ ├── find_interesting_examples.py │ ├── get_normalization_params.py │ ├── make_best_architecture_0minutes.py │ ├── make_best_architecture_120minutes.py │ ├── make_best_architecture_30minutes.py │ ├── make_best_architecture_60minutes.py │ ├── make_best_architecture_90minutes.py │ ├── make_class_activation_maps.py │ ├── make_saliency_maps.py │ ├── plot_class_activation_maps.py │ ├── plot_composite_saliency_map.py │ ├── plot_discard_test.py │ ├── plot_evaluation.py │ ├── plot_evaluation_by_time.py │ ├── plot_gridded_evaluation.py │ ├── plot_label_climo.py │ ├── plot_learning_curves.py │ ├── plot_mean_satellite_fields.py │ ├── plot_permutation_results.py │ ├── plot_predictions.py │ ├── plot_predictions_with_uncertainty.py │ ├── plot_radar.py │ ├── plot_radar_counts.py │ ├── plot_radar_mask.py │ ├── plot_saliency_maps.py │ ├── plot_satellite.py │ ├── plot_satellite_diff_counts.py │ ├── plot_spread_vs_skill.py │ ├── process_radar_data.py │ ├── process_satellite_data.py │ ├── qc_satellite_data.py │ ├── radar_mask_100km_omit-north_partial.nc │ ├── run_discard_test.py │ ├── run_echo_classification.py │ ├── run_permutation.py │ ├── run_prelim_tracking.py │ ├── run_unit_tests.py │ ├── tracks_to_convection_masks.py │ ├── train_neural_net.py │ ├── training_args.py │ └── unzip_example_files.py ├── slurm │ └── tracking │ │ ├── find_prelim_tracking_gaps.qsub │ │ ├── run_final_tracking.qsub │ │ ├── run_prelim_tracking_2016.qsub │ │ ├── run_prelim_tracking_2016_patch.qsub │ │ ├── run_prelim_tracking_2017.qsub │ │ ├── run_prelim_tracking_2017_patch.qsub │ │ ├── run_prelim_tracking_2018.qsub │ │ ├── run_prelim_tracking_2018_patch.qsub │ │ ├── tracks_to_masks_final_2016.qsub │ │ ├── tracks_to_masks_final_2017.qsub │ │ ├── tracks_to_masks_final_2018.qsub │ │ ├── tracks_to_masks_prelim_2016.qsub │ │ ├── tracks_to_masks_prelim_2017.qsub │ │ └── tracks_to_masks_prelim_2018.qsub └── utils │ ├── __init__.py │ ├── evaluation.py │ ├── evaluation_test.py │ ├── fourier_utils.py │ ├── fourier_utils_test.py │ ├── general_utils.py │ ├── general_utils_test.py │ ├── learning_curves.py │ ├── learning_curves_test.py │ ├── normalization.py │ ├── normalization_test.py │ ├── radar_utils.py │ ├── radar_utils_test.py │ ├── uq_evaluation.py │ ├── uq_evaluation_test.py │ ├── wavelet_utils.py │ └── wavelet_utils_test.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled, optimized, and DLL files. 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # Distribution and packaging. 7 | build/ 8 | dist/ 9 | *.egg-info/ 10 | 11 | # IntelliJ things. 12 | .idea/ 13 | *.iml 14 | 15 | # Code that I don't intend to commit. 16 | ml4convection/*.jpg 17 | ml4convection/*.py 18 | ml4convection/io/*.exe 19 | ml4convection/dead_scratch_code/ 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include ml4convection/io/*.LUT 2 | include ml4convection/io/borders.nc 3 | include ml4convection/scripts/radar_mask_100km_omit-north_partial.nc 4 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= /home/ralager/anaconda3/bin/sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | 13 | import os 14 | import sys 15 | sys.path.insert(0, os.path.abspath('../..')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = 'ml4convection' 21 | copyright = '2021, Ryan Lagerquist' 22 | author = 'Ryan Lagerquist' 23 | 24 | # The full version, including alpha/beta/rc tags 25 | release = '1.0.2' 26 | 27 | 28 | # -- General configuration --------------------------------------------------- 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = [ 34 | 'sphinx.ext.napoleon' 35 | ] 36 | napoleon_google_docstring = False 37 | 38 | # Add any paths that contain templates here, relative to this directory. 39 | templates_path = ['_templates'] 40 | 41 | # List of patterns, relative to source directory, that match files and 42 | # directories to ignore when looking for source files. 43 | # This pattern also affects html_static_path and html_extra_path. 44 | exclude_patterns = [] 45 | 46 | 47 | # -- Options for HTML output ------------------------------------------------- 48 | 49 | # The theme to use for HTML and HTML Help pages. See the documentation for 50 | # a list of builtin themes. 51 | # 52 | html_theme = 'alabaster' 53 | 54 | # Add any paths that contain custom static files (such as style sheets) here, 55 | # relative to this directory. They are copied after the builtin static files, 56 | # so a file named "default.css" will overwrite the builtin "default.css". 57 | html_static_path = ['_static'] -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. ml4convection documentation master file, created by 2 | sphinx-quickstart on Mon Apr 26 13:07:37 2021. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to ml4convection's documentation! 7 | ========================================= 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | modules 14 | 15 | 16 | 17 | Indices and tables 18 | ================== 19 | 20 | * :ref:`genindex` 21 | * :ref:`modindex` 22 | * :ref:`search` 23 | -------------------------------------------------------------------------------- /docs/source/ml4convection.figures.rst: -------------------------------------------------------------------------------- 1 | ml4convection.figures package 2 | ============================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | ml4convection.figures.make\_climo\_mask\_figure module 8 | ------------------------------------------------------ 9 | 10 | .. automodule:: ml4convection.figures.make_climo_mask_figure 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | ml4convection.figures.make\_data\_overview\_figure module 16 | --------------------------------------------------------- 17 | 18 | .. automodule:: ml4convection.figures.make_data_overview_figure 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | ml4convection.figures.make\_eval\_figure\_gridded module 24 | -------------------------------------------------------- 25 | 26 | .. automodule:: ml4convection.figures.make_eval_figure_gridded 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | ml4convection.figures.make\_eval\_figure\_ungridded module 32 | ---------------------------------------------------------- 33 | 34 | .. automodule:: ml4convection.figures.make_eval_figure_ungridded 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | ml4convection.figures.make\_fourier\_figure module 40 | -------------------------------------------------- 41 | 42 | .. automodule:: ml4convection.figures.make_fourier_figure 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | ml4convection.figures.make\_fourier\_recomp\_figure module 48 | ---------------------------------------------------------- 49 | 50 | .. automodule:: ml4convection.figures.make_fourier_recomp_figure 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | ml4convection.figures.make\_neigh\_eval\_schematic module 56 | --------------------------------------------------------- 57 | 58 | .. automodule:: ml4convection.figures.make_neigh_eval_schematic 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | ml4convection.figures.make\_permutation\_figure module 64 | ------------------------------------------------------ 65 | 66 | .. automodule:: ml4convection.figures.make_permutation_figure 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | ml4convection.figures.make\_prediction\_comparison\_figure module 72 | ----------------------------------------------------------------- 73 | 74 | .. automodule:: ml4convection.figures.make_prediction_comparison_figure 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | ml4convection.figures.make\_predictor\_prediction\_figure module 80 | ---------------------------------------------------------------- 81 | 82 | .. automodule:: ml4convection.figures.make_predictor_prediction_figure 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | ml4convection.figures.make\_response\_function\_figure module 88 | ------------------------------------------------------------- 89 | 90 | .. automodule:: ml4convection.figures.make_response_function_figure 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | ml4convection.figures.make\_response\_function\_figure\_1d module 96 | ----------------------------------------------------------------- 97 | 98 | .. automodule:: ml4convection.figures.make_response_function_figure_1d 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | ml4convection.figures.make\_satellite\_qc\_figure module 104 | -------------------------------------------------------- 105 | 106 | .. automodule:: ml4convection.figures.make_satellite_qc_figure 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | 112 | Module contents 113 | --------------- 114 | 115 | .. automodule:: ml4convection.figures 116 | :members: 117 | :undoc-members: 118 | :show-inheritance: 119 | -------------------------------------------------------------------------------- /docs/source/ml4convection.io.rst: -------------------------------------------------------------------------------- 1 | ml4convection.io package 2 | ======================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | ml4convection.io.border\_io module 8 | ---------------------------------- 9 | 10 | .. automodule:: ml4convection.io.border_io 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | ml4convection.io.climatology\_io module 16 | --------------------------------------- 17 | 18 | .. automodule:: ml4convection.io.climatology_io 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | ml4convection.io.example\_io module 24 | ----------------------------------- 25 | 26 | .. automodule:: ml4convection.io.example_io 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | ml4convection.io.example\_io\_test module 32 | ----------------------------------------- 33 | 34 | .. automodule:: ml4convection.io.example_io_test 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | ml4convection.io.prediction\_io module 40 | -------------------------------------- 41 | 42 | .. automodule:: ml4convection.io.prediction_io 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | ml4convection.io.prediction\_io\_test module 48 | -------------------------------------------- 49 | 50 | .. automodule:: ml4convection.io.prediction_io_test 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | ml4convection.io.radar\_io module 56 | --------------------------------- 57 | 58 | .. automodule:: ml4convection.io.radar_io 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | ml4convection.io.radar\_io\_test module 64 | --------------------------------------- 65 | 66 | .. automodule:: ml4convection.io.radar_io_test 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | ml4convection.io.satellite\_io module 72 | ------------------------------------- 73 | 74 | .. automodule:: ml4convection.io.satellite_io 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | ml4convection.io.satellite\_io\_test module 80 | ------------------------------------------- 81 | 82 | .. automodule:: ml4convection.io.satellite_io_test 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | ml4convection.io.twb\_radar\_io module 88 | -------------------------------------- 89 | 90 | .. automodule:: ml4convection.io.twb_radar_io 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | ml4convection.io.twb\_radar\_io\_test module 96 | -------------------------------------------- 97 | 98 | .. automodule:: ml4convection.io.twb_radar_io_test 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | ml4convection.io.twb\_satellite\_io module 104 | ------------------------------------------ 105 | 106 | .. automodule:: ml4convection.io.twb_satellite_io 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | ml4convection.io.twb\_satellite\_io\_test module 112 | ------------------------------------------------ 113 | 114 | .. automodule:: ml4convection.io.twb_satellite_io_test 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | 120 | Module contents 121 | --------------- 122 | 123 | .. automodule:: ml4convection.io 124 | :members: 125 | :undoc-members: 126 | :show-inheritance: 127 | -------------------------------------------------------------------------------- /docs/source/ml4convection.machine_learning.rst: -------------------------------------------------------------------------------- 1 | ml4convection.machine\_learning package 2 | ======================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | ml4convection.machine\_learning.chiu\_architecture module 8 | --------------------------------------------------------- 9 | 10 | .. automodule:: ml4convection.machine_learning.chiu_architecture 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | ml4convection.machine\_learning.coord\_conv module 16 | -------------------------------------------------- 17 | 18 | .. automodule:: ml4convection.machine_learning.coord_conv 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | ml4convection.machine\_learning.coord\_conv\_test module 24 | -------------------------------------------------------- 25 | 26 | .. automodule:: ml4convection.machine_learning.coord_conv_test 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | ml4convection.machine\_learning.custom\_losses module 32 | ----------------------------------------------------- 33 | 34 | .. automodule:: ml4convection.machine_learning.custom_losses 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | ml4convection.machine\_learning.custom\_losses\_test module 40 | ----------------------------------------------------------- 41 | 42 | .. automodule:: ml4convection.machine_learning.custom_losses_test 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | ml4convection.machine\_learning.custom\_metrics module 48 | ------------------------------------------------------ 49 | 50 | .. automodule:: ml4convection.machine_learning.custom_metrics 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | ml4convection.machine\_learning.custom\_metrics\_test module 56 | ------------------------------------------------------------ 57 | 58 | .. automodule:: ml4convection.machine_learning.custom_metrics_test 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | ml4convection.machine\_learning.fourier\_metrics module 64 | ------------------------------------------------------- 65 | 66 | .. automodule:: ml4convection.machine_learning.fourier_metrics 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | ml4convection.machine\_learning.fourier\_metrics\_test module 72 | ------------------------------------------------------------- 73 | 74 | .. automodule:: ml4convection.machine_learning.fourier_metrics_test 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | ml4convection.machine\_learning.gradcam module 80 | ---------------------------------------------- 81 | 82 | .. automodule:: ml4convection.machine_learning.gradcam 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | ml4convection.machine\_learning.neural\_net module 88 | -------------------------------------------------- 89 | 90 | .. automodule:: ml4convection.machine_learning.neural_net 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | ml4convection.machine\_learning.neural\_net\_test module 96 | -------------------------------------------------------- 97 | 98 | .. automodule:: ml4convection.machine_learning.neural_net_test 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | ml4convection.machine\_learning.permutation module 104 | -------------------------------------------------- 105 | 106 | .. automodule:: ml4convection.machine_learning.permutation 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | ml4convection.machine\_learning.permutation\_test module 112 | -------------------------------------------------------- 113 | 114 | .. automodule:: ml4convection.machine_learning.permutation_test 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | ml4convection.machine\_learning.saliency module 120 | ----------------------------------------------- 121 | 122 | .. automodule:: ml4convection.machine_learning.saliency 123 | :members: 124 | :undoc-members: 125 | :show-inheritance: 126 | 127 | ml4convection.machine\_learning.standalone\_utils module 128 | -------------------------------------------------------- 129 | 130 | .. automodule:: ml4convection.machine_learning.standalone_utils 131 | :members: 132 | :undoc-members: 133 | :show-inheritance: 134 | 135 | ml4convection.machine\_learning.standalone\_utils\_test module 136 | -------------------------------------------------------------- 137 | 138 | .. automodule:: ml4convection.machine_learning.standalone_utils_test 139 | :members: 140 | :undoc-members: 141 | :show-inheritance: 142 | 143 | ml4convection.machine\_learning.u\_net\_architecture module 144 | ----------------------------------------------------------- 145 | 146 | .. automodule:: ml4convection.machine_learning.u_net_architecture 147 | :members: 148 | :undoc-members: 149 | :show-inheritance: 150 | 151 | ml4convection.machine\_learning.u\_net\_pp\_architecture module 152 | --------------------------------------------------------------- 153 | 154 | .. automodule:: ml4convection.machine_learning.u_net_pp_architecture 155 | :members: 156 | :undoc-members: 157 | :show-inheritance: 158 | 159 | 160 | Module contents 161 | --------------- 162 | 163 | .. automodule:: ml4convection.machine_learning 164 | :members: 165 | :undoc-members: 166 | :show-inheritance: 167 | -------------------------------------------------------------------------------- /docs/source/ml4convection.plotting.rst: -------------------------------------------------------------------------------- 1 | ml4convection.plotting package 2 | ============================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | ml4convection.plotting.cam\_plotting module 8 | ------------------------------------------- 9 | 10 | .. automodule:: ml4convection.plotting.cam_plotting 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | ml4convection.plotting.evaluation\_plotting module 16 | -------------------------------------------------- 17 | 18 | .. automodule:: ml4convection.plotting.evaluation_plotting 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | ml4convection.plotting.evaluation\_plotting\_test module 24 | -------------------------------------------------------- 25 | 26 | .. automodule:: ml4convection.plotting.evaluation_plotting_test 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | ml4convection.plotting.plotting\_utils module 32 | --------------------------------------------- 33 | 34 | .. automodule:: ml4convection.plotting.plotting_utils 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | ml4convection.plotting.prediction\_plotting module 40 | -------------------------------------------------- 41 | 42 | .. automodule:: ml4convection.plotting.prediction_plotting 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | ml4convection.plotting.saliency\_plotting module 48 | ------------------------------------------------ 49 | 50 | .. automodule:: ml4convection.plotting.saliency_plotting 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | ml4convection.plotting.satellite\_plotting module 56 | ------------------------------------------------- 57 | 58 | .. automodule:: ml4convection.plotting.satellite_plotting 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | 64 | Module contents 65 | --------------- 66 | 67 | .. automodule:: ml4convection.plotting 68 | :members: 69 | :undoc-members: 70 | :show-inheritance: 71 | -------------------------------------------------------------------------------- /docs/source/ml4convection.rst: -------------------------------------------------------------------------------- 1 | ml4convection package 2 | ===================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | ml4convection.figures 10 | ml4convection.io 11 | ml4convection.machine_learning 12 | ml4convection.plotting 13 | ml4convection.scripts 14 | ml4convection.utils 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: ml4convection 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/source/ml4convection.utils.rst: -------------------------------------------------------------------------------- 1 | ml4convection.utils package 2 | =========================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | ml4convection.utils.evaluation module 8 | ------------------------------------- 9 | 10 | .. automodule:: ml4convection.utils.evaluation 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | ml4convection.utils.evaluation\_test module 16 | ------------------------------------------- 17 | 18 | .. automodule:: ml4convection.utils.evaluation_test 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | ml4convection.utils.fourier\_utils module 24 | ----------------------------------------- 25 | 26 | .. automodule:: ml4convection.utils.fourier_utils 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | ml4convection.utils.fourier\_utils\_test module 32 | ----------------------------------------------- 33 | 34 | .. automodule:: ml4convection.utils.fourier_utils_test 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | ml4convection.utils.general\_utils module 40 | ----------------------------------------- 41 | 42 | .. automodule:: ml4convection.utils.general_utils 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | ml4convection.utils.general\_utils\_test module 48 | ----------------------------------------------- 49 | 50 | .. automodule:: ml4convection.utils.general_utils_test 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | ml4convection.utils.learning\_curves module 56 | ------------------------------------------- 57 | 58 | .. automodule:: ml4convection.utils.learning_curves 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | ml4convection.utils.learning\_curves\_test module 64 | ------------------------------------------------- 65 | 66 | .. automodule:: ml4convection.utils.learning_curves_test 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | ml4convection.utils.normalization module 72 | ---------------------------------------- 73 | 74 | .. automodule:: ml4convection.utils.normalization 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | ml4convection.utils.normalization\_test module 80 | ---------------------------------------------- 81 | 82 | .. automodule:: ml4convection.utils.normalization_test 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | ml4convection.utils.radar\_utils module 88 | --------------------------------------- 89 | 90 | .. automodule:: ml4convection.utils.radar_utils 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | ml4convection.utils.radar\_utils\_test module 96 | --------------------------------------------- 97 | 98 | .. automodule:: ml4convection.utils.radar_utils_test 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | 104 | Module contents 105 | --------------- 106 | 107 | .. automodule:: ml4convection.utils 108 | :members: 109 | :undoc-members: 110 | :show-inheritance: 111 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | ml4convection 2 | ============= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | ml4convection 8 | setup 9 | -------------------------------------------------------------------------------- /docs/source/setup.rst: -------------------------------------------------------------------------------- 1 | setup module 2 | ============ 3 | 4 | .. automodule:: setup 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /ml4convection/dead_code/read_twb_radar_file.f90: -------------------------------------------------------------------------------- 1 | program read_qpesums_grid 2 | 3 | implicit none 4 | 5 | integer :: ii,jj 6 | character :: filename*256 7 | integer :: yyyy,mm,dd,hh,mn,ss,nx,ny,nz ! 1-9th vars 8 | character :: proj*4 ! 10th vars 9 | integer :: map_scale,projlat1,projlat2,projlon,alon,alat,xy_scale,dx,dy,dxy_scale ! 11-20th vars 10 | integer :: z_scale,i_bb_mode,unkn01(9) 11 | character :: varname*20,varunit*6 12 | integer :: var_scale,missing,nradar 13 | integer,allocatable :: zht(:) 14 | character,allocatable :: mosradar(:,:) 15 | integer*2,allocatable :: var(:,:) 16 | real,allocatable :: var_real(:,:) 17 | real :: xlat,xlon 18 | 19 | call getarg(1,filename) 20 | 21 | open(11,file=trim(filename),form='unformatted',status='old',access='stream') 22 | !open(11,file=trim(filename),form='binary',status='old') 23 | read(11) yyyy,mm,dd,hh,mn,ss,nx,ny,nz,proj,& 24 | map_scale,projlat1,projlat2,projlon,alon,alat,& 25 | xy_scale,dx,dy,dxy_scale 26 | allocate(var(nx,ny),zht(nz)) 27 | allocate(var_real(nx,ny)) 28 | read(11) zht,z_scale,i_bb_mode,unkn01,varname,varunit,& 29 | var_scale,missing,nradar 30 | allocate(mosradar(4,nradar)) 31 | read(11) mosradar,var 32 | close(11) 33 | var_real = float(var)/float(var_scale) 34 | 35 | do jj = 1,ny 36 | do ii = 1,nx 37 | xlat = real(alat)/real(xy_scale)+(jj-ny)*real(dy)/real(dxy_scale) 38 | xlon = real(alon)/real(xy_scale)+(ii-1 )*real(dx)/real(dxy_scale) 39 | write(*,*) xlat,xlon,var_real(ii,jj) 40 | end do 41 | end do 42 | 43 | deallocate(var,var_real,zht,mosradar) 44 | 45 | end 46 | -------------------------------------------------------------------------------- /ml4convection/dead_code/read_twb_satellite_file.f90: -------------------------------------------------------------------------------- 1 | program test 2 | 3 | integer :: ii,jj 4 | character :: filename*256 5 | character*1 :: btmp(921,881) 6 | real :: xlat(921,881),xlon(921,881) 7 | 8 | call getarg(1,filename) 9 | 10 | open(12,file="./Proj_NOAA.GSD",form="unformatted",access="stream",status="old") 11 | read(12) xlat,xlon 12 | close(12) 13 | open(11,file=trim(filename),form="unformatted",access="stream",status="old") 14 | read(11) btmp 15 | close(11) 16 | 17 | do jj = 1,881 18 | do ii = 1,921 19 | write(*,*) xlat(ii,jj),xlon(ii,jj),ichar(btmp(ii,jj)) 20 | end do 21 | end do 22 | 23 | end program 24 | -------------------------------------------------------------------------------- /ml4convection/dead_code/run_spatial_evaluation.py: -------------------------------------------------------------------------------- 1 | """Runs spatially aware (not pixelwise) evaluation.""" 2 | 3 | import argparse 4 | import numpy 5 | from ml4convection.io import prediction_io 6 | from ml4convection.utils import spatial_evaluation as spatial_eval 7 | 8 | SEPARATOR_STRING = '\n\n' + '*' * 50 + '\n\n' 9 | 10 | INPUT_DIR_ARG_NAME = 'input_prediction_dir_name' 11 | FIRST_DATE_ARG_NAME = 'first_date_string' 12 | LAST_DATE_ARG_NAME = 'last_date_string' 13 | HALF_WINDOW_SIZES_ARG_NAME = 'fss_half_window_sizes_px' 14 | OUTPUT_FILE_ARG_NAME = 'output_file_name' 15 | 16 | INPUT_DIR_HELP_STRING = ( 17 | 'Name of input directory. Files therein will be found by ' 18 | '`prediction_io.find_file` and read by `prediction_io.read_file`.' 19 | ) 20 | DATE_HELP_STRING = ( 21 | 'Date (format "yyyymmdd"). Will evaluate predictions for all days in the ' 22 | 'period `{0:s}`...`{1:s}`.' 23 | ).format(FIRST_DATE_ARG_NAME, LAST_DATE_ARG_NAME) 24 | 25 | HALF_WINDOW_SIZES_HELP_STRING = ( 26 | 'List of half-window sizes for fractions skill score (units are pixels).' 27 | ) 28 | OUTPUT_FILE_HELP_STRING = ( 29 | 'Path to output file. Results will be written here by ' 30 | '`spatial_evaluation.write_file`.' 31 | ) 32 | 33 | INPUT_ARG_PARSER = argparse.ArgumentParser() 34 | INPUT_ARG_PARSER.add_argument( 35 | '--' + INPUT_DIR_ARG_NAME, type=str, required=True, 36 | help=INPUT_DIR_HELP_STRING 37 | ) 38 | INPUT_ARG_PARSER.add_argument( 39 | '--' + FIRST_DATE_ARG_NAME, type=str, required=True, help=DATE_HELP_STRING 40 | ) 41 | INPUT_ARG_PARSER.add_argument( 42 | '--' + LAST_DATE_ARG_NAME, type=str, required=True, help=DATE_HELP_STRING 43 | ) 44 | INPUT_ARG_PARSER.add_argument( 45 | '--' + HALF_WINDOW_SIZES_ARG_NAME, type=int, nargs='+', required=True, 46 | help=HALF_WINDOW_SIZES_HELP_STRING 47 | ) 48 | INPUT_ARG_PARSER.add_argument( 49 | '--' + OUTPUT_FILE_ARG_NAME, type=str, required=True, 50 | help=OUTPUT_FILE_HELP_STRING 51 | ) 52 | 53 | 54 | def _run(top_prediction_dir_name, first_date_string, last_date_string, 55 | fss_half_window_sizes_px, output_file_name): 56 | """Runs spatially aware (not pixelwise) evaluation. 57 | 58 | This is effectively the main method. 59 | 60 | :param top_prediction_dir_name: See documentation at top of file. 61 | :param first_date_string: Same. 62 | :param last_date_string: Same. 63 | :param fss_half_window_sizes_px: Same. 64 | :param output_file_name: Same. 65 | """ 66 | 67 | prediction_file_names = prediction_io.find_many_files( 68 | top_directory_name=top_prediction_dir_name, 69 | first_date_string=first_date_string, 70 | last_date_string=last_date_string, 71 | raise_error_if_any_missing=False 72 | ) 73 | 74 | fractions_skill_scores = spatial_eval.get_fractions_skill_scores( 75 | prediction_file_names=prediction_file_names, 76 | half_window_sizes_px=fss_half_window_sizes_px 77 | ) 78 | print(SEPARATOR_STRING) 79 | 80 | num_scales = len(fss_half_window_sizes_px) 81 | for k in range(num_scales): 82 | print(( 83 | 'Fractions skill score for {0:d}-by-{0:d} window = {1:.4f}' 84 | ).format( 85 | int(numpy.round(fss_half_window_sizes_px[k] * 2)) + 1, 86 | fractions_skill_scores[k] 87 | )) 88 | 89 | print(SEPARATOR_STRING) 90 | print('Writing results to: "{0:s}"...'.format(output_file_name)) 91 | spatial_eval.write_file( 92 | fractions_skill_scores=fractions_skill_scores, 93 | half_window_sizes_px=fss_half_window_sizes_px, 94 | pickle_file_name=output_file_name 95 | ) 96 | 97 | 98 | if __name__ == '__main__': 99 | INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args() 100 | 101 | _run( 102 | top_prediction_dir_name=getattr(INPUT_ARG_OBJECT, INPUT_DIR_ARG_NAME), 103 | first_date_string=getattr(INPUT_ARG_OBJECT, FIRST_DATE_ARG_NAME), 104 | last_date_string=getattr(INPUT_ARG_OBJECT, LAST_DATE_ARG_NAME), 105 | fss_half_window_sizes_px=numpy.array( 106 | getattr(INPUT_ARG_OBJECT, HALF_WINDOW_SIZES_ARG_NAME), dtype=int 107 | ), 108 | output_file_name=getattr(INPUT_ARG_OBJECT, OUTPUT_FILE_ARG_NAME) 109 | ) 110 | -------------------------------------------------------------------------------- /ml4convection/figures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thunderhoser/ml4convection/cf5e9682bc3182305274132ae246bc72d994308f/ml4convection/figures/__init__.py -------------------------------------------------------------------------------- /ml4convection/io/Channel_B08_Standary.LUT: -------------------------------------------------------------------------------- 1 | 0 300.00 2 | 1 296.86 3 | 2 293.71 4 | 3 290.57 5 | 4 287.43 6 | 5 284.28 7 | 6 281.14 8 | 7 280.80 9 | 8 280.46 10 | 9 280.12 11 | 10 279.78 12 | 11 279.44 13 | 12 279.10 14 | 13 278.76 15 | 14 278.42 16 | 15 278.08 17 | 16 277.74 18 | 17 277.40 19 | 18 277.05 20 | 19 276.71 21 | 20 276.37 22 | 21 276.03 23 | 22 275.69 24 | 23 275.35 25 | 24 275.01 26 | 25 274.67 27 | 26 274.33 28 | 27 273.99 29 | 28 273.65 30 | 29 273.31 31 | 30 272.97 32 | 31 272.63 33 | 32 272.29 34 | 33 271.95 35 | 34 271.61 36 | 35 271.27 37 | 36 270.93 38 | 37 270.59 39 | 38 270.25 40 | 39 269.91 41 | 40 269.57 42 | 41 269.23 43 | 42 268.88 44 | 43 268.54 45 | 44 268.20 46 | 45 267.86 47 | 46 267.52 48 | 47 267.18 49 | 48 266.84 50 | 49 266.50 51 | 50 266.16 52 | 51 265.82 53 | 52 265.48 54 | 53 265.14 55 | 54 264.79 56 | 55 264.43 57 | 56 264.08 58 | 57 263.73 59 | 58 263.38 60 | 59 263.02 61 | 60 262.67 62 | 61 262.32 63 | 62 261.96 64 | 63 261.61 65 | 64 261.26 66 | 65 260.90 67 | 66 260.55 68 | 67 260.20 69 | 68 259.85 70 | 69 259.49 71 | 70 259.14 72 | 71 258.80 73 | 72 258.46 74 | 73 258.12 75 | 74 257.78 76 | 75 257.44 77 | 76 257.09 78 | 77 256.75 79 | 78 256.41 80 | 79 256.07 81 | 80 255.73 82 | 81 255.39 83 | 82 255.05 84 | 83 254.71 85 | 84 254.37 86 | 85 254.03 87 | 86 253.69 88 | 87 253.34 89 | 88 253.00 90 | 89 252.66 91 | 90 252.32 92 | 91 251.98 93 | 92 251.64 94 | 93 251.29 95 | 94 250.94 96 | 95 250.59 97 | 96 250.24 98 | 97 249.89 99 | 98 249.54 100 | 99 249.19 101 | 100 248.84 102 | 101 248.49 103 | 102 248.14 104 | 103 247.79 105 | 104 247.44 106 | 105 247.09 107 | 106 246.74 108 | 107 246.39 109 | 108 246.04 110 | 109 245.69 111 | 110 245.34 112 | 111 244.99 113 | 112 244.64 114 | 113 244.29 115 | 114 243.94 116 | 115 243.59 117 | 116 243.24 118 | 117 242.89 119 | 118 242.54 120 | 119 242.19 121 | 120 241.84 122 | 121 241.49 123 | 122 241.14 124 | 123 240.81 125 | 124 240.47 126 | 125 240.14 127 | 126 239.81 128 | 127 239.47 129 | 128 239.14 130 | 129 238.81 131 | 130 238.47 132 | 131 238.14 133 | 132 237.81 134 | 133 237.47 135 | 134 237.14 136 | 135 236.79 137 | 136 236.44 138 | 137 236.10 139 | 138 235.75 140 | 139 235.40 141 | 140 235.05 142 | 141 234.71 143 | 142 234.36 144 | 143 234.01 145 | 144 233.66 146 | 145 233.31 147 | 146 232.97 148 | 147 232.62 149 | 148 232.27 150 | 149 231.92 151 | 150 231.57 152 | 151 231.23 153 | 152 230.88 154 | 153 230.53 155 | 154 230.18 156 | 155 229.84 157 | 156 229.49 158 | 157 229.14 159 | 158 228.79 160 | 159 228.45 161 | 160 228.10 162 | 161 227.76 163 | 162 227.41 164 | 163 227.06 165 | 164 226.72 166 | 165 226.37 167 | 166 226.02 168 | 167 225.68 169 | 168 225.33 170 | 169 224.99 171 | 170 224.64 172 | 171 224.29 173 | 172 223.95 174 | 173 223.60 175 | 174 223.26 176 | 175 222.91 177 | 176 222.56 178 | 177 222.22 179 | 178 221.87 180 | 179 221.52 181 | 180 221.18 182 | 181 220.83 183 | 182 220.49 184 | 183 220.14 185 | 184 219.79 186 | 185 219.44 187 | 186 219.09 188 | 187 218.74 189 | 188 218.39 190 | 189 218.04 191 | 190 217.69 192 | 191 217.34 193 | 192 216.99 194 | 193 216.64 195 | 194 216.29 196 | 195 215.94 197 | 196 215.59 198 | 197 215.24 199 | 198 214.89 200 | 199 214.54 201 | 200 214.19 202 | 201 213.84 203 | 202 213.49 204 | 203 213.14 205 | 204 212.79 206 | 205 212.44 207 | 206 212.09 208 | 207 211.74 209 | 208 211.39 210 | 209 211.04 211 | 210 210.69 212 | 211 210.34 213 | 212 209.99 214 | 213 209.64 215 | 214 209.29 216 | 215 208.94 217 | 216 208.59 218 | 217 208.24 219 | 218 207.89 220 | 219 207.54 221 | 220 207.19 222 | 221 206.84 223 | 222 206.49 224 | 223 206.14 225 | 224 205.79 226 | 225 205.44 227 | 226 205.09 228 | 227 204.74 229 | 228 204.39 230 | 229 204.04 231 | 230 203.69 232 | 231 203.34 233 | 232 202.99 234 | 233 202.64 235 | 234 202.28 236 | 235 201.93 237 | 236 201.57 238 | 237 201.21 239 | 238 200.85 240 | 239 200.50 241 | 240 200.14 242 | 241 194.14 243 | 242 188.14 244 | 243 182.14 245 | 244 176.14 246 | 245 170.14 247 | 246 166.13 248 | 247 162.11 249 | 248 158.10 250 | 249 154.08 251 | 250 150.07 252 | 251 146.06 253 | 252 142.04 254 | 253 138.03 255 | 254 134.01 256 | 255 130.00 257 | -------------------------------------------------------------------------------- /ml4convection/io/Channel_B09_Standary.LUT: -------------------------------------------------------------------------------- 1 | 0 300.00 2 | 1 296.86 3 | 2 293.71 4 | 3 290.57 5 | 4 287.43 6 | 5 284.28 7 | 6 281.14 8 | 7 280.80 9 | 8 280.46 10 | 9 280.12 11 | 10 279.78 12 | 11 279.44 13 | 12 279.10 14 | 13 278.76 15 | 14 278.42 16 | 15 278.08 17 | 16 277.74 18 | 17 277.40 19 | 18 277.05 20 | 19 276.71 21 | 20 276.37 22 | 21 276.03 23 | 22 275.69 24 | 23 275.35 25 | 24 275.01 26 | 25 274.67 27 | 26 274.33 28 | 27 273.99 29 | 28 273.65 30 | 29 273.31 31 | 30 272.97 32 | 31 272.63 33 | 32 272.29 34 | 33 271.95 35 | 34 271.61 36 | 35 271.27 37 | 36 270.93 38 | 37 270.59 39 | 38 270.25 40 | 39 269.91 41 | 40 269.57 42 | 41 269.23 43 | 42 268.88 44 | 43 268.54 45 | 44 268.20 46 | 45 267.86 47 | 46 267.52 48 | 47 267.18 49 | 48 266.84 50 | 49 266.50 51 | 50 266.16 52 | 51 265.82 53 | 52 265.48 54 | 53 265.14 55 | 54 264.79 56 | 55 264.43 57 | 56 264.08 58 | 57 263.73 59 | 58 263.38 60 | 59 263.02 61 | 60 262.67 62 | 61 262.32 63 | 62 261.96 64 | 63 261.61 65 | 64 261.26 66 | 65 260.90 67 | 66 260.55 68 | 67 260.20 69 | 68 259.85 70 | 69 259.49 71 | 70 259.14 72 | 71 258.80 73 | 72 258.46 74 | 73 258.12 75 | 74 257.78 76 | 75 257.44 77 | 76 257.09 78 | 77 256.75 79 | 78 256.41 80 | 79 256.07 81 | 80 255.73 82 | 81 255.39 83 | 82 255.05 84 | 83 254.71 85 | 84 254.37 86 | 85 254.03 87 | 86 253.69 88 | 87 253.34 89 | 88 253.00 90 | 89 252.66 91 | 90 252.32 92 | 91 251.98 93 | 92 251.64 94 | 93 251.29 95 | 94 250.94 96 | 95 250.59 97 | 96 250.24 98 | 97 249.89 99 | 98 249.54 100 | 99 249.19 101 | 100 248.84 102 | 101 248.49 103 | 102 248.14 104 | 103 247.79 105 | 104 247.44 106 | 105 247.09 107 | 106 246.74 108 | 107 246.39 109 | 108 246.04 110 | 109 245.69 111 | 110 245.34 112 | 111 244.99 113 | 112 244.64 114 | 113 244.29 115 | 114 243.94 116 | 115 243.59 117 | 116 243.24 118 | 117 242.89 119 | 118 242.54 120 | 119 242.19 121 | 120 241.84 122 | 121 241.49 123 | 122 241.14 124 | 123 240.81 125 | 124 240.47 126 | 125 240.14 127 | 126 239.81 128 | 127 239.47 129 | 128 239.14 130 | 129 238.81 131 | 130 238.47 132 | 131 238.14 133 | 132 237.81 134 | 133 237.47 135 | 134 237.14 136 | 135 236.79 137 | 136 236.44 138 | 137 236.10 139 | 138 235.75 140 | 139 235.40 141 | 140 235.05 142 | 141 234.71 143 | 142 234.36 144 | 143 234.01 145 | 144 233.66 146 | 145 233.31 147 | 146 232.97 148 | 147 232.62 149 | 148 232.27 150 | 149 231.92 151 | 150 231.57 152 | 151 231.23 153 | 152 230.88 154 | 153 230.53 155 | 154 230.18 156 | 155 229.84 157 | 156 229.49 158 | 157 229.14 159 | 158 228.79 160 | 159 228.45 161 | 160 228.10 162 | 161 227.76 163 | 162 227.41 164 | 163 227.06 165 | 164 226.72 166 | 165 226.37 167 | 166 226.02 168 | 167 225.68 169 | 168 225.33 170 | 169 224.99 171 | 170 224.64 172 | 171 224.29 173 | 172 223.95 174 | 173 223.60 175 | 174 223.26 176 | 175 222.91 177 | 176 222.56 178 | 177 222.22 179 | 178 221.87 180 | 179 221.52 181 | 180 221.18 182 | 181 220.83 183 | 182 220.49 184 | 183 220.14 185 | 184 219.79 186 | 185 219.44 187 | 186 219.09 188 | 187 218.74 189 | 188 218.39 190 | 189 218.04 191 | 190 217.69 192 | 191 217.34 193 | 192 216.99 194 | 193 216.64 195 | 194 216.29 196 | 195 215.94 197 | 196 215.59 198 | 197 215.24 199 | 198 214.89 200 | 199 214.54 201 | 200 214.19 202 | 201 213.84 203 | 202 213.49 204 | 203 213.14 205 | 204 212.79 206 | 205 212.44 207 | 206 212.09 208 | 207 211.74 209 | 208 211.39 210 | 209 211.04 211 | 210 210.69 212 | 211 210.34 213 | 212 209.99 214 | 213 209.64 215 | 214 209.29 216 | 215 208.94 217 | 216 208.59 218 | 217 208.24 219 | 218 207.89 220 | 219 207.54 221 | 220 207.19 222 | 221 206.84 223 | 222 206.49 224 | 223 206.14 225 | 224 205.79 226 | 225 205.44 227 | 226 205.09 228 | 227 204.74 229 | 228 204.39 230 | 229 204.04 231 | 230 203.69 232 | 231 203.34 233 | 232 202.99 234 | 233 202.64 235 | 234 202.28 236 | 235 201.93 237 | 236 201.57 238 | 237 201.21 239 | 238 200.85 240 | 239 200.50 241 | 240 200.14 242 | 241 194.14 243 | 242 188.14 244 | 243 182.14 245 | 244 176.14 246 | 245 170.14 247 | 246 166.13 248 | 247 162.11 249 | 248 158.10 250 | 249 154.08 251 | 250 150.07 252 | 251 146.06 253 | 252 142.04 254 | 253 138.03 255 | 254 134.01 256 | 255 130.00 257 | -------------------------------------------------------------------------------- /ml4convection/io/Channel_B10_Standary.LUT: -------------------------------------------------------------------------------- 1 | 0 300.00 2 | 1 296.86 3 | 2 293.71 4 | 3 290.57 5 | 4 287.43 6 | 5 284.28 7 | 6 281.14 8 | 7 280.80 9 | 8 280.46 10 | 9 280.12 11 | 10 279.78 12 | 11 279.44 13 | 12 279.10 14 | 13 278.76 15 | 14 278.42 16 | 15 278.08 17 | 16 277.74 18 | 17 277.40 19 | 18 277.05 20 | 19 276.71 21 | 20 276.37 22 | 21 276.03 23 | 22 275.69 24 | 23 275.35 25 | 24 275.01 26 | 25 274.67 27 | 26 274.33 28 | 27 273.99 29 | 28 273.65 30 | 29 273.31 31 | 30 272.97 32 | 31 272.63 33 | 32 272.29 34 | 33 271.95 35 | 34 271.61 36 | 35 271.27 37 | 36 270.93 38 | 37 270.59 39 | 38 270.25 40 | 39 269.91 41 | 40 269.57 42 | 41 269.23 43 | 42 268.88 44 | 43 268.54 45 | 44 268.20 46 | 45 267.86 47 | 46 267.52 48 | 47 267.18 49 | 48 266.84 50 | 49 266.50 51 | 50 266.16 52 | 51 265.82 53 | 52 265.48 54 | 53 265.14 55 | 54 264.79 56 | 55 264.43 57 | 56 264.08 58 | 57 263.73 59 | 58 263.38 60 | 59 263.02 61 | 60 262.67 62 | 61 262.32 63 | 62 261.96 64 | 63 261.61 65 | 64 261.26 66 | 65 260.90 67 | 66 260.55 68 | 67 260.20 69 | 68 259.85 70 | 69 259.49 71 | 70 259.14 72 | 71 258.80 73 | 72 258.46 74 | 73 258.12 75 | 74 257.78 76 | 75 257.44 77 | 76 257.09 78 | 77 256.75 79 | 78 256.41 80 | 79 256.07 81 | 80 255.73 82 | 81 255.39 83 | 82 255.05 84 | 83 254.71 85 | 84 254.37 86 | 85 254.03 87 | 86 253.69 88 | 87 253.34 89 | 88 253.00 90 | 89 252.66 91 | 90 252.32 92 | 91 251.98 93 | 92 251.64 94 | 93 251.29 95 | 94 250.94 96 | 95 250.59 97 | 96 250.24 98 | 97 249.89 99 | 98 249.54 100 | 99 249.19 101 | 100 248.84 102 | 101 248.49 103 | 102 248.14 104 | 103 247.79 105 | 104 247.44 106 | 105 247.09 107 | 106 246.74 108 | 107 246.39 109 | 108 246.04 110 | 109 245.69 111 | 110 245.34 112 | 111 244.99 113 | 112 244.64 114 | 113 244.29 115 | 114 243.94 116 | 115 243.59 117 | 116 243.24 118 | 117 242.89 119 | 118 242.54 120 | 119 242.19 121 | 120 241.84 122 | 121 241.49 123 | 122 241.14 124 | 123 240.81 125 | 124 240.47 126 | 125 240.14 127 | 126 239.81 128 | 127 239.47 129 | 128 239.14 130 | 129 238.81 131 | 130 238.47 132 | 131 238.14 133 | 132 237.81 134 | 133 237.47 135 | 134 237.14 136 | 135 236.79 137 | 136 236.44 138 | 137 236.10 139 | 138 235.75 140 | 139 235.40 141 | 140 235.05 142 | 141 234.71 143 | 142 234.36 144 | 143 234.01 145 | 144 233.66 146 | 145 233.31 147 | 146 232.97 148 | 147 232.62 149 | 148 232.27 150 | 149 231.92 151 | 150 231.57 152 | 151 231.23 153 | 152 230.88 154 | 153 230.53 155 | 154 230.18 156 | 155 229.84 157 | 156 229.49 158 | 157 229.14 159 | 158 228.79 160 | 159 228.45 161 | 160 228.10 162 | 161 227.76 163 | 162 227.41 164 | 163 227.06 165 | 164 226.72 166 | 165 226.37 167 | 166 226.02 168 | 167 225.68 169 | 168 225.33 170 | 169 224.99 171 | 170 224.64 172 | 171 224.29 173 | 172 223.95 174 | 173 223.60 175 | 174 223.26 176 | 175 222.91 177 | 176 222.56 178 | 177 222.22 179 | 178 221.87 180 | 179 221.52 181 | 180 221.18 182 | 181 220.83 183 | 182 220.49 184 | 183 220.14 185 | 184 219.79 186 | 185 219.44 187 | 186 219.09 188 | 187 218.74 189 | 188 218.39 190 | 189 218.04 191 | 190 217.69 192 | 191 217.34 193 | 192 216.99 194 | 193 216.64 195 | 194 216.29 196 | 195 215.94 197 | 196 215.59 198 | 197 215.24 199 | 198 214.89 200 | 199 214.54 201 | 200 214.19 202 | 201 213.84 203 | 202 213.49 204 | 203 213.14 205 | 204 212.79 206 | 205 212.44 207 | 206 212.09 208 | 207 211.74 209 | 208 211.39 210 | 209 211.04 211 | 210 210.69 212 | 211 210.34 213 | 212 209.99 214 | 213 209.64 215 | 214 209.29 216 | 215 208.94 217 | 216 208.59 218 | 217 208.24 219 | 218 207.89 220 | 219 207.54 221 | 220 207.19 222 | 221 206.84 223 | 222 206.49 224 | 223 206.14 225 | 224 205.79 226 | 225 205.44 227 | 226 205.09 228 | 227 204.74 229 | 228 204.39 230 | 229 204.04 231 | 230 203.69 232 | 231 203.34 233 | 232 202.99 234 | 233 202.64 235 | 234 202.28 236 | 235 201.93 237 | 236 201.57 238 | 237 201.21 239 | 238 200.85 240 | 239 200.50 241 | 240 200.14 242 | 241 194.14 243 | 242 188.14 244 | 243 182.14 245 | 244 176.14 246 | 245 170.14 247 | 246 166.13 248 | 247 162.11 249 | 248 158.10 250 | 249 154.08 251 | 250 150.07 252 | 251 146.06 253 | 252 142.04 254 | 253 138.03 255 | 254 134.01 256 | 255 130.00 257 | -------------------------------------------------------------------------------- /ml4convection/io/Channel_B11_Standary.LUT: -------------------------------------------------------------------------------- 1 | 0 330.00 2 | 1 329.94 3 | 2 329.52 4 | 3 329.10 5 | 4 328.68 6 | 5 328.26 7 | 6 327.84 8 | 7 327.43 9 | 8 327.01 10 | 9 326.59 11 | 10 326.17 12 | 11 325.75 13 | 12 325.33 14 | 13 324.91 15 | 14 324.49 16 | 15 324.07 17 | 16 323.65 18 | 17 323.23 19 | 18 322.82 20 | 19 322.40 21 | 20 321.98 22 | 21 321.56 23 | 22 321.14 24 | 23 320.72 25 | 24 320.30 26 | 25 319.88 27 | 26 319.46 28 | 27 319.04 29 | 28 318.62 30 | 29 318.20 31 | 30 317.79 32 | 31 317.37 33 | 32 316.95 34 | 33 316.53 35 | 34 316.11 36 | 35 315.69 37 | 36 315.27 38 | 37 314.85 39 | 38 314.43 40 | 39 314.01 41 | 40 313.59 42 | 41 313.18 43 | 42 312.76 44 | 43 312.34 45 | 44 311.92 46 | 45 311.50 47 | 46 311.08 48 | 47 310.66 49 | 48 310.24 50 | 49 309.82 51 | 50 309.40 52 | 51 308.98 53 | 52 308.57 54 | 53 308.15 55 | 54 307.73 56 | 55 307.31 57 | 56 306.89 58 | 57 306.47 59 | 58 306.05 60 | 59 305.63 61 | 60 305.21 62 | 61 304.79 63 | 62 304.37 64 | 63 303.95 65 | 64 303.54 66 | 65 303.12 67 | 66 302.70 68 | 67 302.28 69 | 68 301.86 70 | 69 301.44 71 | 70 301.14 72 | 71 301.00 73 | 72 300.57 74 | 73 300.15 75 | 74 299.72 76 | 75 299.30 77 | 76 298.87 78 | 77 298.44 79 | 78 298.02 80 | 79 297.59 81 | 80 297.17 82 | 81 296.74 83 | 82 296.02 84 | 83 295.31 85 | 84 294.60 86 | 85 293.88 87 | 86 293.16 88 | 87 292.45 89 | 88 291.73 90 | 89 291.02 91 | 90 290.30 92 | 91 289.59 93 | 92 288.88 94 | 93 288.16 95 | 94 287.45 96 | 95 286.73 97 | 96 286.02 98 | 97 285.30 99 | 98 284.58 100 | 99 283.87 101 | 100 283.15 102 | 101 282.44 103 | 102 281.72 104 | 103 281.01 105 | 104 280.29 106 | 105 279.58 107 | 106 278.86 108 | 107 278.15 109 | 108 277.43 110 | 109 276.72 111 | 110 276.00 112 | 111 275.29 113 | 112 274.57 114 | 113 273.86 115 | 114 273.14 116 | 115 272.54 117 | 116 271.93 118 | 117 271.33 119 | 118 270.73 120 | 119 270.12 121 | 120 269.52 122 | 121 268.92 123 | 122 268.31 124 | 123 267.71 125 | 124 267.11 126 | 125 266.50 127 | 126 265.90 128 | 127 265.30 129 | 128 264.70 130 | 129 264.09 131 | 130 263.49 132 | 131 262.89 133 | 132 262.28 134 | 133 261.68 135 | 134 261.08 136 | 135 260.47 137 | 136 259.87 138 | 137 259.27 139 | 138 258.66 140 | 139 258.06 141 | 140 257.46 142 | 141 256.85 143 | 142 256.25 144 | 143 255.21 145 | 144 254.17 146 | 145 253.13 147 | 146 252.10 148 | 147 251.06 149 | 148 250.02 150 | 149 248.98 151 | 150 247.94 152 | 151 247.56 153 | 152 247.17 154 | 153 246.79 155 | 154 246.40 156 | 155 246.02 157 | 156 245.63 158 | 157 245.25 159 | 158 244.86 160 | 159 244.48 161 | 160 244.09 162 | 161 243.71 163 | 162 243.32 164 | 163 242.94 165 | 164 242.47 166 | 165 242.00 167 | 166 241.94 168 | 167 241.21 169 | 168 240.47 170 | 169 239.74 171 | 170 239.01 172 | 171 238.27 173 | 172 237.54 174 | 173 236.81 175 | 174 236.07 176 | 175 235.34 177 | 176 234.61 178 | 177 233.87 179 | 178 233.14 180 | 179 232.41 181 | 180 231.67 182 | 181 230.94 183 | 182 230.24 184 | 183 229.54 185 | 184 228.84 186 | 185 228.14 187 | 186 227.44 188 | 187 226.74 189 | 188 226.04 190 | 189 225.34 191 | 190 224.64 192 | 191 223.94 193 | 192 223.24 194 | 193 222.54 195 | 194 221.84 196 | 195 221.14 197 | 196 220.54 198 | 197 219.94 199 | 198 219.26 200 | 199 218.59 201 | 200 217.91 202 | 201 217.23 203 | 202 216.56 204 | 203 215.88 205 | 204 215.20 206 | 205 214.52 207 | 206 213.85 208 | 207 213.17 209 | 208 212.49 210 | 209 211.82 211 | 210 211.14 212 | 211 210.54 213 | 212 209.94 214 | 213 209.19 215 | 214 208.44 216 | 215 207.69 217 | 216 206.94 218 | 217 206.19 219 | 218 205.44 220 | 219 204.69 221 | 220 203.94 222 | 221 203.34 223 | 222 202.74 224 | 223 202.14 225 | 224 201.44 226 | 225 200.74 227 | 226 200.04 228 | 227 199.34 229 | 228 198.64 230 | 229 197.94 231 | 230 196.74 232 | 231 196.14 233 | 232 195.94 234 | 233 195.27 235 | 234 194.61 236 | 235 193.94 237 | 236 192.94 238 | 237 192.28 239 | 238 191.62 240 | 239 190.96 241 | 240 190.30 242 | 241 189.64 243 | 242 188.98 244 | 243 188.32 245 | 244 187.66 246 | 245 187.00 247 | 246 182.07 248 | 247 177.14 249 | 248 170.04 250 | 249 162.94 251 | 250 157.74 252 | 251 152.54 253 | 252 147.34 254 | 253 142.14 255 | 254 136.94 256 | 255 130.00 257 | -------------------------------------------------------------------------------- /ml4convection/io/Channel_B13_Standary.LUT: -------------------------------------------------------------------------------- 1 | 0 330.00 2 | 1 329.94 3 | 2 329.52 4 | 3 329.10 5 | 4 328.68 6 | 5 328.26 7 | 6 327.84 8 | 7 327.43 9 | 8 327.01 10 | 9 326.59 11 | 10 326.17 12 | 11 325.75 13 | 12 325.33 14 | 13 324.91 15 | 14 324.49 16 | 15 324.07 17 | 16 323.65 18 | 17 323.23 19 | 18 322.82 20 | 19 322.40 21 | 20 321.98 22 | 21 321.56 23 | 22 321.14 24 | 23 320.72 25 | 24 320.30 26 | 25 319.88 27 | 26 319.46 28 | 27 319.04 29 | 28 318.62 30 | 29 318.20 31 | 30 317.79 32 | 31 317.37 33 | 32 316.95 34 | 33 316.53 35 | 34 316.11 36 | 35 315.69 37 | 36 315.27 38 | 37 314.85 39 | 38 314.43 40 | 39 314.01 41 | 40 313.59 42 | 41 313.18 43 | 42 312.76 44 | 43 312.34 45 | 44 311.92 46 | 45 311.50 47 | 46 311.08 48 | 47 310.66 49 | 48 310.24 50 | 49 309.82 51 | 50 309.40 52 | 51 308.98 53 | 52 308.57 54 | 53 308.15 55 | 54 307.73 56 | 55 307.31 57 | 56 306.89 58 | 57 306.47 59 | 58 306.05 60 | 59 305.63 61 | 60 305.21 62 | 61 304.79 63 | 62 304.37 64 | 63 303.95 65 | 64 303.54 66 | 65 303.12 67 | 66 302.70 68 | 67 302.28 69 | 68 301.86 70 | 69 301.44 71 | 70 301.14 72 | 71 301.00 73 | 72 300.57 74 | 73 300.15 75 | 74 299.72 76 | 75 299.30 77 | 76 298.87 78 | 77 298.44 79 | 78 298.02 80 | 79 297.59 81 | 80 297.17 82 | 81 296.74 83 | 82 296.02 84 | 83 295.31 85 | 84 294.60 86 | 85 293.88 87 | 86 293.16 88 | 87 292.45 89 | 88 291.73 90 | 89 291.02 91 | 90 290.30 92 | 91 289.59 93 | 92 288.88 94 | 93 288.16 95 | 94 287.45 96 | 95 286.73 97 | 96 286.02 98 | 97 285.30 99 | 98 284.58 100 | 99 283.87 101 | 100 283.15 102 | 101 282.44 103 | 102 281.72 104 | 103 281.01 105 | 104 280.29 106 | 105 279.58 107 | 106 278.86 108 | 107 278.15 109 | 108 277.43 110 | 109 276.72 111 | 110 276.00 112 | 111 275.29 113 | 112 274.57 114 | 113 273.86 115 | 114 273.14 116 | 115 272.54 117 | 116 271.93 118 | 117 271.33 119 | 118 270.73 120 | 119 270.12 121 | 120 269.52 122 | 121 268.92 123 | 122 268.31 124 | 123 267.71 125 | 124 267.11 126 | 125 266.50 127 | 126 265.90 128 | 127 265.30 129 | 128 264.70 130 | 129 264.09 131 | 130 263.49 132 | 131 262.89 133 | 132 262.28 134 | 133 261.68 135 | 134 261.08 136 | 135 260.47 137 | 136 259.87 138 | 137 259.27 139 | 138 258.66 140 | 139 258.06 141 | 140 257.46 142 | 141 256.85 143 | 142 256.25 144 | 143 255.21 145 | 144 254.17 146 | 145 253.13 147 | 146 252.10 148 | 147 251.06 149 | 148 250.02 150 | 149 248.98 151 | 150 247.94 152 | 151 247.56 153 | 152 247.17 154 | 153 246.79 155 | 154 246.40 156 | 155 246.02 157 | 156 245.63 158 | 157 245.25 159 | 158 244.86 160 | 159 244.48 161 | 160 244.09 162 | 161 243.71 163 | 162 243.32 164 | 163 242.94 165 | 164 242.47 166 | 165 242.00 167 | 166 241.94 168 | 167 241.21 169 | 168 240.47 170 | 169 239.74 171 | 170 239.01 172 | 171 238.27 173 | 172 237.54 174 | 173 236.81 175 | 174 236.07 176 | 175 235.34 177 | 176 234.61 178 | 177 233.87 179 | 178 233.14 180 | 179 232.41 181 | 180 231.67 182 | 181 230.94 183 | 182 230.24 184 | 183 229.54 185 | 184 228.84 186 | 185 228.14 187 | 186 227.44 188 | 187 226.74 189 | 188 226.04 190 | 189 225.34 191 | 190 224.64 192 | 191 223.94 193 | 192 223.24 194 | 193 222.54 195 | 194 221.84 196 | 195 221.14 197 | 196 220.54 198 | 197 219.94 199 | 198 219.26 200 | 199 218.59 201 | 200 217.91 202 | 201 217.23 203 | 202 216.56 204 | 203 215.88 205 | 204 215.20 206 | 205 214.52 207 | 206 213.85 208 | 207 213.17 209 | 208 212.49 210 | 209 211.82 211 | 210 211.14 212 | 211 210.54 213 | 212 209.94 214 | 213 209.19 215 | 214 208.44 216 | 215 207.69 217 | 216 206.94 218 | 217 206.19 219 | 218 205.44 220 | 219 204.69 221 | 220 203.94 222 | 221 203.34 223 | 222 202.74 224 | 223 202.14 225 | 224 201.44 226 | 225 200.74 227 | 226 200.04 228 | 227 199.34 229 | 228 198.64 230 | 229 197.94 231 | 230 196.74 232 | 231 196.14 233 | 232 195.94 234 | 233 195.27 235 | 234 194.61 236 | 235 193.94 237 | 236 192.94 238 | 237 192.28 239 | 238 191.62 240 | 239 190.96 241 | 240 190.30 242 | 241 189.64 243 | 242 188.98 244 | 243 188.32 245 | 244 187.66 246 | 245 187.00 247 | 246 182.07 248 | 247 177.14 249 | 248 170.04 250 | 249 162.94 251 | 250 157.74 252 | 251 152.54 253 | 252 147.34 254 | 253 142.14 255 | 254 136.94 256 | 255 130.00 257 | -------------------------------------------------------------------------------- /ml4convection/io/Channel_B14_Standary.LUT: -------------------------------------------------------------------------------- 1 | 0 330.00 2 | 1 329.94 3 | 2 329.52 4 | 3 329.10 5 | 4 328.68 6 | 5 328.26 7 | 6 327.84 8 | 7 327.43 9 | 8 327.01 10 | 9 326.59 11 | 10 326.17 12 | 11 325.75 13 | 12 325.33 14 | 13 324.91 15 | 14 324.49 16 | 15 324.07 17 | 16 323.65 18 | 17 323.23 19 | 18 322.82 20 | 19 322.40 21 | 20 321.98 22 | 21 321.56 23 | 22 321.14 24 | 23 320.72 25 | 24 320.30 26 | 25 319.88 27 | 26 319.46 28 | 27 319.04 29 | 28 318.62 30 | 29 318.20 31 | 30 317.79 32 | 31 317.37 33 | 32 316.95 34 | 33 316.53 35 | 34 316.11 36 | 35 315.69 37 | 36 315.27 38 | 37 314.85 39 | 38 314.43 40 | 39 314.01 41 | 40 313.59 42 | 41 313.18 43 | 42 312.76 44 | 43 312.34 45 | 44 311.92 46 | 45 311.50 47 | 46 311.08 48 | 47 310.66 49 | 48 310.24 50 | 49 309.82 51 | 50 309.40 52 | 51 308.98 53 | 52 308.57 54 | 53 308.15 55 | 54 307.73 56 | 55 307.31 57 | 56 306.89 58 | 57 306.47 59 | 58 306.05 60 | 59 305.63 61 | 60 305.21 62 | 61 304.79 63 | 62 304.37 64 | 63 303.95 65 | 64 303.54 66 | 65 303.12 67 | 66 302.70 68 | 67 302.28 69 | 68 301.86 70 | 69 301.44 71 | 70 301.14 72 | 71 301.00 73 | 72 300.57 74 | 73 300.15 75 | 74 299.72 76 | 75 299.30 77 | 76 298.87 78 | 77 298.44 79 | 78 298.02 80 | 79 297.59 81 | 80 297.17 82 | 81 296.74 83 | 82 296.02 84 | 83 295.31 85 | 84 294.60 86 | 85 293.88 87 | 86 293.16 88 | 87 292.45 89 | 88 291.73 90 | 89 291.02 91 | 90 290.30 92 | 91 289.59 93 | 92 288.88 94 | 93 288.16 95 | 94 287.45 96 | 95 286.73 97 | 96 286.02 98 | 97 285.30 99 | 98 284.58 100 | 99 283.87 101 | 100 283.15 102 | 101 282.44 103 | 102 281.72 104 | 103 281.01 105 | 104 280.29 106 | 105 279.58 107 | 106 278.86 108 | 107 278.15 109 | 108 277.43 110 | 109 276.72 111 | 110 276.00 112 | 111 275.29 113 | 112 274.57 114 | 113 273.86 115 | 114 273.14 116 | 115 272.54 117 | 116 271.93 118 | 117 271.33 119 | 118 270.73 120 | 119 270.12 121 | 120 269.52 122 | 121 268.92 123 | 122 268.31 124 | 123 267.71 125 | 124 267.11 126 | 125 266.50 127 | 126 265.90 128 | 127 265.30 129 | 128 264.70 130 | 129 264.09 131 | 130 263.49 132 | 131 262.89 133 | 132 262.28 134 | 133 261.68 135 | 134 261.08 136 | 135 260.47 137 | 136 259.87 138 | 137 259.27 139 | 138 258.66 140 | 139 258.06 141 | 140 257.46 142 | 141 256.85 143 | 142 256.25 144 | 143 255.21 145 | 144 254.17 146 | 145 253.13 147 | 146 252.10 148 | 147 251.06 149 | 148 250.02 150 | 149 248.98 151 | 150 247.94 152 | 151 247.56 153 | 152 247.17 154 | 153 246.79 155 | 154 246.40 156 | 155 246.02 157 | 156 245.63 158 | 157 245.25 159 | 158 244.86 160 | 159 244.48 161 | 160 244.09 162 | 161 243.71 163 | 162 243.32 164 | 163 242.94 165 | 164 242.47 166 | 165 242.00 167 | 166 241.94 168 | 167 241.21 169 | 168 240.47 170 | 169 239.74 171 | 170 239.01 172 | 171 238.27 173 | 172 237.54 174 | 173 236.81 175 | 174 236.07 176 | 175 235.34 177 | 176 234.61 178 | 177 233.87 179 | 178 233.14 180 | 179 232.41 181 | 180 231.67 182 | 181 230.94 183 | 182 230.24 184 | 183 229.54 185 | 184 228.84 186 | 185 228.14 187 | 186 227.44 188 | 187 226.74 189 | 188 226.04 190 | 189 225.34 191 | 190 224.64 192 | 191 223.94 193 | 192 223.24 194 | 193 222.54 195 | 194 221.84 196 | 195 221.14 197 | 196 220.54 198 | 197 219.94 199 | 198 219.26 200 | 199 218.59 201 | 200 217.91 202 | 201 217.23 203 | 202 216.56 204 | 203 215.88 205 | 204 215.20 206 | 205 214.52 207 | 206 213.85 208 | 207 213.17 209 | 208 212.49 210 | 209 211.82 211 | 210 211.14 212 | 211 210.54 213 | 212 209.94 214 | 213 209.19 215 | 214 208.44 216 | 215 207.69 217 | 216 206.94 218 | 217 206.19 219 | 218 205.44 220 | 219 204.69 221 | 220 203.94 222 | 221 203.34 223 | 222 202.74 224 | 223 202.14 225 | 224 201.44 226 | 225 200.74 227 | 226 200.04 228 | 227 199.34 229 | 228 198.64 230 | 229 197.94 231 | 230 196.74 232 | 231 196.14 233 | 232 195.94 234 | 233 195.27 235 | 234 194.61 236 | 235 193.94 237 | 236 192.94 238 | 237 192.28 239 | 238 191.62 240 | 239 190.96 241 | 240 190.30 242 | 241 189.64 243 | 242 188.98 244 | 243 188.32 245 | 244 187.66 246 | 245 187.00 247 | 246 182.07 248 | 247 177.14 249 | 248 170.04 250 | 249 162.94 251 | 250 157.74 252 | 251 152.54 253 | 252 147.34 254 | 253 142.14 255 | 254 136.94 256 | 255 130.00 257 | -------------------------------------------------------------------------------- /ml4convection/io/Channel_B16_Standary.LUT: -------------------------------------------------------------------------------- 1 | 0 330.00 2 | 1 329.94 3 | 2 329.52 4 | 3 329.10 5 | 4 328.68 6 | 5 328.26 7 | 6 327.84 8 | 7 327.43 9 | 8 327.01 10 | 9 326.59 11 | 10 326.17 12 | 11 325.75 13 | 12 325.33 14 | 13 324.91 15 | 14 324.49 16 | 15 324.07 17 | 16 323.65 18 | 17 323.23 19 | 18 322.82 20 | 19 322.40 21 | 20 321.98 22 | 21 321.56 23 | 22 321.14 24 | 23 320.72 25 | 24 320.30 26 | 25 319.88 27 | 26 319.46 28 | 27 319.04 29 | 28 318.62 30 | 29 318.20 31 | 30 317.79 32 | 31 317.37 33 | 32 316.95 34 | 33 316.53 35 | 34 316.11 36 | 35 315.69 37 | 36 315.27 38 | 37 314.85 39 | 38 314.43 40 | 39 314.01 41 | 40 313.59 42 | 41 313.18 43 | 42 312.76 44 | 43 312.34 45 | 44 311.92 46 | 45 311.50 47 | 46 311.08 48 | 47 310.66 49 | 48 310.24 50 | 49 309.82 51 | 50 309.40 52 | 51 308.98 53 | 52 308.57 54 | 53 308.15 55 | 54 307.73 56 | 55 307.31 57 | 56 306.89 58 | 57 306.47 59 | 58 306.05 60 | 59 305.63 61 | 60 305.21 62 | 61 304.79 63 | 62 304.37 64 | 63 303.95 65 | 64 303.54 66 | 65 303.12 67 | 66 302.70 68 | 67 302.28 69 | 68 301.86 70 | 69 301.44 71 | 70 301.14 72 | 71 301.00 73 | 72 300.57 74 | 73 300.15 75 | 74 299.72 76 | 75 299.30 77 | 76 298.87 78 | 77 298.44 79 | 78 298.02 80 | 79 297.59 81 | 80 297.17 82 | 81 296.74 83 | 82 296.02 84 | 83 295.31 85 | 84 294.60 86 | 85 293.88 87 | 86 293.16 88 | 87 292.45 89 | 88 291.73 90 | 89 291.02 91 | 90 290.30 92 | 91 289.59 93 | 92 288.88 94 | 93 288.16 95 | 94 287.45 96 | 95 286.73 97 | 96 286.02 98 | 97 285.30 99 | 98 284.58 100 | 99 283.87 101 | 100 283.15 102 | 101 282.44 103 | 102 281.72 104 | 103 281.01 105 | 104 280.29 106 | 105 279.58 107 | 106 278.86 108 | 107 278.15 109 | 108 277.43 110 | 109 276.72 111 | 110 276.00 112 | 111 275.29 113 | 112 274.57 114 | 113 273.86 115 | 114 273.14 116 | 115 272.54 117 | 116 271.93 118 | 117 271.33 119 | 118 270.73 120 | 119 270.12 121 | 120 269.52 122 | 121 268.92 123 | 122 268.31 124 | 123 267.71 125 | 124 267.11 126 | 125 266.50 127 | 126 265.90 128 | 127 265.30 129 | 128 264.70 130 | 129 264.09 131 | 130 263.49 132 | 131 262.89 133 | 132 262.28 134 | 133 261.68 135 | 134 261.08 136 | 135 260.47 137 | 136 259.87 138 | 137 259.27 139 | 138 258.66 140 | 139 258.06 141 | 140 257.46 142 | 141 256.85 143 | 142 256.25 144 | 143 255.21 145 | 144 254.17 146 | 145 253.13 147 | 146 252.10 148 | 147 251.06 149 | 148 250.02 150 | 149 248.98 151 | 150 247.94 152 | 151 247.56 153 | 152 247.17 154 | 153 246.79 155 | 154 246.40 156 | 155 246.02 157 | 156 245.63 158 | 157 245.25 159 | 158 244.86 160 | 159 244.48 161 | 160 244.09 162 | 161 243.71 163 | 162 243.32 164 | 163 242.94 165 | 164 242.47 166 | 165 242.00 167 | 166 241.94 168 | 167 241.21 169 | 168 240.47 170 | 169 239.74 171 | 170 239.01 172 | 171 238.27 173 | 172 237.54 174 | 173 236.81 175 | 174 236.07 176 | 175 235.34 177 | 176 234.61 178 | 177 233.87 179 | 178 233.14 180 | 179 232.41 181 | 180 231.67 182 | 181 230.94 183 | 182 230.24 184 | 183 229.54 185 | 184 228.84 186 | 185 228.14 187 | 186 227.44 188 | 187 226.74 189 | 188 226.04 190 | 189 225.34 191 | 190 224.64 192 | 191 223.94 193 | 192 223.24 194 | 193 222.54 195 | 194 221.84 196 | 195 221.14 197 | 196 220.54 198 | 197 219.94 199 | 198 219.26 200 | 199 218.59 201 | 200 217.91 202 | 201 217.23 203 | 202 216.56 204 | 203 215.88 205 | 204 215.20 206 | 205 214.52 207 | 206 213.85 208 | 207 213.17 209 | 208 212.49 210 | 209 211.82 211 | 210 211.14 212 | 211 210.54 213 | 212 209.94 214 | 213 209.19 215 | 214 208.44 216 | 215 207.69 217 | 216 206.94 218 | 217 206.19 219 | 218 205.44 220 | 219 204.69 221 | 220 203.94 222 | 221 203.34 223 | 222 202.74 224 | 223 202.14 225 | 224 201.44 226 | 225 200.74 227 | 226 200.04 228 | 227 199.34 229 | 228 198.64 230 | 229 197.94 231 | 230 196.74 232 | 231 196.14 233 | 232 195.94 234 | 233 195.27 235 | 234 194.61 236 | 235 193.94 237 | 236 192.94 238 | 237 192.28 239 | 238 191.62 240 | 239 190.96 241 | 240 190.30 242 | 241 189.64 243 | 242 188.98 244 | 243 188.32 245 | 244 187.66 246 | 245 187.00 247 | 246 182.07 248 | 247 177.14 249 | 248 170.04 250 | 249 162.94 251 | 250 157.74 252 | 251 152.54 253 | 252 147.34 254 | 253 142.14 255 | 254 136.94 256 | 255 130.00 257 | -------------------------------------------------------------------------------- /ml4convection/io/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thunderhoser/ml4convection/cf5e9682bc3182305274132ae246bc72d994308f/ml4convection/io/__init__.py -------------------------------------------------------------------------------- /ml4convection/io/border_io.py: -------------------------------------------------------------------------------- 1 | """Input/output methods for political borders.""" 2 | 3 | import os 4 | import sys 5 | import numpy 6 | import netCDF4 7 | from gewittergefahr.gg_utils import file_system_utils 8 | 9 | THIS_DIRECTORY_NAME = os.path.dirname(os.path.realpath( 10 | os.path.join(os.getcwd(), os.path.expanduser(__file__)) 11 | )) 12 | sys.path.append(os.path.normpath(os.path.join(THIS_DIRECTORY_NAME, '..'))) 13 | 14 | VERTEX_DIMENSION_KEY = 'vertex' 15 | LATITUDES_KEY = 'latitudes_deg_n' 16 | LONGITUDES_KEY = 'longitudes_deg_e' 17 | 18 | 19 | def _write_file(latitudes_deg_n, longitudes_deg_e, netcdf_file_name): 20 | """Writes borders to NetCDF file. 21 | 22 | P = number of points 23 | 24 | :param latitudes_deg_n: length-P numpy array of latitudes (deg N). 25 | :param longitudes_deg_e: length-P numpy array of longitudes (deg E). 26 | :param netcdf_file_name: Path to output file. 27 | """ 28 | 29 | file_system_utils.mkdir_recursive_if_necessary(file_name=netcdf_file_name) 30 | dataset_object = netCDF4.Dataset( 31 | netcdf_file_name, 'w', format='NETCDF3_64BIT_OFFSET' 32 | ) 33 | 34 | num_points = len(latitudes_deg_n) 35 | dataset_object.createDimension(VERTEX_DIMENSION_KEY, num_points) 36 | 37 | dataset_object.createVariable( 38 | LATITUDES_KEY, datatype=numpy.float32, 39 | dimensions=VERTEX_DIMENSION_KEY 40 | ) 41 | dataset_object.variables[LATITUDES_KEY][:] = latitudes_deg_n 42 | 43 | dataset_object.createVariable( 44 | LONGITUDES_KEY, datatype=numpy.float32, 45 | dimensions=VERTEX_DIMENSION_KEY 46 | ) 47 | dataset_object.variables[LONGITUDES_KEY][:] = longitudes_deg_e 48 | 49 | dataset_object.close() 50 | 51 | 52 | def read_file(netcdf_file_name=None): 53 | """Reads borders from NetCDF file. 54 | 55 | :param netcdf_file_name: Path to input file. If None, will look for file in 56 | repository. 57 | :return: latitudes_deg_n: See doc for `write_file`. 58 | :return: longitudes_deg_e: Same. 59 | """ 60 | 61 | if netcdf_file_name is None: 62 | netcdf_file_name = '{0:s}/borders.nc'.format(THIS_DIRECTORY_NAME) 63 | 64 | dataset_object = netCDF4.Dataset(netcdf_file_name) 65 | latitudes_deg_n = numpy.array( 66 | dataset_object.variables[LATITUDES_KEY][:] 67 | ) 68 | longitudes_deg_e = numpy.array( 69 | dataset_object.variables[LONGITUDES_KEY][:] 70 | ) 71 | dataset_object.close() 72 | 73 | return latitudes_deg_n, longitudes_deg_e 74 | -------------------------------------------------------------------------------- /ml4convection/io/borders.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thunderhoser/ml4convection/cf5e9682bc3182305274132ae246bc72d994308f/ml4convection/io/borders.nc -------------------------------------------------------------------------------- /ml4convection/io/new_satellite_io_test.py: -------------------------------------------------------------------------------- 1 | """Unit tests for new_satellite_io.py.""" 2 | 3 | import unittest 4 | from ml4convection.io import new_satellite_io 5 | 6 | TOLERANCE = 1e-6 7 | 8 | # The following constants are used to test get_cyclone_id, parse_cyclone_id, 9 | # find_file, and file_name_to_cyclone_id. 10 | YEAR = 1998 11 | BASIN_ID_STRING = 'AL' 12 | CYCLONE_NUMBER = 5 13 | CYCLONE_ID_STRING = '1998AL05' 14 | 15 | TOP_DIRECTORY_NAME = 'foo' 16 | VALID_TIME_UNIX_SEC = 907411500 17 | SATELLITE_FILE_NAME = 'foo/1998/1998AL05/AL0598_19982761045M.nc' 18 | 19 | 20 | class NewSatelliteIoTests(unittest.TestCase): 21 | """Each method is a unit test for new_satellite_io.py.""" 22 | 23 | def test_get_cyclone_id(self): 24 | """Ensures correct output from get_cyclone_id.""" 25 | 26 | this_id_string = new_satellite_io.get_cyclone_id( 27 | year=YEAR, basin_id_string=BASIN_ID_STRING, 28 | cyclone_number=CYCLONE_NUMBER 29 | ) 30 | 31 | self.assertTrue(this_id_string == CYCLONE_ID_STRING) 32 | 33 | def test_parse_cyclone_id(self): 34 | """Ensures correct output from parse_cyclone_id.""" 35 | 36 | this_year, this_basin_id_string, this_cyclone_number = ( 37 | new_satellite_io.parse_cyclone_id(CYCLONE_ID_STRING) 38 | ) 39 | 40 | self.assertTrue(this_year == YEAR) 41 | self.assertTrue(this_basin_id_string == BASIN_ID_STRING) 42 | self.assertTrue(this_cyclone_number == CYCLONE_NUMBER) 43 | 44 | def test_find_file(self): 45 | """Ensures correct output from find_file.""" 46 | 47 | this_file_name = new_satellite_io.find_file( 48 | top_directory_name=TOP_DIRECTORY_NAME, 49 | cyclone_id_string=CYCLONE_ID_STRING, 50 | valid_time_unix_sec=VALID_TIME_UNIX_SEC, 51 | raise_error_if_missing=False 52 | ) 53 | 54 | self.assertTrue(this_file_name == SATELLITE_FILE_NAME) 55 | 56 | def test_file_name_to_cyclone_id(self): 57 | """Ensures correct output from file_name_to_cyclone_id.""" 58 | 59 | this_id_string = new_satellite_io.file_name_to_cyclone_id( 60 | SATELLITE_FILE_NAME 61 | ) 62 | 63 | self.assertTrue(this_id_string == CYCLONE_ID_STRING) 64 | 65 | 66 | if __name__ == '__main__': 67 | unittest.main() 68 | -------------------------------------------------------------------------------- /ml4convection/io/twb_radar_io_test.py: -------------------------------------------------------------------------------- 1 | """Unit tests for twb_radar_io.py.""" 2 | 3 | import unittest 4 | from gewittergefahr.gg_utils import time_conversion 5 | from ml4convection.io import twb_radar_io 6 | 7 | # The following constants are used to test find_file and file_name_to_time. 8 | TOP_DIRECTORY_NAME = 'stuff' 9 | VALID_TIME_UNIX_SEC = time_conversion.string_to_unix_sec( 10 | '2020-08-20-22', '%Y-%m-%d-%H' 11 | ) 12 | 13 | FILE_NAME_3D = 'stuff/20200820/MREF3D21L.20200820.2200.gz' 14 | FILE_NAME_2D = 'stuff/20200820/compref_mosaic/COMPREF.20200820.2200.gz' 15 | 16 | # The following constants are used to test find_many_files. 17 | FIRST_TIME_UNIX_SEC = time_conversion.string_to_unix_sec( 18 | '2020-08-20-235959', '%Y-%m-%d-%H%M%S' 19 | ) 20 | LAST_TIME_UNIX_SEC = time_conversion.string_to_unix_sec( 21 | '2020-08-21-001712', '%Y-%m-%d-%H%M%S' 22 | ) 23 | 24 | FILE_NAMES_3D = [ 25 | 'stuff/20200820/MREF3D21L.20200820.2350.gz', 26 | 'stuff/20200821/MREF3D21L.20200821.0000.gz', 27 | 'stuff/20200821/MREF3D21L.20200821.0010.gz', 28 | 'stuff/20200821/MREF3D21L.20200821.0020.gz' 29 | ] 30 | FILE_NAMES_2D = [ 31 | 'stuff/20200820/compref_mosaic/COMPREF.20200820.2350.gz', 32 | 'stuff/20200821/compref_mosaic/COMPREF.20200821.0000.gz', 33 | 'stuff/20200821/compref_mosaic/COMPREF.20200821.0010.gz', 34 | 'stuff/20200821/compref_mosaic/COMPREF.20200821.0020.gz' 35 | ] 36 | 37 | 38 | class TwbRadarIoTests(unittest.TestCase): 39 | """Each method is a unit test for twb_radar_io.py.""" 40 | 41 | def test_find_file_3d(self): 42 | """Ensures correct output from find_file. 43 | 44 | In this case, looking for file with 3-D data. 45 | """ 46 | 47 | this_file_name = twb_radar_io.find_file( 48 | top_directory_name=TOP_DIRECTORY_NAME, 49 | valid_time_unix_sec=VALID_TIME_UNIX_SEC, 50 | with_3d=True, raise_error_if_missing=False 51 | ) 52 | 53 | self.assertTrue(this_file_name == FILE_NAME_3D) 54 | 55 | def test_find_file_2d(self): 56 | """Ensures correct output from find_file. 57 | 58 | In this case, looking for file with 2-D data. 59 | """ 60 | 61 | this_file_name = twb_radar_io.find_file( 62 | top_directory_name=TOP_DIRECTORY_NAME, 63 | valid_time_unix_sec=VALID_TIME_UNIX_SEC, 64 | with_3d=False, raise_error_if_missing=False 65 | ) 66 | 67 | self.assertTrue(this_file_name == FILE_NAME_2D) 68 | 69 | def test_find_many_files_3d(self): 70 | """Ensures correct output from find_many_files. 71 | 72 | In this case, looking for files with 3-D data. 73 | """ 74 | 75 | these_file_names = twb_radar_io.find_many_files( 76 | top_directory_name=TOP_DIRECTORY_NAME, 77 | first_time_unix_sec=FIRST_TIME_UNIX_SEC, 78 | last_time_unix_sec=LAST_TIME_UNIX_SEC, 79 | with_3d=True, test_mode=True 80 | ) 81 | 82 | self.assertTrue(these_file_names == FILE_NAMES_3D) 83 | 84 | def test_find_many_files_2d(self): 85 | """Ensures correct output from find_many_files. 86 | 87 | In this case, looking for files with 2-D data. 88 | """ 89 | 90 | these_file_names = twb_radar_io.find_many_files( 91 | top_directory_name=TOP_DIRECTORY_NAME, 92 | first_time_unix_sec=FIRST_TIME_UNIX_SEC, 93 | last_time_unix_sec=LAST_TIME_UNIX_SEC, 94 | with_3d=False, test_mode=True 95 | ) 96 | 97 | self.assertTrue(these_file_names == FILE_NAMES_2D) 98 | 99 | def test_file_name_to_time_3d(self): 100 | """Ensures correct output from file_name_to_time. 101 | 102 | In this case, file contains 3-D data. 103 | """ 104 | 105 | self.assertTrue( 106 | twb_radar_io.file_name_to_time(FILE_NAME_3D) == 107 | VALID_TIME_UNIX_SEC 108 | ) 109 | 110 | def test_file_name_to_time_2d(self): 111 | """Ensures correct output from file_name_to_time. 112 | 113 | In this case, file contains 2-D data. 114 | """ 115 | 116 | self.assertTrue( 117 | twb_radar_io.file_name_to_time(FILE_NAME_2D) == 118 | VALID_TIME_UNIX_SEC 119 | ) 120 | 121 | 122 | if __name__ == '__main__': 123 | unittest.main() 124 | -------------------------------------------------------------------------------- /ml4convection/io/twb_satellite_io_test.py: -------------------------------------------------------------------------------- 1 | """Unit tests for twb_satellite_io.py.""" 2 | 3 | import unittest 4 | import numpy 5 | from gewittergefahr.gg_utils import time_conversion 6 | from ml4convection.io import twb_satellite_io 7 | 8 | TOLERANCE = 1e-6 9 | 10 | # The following constants are used to test find_file, file_name_to_time, and 11 | # file_name_to_band_number. 12 | TOP_DIRECTORY_NAME = 'stuff' 13 | VALID_TIME_UNIX_SEC = time_conversion.string_to_unix_sec( 14 | '2020-08-20-22', '%Y-%m-%d-%H' 15 | ) 16 | BAND_NUMBER = 8 17 | 18 | FILE_NAME = 'stuff/2020-08/2020-08-20_2200.B08.GSD.Cnt' 19 | 20 | # The following constants are used to test find_many_files. 21 | FIRST_TIME_UNIX_SEC = time_conversion.string_to_unix_sec( 22 | '2020-08-20-235959', '%Y-%m-%d-%H%M%S' 23 | ) 24 | LAST_TIME_UNIX_SEC = time_conversion.string_to_unix_sec( 25 | '2020-08-21-001712', '%Y-%m-%d-%H%M%S' 26 | ) 27 | 28 | FILE_NAMES = [ 29 | 'stuff/2020-08/2020-08-20_2350.B08.GSD.Cnt', 30 | 'stuff/2020-08/2020-08-21_0000.B08.GSD.Cnt', 31 | 'stuff/2020-08/2020-08-21_0010.B08.GSD.Cnt', 32 | 'stuff/2020-08/2020-08-21_0020.B08.GSD.Cnt' 33 | ] 34 | 35 | # The following constants are used to test _count_to_temperature. 36 | MISSING_COUNT = twb_satellite_io.MISSING_COUNT 37 | 38 | BRIGHTNESS_COUNT_MATRIX = numpy.array([ 39 | [0, 5, 10, 15, 20, 25], 40 | [100, 105, 110, 115, 120, 125], 41 | [200, 205, 210, 215, 220, 225], 42 | [MISSING_COUNT, 8, 4, 6, 2, MISSING_COUNT] 43 | ], dtype=int) 44 | 45 | BRIGHTNESS_TEMP_MATRIX_BAND8_KELVINS = numpy.array([ 46 | [300, 284.28, 279.78, 278.08, 276.37, 274.67], 47 | [248.84, 247.09, 245.34, 243.59, 241.84, 240.14], 48 | [214.19, 212.44, 210.69, 208.94, 207.19, 205.44], 49 | [numpy.nan, 280.46, 287.43, 281.14, 293.71, numpy.nan] 50 | ]) 51 | 52 | BRIGHTNESS_TEMP_MATRIX_BAND13_KELVINS = numpy.array([ 53 | [330, 328.26, 326.17, 324.07, 321.98, 319.88], 54 | [283.15, 279.58, 276, 272.54, 269.52, 266.5], 55 | [217.91, 214.52, 211.14, 207.69, 203.94, 200.74], 56 | [numpy.nan, 327.01, 328.68, 327.84, 329.52, numpy.nan] 57 | ]) 58 | 59 | 60 | class TwbSatelliteIoTests(unittest.TestCase): 61 | """Each method is a unit test for twb_satellite_io.py.""" 62 | 63 | def test_find_file(self): 64 | """Ensures correct output from find_file.""" 65 | 66 | this_file_name = twb_satellite_io.find_file( 67 | top_directory_name=TOP_DIRECTORY_NAME, 68 | valid_time_unix_sec=VALID_TIME_UNIX_SEC, band_number=BAND_NUMBER, 69 | raise_error_if_missing=False 70 | ) 71 | 72 | self.assertTrue(this_file_name == FILE_NAME) 73 | 74 | def test_find_many_files(self): 75 | """Ensures correct output from find_many_files.""" 76 | 77 | these_file_names = twb_satellite_io.find_many_files( 78 | top_directory_name=TOP_DIRECTORY_NAME, 79 | first_time_unix_sec=FIRST_TIME_UNIX_SEC, 80 | last_time_unix_sec=LAST_TIME_UNIX_SEC, 81 | band_number=BAND_NUMBER, test_mode=True 82 | ) 83 | 84 | self.assertTrue(these_file_names == FILE_NAMES) 85 | 86 | def test_file_name_to_time(self): 87 | """Ensures correct output from file_name_to_time.""" 88 | 89 | self.assertTrue( 90 | twb_satellite_io.file_name_to_time(FILE_NAME) == 91 | VALID_TIME_UNIX_SEC 92 | ) 93 | 94 | def test_file_name_to_band_number(self): 95 | """Ensures correct output from file_name_to_band_number.""" 96 | 97 | self.assertTrue( 98 | twb_satellite_io.file_name_to_band_number(FILE_NAME) == 99 | BAND_NUMBER 100 | ) 101 | 102 | def test_count_to_temperature_band8(self): 103 | """Ensures correct output from _count_to_temperature. 104 | 105 | In this case, band number is 8. 106 | """ 107 | 108 | this_temp_matrix_kelvins = twb_satellite_io._count_to_temperature( 109 | brightness_counts=BRIGHTNESS_COUNT_MATRIX, band_number=8 110 | ) 111 | 112 | self.assertTrue(numpy.allclose( 113 | this_temp_matrix_kelvins, BRIGHTNESS_TEMP_MATRIX_BAND8_KELVINS, 114 | atol=TOLERANCE, equal_nan=True 115 | )) 116 | 117 | def test_count_to_temperature_band13(self): 118 | """Ensures correct output from _count_to_temperature. 119 | 120 | In this case, band number is 13. 121 | """ 122 | 123 | this_temp_matrix_kelvins = twb_satellite_io._count_to_temperature( 124 | brightness_counts=BRIGHTNESS_COUNT_MATRIX, band_number=13 125 | ) 126 | 127 | self.assertTrue(numpy.allclose( 128 | this_temp_matrix_kelvins, BRIGHTNESS_TEMP_MATRIX_BAND13_KELVINS, 129 | atol=TOLERANCE, equal_nan=True 130 | )) 131 | 132 | 133 | if __name__ == '__main__': 134 | unittest.main() 135 | -------------------------------------------------------------------------------- /ml4convection/machine_learning/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thunderhoser/ml4convection/cf5e9682bc3182305274132ae246bc72d994308f/ml4convection/machine_learning/__init__.py -------------------------------------------------------------------------------- /ml4convection/machine_learning/coord_conv_test.py: -------------------------------------------------------------------------------- 1 | """Unit tests for coord_conv.py.""" 2 | 3 | import unittest 4 | import numpy 5 | from keras import backend as K 6 | from ml4convection.machine_learning import coord_conv 7 | 8 | TOLERANCE = 1e-6 9 | 10 | NUM_EXAMPLES = 32 11 | NUM_GRID_ROWS = 5 12 | NUM_GRID_COLUMNS = 7 13 | NUM_INPUT_CHANNELS_SANS_TIME = 14 14 | 15 | INPUT_DIMENSIONS_SANS_TIME = numpy.array([ 16 | NUM_EXAMPLES, NUM_GRID_ROWS, NUM_GRID_COLUMNS, NUM_INPUT_CHANNELS_SANS_TIME 17 | ], dtype=int) 18 | 19 | Y_COORD_MATRIX_SANS_TIME = numpy.array([ 20 | [-1, -1, -1, -1, -1, -1, -1], 21 | [-0.5, -0.5, -0.5, -0.5, -0.5, -0.5, -0.5], 22 | [0, 0, 0, 0, 0, 0, 0], 23 | [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5], 24 | [1, 1, 1, 1, 1, 1, 1] 25 | ]) 26 | 27 | X_COORD_MATRIX_SANS_TIME = numpy.array([ 28 | [-1, -2. / 3, -1. / 3, 0, 1. / 3, 2. / 3, 1], 29 | [-1, -2. / 3, -1. / 3, 0, 1. / 3, 2. / 3, 1], 30 | [-1, -2. / 3, -1. / 3, 0, 1. / 3, 2. / 3, 1], 31 | [-1, -2. / 3, -1. / 3, 0, 1. / 3, 2. / 3, 1], 32 | [-1, -2. / 3, -1. / 3, 0, 1. / 3, 2. / 3, 1] 33 | ]) 34 | 35 | Y_COORD_MATRIX_SANS_TIME = numpy.expand_dims(Y_COORD_MATRIX_SANS_TIME, axis=0) 36 | Y_COORD_MATRIX_SANS_TIME = numpy.repeat( 37 | Y_COORD_MATRIX_SANS_TIME, axis=0, repeats=NUM_EXAMPLES 38 | ) 39 | Y_COORD_MATRIX_SANS_TIME = numpy.expand_dims(Y_COORD_MATRIX_SANS_TIME, axis=-1) 40 | 41 | X_COORD_MATRIX_SANS_TIME = numpy.expand_dims(X_COORD_MATRIX_SANS_TIME, axis=0) 42 | X_COORD_MATRIX_SANS_TIME = numpy.repeat( 43 | X_COORD_MATRIX_SANS_TIME, axis=0, repeats=NUM_EXAMPLES 44 | ) 45 | X_COORD_MATRIX_SANS_TIME = numpy.expand_dims(X_COORD_MATRIX_SANS_TIME, axis=-1) 46 | 47 | OUTPUT_MATRIX_SANS_TIME = numpy.concatenate( 48 | (X_COORD_MATRIX_SANS_TIME, Y_COORD_MATRIX_SANS_TIME), axis=-1 49 | ) 50 | 51 | NUM_TIMES = 2 52 | NUM_INPUT_CHANNELS_WITH_TIME = 7 53 | 54 | INPUT_DIMENSIONS_WITH_TIME = numpy.array([ 55 | NUM_EXAMPLES, NUM_GRID_ROWS, NUM_GRID_COLUMNS, NUM_TIMES, 56 | NUM_INPUT_CHANNELS_WITH_TIME 57 | ], dtype=int) 58 | 59 | Y_COORD_MATRIX_WITH_TIME = numpy.expand_dims(Y_COORD_MATRIX_SANS_TIME, axis=-2) 60 | Y_COORD_MATRIX_WITH_TIME = numpy.repeat( 61 | Y_COORD_MATRIX_WITH_TIME, axis=-2, repeats=NUM_TIMES 62 | ) 63 | 64 | X_COORD_MATRIX_WITH_TIME = numpy.expand_dims(X_COORD_MATRIX_SANS_TIME, axis=-2) 65 | X_COORD_MATRIX_WITH_TIME = numpy.repeat( 66 | X_COORD_MATRIX_WITH_TIME, axis=-2, repeats=NUM_TIMES 67 | ) 68 | 69 | OUTPUT_MATRIX_WITH_TIME = numpy.concatenate( 70 | (X_COORD_MATRIX_WITH_TIME, Y_COORD_MATRIX_WITH_TIME), axis=-1 71 | ) 72 | 73 | 74 | class CoordConvTests(unittest.TestCase): 75 | """Each method is a unit test for coord_conv.py.""" 76 | 77 | def test_add_spatial_coords_2d(self): 78 | """Ensures correct output from add_spatial_coords_2d.""" 79 | 80 | output_tensor = coord_conv.add_spatial_coords_2d( 81 | input_layer_object=None, test_mode=True, 82 | input_dimensions=INPUT_DIMENSIONS_SANS_TIME 83 | ) 84 | 85 | self.assertTrue(numpy.allclose( 86 | OUTPUT_MATRIX_SANS_TIME, K.eval(output_tensor), atol=TOLERANCE 87 | )) 88 | 89 | def test_add_spatial_coords_2d_with_time(self): 90 | """Ensures correct output from add_spatial_coords_2d_with_time.""" 91 | 92 | output_tensor = coord_conv.add_spatial_coords_2d_with_time( 93 | input_layer_object=None, num_times=0, test_mode=True, 94 | input_dimensions=INPUT_DIMENSIONS_WITH_TIME 95 | ) 96 | 97 | self.assertTrue(numpy.allclose( 98 | OUTPUT_MATRIX_WITH_TIME, K.eval(output_tensor), atol=TOLERANCE 99 | )) 100 | 101 | 102 | if __name__ == '__main__': 103 | unittest.main() 104 | -------------------------------------------------------------------------------- /ml4convection/machine_learning/permutation_test.py: -------------------------------------------------------------------------------- 1 | """Unit tests for permutation.py.""" 2 | 3 | import unittest 4 | import numpy 5 | from ml4convection.machine_learning import neural_net 6 | from ml4convection.machine_learning import permutation 7 | 8 | TOLERANCE = 1e-6 9 | 10 | # The following constants are used to _permute_values and _depermute_values. 11 | PREDICTOR_MATRIX = numpy.random.uniform(low=0., high=1., size=(10, 205, 205, 7)) 12 | CHANNEL_INDEX_TO_PERMUTE = 5 13 | DUMMY_DATA_OPTION_DICT = { 14 | neural_net.INCLUDE_TIME_DIM_KEY: True 15 | } 16 | 17 | 18 | class PermutationTests(unittest.TestCase): 19 | """Each method is a unit test for permutation.py.""" 20 | 21 | def test_permute_values(self): 22 | """Ensures correct output from _permute_values.""" 23 | 24 | new_predictor_matrix, permuted_example_indices = ( 25 | permutation._permute_values( 26 | predictor_matrix=PREDICTOR_MATRIX + 0., 27 | channel_index=CHANNEL_INDEX_TO_PERMUTE, 28 | data_option_dict=DUMMY_DATA_OPTION_DICT, 29 | permuted_example_indices=None 30 | ) 31 | ) 32 | 33 | num_channels = new_predictor_matrix.shape[-1] 34 | indices_to_compare = ( 35 | numpy.arange(num_channels) != CHANNEL_INDEX_TO_PERMUTE 36 | ) 37 | 38 | self.assertTrue(numpy.allclose( 39 | new_predictor_matrix[..., indices_to_compare], 40 | PREDICTOR_MATRIX[..., indices_to_compare], atol=TOLERANCE 41 | )) 42 | 43 | newnew_predictor_matrix = permutation._permute_values( 44 | predictor_matrix=PREDICTOR_MATRIX + 0., 45 | channel_index=CHANNEL_INDEX_TO_PERMUTE, 46 | data_option_dict=DUMMY_DATA_OPTION_DICT, 47 | permuted_example_indices=permuted_example_indices 48 | )[0] 49 | 50 | self.assertTrue(numpy.allclose( 51 | new_predictor_matrix, newnew_predictor_matrix, atol=TOLERANCE 52 | )) 53 | 54 | def test_depermute_values(self): 55 | """Ensures correct output from _depermute_values.""" 56 | 57 | new_predictor_matrix, permuted_example_indices = ( 58 | permutation._permute_values( 59 | predictor_matrix=PREDICTOR_MATRIX + 0., 60 | channel_index=CHANNEL_INDEX_TO_PERMUTE, 61 | data_option_dict=DUMMY_DATA_OPTION_DICT, 62 | permuted_example_indices=None 63 | ) 64 | ) 65 | 66 | self.assertFalse(numpy.allclose( 67 | new_predictor_matrix, PREDICTOR_MATRIX, atol=TOLERANCE 68 | )) 69 | 70 | new_predictor_matrix = permutation._depermute_values( 71 | predictor_matrix=new_predictor_matrix, 72 | channel_index=CHANNEL_INDEX_TO_PERMUTE, 73 | data_option_dict=DUMMY_DATA_OPTION_DICT, 74 | permuted_example_indices=permuted_example_indices 75 | ) 76 | 77 | self.assertTrue(numpy.allclose( 78 | new_predictor_matrix, PREDICTOR_MATRIX, atol=TOLERANCE 79 | )) 80 | 81 | 82 | if __name__ == '__main__': 83 | unittest.main() 84 | -------------------------------------------------------------------------------- /ml4convection/machine_learning/standalone_utils.py: -------------------------------------------------------------------------------- 1 | """Stand-alone Keras operations.""" 2 | 3 | import numpy 4 | import tensorflow.python.keras.backend as K 5 | from gewittergefahr.gg_utils import error_checking 6 | 7 | 8 | def do_2d_pooling(feature_matrix, do_max_pooling, window_size_px=2, 9 | stride_length_px=None, pad_edges=False): 10 | """Pools 2-D feature maps. 11 | 12 | E = number of examples 13 | M = number of rows before pooling 14 | N = number of columns before pooling 15 | C = number of channels 16 | m = number of rows after pooling 17 | n = number of columns after pooling 18 | 19 | :param feature_matrix: E-by-M-by-N-by-C numpy array of feature values. 20 | :param do_max_pooling: Boolean flag. If True, will do max-pooling. If 21 | False, will do average-pooling. 22 | :param window_size_px: Window size (pixels). The pooling window will be 23 | K rows x K columns, where K = `window_size_px`. 24 | :param stride_length_px: Stride length (pixels). The pooling window will 25 | move k rows or columns with each stride, where k = `stride_length_px`. 26 | :param pad_edges: See doc for `do_2d_convolution`. 27 | :return: feature_matrix: E-by-m-by-n-by-C numpy array of new feature values. 28 | """ 29 | 30 | error_checking.assert_is_numpy_array_without_nan(feature_matrix) 31 | error_checking.assert_is_numpy_array(feature_matrix, num_dimensions=4) 32 | error_checking.assert_is_boolean(do_max_pooling) 33 | error_checking.assert_is_boolean(pad_edges) 34 | error_checking.assert_is_integer(window_size_px) 35 | error_checking.assert_is_geq(window_size_px, 2) 36 | 37 | if stride_length_px is None: 38 | stride_length_px = window_size_px 39 | 40 | error_checking.assert_is_integer(stride_length_px) 41 | error_checking.assert_is_geq(stride_length_px, 1) 42 | 43 | feature_tensor = K.pool2d( 44 | x=K.variable(feature_matrix), 45 | pool_mode='max' if do_max_pooling else 'avg', 46 | pool_size=(window_size_px, window_size_px), 47 | strides=(stride_length_px, stride_length_px), 48 | padding='same' if pad_edges else 'valid', data_format='channels_last' 49 | ) 50 | 51 | return feature_tensor.numpy() 52 | 53 | 54 | def do_1d_pooling(feature_matrix, do_max_pooling, window_size_px=2): 55 | """Pools 1-D feature maps. 56 | 57 | E = number of examples 58 | P = number of pixels before pooling 59 | C = number of channels 60 | p = number of pixels after pooling 61 | 62 | :param feature_matrix: E-by-P-by-C numpy array of feature values. 63 | :param do_max_pooling: See doc for `do_2d_pooling`. 64 | :param window_size_px: Same. 65 | :return: feature_matrix: E-by-p-by-C numpy array of new feature values. 66 | """ 67 | 68 | error_checking.assert_is_numpy_array_without_nan(feature_matrix) 69 | error_checking.assert_is_numpy_array(feature_matrix, num_dimensions=3) 70 | error_checking.assert_is_integer(window_size_px) 71 | error_checking.assert_is_geq(window_size_px, 2) 72 | 73 | feature_matrix = numpy.expand_dims(feature_matrix, axis=-2) 74 | feature_matrix = numpy.repeat( 75 | feature_matrix, repeats=window_size_px, axis=-2 76 | ) 77 | 78 | feature_matrix = do_2d_pooling( 79 | feature_matrix=feature_matrix, do_max_pooling=do_max_pooling, 80 | window_size_px=window_size_px 81 | ) 82 | 83 | return feature_matrix[..., 0, :] 84 | 85 | 86 | def do_2d_convolution( 87 | feature_matrix, kernel_matrix, pad_edges=False, stride_length_px=1): 88 | """Convolves 2-D feature maps. 89 | 90 | E = number of examples 91 | M = number of input rows 92 | N = number of input columns 93 | C = number of input channels 94 | m = number of rows in kernel 95 | n = number of columns in kernel 96 | c = number of output channels 97 | 98 | :param feature_matrix: E-by-M-by-N-by-C numpy array of feature values. 99 | :param kernel_matrix: m-by-n-by-C-by-c numpy array of kernel weights (filter 100 | weights). 101 | :param pad_edges: Boolean flag. If True, edges will be padded so that the 102 | output feature matrix has the same dimensions as the input matrix. If 103 | False, edges will not be padded, so the output feature matrix will be 104 | smaller. 105 | :param stride_length_px: Stride length in pixels. The kernel (filter) will 106 | move this far at each step. 107 | :return: feature_matrix: E-by-?-by-?-by-c numpy array of feature values. 108 | The spatial dimensions will be determined by filter size and 109 | edge-padding. 110 | """ 111 | 112 | error_checking.assert_is_numpy_array_without_nan(feature_matrix) 113 | error_checking.assert_is_numpy_array(feature_matrix, num_dimensions=4) 114 | error_checking.assert_is_numpy_array_without_nan(kernel_matrix) 115 | error_checking.assert_is_numpy_array(kernel_matrix, num_dimensions=4) 116 | error_checking.assert_is_boolean(pad_edges) 117 | error_checking.assert_is_integer(stride_length_px) 118 | error_checking.assert_is_geq(stride_length_px, 1) 119 | 120 | feature_tensor = K.conv2d( 121 | x=K.variable(feature_matrix), 122 | kernel=K.variable(kernel_matrix), 123 | strides=(stride_length_px, stride_length_px), 124 | padding='same' if pad_edges else 'valid', 125 | data_format='channels_last' 126 | ) 127 | 128 | return feature_tensor.numpy() 129 | -------------------------------------------------------------------------------- /ml4convection/machine_learning/standalone_utils_test.py: -------------------------------------------------------------------------------- 1 | """Unit tests for standalone_utils.py.""" 2 | 3 | import unittest 4 | import numpy 5 | from ml4convection.machine_learning import standalone_utils 6 | from ml4convection.machine_learning import custom_losses_test 7 | 8 | TOLERANCE = 1e-6 9 | 10 | # The following constants are used to test do_2d_convolution. 11 | TARGET_MATRIX = numpy.expand_dims(custom_losses_test.TARGET_MATRIX, axis=0) 12 | TARGET_MATRIX = numpy.expand_dims(TARGET_MATRIX, axis=-1) 13 | TARGET_MATRIX = numpy.repeat(TARGET_MATRIX, repeats=3, axis=0) 14 | 15 | PREDICTION_MATRIX = numpy.expand_dims( 16 | custom_losses_test.PREDICTION_MATRIX, axis=0 17 | ) 18 | PREDICTION_MATRIX = numpy.expand_dims(PREDICTION_MATRIX, axis=-1) 19 | PREDICTION_MATRIX = numpy.repeat(PREDICTION_MATRIX, repeats=3, axis=0) 20 | 21 | TARGET_MATRIX_SMOOTHED1 = numpy.expand_dims( 22 | custom_losses_test.TARGET_MATRIX_SMOOTHED1, axis=0 23 | ) 24 | TARGET_MATRIX_SMOOTHED1 = numpy.expand_dims(TARGET_MATRIX_SMOOTHED1, axis=-1) 25 | TARGET_MATRIX_SMOOTHED1 = numpy.repeat( 26 | TARGET_MATRIX_SMOOTHED1, repeats=3, axis=0 27 | ) 28 | 29 | PREDICTION_MATRIX_SMOOTHED1 = numpy.expand_dims( 30 | custom_losses_test.PREDICTION_MATRIX_SMOOTHED1, axis=0 31 | ) 32 | PREDICTION_MATRIX_SMOOTHED1 = numpy.expand_dims( 33 | PREDICTION_MATRIX_SMOOTHED1, axis=-1 34 | ) 35 | PREDICTION_MATRIX_SMOOTHED1 = numpy.repeat( 36 | PREDICTION_MATRIX_SMOOTHED1, repeats=3, axis=0 37 | ) 38 | 39 | TARGET_MATRIX_SMOOTHED2 = numpy.expand_dims( 40 | custom_losses_test.TARGET_MATRIX_SMOOTHED2, axis=0 41 | ) 42 | TARGET_MATRIX_SMOOTHED2 = numpy.expand_dims(TARGET_MATRIX_SMOOTHED2, axis=-1) 43 | TARGET_MATRIX_SMOOTHED2 = numpy.repeat( 44 | TARGET_MATRIX_SMOOTHED2, repeats=3, axis=0 45 | ) 46 | 47 | PREDICTION_MATRIX_SMOOTHED2 = numpy.expand_dims( 48 | custom_losses_test.PREDICTION_MATRIX_SMOOTHED2, axis=0 49 | ) 50 | PREDICTION_MATRIX_SMOOTHED2 = numpy.expand_dims( 51 | PREDICTION_MATRIX_SMOOTHED2, axis=-1 52 | ) 53 | PREDICTION_MATRIX_SMOOTHED2 = numpy.repeat( 54 | PREDICTION_MATRIX_SMOOTHED2, repeats=3, axis=0 55 | ) 56 | 57 | WEIGHT_MATRIX_SIZE0 = numpy.full((1, 1, 1, 1), 1.) 58 | WEIGHT_MATRIX_SIZE1 = numpy.full((3, 3, 1, 1), 1. / 9) 59 | WEIGHT_MATRIX_SIZE2 = numpy.full((5, 5, 1, 1), 1. / 25) 60 | 61 | 62 | class StandaloneUtilsTests(unittest.TestCase): 63 | """Each method is a unit test for standalone_utils.py.""" 64 | 65 | def test_do_2d_convolution_size0(self): 66 | """Ensures correct output from do_2d_convolution. 67 | 68 | In this case, half-window size is 0 pixels. 69 | """ 70 | 71 | this_target_matrix = standalone_utils.do_2d_convolution( 72 | feature_matrix=TARGET_MATRIX, kernel_matrix=WEIGHT_MATRIX_SIZE0, 73 | pad_edges=True, stride_length_px=1 74 | ) 75 | self.assertTrue(numpy.allclose( 76 | this_target_matrix, TARGET_MATRIX, atol=TOLERANCE 77 | )) 78 | 79 | this_prediction_matrix = standalone_utils.do_2d_convolution( 80 | feature_matrix=PREDICTION_MATRIX, kernel_matrix=WEIGHT_MATRIX_SIZE0, 81 | pad_edges=True, stride_length_px=1 82 | ) 83 | self.assertTrue(numpy.allclose( 84 | this_prediction_matrix, PREDICTION_MATRIX, atol=TOLERANCE 85 | )) 86 | 87 | def test_do_2d_convolution_size1(self): 88 | """Ensures correct output from do_2d_convolution. 89 | 90 | In this case, half-window size is 1 pixel. 91 | """ 92 | 93 | this_target_matrix = standalone_utils.do_2d_convolution( 94 | feature_matrix=TARGET_MATRIX, kernel_matrix=WEIGHT_MATRIX_SIZE1, 95 | pad_edges=True, stride_length_px=1 96 | ) 97 | self.assertTrue(numpy.allclose( 98 | this_target_matrix, TARGET_MATRIX_SMOOTHED1, atol=TOLERANCE 99 | )) 100 | 101 | this_prediction_matrix = standalone_utils.do_2d_convolution( 102 | feature_matrix=PREDICTION_MATRIX, kernel_matrix=WEIGHT_MATRIX_SIZE1, 103 | pad_edges=True, stride_length_px=1 104 | ) 105 | self.assertTrue(numpy.allclose( 106 | this_prediction_matrix, PREDICTION_MATRIX_SMOOTHED1, atol=TOLERANCE 107 | )) 108 | 109 | def test_do_2d_convolution_size2(self): 110 | """Ensures correct output from do_2d_convolution. 111 | 112 | In this case, half-window size is 2 pixels. 113 | """ 114 | 115 | this_target_matrix = standalone_utils.do_2d_convolution( 116 | feature_matrix=TARGET_MATRIX, kernel_matrix=WEIGHT_MATRIX_SIZE2, 117 | pad_edges=True, stride_length_px=1 118 | ) 119 | self.assertTrue(numpy.allclose( 120 | this_target_matrix, TARGET_MATRIX_SMOOTHED2, atol=TOLERANCE 121 | )) 122 | 123 | this_prediction_matrix = standalone_utils.do_2d_convolution( 124 | feature_matrix=PREDICTION_MATRIX, kernel_matrix=WEIGHT_MATRIX_SIZE2, 125 | pad_edges=True, stride_length_px=1 126 | ) 127 | self.assertTrue(numpy.allclose( 128 | this_prediction_matrix, PREDICTION_MATRIX_SMOOTHED2, atol=TOLERANCE 129 | )) 130 | 131 | 132 | if __name__ == '__main__': 133 | unittest.main() 134 | -------------------------------------------------------------------------------- /ml4convection/plotting/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thunderhoser/ml4convection/cf5e9682bc3182305274132ae246bc72d994308f/ml4convection/plotting/__init__.py -------------------------------------------------------------------------------- /ml4convection/plotting/evaluation_plotting_test.py: -------------------------------------------------------------------------------- 1 | """Unit tests for evaluation_plotting.py.""" 2 | 3 | import unittest 4 | import numpy 5 | from ml4convection.plotting import evaluation_plotting as eval_plotting 6 | 7 | TOLERANCE = 1e-6 8 | 9 | # The following constants are used to test _get_positive_skill_area and 10 | # _get_zero_skill_line. 11 | MEAN_VALUE_IN_TRAINING = 0.2 12 | 13 | X_COORDS_NO_SKILL = numpy.array([0, 1], dtype=float) 14 | Y_COORDS_NO_SKILL = numpy.array([0.1, 0.6]) 15 | 16 | X_COORDS_LEFT = numpy.array([0, 0.2, 0.2, 0, 0]) 17 | Y_COORDS_LEFT = numpy.array([0, 0, 0.2, 0.1, 0]) 18 | X_COORDS_RIGHT = numpy.array([0.2, 1, 1, 0.2, 0.2]) 19 | Y_COORDS_RIGHT = numpy.array([0.2, 0.6, 1, 1, 0.2]) 20 | 21 | # The following constants are used to test _get_pofd_pod_grid, _get_sr_pod_grid, 22 | # _csi_from_sr_and_pod, and _bias_from_sr_and_pod. 23 | POD_GRID_SPACING = 0.5 24 | POFD_GRID_SPACING = 0.25 25 | SUCCESS_RATIO_GRID_SPACING = 0.25 26 | 27 | POD_MATRIX = numpy.array([ 28 | [0.75, 0.75, 0.75, 0.75], 29 | [0.25, 0.25, 0.25, 0.25] 30 | ]) 31 | 32 | SUCCESS_RATIO_MATRIX = numpy.array([ 33 | [0.125, 0.375, 0.625, 0.875], 34 | [0.125, 0.375, 0.625, 0.875] 35 | ]) 36 | 37 | POFD_MATRIX = SUCCESS_RATIO_MATRIX + 0. 38 | 39 | FREQUENCY_BIAS_MATRIX = numpy.array([ 40 | [6, 2, 1.2, 6. / 7], 41 | [2, 2. / 3, 0.4, 2. / 7] 42 | ]) 43 | 44 | CSI_MATRIX = numpy.array([ 45 | [25. / 3, 3, 0.6 + 4. / 3, 1. / 7 + 4. / 3], 46 | [11, 17. / 3, 4.6, 29. / 7] 47 | ]) ** -1 48 | 49 | 50 | class EvaluationPlottingTests(unittest.TestCase): 51 | """Each method is a unit test for evaluation_plotting.py.""" 52 | 53 | def test_get_positive_skill_area(self): 54 | """Ensures correct output from _get_positive_skill_area.""" 55 | 56 | ( 57 | these_x_coords_left, these_y_coords_left, 58 | these_x_coords_right, these_y_coords_right 59 | ) = eval_plotting._get_positive_skill_area( 60 | mean_value_in_training=MEAN_VALUE_IN_TRAINING, 61 | min_value_in_plot=0., max_value_in_plot=1. 62 | ) 63 | 64 | self.assertTrue(numpy.allclose( 65 | these_x_coords_left, X_COORDS_LEFT, atol=TOLERANCE 66 | )) 67 | self.assertTrue(numpy.allclose( 68 | these_y_coords_left, Y_COORDS_LEFT, atol=TOLERANCE 69 | )) 70 | self.assertTrue(numpy.allclose( 71 | these_x_coords_right, X_COORDS_RIGHT, atol=TOLERANCE 72 | )) 73 | self.assertTrue(numpy.allclose( 74 | these_y_coords_right, Y_COORDS_RIGHT, atol=TOLERANCE 75 | )) 76 | 77 | def test_get_zero_skill_line(self): 78 | """Ensures correct output from _get_zero_skill_line.""" 79 | 80 | these_x_coords, these_y_coords = eval_plotting._get_zero_skill_line( 81 | mean_value_in_training=MEAN_VALUE_IN_TRAINING, 82 | min_value_in_plot=0., max_value_in_plot=1. 83 | ) 84 | 85 | self.assertTrue(numpy.allclose( 86 | these_x_coords, X_COORDS_NO_SKILL, atol=TOLERANCE 87 | )) 88 | self.assertTrue(numpy.allclose( 89 | these_y_coords, Y_COORDS_NO_SKILL, atol=TOLERANCE 90 | )) 91 | 92 | def test_get_pofd_pod_grid(self): 93 | """Ensures correct output from _get_pofd_pod_grid.""" 94 | 95 | this_pofd_matrix, this_pod_matrix = eval_plotting._get_pofd_pod_grid( 96 | pofd_spacing=POFD_GRID_SPACING, pod_spacing=POD_GRID_SPACING 97 | ) 98 | 99 | self.assertTrue(numpy.allclose( 100 | this_pofd_matrix, POFD_MATRIX, atol=TOLERANCE 101 | )) 102 | self.assertTrue(numpy.allclose( 103 | this_pod_matrix, POD_MATRIX, atol=TOLERANCE 104 | )) 105 | 106 | def test_get_sr_pod_grid(self): 107 | """Ensures correct output from _get_sr_pod_grid.""" 108 | 109 | this_success_ratio_matrix, this_pod_matrix = ( 110 | eval_plotting._get_sr_pod_grid( 111 | success_ratio_spacing=SUCCESS_RATIO_GRID_SPACING, 112 | pod_spacing=POD_GRID_SPACING 113 | ) 114 | ) 115 | 116 | self.assertTrue(numpy.allclose( 117 | this_success_ratio_matrix, SUCCESS_RATIO_MATRIX, atol=TOLERANCE 118 | )) 119 | self.assertTrue(numpy.allclose( 120 | this_pod_matrix, POD_MATRIX, atol=TOLERANCE 121 | )) 122 | 123 | def test_csi_from_sr_and_pod(self): 124 | """Ensures correct output from _csi_from_sr_and_pod.""" 125 | 126 | this_csi_matrix = eval_plotting._csi_from_sr_and_pod( 127 | success_ratio_array=SUCCESS_RATIO_MATRIX, pod_array=POD_MATRIX 128 | ) 129 | 130 | self.assertTrue(numpy.allclose( 131 | this_csi_matrix, CSI_MATRIX, atol=TOLERANCE 132 | )) 133 | 134 | def test_bias_from_sr_and_pod(self): 135 | """Ensures correct output from _bias_from_sr_and_pod.""" 136 | 137 | this_bias_matrix = eval_plotting._bias_from_sr_and_pod( 138 | success_ratio_array=SUCCESS_RATIO_MATRIX, pod_array=POD_MATRIX 139 | ) 140 | 141 | self.assertTrue(numpy.allclose( 142 | this_bias_matrix, FREQUENCY_BIAS_MATRIX, atol=TOLERANCE 143 | )) 144 | 145 | 146 | if __name__ == '__main__': 147 | unittest.main() 148 | -------------------------------------------------------------------------------- /ml4convection/plotting/new_satellite_plotting.py: -------------------------------------------------------------------------------- 1 | """Plotting methods for satellite data.""" 2 | 3 | import numpy 4 | import matplotlib 5 | matplotlib.use('agg') 6 | from gewittergefahr.gg_utils import grids 7 | from gewittergefahr.gg_utils import error_checking 8 | from ml4convection.plotting import satellite_plotting 9 | 10 | 11 | def _grid_points_to_edges(grid_point_coords): 12 | """Converts grid-point coordinates to grid-cell-edge coordinates. 13 | 14 | P = number of grid points 15 | 16 | :param grid_point_coords: length-P numpy array of grid-point coordinates, in 17 | increasing order. 18 | :return: grid_cell_edge_coords: length-(P + 1) numpy array of grid-cell-edge 19 | coordinates, also in increasing order. 20 | """ 21 | 22 | grid_cell_edge_coords = (grid_point_coords[:-1] + grid_point_coords[1:]) / 2 23 | first_edge_coords = ( 24 | grid_point_coords[0] - numpy.diff(grid_point_coords[:2]) / 2 25 | ) 26 | last_edge_coords = ( 27 | grid_point_coords[-1] + numpy.diff(grid_point_coords[-2:]) / 2 28 | ) 29 | 30 | return numpy.concatenate(( 31 | first_edge_coords, grid_cell_edge_coords, last_edge_coords 32 | )) 33 | 34 | 35 | def plot_2d_grid_regular( 36 | brightness_temp_matrix_kelvins, axes_object, latitudes_deg_n, 37 | longitudes_deg_e, cbar_orientation_string='vertical', font_size=30.): 38 | """Plots brightness temperature on 2-D grid. 39 | 40 | M = number of rows in grid 41 | N = number of columns in grid 42 | 43 | :param brightness_temp_matrix_kelvins: M-by-N numpy array of brightness 44 | temperatures. 45 | :param axes_object: Will plot on these axes (instance of 46 | `matplotlib.axes._subplots.AxesSubplot`). 47 | :param latitudes_deg_n: length-M numpy array of grid-point latitudes (deg 48 | north). 49 | :param longitudes_deg_e: length-N numpy array of grid-point longitudes (deg 50 | east). 51 | :param cbar_orientation_string: Colour-bar orientation. May be 52 | "horizontal", "vertical", or None. 53 | :param font_size: Font size. 54 | :return: colour_bar_object: Colour-bar handle (instance of 55 | `matplotlib.pyplot.colorbar`). 56 | """ 57 | 58 | error_checking.assert_is_numpy_array(latitudes_deg_n, num_dimensions=1) 59 | error_checking.assert_is_valid_lat_numpy_array(latitudes_deg_n) 60 | error_checking.assert_is_greater_numpy_array( 61 | numpy.diff(latitudes_deg_n), 0. 62 | ) 63 | 64 | error_checking.assert_is_numpy_array(longitudes_deg_e, num_dimensions=1) 65 | error_checking.assert_is_valid_lng_numpy_array( 66 | longitudes_deg_e, positive_in_west_flag=True 67 | ) 68 | error_checking.assert_is_greater_numpy_array( 69 | numpy.diff(longitudes_deg_e), 0. 70 | ) 71 | 72 | num_rows = len(latitudes_deg_n) 73 | num_columns = len(longitudes_deg_e) 74 | expected_dim = numpy.array([num_rows, num_columns], dtype=int) 75 | 76 | error_checking.assert_is_numpy_array( 77 | brightness_temp_matrix_kelvins, exact_dimensions=expected_dim 78 | ) 79 | error_checking.assert_is_greater_numpy_array( 80 | brightness_temp_matrix_kelvins, 0., allow_nan=True 81 | ) 82 | 83 | if cbar_orientation_string is not None: 84 | error_checking.assert_is_string(cbar_orientation_string) 85 | 86 | edge_latitudes_deg_n = _grid_points_to_edges(latitudes_deg_n) 87 | edge_longitudes_deg_e = _grid_points_to_edges(longitudes_deg_e) 88 | edge_temp_matrix_kelvins = grids.latlng_field_grid_points_to_edges( 89 | field_matrix=brightness_temp_matrix_kelvins, 90 | min_latitude_deg=1., min_longitude_deg=1., 91 | lat_spacing_deg=1e-6, lng_spacing_deg=1e-6 92 | )[0] 93 | 94 | edge_temp_matrix_kelvins = numpy.ma.masked_where( 95 | numpy.isnan(edge_temp_matrix_kelvins), edge_temp_matrix_kelvins 96 | ) 97 | colour_map_object, colour_norm_object = ( 98 | satellite_plotting.get_colour_scheme() 99 | ) 100 | 101 | if hasattr(colour_norm_object, 'boundaries'): 102 | min_colour_value = colour_norm_object.boundaries[0] 103 | max_colour_value = colour_norm_object.boundaries[-1] 104 | else: 105 | min_colour_value = colour_norm_object.vmin 106 | max_colour_value = colour_norm_object.vmax 107 | 108 | axes_object.pcolormesh( 109 | edge_longitudes_deg_e, edge_latitudes_deg_n, edge_temp_matrix_kelvins, 110 | cmap=colour_map_object, norm=colour_norm_object, 111 | vmin=min_colour_value, vmax=max_colour_value, shading='flat', 112 | edgecolors='None', zorder=-1e11 113 | ) 114 | 115 | if cbar_orientation_string is None: 116 | return None 117 | 118 | return satellite_plotting.add_colour_bar( 119 | brightness_temp_matrix_kelvins=brightness_temp_matrix_kelvins, 120 | axes_object=axes_object, colour_map_object=colour_map_object, 121 | colour_norm_object=colour_norm_object, 122 | orientation_string=cbar_orientation_string, font_size=font_size 123 | ) 124 | -------------------------------------------------------------------------------- /ml4convection/plotting/new_satellite_plotting_test.py: -------------------------------------------------------------------------------- 1 | """Unit tests for new_satellite_plotting.py.""" 2 | 3 | import unittest 4 | import numpy 5 | from ml4convection.plotting import new_satellite_plotting 6 | 7 | TOLERANCE = 1e-6 8 | CENTER_LATITUDES_DEG_N = numpy.array([30, 35, 42, 47], dtype=float) 9 | EDGE_LATITUDES_DEG_N = numpy.array([27.5, 32.5, 38.5, 44.5, 49.5]) 10 | 11 | 12 | class NewSatellitePlottingTests(unittest.TestCase): 13 | """Each method is a unit test for new_satellite_plotting.py.""" 14 | 15 | def test_grid_points_to_edges(self): 16 | """Ensures correct output from _grid_points_to_edges.""" 17 | 18 | these_latitudes_deg_n = new_satellite_plotting._grid_points_to_edges( 19 | CENTER_LATITUDES_DEG_N 20 | ) 21 | self.assertTrue(numpy.allclose( 22 | these_latitudes_deg_n, EDGE_LATITUDES_DEG_N, atol=TOLERANCE 23 | )) 24 | 25 | 26 | if __name__ == '__main__': 27 | unittest.main() 28 | -------------------------------------------------------------------------------- /ml4convection/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thunderhoser/ml4convection/cf5e9682bc3182305274132ae246bc72d994308f/ml4convection/scripts/__init__.py -------------------------------------------------------------------------------- /ml4convection/scripts/delete_basic_scores.py: -------------------------------------------------------------------------------- 1 | """Deletes files with basic scores.""" 2 | 3 | import os 4 | import argparse 5 | from ml4convection.utils import evaluation 6 | from ml4convection.utils import radar_utils 7 | 8 | NUM_RADARS = len(radar_utils.RADAR_LATITUDES_DEG_N) 9 | 10 | EVALUATION_DIR_ARG_NAME = 'evaluation_dir_name' 11 | FIRST_DATE_ARG_NAME = 'first_date_string' 12 | LAST_DATE_ARG_NAME = 'last_date_string' 13 | GRIDDED_ARG_NAME = 'gridded' 14 | PARTIAL_GRIDS_ARG_NAME = 'use_partial_grids' 15 | 16 | EVALUATION_DIR_HELP_STRING = ( 17 | 'Name of top-level directory with score files. Files therein will be ' 18 | 'found by `evaluation.find_basic_score_file` and read by ' 19 | '`evaluation.read_basic_score_file`.' 20 | ) 21 | DATE_HELP_STRING = ( 22 | 'Date (format "yyyymmdd"). Will delete score files for the period ' 23 | '`{0:s}`...`{1:s}`.' 24 | ).format(FIRST_DATE_ARG_NAME, LAST_DATE_ARG_NAME) 25 | 26 | GRIDDED_HELP_STRING = ( 27 | 'Boolean flag. If 1 (0), will delete files with (un)gridded scores.' 28 | ) 29 | PARTIAL_GRIDS_HELP_STRING = ( 30 | 'Boolean flag. If 1 (0), will delete files on partial grids (full grid).' 31 | ) 32 | 33 | INPUT_ARG_PARSER = argparse.ArgumentParser() 34 | INPUT_ARG_PARSER.add_argument( 35 | '--' + EVALUATION_DIR_ARG_NAME, type=str, required=True, 36 | help=EVALUATION_DIR_HELP_STRING 37 | ) 38 | INPUT_ARG_PARSER.add_argument( 39 | '--' + FIRST_DATE_ARG_NAME, type=str, required=True, help=DATE_HELP_STRING 40 | ) 41 | INPUT_ARG_PARSER.add_argument( 42 | '--' + LAST_DATE_ARG_NAME, type=str, required=True, help=DATE_HELP_STRING 43 | ) 44 | INPUT_ARG_PARSER.add_argument( 45 | '--' + GRIDDED_ARG_NAME, type=int, required=True, help=GRIDDED_HELP_STRING 46 | ) 47 | INPUT_ARG_PARSER.add_argument( 48 | '--' + PARTIAL_GRIDS_ARG_NAME, type=int, required=True, 49 | help=PARTIAL_GRIDS_HELP_STRING 50 | ) 51 | 52 | 53 | def _run(top_evaluation_dir_name, first_date_string, last_date_string, gridded, 54 | use_partial_grids): 55 | """Deletes files with basic gridded scores. 56 | 57 | This is effectively the main method. 58 | 59 | :param top_evaluation_dir_name: See documentation at top of file. 60 | :param first_date_string: Same. 61 | :param last_date_string: Same. 62 | :param gridded: Same. 63 | :param use_partial_grids: Same. 64 | """ 65 | 66 | if use_partial_grids: 67 | date_strings = [] 68 | basic_score_file_names = [] 69 | 70 | for k in range(NUM_RADARS): 71 | if len(date_strings) == 0: 72 | basic_score_file_names += ( 73 | evaluation.find_many_basic_score_files( 74 | top_directory_name=top_evaluation_dir_name, 75 | first_date_string=first_date_string, 76 | last_date_string=last_date_string, 77 | gridded=gridded, radar_number=k, 78 | raise_error_if_any_missing=False, 79 | raise_error_if_all_missing=k > 0 80 | ) 81 | ) 82 | 83 | if len(basic_score_file_names) > 0: 84 | date_strings = [ 85 | evaluation.basic_file_name_to_date(f) 86 | for f in basic_score_file_names 87 | ] 88 | else: 89 | basic_score_file_names += [ 90 | evaluation.find_basic_score_file( 91 | top_directory_name=top_evaluation_dir_name, 92 | valid_date_string=d, gridded=gridded, radar_number=k, 93 | raise_error_if_missing=True 94 | ) for d in date_strings 95 | ] 96 | else: 97 | basic_score_file_names = evaluation.find_many_basic_score_files( 98 | top_directory_name=top_evaluation_dir_name, 99 | first_date_string=first_date_string, 100 | last_date_string=last_date_string, 101 | gridded=gridded, radar_number=None, raise_error_if_all_missing=True 102 | ) 103 | 104 | for this_file_name in basic_score_file_names: 105 | print('Deleting file: "{0:s}"...'.format(this_file_name)) 106 | os.remove(this_file_name) 107 | 108 | 109 | if __name__ == '__main__': 110 | INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args() 111 | 112 | _run( 113 | top_evaluation_dir_name=getattr( 114 | INPUT_ARG_OBJECT, EVALUATION_DIR_ARG_NAME 115 | ), 116 | first_date_string=getattr(INPUT_ARG_OBJECT, FIRST_DATE_ARG_NAME), 117 | last_date_string=getattr(INPUT_ARG_OBJECT, LAST_DATE_ARG_NAME), 118 | gridded=bool(getattr(INPUT_ARG_OBJECT, GRIDDED_ARG_NAME)), 119 | use_partial_grids=bool( 120 | getattr(INPUT_ARG_OBJECT, PARTIAL_GRIDS_ARG_NAME) 121 | ) 122 | ) 123 | -------------------------------------------------------------------------------- /ml4convection/scripts/delete_prediction_files.py: -------------------------------------------------------------------------------- 1 | """Deletes prediction files.""" 2 | 3 | import os 4 | import argparse 5 | from ml4convection.io import prediction_io 6 | from ml4convection.utils import radar_utils 7 | 8 | NUM_RADARS = len(radar_utils.RADAR_LATITUDES_DEG_N) 9 | 10 | PREDICTION_DIR_ARG_NAME = 'prediction_dir_name' 11 | FIRST_DATE_ARG_NAME = 'first_date_string' 12 | LAST_DATE_ARG_NAME = 'last_date_string' 13 | PARTIAL_GRIDS_ARG_NAME = 'use_partial_grids' 14 | 15 | PREDICTION_DIR_HELP_STRING = ( 16 | 'Name of top-level directory with prediction files. Files therein will be ' 17 | 'found by `prediction_io.find_file` and read by `prediction_io.read_file`.' 18 | ) 19 | DATE_HELP_STRING = ( 20 | 'Date (format "yyyymmdd"). Will delete prediction files for the period ' 21 | '`{0:s}`...`{1:s}`.' 22 | ).format(FIRST_DATE_ARG_NAME, LAST_DATE_ARG_NAME) 23 | 24 | PARTIAL_GRIDS_HELP_STRING = ( 25 | 'Boolean flag. If 1 (0), will delete files with predictions on partial ' 26 | '(full) grid.' 27 | ) 28 | 29 | INPUT_ARG_PARSER = argparse.ArgumentParser() 30 | INPUT_ARG_PARSER.add_argument( 31 | '--' + PREDICTION_DIR_ARG_NAME, type=str, required=True, 32 | help=PREDICTION_DIR_HELP_STRING 33 | ) 34 | INPUT_ARG_PARSER.add_argument( 35 | '--' + FIRST_DATE_ARG_NAME, type=str, required=True, help=DATE_HELP_STRING 36 | ) 37 | INPUT_ARG_PARSER.add_argument( 38 | '--' + LAST_DATE_ARG_NAME, type=str, required=True, help=DATE_HELP_STRING 39 | ) 40 | INPUT_ARG_PARSER.add_argument( 41 | '--' + PARTIAL_GRIDS_ARG_NAME, type=int, required=True, 42 | help=PARTIAL_GRIDS_HELP_STRING 43 | ) 44 | 45 | 46 | def _run(top_prediction_dir_name, first_date_string, last_date_string, 47 | use_partial_grids): 48 | """Deletes prediction files. 49 | 50 | This is effectively the main method. 51 | 52 | :param top_prediction_dir_name: See documentation at top of file. 53 | :param first_date_string: Same. 54 | :param last_date_string: Same. 55 | :param use_partial_grids: Same. 56 | """ 57 | 58 | prediction_file_names = [] 59 | 60 | if not use_partial_grids: 61 | prediction_file_names += prediction_io.find_many_files( 62 | top_directory_name=top_prediction_dir_name, 63 | first_date_string=first_date_string, 64 | last_date_string=last_date_string, 65 | prefer_zipped=True, allow_other_format=False, radar_number=None, 66 | raise_error_if_any_missing=False, raise_error_if_all_missing=False 67 | ) 68 | 69 | prediction_file_names += prediction_io.find_many_files( 70 | top_directory_name=top_prediction_dir_name, 71 | first_date_string=first_date_string, 72 | last_date_string=last_date_string, 73 | prefer_zipped=False, allow_other_format=False, radar_number=None, 74 | raise_error_if_any_missing=False, 75 | raise_error_if_all_missing=len(prediction_file_names) == 0 76 | ) 77 | else: 78 | for k in range(NUM_RADARS): 79 | prediction_file_names += prediction_io.find_many_files( 80 | top_directory_name=top_prediction_dir_name, 81 | first_date_string=first_date_string, 82 | last_date_string=last_date_string, 83 | prefer_zipped=True, allow_other_format=False, radar_number=k, 84 | raise_error_if_any_missing=False, 85 | raise_error_if_all_missing=False 86 | ) 87 | 88 | this_flag = len(prediction_file_names) == 0 and k == NUM_RADARS - 1 89 | 90 | prediction_file_names += prediction_io.find_many_files( 91 | top_directory_name=top_prediction_dir_name, 92 | first_date_string=first_date_string, 93 | last_date_string=last_date_string, 94 | prefer_zipped=False, allow_other_format=False, radar_number=k, 95 | raise_error_if_any_missing=False, 96 | raise_error_if_all_missing=this_flag 97 | ) 98 | 99 | for this_file_name in prediction_file_names: 100 | print('Deleting file: "{0:s}"...'.format(this_file_name)) 101 | os.remove(this_file_name) 102 | 103 | 104 | if __name__ == '__main__': 105 | INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args() 106 | 107 | _run( 108 | top_prediction_dir_name=getattr( 109 | INPUT_ARG_OBJECT, PREDICTION_DIR_ARG_NAME 110 | ), 111 | first_date_string=getattr(INPUT_ARG_OBJECT, FIRST_DATE_ARG_NAME), 112 | last_date_string=getattr(INPUT_ARG_OBJECT, LAST_DATE_ARG_NAME), 113 | use_partial_grids=bool( 114 | getattr(INPUT_ARG_OBJECT, PARTIAL_GRIDS_ARG_NAME) 115 | ) 116 | ) 117 | -------------------------------------------------------------------------------- /ml4convection/scripts/get_normalization_params.py: -------------------------------------------------------------------------------- 1 | """Finds normalization parameters for satellite variables (predictors).""" 2 | 3 | import argparse 4 | from ml4convection.utils import normalization 5 | 6 | SEPARATOR_STRING = '\n\n' + '*' * 50 + '\n\n' 7 | 8 | SATELLITE_DIR_ARG_NAME = 'input_satellite_dir_name' 9 | FIRST_DATE_ARG_NAME = 'first_date_string' 10 | LAST_DATE_ARG_NAME = 'last_date_string' 11 | NUM_VALUES_PER_BAND_ARG_NAME = 'num_values_per_band' 12 | OUTPUT_FILE_ARG_NAME = 'output_file_name' 13 | 14 | SATELLITE_DIR_HELP_STRING = ( 15 | 'Name of top-level directory with satellite data. Files therein will be ' 16 | 'found by `satellite_io.find_file` and read by `satellite_io.read_file`.' 17 | ) 18 | DATE_HELP_STRING = ( 19 | 'Date (format "yyyymmdd"). Normalization parameters will be based on data ' 20 | 'from the period `{0:s}`...`{1:s}`.' 21 | ).format(FIRST_DATE_ARG_NAME, LAST_DATE_ARG_NAME) 22 | 23 | NUM_VALUES_PER_BAND_HELP_STRING = ( 24 | 'Number of values to save from each spectral band. These will be randomly ' 25 | 'sampled.' 26 | ) 27 | OUTPUT_FILE_HELP_STRING = ( 28 | 'Path to output file. Normalization parameters will be written here by ' 29 | '`normalization.write_file`.' 30 | ) 31 | 32 | INPUT_ARG_PARSER = argparse.ArgumentParser() 33 | INPUT_ARG_PARSER.add_argument( 34 | '--' + SATELLITE_DIR_ARG_NAME, type=str, required=True, 35 | help=SATELLITE_DIR_HELP_STRING 36 | ) 37 | INPUT_ARG_PARSER.add_argument( 38 | '--' + FIRST_DATE_ARG_NAME, type=str, required=True, help=DATE_HELP_STRING 39 | ) 40 | INPUT_ARG_PARSER.add_argument( 41 | '--' + LAST_DATE_ARG_NAME, type=str, required=True, help=DATE_HELP_STRING 42 | ) 43 | INPUT_ARG_PARSER.add_argument( 44 | '--' + NUM_VALUES_PER_BAND_ARG_NAME, type=int, required=False, default=2e5, 45 | help=NUM_VALUES_PER_BAND_HELP_STRING 46 | ) 47 | INPUT_ARG_PARSER.add_argument( 48 | '--' + OUTPUT_FILE_ARG_NAME, type=str, required=True, 49 | help=OUTPUT_FILE_HELP_STRING 50 | ) 51 | 52 | 53 | def _run(top_satellite_dir_name, first_date_string, last_date_string, 54 | num_values_per_band, output_file_name): 55 | """Finds normalization parameters for satellite variables (predictors). 56 | 57 | This is effectively the main method. 58 | 59 | :param top_satellite_dir_name: See documentation at top of file. 60 | :param first_date_string: Same. 61 | :param last_date_string: Same. 62 | :param num_values_per_band: Same. 63 | :param output_file_name: Same. 64 | """ 65 | 66 | normalization_dict = normalization.get_normalization_params( 67 | top_satellite_dir_name=top_satellite_dir_name, 68 | first_date_string=first_date_string, 69 | last_date_string=last_date_string, 70 | num_values_per_band=num_values_per_band 71 | ) 72 | 73 | print(SEPARATOR_STRING) 74 | print('Writing normalization parameters to: "{0:s}"...'.format( 75 | output_file_name 76 | )) 77 | 78 | normalization.write_file( 79 | pickle_file_name=output_file_name, normalization_dict=normalization_dict 80 | ) 81 | 82 | 83 | if __name__ == '__main__': 84 | INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args() 85 | 86 | _run( 87 | top_satellite_dir_name=getattr( 88 | INPUT_ARG_OBJECT, SATELLITE_DIR_ARG_NAME 89 | ), 90 | first_date_string=getattr(INPUT_ARG_OBJECT, FIRST_DATE_ARG_NAME), 91 | last_date_string=getattr(INPUT_ARG_OBJECT, LAST_DATE_ARG_NAME), 92 | num_values_per_band=getattr( 93 | INPUT_ARG_OBJECT, NUM_VALUES_PER_BAND_ARG_NAME 94 | ), 95 | output_file_name=getattr(INPUT_ARG_OBJECT, OUTPUT_FILE_ARG_NAME) 96 | ) 97 | -------------------------------------------------------------------------------- /ml4convection/scripts/make_best_architecture_0minutes.py: -------------------------------------------------------------------------------- 1 | """Makes architecture for best 0-minute U-net in journal paper.""" 2 | 3 | import os 4 | import copy 5 | import numpy 6 | from gewittergefahr.deep_learning import architecture_utils 7 | from ml4convection.io import radar_io 8 | from ml4convection.machine_learning import neural_net 9 | from ml4convection.machine_learning import u_net_architecture 10 | 11 | # Dimensions of predictors (satellite images). 12 | NUM_GRID_ROWS = 205 13 | NUM_GRID_COLUMNS = 205 14 | NUM_SPECTRAL_BANDS = 7 15 | 16 | # Best L_2 weight and number of lag times. The exact lag times are {0, 10, 20} 17 | # minutes, but these are not specified in the architecture; they are specified 18 | # only at training time, when feeding the input data to the model. 19 | L2_WEIGHT = 1e-6 20 | NUM_LAG_TIMES = 3 21 | 22 | # Number of levels in U-net. 23 | NUM_LEVELS = 5 24 | 25 | # File containing mask, which ensures that only grid points within 100 km of the 26 | # three southernmost radars are used in the loss function. The mask is 27 | # referenced to a radar-centered domain, with 205 x 205 grid points. 28 | THIS_DIRECTORY_NAME = os.path.dirname(os.path.realpath( 29 | os.path.join(os.getcwd(), os.path.expanduser(__file__)) 30 | )) 31 | MASK_FILE_NAME = '{0:s}/radar_mask_100km_omit-north_partial.nc'.format( 32 | THIS_DIRECTORY_NAME 33 | ) 34 | 35 | # Loss function (fractions skill score with 9-by-9 window, or half-window size 36 | # of grid points). 37 | LOSS_FUNCTION_NAME = 'fss_neigh4' 38 | 39 | # Metrics (other evaluation scores tracked during training). 40 | METRIC_NAMES = [ 41 | 'fss_neigh0', 'fss_neigh1', 'fss_neigh2', 'fss_neigh3', 42 | 'csi_neigh0', 'csi_neigh1', 'csi_neigh2', 'csi_neigh3', 43 | 'bias_neigh0', 'bias_neigh1', 'bias_neigh2', 'bias_neigh3', 44 | 'iou_neigh0', 'iou_neigh1', 'iou_neigh2', 'iou_neigh3', 45 | 'dice_neigh0', 'dice_neigh1', 'dice_neigh2', 'dice_neigh3' 46 | ] 47 | 48 | # DEFAULT_OPTION_DICT contains fixed hyperparameters, which were not varied 49 | # during the hyperparameter experiment presented in the paper. 50 | DEFAULT_OPTION_DICT = { 51 | u_net_architecture.NUM_LEVELS_KEY: NUM_LEVELS, 52 | u_net_architecture.NUM_CONV_LAYERS_KEY: 2, 53 | u_net_architecture.CONV_LAYER_CHANNEL_COUNTS_KEY: 54 | numpy.array([16, 24, 32, 48, 64, 96], dtype=int), 55 | u_net_architecture.CONV_LAYER_DROPOUT_RATES_KEY: 56 | numpy.full(NUM_LEVELS + 1, 0.), 57 | u_net_architecture.UPCONV_LAYER_DROPOUT_RATES_KEY: 58 | numpy.full(NUM_LEVELS, 0.), 59 | u_net_architecture.SKIP_LAYER_DROPOUT_RATES_KEY: 60 | numpy.full(NUM_LEVELS, 0.), 61 | u_net_architecture.INNER_ACTIV_FUNCTION_KEY: 62 | architecture_utils.RELU_FUNCTION_STRING, 63 | u_net_architecture.INNER_ACTIV_FUNCTION_ALPHA_KEY: 0.2, 64 | u_net_architecture.OUTPUT_ACTIV_FUNCTION_KEY: 65 | architecture_utils.SIGMOID_FUNCTION_STRING, 66 | u_net_architecture.OUTPUT_ACTIV_FUNCTION_ALPHA_KEY: 0., 67 | u_net_architecture.L1_WEIGHT_KEY: 0., 68 | u_net_architecture.USE_BATCH_NORM_KEY: True 69 | } 70 | 71 | 72 | def _run(): 73 | """Makes architecture for best 0-minute U-net in journal paper. 74 | 75 | This is effectively the main method. 76 | """ 77 | 78 | # Read mask. 79 | print('Reading partial mask from: "{0:s}"...'.format( 80 | MASK_FILE_NAME 81 | )) 82 | mask_matrix = radar_io.read_mask_file(MASK_FILE_NAME)[ 83 | radar_io.MASK_MATRIX_KEY 84 | ] 85 | 86 | # Create loss function. 87 | loss_function = neural_net.get_metrics( 88 | metric_names=[LOSS_FUNCTION_NAME], mask_matrix=mask_matrix, 89 | use_as_loss_function=True 90 | )[0][0] 91 | 92 | # Assign experimental hyperparameters. 93 | option_dict = copy.deepcopy(DEFAULT_OPTION_DICT) 94 | option_dict[u_net_architecture.L2_WEIGHT_KEY] = L2_WEIGHT 95 | option_dict[u_net_architecture.INPUT_DIMENSIONS_KEY] = numpy.array([ 96 | NUM_GRID_ROWS, NUM_GRID_COLUMNS, NUM_LAG_TIMES * NUM_SPECTRAL_BANDS 97 | ], dtype=int) 98 | 99 | # This next command creates the U-net architecture and compiles the U-net. 100 | # Thus, it returns a U-net that is completely ready to train. It also 101 | # prints a very long flow chart, containing details on each layer in the 102 | # model. I have not included this flow chart in the paper, because it is 103 | # difficult to read and I feel that Figure 5 does a better job of 104 | # documenting the architecture. 105 | 106 | model_object = u_net_architecture.create_model( 107 | option_dict=option_dict, loss_function=loss_function, 108 | mask_matrix=mask_matrix, metric_names=METRIC_NAMES 109 | ) 110 | 111 | 112 | if __name__ == '__main__': 113 | _run() 114 | -------------------------------------------------------------------------------- /ml4convection/scripts/make_best_architecture_120minutes.py: -------------------------------------------------------------------------------- 1 | """Makes architecture for best 120-minute U-net in journal paper.""" 2 | 3 | import os 4 | import copy 5 | import numpy 6 | from gewittergefahr.deep_learning import architecture_utils 7 | from ml4convection.io import radar_io 8 | from ml4convection.machine_learning import neural_net 9 | from ml4convection.machine_learning import u_net_architecture 10 | 11 | # Dimensions of predictors (satellite images). 12 | NUM_GRID_ROWS = 205 13 | NUM_GRID_COLUMNS = 205 14 | NUM_SPECTRAL_BANDS = 7 15 | 16 | # Best L_2 weight and number of lag times. The exact lag times are {0, 20} 17 | # minutes, but these are not specified in the architecture; they are specified 18 | # only at training time, when feeding the input data to the model. 19 | L2_WEIGHT = 10 ** -5.5 20 | NUM_LAG_TIMES = 2 21 | 22 | # Number of levels in U-net. 23 | NUM_LEVELS = 5 24 | 25 | # File containing mask, which ensures that only grid points within 100 km of the 26 | # three southernmost radars are used in the loss function. The mask is 27 | # referenced to a radar-centered domain, with 205 x 205 grid points. 28 | THIS_DIRECTORY_NAME = os.path.dirname(os.path.realpath( 29 | os.path.join(os.getcwd(), os.path.expanduser(__file__)) 30 | )) 31 | MASK_FILE_NAME = '{0:s}/radar_mask_100km_omit-north_partial.nc'.format( 32 | THIS_DIRECTORY_NAME 33 | ) 34 | 35 | # Loss function (fractions skill score with 9-by-9 window, or half-window size 36 | # of grid points). 37 | LOSS_FUNCTION_NAME = 'fss_neigh4' 38 | 39 | # Metrics (other evaluation scores tracked during training). 40 | METRIC_NAMES = [ 41 | 'fss_neigh0', 'fss_neigh1', 'fss_neigh2', 'fss_neigh3', 42 | 'csi_neigh0', 'csi_neigh1', 'csi_neigh2', 'csi_neigh3', 43 | 'bias_neigh0', 'bias_neigh1', 'bias_neigh2', 'bias_neigh3', 44 | 'iou_neigh0', 'iou_neigh1', 'iou_neigh2', 'iou_neigh3', 45 | 'dice_neigh0', 'dice_neigh1', 'dice_neigh2', 'dice_neigh3' 46 | ] 47 | 48 | # DEFAULT_OPTION_DICT contains fixed hyperparameters, which were not varied 49 | # during the hyperparameter experiment presented in the paper. 50 | DEFAULT_OPTION_DICT = { 51 | u_net_architecture.NUM_LEVELS_KEY: NUM_LEVELS, 52 | u_net_architecture.NUM_CONV_LAYERS_KEY: 2, 53 | u_net_architecture.CONV_LAYER_CHANNEL_COUNTS_KEY: 54 | numpy.array([16, 24, 32, 48, 64, 96], dtype=int), 55 | u_net_architecture.CONV_LAYER_DROPOUT_RATES_KEY: 56 | numpy.full(NUM_LEVELS + 1, 0.), 57 | u_net_architecture.UPCONV_LAYER_DROPOUT_RATES_KEY: 58 | numpy.full(NUM_LEVELS, 0.), 59 | u_net_architecture.SKIP_LAYER_DROPOUT_RATES_KEY: 60 | numpy.full(NUM_LEVELS, 0.), 61 | u_net_architecture.INNER_ACTIV_FUNCTION_KEY: 62 | architecture_utils.RELU_FUNCTION_STRING, 63 | u_net_architecture.INNER_ACTIV_FUNCTION_ALPHA_KEY: 0.2, 64 | u_net_architecture.OUTPUT_ACTIV_FUNCTION_KEY: 65 | architecture_utils.SIGMOID_FUNCTION_STRING, 66 | u_net_architecture.OUTPUT_ACTIV_FUNCTION_ALPHA_KEY: 0., 67 | u_net_architecture.L1_WEIGHT_KEY: 0., 68 | u_net_architecture.USE_BATCH_NORM_KEY: True 69 | } 70 | 71 | 72 | def _run(): 73 | """Makes architecture for best 120-minute U-net in journal paper. 74 | 75 | This is effectively the main method. 76 | """ 77 | 78 | # Read mask. 79 | print('Reading partial mask from: "{0:s}"...'.format( 80 | MASK_FILE_NAME 81 | )) 82 | mask_matrix = radar_io.read_mask_file(MASK_FILE_NAME)[ 83 | radar_io.MASK_MATRIX_KEY 84 | ] 85 | 86 | # Create loss function. 87 | loss_function = neural_net.get_metrics( 88 | metric_names=[LOSS_FUNCTION_NAME], mask_matrix=mask_matrix, 89 | use_as_loss_function=True 90 | )[0][0] 91 | 92 | # Assign experimental hyperparameters. 93 | option_dict = copy.deepcopy(DEFAULT_OPTION_DICT) 94 | option_dict[u_net_architecture.L2_WEIGHT_KEY] = L2_WEIGHT 95 | option_dict[u_net_architecture.INPUT_DIMENSIONS_KEY] = numpy.array([ 96 | NUM_GRID_ROWS, NUM_GRID_COLUMNS, NUM_LAG_TIMES * NUM_SPECTRAL_BANDS 97 | ], dtype=int) 98 | 99 | # This next command creates the U-net architecture and compiles the U-net. 100 | # Thus, it returns a U-net that is completely ready to train. It also 101 | # prints a very long flow chart, containing details on each layer in the 102 | # model. I have not included this flow chart in the paper, because it is 103 | # difficult to read and I feel that Figure 5 does a better job of 104 | # documenting the architecture. 105 | 106 | model_object = u_net_architecture.create_model( 107 | option_dict=option_dict, loss_function=loss_function, 108 | mask_matrix=mask_matrix, metric_names=METRIC_NAMES 109 | ) 110 | 111 | 112 | if __name__ == '__main__': 113 | _run() 114 | -------------------------------------------------------------------------------- /ml4convection/scripts/make_best_architecture_30minutes.py: -------------------------------------------------------------------------------- 1 | """Makes architecture for best 30-minute U-net in journal paper.""" 2 | 3 | import os 4 | import copy 5 | import numpy 6 | from gewittergefahr.deep_learning import architecture_utils 7 | from ml4convection.io import radar_io 8 | from ml4convection.machine_learning import neural_net 9 | from ml4convection.machine_learning import u_net_architecture 10 | 11 | # Dimensions of predictors (satellite images). 12 | NUM_GRID_ROWS = 205 13 | NUM_GRID_COLUMNS = 205 14 | NUM_SPECTRAL_BANDS = 7 15 | 16 | # Best L_2 weight and number of lag times. The exact lag times are 17 | # {0, 20, 40, 60} minutes, but these are not specified in the architecture; they 18 | # are specified only at training time, when feeding the input data to the model. 19 | L2_WEIGHT = 10 ** -5.5 20 | NUM_LAG_TIMES = 4 21 | 22 | # Number of levels in U-net. 23 | NUM_LEVELS = 5 24 | 25 | # File containing mask, which ensures that only grid points within 100 km of the 26 | # three southernmost radars are used in the loss function. The mask is 27 | # referenced to a radar-centered domain, with 205 x 205 grid points. 28 | THIS_DIRECTORY_NAME = os.path.dirname(os.path.realpath( 29 | os.path.join(os.getcwd(), os.path.expanduser(__file__)) 30 | )) 31 | MASK_FILE_NAME = '{0:s}/radar_mask_100km_omit-north_partial.nc'.format( 32 | THIS_DIRECTORY_NAME 33 | ) 34 | 35 | # Loss function (fractions skill score with 9-by-9 window, or half-window size 36 | # of grid points). 37 | LOSS_FUNCTION_NAME = 'fss_neigh4' 38 | 39 | # Metrics (other evaluation scores tracked during training). 40 | METRIC_NAMES = [ 41 | 'fss_neigh0', 'fss_neigh1', 'fss_neigh2', 'fss_neigh3', 42 | 'csi_neigh0', 'csi_neigh1', 'csi_neigh2', 'csi_neigh3', 43 | 'bias_neigh0', 'bias_neigh1', 'bias_neigh2', 'bias_neigh3', 44 | 'iou_neigh0', 'iou_neigh1', 'iou_neigh2', 'iou_neigh3', 45 | 'dice_neigh0', 'dice_neigh1', 'dice_neigh2', 'dice_neigh3' 46 | ] 47 | 48 | # DEFAULT_OPTION_DICT contains fixed hyperparameters, which were not varied 49 | # during the hyperparameter experiment presented in the paper. 50 | DEFAULT_OPTION_DICT = { 51 | u_net_architecture.NUM_LEVELS_KEY: NUM_LEVELS, 52 | u_net_architecture.NUM_CONV_LAYERS_KEY: 2, 53 | u_net_architecture.CONV_LAYER_CHANNEL_COUNTS_KEY: 54 | numpy.array([16, 24, 32, 48, 64, 96], dtype=int), 55 | u_net_architecture.CONV_LAYER_DROPOUT_RATES_KEY: 56 | numpy.full(NUM_LEVELS + 1, 0.), 57 | u_net_architecture.UPCONV_LAYER_DROPOUT_RATES_KEY: 58 | numpy.full(NUM_LEVELS, 0.), 59 | u_net_architecture.SKIP_LAYER_DROPOUT_RATES_KEY: 60 | numpy.full(NUM_LEVELS, 0.), 61 | u_net_architecture.INNER_ACTIV_FUNCTION_KEY: 62 | architecture_utils.RELU_FUNCTION_STRING, 63 | u_net_architecture.INNER_ACTIV_FUNCTION_ALPHA_KEY: 0.2, 64 | u_net_architecture.OUTPUT_ACTIV_FUNCTION_KEY: 65 | architecture_utils.SIGMOID_FUNCTION_STRING, 66 | u_net_architecture.OUTPUT_ACTIV_FUNCTION_ALPHA_KEY: 0., 67 | u_net_architecture.L1_WEIGHT_KEY: 0., 68 | u_net_architecture.USE_BATCH_NORM_KEY: True 69 | } 70 | 71 | 72 | def _run(): 73 | """Makes architecture for best 30-minute U-net in journal paper. 74 | 75 | This is effectively the main method. 76 | """ 77 | 78 | # Read mask. 79 | print('Reading partial mask from: "{0:s}"...'.format( 80 | MASK_FILE_NAME 81 | )) 82 | mask_matrix = radar_io.read_mask_file(MASK_FILE_NAME)[ 83 | radar_io.MASK_MATRIX_KEY 84 | ] 85 | 86 | # Create loss function. 87 | loss_function = neural_net.get_metrics( 88 | metric_names=[LOSS_FUNCTION_NAME], mask_matrix=mask_matrix, 89 | use_as_loss_function=True 90 | )[0][0] 91 | 92 | # Assign experimental hyperparameters. 93 | option_dict = copy.deepcopy(DEFAULT_OPTION_DICT) 94 | option_dict[u_net_architecture.L2_WEIGHT_KEY] = L2_WEIGHT 95 | option_dict[u_net_architecture.INPUT_DIMENSIONS_KEY] = numpy.array([ 96 | NUM_GRID_ROWS, NUM_GRID_COLUMNS, NUM_LAG_TIMES * NUM_SPECTRAL_BANDS 97 | ], dtype=int) 98 | 99 | # This next command creates the U-net architecture and compiles the U-net. 100 | # Thus, it returns a U-net that is completely ready to train. It also 101 | # prints a very long flow chart, containing details on each layer in the 102 | # model. I have not included this flow chart in the paper, because it is 103 | # difficult to read and I feel that Figure 5 does a better job of 104 | # documenting the architecture. 105 | 106 | model_object = u_net_architecture.create_model( 107 | option_dict=option_dict, loss_function=loss_function, 108 | mask_matrix=mask_matrix, metric_names=METRIC_NAMES 109 | ) 110 | 111 | 112 | if __name__ == '__main__': 113 | _run() 114 | -------------------------------------------------------------------------------- /ml4convection/scripts/make_best_architecture_60minutes.py: -------------------------------------------------------------------------------- 1 | """Makes architecture for best 60-minute U-net in journal paper.""" 2 | 3 | import os 4 | import copy 5 | import numpy 6 | from gewittergefahr.deep_learning import architecture_utils 7 | from ml4convection.io import radar_io 8 | from ml4convection.machine_learning import neural_net 9 | from ml4convection.machine_learning import u_net_architecture 10 | 11 | # Dimensions of predictors (satellite images). 12 | NUM_GRID_ROWS = 205 13 | NUM_GRID_COLUMNS = 205 14 | NUM_SPECTRAL_BANDS = 7 15 | 16 | # Best L_2 weight and number of lag times. The exact lag times are {0, 20, 40} 17 | # minutes, but these are not specified in the architecture; they are specified 18 | # only at training time, when feeding the input data to the model. 19 | L2_WEIGHT = 10 ** -5.5 20 | NUM_LAG_TIMES = 3 21 | 22 | # Number of levels in U-net. 23 | NUM_LEVELS = 5 24 | 25 | # File containing mask, which ensures that only grid points within 100 km of the 26 | # three southernmost radars are used in the loss function. The mask is 27 | # referenced to a radar-centered domain, with 205 x 205 grid points. 28 | THIS_DIRECTORY_NAME = os.path.dirname(os.path.realpath( 29 | os.path.join(os.getcwd(), os.path.expanduser(__file__)) 30 | )) 31 | MASK_FILE_NAME = '{0:s}/radar_mask_100km_omit-north_partial.nc'.format( 32 | THIS_DIRECTORY_NAME 33 | ) 34 | 35 | # Loss function (fractions skill score with 9-by-9 window, or half-window size 36 | # of grid points). 37 | LOSS_FUNCTION_NAME = 'fss_neigh4' 38 | 39 | # Metrics (other evaluation scores tracked during training). 40 | METRIC_NAMES = [ 41 | 'fss_neigh0', 'fss_neigh1', 'fss_neigh2', 'fss_neigh3', 42 | 'csi_neigh0', 'csi_neigh1', 'csi_neigh2', 'csi_neigh3', 43 | 'bias_neigh0', 'bias_neigh1', 'bias_neigh2', 'bias_neigh3', 44 | 'iou_neigh0', 'iou_neigh1', 'iou_neigh2', 'iou_neigh3', 45 | 'dice_neigh0', 'dice_neigh1', 'dice_neigh2', 'dice_neigh3' 46 | ] 47 | 48 | # DEFAULT_OPTION_DICT contains fixed hyperparameters, which were not varied 49 | # during the hyperparameter experiment presented in the paper. 50 | DEFAULT_OPTION_DICT = { 51 | u_net_architecture.NUM_LEVELS_KEY: NUM_LEVELS, 52 | u_net_architecture.NUM_CONV_LAYERS_KEY: 2, 53 | u_net_architecture.CONV_LAYER_CHANNEL_COUNTS_KEY: 54 | numpy.array([16, 24, 32, 48, 64, 96], dtype=int), 55 | u_net_architecture.CONV_LAYER_DROPOUT_RATES_KEY: 56 | numpy.full(NUM_LEVELS + 1, 0.), 57 | u_net_architecture.UPCONV_LAYER_DROPOUT_RATES_KEY: 58 | numpy.full(NUM_LEVELS, 0.), 59 | u_net_architecture.SKIP_LAYER_DROPOUT_RATES_KEY: 60 | numpy.full(NUM_LEVELS, 0.), 61 | u_net_architecture.INNER_ACTIV_FUNCTION_KEY: 62 | architecture_utils.RELU_FUNCTION_STRING, 63 | u_net_architecture.INNER_ACTIV_FUNCTION_ALPHA_KEY: 0.2, 64 | u_net_architecture.OUTPUT_ACTIV_FUNCTION_KEY: 65 | architecture_utils.SIGMOID_FUNCTION_STRING, 66 | u_net_architecture.OUTPUT_ACTIV_FUNCTION_ALPHA_KEY: 0., 67 | u_net_architecture.L1_WEIGHT_KEY: 0., 68 | u_net_architecture.USE_BATCH_NORM_KEY: True 69 | } 70 | 71 | 72 | def _run(): 73 | """Makes architecture for best 60-minute U-net in journal paper. 74 | 75 | This is effectively the main method. 76 | """ 77 | 78 | # Read mask. 79 | print('Reading partial mask from: "{0:s}"...'.format( 80 | MASK_FILE_NAME 81 | )) 82 | mask_matrix = radar_io.read_mask_file(MASK_FILE_NAME)[ 83 | radar_io.MASK_MATRIX_KEY 84 | ] 85 | 86 | # Create loss function. 87 | loss_function = neural_net.get_metrics( 88 | metric_names=[LOSS_FUNCTION_NAME], mask_matrix=mask_matrix, 89 | use_as_loss_function=True 90 | )[0][0] 91 | 92 | # Assign experimental hyperparameters. 93 | option_dict = copy.deepcopy(DEFAULT_OPTION_DICT) 94 | option_dict[u_net_architecture.L2_WEIGHT_KEY] = L2_WEIGHT 95 | option_dict[u_net_architecture.INPUT_DIMENSIONS_KEY] = numpy.array([ 96 | NUM_GRID_ROWS, NUM_GRID_COLUMNS, NUM_LAG_TIMES * NUM_SPECTRAL_BANDS 97 | ], dtype=int) 98 | 99 | # This next command creates the U-net architecture and compiles the U-net. 100 | # Thus, it returns a U-net that is completely ready to train. It also 101 | # prints a very long flow chart, containing details on each layer in the 102 | # model. I have not included this flow chart in the paper, because it is 103 | # difficult to read and I feel that Figure 5 does a better job of 104 | # documenting the architecture. 105 | 106 | model_object = u_net_architecture.create_model( 107 | option_dict=option_dict, loss_function=loss_function, 108 | mask_matrix=mask_matrix, metric_names=METRIC_NAMES 109 | ) 110 | 111 | 112 | if __name__ == '__main__': 113 | _run() 114 | -------------------------------------------------------------------------------- /ml4convection/scripts/make_best_architecture_90minutes.py: -------------------------------------------------------------------------------- 1 | """Makes architecture for best 90-minute U-net in journal paper.""" 2 | 3 | import os 4 | import copy 5 | import numpy 6 | from gewittergefahr.deep_learning import architecture_utils 7 | from ml4convection.io import radar_io 8 | from ml4convection.machine_learning import neural_net 9 | from ml4convection.machine_learning import u_net_architecture 10 | 11 | # Dimensions of predictors (satellite images). 12 | NUM_GRID_ROWS = 205 13 | NUM_GRID_COLUMNS = 205 14 | NUM_SPECTRAL_BANDS = 7 15 | 16 | # Best L_2 weight and number of lag times. The exact lag times are 17 | # {0, 20, 40, 60} minutes, but these are not specified in the architecture; they 18 | # are specified only at training time, when feeding the input data to the model. 19 | L2_WEIGHT = 1e-6 20 | NUM_LAG_TIMES = 4 21 | 22 | # Number of levels in U-net. 23 | NUM_LEVELS = 5 24 | 25 | # File containing mask, which ensures that only grid points within 100 km of the 26 | # three southernmost radars are used in the loss function. The mask is 27 | # referenced to a radar-centered domain, with 205 x 205 grid points. 28 | THIS_DIRECTORY_NAME = os.path.dirname(os.path.realpath( 29 | os.path.join(os.getcwd(), os.path.expanduser(__file__)) 30 | )) 31 | MASK_FILE_NAME = '{0:s}/radar_mask_100km_omit-north_partial.nc'.format( 32 | THIS_DIRECTORY_NAME 33 | ) 34 | 35 | # Loss function (fractions skill score with 9-by-9 window, or half-window size 36 | # of grid points). 37 | LOSS_FUNCTION_NAME = 'fss_neigh4' 38 | 39 | # Metrics (other evaluation scores tracked during training). 40 | METRIC_NAMES = [ 41 | 'fss_neigh0', 'fss_neigh1', 'fss_neigh2', 'fss_neigh3', 42 | 'csi_neigh0', 'csi_neigh1', 'csi_neigh2', 'csi_neigh3', 43 | 'bias_neigh0', 'bias_neigh1', 'bias_neigh2', 'bias_neigh3', 44 | 'iou_neigh0', 'iou_neigh1', 'iou_neigh2', 'iou_neigh3', 45 | 'dice_neigh0', 'dice_neigh1', 'dice_neigh2', 'dice_neigh3' 46 | ] 47 | 48 | # DEFAULT_OPTION_DICT contains fixed hyperparameters, which were not varied 49 | # during the hyperparameter experiment presented in the paper. 50 | DEFAULT_OPTION_DICT = { 51 | u_net_architecture.NUM_LEVELS_KEY: NUM_LEVELS, 52 | u_net_architecture.NUM_CONV_LAYERS_KEY: 2, 53 | u_net_architecture.CONV_LAYER_CHANNEL_COUNTS_KEY: 54 | numpy.array([16, 24, 32, 48, 64, 96], dtype=int), 55 | u_net_architecture.CONV_LAYER_DROPOUT_RATES_KEY: 56 | numpy.full(NUM_LEVELS + 1, 0.), 57 | u_net_architecture.UPCONV_LAYER_DROPOUT_RATES_KEY: 58 | numpy.full(NUM_LEVELS, 0.), 59 | u_net_architecture.SKIP_LAYER_DROPOUT_RATES_KEY: 60 | numpy.full(NUM_LEVELS, 0.), 61 | u_net_architecture.INNER_ACTIV_FUNCTION_KEY: 62 | architecture_utils.RELU_FUNCTION_STRING, 63 | u_net_architecture.INNER_ACTIV_FUNCTION_ALPHA_KEY: 0.2, 64 | u_net_architecture.OUTPUT_ACTIV_FUNCTION_KEY: 65 | architecture_utils.SIGMOID_FUNCTION_STRING, 66 | u_net_architecture.OUTPUT_ACTIV_FUNCTION_ALPHA_KEY: 0., 67 | u_net_architecture.L1_WEIGHT_KEY: 0., 68 | u_net_architecture.USE_BATCH_NORM_KEY: True 69 | } 70 | 71 | 72 | def _run(): 73 | """Makes architecture for best 90-minute U-net in journal paper. 74 | 75 | This is effectively the main method. 76 | """ 77 | 78 | # Read mask. 79 | print('Reading partial mask from: "{0:s}"...'.format( 80 | MASK_FILE_NAME 81 | )) 82 | mask_matrix = radar_io.read_mask_file(MASK_FILE_NAME)[ 83 | radar_io.MASK_MATRIX_KEY 84 | ] 85 | 86 | # Create loss function. 87 | loss_function = neural_net.get_metrics( 88 | metric_names=[LOSS_FUNCTION_NAME], mask_matrix=mask_matrix, 89 | use_as_loss_function=True 90 | )[0][0] 91 | 92 | # Assign experimental hyperparameters. 93 | option_dict = copy.deepcopy(DEFAULT_OPTION_DICT) 94 | option_dict[u_net_architecture.L2_WEIGHT_KEY] = L2_WEIGHT 95 | option_dict[u_net_architecture.INPUT_DIMENSIONS_KEY] = numpy.array([ 96 | NUM_GRID_ROWS, NUM_GRID_COLUMNS, NUM_LAG_TIMES * NUM_SPECTRAL_BANDS 97 | ], dtype=int) 98 | 99 | # This next command creates the U-net architecture and compiles the U-net. 100 | # Thus, it returns a U-net that is completely ready to train. It also 101 | # prints a very long flow chart, containing details on each layer in the 102 | # model. I have not included this flow chart in the paper, because it is 103 | # difficult to read and I feel that Figure 5 does a better job of 104 | # documenting the architecture. 105 | 106 | model_object = u_net_architecture.create_model( 107 | option_dict=option_dict, loss_function=loss_function, 108 | mask_matrix=mask_matrix, metric_names=METRIC_NAMES 109 | ) 110 | 111 | 112 | if __name__ == '__main__': 113 | _run() 114 | -------------------------------------------------------------------------------- /ml4convection/scripts/plot_discard_test.py: -------------------------------------------------------------------------------- 1 | """Plots results of discard test (error vs. discard fraction).""" 2 | 3 | import argparse 4 | import matplotlib 5 | matplotlib.use('agg') 6 | from matplotlib import pyplot 7 | from gewittergefahr.gg_utils import file_system_utils 8 | from ml4convection.utils import uq_evaluation 9 | from ml4convection.plotting import uq_evaluation_plotting as uq_eval_plotting 10 | 11 | FIGURE_RESOLUTION_DPI = 300 12 | 13 | INPUT_FILE_ARG_NAME = 'input_file_name' 14 | OUTPUT_FILE_ARG_NAME = 'output_file_name' 15 | 16 | INPUT_FILE_HELP_STRING = ( 17 | 'Path to input file. Will be read by `uq_evaluation.read_discard_results`.' 18 | ) 19 | OUTPUT_FILE_HELP_STRING = ( 20 | 'Path to output file. Figure will be saved as an image here.' 21 | ) 22 | 23 | INPUT_ARG_PARSER = argparse.ArgumentParser() 24 | INPUT_ARG_PARSER.add_argument( 25 | '--' + INPUT_FILE_ARG_NAME, type=str, required=True, 26 | help=INPUT_FILE_HELP_STRING 27 | ) 28 | INPUT_ARG_PARSER.add_argument( 29 | '--' + OUTPUT_FILE_ARG_NAME, type=str, required=True, 30 | help=OUTPUT_FILE_HELP_STRING 31 | ) 32 | 33 | 34 | def _run(input_file_name, output_file_name): 35 | """Plots results of discard test (error vs. discard fraction). 36 | 37 | This is effectively the main method. 38 | 39 | :param input_file_name: See documentation at top of file. 40 | :param output_file_name: Same. 41 | """ 42 | 43 | file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) 44 | 45 | print('Reading data from: "{0:s}"...'.format(input_file_name)) 46 | result_dict = uq_evaluation.read_discard_results(input_file_name) 47 | 48 | figure_object, axes_object = uq_eval_plotting.plot_discard_test( 49 | result_dict=result_dict 50 | ) 51 | 52 | this_string = ( 53 | 'cross-entropy' 54 | if 'X-entropy' in result_dict[uq_evaluation.ERROR_FUNCTION_KEY] 55 | else 'FSS' 56 | ) 57 | axes_object.set_ylabel('Model performance ({0:s})'.format(this_string)) 58 | 59 | title_string = 'Discard test\n(MF = {0:.2f}%; DI = {1:.2g})'.format( 60 | 100 * result_dict[uq_evaluation.MONOTONICITY_FRACTION_KEY], 61 | result_dict[uq_evaluation.DISCARD_IMPROVEMENT_KEY] 62 | ) 63 | print(title_string) 64 | axes_object.set_title(title_string) 65 | 66 | print('Saving figure to file: "{0:s}"...'.format(output_file_name)) 67 | figure_object.savefig( 68 | output_file_name, dpi=FIGURE_RESOLUTION_DPI, 69 | pad_inches=0, bbox_inches='tight' 70 | ) 71 | pyplot.close(figure_object) 72 | 73 | 74 | if __name__ == '__main__': 75 | INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args() 76 | 77 | _run( 78 | input_file_name=getattr(INPUT_ARG_OBJECT, INPUT_FILE_ARG_NAME), 79 | output_file_name=getattr(INPUT_ARG_OBJECT, OUTPUT_FILE_ARG_NAME) 80 | ) 81 | -------------------------------------------------------------------------------- /ml4convection/scripts/plot_radar_mask.py: -------------------------------------------------------------------------------- 1 | """Plots mask for radar data.""" 2 | 3 | import argparse 4 | import numpy 5 | import matplotlib 6 | matplotlib.use('agg') 7 | from matplotlib import pyplot 8 | from gewittergefahr.gg_utils import file_system_utils 9 | from gewittergefahr.plotting import radar_plotting 10 | from ml4convection.io import radar_io 11 | from ml4convection.io import border_io 12 | from ml4convection.plotting import plotting_utils 13 | 14 | METRES_TO_KM = 0.001 15 | 16 | DUMMY_FIELD_NAME = 'reflectivity_column_max_dbz' 17 | COLOUR_MAP_OBJECT = pyplot.get_cmap('winter') 18 | COLOUR_NORM_OBJECT = pyplot.Normalize(vmin=0., vmax=1.) 19 | 20 | FIGURE_RESOLUTION_DPI = 300 21 | FIGURE_WIDTH_INCHES = 15 22 | FIGURE_HEIGHT_INCHES = 15 23 | 24 | INPUT_FILE_ARG_NAME = 'input_mask_file_name' 25 | OUTPUT_FILE_ARG_NAME = 'output_file_name' 26 | 27 | INPUT_FILE_HELP_STRING = ( 28 | 'Path to input file. Will be read by `radar_io.read_mask_file`.' 29 | ) 30 | OUTPUT_FILE_HELP_STRING = 'Path to output file. Image will be saved here.' 31 | 32 | INPUT_ARG_PARSER = argparse.ArgumentParser() 33 | INPUT_ARG_PARSER.add_argument( 34 | '--' + INPUT_FILE_ARG_NAME, type=str, required=True, 35 | help=INPUT_FILE_HELP_STRING 36 | ) 37 | INPUT_ARG_PARSER.add_argument( 38 | '--' + OUTPUT_FILE_ARG_NAME, type=str, required=True, 39 | help=OUTPUT_FILE_HELP_STRING 40 | ) 41 | 42 | 43 | def _run(mask_file_name, output_file_name): 44 | """Plots mask for radar data. 45 | 46 | This is effectively the main method. 47 | 48 | :param mask_file_name: See documentation at top of file. 49 | :param output_file_name: Same. 50 | """ 51 | 52 | border_latitudes_deg_n, border_longitudes_deg_e = border_io.read_file() 53 | 54 | file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) 55 | 56 | print('Reading data from: "{0:s}"...'.format(mask_file_name)) 57 | mask_dict = radar_io.read_mask_file(mask_file_name) 58 | 59 | latitudes_deg_n = mask_dict[radar_io.LATITUDES_KEY] 60 | longitudes_deg_e = mask_dict[radar_io.LONGITUDES_KEY] 61 | 62 | figure_object, axes_object = pyplot.subplots( 63 | 1, 1, figsize=(FIGURE_WIDTH_INCHES, FIGURE_HEIGHT_INCHES) 64 | ) 65 | 66 | plotting_utils.plot_borders( 67 | border_latitudes_deg_n=border_latitudes_deg_n, 68 | border_longitudes_deg_e=border_longitudes_deg_e, 69 | axes_object=axes_object 70 | ) 71 | 72 | mask_matrix = mask_dict[radar_io.MASK_MATRIX_KEY].astype(float) 73 | mask_matrix[mask_matrix < 0.5] = numpy.nan 74 | 75 | radar_plotting.plot_latlng_grid( 76 | field_matrix=mask_matrix, field_name=DUMMY_FIELD_NAME, 77 | axes_object=axes_object, 78 | min_grid_point_latitude_deg=numpy.min(latitudes_deg_n), 79 | min_grid_point_longitude_deg=numpy.min(longitudes_deg_e), 80 | latitude_spacing_deg=numpy.diff(latitudes_deg_n[:2])[0], 81 | longitude_spacing_deg=numpy.diff(longitudes_deg_e[:2])[0], 82 | colour_map_object=COLOUR_MAP_OBJECT, 83 | colour_norm_object=COLOUR_NORM_OBJECT 84 | ) 85 | 86 | plotting_utils.plot_grid_lines( 87 | plot_latitudes_deg_n=latitudes_deg_n, 88 | plot_longitudes_deg_e=longitudes_deg_e, axes_object=axes_object, 89 | parallel_spacing_deg=2., meridian_spacing_deg=2. 90 | ) 91 | 92 | max_distance_km = int(numpy.round( 93 | METRES_TO_KM * mask_dict[radar_io.MAX_MASK_DISTANCE_KEY] 94 | )) 95 | title_string = ( 96 | 'Mask (based on {0:d}-km distance from nearest radar)' 97 | ).format(max_distance_km) 98 | 99 | axes_object.set_title(title_string) 100 | 101 | print('Saving figure to file: "{0:s}"...'.format(output_file_name)) 102 | figure_object.savefig( 103 | output_file_name, dpi=FIGURE_RESOLUTION_DPI, 104 | pad_inches=0, bbox_inches='tight' 105 | ) 106 | pyplot.close(figure_object) 107 | 108 | 109 | if __name__ == '__main__': 110 | INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args() 111 | 112 | _run( 113 | mask_file_name=getattr(INPUT_ARG_OBJECT, INPUT_FILE_ARG_NAME), 114 | output_file_name=getattr(INPUT_ARG_OBJECT, OUTPUT_FILE_ARG_NAME) 115 | ) 116 | -------------------------------------------------------------------------------- /ml4convection/scripts/plot_spread_vs_skill.py: -------------------------------------------------------------------------------- 1 | """Creates spread-skill plot and saves plot to image file.""" 2 | 3 | import argparse 4 | import numpy 5 | import matplotlib 6 | matplotlib.use('agg') 7 | from matplotlib import pyplot 8 | from gewittergefahr.gg_utils import file_system_utils 9 | from ml4convection.utils import uq_evaluation 10 | from ml4convection.plotting import uq_evaluation_plotting as uq_eval_plotting 11 | 12 | FIGURE_RESOLUTION_DPI = 300 13 | 14 | INPUT_FILE_ARG_NAME = 'input_file_name' 15 | OUTPUT_FILE_ARG_NAME = 'output_file_name' 16 | 17 | INPUT_FILE_HELP_STRING = ( 18 | 'Path to input file. Will be read by `uq_evaluation.read_spread_vs_skill`.' 19 | ) 20 | OUTPUT_FILE_HELP_STRING = ( 21 | 'Path to output file. Figure will be saved as an image here.' 22 | ) 23 | 24 | INPUT_ARG_PARSER = argparse.ArgumentParser() 25 | INPUT_ARG_PARSER.add_argument( 26 | '--' + INPUT_FILE_ARG_NAME, type=str, required=True, 27 | help=INPUT_FILE_HELP_STRING 28 | ) 29 | INPUT_ARG_PARSER.add_argument( 30 | '--' + OUTPUT_FILE_ARG_NAME, type=str, required=True, 31 | help=OUTPUT_FILE_HELP_STRING 32 | ) 33 | 34 | 35 | def _run(input_file_name, output_file_name): 36 | """Creates spread-skill plot and saves plot to image file. 37 | 38 | This is effectively the main method. 39 | 40 | :param input_file_name: See documentation at top of file. 41 | :param output_file_name: Same. 42 | """ 43 | 44 | file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name) 45 | 46 | print('Reading data from: "{0:s}"...'.format(input_file_name)) 47 | result_dict = uq_evaluation.read_spread_vs_skill(input_file_name) 48 | 49 | figure_object, axes_object = uq_eval_plotting.plot_spread_vs_skill( 50 | result_dict=result_dict 51 | ) 52 | 53 | if result_dict[uq_evaluation.USE_MEDIAN_KEY]: 54 | axes_object.set_ylabel('Skill (RMSE of median prediction)') 55 | else: 56 | axes_object.set_ylabel('Skill (RMSE of mean prediction)') 57 | 58 | axes_object.set_xlabel('Spread (stdev of predictive distribution)') 59 | 60 | half_window_size_px = int(numpy.round( 61 | result_dict[uq_evaluation.HALF_WINDOW_SIZE_KEY] 62 | )) 63 | title_string = ( 64 | 'Spread-skill plot\n(SSREL = {0:.2g}; SSRAT = {1:.2g})' 65 | ).format( 66 | result_dict[uq_evaluation.SPREAD_SKILL_RELIABILITY_KEY], 67 | result_dict[uq_evaluation.OVERALL_SPREAD_SKILL_RATIO_KEY] 68 | ) 69 | print(title_string) 70 | axes_object.set_title(title_string) 71 | 72 | print('Saving figure to file: "{0:s}"...'.format(output_file_name)) 73 | figure_object.savefig( 74 | output_file_name, dpi=FIGURE_RESOLUTION_DPI, 75 | pad_inches=0, bbox_inches='tight' 76 | ) 77 | pyplot.close(figure_object) 78 | 79 | 80 | if __name__ == '__main__': 81 | INPUT_ARG_OBJECT = INPUT_ARG_PARSER.parse_args() 82 | 83 | _run( 84 | input_file_name=getattr(INPUT_ARG_OBJECT, INPUT_FILE_ARG_NAME), 85 | output_file_name=getattr(INPUT_ARG_OBJECT, OUTPUT_FILE_ARG_NAME) 86 | ) 87 | -------------------------------------------------------------------------------- /ml4convection/scripts/radar_mask_100km_omit-north_partial.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thunderhoser/ml4convection/cf5e9682bc3182305274132ae246bc72d994308f/ml4convection/scripts/radar_mask_100km_omit-north_partial.nc -------------------------------------------------------------------------------- /ml4convection/scripts/run_unit_tests.py: -------------------------------------------------------------------------------- 1 | """Runs all unit tests in ml4convection library.""" 2 | 3 | import os 4 | import sys 5 | import glob 6 | 7 | SEPARATOR_STRING = '\n\n' + '*' * 50 + '\n\n' 8 | 9 | 10 | def _run(): 11 | """Runs all unit tests in ml4rt library. 12 | 13 | This is effectively the main method. 14 | """ 15 | 16 | script_dir_name = os.getcwd() 17 | top_library_dir_name = '/'.join(script_dir_name.split('/')[:-1]) 18 | 19 | test_file_pattern = '{0:s}/*/*_test.py'.format(top_library_dir_name) 20 | test_file_names = glob.glob(test_file_pattern) 21 | 22 | for this_file_name in test_file_names: 23 | if 'dead_code/' in this_file_name: 24 | continue 25 | 26 | print('Running file: "{0:s}"...'.format(this_file_name)) 27 | 28 | command_string = '"{0:s}" "{1:s}"'.format( 29 | sys.executable, this_file_name 30 | ) 31 | 32 | os.system(command_string) 33 | print(SEPARATOR_STRING) 34 | 35 | 36 | if __name__ == '__main__': 37 | _run() 38 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/find_prelim_tracking_gaps.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="find_prelim_tracking_gaps" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=8G 7 | #SBATCH --partition="normal" 8 | #SBATCH --time=12:00:00 9 | #SBATCH --mail-type=BEGIN,END,FAIL 10 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 11 | #SBATCH --output=find_prelim_tracking_gaps_%A.out 12 | 13 | TOP_TRACKING_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data/storm_tracking/prelim" 14 | 15 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/gg_master/gewittergefahr/scripts/find_storm_tracking_gaps.py" \ 16 | --top_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" \ 17 | --first_spc_date_string="20151231" \ 18 | --last_spc_date_string="20181231" \ 19 | --min_time_diff_seconds=1800 20 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/run_final_tracking.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="run_final_tracking" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=8G 7 | #SBATCH --partition="normal" 8 | #SBATCH --time=24:00:00 9 | #SBATCH --array=0-67 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=run_final_tracking_%A_%a.out 13 | 14 | TOP_INPUT_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data/storm_tracking/prelim" 15 | TOP_OUTPUT_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data/storm_tracking/final" 16 | 17 | FIRST_SPC_DATE_STRINGS=("20151231" "20160113" "20160126" "20160210" "20160223" "20160314" "20160325" "20160416" "20160501" "20160517" "20160531" "20160615" "20160623" "20160715" "20160727" "20160804" "20160822" "20161003" "20161022" "20161107" "20161121" "20161204" "20161216" "20161228" "20170110" "20170124" "20170206" "20170219" "20170305" "20170318" "20170401" "20170414" "20170428" "20170520" "20170604" "20170624" "20170710" "20170722" "20170804" "20170813" "20170915" "20170926" "20171017" "20171031" "20171113" "20171127" "20171209" "20171224" "20180106" "20180119" "20180202" "20180215" "20180301" "20180314" "20180331" "20180417" "20180501" "20180515" "20180525" "20180624" "20180709" "20180903" "20180926" "20181012" "20181025" "20181108" "20181122" "20181207") 18 | LAST_SPC_DATE_STRINGS=("20160112" "20160125" "20160209" "20160222" "20160313" "20160324" "20160415" "20160430" "20160516" "20160530" "20160614" "20160622" "20160714" "20160726" "20160803" "20160821" "20161002" "20161021" "20161106" "20161120" "20161203" "20161215" "20161227" "20170109" "20170123" "20170205" "20170218" "20170304" "20170317" "20170331" "20170413" "20170427" "20170519" "20170603" "20170623" "20170709" "20170721" "20170803" "20170812" "20170914" "20170925" "20171016" "20171030" "20171112" "20171126" "20171208" "20171223" "20180105" "20180118" "20180201" "20180214" "20180228" "20180313" "20180330" "20180416" "20180430" "20180514" "20180524" "20180623" "20180708" "20180902" "20180925" "20181011" "20181024" "20181107" "20181121" "20181206" "20181231") 19 | 20 | this_first_date_string=${FIRST_SPC_DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 21 | this_last_date_string=${LAST_SPC_DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 22 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... SPC dates = ${this_first_date_string} to ${this_last_date_string}" 23 | 24 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/gg_master/gewittergefahr/scripts/reanalyze_storm_tracks.py" \ 25 | --input_tracking_dir_name="${TOP_INPUT_DIR_NAME}" \ 26 | --first_spc_date_string="${this_first_date_string}" \ 27 | --last_spc_date_string="${this_last_date_string}" \ 28 | --max_velocity_diff_m_s01=30 \ 29 | --max_link_distance_m_s01=38.5833333333333 \ 30 | --max_join_time_seconds=1200 \ 31 | --max_join_error_m_s01=30 \ 32 | --min_duration_seconds=1200 \ 33 | --output_tracking_dir_name="${TOP_OUTPUT_DIR_NAME}" 34 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/run_prelim_tracking_2016.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="run_prelim_tracking_2016" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=16G 7 | #SBATCH --partition="normal" 8 | #SBATCH --time=04:00:00 9 | #SBATCH --array=0-365 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=run_prelim_tracking_2016_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/prelim" 16 | 17 | DATE_STRINGS=("20160101" "20160102" "20160103" "20160104" "20160105" "20160106" "20160107" "20160108" "20160109" "20160110" "20160111" "20160112" "20160113" "20160114" "20160115" "20160116" "20160117" "20160118" "20160119" "20160120" "20160121" "20160122" "20160123" "20160124" "20160125" "20160126" "20160127" "20160128" "20160129" "20160130" "20160131" "20160201" "20160202" "20160203" "20160204" "20160205" "20160206" "20160207" "20160208" "20160209" "20160210" "20160211" "20160212" "20160213" "20160214" "20160215" "20160216" "20160217" "20160218" "20160219" "20160220" "20160221" "20160222" "20160223" "20160224" "20160225" "20160226" "20160227" "20160228" "20160229" "20160301" "20160302" "20160303" "20160304" "20160305" "20160306" "20160307" "20160308" "20160309" "20160310" "20160311" "20160312" "20160313" "20160314" "20160315" "20160316" "20160317" "20160318" "20160319" "20160320" "20160321" "20160322" "20160323" "20160324" "20160325" "20160326" "20160327" "20160328" "20160329" "20160330" "20160331" "20160401" "20160402" "20160403" "20160404" "20160405" "20160406" "20160407" "20160408" "20160409" "20160410" "20160411" "20160412" "20160413" "20160414" "20160415" "20160416" "20160417" "20160418" "20160419" "20160420" "20160421" "20160422" "20160423" "20160424" "20160425" "20160426" "20160427" "20160428" "20160429" "20160430" "20160501" "20160502" "20160503" "20160504" "20160505" "20160506" "20160507" "20160508" "20160509" "20160510" "20160511" "20160512" "20160513" "20160514" "20160515" "20160516" "20160517" "20160518" "20160519" "20160520" "20160521" "20160522" "20160523" "20160524" "20160525" "20160526" "20160527" "20160528" "20160529" "20160530" "20160531" "20160601" "20160602" "20160603" "20160604" "20160605" "20160606" "20160607" "20160608" "20160609" "20160610" "20160611" "20160612" "20160613" "20160614" "20160615" "20160616" "20160617" "20160618" "20160619" "20160620" "20160621" "20160622" "20160623" "20160624" "20160625" "20160626" "20160627" "20160628" "20160629" "20160630" "20160701" "20160702" "20160703" "20160704" "20160705" "20160706" "20160707" "20160708" "20160709" "20160710" "20160711" "20160712" "20160713" "20160714" "20160715" "20160716" "20160717" "20160718" "20160719" "20160720" "20160721" "20160722" "20160723" "20160724" "20160725" "20160726" "20160727" "20160728" "20160729" "20160730" "20160731" "20160801" "20160802" "20160803" "20160804" "20160805" "20160806" "20160807" "20160808" "20160809" "20160810" "20160811" "20160812" "20160813" "20160814" "20160815" "20160816" "20160817" "20160818" "20160819" "20160820" "20160821" "20160822" "20160823" "20160824" "20160825" "20160826" "20160827" "20160828" "20160829" "20160830" "20160831" "20160901" "20160902" "20160903" "20160904" "20160905" "20160906" "20160907" "20160908" "20160909" "20160910" "20160911" "20160912" "20160913" "20160914" "20160915" "20160916" "20160917" "20160918" "20160919" "20160920" "20160921" "20160922" "20160923" "20160924" "20160925" "20160926" "20160927" "20160928" "20160929" "20160930" "20161001" "20161002" "20161003" "20161004" "20161005" "20161006" "20161007" "20161008" "20161009" "20161010" "20161011" "20161012" "20161013" "20161014" "20161015" "20161016" "20161017" "20161018" "20161019" "20161020" "20161021" "20161022" "20161023" "20161024" "20161025" "20161026" "20161027" "20161028" "20161029" "20161030" "20161031" "20161101" "20161102" "20161103" "20161104" "20161105" "20161106" "20161107" "20161108" "20161109" "20161110" "20161111" "20161112" "20161113" "20161114" "20161115" "20161116" "20161117" "20161118" "20161119" "20161120" "20161121" "20161122" "20161123" "20161124" "20161125" "20161126" "20161127" "20161128" "20161129" "20161130" "20161201" "20161202" "20161203" "20161204" "20161205" "20161206" "20161207" "20161208" "20161209" "20161210" "20161211" "20161212" "20161213" "20161214" "20161215" "20161216" "20161217" "20161218" "20161219" "20161220" "20161221" "20161222" "20161223" "20161224" "20161225" "20161226" "20161227" "20161228" "20161229" "20161230" "20161231") 18 | 19 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 20 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 21 | 22 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/run_prelim_tracking.py" \ 23 | --input_refl_dir_name="${TOP_RADAR_DIR_NAME}" \ 24 | --input_echo_classifn_dir_name="${TOP_RADAR_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --critical_refl_dbz=40 \ 28 | --min_echo_top_m_asl=4000 \ 29 | --min_size_pixels=5 \ 30 | --min_intermax_distance_metres=11112 \ 31 | --max_velocity_diff_m_s01=30 \ 32 | --max_link_distance_m_s01=38.5833333333333 \ 33 | --output_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" 34 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/run_prelim_tracking_2016_patch.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="run_prelim_tracking_2016_patch" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=16G 7 | #SBATCH --partition="normal" 8 | #SBATCH --time=04:00:00 9 | #SBATCH --array=125,131,179,191,196,210,214,218,232,258,293,323,332,352 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=run_prelim_tracking_2016_patch_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/prelim" 16 | 17 | DATE_STRINGS=("20160101" "20160102" "20160103" "20160104" "20160105" "20160106" "20160107" "20160108" "20160109" "20160110" "20160111" "20160112" "20160113" "20160114" "20160115" "20160116" "20160117" "20160118" "20160119" "20160120" "20160121" "20160122" "20160123" "20160124" "20160125" "20160126" "20160127" "20160128" "20160129" "20160130" "20160131" "20160201" "20160202" "20160203" "20160204" "20160205" "20160206" "20160207" "20160208" "20160209" "20160210" "20160211" "20160212" "20160213" "20160214" "20160215" "20160216" "20160217" "20160218" "20160219" "20160220" "20160221" "20160222" "20160223" "20160224" "20160225" "20160226" "20160227" "20160228" "20160229" "20160301" "20160302" "20160303" "20160304" "20160305" "20160306" "20160307" "20160308" "20160309" "20160310" "20160311" "20160312" "20160313" "20160314" "20160315" "20160316" "20160317" "20160318" "20160319" "20160320" "20160321" "20160322" "20160323" "20160324" "20160325" "20160326" "20160327" "20160328" "20160329" "20160330" "20160331" "20160401" "20160402" "20160403" "20160404" "20160405" "20160406" "20160407" "20160408" "20160409" "20160410" "20160411" "20160412" "20160413" "20160414" "20160415" "20160416" "20160417" "20160418" "20160419" "20160420" "20160421" "20160422" "20160423" "20160424" "20160425" "20160426" "20160427" "20160428" "20160429" "20160430" "20160501" "20160502" "20160503" "20160504" "20160505" "20160506" "20160507" "20160508" "20160509" "20160510" "20160511" "20160512" "20160513" "20160514" "20160515" "20160516" "20160517" "20160518" "20160519" "20160520" "20160521" "20160522" "20160523" "20160524" "20160525" "20160526" "20160527" "20160528" "20160529" "20160530" "20160531" "20160601" "20160602" "20160603" "20160604" "20160605" "20160606" "20160607" "20160608" "20160609" "20160610" "20160611" "20160612" "20160613" "20160614" "20160615" "20160616" "20160617" "20160618" "20160619" "20160620" "20160621" "20160622" "20160623" "20160624" "20160625" "20160626" "20160627" "20160628" "20160629" "20160630" "20160701" "20160702" "20160703" "20160704" "20160705" "20160706" "20160707" "20160708" "20160709" "20160710" "20160711" "20160712" "20160713" "20160714" "20160715" "20160716" "20160717" "20160718" "20160719" "20160720" "20160721" "20160722" "20160723" "20160724" "20160725" "20160726" "20160727" "20160728" "20160729" "20160730" "20160731" "20160801" "20160802" "20160803" "20160804" "20160805" "20160806" "20160807" "20160808" "20160809" "20160810" "20160811" "20160812" "20160813" "20160814" "20160815" "20160816" "20160817" "20160818" "20160819" "20160820" "20160821" "20160822" "20160823" "20160824" "20160825" "20160826" "20160827" "20160828" "20160829" "20160830" "20160831" "20160901" "20160902" "20160903" "20160904" "20160905" "20160906" "20160907" "20160908" "20160909" "20160910" "20160911" "20160912" "20160913" "20160914" "20160915" "20160916" "20160917" "20160918" "20160919" "20160920" "20160921" "20160922" "20160923" "20160924" "20160925" "20160926" "20160927" "20160928" "20160929" "20160930" "20161001" "20161002" "20161003" "20161004" "20161005" "20161006" "20161007" "20161008" "20161009" "20161010" "20161011" "20161012" "20161013" "20161014" "20161015" "20161016" "20161017" "20161018" "20161019" "20161020" "20161021" "20161022" "20161023" "20161024" "20161025" "20161026" "20161027" "20161028" "20161029" "20161030" "20161031" "20161101" "20161102" "20161103" "20161104" "20161105" "20161106" "20161107" "20161108" "20161109" "20161110" "20161111" "20161112" "20161113" "20161114" "20161115" "20161116" "20161117" "20161118" "20161119" "20161120" "20161121" "20161122" "20161123" "20161124" "20161125" "20161126" "20161127" "20161128" "20161129" "20161130" "20161201" "20161202" "20161203" "20161204" "20161205" "20161206" "20161207" "20161208" "20161209" "20161210" "20161211" "20161212" "20161213" "20161214" "20161215" "20161216" "20161217" "20161218" "20161219" "20161220" "20161221" "20161222" "20161223" "20161224" "20161225" "20161226" "20161227" "20161228" "20161229" "20161230" "20161231") 18 | 19 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 20 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 21 | 22 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/run_prelim_tracking.py" \ 23 | --input_refl_dir_name="${TOP_RADAR_DIR_NAME}" \ 24 | --input_echo_classifn_dir_name="${TOP_RADAR_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --critical_refl_dbz=40 \ 28 | --min_echo_top_m_asl=4000 \ 29 | --min_size_pixels=5 \ 30 | --min_intermax_distance_metres=11112 \ 31 | --max_velocity_diff_m_s01=30 \ 32 | --max_link_distance_m_s01=38.5833333333333 \ 33 | --output_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" 34 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/run_prelim_tracking_2017.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="run_prelim_tracking_2017" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=16G 7 | #SBATCH --partition="normal" 8 | #SBATCH --time=04:00:00 9 | #SBATCH --array=0-364 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=run_prelim_tracking_2017_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/prelim" 16 | 17 | DATE_STRINGS=("20170101" "20170102" "20170103" "20170104" "20170105" "20170106" "20170107" "20170108" "20170109" "20170110" "20170111" "20170112" "20170113" "20170114" "20170115" "20170116" "20170117" "20170118" "20170119" "20170120" "20170121" "20170122" "20170123" "20170124" "20170125" "20170126" "20170127" "20170128" "20170129" "20170130" "20170131" "20170201" "20170202" "20170203" "20170204" "20170205" "20170206" "20170207" "20170208" "20170209" "20170210" "20170211" "20170212" "20170213" "20170214" "20170215" "20170216" "20170217" "20170218" "20170219" "20170220" "20170221" "20170222" "20170223" "20170224" "20170225" "20170226" "20170227" "20170228" "20170301" "20170302" "20170303" "20170304" "20170305" "20170306" "20170307" "20170308" "20170309" "20170310" "20170311" "20170312" "20170313" "20170314" "20170315" "20170316" "20170317" "20170318" "20170319" "20170320" "20170321" "20170322" "20170323" "20170324" "20170325" "20170326" "20170327" "20170328" "20170329" "20170330" "20170331" "20170401" "20170402" "20170403" "20170404" "20170405" "20170406" "20170407" "20170408" "20170409" "20170410" "20170411" "20170412" "20170413" "20170414" "20170415" "20170416" "20170417" "20170418" "20170419" "20170420" "20170421" "20170422" "20170423" "20170424" "20170425" "20170426" "20170427" "20170428" "20170429" "20170430" "20170501" "20170502" "20170503" "20170504" "20170505" "20170506" "20170507" "20170508" "20170509" "20170510" "20170511" "20170512" "20170513" "20170514" "20170515" "20170516" "20170517" "20170518" "20170519" "20170520" "20170521" "20170522" "20170523" "20170524" "20170525" "20170526" "20170527" "20170528" "20170529" "20170530" "20170531" "20170601" "20170602" "20170603" "20170604" "20170605" "20170606" "20170607" "20170608" "20170609" "20170610" "20170611" "20170612" "20170613" "20170614" "20170615" "20170616" "20170617" "20170618" "20170619" "20170620" "20170621" "20170622" "20170623" "20170624" "20170625" "20170626" "20170627" "20170628" "20170629" "20170630" "20170701" "20170702" "20170703" "20170704" "20170705" "20170706" "20170707" "20170708" "20170709" "20170710" "20170711" "20170712" "20170713" "20170714" "20170715" "20170716" "20170717" "20170718" "20170719" "20170720" "20170721" "20170722" "20170723" "20170724" "20170725" "20170726" "20170727" "20170728" "20170729" "20170730" "20170731" "20170801" "20170802" "20170803" "20170804" "20170805" "20170806" "20170807" "20170808" "20170809" "20170810" "20170811" "20170812" "20170813" "20170814" "20170815" "20170816" "20170817" "20170818" "20170819" "20170820" "20170821" "20170822" "20170823" "20170824" "20170825" "20170826" "20170827" "20170828" "20170829" "20170830" "20170831" "20170901" "20170902" "20170903" "20170904" "20170905" "20170906" "20170907" "20170908" "20170909" "20170910" "20170911" "20170912" "20170913" "20170914" "20170915" "20170916" "20170917" "20170918" "20170919" "20170920" "20170921" "20170922" "20170923" "20170924" "20170925" "20170926" "20170927" "20170928" "20170929" "20170930" "20171001" "20171002" "20171003" "20171004" "20171005" "20171006" "20171007" "20171008" "20171009" "20171010" "20171011" "20171012" "20171013" "20171014" "20171015" "20171016" "20171017" "20171018" "20171019" "20171020" "20171021" "20171022" "20171023" "20171024" "20171025" "20171026" "20171027" "20171028" "20171029" "20171030" "20171031" "20171101" "20171102" "20171103" "20171104" "20171105" "20171106" "20171107" "20171108" "20171109" "20171110" "20171111" "20171112" "20171113" "20171114" "20171115" "20171116" "20171117" "20171118" "20171119" "20171120" "20171121" "20171122" "20171123" "20171124" "20171125" "20171126" "20171127" "20171128" "20171129" "20171130" "20171201" "20171202" "20171203" "20171204" "20171205" "20171206" "20171207" "20171208" "20171209" "20171210" "20171211" "20171212" "20171213" "20171214" "20171215" "20171216" "20171217" "20171218" "20171219" "20171220" "20171221" "20171222" "20171223" "20171224" "20171225" "20171226" "20171227" "20171228" "20171229" "20171230" "20171231") 18 | 19 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 20 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 21 | 22 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/run_prelim_tracking.py" \ 23 | --input_refl_dir_name="${TOP_RADAR_DIR_NAME}" \ 24 | --input_echo_classifn_dir_name="${TOP_RADAR_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --critical_refl_dbz=40 \ 28 | --min_echo_top_m_asl=4000 \ 29 | --min_size_pixels=5 \ 30 | --min_intermax_distance_metres=11112 \ 31 | --max_velocity_diff_m_s01=30 \ 32 | --max_link_distance_m_s01=38.5833333333333 \ 33 | --output_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" 34 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/run_prelim_tracking_2017_patch.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="run_prelim_tracking_2017_patch" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=16G 7 | #SBATCH --partition="normal" 8 | #SBATCH --time=04:00:00 9 | #SBATCH --array=16,31,53,57,68,75,79,90,106,142,170,197,214,251,256,273,286,297,302,305,318,332,359 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=run_prelim_tracking_2017_patch_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/prelim" 16 | 17 | DATE_STRINGS=("20170101" "20170102" "20170103" "20170104" "20170105" "20170106" "20170107" "20170108" "20170109" "20170110" "20170111" "20170112" "20170113" "20170114" "20170115" "20170116" "20170117" "20170118" "20170119" "20170120" "20170121" "20170122" "20170123" "20170124" "20170125" "20170126" "20170127" "20170128" "20170129" "20170130" "20170131" "20170201" "20170202" "20170203" "20170204" "20170205" "20170206" "20170207" "20170208" "20170209" "20170210" "20170211" "20170212" "20170213" "20170214" "20170215" "20170216" "20170217" "20170218" "20170219" "20170220" "20170221" "20170222" "20170223" "20170224" "20170225" "20170226" "20170227" "20170228" "20170301" "20170302" "20170303" "20170304" "20170305" "20170306" "20170307" "20170308" "20170309" "20170310" "20170311" "20170312" "20170313" "20170314" "20170315" "20170316" "20170317" "20170318" "20170319" "20170320" "20170321" "20170322" "20170323" "20170324" "20170325" "20170326" "20170327" "20170328" "20170329" "20170330" "20170331" "20170401" "20170402" "20170403" "20170404" "20170405" "20170406" "20170407" "20170408" "20170409" "20170410" "20170411" "20170412" "20170413" "20170414" "20170415" "20170416" "20170417" "20170418" "20170419" "20170420" "20170421" "20170422" "20170423" "20170424" "20170425" "20170426" "20170427" "20170428" "20170429" "20170430" "20170501" "20170502" "20170503" "20170504" "20170505" "20170506" "20170507" "20170508" "20170509" "20170510" "20170511" "20170512" "20170513" "20170514" "20170515" "20170516" "20170517" "20170518" "20170519" "20170520" "20170521" "20170522" "20170523" "20170524" "20170525" "20170526" "20170527" "20170528" "20170529" "20170530" "20170531" "20170601" "20170602" "20170603" "20170604" "20170605" "20170606" "20170607" "20170608" "20170609" "20170610" "20170611" "20170612" "20170613" "20170614" "20170615" "20170616" "20170617" "20170618" "20170619" "20170620" "20170621" "20170622" "20170623" "20170624" "20170625" "20170626" "20170627" "20170628" "20170629" "20170630" "20170701" "20170702" "20170703" "20170704" "20170705" "20170706" "20170707" "20170708" "20170709" "20170710" "20170711" "20170712" "20170713" "20170714" "20170715" "20170716" "20170717" "20170718" "20170719" "20170720" "20170721" "20170722" "20170723" "20170724" "20170725" "20170726" "20170727" "20170728" "20170729" "20170730" "20170731" "20170801" "20170802" "20170803" "20170804" "20170805" "20170806" "20170807" "20170808" "20170809" "20170810" "20170811" "20170812" "20170813" "20170814" "20170815" "20170816" "20170817" "20170818" "20170819" "20170820" "20170821" "20170822" "20170823" "20170824" "20170825" "20170826" "20170827" "20170828" "20170829" "20170830" "20170831" "20170901" "20170902" "20170903" "20170904" "20170905" "20170906" "20170907" "20170908" "20170909" "20170910" "20170911" "20170912" "20170913" "20170914" "20170915" "20170916" "20170917" "20170918" "20170919" "20170920" "20170921" "20170922" "20170923" "20170924" "20170925" "20170926" "20170927" "20170928" "20170929" "20170930" "20171001" "20171002" "20171003" "20171004" "20171005" "20171006" "20171007" "20171008" "20171009" "20171010" "20171011" "20171012" "20171013" "20171014" "20171015" "20171016" "20171017" "20171018" "20171019" "20171020" "20171021" "20171022" "20171023" "20171024" "20171025" "20171026" "20171027" "20171028" "20171029" "20171030" "20171031" "20171101" "20171102" "20171103" "20171104" "20171105" "20171106" "20171107" "20171108" "20171109" "20171110" "20171111" "20171112" "20171113" "20171114" "20171115" "20171116" "20171117" "20171118" "20171119" "20171120" "20171121" "20171122" "20171123" "20171124" "20171125" "20171126" "20171127" "20171128" "20171129" "20171130" "20171201" "20171202" "20171203" "20171204" "20171205" "20171206" "20171207" "20171208" "20171209" "20171210" "20171211" "20171212" "20171213" "20171214" "20171215" "20171216" "20171217" "20171218" "20171219" "20171220" "20171221" "20171222" "20171223" "20171224" "20171225" "20171226" "20171227" "20171228" "20171229" "20171230" "20171231") 18 | 19 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 20 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 21 | 22 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/run_prelim_tracking.py" \ 23 | --input_refl_dir_name="${TOP_RADAR_DIR_NAME}" \ 24 | --input_echo_classifn_dir_name="${TOP_RADAR_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --critical_refl_dbz=40 \ 28 | --min_echo_top_m_asl=4000 \ 29 | --min_size_pixels=5 \ 30 | --min_intermax_distance_metres=11112 \ 31 | --max_velocity_diff_m_s01=30 \ 32 | --max_link_distance_m_s01=38.5833333333333 \ 33 | --output_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" 34 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/run_prelim_tracking_2018.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="run_prelim_tracking_2018" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=16G 7 | #SBATCH --partition="normal" 8 | #SBATCH --time=04:00:00 9 | #SBATCH --array=0-365 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=run_prelim_tracking_2018_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/prelim" 16 | 17 | DATE_STRINGS=("20180101" "20180102" "20180103" "20180104" "20180105" "20180106" "20180107" "20180108" "20180109" "20180110" "20180111" "20180112" "20180113" "20180114" "20180115" "20180116" "20180117" "20180118" "20180119" "20180120" "20180121" "20180122" "20180123" "20180124" "20180125" "20180126" "20180127" "20180128" "20180129" "20180130" "20180131" "20180201" "20180202" "20180203" "20180204" "20180205" "20180206" "20180207" "20180208" "20180209" "20180210" "20180211" "20180212" "20180213" "20180214" "20180215" "20180216" "20180217" "20180218" "20180219" "20180220" "20180221" "20180222" "20180223" "20180224" "20180225" "20180226" "20180227" "20180228" "20180301" "20180302" "20180303" "20180304" "20180305" "20180306" "20180307" "20180308" "20180309" "20180310" "20180311" "20180312" "20180313" "20180314" "20180315" "20180316" "20180317" "20180318" "20180319" "20180320" "20180321" "20180322" "20180323" "20180324" "20180325" "20180326" "20180327" "20180328" "20180329" "20180330" "20180331" "20180401" "20180402" "20180403" "20180404" "20180405" "20180406" "20180407" "20180408" "20180409" "20180410" "20180411" "20180412" "20180413" "20180414" "20180415" "20180416" "20180417" "20180418" "20180419" "20180420" "20180421" "20180422" "20180423" "20180424" "20180425" "20180426" "20180427" "20180428" "20180429" "20180430" "20180501" "20180502" "20180503" "20180504" "20180505" "20180506" "20180507" "20180508" "20180509" "20180510" "20180511" "20180512" "20180513" "20180514" "20180515" "20180516" "20180517" "20180518" "20180519" "20180520" "20180521" "20180522" "20180523" "20180524" "20180525" "20180526" "20180527" "20180528" "20180529" "20180530" "20180531" "20180601" "20180602" "20180603" "20180604" "20180605" "20180606" "20180607" "20180608" "20180609" "20180610" "20180611" "20180612" "20180613" "20180614" "20180615" "20180616" "20180617" "20180618" "20180619" "20180620" "20180621" "20180622" "20180623" "20180624" "20180625" "20180626" "20180627" "20180628" "20180629" "20180630" "20180701" "20180702" "20180703" "20180704" "20180705" "20180706" "20180707" "20180708" "20180709" "20180710" "20180711" "20180712" "20180713" "20180714" "20180715" "20180716" "20180717" "20180718" "20180719" "20180720" "20180721" "20180722" "20180723" "20180724" "20180725" "20180726" "20180727" "20180728" "20180729" "20180730" "20180731" "20180801" "20180802" "20180803" "20180804" "20180805" "20180806" "20180807" "20180808" "20180809" "20180810" "20180811" "20180812" "20180813" "20180814" "20180815" "20180816" "20180817" "20180818" "20180819" "20180820" "20180821" "20180822" "20180823" "20180824" "20180825" "20180826" "20180827" "20180828" "20180829" "20180830" "20180831" "20180901" "20180902" "20180903" "20180904" "20180905" "20180906" "20180907" "20180908" "20180909" "20180910" "20180911" "20180912" "20180913" "20180914" "20180915" "20180916" "20180917" "20180918" "20180919" "20180920" "20180921" "20180922" "20180923" "20180924" "20180925" "20180926" "20180927" "20180928" "20180929" "20180930" "20181001" "20181002" "20181003" "20181004" "20181005" "20181006" "20181007" "20181008" "20181009" "20181010" "20181011" "20181012" "20181013" "20181014" "20181015" "20181016" "20181017" "20181018" "20181019" "20181020" "20181021" "20181022" "20181023" "20181024" "20181025" "20181026" "20181027" "20181028" "20181029" "20181030" "20181031" "20181101" "20181102" "20181103" "20181104" "20181105" "20181106" "20181107" "20181108" "20181109" "20181110" "20181111" "20181112" "20181113" "20181114" "20181115" "20181116" "20181117" "20181118" "20181119" "20181120" "20181121" "20181122" "20181123" "20181124" "20181125" "20181126" "20181127" "20181128" "20181129" "20181130" "20181201" "20181202" "20181203" "20181204" "20181205" "20181206" "20181207" "20181208" "20181209" "20181210" "20181211" "20181212" "20181213" "20181214" "20181215" "20181216" "20181217" "20181218" "20181219" "20181220" "20181221" "20181222" "20181223" "20181224" "20181225" "20181226" "20181227" "20181228" "20181229" "20181230" "20181231") 18 | 19 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 20 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 21 | 22 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/run_prelim_tracking.py" \ 23 | --input_refl_dir_name="${TOP_RADAR_DIR_NAME}" \ 24 | --input_echo_classifn_dir_name="${TOP_RADAR_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --critical_refl_dbz=40 \ 28 | --min_echo_top_m_asl=4000 \ 29 | --min_size_pixels=5 \ 30 | --min_intermax_distance_metres=11112 \ 31 | --max_velocity_diff_m_s01=30 \ 32 | --max_link_distance_m_s01=38.5833333333333 \ 33 | --output_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" 34 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/run_prelim_tracking_2018_patch.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="run_prelim_tracking_2018_patch" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=16G 7 | #SBATCH --partition="normal" 8 | #SBATCH --time=04:00:00 9 | #SBATCH --array=33,67,102,131,147,156,161 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=run_prelim_tracking_2018_patch_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/prelim" 16 | 17 | DATE_STRINGS=("20180101" "20180102" "20180103" "20180104" "20180105" "20180106" "20180107" "20180108" "20180109" "20180110" "20180111" "20180112" "20180113" "20180114" "20180115" "20180116" "20180117" "20180118" "20180119" "20180120" "20180121" "20180122" "20180123" "20180124" "20180125" "20180126" "20180127" "20180128" "20180129" "20180130" "20180131" "20180201" "20180202" "20180203" "20180204" "20180205" "20180206" "20180207" "20180208" "20180209" "20180210" "20180211" "20180212" "20180213" "20180214" "20180215" "20180216" "20180217" "20180218" "20180219" "20180220" "20180221" "20180222" "20180223" "20180224" "20180225" "20180226" "20180227" "20180228" "20180301" "20180302" "20180303" "20180304" "20180305" "20180306" "20180307" "20180308" "20180309" "20180310" "20180311" "20180312" "20180313" "20180314" "20180315" "20180316" "20180317" "20180318" "20180319" "20180320" "20180321" "20180322" "20180323" "20180324" "20180325" "20180326" "20180327" "20180328" "20180329" "20180330" "20180331" "20180401" "20180402" "20180403" "20180404" "20180405" "20180406" "20180407" "20180408" "20180409" "20180410" "20180411" "20180412" "20180413" "20180414" "20180415" "20180416" "20180417" "20180418" "20180419" "20180420" "20180421" "20180422" "20180423" "20180424" "20180425" "20180426" "20180427" "20180428" "20180429" "20180430" "20180501" "20180502" "20180503" "20180504" "20180505" "20180506" "20180507" "20180508" "20180509" "20180510" "20180511" "20180512" "20180513" "20180514" "20180515" "20180516" "20180517" "20180518" "20180519" "20180520" "20180521" "20180522" "20180523" "20180524" "20180525" "20180526" "20180527" "20180528" "20180529" "20180530" "20180531" "20180601" "20180602" "20180603" "20180604" "20180605" "20180606" "20180607" "20180608" "20180609" "20180610" "20180611" "20180612" "20180613" "20180614" "20180615" "20180616" "20180617" "20180618" "20180619" "20180620" "20180621" "20180622" "20180623" "20180624" "20180625" "20180626" "20180627" "20180628" "20180629" "20180630" "20180701" "20180702" "20180703" "20180704" "20180705" "20180706" "20180707" "20180708" "20180709" "20180710" "20180711" "20180712" "20180713" "20180714" "20180715" "20180716" "20180717" "20180718" "20180719" "20180720" "20180721" "20180722" "20180723" "20180724" "20180725" "20180726" "20180727" "20180728" "20180729" "20180730" "20180731" "20180801" "20180802" "20180803" "20180804" "20180805" "20180806" "20180807" "20180808" "20180809" "20180810" "20180811" "20180812" "20180813" "20180814" "20180815" "20180816" "20180817" "20180818" "20180819" "20180820" "20180821" "20180822" "20180823" "20180824" "20180825" "20180826" "20180827" "20180828" "20180829" "20180830" "20180831" "20180901" "20180902" "20180903" "20180904" "20180905" "20180906" "20180907" "20180908" "20180909" "20180910" "20180911" "20180912" "20180913" "20180914" "20180915" "20180916" "20180917" "20180918" "20180919" "20180920" "20180921" "20180922" "20180923" "20180924" "20180925" "20180926" "20180927" "20180928" "20180929" "20180930" "20181001" "20181002" "20181003" "20181004" "20181005" "20181006" "20181007" "20181008" "20181009" "20181010" "20181011" "20181012" "20181013" "20181014" "20181015" "20181016" "20181017" "20181018" "20181019" "20181020" "20181021" "20181022" "20181023" "20181024" "20181025" "20181026" "20181027" "20181028" "20181029" "20181030" "20181031" "20181101" "20181102" "20181103" "20181104" "20181105" "20181106" "20181107" "20181108" "20181109" "20181110" "20181111" "20181112" "20181113" "20181114" "20181115" "20181116" "20181117" "20181118" "20181119" "20181120" "20181121" "20181122" "20181123" "20181124" "20181125" "20181126" "20181127" "20181128" "20181129" "20181130" "20181201" "20181202" "20181203" "20181204" "20181205" "20181206" "20181207" "20181208" "20181209" "20181210" "20181211" "20181212" "20181213" "20181214" "20181215" "20181216" "20181217" "20181218" "20181219" "20181220" "20181221" "20181222" "20181223" "20181224" "20181225" "20181226" "20181227" "20181228" "20181229" "20181230" "20181231") 18 | 19 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 20 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 21 | 22 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/run_prelim_tracking.py" \ 23 | --input_refl_dir_name="${TOP_RADAR_DIR_NAME}" \ 24 | --input_echo_classifn_dir_name="${TOP_RADAR_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --critical_refl_dbz=40 \ 28 | --min_echo_top_m_asl=4000 \ 29 | --min_size_pixels=5 \ 30 | --min_intermax_distance_metres=11112 \ 31 | --max_velocity_diff_m_s01=30 \ 32 | --max_link_distance_m_s01=38.5833333333333 \ 33 | --output_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" 34 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/tracks_to_masks_final_2016.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="tracks_to_masks_final_2016" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=8G 7 | #SBATCH --partition="swat_plus" 8 | #SBATCH --time=04:00:00 9 | #SBATCH --array=0-365 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=tracks_to_masks_final_2016_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/final" 16 | TOP_MASK_DIR_NAME="${TOP_TRACKING_DIR_NAME}/masks" 17 | 18 | DATE_STRINGS=("20160101" "20160102" "20160103" "20160104" "20160105" "20160106" "20160107" "20160108" "20160109" "20160110" "20160111" "20160112" "20160113" "20160114" "20160115" "20160116" "20160117" "20160118" "20160119" "20160120" "20160121" "20160122" "20160123" "20160124" "20160125" "20160126" "20160127" "20160128" "20160129" "20160130" "20160131" "20160201" "20160202" "20160203" "20160204" "20160205" "20160206" "20160207" "20160208" "20160209" "20160210" "20160211" "20160212" "20160213" "20160214" "20160215" "20160216" "20160217" "20160218" "20160219" "20160220" "20160221" "20160222" "20160223" "20160224" "20160225" "20160226" "20160227" "20160228" "20160229" "20160301" "20160302" "20160303" "20160304" "20160305" "20160306" "20160307" "20160308" "20160309" "20160310" "20160311" "20160312" "20160313" "20160314" "20160315" "20160316" "20160317" "20160318" "20160319" "20160320" "20160321" "20160322" "20160323" "20160324" "20160325" "20160326" "20160327" "20160328" "20160329" "20160330" "20160331" "20160401" "20160402" "20160403" "20160404" "20160405" "20160406" "20160407" "20160408" "20160409" "20160410" "20160411" "20160412" "20160413" "20160414" "20160415" "20160416" "20160417" "20160418" "20160419" "20160420" "20160421" "20160422" "20160423" "20160424" "20160425" "20160426" "20160427" "20160428" "20160429" "20160430" "20160501" "20160502" "20160503" "20160504" "20160505" "20160506" "20160507" "20160508" "20160509" "20160510" "20160511" "20160512" "20160513" "20160514" "20160515" "20160516" "20160517" "20160518" "20160519" "20160520" "20160521" "20160522" "20160523" "20160524" "20160525" "20160526" "20160527" "20160528" "20160529" "20160530" "20160531" "20160601" "20160602" "20160603" "20160604" "20160605" "20160606" "20160607" "20160608" "20160609" "20160610" "20160611" "20160612" "20160613" "20160614" "20160615" "20160616" "20160617" "20160618" "20160619" "20160620" "20160621" "20160622" "20160623" "20160624" "20160625" "20160626" "20160627" "20160628" "20160629" "20160630" "20160701" "20160702" "20160703" "20160704" "20160705" "20160706" "20160707" "20160708" "20160709" "20160710" "20160711" "20160712" "20160713" "20160714" "20160715" "20160716" "20160717" "20160718" "20160719" "20160720" "20160721" "20160722" "20160723" "20160724" "20160725" "20160726" "20160727" "20160728" "20160729" "20160730" "20160731" "20160801" "20160802" "20160803" "20160804" "20160805" "20160806" "20160807" "20160808" "20160809" "20160810" "20160811" "20160812" "20160813" "20160814" "20160815" "20160816" "20160817" "20160818" "20160819" "20160820" "20160821" "20160822" "20160823" "20160824" "20160825" "20160826" "20160827" "20160828" "20160829" "20160830" "20160831" "20160901" "20160902" "20160903" "20160904" "20160905" "20160906" "20160907" "20160908" "20160909" "20160910" "20160911" "20160912" "20160913" "20160914" "20160915" "20160916" "20160917" "20160918" "20160919" "20160920" "20160921" "20160922" "20160923" "20160924" "20160925" "20160926" "20160927" "20160928" "20160929" "20160930" "20161001" "20161002" "20161003" "20161004" "20161005" "20161006" "20161007" "20161008" "20161009" "20161010" "20161011" "20161012" "20161013" "20161014" "20161015" "20161016" "20161017" "20161018" "20161019" "20161020" "20161021" "20161022" "20161023" "20161024" "20161025" "20161026" "20161027" "20161028" "20161029" "20161030" "20161031" "20161101" "20161102" "20161103" "20161104" "20161105" "20161106" "20161107" "20161108" "20161109" "20161110" "20161111" "20161112" "20161113" "20161114" "20161115" "20161116" "20161117" "20161118" "20161119" "20161120" "20161121" "20161122" "20161123" "20161124" "20161125" "20161126" "20161127" "20161128" "20161129" "20161130" "20161201" "20161202" "20161203" "20161204" "20161205" "20161206" "20161207" "20161208" "20161209" "20161210" "20161211" "20161212" "20161213" "20161214" "20161215" "20161216" "20161217" "20161218" "20161219" "20161220" "20161221" "20161222" "20161223" "20161224" "20161225" "20161226" "20161227" "20161228" "20161229" "20161230" "20161231") 19 | 20 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 21 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 22 | 23 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/tracks_to_convection_masks.py" \ 24 | --input_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --output_mask_dir_name="${TOP_MASK_DIR_NAME}" 28 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/tracks_to_masks_final_2017.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="tracks_to_masks_final_2017" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=8G 7 | #SBATCH --partition="swat_plus" 8 | #SBATCH --time=04:00:00 9 | #SBATCH --array=0-364 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=tracks_to_masks_final_2017_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/final" 16 | TOP_MASK_DIR_NAME="${TOP_TRACKING_DIR_NAME}/masks" 17 | 18 | DATE_STRINGS=("20170101" "20170102" "20170103" "20170104" "20170105" "20170106" "20170107" "20170108" "20170109" "20170110" "20170111" "20170112" "20170113" "20170114" "20170115" "20170116" "20170117" "20170118" "20170119" "20170120" "20170121" "20170122" "20170123" "20170124" "20170125" "20170126" "20170127" "20170128" "20170129" "20170130" "20170131" "20170201" "20170202" "20170203" "20170204" "20170205" "20170206" "20170207" "20170208" "20170209" "20170210" "20170211" "20170212" "20170213" "20170214" "20170215" "20170216" "20170217" "20170218" "20170219" "20170220" "20170221" "20170222" "20170223" "20170224" "20170225" "20170226" "20170227" "20170228" "20170301" "20170302" "20170303" "20170304" "20170305" "20170306" "20170307" "20170308" "20170309" "20170310" "20170311" "20170312" "20170313" "20170314" "20170315" "20170316" "20170317" "20170318" "20170319" "20170320" "20170321" "20170322" "20170323" "20170324" "20170325" "20170326" "20170327" "20170328" "20170329" "20170330" "20170331" "20170401" "20170402" "20170403" "20170404" "20170405" "20170406" "20170407" "20170408" "20170409" "20170410" "20170411" "20170412" "20170413" "20170414" "20170415" "20170416" "20170417" "20170418" "20170419" "20170420" "20170421" "20170422" "20170423" "20170424" "20170425" "20170426" "20170427" "20170428" "20170429" "20170430" "20170501" "20170502" "20170503" "20170504" "20170505" "20170506" "20170507" "20170508" "20170509" "20170510" "20170511" "20170512" "20170513" "20170514" "20170515" "20170516" "20170517" "20170518" "20170519" "20170520" "20170521" "20170522" "20170523" "20170524" "20170525" "20170526" "20170527" "20170528" "20170529" "20170530" "20170531" "20170601" "20170602" "20170603" "20170604" "20170605" "20170606" "20170607" "20170608" "20170609" "20170610" "20170611" "20170612" "20170613" "20170614" "20170615" "20170616" "20170617" "20170618" "20170619" "20170620" "20170621" "20170622" "20170623" "20170624" "20170625" "20170626" "20170627" "20170628" "20170629" "20170630" "20170701" "20170702" "20170703" "20170704" "20170705" "20170706" "20170707" "20170708" "20170709" "20170710" "20170711" "20170712" "20170713" "20170714" "20170715" "20170716" "20170717" "20170718" "20170719" "20170720" "20170721" "20170722" "20170723" "20170724" "20170725" "20170726" "20170727" "20170728" "20170729" "20170730" "20170731" "20170801" "20170802" "20170803" "20170804" "20170805" "20170806" "20170807" "20170808" "20170809" "20170810" "20170811" "20170812" "20170813" "20170814" "20170815" "20170816" "20170817" "20170818" "20170819" "20170820" "20170821" "20170822" "20170823" "20170824" "20170825" "20170826" "20170827" "20170828" "20170829" "20170830" "20170831" "20170901" "20170902" "20170903" "20170904" "20170905" "20170906" "20170907" "20170908" "20170909" "20170910" "20170911" "20170912" "20170913" "20170914" "20170915" "20170916" "20170917" "20170918" "20170919" "20170920" "20170921" "20170922" "20170923" "20170924" "20170925" "20170926" "20170927" "20170928" "20170929" "20170930" "20171001" "20171002" "20171003" "20171004" "20171005" "20171006" "20171007" "20171008" "20171009" "20171010" "20171011" "20171012" "20171013" "20171014" "20171015" "20171016" "20171017" "20171018" "20171019" "20171020" "20171021" "20171022" "20171023" "20171024" "20171025" "20171026" "20171027" "20171028" "20171029" "20171030" "20171031" "20171101" "20171102" "20171103" "20171104" "20171105" "20171106" "20171107" "20171108" "20171109" "20171110" "20171111" "20171112" "20171113" "20171114" "20171115" "20171116" "20171117" "20171118" "20171119" "20171120" "20171121" "20171122" "20171123" "20171124" "20171125" "20171126" "20171127" "20171128" "20171129" "20171130" "20171201" "20171202" "20171203" "20171204" "20171205" "20171206" "20171207" "20171208" "20171209" "20171210" "20171211" "20171212" "20171213" "20171214" "20171215" "20171216" "20171217" "20171218" "20171219" "20171220" "20171221" "20171222" "20171223" "20171224" "20171225" "20171226" "20171227" "20171228" "20171229" "20171230" "20171231") 19 | 20 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 21 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 22 | 23 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/tracks_to_convection_masks.py" \ 24 | --input_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --output_mask_dir_name="${TOP_MASK_DIR_NAME}" 28 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/tracks_to_masks_final_2018.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="tracks_to_masks_final_2018" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=8G 7 | #SBATCH --partition="swat_plus" 8 | #SBATCH --time=04:00:00 9 | #SBATCH --array=0-364 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=tracks_to_masks_final_2018_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/final" 16 | TOP_MASK_DIR_NAME="${TOP_TRACKING_DIR_NAME}/masks" 17 | 18 | DATE_STRINGS=("20180101" "20180102" "20180103" "20180104" "20180105" "20180106" "20180107" "20180108" "20180109" "20180110" "20180111" "20180112" "20180113" "20180114" "20180115" "20180116" "20180117" "20180118" "20180119" "20180120" "20180121" "20180122" "20180123" "20180124" "20180125" "20180126" "20180127" "20180128" "20180129" "20180130" "20180131" "20180201" "20180202" "20180203" "20180204" "20180205" "20180206" "20180207" "20180208" "20180209" "20180210" "20180211" "20180212" "20180213" "20180214" "20180215" "20180216" "20180217" "20180218" "20180219" "20180220" "20180221" "20180222" "20180223" "20180224" "20180225" "20180226" "20180227" "20180228" "20180301" "20180302" "20180303" "20180304" "20180305" "20180306" "20180307" "20180308" "20180309" "20180310" "20180311" "20180312" "20180313" "20180314" "20180315" "20180316" "20180317" "20180318" "20180319" "20180320" "20180321" "20180322" "20180323" "20180324" "20180325" "20180326" "20180327" "20180328" "20180329" "20180330" "20180331" "20180401" "20180402" "20180403" "20180404" "20180405" "20180406" "20180407" "20180408" "20180409" "20180410" "20180411" "20180412" "20180413" "20180414" "20180415" "20180416" "20180417" "20180418" "20180419" "20180420" "20180421" "20180422" "20180423" "20180424" "20180425" "20180426" "20180427" "20180428" "20180429" "20180430" "20180501" "20180502" "20180503" "20180504" "20180505" "20180506" "20180507" "20180508" "20180509" "20180510" "20180511" "20180512" "20180513" "20180514" "20180515" "20180516" "20180517" "20180518" "20180519" "20180520" "20180521" "20180522" "20180523" "20180524" "20180525" "20180526" "20180527" "20180528" "20180529" "20180530" "20180531" "20180601" "20180602" "20180603" "20180604" "20180605" "20180606" "20180607" "20180608" "20180609" "20180610" "20180611" "20180612" "20180613" "20180614" "20180615" "20180616" "20180617" "20180618" "20180619" "20180620" "20180621" "20180622" "20180623" "20180624" "20180625" "20180626" "20180627" "20180628" "20180629" "20180630" "20180701" "20180702" "20180703" "20180704" "20180705" "20180706" "20180707" "20180708" "20180709" "20180710" "20180711" "20180712" "20180713" "20180714" "20180715" "20180716" "20180717" "20180718" "20180719" "20180720" "20180721" "20180722" "20180723" "20180724" "20180725" "20180726" "20180727" "20180728" "20180729" "20180730" "20180731" "20180801" "20180802" "20180803" "20180804" "20180805" "20180806" "20180807" "20180808" "20180809" "20180810" "20180811" "20180812" "20180813" "20180814" "20180815" "20180816" "20180817" "20180818" "20180819" "20180820" "20180821" "20180822" "20180823" "20180824" "20180825" "20180826" "20180827" "20180828" "20180829" "20180830" "20180831" "20180901" "20180902" "20180903" "20180904" "20180905" "20180906" "20180907" "20180908" "20180909" "20180910" "20180911" "20180912" "20180913" "20180914" "20180915" "20180916" "20180917" "20180918" "20180919" "20180920" "20180921" "20180922" "20180923" "20180924" "20180925" "20180926" "20180927" "20180928" "20180929" "20180930" "20181001" "20181002" "20181003" "20181004" "20181005" "20181006" "20181007" "20181008" "20181009" "20181010" "20181011" "20181012" "20181013" "20181014" "20181015" "20181016" "20181017" "20181018" "20181019" "20181020" "20181021" "20181022" "20181023" "20181024" "20181025" "20181026" "20181027" "20181028" "20181029" "20181030" "20181031" "20181101" "20181102" "20181103" "20181104" "20181105" "20181106" "20181107" "20181108" "20181109" "20181110" "20181111" "20181112" "20181113" "20181114" "20181115" "20181116" "20181117" "20181118" "20181119" "20181120" "20181121" "20181122" "20181123" "20181124" "20181125" "20181126" "20181127" "20181128" "20181129" "20181130" "20181201" "20181202" "20181203" "20181204" "20181205" "20181206" "20181207" "20181208" "20181209" "20181210" "20181211" "20181212" "20181213" "20181214" "20181215" "20181216" "20181217" "20181218" "20181219" "20181220" "20181221" "20181222" "20181223" "20181224" "20181225" "20181226" "20181227" "20181228" "20181229" "20181230" "20181231") 19 | 20 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 21 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 22 | 23 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/tracks_to_convection_masks.py" \ 24 | --input_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --output_mask_dir_name="${TOP_MASK_DIR_NAME}" 28 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/tracks_to_masks_prelim_2016.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="tracks_to_masks_prelim_2016" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=8G 7 | #SBATCH --partition="swat_plus" 8 | #SBATCH --time=01:00:00 9 | #SBATCH --array=0-365 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=tracks_to_masks_prelim_2016_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/prelim" 16 | TOP_MASK_DIR_NAME="${TOP_TRACKING_DIR_NAME}/masks" 17 | 18 | DATE_STRINGS=("20160101" "20160102" "20160103" "20160104" "20160105" "20160106" "20160107" "20160108" "20160109" "20160110" "20160111" "20160112" "20160113" "20160114" "20160115" "20160116" "20160117" "20160118" "20160119" "20160120" "20160121" "20160122" "20160123" "20160124" "20160125" "20160126" "20160127" "20160128" "20160129" "20160130" "20160131" "20160201" "20160202" "20160203" "20160204" "20160205" "20160206" "20160207" "20160208" "20160209" "20160210" "20160211" "20160212" "20160213" "20160214" "20160215" "20160216" "20160217" "20160218" "20160219" "20160220" "20160221" "20160222" "20160223" "20160224" "20160225" "20160226" "20160227" "20160228" "20160229" "20160301" "20160302" "20160303" "20160304" "20160305" "20160306" "20160307" "20160308" "20160309" "20160310" "20160311" "20160312" "20160313" "20160314" "20160315" "20160316" "20160317" "20160318" "20160319" "20160320" "20160321" "20160322" "20160323" "20160324" "20160325" "20160326" "20160327" "20160328" "20160329" "20160330" "20160331" "20160401" "20160402" "20160403" "20160404" "20160405" "20160406" "20160407" "20160408" "20160409" "20160410" "20160411" "20160412" "20160413" "20160414" "20160415" "20160416" "20160417" "20160418" "20160419" "20160420" "20160421" "20160422" "20160423" "20160424" "20160425" "20160426" "20160427" "20160428" "20160429" "20160430" "20160501" "20160502" "20160503" "20160504" "20160505" "20160506" "20160507" "20160508" "20160509" "20160510" "20160511" "20160512" "20160513" "20160514" "20160515" "20160516" "20160517" "20160518" "20160519" "20160520" "20160521" "20160522" "20160523" "20160524" "20160525" "20160526" "20160527" "20160528" "20160529" "20160530" "20160531" "20160601" "20160602" "20160603" "20160604" "20160605" "20160606" "20160607" "20160608" "20160609" "20160610" "20160611" "20160612" "20160613" "20160614" "20160615" "20160616" "20160617" "20160618" "20160619" "20160620" "20160621" "20160622" "20160623" "20160624" "20160625" "20160626" "20160627" "20160628" "20160629" "20160630" "20160701" "20160702" "20160703" "20160704" "20160705" "20160706" "20160707" "20160708" "20160709" "20160710" "20160711" "20160712" "20160713" "20160714" "20160715" "20160716" "20160717" "20160718" "20160719" "20160720" "20160721" "20160722" "20160723" "20160724" "20160725" "20160726" "20160727" "20160728" "20160729" "20160730" "20160731" "20160801" "20160802" "20160803" "20160804" "20160805" "20160806" "20160807" "20160808" "20160809" "20160810" "20160811" "20160812" "20160813" "20160814" "20160815" "20160816" "20160817" "20160818" "20160819" "20160820" "20160821" "20160822" "20160823" "20160824" "20160825" "20160826" "20160827" "20160828" "20160829" "20160830" "20160831" "20160901" "20160902" "20160903" "20160904" "20160905" "20160906" "20160907" "20160908" "20160909" "20160910" "20160911" "20160912" "20160913" "20160914" "20160915" "20160916" "20160917" "20160918" "20160919" "20160920" "20160921" "20160922" "20160923" "20160924" "20160925" "20160926" "20160927" "20160928" "20160929" "20160930" "20161001" "20161002" "20161003" "20161004" "20161005" "20161006" "20161007" "20161008" "20161009" "20161010" "20161011" "20161012" "20161013" "20161014" "20161015" "20161016" "20161017" "20161018" "20161019" "20161020" "20161021" "20161022" "20161023" "20161024" "20161025" "20161026" "20161027" "20161028" "20161029" "20161030" "20161031" "20161101" "20161102" "20161103" "20161104" "20161105" "20161106" "20161107" "20161108" "20161109" "20161110" "20161111" "20161112" "20161113" "20161114" "20161115" "20161116" "20161117" "20161118" "20161119" "20161120" "20161121" "20161122" "20161123" "20161124" "20161125" "20161126" "20161127" "20161128" "20161129" "20161130" "20161201" "20161202" "20161203" "20161204" "20161205" "20161206" "20161207" "20161208" "20161209" "20161210" "20161211" "20161212" "20161213" "20161214" "20161215" "20161216" "20161217" "20161218" "20161219" "20161220" "20161221" "20161222" "20161223" "20161224" "20161225" "20161226" "20161227" "20161228" "20161229" "20161230" "20161231") 19 | 20 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 21 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 22 | 23 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/tracks_to_convection_masks.py" \ 24 | --input_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --output_mask_dir_name="${TOP_MASK_DIR_NAME}" 28 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/tracks_to_masks_prelim_2017.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="tracks_to_masks_prelim_2017" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=8G 7 | #SBATCH --partition="swat_plus" 8 | #SBATCH --time=01:00:00 9 | #SBATCH --array=0-364 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=tracks_to_masks_prelim_2017_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/prelim" 16 | TOP_MASK_DIR_NAME="${TOP_TRACKING_DIR_NAME}/masks" 17 | 18 | DATE_STRINGS=("20170101" "20170102" "20170103" "20170104" "20170105" "20170106" "20170107" "20170108" "20170109" "20170110" "20170111" "20170112" "20170113" "20170114" "20170115" "20170116" "20170117" "20170118" "20170119" "20170120" "20170121" "20170122" "20170123" "20170124" "20170125" "20170126" "20170127" "20170128" "20170129" "20170130" "20170131" "20170201" "20170202" "20170203" "20170204" "20170205" "20170206" "20170207" "20170208" "20170209" "20170210" "20170211" "20170212" "20170213" "20170214" "20170215" "20170216" "20170217" "20170218" "20170219" "20170220" "20170221" "20170222" "20170223" "20170224" "20170225" "20170226" "20170227" "20170228" "20170301" "20170302" "20170303" "20170304" "20170305" "20170306" "20170307" "20170308" "20170309" "20170310" "20170311" "20170312" "20170313" "20170314" "20170315" "20170316" "20170317" "20170318" "20170319" "20170320" "20170321" "20170322" "20170323" "20170324" "20170325" "20170326" "20170327" "20170328" "20170329" "20170330" "20170331" "20170401" "20170402" "20170403" "20170404" "20170405" "20170406" "20170407" "20170408" "20170409" "20170410" "20170411" "20170412" "20170413" "20170414" "20170415" "20170416" "20170417" "20170418" "20170419" "20170420" "20170421" "20170422" "20170423" "20170424" "20170425" "20170426" "20170427" "20170428" "20170429" "20170430" "20170501" "20170502" "20170503" "20170504" "20170505" "20170506" "20170507" "20170508" "20170509" "20170510" "20170511" "20170512" "20170513" "20170514" "20170515" "20170516" "20170517" "20170518" "20170519" "20170520" "20170521" "20170522" "20170523" "20170524" "20170525" "20170526" "20170527" "20170528" "20170529" "20170530" "20170531" "20170601" "20170602" "20170603" "20170604" "20170605" "20170606" "20170607" "20170608" "20170609" "20170610" "20170611" "20170612" "20170613" "20170614" "20170615" "20170616" "20170617" "20170618" "20170619" "20170620" "20170621" "20170622" "20170623" "20170624" "20170625" "20170626" "20170627" "20170628" "20170629" "20170630" "20170701" "20170702" "20170703" "20170704" "20170705" "20170706" "20170707" "20170708" "20170709" "20170710" "20170711" "20170712" "20170713" "20170714" "20170715" "20170716" "20170717" "20170718" "20170719" "20170720" "20170721" "20170722" "20170723" "20170724" "20170725" "20170726" "20170727" "20170728" "20170729" "20170730" "20170731" "20170801" "20170802" "20170803" "20170804" "20170805" "20170806" "20170807" "20170808" "20170809" "20170810" "20170811" "20170812" "20170813" "20170814" "20170815" "20170816" "20170817" "20170818" "20170819" "20170820" "20170821" "20170822" "20170823" "20170824" "20170825" "20170826" "20170827" "20170828" "20170829" "20170830" "20170831" "20170901" "20170902" "20170903" "20170904" "20170905" "20170906" "20170907" "20170908" "20170909" "20170910" "20170911" "20170912" "20170913" "20170914" "20170915" "20170916" "20170917" "20170918" "20170919" "20170920" "20170921" "20170922" "20170923" "20170924" "20170925" "20170926" "20170927" "20170928" "20170929" "20170930" "20171001" "20171002" "20171003" "20171004" "20171005" "20171006" "20171007" "20171008" "20171009" "20171010" "20171011" "20171012" "20171013" "20171014" "20171015" "20171016" "20171017" "20171018" "20171019" "20171020" "20171021" "20171022" "20171023" "20171024" "20171025" "20171026" "20171027" "20171028" "20171029" "20171030" "20171031" "20171101" "20171102" "20171103" "20171104" "20171105" "20171106" "20171107" "20171108" "20171109" "20171110" "20171111" "20171112" "20171113" "20171114" "20171115" "20171116" "20171117" "20171118" "20171119" "20171120" "20171121" "20171122" "20171123" "20171124" "20171125" "20171126" "20171127" "20171128" "20171129" "20171130" "20171201" "20171202" "20171203" "20171204" "20171205" "20171206" "20171207" "20171208" "20171209" "20171210" "20171211" "20171212" "20171213" "20171214" "20171215" "20171216" "20171217" "20171218" "20171219" "20171220" "20171221" "20171222" "20171223" "20171224" "20171225" "20171226" "20171227" "20171228" "20171229" "20171230" "20171231") 19 | 20 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 21 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 22 | 23 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/tracks_to_convection_masks.py" \ 24 | --input_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --output_mask_dir_name="${TOP_MASK_DIR_NAME}" 28 | -------------------------------------------------------------------------------- /ml4convection/slurm/tracking/tracks_to_masks_prelim_2018.qsub: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | #SBATCH --job-name="tracks_to_masks_prelim_2018" 4 | #SBATCH --ntasks=1 5 | #SBATCH --nodes=1 6 | #SBATCH --mem=8G 7 | #SBATCH --partition="swat_plus" 8 | #SBATCH --time=01:00:00 9 | #SBATCH --array=0-364 10 | #SBATCH --mail-type=BEGIN,END,FAIL 11 | #SBATCH --mail-user=ryan.lagerquist@ou.edu 12 | #SBATCH --output=tracks_to_masks_prelim_2018_%A_%a.out 13 | 14 | TOP_RADAR_DIR_NAME="/condo/swatwork/ralager/taiwan/radar_data" 15 | TOP_TRACKING_DIR_NAME="${TOP_RADAR_DIR_NAME}/storm_tracking/prelim" 16 | TOP_MASK_DIR_NAME="${TOP_TRACKING_DIR_NAME}/masks" 17 | 18 | DATE_STRINGS=("20180101" "20180102" "20180103" "20180104" "20180105" "20180106" "20180107" "20180108" "20180109" "20180110" "20180111" "20180112" "20180113" "20180114" "20180115" "20180116" "20180117" "20180118" "20180119" "20180120" "20180121" "20180122" "20180123" "20180124" "20180125" "20180126" "20180127" "20180128" "20180129" "20180130" "20180131" "20180201" "20180202" "20180203" "20180204" "20180205" "20180206" "20180207" "20180208" "20180209" "20180210" "20180211" "20180212" "20180213" "20180214" "20180215" "20180216" "20180217" "20180218" "20180219" "20180220" "20180221" "20180222" "20180223" "20180224" "20180225" "20180226" "20180227" "20180228" "20180301" "20180302" "20180303" "20180304" "20180305" "20180306" "20180307" "20180308" "20180309" "20180310" "20180311" "20180312" "20180313" "20180314" "20180315" "20180316" "20180317" "20180318" "20180319" "20180320" "20180321" "20180322" "20180323" "20180324" "20180325" "20180326" "20180327" "20180328" "20180329" "20180330" "20180331" "20180401" "20180402" "20180403" "20180404" "20180405" "20180406" "20180407" "20180408" "20180409" "20180410" "20180411" "20180412" "20180413" "20180414" "20180415" "20180416" "20180417" "20180418" "20180419" "20180420" "20180421" "20180422" "20180423" "20180424" "20180425" "20180426" "20180427" "20180428" "20180429" "20180430" "20180501" "20180502" "20180503" "20180504" "20180505" "20180506" "20180507" "20180508" "20180509" "20180510" "20180511" "20180512" "20180513" "20180514" "20180515" "20180516" "20180517" "20180518" "20180519" "20180520" "20180521" "20180522" "20180523" "20180524" "20180525" "20180526" "20180527" "20180528" "20180529" "20180530" "20180531" "20180601" "20180602" "20180603" "20180604" "20180605" "20180606" "20180607" "20180608" "20180609" "20180610" "20180611" "20180612" "20180613" "20180614" "20180615" "20180616" "20180617" "20180618" "20180619" "20180620" "20180621" "20180622" "20180623" "20180624" "20180625" "20180626" "20180627" "20180628" "20180629" "20180630" "20180701" "20180702" "20180703" "20180704" "20180705" "20180706" "20180707" "20180708" "20180709" "20180710" "20180711" "20180712" "20180713" "20180714" "20180715" "20180716" "20180717" "20180718" "20180719" "20180720" "20180721" "20180722" "20180723" "20180724" "20180725" "20180726" "20180727" "20180728" "20180729" "20180730" "20180731" "20180801" "20180802" "20180803" "20180804" "20180805" "20180806" "20180807" "20180808" "20180809" "20180810" "20180811" "20180812" "20180813" "20180814" "20180815" "20180816" "20180817" "20180818" "20180819" "20180820" "20180821" "20180822" "20180823" "20180824" "20180825" "20180826" "20180827" "20180828" "20180829" "20180830" "20180831" "20180901" "20180902" "20180903" "20180904" "20180905" "20180906" "20180907" "20180908" "20180909" "20180910" "20180911" "20180912" "20180913" "20180914" "20180915" "20180916" "20180917" "20180918" "20180919" "20180920" "20180921" "20180922" "20180923" "20180924" "20180925" "20180926" "20180927" "20180928" "20180929" "20180930" "20181001" "20181002" "20181003" "20181004" "20181005" "20181006" "20181007" "20181008" "20181009" "20181010" "20181011" "20181012" "20181013" "20181014" "20181015" "20181016" "20181017" "20181018" "20181019" "20181020" "20181021" "20181022" "20181023" "20181024" "20181025" "20181026" "20181027" "20181028" "20181029" "20181030" "20181031" "20181101" "20181102" "20181103" "20181104" "20181105" "20181106" "20181107" "20181108" "20181109" "20181110" "20181111" "20181112" "20181113" "20181114" "20181115" "20181116" "20181117" "20181118" "20181119" "20181120" "20181121" "20181122" "20181123" "20181124" "20181125" "20181126" "20181127" "20181128" "20181129" "20181130" "20181201" "20181202" "20181203" "20181204" "20181205" "20181206" "20181207" "20181208" "20181209" "20181210" "20181211" "20181212" "20181213" "20181214" "20181215" "20181216" "20181217" "20181218" "20181219" "20181220" "20181221" "20181222" "20181223" "20181224" "20181225" "20181226" "20181227" "20181228" "20181229" "20181230" "20181231") 19 | 20 | this_date_string=${DATE_STRINGS[$SLURM_ARRAY_TASK_ID]} 21 | echo "Array index = ${SLURM_ARRAY_TASK_ID} ... date = ${this_date_string}" 22 | 23 | /home/ralager/anaconda3/bin/python3.6 -u "/condo/swatwork/ralager/ml4convection/ml4convection/scripts/tracks_to_convection_masks.py" \ 24 | --input_tracking_dir_name="${TOP_TRACKING_DIR_NAME}" \ 25 | --first_date_string="${this_date_string}" \ 26 | --last_date_string="${this_date_string}" \ 27 | --output_mask_dir_name="${TOP_MASK_DIR_NAME}" 28 | -------------------------------------------------------------------------------- /ml4convection/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thunderhoser/ml4convection/cf5e9682bc3182305274132ae246bc72d994308f/ml4convection/utils/__init__.py -------------------------------------------------------------------------------- /ml4convection/utils/radar_utils.py: -------------------------------------------------------------------------------- 1 | """Helper methods for radar data.""" 2 | 3 | import numpy 4 | from gewittergefahr.gg_utils import longitude_conversion as lng_conversion 5 | from gewittergefahr.gg_utils import error_checking 6 | 7 | RADAR_LATITUDES_DEG_N = numpy.array([25.07, 23.989, 23.1467, 21.899]) 8 | RADAR_LONGITUDES_DEG_E = numpy.array([121.77, 121.619, 120.086, 120.849]) 9 | 10 | 11 | def radar_sites_to_grid_points(grid_latitudes_deg_n, grid_longitudes_deg_e): 12 | """For each radar site, finds nearest grid point. 13 | 14 | M = number of rows in grid 15 | N = number of columns in grid 16 | R = number of radar sites 17 | 18 | :param grid_latitudes_deg_n: length-M numpy array of latitudes (deg N). 19 | :param grid_longitudes_deg_e: length-N numpy array of longitudes (deg E). 20 | :return: row_indices: length-R numpy array of grid rows. 21 | :return: column_indices: length-R numpy array of grid columns. 22 | """ 23 | 24 | # Basic input-checking. 25 | error_checking.assert_is_numpy_array(grid_latitudes_deg_n, num_dimensions=1) 26 | error_checking.assert_is_valid_lat_numpy_array(grid_latitudes_deg_n) 27 | error_checking.assert_is_greater_numpy_array( 28 | numpy.diff(grid_latitudes_deg_n), 0. 29 | ) 30 | 31 | error_checking.assert_is_numpy_array( 32 | grid_longitudes_deg_e, num_dimensions=1 33 | ) 34 | grid_longitudes_deg_e = ( 35 | lng_conversion.convert_lng_positive_in_west(grid_longitudes_deg_e) 36 | ) 37 | error_checking.assert_is_greater_numpy_array( 38 | numpy.diff(grid_longitudes_deg_e), 0. 39 | ) 40 | 41 | # Make sure that all radars are inside the domain. 42 | error_checking.assert_is_greater( 43 | numpy.min(RADAR_LATITUDES_DEG_N), grid_latitudes_deg_n[0] 44 | ) 45 | error_checking.assert_is_less_than( 46 | numpy.max(RADAR_LATITUDES_DEG_N), grid_latitudes_deg_n[-1] 47 | ) 48 | error_checking.assert_is_greater( 49 | numpy.min(RADAR_LONGITUDES_DEG_E), grid_longitudes_deg_e[0] 50 | ) 51 | error_checking.assert_is_less_than( 52 | numpy.max(RADAR_LONGITUDES_DEG_E), grid_longitudes_deg_e[-1] 53 | ) 54 | 55 | row_indices = numpy.array([ 56 | numpy.argmin(numpy.absolute(l - grid_latitudes_deg_n)) 57 | for l in RADAR_LATITUDES_DEG_N 58 | ], dtype=int) 59 | 60 | column_indices = numpy.array([ 61 | numpy.argmin(numpy.absolute(l - grid_longitudes_deg_e)) 62 | for l in RADAR_LONGITUDES_DEG_E 63 | ], dtype=int) 64 | 65 | return row_indices, column_indices 66 | -------------------------------------------------------------------------------- /ml4convection/utils/radar_utils_test.py: -------------------------------------------------------------------------------- 1 | """Unit tests for radar_utils.py.""" 2 | 3 | import unittest 4 | import numpy 5 | from ml4convection.utils import radar_utils 6 | 7 | FIRST_GRID_LATITUDES_DEG_N = numpy.linspace(20, 30, num=21, dtype=float) 8 | FIRST_GRID_LONGITUDES_DEG_E = numpy.linspace(115, 125, num=11, dtype=float) 9 | FIRST_ROW_INDICES = numpy.array([10, 8, 6, 4], dtype=int) 10 | FIRST_COLUMN_INDICES = numpy.array([7, 7, 5, 6], dtype=int) 11 | 12 | SECOND_GRID_LATITUDES_DEG_N = numpy.linspace(20, 30, num=41, dtype=float) 13 | SECOND_GRID_LONGITUDES_DEG_E = numpy.linspace(115, 125, num=41, dtype=float) 14 | SECOND_ROW_INDICES = numpy.array([20, 16, 13, 8], dtype=int) 15 | SECOND_COLUMN_INDICES = numpy.array([27, 26, 20, 23], dtype=int) 16 | 17 | 18 | class RadarUtilsTests(unittest.TestCase): 19 | """Each method is a unit test for radar_utils.py.""" 20 | 21 | def test_radar_sites_to_grid_points_first(self): 22 | """Ensures correct output from radar_sites_to_grid_points. 23 | 24 | In this case, using first grid. 25 | """ 26 | 27 | these_row_indices, these_column_indices = ( 28 | radar_utils.radar_sites_to_grid_points( 29 | grid_latitudes_deg_n=FIRST_GRID_LATITUDES_DEG_N, 30 | grid_longitudes_deg_e=FIRST_GRID_LONGITUDES_DEG_E 31 | ) 32 | ) 33 | 34 | self.assertTrue(numpy.array_equal( 35 | these_row_indices, FIRST_ROW_INDICES 36 | )) 37 | self.assertTrue(numpy.array_equal( 38 | these_column_indices, FIRST_COLUMN_INDICES 39 | )) 40 | 41 | def test_radar_sites_to_grid_points_second(self): 42 | """Ensures correct output from radar_sites_to_grid_points. 43 | 44 | In this case, using second grid. 45 | """ 46 | 47 | these_row_indices, these_column_indices = ( 48 | radar_utils.radar_sites_to_grid_points( 49 | grid_latitudes_deg_n=SECOND_GRID_LATITUDES_DEG_N, 50 | grid_longitudes_deg_e=SECOND_GRID_LONGITUDES_DEG_E 51 | ) 52 | ) 53 | 54 | self.assertTrue(numpy.array_equal( 55 | these_row_indices, SECOND_ROW_INDICES 56 | )) 57 | self.assertTrue(numpy.array_equal( 58 | these_column_indices, SECOND_COLUMN_INDICES 59 | )) 60 | 61 | 62 | if __name__ == '__main__': 63 | unittest.main() 64 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Setup file for ml4convection.""" 2 | 3 | from setuptools import setup 4 | 5 | PACKAGE_NAMES = [ 6 | 'ml4convection', 'ml4convection.io', 'ml4convection.machine_learning', 7 | 'ml4convection.plotting', 'ml4convection.utils', 'ml4convection.scripts', 8 | 'ml4convection.figures' 9 | ] 10 | KEYWORDS = [ 11 | 'machine learning', 'deep learning', 'artificial intelligence', 12 | 'data science', 'weather', 'meteorology', 'thunderstorm', 'convection', 13 | 'satellite', 'radar' 14 | ] 15 | SHORT_DESCRIPTION = ( 16 | 'Uses machine learning to predict convective initiation and decay from ' 17 | 'satellite data.' 18 | ) 19 | LONG_DESCRIPTION = SHORT_DESCRIPTION 20 | CLASSIFIERS = [ 21 | 'Development Status :: 2 - Pre-Alpha', 22 | 'Intended Audience :: Science/Research', 23 | 'Programming Language :: Python :: 3' 24 | ] 25 | 26 | # You also need to install the following packages, which are not available in 27 | # pip. They can both be installed by "git clone" and "python setup.py install", 28 | # the normal way one installs a GitHub package. 29 | # 30 | # https://github.com/matplotlib/basemap 31 | # https://github.com/tkrajina/srtm.py 32 | 33 | PACKAGE_REQUIREMENTS = [ 34 | 'numpy', 35 | 'scipy', 36 | 'tensorflow', 37 | 'keras', 38 | 'scikit-learn', 39 | 'scikit-image', 40 | 'netCDF4', 41 | 'pyproj', 42 | 'opencv-python', 43 | 'matplotlib', 44 | 'pandas', 45 | 'shapely', 46 | 'geopy', 47 | 'metpy' 48 | ] 49 | 50 | if __name__ == '__main__': 51 | setup( 52 | name='ml4convection', 53 | version='0.1', 54 | description=SHORT_DESCRIPTION, 55 | long_description=LONG_DESCRIPTION, 56 | author='Ryan Lagerquist', 57 | author_email='ryan.lagerquist@noaa.gov', 58 | url='https://github.com/thunderhoser/ml4convection', 59 | packages=PACKAGE_NAMES, 60 | scripts=[], 61 | keywords=KEYWORDS, 62 | classifiers=CLASSIFIERS, 63 | include_package_data=True, 64 | zip_safe=False, 65 | install_requires=PACKAGE_REQUIREMENTS 66 | ) 67 | --------------------------------------------------------------------------------