├── tests ├── __init__.py ├── utils │ ├── __init__.py │ └── get_test.py ├── simulation │ ├── __init__.py │ ├── test_base_simulation_problem.py │ └── test_plot_mixin.py ├── closed_loop │ ├── __init__.py │ ├── test_models │ │ ├── goal_programming_csv │ │ │ ├── input │ │ │ │ ├── fixed_inputs.json │ │ │ │ ├── initial_state.csv │ │ │ │ ├── closed_loop_dates.csv │ │ │ │ └── timeseries_import.csv │ │ │ ├── output_fixed_periods │ │ │ │ └── .gitignore │ │ │ ├── output │ │ │ │ ├── output_modelling_periods_reference │ │ │ │ │ ├── period_1 │ │ │ │ │ │ └── timeseries_export.csv │ │ │ │ │ ├── period_0 │ │ │ │ │ │ └── timeseries_export.csv │ │ │ │ │ └── period_2 │ │ │ │ │ │ └── timeseries_export.csv │ │ │ │ └── timeseries_export_reference.csv │ │ │ ├── model │ │ │ │ └── Example.mo │ │ │ └── src │ │ │ │ └── example.py │ │ └── goal_programming_xml │ │ │ ├── input │ │ │ ├── fixed_inputs.json │ │ │ ├── closed_loop_dates.csv │ │ │ ├── rtcParameterConfig.xml │ │ │ ├── rtcDataConfig.xml │ │ │ └── timeseries_import.xml │ │ │ ├── input_with_forecast_date │ │ │ ├── fixed_inputs.json │ │ │ ├── closed_loop_dates.csv │ │ │ ├── rtcParameterConfig.xml │ │ │ ├── rtcDataConfig.xml │ │ │ └── timeseries_import.xml │ │ │ ├── input_with_forecast_date_equal_first_date │ │ │ ├── fixed_inputs.json │ │ │ ├── closed_loop_dates.csv │ │ │ ├── rtcParameterConfig.xml │ │ │ ├── rtcDataConfig.xml │ │ │ └── timeseries_import.xml │ │ │ ├── output_fixed_periods │ │ │ └── .gitignore │ │ │ ├── model │ │ │ └── Example.mo │ │ │ ├── src │ │ │ └── example.py │ │ │ └── output │ │ │ └── output_modelling_periods_reference │ │ │ ├── period_0 │ │ │ └── timeseries_export.xml │ │ │ ├── period_2 │ │ │ └── timeseries_export.xml │ │ │ └── period_1 │ │ │ └── timeseries_export.xml │ ├── test_read_xml.py │ ├── data │ │ └── rtcDataConfig.xml │ ├── test_optimization_ranges.py │ └── test_run_optization_problem_closed_loop.py ├── optimization │ ├── __init__.py │ ├── test_read_goals.py │ ├── test_base_optimization_problem.py │ ├── test_passing_goals_directly.py │ └── test_plot_goals_mixin.py └── data │ ├── model_input │ ├── target_bounds_as_parameters │ │ ├── initial_state.csv │ │ ├── parameters.csv │ │ └── timeseries_import.csv │ ├── target_bounds_as_timeseries │ │ ├── initial_state.csv │ │ └── timeseries_import.csv │ └── basic │ │ └── timeseries_import.csv │ ├── plot_results │ ├── run_1.pickle │ └── run_2.pickle │ ├── simulation │ ├── tests.csv │ └── output │ │ └── basic │ │ └── timeseries_export.csv │ ├── goals │ ├── target_bounds_as_timeseries.csv │ ├── target_bounds_as_parameters.csv │ └── basic.csv │ ├── plot_table │ └── plot_table.csv │ ├── models │ └── BasicModel.mo │ └── optimization │ ├── tests.csv │ └── output │ ├── basic │ └── timeseries_export.csv │ ├── target_bounds_as_parameters │ └── timeseries_export.csv │ └── target_bounds_as_timeseries │ └── timeseries_export.csv ├── rtctools_interface ├── utils │ ├── __init__.py │ ├── read_goals_mixin.py │ ├── type_definitions.py │ ├── plot_table_schema.py │ ├── read_plot_table.py │ ├── serialization.py │ └── results_collection.py ├── closed_loop │ ├── __init__.py │ ├── config.py │ ├── results_construction.py │ ├── optimization_ranges.py │ ├── time_series_handler.py │ └── runner.py ├── optimization │ ├── __init__.py │ ├── helpers │ │ ├── __init__.py │ │ └── statistics_mixin.py │ ├── plot_goals_mixin.py │ ├── base_optimization_problem.py │ ├── plot_mixin.py │ ├── read_goals.py │ ├── goal_table_schema.py │ ├── goal_generator_mixin.py │ ├── goal_performance_metrics.py │ └── base_goal.py ├── plotting │ └── __init__.py ├── simulation │ ├── __init__.py │ ├── base_simulation_problem.py │ └── plot_mixin.py └── __init__.py ├── .gitattributes ├── .pre-commit-config.yaml ├── ruff.toml ├── .gitignore ├── setup.cfg ├── setup.py ├── .github └── workflows │ └── ci.yml └── COPYING.LESSER /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/simulation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/closed_loop/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/optimization/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rtctools_interface/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rtctools_interface/closed_loop/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rtctools_interface/optimization/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rtctools_interface/plotting/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rtctools_interface/simulation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rtctools_interface/optimization/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | rtctools_interface/_version.py export-subst 2 | -------------------------------------------------------------------------------- /tests/data/model_input/target_bounds_as_parameters/initial_state.csv: -------------------------------------------------------------------------------- 1 | x, 2 | 5.0, 3 | -------------------------------------------------------------------------------- /tests/data/model_input/target_bounds_as_timeseries/initial_state.csv: -------------------------------------------------------------------------------- 1 | x, 2 | 5.0, 3 | -------------------------------------------------------------------------------- /tests/data/model_input/target_bounds_as_parameters/parameters.csv: -------------------------------------------------------------------------------- 1 | p_min,p_max 2 | 5.0,15.0 -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_csv/input/fixed_inputs.json: -------------------------------------------------------------------------------- 1 | [ 2 | "target_1" 3 | ] -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input/fixed_inputs.json: -------------------------------------------------------------------------------- 1 | [ 2 | "target_1" 3 | ] -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input_with_forecast_date/fixed_inputs.json: -------------------------------------------------------------------------------- 1 | [ 2 | "target_1" 3 | ] -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_csv/input/initial_state.csv: -------------------------------------------------------------------------------- 1 | storage.V,Q_pump,Q_orifice 2 | 400000.0,0.0,0.0 3 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input_with_forecast_date_equal_first_date/fixed_inputs.json: -------------------------------------------------------------------------------- 1 | [ 2 | "target_1" 3 | ] -------------------------------------------------------------------------------- /tests/data/plot_results/run_1.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Deltares/rtc-tools-interface/main/tests/data/plot_results/run_1.pickle -------------------------------------------------------------------------------- /tests/data/plot_results/run_2.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Deltares/rtc-tools-interface/main/tests/data/plot_results/run_2.pickle -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_csv/output_fixed_periods/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore all files except this one. 2 | * 3 | !.gitignore 4 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/output_fixed_periods/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore all files except this one. 2 | * 3 | !.gitignore 4 | -------------------------------------------------------------------------------- /tests/data/simulation/tests.csv: -------------------------------------------------------------------------------- 1 | test,model_folder,model_name,model_input_folder,plot_table_file,output_folder 2 | basic,.,BasicModel,basic,plot_table.csv,basic -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input/closed_loop_dates.csv: -------------------------------------------------------------------------------- 1 | start_date,end_date 2 | 2024-05-01,2024-05-03 3 | 2024-05-03,2024-05-05 4 | 2024-05-05,2024-05-07 5 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_csv/input/closed_loop_dates.csv: -------------------------------------------------------------------------------- 1 | start_date,end_date 2 | 2024-05-01,2024-05-03 3 | 2024-05-03,2024-05-04 4 | 2024-05-04,2024-05-07 5 | 6 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input_with_forecast_date/closed_loop_dates.csv: -------------------------------------------------------------------------------- 1 | start_date,end_date 2 | 2024-05-01,2024-05-03 3 | 2024-05-03,2024-05-05 4 | 2024-05-05,2024-05-07 5 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input_with_forecast_date_equal_first_date/closed_loop_dates.csv: -------------------------------------------------------------------------------- 1 | start_date,end_date 2 | 2024-05-01,2024-05-03 3 | 2024-05-03,2024-05-05 4 | 2024-05-05,2024-05-07 5 | -------------------------------------------------------------------------------- /rtctools_interface/__init__.py: -------------------------------------------------------------------------------- 1 | from . import _version 2 | 3 | __version__ = _version.get_versions()["version"] 4 | 5 | from rtctools_interface.closed_loop.runner import run_optimization_problem_closed_loop 6 | 7 | __all__ = ["run_optimization_problem_closed_loop"] 8 | -------------------------------------------------------------------------------- /tests/data/goals/target_bounds_as_timeseries.csv: -------------------------------------------------------------------------------- 1 | id,state,active,goal_type,function_min,function_max,function_nominal,target_data_type,target_min,target_max,priority,weight,order 2 | goal_1,x,1,range,0,15,10,timeseries,s,s,10,, 3 | goal_2,u,1,minimization_path,,,,,,,20,,2 4 | -------------------------------------------------------------------------------- /tests/data/goals/target_bounds_as_parameters.csv: -------------------------------------------------------------------------------- 1 | id,state,active,goal_type,function_min,function_max,function_nominal,target_data_type,target_min,target_max,priority,weight,order 2 | goal_1,x,1,range,0,20,10,parameter,p_min,p_max,10,, 3 | goal_2,u,1,minimization_path,,,,,,,20,,2 4 | -------------------------------------------------------------------------------- /tests/data/plot_table/plot_table.csv: -------------------------------------------------------------------------------- 1 | id,y_axis_title,variables_style_1,variables_style_2,variables_with_previous_result,custom_title,specified_in 2 | goal_1,Volume ($m^3$),,,,,goal_generator 3 | goal_2,Volume ($m^3$),,,,,goal_generator 4 | goal_3,"y-axis label",,,x,Test Title,python -------------------------------------------------------------------------------- /tests/closed_loop/test_read_xml.py: -------------------------------------------------------------------------------- 1 | """Tests for the base optimization problem class.""" 2 | 3 | import unittest 4 | 5 | 6 | class TestReadXml(unittest.TestCase): 7 | """Placeholder for testing reading XML files.""" 8 | 9 | def test_read_xml(self): 10 | """TODO""" 11 | -------------------------------------------------------------------------------- /tests/data/models/BasicModel.mo: -------------------------------------------------------------------------------- 1 | model BasicModel 2 | input Real u(fixed=false, min=-10, max=15); 3 | input Real f(fixed=true, min=-10, max=15); 4 | parameter Real scale = 3600; 5 | output Real x(start=5); 6 | 7 | equation 8 | der(x) = (-u + f) / scale; 9 | 10 | end BasicModel; 11 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/astral-sh/ruff-pre-commit 3 | # Ruff version. 4 | rev: v0.14.8 5 | hooks: 6 | # Run the linter. 7 | - id: ruff-check 8 | args: [ --fix ] 9 | # Run the formatter. 10 | - id: ruff-format 11 | -------------------------------------------------------------------------------- /tests/data/goals/basic.csv: -------------------------------------------------------------------------------- 1 | id,state,active,goal_type,function_min,function_max,function_nominal,target_data_type,target_min,target_max,priority,weight,order 2 | goal_1,x,1,range,0,15,10,value,5.0,10.0,10,, 3 | goal_2,u,1,minimization_path,,,,,,,20,,2 4 | goal_3,u,1,range_rate_of_change,,,,value,-3,3,15,,1 5 | -------------------------------------------------------------------------------- /tests/data/model_input/basic/timeseries_import.csv: -------------------------------------------------------------------------------- 1 | Time,f, 2 | 2023-06-20 00:00:00,4.0, 3 | 2023-06-20 01:00:00,4.0, 4 | 2023-06-20 02:00:00,4.0, 5 | 2023-06-20 03:00:00,4.0, 6 | 2023-06-20 04:00:00,4.0, 7 | 2023-06-20 05:00:00,4.0, 8 | 2023-06-20 06:00:00,0.0, 9 | 2023-06-20 07:00:00,0.0, 10 | 2023-06-20 08:00:00,0.0, 11 | 2023-06-20 09:00:00,0.0, 12 | 2023-06-20 10:00:00,0.0, -------------------------------------------------------------------------------- /tests/data/model_input/target_bounds_as_parameters/timeseries_import.csv: -------------------------------------------------------------------------------- 1 | Time,f, 2 | 2023-06-20 00:00:00,4.0, 3 | 2023-06-20 01:00:00,4.0, 4 | 2023-06-20 02:00:00,4.0, 5 | 2023-06-20 03:00:00,4.0, 6 | 2023-06-20 04:00:00,4.0, 7 | 2023-06-20 05:00:00,4.0, 8 | 2023-06-20 06:00:00,0.0, 9 | 2023-06-20 07:00:00,0.0, 10 | 2023-06-20 08:00:00,0.0, 11 | 2023-06-20 09:00:00,0.0, 12 | 2023-06-20 10:00:00,0.0, -------------------------------------------------------------------------------- /tests/data/simulation/output/basic/timeseries_export.csv: -------------------------------------------------------------------------------- 1 | time,x 2 | 2023-06-20 00:00:00,5.000000 3 | 2023-06-20 01:00:00,8.000000 4 | 2023-06-20 02:00:00,11.000000 5 | 2023-06-20 03:00:00,14.000000 6 | 2023-06-20 04:00:00,17.000000 7 | 2023-06-20 05:00:00,20.000000 8 | 2023-06-20 06:00:00,19.000000 9 | 2023-06-20 07:00:00,18.000000 10 | 2023-06-20 08:00:00,17.000000 11 | 2023-06-20 09:00:00,16.000000 12 | 2023-06-20 10:00:00,15.000000 13 | -------------------------------------------------------------------------------- /tests/data/model_input/target_bounds_as_timeseries/timeseries_import.csv: -------------------------------------------------------------------------------- 1 | Time,f,s, 2 | 2023-06-20 00:00:00,4.0,5.0, 3 | 2023-06-20 01:00:00,4.0,5.0, 4 | 2023-06-20 02:00:00,4.0,7.0, 5 | 2023-06-20 03:00:00,4.0,9.0, 6 | 2023-06-20 04:00:00,4.0,9.0, 7 | 2023-06-20 05:00:00,4.0,10.0, 8 | 2023-06-20 06:00:00,0.0,10.0, 9 | 2023-06-20 07:00:00,0.0,10.0, 10 | 2023-06-20 08:00:00,0.0,10.0, 11 | 2023-06-20 09:00:00,0.0,10.0, 12 | 2023-06-20 10:00:00,0.0,10.0, -------------------------------------------------------------------------------- /ruff.toml: -------------------------------------------------------------------------------- 1 | line-length = 100 2 | exclude = ["versioneer.py", "rtctools_interface/_version.py"] 3 | 4 | target-version = "py310" 5 | 6 | [lint] 7 | 8 | ignore = [ 9 | "B904", # Fix someday: raising exceptions within except 10 | ] 11 | select = [ 12 | "B", # flake8-bugbear 13 | "C4", # flake8-comprehensions 14 | "E", # default / pycodestyle 15 | "F", # default / pyflakes 16 | "I", # isort 17 | "UP", # pyupgrade 18 | "W", # pycodestyle 19 | ] 20 | -------------------------------------------------------------------------------- /tests/data/optimization/tests.csv: -------------------------------------------------------------------------------- 1 | test,model_folder,model_name,model_input_folder,goals_file,plot_table_file,output_folder 2 | basic,.,BasicModel,basic,basic.csv,plot_table.csv,basic 3 | target_bounds_as_parameters,.,BasicModel,target_bounds_as_parameters,target_bounds_as_parameters.csv,plot_table.csv,target_bounds_as_parameters 4 | target_bounds_as_timeseries,.,BasicModel,target_bounds_as_timeseries,target_bounds_as_timeseries.csv,plot_table.csv,target_bounds_as_timeseries -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.c 7 | *.so 8 | 9 | # Distribution & Packaging 10 | .eggs 11 | build 12 | dist 13 | 14 | # Testing & Coverage 15 | .pytest_cache 16 | .tox 17 | .coverage 18 | *.egg-info 19 | 20 | # Model caches 21 | *.pymoca_cache 22 | 23 | # Editors 24 | .idea 25 | .ipynb_checkpoints 26 | .vscode 27 | 28 | # Figures & results 29 | *.png 30 | *.html 31 | *.pickle 32 | *.csv 33 | *.json 34 | output/ 35 | input_modelling_periods/ -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | license_file = COPYING.LESSER 3 | 4 | # See the docstring in versioneer.py for instructions. Note that you must 5 | # re-run 'versioneer.py setup' after changing this section, and commit the 6 | # resulting files. 7 | 8 | [versioneer] 9 | VCS = git 10 | style = pep440 11 | versionfile_source = rtctools_interface/_version.py 12 | parentdir_prefix = rtctools_interface- 13 | 14 | [flake8] 15 | max-line-length = 120 16 | exclude = 17 | # generated files 18 | rtctools_interface/_version.py 19 | -------------------------------------------------------------------------------- /tests/data/optimization/output/basic/timeseries_export.csv: -------------------------------------------------------------------------------- 1 | time,u,x 2 | 2023-06-20 00:00:00,1.199915,4.999782 3 | 2023-06-20 01:00:00,2.324915,6.674866 4 | 2023-06-20 02:00:00,3.449916,7.224951 5 | 2023-06-20 03:00:00,3.487373,7.737578 6 | 2023-06-20 04:00:00,3.431187,8.306391 7 | 2023-06-20 05:00:00,2.306187,10.000204 8 | 2023-06-20 06:00:00,1.181186,8.819018 9 | 2023-06-20 07:00:00,0.056186,8.762831 10 | 2023-06-20 08:00:00,0.000000,8.762831 11 | 2023-06-20 09:00:00,0.000000,8.762831 12 | 2023-06-20 10:00:00,0.000000,8.762831 13 | -------------------------------------------------------------------------------- /rtctools_interface/optimization/plot_goals_mixin.py: -------------------------------------------------------------------------------- 1 | """Deprecated, use PlotMixin.""" 2 | 3 | import warnings 4 | 5 | from rtctools_interface.optimization.plot_mixin import PlotMixin 6 | 7 | 8 | class PlotGoalsMixin(PlotMixin): 9 | """ 10 | Deprecated class, use PlotMixin instead. 11 | """ 12 | 13 | def __init__(self, *args, **kwargs): 14 | warnings.warn( 15 | "PlotGoalsMixin is deprecated, use PlotMixin instead", FutureWarning, stacklevel=1 16 | ) 17 | super().__init__(*args, **kwargs) 18 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input/rtcParameterConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 0.0 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /tests/data/optimization/output/target_bounds_as_parameters/timeseries_export.csv: -------------------------------------------------------------------------------- 1 | time,u,x 2 | 2023-06-20 00:00:00,0.000000,5.000000 3 | 2023-06-20 01:00:00,1.999950,7.000050 4 | 2023-06-20 02:00:00,1.999950,9.000101 5 | 2023-06-20 03:00:00,1.999950,11.000151 6 | 2023-06-20 04:00:00,1.999950,13.000202 7 | 2023-06-20 05:00:00,1.999950,15.000252 8 | 2023-06-20 06:00:00,0.000070,15.000183 9 | 2023-06-20 07:00:00,0.000046,15.000136 10 | 2023-06-20 08:00:00,0.000032,15.000104 11 | 2023-06-20 09:00:00,0.000020,15.000084 12 | 2023-06-20 10:00:00,0.000010,15.000074 13 | -------------------------------------------------------------------------------- /tests/data/optimization/output/target_bounds_as_timeseries/timeseries_export.csv: -------------------------------------------------------------------------------- 1 | time,u,x 2 | 2023-06-20 00:00:00,0.000000,5.000000 3 | 2023-06-20 01:00:00,3.998886,5.001114 4 | 2023-06-20 02:00:00,2.001053,7.000061 5 | 2023-06-20 03:00:00,2.001053,8.999009 6 | 2023-06-20 04:00:00,3.998352,9.000657 7 | 2023-06-20 05:00:00,3.000139,10.000517 8 | 2023-06-20 06:00:00,0.000055,10.000462 9 | 2023-06-20 07:00:00,0.000031,10.000430 10 | 2023-06-20 08:00:00,0.000020,10.000410 11 | 2023-06-20 09:00:00,0.000013,10.000397 12 | 2023-06-20 10:00:00,0.000006,10.000391 13 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input_with_forecast_date/rtcParameterConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 0.0 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input_with_forecast_date_equal_first_date/rtcParameterConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 0.0 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /tests/optimization/test_read_goals.py: -------------------------------------------------------------------------------- 1 | """Tests for reading goals from a csv file.""" 2 | 3 | import pathlib 4 | import unittest 5 | 6 | from rtctools_interface.optimization.read_goals import read_goals 7 | 8 | CSV_FILE = pathlib.Path(__file__).parent.parent / "data" / "goals" / "basic.csv" 9 | 10 | 11 | class TestGoalReader(unittest.TestCase): 12 | """Test for reading goals.""" 13 | 14 | def test_read_csv(self): 15 | """Test for reading goals from csv.""" 16 | goals = read_goals(CSV_FILE, path_goal=True) 17 | self.assertEqual(len(goals), 3) 18 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_csv/output/output_modelling_periods_reference/period_1/timeseries_export.csv: -------------------------------------------------------------------------------- 1 | time,Q_orifice,Q_pump,is_downhill,sea_level,storage_level 2 | 2024-05-03 00:00:00,0.000000,5.019110,0.000000,0.600000,0.434967 3 | 2024-05-03 08:00:00,0.000000,5.005019,0.000000,0.700000,0.434823 4 | 2024-05-03 16:00:00,0.000000,5.001069,0.000000,0.800000,0.434792 5 | 2024-05-04 00:00:00,0.000000,4.991490,0.000000,0.900000,0.435037 6 | 2024-05-04 08:00:00,0.000000,4.827632,0.000000,1.000000,0.440001 7 | 2024-05-04 16:00:00,0.000000,4.999987,0.000000,0.900000,0.440002 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | import versioneer 4 | 5 | setup( 6 | name="rtc_tools_interface", 7 | version=versioneer.get_version(), 8 | maintainer="Deltares", 9 | packages=find_packages("."), 10 | author="Deltares", 11 | description="Toolbox for user interfaces for RTC-Tools", 12 | install_requires=[ 13 | "matplotlib", 14 | "numpy", 15 | "pandas", 16 | "plotly", 17 | "pydantic", 18 | "casadi != 3.6.6", 19 | "rtc-tools >= 2.7.0a3", 20 | ], 21 | tests_require=["pytest", "pytest-runner"], 22 | python_requires=">=3.9", 23 | cmdclass=versioneer.get_cmdclass(), 24 | ) 25 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_csv/output/output_modelling_periods_reference/period_0/timeseries_export.csv: -------------------------------------------------------------------------------- 1 | time,Q_orifice,Q_pump,is_downhill,sea_level,storage_level 2 | 2024-05-01 00:00:00,0.000000,0.000000,1.000000,0.000000,0.400000 3 | 2024-05-01 08:00:00,3.914858,0.000000,1.000000,0.100000,0.431252 4 | 2024-05-01 16:00:00,5.044142,0.000000,1.000000,0.200000,0.429981 5 | 2024-05-02 00:00:00,3.975639,0.676391,1.000000,0.300000,0.440002 6 | 2024-05-02 08:00:00,2.125126,2.874857,1.000000,0.400000,0.440003 7 | 2024-05-02 16:00:00,0.000000,5.155733,0.000000,0.500000,0.435518 8 | 2024-05-03 00:00:00,0.000000,5.019110,0.000000,0.600000,0.434967 9 | 2024-05-03 08:00:00,0.000000,4.825209,0.000000,0.700000,0.440001 10 | 2024-05-03 16:00:00,0.000000,4.999972,0.000000,0.800000,0.440002 11 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_csv/input/timeseries_import.csv: -------------------------------------------------------------------------------- 1 | UTC,H_sea,Q_in,target_1 2 | 2024-05-01 00:00:00,0.0,5.0,5.0 3 | 2024-05-01 08:00:00,0.1,5.0,5.0 4 | 2024-05-01 16:00:00,0.2,5.0,5.0 5 | 2024-05-02 00:00:00,0.3,5.0, 6 | 2024-05-02 08:00:00,0.4,5.0, 7 | 2024-05-02 16:00:00,0.5,5.0, 8 | 2024-05-03 00:00:00,0.6,5.0, 9 | 2024-05-03 08:00:00,0.7,5.0, 10 | 2024-05-03 16:00:00,0.8,5.0, 11 | 2024-05-04 00:00:00,0.9,5.0, 12 | 2024-05-04 08:00:00,1.0,5.0, 13 | 2024-05-04 16:00:00,0.9,5.0, 14 | 2024-05-05 00:00:00,0.8,5.0, 15 | 2024-05-05 08:00:00,0.7,5.0, 16 | 2024-05-05 16:00:00,0.6,5.0, 17 | 2024-05-06 00:00:00,0.5,5.0, 18 | 2024-05-06 08:00:00,0.4,5.0, 19 | 2024-05-06 16:00:00,0.3,5.0, 20 | 2024-05-07 00:00:00,0.2,5.0, 21 | 2024-05-07 08:00:00,0.1,5.0, 22 | 2024-05-07 16:00:00,0.0,5.0, 23 | -------------------------------------------------------------------------------- /rtctools_interface/simulation/base_simulation_problem.py: -------------------------------------------------------------------------------- 1 | """Module for a basic simulation problem.""" 2 | 3 | from rtctools.simulation.csv_mixin import CSVMixin 4 | from rtctools.simulation.simulation_problem import SimulationProblem 5 | 6 | 7 | class BaseSimulationProblem( 8 | CSVMixin, 9 | SimulationProblem, 10 | ): 11 | # Ignore too many ancestors, since the use of mixin classes is how rtc-tools is set up. 12 | # pylint: disable=too-many-ancestors 13 | """ 14 | Basic simulation problem for a given state. 15 | 16 | :cvar goal_table_file: 17 | path to csv file containing a list of goals. 18 | """ 19 | 20 | def __init__( 21 | self, 22 | **kwargs, 23 | ): 24 | super().__init__(**kwargs) 25 | 26 | def update(self, dt): 27 | self.set_var("u", 1) 28 | super().update(dt) 29 | 30 | def initialize(self): 31 | self.set_var("u", 1) 32 | super().initialize() 33 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_csv/output/output_modelling_periods_reference/period_2/timeseries_export.csv: -------------------------------------------------------------------------------- 1 | time,Q_orifice,Q_pump,is_downhill,sea_level,storage_level 2 | 2024-05-04 00:00:00,0.000000,4.991490,0.000000,0.900000,0.435037 3 | 2024-05-04 08:00:00,0.000000,5.004574,0.000000,1.000000,0.434905 4 | 2024-05-04 16:00:00,0.000000,5.000738,0.000000,0.900000,0.434884 5 | 2024-05-05 00:00:00,0.000000,5.002209,0.000000,0.800000,0.434820 6 | 2024-05-05 08:00:00,0.000000,5.025354,0.000000,0.700000,0.434090 7 | 2024-05-05 16:00:00,0.000000,5.062433,0.000000,0.600000,0.432292 8 | 2024-05-06 00:00:00,0.000000,4.790646,0.000000,0.500000,0.438322 9 | 2024-05-06 08:00:00,2.125115,2.816523,1.000000,0.400000,0.440002 10 | 2024-05-06 16:00:00,3.975637,1.024369,1.000000,0.300000,0.440002 11 | 2024-05-07 00:00:00,5.070749,0.000000,1.000000,0.200000,0.437965 12 | 2024-05-07 08:00:00,5.095738,0.000000,1.000000,0.100000,0.435207 13 | 2024-05-07 16:00:00,5.003009,0.000000,1.000000,0.000000,0.435121 14 | -------------------------------------------------------------------------------- /tests/simulation/test_base_simulation_problem.py: -------------------------------------------------------------------------------- 1 | """Tests for the base simulation problem class.""" 2 | 3 | import unittest 4 | 5 | from rtctools_interface.simulation.base_simulation_problem import BaseSimulationProblem 6 | from tests.utils.get_test import get_test_data 7 | 8 | 9 | class TestBasSimulationProblem(unittest.TestCase): 10 | """Test for the base simulation problem class.""" 11 | 12 | def run_test(self, test): 13 | """Solve an simulation problem.""" 14 | test_data = get_test_data(test, optimization=False) 15 | problem = BaseSimulationProblem( 16 | model_folder=test_data["model_folder"], 17 | model_name=test_data["model_name"], 18 | input_folder=test_data["model_input_folder"], 19 | output_folder=test_data["output_folder"], 20 | ) 21 | problem.simulate() 22 | 23 | # TODO: use pytest instead to parametrise tests. 24 | def test_base_simulation_problem(self): 25 | """Solve several simulation problems.""" 26 | for test in ["basic"]: 27 | self.run_test(test) 28 | -------------------------------------------------------------------------------- /tests/utils/get_test.py: -------------------------------------------------------------------------------- 1 | """Tools for getting optimization or simulation test data.""" 2 | 3 | import pathlib 4 | 5 | import pandas as pd 6 | 7 | DATA_DIR = pathlib.Path(__file__).parent.parent / "data" 8 | 9 | 10 | def get_test_data(test: str, optimization: bool = True) -> dict: 11 | """ 12 | Get the input data and output folder for a given test. 13 | """ 14 | sub_path = "optimization" if optimization else "simulation" 15 | tests_df = pd.read_csv(DATA_DIR / sub_path / "tests.csv", sep=",") 16 | tests_df.set_index("test", inplace=True) 17 | test_data = tests_df.loc[test] 18 | test_data_dict = { 19 | "model_folder": DATA_DIR / "models" / test_data["model_folder"], 20 | "model_name": test_data["model_name"], 21 | "model_input_folder": DATA_DIR / "model_input" / test_data["model_input_folder"], 22 | "plot_table_file": DATA_DIR / "plot_table" / test_data["plot_table_file"], 23 | "output_folder": DATA_DIR / sub_path / "output" / test_data["output_folder"], 24 | } 25 | if optimization: 26 | test_data_dict["goals_file"] = DATA_DIR / "goals" / test_data["goals_file"] 27 | return test_data_dict 28 | -------------------------------------------------------------------------------- /tests/optimization/test_base_optimization_problem.py: -------------------------------------------------------------------------------- 1 | """Tests for the base optimization problem class.""" 2 | 3 | import unittest 4 | 5 | from rtctools_interface.optimization.base_optimization_problem import BaseOptimizationProblem 6 | from tests.utils.get_test import get_test_data 7 | 8 | 9 | class TestBaseOptimizationProblem(unittest.TestCase): 10 | """Test for the base optimization problem class.""" 11 | 12 | def run_test(self, test): 13 | """Solve an optimization problem.""" 14 | test_data = get_test_data(test, optimization=True) 15 | problem = BaseOptimizationProblem( 16 | goal_table_file=test_data["goals_file"], 17 | model_folder=test_data["model_folder"], 18 | model_name=test_data["model_name"], 19 | input_folder=test_data["model_input_folder"], 20 | output_folder=test_data["output_folder"], 21 | ) 22 | problem.optimize() 23 | 24 | # TODO: use pytest instead to parametrise tests. 25 | def test_base_optimization_problem(self): 26 | """Solve several optimization problems.""" 27 | for test in ["basic", "target_bounds_as_parameters", "target_bounds_as_timeseries"]: 28 | self.run_test(test) 29 | -------------------------------------------------------------------------------- /rtctools_interface/optimization/base_optimization_problem.py: -------------------------------------------------------------------------------- 1 | """Module for a basic optimization problem.""" 2 | 3 | from rtctools.optimization.collocated_integrated_optimization_problem import ( 4 | CollocatedIntegratedOptimizationProblem, 5 | ) 6 | from rtctools.optimization.csv_mixin import CSVMixin 7 | from rtctools.optimization.goal_programming_mixin import GoalProgrammingMixin 8 | from rtctools.optimization.modelica_mixin import ModelicaMixin 9 | 10 | from rtctools_interface.optimization.goal_generator_mixin import GoalGeneratorMixin 11 | 12 | 13 | class BaseOptimizationProblem( 14 | GoalGeneratorMixin, 15 | GoalProgrammingMixin, 16 | CSVMixin, 17 | ModelicaMixin, 18 | CollocatedIntegratedOptimizationProblem, 19 | ): 20 | # Ignore too many ancestors, since the use of mixin classes is how rtc-tools is set up. 21 | # pylint: disable=too-many-ancestors 22 | """ 23 | Basic optimization problem for a given state. 24 | 25 | :cvar goal_table_file: 26 | path to csv file containing a list of goals. 27 | """ 28 | 29 | def __init__( 30 | self, 31 | goal_table_file=None, 32 | **kwargs, 33 | ): 34 | self.goal_table_file = goal_table_file 35 | super().__init__(**kwargs) 36 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_csv/output/timeseries_export_reference.csv: -------------------------------------------------------------------------------- 1 | time,Q_orifice,Q_pump,is_downhill,sea_level,storage_level 2 | 2024-05-01 00:00:00,0.0,0.0,1.0,0.0,0.4 3 | 2024-05-01 08:00:00,3.914858,0.0,1.0,0.1,0.431252 4 | 2024-05-01 16:00:00,5.044142,0.0,1.0,0.2,0.429981 5 | 2024-05-02 00:00:00,3.975639,0.676391,1.0,0.3,0.440002 6 | 2024-05-02 08:00:00,2.125126,2.874857,1.0,0.4,0.440003 7 | 2024-05-02 16:00:00,0.0,5.155733,0.0,0.5,0.435518 8 | 2024-05-03 00:00:00,0.0,5.01911,0.0,0.6,0.434967 9 | 2024-05-03 08:00:00,0.0,5.005019,0.0,0.7,0.434823 10 | 2024-05-03 16:00:00,0.0,5.001069,0.0,0.8,0.434792 11 | 2024-05-04 00:00:00,0.0,4.99149,0.0,0.9,0.435037 12 | 2024-05-04 08:00:00,0.0,5.004574,0.0,1.0,0.434905 13 | 2024-05-04 16:00:00,0.0,5.000738,0.0,0.9,0.434884 14 | 2024-05-05 00:00:00,0.0,5.002209,0.0,0.8,0.43482 15 | 2024-05-05 08:00:00,0.0,5.025354,0.0,0.7,0.43409 16 | 2024-05-05 16:00:00,0.0,5.062433,0.0,0.6,0.432292 17 | 2024-05-06 00:00:00,0.0,4.790646,0.0,0.5,0.438322 18 | 2024-05-06 08:00:00,2.125115,2.816523,1.0,0.4,0.440002 19 | 2024-05-06 16:00:00,3.975637,1.024369,1.0,0.3,0.440002 20 | 2024-05-07 00:00:00,5.070749,0.0,1.0,0.2,0.437965 21 | 2024-05-07 08:00:00,5.095738,0.0,1.0,0.1,0.435207 22 | 2024-05-07 16:00:00,5.003009,0.0,1.0,0.0,0.435121 23 | -------------------------------------------------------------------------------- /tests/optimization/test_passing_goals_directly.py: -------------------------------------------------------------------------------- 1 | """Tests for the base optimization problem class.""" 2 | 3 | import unittest 4 | 5 | from rtctools_interface.optimization.base_optimization_problem import BaseOptimizationProblem 6 | from rtctools_interface.optimization.read_goals import read_goals_from_csv 7 | from tests.utils.get_test import get_test_data 8 | 9 | 10 | class TestPassingGoalsDirectly(unittest.TestCase): 11 | """Test for the base optimization problem class.""" 12 | 13 | def run_test(self, test): 14 | """Solve an optimization problem.""" 15 | test_data = get_test_data(test, optimization=True) 16 | 17 | goals_to_generate = read_goals_from_csv(test_data["goals_file"]) 18 | problem = BaseOptimizationProblem( 19 | model_folder=test_data["model_folder"], 20 | model_name=test_data["model_name"], 21 | input_folder=test_data["model_input_folder"], 22 | output_folder=test_data["output_folder"], 23 | goals_to_generate=goals_to_generate, 24 | read_goals_from="passed_list", 25 | ) 26 | problem.optimize() 27 | 28 | # TODO: use pytest instead to parametrise tests. 29 | def test_base_optimization_problem(self): 30 | """Solve several optimization problems.""" 31 | for test in ["basic", "target_bounds_as_parameters", "target_bounds_as_timeseries"]: 32 | self.run_test(test) 33 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input/rtcDataConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | H_sea 6 | par_0 7 | 8 | 9 | 10 | 11 | Q_in 12 | par_0 13 | 14 | 15 | 16 | 17 | Q_orifice 18 | par_0 19 | 20 | 21 | 22 | 23 | Q_pump 24 | par_0 25 | 26 | 27 | 28 | 29 | is_downhill 30 | par_0 31 | 32 | 33 | 34 | 35 | sea_level 36 | par_0 37 | 38 | 39 | 40 | 41 | storage_level 42 | par_0 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input_with_forecast_date/rtcDataConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | H_sea 6 | par_0 7 | 8 | 9 | 10 | 11 | Q_in 12 | par_0 13 | 14 | 15 | 16 | 17 | Q_orifice 18 | par_0 19 | 20 | 21 | 22 | 23 | Q_pump 24 | par_0 25 | 26 | 27 | 28 | 29 | is_downhill 30 | par_0 31 | 32 | 33 | 34 | 35 | sea_level 36 | par_0 37 | 38 | 39 | 40 | 41 | storage_level 42 | par_0 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input_with_forecast_date_equal_first_date/rtcDataConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | H_sea 6 | par_0 7 | 8 | 9 | 10 | 11 | Q_in 12 | par_0 13 | 14 | 15 | 16 | 17 | Q_orifice 18 | par_0 19 | 20 | 21 | 22 | 23 | Q_pump 24 | par_0 25 | 26 | 27 | 28 | 29 | is_downhill 30 | par_0 31 | 32 | 33 | 34 | 35 | sea_level 36 | par_0 37 | 38 | 39 | 40 | 41 | storage_level 42 | par_0 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /rtctools_interface/utils/read_goals_mixin.py: -------------------------------------------------------------------------------- 1 | """Mixin to read the goal table and store as class variables.""" 2 | 3 | import os 4 | from typing import Literal 5 | 6 | from rtctools_interface.optimization.read_goals import read_goals 7 | 8 | 9 | class ReadGoalsMixin: 10 | """Read the goal table either from the default or specified path.""" 11 | 12 | def load_goals( 13 | self, 14 | read_from: Literal["csv_table", "passed_list"] = "csv_table", 15 | goals_to_generate: list | None = None, 16 | ): 17 | """Read goal table and store as instance variable.""" 18 | goals_to_generate = goals_to_generate if goals_to_generate else [] 19 | if not hasattr(self, "goal_table_file"): 20 | self.goal_table_file = os.path.join(self._input_folder, "goal_table.csv") 21 | 22 | if ( 23 | read_from == "csv_table" 24 | and os.path.isfile(self.goal_table_file) 25 | or read_from == "passed_list" 26 | ): 27 | self._goal_generator_path_goals = read_goals( 28 | self.goal_table_file, 29 | path_goal=True, 30 | read_from=read_from, 31 | goals_to_generate=goals_to_generate, 32 | ) 33 | self._goal_generator_non_path_goals = read_goals( 34 | self.goal_table_file, 35 | path_goal=False, 36 | read_from=read_from, 37 | goals_to_generate=goals_to_generate, 38 | ) 39 | self._all_goal_generator_goals = ( 40 | self._goal_generator_path_goals + self._goal_generator_non_path_goals 41 | ) 42 | else: 43 | self._all_goal_generator_goals = [] 44 | -------------------------------------------------------------------------------- /tests/closed_loop/data/rtcDataConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | TroutLake 7 | Inflow 8 | 9 | 10 | 11 | 12 | Alder 13 | Inflow 14 | 15 | 16 | 17 | 18 | TroutLake 19 | Volume 20 | 21 | 22 | 23 | 24 | TroutLake 25 | Q_release_user 26 | 27 | 28 | 29 | 30 | RiverCity 31 | Q 32 | 33 | 34 | 35 | 36 | TroutLake 37 | Q_out 38 | 39 | 40 | 41 | 42 | TroutLake 43 | Q_spill 44 | 45 | 46 | 47 | 48 | TroutLake 49 | Q_turbine 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /rtctools_interface/utils/type_definitions.py: -------------------------------------------------------------------------------- 1 | """Type definitions for rtc_tools interface.""" 2 | 3 | import datetime 4 | import pathlib 5 | from typing import Literal, TypedDict 6 | 7 | import numpy as np 8 | 9 | from rtctools_interface.utils.plot_table_schema import PlotTableRow 10 | 11 | 12 | class TargetDict(TypedDict): 13 | """Target min and max timeseries for a goal.""" 14 | 15 | target_min: np.ndarray 16 | target_max: np.ndarray 17 | 18 | 19 | class GoalConfig(TypedDict): 20 | """Configuration for a goal.""" 21 | 22 | goal_id: str 23 | state: str 24 | goal_type: str 25 | function_min: float | None 26 | function_max: float | None 27 | function_nominal: float | None 28 | target_min: tuple[float, np.ndarray] 29 | target_max: tuple[float, np.ndarray] 30 | target_min_series: np.ndarray | None 31 | target_max_series: np.ndarray | None 32 | priority: int 33 | weight: float 34 | order: int 35 | 36 | 37 | class PrioIndependentData(TypedDict): 38 | """Data for one optimization run, which is independent of the priority.""" 39 | 40 | io_datetimes: list[datetime.datetime] 41 | times: np.ndarray 42 | base_goals: list[GoalConfig] 43 | 44 | 45 | class PlotOptions(TypedDict): 46 | """Plot configuration for on optimization run.""" 47 | 48 | plot_config: list[PlotTableRow] 49 | plot_max_rows: int 50 | output_folder: pathlib.Path 51 | save_plot_to: Literal["image", "stringio"] 52 | 53 | 54 | class IntermediateResult(TypedDict): 55 | """Dict containing the results (timeseries) for one priority optimization.""" 56 | 57 | priority: int 58 | timeseries_data: dict[str, np.ndarray] 59 | 60 | 61 | class PlotDataAndConfig(TypedDict): 62 | """All data and options required to create all plots for one optimization run.""" 63 | 64 | intermediate_results: list[IntermediateResult] 65 | plot_options: PlotOptions 66 | prio_independent_data: PrioIndependentData 67 | config_version: float 68 | -------------------------------------------------------------------------------- /tests/simulation/test_plot_mixin.py: -------------------------------------------------------------------------------- 1 | """Tests for goal-plotting functionalities.""" 2 | 3 | import unittest 4 | 5 | from rtctools_interface.simulation.base_simulation_problem import BaseSimulationProblem 6 | from rtctools_interface.simulation.plot_mixin import PlotMixin 7 | from tests.utils.get_test import get_test_data 8 | 9 | 10 | class BaseSimulationProblemPlotting(PlotMixin, BaseSimulationProblem): 11 | # Ignore too many ancestors, since the use of mixin classes is 12 | # how rtc-tools is set up. 13 | # Ignore abstract-method not implemented, as this is related how 14 | # some todo's are setup in simulation mode. 15 | # pylint: disable=too-many-ancestors, abstract-method 16 | """Simulation problem with plotting functionalities.""" 17 | 18 | def __init__( 19 | self, 20 | plot_table_file, 21 | **kwargs, 22 | ): 23 | self.plot_table_file = plot_table_file 24 | super().__init__(**kwargs) 25 | 26 | 27 | class TestPlotMixin(unittest.TestCase): 28 | """Test for goal-plotting functionalities.""" 29 | 30 | def run_test(self, test, plotting_library): 31 | """Solve an simulation problem.""" 32 | test_data = get_test_data(test, optimization=False) 33 | problem = BaseSimulationProblemPlotting( 34 | plot_table_file=test_data["plot_table_file"], 35 | model_folder=test_data["model_folder"], 36 | model_name=test_data["model_name"], 37 | input_folder=test_data["model_input_folder"], 38 | output_folder=test_data["output_folder"], 39 | plotting_library=plotting_library, 40 | ) 41 | problem.simulate() 42 | 43 | def test_plot_goals_mixin_plotly(self): 44 | """Solve several simulation problems.""" 45 | for test in [ 46 | "basic", 47 | ]: 48 | self.run_test(test, plotting_library="plotly") 49 | 50 | def test_plot_goals_mixin_matplotlib(self): 51 | """Solve several simulation problems.""" 52 | for test in [ 53 | "basic", 54 | ]: 55 | self.run_test(test, plotting_library="matplotlib") 56 | -------------------------------------------------------------------------------- /rtctools_interface/utils/plot_table_schema.py: -------------------------------------------------------------------------------- 1 | """Schema for the plot_table.""" 2 | 3 | from typing import Literal 4 | 5 | import numpy as np 6 | import pandas as pd 7 | from pydantic import BaseModel, field_validator, model_validator 8 | 9 | 10 | def string_to_list(string): 11 | """ 12 | Convert a string to a list of strings 13 | """ 14 | if string == "" or not isinstance(string, str): 15 | return [] 16 | string_without_whitespace = string.replace(" ", "") 17 | list_of_strings = string_without_whitespace.split(",") 18 | return list_of_strings 19 | 20 | 21 | class PlotTableRow(BaseModel): 22 | """Model for one row in the plot table.""" 23 | 24 | specified_in: Literal["python", "goal_generator"] 25 | y_axis_title: str 26 | id: int | str | float = np.nan 27 | variables_style_1: list[str] = [] 28 | variables_style_2: list[str] = [] 29 | variables_with_previous_result: list[str] = [] 30 | custom_title: str | float = np.nan 31 | 32 | @field_validator( 33 | "variables_style_1", "variables_style_2", "variables_with_previous_result", mode="before" 34 | ) 35 | @classmethod 36 | def convert_to_list(cls, value): 37 | """Convert the inputs to a list.""" 38 | if isinstance(value, list): 39 | return value 40 | return string_to_list(value) 41 | 42 | @field_validator("id") 43 | @classmethod 44 | def convert_to_int(cls, value): 45 | """Convert value to integer if possible.""" 46 | try: 47 | return int(value) 48 | except (ValueError, TypeError): 49 | return value 50 | 51 | @model_validator(mode="after") 52 | def check_required_id(self): 53 | """Check if ID is present if specified in goal_generator.""" 54 | if self.specified_in == "goal_generator" and pd.isna(self.id): 55 | raise ValueError("ID is required when goal is specified in the goal generator.") 56 | return self 57 | 58 | def get(self, attribute_name, default=None): 59 | """Similar functionality as dict-get method.""" 60 | try: 61 | return getattr(self, attribute_name) 62 | except AttributeError: 63 | return default 64 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | 9 | linting: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v6 13 | - uses: actions/setup-python@v6 14 | - uses: pre-commit/action@v3.0.1 15 | 16 | test: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v3 20 | - uses: actions/setup-python@v4 21 | with: 22 | python-version: '3.10' 23 | - name: Cache pip 24 | uses: actions/cache@v3 25 | with: 26 | path: ~/.cache/pip 27 | key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} 28 | restore-keys: | 29 | ${{ runner.os }}-pip- 30 | - run: pip install -e . 31 | - run: pip install pytest 32 | - run: pytest tests 33 | 34 | build: 35 | runs-on: ubuntu-latest 36 | steps: 37 | - uses: actions/checkout@v3 38 | - uses: actions/setup-python@v4 39 | with: 40 | python-version: '3.10' 41 | - name: Cache pip 42 | uses: actions/cache@v3 43 | with: 44 | path: ~/.cache/pip 45 | key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} 46 | restore-keys: | 47 | ${{ runner.os }}-pip- 48 | - run: pip install setuptools wheel 49 | - run: python setup.py sdist bdist_wheel 50 | - uses: actions/upload-artifact@v4 51 | with: 52 | name: dist 53 | path: dist/ 54 | 55 | deploy: 56 | needs: build 57 | runs-on: ubuntu-latest 58 | if: startsWith(github.ref, 'refs/tags/') 59 | steps: 60 | - uses: actions/checkout@v3 61 | - uses: actions/setup-python@v4 62 | with: 63 | python-version: '3.10' 64 | - name: Cache pip 65 | uses: actions/cache@v3 66 | with: 67 | path: ~/.cache/pip 68 | key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} 69 | restore-keys: | 70 | ${{ runner.os }}-pip- 71 | - run: pip install twine 72 | - uses: actions/download-artifact@v4 73 | with: 74 | name: dist 75 | path: dist 76 | - run: twine upload -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }} dist/* 77 | -------------------------------------------------------------------------------- /rtctools_interface/utils/read_plot_table.py: -------------------------------------------------------------------------------- 1 | """Module for reading goals from a csv file.""" 2 | 3 | import logging 4 | from pathlib import Path 5 | 6 | import pandas as pd 7 | 8 | from rtctools_interface.utils.plot_table_schema import PlotTableRow 9 | 10 | logger = logging.getLogger("rtctools") 11 | 12 | 13 | def read_plot_config_from_csv(plot_table_file: Path | str) -> list[PlotTableRow]: 14 | """Read plot information from csv file and check values""" 15 | plot_table_file = Path(plot_table_file) 16 | if plot_table_file.is_file(): 17 | try: 18 | raw_plot_table = pd.read_csv(plot_table_file, sep=",") 19 | except pd.errors.EmptyDataError: # Empty plot table 20 | raw_plot_table = pd.DataFrame() 21 | parsed_rows: list[PlotTableRow] = [] 22 | for _, row in raw_plot_table.iterrows(): 23 | parsed_rows.append(PlotTableRow(**row)) 24 | return parsed_rows 25 | message = ( 26 | f"No plot table was found at the default location ({plot_table_file.resolve()})." 27 | + " Please create one before using the PlotMixin." 28 | + f" It should have the following columns: '{list(PlotTableRow.model_fields.keys())}'" 29 | ) 30 | raise FileNotFoundError(message) 31 | 32 | 33 | def read_plot_config_from_list(plot_config: list[PlotTableRow]) -> list[PlotTableRow]: 34 | """Read plot config from a list. Validates whether the elements are of correct type.""" 35 | if not isinstance(plot_config, list): 36 | raise TypeError(f"Pass a list of PlotTableRow elements, not a {type(plot_config)}") 37 | for plot_table_row in plot_config: 38 | if not isinstance(plot_table_row, PlotTableRow): 39 | raise TypeError( 40 | "Each element in the passed plot table should be of type 'PlotTableRow'" 41 | ) 42 | return plot_config 43 | 44 | 45 | def get_plot_config( 46 | plot_table_file=None, plot_config_list=None, read_from="csv_table" 47 | ) -> list[PlotTableRow]: 48 | """Get plot config rows.""" 49 | if read_from == "csv_table": 50 | return read_plot_config_from_csv(plot_table_file) 51 | if read_from == "passed_list": 52 | return read_plot_config_from_list(plot_config_list) 53 | raise ValueError("PlotMixin should either read from 'csv_table' or 'passed_list'") 54 | -------------------------------------------------------------------------------- /tests/optimization/test_plot_goals_mixin.py: -------------------------------------------------------------------------------- 1 | """Tests for goal-plotting functionalities.""" 2 | 3 | import unittest 4 | 5 | from rtctools_interface.optimization.base_optimization_problem import ( 6 | BaseOptimizationProblem, 7 | ) 8 | from rtctools_interface.optimization.plot_mixin import PlotMixin 9 | from tests.utils.get_test import get_test_data 10 | 11 | 12 | class BaseOptimizationProblemPlotting(PlotMixin, BaseOptimizationProblem): 13 | # Ignore too many ancestors, since the use of mixin classes is how rtc-tools is set up. 14 | # pylint: disable=too-many-ancestors 15 | """Optimization problem with plotting functionalities.""" 16 | 17 | def __init__( 18 | self, 19 | plot_table_file, 20 | goal_table_file, 21 | **kwargs, 22 | ): 23 | self.plot_table_file = plot_table_file 24 | super().__init__(goal_table_file=goal_table_file, **kwargs) 25 | 26 | 27 | class TestPlotMixin(unittest.TestCase): 28 | """Test for goal-plotting functionalities.""" 29 | 30 | def run_test(self, test, plotting_library): 31 | """Solve an optimization problem.""" 32 | test_data = get_test_data(test, optimization=True) 33 | problem = BaseOptimizationProblemPlotting( 34 | goal_table_file=test_data["goals_file"], 35 | plot_table_file=test_data["plot_table_file"], 36 | model_folder=test_data["model_folder"], 37 | model_name=test_data["model_name"], 38 | input_folder=test_data["model_input_folder"], 39 | output_folder=test_data["output_folder"], 40 | plotting_library=plotting_library, 41 | ) 42 | problem.optimize() 43 | 44 | def test_plot_goals_mixin_matplotlib(self): 45 | """Solve several optimization problems.""" 46 | for test in [ 47 | "basic", 48 | "target_bounds_as_parameters", 49 | "target_bounds_as_timeseries", 50 | ]: 51 | self.run_test(test, plotting_library="matplotlib") 52 | 53 | def test_plot_goals_mixin_plotly(self): 54 | """Solve several optimization problems.""" 55 | for test in [ 56 | "basic", 57 | "target_bounds_as_parameters", 58 | "target_bounds_as_timeseries", 59 | ]: 60 | self.run_test(test, plotting_library="plotly") 61 | -------------------------------------------------------------------------------- /rtctools_interface/optimization/plot_mixin.py: -------------------------------------------------------------------------------- 1 | """Mixin to store all required data for plotting. Can also call the plot function.""" 2 | 3 | import logging 4 | 5 | from rtctools_interface.optimization.base_goal import BaseGoal 6 | from rtctools_interface.optimization.helpers.statistics_mixin import StatisticsMixin 7 | from rtctools_interface.plotting.plot_tools import ( 8 | create_plot_each_priority, 9 | create_plot_final_results, 10 | ) 11 | from rtctools_interface.utils.results_collection import PlottingBaseMixin 12 | 13 | logger = logging.getLogger("rtctools") 14 | 15 | 16 | class PlotMixin(PlottingBaseMixin, StatisticsMixin): 17 | """ 18 | Class for plotting results. 19 | """ 20 | 21 | optimization_problem = True 22 | 23 | def priority_completed(self, priority: int) -> None: 24 | """Store priority-dependent results required for plotting.""" 25 | timeseries_data = self.collect_timeseries_data( 26 | list(set(self.custom_variables + self.state_variables)) 27 | ) 28 | to_store = {"timeseries_data": timeseries_data, "priority": priority} 29 | self._intermediate_results.append(to_store) 30 | super().priority_completed(priority) 31 | 32 | def post(self): 33 | """Tasks after optimizing. Creates a plot for for each priority.""" 34 | super().post() 35 | 36 | if self.solver_stats["success"]: 37 | base_goals = [ 38 | goal.get_goal_config() 39 | for goal in self.goals() + self.path_goals() 40 | if isinstance(goal, BaseGoal) 41 | ] 42 | current_run = self.create_plot_data_and_config(base_goals) 43 | # Cache results, such that in a next run they can be used for comparison 44 | self._store_current_results(self._cache_folder, current_run) 45 | 46 | # Create the plots 47 | plot_data = {} 48 | if self.plot_results_each_priority: 49 | plot_data = plot_data | create_plot_each_priority( 50 | current_run, plotting_library=self.plotting_library 51 | ) 52 | 53 | if self.plot_final_results: 54 | plot_data = plot_data | create_plot_final_results( 55 | current_run, self._previous_run, plotting_library=self.plotting_library 56 | ) 57 | -------------------------------------------------------------------------------- /rtctools_interface/utils/serialization.py: -------------------------------------------------------------------------------- 1 | """Methods to serialize and deserialize the PlotDataAndConfig objects.""" 2 | 3 | import datetime 4 | import json 5 | from pathlib import Path 6 | from typing import Any 7 | 8 | import numpy as np 9 | import pandas as pd 10 | 11 | from rtctools_interface.utils.plot_table_schema import PlotTableRow 12 | 13 | 14 | def custom_encoder(obj): 15 | """Custom JSON encoder for types not supported by default.""" 16 | if isinstance(obj, (datetime.datetime, datetime.date)): 17 | return { 18 | "__type__": "datetime" if isinstance(obj, datetime.datetime) else "date", 19 | "value": obj.isoformat(), 20 | } 21 | if isinstance(obj, np.ndarray): 22 | return {"__type__": "ndarray", "data": obj.tolist()} 23 | if isinstance(obj, Path): 24 | return {"__type__": "path", "value": str(obj)} 25 | if isinstance(obj, pd.DataFrame): 26 | return {"__type__": "pandas_dataframe", "value": obj.to_json()} 27 | if isinstance(obj, PlotTableRow): 28 | return {"__type__": "plot_table_row", "value": obj.model_dump()} 29 | if hasattr(obj, "dict"): 30 | return obj.dict() 31 | raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable") 32 | 33 | 34 | def serialize(object_to_serialize: Any) -> str: 35 | """Serialize an object to a JSON string.""" 36 | return json.dumps(object_to_serialize, default=custom_encoder) 37 | 38 | 39 | def custom_decoder(dct: Any) -> Any: 40 | """Custom JSON decoder for types not supported by default.""" 41 | # pylint: disable=too-many-return-statements 42 | if isinstance(dct, dict): 43 | for key, value in dct.items(): 44 | dct[key] = custom_decoder(value) # Recursively process each value 45 | if dct.get("__type__") == "datetime": 46 | return datetime.datetime.fromisoformat(dct["value"]) 47 | if dct.get("__type__") == "date": 48 | return datetime.date.fromisoformat(dct["value"]) 49 | if dct.get("__type__") == "ndarray": 50 | return np.array(dct["data"]) 51 | if dct.get("__type__") == "path" and dct["value"]: 52 | return Path(dct["value"]) 53 | if dct.get("__type__") == "pandas_dataframe": 54 | return pd.read_json(dct["value"]) 55 | if dct.get("__type__") == "plot_table_row": 56 | return PlotTableRow(**dct["value"]) 57 | return dct 58 | if isinstance(dct, list): 59 | return [custom_decoder(item) for item in dct] # Recursively process each item in the list 60 | return dct 61 | 62 | 63 | def deserialize(serialized_str: str) -> dict: 64 | """Deserialize the JSON string.""" 65 | return json.loads(serialized_str, object_hook=custom_decoder) 66 | -------------------------------------------------------------------------------- /tests/closed_loop/test_optimization_ranges.py: -------------------------------------------------------------------------------- 1 | """Tests for getting optimization periods.""" 2 | 3 | import unittest 4 | from datetime import datetime, timedelta 5 | 6 | import rtctools_interface.closed_loop.optimization_ranges as opt_ranges 7 | 8 | 9 | class TestGetOptimizationPeriods(unittest.TestCase): 10 | """Test calculating optimization periods.""" 11 | 12 | def check_ranges( 13 | self, 14 | ranges: list[tuple[datetime, datetime]], 15 | expected_ranges: list[tuple[datetime, datetime]], 16 | ): 17 | """Check if two lists of ranges are the same.""" 18 | n_exp_ranges = len(expected_ranges) 19 | n_ranges = len(ranges) 20 | self.assertEqual(n_ranges, n_exp_ranges) 21 | for i_period in range(n_ranges): 22 | exp_start, exp_end = expected_ranges[i_period] 23 | start, end = expected_ranges[i_period] 24 | self.assertEqual(start, exp_start) 25 | self.assertEqual(end, exp_end) 26 | 27 | def test_get_optimization_ranges(self): 28 | """Test get_optimization_ranges.""" 29 | model_times = [ 30 | datetime(2024, 1, 1), 31 | datetime(2024, 1, 2), 32 | datetime(2024, 1, 4), 33 | datetime(2024, 1, 5), 34 | datetime(2024, 1, 7), 35 | datetime(2024, 1, 8), 36 | datetime(2024, 1, 10), 37 | ] 38 | start_time = datetime(2024, 1, 2) 39 | forecast_timestep = timedelta(days=2) 40 | optimization_period = timedelta(days=5) 41 | ranges = opt_ranges.get_optimization_ranges( 42 | model_times=model_times, 43 | start_time=start_time, 44 | forecast_timestep=forecast_timestep, 45 | optimization_period=optimization_period, 46 | ) 47 | expected_ranges = [ 48 | (datetime(2024, 1, 2), datetime(2024, 1, 7)), 49 | (datetime(2024, 1, 4), datetime(2024, 1, 8)), 50 | (datetime(2024, 1, 5), datetime(2024, 1, 10)), 51 | ] 52 | self.check_ranges(ranges, expected_ranges) 53 | 54 | def test_round_datetime_ranges_to_days(self): 55 | """Test round_datetime_ranges_to_days.""" 56 | ranges = [ 57 | (datetime(2020, 1, 1), datetime(2020, 1, 2)), 58 | (datetime(2020, 1, 2), datetime(2020, 1, 2)), 59 | (datetime(2020, 1, 2, 12), datetime(2020, 1, 5, 23)), 60 | ] 61 | rounded_ranges = opt_ranges.round_datetime_ranges_to_days(ranges) 62 | expected_ranges = [ 63 | (datetime(2020, 1, 1), datetime(2020, 1, 2, 23, 59, 59, 999999)), 64 | (datetime(2020, 1, 2, 12), datetime(2020, 1, 5, 23, 59, 59, 999999)), 65 | ] 66 | self.check_ranges(rounded_ranges, expected_ranges) 67 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_csv/model/Example.mo: -------------------------------------------------------------------------------- 1 | // This example model is a modified version of the goal_programming example model of rtc-tools: https://gitlab.com/deltares/rtc-tools 2 | model Example 3 | // Declare Model Elements 4 | Deltares.ChannelFlow.Hydraulic.Storage.Linear storage(A=1.0e6, H_b=0.0, HQ.H(min=0.0, max=0.5)) annotation(Placement(visible = true, transformation(origin = {32.022, -0}, extent = {{-10, -10}, {10, 10}}, rotation = -270))); 5 | Deltares.ChannelFlow.Hydraulic.BoundaryConditions.Discharge discharge annotation(Placement(visible = true, transformation(origin = {60, -0}, extent = {{10, -10}, {-10, 10}}, rotation = 270))); 6 | Deltares.ChannelFlow.Hydraulic.BoundaryConditions.Level level annotation(Placement(visible = true, transformation(origin = {-52.7, 0}, extent = {{-10, -10}, {10, 10}}, rotation = -270))); 7 | Deltares.ChannelFlow.Hydraulic.Structures.Pump pump annotation(Placement(visible = true, transformation(origin = {0, -20}, extent = {{10, -10}, {-10, 10}}, rotation = 0))); 8 | Deltares.ChannelFlow.Hydraulic.Structures.Pump orifice annotation(Placement(visible = true, transformation(origin = {0, 20}, extent = {{10, -10}, {-10, 10}}, rotation = 0))); 9 | 10 | // Define Input/Output Variables and set them equal to model variables 11 | input Modelica.SIunits.VolumeFlowRate Q_pump(fixed=false, min=0.0, max=7.0) = pump.Q; 12 | input Boolean is_downhill; 13 | input Modelica.SIunits.VolumeFlowRate Q_in(fixed=true) = discharge.Q; 14 | input Modelica.SIunits.Position H_sea(fixed=true) = level.H; 15 | input Modelica.SIunits.VolumeFlowRate Q_orifice(fixed=false, min=0.0, max=10.0) = orifice.Q; 16 | output Modelica.SIunits.Position storage_level = storage.HQ.H; 17 | output Modelica.SIunits.Position sea_level = level.H; 18 | equation 19 | // Connect Model Elements 20 | connect(orifice.HQDown, level.HQ) annotation(Line(visible = true, origin = {-33.025, 10}, points = {{25.025, 10}, {-6.675, 10}, {-6.675, -10}, {-11.675, -10}}, color = {0, 0, 255})); 21 | connect(storage.HQ, orifice.HQUp) annotation(Line(visible = true, origin = {43.737, 48.702}, points = {{-3.715, -48.702}, {-3.737, -28.702}, {-35.737, -28.702}}, color = {0, 0, 255})); 22 | connect(storage.HQ, pump.HQUp) annotation(Line(visible = true, origin = {4.669, -31.115}, points = {{35.353, 31.115}, {35.331, 11.115}, {3.331, 11.115}}, color = {0, 0, 255})); 23 | connect(discharge.HQ, storage.HQ) annotation(Line(visible = true, origin = {46.011, -0}, points = {{5.989, 0}, {-5.989, -0}}, color = {0, 0, 255})); 24 | connect(pump.HQDown, level.HQ) annotation(Line(visible = true, origin = {-33.025, -10}, points = {{25.025, -10}, {-6.675, -10}, {-6.675, 10}, {-11.675, 10}}, color = {0, 0, 255})); 25 | annotation(Diagram(coordinateSystem(extent = {{-148.5, -105}, {148.5, 105}}, preserveAspectRatio = true, initialScale = 0.1, grid = {5, 5}))); 26 | end Example; 27 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/model/Example.mo: -------------------------------------------------------------------------------- 1 | // This example model is a modified version of the goal_programming example model of rtc-tools: https://gitlab.com/deltares/rtc-tools 2 | model Example 3 | // Declare Model Elements 4 | Deltares.ChannelFlow.Hydraulic.Storage.Linear storage(A=1.0e6, H_b=0.0, HQ.H(min=0.0, max=0.5)) annotation(Placement(visible = true, transformation(origin = {32.022, -0}, extent = {{-10, -10}, {10, 10}}, rotation = -270))); 5 | Deltares.ChannelFlow.Hydraulic.BoundaryConditions.Discharge discharge annotation(Placement(visible = true, transformation(origin = {60, -0}, extent = {{10, -10}, {-10, 10}}, rotation = 270))); 6 | Deltares.ChannelFlow.Hydraulic.BoundaryConditions.Level level annotation(Placement(visible = true, transformation(origin = {-52.7, 0}, extent = {{-10, -10}, {10, 10}}, rotation = -270))); 7 | Deltares.ChannelFlow.Hydraulic.Structures.Pump pump annotation(Placement(visible = true, transformation(origin = {0, -20}, extent = {{10, -10}, {-10, 10}}, rotation = 0))); 8 | Deltares.ChannelFlow.Hydraulic.Structures.Pump orifice annotation(Placement(visible = true, transformation(origin = {0, 20}, extent = {{10, -10}, {-10, 10}}, rotation = 0))); 9 | 10 | // Define Input/Output Variables and set them equal to model variables 11 | input Modelica.SIunits.VolumeFlowRate Q_pump(fixed=false, min=0.0, max=7.0) = pump.Q; 12 | input Boolean is_downhill; 13 | input Modelica.SIunits.VolumeFlowRate Q_in(fixed=true) = discharge.Q; 14 | input Modelica.SIunits.Position H_sea(fixed=true) = level.H; 15 | input Modelica.SIunits.VolumeFlowRate Q_orifice(fixed=false, min=0.0, max=10.0) = orifice.Q; 16 | output Modelica.SIunits.Position storage_level = storage.HQ.H; 17 | output Modelica.SIunits.Position sea_level = level.H; 18 | equation 19 | // Connect Model Elements 20 | connect(orifice.HQDown, level.HQ) annotation(Line(visible = true, origin = {-33.025, 10}, points = {{25.025, 10}, {-6.675, 10}, {-6.675, -10}, {-11.675, -10}}, color = {0, 0, 255})); 21 | connect(storage.HQ, orifice.HQUp) annotation(Line(visible = true, origin = {43.737, 48.702}, points = {{-3.715, -48.702}, {-3.737, -28.702}, {-35.737, -28.702}}, color = {0, 0, 255})); 22 | connect(storage.HQ, pump.HQUp) annotation(Line(visible = true, origin = {4.669, -31.115}, points = {{35.353, 31.115}, {35.331, 11.115}, {3.331, 11.115}}, color = {0, 0, 255})); 23 | connect(discharge.HQ, storage.HQ) annotation(Line(visible = true, origin = {46.011, -0}, points = {{5.989, 0}, {-5.989, -0}}, color = {0, 0, 255})); 24 | connect(pump.HQDown, level.HQ) annotation(Line(visible = true, origin = {-33.025, -10}, points = {{25.025, -10}, {-6.675, -10}, {-6.675, 10}, {-11.675, 10}}, color = {0, 0, 255})); 25 | annotation(Diagram(coordinateSystem(extent = {{-148.5, -105}, {148.5, 105}}, preserveAspectRatio = true, initialScale = 0.1, grid = {5, 5}))); 26 | end Example; 27 | -------------------------------------------------------------------------------- /rtctools_interface/simulation/plot_mixin.py: -------------------------------------------------------------------------------- 1 | """Mixin to store all required data for plotting. Can also call the plot function.""" 2 | 3 | import logging 4 | 5 | import numpy as np 6 | 7 | from rtctools_interface.plotting.plot_tools import create_plot_final_results 8 | from rtctools_interface.utils.results_collection import PlottingBaseMixin 9 | 10 | logger = logging.getLogger("rtctools") 11 | 12 | 13 | class PlotMixin(PlottingBaseMixin): 14 | """ 15 | Class for plotting results based on the plot_table. 16 | """ 17 | 18 | optimization_problem = False 19 | _manual_extracted_states: dict[str, list] = {} 20 | 21 | def manual_extraction_from_state_vector(self): 22 | for variable in self.custom_variables: 23 | try: 24 | self._manual_extracted_states[variable].append(self.get_var(variable)) 25 | except KeyError: 26 | logger.debug(f"Variable {variable} not found in output of model.") 27 | 28 | def initialize(self, *args, **kwargs): 29 | super().initialize(*args, **kwargs) 30 | self._manual_extracted_states = {variable: [] for variable in self.custom_variables} 31 | self.manual_extraction_from_state_vector() 32 | 33 | def update(self, dt): 34 | super().update(dt) 35 | self.manual_extraction_from_state_vector() 36 | 37 | def post(self): 38 | """Tasks after optimizing.""" 39 | super().post() 40 | 41 | # find empty arrays in self._manual_extracted_states 42 | # for these variables try to use self.get_timeseries(variable) 43 | for variable in self.custom_variables: 44 | if ( 45 | not self._manual_extracted_states[variable] 46 | or len(self._manual_extracted_states[variable]) == 0 47 | ): 48 | logger.debug(f"Variable {variable} has empty data collected.") 49 | try: 50 | self._manual_extracted_states[variable] = self.get_timeseries(variable) 51 | except KeyError: 52 | logger.warning(f"Variable {variable} not found in output of model.") 53 | 54 | timeseries_data = self.collect_timeseries_data(self.custom_variables) 55 | self._intermediate_results.append({"timeseries_data": timeseries_data, "priority": 0}) 56 | current_run = self.create_plot_data_and_config([]) 57 | self._store_current_results(self._cache_folder, current_run) 58 | 59 | if self.plot_final_results: 60 | create_plot_final_results( 61 | current_run, self._previous_run, plotting_library=self.plotting_library 62 | ) 63 | 64 | def collect_timeseries_data(self, all_variables_to_store: list[str]) -> dict[str, np.ndarray]: 65 | return { 66 | variable: np.array(self._manual_extracted_states[variable]) 67 | for variable in all_variables_to_store 68 | } 69 | -------------------------------------------------------------------------------- /rtctools_interface/closed_loop/config.py: -------------------------------------------------------------------------------- 1 | """Module for configuring a closed-loop optimization problem.""" 2 | 3 | from datetime import timedelta 4 | from pathlib import Path 5 | 6 | 7 | class ClosedLoopConfig: 8 | """Configuration of a closed-loop optimization problem.""" 9 | 10 | def __init__( 11 | self, 12 | file: Path = None, 13 | round_to_dates: bool = False, 14 | ): 15 | """ 16 | Create a configuration for closed-loop optimization. 17 | 18 | :param file: CSV file with two columns 'start_date' and 'end_date'. 19 | Each row indicates a time range for which to optimize a given problem. 20 | Note: 21 | * The start time of the first time range should coincide with the 22 | start time of the input timeseries. 23 | * The start time of the next time range should be less or equal 24 | to the end time of the current time range. 25 | :param round_to_dates: If true, the start time and end time of each time range 26 | is rounded to just the date. 27 | In particular, the start time is rounded to the start of the day 28 | and the end time is rounded to the end of the day. 29 | """ 30 | if file is not None: 31 | file = Path(file).resolve() 32 | self._file = file 33 | self._forecast_timestep: timedelta | None = None 34 | self._optimization_period: timedelta | None = None 35 | self.round_to_dates = round_to_dates 36 | 37 | @classmethod 38 | def from_fixed_periods( 39 | cls, 40 | forecast_timestep: timedelta, 41 | optimization_period: timedelta, 42 | round_to_dates: bool = False, 43 | ): 44 | """ 45 | Create a closed loop configuration based on fixed periods. 46 | 47 | :param forecast_timestep: The time between the start of each optimization time range. 48 | :param optimization_period: the duration of each optimization time range 49 | """ 50 | if forecast_timestep > optimization_period: 51 | raise ValueError( 52 | f"The forecast timestep ({forecast_timestep}) should be less than or equal to" 53 | f" the optimization period ({optimization_period})." 54 | ) 55 | config = cls() 56 | config._forecast_timestep = forecast_timestep 57 | config._optimization_period = optimization_period 58 | config.round_to_dates = round_to_dates 59 | return config 60 | 61 | @property 62 | def file(self): 63 | """Get the file that defines the closed-loop periods.""" 64 | return self._file 65 | 66 | @property 67 | def forecast_timestep(self): 68 | """Get the forecast timestep of the closed-loop periods.""" 69 | return self._forecast_timestep 70 | 71 | @property 72 | def optimization_period(self): 73 | """Get the optimization period of the closed-loop periods.""" 74 | return self._optimization_period 75 | -------------------------------------------------------------------------------- /rtctools_interface/optimization/helpers/statistics_mixin.py: -------------------------------------------------------------------------------- 1 | """Base mixin for retrieving particular stats for goals and plotting.""" 2 | 3 | import logging 4 | 5 | import numpy as np 6 | 7 | from rtctools_interface.optimization.base_goal import BaseGoal 8 | from rtctools_interface.utils.type_definitions import TargetDict 9 | 10 | logger = logging.getLogger("rtctools") 11 | 12 | 13 | class StatisticsMixin: 14 | # TODO: remove pylint disable below once we have more public functions. 15 | # pylint: disable=too-few-public-methods 16 | """A mixin class providing methods for collecting data and statistics from optimization results, 17 | useful for solution performance analysis.""" 18 | 19 | def collect_range_target_values( 20 | self, 21 | base_goals: list[BaseGoal], 22 | ) -> dict[str, TargetDict]: 23 | """For the goals with targets, collect the actual timeseries with these targets.""" 24 | target_series: dict[str, TargetDict] = {} 25 | for goal in base_goals: 26 | if goal.goal_type in ["range", "range_rate_of_change"]: 27 | target_dict = self.collect_range_target_values_from_basegoal(goal) 28 | target_series[str(goal.goal_id)] = target_dict 29 | return target_series 30 | 31 | def collect_range_target_values_from_basegoal(self, goal: BaseGoal) -> TargetDict: 32 | """Collect the target timeseries for a single basegoal.""" 33 | t = self.times() 34 | 35 | def get_parameter_ranges(goal) -> tuple[np.ndarray, np.ndarray]: 36 | target_min = np.full_like(t, 1) * float(goal.target_min) 37 | target_max = np.full_like(t, 1) * float(goal.target_max) 38 | return target_min, target_max 39 | 40 | def get_value_ranges(goal) -> tuple[np.ndarray, np.ndarray]: 41 | target_min = np.full_like(t, 1) * float(goal.target_min) 42 | target_max = np.full_like(t, 1) * float(goal.target_max) 43 | return target_min, target_max 44 | 45 | def get_timeseries_ranges(goal) -> tuple[np.ndarray, np.ndarray]: 46 | try: 47 | target_min = goal.target_min.values 48 | except AttributeError: 49 | target_min = goal.target_min 50 | try: 51 | target_max = goal.target_max.values 52 | except AttributeError: 53 | target_max = goal.target_max 54 | return target_min, target_max 55 | 56 | supported_goal_types = ["range", "range_rate_of_change"] 57 | if goal.goal_type in supported_goal_types: 58 | if goal.target_data_type == "parameter": 59 | target_min, target_max = get_parameter_ranges(goal) 60 | elif goal.target_data_type == "value": 61 | target_min, target_max = get_value_ranges(goal) 62 | elif goal.target_data_type == "timeseries": 63 | target_min, target_max = get_timeseries_ranges(goal) 64 | else: 65 | message = f"Target type {goal.target_data_type} not known for goal {goal.goal_id}." 66 | logger.error(message) 67 | raise ValueError(message) 68 | else: 69 | message = f"Goal type {goal.goal_type} not supported for target collection." 70 | logger.error(message) 71 | raise ValueError(message) 72 | target_dict: TargetDict = {"target_min": target_min, "target_max": target_max} 73 | return target_dict 74 | -------------------------------------------------------------------------------- /rtctools_interface/optimization/read_goals.py: -------------------------------------------------------------------------------- 1 | """Module for reading goals from a csv file.""" 2 | 3 | import pandas as pd 4 | 5 | from rtctools_interface.optimization.goal_table_schema import ( 6 | GOAL_TYPES, 7 | NON_PATH_GOALS, 8 | PATH_GOALS, 9 | BaseGoalModel, 10 | MaximizationGoalModel, 11 | MinimizationGoalModel, 12 | RangeGoalModel, 13 | RangeRateOfChangeGoalModel, 14 | ) 15 | 16 | 17 | def goal_table_checks(goal_table): 18 | """Validate input goal table.""" 19 | if "goal_type" not in goal_table: 20 | raise ValueError("Goal type column not in goal table.") 21 | if "active" not in goal_table: 22 | raise ValueError("Active column not in goal table.") 23 | for _, row in goal_table.iterrows(): 24 | if row["goal_type"] not in GOAL_TYPES.keys(): 25 | raise ValueError( 26 | f"Goal of type {row['goal_type']} is not allowed. Allowed are {GOAL_TYPES.keys()}" 27 | ) 28 | if int(row["active"]) not in [0, 1]: 29 | raise ValueError("Value in active column should be either 0 or 1.") 30 | 31 | 32 | def validate_goal_list(goal_list): 33 | """Validate list of goals on correct type and uniqueness of id's""" 34 | ids = [goal.goal_id for goal in goal_list] 35 | if len(ids) != len(set(ids)): 36 | raise ValueError("ID's in goal generator table should be unique!") 37 | 38 | 39 | def read_goals_from_csv( 40 | file, 41 | ) -> list[ 42 | RangeGoalModel | RangeRateOfChangeGoalModel | MinimizationGoalModel | MaximizationGoalModel 43 | ]: 44 | """Read goals from csv file and validate values.""" 45 | raw_goal_table = pd.read_csv(file, sep=",") 46 | goal_table_checks(raw_goal_table) 47 | 48 | parsed_goals = [] 49 | for _, row in raw_goal_table.iterrows(): 50 | if int(row["active"]) == 1: 51 | parsed_goals.append(GOAL_TYPES[row["goal_type"]](**row)) 52 | return parsed_goals 53 | 54 | 55 | def read_goals_from_list( 56 | goals_to_generate, 57 | ) -> list[ 58 | RangeGoalModel | RangeRateOfChangeGoalModel | MinimizationGoalModel | MaximizationGoalModel 59 | ]: 60 | """Read goals from a list. Validates whether the goals are of correct type.""" 61 | if not isinstance(goals_to_generate, list): 62 | raise TypeError(f"Pass a list of goal elements, not a {type(goals_to_generate)}") 63 | for base_goal in goals_to_generate: 64 | if not isinstance(base_goal, BaseGoalModel): 65 | raise TypeError( 66 | "Each element in the list of goals to generate should be a child of BaseGoalModel" 67 | ) 68 | active_goals = [] 69 | for goal in goals_to_generate: 70 | if int(goal.active) == 1: 71 | active_goals.append(goal) 72 | return active_goals 73 | 74 | 75 | def read_goals( 76 | file=None, path_goal: bool = True, read_from="csv_table", goals_to_generate=None 77 | ) -> list[ 78 | RangeGoalModel | RangeRateOfChangeGoalModel | MinimizationGoalModel | MaximizationGoalModel 79 | ]: 80 | """Read goals from a csv file 81 | 82 | Returns either only the path_goals or only the non_path goals. 83 | In either case only the active goals. 84 | """ 85 | if read_from == "csv_table": 86 | parsed_goals = read_goals_from_csv(file) 87 | elif read_from == "passed_list": 88 | parsed_goals = read_goals_from_list(goals_to_generate) 89 | else: 90 | raise ValueError("GoalGeneratorMixin should either read from 'csv_table' or 'passed_list'") 91 | validate_goal_list(parsed_goals) 92 | requested_goal_types = PATH_GOALS.keys() if path_goal else NON_PATH_GOALS.keys() 93 | return [goal for goal in parsed_goals if goal.goal_type in requested_goal_types] 94 | -------------------------------------------------------------------------------- /rtctools_interface/optimization/goal_table_schema.py: -------------------------------------------------------------------------------- 1 | """Schema for the goal_table.""" 2 | 3 | from typing import Literal 4 | 5 | import numpy as np 6 | import pandas as pd 7 | from pydantic import BaseModel, Field, field_validator, model_validator 8 | 9 | 10 | class BaseGoalModel(BaseModel): 11 | """BaseModel for a goal.""" 12 | 13 | goal_id: int | str = Field(..., alias="id") 14 | active: Literal[0, 1] 15 | state: str 16 | goal_type: str 17 | priority: int 18 | function_nominal: float = np.nan 19 | weight: float = np.nan 20 | order: float = np.nan 21 | 22 | @field_validator("goal_type") 23 | @classmethod 24 | def validate_goal_type(cls, value): 25 | """Check whether the supplied goal type is supported""" 26 | if value not in GOAL_TYPES.keys(): 27 | raise ValueError( 28 | f"Invalid goal_type '{value}'. Allowed values are {GOAL_TYPES.keys()}." 29 | ) 30 | return value 31 | 32 | @field_validator("goal_id", "active") 33 | @classmethod 34 | def convert_to_int(cls, value): 35 | """Convert value to integer if possible.""" 36 | try: 37 | return int(value) 38 | except (ValueError, TypeError): 39 | return value 40 | 41 | def get(self, attribute_name, default=None): 42 | """Similar functionality as dict-get method.""" 43 | try: 44 | return getattr(self, attribute_name) 45 | except AttributeError: 46 | return default 47 | 48 | 49 | class MaximizationGoalModel(BaseGoalModel): 50 | """Model for a minimization and maximization goal.""" 51 | 52 | 53 | MinimizationGoalModel = MaximizationGoalModel 54 | 55 | 56 | class RangeGoalModel(BaseGoalModel): 57 | """Model for a range goal.""" 58 | 59 | target_data_type: str 60 | function_min: float = np.nan 61 | function_max: float = np.nan 62 | target_min: float | str = np.nan 63 | target_max: float | str = np.nan 64 | 65 | @field_validator("target_min", "target_max") 66 | @classmethod 67 | def convert_to_float(cls, value): 68 | """Convert value to float if possible.""" 69 | try: 70 | return float(value) 71 | except (ValueError, TypeError): 72 | return value 73 | 74 | @model_validator(mode="after") 75 | def validate_targets(self): 76 | """Check whether required columns for the range_goal are available.""" 77 | try: 78 | assert not (pd.isna(self.target_min) and pd.isna(self.target_max)) 79 | except AssertionError as exc: 80 | raise ValueError( 81 | "For a range goal, at least one of target_min and target_max should be set." 82 | ) from exc 83 | return self 84 | 85 | @model_validator(mode="after") 86 | def validate_target_type_and_value(self): 87 | """Check if target_min and target_max dtype correspond to target_data_type.""" 88 | try: 89 | if self.target_data_type == "value": 90 | assert isinstance(self.target_min, float) 91 | assert isinstance(self.target_max, float) 92 | elif self.target_data_type in ["parameter", "timeseries"]: 93 | assert isinstance(self.target_min, str) or pd.isna(self.target_min) 94 | assert isinstance(self.target_max, str) or pd.isna(self.target_max) 95 | except AssertionError as exc: 96 | raise ValueError( 97 | "The type in the target_min/target_max column does not " 98 | "correspond to the target_data_type." 99 | ) from exc 100 | return self 101 | 102 | 103 | class RangeRateOfChangeGoalModel(RangeGoalModel): 104 | """Model for a rate of change range goal.""" 105 | 106 | 107 | PATH_GOALS = { 108 | "minimization_path": MinimizationGoalModel, 109 | "maximization_path": MaximizationGoalModel, 110 | "range": RangeGoalModel, 111 | "range_rate_of_change": RangeRateOfChangeGoalModel, 112 | } 113 | NON_PATH_GOALS: dict = {} 114 | GOAL_TYPES = PATH_GOALS | NON_PATH_GOALS 115 | TARGET_DATA_TYPES = [ 116 | "value", 117 | "parameter", 118 | "timeseries", 119 | ] 120 | -------------------------------------------------------------------------------- /rtctools_interface/optimization/goal_generator_mixin.py: -------------------------------------------------------------------------------- 1 | """Module for a basic optimization problem.""" 2 | 3 | import logging 4 | from pathlib import Path 5 | 6 | import pandas as pd 7 | 8 | from rtctools_interface.optimization.base_goal import BaseGoal 9 | from rtctools_interface.optimization.goal_performance_metrics import get_performance_metrics 10 | from rtctools_interface.optimization.helpers.statistics_mixin import StatisticsMixin 11 | from rtctools_interface.utils.read_goals_mixin import ReadGoalsMixin 12 | 13 | logger = logging.getLogger("rtctools") 14 | 15 | 16 | def write_performance_metrics( 17 | performance_metrics: dict[str, pd.DataFrame], output_path: str | Path 18 | ): 19 | """Write the performance metrics for each goal to a csv file.""" 20 | output_path = Path(output_path) / "performance_metrics" 21 | output_path.mkdir(parents=True, exist_ok=True) 22 | for goal_id, performance_metric_table in performance_metrics.items(): 23 | performance_metric_table.to_csv(output_path / f"{goal_id}.csv") 24 | 25 | 26 | class GoalGeneratorMixin(ReadGoalsMixin, StatisticsMixin): 27 | # TODO: remove pylint disable below once we have more public functions. 28 | # pylint: disable=too-few-public-methods 29 | """Add path goals as specified in the goal_table. 30 | 31 | By default, the mixin looks for the csv in the in the default input 32 | folder. One can also set the path to the goal_table_file manually 33 | with the `goal_table_file` class variable. 34 | """ 35 | 36 | calculate_performance_metrics = True 37 | 38 | def __init__(self, **kwargs): 39 | super().__init__(**kwargs) 40 | if not hasattr(self, "_all_goal_generator_goals"): 41 | goals_to_generate = kwargs.get("goals_to_generate", []) 42 | read_from = kwargs.get("read_goals_from", "csv_table") 43 | self.load_goals(read_from, goals_to_generate) 44 | if self.calculate_performance_metrics: 45 | # A dataframe for each goal defined by the goal generator 46 | self._performance_metrics = {} 47 | for goal in self._all_goal_generator_goals: 48 | self._performance_metrics[goal.goal_id] = pd.DataFrame() 49 | 50 | def path_goals(self): 51 | """Return the list of path goals.""" 52 | goals = super().path_goals() 53 | new_goals = self._goal_generator_path_goals 54 | if new_goals: 55 | goals = goals + [ 56 | BaseGoal(optimization_problem=self, **goal.__dict__) for goal in new_goals 57 | ] 58 | return goals 59 | 60 | def goals(self): 61 | """Return the list of goals.""" 62 | goals = super().goals() 63 | new_goals = self._goal_generator_non_path_goals 64 | if new_goals: 65 | goals = goals + [ 66 | BaseGoal(optimization_problem=self, **goal.__dict__) for goal in new_goals 67 | ] 68 | return goals 69 | 70 | def store_performance_metrics(self, label): 71 | """Calculate and store performance metrics.""" 72 | results = self.extract_results() 73 | goal_generator_goals = self._all_goal_generator_goals 74 | all_base_goals = [ 75 | goal for goal in self.goals() + self.path_goals() if isinstance(goal, BaseGoal) 76 | ] 77 | targets = self.collect_range_target_values(all_base_goals) 78 | for goal in goal_generator_goals: 79 | next_row = get_performance_metrics(results, goal, targets.get(str(goal.goal_id))) 80 | if next_row is not None: 81 | next_row.rename(label, inplace=True) 82 | self._performance_metrics[goal.goal_id] = pd.concat( 83 | [self._performance_metrics[goal.goal_id].T, next_row], axis=1 84 | ).T 85 | 86 | def priority_completed(self, priority): 87 | """Tasks after priority optimization.""" 88 | super().priority_completed(priority) 89 | if self.calculate_performance_metrics: 90 | self.store_performance_metrics(priority) 91 | 92 | def post(self): 93 | """Tasks after all optimization steps.""" 94 | super().post() 95 | if self.calculate_performance_metrics: 96 | self.store_performance_metrics("final_results") 97 | write_performance_metrics(self._performance_metrics, self._output_folder) 98 | 99 | def get_performance_metrics(self): 100 | """Get the plot data and config from the current run.""" 101 | return self._performance_metrics 102 | -------------------------------------------------------------------------------- /rtctools_interface/optimization/goal_performance_metrics.py: -------------------------------------------------------------------------------- 1 | """This file contains functions to get performance metrics for the BaseGoal.""" 2 | 3 | import logging 4 | 5 | import numpy as np 6 | import pandas as pd 7 | 8 | from rtctools_interface.optimization.goal_table_schema import ( 9 | BaseGoalModel, 10 | MaximizationGoalModel, 11 | MinimizationGoalModel, 12 | RangeGoalModel, 13 | RangeRateOfChangeGoalModel, 14 | ) 15 | from rtctools_interface.utils.type_definitions import TargetDict 16 | 17 | logger = logging.getLogger("rtctools") 18 | 19 | ABS_TOL = 0.001 20 | 21 | 22 | def get_mean_absolute_percentual_difference(timeseries: np.ndarray) -> float: 23 | """Calculate the mean absolute percentual difference, ignoring entries where timeseries = 0.""" 24 | nonzero_indices = np.nonzero(timeseries) 25 | timeseries = timeseries[nonzero_indices] 26 | differences = np.diff(timeseries) 27 | if len(timeseries) <= 1: 28 | return 0 29 | mapd = np.mean(np.abs(differences / timeseries[:-1])) 30 | return mapd 31 | 32 | 33 | def get_absolute_sum_difference(timeseries: np.ndarray) -> float: 34 | """Calculate the mean of absolute first-order difference.""" 35 | if len(timeseries) <= 1: 36 | return 0 37 | mad = np.mean(np.abs(np.diff(timeseries))) 38 | return mad 39 | 40 | 41 | def get_max_difference(timeseries: np.ndarray) -> float: 42 | """Get maximum one step difference""" 43 | return max(np.diff(timeseries)) 44 | 45 | 46 | def get_basic_metrics(timeseries: np.ndarray) -> dict[str, float]: 47 | """Get general metrics applicable for each goal type.""" 48 | metrics = { 49 | "timeseries_sum": sum(timeseries), 50 | "timeseries_min": min(timeseries), 51 | "timeseries_max": max(timeseries), 52 | "timeseries_avg": np.mean(timeseries), 53 | "mean_absolute_percentual_difference": get_mean_absolute_percentual_difference(timeseries), 54 | "mean_absolute_difference": get_absolute_sum_difference(timeseries), 55 | "max_difference": get_max_difference(timeseries), 56 | } 57 | return metrics 58 | 59 | 60 | def performance_metrics_minmaximization( 61 | results: dict[str, np.ndarray], goal: MinimizationGoalModel 62 | ) -> pd.Series: 63 | """Get all relevant statistics for a min/maximization goal.""" 64 | state_timeseries = results[goal.state] 65 | metrics = get_basic_metrics(state_timeseries) 66 | return pd.Series(metrics) 67 | 68 | 69 | def get_range_percentual_exceedance( 70 | timeseries: np.ndarray, goal: RangeGoalModel, targets: TargetDict 71 | ) -> dict[str, float | None] | None: 72 | """Calculate percentage of timesteps in which target is exceeded""" 73 | if goal.goal_type not in ["range", "range_rate_of_change"]: 74 | below_target = None 75 | above_target = None 76 | else: 77 | below_target = float( 78 | sum(np.where(timeseries + ABS_TOL < targets["target_min"], 1, 0)) / len(timeseries) 79 | ) 80 | above_target = float( 81 | sum(np.where(timeseries - ABS_TOL > targets["target_max"], 1, 0)) / len(timeseries) 82 | ) 83 | return {"perc_below_target": below_target, "perc_above_target": above_target} 84 | 85 | 86 | def get_range_total_exceedance( 87 | timeseries: np.ndarray, goal: RangeGoalModel, targets: TargetDict 88 | ) -> dict[str, float | None] | None: 89 | """Calculate sum of absolute exceedances of the target""" 90 | if goal.goal_type not in ["range", "range_rate_of_change"]: 91 | below_target = None 92 | above_target = None 93 | else: 94 | below_target = float( 95 | sum( 96 | np.abs( 97 | np.where( 98 | timeseries < targets["target_min"], timeseries - targets["target_min"], 0 99 | ) 100 | ) 101 | ) 102 | ) 103 | above_target = float( 104 | sum( 105 | np.abs( 106 | np.where( 107 | timeseries > targets["target_max"], timeseries - targets["target_max"], 0 108 | ) 109 | ) 110 | ) 111 | ) 112 | return {"sum_below_target": below_target, "sum_above_target": above_target} 113 | 114 | 115 | def performance_metrics_range( 116 | results: dict[str, np.ndarray], goal: RangeGoalModel, targets: TargetDict 117 | ) -> pd.Series: 118 | """Get all relevant statistics for a range goal.""" 119 | metrics: dict = {} 120 | state_timeseries = results[goal.state] 121 | metrics = metrics | get_basic_metrics(state_timeseries) 122 | metrics = metrics | get_range_percentual_exceedance(state_timeseries, goal, targets) 123 | metrics = metrics | get_range_total_exceedance(state_timeseries, goal, targets) 124 | return pd.Series(metrics) 125 | 126 | 127 | def performance_metrics_rangerateofchange( 128 | results: dict[str, np.ndarray], goal: RangeGoalModel, _targets: TargetDict 129 | ) -> pd.Series: 130 | """Get all relevant statistics for a range-rate-of-change goal.""" 131 | metrics: dict[str, float | None] = {} 132 | state_timeseries = results[goal.state] 133 | metrics = metrics | get_basic_metrics(state_timeseries) 134 | return pd.Series(metrics) 135 | 136 | 137 | def get_performance_metrics(results, goal: BaseGoalModel, targets: TargetDict) -> pd.Series | None: 138 | """Returns a series with performance metrics for each goal.""" 139 | if type(goal) in [MinimizationGoalModel, MaximizationGoalModel]: # pylint: disable=unidiomatic-typecheck 140 | return performance_metrics_minmaximization(results, goal) 141 | if type(goal) in [RangeGoalModel]: 142 | return performance_metrics_range(results, goal, targets) 143 | if type(goal) in [RangeRateOfChangeGoalModel]: 144 | return performance_metrics_rangerateofchange(results, goal, targets) 145 | logger.info("No performance metrics are implemented for goal of type: %s", str(type(goal))) 146 | return None 147 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/src/example.py: -------------------------------------------------------------------------------- 1 | """This example model is a modified version of the goal_programming example model of 2 | rtc-tools: https://gitlab.com/deltares/rtc-tools""" 3 | 4 | import numpy as np 5 | from rtctools.optimization.collocated_integrated_optimization_problem import ( 6 | CollocatedIntegratedOptimizationProblem, 7 | ) 8 | from rtctools.optimization.goal_programming_mixin import Goal, GoalProgrammingMixin, StateGoal 9 | from rtctools.optimization.modelica_mixin import ModelicaMixin 10 | from rtctools.optimization.pi_mixin import PIMixin 11 | 12 | from rtctools_interface.closed_loop.runner import run_optimization_problem_closed_loop 13 | 14 | 15 | class WaterLevelRangeGoal(StateGoal): 16 | # Applying a state goal to every time step is easily done by defining a goal 17 | # that inherits StateGoal. StateGoal is a helper class that uses the state 18 | # to determine the function, function range, and function nominal 19 | # automatically. 20 | state = "storage.HQ.H" 21 | # One goal can introduce a single or two constraints (min and/or max). Our 22 | # target water level range is 0.43 - 0.44. We might not always be able to 23 | # realize this, but we want to try. 24 | target_min = 0.43 25 | target_max = 0.44 26 | 27 | # Because we want to satisfy our water level target first, this has a 28 | # higher priority (=lower number). 29 | priority = 1 30 | 31 | 32 | class MinimizeQpumpGoal(Goal): 33 | # This goal does not use a helper class, so we have to define the function 34 | # method, range and nominal explicitly. We do not specify a target_min or 35 | # target_max in this class, so the goal programming mixin will try to 36 | # minimize the expression returned by the function method. 37 | def function(self, optimization_problem, ensemble_member): 38 | return optimization_problem.integral("Q_pump") 39 | 40 | # The nominal is used to scale the value returned by 41 | # the function method so that the value is on the order of 1. 42 | function_nominal = 100.0 43 | # The lower the number returned by this function, the higher the priority. 44 | priority = 2 45 | # The penalty variable is taken to the order'th power. 46 | order = 1 47 | 48 | 49 | class MinimizeChangeInQpumpGoal(Goal): 50 | # To reduce pump power cycles, we add a third goal to minimize changes in 51 | # Q_pump. This will be passed into the optimization problem as a path goal 52 | # because it is an an individual goal that should be applied at every time 53 | # step. 54 | def function(self, optimization_problem, ensemble_member): 55 | return optimization_problem.der("Q_pump") 56 | 57 | function_nominal = 5.0 58 | priority = 3 59 | # Default order is 2, but we want to be explicit 60 | order = 2 61 | 62 | 63 | class Example( 64 | GoalProgrammingMixin, PIMixin, ModelicaMixin, CollocatedIntegratedOptimizationProblem 65 | ): 66 | """ 67 | An introductory example to goal programming in RTC-Tools 68 | """ 69 | 70 | def path_constraints(self, ensemble_member): 71 | # We want to add a few hard constraints to our problem. The goal 72 | # programming mixin however also generates constraints (and objectives) 73 | # from on our goals, so we have to call super() here. 74 | constraints = super().path_constraints(ensemble_member) 75 | 76 | # Release through orifice downhill only. This constraint enforces the 77 | # fact that water only flows downhill 78 | constraints.append( 79 | (self.state("Q_orifice") + (1 - self.state("is_downhill")) * 10, 0.0, 10.0) 80 | ) 81 | 82 | # Make sure is_downhill is true only when the sea is lower than the 83 | # water level in the storage. 84 | M = 2 # The so-called "big-M" 85 | constraints.append( 86 | ( 87 | self.state("H_sea") 88 | - self.state("storage.HQ.H") 89 | - (1 - self.state("is_downhill")) * M, 90 | -np.inf, 91 | 0.0, 92 | ) 93 | ) 94 | constraints.append( 95 | ( 96 | self.state("H_sea") - self.state("storage.HQ.H") + self.state("is_downhill") * M, 97 | 0.0, 98 | np.inf, 99 | ) 100 | ) 101 | 102 | # Orifice flow constraint. Uses the equation: 103 | # Q(HUp, HDown, d) = width * C * d * (2 * g * (HUp - HDown)) ^ 0.5 104 | # Note that this equation is only valid for orifices that are submerged 105 | # units: description: 106 | w = 3.0 # m width of orifice 107 | d = 0.8 # m hight of orifice 108 | C = 1.0 # none orifice constant 109 | g = 9.8 # m/s^2 gravitational acceleration 110 | constraints.append( 111 | ( 112 | ((self.state("Q_orifice") / (w * C * d)) ** 2) / (2 * g) 113 | + self.state("orifice.HQDown.H") 114 | - self.state("orifice.HQUp.H") 115 | - M * (1 - self.state("is_downhill")), 116 | -np.inf, 117 | 0.0, 118 | ) 119 | ) 120 | 121 | return constraints 122 | 123 | def goals(self): 124 | return [MinimizeQpumpGoal()] 125 | 126 | def path_goals(self): 127 | # Sorting goals on priority is done in the goal programming mixin. We 128 | # do not have to worry about order here. 129 | return [WaterLevelRangeGoal(self), MinimizeChangeInQpumpGoal()] 130 | 131 | def pre(self): 132 | # Call super() class to not overwrite default behaviour 133 | super().pre() 134 | # We keep track of our intermediate results, so that we can print some 135 | # information about the progress of goals at the end of our run. 136 | self.intermediate_results = [] 137 | 138 | # Any solver options can be set here 139 | def solver_options(self): 140 | options = super().solver_options() 141 | solver = options["solver"] 142 | options[solver]["print_level"] = 1 143 | return options 144 | 145 | 146 | # Run 147 | if __name__ == "__main__": 148 | run_optimization_problem_closed_loop(Example) 149 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_csv/src/example.py: -------------------------------------------------------------------------------- 1 | """This example model is a modified version of the goal_programming example model of 2 | rtc-tools: https://gitlab.com/deltares/rtc-tools""" 3 | 4 | import numpy as np 5 | from rtctools.optimization.collocated_integrated_optimization_problem import ( 6 | CollocatedIntegratedOptimizationProblem, 7 | ) 8 | from rtctools.optimization.csv_mixin import CSVMixin 9 | from rtctools.optimization.goal_programming_mixin import Goal, GoalProgrammingMixin, StateGoal 10 | from rtctools.optimization.modelica_mixin import ModelicaMixin 11 | 12 | from rtctools_interface.closed_loop.runner import run_optimization_problem_closed_loop 13 | 14 | 15 | class WaterLevelRangeGoal(StateGoal): 16 | # Applying a state goal to every time step is easily done by defining a goal 17 | # that inherits StateGoal. StateGoal is a helper class that uses the state 18 | # to determine the function, function range, and function nominal 19 | # automatically. 20 | state = "storage.HQ.H" 21 | # One goal can introduce a single or two constraints (min and/or max). Our 22 | # target water level range is 0.43 - 0.44. We might not always be able to 23 | # realize this, but we want to try. 24 | target_min = 0.43 25 | target_max = 0.44 26 | 27 | # Because we want to satisfy our water level target first, this has a 28 | # higher priority (=lower number). 29 | priority = 1 30 | 31 | 32 | class MinimizeQpumpGoal(Goal): 33 | # This goal does not use a helper class, so we have to define the function 34 | # method, range and nominal explicitly. We do not specify a target_min or 35 | # target_max in this class, so the goal programming mixin will try to 36 | # minimize the expression returned by the function method. 37 | def function(self, optimization_problem, ensemble_member): 38 | return optimization_problem.integral("Q_pump") 39 | 40 | # The nominal is used to scale the value returned by 41 | # the function method so that the value is on the order of 1. 42 | function_nominal = 100.0 43 | # The lower the number returned by this function, the higher the priority. 44 | priority = 2 45 | # The penalty variable is taken to the order'th power. 46 | order = 1 47 | 48 | 49 | class MinimizeChangeInQpumpGoal(Goal): 50 | # To reduce pump power cycles, we add a third goal to minimize changes in 51 | # Q_pump. This will be passed into the optimization problem as a path goal 52 | # because it is an an individual goal that should be applied at every time 53 | # step. 54 | def function(self, optimization_problem, ensemble_member): 55 | return optimization_problem.der("Q_pump") 56 | 57 | function_nominal = 5.0 58 | priority = 3 59 | # Default order is 2, but we want to be explicit 60 | order = 2 61 | 62 | 63 | class Example( 64 | GoalProgrammingMixin, CSVMixin, ModelicaMixin, CollocatedIntegratedOptimizationProblem 65 | ): 66 | """ 67 | An introductory example to goal programming in RTC-Tools 68 | """ 69 | 70 | def path_constraints(self, ensemble_member): 71 | # We want to add a few hard constraints to our problem. The goal 72 | # programming mixin however also generates constraints (and objectives) 73 | # from on our goals, so we have to call super() here. 74 | constraints = super().path_constraints(ensemble_member) 75 | 76 | # Release through orifice downhill only. This constraint enforces the 77 | # fact that water only flows downhill 78 | constraints.append( 79 | (self.state("Q_orifice") + (1 - self.state("is_downhill")) * 10, 0.0, 10.0) 80 | ) 81 | 82 | # Make sure is_downhill is true only when the sea is lower than the 83 | # water level in the storage. 84 | M = 2 # The so-called "big-M" 85 | constraints.append( 86 | ( 87 | self.state("H_sea") 88 | - self.state("storage.HQ.H") 89 | - (1 - self.state("is_downhill")) * M, 90 | -np.inf, 91 | 0.0, 92 | ) 93 | ) 94 | constraints.append( 95 | ( 96 | self.state("H_sea") - self.state("storage.HQ.H") + self.state("is_downhill") * M, 97 | 0.0, 98 | np.inf, 99 | ) 100 | ) 101 | 102 | # Orifice flow constraint. Uses the equation: 103 | # Q(HUp, HDown, d) = width * C * d * (2 * g * (HUp - HDown)) ^ 0.5 104 | # Note that this equation is only valid for orifices that are submerged 105 | # units: description: 106 | w = 3.0 # m width of orifice 107 | d = 0.8 # m hight of orifice 108 | C = 1.0 # none orifice constant 109 | g = 9.8 # m/s^2 gravitational acceleration 110 | constraints.append( 111 | ( 112 | ((self.state("Q_orifice") / (w * C * d)) ** 2) / (2 * g) 113 | + self.state("orifice.HQDown.H") 114 | - self.state("orifice.HQUp.H") 115 | - M * (1 - self.state("is_downhill")), 116 | -np.inf, 117 | 0.0, 118 | ) 119 | ) 120 | 121 | return constraints 122 | 123 | def goals(self): 124 | return [MinimizeQpumpGoal()] 125 | 126 | def path_goals(self): 127 | # Sorting goals on priority is done in the goal programming mixin. We 128 | # do not have to worry about order here. 129 | return [WaterLevelRangeGoal(self), MinimizeChangeInQpumpGoal()] 130 | 131 | def pre(self): 132 | # Call super() class to not overwrite default behaviour 133 | super().pre() 134 | # We keep track of our intermediate results, so that we can print some 135 | # information about the progress of goals at the end of our run. 136 | self.intermediate_results = [] 137 | 138 | # Any solver options can be set here 139 | def solver_options(self): 140 | options = super().solver_options() 141 | solver = options["solver"] 142 | options[solver]["print_level"] = 1 143 | return options 144 | 145 | 146 | # Run 147 | if __name__ == "__main__": 148 | run_optimization_problem_closed_loop(Example) 149 | -------------------------------------------------------------------------------- /rtctools_interface/closed_loop/results_construction.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import logging 3 | import os 4 | from pathlib import Path 5 | 6 | import pandas as pd 7 | from rtctools.data import pi, rtc 8 | 9 | logger = logging.getLogger("rtctools") 10 | 11 | 12 | def _get_variables_from_pi(data_config: rtc.DataConfig, timeseries: pi.Timeseries): 13 | """Get all variables of a PI timeseries that are in the data configuration.""" 14 | variables = [] 15 | for var, _ in timeseries.items(): 16 | try: 17 | data_config.pi_variable_ids(var) 18 | variables.append(var) 19 | except KeyError: 20 | pass 21 | return variables 22 | 23 | 24 | def combine_xml_exports( 25 | output_base_path: Path, original_input_timeseries_path: Path, write_csv_out: bool = False 26 | ): 27 | """Combine the xml exports of multiple periods into a single xml file.""" 28 | logger.info("Combining XML exports.") 29 | dataconfig = rtc.DataConfig(folder=original_input_timeseries_path) 30 | 31 | ts_import_orig = pi.Timeseries( 32 | data_config=dataconfig, 33 | folder=original_input_timeseries_path, 34 | basename="timeseries_import", 35 | binary=False, 36 | ) 37 | if ts_import_orig.forecast_datetime > ts_import_orig.start_datetime: 38 | logger.info( 39 | "Timeseries export will start at original forecast date, " 40 | "disregarding data before forecast date." 41 | ) 42 | ts_import_orig.resize(ts_import_orig.forecast_datetime, ts_import_orig.end_datetime) 43 | ts_import_orig.times = ts_import_orig.times[ 44 | ts_import_orig.times.index(ts_import_orig.forecast_datetime) : 45 | ] 46 | orig_start_datetime = ts_import_orig.start_datetime 47 | orig_end_datetime = ts_import_orig.end_datetime 48 | 49 | ts_export = pi.Timeseries( 50 | data_config=dataconfig, 51 | folder=output_base_path / "period_0", 52 | basename="timeseries_export", 53 | binary=False, 54 | ) # Use the first timeseries export as a starting point for the combined timeseries export. 55 | ts_export.resize(orig_start_datetime, orig_end_datetime) 56 | 57 | variables = _get_variables_from_pi(data_config=dataconfig, timeseries=ts_export) 58 | 59 | i = 0 60 | while os.path.isfile(os.path.join(output_base_path, f"period_{i}", "timeseries_export.xml")): 61 | ts_export_step = pi.Timeseries( 62 | data_config=dataconfig, 63 | folder=os.path.join(output_base_path, f"period_{i}"), 64 | basename="timeseries_export", 65 | binary=False, 66 | ) 67 | all_times = ts_import_orig.times # Workaround to map indices to times, as ts_export does 68 | # not contain all times. TODO Check whether the assumption that these times map to 69 | # the correct indices for ts_export always holds. 70 | for loc_par in variables: 71 | try: 72 | current_values = ts_export.get(loc_par) 73 | new_values = ts_export_step.get(loc_par) 74 | except KeyError: 75 | logger.debug(f"Variable {loc_par} not found in output of model horizon: {i}") 76 | continue 77 | new_times = ts_export_step.times 78 | try: 79 | start_new_data_index = all_times.index(new_times[0]) 80 | except ValueError: 81 | if all_times[-1] + ts_export.dt == new_times[0]: 82 | start_new_data_index = len(all_times) 83 | else: 84 | raise ValueError( 85 | "Could not match the start data of the timeseries export file " 86 | + "with the end of the previous." 87 | ) 88 | combined_values = copy.deepcopy(current_values) 89 | combined_values[start_new_data_index : start_new_data_index + len(new_values)] = ( 90 | new_values # noqa 91 | ) 92 | ts_export.set(loc_par, combined_values) 93 | i += 1 94 | ts_export.write(output_folder=output_base_path.parent, output_filename="timeseries_export") 95 | 96 | if write_csv_out: 97 | data = pd.DataFrame({"date": all_times}) 98 | new_columns = [] 99 | for timeseries_id in variables: 100 | try: 101 | values = ts_export.get(timeseries_id) 102 | new_columns.append(pd.Series(values, name=timeseries_id)) 103 | except KeyError: 104 | logger.debug(f"Variable {timeseries_id} not found in output of model horizon: {i}") 105 | continue 106 | data = pd.concat([data] + new_columns, axis=1) 107 | data.round(6).to_csv(output_base_path.parent / "timeseries_export.csv", index=False) 108 | 109 | 110 | def combine_dataframes(dfs: list[pd.DataFrame], index_col: str = "time"): 111 | """Combine multiple dataframes with the same index column. 112 | The dataframes are combined in the order they are passed, 113 | with the last dataframe taking precedence in case of overlapping indices.""" 114 | combined_df = pd.DataFrame() 115 | for df in dfs: 116 | df.set_index(index_col, inplace=True) 117 | combined_df = df.combine_first(combined_df) 118 | combined_df.reset_index(inplace=True) 119 | return combined_df 120 | 121 | 122 | def combine_csv_exports(output_base_path: Path): 123 | """Combine the csv exports of multiple periods into a single csv file.""" 124 | i = 0 125 | dfs = [] 126 | while os.path.isfile(os.path.join(output_base_path, f"period_{i}", "timeseries_export.csv")): 127 | df = pd.read_csv(os.path.join(output_base_path, f"period_{i}", "timeseries_export.csv")) 128 | dfs.append(df) 129 | i += 1 130 | combined_df = combine_dataframes(dfs) 131 | combined_df.round(6).to_csv(output_base_path.parent / "timeseries_export.csv", index=False) 132 | 133 | 134 | if __name__ == "__main__": 135 | closed_loop_test_folder = Path(__file__).parents[2] / "tests" / "closed_loop" 136 | output_base_path = closed_loop_test_folder / Path( 137 | r"test_models\goal_programming_xml\output\output_modelling_periods_reference" 138 | ) 139 | original_input_timeseries_path = closed_loop_test_folder / Path( 140 | r"test_models\goal_programming_xml\input" 141 | ) 142 | combine_xml_exports(output_base_path, original_input_timeseries_path, True) 143 | -------------------------------------------------------------------------------- /rtctools_interface/closed_loop/optimization_ranges.py: -------------------------------------------------------------------------------- 1 | """Module for calculating optimization periods.""" 2 | 3 | import bisect 4 | import datetime 5 | from pathlib import Path 6 | 7 | import pandas as pd 8 | 9 | 10 | def get_optimization_ranges_from_file( 11 | file_path: Path, model_time_range: tuple[datetime.datetime, datetime.datetime] 12 | ): 13 | """Read horizon config from a csv file""" 14 | if not file_path.exists(): 15 | raise FileNotFoundError( 16 | "The closed_loop_dates csv does not exist. Please create a horizon " 17 | f"config file in {file_path}." 18 | ) 19 | try: 20 | closed_loop_dates = pd.read_csv(file_path) 21 | except pd.errors.EmptyDataError: 22 | raise ValueError( 23 | "The closed_loop_dates csv is empty. Please provide a valid file with " 24 | "start_date and end_date column." 25 | ) 26 | closed_loop_dates.columns = closed_loop_dates.columns.str.replace(" ", "") 27 | if not all(col in closed_loop_dates.columns for col in ["start_date", "end_date"]): 28 | raise ValueError( 29 | "The closed_loop_dates csv should have both 'start_date' and 'end_date' columns." 30 | ) 31 | closed_loop_dates["start_date"] = pd.to_datetime(closed_loop_dates["start_date"]) 32 | closed_loop_dates["end_date"] = pd.to_datetime(closed_loop_dates["end_date"]) 33 | for i in range(1, len(closed_loop_dates)): 34 | if closed_loop_dates["start_date"].iloc[i] > closed_loop_dates["end_date"].iloc[i - 1]: 35 | raise ValueError( 36 | f"Closed loop date table: Start date at row {i} is later than previous end date." 37 | ) 38 | if any(closed_loop_dates["start_date"] < closed_loop_dates["start_date"].shift(1)): 39 | raise ValueError("Closed loop date table: The start dates are not in ascending order.") 40 | if any(closed_loop_dates["end_date"] < closed_loop_dates["end_date"].shift(1)): 41 | raise ValueError("Closed loop date table: The end dates are not in ascending order.") 42 | if any(closed_loop_dates["end_date"] < closed_loop_dates["start_date"]): 43 | raise ValueError( 44 | "Closed loop date table: For one or more rows the end date is before the start date." 45 | ) 46 | if any(closed_loop_dates["start_date"] > closed_loop_dates["end_date"]): 47 | raise ValueError( 48 | "Closed loop date table: For one or more rows the start date is after the end date." 49 | ) 50 | if ( 51 | any(closed_loop_dates["start_date"].dt.hour != 0) 52 | or any(closed_loop_dates["start_date"].dt.minute != 0) 53 | or any(closed_loop_dates["end_date"].dt.hour != 0) 54 | or any(closed_loop_dates["end_date"].dt.minute != 0) 55 | ): 56 | raise ValueError( 57 | "Closed loop date table: Currently, the date ranges can only be specific " 58 | "up to the level of days." 59 | ) 60 | assert min(closed_loop_dates["start_date"]).date() == model_time_range[0].date(), ( 61 | "The start day of the first optimization run is not equal" 62 | " to the start day of the forecast date (or first timestep)." 63 | ) 64 | assert max(closed_loop_dates["end_date"]).date() <= model_time_range[1].date(), ( 65 | "The end date of one or more optimization runs is later" 66 | " than the end date of the timeseries import." 67 | ) 68 | closed_loop_dates = [ 69 | (optimization_range["start_date"], optimization_range["end_date"]) 70 | for _, optimization_range in closed_loop_dates.iterrows() 71 | ] 72 | return closed_loop_dates 73 | 74 | 75 | def _get_next_time_index( 76 | times: list[datetime.date], i_current: int, timestep_size: datetime.timedelta 77 | ) -> int: 78 | """ 79 | Get the next timestep index. 80 | 81 | The next timestep index i_next is the highest index such that: 82 | * times[i_next] > times[i_current] 83 | * times[i_next] <= times[i_current] + timestep_size 84 | In case i_current >= i_max, i_next will be set to i_max, where i_max = len(times) - 1. 85 | """ 86 | i_max = len(times) - 1 87 | if i_current >= i_max: 88 | return i_max 89 | current_time = times[i_current] 90 | next_time = current_time + timestep_size 91 | i_next = bisect.bisect_right(times, next_time) - 1 92 | i_next = max(i_current + 1, i_next) 93 | return i_next 94 | 95 | 96 | def get_optimization_ranges( 97 | model_times: list[datetime.date], 98 | start_time: datetime.datetime, 99 | forecast_timestep: datetime.timedelta, 100 | optimization_period: datetime.timedelta, 101 | ) -> list[tuple[datetime.datetime, datetime.datetime]]: 102 | """Calculate a list of optimization periods.""" 103 | if forecast_timestep > optimization_period: 104 | raise ValueError( 105 | f"Forecast timestep {forecast_timestep} cannot be larger than" 106 | f" the optimization period {optimization_period}." 107 | ) 108 | if start_time not in model_times: 109 | raise ValueError(f"Start time {start_time} is not in the given model times.") 110 | i_max = len(model_times) - 1 111 | i_start = model_times.index(start_time) 112 | i_end = _get_next_time_index(model_times, i_start, optimization_period) 113 | optimization_periods = [] 114 | optimization_periods.append((model_times[i_start], model_times[i_end])) 115 | while i_end < i_max: 116 | i_start = _get_next_time_index(model_times, i_start, forecast_timestep) 117 | i_end = _get_next_time_index(model_times, i_start, optimization_period) 118 | optimization_periods.append((model_times[i_start], model_times[i_end])) 119 | return optimization_periods 120 | 121 | 122 | def round_datetime_ranges_to_days( 123 | datetime_ranges: list[tuple[datetime.datetime, datetime.datetime]], 124 | ) -> list[tuple[datetime.datetime, datetime.datetime]]: 125 | """Round datetimes to dats in datetime ranges. 126 | 127 | The start of the range is rounded to the start of the day 128 | and the end of the range is rounded to the end of the day. 129 | """ 130 | rounded_ranges = [] 131 | for start, end in datetime_ranges: 132 | if start.date() == end.date(): 133 | continue 134 | start = datetime.datetime.combine(start.date(), datetime.time.min) 135 | end = datetime.datetime.combine(end.date(), datetime.time.max) 136 | rounded_ranges.append((start, end)) 137 | return rounded_ranges 138 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input/timeseries_import.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -6.0 4 | 5 |
6 | instantaneous 7 | H_sea 8 | par_0 9 | 10 | 11 | 12 | -999.0 13 | test 14 | 2024-01-01 15 | 11:00:00 16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 |
41 | instantaneous 42 | Q_in 43 | par_0 44 | 45 | 46 | 47 | -999.0 48 | test 49 | 2024-01-01 50 | 11:00:00 51 |
52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 |
76 | instantaneous 77 | storage_level 78 | par_0 79 | 80 | 81 | 82 | -999.0 83 | test 84 | 2024-01-01 85 | 11:00:00 86 |
87 | 88 | 89 | 90 |
91 | instantaneous 92 | Q_orifice 93 | par_0 94 | 95 | 96 | 97 | -999.0 98 | test 99 | 2024-01-01 100 | 11:00:00 101 |
102 | 103 | 104 | 105 |
106 | instantaneous 107 | Q_pump 108 | par_0 109 | 110 | 111 | 112 | -999.0 113 | test 114 | 2024-01-01 115 | 11:00:00 116 |
117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input_with_forecast_date_equal_first_date/timeseries_import.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -6.0 4 | 5 |
6 | instantaneous 7 | H_sea 8 | par_0 9 | 10 | 11 | 12 | 13 | -999.0 14 | test 15 | 2024-01-01 16 | 11:00:00 17 |
18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 |
42 | instantaneous 43 | Q_in 44 | par_0 45 | 46 | 47 | 48 | 49 | -999.0 50 | test 51 | 2024-01-01 52 | 11:00:00 53 |
54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 |
78 | instantaneous 79 | storage_level 80 | par_0 81 | 82 | 83 | 84 | 85 | -999.0 86 | test 87 | 2024-01-01 88 | 11:00:00 89 |
90 | 91 | 92 | 93 |
94 | instantaneous 95 | Q_orifice 96 | par_0 97 | 98 | 99 | 100 | 101 | -999.0 102 | test 103 | 2024-01-01 104 | 11:00:00 105 |
106 | 107 | 108 | 109 |
110 | instantaneous 111 | Q_pump 112 | par_0 113 | 114 | 115 | 116 | 117 | -999.0 118 | test 119 | 2024-01-01 120 | 11:00:00 121 |
122 | 123 | 124 | 125 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/input_with_forecast_date/timeseries_import.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -6.0 4 | 5 |
6 | instantaneous 7 | H_sea 8 | par_0 9 | 10 | 11 | 12 | 13 | -999.0 14 | test 15 | 2024-01-01 16 | 11:00:00 17 |
18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 |
43 | instantaneous 44 | Q_in 45 | par_0 46 | 47 | 48 | 49 | 50 | -999.0 51 | test 52 | 2024-01-01 53 | 11:00:00 54 |
55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 |
80 | instantaneous 81 | storage_level 82 | par_0 83 | 84 | 85 | 86 | 87 | -999.0 88 | test 89 | 2024-01-01 90 | 11:00:00 91 |
92 | 93 | 94 | 95 | 96 |
97 | instantaneous 98 | Q_orifice 99 | par_0 100 | 101 | 102 | 103 | 104 | -999.0 105 | test 106 | 2024-01-01 107 | 11:00:00 108 |
109 | 110 | 111 | 112 | 113 |
114 | instantaneous 115 | Q_pump 116 | par_0 117 | 118 | 119 | 120 | 121 | -999.0 122 | test 123 | 2024-01-01 124 | 11:00:00 125 |
126 | 127 | 128 | 129 | 130 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/output/output_modelling_periods_reference/period_0/timeseries_export.xml: -------------------------------------------------------------------------------- 1 | 2 | -6.0 3 | 4 | 5 | instantaneous 6 | H_sea 7 | par_0 8 | 9 | 10 | 11 | -999 12 | H_sea 13 | test 14 | 2024-06-13 15 | 17:01:45 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | instantaneous 30 | Q_orifice 31 | par_0 32 | 33 | 34 | 35 | -999 36 | Q_orifice 37 | test 38 | 2024-06-13 39 | 17:01:45 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | instantaneous 54 | Q_pump 55 | par_0 56 | 57 | 58 | 59 | -999 60 | Q_pump 61 | test 62 | 2024-06-13 63 | 17:01:45 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | instantaneous 78 | is_downhill 79 | par_0 80 | 81 | 82 | 83 | -999 84 | is_downhill 85 | unit_unknown 86 | 2024-06-13 87 | 17:01:45 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | instantaneous 102 | sea_level 103 | par_0 104 | 105 | 106 | 107 | -999 108 | sea_level 109 | unit_unknown 110 | 2024-06-13 111 | 17:01:45 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | instantaneous 126 | storage_level 127 | par_0 128 | 129 | 130 | 131 | -999 132 | storage_level 133 | test 134 | 2024-06-13 135 | 17:01:45 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/output/output_modelling_periods_reference/period_2/timeseries_export.xml: -------------------------------------------------------------------------------- 1 | 2 | -6.0 3 | 4 | 5 | instantaneous 6 | H_sea 7 | par_0 8 | 9 | 10 | 11 | -999 12 | H_sea 13 | test 14 | 2024-06-13 15 | 17:01:49 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | instantaneous 30 | Q_orifice 31 | par_0 32 | 33 | 34 | 35 | -999 36 | Q_orifice 37 | test 38 | 2024-06-13 39 | 17:01:49 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | instantaneous 54 | Q_pump 55 | par_0 56 | 57 | 58 | 59 | -999 60 | Q_pump 61 | test 62 | 2024-06-13 63 | 17:01:49 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | instantaneous 78 | is_downhill 79 | par_0 80 | 81 | 82 | 83 | -999 84 | is_downhill 85 | unit_unknown 86 | 2024-06-13 87 | 17:01:49 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | instantaneous 102 | sea_level 103 | par_0 104 | 105 | 106 | 107 | -999 108 | sea_level 109 | unit_unknown 110 | 2024-06-13 111 | 17:01:49 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | instantaneous 126 | storage_level 127 | par_0 128 | 129 | 130 | 131 | -999 132 | storage_level 133 | test 134 | 2024-06-13 135 | 17:01:49 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /tests/closed_loop/test_models/goal_programming_xml/output/output_modelling_periods_reference/period_1/timeseries_export.xml: -------------------------------------------------------------------------------- 1 | 2 | -6.0 3 | 4 | 5 | instantaneous 6 | H_sea 7 | par_0 8 | 9 | 10 | 11 | -999 12 | H_sea 13 | test 14 | 2024-06-13 15 | 17:01:47 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | instantaneous 30 | Q_orifice 31 | par_0 32 | 33 | 34 | 35 | -999 36 | Q_orifice 37 | test 38 | 2024-06-13 39 | 17:01:47 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | instantaneous 54 | Q_pump 55 | par_0 56 | 57 | 58 | 59 | -999 60 | Q_pump 61 | test 62 | 2024-06-13 63 | 17:01:47 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | instantaneous 78 | is_downhill 79 | par_0 80 | 81 | 82 | 83 | -999 84 | is_downhill 85 | unit_unknown 86 | 2024-06-13 87 | 17:01:47 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | instantaneous 102 | sea_level 103 | par_0 104 | 105 | 106 | 107 | -999 108 | sea_level 109 | unit_unknown 110 | 2024-06-13 111 | 17:01:47 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | instantaneous 126 | storage_level 127 | par_0 128 | 129 | 130 | 131 | -999 132 | storage_level 133 | test 134 | 2024-06-13 135 | 17:01:47 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /COPYING.LESSER: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /rtctools_interface/utils/results_collection.py: -------------------------------------------------------------------------------- 1 | """Mixin to store all required data for plotting.""" 2 | 3 | import copy 4 | import logging 5 | import os 6 | import time 7 | from pathlib import Path 8 | 9 | import numpy as np 10 | 11 | from rtctools_interface.utils.plot_table_schema import PlotTableRow 12 | from rtctools_interface.utils.read_goals_mixin import ReadGoalsMixin 13 | from rtctools_interface.utils.read_plot_table import get_plot_config 14 | from rtctools_interface.utils.serialization import deserialize, serialize 15 | from rtctools_interface.utils.type_definitions import ( 16 | PlotDataAndConfig, 17 | PlotOptions, 18 | PrioIndependentData, 19 | ) 20 | 21 | MAX_NUM_CACHED_FILES = 5 22 | CONFIG_VERSION: float = 1.0 23 | 24 | logger = logging.getLogger("rtctools") 25 | 26 | 27 | def get_most_recent_cache(cache_folder): 28 | """Get the most recent pickle file, based on its name.""" 29 | cache_folder = Path(cache_folder) 30 | json_files = list(cache_folder.glob("*.json")) 31 | 32 | if json_files: 33 | return max(json_files, key=lambda file: int(file.stem), default=None) 34 | return None 35 | 36 | 37 | def clean_cache_folder(cache_folder, max_files=10): 38 | """Clean the cache folder with pickles. 39 | 40 | remove the oldest ones when there are more than `max_files`. 41 | """ 42 | cache_path = Path(cache_folder) 43 | files = [f for f in cache_path.iterdir() if f.suffix == ".json"] 44 | 45 | if len(files) > max_files: 46 | files.sort(key=lambda x: int(x.stem)) 47 | files_to_delete = len(files) - max_files 48 | for i in range(files_to_delete): 49 | file_to_delete = cache_path / files[i] 50 | file_to_delete.unlink() 51 | 52 | 53 | def write_cache_file(cache_folder: Path, results_to_store: PlotDataAndConfig): 54 | """Write a file to the cache folder as a pickle file with the linux time as name.""" 55 | os.makedirs(cache_folder, exist_ok=True) 56 | file_name = int(time.time()) 57 | with open(cache_folder / f"{file_name}.json", "w", encoding="utf-8") as json_file: 58 | json_file.write(serialize(results_to_store)) 59 | 60 | clean_cache_folder(cache_folder, MAX_NUM_CACHED_FILES) 61 | 62 | 63 | def read_cache_file_from_folder(cache_folder: Path) -> PlotDataAndConfig | None: 64 | """Read the most recent file from the cache folder.""" 65 | cache_file_path = get_most_recent_cache(cache_folder) 66 | loaded_data: PlotDataAndConfig | None 67 | if cache_file_path: 68 | with open(cache_file_path, encoding="utf-8") as handle: 69 | loaded_data: dict = deserialize(handle.read()) 70 | if loaded_data.get("config_version", 0) < CONFIG_VERSION: 71 | logger.warning( 72 | """The cache file that was found is not supported by 73 | the current version of rtc-tools-interface!""" 74 | ) 75 | loaded_data = None 76 | else: 77 | loaded_data = None 78 | return loaded_data 79 | 80 | 81 | def get_plot_variables(plot_config: list[PlotTableRow]) -> list[str]: 82 | """Get list of variable-names that are in the plot table.""" 83 | variables_style_1 = [ 84 | var for subplot_config in plot_config for var in subplot_config.variables_style_1 85 | ] 86 | variables_style_2 = [ 87 | var for subplot_config in plot_config for var in subplot_config.variables_style_2 88 | ] 89 | variables_with_previous_result = [ 90 | var 91 | for subplot_config in plot_config 92 | for var in subplot_config.variables_with_previous_result 93 | ] 94 | return list(set(variables_style_1 + variables_style_2 + variables_with_previous_result)) 95 | 96 | 97 | def filter_plot_config( 98 | plot_config: list[PlotTableRow], all_goal_generator_goals 99 | ) -> list[PlotTableRow]: 100 | """ "Remove PlotTableRows corresponding to non-existing goals in the goal generator.""" 101 | goal_generator_goal_ids = [goal.goal_id for goal in all_goal_generator_goals] 102 | new_plot_config = [ 103 | plot_table_row 104 | for plot_table_row in plot_config 105 | if plot_table_row.id in goal_generator_goal_ids or plot_table_row.specified_in == "python" 106 | ] 107 | return new_plot_config 108 | 109 | 110 | class PlottingBaseMixin(ReadGoalsMixin): 111 | """Base class for creating plots. 112 | 113 | Reads the plot table, if available the goal table, and contains 114 | functions to store all required data for plots.""" 115 | 116 | plot_max_rows = 4 117 | plot_results_each_priority = True 118 | plot_final_results = True 119 | 120 | def __init__(self, **kwargs): 121 | super().__init__(**kwargs) 122 | try: 123 | plot_table_file = self.plot_table_file 124 | except AttributeError: 125 | plot_table_file = os.path.join(self._input_folder, "plot_table.csv") 126 | plot_config_list = kwargs.get("plot_config_list", []) 127 | read_from = kwargs.get("read_goals_from", "csv_table") 128 | self.save_plot_to = kwargs.get("save_plot_to", "image") 129 | self.plotting_library = kwargs.get("plotting_library", "plotly") 130 | self.plot_config = get_plot_config(plot_table_file, plot_config_list, read_from) 131 | 132 | self.custom_variables = get_plot_variables(self.plot_config) 133 | 134 | if not hasattr(self, "_all_goal_generator_goals") and self.optimization_problem: 135 | goals_to_generate = kwargs.get("goals_to_generate", []) 136 | read_from = kwargs.get("read_goals_from", "csv_table") 137 | self.load_goals(read_from, goals_to_generate) 138 | 139 | if self.optimization_problem: 140 | all_goal_generator_goals = self._all_goal_generator_goals 141 | self.state_variables = list({base_goal.state for base_goal in all_goal_generator_goals}) 142 | else: 143 | self.state_variables = [] 144 | all_goal_generator_goals = [] 145 | 146 | self.plot_config = filter_plot_config(self.plot_config, all_goal_generator_goals) 147 | 148 | self._cache_folder = Path(self._output_folder) / "cached_results" 149 | if "previous_run_plot_config" in kwargs: 150 | self._previous_run = kwargs["previous_run_plot_config"] 151 | else: 152 | self._previous_run = read_cache_file_from_folder(self._cache_folder) 153 | 154 | def pre(self): 155 | """Tasks before optimizing.""" 156 | super().pre() 157 | self._intermediate_results = [] 158 | 159 | def collect_timeseries_data(self, all_variables_to_store: list[str]) -> dict[str, np.ndarray]: 160 | """Collect the timeseries data for a list of variables.""" 161 | extracted_results = copy.deepcopy(self.extract_results()) 162 | timeseries_data = {} 163 | for timeseries_name in all_variables_to_store: 164 | try: 165 | timeseries_data[timeseries_name] = extracted_results[timeseries_name] 166 | except KeyError: 167 | try: 168 | timeseries_data[timeseries_name] = self.io.get_timeseries(timeseries_name)[1] 169 | except KeyError as exc: 170 | raise KeyError(f"Cannot find timeseries for {timeseries_name}") from exc 171 | return timeseries_data 172 | 173 | def create_plot_data_and_config(self, base_goals: list) -> PlotDataAndConfig: 174 | """Create the PlotDataAndConfig dict.""" 175 | prio_independent_data: PrioIndependentData = { 176 | "io_datetimes": [ 177 | dt 178 | for idx, dt in enumerate(self.io.datetimes) 179 | if self.io.times_sec[idx] in self.times() 180 | ], 181 | "times": self.times(), 182 | "base_goals": base_goals, 183 | } 184 | plot_options: PlotOptions = { 185 | "plot_config": self.plot_config, 186 | "plot_max_rows": self.plot_max_rows, 187 | "output_folder": self._output_folder, 188 | "save_plot_to": self.save_plot_to, 189 | } 190 | plot_data_and_config: PlotDataAndConfig = { 191 | "intermediate_results": self._intermediate_results, 192 | "plot_options": plot_options, 193 | "prio_independent_data": prio_independent_data, 194 | "config_version": CONFIG_VERSION, 195 | } 196 | return plot_data_and_config 197 | 198 | def _store_current_results(self, cache_folder, results_to_store): 199 | write_cache_file(cache_folder, results_to_store) 200 | self._plot_data_and_config = results_to_store 201 | 202 | @property 203 | def get_plot_data_and_config(self): 204 | """Get the plot data and config from the current run.""" 205 | return self._plot_data_and_config 206 | -------------------------------------------------------------------------------- /rtctools_interface/closed_loop/time_series_handler.py: -------------------------------------------------------------------------------- 1 | import bisect 2 | import datetime 3 | import logging 4 | from abc import ABC, abstractmethod 5 | from pathlib import Path 6 | 7 | import numpy as np 8 | import pandas as pd 9 | from rtctools.data import csv, pi, rtc 10 | 11 | ns = {"fews": "http://www.wldelft.nl/fews", "pi": "http://www.wldelft.nl/fews/PI"} 12 | 13 | logger = logging.getLogger("rtctools") 14 | 15 | 16 | class TimeSeriesHandler(ABC): 17 | """ABC for handling timeseries data.""" 18 | 19 | # The forecast date determines at which date the optimization starts. 20 | forecast_date: datetime.datetime | None = None 21 | 22 | @abstractmethod 23 | def read(self, file_name: str) -> None: 24 | """Read the timeseries.""" 25 | 26 | @abstractmethod 27 | def select_time_range(self, start_date: datetime.datetime, end_date: datetime.datetime) -> None: 28 | """Select a time range from the timeseries data. Removes data outside the interval. 29 | The specified range is inclusive on both sides.""" 30 | 31 | @abstractmethod 32 | def write(self, file_path: Path) -> None: 33 | """Write the timeseries data to a file.""" 34 | 35 | @abstractmethod 36 | def get_timestep(self) -> datetime.timedelta: 37 | """Get the timestep of the timeseries data.""" 38 | 39 | @abstractmethod 40 | def get_datetimes(self) -> list[datetime.datetime]: 41 | """Get the dates of the timeseries.""" 42 | 43 | @abstractmethod 44 | def get_datetime_range(self) -> tuple[datetime.datetime, datetime.datetime]: 45 | """Get the date range of the timeseries data (min, max).""" 46 | 47 | @abstractmethod 48 | def get_all_internal_ids(self) -> list[str]: 49 | """Get all internal id's of the timeseries data.""" 50 | 51 | @abstractmethod 52 | def set_initial_value(self, internal_id: str, value: float) -> None: 53 | """Set the initial value of a variable in the timeseries data.""" 54 | 55 | @abstractmethod 56 | def is_set(self, internal_id: str) -> bool: 57 | """Check whether the variable exists in the timeseries data 58 | and whether it has a least one non-nan value""" 59 | 60 | 61 | class CSVTimeSeriesFile(TimeSeriesHandler): 62 | """Timeseries handler for csv files.""" 63 | 64 | def __init__( 65 | self, 66 | input_folder: Path, 67 | timeseries_import_basename: str = "timeseries_import", 68 | csv_delimiter=",", 69 | initial_state_base_name: str = "initial_state", 70 | ): 71 | self.data = pd.DataFrame() 72 | self.input_folder = input_folder 73 | self.csv_delimiter = csv_delimiter 74 | self.data_col = None 75 | self.initial_state = None 76 | self.read(timeseries_import_basename, initial_state_base_name) 77 | 78 | def read(self, file_name: str, initial_state_base_name=None): 79 | timeseries = csv.load( 80 | (self.input_folder / file_name).with_suffix(".csv"), 81 | delimiter=self.csv_delimiter, 82 | with_time=True, 83 | ) 84 | self.data = pd.DataFrame(timeseries) 85 | if self.data is not None: 86 | self.date_col = self.data.columns[0] 87 | self.forecast_date = self.data[self.date_col].iloc[0] 88 | else: 89 | raise ValueError("No data to read.") 90 | if initial_state_base_name is not None: 91 | initial_state_file = self.input_folder / initial_state_base_name 92 | if initial_state_file.with_suffix(".csv").exists(): 93 | initial_state = csv.load( 94 | initial_state_file.with_suffix(".csv"), 95 | delimiter=self.csv_delimiter, 96 | with_time=False, 97 | ) 98 | self.initial_state: dict | None = { 99 | field: float(initial_state[field]) for field in initial_state.dtype.names 100 | } 101 | else: 102 | self.initial_state = None 103 | 104 | def select_time_range(self, start_date: datetime.datetime, end_date: datetime.datetime): 105 | mask = (self.data[self.date_col] >= start_date) & (self.data[self.date_col] <= end_date) 106 | self.data = self.data.loc[mask] 107 | self.forecast_date = start_date 108 | 109 | def write(self, file_path: Path): 110 | self.write_timeseries(file_path) 111 | self.write_initial_state(file_path) 112 | 113 | def write_timeseries(self, file_path: Path, file_name: str = "timeseries_import"): 114 | self.data.to_csv( 115 | (file_path / file_name).with_suffix(".csv"), 116 | index=False, 117 | date_format="%Y-%m-%d %H:%M:%S", 118 | ) 119 | 120 | def write_initial_state(self, file_path: Path, file_name: str = "initial_state"): 121 | if self.initial_state is not None: 122 | initial_state = pd.DataFrame(self.initial_state, index=[0]) 123 | initial_state.to_csv( 124 | (file_path / file_name).with_suffix(".csv"), header=True, index=False 125 | ) 126 | 127 | def get_timestep(self): 128 | return self.data[self.date_col].diff().min() 129 | 130 | def get_datetimes(self): 131 | return self.data[self.date_col].to_list() 132 | 133 | def get_datetime_range(self): 134 | return self.data[self.date_col].min(), self.data[self.date_col].max() 135 | 136 | def get_all_internal_ids(self): 137 | ids = list(self.data.columns[1:]) 138 | if self.initial_state is not None: 139 | ids.extend(list(self.initial_state.keys())) 140 | return ids 141 | 142 | def set_initial_value(self, internal_id, value): 143 | if self.initial_state is None or internal_id not in self.initial_state: 144 | self.data[internal_id].iloc[0] = value 145 | else: 146 | self.initial_state[internal_id] = value 147 | 148 | def is_set(self, internal_id): 149 | val_is_set = False 150 | if internal_id in self.data.columns: 151 | val_is_set = not self.data[internal_id].isna().all() 152 | if self.initial_state is not None and internal_id in self.initial_state: 153 | val_is_set = False 154 | return val_is_set 155 | 156 | 157 | class XMLTimeSeriesFile(TimeSeriesHandler): 158 | """ "Timeseries handler for xml files""" 159 | 160 | def __init__( 161 | self, 162 | input_folder: Path, 163 | timeseries_import_basename: str = "timeseries_import", 164 | ): 165 | self.input_folder = input_folder 166 | self.pi_binary_timeseries = False 167 | self.pi_validate_timeseries = True 168 | self.data_config = None 169 | self.pi_timeseries = None 170 | self.read(timeseries_import_basename) 171 | 172 | if self.get_datetime_range()[0] < self.forecast_date: 173 | logger.warning( 174 | "Currently, the closed loop runner does support data before the forecast date." 175 | ) 176 | logger.warning("Removing data before forecast date.") 177 | self.select_time_range(self.forecast_date, self.pi_timeseries.times[-1]) 178 | 179 | def read(self, file_name: str): 180 | """Read the timeseries data from a file.""" 181 | timeseries_import_basename = file_name 182 | self.data_config = rtc.DataConfig(self.input_folder) 183 | self.pi_timeseries = pi.Timeseries( 184 | self.data_config, 185 | self.input_folder, 186 | timeseries_import_basename, 187 | binary=self.pi_binary_timeseries, 188 | pi_validate_times=self.pi_validate_timeseries, 189 | ) 190 | self.forecast_date = self.pi_timeseries.forecast_datetime 191 | 192 | def is_set(self, internal_id): 193 | """Check whether the variable exists in the timeseries data. 194 | 195 | and whether it has a value at at least one of time steps.""" 196 | try: 197 | var: np.ndarray = self.pi_timeseries.get(variable=internal_id) 198 | return not np.isnan(var).all() 199 | except KeyError: 200 | return False 201 | 202 | def _is_in_dataconfig(self, internal_id): 203 | """Check if an internal id is in the data configuration.""" 204 | try: 205 | self.data_config.pi_variable_ids(internal_id) 206 | return True 207 | except KeyError: 208 | return False 209 | 210 | def get_all_internal_ids(self): 211 | """Get all internal id's of the timeseries data. 212 | 213 | Only returns the id's that are also in the dataconfig.""" 214 | all_ids = [var for var, _ in self.pi_timeseries.items() if self._is_in_dataconfig(var)] 215 | return all_ids 216 | 217 | def select_time_range(self, start_date: datetime.datetime, end_date: datetime.datetime): 218 | times = self.pi_timeseries.times 219 | i_start = bisect.bisect_left(times, start_date) 220 | i_end = bisect.bisect_right(times, end_date) - 1 221 | self.pi_timeseries.resize(start_datetime=times[i_start], end_datetime=times[i_end]) 222 | self.pi_timeseries.times = times[i_start : i_end + 1] 223 | self.pi_timeseries.forecast_datetime = times[i_start] 224 | self.forecast_date = times[i_start] 225 | 226 | def write(self, file_path: Path, file_name: str = "timeseries_import"): 227 | # By setting make_new_file headers will be recreated, 228 | # neceesary for writing new forecast date 229 | self.pi_timeseries.make_new_file = True 230 | self.pi_timeseries.write(output_folder=file_path, output_filename=file_name) 231 | 232 | def get_datetimes(self): 233 | """Get the dates of all timeseries data.""" 234 | return self.pi_timeseries.times 235 | 236 | def get_datetime_range(self): 237 | """Get the date range of the timeseries data, minimum and maximum over all series""" 238 | times = self.pi_timeseries.times 239 | return times[0], times[-1] 240 | 241 | def get_timestep(self): 242 | """Get the timestep of the timeseries data, raise error if different stepsizes""" 243 | return self.pi_timeseries.dt 244 | 245 | def set_initial_value(self, internal_id: str, value: float): 246 | self.pi_timeseries.set(internal_id, [value]) 247 | -------------------------------------------------------------------------------- /rtctools_interface/closed_loop/runner.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import datetime 3 | import json 4 | import logging 5 | import os 6 | import shutil 7 | import sys 8 | from pathlib import Path 9 | 10 | from rtctools.data.pi import DiagHandler 11 | from rtctools.optimization.csv_mixin import CSVMixin 12 | from rtctools.optimization.pi_mixin import PIMixin 13 | from rtctools.util import _resolve_folder, run_optimization_problem 14 | 15 | import rtctools_interface.closed_loop.optimization_ranges as opt_ranges 16 | from rtctools_interface.closed_loop.config import ClosedLoopConfig 17 | from rtctools_interface.closed_loop.results_construction import ( 18 | combine_csv_exports, 19 | combine_xml_exports, 20 | ) 21 | from rtctools_interface.closed_loop.time_series_handler import ( 22 | CSVTimeSeriesFile, 23 | TimeSeriesHandler, 24 | XMLTimeSeriesFile, 25 | ) 26 | 27 | logger = logging.getLogger("rtctools") 28 | 29 | 30 | def set_initial_values_from_previous_run( 31 | results_previous_run: dict | None, 32 | timeseries: TimeSeriesHandler, 33 | previous_run_datetimes: list[datetime.datetime], 34 | ) -> None: 35 | """Modifies the initial values of `timeseries` based on the results of the previous run 36 | (if any)""" 37 | if results_previous_run is not None: 38 | variables_to_set = { 39 | key: value for key, value in results_previous_run.items() if not timeseries.is_set(key) 40 | } 41 | if timeseries.forecast_date: 42 | index_of_initial_value = previous_run_datetimes.index(timeseries.forecast_date) 43 | else: 44 | raise ValueError("Could not find forecast date in timeseries import.") 45 | for key, values in variables_to_set.items(): 46 | if values is not None: 47 | timeseries.set_initial_value(key, values[index_of_initial_value]) 48 | else: 49 | raise ValueError(f"Could not find initial value for {key}.") 50 | 51 | 52 | def write_input_folder( 53 | modelling_period_input_folder_i: Path, 54 | original_input_folder: Path, 55 | timeseries_import: TimeSeriesHandler, 56 | ) -> None: 57 | """Write the input folder for the current modelling period. 58 | Copies the original input folder to the modelling period input folder and writes the new 59 | timeseries_import.""" 60 | modelling_period_input_folder_i.mkdir(exist_ok=True) 61 | for file in original_input_folder.iterdir(): 62 | if file.is_file(): 63 | shutil.copy(file, modelling_period_input_folder_i / file.name) 64 | elif file.is_dir(): 65 | shutil.copytree(file, modelling_period_input_folder_i / file.name) 66 | timeseries_import.write(modelling_period_input_folder_i) 67 | 68 | 69 | def _get_optimization_ranges( 70 | config: ClosedLoopConfig, 71 | input_timeseries: TimeSeriesHandler, 72 | ) -> list[tuple[datetime.datetime, datetime.datetime]]: 73 | """Return a list of optimization periods.""" 74 | if config.file is not None: 75 | datetime_range = input_timeseries.get_datetime_range() 76 | optimization_ranges = opt_ranges.get_optimization_ranges_from_file( 77 | config.file, datetime_range 78 | ) 79 | elif config.optimization_period is not None: 80 | datetimes = input_timeseries.get_datetimes() 81 | optimization_ranges = opt_ranges.get_optimization_ranges( 82 | model_times=datetimes, 83 | start_time=datetimes[0], 84 | forecast_timestep=config.forecast_timestep, 85 | optimization_period=config.optimization_period, 86 | ) 87 | else: 88 | raise ValueError( 89 | "The closed-loop configuration should have either a file or optimization_period set." 90 | ) 91 | if config.round_to_dates: 92 | optimization_ranges = opt_ranges.round_datetime_ranges_to_days(optimization_ranges) 93 | return optimization_ranges 94 | 95 | 96 | def run_optimization_problem_closed_loop( 97 | optimization_problem_class, 98 | base_folder="..", 99 | log_level=logging.INFO, 100 | profile=False, 101 | config: ClosedLoopConfig | None = None, 102 | modelling_period_input_folder: str | None = None, 103 | **kwargs, 104 | ) -> dict: 105 | """ 106 | Runs an optimization problem in closed loop mode. 107 | 108 | This function is a drop-in replacement for the run_optimization_problem of rtc-tools. 109 | The user needs to specify a closed-loop configuration that describes the time ranges 110 | for which to subsequentially solve the optimization problem. 111 | The results from the previous run will be used to set the initial values of the next run. 112 | See README.md for more details. 113 | """ 114 | base_folder = Path(base_folder) 115 | if not os.path.isabs(base_folder): 116 | base_folder = Path(sys.path[0]) / base_folder 117 | original_input_folder = Path(_resolve_folder(kwargs, base_folder, "input_folder", "input")) 118 | original_output_folder = Path(_resolve_folder(kwargs, base_folder, "output_folder", "output")) 119 | original_output_folder.mkdir(exist_ok=True) 120 | 121 | # Set logging handlers. 122 | if not logger.hasHandlers(): 123 | handler = logging.StreamHandler() 124 | formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") 125 | handler.setFormatter(formatter) 126 | logger.addHandler(handler) 127 | if issubclass(optimization_problem_class, PIMixin): 128 | if not any(isinstance(h, DiagHandler) for h in logger.handlers): 129 | diag_handler = DiagHandler(original_output_folder) 130 | logger.addHandler(diag_handler) 131 | logger.setLevel(log_level) 132 | 133 | if issubclass(optimization_problem_class, PIMixin): 134 | original_import = XMLTimeSeriesFile(original_input_folder) 135 | elif issubclass(optimization_problem_class, CSVMixin): 136 | original_import = CSVTimeSeriesFile(original_input_folder) 137 | else: 138 | raise ValueError("Optimization problem class must be derived from PIMixin or CSVMixin.") 139 | 140 | variables_in_import = original_import.get_all_internal_ids() 141 | 142 | fixed_input_config_file = (original_input_folder / "fixed_inputs.json").resolve() 143 | if not fixed_input_config_file.exists(): 144 | raise FileNotFoundError( 145 | f"Could not find fixed inputs configuration file: {fixed_input_config_file}" 146 | "Create a file with a list of strings that represent the fixed inputs" 147 | " (can be an empty list)." 148 | ) 149 | with open(fixed_input_config_file) as file: 150 | fixed_input_series = json.load(file) 151 | if not isinstance(fixed_input_series, list) and all( 152 | isinstance(item, str) for item in fixed_input_series 153 | ): 154 | raise ValueError("Fixed input config file should be a list of strings (or an empty list).") 155 | 156 | if modelling_period_input_folder is None: 157 | modelling_period_input_folder = base_folder / "input_modelling_periods" 158 | if modelling_period_input_folder.exists(): 159 | shutil.rmtree(modelling_period_input_folder) 160 | modelling_period_input_folder.mkdir(exist_ok=True) 161 | 162 | modelling_periods_output_folder = original_output_folder / "output_modelling_periods" 163 | if modelling_periods_output_folder.exists(): 164 | shutil.rmtree(modelling_periods_output_folder) 165 | modelling_periods_output_folder.mkdir(exist_ok=True) 166 | 167 | if config is None: 168 | config = ClosedLoopConfig(original_input_folder / "closed_loop_dates.csv") 169 | optimization_ranges = _get_optimization_ranges(config, original_import) 170 | results_previous_run = None 171 | previous_run_datetimes = None 172 | for i, (start_time, end_time) in enumerate(optimization_ranges): 173 | timeseries_import = copy.deepcopy(original_import) 174 | 175 | timeseries_import.select_time_range(start_date=start_time, end_date=end_time) 176 | 177 | set_initial_values_from_previous_run( 178 | results_previous_run, timeseries_import, previous_run_datetimes 179 | ) 180 | 181 | modelling_period_name = f"period_{i}" 182 | modelling_period_output_folder_i = modelling_periods_output_folder / modelling_period_name 183 | modelling_period_output_folder_i.mkdir(exist_ok=True) 184 | modelling_period_input_folder_i = modelling_period_input_folder / modelling_period_name 185 | write_input_folder( 186 | modelling_period_input_folder_i, original_input_folder, timeseries_import 187 | ) 188 | 189 | logger.info(f"Running optimization for period {i}: {(str(start_time), str(end_time))}.") 190 | result = run_optimization_problem( 191 | optimization_problem_class, 192 | base_folder, 193 | log_level, 194 | profile, 195 | input_folder=modelling_period_input_folder_i, 196 | output_folder=modelling_period_output_folder_i, 197 | **kwargs, 198 | ) 199 | period = f"period {i} {(str(start_time), str(end_time))}" 200 | if result.solver_stats["success"]: 201 | logger.info(f"Successful optimization for {period}.") 202 | else: 203 | message = ( 204 | f"Failed optimization for {period} with status" 205 | + f"{result.solver_stats['return_status']}'." 206 | ) 207 | logger.error(message) 208 | raise Exception(message) 209 | 210 | results_previous_run = { 211 | key: result.extract_results().get(key) 212 | for key in variables_in_import 213 | if key not in fixed_input_series 214 | } 215 | previous_run_datetimes = result.io.datetimes 216 | 217 | logger.info("Finished all optimization runs.") 218 | if issubclass(optimization_problem_class, PIMixin): 219 | combine_xml_exports( 220 | modelling_periods_output_folder, original_input_folder, write_csv_out=True 221 | ) 222 | elif issubclass(optimization_problem_class, CSVMixin): 223 | combine_csv_exports(modelling_periods_output_folder) 224 | else: 225 | logger.warning( 226 | "Could not combine exports because the optimization problem class is not " 227 | "derived from PIMixin or CSVMixin." 228 | ) 229 | return result.solver_stats 230 | -------------------------------------------------------------------------------- /tests/closed_loop/test_run_optization_problem_closed_loop.py: -------------------------------------------------------------------------------- 1 | """Test the closed loop runner""" 2 | 3 | import math 4 | import xml.etree.ElementTree as ET 5 | from datetime import timedelta 6 | from pathlib import Path 7 | from unittest import TestCase 8 | 9 | import pandas as pd 10 | 11 | from rtctools_interface.closed_loop.config import ClosedLoopConfig 12 | from rtctools_interface.closed_loop.runner import run_optimization_problem_closed_loop 13 | 14 | from .test_models.goal_programming_csv.src.example import Example as ExampleCsv 15 | from .test_models.goal_programming_xml.src.example import Example as ExampleXml 16 | 17 | ns = {"fews": "http://www.wldelft.nl/fews", "pi": "http://www.wldelft.nl/fews/PI"} 18 | 19 | # Elementwise comparisons are practially disabled. 20 | A_TOL = 0.1 21 | R_TOL = 0.1 22 | 23 | 24 | def compare_xml_file(file_result: Path, file_ref: Path): 25 | """Compare two timeseries_export files elementwise.""" 26 | tree_result = ET.parse(file_result) 27 | tree_ref = ET.parse(file_ref) 28 | series_result = tree_result.findall("pi:series", ns) 29 | series_ref = tree_ref.findall("pi:series", ns) 30 | assert len(series_result) == len(series_ref), "Different number of series found in exports." 31 | for serie_result, serie_ref in zip(series_result, series_ref, strict=False): 32 | for event_result, event_ref in zip( 33 | serie_result.findall("pi:event", ns), serie_ref.findall("pi:event", ns), strict=False 34 | ): 35 | value_result = float(event_result.attrib["value"]) 36 | value_ref = float(event_ref.attrib["value"]) 37 | assert math.isclose(value_result, value_ref, rel_tol=R_TOL, abs_tol=A_TOL), ( 38 | f"Difference found in event: {value_result} != {value_ref}" 39 | ) 40 | 41 | 42 | def compare_xml_files(output_modelling_period_folder: Path, reference_folder: Path): 43 | """Compare the timeseries_export.xml files in the output and reference folders.""" 44 | for folder in output_modelling_period_folder.iterdir(): 45 | if not folder.is_dir(): 46 | continue 47 | file_name = "timeseries_export.xml" 48 | file_result = folder / file_name 49 | file_ref = reference_folder / folder.name / file_name 50 | compare_xml_file(file_result, file_ref) 51 | 52 | 53 | class TestClosedLoop(TestCase): 54 | """ 55 | Class for testing closed loop runner. 56 | """ 57 | 58 | def compare_csv_files(self, output_folder: Path, reference_folder: Path, n_periods: int): 59 | """Compare the csv files in output and reference subfolders.""" 60 | self.assertTrue(output_folder.exists(), "Output modelling period folder should be created.") 61 | self.assertEqual( 62 | len(list(output_folder.iterdir())), 63 | n_periods, 64 | f"Error: {n_periods} modelling periods should be created.", 65 | ) 66 | for folder in output_folder.iterdir(): 67 | self.assertTrue((folder / "timeseries_export.csv").exists()) 68 | for folder in output_folder.iterdir(): 69 | reference_folder_i = reference_folder / folder.name 70 | for file in folder.iterdir(): 71 | df_result = pd.read_csv(file) 72 | df_ref = pd.read_csv(reference_folder_i / file.name) 73 | pd.testing.assert_frame_equal(df_result, df_ref, atol=A_TOL, rtol=R_TOL) 74 | 75 | def compare_csv_file(self, output_file: Path, reference_file: Path): 76 | """Compare the main csv output file in output and reference folder.""" 77 | df_result = pd.read_csv(output_file) 78 | df_ref = pd.read_csv(reference_file) 79 | pd.testing.assert_frame_equal(df_result, df_ref, atol=A_TOL, rtol=R_TOL) 80 | 81 | def compare_xml_files(self, output_folder: Path, reference_folder: Path, n_periods: int): 82 | """Compare the xml files in output and reference subfolders.""" 83 | self.assertTrue(output_folder.exists(), "Output modelling period folder should be created.") 84 | self.assertEqual( 85 | len([f for f in output_folder.iterdir() if f.is_dir()]), 86 | n_periods, 87 | f"Error: {n_periods} modelling periods should be created.", 88 | ) 89 | for folder in output_folder.iterdir(): 90 | if folder.is_dir(): 91 | self.assertTrue((folder / "timeseries_export.xml").exists()) 92 | compare_xml_files(output_folder, reference_folder) 93 | 94 | def compare_xml_file(self, output_file: Path, reference_file: Path): 95 | """Compare the main xml output file in output and reference folder.""" 96 | compare_xml_file(output_file, reference_file) 97 | 98 | def test_running_closed_loop_csv(self): 99 | """ 100 | Check if test model runs without problems and generates same results. 101 | """ 102 | base_folder = Path(__file__).parent / "test_models" / "goal_programming_csv" 103 | config = ClosedLoopConfig( 104 | file=base_folder / "input" / "closed_loop_dates.csv", round_to_dates=True 105 | ) 106 | run_optimization_problem_closed_loop(ExampleCsv, base_folder=base_folder, config=config) 107 | self.compare_csv_files( 108 | output_folder=base_folder / "output" / "output_modelling_periods", 109 | reference_folder=base_folder / "output" / "output_modelling_periods_reference", 110 | n_periods=3, 111 | ) 112 | self.compare_csv_file( 113 | output_file=base_folder / "output" / "timeseries_export.csv", 114 | reference_file=base_folder / "output" / "timeseries_export_reference.csv", 115 | ) 116 | 117 | def test_running_closed_loop_csv_fixed_periods(self): 118 | """ 119 | Check if test model runs for fixed optimization periods. 120 | """ 121 | base_folder = Path(__file__).parent / "test_models" / "goal_programming_csv" 122 | output_folder = "output_fixed_periods" 123 | config = ClosedLoopConfig.from_fixed_periods( 124 | optimization_period=timedelta(days=3), forecast_timestep=timedelta(days=2) 125 | ) 126 | run_optimization_problem_closed_loop( 127 | ExampleCsv, base_folder=base_folder, config=config, output_folder=output_folder 128 | ) 129 | self.compare_csv_file( 130 | output_file=base_folder / output_folder / "timeseries_export.csv", 131 | reference_file=base_folder / "output" / "timeseries_export_reference.csv", 132 | ) 133 | 134 | def test_running_closed_loop_xml(self): 135 | """ 136 | Check if test model runs without problems and generates same results. 137 | """ 138 | test_cases = [ 139 | { 140 | "description": "without forecast date", 141 | "input_folder": "input", 142 | }, 143 | { 144 | "description": "with forecast date unequal to first date", 145 | "input_folder": "input_with_forecast_date", 146 | }, 147 | { 148 | "description": "with forecast date equal to first date", 149 | "input_folder": "input_with_forecast_date_equal_first_date", 150 | }, 151 | ] 152 | 153 | base_folder = Path(__file__).parent / "test_models" / "goal_programming_xml" 154 | 155 | for case in test_cases: 156 | with self.subTest(case["description"]): 157 | config = ClosedLoopConfig( 158 | file=base_folder / case["input_folder"] / "closed_loop_dates.csv", 159 | round_to_dates=True, 160 | ) 161 | run_optimization_problem_closed_loop( 162 | ExampleXml, 163 | base_folder=base_folder, 164 | config=config, 165 | input_folder=case["input_folder"], 166 | ) 167 | 168 | self.compare_xml_files( 169 | output_folder=base_folder / "output" / "output_modelling_periods", 170 | reference_folder=base_folder / "output" / "output_modelling_periods_reference", 171 | n_periods=3, 172 | ) 173 | self.compare_xml_file( 174 | output_file=base_folder / "output" / "timeseries_export.xml", 175 | reference_file=base_folder / "output" / "timeseries_export_reference.xml", 176 | ) 177 | 178 | def test_running_closed_loop_xml_fixed_periods(self): 179 | """ 180 | Check if test model runs for fixed optimization periods. 181 | """ 182 | test_cases = [ 183 | { 184 | "description": "without forecast date", 185 | "output_folder": "output_fixed_periods", 186 | "input_folder": "input", 187 | "optimization_period": timedelta(days=3), 188 | "forecast_timestep": timedelta(days=2), 189 | }, 190 | { 191 | "description": "with forecast date unequal to first date", 192 | "output_folder": "output_fixed_periods", 193 | "input_folder": "input_with_forecast_date", 194 | "optimization_period": timedelta(days=3), 195 | "forecast_timestep": timedelta(days=2), 196 | }, 197 | { 198 | "description": "with forecast date equal to first date", 199 | "output_folder": "output_fixed_periods", 200 | "input_folder": "input_with_forecast_date_equal_first_date", 201 | "optimization_period": timedelta(days=3), 202 | "forecast_timestep": timedelta(days=2), 203 | }, 204 | ] 205 | 206 | base_folder = Path(__file__).parent / "test_models" / "goal_programming_xml" 207 | 208 | for case in test_cases: 209 | with self.subTest(case["description"]): 210 | config = ClosedLoopConfig.from_fixed_periods( 211 | optimization_period=case["optimization_period"], 212 | forecast_timestep=case["forecast_timestep"], 213 | ) 214 | run_optimization_problem_closed_loop( 215 | ExampleXml, 216 | base_folder=base_folder, 217 | config=config, 218 | output_folder=case["output_folder"], 219 | input_folder=case["input_folder"], 220 | ) 221 | self.compare_xml_file( 222 | output_file=base_folder / case["output_folder"] / "timeseries_export.xml", 223 | reference_file=base_folder / "output" / "timeseries_export_reference.xml", 224 | ) 225 | -------------------------------------------------------------------------------- /rtctools_interface/optimization/base_goal.py: -------------------------------------------------------------------------------- 1 | """Module for a basic Goal.""" 2 | 3 | import logging 4 | 5 | import numpy as np 6 | from rtctools.optimization.goal_programming_mixin import Goal 7 | from rtctools.optimization.optimization_problem import OptimizationProblem 8 | from rtctools.optimization.timeseries import Timeseries 9 | 10 | from rtctools_interface.optimization.goal_table_schema import GOAL_TYPES, TARGET_DATA_TYPES 11 | from rtctools_interface.utils.type_definitions import GoalConfig 12 | 13 | logger = logging.getLogger("rtctools") 14 | 15 | 16 | class BaseGoal(Goal): 17 | """ 18 | Basic optimization goal for a given state. 19 | 20 | :cvar goal_type: 21 | Type of goal ('range' or 'minimization_path' or 'maximization_path') 22 | :cvar target_data_type: 23 | Type of target data ('value', 'parameter', 'timeseries'). 24 | If 'value', set the target bounds by value. 25 | If 'parameter', set the bounds by a parameter. The target_min 26 | and/or target_max are expected to be the name of the parameter. 27 | If 'timeseries', set the bounds by a timeseries. The target_min 28 | and/or target_max are expected to be the name of the timeseries. 29 | """ 30 | 31 | def __init__( 32 | self, 33 | optimization_problem: OptimizationProblem, 34 | state, 35 | *, 36 | goal_type="minimization_path", 37 | function_min=np.nan, 38 | function_max=np.nan, 39 | function_nominal=np.nan, 40 | target_data_type="value", 41 | target_min=np.nan, 42 | target_max=np.nan, 43 | priority=1, 44 | weight=1.0, 45 | order=2, 46 | goal_id=None, 47 | **_kwargs, 48 | ): 49 | self.goal_id = goal_id 50 | self.state = state 51 | self.target_data_type = target_data_type 52 | self._set_goal_type(goal_type) 53 | if goal_type in ["range", "range_rate_of_change"]: 54 | self._set_function_bounds( 55 | optimization_problem=optimization_problem, 56 | function_min=function_min, 57 | function_max=function_max, 58 | ) 59 | self._set_function_nominal(function_nominal) 60 | if goal_type in ["range", "range_rate_of_change"]: 61 | self._set_target_bounds( 62 | optimization_problem=optimization_problem, 63 | target_min=target_min, 64 | target_max=target_max, 65 | ) 66 | self.priority = priority if np.isfinite(priority) else 1 67 | self.weight = weight if np.isfinite(weight) else 1.0 68 | self._set_order(order) 69 | 70 | def function(self, optimization_problem, ensemble_member): 71 | del ensemble_member 72 | if self.goal_type == "maximization_path": 73 | return -optimization_problem.state(self.state) 74 | if self.goal_type in ["minimization_path", "range"]: 75 | return optimization_problem.state(self.state) 76 | if self.goal_type in ["range_rate_of_change"]: 77 | return optimization_problem.der(self.state) 78 | raise ValueError( 79 | f"Unsupported goal type '{self.goal_type}', supported are {GOAL_TYPES.keys()}" 80 | ) 81 | 82 | def _set_order(self, order): 83 | """Set the order of the goal.""" 84 | if np.isfinite(order): 85 | self.order = order 86 | elif self.goal_type in ["maximization_path", "minimization_path"]: 87 | self.order = 1 88 | else: 89 | self.order = 2 90 | if self.goal_type == "maximization_path" and self.order % 2 == 0: 91 | logger.warning( 92 | "Using even order '%i' for a maximization_path goal" 93 | + " results in a minimization_path goal.", 94 | self.order, 95 | ) 96 | 97 | def _set_goal_type( 98 | self, 99 | goal_type, 100 | ): 101 | """Set the goal type.""" 102 | if goal_type in GOAL_TYPES: 103 | self.goal_type = goal_type 104 | else: 105 | raise ValueError(f"goal_type should be one of {GOAL_TYPES.keys()}.") 106 | 107 | def _get_state_range(self, optimization_problem, state_name): 108 | if isinstance(optimization_problem.bounds()[state_name][0], float): 109 | state_range_0 = optimization_problem.bounds()[state_name][0] 110 | elif isinstance(optimization_problem.bounds()[state_name][0], Timeseries): 111 | state_range_0 = optimization_problem.bounds()[state_name][0].values 112 | else: 113 | state_range_0 = np.nan 114 | if isinstance(optimization_problem.bounds()[state_name][1], float): 115 | state_range_1 = optimization_problem.bounds()[state_name][1] 116 | elif isinstance(optimization_problem.bounds()[state_name][1], Timeseries): 117 | state_range_1 = optimization_problem.bounds()[state_name][1].values 118 | else: 119 | state_range_1 = np.nan 120 | return (state_range_0, state_range_1) 121 | 122 | def _set_function_bounds( 123 | self, 124 | optimization_problem: OptimizationProblem, 125 | function_min=np.nan, 126 | function_max=np.nan, 127 | ): 128 | """Set function bounds either by user specified value or calculated""" 129 | state_range = self._get_state_range(optimization_problem, self.state) 130 | if (~np.isfinite(function_min) & ~np.isfinite(state_range[0])).any() or ( 131 | ~np.isfinite(function_max) & ~np.isfinite(state_range[1]) 132 | ).any(): 133 | raise ValueError( 134 | f"The upper/lower bound for state {self.state} for " 135 | + f"goal with id={self.goal_id} is not specified" 136 | + " so the function range should be specified!" 137 | ) 138 | if self.goal_type in ["range_rate_of_change"]: 139 | maximum_scaled_difference = (state_range[1] - state_range[0]) / np.diff( 140 | optimization_problem.times() 141 | ).min() 142 | calculated_range = (-maximum_scaled_difference, maximum_scaled_difference) 143 | else: 144 | calculated_range = state_range 145 | 146 | function_range_0 = function_min if np.isfinite(function_min) else calculated_range[0] 147 | function_range_1 = function_max if np.isfinite(function_max) else calculated_range[1] 148 | self.function_range = (function_range_0, function_range_1) 149 | 150 | def _set_function_nominal(self, function_nominal): 151 | """Set function nominal""" 152 | self.function_nominal = function_nominal 153 | if not np.isfinite(self.function_nominal): 154 | if isinstance(self.function_range, (list, tuple)): 155 | if np.all(np.isfinite(self.function_range)): 156 | self.function_nominal = ( 157 | abs(self.function_range[0]) + abs(self.function_range[1]) 158 | ) / 2 159 | return 160 | self.function_nominal = 1.0 161 | logger.warning( 162 | "Function nominal for goal with id '%s' not specified, nominal is set to 1.0", 163 | self.goal_id, 164 | ) 165 | 166 | def _set_target_bounds( 167 | self, 168 | optimization_problem: OptimizationProblem, 169 | target_min=np.nan, 170 | target_max=np.nan, 171 | ): 172 | """Set the target bounds.""" 173 | 174 | def set_value_target(): 175 | if self.goal_type == "range_rate_of_change": 176 | self.target_min = float(target_min) / 100 * self.function_nominal 177 | self.target_max = float(target_max) / 100 * self.function_nominal 178 | else: 179 | self.target_min = float(target_min) 180 | self.target_max = float(target_max) 181 | 182 | def set_parameter_target(): 183 | if isinstance(target_max, str): 184 | self.target_max = optimization_problem.parameters(0)[target_max] 185 | if self.target_max is None: 186 | self.target_max = optimization_problem.io.get_parameter(target_max) 187 | elif np.isnan(target_max): 188 | self.target_max = np.nan 189 | if isinstance(target_min, str): 190 | self.target_min = optimization_problem.parameters(0)[target_min] 191 | if self.target_min is None: 192 | self.target_min = optimization_problem.io.get_parameter(target_min) 193 | elif np.isnan(target_min): 194 | self.target_min = np.nan 195 | 196 | def set_timeseries_target(): 197 | if isinstance(target_max, str): 198 | self.target_max = optimization_problem.get_timeseries(target_max) 199 | elif np.isnan(target_max): 200 | self.target_max = np.nan 201 | if isinstance(target_min, str): 202 | self.target_min = optimization_problem.get_timeseries(target_min) 203 | elif np.isnan(target_min): 204 | self.target_min = np.nan 205 | 206 | if self.target_data_type not in TARGET_DATA_TYPES: 207 | raise ValueError(f"target_data_type should be one of {TARGET_DATA_TYPES}.") 208 | 209 | if self.goal_type == "range_rate_of_change" and self.target_data_type != "value": 210 | raise ValueError( 211 | "For range_rate_of_change goal only the `value` target type is supported." 212 | ) 213 | 214 | if self.target_data_type == "value": 215 | set_value_target() 216 | elif self.target_data_type == "parameter": 217 | set_parameter_target() 218 | elif self.target_data_type == "timeseries": 219 | set_timeseries_target() 220 | 221 | self._target_dict = optimization_problem.collect_range_target_values_from_basegoal(self) 222 | 223 | def get_goal_config(self) -> GoalConfig: 224 | """ 225 | Serialize the goal configuration into a dictionary. 226 | """ 227 | goal_config: GoalConfig = { 228 | "goal_id": self.goal_id, 229 | "state": self.state, 230 | "goal_type": self.goal_type, 231 | "function_min": self.function_range[0] 232 | if np.any(np.isfinite(self.function_range[0])) 233 | else None, 234 | "function_max": self.function_range[1] 235 | if np.any(np.isfinite(self.function_range[1])) 236 | else None, 237 | "function_nominal": self.function_nominal 238 | if np.any(np.isfinite(self.function_nominal)) 239 | else None, 240 | "target_min_series": None, 241 | "target_max_series": None, 242 | "target_min": self.target_min, 243 | "target_max": self.target_max, 244 | "priority": self.priority, 245 | "weight": self.weight, 246 | "order": self.order, 247 | } 248 | if isinstance(self.target_min, Timeseries): 249 | goal_config["target_min"] = self.target_min.values 250 | if isinstance(self.target_max, Timeseries): 251 | goal_config["target_max"] = self.target_max.values 252 | 253 | if hasattr(self, "_target_dict"): 254 | goal_config["target_min_series"] = self._target_dict["target_min"] 255 | goal_config["target_max_series"] = self._target_dict["target_max"] 256 | return goal_config 257 | --------------------------------------------------------------------------------