├── Dockerfile
├── experiments
    ├── dataloader.py
    ├── visualization.py
    ├── rnn.py
    ├── hidden_physics.py
    ├── direct_solution.py
    ├── time_stepper.py
    └── latent_neural_odes.py
├── .gitignore
├── requirements.txt
├── README.md
└── docs
    ├── survey_structure.svg
    └── time_vs_state_plot.svg


/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM python:3.8-slim-buster
 2 | 
 3 | WORKDIR /app
 4 | 
 5 | COPY requirements.txt requirements.txt
 6 | 
 7 | RUN apt-get update && apt-get install build-essential -y
 8 | RUN pip3 install --upgrade pip
 9 | RUN pip3 install --upgrade pip setuptools wheel
10 | RUN pip3 install -r requirements.txt
11 | 
12 | COPY . .


--------------------------------------------------------------------------------
/experiments/dataloader.py:
--------------------------------------------------------------------------------
 1 | from typing import Tuple
 2 | import numpy as np
 3 | import torch
 4 | 
 5 | from enum import Enum
 6 | from smt.sampling_methods import LHS
 7 | from torchdyn.numerics import odeint
 8 | 
 9 | class Sampling(Enum):
10 |     RANDOM = 0
11 |     GRID = 1
12 | 
13 | 
14 | def grid_init_samples(domain, n_trajectories: int) -> np.ndarray:
15 |     x = np.linspace(domain[0][0], domain[0][1], n_trajectories)
16 |     y = np.linspace(domain[1][0], domain[1][1], n_trajectories)
17 | 
18 |     xx, yy = np.meshgrid(x, y)
19 |     return np.concatenate((xx.flatten()[..., np.newaxis], yy.flatten()[..., np.newaxis]), axis=1)
20 | 
21 | 
22 | def random_init_samples(domain, n_trajectories: int) -> np.ndarray:
23 |     values = LHS(xlimits=np.array(domain))
24 |     return values(n_trajectories)
25 | 
26 | 
27 | def pendulum(t, y):
28 |     θ = y[:, 0]
29 |     ω = y[:, 1]
30 | 
31 |     dθ = ω
32 |     dω = -torch.sin(θ)
33 | 
34 |     return torch.stack((dθ, dω), dim=1)
35 | 
36 | 
37 | def load_pendulum_data(t_span, y0s_domain=None, n_trajectories=100, sampling=Sampling.RANDOM, solver='rk4') -> Tuple[torch.Tensor, torch.Tensor]:
38 |     if not y0s_domain:
39 |         y0s_domain = [[-1., 1.], [-1., 1.]]
40 | 
41 |     if sampling == Sampling.RANDOM:
42 |         y0s = random_init_samples(y0s_domain, n_trajectories)
43 |     elif sampling == Sampling.GRID:
44 |         y0s = grid_init_samples(y0s_domain, n_trajectories)
45 | 
46 |     y0s = torch.tensor(y0s)
47 |     _, ys = odeint(pendulum, y0s, t_span, solver) 
48 | 
49 |     return y0s, ys
50 | 


--------------------------------------------------------------------------------
/experiments/visualization.py:
--------------------------------------------------------------------------------
 1 | from matplotlib import cm, pyplot as plt
 2 | from matplotlib.collections import LineCollection
 3 | import torch
 4 | import numpy as np
 5 | 
 6 | 
 7 | def get_meshgrid(step_per_axis, domain=1.0, flatten=False, dtype=torch.float64):
 8 |     ε = 1e-10
 9 |     x = torch.arange(-domain, domain + ε, step_per_axis, dtype=dtype)
10 |     y = x = torch.arange(-domain, domain + ε, step_per_axis, dtype=dtype)
11 | 
12 |     xy = torch.stack(torch.meshgrid((x, y)))
13 |     if flatten:
14 |         return xy.flatten(1).T
15 |     else:
16 |         return xy.T
17 | 
18 | 
19 | def plot_colored(fig, ax, t, x, cmap="jet", label=None, colorbar=False, **kwargs):
20 |     """
21 |     t : (time x traj)
22 |     x : (time x traj x state)
23 |     """
24 |     # x = torch.tensor(x)
25 |     # t = torch.tensor(t)
26 |     x = torch.as_tensor(x)
27 |     t = torch.as_tensor(t)
28 | 
29 |     if t.ndim == 1:
30 |         t = t.unsqueeze(-1).expand(-1, x.shape[1])
31 | 
32 |     norm = plt.Normalize(t.min(), t.max())
33 | 
34 |     for i in range(t.shape[1]):
35 |         xi = x[:, i]
36 |         ti = t[:, i]
37 |         segments = torch.stack([xi[:-1], xi[1:]], axis=1)
38 | 
39 |         if i == t.shape[1] - 1:
40 |             lc = LineCollection(segments, cmap=cmap, norm=norm, label=label, **kwargs)
41 |         else:
42 |             lc = LineCollection(segments, cmap=cmap, norm=norm, **kwargs)
43 |         lc.set_array(ti)
44 |         ax.add_collection(lc)
45 | 
46 |     if colorbar:
47 |         fig.colorbar(cm.ScalarMappable(norm=norm, cmap=cmap), ax=ax)
48 |     ax.set_xlim(x.min(), x.max())
49 |     ax.set_ylim(x.min(), x.max())
50 | 


--------------------------------------------------------------------------------
/experiments/rnn.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | import torch
 3 | 
 4 | from matplotlib import pyplot as plt
 5 | 
 6 | from tqdm import tqdm
 7 | from torch import nn
 8 | from torch.nn.functional import mse_loss
 9 | from torchdyn.numerics import odeint
10 | 
11 | from dataloader import Sampling, load_pendulum_data
12 | 
13 | def run():
14 |     """
15 |     Training data
16 | 
17 |     """
18 |     y0s_domain = [[-1., 1.], [-1., 1.]]
19 |     n_steps = 1
20 |     step_size = 0.01  
21 |     t_span = torch.arange(0., step_size * (n_steps + 1), step_size)
22 | 
23 |     y0s, ys = load_pendulum_data(t_span, y0s_domain, n_trajectories=20, sampling=Sampling.RANDOM)
24 | 
25 |     """
26 |     Network
27 |     
28 |     """
29 |     class RNN(nn.Module):
30 | 
31 |         def __init__(self, n_states):
32 |             super().__init__()
33 |             self.rnn = nn.RNN(n_states, n_states, batch_first=True).double()
34 | 
35 |         def forward(self, y0s):
36 |             y0s = y0s.unsqueeze(dim=1)
37 |             out, hidden = self.rnn(y0s, torch.swapaxes(y0s, 0, 1))
38 |             return hidden.squeeze(dim=0)
39 | 
40 | 
41 |     model = RNN(n_states=2)
42 |     opt = torch.optim.Adam(model.parameters())
43 | 
44 |     """
45 |     Training
46 | 
47 |     """
48 |     epochs = 20
49 |     progress = tqdm(range(epochs), 'Training')
50 |     losses = []
51 | 
52 |     for _ in progress:
53 |         _, y_pred = odeint(lambda t, y: model(y), y0s, t_span, 'euler')
54 | 
55 |         loss = mse_loss(y_pred, ys)
56 |         loss.backward()
57 |         opt.step()
58 |         opt.zero_grad()
59 | 
60 |         losses.append(loss.item())
61 |         progress.set_description(f'loss: {loss.item()}')
62 | 
63 |     """
64 |     Test data
65 | 
66 |     """
67 |     y0s_domain = [[-1., 1.], [-1., 1.]]
68 |     n_steps = 20
69 |     step_size = 0.01  
70 |     t_span = torch.arange(0., step_size * (n_steps + 1), step_size)
71 | 
72 |     y0s, ys = load_pendulum_data(t_span, y0s_domain, n_trajectories=10, sampling=Sampling.GRID)
73 |     _, y_pred = odeint(lambda t, y: model(y), y0s, t_span, 'euler')
74 | 
75 |     """
76 |     Plot
77 | 
78 |     """
79 |     plt.plot(y_pred.detach().numpy()[:, :, 1], y_pred.detach().numpy()[:, :, 0], color='r')
80 |     plt.plot(ys.numpy()[:, :, 1], ys.numpy()[:, :, 0], color='b')
81 |     plt.scatter(ys[0, :, 1], ys[0, :, 0], color='g')
82 |     plt.ylim(y0s_domain[0])
83 |     plt.xlim(y0s_domain[1])
84 |     plt.show()
85 | 
86 | 
87 | 
88 | if __name__ == "__main__":
89 |     run()
90 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | pip-wheel-metadata/
 24 | share/python-wheels/
 25 | *.egg-info/
 26 | .installed.cfg
 27 | *.egg
 28 | MANIFEST
 29 | 
 30 | # PyInstaller
 31 | #  Usually these files are written by a python script from a template
 32 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 33 | *.manifest
 34 | *.spec
 35 | 
 36 | # Installer logs
 37 | pip-log.txt
 38 | pip-delete-this-directory.txt
 39 | 
 40 | # Unit test / coverage reports
 41 | htmlcov/
 42 | .tox/
 43 | .nox/
 44 | .coverage
 45 | .coverage.*
 46 | .cache
 47 | nosetests.xml
 48 | coverage.xml
 49 | *.cover
 50 | *.py,cover
 51 | .hypothesis/
 52 | .pytest_cache/
 53 | 
 54 | # Translations
 55 | *.mo
 56 | *.pot
 57 | 
 58 | # Django stuff:
 59 | *.log
 60 | local_settings.py
 61 | db.sqlite3
 62 | db.sqlite3-journal
 63 | 
 64 | # Flask stuff:
 65 | instance/
 66 | .webassets-cache
 67 | 
 68 | # Scrapy stuff:
 69 | .scrapy
 70 | 
 71 | # Sphinx documentation
 72 | docs/_build/
 73 | 
 74 | # PyBuilder
 75 | target/
 76 | 
 77 | # Jupyter Notebook
 78 | .ipynb_checkpoints
 79 | 
 80 | # IPython
 81 | profile_default/
 82 | ipython_config.py
 83 | 
 84 | # pyenv
 85 | .python-version
 86 | 
 87 | # pipenv
 88 | #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 89 | #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 90 | #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 91 | #   install all needed dependencies.
 92 | #Pipfile.lock
 93 | 
 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 95 | __pypackages__/
 96 | 
 97 | # Celery stuff
 98 | celerybeat-schedule
 99 | celerybeat.pid
100 | 
101 | # SageMath parsed files
102 | *.sage.py
103 | 
104 | # Environments
105 | .env
106 | .venv
107 | env/
108 | venv/
109 | ENV/
110 | env.bak/
111 | venv.bak/
112 | 
113 | # Spyder project settings
114 | .spyderproject
115 | .spyproject
116 | 
117 | # Rope project settings
118 | .ropeproject
119 | 
120 | # mkdocs documentation
121 | /site
122 | 
123 | # mypy
124 | .mypy_cache/
125 | .dmypy.json
126 | dmypy.json
127 | 
128 | # Pyre type checker
129 | .pyre/
130 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
  1 | absl-py==0.13.0
  2 | aiohttp==3.7.4.post0
  3 | argon2-cffi==20.1.0
  4 | async-timeout==3.0.1
  5 | attrs==21.2.0
  6 | backcall==0.2.0
  7 | bleach==4.0.0
  8 | boltons==21.0.0
  9 | cachetools==4.2.2
 10 | certifi==2021.5.30
 11 | cffi==1.14.6
 12 | chardet==4.0.0
 13 | charset-normalizer==2.0.4
 14 | cycler==0.10.0
 15 | debugpy==1.4.1
 16 | decorator==5.0.9
 17 | defusedxml==0.7.1
 18 | einops==0.3.0
 19 | entrypoints==0.3
 20 | fsspec==2021.7.0
 21 | future==0.18.2
 22 | google-auth==1.35.0
 23 | google-auth-oauthlib==0.4.5
 24 | grpcio==1.39.0
 25 | idna==3.2
 26 | ipykernel==6.2.0
 27 | ipython==7.26.0
 28 | ipython-genutils==0.2.0
 29 | ipywidgets==7.6.3
 30 | jedi==0.18.0
 31 | Jinja2==3.0.1
 32 | joblib==1.0.1
 33 | jsonschema==3.2.0
 34 | jupyter-client==6.1.12
 35 | jupyter-core==4.7.1
 36 | jupyterlab-pygments==0.1.2
 37 | jupyterlab-widgets==1.0.0
 38 | kiwisolver==1.3.1
 39 | Markdown==3.3.4
 40 | MarkupSafe==2.0.1
 41 | matplotlib==3.4.3
 42 | matplotlib-inline==0.1.2
 43 | mistune==0.8.4
 44 | multidict==5.1.0
 45 | nbclient==0.5.4
 46 | nbconvert==6.1.0
 47 | nbformat==5.1.3
 48 | nest-asyncio==1.5.1
 49 | notebook==6.4.3
 50 | numpy==1.21.2
 51 | oauthlib==3.1.1
 52 | packaging==21.0
 53 | pandocfilters==1.4.3
 54 | parso==0.8.2
 55 | pastel==0.2.1
 56 | pexpect==4.8.0
 57 | pickleshare==0.7.5
 58 | Pillow==8.3.1
 59 | poethepoet==0.10.0
 60 | prometheus-client==0.11.0
 61 | prompt-toolkit==3.0.19
 62 | protobuf==3.17.3
 63 | ptyprocess==0.7.0
 64 | pyasn1==0.4.8
 65 | pyasn1-modules==0.2.8
 66 | pycparser==2.20
 67 | pyDeprecate==0.3.1
 68 | pyDOE==0.3.8
 69 | Pygments==2.10.0
 70 | pyparsing==2.4.7
 71 | pyrsistent==0.18.0
 72 | python-dateutil==2.8.2
 73 | pytorch-lightning==1.4.2
 74 | PyYAML==5.4.1
 75 | pyzmq==22.2.1
 76 | requests==2.26.0
 77 | requests-oauthlib==1.3.0
 78 | rsa==4.7.2
 79 | scikit-learn==0.24.2
 80 | scipy==1.7.1
 81 | Send2Trash==1.8.0
 82 | six==1.16.0
 83 | sklearn==0.0
 84 | tensorboard==2.6.0
 85 | tensorboard-data-server==0.6.1
 86 | tensorboard-plugin-wit==1.8.0
 87 | terminado==0.11.0
 88 | testpath==0.5.0
 89 | threadpoolctl==2.2.0
 90 | tomlkit==0.7.2
 91 | torch==1.9.0
 92 | torchaudio==0.9.0
 93 | torchcde==0.2.3
 94 | torchdiffeq==0.2.2
 95 | torchdyn==1.0.1
 96 | torchmetrics==0.5.0
 97 | torchsde==0.2.5
 98 | torchvision==0.10.0
 99 | tornado==6.1
100 | tqdm==4.62.1
101 | traitlets==5.0.5
102 | trampoline==0.1.2
103 | typing-extensions==3.10.0.0
104 | urllib3==1.26.6
105 | wcwidth==0.2.5
106 | webencodings==0.5.1
107 | Werkzeug==2.0.1
108 | widgetsnbextension==3.5.1
109 | yarl==1.6.3
110 | absl-py==0.13.0
111 | aiohttp==3.7.4.post0
112 | alabaster==0.7.12
113 | appdirs==1.4.4
114 | argon2-cffi==20.1.0
115 | async-timeout==3.0.1
116 | attrs==21.2.0
117 | Babel==2.9.1
118 | backcall==0.2.0
119 | black==21.7b0
120 | bleach==4.0.0
121 | boltons==21.0.0
122 | cachetools==4.2.2
123 | certifi==2021.5.30
124 | cffi==1.14.6
125 | chardet==4.0.0
126 | charset-normalizer==2.0.4
127 | click==8.0.1
128 | cycler==0.10.0
129 | debugpy==1.4.1
130 | decorator==5.0.9
131 | defusedxml==0.7.1
132 | docutils==0.17.1
133 | einops==0.3.0
134 | entrypoints==0.3
135 | fsspec==2021.7.0
136 | future==0.18.2
137 | google-auth==1.35.0
138 | google-auth-oauthlib==0.4.5
139 | grpcio==1.39.0
140 | idna==3.2
141 | imagesize==1.2.0
142 | ipykernel==6.2.0
143 | ipython==7.26.0
144 | ipython-genutils==0.2.0
145 | ipywidgets==7.6.3
146 | jedi==0.18.0
147 | Jinja2==3.0.1
148 | joblib==1.0.1
149 | jsonschema==3.2.0
150 | jupyter-client==6.1.12
151 | jupyter-core==4.7.1
152 | jupyterlab-pygments==0.1.2
153 | jupyterlab-widgets==1.0.0
154 | kiwisolver==1.3.1
155 | Markdown==3.3.4
156 | MarkupSafe==2.0.1
157 | matplotlib==3.4.3
158 | matplotlib-inline==0.1.2
159 | mistune==0.8.4
160 | multidict==5.1.0
161 | mypy-extensions==0.4.3
162 | nbclient==0.5.4
163 | nbconvert==6.1.0
164 | nbformat==5.1.3
165 | nest-asyncio==1.5.1
166 | notebook==6.4.3
167 | numpy==1.21.2
168 | numpydoc==1.1.0
169 | oauthlib==3.1.1
170 | packaging==21.0
171 | pandocfilters==1.4.3
172 | parso==0.8.2
173 | pastel==0.2.1
174 | pathspec==0.9.0
175 | pexpect==4.8.0
176 | pickleshare==0.7.5
177 | Pillow==8.3.1
178 | poethepoet==0.10.0
179 | prometheus-client==0.11.0
180 | prompt-toolkit==3.0.19
181 | protobuf==3.17.3
182 | ptyprocess==0.7.0
183 | pyasn1==0.4.8
184 | pyasn1-modules==0.2.8
185 | pycparser==2.20
186 | pyDeprecate==0.3.1
187 | pyDOE==0.3.8
188 | pyDOE2==1.3.0
189 | Pygments==2.10.0
190 | pyparsing==2.4.7
191 | pyrsistent==0.18.0
192 | python-dateutil==2.8.2
193 | pytorch-lightning==1.4.2
194 | pytz==2021.1
195 | PyYAML==5.4.1
196 | pyzmq==22.2.1
197 | regex==2021.8.3
198 | requests==2.26.0
199 | requests-oauthlib==1.3.0
200 | rsa==4.7.2
201 | scikit-learn==0.24.2
202 | scipy==1.7.1
203 | Send2Trash==1.8.0
204 | six==1.16.0
205 | sklearn==0.0
206 | smt==1.0.0
207 | snowballstemmer==2.1.0
208 | Sphinx==4.1.2
209 | sphinxcontrib-applehelp==1.0.2
210 | sphinxcontrib-devhelp==1.0.2
211 | sphinxcontrib-htmlhelp==2.0.0
212 | sphinxcontrib-jsmath==1.0.1
213 | sphinxcontrib-qthelp==1.0.3
214 | sphinxcontrib-serializinghtml==1.1.5
215 | tensorboard==2.6.0
216 | tensorboard-data-server==0.6.1
217 | tensorboard-plugin-wit==1.8.0
218 | terminado==0.11.0
219 | testpath==0.5.0
220 | threadpoolctl==2.2.0
221 | tomli==1.2.1
222 | tomlkit==0.7.2
223 | torch==1.9.0
224 | torchaudio==0.9.0
225 | torchcde==0.2.3
226 | torchdiffeq==0.2.2
227 | torchdyn==1.0.1
228 | torchmetrics==0.5.0
229 | torchsde==0.2.5
230 | torchvision==0.10.0
231 | tornado==6.1
232 | tqdm==4.62.1
233 | traitlets==5.0.5
234 | trampoline==0.1.2
235 | typing-extensions==3.10.0.0
236 | urllib3==1.26.6
237 | wcwidth==0.2.5
238 | webencodings==0.5.1
239 | Werkzeug==2.0.1
240 | widgetsnbextension==3.5.1
241 | yarl==1.6.3
242 | 


--------------------------------------------------------------------------------
/experiments/hidden_physics.py:
--------------------------------------------------------------------------------
  1 | from argparse import ArgumentParser
  2 | from collections import defaultdict
  3 | 
  4 | import matplotlib.pyplot as plt
  5 | import numpy as np
  6 | import torch
  7 | import torch.nn as nn
  8 | import torch.nn.functional as F
  9 | from scipy.integrate import solve_ivp
 10 | from torch.autograd import grad
 11 | from tqdm import tqdm
 12 | 
 13 | 
 14 | def listify(A):
 15 |     return [a for a in A.flatten()]
 16 | 
 17 | 
 18 | def xavier_init(module):
 19 |     for m in module.modules():
 20 |         if type(m) == nn.Linear:
 21 |             nn.init.xavier_uniform_(m.weight)
 22 | 
 23 | 
 24 | def plot_loss(losses):
 25 |     fig, ax = plt.subplots()
 26 |     fig.canvas.manager.set_window_title(f"loss terms")
 27 | 
 28 |     for loss_name, loss in losses.items():
 29 |         ax.plot(loss, label=loss_name)
 30 | 
 31 |     ax.legend()
 32 |     ax.set_xlabel("epoch")
 33 | 
 34 | 
 35 | def l_fun(t):
 36 |     return np.sin(t) + 2
 37 | 
 38 | 
 39 | def f(t, y):
 40 |     θ, ω = y
 41 |     g = 1.0
 42 |     l = l_fun(t)
 43 | 
 44 |     dω = -(g / l) * np.sin(θ)
 45 |     dθ = ω
 46 | 
 47 |     return dθ, dω
 48 | 
 49 | 
 50 | if __name__ == "__main__":
 51 | 
 52 |     parser = ArgumentParser()
 53 |     parser.add_argument("--device", default="cpu")
 54 |     parser.add_argument("--n_epochs", default=6000, type=int)
 55 |     parser.add_argument("--t_start", default=0.0, type=float)
 56 |     parser.add_argument("--t_end", type=float, default=np.pi * 4)
 57 |     args = parser.parse_args()
 58 | 
 59 |     device = args.device
 60 |     n_epochs = args.n_epochs
 61 | 
 62 |     """
 63 |     generate data
 64 | 
 65 |     """
 66 |     y0 = [np.pi / 4, 0]
 67 |     step_size = 0.01
 68 | 
 69 |     t_start = args.t_start
 70 |     t_end = args.t_end
 71 | 
 72 |     t_eval = np.arange(t_start, t_end, step_size)
 73 | 
 74 |     y_true = solve_ivp(
 75 |         f, t_span=(t_start, t_end), t_eval=t_eval, y0=y0, method="RK45"
 76 |     ).y
 77 |     θ, ω = y_true
 78 |     l = torch.tensor(l_fun(t_eval)).to(device)
 79 | 
 80 |     """
 81 |     network
 82 |     
 83 |     """
 84 |     nn_hidden = (
 85 |         nn.Sequential(
 86 |             nn.Linear(1, 32),
 87 |             nn.Softplus(),
 88 |             nn.Linear(32, 32),
 89 |             nn.Softplus(),
 90 |             nn.Linear(32, 32),
 91 |             nn.Softplus(),
 92 |             nn.Linear(32, 32),
 93 |             nn.Softplus(),
 94 |             nn.Linear(32, 2),
 95 |         )
 96 |         .double()
 97 |         .to(device)
 98 |     )
 99 | 
100 |     xavier_init(nn_hidden)
101 |     optimizer = torch.optim.Adam(nn_hidden.parameters())
102 | 
103 |     t_train = torch.tensor(t_eval, requires_grad=True).to(device)
104 |     y_train = torch.tensor(y_true).to(device)
105 | 
106 |     subsample_every = int(2.5 / step_size)
107 |     losses = defaultdict(list)
108 | 
109 |     """
110 |     training
111 |     
112 |     """
113 |     for _ in tqdm(range(n_epochs), "training hidden physics model"):
114 |         θ_pred, l_pred = nn_hidden(t_train[..., None]).T
115 | 
116 |         ω_pred = grad(
117 |             listify(θ_pred),
118 |             t_train,
119 |             only_inputs=True,
120 |             retain_graph=True,
121 |             create_graph=True,
122 |         )[0]
123 | 
124 |         dω_pred = grad(
125 |             listify(ω_pred),
126 |             t_train,
127 |             only_inputs=True,
128 |             retain_graph=True,
129 |             create_graph=True,
130 |         )[0]
131 | 
132 |         dω_eq = -(1.0 / l_pred) * torch.sin(θ_pred)
133 | 
134 |         y_pred = torch.column_stack((θ_pred, ω_pred)).T
135 | 
136 |         loss_collocation = F.mse_loss(
137 |             y_pred[:, ::subsample_every], y_train[:, ::subsample_every]
138 |         )
139 |         loss_hidden = F.mse_loss(dω_pred, dω_eq)
140 |         loss_length = F.mse_loss(l_pred, l)
141 | 
142 |         loss = loss_collocation + loss_hidden
143 | 
144 |         loss.backward()
145 |         optimizer.step()
146 |         optimizer.zero_grad()
147 | 
148 |         losses["collocation"].append(loss_collocation.item())
149 |         losses["hidden"].append(loss_hidden.item())
150 | 
151 |     plot_loss(losses)
152 | 
153 |     predicted = {
154 |         "θ(t)": θ_pred.detach().cpu().flatten(),
155 |         "ω(t)": ω_pred.detach().cpu().flatten(),
156 |         "l(t)": l_pred.detach().cpu().flatten(),
157 |     }
158 |     true = {
159 |         "θ(t)": θ,
160 |         "ω(t)": ω,
161 |         "l(t)": l.detach().cpu(),
162 |     }
163 | 
164 |     fig, (ax0, ax1, ax2) = plt.subplots(3, 1, sharex=True)
165 |     fig.canvas.manager.set_window_title(f"states")
166 | 
167 |     ax0.set_ylabel("θ(t)")
168 |     ax0.plot(t_eval, θ, c="black", label="true")
169 |     ax0.plot(
170 |         t_eval,
171 |         θ_pred.detach().cpu().flatten(),
172 |         c="b",
173 |         linestyle="--",
174 |         label="predicted",
175 |     )
176 |     ax0.scatter(
177 |         t_eval[::subsample_every],
178 |         θ[::subsample_every],
179 |         c="black",
180 |         linestyle="None",
181 |         label="collocation point",
182 |     )
183 | 
184 |     ax1.set_ylabel("ω(t)")
185 |     ax1.plot(t_eval, ω, c="black", label="true")
186 |     ax1.plot(
187 |         t_eval,
188 |         ω_pred.detach().cpu().flatten(),
189 |         c="r",
190 |         linestyle="--",
191 |         label="predicted",
192 |     )
193 |     ax1.scatter(
194 |         t_eval[::subsample_every],
195 |         ω[::subsample_every],
196 |         c="black",
197 |         linestyle="None",
198 |         label="collocation point",
199 |     )
200 | 
201 |     ax2.set_ylabel("l(t)")
202 |     ax2.set_xlabel("t")
203 |     ax2.plot(t_eval, l, c="black", label="true")
204 |     ax2.plot(
205 |         t_eval,
206 |         l_pred.detach().cpu().flatten(),
207 |         c="g",
208 |         linestyle="--",
209 |         label="predicted",
210 |     )
211 |     # skip drawing misleading collocation points, since none are used for the pendulum length
212 | 
213 |     ax1.legend()
214 |     plt.tight_layout()
215 | 
216 |     plt.show()
217 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Constructing Neural Network-Based Models for Simulating Dynamical Systems
 2 | 
 3 | This is a companion repo for the review paper **Constructing Neural Network-Based Models for Simulating Dynamical Systems** which provides a practical description on how models like *Neural Ordinary Differential Equations* and *Physics-informed Neural Networks* can be implemented.
 4 | The full paper can be accessed at: https://dl.acm.org/doi/10.1145/3567591
 5 | 
 6 | The code in the repo is implemented in Python using PyTorch for defining and training the models.
 7 | The scripts can be run using default parameters to reproduce the plots seen in the paper, as well as things like loss curves which were cut due to space requirements:
 8 | 
 9 | <img src="docs/stream_plot.svg" width=49%><img src="docs/time_vs_state_plot.svg" width=49%>
10 | 
11 | 
12 | <img src="docs/survey_structure.svg">
13 | 
14 | 
15 | # Installing dependencies
16 | 
17 | The dependencies necessary to run the scripts can be installed through pip using the `requirements.txt` file as follows:
18 | ``` bash
19 | python3 -m pip install -r requirements.txt
20 | ```
21 | In case you are using Conda you can run the following command from a fresh environment:
22 | ``` bash
23 | conda install --file requirements.txt
24 | ```
25 | 
26 | # Running the experiments
27 | 
28 | Each experiment can be run using default parameters by executing the script in the python interpreter as follows:
29 | ```
30 | python3 experiments/<name of experiment>.py ...
31 | ```
32 | The table below contains the commands necessary to train and evaluate the models described in the review paper.
33 | 
34 | | Name                                         | Section | Command                                                         |
35 | | -------------------------------------------- | ------- | --------------------------------------------------------------- |
36 | | Vanilla Direct-Solution                      | 3.2     | python3 experiments/direct_solution.py --model vanilla          |
37 | | Automatic Differentiation in Direct-Solution | 3.3     | python3 experiments/direct_solution.py --model autodiff         |
38 | | Physics Informed Neural Networks             | 3.4     | python3 experiments/direct_solution.py --model pinn             |
39 | | Hidden Physics Networks                      | 3.5     | python3 experiments/hidden_physics.py                           |
40 | | Direct Time-Stepper                          | 4.2.1   | python3 experiments/time_stepper.py --solver direct             |
41 | | Residual Time-Stepper                        | 4.2.2   | python3 experiments/time_stepper.py --solver resnet             |
42 | | Euler Time-Stepper                           | 4.2.3   | python3 experiments/time_stepper.py --solver euler              |
43 | | Neural ODEs Time-Stepper                     | 4.2.4   | python3 experiments/time_stepper.py --solver {rk4,dopri5,tsit5} |
44 | | Neural State-Space Model                     | 4.3.1   | ...                                                             |
45 | | Neural ODEs with input                       | 4.3.2-3 | ...                                                             |
46 | | Lagrangian Time-Stepper                      | 4.4.1   | ...                                                             |
47 | | Hamiltonian Time-Stepper                     | 4.4.1   | ...                                                             |
48 | | Deep Potential Time-Stepper                  | 4.4.2   | ...                                                             |
49 | | Deep Markov-Model                            | 4.5.1   | ...                                                             |
50 | | Latent Neural ODEs                           | 4.5.2   | python3 experiments/latent_neural_odes.py                       |
51 | | Bayesian Neural ODEs                         | 4.5.3   | ...                                                             |
52 | | Neural SDEs                                  | 4.5.4   | ...                                                             |
53 | 
54 | 
55 | 
56 | # Docker Image
57 | In an effort to ensure that the code can be executed in the future, we provide a docker image.
58 | The Docker image allows the code to be run in a Linux based virtual machine on any platform supported by Docker.
59 | 
60 | To use the docker image, invoke the build command in the root of this repository:
61 | ``` bash
62 | docker build . -t python_dynamical_systems
63 | ```
64 | 
65 | Following this "containers" containing the code and all dependencies can be instantiated via the "run" command:
66 | ``` bash
67 | docker run -ti python_dynamical_systems bash
68 | ```
69 | The command will establish an interactive connection to the container.
70 | Following this you can execute the code as if it was running on your host machine:
71 | ``` bash
72 | python3 experiments/time_stepper.py ...
73 | ```
74 | 
75 | # Citing the paper
76 | 
77 | If you use the work please consider citing it:
78 | ``` bibtex
79 | @article{10.1145/3567591,
80 | author = {Legaard, Christian and Schranz, Thomas and Schweiger, Gerald and Drgo\v{n}a, J\'{a}n and Falay, Basak and Gomes, Cl\'{a}udio and Iosifidis, Alexandros and Abkar, Mahdi and Larsen, Peter},
81 | title = {Constructing Neural Network Based Models for Simulating Dynamical Systems},
82 | year = {2023},
83 | issue_date = {November 2023},
84 | publisher = {Association for Computing Machinery},
85 | address = {New York, NY, USA},
86 | volume = {55},
87 | number = {11},
88 | issn = {0360-0300},
89 | url = {https://doi.org/10.1145/3567591},
90 | doi = {10.1145/3567591},
91 | abstract = {Dynamical systems see widespread use in natural sciences like physics, biology, and chemistry, as well as engineering disciplines such as circuit analysis, computational fluid dynamics, and control. For simple systems, the differential equations governing the dynamics can be derived by applying fundamental physical laws. However, for more complex systems, this approach becomes exceedingly difficult. Data-driven modeling is an alternative paradigm that seeks to learn an approximation of the dynamics of a system using observations of the true system. In recent years, there has been an increased interest in applying data-driven modeling techniques to solve a wide range of problems in physics and engineering. This article provides a survey of the different ways to construct models of dynamical systems using neural networks. In addition to the basic overview, we review the related literature and outline the most significant challenges from numerical simulations that this modeling paradigm must overcome. Based on the reviewed literature and identified challenges, we provide a discussion on promising research areas.},
92 | journal = {ACM Comput. Surv.},
93 | month = {feb},
94 | articleno = {236},
95 | numpages = {34},
96 | keywords = {physics-informed neural networks, physics-based regularization, Neural ODEs}
97 | }
98 | ```
99 | 


--------------------------------------------------------------------------------
/experiments/direct_solution.py:
--------------------------------------------------------------------------------
  1 | from argparse import ArgumentParser
  2 | from collections import defaultdict
  3 | from math import ceil, sin
  4 | from math import floor
  5 | 
  6 | 
  7 | import numpy as np
  8 | import torch
  9 | import torch.nn as nn
 10 | import torch.nn.functional as F
 11 | from torch.autograd import grad
 12 | from torch.nn.modules.container import T
 13 | 
 14 | from tqdm import tqdm
 15 | import matplotlib.pyplot as plt
 16 | from scipy.integrate import solve_ivp
 17 | import matplotlib as mpl
 18 | 
 19 | 
 20 | if __name__ == "__main__":
 21 | 
 22 |     parser = ArgumentParser()
 23 |     parser.add_argument(
 24 |         "--model",
 25 |         choices=["vanilla", "autodiff", "pinn"],
 26 |         default="vanilla",
 27 |     )
 28 |     parser.add_argument("--hidden_dim", default=32, type=int)
 29 |     parser.add_argument("--n_layers", default=5, type=int)
 30 |     parser.add_argument("--device", default="cpu")
 31 |     parser.add_argument("--n_epochs", default=2000, type=int)
 32 |     parser.add_argument("--t_start", default=0.0, type=float)
 33 |     parser.add_argument("--t_end", type=float, default=np.pi * 4)
 34 |     args = parser.parse_args()
 35 | 
 36 |     ############### Setup Experiment ###############
 37 |     y0 = [np.pi / 4, 0]
 38 |     step_size = 0.01
 39 |     t_start = args.t_start
 40 |     t_end = args.t_end
 41 |     t_span = (t_start, t_end)
 42 |     t_eval = np.arange(t_start, t_end, step_size)  # 0.0 , 0.01
 43 | 
 44 |     g = 1.0  # gravitational acceleration [m/s^2]
 45 |     l = 1.0  # length of pendulum [m]
 46 | 
 47 |     model = args.model
 48 |     n_epochs = args.n_epochs
 49 |     device = args.device
 50 |     subsample_every = int(2.5 / step_size)
 51 | 
 52 |     ############### Define Derivative ###############
 53 |     def f(t, y):
 54 |         θ, ω = y  # state variables go in
 55 |         g = 1.0
 56 |         l = 1.0
 57 |         dω = -(g / l) * np.sin(θ)
 58 |         dθ = ω  # special case (common for mechanical systems), the state variable ω is per definition dθ
 59 | 
 60 |         return dθ, dω  # derivatives of state variables go out
 61 | 
 62 |     ############### Solve ODE ###############
 63 | 
 64 |     res = solve_ivp(f, t_span, t_eval=t_eval, y0=y0, method="RK45")
 65 | 
 66 |     ############### Plot ###############
 67 | 
 68 |     def plot_colored(ax, x, y, c, cmap=plt.cm.jet, steps=10, **kwargs):
 69 |         a = c.size
 70 |         c = np.asarray(c)
 71 |         c -= c.min()
 72 |         c = c / c.max()
 73 |         it = 0
 74 |         while it < c.size - steps:
 75 |             x_segm = x[it : it + steps + 1]
 76 |             y_segm = y[it : it + steps + 1]
 77 |             c_segm = cmap(c[it + steps // 2])
 78 |             ax.plot(x_segm, y_segm, c=c_segm, **kwargs)
 79 |             it += steps
 80 | 
 81 |     θ, ω = res.y
 82 | 
 83 |     def xavier_init(module):
 84 |         for m in module.modules():
 85 |             if type(m) == nn.Linear:
 86 |                 nn.init.xavier_uniform_(m.weight)
 87 | 
 88 |     def construct_network(input_dim, output_dim, hidden_dim, hidden_layers):
 89 | 
 90 |         layers = [nn.Linear(input_dim, hidden_dim), nn.Softplus()]
 91 |         for _ in range(hidden_layers):
 92 |             layers.extend([nn.Linear(hidden_dim, hidden_dim), nn.Softplus()])
 93 |         layers.append(nn.Linear(hidden_dim, output_dim))
 94 | 
 95 |         net = nn.Sequential(*layers).double().to(device)
 96 |         xavier_init(net)
 97 |         return net
 98 | 
 99 |     hidden_dim = args.hidden_dim
100 |     hidden_layers = args.n_layers
101 | 
102 |     t = torch.tensor(t_eval, device=device, requires_grad=True)
103 |     losses = defaultdict(lambda: defaultdict(list))
104 | 
105 |     # `torch.autograd.grad` supports only lists of scalar values (e.g. single evaluation of network).
106 |     # however the function accepts a list of these.
107 |     def listify(A):
108 |         return [a for a in A.flatten()]
109 | 
110 |     y_train = torch.tensor(res.y[:, ::subsample_every]).to(device)
111 |     t_train = torch.tensor(t_eval[::subsample_every], requires_grad=True).to(device)
112 |     θω_pred = None
113 | 
114 |     if model == "vanilla":
115 |         nn_vanilla = construct_network(1, 2, hidden_dim, hidden_layers)
116 | 
117 |         opt_vanilla = torch.optim.Adam(nn_vanilla.parameters())
118 | 
119 |         for epoch in tqdm(range(n_epochs), desc="vanilla: training epoch"):
120 |             out = nn_vanilla(t_train.unsqueeze(-1)).T
121 | 
122 |             loss_collocation = F.mse_loss(out, y_train)
123 | 
124 |             loss_collocation.backward()
125 |             opt_vanilla.step()
126 |             nn_vanilla.zero_grad()
127 |             losses["vanilla"]["collocation"].append(loss_collocation.item())
128 | 
129 |         θω_pred = nn_vanilla(t.unsqueeze(-1)).detach().detach().cpu().T
130 | 
131 |     # autodiff
132 |     elif model == "autodiff":
133 |         nn_autodiff = construct_network(1, 1, hidden_dim, hidden_layers)
134 |         opt_autodiff = torch.optim.Adam(nn_autodiff.parameters())
135 | 
136 |         for epoch in tqdm(range(n_epochs), desc="autodiff: training epoch"):
137 |             θ_pred = nn_autodiff(t_train.unsqueeze(-1)).T
138 | 
139 |             θ_listed = listify(θ_pred)
140 | 
141 |             # [0] since we differentiate with respect to an "single input",
142 |             # which is coincidentially a tensor.
143 |             # in this case  ω ≜ dθ
144 |             ω_pred = grad(
145 |                 θ_listed,
146 |                 t_train,
147 |                 only_inputs=True,
148 |                 retain_graph=True,
149 |                 create_graph=True,
150 |             )[0].unsqueeze(0)
151 | 
152 |             θω = torch.cat((θ_pred, ω_pred), dim=0)
153 | 
154 |             loss_collocation = F.mse_loss(θω, y_train)
155 |             loss_collocation.backward()
156 | 
157 |             # sanity check
158 |             max_grad = next(nn_autodiff.modules())[0].weight.grad.max()
159 |             assert (
160 |                 max_grad != 0.0
161 |             ), "maximal gradient of first layer was zero, something is up!"
162 | 
163 |             opt_autodiff.step()
164 |             nn_autodiff.zero_grad()
165 | 
166 |             losses["autodiff"]["collocation"].append(loss_collocation.item())
167 | 
168 |         θ_autodiff = nn_autodiff(t.unsqueeze(-1)).T
169 |         θ_autodiff_listed = listify(θ_autodiff)
170 |         ω_autodiff = grad(θ_autodiff_listed, t, only_inputs=True)[0].unsqueeze(0)
171 |         θω_pred = torch.cat((θ_autodiff, ω_autodiff), dim=0).detach().cpu()
172 | 
173 |     # pinn
174 |     elif model == "pinn":
175 |         nn_pinn = construct_network(1, 1, hidden_dim, hidden_layers)
176 | 
177 |         opt_pinn = torch.optim.Adam(nn_pinn.parameters())
178 |         t_train_dense = torch.tensor(t_eval, requires_grad=True).to(device)
179 |         losses_pinn = {"collocation": [], "equation": []}
180 |         for epoch in tqdm(range(n_epochs), desc="pinn: training epoch"):
181 |             θ_pred = nn_pinn(t_train_dense.unsqueeze(-1)).T
182 |             θ_listed = listify(θ_pred)
183 | 
184 |             ω_pred = grad(
185 |                 θ_listed,
186 |                 t_train_dense,
187 |                 only_inputs=True,
188 |                 retain_graph=True,
189 |                 create_graph=True,
190 |             )[0].unsqueeze(0)
191 |             ω_listed = listify(ω_pred)
192 |             dω_pred = grad(
193 |                 ω_listed,
194 |                 t_train_dense,
195 |                 only_inputs=True,
196 |                 retain_graph=True,
197 |                 create_graph=True,
198 |             )[0].unsqueeze(0)
199 | 
200 |             θω_dense = torch.cat((θ_pred, ω_pred), dim=0)
201 | 
202 |             # collocation loss is defined for sparse set of points
203 |             θω = θω_dense[:, ::subsample_every]
204 |             loss_collocation = F.mse_loss(θω, y_train)
205 | 
206 |             # equation based loss is defined for dense samples
207 |             dω_eq = -torch.sin(θ_pred)
208 |             loss_equation = F.mse_loss(dω_pred, dω_eq)
209 | 
210 |             loss_total = loss_collocation + loss_equation
211 |             loss_total.backward()
212 |             # next(net.modules())[0].weight.grad => this gives you gradients of the loss
213 | 
214 |             # sanity check
215 |             max_grad = next(nn_pinn.modules())[0].weight.grad.max()
216 |             assert (
217 |                 max_grad != 0.0
218 |             ), "maximal gradient of first layer was zero, something is up!"
219 | 
220 |             opt_pinn.step()
221 |             nn_pinn.zero_grad()
222 | 
223 |             losses["pinn"]["collocation"].append(loss_collocation.item())
224 |             losses["pinn"]["equation"].append(loss_equation.item())
225 | 
226 |         θ_pinn = nn_pinn(t.unsqueeze(-1)).T
227 |         θ_pinn_listed = listify(θ_pinn)
228 |         ω_pinn = grad(θ_pinn_listed, t, only_inputs=True)[0].unsqueeze(0)
229 |         θω_pred = torch.cat((θ_pinn, ω_pinn), dim=0).detach().cpu()
230 | 
231 |     def plot_predictions(model, θω_pred):
232 |         ω_numerical = np.diff(θω_pred[:1]) / step_size
233 |         fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True)
234 |         fig.canvas.manager.set_window_title(model)
235 |         ax1.set_ylabel("θ(t)")
236 |         ax2.set_ylabel("ω(t)")
237 |         ax2.set_xlabel("t")
238 | 
239 |         ax1.plot(t_eval, θ, c="black", label="true")
240 |         ax1.plot(t_eval, θω_pred[0], c="b", linestyle="--", label="predicted")
241 | 
242 |         ax2.plot(t_eval, ω, c="black", label="true")
243 |         ax2.plot(t_eval, θω_pred[1], c="r", linestyle="--", label="predicted")
244 |         ax2.plot(
245 |             t_eval[1:],
246 |             ω_numerical.T,
247 |             c="r",
248 |             linestyle="dotted",
249 |             label="numerical",
250 |         )
251 | 
252 |         ax1.scatter(
253 |             t_eval[::subsample_every],
254 |             res.y[:, ::subsample_every][0],
255 |             c="black",
256 |             linestyle="None",
257 |             label="collocation point",
258 |         )
259 |         ax2.scatter(
260 |             t_eval[::subsample_every],
261 |             res.y[:, ::subsample_every][1],
262 |             c="black",
263 |             linestyle="None",
264 |             label="collocation point",
265 |         )
266 |         ax2.legend()
267 |         plt.tight_layout()
268 | 
269 |     # plot loss functions as function of training steps
270 |     def plot_losses(model, losses):
271 |         fig, ax = plt.subplots()
272 |         fig.canvas.manager.set_window_title(f"loss terms '{model}'")
273 | 
274 |         for loss_name, loss in losses.items():
275 |             ax.plot(loss, label=loss_name)
276 | 
277 |         ax.legend()
278 |         ax.set_xlabel("epoch")
279 | 
280 |     plot_predictions(model, θω_pred)
281 |     plot_losses(model, losses[model])
282 | 
283 |     fig = plt.figure()
284 |     x, y = np.meshgrid(
285 |         np.arange(-np.pi, np.pi, 0.01),
286 |         np.arange(-np.pi, np.pi, 0.01),
287 |     )
288 |     dθ, dω = f(None, (x, y))
289 |     plt.streamplot(x, y, dθ, dω, density=2)
290 |     plt.xlabel("θ")
291 |     plt.ylabel("ω")
292 |     fig.canvas.manager.set_window_title(f"phase portrait '{model}'")
293 | 
294 |     ax = plt.gca()
295 |     plot_colored(ax, θ, ω, t_eval)
296 |     cmap = plt.cm.jet
297 | 
298 |     norm = mpl.colors.Normalize(vmin=t_eval.min(), vmax=t_eval.max())
299 |     fig.colorbar(mpl.cm.ScalarMappable(norm=norm, cmap=cmap), ax=ax)
300 | 
301 |     # draw initial state
302 |     plt.scatter(θ[0], ω[0], label="$y_0$", marker="*", c="g", s=200, zorder=100)
303 |     plt.legend(loc="upper right")
304 |     ax.set_aspect(1)
305 | 
306 |     plt.show()
307 | 


--------------------------------------------------------------------------------
/experiments/time_stepper.py:
--------------------------------------------------------------------------------
  1 | from argparse import ArgumentParser
  2 | from numpy import double
  3 | import torch.nn as nn
  4 | from torch.optim import Adam
  5 | from tqdm import tqdm
  6 | from pyDOE import lhs
  7 | import matplotlib.pyplot as plt
  8 | from torch.nn.functional import mse_loss
  9 | import numpy as np
 10 | 
 11 | import torch
 12 | from torchdyn.numerics import odeint
 13 | from torchdyn.numerics.solvers import SolverTemplate
 14 | import matplotlib.patches as mpatches
 15 | from matplotlib.collections import LineCollection
 16 | 
 17 | from visualization import get_meshgrid, plot_colored
 18 | 
 19 | 
 20 | class DirectSolver(SolverTemplate):
 21 |     def __init__(self, dtype=torch.float32):
 22 |         super().__init__(order=1)
 23 |         self.dtype = dtype
 24 |         self.stepping_class = "fixed"
 25 | 
 26 |     def step(self, f, x, t, dt, k1=None):
 27 | 
 28 |         x_sol = f(t, x)
 29 |         return None, x_sol, None
 30 | 
 31 | 
 32 | class ResnetSolver(SolverTemplate):
 33 |     def __init__(self, step_size=None, dtype=torch.float32):
 34 |         super().__init__(order=1)
 35 |         self.dtype = dtype
 36 |         self.stepping_class = "fixed"
 37 | 
 38 |         self.step_size = 1 if step_size is None else step_size
 39 | 
 40 |     def step(self, f, x, t, dt, k1=None):
 41 |         x_sol = x + f(t, x) * self.step_size
 42 |         return None, x_sol, None
 43 | 
 44 | 
 45 | if __name__ == "__main__":
 46 | 
 47 |     parser = ArgumentParser()
 48 |     parser.add_argument(
 49 |         "--solver",
 50 |         choices=["direct", "resnet", "euler", "rk4", "dopri5", "tsit5"],
 51 |         default="euler",
 52 |     )
 53 |     parser.add_argument("--hidden_dim", default=32, type=int)
 54 |     parser.add_argument("--n_layers", default=8, type=int)
 55 |     parser.add_argument("--device", default="cpu")
 56 |     parser.add_argument("--n_epochs", default=2000, type=int)
 57 |     parser.add_argument("--n_traj_train", default=100, type=int)
 58 |     parser.add_argument("--n_traj_validate", default=10, type=int)
 59 |     parser.add_argument("--t_start_train", default=0.0, type=float)
 60 |     parser.add_argument("--t_end_train", type=float, default=0.001)
 61 |     parser.add_argument("--t_start_validate", default=0.0, type=float)
 62 |     parser.add_argument("--t_end_validate", type=float, default=4 * np.pi)
 63 |     parser.add_argument("--step_size_train", default=0.001, type=float)
 64 |     parser.add_argument("--step_size_validate", default=0.001, type=float)
 65 |     parser.add_argument("--noise_std", default=0.0, type=float)
 66 |     parser.add_argument("--domain_train", default=1.0, type=float)
 67 |     parser.add_argument("--domain_validate", default=1.0, type=float)
 68 |     args = parser.parse_args()
 69 | 
 70 |     # generate data
 71 | 
 72 |     def f(t, x):
 73 |         θ = x[..., 0]
 74 |         ω = x[..., 1]
 75 | 
 76 |         dθ = ω
 77 |         dω = -torch.sin(θ)
 78 | 
 79 |         return torch.stack((dθ, dω), dim=-1)
 80 | 
 81 |     domain_draw_factor = 1.3
 82 | 
 83 |     domain_train = args.domain_train
 84 |     domain_validate = args.domain_validate
 85 |     x0_train = (
 86 |         torch.tensor(lhs(2, args.n_traj_train), device=args.device) * 2 - 1
 87 |     ) * domain_train
 88 |     x0_validate = (
 89 |         torch.tensor(lhs(2, args.n_traj_validate), device=args.device) * 2 - 1
 90 |     ) * domain_validate
 91 |     x0_grid = get_meshgrid(step_per_axis=0.01, domain=domain_validate)
 92 |     x0_example = torch.tensor((0.6, 0)).double().unsqueeze(0).to(args.device)
 93 | 
 94 |     step_size_train = args.step_size_train
 95 |     ε = 1e-10
 96 |     t_span_train = torch.arange(
 97 |         args.t_start_train, args.t_end_train + ε, step_size_train
 98 |     )
 99 |     t_span_validate = torch.arange(
100 |         args.t_start_validate,
101 |         args.t_end_validate + ε,
102 |         args.step_size_validate,
103 |     )
104 | 
105 |     if args.solver.lower() == "direct":
106 |         solver = DirectSolver()
107 |     elif args.solver.lower() == "resnet":
108 |         solver = ResnetSolver()
109 |     else:
110 |         solver = args.solver
111 | 
112 |     _, x_train = odeint(f, x0_train, t_span_train, solver="rk4")
113 |     x_true = x_train
114 |     x_train = x_train + torch.randn_like(x_train) * args.noise_std
115 | 
116 |     _, x_validate = odeint(f, x0_validate, t_span_validate, solver="rk4")
117 |     _, x_example = odeint(f, x0_example, t_span_validate, solver="rk4")
118 | 
119 |     ##################### model ##########################
120 |     layers = []
121 |     layers.append(nn.Linear(2, args.hidden_dim))
122 |     for _ in range(args.n_layers):
123 |         layers.append(nn.Linear(args.hidden_dim, args.hidden_dim))
124 |         layers.append(nn.Softplus())
125 | 
126 |     layers.append(nn.Linear(args.hidden_dim, 2))
127 | 
128 |     net = nn.Sequential(*layers)
129 |     net.to(args.device).double()
130 | 
131 |     for m in net.modules():
132 |         if type(m) == nn.Linear:
133 |             nn.init.xavier_uniform_(m.weight)
134 | 
135 |     # optimizer
136 | 
137 |     opt = Adam(net.parameters())
138 | 
139 |     # train
140 |     losses = []
141 | 
142 |     for _ in tqdm(range(args.n_epochs)):
143 | 
144 |         _, x_pred_train = odeint(
145 |             lambda t, x: net(x), x0_train, t_span_train, solver=solver
146 |         )
147 |         loss = mse_loss(x_pred_train, x_train)
148 |         loss.backward()
149 |         opt.step()
150 |         opt.zero_grad()
151 |         losses.append(loss.item())
152 | 
153 |     _, x_pred_train = odeint(lambda t, x: net(x), x0_train, t_span_train, solver=solver)
154 | 
155 |     _, x_pred_validate = odeint(
156 |         lambda t, x: net(x), x0_validate, t_span_validate, solver=solver
157 |     )
158 | 
159 |     _, x_pred_example = odeint(
160 |         lambda t, x: net(x), x0_example, t_span_validate, solver=solver
161 |     )
162 | 
163 |     # derivatives
164 |     # x0_grid_before = get_meshgrid(step_per_axis=0.01, domain=domain)
165 |     x_derivative = f(None, x0_grid)
166 | 
167 |     if args.solver == "direct":
168 | 
169 |         out = net(x0_grid)
170 |         # normalize for the step size used during training. If the network is trained with a step-size of 1/100 of a second
171 |         # it will predict changes that are 100 times as small as those for 1 second.
172 |         x_derivative_pred = (out - x0_grid) / step_size_train
173 | 
174 |     elif args.solver == "resnet":
175 |         x_derivative_pred = net(x0_grid) / step_size_train
176 |     else:
177 |         x_derivative_pred = net(x0_grid)
178 | 
179 |     # plot
180 |     x_pred_train = x_pred_train.detach().numpy()
181 |     x_pred_validate = x_pred_validate.detach().numpy()
182 |     x_pred_example = x_pred_example.detach().numpy()
183 |     x_derivative_pred = x_derivative_pred.detach().numpy()
184 |     x_derivative = x_derivative.detach().numpy()
185 |     x0_grid = x0_grid.detach().numpy()
186 | 
187 |     # streamplot
188 |     density = 1
189 |     fig, ax = plt.subplots()
190 |     fig.canvas.manager.set_window_title("stream plot")
191 |     ode_patch = mpatches.Patch(color="black", label="true")
192 |     nn_patch = mpatches.Patch(color="blue", label="pred")
193 | 
194 |     ax.streamplot(
195 |         x0_grid[..., 0],
196 |         x0_grid[..., 1],
197 |         x_derivative[..., 0],
198 |         x_derivative[..., 1],
199 |         color="black",
200 |         density=density,
201 |     )
202 | 
203 |     ax.streamplot(
204 |         x0_grid[..., 0],
205 |         x0_grid[..., 1],
206 |         x_derivative_pred[..., 0],
207 |         x_derivative_pred[..., 1],
208 |         color="blue",
209 |         density=density,
210 |     )
211 | 
212 |     ax.set_xlabel("θ")
213 |     ax.set_ylabel("ω")
214 |     ax.set_xlim(-domain_validate, domain_validate)
215 |     ax.set_ylim(-domain_validate, domain_validate)
216 | 
217 |     ax.legend(handles=[ode_patch, nn_patch])
218 | 
219 |     # quiver
220 |     fig, ax = plt.subplots()
221 |     fig.canvas.manager.set_window_title("quiver")
222 |     ax.quiver(
223 |         x0_grid[..., 0],
224 |         x0_grid[..., 1],
225 |         x_derivative[..., 0],
226 |         x_derivative[..., 1],
227 |         color="black",
228 |         angles="xy",
229 |         scale_units="xy",
230 |         label="true"
231 |         # scale=1,
232 |     )
233 | 
234 |     ax.quiver(
235 |         x0_grid[..., 0],
236 |         x0_grid[..., 1],
237 |         x_derivative_pred[..., 0],
238 |         x_derivative_pred[..., 1],
239 |         color="blue",
240 |         angles="xy",
241 |         scale_units="xy",
242 |         # scale=1,
243 |         label="pred",
244 |     )
245 |     ax.legend()
246 | 
247 |     ax.set_xlabel("θ")
248 |     ax.set_ylabel("ω")
249 |     ax.set_ylim(-domain_validate, domain_validate)
250 |     ax.set_xlim(-domain_validate, domain_validate)
251 | 
252 |     # phase space, training
253 |     fig, ax = plt.subplots()
254 |     fig.canvas.manager.set_window_title("phase space: training")
255 | 
256 |     lines_true = LineCollection(
257 |         [x for x in x_true.swapaxes(0, 1)], color="black", label="true"
258 |     )
259 | 
260 |     # lines_noise = LineCollection(
261 |     #     [x for x in x_train.swapaxes(0, 1)], color="red", label="noisy"
262 |     # )
263 | 
264 |     lines_pred = LineCollection(
265 |         [x for x in x_pred_train.swapaxes(0, 1)], color="blue", label="pred"
266 |     )
267 | 
268 |     ax.add_collection(lines_true)
269 |     if args.noise_std != 0.0:
270 |         ax.scatter(x_train[..., 0], x_train[..., 1], label="observations")
271 |         # ax.add_collection(lines_noise)
272 |     ax.add_collection(lines_pred)
273 | 
274 |     ax.set_xlabel("θ")
275 |     ax.set_ylabel("ω")
276 |     ax.set_xlim(-domain_draw_factor * domain_train, domain_draw_factor * domain_train)
277 |     ax.set_ylim(-domain_draw_factor * domain_train, domain_draw_factor * domain_train)
278 |     ax.legend()
279 | 
280 |     # phase space, validation
281 |     fig, ax = plt.subplots()
282 |     fig.canvas.manager.set_window_title("phase space: validation")
283 |     lines_true = LineCollection(
284 |         [x for x in x_validate.swapaxes(0, 1)], color="black", label="true"
285 |     )
286 |     # lines_pred = LineCollection(
287 |     #     [x for x in x_pred_validate.swapaxes(0, 1)], color="blue", label="pred"
288 |     # )
289 |     # plot_colored(fig, ax, t_span_validate, x_validate, label="true")
290 |     ax.add_collection(lines_true)
291 | 
292 |     plot_colored(
293 |         fig,
294 |         ax,
295 |         t_span_validate,
296 |         x_pred_validate,
297 |         label="pred",
298 |         colorbar=True,
299 |         # linestyle="dashed",
300 |     )
301 |     # ax.add_collection(lines_pred)
302 |     ax.set_xlabel("θ")
303 |     ax.set_ylabel("ω")
304 |     ax.set_xlim(-2 * domain_validate, 2 * domain_validate)
305 |     ax.set_ylim(-2 * domain_validate, 2 * domain_validate)
306 |     ax.legend()
307 | 
308 |     # time series validation, specific idx
309 |     fig, (ax1, ax2) = plt.subplots(2, sharex=True)
310 |     example_idx = 0
311 |     fig.canvas.manager.set_window_title(f"states vs time: validation idx={example_idx}")
312 | 
313 |     ax1.plot(t_span_validate, x_validate[..., example_idx, 0], color="black")
314 |     ax1.plot(
315 |         t_span_validate,
316 |         x_pred_validate[..., example_idx, 0],
317 |         linestyle="dashed",
318 |         color="blue",
319 |     )
320 |     ax2.plot(
321 |         t_span_validate, x_validate[..., example_idx, 1], color="black", label="true"
322 |     )
323 |     ax2.plot(
324 |         t_span_validate,
325 |         x_pred_validate[..., example_idx, 1],
326 |         linestyle="dashed",
327 |         color="blue",
328 |         label="pred",
329 |     )
330 |     ax1.set_ylabel("θ(t)")
331 |     ax2.set_ylabel("ω(t)")
332 |     ax2.set_xlabel("t")
333 |     ax2.legend()
334 | 
335 |     # time series validation, example (0.6,0)
336 |     fig, (ax1, ax2) = plt.subplots(2, sharex=True)
337 |     fig.canvas.manager.set_window_title(f"states vs time: validation example")
338 | 
339 |     ax1.plot(t_span_validate, x_example[..., 0], color="black")
340 |     ax1.plot(
341 |         t_span_validate,
342 |         x_pred_example[..., 0],
343 |         linestyle="dashed",
344 |         color="blue",
345 |     )
346 |     ax2.plot(t_span_validate, x_example[..., 1], label="true", color="black")
347 |     ax2.plot(
348 |         t_span_validate,
349 |         x_pred_example[..., 1],
350 |         linestyle="dashed",
351 |         color="blue",
352 |         label="predicted",
353 |     )
354 |     ax1.set_ylabel("θ(t)")
355 |     ax2.set_ylabel("ω(t)")
356 |     ax2.set_xlabel("t")
357 |     ax2.legend()
358 | 
359 |     # show
360 |     fig, ax = plt.subplots()
361 |     ax.plot(losses)
362 |     ax.set_xlabel("epoch")
363 |     ax.set_ylabel("MSE")
364 | 
365 |     plt.show()
366 | 


--------------------------------------------------------------------------------
/experiments/latent_neural_odes.py:
--------------------------------------------------------------------------------
  1 | from argparse import ArgumentParser
  2 | from collections import defaultdict
  3 | from itertools import chain
  4 | from numpy.random.mtrand import choice
  5 | 
  6 | import torch.nn as nn
  7 | from torch.nn.modules.rnn import RNN
  8 | from torch.optim import Adam
  9 | from tqdm import tqdm
 10 | from pyDOE import lhs
 11 | import matplotlib.pyplot as plt
 12 | from torch.nn.functional import mse_loss
 13 | import numpy as np
 14 | 
 15 | import torch
 16 | from torchdyn.numerics import odeint
 17 | from torchdyn.numerics.solvers import SolverTemplate
 18 | import matplotlib.patches as mpatches
 19 | from matplotlib.collections import LineCollection
 20 | 
 21 | from visualization import get_meshgrid, plot_colored
 22 | 
 23 | 
 24 | class DirectSolver(SolverTemplate):
 25 |     def __init__(self, dtype=torch.float32):
 26 |         super().__init__(order=1)
 27 |         self.dtype = dtype
 28 |         self.stepping_class = "fixed"
 29 | 
 30 |     def step(self, f, x, t, dt, k1=None):
 31 | 
 32 |         x_sol = f(t, x)
 33 |         return None, x_sol, None
 34 | 
 35 | 
 36 | class ResnetSolver(SolverTemplate):
 37 |     def __init__(self, step_size=None, dtype=torch.float32):
 38 |         super().__init__(order=1)
 39 |         self.dtype = dtype
 40 |         self.stepping_class = "fixed"
 41 | 
 42 |         self.step_size = 1 if step_size is None else step_size
 43 | 
 44 |     def step(self, f, x, t, dt, k1=None):
 45 |         x_sol = x + f(t, x) * self.step_size
 46 |         return None, x_sol, None
 47 | 
 48 | 
 49 | if __name__ == "__main__":
 50 | 
 51 |     parser = ArgumentParser()
 52 |     parser.add_argument(
 53 |         "--solver", choices=["direct", "resnet", "euler", "rk4"], default="euler"
 54 |     )
 55 |     parser.add_argument("--hidden_dim", default=32, type=int)
 56 |     parser.add_argument("--n_layers", default=8, type=int)
 57 |     parser.add_argument("--device", default="cpu")
 58 |     parser.add_argument("--n_epochs", default=100, type=int)
 59 |     parser.add_argument("--n_traj_train", default=55, type=int)
 60 |     parser.add_argument("--n_traj_validate", default=10, type=int)
 61 |     parser.add_argument("--t_start_train", default=0.0, type=float)
 62 |     parser.add_argument("--t_end_train", default=1.0, type=float)
 63 |     parser.add_argument("--t_start_validate", default=0.0, type=float)
 64 |     parser.add_argument("--t_end_validate", default=4 * np.pi, type=float)
 65 |     parser.add_argument("--step_size_train", default=0.01, type=float)
 66 |     parser.add_argument("--step_size_validate", default=0.01, type=float)
 67 |     parser.add_argument("--noise_std", default=0.000001, type=float)
 68 |     parser.add_argument("--domain_train", default=1.0, type=float)
 69 |     parser.add_argument("--domain_validate", default=1.0, type=float)
 70 |     parser.add_argument("--latent_dim", default=4, type=int)
 71 |     parser.add_argument("--scatter", default=False, type=bool)
 72 |     parser.add_argument(
 73 |         "--criterion", default="elbo", choices=["elbo", "mse"], type=str
 74 |     )
 75 |     args = parser.parse_args()
 76 | 
 77 |     # generate data
 78 | 
 79 |     def f(t, x):
 80 |         θ = x[..., 0]
 81 |         ω = x[..., 1]
 82 | 
 83 |         dθ = ω
 84 |         dω = -torch.sin(θ)
 85 | 
 86 |         return torch.stack((dθ, dω), dim=-1)
 87 | 
 88 |     domain_draw_factor = 1.3
 89 | 
 90 |     noise_log_var = 2 * torch.log(torch.tensor(args.noise_std))
 91 | 
 92 |     latent_dim = args.latent_dim
 93 |     domain_train = args.domain_train
 94 |     domain_validate = args.domain_validate
 95 | 
 96 |     x0_train = (
 97 |         torch.tensor(lhs(2, args.n_traj_train), device=args.device) * 2 - 1
 98 |     ) * domain_train
 99 |     x0_validate = (
100 |         torch.tensor(lhs(2, args.n_traj_validate), device=args.device) * 2 - 1
101 |     ) * domain_validate
102 |     x0_grid = get_meshgrid(step_per_axis=0.01, domain=domain_validate)
103 |     x0_example = torch.tensor((0.6, 0)).double().unsqueeze(0).to(args.device)
104 | 
105 |     step_size_train = args.step_size_train
106 |     ε = 1e-10
107 |     t_span_train = torch.arange(
108 |         args.t_start_train, args.t_end_train + ε, step_size_train
109 |     )
110 |     t_span_validate = torch.arange(
111 |         args.t_start_validate,
112 |         args.t_end_validate + ε,
113 |         args.step_size_validate,
114 |     )
115 | 
116 |     if args.solver.lower() == "direct":
117 |         solver = DirectSolver()
118 |     elif args.solver.lower() == "resnet":
119 |         solver = ResnetSolver()
120 |     else:
121 |         solver = args.solver
122 | 
123 |     _, x_train = odeint(f, x0_train, t_span_train, solver="rk4")
124 |     x_train = x_train + torch.randn_like(x_train) * args.noise_std
125 | 
126 |     _, x_validate = odeint(f, x0_validate, t_span_train, solver="rk4")
127 |     _, x_example = odeint(f, x0_example, t_span_validate, solver="rk4")
128 | 
129 |     ##################### model ##########################
130 |     device = args.device
131 | 
132 |     class Encoder(nn.Module):
133 |         def __init__(self, input_size, hidden_size, output_size) -> None:
134 |             super().__init__()
135 |             self.rnn = nn.RNN(input_size, hidden_size)
136 |             self.h2o = nn.Linear(hidden_size, output_size)
137 | 
138 |         def forward(self, x):
139 |             x_flipped = torch.flip(x, (0,))
140 |             _, h = self.rnn(x_flipped)
141 |             z0 = self.h2o(h)
142 |             return z0
143 | 
144 |     class LatentODE(nn.Module):
145 |         def __init__(self) -> None:
146 |             super().__init__()
147 | 
148 |             self.encoder = Encoder(2, 25, latent_dim * 2)
149 | 
150 |             layers = []
151 |             layers.append(nn.Linear(latent_dim, args.hidden_dim))
152 |             for _ in range(args.n_layers):
153 |                 layers.append(nn.Linear(args.hidden_dim, args.hidden_dim))
154 |                 layers.append(nn.Softplus())
155 | 
156 |             layers.append(nn.Linear(args.hidden_dim, 4))
157 |             self.dynamics = nn.Sequential(*layers)
158 | 
159 |             self.decoder = (
160 |                 nn.Sequential(
161 |                     nn.Linear(latent_dim, 20), nn.Softplus(), nn.Linear(20, 2)
162 |                 )
163 |                 .to(device)
164 |                 .double()
165 |             )
166 |             self.inference = False
167 | 
168 |         def forward(self, t, x):
169 |             z = self.encoder(x)[-1]
170 |             qz0_mean, qz0_logvar = z[:, :latent_dim], z[:, latent_dim:]
171 |             qz0_std = torch.exp(0.5 * qz0_logvar)
172 |             ε = torch.randn_like(qz0_mean)
173 | 
174 |             if self.inference:
175 |                 z0 = qz0_mean
176 |             else:
177 |                 z0 = qz0_mean + ε * qz0_std  # TODO
178 |             # z0 = qz0_mean  # TODO
179 | 
180 |             _, z_pred = odeint(lambda t, z: self.dynamics(z), z0, t, solver=solver)
181 | 
182 |             x_pred = self.decoder(z_pred)
183 | 
184 |             return qz0_mean, qz0_logvar, x_pred
185 | 
186 |     net = LatentODE().to(args.device).double()
187 | 
188 |     for m in chain(net.modules()):
189 |         if type(m) == nn.Linear:
190 |             nn.init.xavier_uniform_(m.weight)
191 | 
192 |     # optimizer
193 | 
194 |     opt = Adam(net.parameters())
195 | 
196 |     # train
197 |     losses = defaultdict(list)
198 | 
199 |     def log_normal_pdf(x, mean, logvar):
200 |         px = -0.5 * (
201 |             torch.log(torch.tensor(np.pi))
202 |             + logvar
203 |             + (x - mean) ** 2.0 / torch.exp(logvar)
204 |         )
205 |         return px.sum((0, 2))
206 | 
207 |         # return -.5 * (const + logvar + (x - mean) ** 2. / torch.exp(logvar))
208 | 
209 |     def normal_kl(μ1, log_var1, μ2, log_var2):
210 |         v1 = torch.exp(log_var1)
211 |         v2 = torch.exp(log_var2)
212 |         lstd1 = log_var1 / 2.0
213 |         lstd2 = log_var2 / 2.0
214 | 
215 |         kl = lstd2 - lstd1 + ((v1 + (μ1 - μ2) ** 2.0) / (2.0 * v2)) - 0.5
216 |         return kl.sum(-1)
217 | 
218 |     for _ in tqdm(range(args.n_epochs)):
219 | 
220 |         qz0_mean, qz0_logvar, x_pred_train = net(t_span_train, x_train)
221 | 
222 |         log_px = log_normal_pdf(x_train, x_pred_train, noise_log_var)
223 |         log_normal_kl = normal_kl(
224 |             qz0_mean,
225 |             qz0_logvar,
226 |             torch.zeros_like(qz0_mean),
227 |             torch.zeros_like(qz0_logvar),  # we want logvar = 1 => var = 1
228 |         )
229 | 
230 |         loss_elbo = torch.mean(log_normal_kl - log_px, dim=0)
231 |         loss_mse = mse_loss(x_pred_train, x_train)
232 | 
233 |         if args.criterion == "mse":
234 |             loss = loss_mse
235 |         elif args.criterion == "elbo":
236 |             loss = loss_elbo
237 | 
238 |         loss.backward()
239 |         opt.step()
240 |         opt.zero_grad()
241 |         losses["log_px"].append(
242 |             torch.mean(log_px).detach().numpy()
243 |         )  # how likely is it to observe the targets, if the output distributions produced by the network, is as they claim.
244 |         losses["kl"].append(torch.mean(log_normal_kl).detach().numpy())
245 |         losses["mse"].append(loss_mse.detach().numpy())
246 |         losses["elbo"].append(loss_elbo.detach().numpy())
247 | 
248 |     net.inference = True
249 |     _, _, x_pred_train = net(t_span_train, x_train)
250 |     _, _, x_pred_validate = net(t_span_validate, x_validate)
251 | 
252 |     # _, x_pred_example = odeint(
253 |     #     lambda t, x: dynamics(x), x0_example, t_span_validate, solver=solver
254 |     # )
255 | 
256 |     # derivatives
257 |     # x0_grid_before = get_meshgrid(step_per_axis=0.01, domain=domain)
258 |     x_derivative = f(None, x0_grid)
259 | 
260 |     # plot
261 |     x_pred_train = x_pred_train.detach().numpy()
262 |     x_pred_validate = x_pred_validate.detach().numpy()
263 | 
264 |     # phase space, training
265 |     fig, ax = plt.subplots()
266 |     fig.canvas.manager.set_window_title("phase space: training")
267 | 
268 |     lines_true = LineCollection(
269 |         [x for x in x_train.swapaxes(0, 1)],
270 |         color="black",
271 |         label="true",
272 |     )
273 |     # lines_pred = LineCollection(
274 |     #     [x for x in x_pred_train.swapaxes(0, 1)], color="blue", label="pred"
275 |     # )
276 | 
277 |     if args.scatter:
278 |         ax.scatter(x_pred_train[..., 0], x_pred_train[..., 1])
279 |     plot_colored(
280 |         fig,
281 |         ax,
282 |         t_span_train,
283 |         x_pred_train,
284 |         label="pred",
285 |         colorbar=True,
286 |         # linestyle="dashed",
287 |     )
288 |     if args.scatter:
289 |         ax.scatter(x_train[..., 0], x_train[..., 1])
290 |     ax.add_collection(lines_true)
291 |     # ax.add_collection(lines_pred)
292 | 
293 |     ax.set_xlabel("θ")
294 |     ax.set_ylabel("ω")
295 |     ax.set_xlim(-domain_draw_factor * domain_train, domain_draw_factor * domain_train)
296 |     ax.set_ylim(-domain_draw_factor * domain_train, domain_draw_factor * domain_train)
297 |     ax.legend()
298 | 
299 |     # # phase space, validation
300 |     fig, ax = plt.subplots()
301 |     fig.canvas.manager.set_window_title("phase space: validation")
302 | 
303 |     lines_true = LineCollection(
304 |         [x for x in x_validate.swapaxes(0, 1)],
305 |         color="black",
306 |         label="true",
307 |     )
308 |     # lines_pred = LineCollection(
309 |     #     [x for x in x_pred_train.swapaxes(0, 1)], color="blue", label="pred"
310 |     # )
311 |     if args.scatter:
312 |         ax.scatter(x_pred_validate[..., 0], x_pred_validate[..., 1])
313 |     plot_colored(
314 |         fig,
315 |         ax,
316 |         t_span_validate,
317 |         x_pred_validate,
318 |         label="pred",
319 |         colorbar=True,
320 |         # linestyle="dashed",
321 |     )
322 | 
323 |     if args.scatter:
324 |         ax.scatter(x_validate[..., 0], x_validate[..., 1])
325 |     ax.add_collection(lines_true)
326 |     # ax.add_collection(lines_pred)
327 | 
328 |     ax.set_xlabel("θ")
329 |     ax.set_ylabel("ω")
330 |     ax.set_xlim(
331 |         -domain_draw_factor * domain_validate, domain_draw_factor * domain_validate
332 |     )
333 |     ax.set_ylim(
334 |         -domain_draw_factor * domain_validate, domain_draw_factor * domain_validate
335 |     )
336 |     ax.legend()
337 | 
338 |     # # time series validation, example (0.6,0)
339 |     # fig, (ax1, ax2) = plt.subplots(2, sharex=True)
340 |     # fig.canvas.manager.set_window_title(f"states vs time: validation example")
341 | 
342 |     # ax1.plot(t_span_validate, x_example[..., 0], color="black")
343 |     # ax1.plot(
344 |     #     t_span_validate,
345 |     #     x_pred_example[..., 0],
346 |     #     linestyle="dashed",
347 |     #     color="blue",
348 |     # )
349 |     # ax2.plot(t_span_validate, x_example[..., 1], label="true", color="black")
350 |     # ax2.plot(
351 |     #     t_span_validate,
352 |     #     x_pred_example[..., 1],
353 |     #     linestyle="dashed",
354 |     #     color="blue",
355 |     #     label="predicted",
356 |     # )
357 |     # ax1.set_ylabel("θ(t)")
358 |     # ax2.set_ylabel("ω(t)")
359 |     # ax2.set_xlabel("t")
360 |     # ax2.legend()
361 | 
362 |     # show
363 |     fig, ax = plt.subplots()
364 |     ax.set_xlabel("epoch")
365 |     ax.set_ylabel("loss")
366 |     for name, values in losses.items():
367 |         ax.plot(values, label=name)
368 | 
369 |     ax.legend()
370 |     plt.show()
371 | 


--------------------------------------------------------------------------------
/docs/survey_structure.svg:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
3 | <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="737px" height="339px" viewBox="-0.5 -0.5 737 339"><defs/><g><path d="M 370.02 293.74 L 399.8 222.83" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 182 215.57 L 154.16 274.87" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 276 192.34 L 251 195.57" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 276 171.91 L 226 147.43" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 339.82 168.81 L 369 156.35" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 465.4 159.15 L 485 170.23" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><rect x="276" y="165.57" width="66" height="45" rx="6.75" ry="6.75" fill="#ffffff" stroke="#000000" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 64px; height: 1px; padding-top: 188px; margin-left: 277px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div><span>Survey<br /></span></div><div><span>Structure</span></div></div></div></div></foreignObject><text x="309" y="192" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Survey...</text></switch></g><path d="M 174.33 175.57 L 136 150.57" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><rect x="159" y="175.57" width="92" height="40" rx="6" ry="6" fill="#ffffff" stroke="#0000ff" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 90px; height: 1px; padding-top: 196px; margin-left: 160px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Background <br /></div><div><font style="font-size: 9px">(Sec. 2)</font></div></div></div></div></foreignObject><text x="205" y="199" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Background...</text></switch></g><rect x="1" y="135.57" width="135" height="30" rx="4.5" ry="4.5" fill="#ffffff" stroke="#0000ff" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 133px; height: 1px; padding-top: 151px; margin-left: 2px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">Differential Equations <font style="font-size: 9px">(Sec. 2.1)</font></div></div></div></foreignObject><text x="69" y="154" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Differential Equations...</text></switch></g><rect x="1" y="210.57" width="135" height="30" rx="4.5" ry="4.5" fill="#ffffff" stroke="#0000ff" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 133px; height: 1px; padding-top: 226px; margin-left: 2px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Neural Networks</div><div><font style="font-size: 9px">(Sec. 2.2)</font><br /></div></div></div></div></foreignObject><text x="69" y="229" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Neural Networks...</text></switch></g><path d="M 159 205.57 L 135.32 221.85" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 258.55 100 L 286 80" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 195.89 100 L 170.5 80" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><rect x="176" y="100" width="100" height="47.43" rx="7.11" ry="7.11" fill="#ffffff" stroke="#ff8000" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 124px; margin-left: 177px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Direct-Solution</div><div>Models<br /></div><div><font style="font-size: 9px">(Sec. 3)</font></div></div></div></div></foreignObject><text x="226" y="127" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Direct-Solution...</text></switch></g><rect x="236" y="30" width="100" height="50" rx="7.5" ry="7.5" fill="#ffffff" stroke="#ff8000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 55px; margin-left: 237px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Hidden Physics Networks</div><div><font style="font-size: 9px">(Sec. 3.5)</font><br /></div></div></div></div></foreignObject><text x="286" y="59" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Hidden Physics N...</text></switch></g><path d="M 419 110 L 419 84.43" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 469 122.31 L 511 111.64" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><rect x="369" y="110" width="100" height="50" rx="7.5" ry="7.5" fill="#ffffff" stroke="#00cc00" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 135px; margin-left: 370px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Time-Stepper</div><div>Models<br /></div><div><font style="font-size: 9px">(Sec. 4)</font></div></div></div></div></foreignObject><text x="419" y="139" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Time-Stepper...</text></switch></g><rect x="116" y="30" width="109" height="50" rx="7.5" ry="7.5" fill="#ffffff" stroke="#ff8000" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 107px; height: 1px; padding-top: 55px; margin-left: 117px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;"><div>Physics Informed Neural Networks</div><div><font style="font-size: 9px">(Sec. 3.1-4)</font><br /></div></div></div></div></foreignObject><text x="171" y="59" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Physics Informed N...</text></switch></g><path d="M 419 52.28 L 419 30" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><rect x="356.5" y="52.28" width="125" height="32.15" rx="4.82" ry="4.82" fill="#ffffff" stroke="#00cc00" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 123px; height: 1px; padding-top: 68px; margin-left: 358px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Integration Schemes <br /></div><div><font style="font-size: 9px">(Sec. 4.2)</font><br /></div></div></div></div></foreignObject><text x="419" y="72" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Integration Schemes...</text></switch></g><rect x="369" y="0" width="100" height="30" rx="4.5" ry="4.5" fill="#ffffff" stroke="#00cc00" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 15px; margin-left: 370px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;"><div><span>Neural ODEs<br /></span></div><div><span><font style="font-size: 9px">(Sec. 4.2.1-4.2.4)</font><br /></span></div></div></div></div></foreignObject><text x="419" y="19" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Neural ODEs...</text></switch></g><path d="M 533.97 198.08 L 610.91 266.22" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 514.5 197.29 L 686 150" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 597.81 197.61 L 613.7 217.42" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><rect x="485" y="165.57" width="118" height="31.72" rx="4.76" ry="4.76" fill="#ffffff" stroke="#00cc00" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 116px; height: 1px; padding-top: 181px; margin-left: 486px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Network Architecture</div><div><font style="font-size: 9px">(Sec. 4.4)</font><br /></div></div></div></div></foreignObject><text x="544" y="185" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Network Architectur...</text></switch></g><path d="M 551 94.43 L 586 35.54" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 591 120.25 L 636 106.64" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><rect x="511" y="94.43" width="80" height="34.43" rx="5.16" ry="5.16" fill="#ffffff" stroke="#00cc00" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 78px; height: 1px; padding-top: 112px; margin-left: 512px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>External Input</div><div><font style="font-size: 9px">(Sec. 4.3)</font><br /></div></div></div></div></foreignObject><text x="551" y="115" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">External Inpu...</text></switch></g><rect x="586" y="24.43" width="100" height="44.43" rx="6.66" ry="6.66" fill="#ffffff" stroke="#00cc00" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 47px; margin-left: 587px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Neural State-Space Models</div><div><font style="font-size: 9px">(Sec. 4.3.1)</font><br /></div></div></div></div></foreignObject><text x="636" y="50" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Neural State-Spa...</text></switch></g><rect x="636" y="84.43" width="100" height="44.43" rx="6.66" ry="6.66" fill="#ffffff" stroke="#00cc00" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 107px; margin-left: 637px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Neural ODEs</div><div>with input<br /></div><div><font style="font-size: 9px">(Sec. 4.3.2-3)</font><br /></div></div></div></div></foreignObject><text x="686" y="110" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Neural ODEs...</text></switch></g><rect x="586" y="266.22" width="100" height="44.43" rx="6.66" ry="6.66" fill="#ffffff" stroke="#00cc00" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 288px; margin-left: 587px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Graph Neural Networks</div><div><font style="font-size: 9px">(Sec. 4.4.3)</font><br /></div></div></div></div></foreignObject><text x="636" y="292" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Graph Neural Net...</text></switch></g><rect x="636" y="150" width="100" height="44.29" rx="6.64" ry="6.64" fill="#ffffff" stroke="#00cc00" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 172px; margin-left: 637px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Hamiltonian/ Lagrangian NN<br /></div><font style="font-size: 9px">(Sec 4.4.1)</font></div></div></div></foreignObject><text x="686" y="176" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Hamiltonian/ Lag...</text></switch></g><rect x="615" y="210.57" width="100" height="44.79" rx="6.72" ry="6.72" fill="#ffffff" stroke="#00cc00" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 233px; margin-left: 616px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Deep Potential NNs</div><div><font style="font-size: 9px">(Sec. 4.4.2)</font><br /></div></div></div></div></foreignObject><text x="665" y="237" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Deep Potential N...</text></switch></g><path d="M 417.39 194.29 L 418.32 160" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><rect x="367" y="194.29" width="100" height="28.86" rx="4.33" ry="4.33" fill="#ffffff" stroke="#00cc00" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 209px; margin-left: 368px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">Uncertainty<div><font style="font-size: 9px">(Sec. 4.5)</font><br /></div></div></div></div></foreignObject><text x="417" y="212" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Uncertainty...</text></switch></g><path d="M 336 244.71 L 384.52 223.15" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><rect x="266" y="238.04" width="70" height="44.43" rx="6.66" ry="6.66" fill="#ffffff" stroke="#00cc00" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 68px; height: 1px; padding-top: 260px; margin-left: 267px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 11px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div style="font-size: 11px">Deep Markov Models<br style="font-size: 11px" /></div><div style="font-size: 11px"><font style="font-size: 11px">(Sec. 4.5.1)</font><br style="font-size: 11px" /></div></div></div></div></foreignObject><text x="301" y="264" fill="#000000" font-family="Helvetica" font-size="11px" text-anchor="middle">Deep Markov...</text></switch></g><path d="M 443.54 293.43 L 421.52 223.15" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><rect x="416" y="293.43" width="69" height="44.43" rx="6.66" ry="6.66" fill="#ffffff" stroke="#00cc00" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 67px; height: 1px; padding-top: 316px; margin-left: 417px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 11px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div style="font-size: 11px">Baysian Neural ODEs<br style="font-size: 11px" /></div><div style="font-size: 11px"><font style="font-size: 11px">(Sec. 4.5.3)</font><br style="font-size: 11px" /></div></div></div></div></foreignObject><text x="451" y="319" fill="#000000" font-family="Helvetica" font-size="11px" text-anchor="middle">Baysian Neur...</text></switch></g><path d="M 481.5 236.87 L 450.06 223.15" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="stroke"/><rect x="481.5" y="230" width="68" height="43.43" rx="6.51" ry="6.51" fill="#ffffff" stroke="#00cc00" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 66px; height: 1px; padding-top: 252px; margin-left: 483px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div>Neural SDEs<br /></div><div><font style="font-size: 9px">(Sec. 4.5.4)</font><br /></div></div></div></div></foreignObject><text x="516" y="255" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle">Neural SDEs...</text></switch></g><rect x="49" y="273.43" width="110" height="30" rx="4.5" ry="4.5" fill="#ffffff" stroke="#0000ff" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 108px; height: 1px; padding-top: 288px; margin-left: 50px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;"><div>Model Taxonomy<br /></div><div><font style="font-size: 9px">(Sec. 2.3)</font><br /></div></div></div></div></foreignObject><text x="104" y="292" fill="#000000" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Model Taxonomy...</text></switch></g><rect x="315" y="293.43" width="70" height="44.43" rx="6.66" ry="6.66" fill="#ffffff" stroke="#00cc00" pointer-events="all"/><g transform="translate(-0.5 -0.5)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 68px; height: 1px; padding-top: 316px; margin-left: 316px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;"><div style="display: inline-block; font-size: 11px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><div style="font-size: 11px">Latent Neural ODEs</div><div style="font-size: 11px">(Sec. 4.5.2)<br style="font-size: 11px" /></div></div></div></div></foreignObject><text x="350" y="319" fill="#000000" font-family="Helvetica" font-size="11px" text-anchor="middle">Latent Neura...</text></switch></g></g><switch><g requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"/><a transform="translate(0,-5)" xlink:href="https://www.diagrams.net/doc/faq/svg-export-text-problems" target="_blank"><text text-anchor="middle" font-size="10px" x="50%" y="100%">Viewer does not support full SVG 1.1</text></a></switch></svg>


--------------------------------------------------------------------------------
/docs/time_vs_state_plot.svg:
--------------------------------------------------------------------------------
   1 | <?xml version="1.0" encoding="utf-8" standalone="no"?>
   2 | <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
   3 |   "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
   4 | <svg height="345.6pt" version="1.1" viewBox="0 0 460.8 345.6" width="460.8pt" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
   5 |  <metadata>
   6 |   <rdf:RDF xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
   7 |    <cc:Work>
   8 |     <dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
   9 |     <dc:date>2022-11-19T11:39:59.949724</dc:date>
  10 |     <dc:format>image/svg+xml</dc:format>
  11 |     <dc:creator>
  12 |      <cc:Agent>
  13 |       <dc:title>Matplotlib v3.4.3, https://matplotlib.org/</dc:title>
  14 |      </cc:Agent>
  15 |     </dc:creator>
  16 |    </cc:Work>
  17 |   </rdf:RDF>
  18 |  </metadata>
  19 |  <defs>
  20 |   <style type="text/css">*{stroke-linecap:butt;stroke-linejoin:round;}</style>
  21 |  </defs>
  22 |  <g id="figure_1">
  23 |   <g id="patch_1">
  24 |    <path d="M 0 345.6 
  25 | L 460.8 345.6 
  26 | L 460.8 0 
  27 | L 0 0 
  28 | z
  29 | " style="fill:#ffffff;"/>
  30 |   </g>
  31 |   <g id="axes_1">
  32 |    <g id="patch_2">
  33 |     <path d="M 57.6 162.432 
  34 | L 414.72 162.432 
  35 | L 414.72 41.472 
  36 | L 57.6 41.472 
  37 | z
  38 | " style="fill:#ffffff;"/>
  39 |    </g>
  40 |    <g id="matplotlib.axis_1">
  41 |     <g id="xtick_1">
  42 |      <g id="line2d_1">
  43 |       <defs>
  44 |        <path d="M 0 0 
  45 | L 0 3.5 
  46 | " id="mb88277fd6a" style="stroke:#000000;stroke-width:0.8;"/>
  47 |       </defs>
  48 |       <g>
  49 |        <use style="stroke:#000000;stroke-width:0.8;" x="73.832727" xlink:href="#mb88277fd6a" y="162.432"/>
  50 |       </g>
  51 |      </g>
  52 |     </g>
  53 |     <g id="xtick_2">
  54 |      <g id="line2d_2">
  55 |       <g>
  56 |        <use style="stroke:#000000;stroke-width:0.8;" x="125.504627" xlink:href="#mb88277fd6a" y="162.432"/>
  57 |       </g>
  58 |      </g>
  59 |     </g>
  60 |     <g id="xtick_3">
  61 |      <g id="line2d_3">
  62 |       <g>
  63 |        <use style="stroke:#000000;stroke-width:0.8;" x="177.176527" xlink:href="#mb88277fd6a" y="162.432"/>
  64 |       </g>
  65 |      </g>
  66 |     </g>
  67 |     <g id="xtick_4">
  68 |      <g id="line2d_4">
  69 |       <g>
  70 |        <use style="stroke:#000000;stroke-width:0.8;" x="228.848426" xlink:href="#mb88277fd6a" y="162.432"/>
  71 |       </g>
  72 |      </g>
  73 |     </g>
  74 |     <g id="xtick_5">
  75 |      <g id="line2d_5">
  76 |       <g>
  77 |        <use style="stroke:#000000;stroke-width:0.8;" x="280.520326" xlink:href="#mb88277fd6a" y="162.432"/>
  78 |       </g>
  79 |      </g>
  80 |     </g>
  81 |     <g id="xtick_6">
  82 |      <g id="line2d_6">
  83 |       <g>
  84 |        <use style="stroke:#000000;stroke-width:0.8;" x="332.192226" xlink:href="#mb88277fd6a" y="162.432"/>
  85 |       </g>
  86 |      </g>
  87 |     </g>
  88 |     <g id="xtick_7">
  89 |      <g id="line2d_7">
  90 |       <g>
  91 |        <use style="stroke:#000000;stroke-width:0.8;" x="383.864126" xlink:href="#mb88277fd6a" y="162.432"/>
  92 |       </g>
  93 |      </g>
  94 |     </g>
  95 |    </g>
  96 |    <g id="matplotlib.axis_2">
  97 |     <g id="ytick_1">
  98 |      <g id="line2d_8">
  99 |       <defs>
 100 |        <path d="M 0 0 
 101 | L -3.5 0 
 102 | " id="m6a8ec1dfe5" style="stroke:#000000;stroke-width:0.8;"/>
 103 |       </defs>
 104 |       <g>
 105 |        <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m6a8ec1dfe5" y="133.609051"/>
 106 |       </g>
 107 |      </g>
 108 |      <g id="text_1">
 109 |       <!-- −0.5 -->
 110 |       <g transform="translate(26.317187 137.40827)scale(0.1 -0.1)">
 111 |        <defs>
 112 |         <path d="M 678 2272 
 113 | L 4684 2272 
 114 | L 4684 1741 
 115 | L 678 1741 
 116 | L 678 2272 
 117 | z
 118 | " id="DejaVuSans-2212" transform="scale(0.015625)"/>
 119 |         <path d="M 2034 4250 
 120 | Q 1547 4250 1301 3770 
 121 | Q 1056 3291 1056 2328 
 122 | Q 1056 1369 1301 889 
 123 | Q 1547 409 2034 409 
 124 | Q 2525 409 2770 889 
 125 | Q 3016 1369 3016 2328 
 126 | Q 3016 3291 2770 3770 
 127 | Q 2525 4250 2034 4250 
 128 | z
 129 | M 2034 4750 
 130 | Q 2819 4750 3233 4129 
 131 | Q 3647 3509 3647 2328 
 132 | Q 3647 1150 3233 529 
 133 | Q 2819 -91 2034 -91 
 134 | Q 1250 -91 836 529 
 135 | Q 422 1150 422 2328 
 136 | Q 422 3509 836 4129 
 137 | Q 1250 4750 2034 4750 
 138 | z
 139 | " id="DejaVuSans-30" transform="scale(0.015625)"/>
 140 |         <path d="M 684 794 
 141 | L 1344 794 
 142 | L 1344 0 
 143 | L 684 0 
 144 | L 684 794 
 145 | z
 146 | " id="DejaVuSans-2e" transform="scale(0.015625)"/>
 147 |         <path d="M 691 4666 
 148 | L 3169 4666 
 149 | L 3169 4134 
 150 | L 1269 4134 
 151 | L 1269 2991 
 152 | Q 1406 3038 1543 3061 
 153 | Q 1681 3084 1819 3084 
 154 | Q 2600 3084 3056 2656 
 155 | Q 3513 2228 3513 1497 
 156 | Q 3513 744 3044 326 
 157 | Q 2575 -91 1722 -91 
 158 | Q 1428 -91 1123 -41 
 159 | Q 819 9 494 109 
 160 | L 494 744 
 161 | Q 775 591 1075 516 
 162 | Q 1375 441 1709 441 
 163 | Q 2250 441 2565 725 
 164 | Q 2881 1009 2881 1497 
 165 | Q 2881 1984 2565 2268 
 166 | Q 2250 2553 1709 2553 
 167 | Q 1456 2553 1204 2497 
 168 | Q 953 2441 691 2322 
 169 | L 691 4666 
 170 | z
 171 | " id="DejaVuSans-35" transform="scale(0.015625)"/>
 172 |        </defs>
 173 |        <use xlink:href="#DejaVuSans-2212"/>
 174 |        <use x="83.789062" xlink:href="#DejaVuSans-30"/>
 175 |        <use x="147.412109" xlink:href="#DejaVuSans-2e"/>
 176 |        <use x="179.199219" xlink:href="#DejaVuSans-35"/>
 177 |       </g>
 178 |      </g>
 179 |     </g>
 180 |     <g id="ytick_2">
 181 |      <g id="line2d_9">
 182 |       <g>
 183 |        <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m6a8ec1dfe5" y="101.810034"/>
 184 |       </g>
 185 |      </g>
 186 |      <g id="text_2">
 187 |       <!-- 0.0 -->
 188 |       <g transform="translate(34.696875 105.609253)scale(0.1 -0.1)">
 189 |        <use xlink:href="#DejaVuSans-30"/>
 190 |        <use x="63.623047" xlink:href="#DejaVuSans-2e"/>
 191 |        <use x="95.410156" xlink:href="#DejaVuSans-30"/>
 192 |       </g>
 193 |      </g>
 194 |     </g>
 195 |     <g id="ytick_3">
 196 |      <g id="line2d_10">
 197 |       <g>
 198 |        <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m6a8ec1dfe5" y="70.011017"/>
 199 |       </g>
 200 |      </g>
 201 |      <g id="text_3">
 202 |       <!-- 0.5 -->
 203 |       <g transform="translate(34.696875 73.810236)scale(0.1 -0.1)">
 204 |        <use xlink:href="#DejaVuSans-30"/>
 205 |        <use x="63.623047" xlink:href="#DejaVuSans-2e"/>
 206 |        <use x="95.410156" xlink:href="#DejaVuSans-35"/>
 207 |       </g>
 208 |      </g>
 209 |     </g>
 210 |     <g id="text_4">
 211 |      <!-- θ(t) -->
 212 |      <g transform="translate(20.2375 110.873094)rotate(-90)scale(0.1 -0.1)">
 213 |       <defs>
 214 |        <path d="M 2953 2219 
 215 | L 963 2219 
 216 | Q 1009 1284 1225 888 
 217 | Q 1494 400 1959 400 
 218 | Q 2428 400 2688 891 
 219 | Q 2916 1322 2953 2219 
 220 | z
 221 | M 2944 2750 
 222 | Q 2856 3638 2688 3950 
 223 | Q 2419 4444 1959 4444 
 224 | Q 1481 4444 1228 3956 
 225 | Q 1028 3563 969 2750 
 226 | L 2944 2750 
 227 | z
 228 | M 1959 4913 
 229 | Q 2709 4913 3138 4250 
 230 | Q 3566 3591 3566 2422 
 231 | Q 3566 1256 3138 594 
 232 | Q 2709 -72 1959 -72 
 233 | Q 1206 -72 781 594 
 234 | Q 353 1256 353 2422 
 235 | Q 353 3591 781 4250 
 236 | Q 1206 4913 1959 4913 
 237 | z
 238 | " id="DejaVuSans-3b8" transform="scale(0.015625)"/>
 239 |        <path d="M 1984 4856 
 240 | Q 1566 4138 1362 3434 
 241 | Q 1159 2731 1159 2009 
 242 | Q 1159 1288 1364 580 
 243 | Q 1569 -128 1984 -844 
 244 | L 1484 -844 
 245 | Q 1016 -109 783 600 
 246 | Q 550 1309 550 2009 
 247 | Q 550 2706 781 3412 
 248 | Q 1013 4119 1484 4856 
 249 | L 1984 4856 
 250 | z
 251 | " id="DejaVuSans-28" transform="scale(0.015625)"/>
 252 |        <path d="M 1172 4494 
 253 | L 1172 3500 
 254 | L 2356 3500 
 255 | L 2356 3053 
 256 | L 1172 3053 
 257 | L 1172 1153 
 258 | Q 1172 725 1289 603 
 259 | Q 1406 481 1766 481 
 260 | L 2356 481 
 261 | L 2356 0 
 262 | L 1766 0 
 263 | Q 1100 0 847 248 
 264 | Q 594 497 594 1153 
 265 | L 594 3053 
 266 | L 172 3053 
 267 | L 172 3500 
 268 | L 594 3500 
 269 | L 594 4494 
 270 | L 1172 4494 
 271 | z
 272 | " id="DejaVuSans-74" transform="scale(0.015625)"/>
 273 |        <path d="M 513 4856 
 274 | L 1013 4856 
 275 | Q 1481 4119 1714 3412 
 276 | Q 1947 2706 1947 2009 
 277 | Q 1947 1309 1714 600 
 278 | Q 1481 -109 1013 -844 
 279 | L 513 -844 
 280 | Q 928 -128 1133 580 
 281 | Q 1338 1288 1338 2009 
 282 | Q 1338 2731 1133 3434 
 283 | Q 928 4138 513 4856 
 284 | z
 285 | " id="DejaVuSans-29" transform="scale(0.015625)"/>
 286 |       </defs>
 287 |       <use xlink:href="#DejaVuSans-3b8"/>
 288 |       <use x="61.181641" xlink:href="#DejaVuSans-28"/>
 289 |       <use x="100.195312" xlink:href="#DejaVuSans-74"/>
 290 |       <use x="139.404297" xlink:href="#DejaVuSans-29"/>
 291 |      </g>
 292 |     </g>
 293 |    </g>
 294 |    <g id="line2d_11">
 295 |     <path clip-path="url(#p24b09aa7a2)" d="M 73.832727 141.572893 
 296 | L 76.467994 137.749451 
 297 | L 79.335785 133.187687 
 298 | L 82.513606 127.702136 
 299 | L 86.13064 121.000205 
 300 | L 90.600258 112.222479 
 301 | L 99.07445 94.952919 
 302 | L 104.706689 83.765192 
 303 | L 108.685423 76.340351 
 304 | L 112.069934 70.479986 
 305 | L 115.092739 65.678987 
 306 | L 117.857186 61.696562 
 307 | L 120.414944 58.393187 
 308 | L 122.817689 55.647508 
 309 | L 125.065414 53.408981 
 310 | L 127.209799 51.582494 
 311 | L 129.250839 50.132805 
 312 | L 131.188537 49.022992 
 313 | L 133.074566 48.196183 
 314 | L 134.908909 47.634681 
 315 | L 136.691589 47.320067 
 316 | L 138.448442 47.233833 
 317 | L 140.205283 47.370051 
 318 | L 141.962129 47.728308 
 319 | L 143.74481 48.317672 
 320 | L 145.579159 49.159629 
 321 | L 147.465188 50.271384 
 322 | L 149.42872 51.689572 
 323 | L 151.495594 53.463534 
 324 | L 153.665816 55.627743 
 325 | L 155.939381 58.214207 
 326 | L 158.367957 61.32075 
 327 | L 160.97739 65.029993 
 328 | L 163.793508 69.429479 
 329 | L 166.893819 74.696415 
 330 | L 170.407512 81.118703 
 331 | L 174.644602 89.351234 
 332 | L 181.026088 102.304208 
 333 | L 188.725197 117.819822 
 334 | L 192.936447 125.798531 
 335 | L 196.424306 131.945069 
 336 | L 199.524617 136.968436 
 337 | L 202.340741 141.116681 
 338 | L 204.924334 144.538624 
 339 | L 207.352922 147.395076 
 340 | L 209.626481 149.73517 
 341 | L 211.796703 151.654885 
 342 | L 213.863577 153.188927 
 343 | L 215.827103 154.373957 
 344 | L 217.713126 155.257933 
 345 | L 219.547481 155.875608 
 346 | L 221.330168 156.245162 
 347 | L 223.087008 156.385706 
 348 | L 224.843861 156.3038 
 349 | L 226.600702 155.999693 
 350 | L 228.383376 155.46494 
 351 | L 230.217731 154.678632 
 352 | L 232.103754 153.623322 
 353 | L 234.06728 152.262828 
 354 | L 236.108332 150.571309 
 355 | L 238.25272 148.497801 
 356 | L 240.500433 146.009687 
 357 | L 242.903187 143.009689 
 358 | L 245.460933 139.452253 
 359 | L 248.225389 135.217591 
 360 | L 251.248185 130.171409 
 361 | L 254.658542 124.031587 
 362 | L 258.688942 116.295783 
 363 | L 264.217841 105.155825 
 364 | L 274.138841 85.12596 
 365 | L 278.272601 77.331781 
 366 | L 281.734614 71.265841 
 367 | L 284.809079 66.31756 
 368 | L 287.599381 62.237406 
 369 | L 290.182961 58.843837 
 370 | L 292.585715 56.043907 
 371 | L 294.859286 53.727846 
 372 | L 297.00365 51.852653 
 373 | L 299.044702 50.355353 
 374 | L 301.00824 49.185816 
 375 | L 302.894263 48.316951 
 376 | L 304.728618 47.714128 
 377 | L 306.51128 47.359109 
 378 | L 308.268121 47.232931 
 379 | L 310.024961 47.329206 
 380 | L 311.781826 47.647649 
 381 | L 313.564513 48.196865 
 382 | L 315.398844 48.997901 
 383 | L 317.284891 50.068177 
 384 | L 319.248429 51.443972 
 385 | L 321.289457 53.150991 
 386 | L 323.433845 55.240306 
 387 | L 325.707417 57.774963 
 388 | L 328.110146 60.794752 
 389 | L 330.693726 64.40983 
 390 | L 333.484028 68.709075 
 391 | L 336.532671 73.825892 
 392 | L 339.968837 80.041053 
 393 | L 344.050942 87.905258 
 394 | L 349.786526 99.489078 
 395 | L 359.00992 118.111502 
 396 | L 363.169539 125.977321 
 397 | L 366.657373 132.110338 
 398 | L 369.731863 137.079658 
 399 | L 372.522165 141.181199 
 400 | L 375.105745 144.596281 
 401 | L 377.534321 147.445873 
 402 | L 379.807892 149.779264 
 403 | L 381.952256 151.671388 
 404 | L 383.993308 153.18524 
 405 | L 385.956846 154.371025 
 406 | L 387.842869 155.255731 
 407 | L 389.677224 155.874122 
 408 | L 391.459886 156.244373 
 409 | L 393.216727 156.385613 
 410 | L 394.973567 156.304405 
 411 | L 396.730432 156.000988 
 412 | L 398.487273 155.476293 
 413 | L 398.487273 155.476293 
 414 | " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-width:1.5;"/>
 415 |    </g>
 416 |    <g id="line2d_12">
 417 |     <path clip-path="url(#p24b09aa7a2)" d="M 73.832727 141.572893 
 418 | L 76.49383 137.715717 
 419 | L 79.387457 133.12043 
 420 | L 82.591114 127.605329 
 421 | L 86.259819 120.83325 
 422 | L 90.806946 111.945038 
 423 | L 100.314578 92.698988 
 424 | L 105.533438 82.476401 
 425 | L 109.434667 75.307443 
 426 | L 112.767503 69.63275 
 427 | L 115.764475 64.958567 
 428 | L 118.503087 61.090818 
 429 | L 121.035008 57.890807 
 430 | L 123.411913 55.238605 
 431 | L 125.63381 53.083956 
 432 | L 127.752352 51.333532 
 433 | L 129.767564 49.952453 
 434 | L 131.705255 48.891782 
 435 | L 133.565444 48.124391 
 436 | L 135.373959 47.616849 
 437 | L 137.130806 47.350811 
 438 | L 138.887652 47.309417 
 439 | L 140.644499 47.492775 
 440 | L 142.401339 47.900183 
 441 | L 144.209854 48.551995 
 442 | L 146.044209 49.451362 
 443 | L 147.956072 50.640133 
 444 | L 149.945438 52.143863 
 445 | L 152.038146 54.011387 
 446 | L 154.234203 56.275154 
 447 | L 156.559442 58.996409 
 448 | L 159.039692 62.246056 
 449 | L 161.726634 66.14145 
 450 | L 164.646094 70.774266 
 451 | L 167.901422 76.369523 
 452 | L 171.647634 83.269243 
 453 | L 176.272272 92.27969 
 454 | L 193.168979 125.668052 
 455 | L 196.786008 132.053754 
 456 | L 199.938 137.180205 
 457 | L 202.779958 141.390003 
 458 | L 205.389384 144.868604 
 459 | L 207.81796 147.743641 
 460 | L 210.091519 150.097165 
 461 | L 212.23592 152.002185 
 462 | L 214.251113 153.503127 
 463 | L 216.188817 154.674326 
 464 | L 218.049006 155.542395 
 465 | L 219.857515 156.142008 
 466 | L 221.614355 156.491541 
 467 | L 223.345361 156.610218 
 468 | L 225.07638 156.504607 
 469 | L 226.807386 156.17535 
 470 | L 228.564239 155.614041 
 471 | L 230.372748 154.799801 
 472 | L 232.232925 153.715878 
 473 | L 234.170628 152.326414 
 474 | L 236.21168 150.582999 
 475 | L 238.356057 148.452246 
 476 | L 240.629616 145.872323 
 477 | L 243.058204 142.770839 
 478 | L 245.667618 139.066975 
 479 | L 248.483754 134.674909 
 480 | L 251.609899 129.373761 
 481 | L 255.149414 122.915879 
 482 | L 259.386523 114.698631 
 483 | L 265.638813 102.025471 
 484 | L 273.777139 85.614114 
 485 | L 278.014236 77.583331 
 486 | L 281.527917 71.387011 
 487 | L 284.628252 66.358376 
 488 | L 287.444352 62.203603 
 489 | L 290.053803 58.742627 
 490 | L 292.482379 55.884167 
 491 | L 294.755925 53.543707 
 492 | L 296.900314 51.646856 
 493 | L 298.941366 50.131116 
 494 | L 300.904904 48.946571 
 495 | L 302.790902 48.066418 
 496 | L 304.599435 47.462884 
 497 | L 306.382122 47.100605 
 498 | L 308.138963 46.970806 
 499 | L 309.869957 47.063955 
 500 | L 311.626797 47.382458 
 501 | L 313.409509 47.93506 
 502 | L 315.218017 48.729616 
 503 | L 317.104041 49.805846 
 504 | L 319.067554 51.189944 
 505 | L 321.108606 52.906857 
 506 | L 323.25297 55.006528 
 507 | L 325.526541 57.55057 
 508 | L 327.955117 60.610977 
 509 | L 330.564568 64.26825 
 510 | L 333.380667 68.607605 
 511 | L 336.480978 73.803359 
 512 | L 339.994659 80.140675 
 513 | L 344.1801 88.171026 
 514 | L 350.044867 99.95475 
 515 | L 359.552497 119.054533 
 516 | L 363.789581 127.032946 
 517 | L 367.303287 133.189532 
 518 | L 370.403598 138.183953 
 519 | L 373.193875 142.269584 
 520 | L 375.77748 145.666626 
 521 | L 378.18021 148.464414 
 522 | L 380.427959 150.746192 
 523 | L 382.546501 152.585458 
 524 | L 384.561707 154.044304 
 525 | L 386.473552 155.159095 
 526 | L 388.307907 155.977314 
 527 | L 390.090594 156.53328 
 528 | L 391.847434 156.848423 
 529 | L 393.578453 156.93192 
 530 | L 395.309447 156.789994 
 531 | L 397.040441 156.423437 
 532 | L 398.487273 155.94588 
 533 | L 398.487273 155.94588 
 534 | " style="fill:none;stroke:#0000ff;stroke-dasharray:5.55,2.4;stroke-dashoffset:0;stroke-width:1.5;"/>
 535 |    </g>
 536 |    <g id="patch_3">
 537 |     <path d="M 57.6 162.432 
 538 | L 57.6 41.472 
 539 | " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
 540 |    </g>
 541 |    <g id="patch_4">
 542 |     <path d="M 414.72 162.432 
 543 | L 414.72 41.472 
 544 | " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
 545 |    </g>
 546 |    <g id="patch_5">
 547 |     <path d="M 57.6 162.432 
 548 | L 414.72 162.432 
 549 | " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
 550 |    </g>
 551 |    <g id="patch_6">
 552 |     <path d="M 57.6 41.472 
 553 | L 414.72 41.472 
 554 | " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
 555 |    </g>
 556 |   </g>
 557 |   <g id="axes_2">
 558 |    <g id="patch_7">
 559 |     <path d="M 57.6 307.584 
 560 | L 414.72 307.584 
 561 | L 414.72 186.624 
 562 | L 57.6 186.624 
 563 | z
 564 | " style="fill:#ffffff;"/>
 565 |    </g>
 566 |    <g id="matplotlib.axis_3">
 567 |     <g id="xtick_8">
 568 |      <g id="line2d_13">
 569 |       <g>
 570 |        <use style="stroke:#000000;stroke-width:0.8;" x="73.832727" xlink:href="#mb88277fd6a" y="307.584"/>
 571 |       </g>
 572 |      </g>
 573 |      <g id="text_5">
 574 |       <!-- 0 -->
 575 |       <g transform="translate(70.651477 322.182437)scale(0.1 -0.1)">
 576 |        <use xlink:href="#DejaVuSans-30"/>
 577 |       </g>
 578 |      </g>
 579 |     </g>
 580 |     <g id="xtick_9">
 581 |      <g id="line2d_14">
 582 |       <g>
 583 |        <use style="stroke:#000000;stroke-width:0.8;" x="125.504627" xlink:href="#mb88277fd6a" y="307.584"/>
 584 |       </g>
 585 |      </g>
 586 |      <g id="text_6">
 587 |       <!-- 2 -->
 588 |       <g transform="translate(122.323377 322.182437)scale(0.1 -0.1)">
 589 |        <defs>
 590 |         <path d="M 1228 531 
 591 | L 3431 531 
 592 | L 3431 0 
 593 | L 469 0 
 594 | L 469 531 
 595 | Q 828 903 1448 1529 
 596 | Q 2069 2156 2228 2338 
 597 | Q 2531 2678 2651 2914 
 598 | Q 2772 3150 2772 3378 
 599 | Q 2772 3750 2511 3984 
 600 | Q 2250 4219 1831 4219 
 601 | Q 1534 4219 1204 4116 
 602 | Q 875 4013 500 3803 
 603 | L 500 4441 
 604 | Q 881 4594 1212 4672 
 605 | Q 1544 4750 1819 4750 
 606 | Q 2544 4750 2975 4387 
 607 | Q 3406 4025 3406 3419 
 608 | Q 3406 3131 3298 2873 
 609 | Q 3191 2616 2906 2266 
 610 | Q 2828 2175 2409 1742 
 611 | Q 1991 1309 1228 531 
 612 | z
 613 | " id="DejaVuSans-32" transform="scale(0.015625)"/>
 614 |        </defs>
 615 |        <use xlink:href="#DejaVuSans-32"/>
 616 |       </g>
 617 |      </g>
 618 |     </g>
 619 |     <g id="xtick_10">
 620 |      <g id="line2d_15">
 621 |       <g>
 622 |        <use style="stroke:#000000;stroke-width:0.8;" x="177.176527" xlink:href="#mb88277fd6a" y="307.584"/>
 623 |       </g>
 624 |      </g>
 625 |      <g id="text_7">
 626 |       <!-- 4 -->
 627 |       <g transform="translate(173.995277 322.182437)scale(0.1 -0.1)">
 628 |        <defs>
 629 |         <path d="M 2419 4116 
 630 | L 825 1625 
 631 | L 2419 1625 
 632 | L 2419 4116 
 633 | z
 634 | M 2253 4666 
 635 | L 3047 4666 
 636 | L 3047 1625 
 637 | L 3713 1625 
 638 | L 3713 1100 
 639 | L 3047 1100 
 640 | L 3047 0 
 641 | L 2419 0 
 642 | L 2419 1100 
 643 | L 313 1100 
 644 | L 313 1709 
 645 | L 2253 4666 
 646 | z
 647 | " id="DejaVuSans-34" transform="scale(0.015625)"/>
 648 |        </defs>
 649 |        <use xlink:href="#DejaVuSans-34"/>
 650 |       </g>
 651 |      </g>
 652 |     </g>
 653 |     <g id="xtick_11">
 654 |      <g id="line2d_16">
 655 |       <g>
 656 |        <use style="stroke:#000000;stroke-width:0.8;" x="228.848426" xlink:href="#mb88277fd6a" y="307.584"/>
 657 |       </g>
 658 |      </g>
 659 |      <g id="text_8">
 660 |       <!-- 6 -->
 661 |       <g transform="translate(225.667176 322.182437)scale(0.1 -0.1)">
 662 |        <defs>
 663 |         <path d="M 2113 2584 
 664 | Q 1688 2584 1439 2293 
 665 | Q 1191 2003 1191 1497 
 666 | Q 1191 994 1439 701 
 667 | Q 1688 409 2113 409 
 668 | Q 2538 409 2786 701 
 669 | Q 3034 994 3034 1497 
 670 | Q 3034 2003 2786 2293 
 671 | Q 2538 2584 2113 2584 
 672 | z
 673 | M 3366 4563 
 674 | L 3366 3988 
 675 | Q 3128 4100 2886 4159 
 676 | Q 2644 4219 2406 4219 
 677 | Q 1781 4219 1451 3797 
 678 | Q 1122 3375 1075 2522 
 679 | Q 1259 2794 1537 2939 
 680 | Q 1816 3084 2150 3084 
 681 | Q 2853 3084 3261 2657 
 682 | Q 3669 2231 3669 1497 
 683 | Q 3669 778 3244 343 
 684 | Q 2819 -91 2113 -91 
 685 | Q 1303 -91 875 529 
 686 | Q 447 1150 447 2328 
 687 | Q 447 3434 972 4092 
 688 | Q 1497 4750 2381 4750 
 689 | Q 2619 4750 2861 4703 
 690 | Q 3103 4656 3366 4563 
 691 | z
 692 | " id="DejaVuSans-36" transform="scale(0.015625)"/>
 693 |        </defs>
 694 |        <use xlink:href="#DejaVuSans-36"/>
 695 |       </g>
 696 |      </g>
 697 |     </g>
 698 |     <g id="xtick_12">
 699 |      <g id="line2d_17">
 700 |       <g>
 701 |        <use style="stroke:#000000;stroke-width:0.8;" x="280.520326" xlink:href="#mb88277fd6a" y="307.584"/>
 702 |       </g>
 703 |      </g>
 704 |      <g id="text_9">
 705 |       <!-- 8 -->
 706 |       <g transform="translate(277.339076 322.182437)scale(0.1 -0.1)">
 707 |        <defs>
 708 |         <path d="M 2034 2216 
 709 | Q 1584 2216 1326 1975 
 710 | Q 1069 1734 1069 1313 
 711 | Q 1069 891 1326 650 
 712 | Q 1584 409 2034 409 
 713 | Q 2484 409 2743 651 
 714 | Q 3003 894 3003 1313 
 715 | Q 3003 1734 2745 1975 
 716 | Q 2488 2216 2034 2216 
 717 | z
 718 | M 1403 2484 
 719 | Q 997 2584 770 2862 
 720 | Q 544 3141 544 3541 
 721 | Q 544 4100 942 4425 
 722 | Q 1341 4750 2034 4750 
 723 | Q 2731 4750 3128 4425 
 724 | Q 3525 4100 3525 3541 
 725 | Q 3525 3141 3298 2862 
 726 | Q 3072 2584 2669 2484 
 727 | Q 3125 2378 3379 2068 
 728 | Q 3634 1759 3634 1313 
 729 | Q 3634 634 3220 271 
 730 | Q 2806 -91 2034 -91 
 731 | Q 1263 -91 848 271 
 732 | Q 434 634 434 1313 
 733 | Q 434 1759 690 2068 
 734 | Q 947 2378 1403 2484 
 735 | z
 736 | M 1172 3481 
 737 | Q 1172 3119 1398 2916 
 738 | Q 1625 2713 2034 2713 
 739 | Q 2441 2713 2670 2916 
 740 | Q 2900 3119 2900 3481 
 741 | Q 2900 3844 2670 4047 
 742 | Q 2441 4250 2034 4250 
 743 | Q 1625 4250 1398 4047 
 744 | Q 1172 3844 1172 3481 
 745 | z
 746 | " id="DejaVuSans-38" transform="scale(0.015625)"/>
 747 |        </defs>
 748 |        <use xlink:href="#DejaVuSans-38"/>
 749 |       </g>
 750 |      </g>
 751 |     </g>
 752 |     <g id="xtick_13">
 753 |      <g id="line2d_18">
 754 |       <g>
 755 |        <use style="stroke:#000000;stroke-width:0.8;" x="332.192226" xlink:href="#mb88277fd6a" y="307.584"/>
 756 |       </g>
 757 |      </g>
 758 |      <g id="text_10">
 759 |       <!-- 10 -->
 760 |       <g transform="translate(325.829726 322.182437)scale(0.1 -0.1)">
 761 |        <defs>
 762 |         <path d="M 794 531 
 763 | L 1825 531 
 764 | L 1825 4091 
 765 | L 703 3866 
 766 | L 703 4441 
 767 | L 1819 4666 
 768 | L 2450 4666 
 769 | L 2450 531 
 770 | L 3481 531 
 771 | L 3481 0 
 772 | L 794 0 
 773 | L 794 531 
 774 | z
 775 | " id="DejaVuSans-31" transform="scale(0.015625)"/>
 776 |        </defs>
 777 |        <use xlink:href="#DejaVuSans-31"/>
 778 |        <use x="63.623047" xlink:href="#DejaVuSans-30"/>
 779 |       </g>
 780 |      </g>
 781 |     </g>
 782 |     <g id="xtick_14">
 783 |      <g id="line2d_19">
 784 |       <g>
 785 |        <use style="stroke:#000000;stroke-width:0.8;" x="383.864126" xlink:href="#mb88277fd6a" y="307.584"/>
 786 |       </g>
 787 |      </g>
 788 |      <g id="text_11">
 789 |       <!-- 12 -->
 790 |       <g transform="translate(377.501626 322.182437)scale(0.1 -0.1)">
 791 |        <use xlink:href="#DejaVuSans-31"/>
 792 |        <use x="63.623047" xlink:href="#DejaVuSans-32"/>
 793 |       </g>
 794 |      </g>
 795 |     </g>
 796 |     <g id="text_12">
 797 |      <!-- t -->
 798 |      <g transform="translate(234.199844 335.860562)scale(0.1 -0.1)">
 799 |       <use xlink:href="#DejaVuSans-74"/>
 800 |      </g>
 801 |     </g>
 802 |    </g>
 803 |    <g id="matplotlib.axis_4">
 804 |     <g id="ytick_4">
 805 |      <g id="line2d_20">
 806 |       <g>
 807 |        <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m6a8ec1dfe5" y="280.143048"/>
 808 |       </g>
 809 |      </g>
 810 |      <g id="text_13">
 811 |       <!-- −0.5 -->
 812 |       <g transform="translate(26.317187 283.942267)scale(0.1 -0.1)">
 813 |        <use xlink:href="#DejaVuSans-2212"/>
 814 |        <use x="83.789062" xlink:href="#DejaVuSans-30"/>
 815 |        <use x="147.412109" xlink:href="#DejaVuSans-2e"/>
 816 |        <use x="179.199219" xlink:href="#DejaVuSans-35"/>
 817 |       </g>
 818 |      </g>
 819 |     </g>
 820 |     <g id="ytick_5">
 821 |      <g id="line2d_21">
 822 |       <g>
 823 |        <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m6a8ec1dfe5" y="247.104"/>
 824 |       </g>
 825 |      </g>
 826 |      <g id="text_14">
 827 |       <!-- 0.0 -->
 828 |       <g transform="translate(34.696875 250.903219)scale(0.1 -0.1)">
 829 |        <use xlink:href="#DejaVuSans-30"/>
 830 |        <use x="63.623047" xlink:href="#DejaVuSans-2e"/>
 831 |        <use x="95.410156" xlink:href="#DejaVuSans-30"/>
 832 |       </g>
 833 |      </g>
 834 |     </g>
 835 |     <g id="ytick_6">
 836 |      <g id="line2d_22">
 837 |       <g>
 838 |        <use style="stroke:#000000;stroke-width:0.8;" x="57.6" xlink:href="#m6a8ec1dfe5" y="214.064952"/>
 839 |       </g>
 840 |      </g>
 841 |      <g id="text_15">
 842 |       <!-- 0.5 -->
 843 |       <g transform="translate(34.696875 217.864171)scale(0.1 -0.1)">
 844 |        <use xlink:href="#DejaVuSans-30"/>
 845 |        <use x="63.623047" xlink:href="#DejaVuSans-2e"/>
 846 |        <use x="95.410156" xlink:href="#DejaVuSans-35"/>
 847 |       </g>
 848 |      </g>
 849 |     </g>
 850 |     <g id="text_16">
 851 |      <!-- ω(t) -->
 852 |      <g transform="translate(20.2375 257.152437)rotate(-90)scale(0.1 -0.1)">
 853 |       <defs>
 854 |        <path d="M 1719 -91 
 855 | Q 422 -91 422 1766 
 856 | Q 422 2500 906 3500 
 857 | L 1525 3500 
 858 | Q 1078 2500 1078 1750 
 859 | Q 1078 397 1772 397 
 860 | Q 2406 397 2406 2078 
 861 | L 2938 2078 
 862 | Q 2938 397 3572 397 
 863 | Q 4266 397 4266 1750 
 864 | Q 4266 2500 3819 3500 
 865 | L 4438 3500 
 866 | Q 4922 2500 4922 1766 
 867 | Q 4922 -91 3625 -91 
 868 | Q 2775 -91 2672 844 
 869 | Q 2544 -91 1719 -91 
 870 | z
 871 | " id="DejaVuSans-3c9" transform="scale(0.015625)"/>
 872 |       </defs>
 873 |       <use xlink:href="#DejaVuSans-3c9"/>
 874 |       <use x="83.740234" xlink:href="#DejaVuSans-28"/>
 875 |       <use x="122.753906" xlink:href="#DejaVuSans-74"/>
 876 |       <use x="161.962891" xlink:href="#DejaVuSans-29"/>
 877 |      </g>
 878 |     </g>
 879 |    </g>
 880 |    <g id="line2d_23">
 881 |     <path clip-path="url(#pf177800d0e)" d="M 73.832727 210.075911 
 882 | L 76.726354 205.942932 
 883 | L 79.335785 202.589462 
 884 | L 81.738528 199.854127 
 885 | L 83.960419 197.654553 
 886 | L 86.027296 195.914399 
 887 | L 87.964993 194.566347 
 888 | L 89.799344 193.553957 
 889 | L 91.556189 192.832667 
 890 | L 93.261361 192.370261 
 891 | L 94.914862 192.148691 
 892 | L 96.568363 192.152014 
 893 | L 98.221864 192.380207 
 894 | L 99.901202 192.840507 
 895 | L 101.606375 193.540206 
 896 | L 103.389055 194.516549 
 897 | L 105.249241 195.794118 
 898 | L 107.212773 197.417945 
 899 | L 109.305487 199.443944 
 900 | L 111.553212 201.937609 
 901 | L 113.981794 204.971934 
 902 | L 116.642898 208.66174 
 903 | L 119.588195 213.134989 
 904 | L 122.921031 218.612136 
 905 | L 126.822263 225.467464 
 906 | L 131.679421 234.476916 
 907 | L 139.662724 249.835049 
 908 | L 147.542696 264.828643 
 909 | L 152.193163 273.205567 
 910 | L 155.939381 279.513513 
 911 | L 159.168869 284.532427 
 912 | L 162.036661 288.592664 
 913 | L 164.646094 291.910781 
 914 | L 167.022995 294.582941 
 915 | L 169.219052 296.726937 
 916 | L 171.260092 298.419407 
 917 | L 173.197789 299.743475 
 918 | L 175.032144 300.732316 
 919 | L 176.788985 301.430438 
 920 | L 178.494151 301.869953 
 921 | L 180.147655 302.0691 
 922 | L 181.801159 302.043286 
 923 | L 183.454651 301.792692 
 924 | L 185.134002 301.309883 
 925 | L 186.865008 300.575033 
 926 | L 188.647695 299.572273 
 927 | L 190.507884 298.268109 
 928 | L 192.471409 296.617494 
 929 | L 194.564117 294.56459 
 930 | L 196.837689 292.013322 
 931 | L 199.292098 288.914717 
 932 | L 201.97904 285.154485 
 933 | L 204.976014 280.564628 
 934 | L 208.360512 274.961139 
 935 | L 212.339256 267.925924 
 936 | L 217.351424 258.584213 
 937 | L 226.755718 240.465398 
 938 | L 233.395557 227.93113 
 939 | L 237.865172 219.954317 
 940 | L 241.53387 213.846697 
 941 | L 244.711696 208.975203 
 942 | L 247.527807 205.051126 
 943 | L 250.085578 201.856225 
 944 | L 252.436651 199.265858 
 945 | L 254.606874 197.195944 
 946 | L 256.647914 195.550436 
 947 | L 258.559771 194.288256 
 948 | L 260.368292 193.353046 
 949 | L 262.125133 192.692682 
 950 | L 263.804471 192.294764 
 951 | L 265.457963 192.12859 
 952 | L 267.111467 192.187446 
 953 | L 268.764971 192.470927 
 954 | L 270.444297 192.986734 
 955 | L 272.175315 193.754958 
 956 | L 273.95799 194.791139 
 957 | L 275.844013 196.1493 
 958 | L 277.833385 197.861366 
 959 | L 279.951927 199.983066 
 960 | L 282.251345 202.610663 
 961 | L 284.731589 205.791786 
 962 | L 287.470198 209.678282 
 963 | L 290.518841 214.405241 
 964 | L 293.980878 220.198212 
 965 | L 298.062933 227.479258 
 966 | L 303.307633 237.317462 
 967 | L 320.850265 270.59928 
 968 | L 324.854806 277.511326 
 969 | L 328.239329 282.926165 
 970 | L 331.236303 287.312989 
 971 | L 333.923244 290.862126 
 972 | L 336.377642 293.744315 
 973 | L 338.625392 296.051445 
 974 | L 340.718087 297.890988 
 975 | L 342.6816 299.329879 
 976 | L 344.541802 300.424577 
 977 | L 346.324489 301.219887 
 978 | L 348.029661 301.742353 
 979 | L 349.708987 302.025434 
 980 | L 351.362491 302.077756 
 981 | L 353.015995 301.905058 
 982 | L 354.669499 301.508532 
 983 | L 356.374672 300.868078 
 984 | L 358.131512 299.967182 
 985 | L 359.965867 298.772776 
 986 | L 361.903558 297.240017 
 987 | L 363.970432 295.312879 
 988 | L 366.166514 292.954862 
 989 | L 368.543397 290.069681 
 990 | L 371.152823 286.541267 
 991 | L 374.02064 282.27916 
 992 | L 377.250134 277.069613 
 993 | L 380.996333 270.587644 
 994 | L 385.543476 262.254303 
 995 | L 392.079953 249.761921 
 996 | L 398.487273 237.403767 
 997 | L 398.487273 237.403767 
 998 | " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-width:1.5;"/>
 999 |    </g>
1000 |    <g id="line2d_24">
1001 |     <path clip-path="url(#pf177800d0e)" d="M 73.832727 210.075911 
1002 | L 76.62301 206.168798 
1003 | L 79.180769 202.958802 
1004 | L 81.557676 200.323098 
1005 | L 83.779568 198.183344 
1006 | L 85.87228 196.470499 
1007 | L 87.861649 195.126478 
1008 | L 89.747672 194.117069 
1009 | L 91.556189 193.397677 
1010 | L 93.313032 192.936391 
1011 | L 95.018206 192.7155 
1012 | L 96.723378 192.719687 
1013 | L 98.428552 192.949331 
1014 | L 100.133724 193.403548 
1015 | L 101.890571 194.10412 
1016 | L 103.699083 195.067914 
1017 | L 105.585109 196.329138 
1018 | L 107.574478 197.93399 
1019 | L 109.667189 199.914008 
1020 | L 111.889083 202.32673 
1021 | L 114.291824 205.272173 
1022 | L 116.901257 208.834261 
1023 | L 119.743212 213.099255 
1024 | L 122.921031 218.280662 
1025 | L 126.563898 224.660904 
1026 | L 130.981846 232.870006 
1027 | L 137.492507 245.495664 
1028 | L 146.483419 262.869548 
1029 | L 150.978875 271.05707 
1030 | L 154.673413 277.337666 
1031 | L 157.902907 282.401823 
1032 | L 160.822367 286.575077 
1033 | L 163.483474 290.000115 
1034 | L 165.93789 292.806985 
1035 | L 168.237289 295.106473 
1036 | L 170.381672 296.945855 
1037 | L 172.422712 298.410103 
1038 | L 174.360409 299.532017 
1039 | L 176.220598 300.356158 
1040 | L 178.003279 300.908391 
1041 | L 179.760132 301.221518 
1042 | L 181.491126 301.303289 
1043 | L 183.196298 301.162559 
1044 | L 184.927317 300.794883 
1045 | L 186.684157 300.191071 
1046 | L 188.492678 299.329032 
1047 | L 190.352867 298.191666 
1048 | L 192.290558 296.742364 
1049 | L 194.331598 294.932299 
1050 | L 196.475975 292.729228 
1051 | L 198.77538 290.040699 
1052 | L 201.22979 286.822042 
1053 | L 203.890897 282.958476 
1054 | L 206.810357 278.319846 
1055 | L 210.091519 272.677791 
1056 | L 213.915246 265.643837 
1057 | L 218.798231 256.164452 
1058 | L 234.222309 225.87184 
1059 | L 238.123538 218.897876 
1060 | L 241.482202 213.336926 
1061 | L 244.479177 208.793496 
1062 | L 247.217786 205.037547 
1063 | L 249.723876 201.967961 
1064 | L 252.049103 199.460642 
1065 | L 254.24516 197.413948 
1066 | L 256.312046 195.788006 
1067 | L 258.249738 194.539371 
1068 | L 260.109927 193.599898 
1069 | L 261.892601 192.943768 
1070 | L 263.623608 192.539477 
1071 | L 265.328792 192.368152 
1072 | L 267.00813 192.420892 
1073 | L 268.71329 192.699428 
1074 | L 270.418463 193.203515 
1075 | L 272.175315 193.956193 
1076 | L 273.983824 194.974057 
1077 | L 275.895694 196.311607 
1078 | L 277.885053 197.979316 
1079 | L 280.003608 200.051543 
1080 | L 282.251345 202.566073 
1081 | L 284.679921 205.623533 
1082 | L 287.315169 209.30708 
1083 | L 290.208807 213.742809 
1084 | L 293.464147 219.153613 
1085 | L 297.210347 225.827835 
1086 | L 301.835004 234.546676 
1087 | L 309.482433 249.52569 
1088 | L 316.664824 263.411532 
1089 | L 321.108606 271.525372 
1090 | L 324.777316 277.774483 
1091 | L 327.980963 282.806253 
1092 | L 330.874602 286.948923 
1093 | L 333.535696 290.377961 
1094 | L 335.990119 293.185537 
1095 | L 338.26369 295.459066 
1096 | L 340.408054 297.297864 
1097 | L 342.449106 298.759611 
1098 | L 344.386797 299.877351 
1099 | L 346.246974 300.695849 
1100 | L 348.029661 301.241192 
1101 | L 349.786526 301.546119 
1102 | L 351.491698 301.619063 
1103 | L 353.196846 301.471142 
1104 | L 354.927865 301.094921 
1105 | L 356.684705 300.481188 
1106 | L 358.467392 299.621862 
1107 | L 360.327593 298.474919 
1108 | L 362.26526 297.014277 
1109 | L 364.306312 295.190777 
1110 | L 366.450676 292.972101 
1111 | L 368.750094 290.265225 
1112 | L 371.204491 287.025388 
1113 | L 373.865611 283.137168 
1114 | L 376.810917 278.42683 
1115 | L 380.092055 272.747733 
1116 | L 383.94164 265.620795 
1117 | L 388.850484 256.032258 
1118 | L 398.487273 236.71741 
1119 | L 398.487273 236.71741 
1120 | " style="fill:none;stroke:#0000ff;stroke-dasharray:5.55,2.4;stroke-dashoffset:0;stroke-width:1.5;"/>
1121 |    </g>
1122 |    <g id="patch_8">
1123 |     <path d="M 57.6 307.584 
1124 | L 57.6 186.624 
1125 | " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
1126 |    </g>
1127 |    <g id="patch_9">
1128 |     <path d="M 414.72 307.584 
1129 | L 414.72 186.624 
1130 | " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
1131 |    </g>
1132 |    <g id="patch_10">
1133 |     <path d="M 57.6 307.584 
1134 | L 414.72 307.584 
1135 | " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
1136 |    </g>
1137 |    <g id="patch_11">
1138 |     <path d="M 57.6 186.624 
1139 | L 414.72 186.624 
1140 | " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/>
1141 |    </g>
1142 |    <g id="legend_1">
1143 |     <g id="patch_12">
1144 |      <path d="M 352.984063 223.98025 
1145 | L 407.72 223.98025 
1146 | Q 409.72 223.98025 409.72 221.98025 
1147 | L 409.72 193.624 
1148 | Q 409.72 191.624 407.72 191.624 
1149 | L 352.984063 191.624 
1150 | Q 350.984063 191.624 350.984063 193.624 
1151 | L 350.984063 221.98025 
1152 | Q 350.984063 223.98025 352.984063 223.98025 
1153 | z
1154 | " style="fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;"/>
1155 |     </g>
1156 |     <g id="line2d_25">
1157 |      <path d="M 354.984063 199.722437 
1158 | L 374.984063 199.722437 
1159 | " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-width:1.5;"/>
1160 |     </g>
1161 |     <g id="line2d_26"/>
1162 |     <g id="text_17">
1163 |      <!-- true -->
1164 |      <g transform="translate(382.984063 203.222437)scale(0.1 -0.1)">
1165 |       <defs>
1166 |        <path d="M 2631 2963 
1167 | Q 2534 3019 2420 3045 
1168 | Q 2306 3072 2169 3072 
1169 | Q 1681 3072 1420 2755 
1170 | Q 1159 2438 1159 1844 
1171 | L 1159 0 
1172 | L 581 0 
1173 | L 581 3500 
1174 | L 1159 3500 
1175 | L 1159 2956 
1176 | Q 1341 3275 1631 3429 
1177 | Q 1922 3584 2338 3584 
1178 | Q 2397 3584 2469 3576 
1179 | Q 2541 3569 2628 3553 
1180 | L 2631 2963 
1181 | z
1182 | " id="DejaVuSans-72" transform="scale(0.015625)"/>
1183 |        <path d="M 544 1381 
1184 | L 544 3500 
1185 | L 1119 3500 
1186 | L 1119 1403 
1187 | Q 1119 906 1312 657 
1188 | Q 1506 409 1894 409 
1189 | Q 2359 409 2629 706 
1190 | Q 2900 1003 2900 1516 
1191 | L 2900 3500 
1192 | L 3475 3500 
1193 | L 3475 0 
1194 | L 2900 0 
1195 | L 2900 538 
1196 | Q 2691 219 2414 64 
1197 | Q 2138 -91 1772 -91 
1198 | Q 1169 -91 856 284 
1199 | Q 544 659 544 1381 
1200 | z
1201 | M 1991 3584 
1202 | L 1991 3584 
1203 | z
1204 | " id="DejaVuSans-75" transform="scale(0.015625)"/>
1205 |        <path d="M 3597 1894 
1206 | L 3597 1613 
1207 | L 953 1613 
1208 | Q 991 1019 1311 708 
1209 | Q 1631 397 2203 397 
1210 | Q 2534 397 2845 478 
1211 | Q 3156 559 3463 722 
1212 | L 3463 178 
1213 | Q 3153 47 2828 -22 
1214 | Q 2503 -91 2169 -91 
1215 | Q 1331 -91 842 396 
1216 | Q 353 884 353 1716 
1217 | Q 353 2575 817 3079 
1218 | Q 1281 3584 2069 3584 
1219 | Q 2775 3584 3186 3129 
1220 | Q 3597 2675 3597 1894 
1221 | z
1222 | M 3022 2063 
1223 | Q 3016 2534 2758 2815 
1224 | Q 2500 3097 2075 3097 
1225 | Q 1594 3097 1305 2825 
1226 | Q 1016 2553 972 2059 
1227 | L 3022 2063 
1228 | z
1229 | " id="DejaVuSans-65" transform="scale(0.015625)"/>
1230 |       </defs>
1231 |       <use xlink:href="#DejaVuSans-74"/>
1232 |       <use x="39.208984" xlink:href="#DejaVuSans-72"/>
1233 |       <use x="80.322266" xlink:href="#DejaVuSans-75"/>
1234 |       <use x="143.701172" xlink:href="#DejaVuSans-65"/>
1235 |      </g>
1236 |     </g>
1237 |     <g id="line2d_27">
1238 |      <path d="M 354.984063 214.400562 
1239 | L 374.984063 214.400562 
1240 | " style="fill:none;stroke:#0000ff;stroke-dasharray:5.55,2.4;stroke-dashoffset:0;stroke-width:1.5;"/>
1241 |     </g>
1242 |     <g id="line2d_28"/>
1243 |     <g id="text_18">
1244 |      <!-- pred -->
1245 |      <g transform="translate(382.984063 217.900562)scale(0.1 -0.1)">
1246 |       <defs>
1247 |        <path d="M 1159 525 
1248 | L 1159 -1331 
1249 | L 581 -1331 
1250 | L 581 3500 
1251 | L 1159 3500 
1252 | L 1159 2969 
1253 | Q 1341 3281 1617 3432 
1254 | Q 1894 3584 2278 3584 
1255 | Q 2916 3584 3314 3078 
1256 | Q 3713 2572 3713 1747 
1257 | Q 3713 922 3314 415 
1258 | Q 2916 -91 2278 -91 
1259 | Q 1894 -91 1617 61 
1260 | Q 1341 213 1159 525 
1261 | z
1262 | M 3116 1747 
1263 | Q 3116 2381 2855 2742 
1264 | Q 2594 3103 2138 3103 
1265 | Q 1681 3103 1420 2742 
1266 | Q 1159 2381 1159 1747 
1267 | Q 1159 1113 1420 752 
1268 | Q 1681 391 2138 391 
1269 | Q 2594 391 2855 752 
1270 | Q 3116 1113 3116 1747 
1271 | z
1272 | " id="DejaVuSans-70" transform="scale(0.015625)"/>
1273 |        <path d="M 2906 2969 
1274 | L 2906 4863 
1275 | L 3481 4863 
1276 | L 3481 0 
1277 | L 2906 0 
1278 | L 2906 525 
1279 | Q 2725 213 2448 61 
1280 | Q 2172 -91 1784 -91 
1281 | Q 1150 -91 751 415 
1282 | Q 353 922 353 1747 
1283 | Q 353 2572 751 3078 
1284 | Q 1150 3584 1784 3584 
1285 | Q 2172 3584 2448 3432 
1286 | Q 2725 3281 2906 2969 
1287 | z
1288 | M 947 1747 
1289 | Q 947 1113 1208 752 
1290 | Q 1469 391 1925 391 
1291 | Q 2381 391 2643 752 
1292 | Q 2906 1113 2906 1747 
1293 | Q 2906 2381 2643 2742 
1294 | Q 2381 3103 1925 3103 
1295 | Q 1469 3103 1208 2742 
1296 | Q 947 2381 947 1747 
1297 | z
1298 | " id="DejaVuSans-64" transform="scale(0.015625)"/>
1299 |       </defs>
1300 |       <use xlink:href="#DejaVuSans-70"/>
1301 |       <use x="63.476562" xlink:href="#DejaVuSans-72"/>
1302 |       <use x="102.339844" xlink:href="#DejaVuSans-65"/>
1303 |       <use x="163.863281" xlink:href="#DejaVuSans-64"/>
1304 |      </g>
1305 |     </g>
1306 |    </g>
1307 |   </g>
1308 |  </g>
1309 |  <defs>
1310 |   <clipPath id="p24b09aa7a2">
1311 |    <rect height="120.96" width="357.12" x="57.6" y="41.472"/>
1312 |   </clipPath>
1313 |   <clipPath id="pf177800d0e">
1314 |    <rect height="120.96" width="357.12" x="57.6" y="186.624"/>
1315 |   </clipPath>
1316 |  </defs>
1317 | </svg>
1318 | 


--------------------------------------------------------------------------------