├── images ├── BOPINNalgo.PNG └── BOPINN_methodolody.png ├── data ├── u_analytic_c=0.2t=0.25snr=39.36.mat ├── u_analytic_c=0.55t=0.25snr=38.91.mat └── u_analytic_c=0.85t=0.25snr=37.5.mat ├── LICENSE ├── lib ├── network.py ├── layer.py ├── pinn_wave.py └── optimizer.py ├── README.md ├── analytical.py ├── PINN.py ├── BOPINN.py └── BOPINN.ipynb /images/BOPINNalgo.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahindrautela/BOPINN/HEAD/images/BOPINNalgo.PNG -------------------------------------------------------------------------------- /images/BOPINN_methodolody.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahindrautela/BOPINN/HEAD/images/BOPINN_methodolody.png -------------------------------------------------------------------------------- /data/u_analytic_c=0.2t=0.25snr=39.36.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahindrautela/BOPINN/HEAD/data/u_analytic_c=0.2t=0.25snr=39.36.mat -------------------------------------------------------------------------------- /data/u_analytic_c=0.55t=0.25snr=38.91.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahindrautela/BOPINN/HEAD/data/u_analytic_c=0.55t=0.25snr=38.91.mat -------------------------------------------------------------------------------- /data/u_analytic_c=0.85t=0.25snr=37.5.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mahindrautela/BOPINN/HEAD/data/u_analytic_c=0.85t=0.25snr=37.5.mat -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Mahindra Rautela 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/network.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | class Network: 4 | """ 5 | Build a physics informed neural network (PINN) model for the wave equation. 6 | """ 7 | 8 | @classmethod 9 | def build(cls, num_inputs=2, layers=[64, 128, 128, 128, 128, 64], activation='tanh', num_outputs=1): 10 | """ 11 | Build a PINN model for the wave equation with input shape (t, x) and output shape u(t, x). 12 | 13 | Args: 14 | num_inputs: number of input variables. Default is 2 for (t, x). 15 | layers: number of hidden layers. 16 | activation: activation function in hidden layers. 17 | num_outpus: number of output variables. Default is 1 for u(t, x). 18 | 19 | Returns: 20 | keras network model. 21 | """ 22 | 23 | # input layer 24 | inputs = tf.keras.layers.Input(shape=(num_inputs,)) 25 | # hidden layers 26 | x = inputs 27 | for layer in layers: 28 | x = tf.keras.layers.Dense(layer, activation=activation, 29 | kernel_initializer='he_normal')(x) 30 | x = tf.keras.layers.Dropout(0.1)(x) 31 | # output layer 32 | outputs = tf.keras.layers.Dense(num_outputs, 33 | kernel_initializer='he_normal')(x) 34 | 35 | return tf.keras.models.Model(inputs=inputs, outputs=outputs) 36 | -------------------------------------------------------------------------------- /lib/layer.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | class GradientLayer(tf.keras.layers.Layer): 4 | """ 5 | Custom layer to compute 1st and 2nd derivatives for the wave equation. 6 | 7 | Attributes: 8 | model: keras network model. 9 | """ 10 | 11 | def __init__(self, model, **kwargs): 12 | """ 13 | Args: 14 | model: keras network model. 15 | """ 16 | 17 | self.model = model 18 | super().__init__(**kwargs) 19 | 20 | def call(self, tx): 21 | """ 22 | Computing 1st and 2nd derivatives for the wave equation. 23 | 24 | Args: 25 | tx: input variables (t, x). 26 | 27 | Returns: 28 | u: network output. 29 | du_dt: 1st derivative of t. 30 | du_dx: 1st derivative of x. 31 | d2u_dt2: 2nd derivative of t. 32 | d2u_dx2: 2nd derivative of x. 33 | """ 34 | 35 | with tf.GradientTape() as g: 36 | g.watch(tx) 37 | with tf.GradientTape() as gg: 38 | gg.watch(tx) 39 | u = self.model(tx) 40 | du_dtx = gg.batch_jacobian(u, tx) 41 | du_dt = du_dtx[..., 0] 42 | du_dx = du_dtx[..., 1] 43 | d2u_dtx2 = g.batch_jacobian(du_dtx, tx) 44 | d2u_dt2 = d2u_dtx2[..., 0, 0] 45 | d2u_dx2 = d2u_dtx2[..., 1, 1] 46 | 47 | return u, du_dt, du_dx, d2u_dt2, d2u_dx2 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BOPINN (Bayesian optimized physics-informed neural network) 2 | BOPINN presents new paradigm to solve inverse problems by bringing an amalgamation of PINNs and BO. It uses BO (a gradient-free and global optimization scheme) and PINNs (a fast neural surrogate solver for PDEs). In BOPINN, a PINN utilizes a neural surrogate to solve the partial differential equation (wave propagation here). Bayesian optimization runs over the PINN model and estimates the optimum parameters (wave velocity in the medium here) using a single snapshot observation of the field. BOPINN queries the black-box PINN model at different wave velocities until it converges to the true wave velocity. The proposed method is simpler (uses single neural network), robust (capturs uncertainty) and flexible (useful in real-time and online settings) as compared to it's counterparts. 3 | 4 |

5 | 6 |

7 | 8 |

9 | 10 |

11 | 12 | ## About the repository: 13 | 1. Written in tensorflow 2.10 with cuda 11.8 and cudnn 8.x 14 | 2. The code uses BO repository from [bayesian-optimization](https://github.com/bayesian-optimization/BayesianOptimization). 15 | 3. "BOPINN.py" is the python file (run in spyder) and "BOPINN.ipynb" is a notebook (use colab or jupyter) 16 | 4. "PINN.py" is a PINN based solver for forward wave propagation problem. It's an auxillary code to understand the forward problem 17 | 5. "analytical.py" gives the exact solution of the wave equation with dirichlet BC and it is used to collect data (added white noise) 18 | 6. data folder contains the snapshot observation collected from "analytical.py" 19 | 7. lib folder has .py files required to run "PINN.py" and "BOPINN.py" 20 | 21 | For more information: 22 | 1. [Link](https://doi.org/10.48550/arXiv.2312.14064) of the paper: Bayesian optimized physics-informed neural network for estimating wave propagation velocities. 23 | 2. Please cite the paper if you are using code, paper or data. 24 | ``` 25 | @article{rautela2023bayesian, 26 | title={Bayesian optimized physics-informed neural network for estimating wave propagation velocities}, 27 | author={Rautela, Mahindra and Gopalakrishnan, S and Senthilnath, J}, 28 | journal={arXiv preprint arXiv:2312.14064}, 29 | year={2023} 30 | } 31 | ``` 32 | 33 | -------------------------------------------------------------------------------- /lib/pinn_wave.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from .layer import GradientLayer 3 | 4 | class PINN: 5 | """ 6 | Build a physics informed neural network (PINN) model for the wave equation. 7 | 8 | Attributes: 9 | network: keras network model with input (t, x) and output u(t, x). 10 | c: wave velocity. 11 | grads: gradient layer. 12 | """ 13 | 14 | def __init__(self, network, c): 15 | """ 16 | Args: 17 | network: keras network model with input (t, x) and output u(t, x). 18 | c: wave velocity. Default is 1. 19 | """ 20 | 21 | self.network = network 22 | self.c = c 23 | self.grads = GradientLayer(self.network) 24 | 25 | def build(self): 26 | """ 27 | Build a PINN model for the wave equation. 28 | 29 | Returns: 30 | PINN model for the projectile motion with 31 | input: [ (t, x) relative to equation, 32 | (t=0, x) relative to initial condition, 33 | (t, x=bounds) relative to boundary condition ], 34 | output: [ u(t,x) relative to equation, 35 | u(t=0, x) relative to initial condition, 36 | du_dt(t=0, x) relative to initial derivative of t, 37 | u(t, x=bounds) relative to boundary condition ] 38 | """ 39 | 40 | # equation input: (t, x) 41 | tx_eqn = tf.keras.layers.Input(shape=(2,)) 42 | # initial condition input: (t=0, x) 43 | tx_ini = tf.keras.layers.Input(shape=(2,)) 44 | # boundary condition input: (t, x=-1) or (t, x=+1) 45 | tx_bnd = tf.keras.layers.Input(shape=(2,)) 46 | 47 | # compute gradients 48 | _, _, _, d2u_dt2, d2u_dx2 = self.grads(tx_eqn) 49 | 50 | # equation output being zero 51 | u_eqn = d2u_dt2 - self.c*self.c * d2u_dx2 52 | # initial condition output 53 | u_ini, du_dt_ini, _, _, _ = self.grads(tx_ini) 54 | # boundary condition output 55 | u_bnd = self.network(tx_bnd) # dirichlet 56 | #_, _, u_bnd, _, _ = self.grads(tx_bnd) # neumann 57 | 58 | # build the PINN model for the wave equation 59 | return tf.keras.models.Model( 60 | inputs=[tx_eqn, tx_ini, tx_bnd], 61 | outputs=[u_eqn, u_ini, du_dt_ini, u_bnd]) 62 | 63 | -------------------------------------------------------------------------------- /analytical.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import math 3 | import matplotlib.pyplot as plt 4 | from matplotlib.colors import Normalize 5 | from matplotlib.gridspec import GridSpec 6 | import os 7 | 8 | # Other variables 9 | L = 10 10 | n = L 11 | T = 1 12 | 13 | # speed 14 | C = [0.20, 0.55, 0.85] 15 | c = C[0] 16 | 17 | # snapshot time 18 | tfrac = np.array([0.25, 0.50, 0.75]) 19 | tilde_t = tfrac*T 20 | 21 | # number of scan points 22 | num_points = 5000 23 | 24 | # RMS function 25 | def RMS(S): 26 | rms = np.sqrt(np.mean(S**2)) 27 | return rms 28 | 29 | def u_analytic(t,x,c,L,n): 30 | # x and t in mesh form 31 | usol = -np.sin(math.pi*x)*np.cos(n*math.pi*c*t/L) 32 | return usol 33 | 34 | # Full analytical solution of the domain [0,L] and [0,T] 35 | t = np.linspace(0,T,num_points) 36 | x = np.linspace(0,L,num_points) 37 | x_mesh, t_mesh = np.meshgrid(x, t) 38 | usol_full = u_analytic(t_mesh, x_mesh, c, L, n) 39 | 40 | # index of snapshot time in the array 41 | idx = [int(p) for p in num_points*tfrac] 42 | 43 | # plot u(t,x) distribution as a color-map 44 | fig = plt.figure(figsize=(7,4)) 45 | gs = GridSpec(2, 3) # A grid layout to place subplots within a figure. 46 | plt.subplot(gs[0, :]) 47 | vmin, vmax = -1.0, +1.0 48 | plt.pcolormesh(t_mesh, x_mesh, usol_full, cmap='rainbow', shading = 'auto', norm=Normalize(vmin=vmin, vmax=vmax)) 49 | plt.xlabel('t') 50 | plt.ylabel('x') 51 | cbar = plt.colorbar(pad=0.05, aspect=10) 52 | cbar.set_label('u(t,x)') 53 | cbar.mappable.set_clim(vmin, vmax) 54 | 55 | # plot u(t=const, x) cross-sections 56 | for i, t_cs in enumerate(tilde_t): 57 | plt.subplot(gs[1, i]) 58 | plt.plot(x, usol_full[idx[i],:], 'b', linewidth = 2) 59 | plt.title('t={}'.format(t_cs)) 60 | plt.xlabel('x') 61 | plt.ylabel('u(t,x)') 62 | plt.ylim(-1,1) 63 | plt.tight_layout() 64 | plt.show() 65 | 66 | #%% usol at particular snapshot time 67 | from scipy.io import savemat 68 | 69 | # data storing 70 | path = "C:\MSR\data\PINNBO\data" 71 | 72 | # amplitude of noise 73 | beta_range = [0.0075] #[0.0075, 0.01, 0.025, 0.05] 74 | 75 | # these parameters need to be set 76 | usol_app = [] 77 | usol_n_app = [] 78 | for ii in range(len(C)): 79 | for jj in range(len(tilde_t)): 80 | for kk in range(len(beta_range)): 81 | c = C[ii] 82 | t_obs = tilde_t[jj] 83 | beta = beta_range[kk] 84 | 85 | # without noise 86 | tt = np.full(t.shape, t_obs) 87 | xx = np.linspace(0,L,num_points) 88 | xx_mesh, tt_mesh = np.meshgrid(xx, tt) 89 | usol = u_analytic(tt_mesh, xx_mesh, c, L, n) 90 | usol = usol[0,:] # all rows (time) are same 91 | usol_app.append(usol) 92 | 93 | # Add white noise to the data 94 | mu = 0 95 | sigma = 1 96 | noise = beta*(sigma*np.random.randn(num_points,1) + mu) 97 | 98 | # noisy data 99 | usol_n = usol[:,np.newaxis] + noise 100 | usol_n_app.append(usol_n) 101 | 102 | # signal to noise ratio 103 | snr = 20*np.log10(RMS(usol)/RMS(noise)) 104 | snr_percent = RMS(noise)/RMS(usol)*100 105 | 106 | f1 = "c="+str(c) 107 | f2 = "t="+str(t_obs) 108 | f3 = "snr="+str(np.round(snr,2)) 109 | 110 | plt.figure(figsize=(22,4)) 111 | fig, (ax1, ax2) = plt.subplots(1, 2,figsize=(10,4)) 112 | ax1.plot(xx,usol) 113 | ax1.set_title('Without noise', fontsize = 15) 114 | ax1.set(xlabel='x', ylabel='Normalized u(x,t)') 115 | ax1.set_ylim(-1.2,1.2) 116 | 117 | ax2.plot(xx,usol_n) 118 | ax2.set_title(f1+f2+f3, fontsize = 15) 119 | ax2.set(xlabel='x') 120 | ax2.set_ylim(-1.2,1.2) 121 | 122 | plt.show() 123 | 124 | # save data 125 | filename = "u_analytic_"+f1+f2+f3 126 | filepath = os.path.join(path, filename) 127 | mdic = {"a1": usol_n, "label": "experiment"} 128 | savemat(filepath+".mat", mdic) 129 | 130 | #%% analysis of experimental data 131 | usol_app = np.array(usol_app) 132 | usol_n_app = np.array(usol_n_app) 133 | color = ['k','tab:blue','tab:orange', 134 | 'tab:green','tab:red','tab:purple', 135 | 'tab:brown','tab:pink','tab:gray','tab:cyan'] 136 | 137 | plt.figure(figsize=(10,4)) 138 | for i in range(usol_app.shape[0]): 139 | plt.plot(xx,usol_app[i,:],color[i]) 140 | plt.legend(['1','2','3','4','5','6','7','8','9']) 141 | #plt.savefig('analytical_9_withoutnoise', dpi = 300) 142 | 143 | plt.figure(figsize=(10,4)) 144 | for i in range(usol_app.shape[0]): 145 | plt.plot(xx,usol_n_app[i,:],color[i]) 146 | plt.legend(['1','2','3','4','5','6','7','8','9']) 147 | #plt.savefig('analytical_9_withnoise', dpi = 300) 148 | -------------------------------------------------------------------------------- /PINN.py: -------------------------------------------------------------------------------- 1 | #import lib.tf_silent 2 | import numpy as np 3 | import tensorflow as tf 4 | import matplotlib.pyplot as plt 5 | from matplotlib.colors import Normalize 6 | from matplotlib.gridspec import GridSpec 7 | from lib.pinn_wave import PINN 8 | from lib.network import Network 9 | from lib.optimizer import L_BFGS_B 10 | import math 11 | import time 12 | 13 | # number of training samples 14 | num_train_samples = 25000 15 | 16 | # number of test samples 17 | num_test_samples = 5000 18 | 19 | c = 0.2 #scaled speed 20 | L = 10 21 | n = L 22 | T = 1 23 | 24 | # Initial conditions 25 | def u0(t): 26 | z = -np.sin(1*math.pi*t) 27 | return z 28 | 29 | def du0_dt(tx): 30 | with tf.GradientTape() as g: 31 | g.watch(tx) 32 | u = u0(tx) 33 | du_dt = g.batch_jacobian(u, tx)[..., 0] 34 | return du_dt 35 | 36 | # Analytical solution 37 | xx = np.linspace(0,L,num_test_samples) 38 | tt = np.linspace(0,T,num_test_samples) 39 | usol = np.zeros((num_test_samples,num_test_samples)) 40 | for i,xi in enumerate(xx): 41 | for j,tj in enumerate(tt): 42 | usol[i,j] = -np.sin(math.pi*xi)*np.cos(n*math.pi*c*tj/L) 43 | 44 | 45 | ######################################################################## 46 | ######################## collocation points ############################ 47 | ######################################################################## 48 | 49 | # create training input 50 | tx_eqn = np.random.rand(num_train_samples, 2) 51 | tx_eqn[..., 0] = T*tx_eqn[..., 0] # t = 0 ~ +1 52 | tx_eqn[..., 1] = L*tx_eqn[..., 1] # x = 0 ~ +10 53 | #print('\nShape of t_eqn ==>',tx_eqn.shape) 54 | 55 | tx_ini = np.random.rand(num_train_samples, 2) 56 | tx_ini[..., 0] = 0 # t = 0 57 | tx_ini[..., 1] = L*tx_ini[..., 1] # x = 0 ~ +10 58 | #print('\nShape of tx_ini ==>',tx_ini.shape) 59 | 60 | tx_bnd = np.random.rand(num_train_samples, 2) 61 | tx_bnd[..., 0] = T*tx_bnd[..., 0] # t = 0 ~ +1 62 | tx_bnd[..., 1] = L*np.round(tx_bnd[..., 1]) # x = 0 or +10 63 | #print('\nShape of tx_bnd ==>',tx_bnd.shape) 64 | 65 | u_zero = np.zeros((num_train_samples, 1)) 66 | u_ini = u0(tx_ini[:,1,None]) 67 | du_dt_ini = np.zeros((num_train_samples, 1)) 68 | 69 | ######################################################################### 70 | ########################### TRAINING PINNs ############################## 71 | ######################################################################### 72 | 73 | # build a core network model 74 | network = Network.build() 75 | #network.summary() 76 | 77 | # build a PINN model 78 | pinn = PINN(network,c).build() 79 | 80 | # train the model using L-BFGS-B algorithm 81 | begin = time.time() 82 | x_train = [tx_eqn, tx_ini, tx_bnd] 83 | y_train = [u_zero, u_ini, du_dt_ini, u_zero] 84 | lbfgs = L_BFGS_B(model=pinn, x_train=x_train, y_train=y_train) 85 | lbfgs.fit() 86 | end = time.time() 87 | totaltime = end-begin 88 | print("\n Total runtime of the program is (min.)",totaltime/60) 89 | 90 | ######################################################################### 91 | ######################## PREDICTION ##################################### 92 | ######################################################################### 93 | 94 | # predict u(t,x) distribution 95 | t_flat = np.linspace(0, T, num_test_samples) 96 | x_flat = np.linspace(0, L, num_test_samples) 97 | t, x = np.meshgrid(t_flat, x_flat) 98 | tx = np.stack([t.flatten(), x.flatten()], axis=-1) 99 | u = network.predict(tx, batch_size=num_test_samples) 100 | u = u.reshape(t.shape) 101 | 102 | # plot u(t,x) distribution as a color-map 103 | fig = plt.figure(figsize=(12,8)) 104 | gs = GridSpec(2, 3) # A grid layout to place subplots within a figure. 105 | plt.subplot(gs[0, :]) 106 | vmin, vmax = -1.0, +1.0 107 | plt.pcolormesh(t, x, u, cmap='rainbow', shading = 'auto', norm=Normalize(vmin=vmin, vmax=vmax)) 108 | plt.xlabel('t',fontsize=20) 109 | plt.ylabel('x',fontsize=20) 110 | plt.xticks(fontsize=20) 111 | plt.yticks(fontsize=20) 112 | cbar = plt.colorbar(pad=0.05, aspect=10) 113 | cbar.set_label('u(t,x)', fontsize=20) 114 | cbar.ax.tick_params(labelsize=20) 115 | cbar.mappable.set_clim(vmin, vmax) 116 | 117 | # plot u(t=const, x) cross-sections 118 | tfrac = np.array([0.25,0.5,0.75]) 119 | t_cross_sections = (T*tfrac).tolist() 120 | idx = [int(x) for x in (num_test_samples*tfrac)] 121 | 122 | for i, t_cs in enumerate(t_cross_sections): 123 | plt.subplot(gs[1, i]) 124 | full = np.full(t_flat.shape, t_cs) 125 | tx = np.stack([np.full(t_flat.shape, t_cs), x_flat], axis=-1) 126 | u = network.predict(tx, batch_size=num_test_samples) 127 | #print(u.shape) 128 | plt.plot(x_flat, u, '.b') 129 | plt.plot(x_flat, usol[:,idx[i]], 'r--', linewidth = 2) 130 | plt.title('t = {}'.format(t_cs),fontsize=20) 131 | plt.xlabel('x',fontsize=20) 132 | plt.ylabel('u(t,x)',fontsize=20) 133 | plt.xticks(fontsize=20) 134 | plt.yticks(fontsize=20) 135 | plt.ylim(-1,1) 136 | plt.legend(['Prediction','Exact'], loc = 'upper right',fontsize=8) 137 | plt.tight_layout() 138 | plt.savefig('PINNs_at_'+str(c)+'.png', transparent=True, dpi = 900) 139 | plt.show() 140 | 141 | -------------------------------------------------------------------------------- /lib/optimizer.py: -------------------------------------------------------------------------------- 1 | import scipy.optimize 2 | import numpy as np 3 | import tensorflow as tf 4 | 5 | class L_BFGS_B: 6 | """ 7 | Optimize the keras network model using L-BFGS-B algorithm. 8 | 9 | Attributes: 10 | model: optimization target model. 11 | samples: training samples. 12 | factr: convergence condition. typical values for factr are: 1e12 for low accuracy; 13 | 1e7 for moderate accuracy; 10 for extremely high accuracy. 14 | m: maximum number of variable metric corrections used to define the limited memory matrix. 15 | maxls: maximum number of line search steps (per iteration). 16 | maxiter: maximum number of iterations. 17 | metris: logging metrics. 18 | progbar: progress bar. 19 | """ 20 | 21 | def __init__(self, model, x_train, y_train, m=10, factr=1e7, pgtol=1e-5, 22 | epsilon=1e-8, maxiter=5000, maxls=50): 23 | """ 24 | Args: 25 | model: optimization target model. 26 | samples: training samples. 27 | factr: convergence condition. typical values for factr are: 1e12 for low accuracy; 28 | 1e7 for moderate accuracy; 10.0 for extremely high accuracy. 29 | m: maximum number of variable metric corrections used to define the limited memory matrix. 30 | maxls: maximum number of line search steps (per iteration). 31 | maxiter: maximum number of iterations. 32 | """ 33 | 34 | # set attributes 35 | self.model = model 36 | self.x_train = [ tf.constant(x, dtype=tf.float32) for x in x_train ] 37 | self.y_train = [ tf.constant(y, dtype=tf.float32) for y in y_train ] 38 | self.factr = factr 39 | self.m = m 40 | self.pgtol = pgtol 41 | self.epsilon = epsilon 42 | self.maxls = maxls 43 | self.maxiter = maxiter 44 | self.metrics = ['loss'] 45 | # initialize the progress bar 46 | self.progbar = tf.keras.callbacks.ProgbarLogger( 47 | count_mode='steps', stateful_metrics=self.metrics) 48 | self.progbar.set_params( { 49 | 'verbose':1, 'epochs':1, 'steps':self.maxiter, 'metrics':self.metrics}) 50 | 51 | def set_weights(self, flat_weights): 52 | """ 53 | Set weights to the model. 54 | 55 | Args: 56 | flat_weights: flatten weights. 57 | """ 58 | 59 | # get model weights 60 | shapes = [ w.shape for w in self.model.get_weights() ] 61 | # compute splitting indices 62 | split_ids = np.cumsum([ np.prod(shape) for shape in [0] + shapes ]) 63 | # reshape weights 64 | weights = [ flat_weights[from_id:to_id].reshape(shape) 65 | for from_id, to_id, shape in zip(split_ids[:-1], split_ids[1:], shapes) ] 66 | # set weights to the model 67 | self.model.set_weights(weights) 68 | 69 | @tf.function 70 | def tf_evaluate(self, x, y): 71 | """ 72 | Evaluate loss and gradients for weights as tf.Tensor. 73 | 74 | Args: 75 | x: input data. 76 | 77 | Returns: 78 | loss and gradients for weights as tf.Tensor. 79 | """ 80 | 81 | with tf.GradientTape() as g: 82 | loss = tf.reduce_mean(tf.keras.losses.mse(self.model(x), y)) 83 | grads = g.gradient(loss, self.model.trainable_variables) 84 | return loss, grads 85 | 86 | def evaluate(self, weights): 87 | """ 88 | Evaluate loss and gradients for weights as ndarray. 89 | 90 | Args: 91 | weights: flatten weights. 92 | 93 | Returns: 94 | loss and gradients for weights as ndarray. 95 | """ 96 | 97 | # update weights 98 | self.set_weights(weights) 99 | # compute loss and gradients for weights 100 | loss, grads = self.tf_evaluate(self.x_train, self.y_train) 101 | # convert tf.Tensor to flatten ndarray 102 | loss = loss.numpy().astype('float64') 103 | grads = np.concatenate([ g.numpy().flatten() for g in grads ]).astype('float64') 104 | 105 | return loss, grads 106 | 107 | def callback(self, weights): 108 | """ 109 | Callback that prints the progress to stdout. 110 | 111 | Args: 112 | weights: flatten weights. 113 | """ 114 | self.progbar.on_batch_begin(0) 115 | loss, _ = self.evaluate(weights) 116 | self.progbar.on_batch_end(0, logs=dict(zip(self.metrics, [loss]))) 117 | 118 | def fit(self): 119 | """ 120 | Train the model using L-BFGS-B algorithm. 121 | """ 122 | 123 | # get initial weights as a flat vector 124 | initial_weights = np.concatenate( 125 | [ w.flatten() for w in self.model.get_weights() ]) 126 | # optimize the weight vector 127 | print('Optimizer: L-BFGS-B (maxiter={})'.format(self.maxiter)) 128 | self.progbar.on_train_begin() 129 | self.progbar.on_epoch_begin(1) 130 | 131 | scipy.optimize.fmin_l_bfgs_b(func=self.evaluate, 132 | x0=initial_weights, 133 | factr=self.factr, 134 | pgtol=self.pgtol, 135 | epsilon=self.epsilon, 136 | m=self.m, 137 | maxls=self.maxls, 138 | maxiter=self.maxiter, 139 | callback=self.callback) 140 | 141 | 142 | # scipy.optimize.least_squares(func = self.evaluate, x0 = initial_weights) 143 | 144 | # scipy.optimize.minimize(fun = self.evaluate, 145 | # x0 = initial_weights, 146 | # method='L-BFGS-B', 147 | # jac= True, # If jac is True, fun is assumed to return the gradient along with the objective function 148 | # callback = self.callback, 149 | # options = {'disp': None, 150 | # 'maxcor': 200, 151 | # 'ftol': 1 * np.finfo(float).eps, #The iteration stops when (f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= ftol 152 | # 'gtol': 5e-5, 153 | # 'maxfun': 50000, 154 | # 'maxiter': 1, 155 | # 'iprint': 50, #print update every 50 iterations 156 | # 'maxls': 50}) 157 | 158 | # scipy.optimize.minimize(fun=self.evaluate, 159 | # x0=initial_weights, 160 | # jac=True, 161 | # method='BFGS', 162 | # callback=self.callback, 163 | # options={'maxiter': 15000, 164 | # 'maxls': 20}) 165 | 166 | # scipy.optimize.minimize(fun = self.evaluate, 167 | # x0 = initial_weights, 168 | # args=(), 169 | # method='L-BFGS-B', 170 | # jac= True, 171 | # callback = self.callback, 172 | # options = {'disp': None, 173 | # 'maxcor': 200, 174 | # 'ftol': 1 * np.finfo(float).eps, #The iteration stops when (f^k - f^{k+1})/max{|f^k|,|f^{k+1}|,1} <= ftol 175 | # 'gtol': 5e-5, 176 | # 'maxfun': 50000, 177 | # 'maxiter': 1, 178 | # 'iprint': 50, #print update every 50 iterations 179 | # 'maxls': 50}) 180 | 181 | self.progbar.on_epoch_end(1) 182 | self.progbar.on_train_end() 183 | -------------------------------------------------------------------------------- /BOPINN.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import matplotlib.pyplot as plt 4 | from lib.pinn_wave import PINN 5 | from lib.network import Network 6 | from lib.optimizer import L_BFGS_B 7 | import math 8 | from bayes_opt import BayesianOptimization, UtilityFunction 9 | import scipy.io 10 | import os 11 | from os.path import join 12 | import time 13 | 14 | # number of training samples 15 | num_train_samples = 25000 16 | 17 | # number of test samples 18 | num_test_samples = 5000 19 | 20 | # Other variables 21 | L = 10 22 | n = L 23 | T = 1 24 | 25 | # define x,t for PINNs prediction 26 | x_test = np.linspace(0,L,num_test_samples) 27 | t_test = np.linspace(0,T,num_test_samples) 28 | 29 | # upload the snapshot observation 30 | path = "E:\MSR\data\BOPINN\data" 31 | dir_list = os.listdir(path) 32 | print("Files in directory",dir_list) 33 | 34 | idx_data = 3 # 0,3,6 35 | data = dir_list[idx_data] 36 | print("Imported file", data) 37 | 38 | file = join(path, data) 39 | u_analy = scipy.io.loadmat(file) 40 | u_analy = u_analy['a1'] 41 | 42 | # plot the snapshot observation 43 | fig = plt.figure(figsize=(7,4)) 44 | plt.plot(x_test,u_analy, '-', linewidth = 2) 45 | plt.xlabel('$x$', fontsize = 15) 46 | plt.ylabel('Normalized u(x,t)', fontsize = 15) 47 | plt.xticks(fontsize = 12) 48 | plt.yticks(fontsize = 12) 49 | 50 | # time of observation 51 | tilde_t = 0.25 52 | 53 | #%% Initial conditions 54 | def u0(t): 55 | z = -np.sin(1*math.pi*t) 56 | return z 57 | 58 | def du0_dt(tx): 59 | with tf.GradientTape() as g: 60 | g.watch(tx) 61 | u = u0(tx) 62 | du_dt = g.batch_jacobian(u, tx)[..., 0] 63 | return du_dt 64 | 65 | def RMS(S): 66 | rms = np.sqrt(np.mean(S**2)) 67 | return rms 68 | 69 | #%% collocation points 70 | # create training input 71 | tx_eqn = np.random.rand(num_train_samples, 2) 72 | tx_eqn[..., 0] = T*tx_eqn[..., 0] # t = 0 ~ +1 73 | tx_eqn[..., 1] = L*tx_eqn[..., 1] # x = 0 ~ +10 74 | #print('\nShape of t_eqn ==>',tx_eqn.shape) 75 | 76 | tx_ini = np.random.rand(num_train_samples, 2) 77 | tx_ini[..., 0] = 0 # t = 0 78 | tx_ini[..., 1] = L*tx_ini[..., 1] # x = 0 ~ +10 79 | #print('\nShape of tx_ini ==>',tx_ini.shape) 80 | 81 | tx_bnd = np.random.rand(num_train_samples, 2) 82 | tx_bnd[..., 0] = T*tx_bnd[..., 0] # t = 0 ~ +1 83 | tx_bnd[..., 1] = L*np.round(tx_bnd[..., 1]) # x = 0 or +10 84 | #print('\nShape of tx_bnd ==>',tx_bnd.shape) 85 | 86 | # initial and boundary conditions 87 | u_zero = np.zeros((num_train_samples, 1)) 88 | u_ini = u0(tx_ini[:,1,None]) 89 | du_dt_ini = np.zeros((num_train_samples, 1)) 90 | 91 | #%% g(c) = (u_pred - u_true)^2; u_pred via PINNs 92 | 93 | def model_builder(ic): 94 | #ic = hp.Float('ic', min_value=0.1, max_value=1, step=10) 95 | print('\n ## ->>>> PINNs simulation at speed = ' + str(ic)) 96 | 97 | # build a PINN model 98 | network = Network.build() 99 | pinn = PINN(network,ic).build() 100 | 101 | # train the model using L-BFGS-B algorithm 102 | begin = time.time() 103 | x_train = [tx_eqn, tx_ini, tx_bnd] 104 | y_train = [u_zero, u_ini, du_dt_ini, u_zero] 105 | lbfgs = L_BFGS_B(model=pinn, x_train=x_train, y_train=y_train) 106 | lbfgs.fit() 107 | end = time.time() 108 | totaltime = end-begin 109 | print("\n Total runtime is (min.)",totaltime/60) 110 | 111 | # test the model 112 | tx = np.stack([np.full(t_test.shape, tilde_t), x_test], axis=-1) 113 | u_pred = network.predict(tx, batch_size=num_test_samples) 114 | 115 | # mse between u_pred via PINN and snapshot observation 116 | mse = -np.mean(np.square(u_analy - u_pred)) 117 | 118 | del network, pinn, lbfgs, u_pred 119 | 120 | return mse 121 | 122 | #%% Bayesian Optimization 123 | # Attributes of BO 124 | itt_explore = 5 125 | itt = 45 126 | itt_all = itt_explore + itt 127 | n_runs = 10 128 | 129 | # bounds of BO 130 | pbounds = {'ic': (0.1, 1)} 131 | 132 | # Start BO 133 | mse_star_all = [] 134 | cstar_all = [] 135 | mse_all_all = [] 136 | ic_all_all = [] 137 | 138 | for r in range(n_runs): 139 | print('\n ## ->>>> Run = ' + str(r)) 140 | 141 | # define the model 142 | optimizer = BayesianOptimization( 143 | f=model_builder, 144 | pbounds=pbounds, 145 | allow_duplicate_points=True) 146 | 147 | # utility function 148 | util = UtilityFunction(kind='ucb', 149 | kappa=2.576, 150 | kappa_decay=1, 151 | kappa_decay_delay=0) 152 | 153 | # run the model 154 | optimizer.maximize(init_points=itt_explore, 155 | n_iter=itt, 156 | acquisition_function=util) 157 | 158 | soln = optimizer.max 159 | resi = optimizer.res 160 | 161 | # optimum values 162 | mse_star = list(soln.values())[0] 163 | cstar = list(soln.values())[1] 164 | cstar2 = list(cstar.values())[0] 165 | 166 | # append all optimum values 167 | mse_star_all.append(mse_star) 168 | cstar_all.append(cstar2) 169 | 170 | # all run values 171 | mse_all = [] 172 | ic_all = [] 173 | for i,res in enumerate(resi): 174 | mse = list(res.values())[0] 175 | ic = list(res.values())[1] 176 | ic2 = list(ic.values())[0] 177 | 178 | # append all run values 179 | mse_all.append(mse) 180 | ic_all.append(ic2) 181 | 182 | mse_all_all.append(np.array(mse_all)) 183 | ic_all_all.append(np.array(ic_all)) 184 | 185 | del optimizer 186 | 187 | mse_all_all = np.array(mse_all_all) 188 | mse_star_all = np.array(mse_star_all) 189 | ic_all_all = np.array(ic_all_all) 190 | cstar_all = np.array(cstar_all) 191 | 192 | #%% Process the BO results 193 | # max, min, mean and sd target function/objective function value across different runs 194 | max_mse_star_allruns, min_mse_star_allruns = np.max(mse_star_all), np.min(mse_star_all) 195 | mean_mse_star_allruns, std_mse_star_allruns = np.mean(mse_star_all), np.std(mse_star_all) 196 | 197 | # optima corresponding to abovementioned optimal points 198 | idx_max_mse_star_allruns = np.where(max_mse_star_allruns == mse_star_all) 199 | idx_min_mse_star_allruns = np.where(min_mse_star_allruns == mse_star_all) 200 | 201 | max_cstar_allruns = cstar_all[idx_max_mse_star_allruns] 202 | min_cstar_allruns = cstar_all[idx_min_mse_star_allruns] 203 | mean_cstar_allruns = np.mean(cstar_all) 204 | std_cstar_allruns = np.std(cstar_all) 205 | 206 | print("Max (best optimal) tf across runs = ",max_mse_star_allruns) 207 | print("Min (least optimal) tf across runs = ",min_mse_star_allruns) 208 | print("Mean tf across runs = ",mean_mse_star_allruns) 209 | print("Std tf across runs = ",std_mse_star_allruns) 210 | 211 | print("Max (best optimal) c* across runs = ",max_cstar_allruns) 212 | print("Min (least optimal) c* across runs = ",min_cstar_allruns) 213 | print("Mean c* across runs = ",mean_cstar_allruns) 214 | print("Std c* across runs = ",std_cstar_allruns) 215 | 216 | #%% plot the BO results 217 | # plot best optimal run with the optima 218 | idx_max_all = [] 219 | for i in range(mse_all_all.shape[0]): 220 | idx_max = np.where(mse_all_all[i,:] == mse_star_all[i]) 221 | idx_max = idx_max[0][0] 222 | idx_max_all.append(idx_max) 223 | 224 | mean_mse_all = np.mean(mse_all_all, axis=0) 225 | std_mse_all = np.std(mse_all_all, axis=0) 226 | mean_ic_all = np.mean(ic_all_all, axis=0) 227 | std_ic_all = np.std(ic_all_all, axis=0) 228 | 229 | opt_mse_run = mse_all_all[idx_max_mse_star_allruns[0][0]] 230 | opt_c_run = ic_all_all[idx_max_mse_star_allruns[0][0]] 231 | opt_mse = mse_star_all[idx_max_mse_star_allruns[0][0]] 232 | opt_c = cstar_all[idx_max_mse_star_allruns[0][0]] 233 | 234 | # tf vs c 235 | txt = 'c* = '+ str(round(opt_c,4)) 236 | plt.figure(figsize = (8, 6)) 237 | plt.plot(opt_c_run,opt_mse_run,'ob',markersize=6) 238 | plt.plot(opt_c,opt_mse,'*r',markersize=8, label = 'Best optima') 239 | #plt.text(0.75, -0.03, txt, fontsize=18, c = 'r') 240 | plt.xlabel("velocity, c",fontsize=20) 241 | plt.ylabel("target function, g(c)",fontsize=20) 242 | plt.xticks(fontsize=20) 243 | plt.yticks(fontsize=20) 244 | plt.legend(fontsize = 14, loc='upper left') 245 | plt.savefig('tfvsc_'+str(idx_data+1)+'.png', bbox_inches='tight', dpi=600) 246 | plt.show() -------------------------------------------------------------------------------- /BOPINN.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "source": [ 6 | "## Colab computing infra available\n", 7 | "### Preliminaries\n", 8 | "#### 1. Runtime -> Change runtime type -> T4 GPU\n", 9 | "#### 2. Upload data folder from github repository on your google drive\n", 10 | "#### 3. Except first three cells, this notebook can be run on jupyter.\n", 11 | "#### 4. For 3., pip install bayesian-optimization is performed in anaconda tensorflow environment." 12 | ], 13 | "metadata": { 14 | "id": "XW60sZt3HGKv" 15 | } 16 | }, 17 | { 18 | "cell_type": "code", 19 | "source": [ 20 | "#GPU count and name\n", 21 | "!nvidia-smi -L\n", 22 | "!lscpu |grep 'Model name'\n", 23 | "#no.of sockets i.e available slots for physical processors\n", 24 | "!lscpu | grep 'Socket(s):'\n", 25 | "#no.of cores each processor is having\n", 26 | "!lscpu | grep 'Core(s) per socket:'\n", 27 | "#no.of threads each core is having\n", 28 | "!lscpu | grep 'Thread(s) per core'\n", 29 | "!lscpu | grep \"L3 cache\"\n", 30 | "#if it had turbo boost it would've shown Min and Max MHz also but it is only showing current frequency this means it always operates at 2.3GHz\n", 31 | "!lscpu | grep \"MHz\"\n", 32 | "#memory that we can use\n", 33 | "!cat /proc/meminfo | grep 'MemAvailable'\n", 34 | "#hard disk that we can use\n", 35 | "!df -h / | awk '{print $4}'" 36 | ], 37 | "metadata": { 38 | "id": "8iHIF9ANB7W-", 39 | "colab": { 40 | "base_uri": "https://localhost:8080/" 41 | }, 42 | "outputId": "677a2eac-9246-478b-f44c-cbeef270bda4" 43 | }, 44 | "execution_count": null, 45 | "outputs": [ 46 | { 47 | "output_type": "stream", 48 | "name": "stdout", 49 | "text": [ 50 | "GPU 0: Tesla T4 (UUID: GPU-2429e038-2c60-04b9-d3a5-5573f49827da)\n", 51 | "Model name: Intel(R) Xeon(R) CPU @ 2.30GHz\n", 52 | "Socket(s): 1\n", 53 | "Core(s) per socket: 1\n", 54 | "Thread(s) per core: 2\n", 55 | "L3 cache: 45 MiB (1 instance)\n", 56 | "MemAvailable: 12416136 kB\n", 57 | "Avail\n", 58 | "52G\n" 59 | ] 60 | } 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "source": [ 66 | "# mount google drive\n", 67 | "from google.colab import drive\n", 68 | "drive.mount('/content/gdrive')\n", 69 | "path = 'gdrive/MyDrive/BOPINN/data'" 70 | ], 71 | "metadata": { 72 | "id": "q_ae9KDsC2Q1", 73 | "colab": { 74 | "base_uri": "https://localhost:8080/" 75 | }, 76 | "outputId": "6d816703-0158-473b-a2c5-95e4b09b1c99" 77 | }, 78 | "execution_count": null, 79 | "outputs": [ 80 | { 81 | "output_type": "stream", 82 | "name": "stdout", 83 | "text": [ 84 | "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n" 85 | ] 86 | } 87 | ] 88 | }, 89 | { 90 | "cell_type": "code", 91 | "source": [ 92 | "# install Bayesian optimization library \"https://github.com/bayesian-optimization/BayesianOptimization\"\n", 93 | "%pip install bayesian-optimization==1.4.1\n", 94 | "from bayes_opt import BayesianOptimization,UtilityFunction" 95 | ], 96 | "metadata": { 97 | "colab": { 98 | "base_uri": "https://localhost:8080/" 99 | }, 100 | "id": "CrbZE4FvZhbf", 101 | "outputId": "7cbc93bf-ae8b-4936-9fa9-70eb3727dcfb" 102 | }, 103 | "execution_count": null, 104 | "outputs": [ 105 | { 106 | "output_type": "stream", 107 | "name": "stdout", 108 | "text": [ 109 | "Requirement already satisfied: bayesian-optimization==1.4.1 in /usr/local/lib/python3.10/dist-packages (1.4.1)\n", 110 | "Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from bayesian-optimization==1.4.1) (1.23.5)\n", 111 | "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from bayesian-optimization==1.4.1) (1.11.4)\n", 112 | "Requirement already satisfied: scikit-learn>=0.18.0 in /usr/local/lib/python3.10/dist-packages (from bayesian-optimization==1.4.1) (1.2.2)\n", 113 | "Requirement already satisfied: colorama in /usr/local/lib/python3.10/dist-packages (from bayesian-optimization==1.4.1) (0.4.6)\n", 114 | "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.18.0->bayesian-optimization==1.4.1) (1.3.2)\n", 115 | "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.18.0->bayesian-optimization==1.4.1) (3.2.0)\n" 116 | ] 117 | } 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "source": [ 123 | "## In-built classes" 124 | ], 125 | "metadata": { 126 | "id": "qECf7VXwCvKO" 127 | } 128 | }, 129 | { 130 | "cell_type": "markdown", 131 | "source": [ 132 | "### Gradient Tape" 133 | ], 134 | "metadata": { 135 | "id": "GQQzlB87IQS1" 136 | } 137 | }, 138 | { 139 | "cell_type": "code", 140 | "source": [ 141 | "import tensorflow as tf\n", 142 | "import numpy as np\n", 143 | "import matplotlib.pyplot as plt\n", 144 | "\n", 145 | "class GradientLayer(tf.keras.layers.Layer):\n", 146 | " \"\"\"\n", 147 | " Custom layer to compute 1st and 2nd derivatives for the wave equation.\n", 148 | "\n", 149 | " Attributes:\n", 150 | " model: keras network model.\n", 151 | " \"\"\"\n", 152 | "\n", 153 | " def __init__(self, model, **kwargs):\n", 154 | " \"\"\"\n", 155 | " Args:\n", 156 | " model: keras network model.\n", 157 | " \"\"\"\n", 158 | "\n", 159 | " self.model = model\n", 160 | " super().__init__(**kwargs)\n", 161 | "\n", 162 | " def call(self, tx):\n", 163 | " \"\"\"\n", 164 | " Computing 1st and 2nd derivatives for the wave equation.\n", 165 | "\n", 166 | " Args:\n", 167 | " tx: input variables (t, x).\n", 168 | "\n", 169 | " Returns:\n", 170 | " u: network output.\n", 171 | " du_dt: 1st derivative of t.\n", 172 | " du_dx: 1st derivative of x.\n", 173 | " d2u_dt2: 2nd derivative of t.\n", 174 | " d2u_dx2: 2nd derivative of x.\n", 175 | " \"\"\"\n", 176 | "\n", 177 | " with tf.GradientTape() as g:\n", 178 | " g.watch(tx)\n", 179 | " with tf.GradientTape() as gg:\n", 180 | " gg.watch(tx)\n", 181 | " u = self.model(tx)\n", 182 | " du_dtx = gg.batch_jacobian(u, tx)\n", 183 | " du_dt = du_dtx[..., 0]\n", 184 | " du_dx = du_dtx[..., 1]\n", 185 | " d2u_dtx2 = g.batch_jacobian(du_dtx, tx)\n", 186 | " d2u_dt2 = d2u_dtx2[..., 0, 0]\n", 187 | " d2u_dx2 = d2u_dtx2[..., 1, 1]\n", 188 | "\n", 189 | " return u, du_dt, du_dx, d2u_dt2, d2u_dx2" 190 | ], 191 | "metadata": { 192 | "id": "8GmwN5gSIOr2" 193 | }, 194 | "execution_count": null, 195 | "outputs": [] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "source": [ 200 | "### PINNs" 201 | ], 202 | "metadata": { 203 | "id": "e7-8pYNbDJEv" 204 | } 205 | }, 206 | { 207 | "cell_type": "code", 208 | "source": [ 209 | "class PINN:\n", 210 | " \"\"\"\n", 211 | " Build a physics informed neural network (PINN) model for the wave equation.\n", 212 | "\n", 213 | " Attributes:\n", 214 | " network: keras network model with input (t, x) and output u(t, x).\n", 215 | " c: wave velocity.\n", 216 | " grads: gradient layer.\n", 217 | " \"\"\"\n", 218 | "\n", 219 | " def __init__(self, network, c):\n", 220 | " \"\"\"\n", 221 | " Args:\n", 222 | " network: keras network model with input (t, x) and output u(t, x).\n", 223 | " c: wave velocity. Default is 1.\n", 224 | " \"\"\"\n", 225 | "\n", 226 | " self.network = network\n", 227 | " self.c = c\n", 228 | " self.grads = GradientLayer(self.network)\n", 229 | "\n", 230 | " def build(self):\n", 231 | " \"\"\"\n", 232 | " Build a PINN model for the wave equation.\n", 233 | "\n", 234 | " Returns:\n", 235 | " PINN model for the projectile motion with\n", 236 | " input: [ (t, x) relative to equation,\n", 237 | " (t=0, x) relative to initial condition,\n", 238 | " (t, x=bounds) relative to boundary condition ],\n", 239 | " output: [ u(t,x) relative to equation,\n", 240 | " u(t=0, x) relative to initial condition,\n", 241 | " du_dt(t=0, x) relative to initial derivative of t,\n", 242 | " u(t, x=bounds) relative to boundary condition ]\n", 243 | " \"\"\"\n", 244 | "\n", 245 | " # equation input: (t, x)\n", 246 | " tx_eqn = tf.keras.layers.Input(shape=(2,))\n", 247 | " # initial condition input: (t=0, x)\n", 248 | " tx_ini = tf.keras.layers.Input(shape=(2,))\n", 249 | " # boundary condition input: (t, x=-1) or (t, x=+1)\n", 250 | " tx_bnd = tf.keras.layers.Input(shape=(2,))\n", 251 | "\n", 252 | " # compute gradients\n", 253 | " _, _, _, d2u_dt2, d2u_dx2 = self.grads(tx_eqn)\n", 254 | "\n", 255 | " # equation output being zero\n", 256 | " u_eqn = d2u_dt2 - self.c*self.c * d2u_dx2\n", 257 | " # initial condition output\n", 258 | " u_ini, du_dt_ini, _, _, _ = self.grads(tx_ini)\n", 259 | " # boundary condition output\n", 260 | " u_bnd = self.network(tx_bnd) # dirichlet\n", 261 | " #_, _, u_bnd, _, _ = self.grads(tx_bnd) # neumann\n", 262 | "\n", 263 | " # build the PINN model for the wave equation\n", 264 | " return tf.keras.models.Model(\n", 265 | " inputs=[tx_eqn, tx_ini, tx_bnd],\n", 266 | " outputs=[u_eqn, u_ini, du_dt_ini, u_bnd])" 267 | ], 268 | "metadata": { 269 | "id": "N_6QYlj4C04v" 270 | }, 271 | "execution_count": null, 272 | "outputs": [] 273 | }, 274 | { 275 | "cell_type": "markdown", 276 | "source": [ 277 | "### Network for PINN" 278 | ], 279 | "metadata": { 280 | "id": "ZWPMeef1DLeA" 281 | } 282 | }, 283 | { 284 | "cell_type": "code", 285 | "source": [ 286 | "class Network:\n", 287 | " \"\"\"\n", 288 | " Build a physics informed neural network (PINN) model for the wave equation.\n", 289 | " \"\"\"\n", 290 | "\n", 291 | " @classmethod\n", 292 | " def build(cls, num_inputs=2, layers=[64, 128, 128, 128, 128, 64], activation='tanh', num_outputs=1):\n", 293 | " \"\"\"\n", 294 | " Build a PINN model for the wave equation with input shape (t, x) and output shape u(t, x).\n", 295 | "\n", 296 | " Args:\n", 297 | " num_inputs: number of input variables. Default is 2 for (t, x).\n", 298 | " layers: number of hidden layers.\n", 299 | " activation: activation function in hidden layers.\n", 300 | " num_outpus: number of output variables. Default is 1 for u(t, x).\n", 301 | "\n", 302 | " Returns:\n", 303 | " keras network model.\n", 304 | " \"\"\"\n", 305 | "\n", 306 | " # input layer\n", 307 | " inputs = tf.keras.layers.Input(shape=(num_inputs,))\n", 308 | " # hidden layers\n", 309 | " x = inputs\n", 310 | " for layer in layers:\n", 311 | " x = tf.keras.layers.Dense(layer, activation=activation,\n", 312 | " kernel_initializer='he_normal')(x)\n", 313 | " x = tf.keras.layers.Dropout(0.1)(x)\n", 314 | " # output layer\n", 315 | " outputs = tf.keras.layers.Dense(num_outputs,\n", 316 | " kernel_initializer='he_normal')(x)\n", 317 | "\n", 318 | " return tf.keras.models.Model(inputs=inputs, outputs=outputs)" 319 | ], 320 | "metadata": { 321 | "id": "YqrofsFgDNXR" 322 | }, 323 | "execution_count": null, 324 | "outputs": [] 325 | }, 326 | { 327 | "cell_type": "markdown", 328 | "source": [ 329 | "### Optimizer for PINN" 330 | ], 331 | "metadata": { 332 | "id": "ipTUkVObDT3M" 333 | } 334 | }, 335 | { 336 | "cell_type": "code", 337 | "source": [ 338 | "import scipy.optimize\n", 339 | "\n", 340 | "class L_BFGS_B:\n", 341 | " \"\"\"\n", 342 | " Optimize the keras network model using L-BFGS-B algorithm.\n", 343 | "\n", 344 | " Attributes:\n", 345 | " model: optimization target model.\n", 346 | " samples: training samples.\n", 347 | " factr: convergence condition. typical values for factr are: 1e12 for low accuracy;\n", 348 | " 1e7 for moderate accuracy; 10 for extremely high accuracy.\n", 349 | " m: maximum number of variable metric corrections used to define the limited memory matrix.\n", 350 | " maxls: maximum number of line search steps (per iteration).\n", 351 | " maxiter: maximum number of iterations.\n", 352 | " metris: logging metrics.\n", 353 | " progbar: progress bar.\n", 354 | " \"\"\"\n", 355 | "\n", 356 | " def __init__(self, model, x_train, y_train, m=10, factr=1e7, pgtol=1e-5,\n", 357 | " epsilon=1e-8, maxiter=5000, maxls=50):\n", 358 | " \"\"\"\n", 359 | " Args:\n", 360 | " model: optimization target model.\n", 361 | " samples: training samples.\n", 362 | " factr: convergence condition. typical values for factr are: 1e12 for low accuracy;\n", 363 | " 1e7 for moderate accuracy; 10.0 for extremely high accuracy.\n", 364 | " m: maximum number of variable metric corrections used to define the limited memory matrix.\n", 365 | " maxls: maximum number of line search steps (per iteration).\n", 366 | " maxiter: maximum number of iterations.\n", 367 | " \"\"\"\n", 368 | "\n", 369 | " # set attributes\n", 370 | " self.model = model\n", 371 | " self.x_train = [ tf.constant(x, dtype=tf.float32) for x in x_train ]\n", 372 | " self.y_train = [ tf.constant(y, dtype=tf.float32) for y in y_train ]\n", 373 | " self.factr = factr\n", 374 | " self.m = m\n", 375 | " self.pgtol = pgtol\n", 376 | " self.epsilon = epsilon\n", 377 | " self.maxls = maxls\n", 378 | " self.maxiter = maxiter\n", 379 | " self.metrics = ['loss']\n", 380 | " # initialize the progress bar\n", 381 | " self.progbar = tf.keras.callbacks.ProgbarLogger(\n", 382 | " count_mode='steps', stateful_metrics=self.metrics)\n", 383 | " self.progbar.set_params( {\n", 384 | " 'verbose':1, 'epochs':1, 'steps':self.maxiter, 'metrics':self.metrics})\n", 385 | "\n", 386 | " def set_weights(self, flat_weights):\n", 387 | " \"\"\"\n", 388 | " Set weights to the model.\n", 389 | "\n", 390 | " Args:\n", 391 | " flat_weights: flatten weights.\n", 392 | " \"\"\"\n", 393 | "\n", 394 | " # get model weights\n", 395 | " shapes = [ w.shape for w in self.model.get_weights() ]\n", 396 | " # compute splitting indices\n", 397 | " split_ids = np.cumsum([ np.prod(shape) for shape in [0] + shapes ])\n", 398 | " # reshape weights\n", 399 | " weights = [ flat_weights[from_id:to_id].reshape(shape)\n", 400 | " for from_id, to_id, shape in zip(split_ids[:-1], split_ids[1:], shapes) ]\n", 401 | " # set weights to the model\n", 402 | " self.model.set_weights(weights)\n", 403 | "\n", 404 | " @tf.function\n", 405 | " def tf_evaluate(self, x, y):\n", 406 | " \"\"\"\n", 407 | " Evaluate loss and gradients for weights as tf.Tensor.\n", 408 | "\n", 409 | " Args:\n", 410 | " x: input data.\n", 411 | "\n", 412 | " Returns:\n", 413 | " loss and gradients for weights as tf.Tensor.\n", 414 | " \"\"\"\n", 415 | "\n", 416 | " with tf.GradientTape() as g:\n", 417 | " loss = tf.reduce_mean(tf.keras.losses.mse(self.model(x), y))\n", 418 | " grads = g.gradient(loss, self.model.trainable_variables)\n", 419 | " return loss, grads\n", 420 | "\n", 421 | " def evaluate(self, weights):\n", 422 | " \"\"\"\n", 423 | " Evaluate loss and gradients for weights as ndarray.\n", 424 | "\n", 425 | " Args:\n", 426 | " weights: flatten weights.\n", 427 | "\n", 428 | " Returns:\n", 429 | " loss and gradients for weights as ndarray.\n", 430 | " \"\"\"\n", 431 | "\n", 432 | " # update weights\n", 433 | " self.set_weights(weights)\n", 434 | " # compute loss and gradients for weights\n", 435 | " loss, grads = self.tf_evaluate(self.x_train, self.y_train)\n", 436 | " # convert tf.Tensor to flatten ndarray\n", 437 | " loss = loss.numpy().astype('float64')\n", 438 | " grads = np.concatenate([ g.numpy().flatten() for g in grads ]).astype('float64')\n", 439 | "\n", 440 | " return loss, grads\n", 441 | "\n", 442 | " def callback(self, weights):\n", 443 | " \"\"\"\n", 444 | " Callback that prints the progress to stdout.\n", 445 | "\n", 446 | " Args:\n", 447 | " weights: flatten weights.\n", 448 | " \"\"\"\n", 449 | " self.progbar.on_batch_begin(0)\n", 450 | " loss, _ = self.evaluate(weights)\n", 451 | " self.progbar.on_batch_end(0, logs=dict(zip(self.metrics, [loss])))\n", 452 | "\n", 453 | " def fit(self):\n", 454 | " \"\"\"\n", 455 | " Train the model using L-BFGS-B algorithm.\n", 456 | " \"\"\"\n", 457 | "\n", 458 | " # get initial weights as a flat vector\n", 459 | " initial_weights = np.concatenate(\n", 460 | " [ w.flatten() for w in self.model.get_weights() ])\n", 461 | " # optimize the weight vector\n", 462 | " print('Optimizer: L-BFGS-B (maxiter={})'.format(self.maxiter))\n", 463 | " self.progbar.on_train_begin()\n", 464 | " self.progbar.on_epoch_begin(1)\n", 465 | "\n", 466 | " scipy.optimize.fmin_l_bfgs_b(func=self.evaluate,\n", 467 | " x0=initial_weights,\n", 468 | " factr=self.factr,\n", 469 | " pgtol=self.pgtol,\n", 470 | " epsilon=self.epsilon,\n", 471 | " m=self.m,\n", 472 | " maxls=self.maxls,\n", 473 | " maxiter=self.maxiter,\n", 474 | " callback=self.callback)\n", 475 | "\n", 476 | " self.progbar.on_epoch_end(1)\n", 477 | " self.progbar.on_train_end()\n" 478 | ], 479 | "metadata": { 480 | "id": "RDynBdklDSqh" 481 | }, 482 | "execution_count": null, 483 | "outputs": [] 484 | }, 485 | { 486 | "cell_type": "markdown", 487 | "source": [ 488 | "## Upload data (snapshot observation)" 489 | ], 490 | "metadata": { 491 | "id": "WpYtKf8CDl7P" 492 | } 493 | }, 494 | { 495 | "cell_type": "code", 496 | "source": [ 497 | "#from bayes_opt import BayesianOptimization, UtilityFunction\n", 498 | "import math\n", 499 | "import scipy.io\n", 500 | "import os\n", 501 | "from os.path import join\n", 502 | "import time\n", 503 | "\n", 504 | "# number of training samples: found 25000 points is optimal for the resolution of u(x,t) in PINN\n", 505 | "num_train_samples = 25000\n", 506 | "\n", 507 | "# number of test samples\n", 508 | "num_test_samples = 5000\n", 509 | "\n", 510 | "# Other variables\n", 511 | "L = 10\n", 512 | "n = L\n", 513 | "T = 1\n", 514 | "\n", 515 | "# define x,t for PINNs prediction\n", 516 | "x_test = np.linspace(0,L,num_test_samples)\n", 517 | "t_test = np.linspace(0,T,num_test_samples)\n", 518 | "\n", 519 | "# upload the snapshot observation\n", 520 | "dir_list = os.listdir(path) # path defined in 2 cell\n", 521 | "print(\"Files in directory\",dir_list)\n", 522 | "idx_data = 0 # 0,1,2 # which file\n", 523 | "data = dir_list[idx_data]\n", 524 | "print(\"Imported file\", data)\n", 525 | "\n", 526 | "file = join(path, data)\n", 527 | "u_analy = scipy.io.loadmat(file)\n", 528 | "u_analy = u_analy['a1']\n", 529 | "\n", 530 | "# plot the snapshot observation\n", 531 | "fig = plt.figure(figsize=(7,4))\n", 532 | "plt.plot(x_test,u_analy, '-', linewidth = 2)\n", 533 | "plt.title(\"Snapshot observation at = 0.25s\",fontsize=15)\n", 534 | "plt.xlabel('$x$', fontsize = 15)\n", 535 | "plt.ylabel('Normalized u(x,t)', fontsize = 15)\n", 536 | "plt.xticks(fontsize = 12)\n", 537 | "plt.yticks(fontsize = 12)\n", 538 | "\n", 539 | "# time of observation = for PINN prediction of u(x,t) at tilde_t\n", 540 | "tilde_t = 0.25" 541 | ], 542 | "metadata": { 543 | "id": "_3cuJlPqCHZd", 544 | "colab": { 545 | "base_uri": "https://localhost:8080/", 546 | "height": 459 547 | }, 548 | "outputId": "d5169459-39aa-4294-d7cb-127b58634e5e" 549 | }, 550 | "execution_count": null, 551 | "outputs": [ 552 | { 553 | "output_type": "stream", 554 | "name": "stdout", 555 | "text": [ 556 | "Files in directory ['u_analytic_c=0.2t=0.25snr=39.36.mat', 'u_analytic_c=0.55t=0.25snr=38.91.mat', 'u_analytic_c=0.85t=0.25snr=37.5.mat']\n", 557 | "Imported file u_analytic_c=0.2t=0.25snr=39.36.mat\n" 558 | ] 559 | }, 560 | { 561 | "output_type": "display_data", 562 | "data": { 563 | "text/plain": [ 564 | "
" 565 | ], 566 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAogAAAGVCAYAAAB96QBiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAC6sElEQVR4nOydd3gUVRfG303vvUNIQklCDy10aVIkgAihCVKkyCeIdEREQEAQFMQCiiAIoggRRKTX0AIIhN4TQkkC6T3ZlJ3vj7izM7O7yWbbzOze3/Psw8zsnZmzZHfmzD3nvEdCURQFAoFAIBAIBALhPyz4NoBAIBAIBAKBICyIg0ggEAgEAoFAYEEcRAKBQCAQCAQCC+IgEggEAoFAIBBYEAeRQCAQCAQCgcCCOIgEAoFAIBAIBBbEQSQQCAQCgUAgsCAOIoFAIBAIBAKBBXEQCQQCgUAgEAgsiINIIKjh1KlTGDx4MGrVqgUbGxu4u7sjLCwMQ4YMwXfffYfc3Fy+TTQIwcHBkEgkfJuhVxYvXgyJRIKtW7fybYroIP93xuXOnTsYMmQIvL29YW9vj6ZNm+Lrr7+GTCar0XHu37+PL774At26dYOXlxesra3h5+eHQYMG4ezZs2r3k0gkVb5KSkp0/YgEkWDFtwEEghD57LPPsGjRIgBAw4YN0bZtW1hbW+PBgwfYs2cPYmJi0Lp1a7Rr145nS8WLRCJBUFAQkpKS+DbFrOnatStiY2Px5MkTBAcH822OQVm8eDGWLFmCLVu2YOzYsXybo0RcXBx69OiB4uJiREZGIjg4GGfOnMGMGTNw4cIF/PHHHxo/vL3++utITk6Gk5MT2rVrBw8PD9y9exd79+7FX3/9hTVr1mD69Okq93V0dER0dLTK9ywtLbX9eASRQRxEAoHD1atXsXjxYlhbW2PXrl0YOHAg6/2XL1/i119/hZubGy/2EQjGZOrUqRg+fDj8/f35NsWkKSsrw8iRI1FcXIw1a9ZgxowZAICCggL06tULu3fvRt++fTV2bMPDw7FixQoMGTIEdnZ29PYff/wRkydPxuzZs9GrVy80atRIaV8vLy8yY0wgIWYCgcuePXtAURSGDh2q5BwCgJ+fH2bPno3w8HDjG0cgGBkvLy+Eh4fD1dWVb1NMmr179+LJkydo3rw57RwCgJOTE7777jsAwFdffaXx8Y4fP4533nmH5RwCwHvvvYdevXqhoqICu3fv1o/xBJOEOIgEAof09HQAgLe3d432Y+bubdq0Cc2aNYO9vT38/Pzw3nvvIScnR2mfx48fY/HixWjfvj38/PxgY2OD2rVrY/To0Xj48KHK80gkEgQHB6O0tBSLFi1CvXr1YGdnh7p16+LTTz9VmSNUUFCAFStWoHnz5nB1dYWTkxPq1auHIUOG4MiRI2o/k6afAwAyMzMxZ84cNGjQAHZ2dvDw8ECfPn1w9OhR1ritW7fS/09Pnz5l5Td17dpVrS1cDh48iJ49e8Ld3R12dnYICwvDRx99pNY+OZcuXULv3r3h5uYGFxcX9OzZExcvXlQ59sKFCxg4cCCCgoJga2sLPz8/REZG4qOPPkJBQYHKYw8ZMgT+/v7033LChAl49uyZ0lhmbt/ly5fRr18/eHp6QiKR4N9//4WXlxfs7OzUfp5bt25BIpGgZcuW9LacnBx8++236N27N22zp6cn+vTpg2PHjrH2T0pKgkQiQWxsLAAgJCSE9bdQZScXTf/mcuTf3YqKCnzxxRcIDQ2Fra0tAgMDMW/ePEilUpX7qYKiKPz+++8YPnw4QkND4ejoCGdnZ0RGRmL9+vVKOXvBwcFYsmQJAGDcuHGsz3r69GmNz2soDhw4AAAqQ7stW7ZE3bp1cfv2bb2kZDRv3hwAkJKSovOxAOD27dsYNWoU6tatCzs7O3h7eyMiIgLTp09HamqqXs5B4AGKQCCw+OyzzygAVGBgIPXq1SuN9wsKCqIAUHPmzKFsbGyoXr16UW+99Rbl4+NDAaA6d+5MyWQy1j7z5s2jJBIJ1bRpU6pfv37U4MGDqYYNG1IAKBcXF+rGjRtK5wFA1alTh+rXrx9lb29P9evXjxo0aBDl6upKAaB69OhBlZeX0+PLy8uptm3bUgAoLy8vasCAAdTQoUOpDh06UA4ODtSYMWN0/hwvXryg6tatS9s2bNgwqnv37pSlpSUFgFqzZg099uzZs9SYMWMoAJSjoyM1ZswY+rVixQqN/q8///xzCgBlZWVF9ejRgxo2bBhVu3ZtCgAVGhpKvXz5kjV+0aJFFABq4sSJlI2NDdWoUSNq+PDhVOvWrSkAlI2NDXXkyBHWPn///TdlYWFBSSQSqm3bttTw4cOpPn36UPXq1aMAUE+ePGGN//777ykLCwvKwsKCatu2LTVkyBCqWbNmFADK29ubunv3rkqbxo0bR1lbW1ONGzemhg8fTr322mvUjRs3qMmTJ1MAqE2bNqn8P5g3bx4FgPrqq6/obYcOHaIAUMHBwVTPnj2pYcOGUe3bt6ckEgklkUiozZs302PT09OpMWPGUL6+vhQAavDgway/BdfOLVu2sM5fk7+5HABUUFAQNXToUMrJyYnq168f1a9fP/q7O3LkSJWfVRXFxcUUAMrT05Pq3LkzNWzYMOr111+nHBwcKABK3+tZs2ZRzZs3pwBQHTt2ZH3We/fuaXxeQyG37cCBAyrfj46OpgBQ+/bt0/lcgwcPpgBQn376qdJ78uvEsmXLqEmTJlHTp0+nfvnlFyo/P1/lsa5cuULZ2dlRAKhmzZpRQ4cOpfr160c1atSIAkCdOnVKZ3sJ/EAcRAKBQ0JCAmVvb08BoJydnakxY8ZQP/30E3Xt2jWW48VF7lj5+flR9+/fp7enp6dT9evXpwBQJ06cYO0TFxdHJSYmKh3r559/pgBQ3bp1U3oPAAWAql27NpWQkEBvT0tLo5o0aUIBoNauXUtvP3nyJAWAatOmDVVcXMw6Vm5uLnXlyhWdP0e/fv0oANTbb79NSaVSevvZs2cpBwcHytLSkoqPj1f6HEFBQUqfrzouX75MWVhYUE5OTtTFixfp7SUlJdSQIUNoZ4eJ3MkBQC1YsIDl4K5fv54CQPn7+1NFRUX09tdee40CQMXExKi0IS8vj16Pi4ujLC0tqVq1ain9f27atIkCQLVt21atTV988YXSOc6ePUsBoLp37670nkwmo+rUqUNZWFhQycnJ9PbExEQqLi5Oafy1a9coNzc3ysXFRelG36VLF5UOL9dOroOo7d8cANWwYUMqNTWVZbebmxsFgHr8+LFKO7iUlZVRe/fupUpLS1nb09LSaMc/NjZWo8+iCVu2bKHt1/TVpUsXjY/v7u5OAVD5UEhRFDV9+nQKAPXNN9/U2HYmjx8/pmxtbSkASt9ViqLUfhZPT0/qn3/+URo/evRoCgD15ZdfKr137949KiUlRSd7CfxBHEQCQQXHjx+nAgMDlS6Sbm5u1P/+9z+VFz25Y/XTTz8pvffll19SAKhFixZpbEPHjh0piURC5eTksLbLbdm4caPSPvIZpHr16tHb/vjjDwoANX36dI3OW9PPkZCQQAGgnJycqMzMTKV9Zs6cSQGgJkyYoPQ5tHEQ5Tek+fPnK7336tUryt7enrKwsKCePXtGb5c7BkFBQVRZWZnSfvIZ1u3bt9Pb5DO53P9/Vbz55psUAGr//v0q3x8wYAAFgLp27ZqSTU2bNlWakaWoSicwODhYyQmkKIqKjY2lZ4s1ZcGCBRQA6u+//2Zt18ZB1OVvDoA6duyY0j5Tp07V2nnjcuzYMQoANXPmzGo/i6bIZ75r8tJ0RpyiKMra2poCQD169Ejl+/K/3/Lly2tsu5yysjKqU6dOFABq2LBhKseMHj2aOnz4MJWcnEwVFBRQ8fHx1DvvvEPPtF++fJk1/o033qAAUNevX9faLoIwIVXMBIIKevTogcePH+PAgQM4evQoLl++jJs3byInJwcbNmzAn3/+iTNnziAsLExp3169eiltCw0NBQCV+TgFBQXYv38/rl+/jqysLJSVldFjKYpCQkICK89MzvDhw5W29enTB+7u7khISEBqair8/f0REREBCwsLbNmyBY0aNcKgQYPg6elZ7f+Bpp/j3Llz9Lk9PDyU9nnnnXewZs2aKrXXaoL8OCNHjlR6z8fHB7169cK+fftw/vx5pf+jwYMHw8pK+bI3YsQIXLp0CWfPnsWoUaMAAK1atcK9e/fwzjvvYOHChWjVqhUsLJTTtmUyGU6cOAEHBwf07t1bpc2dO3fG33//jcuXL6NFixas9/r166dSukQikeDtt9/G559/jp07d2LmzJn0ezt27AAA2lYmFRUVOHHiBC5cuIDU1FQ6r+/Ro0esf3VBl7+5tbU1unXrprS9qt9IVVy/fh1Hjx7F06dPUVRUBIqikJ+fD0A/n1VOp06d0KlTJ70djw+mTZuGc+fOoW7duli/fr3KMb/88gtrPSIiAtu2bUNgYCA+//xzfPLJJ6y85VatWuHQoUOYMmUKli1bhk6dOqn8jRHEB/krEghqsLGxwVtvvYW33noLQGUBwM6dO/Hxxx8jLS0NU6dOVUr8B4DatWsrbXN2dgYApST8kydPYvjw4XRhjCrkNzsm7u7u9DG5BAUFITs7GykpKfD390doaChWrVqF+fPnY9KkSZg8eTKaNGmCHj16YOzYsWjWrJnK42j6OeSJ7uo09OTbk5OT1X7GmqDL+YKCgqrch5m0//nnn+PWrVvYv38/9u/fD3d3d3Tq1AkDBgzAqFGj6OrQjIwMumDFxsamStszMjKUttWpU0ft+JEjR+Lzzz/Hjh07aAextLQUu3fvhp2dHQYNGsQa/+LFC/Tr1w83btxQe0xV36eaosvfwM/PT6WWnrrfiDpKS0sxduxY/P7772rH6OOzGgsnJydkZ2ejqKhI5fuFhYUAoPZ3Xx3Lly/Hhg0b4OvriyNHjqh07Kti7ty5+OKLL3D69GmUlpbS3/U5c+bg3LlzOH36NLp16wYnJye0b98eUVFRGDt2LKl+FzHEQSQQNMTNzQ2TJ09GQEAA3nzzTZw6dQpFRUVwcHBgjVM1y6SKgoICDB06FFlZWfj0008xfPhwBAUFwd7enp49+v3330FRlM62z5o1C0OHDsVff/2FY8eO4ezZs1i7di2+/vprrF27Fh9++KHSPpp+juowdlcWfZ0vMDAQV65cwcmTJ/HPP/8gNjaWdhZXrVqFuLg4eHp60tWyTk5OGDx4cJXHbNy4sdI2rgwJk0aNGqFFixa4du0aHjx4gLCwMBw6dAjZ2dkYMmQIXFxcWOMnTJiAGzduYPDgwZg7dy7CwsLg7OwMCwsLbNy4Ee+9955evk/VUdXfQF/fqzVr1uD3339H06ZNsWrVKrRs2RLu7u6wtrbGw4cPERYWptfPeu7cOWzatKlG+4SHh+Ojjz7SaGydOnWQnZ2NFy9eqHxoe/HiBQD1DzlV8cMPP+CTTz6Bq6srDh8+jPr169f4GK6urvDx8UFqaioyMzNpXUwXFxecPHkS58+fx/79+3H69GmcPHkSx44dw4oVK3D27Fk0aNCgxucj8A9xEAmEGtK9e3cAlaG8nJwcJQdRU86ePYvMzExER0fT8htMEhMT1e6bnZ2N/Px8lbMJckmVgIAA1vbAwEB88MEH+OCDD1BeXo6dO3di3LhxmDt3LkaPHg13d3etPof8PE+fPlX5vlyWo1atWlodX9X5njx5gqdPn6oU+a3qfOpslG/n/p9ZWVmhV69edLj96dOnePfdd3Hy5El88cUXWLVqFS1HIw/j69shHjlyJOLj47Fjxw589tlnasPLhYWFOHbsGHx9ffHHH38ozdJV9X2qKcb+m6ti7969AIDff/9dyfHW52eV8/jxY6Xwa3V06dJFYwexefPmuHHjBq5du4a+ffsqvX/t2jUAUDvjr46dO3diypQpcHBwwIEDBxAREVGj/eXIZDLk5eUBqOy0wkQikbBC8GlpaZg+fTp+//13LFiwALt27dLqnAR+ITqIBAKH6mYdHj9+DKAynOjl5aX1ebKzswGoDuU+fvyYviGoQ9VF9+jRo8jKykLdunWr7HxhZWWFUaNGoU2bNigtLdUpV0t+Uzh8+LBKzb5ff/0VQGUeHhNra2uUl5fX+Hzy46gKLaanp+PIkSOQSCTo2LGj0vt79uxBRUWF0vadO3cCQLU5ZkFBQZg3bx6ASu03oPL/smvXrsjLy8OJEydq9mE0YMSIEbCwsMDvv/+OvLw87N+/Hx4eHnjjjTdY43JzcyGTyeDv76/kHJaVldEOFRd5qLAmfwtt/+b6pKrfjzqHRJvPKmfs2LGgKgs7NX7VRF8xKioKABATE6P0Xnx8PBITE9GkSZMatUM8ePAgRo8eDSsrK+zdu1flb0JTDh8+jMLCQtSrV09p5pqLj48PFi9eDEDxOyGID+IgEggcFi5ciDlz5iAhIUHpveTkZLz33nsAgAEDBlSbc1YV8qT8PXv2sHIQc3JyMH78eLpYRR1LlixhieZmZGRgzpw5AIApU6bQ20+dOoXjx48rCQc/efIE9+7dg0QiUXmT1ZS6desiKioK+fn5+PDDD1l2x8XFYcOGDbC0tGTZBFTOQr169apaYWsuU6ZMgYWFBb755htcuXKF3l5aWooPPvgAxcXFGDRoEAIDA5X2TUpKUpqt3bhxI+Li4uDr68sKEa9duxYvX75UOsbBgwcBgHX8BQsWwMLCAuPGjVPpFBQUFODnn39GcXFxjT4rUPn/1K1bNzx+/Bjz5s1DSUkJhgwZAmtra9Y4Hx8fuLq64vbt2zh//jy9vaKiAvPmzVMrvC6fDXzw4IHGNmn7N9cn8t/PDz/8wNoeExODbdu2qdxHm89qLN566y2EhITgxo0bWLt2Lb29sLCQ/n+cNWuW0n49evRAeHg4Ll++zNp+/vx5REdHg6Io/PHHHyqLzrjs3LkT//77r9L22NhYTJw4EQCU/qY//PADnjx5orSPqt8JQWQYuWqaQBA8H374IS3HERoaSg0cOJAaPnw41alTJ1qKon79+tSLFy9Y+8nlYVRx6tQpleK9PXv2pOVzBg4cSA0cOJByc3Oj6tevT0uncIVmAYVQtoODA9W/f39q0KBBtI5ct27dWFIua9eupfCfWHOfPn2okSNHUr169aK10D744AOdP8eLFy+okJAQWkpm+PDhVI8ePWjRZKaYs5wPPviAAkCFhIRQI0eOpMaPH0+tWrVK5Xm5LF++nAIqhbJff/11avjw4bQsUYMGDaoUypaLUo8YMYJq06YNBYCytramDh06xNrH1dWVsrCwoFq0aEENHTqUGjJkCBUaGkoBoDw8PKiHDx+yxm/YsIH+vE2aNKEGDRpEDRs2jGrbti39f52dna1kkyaSK3JdTPnr7NmzVf6/WFpa0kLZwcHBlL29PTVlyhSVUkt//vknBVQKs0dHR1Pjx4+nxo8fX62d2vzN5WNVIdcZ1FQKKjY2lj5Xq1atqBEjRtD6h7Nnz1apQ5icnEzZ2dlRlpaWVJ8+fah3332XGj9+PEvvk0/Onz9Pa7C2bduWGjp0KOXv708BoKKjo1XKIcl/r9zrhPx6EBISolaGhytlJRewDw0Npd566y1q6NChVEREBP29Gz58OFVRUcHaRy7w3ahRI2rw4MHUsGHD6G12dnbUuXPn9P7/RDAOxEEkEDikp6dT27dvp0aNGkU1bdqU8vT0pKysrCgPDw+qY8eO1KpVq6iCggKl/bRxrIqKiqgFCxZQDRo0oGxtbanAwEBq8uTJVEZGBn2xVuUgBgUFUSUlJdTHH39MBQcHUzY2NlRQUBC1YMECltgzRVHUo0ePqE8++YTq2LEj5e/vT9nY2FC1atWievToQf35559KNx1tPgdFUVRGRgY1a9Ysql69epSNjQ3l5uZG9erVS6lDiZyCggJq6tSpVGBgIGVlZVVjYeF//vmH6tGjB+Xq6krZ2NhQ9evXp+bOnUtlZWUpjWU6ORcuXKB69OhBOTs7U05OTlSPHj2o8+fPK+2zbds26u2336bCwsIoZ2dnytnZmWrUqBE1c+ZMpYcDOfHx8dSYMWOooKAg+v+gcePG1Lvvvkv9888/rP/rmjiIubm5dLeKoKAglY6CnF9++YVq0aIF5eDgQHl6elJvvvkmdePGjSodsLVr11KNGjWiHVnm378qO2v6N9eng0hRlQLl3bt3p9zd3SlnZ2eqQ4cO1J9//kk9efJE7ffpyJEjVMeOHSknJyf6swqp28ft27epwYMHU56enpSdnR3VuHFjas2aNUqOmRx1DiLzgULdi/s7PnjwIDVy5EgqPDyccnNzo6ysrCgfHx/qjTfeoHbv3q3y/H///Tf17rvvUo0bN6bc3NwoBwcHKjQ0lJowYYJgHG+CdkgoygglbQQCQW9IJBIEBQXppScrgUAgEAiqIDmIBAKBQCAQCAQWxEEkEAgEAoFAILAgDiKBQCAQCAQCgQURyiYQRAZJGyYQCASCoSEziAQCgUAgEAgEFsRBJBAIBAKBQCCwICFmHpHJZEhJSYGzs7Pe+7cSCAQCgUAgcKEoCvn5+QgICICFhfp5QuIg8khKSgppQ0QgEAgEAsHoPH/+vMo2q8RB5BFnZ2cAlX+k6pqfEwgEAoFAIOhKXl4eAgMDaR9EHcRB5BF5WNnFxYU4iAQCgUAgEIxGdaltpEiFQCAQCAQCgcCCOIgEAoFAIBAIBBbEQSQQCAQCgUAgsCAOIoFAIBAIBAKBBXEQCQQCgUAgEAgsiINIIBAIBAKBQGBhUg5iQUEBFi1ahD59+sDDwwMSiQRbt27VeP+cnBxMmjQJ3t7ecHR0RLdu3XDt2jWVY//++2+0bNkSdnZ2qFOnDhYtWoTy8nI9fRICgUAgEAgE/jApBzEjIwOfffYZ7t27h+bNm9doX5lMhqioKPz222+YOnUqVq1ahbS0NHTt2hWPHj1ijT106BAGDhwINzc3fPvttxg4cCCWLVuGDz74QJ8fh0AgEAgEAoEXTEoo29/fH6mpqfDz88OVK1fQpk0bjfeNiYnBhQsXsHv3bkRHRwMAhg4ditDQUCxatAi//fYbPXb27Nlo1qwZjh49Ciuryv9CFxcXfP755/jwww8RHh6u3w9GIBAIBAKBYERMagbR1tYWfn5+Wu0bExMDX19fDBo0iN7m7e2NoUOHYt++fZBKpQCAu3fv4u7du5g0aRLtHALA+++/D4qiEBMTo9uHIBgdiqJQUlbBtxkEgmCokFH4/tRjbDidAJmM4tscs+d5VhES0wv4NoOAyvsFRZnHb8KkZhB1IT4+Hi1btoSFBdtnjoyMxMaNG/Hw4UM0bdoU8fHxAIDWrVuzxgUEBKB27dr0+6qQSqW0owlU9kMk8Meaow/wzcnHrG3HZ3ZBfR8nniwyX55lFuHik0wcvfMS91LzcWBaJ7g52PBtltmyNz4Zq488AABYSID3utTj2SLzJfZhOsb8fBkA8PfUjmhW241fg8yUk/df4ftTCbj6NBsAsG9KRzQPdOPXKANjUjOIupCamgp/f3+l7fJtKSkp9Djmdu5Y+ThVrFixAq6urvQrMDBQH6YTtCCzQKrkHALA62tikV9SxoNF5otMRuG11acwN+Ymjt9LQ3JOMSI+O4YEMmNidJJzivH9qceYvfsGvW3Fofv46M+bPFplnlAUheCPDtDOIQAM+O48vjh8n0erzJNNZxPx7tYrtHMIAG9+fx5lFTIerTI8xEH8j+LiYtja2iptt7Ozo99n/qturPx9VcyfPx+5ubn06/nz5/ownaAFSZmFat9ruvgo7qTkGtEa8+arYw9Ubu/xVSyCPzqA28nkb2EsOq48Sc8cMtn573PsvkKuV8Yis0CKkPkHVb634XQCxm/912zCnHyTWSDFsgP3VL53+PZLI1tjXIiD+B/29vas8K+ckpIS+n3mv+rGyt9Xha2tLVxcXFgvAj8M3hBX5ftR35wjF2Aj8f2phCrf7/ftOZSWm/aTuhBIyy+p8v05MTdRVEqkvIzB4A0Xqnz/xP00nH+caSRrzJtvVUSa5HzwezySc9RPCokd4iD+h7wCmot8W0BAAD2OuZ07Vj6OIFyYIZuqSMmt+oZJ0J0/r77QaFzoJ4dQbuLhHL55+6dL1Y4Z+/O/RrCEkJRZVO2YGy9yDG+ImZNdWIqtF5KqHNNx5UlkF5YaxyAjQxzE/4iIiMC1a9cgk7FvQpcuXYKDgwNCQ0PpcQBw5coV1riUlBS8ePGCfp8gTOISMhH7MF2jsR1XniSziAZmFiPXrTo0/bsRas7J+6/wOK36nM/LSVmkqtnA9F13VqNxqlIBCPqlxdJjGo0buan6hysxYpYOYmpqKu7fv4+yMkUxQnR0NF69eoU9e/bQ2zIyMrB7927079+fzjls3LgxwsPDsXHjRlRUKKRRNmzYAIlEQmsoEoTJsgN3lbbV9XbExM4hKsefvJ9maJPMlv03lAu6PBzVVy4XSEl401C8u/WK0rYnK/qib1Nl2bANsVWnBBC0p7i0AndTldUtFvZrxIM15k1qrnLoOKqpP9oEuyttV/U3MwVMTubmu+++Q05ODl1NvH//frx4URnG+uCDD+Dq6or58+fjl19+wZMnTxAcHAyg0kFs164dxo0bh7t378LLywvr169HRUUFlixZwjrH6tWrMWDAAPTq1QvDhw/H7du38d1332HChAlo2LChUT8voWak5yvnjv45uQPcHW3w09knSu+N/+UKHizrA1srS2OYZ1Z88LuyJFTc/O4oLq1Aty9PI7uIXU2+68pzvBlRy1jmmTU/vtMKEokE60e2Qnq+FG2WH6ffW33kAaZ0q8+jdabLodvKqUsHpnVCmK8zriRl4RCnKOLgrVT0baqsqEHQjQoZhfYrTiptXzm4KZztrFFSVoHwhYdZ7+259gKDWtY2lolGweRmEL/88kssXLgQGzZsAADs2bMHCxcuxMKFC5Gdna12P0tLSxw8eBDDhg3DN998gzlz5sDLywsnT55EWFgYa2y/fv2wZ88eZGVl4YMPPsCePXvw8ccf4/vvvzfoZyPoThrHQfwkqiHc/5u12jGhrcqnw7XHHiltI+ifNsHusLWyhJuDDeI/7YXLH/dgvX/+cSZSTDghnC9UhZZ7N1bMHHo7Kys2FJcSYXlDsOIQW8Lm3LxuaBzgCitLC2wY1Qp/T+3Iev/9HdeMaZ7Z8Nulp0rbejbyhbOdNQDAztoSlxewr08zd2meLiMWTM5BTEpKopXOuS/5bOHWrVtZ63Lc3d2xadMmZGRkoLCwEKdPn1YSxJYzcOBAxMfHo6SkBM+fP8fSpUthbW1t4E9H0AVVmlUTOtellzvW98KOCe2UxvxAQmp650W2chJ+mJ8za93HxU5pTIeVyk/1BN2Yues6az3h875KY4a2Zs+MfHuSPDTpm9ziMlaE49N+jVDb3YE1RpVItqkWSPDJwn13lLZ9O6IFa93HWfn6ZGqYnINIIKjjxD12PuEfk5SdQRsrC6wf2dJYJpktnb44xVq3s7bAnN7KPcx/fKeVsUwyS0rKKnDzBVtn0tJCojRuVXRz1vr60+ShSd+sO852ulsFKUczAGDL2Das9cN3TFuLTwg8Wv4G7KyV04wigz1Y67nFptVkgTiIBLNh8q9XWett63qqHNehnvL2QlIgoTeeqZDwuPdZH7jaK8/AM0OdBP3D/U2sHNRU7VhuAdG+68kGsclc+fk8Owe6cYBqndxu4T6s9fl7bpHKcj3CjTRZW0pgbanaVRrVPoi1Pn6raclAEQeRQODg5mCjFFIbtdk0ZQz4YEMsW3i2Y31PSCTKs1ZyenBuiM+zqteII2jG6Qds6aA3qih4ODO3G2v9w53XDWGSWXIhIYO1HhHoBis1TokqnlTRGYpQM6b/cZ21vu3dtmrHtubM8l55qr7OQYwQB5FgFnBnrb57u4WakZVwQ2rxz3L0bZLZ8vtldsu2b0dUHdJfP4r9/tgtmgmdE6rmqQqnQtUsrhwnW5MTvRAMXMmnjdWkVix9szFr/YtDpD+zvjhwk11JrqpwUU6Am3LntPwS0wkzEweRYBaM3cp2KjrX9+bJEgKXqrQPAShJDCWkk9kSfdDvm3Os9eqcEgBoVtuVtW5qOVd8wX1oUlWgxWRkW3Zo8+jdV3q3yRxRFaqvyUwuYFr5ucRBJJg8FEUhkeNUuNjXfDaE5CHqDreX75k53dSMJBiafM73uZcG+Z5/vc+WWYnjhEYJxsFCRSGRqtxeQs3gyqBxpWxUwY1GbSAOIoEgHlR14Kgq503OtncjWevcak9CzcksYEty1PF0UDOSTRBnHGmBqBva/v9xHRNNegYTqoY7C7t1XBs1I9lw1RauPTOt/Dc+WHnoHr1sbSnRSMomyoSFyomDSDB5bjxnO3aayti8FsoOQ4/46SJxTHQkLjGTXra2rN5Jl/PLOLazLi1X1rQkaA5XHNtehYSHOpjVtSsP3Se/CR05dIud89Y1zEfNSDbcDirfnCDalLry13VFLmhdLyeN9pFIJPi4L1uiy1R+E8RBJJg83ArkmrSmmvF6KGs9o4CI0urC3Jib9HJZheYXUW4y+JL9ykK2BM3hhtKuLnxd430/7stuJ0pmrnTjoz23tN63eaAbvZyYQXJz9cmAiACNx47tEMJa33Hpmb7N4QXiIBJMGl2f5A5ynu5JqzftySxgOyV9aqBxaGNlAWZWADepn1Azvjr6gLXuYKN5Tq6vC7v1Hqnw5w9mGkxDf9W6iQTNOMIRHG9Rx03jfW2s2K7UJ3/d1odJvEMcRIJJU86pSpvTO0zNSNX0auzLWv/pbKLONpkr3JmSVUOa1Wj/uZxOK6UkzKw113Rw6oI9HVnr3516rGYkoTrS8kpY69+/XbMuTkxZonupeSpbWBI0473tbNH49moaKaijW5jpKWMQB5Fg0pSUVbDWx3YIrtH+U7rVZ63nl5BKZm1JTGfnvbnY1ax3eQMfdk5QOmdGkqAdH/ZoUKPxVpYWsGXMmOQUEakbbdl4hv3A2bepbp2DuC0sCdqjSSEjk8UDGlc/SGQQB5Fg0nBzQRxrKPZrZ22JtcMUotnc5H6C5tRyV1QiT+lWr8b7d+U8oQ/fGKezTeZIBWdWfVoNHUQAuLawJ2v9Tgqp8NeGTecU7fXqejvW2Ckh6AduKtLtJb1rfAxuxXN6vvgfYImDSDBpVuqhw0DPRoqnejtr8pPRljMPFW3duodrVqnJhCtY+zyL5INqw+UnWax1SxWaetXBfdB6SuRudKZxgGv1gwgGgTsLrk3XIG4eYpvlx3WySQiQux3BbPhtovqemlXBvFgkpBeSQhUtOPuI3fPXybZm4WU59bwdqx9EqJIRP13Uy3GYMkXb4pL0ckxzxs5Ku9vxAk5VOaHmnH6YpvMxtHnQEjrEQSSYLE84sg8d6nnp5bgdVp7Uy3HMiXc2s1sd+rtVL0Crih81aAdHUI+qVmLa8klUI3r5YmJWFSMJqojnyAM1qaXdDOK7ndgSKyQNpuYs2qeQzXK2077n+K732uvDHMFAHESCybJgr/b6YgTDUtMCFTn1fZxZ6/dS8/Rhjtlw8Ukma/2HUTWrmmXi5WRb/SCCWtZxhK25igmawp25en1NrNY2mSt5jOLDhn7aywVFhniw1sUumE0cRILJ8vBVPt8mEFQwoLnmArTVsezAXb0dyxwoLmVX9XcP184pAQB7G/btQ+w3Q2PTItCdtV4TLUqC/uB+b+e9UTMptKo4cueV3o7FB8RBJJgszK4nDjaatxJTRZtg9sX8GUnK1xjuBXhVdM30D6vi/OPM6gcRaLZeSGKtcxPra0LLOuzfxPrTCVofyxw5n5BBL7ev68nSNKwpIyLrsNbJ9Ulz9t9kN0Pg6nzqQszVF3o7Fh8QB5FgFjTSscvAjJ7slnuXnhDHRFNOPWAngNvVoO+vKnRxasyds48yqh+kIW4ONqz11UceqBlJ4FIgLWdVk8+uoYA/l8/eZGvwXU4iOaGaksDJ2XTSIQeRy/F7ZAaRQBAc3A4FH2ih9caEW+DyMclv1Ji4BP0606dnd9Xr8cyVEC9SEc4XSZwCOl0KIwDAmiMBZa/jQ5g5ce4x+6HJ1kq3/7tgT4fqB4kEk3MQpVIp5s2bh4CAANjb26Nt27Y4duxYtfsFBwdDIpGofDVowHYu1I1buXKloT4WoYZcecquEOwSqt82SGUVJN9KUywt9HuZCXCzZ61ffUpmS7RhiR46P3w1pHn1gwhKPEpj50fb6eiUcOF2LSKo5yrjXjG8TaDOx+NGm8SMyWXFjh07FjExMZg+fToaNGiArVu3om/fvjh16hQ6deqkdr+vv/4aBQXsH9XTp0/xySefoFevXkrje/bsidGjR7O2tWjRQj8fgqAzn+3Xf/HCwn6NsPQfUhRRU5hhFm26dlTHz+eT0CrIo/qBBBbunBCxNgxqWQuzdt+g12UyChYmqAenb7iz6p5Ouv8tmHx17KHOURNzpGWQe/WDqqFvU398uPM6vZ5ZIIWnSCv+TcpBvHz5Mnbu3InVq1dj9uzZAIDRo0ejSZMmmDt3Li5cuKB234EDByptW7ZsGQBg5MiRSu+FhoZi1KhR+jGcoHdeckLM+qA+pxfwk4xCEqbTAKYuW/9m/no//oGbqfj+bb0f1uR4xflNyPRQdcxtDVdYWg5nLSWMzIlyRgQiMtijxi1ACYahfzPdFRa44f7VRx5g5WD9FeYZE5MKMcfExMDS0hKTJk2it9nZ2WH8+PGIi4vD8+fPa3S83377DSEhIejQoYPK94uLi1FSon9HhCBMmtdmC9lyu4MQlCmrkLHWuU62tqwS6QWXT9p+foK1rq0wc1X8xul9TlDNnvhkellfIcmdk9qx1gul5WpGEuQkc7pi2euodqGKnf/WzO8QEiblIMbHxyM0NBQuLuyK1cjISADA9evXa3Sse/fu4e23VU9NbN26FY6OjrC3t0ejRo3w22+/VXtMqVSKvLw81otgeNYO00+eFLdqM7uwTM1IgpxMhtRQqyB3pRknbantzs5DJBp8NccQrcFW6KH3ualTwelmo2uBipx2dT1Z6/HPcvRyXFPmioGqvfWp9conJuUgpqamwt9fOYQl35aSkqLxsXbs2AFAdXi5Q4cOWL58Of766y9s2LABlpaWGDlyJDZs2FDlMVesWAFXV1f6FRioe0IsQZnScvas1cCIWgY5z7cnH1U/yMxJy1fMsIf5OVcxsmaUc26yRRwBaAIbbou9mMn6awnm56Jd20RzJaNAylrnPuzoQucGCrWF0grym6iOF9mKGcRabvr7O7zfrZ7ejsUnJuUgFhcXw9ZWORnUzs6Ofl8TZDIZdu7ciRYtWqBhQ+VG6OfPn8eHH36IAQMGYPLkybh69SqaNGmCjz/+uMpzzJ8/H7m5ufSrpiFvgmacvM/WntLXrBUXrpNCUOZPhlCsj7P+ErUDPdhSEs+yiDBwVRSWssONrfSQjC/nB9Ifu0YcufOStc6NTOgCc3byx9hEvR3XVPmFIRzfu7Gf3o4bzmnXx520EAsm5SDa29tDKpUqbZfnCdrba/aEEBsbi+TkZJWzh6qwsbHB1KlTkZOTg6tXr6odZ2trCxcXF9aLoH+Kywz35NyuLqmWrQm/xD2ll6V6vEiGeDmyuuM8SiOyHlVRwMhH83G21etDU5MAch2rCZ/uu0Mv6zvKf+2ZQrLl0hMi/1Qdbg6KgqreWvbC1oT9NzSPXgoJk3IQ/f39kZqaqrRdvi0gQLO8gB07dsDCwgIjRozQ+NzycHFWFvlR8k1KjiKsObR1bb0e29+V/ZBxmVyE1cLNtXq9oX4vwEwNvhfZZAaxKvJLFA5ix/peVYysOVacqs2iUlIcoSlz+4Tr9Xif9mNrW+YWkTzpqnj4SvFg2ciADzpizc01KQcxIiICDx8+VCr+uHTpEv1+dUilUvz555/o2rWrxg4lACQmVk7ne3vrV5CZUHOYLb/ahnhWMbLm1OGENn+79FTNSAK3glmfYU0AcGBIg5zTYws5U+QPRiXlvVTDFsedeUj+FprSsZ5+nXV/N3Y+KLfNJUHBjec5rHUnPUsNNWWoBHDzTsWCSTmI0dHRqKiowMaNG+ltUqkUW7ZsQdu2belZvmfPnuH+fdUe/cGDB5GTk6M2vJyerixtkp+fj6+//hpeXl5o1Yrk4/AJt5pV3+HmcR2DWevM2UoCG2mZwkF0MIB8RCpDouJCQiapZK6Czeee0Mv3X+ZXMVI7OtRTPIhN/lV9mo2585yTKxvirV8d1bocXVax5r4Zg5P32c6zvnPVvxoq/i5DJqXO2bZtWwwZMgTz589HWloa6tevj19++QVJSUnYvHkzPW706NGIjY1VeUPZsWMHbG1tMXjwYJXn+P777/HXX3+hf//+qFOnDlJTU/Hzzz/j2bNn2L59O2xs9KuIT6gZ2ZyQSs9G+g1rchPKc4pL1YwkXH+RQy8bosq4oT87JPQiu1ipeIWgzJzeYXo/JhF61oyfzrILR/Q9axXkyXYQ80pIiFkd3xhYhSLUV3+qDXxhUjOIALBt2zZMnz4d27dvx7Rp01BWVoZ//vkHr732WrX75uXl4cCBA4iKioKrq2oR2Y4dO8LHxwebNm3ClClTsHbtWoSFheH48eMaF7UQDAezW0R9Hyf4GkCCY0z7IHq5fV39hrBNiTE/Xzbo8ZsHurHWD91Wzj8mKMPVy9MHs3vp3+k0RVztDd9lph5jVtLWyuRu8XrD2AEHbk62GDC5xz47OzusXr0aq1evVjvm9OnTKre7uLhUK4XTs2dP9OzZUxcTCQaE6SD20aNsAZPhkXXo6tycYvKELhTsbUzucqYXuJGS8gr9hx0bcDrkUBRlMHkpMcPMYX6/q2G08j5/qymGbbwIgF2EQVDP0oFNDH6OQ7dT0U8PrfyMCXm8IJgUzO4Bvi6GaZDuwpgF2HddnPIFxuajN/RbrSmHGS4tIq3FVMLViGwTrH+pJguOXosheqGbAkxh5nre+mk7yYWpwfcoTf/5pqZACSc3fWRkHYOfU4yKF8RBJJgU604o8kpc9ShAy8SXIfgc7Ely3jRhfKcQgxyX6eyItVLQ0CRns6MiXGfOEDwmupQq+SUuiV42VMDR1cGabqN4MVF8TokxyONEfozxm9gWJz7FC+IgEkwWQwn4WllawOq/C0pSZpFSGzMCcDeFLaVibWmYSw1T6PY6R7aCUMkPZxSFEe4Ohs+BA4B5MTeNch6xkcMoomteW3Weuz5g5rtxK6cJQB5DF1SfrQ656LN7FB8QB5FgMnAdNRcDJoQz2+ztu5FssPOIFWPN5jGT/v9NyiZSNyq4zuiuMaS1cfq/p+SSEDMXroB4AyNVuXJb+xGACwkKrU59K10wmdi5rsGObQyIg0gwGTIK2U6Jl5Nxnt7mxdwyynnExK3kXHp5eBvDOSXcqtDMQiI7xEQmo1izJe+0C6pitG6MiDSO8ylWpv1+nZfzuhso1UbMMNsdFkkN15r1zRbsohSxPcASB5FgMmTkK5yDYQaeKQn3Uzz9lxqgKlTs7Ln2gl5ONeBskp01W4D76tNsNSPNE+53089V/7JPcojUTdUcv/fKaOfyY8h7bb8ovtw3Y+Jgq38RfznenEmKHJG1PiQOIsFkuM2YtfJwMuxT84yeoQY9vthJSC+kl99qUcug5xrbIZheJr1n2XDb6hkqFxQAPBzZv7lcIgHFGzYM/UOSm8uGO4v3PwPJDQGV3VkGRihmEQ35sGwIiINIMBnm71WEel8Z+IfYNkT/UiGmiqG7bLQOVvR4Jp1t2Jx6oNwa1FBwdQ+zSbifhT9j9pb5UGMIpnQznNMjdpIy2UU7Ps6Gm1UH2NqXz7PFVTCk9ZVbKpXi4sWLuH79OtLT05GTkwM3Nzd4e3sjIiIC7dq1g62tuCt4COKCWbnHbYmnb7jHzyiQGi3nUehwn9A71jdstxlmHuIvF55i0mvk5ijH0QA9sKtidPsgWs6DzCCyaVfXE3vjKwvaRrUzrO5ex/perPWyCplBZ4/FhLTccDmHqmD22z5x7xV6G6iBgyGokYMok8nw999/46effsLJkydRWlr5hMi8IcifIm1sbNCjRw9MnDgR/fv3h4UF+XISjIchwwZyLCSA3CfNLCglDuJ/nHucwVp3MHCHk9JyRZ5dck7VnZDMDaarPrhlbYOfz8VO4azfeJGj1A7RnEnNVXw3nWwNKzdU252tz5qWL0UtN8PJuYiJ848zjXo+5mTCrisvsCq6uVHPrwsaX7m3bt2KTz/9FMnJyaAoCoGBgYiMjER4eDg8PDzg4uKC3NxcZGdn4969e7h8+TIOHjyIQ4cOoVatWli6dCnGjBljyM9CMGO4KvXeRtCfGh5ZB79degYAWPjXbeya3N7g5xQDSRmK/MPu4T4GPx/XCbmTkovGAYbTmBMTLxghrbfbGr5bBLP376rDDzC6fbDBzykGZDKKJVptyMIIOUNb18auK5XFYq/ySoiD+B/MCa2oZv4GPx9XgjuvpIz1ICVkNHIQmzVrhjt37iAsLAyfffYZ3n77bYSEVN8ZITExETt27MBvv/2GcePGYe3atbh+/bquNhMIShy8lWr0c8qdQwC4nEQ6FshhSs28bYQWVtyZ26X/3MXOScRZB9iV/YasYJbThpGbW0BaH9I8Tmd3lnE0Qt9wZu5bRj7pMiSH+b0cZOACOqAytYDJwZupGG6E66I+0Cjua2VlhT///BN3797FggULNHIOAaBu3bpYuHAh7t27hz///JOEmQkGoy4jz2OEkX58nTh5PoRKNjI6d9T2MP6sRX4JcUzkHGaIJHs6Gl4Pz5BdKcRMr7VnWOuWRmjtxoyipJM2lDRfH1e0YzV0gQqgLMUlJlk0jTy2a9euYeDAgTqd6K233sK1a9d0OgaBoI4fYxVOSaivk1HO2aux4RT4xUxRqSIJnJsLZQzucNr8mStp+YpKfg9HG6UblSHg4+9NUA1zZp05k0xQYIxZdS4/nU2sfpBAIFN6BJOAWZxwMdE4Scgj2xquK4Wp4GRgiRuCepIyFPmHWTxJzvxLUi+UiGpq+Lw3gD2D+DAt3yjnFDoVnHasxshV5/I8SzyFdFo5iN27d8eqVauqHffll1+ie/fu2pyCQNAaTyNVE3PDRA9ekovw/hspvJzX0FI6YiS/RCEzM7VbfV5seJopLt03QxHipUiBWTqwiVHOyZxBPHDT+DnaQoQv6SVjqGoYAq0cxNOnT+P+/fvVjnvw4AFiY2O1OQWBoDXvdtQsR1bfLD94j5fzColP993m5bxrhkbwcl4hw8zF9DRwZyEmLnaKWWNmdyNzRi7F5O5grdRxxlBwZ8deiEyk2RDwUcwIAGNEWs1v0BBzSUkJrKxIiIlgWMo5Sb8ORhYHlsOcsTFX+AjZAICvi/FziYTO9D+u08vGckoAINRX0ad864Uko51XqFAURafAuNgbT96EK4y94lD1kzqmDvdeYSzcHcUha8PFYA5iXl4eLly4AH9/4+RbEMyXm5xZCn8eEo8B5fwWc6QPo0vAwn6NeLPjVZ64ep4amma13Yx2rvdJmzcWcQmKnGg+Q+5MO8yVxfvv0suRRmyXasNx1sVyr9B4eq9u3bqs9ZiYGJw+fVrl2PLycrx69Qrl5eWYOnWqTgYSCNXBlTXh9oQ1FjdfkHCalPGE3jjAhTc79l1PJi33GAR7Gq+6uE0w+8Z760UumtY2X+Hytzdd4u3cU7rVw/enEgAAdRl5kATj/iYkEgk8HG3oYrGMAqkooh4aO4hJSUn0skQiQUFBAQoKClSOtba2RkBAAAYMGIAVK1bobCSBUBX3UxWyJs5GrpqNauZPEsAZbP+vDy+grP9laCJDPOiOOvZGECIWMtyqZWM+NHEr1w/cSjVrB5FPOtTzoh3EK0+zebZGWLzbybi56m808cOO/5orpOWJw0HUOMQsk8noF0VRGDt2LGsb8yWVSvHkyROsW7cODg7G1cWSSqWYN28eAgICYG9vj7Zt2+LYsWPV7rd48WJIJBKll52d6j/i5s2b0bBhQ9jZ2aFBgwb49ttv9f1RCBrCzK2Z90a4Uc890gjty8QEUwPRyghiwEyYf4vEdNUPr+ZCBo/CyFxn1Nxzcwe1VHTr+HZEC6Oem5kPSgDcHBS5gOF+xo1wMB1CsaTAaPWYvWXLFtSvz49sQnWMHTsWMTExmD59Oho0aICtW7eib9++OHXqFDp16lTt/hs2bICTk0Jo2dJSeRbkxx9/xOTJkzF48GDMnDkTZ8+exbRp01BUVIR58+bp9fMQakZJWUX1g/RIh3qkm4qcNM5Fz9WICfkA0LSWYpYqmyfdP6Fw/VkOvWyMDipVIY5sK8PBzDdrUsu4M6lejOp1Zrcpc4SiKOQUVT6sNPQ3fvqLD6OA71W+CTuIY8aM0bcdeuHy5cvYuXMnVq9ejdmzZwMARo8ejSZNmmDu3Lm4cOFCtceIjo6Gl5f6m35xcTEWLFiAqKgoxMTEAAAmTpwImUyGpUuXYtKkSXB3d9fPByLUmAHNA4x+ztru9niRXVmlWFRaDgczDW++5DiIgR7GjR4wK0QP3ErF18ONO1sjJOb+eZNeNvbfAQDGdgimK5i5vbLNjX3XFdqgbkZ+aGLO5iamFxr13EKDWSB0L9X43ZaYM4jJ2eIQyzZYFfOzZ8+Qmmrc3KyYmBhYWlpi0qRJ9DY7OzuMHz8ecXFxeP78ebXHoCgKeXl5oCjVz72nTp1CZmYm3n//fdb2KVOmoLCwEAcOHNDtQxB0gg+ZlReMH3tKjjieDA0Bs1iIDy1KZu5bWYW5z1sp4KNiktmn/JsTj6oYadpw7yPGlLlRxYXHGbyen08W77/D6/l9XBT3pvWnE3i0RHMM5iCGhISgdu3a6NWrFy5evGio07CIj49HaGgoXFzY08eRkZEAgOvXr1d7jLp168LV1RXOzs4YNWoUXr16pXQOAGjdujVre6tWrWBhYUG/rwqpVIq8vDzWi6AbxaXskDIfFcyTuyiqZW8l5xj9/EKBWaDCR96ZrRX7ciYTiZSEoVnyZmOjn7N5oJvRzylEzj9mS8twuy8Zm22M36i5cfpBOq/nr8OZyVc3CSUkDOYgUhQFiqJw/PhxdOzYEX379jXUqWhSU1NV6i7Kt6WkqG8D5u7ujqlTp+LHH39ETEwMJkyYgD/++AOdO3dmOXKpqamwtLSEj48Pa38bGxt4enpWeY4VK1bA1dWVfgUGBtb0IxI48NW5gwmzGGPGHzd4tIRfDt95SS+ffmj8izH34eDqM1K1CQAteHDW+BJMFxpFpeXVDzIi4f6kaAVg5ysbC2c79uzxozThF9IZzEGUyWSoqKjAtWvXsHr1aqN0VCkuLoatrfKFSV6JXFysPu7/4Ycf4ttvv8Xbb7+NwYMH4+uvv8Yvv/yCR48eYf369axz2NioTvq2s7Or8hzz589Hbm4u/dIk5E2omhP30/g2gVW5S6hkigB6j564x/93gw+YIWVrSwlvuqBM+OqByzcJjLy/d9oF8WLD3vc70Mtc+SNzZVV0M75NwJMM4eeEGrTVnkQiQUREBGbOnIm///7bkKcCANjb20MqVZZ3KCkpod+vCW+//Tb8/Pxw/Phx1jlKS1X/yEpKSqo8h62tLVxcXFgvgm74CGCmYnKXutUPMjOCeRLlZeYhhvk5VTHSdElgSPwIJRdTaDNpxuKLwwoJLr6cZGbeY6HUPB9mueFcPqqYAaBvU0WnqZe5ws9XN6iDaGz8/f1VFsbItwUE1LzCNTAwEFlZWaxzVFRUIC2NPTtRWlqKzMxMrc5B0J77L/Pp5Wnd+ZFe8mFUp/EtKSIU+EqviW5Vm14+dOtlFSNNl+dZ/LVzYzIwQnEtNFfHhEmLOm68nJeZm5ucI4zvhrGRlvPTg5kL87q46G9+i2Y0QSsHURO5GDnr1q3T5hRaERERgYcPHyoVf1y6dIl+vyZQFIWkpCR4e3uzzgEAV65cYY29cuUKZDJZjc9B0B/uPDpnzf7rFJFVVMpbQ3g+4RaluDrwU63JvBkevfuqipGmSx7jbzG3TxhvdjBzrO6kkDaUUU2V8+ONAbOj0cXErCpGmi7M30SoL3+RhXKRFc5p5SB26dIFixYtgkym/kb48uVL9O7dGzNnztTauJoSHR2NiooKbNy4kd4mlUqxZcsWtG3bli4KefbsGe7fv8/aNz1dOal+w4YNSE9PR58+feht3bt3h4eHBzZs2KA01sHBAVFRUfr8SIQaEMZj1wC//2YRKQpIyjS/p3RuPg0fhREAMKQ1Kfx6zHDMuJWTxuROiuJB/e/r6ov3zAW+Cne42ouFUvML92cXKhzEet78OYgf923I27m1QavKEW9vbyxbtgxHjx7Fr7/+inr12Anpe/fuxaRJk5CZmalR9xJ90bZtWwwZMgTz589HWloa6tevj19++QVJSUnYvHkzPW706NGIjY1l5SUEBQVh2LBhaNq0Kezs7HDu3Dns3LkTEREReO+99+hx9vb2WLp0KaZMmYIhQ4agd+/eOHv2LH799VcsX74cHh7sRvUE49G+nidv52aGMJbsv4Pt49vyZgsfLOaES/gqjKhn5t0iANC9dwHAx5m/fq8f9miAdf9pIBq7g4gQ4Hbz4es3YWXJngd6nl1k9DZzfPNjrOI3ceg2f6knwZ7sB7bMAik8BSwkr9UM4q1btzBgwABcunQJERERtPNVVFSECRMmIDo6Gnl5eVi+fDliY2P1anB1bNu2DdOnT8f27dsxbdo0lJWV4Z9//sFrr71W5X4jR47E5cuXsXjxYkyfPh3//vsv5s6dizNnzij1k37//fexceNG3Lp1C1OmTMH58+exdu1azJ8/35AfjcDhbgo7lYDPas0rSYrQzdlH5idGe43R2o1PJBIJAj0UhWLGbr0oNPgs4mLm3K0zQ7Hs1Ucf8G0CzYc9GtDLz7PE0cVDnzzPVkR1+NTo5N6jhHLdVIdWM4ienp7Yu3cvfvrpJ8yYMQOTJk3C3r178eDBAyQkJCA8PBy//vorWrZsqW97q8XOzg6rV6/G6tWr1Y45ffq00raffvqpRueZOHEiJk6cWFPzCHrk1APhyJh0buDN0gE0Z0ZE1uH1/Mwb4L7ryRjWhl97+ITZvYFgXH679IxvE2hCGKoCD1/lo2cjXx6tMT51PBzxb1KlLuri/o14tkaB0PPVdapinjhxIq5cuQJnZ2ccOnQIiYmJGDx4MOLj43lxDgnmxdfHH/JtAg2zW4WznXn2YpbDnK3gm6dmlg9awMkv47MvuLn3YBYSdRihzfR8ZSk4U4cpMVTLvWZyd4bESeD3Cp0cxKdPn2LChAnIy8uDra0tKIrC4cOHsX37dn3ZRyCoZVgbRUFCVDN+KgTlMBuxM3sSmwvMBHw/V/7y3gDAnVFBvensEx4tMT4ZArr5Nw5g57mJQffNUDSvzW8OpoeDQuFh64Uk/gzhidxiRT6oK8/9sKd2U8ixHRe40oLWDuL27dvRvHlzXLhwAVFRUUhKSsKPP/4IAHjvvffw1ltvITMzs5qjEAja8+tFRQhnUItaPFqizNWn5iMnQVEUnZDfiCcBWibjOobQy/5u/Dqrxua0gNIuuPlWienCby1mKAa1rF39IAPCpwSYEJCHlwHA1sqyipGG5+BthVbzLwLvja2Vgzh8+HCMHTsWZWVl+P7777F//374+Phg4sSJuHbtGlq3bo19+/ahWbNmOHLkiL5tJhCUsLESlub7zRfmo/uWV1xO63t5OvF/I+rL0JtrWcedR0uMDzPEPOP1UB4tUSbFjGcQ+c75c+GEMksFIhxtDIQiHC8n3E88/bC1uqvu2rULERERuHr1Kv73v/+x3mvQoAEuXLiATz75BGlpaUQXkGAU2tflT+JGFVYW/Pe/NRaZhYqwphA6yXgwbNgbn8yjJcbny6OKvNw2IcJyjrlSSOZEgBu/eW/c2Vzmb9bUucNRu+CbmT2F9eBWFVo5iHPnzsXFixcRHh6u8n1LS0t89tlnOHPmDIKDg3Wxj0BQi+9/FZpeTrZKWl98w6fkjrFhSjUIIf+Sm2Mk9EpBQ+Fix2+uFQC0CVY4qbYCm+U3JE8zC6sfZGRGRCpytpnC0abOnmsv+DaBhaejeIq3tPrFrly5EtbW1V982rdvjxs3bmhzCgKhSkrLZUj7LyG/lkDyzLqEKloyyvhqRswDTA3ICgF8bkvO7K25therKwDR8Nm9FK3+ejfx49ES45JRILwZOndGoUp2UWkVI00LZstNO2v+H1JcOA+wlACumeow+P+WoyP/FymC6fEsq4hufC4U2QKm/l+h1HwEmpkXvNcbCk9fzVzEsqXl7M/Jp8SNHGZoVUi6gIYmp0h4M3TM1ItbyeaTI83k76nG6+ymDu4DbFGpcK9PGjmId+7oJ3dEX8chEG48z6GX+WwnxoSpf5hjRk/oB28pqvICeez9y2ROb8XMVexD5T7rpsi1pzl8m6CEhwByUvmAOYO4bGATHi1RkMCoIl956D6PlhgXP4YEWaiv8ApEdlwSbiWzRg5is2bNMGLECNy8eVOrk8THx2Po0KFo3ry5VvsTCFwKSxW5bkIRpq7DcI4S0oWXg2QIKIrCi2xF5xKhVOgx5V62XxTuBVifONjwK9+hCkdb9m+T+WBnylxIUEi88dnukIm5Ousv8yqr5+t6CTOa+ThNuPJPGjmIixYtwoEDB9CiRQtERETgiy++wMWLFyGVqs6zKCkpQVxcHFasWIGmTZuidevWOHz4MBYtWqRX4wnmSy4jhNO8tht/hjCo5WYPe+vKm/TjtHyerTEOUo5cBlMwnE8kMJ8iITnMsGGIQG+GQup+ZEj2XU+hlz0F0lFmSKvA6geZGEzZp8QM4Ty0h/o60cthfvxrx6pDo6mXTz/9FJMnT8by5cuxbds2zJ8/HxKJBFZWVggMDIS7uzucnZ2Rn5+PrKwsPH/+HBUVFaAoCq6urvjwww8xf/58eHt7V38yAkEDchitk1wd+K/WBAALCwk8HG2QnFOMJDNp8SbEXCvAPGdLtpxXdI15IqCbIZNUM9RC5GoQ8gVXLDstv0Qw6TmGYodAowcze4Zi8q/XAADFpfwrP6hD42+uj48P1q1bh5UrV2LXrl34559/cO7cOSQmJiqN9fPzQ+fOnREVFYWhQ4fCzs60v4QE43OKEULku3USk+QcRbg1NbcY/q7CKKAxFBtOP+bbBJU42Aov3GpoPBxt6NSGdnU9eLZGNZmF5pObK6e+j1P1g4wA9zr5Mtf0HUShFoAwZ5XvvxRutKnGjzb29vYYM2YMxowZAwBIT09HWloacnNz4erqCh8fHzJTSDA4zozcplo8i9Cq4+aLXJN3EIWa3/dhjwbYc828RLLrejnRLcWYRTp8M6Z9EN1SrEM9YQnaGwKZjC1bIiRN1IERAfjrv/B3XrFwZ670hSPjQVFIzQuY/eL/uZmK797m0Zgq0FnmxtvbG40bN0aHDh3QuHFj4hwSjMKN/1rZSSTKifB80qm+F71sDu2svBhPwkNb89tvlkmQJzsHL69EmKFwfcKsnK3lJoxqcgB4p30wvczMzTNVShnC7ELLBQ1n9Eo3h9/E5wcV1drlMuHoDbo5sMP93IcKocC/aiSBUEOuPlUIHwtNY/StFrXoZXMQo20VpOiU8QajB7IQYOqNpTBC/6YIRVE4cV+RduEmkLxcgD1bYg7EMzoLCS0XlBlmzi02fQeRyfA2winSceOE+7nFfkJBq6mXZ89qJnhap06d6gcRCBqy5phwKyGZxRG3zUCMltlaL0Ig1eRyQrwcaQmJG89zEC7gakFdyZeyw4V21sLJweR2jjB1PvnrFt8mqIXZTSXLzPJBZ/USTtoFtzXs/Zd5aFFHWL3TAS0dxODgYI3zKiQSCcrLTT/XgWA8hNAhQh0u9grbdl15gVXRpq39KZ+FsLSQCKpYCGDri8378xaGtTHdB1Uh9MBWhzXnZphbXCa474o+EbIGqqeTwkHcHvcUU7rV59Ea4+ItED1KVaw6/AC/T2rHtxlKaHWnfe2111Q6iDKZDM+fP8ezZ88gk8nQvn172NiYn9wEwbDUZrTWW9ivEY+WKBPiJYyKRWMhz3vzcLSBhYCSwAGgZR03XGOE+0yZp4xQZvNAN/4M0YDvTj7Cgihh/W4Nxej2QXybwMKR8XAtF5A2VcQUQo9LzKx+EA9o5SCePn26yvcfPnyICRMmgKIoHDp0SJtTEAhqOXL7Jb3cLUxYRVHc3JLMAqlghHL1DUVRyCyoDFN5ClB3MDLE02wcxE/23aaX76fm8WhJ9Vz+r9LaHJjYuS7fJrAQiuSOMXiaKdyZXAAI9nQQvF6uQYpUQkNDsWfPHty9e9fo3VOkUinmzZuHgIAA2Nvbo23btjh27Fi1++3ZswfDhg1D3bp14eDggLCwMMyaNQs5OTlKY+Uhdu5r8uTJBvhEBC4pDLFddwdhOSbcWTRTLlTJl5bTFZteAnSC2wQLL6fHUCQywppCTXiXI6x5ZsPCjHYIARsr9i0/s0B1NzRToKRM8TsQksSNnI/eCOfbhGoxWDKXl5cX2rZti507d+KLL74w1GmUGDt2LGJiYjB9+nQ0aNAAW7duRd++fXHq1Cl06tRJ7X6TJk1CQEAARo0ahTp16uDWrVv47rvvcPDgQVy7dg329uwfekREBGbNmsXaFhoaapDPRFCPkKo15TB138QU5qgp8tlDgJ3bJBS6h/vwbQIvrBrcjG8TlOjd2BdH7rwCADwQsDCwrkjLFcLMwZ4OgtJAVEVRaQVMVZly3QlFMaOTQLrZMOnR0JdvE6rFoP9rFEXh1atXhjwFi8uXL2Pnzp1YvXo1Zs+eDQAYPXo0mjRpgrlz5+LChQtq942JiUHXrl1Z21q1aoUxY8Zgx44dmDBhAuu9WrVqYdSoUXr/DISq4Wp3CfEC7OGomE0zZQfxSpJCbshJQFqUciQSCcL9nHH/ZT6sLCSQllfA1ko41b2GQog5iK2C3GkHsbhMmN0t9EF6PkOLUmCzh6ooFHCbN105/1iR11cswI4q1pYW8HOxw8u8EkFOdAAG1EGMj49HbGwsgoKMl6QbExMDS0tLTJo0id5mZ2eH8ePHIy4uDs+fP1e7L9c5BIC33noLAHDv3j2V+5SWlqKwUNh5DqbGuUcZfJtQLcxK5lsvhJ0PpgsrDylEaC89yapiJH8E/yeYXS6jkFFgmuF+iiMGyg0jCgF7AcnuGJLtcYrOQkwHRUj4uigeYPffMH3hckBYnYWYeDlXRl4KSsqVfsdCQKvH/s8++0ztewUFBXj48CEOHTqE8vJyvPfee1obV1Pi4+MRGhoKFxe23llkZCQA4Pr16wgM1Fws8+XLymIILy8vpfdOnjwJBwcHVFRUICgoCDNmzMCHH36og/UETdhy/gnfJlTLS0aO5NrjD/Hh6w14tMZwNK7lijMP0wEAI9sKU0KGKaeSX1IGQPizOjWF2yGC6QAIhX7NArBw3x2+zTA4P55J5NuEanmVp5jl/P5UAub0Fn4unDY0DnDBnZTKB/ThkcK8PskjL+UyCtJymaD0SwEtHcTFixdDIpFU6fE6ODhg/vz5mDlzptbG1ZTU1FT4+yt3c5BvS0mp2dPSF198AUtLS0RHR7O2N2vWDJ06dUJYWBgyMzOxdetWTJ8+HSkpKVXmW0qlUkilih9nXp7pzi4ZitIK4T1lcRHiDI4hYCa4tw0RZiYTsw1jgYC1AnWhSMoOnwlRJ9SdUeXuLMB8MILpIXcOAcDRRliOlxwnW8UDbHq+FIEewmmRCWjpIG7ZskXtezY2NvD390ebNm3g6GjcPpTFxcWwtVV+erazs6Pf15TffvsNmzdvxty5c9GgAXsG6O+//2atjxs3Dm+88QbWrFmDDz74ALVrq+5Ju2LFCixZskRjGwjKvMxV/A3faScsjTE5/+taD9+efMy3GQaHeQEWYg4iwE5Ov5CQidbBHjxaYxg2xCbwbYJGtA5yx5Wn2cgvKUdJWYXgZkv0QdsQDzrd4v2u9Xi2RjXTejTANyce8W2GQeH2NhZirjoAVMgUldaxD9MxSmD3NK2u6mPGjNG3HXrB3t6eNUMnp6SkhH5fE86ePYvx48ejd+/eWL58ebXjJRIJZsyYgSNHjuD06dNqi1e4M6p5eXk1CnkT2OERZsWgkHCwsUJdb0ckphcK9slV3wg1IZ/Zg3nNsYeY1sP0wv0/iMRBDHCzB55WaiC+yC42SU0+Zi6u0G72cqZ2q2/yDqJYim+KGMUzn/x1W3DfGZOKhfn7+yM1NVVpu3xbQEBAtce4ceMGBgwYgCZNmiAmJgZWVpr50HJHLytLfbK+ra0tXFxcWC+C9jQOcOXbBLXI9RkLSytQViFsXTpteJzGliqxFKDOGACT/L/nwsyz7BIqLOF4JqUMfcafRZBLrCtCrUw1hxQYpkB+XS/jRjJrghAVB5iY1DclIiICDx8+VMrtu3TpEv1+VSQkJKBPnz7w8fHBwYMH4eSk+RNuYmJlcrK3t3Av0KbGkNaqQ/lCwIUR2hRyn1xt+VQkBQfeAhTw1jc9GHqPC/s15NGSqklidLb47dIzHi0xDkLMBZXTrHblw7WFRLkK3hRYfuAuvZyYIVylkfrewp5FNykHMTo6GhUVFdi4cSO9TSqVYsuWLWjbti09y/fs2TPcv3+fte/Lly/Rq1cvWFhY4MiRI2odvaysLFRUsEObZWVlWLlyJWxsbNCtWzc9fyqCOoQsnVHGKKa5/tz0Wosxn3w71BNmgQoAvCOwXriGgBlOc7YT5qwVwO4c0TrI9LrcPBGwI8Kl5D8tShnF7jhiKjA7bLnaC/c3IfTZXOE+4mhB27ZtMWTIEMyfPx9paWmoX78+fvnlFyQlJWHz5s30uNGjRyM2Npb15NSnTx8kJiZi7ty5OHfuHM6dO0e/5+vri549ewKoLFBZtmwZoqOjERISgqysLPz222+4ffs2Pv/8c/j5+RnvA5sZOZy2dUJNPAaAq08VTuH2uKfoHi581fya4OusmJkb3FK4M7lBnuzwUmJ6AeoK/Km9ppy6n04vC7VYCGC3nbvy1PQemtLySqofJBAeviqglw/cSkV0K+H+hrWhga8TnQ+6Klp4nYXkdG/I7vYktOIt4V5NtGTbtm1YuHAhtm/fjuzsbDRr1gz//PMPXnvttSr3u3HjBgBg1apVSu916dKFdhCbNm2KRo0a4ddff0V6ejpsbGwQERGBXbt2YciQIfr/QASa1FzxXID93ezo/rinHqRXM1p8bGbkkFlZCtdR5/I8u9ikHMS8kjK6HzYAOAi4KMpewCFXfSCWwgguR++8NDkH8eCtl/Ry4wDh5vq7cGb8U3NLECKgnEmT+8Xa2dlh9erVWL16tdoxp0+fVtqmaR5Gq1atlGRuCMaB2UWlh8D77Ao5rKEPnmcVM5aLeLSkZkhNrM0bt6+xkGfVa7kJs9JdXzDban4o8Gr5jvU96U4v4f7CdaC0JatQEW1ihpuFiIONJV3NnJBWICgHUdgBcAKBwfF7ir7eLwUezpnQqS7fJhiNIa3FI9V0LzW/+kEiglkZTOCXlBzFNSnIU1iCx1xmvB5KL+eZcL94QNiz6gBb6mbunzd5tEQZ4iASRENkiELkeGyHYP4M0YC+Tc0nF9XHWdiVwi3quNHL2y8m8WaHIWBWyAt48lAlpiZBlJ6v0Gj1c7Hj0ZLq8Wb8ZjMKlLWDxQwzGmghEfasOhfmzKcQ0CjEvG3bNp1OMnr0aJ32JxAAsLqT1BFYSyIuEokEtd3t8SK72ORai3FnrYR+AW5e2w3x/+miZRQI6wKsK3/FJ9PLywc25dGSmnPzRS5amVA1864rz+llb4E/NDHtu//StGbVX2Qr0l/EIODD7L4T6ius/GiN7lxjx47V6iZAURQkEglxEAlmibuDDV5kF6NQWk7/FkyBXy4k8W1CjbAVuJSELhy+o0jG93AUdq4Vl6+PP8T28W35NkNvMEOFjgKuJgcqNRpDfZ3w8FUBkjIKTer6xGwB2qy2G3+GaMhHb4TjrfUXAAAtAoX1wKTRt/jTTz9V+vIkJCTg119/hYODA3r16oXg4GAAwNOnT3H06FEUFhZi1KhRqFdPmP0oCeIm1NeZbxOqRT7TJqOA9AIpfJyFHXbSlOvPc/g2oUaE+Qn/u6IPxOYgMmd6TA0ha7TK8Xa2xcNXBSiXUSiQlgtaQ7MmlDAK0SKDheVwqcLfVVG8lVMsrAiHRg7i4sWLWeuPHj1CZGQkRo0aha+//hoeHh6s97OzszF9+nTs378fFy9e1JuxBPOlnJOvJNQ2VkwevFKEbv736zX8+b8OPFqjP+owEvDFED5/M6IWZu66wbcZBsddBL8JJqbUYaiII3Ejht+FpYViZj0tX2oyDmJeiaLoRgwTCcx72ZE7ryCTUbAQSOtSrWIv8+fPh7u7O7Zs2aLkHAKAu7s7Nm/eDDc3N8yfP19nIwmEwlK2PInYwiFXTUgYmPk/v2qwcEVo5VhaSNC+rqLbC/MGYkq4i2AGcffk9vSyKVXPMruoONtawcpS+GkNZx4q9Fn330jh0RL9wtTLFYOjzhXGvpiYyZMlymj1LT59+jTatWsHS0v10+hWVlZo164dYmNjtTaOQJDDfELv01gcFcJCr+7VFmalHbdTiVAJYGjwJYmoJVpVMKtmAcBNBNqbzBZ7Yrh5a8q7W/+ll/Ol4pgZ7dzAi152NCER85cMB7GWm7CLGVVRLhNOaY1WDmJxcTFSU1OrHffy5UuUlAhbr44gDpIyFGLMQvoBVcU/0zrxbYJBYFYCezkJf9YKAHxcFM56odQ0xLK5YU0xzFpJJBJaCDinuEzjBgVC51WewlkXi0h+/+YB9LKQZq10ZS+jst9DJNcnZlV5iYDE/LW6ojRr1gxnz57F8ePH1Y45ceIEzpw5g2bNhB+CIgifKb9do5eZgtlCxstR8aMXy01DE7IKFTdDMYQ1AXa3l7iEjCpGioeSMkVeLlMjVOjIO6pU/FccYWoIubUbE6ZTIrKMHbVwc9WF3JucSUuGVuuvl57xZwgHrXMQZTIZ+vXrh3fffRdHjhzB/fv3cf/+fRw5cgTjx49HVFQUKIrCRx99pG+bCWYIsypwYERAFSOFAzPROLe4DIUmcjOUzyC62lvDWgSzVgBQj9F/eee/z6sYKR6KGTMNjUTULs2VkZSfU2Q6eYhyKkQS4YhgSMAUC2jWShdyOXmtjgLvoiInOUdR0c/MDeUbra7uAwYMwPr162FhYYGtW7eib9++aNy4MRo3boy+fftiy5YtkEgk+PbbbzFgwAB920wwQ5oHutLL0wTe51QdZx8J54evLRRF0W0OfV3Ek2PZsb4i3yot3zQ6R8Q/UxQ+JaQX8GhJzShhFJydMYHfBJda7uLoOe3CiGrcZWgHiplszgOHGNIuAGBu73B6+a0WtXi0hI3W86+TJ09G3759sXnzZpw7dw4pKZVVUP7+/ujcuTPGjRtHayMSCLpy8JZCEFgsYQMupRXimFmoiuyiMlrf0c9VHDdCQFzOrKYcYYhki7XH9L9PsjCybRDfZugEN49yarf6PFlSMywZEY7sojJByatoSyajbaDQ27EyqS3Qhwqd7rR16tTBkiVL9GULgaARDiJ1ELn5MWLk/kvFTIOviVZpi4Vmtd1wMbGyRdeHPcThlABA36b+OHE/DYDwO45oAjesWdtdfJWzQKWYv6/Ae0hXBzM64Ocqns/C1KDcG5+MtcMi+DOGgTjmXwlmTRnHsXIQQZcCVYitA4kqfohNpJcLS8WTU8mddTaFfNDL//VvBYCejcQh/QQATWop0kW4v20xwswfq+ftCBsRtXasxZB/YsrDiJXvTz2ml8Ug+yRHqJJPOn2T7969ixkzZqBjx44ICwvD3Llz6fcuXLiAb775BllZWVUcgUConswCdvshMYVBmFqI2+Ke8miJfmA6540DXKsYKSw8ndiznY/SxJOzpw65DqK7g7WoZktc7BU3w11XXvBoiX7Ywag6FVsSCVPqRkwPfOpwd1CoKohpdprbL14o8k9aO4hr1qxBREQE1q1bh7i4ODx+/BgZGWz5iBkzZmD37t06G0kwb5hhzXfaiStfqW9Tf75N0CuWlgrnfEBzcVSTq2LT2cTqBwmctPzKGR9vkYX6uS3d8kXe2SaFMYPYREQPTQBwL1VxbT1xL41HS/RDHEPPsX09zypGCgtuZ7BDt1+qGWlctHIQDxw4gNmzZyMwMBB79uxBWlqaksfboUMHeHt7Y9++fXoxlGC+MKUw6nmLo3OHnKGtA+llU9BCZLZHcxHZ52FqBTYSiVadOjILpCj7r+gpVWShQa70SHGpuCVWmP1+h0cGVjFSeDzPVuiDbj73hEdL9A9zNlFszNktjN7xWjmIa9asgaOjI44dO4aBAwfCy8tL5biIiAg8ePBAJwMJBGYSOFNDTQwwO3hwk9nFiNxBlEgqe86KiQmdQuhlmUi06tTx4xnFDGh+ibhCg9zZkiKRO4g5RYoUGE9Hcc3mLnuzCd8m6A3uJJWliFKRuBQK5DehlYN49epVtGvXDnXr1q1ynJeXF16+FMZUKUG8sGat7MTlIJpSj1MAuPEiFwBAUeLKBQXYXV+4emliI8hTUSlrL9KiLTliFy5nPcCKbFa9dbB4OvBUh9jFvpnRscEta/NoiQKtHMTS0lI4OztXOy4tLQ1WVqZ1gyQYH2ZvTbGFNe054TQxV8+KPRTIDDllF5VWMVL4MNMuJnYOqWKk8Pn1oriLt47cUbT+dBNZhMPGygLWluJ60FMHM59SjCyIakgv+wuk6EwrBzEkJAQ3blQdIy8tLcXNmzcRGhqqlWHaIpVKMW/ePAQEBMDe3h5t27bFsWPHNNo3OTkZQ4cOhZubG1xcXPDmm28iMVF1MvvmzZvRsGFD2NnZoUGDBvj222/1+TEIDBIzCullsc0gAuxijnQRd/EQezGBO+Pm/SKruIqRwmf1EUXqzoNX4hTJluNoK+4ZUCZ2IpzNLWMI+GcVivfBKYOhdtE1zJtHS7TDyVZxffrzmjCq+7VutZeUlIQ1a9aoHbNq1Sqkp6dj0KBBWhunDWPHjsWaNWswcuRIrFu3DpaWlujbty/OnTtX5X4FBQXo1q0bYmNj8fHHH2PJkiWIj49Hly5dkJmZyRr7448/YsKECWjcuDG+/fZbtG/fHtOmTcMXX3xhyI9GAOAkUL2oqvBghDazRDxzdTsll14e2loYIZCawAz/XU7KEoyUhK5ENRNfNfmaoc3p5e7hPjxaohum8h2S882JR3yboDXMavL+IvxNMGefhVJ4ptXddu7cudixYwfmzJmDS5cu4a233gIAvHr1Cnv37sXevXuxY8cOhISEYOrUqXo1uCouX76MnTt3YvXq1Zg9ezYAYPTo0WjSpAnmzp2LCxcuqN13/fr1ePToES5fvow2bdoAAN544w00adIEX331FT7//HMAQHFxMRYsWICoqCjExMQAACZOnAiZTIalS5di0qRJcHd3N/AnNR9KOHklTGFXscD84eeI2EHMKlTMIIpJDFgOty9ram4JAkT4feIiRgeLWVEel5BZxUhhI/bZWy5bLyRh8YDGfJuhFcwqbH83YYRoa4IQ721aXeXd3d1x/PhxNG7cGLt378bIkSMBAIcPH0Z0dDR+/fVXNGzYEIcPH9YoV1FfxMTEwNLSEpMmTaK32dnZYfz48YiLi8Pz5+qToWNiYtCmTRvaOQSA8PBw9OjRA7t27aK3nTp1CpmZmXj//fdZ+0+ZMgWFhYU4cOCAHj8R4SHjAhzdSnyzVgB7BjG7ULxhWmaxUBuRJrczw8wv84TxlK4rXNkYMcD8TSRlFlUxUtgcuf2q+kEEo8DMyw0UYbtDB87vWAiTCVpPA4SGhuL69evYu3cvJk+ejDfeeAO9evXCu+++i507d+LGjRuoX9+4/UHj4+MRGhoKFxe2xllkZCQA4Pr16yr3k8lkuHnzJlq3bq30XmRkJBISEpCfn0+fA4DS2FatWsHCwoJ+n6AfmFPtQR7i+9EDgBujOOJWcm4VI4XN+ccKIXwx5oICwITOCuWFNBE7iHbWlZfuWm72SrIxYsCBU90vVtmhEEblaYBACgvMlQJGAWCgCO8VEokEHesrxL1f5fGfr65TQpeFhQXefPNNvPnmm/qyRydSU1Ph76/cuUK+LSUlReV+WVlZkEql1e4bFhaG1NRUWFpawseHHdaxsbGBp6en2nMAlQU0Uqnij56XZ9iqq8T0AozZchm+znb4bWI7UYYFn2cpZhfEGg5kypCIOYRz4r6i0wK3Olss+LoobuJCuABrA0VRKCmr7GEsNlkVddx7mSeq1o1yMgsU36FpPRrwaIn2vN+1HtafTuDbDJ3xcbZFWr4Ufi7iddQjAt1w/nFlysXLvBKE+RkvAqsKrTyGM2fO4Pz589WOe/jwIc6cOaPNKbSiuLgYtrbKQqV2dnb0++r2A6DRvsXFxbCxUa3Qbmdnp/YcALBixQq4urrSr8BAw6ruv7P5Mp5nFePK02ycfZRu0HMZCmYFc4jIuqjICfPl90duCJrXduPbBK3wZQiXvxLpDCIzJHtX5NIecvbfSOXbBK1gpimI9QG2aS3xOeZcKmQU0v5TiBDjRIicIA9H1PdxQqf6XrATwOfQyoKuXbvitddew8CBA1FYWKh23IoVK9CtWzetjasp9vb2rBk6OSUlJfT76vYDoNG+9vb2KC1VnRtQUlKi9hwAMH/+fOTm5tKvqnIidaVCRiGZUdWVI1Jh4JeMELMY80oAoI6nOO1mUlouY62LdQaRmfsmVufqSlIW3yboBaZj0ry2OJ2UIqmiiE5sGohyujdkR8PE+OC05byiQOVZlnhzWoe2CcTxmV3w64S2aFuX/17SWruoFhYW+Pvvv9G+fXs8fSoMoVN/f3+kpio/icq3BQSoLn338PCAra2tRvv6+/ujoqICaWnsxualpaXIzMxUew6gcobSxcWF9TIUlhYSLOrfiF4vKRenyDEzUVesF2AAaMuo2iwQoVj23nhh6HLpiq2VwrE9/UCcs+oVjHy9/s3FJ+chh1l0Jtbr03aGyLejyFpPymH+JgAgTYSpF2J92BM6WjuIb7/9NqZNm4bbt28jMjLSqKFkdURERODhw4dKuX2XLl2i31eFhYUFmjZtiitXrii9d+nSJdStW5euxpYfgzv2ypUrkMlkas/BB8xcjMdpBTxaoj05/1XOOttawdqS/yl3bWGGn17mik+k+dErcX5/uMhMQLdu8f479HJUUz8eLdEN5ix0rkgjHExsBRAS1AflMln1gwRGJENVYVZP4zbnMGW0/kZbWlri66+/xsaNG5GTk4OePXti48aN+rStxkRHR6OiooJlh1QqxZYtW9C2bVs65+/Zs2e4f/++0r7//vsvy/F78OABTp48iSFDhtDbunfvDg8PD2zYsIG1/4YNG+Dg4ICoqChDfDStYD7RbjmfxJ8hOiAPjbuKePYQYLdOSskRXwhnD6PdoZgJ9hRnHisTeYEKoDz7IyY8GeH+hyJ8gL31gq1IIObiCCblIqwoL2PYLNZcUCGi85z4hAkTEBYWhsGDB+N///sfbt26hXXr1sHCwvhPU23btsWQIUMwf/58pKWloX79+vjll1+QlJSEzZs30+NGjx6N2NhYlgr++++/j59++glRUVGYPXs2rK2tsWbNGvj6+mLWrFn0OHt7eyxduhRTpkzBkCFD0Lt3b5w9exa//vorli9fDg8P4ejDWYm8x2aFjKJbP73IFt+sGxN/1gyi+BxEMbfgYmJjZQFfF1u6grmkrEKU7dHkiPnBqYGPongrQ4QtKC8msgW+uULsYsLZ1gr5/6W+JKQViE7n9DbDWRf7fU9I6OUb3blzZ/z7779o0qQJ1q9fj969eyM7O1sfh64x27Ztw/Tp07F9+3ZMmzYNZWVl+Oeff/Daa69VuZ+zszNOnz6N1157DcuWLcPChQvRvHlzxMbGwtub3dfx/fffx8aNG3Hr1i1MmTIF58+fx9q1azF//nxDfrQaw51dyC0WVxjnXxNJxgcAN4YcSZ4IexpbWiguup8wmsqLkRaBik5HYnd8WwS68W2C1jgz2mYevSs+wWkfRkV8iJe4Z6ZHtguilx+JcDa3sFSR1+3pqKxGQtAOvWXVBgUFIS4uDqNGjcJff/2FyMhIlbqChsbOzg6rV6/G6tWr1Y45ffq0yu21a9fG7t27NTrPxIkTMXHiRG1MNBrNOJWBmQVSUemm5ZcofvR1RSpxI4eZP/nX9WSWYLMYYBZGtA3hv7pOF7ycFaHNjAKpqEJSzGry5oFuohTJlsPtq05RlKg+Tx7j+vR+13o8WqI7nep74YfYSi3Esgrx5SCmM2agW9Rx488QE0Ovc+IODg7Ys2cPFixYgISEBI20EgmGg1vUkSOyGURm8/WJInOouKTnK8LKt5PzWOkNYoBZQV5HhF0KmDBnGMTW2YZZrenrLO6ZEmtLC1Z7MWZupRjYHpdELzuLtLOQHG/Gd2lbnDBUSWqC3EF0tLEUbTW5ENHKQRwzZgw6deqk9v2lS5fi999/R3BwMIKCgtSOIxie6a8r1P2zCsQVTnvCEMkWYiPzmtCnCXs2vVRkT+nM4g5nO3FfgJnO+dWn/KTCaAtT9ilNhHl7XCIZ8k9SkUndPGRU9qcXiPtvwe0DLDbkDRWYbU0JuqOVg7hlyxa8++67VY4ZNmwYEhISkJiYqJVhBP3g6aR4MswWQPPvmvDHvwoh8Vru4nYQvZzYFy4xFapQFIXrz3PodQsL8YQBVcEMKe+5Jq7q7OJShRPVp4l4JW7kXGM46PdS83m0RDdCRF4dX5tzfa0QUSUz81qaJ7IomdARb9kVQSOYOYdiK1IpLlPcDF1EHsLh5lYxnV+hc+2ZuGbZqoP5vRIbzC4RjiKf9QEAph9y80UOb3boSsf64s7L5V6fxCTmz7yvibXDk1DRKFYkF8GOjIyEnZ1djUWxq6sgJhgOpoO4/0aK6Ioj5Ig9rMnFSkSzcMxkfFMgulVtLNl/l28ztGLFIYV+q72N+H8T4zuFYN2JRwCACwmZeK+LeIo9AlztkJJbAm9nW1EV12hCobRcNAWNB24pOqCJPdQvNDS6wnTt2hUSiQT37t1DaGgova4pFRXifWIXO0x1/xsvxJWQz0TMWnVyRkTWwe+XnwEA3B3FkyuTzZCCEXsuKCD+ggI5BSKUS+LSKEDRbjRDZDf3lP9Cm84mUhTRso4brj3LASCuGcRv/nvAAACR1f4JHo2+2aNHj4ZEIoGrqytrnSB8HDmzDMWlFaKYhhdTjp6m9GniRzuImSIqGJq56wa9PKR17SpGipNXeSXwFUEXDG7le6/G4s9BjGDoOD7NLFI/UGAwZVVsTeDhFQAyGNekvfHJmNcnnEdrNEc+kwuQNnv6RiMHcevWrVWuE4RLk1ourPV8aZkoHMR0E6jQ5MJsLZYpUoFmMYXGNSUhvUAUDiI3h1hM+o3qYIYxxTRrxXyA5RagiRVmfutvl56JxkG0Y9zP3mpZi0dLTA9SpGLicGd6j9wRR8eCUkZawrsdQ3i0RH94Mm4kmSILp8kZHlmHbxP0AvM79eiVODpHFJaaXqoOMwVGTKQxdE1bBblXMVI8fDG4Kb38hogq5BPTFXJoROZGv4jz10nQmo1nEvg2QSOScxQXYDcR95tl4sGYQRRLizduVwU3kSSuV0fnUC96WSyzuQWMYiFuZECscB9gxSIg/9ulZ/Syj7PwZ581oTkj3L9TRCoLTEyhsl9IaBRi3rZtm04nGT16tE77E/SHvUjyZe4wOlw05bQMFCu2VpawsbJAabkMV0Qi0MydXbOyNI1nSqZsUr5Iij1eZCtCgG2CPaoYKV7230zFgOYBfJtRLSfup9HLjrbiuKZWB1dKTAytD0s4klVCt1dsaOQgjh07Vqv/ePkXjDiI/DK2QzC2XkgCALwZIY4cDWYOYqC7uFu7MWH20k3LK4GPwHPfxCSYWxNcGLJJW84nYVH/xjxaoxk/nVU0HYhLyOTREsMx7fd4UTiITEK8xC2SLcfLid26UVouE7x6BFMYu2cjXx4tMU00chA//fRT4pmLmI71vWgHMSFN+PlWFEVhT7yiw4WpJIFzScwoFL6DKJKQX03xFmEf48eM364Y7TcVZJyHpma13fgxRM/YcPJB0/KkqOMp7Ifzey8V3XdMQYJLaGjkIC5evNjAZhAMCTM0tSc+GWuGRfBnjAY85IQ1xd5FhcnrDX1x/F5loRBzNlGolIusZ7SmcJPZZTJK8C0E2wR74NDtlwCAub3FUWGqCdN6NKC17Op6C382Lp+RC/paqDePlhiWTecS8dmbTfg2o0r+fZJFL4vhuyM2TCOhiFAlYb7OfJtQIwqk7Jwwod+4a0LrYEXF48NXwu89m8YI9dczsQtwp/qKQpXCUuFLrMQlKsLKdTyEPbNTE/7H6J7iIYIqVKbckFi6jWiDGOTQvjv1mF4Wk46mWCAOohnQvp6iT6iHCDp4FEgVicdilcFQB3PWcNmBezxaohmJ6YrZ3EEtTUskm9m+UQwafDlFCsfEyYRaTzIdkStPswVfyXyd0TP6eZZpOSXD2wTSyyUik1XiFqwQdEenq8yzZ8+wf/9+PHr0CPn5+Sp/2BKJBJs3b9blNAQdkUgkaODjhEdpBcgpKkVpuUwp30RIMJ/Q5/QO49ES/cPUTxMDXx59SC+3rGMaem9ynBgt0vJLyuEvomJ5SxOaVefy8FUBwvyEG/XYxZCAuf48hz9DDABTa/OXuKdYIvAQM5MJnevybYLJobWD+Nlnn2Hp0qWQyRQzInIHUV7QIq9iJg4i/9TxcMCjtALIKCCvpEypYk1IfHHoPr3sYmIhnP91rY9fLz6rfqAAMRU5DznMnsx5xcKWuskWiVajPuBqbwqNjvW9cO5xBgDg/a71qhktLopEMJOuDn9XYRf8iRGtppH++OMPLF68GIGBgdi4cSN69uwJADhy5Ag2bNiALl26gKIozJw5EydPntSrwQTtYDpazCRrIZKcU6xYEXa0qcaIqSK7iJOXJ4FpzVr5uCgekpjVkELkbmoe3yYYDYFHmJFTrHDWmb2kTYGZvcTTy5gbsRS6JI8Y0cpBXL9+PWxsbHDq1CmMHz8e/v7+AICePXvivffew8mTJ/HVV19h3bp1sLQkfzQhwAynXX8uDpFmgN1n0xSwtmD/5IRcyfwiu5i1HurnxJMlhoHZT3fhX7d5tKR6mN1e3mkXxKMlhiGqqT+9zHTAhAiztVttE9JoBYBQERU0MnNyCYZBKwfx5s2b6NChA4KCKi9UzJCynBkzZiAsLAzLli3Tg5kEXXmSobio3U4Wz2yEqRWpcCuyt5x/wpMl1cNtB2hrZVrO+muMdntCJ4Uxq97Q3zTa7DFhftc2nkmsYiS/UBSFY3cV/ex9XYSbqqMN1pxOSULOmY65+oJvE0were6+UqkUfn6KZt52dpWx/5ycHNa45s2b499//9XeuhqSk5ODSZMmwdvbG46OjujWrRuuXbtW7X4ymQxbt27FgAEDEBgYCEdHRzRp0gTLli1DSYnyD0Qikah8rVy50hAfSy8wb4abzwnXKeHyWgPT1RkDgH+ThDub+ypP8d3/uK/p6O7J6Rrqw7cJGrOSkZebXSTsGTZtuMGoDD77KIM/Q6qB2eEJUNbTNDV2XxGuE8as5He2NZ2qfiGh1f+qv78/0tIUvShr1aps33bnzh106tSJ3v7ixQtUVBin9FwmkyEqKgo3btzAnDlz4OXlhfXr16Nr1664evUqGjRooHbfoqIijBs3Du3atcPkyZPh4+ODuLg4LFq0CCdOnMDJkyeVOsn07NlTqYVgixYtDPLZ9EFdL3GEB5khV39XO1FocemCkPsAX2KI0Po4m14COHc2N7+kjFW4IlTuCzxfUhvmvxGOhfvu8G1GtZRzuqiYcjW50ClmVFwviGrIoyWmi1YOYtOmTXH16lV6vWvXrqAoCosWLcLff/8NR0dH7Nq1C2fPnkX79u31ZmxVxMTE4MKFC9i9ezeio6MBAEOHDkVoaCgWLVqE3377Te2+NjY2OH/+PDp06EBvmzhxIoKDg2kn8fXXX2ftExoailGjRhnmwxiAHg3FMVtSyKiiE7LUhS4wu6kIWUeNee9zMoMndCHngzIJMiGRbDkRgeKQUPrTzMKaJ++nYUq3+nyboZIvjz6gl31MLNQvFLQKMffv3x/Jycl0hXLHjh3RrVs3nDp1Cu7u7vDy8sKIESMgkUiwcOFCvRqsjpiYGPj6+mLQoEH0Nm9vbwwdOhT79u2DVCpVu6+NjQ3LOZTz1ltvAQDu3VMtaFxcXKwyBC1EuDOgSYycRCFxJ0WRH3k7OZdHSwxH2xAPejklV7jfH2mZwmEKEnhPVm2xtlT8Li4kZFYxkj+4si8TTVDvLYTRpSdcwA+GXx17WP0gE+LqU+GmwBQxZhCFrswhVrRyEEeNGoV79+4hIiKC3rZ3715MmjQJHh4eyM/PR6NGjbB9+3b06dNHX7ZWSXx8PFq2bAkLTpVoZGQkioqK8PBhzX/YL19W9j318lJOZt+6dSscHR1hb2+PRo0aVTlDKUcqlSIvL4/14gupQGdLVhxSOOMZBaaXawUATWuLQ5F5N2O2xNQKVOSUVShChh/8Hs+jJeo5cDOVte7qIPwweE1hOuqmGEIXE18Oac63CTXG1HPV+UIrB9HW1hZhYWHw8FDMhLi4uOCHH37Ay5cvIZVKcevWLbz99tt6M7Q6UlNTabkdJvJtKSkpNT7mqlWr4OLigjfeeIO1vUOHDli+fDn++usvbNiwAZaWlhg5ciQ2bNhQ5fFWrFgBV1dX+hUYGFjleH0zoHkAvSzUcFq4n6JC01Sbr7erq2h96CKSlmm21qZVTS6nVZAitCkRaDpZkchanmkDV/5JqPRtqijOXDW4GY+WGA4hz+Cqw9Rz1flCkHcnmUyG0lLNZo9sbW0hkUhQXFwMW1vlPAR5hXVxcbHSe1Xx+eef4/jx41i/fj3c3NxY750/f561/u6776JVq1b4+OOPMXbsWNjb26s85vz58zFz5kx6PS8vz6hOoh9Dab5YoH0r3RmzIwujGvFoiWHxdbHFqzyp4EWB5Zia3JCcCZ1C6DCaUP8WHo6K3wTTQTEluAVDWYWlguwbf/DWS3q5ZZAbf4YYkMYBwpdRknGKhaxIsZBBEORV/8yZM7C3t9fo9eBBZaKqvb29yjxDeY6gOqdNFX/88Qc++eQTjB8/Hv/73/+qHW9jY4OpU6ciJyeHVbzDxdbWFi4uLqyXMWEWgCRlCjMHMaNA8TesY6J5bwAQ4Fb5fcyXlqNcgK3FpOXsBwgh9+7WBW5urhDZcDqBXm7PmH02ZfbfqHnEx9g42ZpeqB+o/E3IozfOAo1w5HHUH0g1uWHQ+q9/584dfPnll4iNjUVqaqraGT+JRILy8polkIaHh2PLli0ajZWHkP39/ZGamqr0vnxbQECA0nuqOHbsGEaPHo2oqCj88MMPGloMeiYwKyurmpH8wVSe33z2CYa2Nm6IWxOYeYdC7hetK8WcBGt3gc2WpOWxH7ZMNQfRxkoENxaGE+vhaLq/CSa/XnyKMR2C+TajSpwE6jzpA7ncU4G0HDIZpTTDyzeZHBF/MTzoiRGtvuGxsbF44403UFJSAolEAg8PDzg56U9nz8/PD2PHjq3RPhERETh79ixkMhmrUOXSpUtwcHBAaGj1PSYvXbqEt956C61bt8auXbtgZaX5f09iYqX6v7e3cJNlmZWoD14JMxH84X92WVlIRJOfpw3MRPztF59iWg/1Op18wBTJDvdzNtkn9A71hN9NJY3xt3i9kTjkqrRhWOtA/HHlOQAgIb2AZ2uU4c70O5hw7195qg9FVQqzewrsYT0hTfH96N3Yl0dLTBut4kZz585FSUkJPvnkE2RnZyM9PR1PnjxR+zIG0dHRePXqFfbs2UNvy8jIwO7du9G/f39WfmJCQgISEhJY+9+7dw9RUVEIDg7GP//8ozYknZ6errQtPz8fX3/9Nby8vNCqVSs9fSL9M6hlbb5NqJLSchnS/utUUC6jzOapUIiSQ8wn9P7NNZt9FyN2nJu8EIu3HP/ToHSytTLZmVwAmNy1Hr3cs5HwbvqlHAdRaLNq+sTHWXG/TMtXLxHHF//boeiQ1qy2G3+GmDhaTdHcvHkT7dq1w2effaZve7QmOjoa7dq1w7hx43D37l26k0pFRQWWLFnCGtujRw8AQFJSEoBKB693797Izs7GnDlzcODAAdb4evXq0YLf33//Pf766y/0798fderUQWpqKn7++Wc8e/YM27dvh42NsEKFTLwF9hTI5XaKaeoeqmJa9/r45uRjAMrdGYTAY8YTuruJtxNj8iqvBIECE6JO/G82rUBq2lpv/owiuuxC4XUYOnEvrfpBJgKzc1JKTrHg+n9XMK6ZjqSC2WBo5SB6enoiODhYz6bohqWlJQ4ePIg5c+bgm2++QXFxMdq0aYOtW7ciLCysyn0zMzPx/HllaOOjjz5Sen/MmDG0g9ixY0dcuHABmzZtQmZmJhwdHREZGYmff/4Z3bt31/8H0yNc/TSKEtYs3fpTj+llexMO3wCAN+MJ/e8bKfhmhLDaNK4+ouhSwKyiNUUcbCxpKZl7qXmCcxAF+PxgEJizuZeThJfL/YzR9Yip22iKWFsqgotfH3+EHg2FN6Mrx9LSNAvohIBWDmJUVBSOHTuGiooKWFoK50bu7u6OTZs2YdOmTVWOk88cygkODgalocZFz5490bNnT21NFBSFpRWCaqF2nPGEHipCLa6a4OMint7G9jbC+Y4YAqbO4Gf/3EWvxsKRkikqNe1ZQzGRzgi1zusTzqMlhifATXF9uiXwjlau9qb9AMsnWrney5Ytg0Qiwfjx45GbK+wvD0E9e+OT+TZBLU62wnnwMAS9GDlWdgIToeY+LEUEuvFjCA9kCqx7z77rwpd7MRT5JcIKM2+9kEQvm3q4X0x5x6RIxXBoNTXg7e2Ny5cvo0uXLggODkbr1q1Rq1YtpTZ3QGX5+ebNm3U2lKB/uGKjQsLRxGetJBIJfJxtkZYvpSUlhEJ2EfvGbOpP6L0a+eLo3VcAAC9nYeVbfnH4Pt8m8Mb+G6l4u20dvs1QCbPK3xThFm8JidxixfUpMtjDpAu3+Earu3BeXh6GDBmCe/fugaIonDhxQu1Y4iAKi3c7huDn85WV5etPPxas1lhUM+W2iaaGvDowPV+KkrIKwVyUC018doTL7N5htIP4PKtmHZcMTU6RsGbRjAkz509o9G8mnhk2bYkIdMP15zkAKiV+rASS6/fbpWf0MrM7GEH/aOUgzp49G6dPn0aTJk0wceJE1K1bV686iATDwQxnvsoTnnyBHCHKXBiSU/fT8EZTYTjFzPBZPzNw1OsIrChFHVO71efbBIPTvq4n4hIzAQA/xCbgozeEkev34CVbNzZcYFW9hoDZ9jSvpFwwrQ+Zs+oX//uuEAyDVg7ivn37EBgYiLi4ODg6OurbJoIBEcqPvDocTDzEDACN/F1wNzUPADtswjf5JQoH0U9ExTTawp25rZBRghAGL+Po7g1sUYsnS4xHj4Y+tIMoJNLy2SFlAXw9DM5jhlj5nZRcdG4gvCYQDkTixqBoNWdcXFyMdu3aEedQhLwl0JsMs7cmUwLGlJnVS9HdJyVXODlNzGpNLzP5WzB13lJyhBFmZlZXA0B9H9OP0jBzDpvVduXREjYVnHxtF4HlDRsC5mf8ITahipH80TXMdDsLCQGtHMSIiAi8fPlS37YQjACzZZKzgCRuvj3xiF5OF6ByvyHwd1V060nOFoZTAgBbziu6H5nDDCJQqX8oZ8v5JP4MYcDs192hniePlhgPBxsrelaI6yDzSXYRu7rdlLuoyGGmXpx/LJxZ3cYBioe5KWaQdsEnWjmIn376KS5cuIDDhw/r2x6CEcmXluO5QBLBb7wwP7kkZm/sZ1nCabd35Wk2vezjYh4ziC3ruNHLQpnBLi5TOEi+ZuKoA4CnU2UaTGaBcB4UN51VPDQJIf3AGDjbCWcCgQnzwcnUFRb4RqtvgI2NDaZMmYL+/ftj5MiR6Nmzp1qZGwB47bXXdDKSYDjWnXiEL4c059sMhHg64vIT4XVPMCSOtlawt7ZEcVmFoHIQmQS4qu5Jbmq82ykE136LByCc/DJmNblQKtyNgZeTLZ5nFSO7qAxlFTJWVw++sGJ8KcYJVPlB3wxpHYhdV17wbQaLChmFREbvehsr/r8bpoxWDmLXrl0hkUhAURS2bduG7du3Vzm+okI4oQICm2eZwphB9GXMVK0a3IxHS4yLq701issqBCNnwi2MCPYyjzxjZvFWVqEwxLKZeV9yuRFzgDlD9OBlPprU4j8XkRnhaOBr+rmgANCqjjtrvVBaDkee05K41eQEw6LVX3v06NGC6uFLqBlMLcTbKcII7T5n5OA1N6POHS//E9xNy5cKojc2M6zZpJbpS3nI8WGElV8IJB/0n5up9DIzR9LUuc9wAhbuu42973fk0RplXgsVXjWvIeDmWSbnFCPUl98WqC/zhPHbNBe0chC3bt2qZzMIxmRom9q0gyiURHBm2z+h5IAZm5svcnl3jnMZM5kO1sLMQTIE3s6KHL8Dt1LxPY+2qGJY60C+TTAaYb7OePCq0kkMcOM/xYHbetJcCreAyjxEuewVU/6KLxLTFeHlGa+HVjGSoA+0CuC3bNkSQ4YM0bctBCMhNC1EbkjPzUwTj7869pBvE/DnNUXO0eUk88kJdRFoQr6cuX3C+DbBaDA/a6A7/yLmhZyHaL5n+Y3Jux1D6OU8AeRJLztwj14WWg97U0Sr/+EHDx7A2to8b+KmgLOtsP52XIkXc5CQUMWdZP7D/eUVitkScwmlAZU3febXTmjtBvnO/TImzFnDo3f5l1PLZ2i09m5sXh2emJXMTK1aIcDNlyboH60cxAYNGiAzUzi6SISawX3y4oZQjM1TAUm8GJuBEYqerpkCKI7Yd0MR6h/dLohHS4xPFKO/Lt9anCVl7FkrWzOq1qzlrnAQcwVQvMUs5DMnRx0ALBizpT+dTeTREmWGmlHaBV9oddUZP348YmNjcf/+/eoHEwSHRCKBl5MizCwt5/dJbNuFp/SyjQAkLYyJ0NqnBXkoqpbNRQNRjjdDRP7SE34fgK8kZbPWzSms6WJnTWuEZhaWKnUxMTbDNl6kl/ffSOHREuPzLyPN5HaysAqlXMw0FcmYaHU3/uCDDzB27Fh06dIFa9euxePHj1Fayv/sB0FzmtV2o5dv8ixSbc/op/lx33AeLTE+Het78W0CiwJGaDXcz3yqmAEgq1Axa1hawa9TUiDlf+aMTzILFPeTp5nCiTCU8fy9MDZ1PPnPAZXDjXSZkzYoX2jlIFpaWuKnn35Ceno6Zs+ejbCwMNjb28PS0lLpZWVlXlPyYoE5UyeXWuELZkipdbAHj5YYH64IsIzn2RJ5npGzrZXZidC2Ynz3Fv51m0dL2A9t73etx6Ml/MB8UGHO4PHNikFN+TbBqLzfVdHKju80hzxGFXUngT1YmypaeW+BgYFmFfIwRbqH++DwncoE8FsvcjCgeUA1exgOZkstZoN4cyS/pByuDvz9H8hlJPhOO+CD4lLhFKasP60QyT7/OINHS/iH73zQxgEuuJNSGV4d3sa88t5c7a1ha2UBabmM99Z7L3MVExnm1HqST7T6iyclJenZDIKxySlWhHB+OvsEC6Ia8WbLkTuv6GVzy3sDgAHNA/D3f7lNWUWlvDmItxlV1KVmWCH4TrtgfH5QkVctBOFyAMgRgLyIsflySHPM3n0DQOVsNp9k/PcA6+tiK4jvg7GRPyxmFJSiQFoOJ57+HswiGX9X4iAaA/OKIRFozj4SxqxEYnoBvVzX29Es80oepyn+D5iC4cZm4rYrvJ1bCNjbWKKOhyLnSigi8uZYrRnup+jYkc+j5FCFjKJnMMmsFRD7IJ23c8dcVWi0ZheRmgdjoDcHMTs7G9nZ2dUPNCA5OTmYNGkSvL294ejoiG7duuHatWsa7Tt27FhIJBKlV3i4ctGETCbDqlWrEBISAjs7OzRr1gy///67vj+OQfmC0++Yr9y3RwzniKmSb07cZbRR++bEI97sSM3lNxdVCDDbCwpl5q5nI/PS3gOEk2qSWSCF/NLo40wcRGYhF5+Qv4Vx0Gmu+ODBg1i3bh3Onz+P4uJKsWN7e3t06tQJ06ZNQ9++ffVipCbIZDJERUXhxo0bmDNnDry8vLB+/Xp07doVV69eRYMGDao9hq2tLTZt2sTa5uqq3Ch+wYIFWLlyJSZOnIg2bdpg3759ePvttyGRSDB8+HC9fSZDwm1hVVxWwYvGF1Odf+nAJkY/vxAIcLVDisCcs8gQ8yoWkuPmoJB/ysiXohYPrd6k5eyZywY+Tka3gW+EUj177J4i/SU9X1i/UWPRu7EvnQa0/eJTvNM+mF+DAAxqKSx5MFNFa49gxowZ+Oabb+jSc1dXV0gkEuTk5ODo0aM4duwYPvzwQ6xZs0ZvxlZFTEwMLly4gN27dyM6OhoAMHToUISGhmLRokX47bffqj2GlZUVRo0aVeWY5ORkfPXVV5gyZQq+++47AMCECRPQpUsXzJkzB0OGDIGlpfjCpEWl/DiIzP6eQm93Zii+G9kSg9Zf4NsMFhM6hVQ/yARhZphtPJOI70e2NLoNTHHo7uE+Zpn3xkVaXgFbK+NfVw/cTKWXn3M6PpkLvRr50Q7iw1cF1Yw2DoEewniAMHW0CjH/8ccfWLduHby9vfHNN9/Q4eWsrCzk5OTg22+/hY+PD9atW4ddu3bp22aVxMTEwNfXF4MGDaK3eXt7Y+jQodi3bx+kUs2mxisqKpCXp14QdN++fSgrK8P7779Pb5NIJPjf//6HFy9eIC4uTvsPYWTsGfl+STxpjcmrAwHhhJWMTcs67qz1cp4KRKwtFY6IOYY1AbZA9YFbqVWMNBzM0LanwPqm88Xh2/y03GOKdE/uUpcXG/imcyj/kjJ8d/syV7RyENevXw87OzucOXMGU6dOZYVhXVxcMGXKFMTGxsLW1hbr16/Xm7FVER8fj5YtW8LCgv2RIiMjUVRUhIcPH1Z7jKKiIri4uMDV1RUeHh6YMmUKCgrYT0zx8fFwdHREw4YNlc4jf18dUqkUeXl5rBefWDEcgoev8nmx4c9risRjc2tjpQ6m3pexKKuQ0SLAwZ4OZjtr5SeA6siUHMVMlSvpFgEA2Bb3tPpBBuDSE0UnES8n81NYAAAvR/bn5sNZS2NIHQV6GD/tw1zRykG8ceMGunfvjtDQULVjQkND0b17d1y/fl1b22pEamoq/P39lbbLt6WkVN0iyd/fH3PnzsWWLVvw+++/Y8CAAVi/fj369OmD8nLFDTs1NRW+vr5KN1BNzrNixQq4urrSr8BAfqsTJ3RSPBFLwL9DYKY+iRLLDtw1+jk3nX1CLycxes+aG1HNlK8hxmbR33fo5ZRc8wxrAmC1A736lN8CSABoVls5H90csLBgX5j50Ehdxyjee55lvr8JY6OVg1haWgpHR8dqxzk6OmrVgk8mk6GkpESjl/xppri4GLa2yk94dnZ29PtVsWLFCqxcuRJDhw7F8OHDsXXrVixfvhznz59HTEwMPU6X88yfPx+5ubn06/nz59X/ZxiQRgGKis2P997i0ZJKgj2r/06ZA3uuGV/qhs/qaSER3bI23yagpExRpBJpZp2FmGwc3Zpe7s+jkL8cPnIghULvxoqUEz4kZn6//Mzo5yRo6SDWq1cPsbGxKCxUn7dWVFSE2NhY1KtX8zZRZ86cgb29vUavBw8eAKisnlaVZ1hSUkK/X1NmzJgBCwsLHD9+nN6my3lsbW3h4uLCevGJH8+6XswbIQB4O5tnCEcIFJcJQ/OPb7izJXwQ6K5IwB/Rtg6PlvCLO6OifP+NqiNAhqCI01lHCN8NvvBghJmvP8sx+vnfaqGoWo5qyv8sv7mgVdLX0KFDsWjRIgwcOBDr169XkpBJSEjAlClTkJ6ejqlTp9b4+OHh4diyZYtGY+WhXX9/f6SmKieVy7cFBNT8CdTe3h6enp7IylLkofj7++PUqVNKXRZ0OQ9fNA7g10FlttAy9x89s5tKqyD3akbrHy8nG2QUEPFZLik5xUqSUIbmCiOcas6zVvY8i+bv+pcd4eFD8kgoMPMO/7fjGpJWRhn1/I42ClflnfZBRj23OaOVgzh79mzs27cPJ06cQKNGjdCyZUsEBwcDAJ4+fYqrV6+ioqICrVu3xqxZs2p8fD8/P4wdO7ZG+0RERODs2bOQyWSsQpVLly7BwcGhynxJdeTn5yMjIwPe3t6s82zatAn37t1Do0aK9nSXLl2i3xcL3CfixPQC1PU2nuYaM1Th7mjeyfjjO4XQDmKor3M1o/XPG038sf1iZSHAd2+3MPr5hcqMP67jj/faG+18ZWbY4lAd9jZsB7GkrMKonZbkvwcCkMuzaDzzb+FgY74PTcZGqxCzvb09Tp8+jSlTpsDGxgb//vsvdu/ejd27d+Py5cuwsbHBlClTcPLkSa1Cu9oQHR2NV69eYc+ePfS2jIwM7N69G/3792flDSYkJCAhIYFeLykpQX6+chXv0qVLQVEU+vTpQ2978803YW1tzarOpigKP/zwA2rVqoUOHTro+6MZDWbFnjFgtk5KNlONMTm13RW/k6QM40sOMS/AzWu7Gf38QsXYv4mXAhNM5xNuBbex21BGhnjSy11CvasYafoIqc2gB5F+Mhpa64o4OTnh22+/xRdffIGrV6/S1bsBAQFo1aoVHByMK2QZHR2Ndu3aYdy4cbh79y7dSaWiogJLlixhje3RowcAICkpCQDw8uVLtGjRAiNGjKBb6x05cgQHDx5Enz598Oabb9L71q5dG9OnT8fq1atRVlaGNm3a4K+//sLZs2exY8cOUYpky7GxNG5rbqZ0xSkee3wKAU+GhEZcYqZRz81ts2jskKrQCPdzxv2XlQ+MxtYhfH1NrFHPJyaeGrm6nlkYMSLSfHNBAeB/Xeth64UkXs7N7SxU252IZBsLnYXnHBwc0LlzZ33YohOWlpY4ePAg5syZg2+++QbFxcVo06YNtm7dirCwsCr3dXNzQ79+/XDs2DH88ssvqKioQP369fH5559j9uzZStqKK1euhLu7O3788Uds3boVDRo0wK+//oq3337bkB/RIIxuH0Q7areSczG4FT9VnB3re1Y/yIwwZu4bV0rF0oyT8QFgdXRz9P/uHACgR0Mfo56bDwkRseDMY6elTIH0IOYL7gzii+wiozlqqw4/MMp5CMqYlDKxu7s7Nm3apNRPmYt85lCOm5sbtm/frvF5LCwsMH/+fMyfP18bMwWFD6NyeOuFJCwe0Ngo5+VWCK4f2coo5xUyFhJAPpl3NyXPaA7ip/vuVD/IjPBg6O8VSI0rWs78Dow04wpmOcvfaoIFe28DAAqN/LdgQtIu2Nx6kWs0B3HzuSfVDyIYBI0cxM8++0ynk3z66ac67U8wHAOa18KXRyu7zBizSq+kjD1TQjpGAO3reeL848rw8oRtV4xWKXjyfppRziMWmN/Fg7deKikWGJLODbwR+7Ay3WJWr6ojH+ZA01oKcWpjOuvZheyK/ia1zFMkWx18zXQv7t+o+kEEvaGRg7h48WJIJJIatdhhXlCJgyhcAj3sYW0pQVkFBTtr4+UgkmpNZS4mGrcggqAaR06V5KHbL9HXSDJMcucQAFx4DKkKBSdG+81tcU/x2ZtNjHLeUZsvGeU8YuX4vVcYyNAmNBZCaIVpTmh0BVqxYkWNDpqcnIzNmzejuLjYbHu6igWJRAIXO2tkFpYiIb3QaLMlB24qa1aaO81ru+IaDyK0jfxdcDe1si/4kemvGf38QoP7/f/oz5tGcRC5wvFWRi4aEyLc/uw5RaVwczB84VCYrzPupOQZ/DxiYkq3evj+VKX6B199qf1czbuAztho5CDOmzdPo4O9evUKn3/+OTZt2oSSkhK4uLhg+vTputhHMAKZjHDKuccZ6NzA8JIOL/MUch6hvsbTXhQynw9qij5fnzX6efOllRpnHo42CPMzvgaj0MkrMU5ok88cO6HCdUReZBcbxUFkXp++HNLc4OcTA/2bB9AOorHy1bkPTRGBbgY/J0GBXmIY6enpWLlyJX744QeUlJTAyckJs2bNwqxZs+Dm5qaPUxCMxJ5ryUZxEJmVstN6NKhipPkQ7sfubGOs2dzcokoHkeSBKnBzsEbOf/8vxqrq3vkvv73ZhQj3/97GyjizqhcSFFJTJNRfiT8Ps3cXjSz5RWCj068tMzMTc+fORd26dbF27VpYWlrio48+wpMnT7B06VLiHIqQQ7eNE/pNy1PIRvDROUQM3HiRa/BzyGQUPUPmQhxEGmaovYWRZi0evFQW6yew+TfJ+Hm6nRp4Gf2cQoT7AFluhDxyC8YDclNSKGR0tHIQs7Oz8fHHHyMkJARffvklAGDu3Ll48uQJli9fDg8PD70aSTAezWq5GeU8afmKEA5TaoegIC7B8E/PR+68pJdvPM8x+PnEAlP3jdkb2ZAwew+vim5mlHOKDbnkjSFJzmHrgjrYkBlEOa839KWXjZGj+f2px/Ryv2bGKRQjKKiRg5ibm4uFCxciJCQEK1euhEwmw6xZs/DkyROsXLkSnp5E7FiMfD0sgl6+bKQn9Ov/OSM2lhYktKmGLw7fN/g5/rfjmsHPYQrURMFBW/64oggxdwszrkC3kDGm/BYAJKQVGPV8YiKnSJGv/izL8J1tmK0uMznSQwTDo5GDmJeXh8WLFyM4OBjLly9HWVkZZsyYgcTERKxevRpeXmQKXsx0rG/cv19OUSny/wtrllbISKU7QfDkFpcZ9Pjcdoek36yCzwc1Ner5SDcb9bzTPoheTuV0YDI0YSQVyeho5CAGBQVh6dKlKC0txYcffojExER89dVX8PEhT7mmgDcnxGtojcLdV14Y9Phi5pOohrydm4jQqmfWrhsGPf7VZ+wwtrm3O2TSNoSdspSUUWjQ852494pe7lCPRMWYuDMqyPONVN0vh0SajI9GDmJubmWyvFQqxffff4/AwEDY2Nho9LK1JfllYiM937B9R5cfvEcv1/V2NOi5xEYUj3k277QP5u3cQueEgbvNnH5Autmow5ZTuWzozj/ManJyfWLD7IfNzF82Bl3DDK+uQWCjcQ4iRVGQyWQoLy+v0auszLChGYJ+YCbId1192mjnHd8pxGjnEgNcKYni0go1I3WHm1dHZq3YGDP3zdAhbDHDTUHh9nHXJ9zfhJ2VpZqR5gnTQXz4yrC5mtwqaSIcb3w0+h+XyWQ6vQjCh5nzVGrENnjGDlOIjd1XDaeNdyvZ8DI6YsbNwXghrWN3X1U/yIxhPrzsu55isPNwCyHe61LPYOcSI7XdHVjrzw1YqEJaj/IPcckJAIDJXerycl4iXaBMJCPnKiWnpIqRujHt93iDHdsUYEp6AIatZH7F0AV9o4mfwc4jVoa3CaSXHxmwyri8gv035uZnmzt21uwZVUPOfBs6lYBQPcRBJAAwnkjyY87FnftESgB6MhyTH2ITDHaepEzDy1SImQ+612etHzXSLN+sXqFGOY+YCDBSuL+4zHApHabI5F+vGuzYP59/YrBjEzSDOIgEAMoOoqG6OhhbGkGM5BQTvS8hwM15em+7YW6G3JlJPx5amgmdsR2CjXIeZrHQwIgAo5xTzLzINs71vDPpZsMLxEEkAABe4/RfNpRg9l2G+j4JL6uGD5HkL4c0N/o5CZVw894crElhBBdHW3Y3E0OF+5fsv0svn36YbpBziJ2lA5sY/ZzvkmJGXiAOIgGAcgXrjotPDXKeFYcU3UG4TimhkuZG6P3LvcESZ101EUb4W3x+4B5r3YJUk1eLMQqshrUOrH6QGdIm2J21XmKAsPyLbHb6S+sgdzUjCYaEOIgEldw3UIiZCekWoRprI8g55BWzq8e5yeeESmb2NHw+4N83DFeVa6psOG243Fw5XYjunkokYD/AXH6i/2jTxjOJrHVnOyKSzQfEQSQYDW6Hlm7hpBOPJmQboAfpL3FJej+mKeLjYvgq1nKZ4fs8mxoWBmjPGc/pZtM2hHRRUUUDHyfW+rnHGXo/x7Y4w0SwCDWDOIgEGm5bKX07JtmMRu9hvs5EmFlD9sQn6/2YzAizgw2ZPVRHuJ8La/1ppmHbvH3+lnH7DouJEZGKkO+BW6l6P/7/fr3GWifXJ9VwUyDqezupGakfmtV2NejxCeoxKQcxJycHkyZNgre3NxwdHdGtWzdcu3at+h1Rqdav7tWzZ096XFJSktpxO3fuNNRHMwqbxrRmrSfn6LdC7buTj+nlzELDtvMTOy6MjgVcaSB9EHNNIcC9cnAzvR/fVGHqFeqLeox2bv2bk1xQdfQIZ+tSZun5AfZlnuE0R02Zh68Mm44U4kXaHfKFVfVDxIFMJkNUVBRu3LiBOXPmwMvLC+vXr0fXrl1x9epVNGjQoMr9t2/frrTtypUrWLduHXr16qX03ogRI9C3b1/Wtvbt2+v2IXjGwcawXwdm2CCjgEi5VMWWcZEYvOECgMouGysG6Xdm6XmWwvlv6Oes12ObGhM7h+Cns5WabBkF+nUQS8oqkJCumJUkuVbqkZazU1Ryi8tIHjNPTO1WH9+dqnzg33TuCT7p18hg55rXJ9xgxyZUjck4iDExMbhw4QJ2796N6OhoAMDQoUMRGhqKRYsW4bfffqty/1GjRiltO336NCQSCUaMGKH0XsuWLVXuY0oYs+UegU0rRtWevp0SLn6udgY9vthp6K8IM994kYO+TfU3y/fx3lt6O5ap06MhO2e5UGq4Np1zeocZ7NimgCGVFri56sYSSScoYzIh5piYGPj6+mLQoEH0Nm9vbwwdOhT79u2DVFqzm6xUKsWff/6JLl26oHbt2irHFBYWorTUtGbCHBn5aIPWX+DREgKTolL93Qy5Iuhk1qpqfF0UDvSPsYlKNzBd2HNN//mlpoqdtSXa1VW0oVx77CGP1pg3XTkV3vrsyXzzBekRLxRMxkGMj49Hy5YtYWHB/kiRkZEoKirCw4c1u5gcPHgQOTk5GDlypMr3lyxZAicnJ9jZ2aFNmzY4evRotceUSqXIy8tjvYRGYalxWk199mZjo5zHVHimxwvw8oP3qh9EoOHOsD56ZZhewOM6BhvkuKZEKSPMfEKPvXq5+dZhviTtoiq4UlyFenyAPW+AqmiCdpiMg5iamgp/f+XQj3xbSkrNtMZ27NgBW1tbOlwtx8LCAr169cLq1avx999/Y+3atUhLS8Mbb7yBAwcOVHnMFStWwNXVlX4FBgpPiHVuH8OEVu6ksJ8KhxIR2mph5t7cS9Xfw8QZ0iGiRtTihLguJBjmBjaqXZBBjmtKDGtjmOvG+Ufsvyk3nE2omoQ0/VX3r2HMDIeT/GheEaSDKJPJUFJSotFL3hGiuLgYtrbKmmV2dnb0+5qSl5eHAwcOoG/fvnBzc2O9V6dOHRw5cgSTJ09G//798eGHHyI+Ph7e3t6YNWtWlcedP38+cnNz6dfz58+rHM8HI9uyb1LScv3MKP4YyxY+JcLM1RPs6UAv77j4zCDnYIbsCKqxtWJfJo/fe6WX43I7UNTxcFAzkiCHKzukr5Z7XNkciQF0Fk2ZQ7f1LzsEAN2JVi6vCNJBPHPmDOzt7TV6PXjwAABgb2+vMs+wpKSEfl9T/vzzT5SUlKgNL3Px8PDAuHHj8ODBA7x48ULtOFtbW7i4uLBeQoMprwIAt/XU0iouMVMvxzFXrjzNrn6QFqyOJj2Yq4PrLHSs56WX43JzQY3RQUfsNK3F1sRjVuPrQss6iqKw916rq5djmjrL31L0ZP7npn4cxJwidk7/1O719XJcgnYIsoo5PDwcW7Zs0WisPITs7++P1FTlL6l8W0BAgMbn37FjB1xdXdGvXz+N95GHi7OystQWtYgB7s1w8IY4JK2M0vm46fkK593XCN0pTAFjdJqp7U4qBDVh/ciWeH9HpaZqgZ7yrX48Y/h2caYGV6R55eF7WD+ylc7HZYr4kw5PmvFaA/23IswvYf+2DC29RqgaQf7v+/n5YezYsTXaJyIiAmfPnoVMJmMVqly6dAkODg4IDdWsp2pqaipOnTqFsWPHqgxZqyMxsTKE6u0t/v6dzrZWyDeghMSP77SufhBBKQxfKC2Ho61uP9n8kjLWOgmlaUY9RrcIfRWpHLz1kl7m5jkSNIP5f6gLWy8k0cs+zuQBVhO439kX2UWo7a5bmgSp6hcWJhPTiI6OxqtXr7Bnzx56W0ZGBnbv3o3+/fuznL2EhAQkJKh+et+5cydkMpna8HJ6unKCf3JyMn7++Wc0a9ZMZaGM2Pg4qiFrvUBHZ7GUI3DbnLRO0oq31p/X+RirDj/QgyXmB7Pbycn7aXiZq9+uG6SCWXNGtq1DLzfy1z1N5xWng4qrPZF90gTubO7Sf+7qfMy1x4l0kZAQ5AyiNkRHR6Ndu3YYN24c7t69S3dSqaiowJIlS1hje/ToAaCybR6XHTt2ICAgAF27dlV5nrlz5yIhIQE9evRAQEAAkpKS8OOPP6KwsBDr1q3T98fiheFtAjF/j0LAN6eoFE46zFz9fpldYEFmrbTjoR5mrpjJ+KRCUHOsOPmBb2+6iJOzump9PK7I89sMp4dQNR/2aIAdlyqvKZ5OundSOcWRy3EhDqLG1PFwoCW47qTorrRgIQFk/9UdrRlK8qP5xmRmEC0tLXHw4EEMGzYM33zzDd1u7+TJkwgL00y65cGDB7h69SqGDx+upKcop1evXpBIJPj+++/x/vvvY+PGjXjttdcQFxen1qkUG1wH7kmGbhIGi/6+o9P+5syM19mpETKZblWbzP61Xw4hF2BtSUzX7TdxlVN0RHKtNMfb2RY2/1WW60Mf9KM97G42pFhIcxYyWuxx8we1oYGP4qF1YEQtnY9H0A2Tuiq5u7tj06ZN2LRpU5XjVM0cAkBYWFi1sgkjRoxQ2XrPlHln82W9FKoQas57Xeqywi6Hbr9EVDPt0hiyC9kVgvV9nNSMJBia0T9f5tsE0SKRSODtZIvknGI8zSxCgbRcpwgHk0APkgtaE5jXkNzisipGVk9mgRQPXikq+7khbILxIY9KBKNy/qPufJsgKriFKofvaJ+U/zidHaImWpQ1Y8kA0v1HKDA7n2yM1b4avJzTNnH3ex20PpY54mDDvobo4iT2XHtGV3MIeoY4iASVLBvYpPpBGsAscGlW25VUa2qBu4MiJ0oXYeAhP8TpwxyzZSQnTzAx3TAt9wg145uTj7Xel1sUwW2rSKgaZp9yANh/o2Ydy5hkcSIcBP4hDiJBJSMi2TdDbXPfLj9RCGR7OxH5CG2Y2UuRQ6svQVprSxK+qSncQpXuX8VqdZwEjmNpqPaWpoy+ZnO/P0W0KPXJzRc5Wu3HffB9LVT8cnGmAHEQCSqx5OR/7L+p3ZPhu1uv0MsnONWCBM0wxExVnybil2MSKzN33WCt/69LPZ4sES9vtWQXMOgqxUXQno71PenlXVfUdxKrikzO7OFk0s1GEBAHkaARH+68rvMxejXy1d0QM8TZji27oc1sLrdApRPjok4wLjee57DWiexTzXHh/CZWH77PkyWE3o39dD5GRgG7TS53tp7AD+SvQDAY3ATwxSTJXyvGtA9irc/982aNjxH1zVnW+qCW4m0HySe/vBup0/5lnN8EQT/8Eve0xvtkcpySxf0bqRlJqIphbQJZ61wJJ03gary2CnJXM5JgTIiDSNCYUw9qFiLmapQFkAIVrfDk5G7GXK1ZGKdCRiGF0/mDaL1pR8d67JnXS4mZakaq5iJn/N9TO+psk7nSQEeZplbLjrPWx3YM0el45oqtFbuS+fi9VzU+xt/XFS32ZrweqpTiROAHcpcgqOXGol6s9XFb/q3R/v2+PadPc8wa7ixiTXjwMp+1TlqJaQ839LWvhlWbOUVsGRAvUrilNdz/O10q/An6Y8PpmhX+FEjLcfyeYvKhbV0PfZtE0BLiIBLUoosjkV9ShqLSCj1aY94MbsUOCZeUaf5/W1jKTuD/7E0S6tcXv116Vv0gBjlF7FxQdwfdW8WZK9yUlbXHtO/j+3pDkh+tC9Nfb8Bar8nM+nxOJ5vIYOIgCgXiIBIMQm8ieqpXQn3ZfZOTMjVv9cZ11Ac0D9CLTYSas3Afu+2kvQ0RK9eWME4v8ZroIVZwCr2+H9lCLzaZK0Nas/MQh228qPG+XO1E0kFFOBAHkVAlIyIDqx+kAm7O29m53fRhjtnC7Xpy5mG6xvvO2c2WVSFVs7oRwBFTTs+XqhnJhlupuSq6md5sIlSiaZg5LZ99fbIhObk64UR6iZsk5FdBqJJPotiVfdc5Eh2aEujhoAdrCHI+P3i/BjdDzRwYgmbseZ9dWPKjhq3evjzygLXePdxHbzaZK1O71WetX3umWQXtvD8VYc0QL0fy0KQjqmbCc4uqb7vHfWj6eWxrvdlE0B3iIBKqxNGW/WQ48Pvz1e5TyBGt/V9XIgSsD97lVFkO1yCMwxUQjmpKBLJ1xc/VDsGeigeeTeeeaLTfzn+fs9ZJgYruzOoVylofvCGu2vzcV3klrBl40v5Td2yslF2JOA3yEO+l5rHWu4WRhyYhQRxEQo25/zKvyvfPPc5grc/qGapmJKEmfPRGOGv90pOsavfhhpeXkAIVvdC5AbsVWFFp1Z08uI56p/peerfJHJFIJOjJEeD/MTaxyn1OcTo6vU8eYPXCkxV9WeuTf71a7T7vbL7MWiczucKCOIiEamkc4MJa7/P1WTUjK5nHEXImqvj6QdVTelVdVUrLZTh0+yVrG5m10g9uDuwK/x+qkfbYeIbttKwc3FTvNpkr3FD96YdV67Vy+5m3rUu6CukDVc4dt1kCE27f5lBf3XQtCfqH3LkJ1bJ9fNsajedqvRH0x/GZr7HWzzxSX6ySnFNsaHPMlnc4upTVVdDeTcllrdd2Jzm5+mIIRwKqqqKhq0+zlSIcRJTZcBy4lar2vb/i2dXLa4ZGGNgaQk0hDiKhWjwcNddqe8mpXm4e6KZna8ybet7sp+wfqiiQ4BZF3FnS2yA2mSM+znZK207eV91BgqIolhAwQb9wIxQvsovxVI0M1OANF1jrC/uR9nr6xImTs/7hzutqx/58np27Sxx14UEcRIJGjObMmOy8rCwQLC2vQLsVJ1jb1g2LMKRZZgc3jHMxMUtlNXNZhUzp6Z1bcETQL+9uvaJy+6xd7DzQRv4uKscR9EeX1aeVtl1JUs7ZHd+JtNfTJ39NUW4dmVusHFFSdc0K8iSz6kKDOIgEjZjTO4y1/tGeW0o/8s8P3FPaL9jL0aB2mSPrhkew1if8ouyYvLuV3RYxmFx89c6vGqZe7IlPZq3X17GHMEGZzg2Ui36eZLBnEaN/iDOWOWaLqu928yVHlbZN3Ma+Zm0d1wYOREtRcBAHkaARznbW8HVhFziEzD/IWv8l7qkxTTJbWnNaUZ24rxy+PPuInWf10RsNDWqTOdKxvnJxw9+crhC/XlT+TUzr0UBpG0E3vhrSXGnbLxeS6OWrT5X1Ec9/1N2QJpktx2d2Udq26Wwi7bCff5yhlHLRlcjbCBLiIBI05sJHPZS2rTx0HwDQ6YuTSu/NJPI2BkGVbtvCv24j9mE6br7IUfnETmat9I9EIsGnnBy2ab/HY9k/dyEtr8Cms4n45K/brPc/7deI/C0MgI+LHWLndGVt23ohCbEP0/Eko1Ap9xAg+oeGQtX3e9mBe+j25Wl8vPcWRm66xINVBG0wGQcxNTUVH330Ebp16wZnZ2dIJBKcPn26RsdITk7G0KFD4ebmBhcXF7z55ptITFStqbV582Y0bNgQdnZ2aNCgAb799ls9fAphoyqJ+IfYBIzf+i9eZCtXzE7hdDkg6A9u68LtF59izM+XMeC780o5P3W9HYlTYiDeVZHDtuncE4R9chjLVKRcqBpP0A9BnsrpLGN+voxuX542vjEElfx2STl3/cC0TjxYQtAEk3EQHzx4gC+++ALJyclo2rTmGmMFBQXo1q0bYmNj8fHHH2PJkiWIj49Hly5dkJnJVoT/8ccfMWHCBDRu3Bjffvst2rdvj2nTpuGLL77Q18cRLKqSkFWFOC8v6EGq0gxIbXfNZz9OzupqOEMIuPyx8sw6QdhM7EwcdUMyu5fm0aPGAa4GtISgCybjILZq1QqZmZl4+PAhZs6cWeP9169fj0ePHuGff/7B3LlzMWPGDBw9ehSpqan46quv6HHFxcVYsGABoqKiEBMTg4kTJ2Lbtm0YOXIkli5diuxszXqBipUIDWVrVMmAEPSHRCLByVnKuT5cklZGGcEa88bHRbPv+rKBTQxsCeGTKM1ybWf1Cqt+EEFrxneqq9G4c/O6VT+IwBsm4yA6OzvDw8Oj+oFqiImJQZs2bdCmTRt6W3h4OHr06IFdu3bR206dOoXMzEy8//77rP2nTJmCwsJCHDhwQGsbTIW4+ST52xgEelRdmUzauRmPH0a1rPL9TvW9MKpdUJVjCLozoXPdamevNo9pDTtrSyNZZJ7Y21hqNEtLBOOFjck4iLogk8lw8+ZNtG7dWum9yMhIJCQkID8/HwAQHx8PAEpjW7VqBQsLC/p9VUilUuTl5bFeYmRqFbmF64ZHwN+VJH8bA2tLC4yIDFT7/qL+RATYWPRp4q+yklbOl1W8R9AvU7s3wFI1PceTVkahR0Nfle8R9MuCqEY4M0f9DOH1T3sa0RqCNhAHEUBWVhakUin8/f2V3pNvS0mplK9ITU2FpaUlfHzYZfk2Njbw9PSkx6lixYoVcHV1pV+Bgepv7kJmYue6mNajAXo3Vr7QDmgewINF5suygU0xIrIOvf7d2y1gbSnBwn6N0MDXmUfLzI9BLWvhQxUSNlvGtYGfK0m5MCbvtA/Gxfns3NArn7zOkzXmSx01+qu3l/SGm4PmHboI/CBIZUqZTIbS0lKNxv6/vfuPqbJs+AD+5SQcDiLHHSSBAAURKZRxXkvnjwkP61HQsKUIElkqjukEtGYzS7Kh2J7MyOZEtILaSgqdb5stFmqGc3SEKXunSYf5iiHgjxT5IRyQc67nD+M83g9YuuJch3N/Pxv/XPet54uXen+57l9arXbQl4Q/iu7ubvvv9d88PT0V+3R3d8PDY/C/2J6envb9BrNp0ybF9ZHt7e3DsiTqvdztj7Dp7bNh46H/w2+dPSh8aepfngt6NI9p3PDuoil4d9F/bsx6LpolXQY3Nze8+s8IrJg1Hv8qr8Plm13InBOG2Ag/2dFUyV/viX3LpmJf5f9jxaxQjPEe+P87Db096f+D9aW18NGNwP+unYVAvQ4a3sA4LDhlQaysrMQ//vFwF69euHABkZGRf+nzdLp7p0R7ega+5N1isSj20el0DyyvFovFvt9gtFrtoCV0OPMYoUEBX6dHZDfaywPvLoqWHYMAzI3yx9wof9kxVG3+lADERz7O6z6HIacsiJGRkSguLn6ofQc7LfyoDAYDtFotWlpaBmzrHwsMDLR/ntVqxfXr1xWnmXt7e3Hz5k37fkRERASWw2HKKQuiv78/li9f7rDP02g0mDJlCmpqBr7T1mQyISwsDKNG3bueKyYmBgBQU1OD+fPn2/erqamBzWazbyciIiIarlR5k8qvv/6Kuro6xVhycjKqq6sVJfGXX37B8ePHsWTJEvtYfHw8DAYDCgsLFb++sLAQXl5eWLCAz50jIiKi4c1NCCFkh/i7bNu2DQBw/vx5lJaWYuXKlQgNvfcsps2bN9v3i4uLw48//oj7v/WOjg4YjUZ0dHRgw4YNcHd3xwcffACr1Yra2lr4+f3nQvM9e/Zg7dq1SE5Oxrx583Dy5El8/vnnyM/Px5tvvvnQedvb26HX69HW1gYfH5+/+u0TERER/aGH7R4uVRD/6A7a+7/NwQoiAFy5csX+BhWbzYa4uDgUFBQgPHzgc//279+PnTt34tKlSwgODkZWVhbWrVv3SHfxsiASERGRI6myIA43LIhERETkSA/bPVR5DSIRERERPRgLIhEREREpsCASERERkQILIhEREREpsCASERERkYJTvklFLfpvIG9vb5echIiIiNSgv3P82UNsWBAl6ujoAAAEBwdLTkJERERq0tHRAb1e/8DtfA6iRDabDc3NzRg1atQjPWD7UbS3tyM4OBiNjY181qJEnAfnwblwHpwL58B5cB6OmAshBDo6OhAYGAiN5sFXGnIFUSKNRoOgoCCHfJaPjw//4TsBzoPz4Fw4D86Fc+A8OI+hnos/Wjnsx5tUiIiIiEiBBZGIiIiIFFgQXZxWq8WWLVug1WplR1E1zoPz4Fw4D86Fc+A8OA9nmgvepEJEREREClxBJCIiIiIFFkQiIiIiUmBBJCIiIiIFFkQiIiIiUmBBdFE9PT3YuHEjAgMDodPpMH36dFRUVMiOpSrV1dXIyspCVFQURo4ciZCQEKSkpMBsNsuOpnr5+flwc3PD5MmTZUdRpTNnzmDhwoUwGAzw8vLC5MmT8dFHH8mOpTr19fVYunQpgoKC4OXlhcjISOTl5aGrq0t2NJfV2dmJLVu2ICEhAQaDAW5ubigpKRl03wsXLiAhIQHe3t4wGAxYtmwZbty44bCsvIvZRaWlpeHgwYNYv349Jk6ciJKSElRXV+OHH37A7NmzZcdTheTkZJw6dQpLlixBdHQ0rl69it27d6OzsxM//fQTy4kkV65cwaRJk+Dm5obx48fj3LlzsiOpyvfff4+kpCQYjUakpqbC29sbFy9ehM1mw3vvvSc7nmo0NjYiOjoaer0eq1evhsFgQFVVFUpKSrBw4UJ88803siO6pIaGBoSGhiIkJARhYWE4ceIEiouLsXz5csV+V65cgdFohF6vR05ODjo7O/H+++8jJCQEp0+fhoeHx9CHFeRyTCaTACB27NhhH+vu7hYTJkwQM2bMkJhMXU6dOiV6enoUY2azWWi1WpGeni4pFaWmpor4+HgRGxsroqKiZMdRlba2NjF27FjxwgsvCKvVKjuOquXn5wsA4ty5c4rxl19+WQAQt27dkpTMtVksFtHS0iKEEKK6uloAEMXFxQP2W7NmjdDpdOLy5cv2sYqKCgFAFBUVOSQrTzG7oIMHD+Kxxx5DZmamfczT0xMZGRmoqqpCY2OjxHTqMXPmzAE/5U2cOBFRUVG4cOGCpFTqVllZiYMHD+LDDz+UHUWVvvzyS1y7dg35+fnQaDS4c+cObDab7Fiq1N7eDgAYO3asYjwgIAAajcYxK1QqpNVq4e/v/6f7HTp0CM899xxCQkLsY88++ywiIiLw9ddfD2VEOxZEF3T27FlEREQMeNH3tGnTAAC1tbUSUhEACCFw7do1jBkzRnYU1bFarcjOzsaqVaswZcoU2XFU6ejRo/Dx8UFTUxMmTZoEb29v+Pj4YM2aNbBYLLLjqUpcXBwAICMjA7W1tWhsbMRXX32FwsJC5OTkYOTIkXIDqlhTUxOuX7+Op59+esC2adOm4ezZsw7JwYLoglpaWhAQEDBgvH+subnZ0ZHod1988QWampqQmpoqO4rq7N27F5cvX8bWrVtlR1Gt+vp69PX14fnnn8e8efNw6NAhrFy5Env37sWKFStkx1OVhIQEbN26FRUVFTAajQgJCcHSpUuRnZ2NgoIC2fFUraWlBQAeeBy/desWenp6hjzHiCH/BHK47u7uQd/j6Onpad9OjldXV4e1a9dixowZeOWVV2THUZWbN2/i7bffRm5uLvz8/GTHUa3Ozk50dXVh9erV9ruWFy1ahN7eXhQVFSEvLw8TJ06UnFI9xo8fjzlz5mDx4sXw9fXFt99+i+3bt8Pf3x9ZWVmy46lW/zH6z47jQ/2+ZhZEF6TT6Qb96aL/FI5Op3N0JNW7evUqFixYAL1eb79GlBxn8+bNMBgMyM7Olh1F1fr/70lLS1OMv/jiiygqKkJVVRULooOUlpYiMzMTZrMZQUFBAO6VdZvNho0bNyItLQ2+vr6SU6pT/78T2cdxnmJ2QQEBAfYl6vv1jwUGBjo6kqq1tbUhMTERt2/fRnl5Of/8Hay+vh779u1DTk4Ompub0dDQgIaGBlgsFty9excNDQ24deuW7Jiq0P93/79vjHj88ccBAK2trQ7PpFZ79uyB0Wi0l8N+CxcuRFdXl8Ouc6OB+k8tP+g4bjAYhnz1EGBBdEkxMTEwm832u9T6mUwm+3ZyDIvFgqSkJJjNZhw5cgRPPfWU7Eiq09TUBJvNhpycHISGhtq/TCYTzGYzQkNDkZeXJzumKkydOhXAvTm5X/910Tz97zjXrl2D1WodMH737l0AQF9fn6Mj0e+eeOIJ+Pn5oaamZsC206dPO+wYzoLogpKTk2G1WrFv3z77WE9PD4qLizF9+nQEBwdLTKceVqsVqampqKqqQllZGWbMmCE7kipNnjwZhw8fHvAVFRWFkJAQHD58GBkZGbJjqkJKSgoA4JNPPlGMf/zxxxgxYoT9zloaehERETh79uyANzsdOHAAGo0G0dHRkpIRACxevBhHjhxRPJbu2LFjMJvNWLJkiUMy8E0qLiolJQWHDx/Gq6++ivDwcHz22Wc4ffo0jh07hjlz5siOpwrr16/Hrl27kJSUZD8w3u+ll16SkIr6xcXF4bfffuObVBwsIyMDn376KVJSUhAbG4sTJ06grKwMmzZtwvbt22XHU43KykrEx8fD19cXWVlZ8PX1xZEjR/Ddd99h1apV2L9/v+yILmv37t24ffs2mpubUVhYiEWLFsFoNAIAsrOzodfr0djYCKPRiNGjR2PdunXo7OzEjh07EBQUhOrqaoecYuabVFxUd3e32LBhg/D39xdarVY888wzory8XHYsVYmNjRUAHvhFcvFNKnL09vaKd955R4wbN064u7uL8PBwUVBQIDuWKplMJpGYmCj8/f2Fu7u7iIiIEPn5+eLu3buyo7m0cePGPfC4cOnSJft+586dE3PnzhVeXl5i9OjRIj09XVy9etVhObmCSEREREQKvAaRiIiIiBRYEImIiIhIgQWRiIiIiBRYEImIiIhIgQWRiIiIiBRYEImIiIhIgQWRiIiIiBRYEImIiIhIgQWRiIiIiBRYEImIiIhIgQWRiIiIiBRYEImIiIhIgQWRiIiIiBRYEImIhoH09HS4ublh27ZtA7ZVVVXBy8sLvr6+qKurk5COiFyNmxBCyA5BRER/7OLFi3jyySfh7e2NS5cuQa/XAwDq6+sxc+ZM3LlzB0ePHsXMmTMlJyUiV8AVRCKiYWDChAnIyMhAa2srCgoKAAA3btxAYmIiWltbceDAAZZDIvrbcAWRiGiYaG5uRnh4ODw8PHD+/HksXrwYJpMJRUVFyMzMlB2PiFwIVxCJiIaJwMBAZGVloa2tDTExMTCZTMjNzWU5JKK/HVcQiYiGkZaWFgQFBcFms2H58uUoLi6WHYmIXBBXEImIhgkhBF577TXYbDYAwIgRIyQnIiJXxYJIRDRMvP766ygtLcX8+fMREBCAkpIS1NfXy45FRC6IBZGIaBjYtWsXdu7ciWnTpqGsrAxvvPEG+vr6kJubKzsaEbkgXoNIROTkysrKkJqairCwMFRVVcHPzw8WiwXh4eFobm7GmTNnEBMTIzsmEbkQriASETmxyspKLFu2DGPGjEF5eTn8/PwAAJ6enti0aROEEHjrrbckpyQiV8MVRCIiJ/Xzzz9j1qxZ6O3txfHjxzF9+nTF9t7eXoSHh6OxsREnT57E7NmzJSUlIlfDgkhERERECjzFTEREREQKLIhEREREpMCCSEREREQKLIhEREREpMCCSEREREQKLIhEREREpMCCSEREREQKLIhEREREpMCCSEREREQKLIhEREREpMCCSEREREQKLIhEREREpPBvw90Hk8PI4kYAAAAASUVORK5CYII=\n" 567 | }, 568 | "metadata": {} 569 | } 570 | ] 571 | }, 572 | { 573 | "cell_type": "markdown", 574 | "source": [ 575 | "## Collocation points" 576 | ], 577 | "metadata": { 578 | "id": "mPj-1ODfEFhe" 579 | } 580 | }, 581 | { 582 | "cell_type": "code", 583 | "source": [ 584 | "#%% Initial conditions\n", 585 | "def u0(t):\n", 586 | " z = -np.sin(1*math.pi*t)\n", 587 | " return z\n", 588 | "\n", 589 | "def du0_dt(tx):\n", 590 | " with tf.GradientTape() as g:\n", 591 | " g.watch(tx)\n", 592 | " u = u0(tx)\n", 593 | " du_dt = g.batch_jacobian(u, tx)[..., 0]\n", 594 | " return du_dt\n", 595 | "\n", 596 | "def RMS(S):\n", 597 | " rms = np.sqrt(np.mean(S**2))\n", 598 | " return rms\n", 599 | "\n", 600 | "#%% collocation points\n", 601 | "# create training input\n", 602 | "tx_eqn = np.random.rand(num_train_samples, 2)\n", 603 | "tx_eqn[..., 0] = T*tx_eqn[..., 0] # t = 0 ~ +1\n", 604 | "tx_eqn[..., 1] = L*tx_eqn[..., 1] # x = 0 ~ +10\n", 605 | "#print('\\nShape of t_eqn ==>',tx_eqn.shape)\n", 606 | "\n", 607 | "tx_ini = np.random.rand(num_train_samples, 2)\n", 608 | "tx_ini[..., 0] = 0 # t = 0\n", 609 | "tx_ini[..., 1] = L*tx_ini[..., 1] # x = 0 ~ +10\n", 610 | "#print('\\nShape of tx_ini ==>',tx_ini.shape)\n", 611 | "\n", 612 | "tx_bnd = np.random.rand(num_train_samples, 2)\n", 613 | "tx_bnd[..., 0] = T*tx_bnd[..., 0] # t = 0 ~ +1\n", 614 | "tx_bnd[..., 1] = L*np.round(tx_bnd[..., 1]) # x = 0 or +10\n", 615 | "#print('\\nShape of tx_bnd ==>',tx_bnd.shape)\n", 616 | "\n", 617 | "# initial and boundary conditions\n", 618 | "u_zero = np.zeros((num_train_samples, 1))\n", 619 | "u_ini = u0(tx_ini[:,1,None])\n", 620 | "du_dt_ini = np.zeros((num_train_samples, 1))" 621 | ], 622 | "metadata": { 623 | "id": "xHHrWjfvCeaP" 624 | }, 625 | "execution_count": null, 626 | "outputs": [] 627 | }, 628 | { 629 | "cell_type": "markdown", 630 | "source": [ 631 | "## Target function or objective function for BOPINN" 632 | ], 633 | "metadata": { 634 | "id": "_3hVo0wKEOyZ" 635 | } 636 | }, 637 | { 638 | "cell_type": "code", 639 | "source": [ 640 | "#%% Target function or ojective function, g(c) = (u_pred - u_true)^2; u_pred via PINNs\n", 641 | "def model_builder(ic):\n", 642 | " #ic = hp.Float('ic', min_value=0.1, max_value=1, step=10)\n", 643 | " print('\\n ## ->>>> PINNs simulation at speed = ' + str(ic))\n", 644 | "\n", 645 | " # build a PINN model\n", 646 | " network = Network.build()\n", 647 | " pinn = PINN(network,ic).build()\n", 648 | "\n", 649 | " # train the model using L-BFGS-B algorithm\n", 650 | " begin = time.time()\n", 651 | " x_train = [tx_eqn, tx_ini, tx_bnd]\n", 652 | " y_train = [u_zero, u_ini, du_dt_ini, u_zero]\n", 653 | " lbfgs = L_BFGS_B(model=pinn, x_train=x_train, y_train=y_train)\n", 654 | " lbfgs.fit()\n", 655 | " end = time.time()\n", 656 | " totaltime = end-begin\n", 657 | " print(\"\\n Total runtime is (min.)\",totaltime/60)\n", 658 | "\n", 659 | " # test the model\n", 660 | " tx = np.stack([np.full(t_test.shape, tilde_t), x_test], axis=-1)\n", 661 | " u_pred = network.predict(tx, batch_size=num_test_samples)\n", 662 | "\n", 663 | " # mse between u_pred via PINN and snapshot observation\n", 664 | " mse = -np.mean(np.square(u_analy - u_pred))\n", 665 | "\n", 666 | " del network, pinn, lbfgs, u_pred\n", 667 | "\n", 668 | " return mse" 669 | ], 670 | "metadata": { 671 | "id": "Q2NA5OvMCg64" 672 | }, 673 | "execution_count": null, 674 | "outputs": [] 675 | }, 676 | { 677 | "cell_type": "markdown", 678 | "source": [ 679 | "## Training" 680 | ], 681 | "metadata": { 682 | "id": "bJ-eT6gwEYSH" 683 | } 684 | }, 685 | { 686 | "cell_type": "code", 687 | "source": [ 688 | "#%% Bayesian Optimization\n", 689 | "# Attributes of BO\n", 690 | "itt_explore = 5\n", 691 | "itt = 45\n", 692 | "itt_all = itt_explore + itt\n", 693 | "n_runs = 10 # reduce this if colab has limited capability\n", 694 | "\n", 695 | "# bounds of BO\n", 696 | "pbounds = {'ic': (0.1, 1)}\n", 697 | "\n", 698 | "# Start BO\n", 699 | "mse_star_all = []\n", 700 | "cstar_all = []\n", 701 | "mse_all_all = []\n", 702 | "ic_all_all = []\n", 703 | "\n", 704 | "for r in range(n_runs):\n", 705 | " print('\\n ## ->>>> Run = ' + str(r))\n", 706 | "\n", 707 | " # define the model\n", 708 | " optimizer = BayesianOptimization(\n", 709 | " f=model_builder,\n", 710 | " pbounds=pbounds,\n", 711 | " allow_duplicate_points=True)\n", 712 | "\n", 713 | " # utility function\n", 714 | " util = UtilityFunction(kind='ucb',\n", 715 | " kappa=2.576,\n", 716 | " xi=0.0,\n", 717 | " kappa_decay=1,\n", 718 | " kappa_decay_delay=0)\n", 719 | "\n", 720 | " # run the model\n", 721 | " optimizer.maximize(init_points=itt_explore,\n", 722 | " n_iter=itt,\n", 723 | " acquisition_function=util)\n", 724 | "\n", 725 | " soln = optimizer.max\n", 726 | " resi = optimizer.res\n", 727 | "\n", 728 | " # optimum values\n", 729 | " mse_star = list(soln.values())[0]\n", 730 | " cstar = list(soln.values())[1]\n", 731 | " cstar2 = list(cstar.values())[0]\n", 732 | "\n", 733 | " # append all optimum values\n", 734 | " mse_star_all.append(mse_star)\n", 735 | " cstar_all.append(cstar2)\n", 736 | "\n", 737 | " # all run values\n", 738 | " mse_all, ic_all = [], []\n", 739 | " for i,res in enumerate(resi):\n", 740 | " mse = list(res.values())[0]\n", 741 | " ic = list(res.values())[1]\n", 742 | " ic2 = list(ic.values())[0]\n", 743 | "\n", 744 | " # append all run values\n", 745 | " mse_all.append(mse)\n", 746 | " ic_all.append(ic2)\n", 747 | "\n", 748 | " mse_all_all.append(np.array(mse_all))\n", 749 | " ic_all_all.append(np.array(ic_all))\n", 750 | "\n", 751 | " del optimizer\n", 752 | "\n", 753 | "mse_all_all = np.array(mse_all_all)\n", 754 | "mse_star_all = np.array(mse_star_all)\n", 755 | "ic_all_all = np.array(ic_all_all)\n", 756 | "cstar_all = np.array(cstar_all)" 757 | ], 758 | "metadata": { 759 | "id": "Mp27Bsy8Cl4P", 760 | "colab": { 761 | "base_uri": "https://localhost:8080/" 762 | }, 763 | "outputId": "27b03b23-b1dc-4b7b-c550-c7633601a453" 764 | }, 765 | "execution_count": null, 766 | "outputs": [ 767 | { 768 | "metadata": { 769 | "tags": null 770 | }, 771 | "name": "stdout", 772 | "output_type": "stream", 773 | "text": [ 774 | "\n", 775 | " ## ->>>> Run = 0\n", 776 | "| iter | target | ic |\n", 777 | "-------------------------------------\n", 778 | "\n", 779 | " ## ->>>> PINNs simulation at speed = 0.4372813898716186\n", 780 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 781 | "5000/5000 [==============================] - 523s 105ms/step\n", 782 | "\n", 783 | " Total runtime is (min.) 8.710416158040365\n", 784 | "1/1 [==============================] - 0s 127ms/step\n", 785 | "| \u001b[0m1 \u001b[0m | \u001b[0m-0.001148\u001b[0m | \u001b[0m0.4373 \u001b[0m |\n", 786 | "\n", 787 | " ## ->>>> PINNs simulation at speed = 0.353260871860698\n", 788 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 789 | "5000/5000 [==============================] - 457s 91ms/step\n", 790 | "\n", 791 | " Total runtime is (min.) 7.620774328708649\n", 792 | "1/1 [==============================] - 0s 108ms/step\n", 793 | "| \u001b[95m2 \u001b[0m | \u001b[95m-0.000404\u001b[0m | \u001b[95m0.3533 \u001b[0m |\n", 794 | "\n", 795 | " ## ->>>> PINNs simulation at speed = 0.9598970605863492\n", 796 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 797 | "5000/5000 [==============================] - 1106s 221ms/step\n", 798 | "\n", 799 | " Total runtime is (min.) 18.436524299780526\n", 800 | "1/1 [==============================] - 0s 105ms/step\n", 801 | "| \u001b[0m3 \u001b[0m | \u001b[0m-0.03358 \u001b[0m | \u001b[0m0.9599 \u001b[0m |\n", 802 | "\n", 803 | " ## ->>>> PINNs simulation at speed = 0.868646025552868\n", 804 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 805 | "5000/5000 [==============================] - 779s 156ms/step\n", 806 | "\n", 807 | " Total runtime is (min.) 12.977701457341512\n", 808 | "1/1 [==============================] - 0s 68ms/step\n", 809 | "| \u001b[0m4 \u001b[0m | \u001b[0m-0.02251 \u001b[0m | \u001b[0m0.8686 \u001b[0m |\n", 810 | "\n", 811 | " ## ->>>> PINNs simulation at speed = 0.15795893544754053\n", 812 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 813 | "5000/5000 [==============================] - 502s 100ms/step\n" 814 | ] 815 | }, 816 | { 817 | "metadata": { 818 | "tags": null 819 | }, 820 | "name": "stderr", 821 | "output_type": "stream", 822 | "text": [ 823 | "WARNING:tensorflow:5 out of the last 5 calls to .predict_function at 0x79d518d530a0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" 824 | ] 825 | }, 826 | { 827 | "metadata": { 828 | "tags": null 829 | }, 830 | "name": "stdout", 831 | "output_type": "stream", 832 | "text": [ 833 | "\n", 834 | " Total runtime is (min.) 8.373841834068298\n", 835 | "1/1 [==============================] - 0s 109ms/step\n", 836 | "| \u001b[95m5 \u001b[0m | \u001b[95m-7.06e-05\u001b[0m | \u001b[95m0.158 \u001b[0m |\n", 837 | "\n", 838 | " ## ->>>> PINNs simulation at speed = 0.35322383568991045\n", 839 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 840 | "5000/5000 [==============================] - 616s 123ms/step\n" 841 | ] 842 | }, 843 | { 844 | "metadata": { 845 | "tags": null 846 | }, 847 | "name": "stderr", 848 | "output_type": "stream", 849 | "text": [ 850 | "WARNING:tensorflow:6 out of the last 6 calls to .predict_function at 0x79d50627e170> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n" 851 | ] 852 | }, 853 | { 854 | "output_type": "stream", 855 | "name": "stdout", 856 | "text": [ 857 | "\n", 858 | " Total runtime is (min.) 10.258907715479532\n", 859 | "1/1 [==============================] - 0s 107ms/step\n", 860 | "| \u001b[0m6 \u001b[0m | \u001b[0m-0.000410\u001b[0m | \u001b[0m0.3532 \u001b[0m |\n", 861 | "\n", 862 | " ## ->>>> PINNs simulation at speed = 0.6265603597846434\n", 863 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 864 | "5000/5000 [==============================] - 702s 140ms/step\n", 865 | "\n", 866 | " Total runtime is (min.) 11.704780928293864\n", 867 | "1/1 [==============================] - 0s 105ms/step\n", 868 | "| \u001b[0m7 \u001b[0m | \u001b[0m-0.005768\u001b[0m | \u001b[0m0.6266 \u001b[0m |\n", 869 | "\n", 870 | " ## ->>>> PINNs simulation at speed = 0.10004206006951194\n", 871 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 872 | "5000/5000 [==============================] - 364s 73ms/step\n", 873 | "\n", 874 | " Total runtime is (min.) 6.072018718719482\n", 875 | "1/1 [==============================] - 0s 112ms/step\n", 876 | "| \u001b[0m8 \u001b[0m | \u001b[0m-0.000105\u001b[0m | \u001b[0m0.1 \u001b[0m |\n", 877 | "\n", 878 | " ## ->>>> PINNs simulation at speed = 0.25506179377329546\n", 879 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 880 | "5000/5000 [==============================] - 443s 89ms/step\n", 881 | "\n", 882 | " Total runtime is (min.) 7.377660123507182\n", 883 | "1/1 [==============================] - 0s 83ms/step\n", 884 | "| \u001b[0m9 \u001b[0m | \u001b[0m-9.066e-0\u001b[0m | \u001b[0m0.2551 \u001b[0m |\n", 885 | "\n", 886 | " ## ->>>> PINNs simulation at speed = 0.20751207571890762\n", 887 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 888 | "5000/5000 [==============================] - 375s 75ms/step\n", 889 | "\n", 890 | " Total runtime is (min.) 6.241973801453908\n", 891 | "1/1 [==============================] - 0s 119ms/step\n", 892 | "| \u001b[0m10 \u001b[0m | \u001b[0m-7.107e-0\u001b[0m | \u001b[0m0.2075 \u001b[0m |\n", 893 | "\n", 894 | " ## ->>>> PINNs simulation at speed = 0.29827013284790804\n", 895 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 896 | "5000/5000 [==============================] - 419s 84ms/step\n", 897 | "\n", 898 | " Total runtime is (min.) 6.982113985220591\n", 899 | "1/1 [==============================] - 0s 76ms/step\n", 900 | "| \u001b[0m11 \u001b[0m | \u001b[0m-0.000182\u001b[0m | \u001b[0m0.2983 \u001b[0m |\n", 901 | "\n", 902 | " ## ->>>> PINNs simulation at speed = 0.12646718809275764\n", 903 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 904 | "5000/5000 [==============================] - 481s 96ms/step\n", 905 | "\n", 906 | " Total runtime is (min.) 8.016359814008077\n", 907 | "1/1 [==============================] - 0s 81ms/step\n", 908 | "| \u001b[0m12 \u001b[0m | \u001b[0m-8.387e-0\u001b[0m | \u001b[0m0.1265 \u001b[0m |\n", 909 | "\n", 910 | " ## ->>>> PINNs simulation at speed = 0.18291510701869837\n", 911 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 912 | "5000/5000 [==============================] - 367s 73ms/step\n", 913 | "\n", 914 | " Total runtime is (min.) 6.108590332667033\n", 915 | "1/1 [==============================] - 0s 80ms/step\n", 916 | "| \u001b[95m13 \u001b[0m | \u001b[95m-6.506e-0\u001b[0m | \u001b[95m0.1829 \u001b[0m |\n", 917 | "\n", 918 | " ## ->>>> PINNs simulation at speed = 0.2295051729538426\n", 919 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 920 | "5000/5000 [==============================] - 510s 102ms/step\n", 921 | "\n", 922 | " Total runtime is (min.) 8.495574649175008\n", 923 | "1/1 [==============================] - 0s 133ms/step\n", 924 | "| \u001b[0m14 \u001b[0m | \u001b[0m-7.231e-0\u001b[0m | \u001b[0m0.2295 \u001b[0m |\n", 925 | "\n", 926 | " ## ->>>> PINNs simulation at speed = 0.17489332261182525\n", 927 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 928 | "5000/5000 [==============================] - 480s 96ms/step\n", 929 | "\n", 930 | " Total runtime is (min.) 8.002258733908336\n", 931 | "1/1 [==============================] - 0s 67ms/step\n", 932 | "| \u001b[0m15 \u001b[0m | \u001b[0m-6.664e-0\u001b[0m | \u001b[0m0.1749 \u001b[0m |\n", 933 | "\n", 934 | " ## ->>>> PINNs simulation at speed = 0.19438706454255616\n", 935 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 936 | "5000/5000 [==============================] - 365s 73ms/step\n", 937 | "\n", 938 | " Total runtime is (min.) 6.076679023106893\n", 939 | "1/1 [==============================] - 0s 67ms/step\n", 940 | "| \u001b[95m16 \u001b[0m | \u001b[95m-5.969e-0\u001b[0m | \u001b[95m0.1944 \u001b[0m |\n", 941 | "\n", 942 | " ## ->>>> PINNs simulation at speed = 0.1931523178425959\n", 943 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 944 | "5000/5000 [==============================] - 478s 96ms/step\n", 945 | "\n", 946 | " Total runtime is (min.) 7.9739015102386475\n", 947 | "1/1 [==============================] - 0s 71ms/step\n", 948 | "| \u001b[95m17 \u001b[0m | \u001b[95m-5.967e-0\u001b[0m | \u001b[95m0.1932 \u001b[0m |\n", 949 | "\n", 950 | " ## ->>>> PINNs simulation at speed = 0.1472501091609439\n", 951 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 952 | "5000/5000 [==============================] - 498s 100ms/step\n", 953 | "\n", 954 | " Total runtime is (min.) 8.299330997467042\n", 955 | "1/1 [==============================] - 0s 67ms/step\n", 956 | "| \u001b[0m18 \u001b[0m | \u001b[0m-7.572e-0\u001b[0m | \u001b[0m0.1473 \u001b[0m |\n", 957 | "\n", 958 | " ## ->>>> PINNs simulation at speed = 0.1864475687192167\n", 959 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 960 | "5000/5000 [==============================] - 395s 79ms/step\n", 961 | "\n", 962 | " Total runtime is (min.) 6.576291259129842\n", 963 | "1/1 [==============================] - 0s 106ms/step\n", 964 | "| \u001b[0m19 \u001b[0m | \u001b[0m-6.111e-0\u001b[0m | \u001b[0m0.1864 \u001b[0m |\n", 965 | "\n", 966 | " ## ->>>> PINNs simulation at speed = 0.20641322721099062\n", 967 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 968 | "5000/5000 [==============================] - 415s 83ms/step\n", 969 | "\n", 970 | " Total runtime is (min.) 6.909338291486105\n", 971 | "1/1 [==============================] - 0s 70ms/step\n", 972 | "| \u001b[0m20 \u001b[0m | \u001b[0m-6.846e-0\u001b[0m | \u001b[0m0.2064 \u001b[0m |\n", 973 | "\n", 974 | " ## ->>>> PINNs simulation at speed = 0.17467333008463468\n", 975 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 976 | "5000/5000 [==============================] - 407s 81ms/step\n", 977 | "\n", 978 | " Total runtime is (min.) 6.785109670956929\n", 979 | "1/1 [==============================] - 0s 98ms/step\n", 980 | "| \u001b[0m21 \u001b[0m | \u001b[0m-6.953e-0\u001b[0m | \u001b[0m0.1747 \u001b[0m |\n", 981 | "\n", 982 | " ## ->>>> PINNs simulation at speed = 0.19224016373696584\n", 983 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 984 | "5000/5000 [==============================] - 506s 101ms/step\n", 985 | "\n", 986 | " Total runtime is (min.) 8.428928422927857\n", 987 | "1/1 [==============================] - 0s 69ms/step\n", 988 | "| \u001b[0m22 \u001b[0m | \u001b[0m-6.054e-0\u001b[0m | \u001b[0m0.1922 \u001b[0m |\n", 989 | "\n", 990 | " ## ->>>> PINNs simulation at speed = 0.19200099153944267\n", 991 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 992 | "5000/5000 [==============================] - 437s 87ms/step\n", 993 | "\n", 994 | " Total runtime is (min.) 7.2850401679674786\n", 995 | "1/1 [==============================] - 0s 103ms/step\n", 996 | "| \u001b[0m23 \u001b[0m | \u001b[0m-6.077e-0\u001b[0m | \u001b[0m0.192 \u001b[0m |\n", 997 | "\n", 998 | " ## ->>>> PINNs simulation at speed = 0.19115332829277382\n", 999 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 1000 | "5000/5000 [==============================] - 335s 67ms/step\n", 1001 | "\n", 1002 | " Total runtime is (min.) 5.587451763947805\n", 1003 | "1/1 [==============================] - 0s 68ms/step\n", 1004 | "| \u001b[0m24 \u001b[0m | \u001b[0m-6.288e-0\u001b[0m | \u001b[0m0.1912 \u001b[0m |\n", 1005 | "\n", 1006 | " ## ->>>> PINNs simulation at speed = 0.19599276179575198\n", 1007 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 1008 | "5000/5000 [==============================] - 454s 91ms/step\n", 1009 | "\n", 1010 | " Total runtime is (min.) 7.566520047187805\n", 1011 | "1/1 [==============================] - 0s 101ms/step\n", 1012 | "| \u001b[0m25 \u001b[0m | \u001b[0m-6.409e-0\u001b[0m | \u001b[0m0.196 \u001b[0m |\n", 1013 | "\n", 1014 | " ## ->>>> PINNs simulation at speed = 0.1854055143664269\n", 1015 | "Optimizer: L-BFGS-B (maxiter=5000)\n", 1016 | "5000/5000 [==============================] - 456s 91ms/step\n", 1017 | "\n", 1018 | " Total runtime is (min.) 7.592611781756083\n", 1019 | "1/1 [==============================] - 0s 69ms/step\n", 1020 | "| \u001b[0m26 \u001b[0m | \u001b[0m-6.051e-0\u001b[0m | \u001b[0m0.1854 \u001b[0m |\n", 1021 | "\n", 1022 | " ## ->>>> PINNs simulation at speed = 0.186162721420857\n", 1023 | "Optimizer: L-BFGS-B (maxiter=5000)\n" 1024 | ] 1025 | } 1026 | ] 1027 | }, 1028 | { 1029 | "cell_type": "markdown", 1030 | "source": [ 1031 | "## Processing and plotting" 1032 | ], 1033 | "metadata": { 1034 | "id": "Bp5UWXSnE_YX" 1035 | } 1036 | }, 1037 | { 1038 | "cell_type": "code", 1039 | "source": [ 1040 | "#%% Process the BO results\n", 1041 | "# max, min, mean and sd target function/objective function value across different runs\n", 1042 | "max_mse_star_allruns, min_mse_star_allruns = np.max(mse_star_all), np.min(mse_star_all)\n", 1043 | "mean_mse_star_allruns, std_mse_star_allruns = np.mean(mse_star_all), np.std(mse_star_all)\n", 1044 | "\n", 1045 | "# optima corresponding to abovementioned optimal points\n", 1046 | "idx_max_mse_star_allruns = np.where(max_mse_star_allruns == mse_star_all)\n", 1047 | "idx_min_mse_star_allruns = np.where(min_mse_star_allruns == mse_star_all)\n", 1048 | "\n", 1049 | "max_cstar_allruns = cstar_all[idx_max_mse_star_allruns]\n", 1050 | "min_cstar_allruns = cstar_all[idx_min_mse_star_allruns]\n", 1051 | "mean_cstar_allruns = np.mean(cstar_all)\n", 1052 | "std_cstar_allruns = np.std(cstar_all)\n", 1053 | "\n", 1054 | "print(\"Max (best optimal) tf across runs = \",max_mse_star_allruns)\n", 1055 | "print(\"Min (worst optimal) tf across runs = \",min_mse_star_allruns)\n", 1056 | "print(\"Mean tf across runs = \",mean_mse_star_allruns)\n", 1057 | "print(\"Std tf across runs = \",std_mse_star_allruns)\n", 1058 | "\n", 1059 | "print(\"Max (best optimal) c* across runs = \",max_cstar_allruns)\n", 1060 | "print(\"Min (worst optimal) c* across runs = \",min_cstar_allruns)\n", 1061 | "print(\"Mean c* across runs = \",mean_cstar_allruns)\n", 1062 | "print(\"Std c* across runs = \",std_cstar_allruns)" 1063 | ], 1064 | "metadata": { 1065 | "id": "m45_nEHXEcel" 1066 | }, 1067 | "execution_count": null, 1068 | "outputs": [] 1069 | }, 1070 | { 1071 | "cell_type": "code", 1072 | "source": [ 1073 | "# plot best optimal run with the optima\n", 1074 | "idx_max_all = []\n", 1075 | "for i in range(mse_all_all.shape[0]):\n", 1076 | " idx_max = np.where(mse_all_all[i,:] == mse_star_all[i])\n", 1077 | " idx_max = idx_max[0][0]\n", 1078 | " idx_max_all.append(idx_max)\n", 1079 | "\n", 1080 | "mean_mse_all = np.mean(mse_all_all, axis=0)\n", 1081 | "std_mse_all = np.std(mse_all_all, axis=0)\n", 1082 | "mean_ic_all = np.mean(ic_all_all, axis=0)\n", 1083 | "std_ic_all = np.std(ic_all_all, axis=0)\n", 1084 | "\n", 1085 | "opt_mse_run = mse_all_all[idx_max_mse_star_allruns[0][0]]\n", 1086 | "opt_c_run = ic_all_all[idx_max_mse_star_allruns[0][0]]\n", 1087 | "opt_mse = mse_star_all[idx_max_mse_star_allruns[0][0]]\n", 1088 | "opt_c = cstar_all[idx_max_mse_star_allruns[0][0]]\n", 1089 | "\n", 1090 | "txt = 'c* = '+ str(round(opt_c,4))\n", 1091 | "plt.figure(figsize = (8, 6))\n", 1092 | "plt.plot(opt_c_run,opt_mse_run,'ob',markersize=6)\n", 1093 | "plt.plot(opt_c,opt_mse,'*r',markersize=8, label = 'Best optima')\n", 1094 | "#plt.text(0.75, -0.03, txt, fontsize=18, c = 'r')\n", 1095 | "plt.xlabel(\"velocity, c\",fontsize=20)\n", 1096 | "plt.ylabel(\"target function, g(c)\",fontsize=20)\n", 1097 | "plt.xticks(fontsize=20)\n", 1098 | "plt.yticks(fontsize=20)\n", 1099 | "plt.legend(fontsize = 14, loc='upper left')\n", 1100 | "plt.savefig('tfvsc_'+str(idx_data+1)+'.png', bbox_inches='tight', dpi=600)\n", 1101 | "plt.show()" 1102 | ], 1103 | "metadata": { 1104 | "id": "qaTRHKjgFDP3" 1105 | }, 1106 | "execution_count": null, 1107 | "outputs": [] 1108 | } 1109 | ], 1110 | "metadata": { 1111 | "colab": { 1112 | "provenance": [], 1113 | "gpuType": "T4" 1114 | }, 1115 | "kernelspec": { 1116 | "display_name": "Python 3", 1117 | "name": "python3" 1118 | }, 1119 | "accelerator": "GPU" 1120 | }, 1121 | "nbformat": 4, 1122 | "nbformat_minor": 0 1123 | } --------------------------------------------------------------------------------