├── .gitignore ├── .github └── workflows │ └── codeql.yml ├── README.md ├── LICENSE └── experiments ├── lts.py ├── ltc.py └── experiments.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | venv 3 | __pycache__ -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | schedule: 7 | - cron: '24 18 * * 0' 8 | 9 | jobs: 10 | analyze: 11 | name: Analyze 12 | runs-on: ubuntu-latest 13 | 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | language: [ 'python', 'javascript' ] 18 | 19 | steps: 20 | - name: Checkout repository 21 | uses: actions/checkout@v2 22 | 23 | - name: Initialize CodeQL 24 | uses: github/codeql-action/init@v2 25 | with: 26 | languages: ${{ matrix.language }} 27 | 28 | - name: Autobuild 29 | uses: github/codeql-action/autobuild@v2 30 | 31 | - name: Perform CodeQL Analysis 32 | uses: github/codeql-action/analyze@v2 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Liquid Time-stochasticity Networks (LTSs) 2 | 3 | [![CodeQL](https://github.com/Ammar-Raneez/FYP_Algorithm/actions/workflows/codeql.yml/badge.svg)](https://github.com/Ammar-Raneez/FYP_Algorithm/actions/workflows/codeql.yml) 4 | [![CodeFactor](https://www.codefactor.io/repository/github/ammar-raneez/liquid-time-stochasticity-networks/badge)](https://www.codefactor.io/repository/github/ammar-raneez/liquid-time-stochasticity-networks) 5 | 6 | This is the official repository for Liquid TIme-stochasticity networks described in paper: https://doi.org/10.1109/CCWC57344.2023.10099071 7 | 8 | This implementation utilizes the Euler Maruyama solver to perform forward propagation and relies on the conventional backpropagation through-time (BPTT) to train the models. 9 | 10 | ## Prerequisites 11 | 12 | The architecture was built using Keras and TensorFlow 2.0+ and Python 3+ on the Windows 11 machine. 13 | 14 | ## Experiments 15 | 16 | `experiments/experiments.ipynb` demonstrates a couple of experiments attempting to model bitcoin prices. 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Ammar 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /experiments/lts.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A TensorFlow V2 implementation of the Liquid Time-Stochasticity cell proposed 3 | by Raneez and Wirasingha (2023) at https://doi.org/10.1109/CCWC57344.2023.10099071 4 | 5 | An RNN with continuous-time hidden 6 | states determined by stochastic differential equations 7 | ''' 8 | 9 | import tensorflow as tf 10 | import numpy as np 11 | from enum import Enum 12 | 13 | class MappingType(Enum): 14 | Affine = 0 15 | 16 | class SDESolver(Enum): 17 | EulerMaruyama = 0 18 | 19 | class NoiseType(Enum): 20 | diagonal = 0 21 | 22 | class LTSCell(tf.keras.layers.Layer): 23 | 24 | def __init__(self, units, **kwargs): 25 | ''' 26 | Initializes the LTS cell & parameters 27 | Calls parent Layer constructor to initialize required fields 28 | Variables adapted from ltc.py 29 | ''' 30 | 31 | super(LTSCell, self).__init__(**kwargs) 32 | self.input_size = -1 33 | self.units = units 34 | self.built = False 35 | 36 | self._time_step = 1.0 37 | self._brownian_motion = None 38 | 39 | # Number of SDE solver steps in one RNN step 40 | self._sde_solver_unfolds = 6 41 | self._solver = SDESolver.EulerMaruyama 42 | self._noise_type = NoiseType.diagonal 43 | 44 | self._input_mapping = MappingType.Affine 45 | 46 | self._erev_init_factor = 1 47 | 48 | self._w_init_max = 1.0 49 | self._w_init_min = 0.01 50 | self._cm_init_min = 0.5 51 | self._cm_init_max = 0.5 52 | self._gleak_init_min = 1 53 | self._gleak_init_max = 1 54 | 55 | self._w_min_value = 0.00001 56 | self._w_max_value = 1000 57 | self._gleak_min_value = 0.00001 58 | self._gleak_max_value = 1000 59 | self._cm_t_min_value = 0.000001 60 | self._cm_t_max_value = 1000 61 | 62 | self._fix_cm = None 63 | self._fix_gleak = None 64 | self._fix_vleak = None 65 | 66 | self._input_weights = None 67 | self._input_biases = None 68 | 69 | @property 70 | def state_size(self): 71 | return self.units 72 | 73 | def build(self, input_shape): 74 | ''' 75 | Automatically triggered the first time __call__ is run 76 | ''' 77 | 78 | self.input_size = int(input_shape[-1]) 79 | self._init_variables() 80 | self.built = True 81 | 82 | @tf.function 83 | def call(self, inputs, states): 84 | ''' 85 | Automatically calls build() the first time. 86 | Runs the LTS cell for one step using the previous RNN cell output & state 87 | by calculating the SDE solver to generate the next output and state 88 | ''' 89 | 90 | inputs = self._map_weights_and_biases(inputs) 91 | next_state = self._sde_solver_euler_maruyama(inputs, states) 92 | output = next_state 93 | return output, next_state 94 | 95 | def get_config(self): 96 | ''' 97 | Enable serialization 98 | ''' 99 | 100 | config = super(LTSCell, self).get_config() 101 | config.update({ 'units': self.units }) 102 | return config 103 | 104 | ### Helper methods ### 105 | def _init_variables(self): 106 | ''' 107 | Creates the variables to be used within __call__ 108 | ''' 109 | 110 | # Define sensory variables 111 | self.sensory_mu = tf.Variable( 112 | tf.random.uniform( 113 | [self.input_size, self.units], 114 | minval = 0.3, 115 | maxval = 0.8, 116 | dtype = tf.float32 117 | ), 118 | name = 'sensory_mu', 119 | trainable = True, 120 | ) 121 | 122 | self.sensory_sigma = tf.Variable( 123 | tf.random.uniform( 124 | [self.input_size, self.units], 125 | minval = 3.0, 126 | maxval = 8.0, 127 | dtype = tf.float32 128 | ), 129 | name = 'sensory_sigma', 130 | trainable = True, 131 | ) 132 | 133 | self.sensory_W = tf.Variable( 134 | tf.constant( 135 | np.random.uniform( 136 | low = self._w_init_min, 137 | high = self._w_init_max, 138 | size = [self.input_size, self.units] 139 | ), 140 | dtype = tf.float32 141 | ), 142 | name = 'sensory_W', 143 | trainable = True, 144 | shape = [self.input_size, self.units] 145 | ) 146 | 147 | sensory_erev_init = 2 * np.random.randint( 148 | low = 0, 149 | high = 2, 150 | size = [self.input_size, self.units] 151 | ) - 1 152 | self.sensory_erev = tf.Variable( 153 | tf.constant( 154 | sensory_erev_init * self._erev_init_factor, 155 | dtype = tf.float32 156 | ), 157 | name = 'sensory_erev', 158 | trainable = True, 159 | shape = [self.input_size, self.units] 160 | ) 161 | 162 | # Define base stochastic differential equation variables 163 | self.mu = tf.Variable( 164 | tf.random.uniform( 165 | [self.units, self.units], 166 | minval = 0.3, 167 | maxval = 0.8, 168 | dtype = tf.float32 169 | ), 170 | name = 'mu', 171 | trainable = True, 172 | ) 173 | 174 | self.sigma = tf.Variable( 175 | tf.random.uniform( 176 | [self.units, self.units], 177 | minval = 3.0, 178 | maxval = 8.0, 179 | dtype = tf.float32 180 | ), 181 | name = 'sigma', 182 | trainable = True, 183 | ) 184 | 185 | self.W = tf.Variable( 186 | tf.constant( 187 | np.random.uniform( 188 | low = self._w_init_min, 189 | high = self._w_init_max, 190 | size = [self.units, self.units] 191 | ), 192 | dtype = tf.float32 193 | ), 194 | name = 'W', 195 | trainable = True, 196 | shape = [self.units, self.units] 197 | ) 198 | 199 | erev_init = 2 * np.random.randint( 200 | low = 0, 201 | high = 2, 202 | size = [self.units, self.units] 203 | ) - 1 204 | self.erev = tf.Variable( 205 | tf.constant( 206 | erev_init * self._erev_init_factor, 207 | dtype = tf.float32 208 | ), 209 | name = 'erev', 210 | trainable = True, 211 | shape = [self.units, self.units] 212 | ) 213 | 214 | # Define a simple Wiener process (Brownian motion) 215 | self._brownian_motion = tf.Variable( 216 | tf.random.normal( 217 | [self.units], 218 | mean = 0.0, 219 | stddev = tf.sqrt(self._time_step), 220 | dtype = tf.float32 221 | ) 222 | ) 223 | 224 | # Synaptic leakage conductance variables of the neural dynamics of small species 225 | if self._fix_vleak is None: 226 | self.vleak = tf.Variable( 227 | tf.random.uniform( 228 | [self.units], 229 | minval = -0.2, 230 | maxval = 0.2, 231 | dtype = tf.float32 232 | ), 233 | name = 'vleak', 234 | trainable = True, 235 | ) 236 | else: 237 | self.vleak = tf.Variable( 238 | tf.constant(self._fix_vleak, dtype = tf.float32), 239 | name = 'vleak', 240 | trainable = False, 241 | shape = [self.units] 242 | ) 243 | 244 | if self._fix_gleak is None: 245 | initializer = tf.constant(self._gleak_init_min, dtype = tf.float32) 246 | 247 | if self._gleak_init_max > self._gleak_init_min: 248 | initializer = tf.random.uniform( 249 | [self.units], 250 | minval = self._gleak_init_min, 251 | maxval = self._gleak_init_max, 252 | dtype = tf.float32 253 | ) 254 | 255 | self.gleak = tf.Variable( 256 | initializer, 257 | name = 'gleak', 258 | trainable = True, 259 | ) 260 | else: 261 | self.gleak = tf.Variable( 262 | tf.constant(self._fix_gleak), 263 | name = 'gleak', 264 | trainable = False, 265 | shape = [self.units] 266 | ) 267 | 268 | if self._fix_cm is None: 269 | initializer = tf.constant(self._cm_init_min, dtype = tf.float32) 270 | 271 | if self._cm_init_max > self._cm_init_min: 272 | initializer = tf.random.uniform( 273 | [self.units], 274 | minval = self._cm_init_min, 275 | maxval = self._cm_init_max, 276 | dtype = tf.float32 277 | ) 278 | 279 | self.cm_t = tf.Variable( 280 | initializer, 281 | name = 'cm_t', 282 | trainable = True, 283 | ) 284 | else: 285 | self.cm_t = tf.Variable( 286 | tf.constant(self._fix_cm), 287 | name = 'cm_t', 288 | trainable = False, 289 | shape = [self.units] 290 | ) 291 | 292 | def _map_weights_and_biases(self, inputs): 293 | ''' 294 | Initializes weights & biases to be used 295 | ''' 296 | 297 | # Create a workaround from creating tf Variables every function call 298 | # init with None and set only if not None - aka only first time 299 | if self._input_weights is None: 300 | self._input_weights = tf.Variable( 301 | lambda: tf.ones( 302 | [self.input_size], 303 | dtype = tf.float32 304 | ), 305 | name = 'input_weights', 306 | trainable = True 307 | ) 308 | 309 | if self._input_biases is None: 310 | self._input_biases = tf.Variable( 311 | lambda: tf.zeros( 312 | [self.input_size], 313 | dtype = tf.float32 314 | ), 315 | name = 'input_biases', 316 | trainable = True 317 | ) 318 | 319 | inputs = inputs * self._input_weights 320 | inputs = inputs + self._input_biases 321 | 322 | return inputs 323 | 324 | @tf.function 325 | def _sde_solver_euler_maruyama(self, inputs, states): 326 | ''' 327 | Implement Euler Maruyama implicit SDE solver 328 | ''' 329 | 330 | for _ in range(self._sde_solver_unfolds): 331 | # Compute drift and diffusion terms 332 | drift = self._sde_solver_drift(inputs, states) 333 | diffusion = self._sde_solver_diffusion(inputs, states) 334 | 335 | # Compute the next state 336 | states = states + drift * self._time_step + diffusion * self._brownian_motion 337 | states = tf.reshape(states, shape=[int(self._time_step), self.units]) 338 | 339 | return states 340 | 341 | @tf.function 342 | def _sde_solver_drift(self, inputs, states): 343 | ''' 344 | Compute the drift term of the Euler-Maruyama SDE solver 345 | Implement custom Euler ODE solver - first-order numerical procedure 346 | Utilize the LTC's deterministic solver 347 | ''' 348 | 349 | # State returned as -> tuple(Tensor); previous return state x(t), to produce x(t+1) 350 | v_pre = states[0] 351 | 352 | sensory_w_activation = self.sensory_W * self._sigmoid(inputs, self.sensory_mu, self.sensory_sigma) 353 | sensory_rev_activation = sensory_w_activation * self.sensory_erev 354 | 355 | w_numerator_sensory = tf.reduce_sum(input_tensor = sensory_rev_activation, axis = 1) 356 | w_denominator_sensory = tf.reduce_sum(input_tensor = sensory_w_activation, axis = 1) 357 | 358 | for _ in range(self._sde_solver_unfolds): 359 | w_activation = self.W * self._sigmoid(v_pre, self.mu, self.sigma) 360 | 361 | rev_activation = w_activation * self.erev 362 | 363 | w_numerator = tf.reduce_sum(input_tensor = rev_activation, axis = 1) + w_numerator_sensory 364 | w_denominator = tf.reduce_sum(input_tensor = w_activation, axis = 1) + w_denominator_sensory 365 | 366 | numerator = self.cm_t * v_pre + self.gleak * self.vleak + w_numerator 367 | denominator = self.cm_t + self.gleak + w_denominator 368 | 369 | v_pre = numerator / denominator 370 | 371 | return v_pre 372 | 373 | @tf.function 374 | def _sde_solver_diffusion(self, inputs, states): 375 | ''' 376 | Compute the diffusion term of the Euler-Maruyama SDE solver 377 | ''' 378 | 379 | return 1.0 380 | 381 | @tf.function 382 | def _sigmoid(self, v_pre, mu, sigma): 383 | v_pre = tf.reshape(v_pre, [-1, v_pre.shape[-1], 1]) 384 | mues = v_pre - mu 385 | x = sigma * mues 386 | return tf.nn.sigmoid(x) 387 | 388 | # References 389 | # https://splunktool.com/how-can-i-implement-a-custom-rnn-specifically-an-esn-in-tensorflow 390 | # https://colab.research.google.com/github/luckykadam/adder/blob/master/rnn_full_adder.ipynb 391 | # https://www.tutorialexample.com/build-custom-rnn-by-inheriting-rnncell-in-tensorflow-tensorflow-tutorial/ 392 | # https://notebook.community/tensorflow/docs-l10n/site/en-snapshot/guide/migrate 393 | # https://www.tensorflow.org/api_docs/python/tf/keras/layers/AbstractRNNCell 394 | # https://www.tensorflow.org/guide/keras/custom_layers_and_models/#layers_are_recursively_composable 395 | # https://www.tensorflow.org/guide/function#creating_tfvariables 396 | # https://docs.sciml.ai/DiffEqDocs/stable/solvers/sde_solve/#Full-List-of-Methods 397 | -------------------------------------------------------------------------------- /experiments/ltc.py: -------------------------------------------------------------------------------- 1 | ''' 2 | A TensorFlow V2 implementation of the Liquid Time-Constant cell proposed 3 | by Hasani et al. (2020) at https://arxiv.org/pdf/2006.04439.pdf 4 | 5 | An RNN with continuous-time hidden 6 | states determined by ordinary differential equations 7 | ''' 8 | 9 | import tensorflow as tf 10 | import numpy as np 11 | from enum import Enum 12 | 13 | class MappingType(Enum): 14 | Identity = 0 15 | Linear = 1 16 | Affine = 2 17 | 18 | class ODESolver(Enum): 19 | SemiImplicit = 0 20 | Explicit = 1 21 | RungeKutta = 2 22 | 23 | class LTCCell(tf.keras.layers.Layer): 24 | 25 | def __init__(self, units, **kwargs): 26 | ''' 27 | Initializes the LTC cell & parameters 28 | Calls parent Layer constructor to initialize required fields 29 | ''' 30 | 31 | super(LTCCell, self).__init__(**kwargs) 32 | self.input_size = -1 33 | self.units = units 34 | self.built = False 35 | 36 | # Number of ODE solver steps in one RNN step 37 | self._ode_solver_unfolds = 6 38 | self._solver = ODESolver.SemiImplicit 39 | 40 | self._input_mapping = MappingType.Affine 41 | 42 | self._erev_init_factor = 1 43 | 44 | self._w_init_max = 1.0 45 | self._w_init_min = 0.01 46 | self._cm_init_min = 0.5 47 | self._cm_init_max = 0.5 48 | self._gleak_init_min = 1 49 | self._gleak_init_max = 1 50 | 51 | self._w_min_value = 0.00001 52 | self._w_max_value = 1000 53 | self._gleak_min_value = 0.00001 54 | self._gleak_max_value = 1000 55 | self._cm_t_min_value = 0.000001 56 | self._cm_t_max_value = 1000 57 | 58 | self._fix_cm = None 59 | self._fix_gleak = None 60 | self._fix_vleak = None 61 | 62 | self._input_weights = None 63 | self._input_biases = None 64 | 65 | @property 66 | def state_size(self): 67 | return self.units 68 | 69 | def build(self, input_shape): 70 | ''' 71 | Automatically triggered the first time __call__ is run 72 | ''' 73 | 74 | self.input_size = int(input_shape[-1]) 75 | self._get_variables() 76 | self.built = True 77 | 78 | @tf.function 79 | def call(self, inputs, states): 80 | ''' 81 | Automatically calls build() the first time. 82 | Runs the LTC cell for one step using the previous RNN cell output & state 83 | by calculating a type of ODE solver to generate the next output and state 84 | ''' 85 | 86 | inputs = self._map_inputs(inputs) 87 | 88 | if self._solver == ODESolver.Explicit: 89 | next_state = self._ode_step_explicit( 90 | inputs, 91 | states, 92 | _ode_solver_unfolds = self._ode_solver_unfolds 93 | ) 94 | elif self._solver == ODESolver.SemiImplicit: 95 | next_state = self._ode_step_hybrid_euler(inputs, states) 96 | elif self._solver == ODESolver.RungeKutta: 97 | next_state = self._ode_step_runge_kutta(inputs, states) 98 | else: 99 | raise ValueError(f'Unknown ODE solver \'{str(self._solver)}\'') 100 | 101 | output = next_state 102 | return output, next_state 103 | 104 | def get_param_constrain_op(self): 105 | ''' 106 | Clips variable values to prevent exploding gradients 107 | ''' 108 | 109 | cm_clipping_op = tf.Variable.assign( 110 | self.cm_t, 111 | tf.clip_by_value(self.cm_t, self._cm_t_min_value, self._cm_t_max_value) 112 | ) 113 | 114 | gleak_clipping_op = tf.Variable.assign( 115 | self.gleak, 116 | tf.clip_by_value(self.gleak, self._gleak_min_value, self._gleak_max_value) 117 | ) 118 | 119 | w_clipping_op = tf.Variable.assign( 120 | self.W, 121 | tf.clip_by_value(self.W, self._w_min_value, self._w_max_value) 122 | ) 123 | 124 | sensory_w_clipping_op = tf.Variable.assign( 125 | self.sensory_W, 126 | tf.clip_by_value(self.sensory_W, self._w_min_value, self._w_max_value) 127 | ) 128 | 129 | return [ 130 | cm_clipping_op, 131 | gleak_clipping_op, 132 | w_clipping_op, 133 | sensory_w_clipping_op 134 | ] 135 | 136 | def get_config(self): 137 | ''' 138 | Enable serialization 139 | ''' 140 | 141 | config = super(LTCCell, self).get_config() 142 | config.update({ 'units': self.units }) 143 | return config 144 | 145 | ### Helper methods ### 146 | def _get_variables(self): 147 | ''' 148 | Creates the variables to be used within __call__ 149 | ''' 150 | 151 | self.sensory_mu = tf.Variable( 152 | tf.random.uniform( 153 | [self.input_size, self.units], 154 | minval = 0.3, 155 | maxval = 0.8, 156 | dtype = tf.float32 157 | ), 158 | name = 'sensory_mu', 159 | trainable = True, 160 | ) 161 | 162 | self.sensory_sigma = tf.Variable( 163 | tf.random.uniform( 164 | [self.input_size, self.units], 165 | minval = 3.0, 166 | maxval = 8.0, 167 | dtype = tf.float32 168 | ), 169 | name = 'sensory_sigma', 170 | trainable = True, 171 | ) 172 | 173 | self.sensory_W = tf.Variable( 174 | tf.constant( 175 | np.random.uniform( 176 | low = self._w_init_min, 177 | high = self._w_init_max, 178 | size = [self.input_size, self.units] 179 | ), 180 | dtype = tf.float32 181 | ), 182 | name = 'sensory_W', 183 | trainable = True, 184 | shape = [self.input_size, self.units] 185 | ) 186 | 187 | sensory_erev_init = 2 * np.random.randint( 188 | low = 0, 189 | high = 2, 190 | size = [self.input_size, self.units] 191 | ) - 1 192 | self.sensory_erev = tf.Variable( 193 | tf.constant( 194 | sensory_erev_init * self._erev_init_factor, 195 | dtype = tf.float32 196 | ), 197 | name = 'sensory_erev', 198 | trainable = True, 199 | shape = [self.input_size, self.units] 200 | ) 201 | 202 | self.mu = tf.Variable( 203 | tf.random.uniform( 204 | [self.units, self.units], 205 | minval = 0.3, 206 | maxval = 0.8, 207 | dtype = tf.float32) 208 | , 209 | name = 'mu', 210 | trainable = True, 211 | ) 212 | 213 | self.sigma = tf.Variable( 214 | tf.random.uniform( 215 | [self.units, self.units], 216 | minval = 3.0, 217 | maxval = 8.0, 218 | dtype = tf.float32 219 | ), 220 | name = 'sigma', 221 | trainable = True, 222 | ) 223 | 224 | self.W = tf.Variable( 225 | tf.constant( 226 | np.random.uniform( 227 | low = self._w_init_min, 228 | high = self._w_init_max, 229 | size = [self.units, self.units] 230 | ), 231 | dtype = tf.float32 232 | ), 233 | name = 'W', 234 | trainable = True, 235 | shape = [self.units, self.units] 236 | ) 237 | 238 | erev_init = 2 * np.random.randint( 239 | low = 0, 240 | high = 2, 241 | size = [self.units, self.units] 242 | ) - 1 243 | self.erev = tf.Variable( 244 | tf.constant( 245 | erev_init * self._erev_init_factor, 246 | dtype = tf.float32 247 | ), 248 | name = 'erev', 249 | trainable = True, 250 | shape = [self.units, self.units] 251 | ) 252 | 253 | if self._fix_vleak is None: 254 | self.vleak = tf.Variable( 255 | tf.random.uniform( 256 | [self.units], 257 | minval = -0.2, 258 | maxval = 0.2, 259 | dtype = tf.float32 260 | ), 261 | name = 'vleak', 262 | trainable = True, 263 | ) 264 | else: 265 | self.vleak = tf.Variable( 266 | tf.constant(self._fix_vleak, dtype = tf.float32), 267 | name = 'vleak', 268 | trainable = False, 269 | shape = [self.units] 270 | ) 271 | 272 | if self._fix_gleak is None: 273 | initializer = tf.constant(self._gleak_init_min, dtype = tf.float32) 274 | 275 | if self._gleak_init_max > self._gleak_init_min: 276 | initializer = tf.random.uniform( 277 | [self.units], 278 | minval = self._gleak_init_min, 279 | maxval = self._gleak_init_max, 280 | dtype = tf.float32 281 | ) 282 | 283 | self.gleak = tf.Variable( 284 | initializer, 285 | name = 'gleak', 286 | trainable = True, 287 | ) 288 | else: 289 | self.gleak = tf.Variable( 290 | tf.constant(self._fix_gleak), 291 | name = 'gleak', 292 | trainable = False, 293 | shape = [self.units] 294 | ) 295 | 296 | if self._fix_cm is None: 297 | initializer = tf.constant(self._cm_init_min, dtype = tf.float32) 298 | 299 | if self._cm_init_max > self._cm_init_min: 300 | initializer = tf.random.uniform( 301 | [self.units], 302 | minval = self._cm_init_min, 303 | maxval = self._cm_init_max, 304 | dtype = tf.float32 305 | ) 306 | 307 | self.cm_t = tf.Variable( 308 | initializer, 309 | name = 'cm_t', 310 | trainable = True, 311 | ) 312 | else: 313 | self.cm_t = tf.Variable( 314 | tf.constant(self._fix_cm), 315 | name = 'cm_t', 316 | trainable = False, 317 | shape = [self.units] 318 | ) 319 | 320 | def _map_inputs(self, inputs): 321 | ''' 322 | Maps the inputs to the sensory layer 323 | Initializes weights & biases to be used 324 | ''' 325 | 326 | # Create a workaround from creating tf Variables every function call 327 | # init with None and set only if not None - aka only first time 328 | if self._input_weights is None: 329 | self._input_weights = tf.Variable( 330 | lambda: tf.ones( 331 | [self.input_size], 332 | dtype = tf.float32 333 | ), 334 | name = 'input_weights', 335 | trainable = True 336 | ) 337 | 338 | if self._input_biases is None: 339 | self._input_biases = tf.Variable( 340 | lambda: tf.zeros( 341 | [self.input_size], 342 | dtype = tf.float32 343 | ), 344 | name = 'input_biases', 345 | trainable = True 346 | ) 347 | 348 | if self._input_mapping == MappingType.Affine or self._input_mapping == MappingType.Linear: 349 | inputs = inputs * self._input_weights 350 | if self._input_mapping == MappingType.Affine: 351 | inputs = inputs + self._input_biases 352 | 353 | return inputs 354 | 355 | @tf.function 356 | def _ode_step_hybrid_euler(self, inputs, states): 357 | ''' 358 | Implement custom Euler ODE solver - first-order numerical procedure 359 | ''' 360 | 361 | # State returned as -> tuple(Tensor); previous return state x(t), to produce x(t+1) 362 | v_pre = states[0] 363 | 364 | sensory_w_activation = self.sensory_W * self._sigmoid(inputs, self.sensory_mu, self.sensory_sigma) 365 | sensory_rev_activation = sensory_w_activation * self.sensory_erev 366 | 367 | w_numerator_sensory = tf.reduce_sum(input_tensor = sensory_rev_activation, axis = 1) 368 | w_denominator_sensory = tf.reduce_sum(input_tensor = sensory_w_activation, axis = 1) 369 | 370 | for _ in range(self._ode_solver_unfolds): 371 | w_activation = self.W * self._sigmoid(v_pre, self.mu, self.sigma) 372 | 373 | rev_activation = w_activation * self.erev 374 | 375 | w_numerator = tf.reduce_sum(input_tensor = rev_activation, axis = 1) + w_numerator_sensory 376 | w_denominator = tf.reduce_sum(input_tensor = w_activation, axis = 1) + w_denominator_sensory 377 | 378 | numerator = self.cm_t * v_pre + self.gleak * self.vleak + w_numerator 379 | denominator = self.cm_t + self.gleak + w_denominator 380 | 381 | v_pre = numerator / denominator 382 | 383 | return v_pre 384 | 385 | @tf.function 386 | def _ode_step_runge_kutta(self, inputs, states): 387 | ''' 388 | Implement Runge-Kutta ODE solver - RK4, fourth-order numerical procedure 389 | ''' 390 | 391 | h = 0.1 392 | for _ in range(self._ode_solver_unfolds): 393 | k1 = h * self._f_prime(inputs, states) 394 | k2 = h * self._f_prime(inputs, states + k1 * 0.5) 395 | k3 = h * self._f_prime(inputs, states + k2 * 0.5) 396 | k4 = h * self._f_prime(inputs, states + k3) 397 | 398 | states = states + 1.0 / 6 * (k1 + 2 * k2 + 2 * k3 + k4) 399 | 400 | return states 401 | 402 | @tf.function 403 | def _ode_step_explicit(self, inputs, states, _ode_solver_unfolds): 404 | ''' 405 | Implement ODE explicit iterative solver - a generalization of RK4 406 | ''' 407 | 408 | v_pre = states[0] 409 | 410 | # Pre-compute the effects of the sensory neurons 411 | sensory_w_activation = self.sensory_W * self._sigmoid(inputs, self.sensory_mu, self.sensory_sigma) 412 | w_reduced_sensory = tf.reduce_sum(input_tensor = sensory_w_activation, axis = 1) 413 | 414 | # Unfold the ODE multiple times into one RNN step 415 | for _ in range(_ode_solver_unfolds): 416 | f_prime = self._calculate_f_prime(v_pre, sensory_w_activation, w_reduced_sensory) 417 | v_pre = v_pre + 0.1 * f_prime 418 | 419 | return v_pre 420 | 421 | @tf.function 422 | def _f_prime(self, inputs, states): 423 | ''' 424 | Obtain f' for the ODE solvers 425 | ''' 426 | 427 | v_pre = states[0] 428 | 429 | # Pre-compute the effects of the sensory neurons 430 | sensory_w_activation = self.sensory_W * self._sigmoid(inputs, self.sensory_mu, self.sensory_sigma) 431 | w_reduced_sensory = tf.reduce_sum(input_tensor = sensory_w_activation, axis = 1) 432 | f_prime = self._calculate_f_prime(v_pre, sensory_w_activation, w_reduced_sensory) 433 | 434 | return f_prime 435 | 436 | @tf.function 437 | def _calculate_f_prime(self, v_pre, sensory_w_activation, w_reduced_sensory): 438 | ''' 439 | Helper function to calculate f' 440 | ''' 441 | 442 | # Unfold the ODE multiple times into one RNN step 443 | w_activation = self.W * self._sigmoid(v_pre, self.mu, self.sigma) 444 | w_reduced_synapse = tf.reduce_sum(input_tensor = w_activation, axis = 1) 445 | sensory_in = self.sensory_erev * sensory_w_activation 446 | synapse_in = self.erev * w_activation 447 | sum_in = ( 448 | tf.reduce_sum(input_tensor = sensory_in, axis = 1) - 449 | v_pre * w_reduced_synapse + tf.reduce_sum(input_tensor = synapse_in, axis = 1) - 450 | v_pre * w_reduced_sensory 451 | ) 452 | 453 | return 1 / self.cm_t * (self.gleak * (self.vleak - v_pre) + sum_in) 454 | 455 | @tf.function 456 | def _sigmoid(self, v_pre, mu, sigma): 457 | v_pre = tf.reshape(v_pre, [-1, v_pre.shape[-1], 1]) 458 | mues = v_pre - mu 459 | x = sigma * mues 460 | return tf.nn.sigmoid(x) 461 | 462 | # References 463 | # https://splunktool.com/how-can-i-implement-a-custom-rnn-specifically-an-esn-in-tensorflow 464 | # https://colab.research.google.com/github/luckykadam/adder/blob/master/rnn_full_adder.ipynb 465 | # https://www.tutorialexample.com/build-custom-rnn-by-inheriting-rnncell-in-tensorflow-tensorflow-tutorial/ 466 | # https://notebook.community/tensorflow/docs-l10n/site/en-snapshot/guide/migrate 467 | # https://www.tensorflow.org/api_docs/python/tf/keras/layers/AbstractRNNCell 468 | # https://www.tensorflow.org/guide/keras/custom_layers_and_models/#layers_are_recursively_composable 469 | # https://www.tensorflow.org/guide/function#creating_tfvariables 470 | -------------------------------------------------------------------------------- /experiments/experiments.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "3bd1578e", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import numpy as np\n", 11 | "import pandas as pd\n", 12 | "import matplotlib.pyplot as plt\n", 13 | "import tensorflow as tf\n", 14 | "import requests\n", 15 | "\n", 16 | "from lts import LTSCell" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 4, 22 | "id": "c9ba0f99", 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "url = 'http://api.scraperlink.com/investpy/?email=your@email.com&type=historical_data&product=cryptos&symbol=BTC&from_date=10/01/2013&to_date=2/1/2023'\n", 27 | "response = requests.request('GET', url)\n", 28 | "prices = response.json()['data']\n", 29 | "data = pd.DataFrame(prices)" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": 5, 35 | "id": "9d9f6cdd", 36 | "metadata": {}, 37 | "outputs": [ 38 | { 39 | "data": { 40 | "text/html": [ 41 | "
\n", 42 | "\n", 55 | "\n", 56 | " \n", 57 | " \n", 58 | " \n", 59 | " \n", 60 | " \n", 61 | " \n", 62 | " \n", 63 | " \n", 64 | " \n", 65 | " \n", 66 | " \n", 67 | " \n", 68 | " \n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | "
direction_colorrowDaterowDateRawrowDateTimestamplast_closelast_openlast_maxlast_minvolumevolumeRawchange_precentlast_closeRawlast_openRawlast_maxRawlast_minRawchange_precentRaw
0redFontJan 02, 202316726176002023-01-02T00:00:00Z16,576.216,618.416,625.916,551.0108.75K108749-0.2516576.2480468750000016618.4062500000000016625.9003906250000016550.99414062500000-0.253684
1greenFontJan 01, 202316725312002023-01-01T00:00:00Z16,618.416,537.516,621.916,499.7107.84K1078370.4916618.4062500000000016537.5429687500000016621.8984375000000016499.666015625000000.489668
2redFontDec 31, 202216724448002022-12-31T00:00:00Z16,537.416,607.216,635.916,487.3130.44K130440-0.4216537.4277343750000016607.1992187500000016635.9121093750000016487.26757812500000-0.420128
3redFontDec 30, 202216723584002022-12-30T00:00:00Z16,607.216,636.416,644.416,360.0192.76K192763-0.1816607.1992187500000016636.4160156250000016644.3535156250000016360.02832031250000-0.175620
4greenFontDec 29, 202216722720002022-12-29T00:00:00Z16,636.416,546.216,659.116,496.6181.47K1814660.5516636.4160156250000016546.1855468750000016659.0566406250000016496.562500000000000.545194
\n", 175 | "
" 176 | ], 177 | "text/plain": [ 178 | " direction_color rowDate rowDateRaw rowDateTimestamp last_close \\\n", 179 | "0 redFont Jan 02, 2023 1672617600 2023-01-02T00:00:00Z 16,576.2 \n", 180 | "1 greenFont Jan 01, 2023 1672531200 2023-01-01T00:00:00Z 16,618.4 \n", 181 | "2 redFont Dec 31, 2022 1672444800 2022-12-31T00:00:00Z 16,537.4 \n", 182 | "3 redFont Dec 30, 2022 1672358400 2022-12-30T00:00:00Z 16,607.2 \n", 183 | "4 greenFont Dec 29, 2022 1672272000 2022-12-29T00:00:00Z 16,636.4 \n", 184 | "\n", 185 | " last_open last_max last_min volume volumeRaw change_precent \\\n", 186 | "0 16,618.4 16,625.9 16,551.0 108.75K 108749 -0.25 \n", 187 | "1 16,537.5 16,621.9 16,499.7 107.84K 107837 0.49 \n", 188 | "2 16,607.2 16,635.9 16,487.3 130.44K 130440 -0.42 \n", 189 | "3 16,636.4 16,644.4 16,360.0 192.76K 192763 -0.18 \n", 190 | "4 16,546.2 16,659.1 16,496.6 181.47K 181466 0.55 \n", 191 | "\n", 192 | " last_closeRaw last_openRaw last_maxRaw \\\n", 193 | "0 16576.24804687500000 16618.40625000000000 16625.90039062500000 \n", 194 | "1 16618.40625000000000 16537.54296875000000 16621.89843750000000 \n", 195 | "2 16537.42773437500000 16607.19921875000000 16635.91210937500000 \n", 196 | "3 16607.19921875000000 16636.41601562500000 16644.35351562500000 \n", 197 | "4 16636.41601562500000 16546.18554687500000 16659.05664062500000 \n", 198 | "\n", 199 | " last_minRaw change_precentRaw \n", 200 | "0 16550.99414062500000 -0.253684 \n", 201 | "1 16499.66601562500000 0.489668 \n", 202 | "2 16487.26757812500000 -0.420128 \n", 203 | "3 16360.02832031250000 -0.175620 \n", 204 | "4 16496.56250000000000 0.545194 " 205 | ] 206 | }, 207 | "execution_count": 5, 208 | "metadata": {}, 209 | "output_type": "execute_result" 210 | } 211 | ], 212 | "source": [ 213 | "data.head()" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": 6, 219 | "id": "5643aa5d", 220 | "metadata": {}, 221 | "outputs": [ 222 | { 223 | "data": { 224 | "text/html": [ 225 | "
\n", 226 | "\n", 239 | "\n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | "
rowDatelast_closeRawlast_openRawlast_maxRawlast_minRaw
0Jan 02, 202316576.2480468750000016618.4062500000000016625.9003906250000016550.99414062500000
1Jan 01, 202316618.4062500000000016537.5429687500000016621.8984375000000016499.66601562500000
2Dec 31, 202216537.4277343750000016607.1992187500000016635.9121093750000016487.26757812500000
3Dec 30, 202216607.1992187500000016636.4160156250000016644.3535156250000016360.02832031250000
4Dec 29, 202216636.4160156250000016546.1855468750000016659.0566406250000016496.56250000000000
\n", 293 | "
" 294 | ], 295 | "text/plain": [ 296 | " rowDate last_closeRaw last_openRaw \\\n", 297 | "0 Jan 02, 2023 16576.24804687500000 16618.40625000000000 \n", 298 | "1 Jan 01, 2023 16618.40625000000000 16537.54296875000000 \n", 299 | "2 Dec 31, 2022 16537.42773437500000 16607.19921875000000 \n", 300 | "3 Dec 30, 2022 16607.19921875000000 16636.41601562500000 \n", 301 | "4 Dec 29, 2022 16636.41601562500000 16546.18554687500000 \n", 302 | "\n", 303 | " last_maxRaw last_minRaw \n", 304 | "0 16625.90039062500000 16550.99414062500000 \n", 305 | "1 16621.89843750000000 16499.66601562500000 \n", 306 | "2 16635.91210937500000 16487.26757812500000 \n", 307 | "3 16644.35351562500000 16360.02832031250000 \n", 308 | "4 16659.05664062500000 16496.56250000000000 " 309 | ] 310 | }, 311 | "execution_count": 6, 312 | "metadata": {}, 313 | "output_type": "execute_result" 314 | } 315 | ], 316 | "source": [ 317 | "# Remove unnecessary columns\n", 318 | "data.drop(columns=[\n", 319 | " 'direction_color',\n", 320 | " 'rowDateRaw',\n", 321 | " 'rowDateTimestamp',\n", 322 | "\n", 323 | " # Quantity of btc bought or sold\n", 324 | " 'volume',\n", 325 | " 'volumeRaw',\n", 326 | " 'change_precent',\n", 327 | " 'change_precentRaw',\n", 328 | " 'last_close',\n", 329 | " 'last_open',\n", 330 | " 'last_max',\n", 331 | " 'last_min'\n", 332 | " ],\n", 333 | " inplace=True\n", 334 | ")\n", 335 | "\n", 336 | "data.head()" 337 | ] 338 | }, 339 | { 340 | "cell_type": "code", 341 | "execution_count": 7, 342 | "id": "d7c2c5af", 343 | "metadata": {}, 344 | "outputs": [ 345 | { 346 | "name": "stdout", 347 | "output_type": "stream", 348 | "text": [ 349 | "\n", 350 | "RangeIndex: 3381 entries, 0 to 3380\n", 351 | "Data columns (total 5 columns):\n", 352 | " # Column Non-Null Count Dtype \n", 353 | "--- ------ -------------- ----- \n", 354 | " 0 rowDate 3381 non-null object\n", 355 | " 1 last_closeRaw 3381 non-null object\n", 356 | " 2 last_openRaw 3381 non-null object\n", 357 | " 3 last_maxRaw 3381 non-null object\n", 358 | " 4 last_minRaw 3381 non-null object\n", 359 | "dtypes: object(5)\n", 360 | "memory usage: 132.2+ KB\n" 361 | ] 362 | } 363 | ], 364 | "source": [ 365 | "data.info()" 366 | ] 367 | }, 368 | { 369 | "cell_type": "code", 370 | "execution_count": 8, 371 | "id": "428e484a", 372 | "metadata": {}, 373 | "outputs": [ 374 | { 375 | "data": { 376 | "text/plain": [ 377 | "rowDate 0\n", 378 | "last_closeRaw 0\n", 379 | "last_openRaw 0\n", 380 | "last_maxRaw 0\n", 381 | "last_minRaw 0\n", 382 | "dtype: int64" 383 | ] 384 | }, 385 | "execution_count": 8, 386 | "metadata": {}, 387 | "output_type": "execute_result" 388 | } 389 | ], 390 | "source": [ 391 | "data.isnull().sum()" 392 | ] 393 | }, 394 | { 395 | "cell_type": "code", 396 | "execution_count": 9, 397 | "id": "fd6defd7", 398 | "metadata": {}, 399 | "outputs": [], 400 | "source": [ 401 | "# Convert date object to datetime\n", 402 | "data['rowDate'] = pd.to_datetime(data['rowDate'])\n", 403 | "\n", 404 | "# Convert values to floats\n", 405 | "data = data.astype({\n", 406 | " 'last_closeRaw': 'float',\n", 407 | " 'last_openRaw': 'float',\n", 408 | " 'last_maxRaw': 'float',\n", 409 | " 'last_minRaw': 'float'\n", 410 | "})" 411 | ] 412 | }, 413 | { 414 | "cell_type": "code", 415 | "execution_count": 10, 416 | "id": "1eb21dbb", 417 | "metadata": {}, 418 | "outputs": [ 419 | { 420 | "name": "stdout", 421 | "output_type": "stream", 422 | "text": [ 423 | "\n", 424 | "RangeIndex: 3381 entries, 0 to 3380\n", 425 | "Data columns (total 5 columns):\n", 426 | " # Column Non-Null Count Dtype \n", 427 | "--- ------ -------------- ----- \n", 428 | " 0 rowDate 3381 non-null datetime64[ns]\n", 429 | " 1 last_closeRaw 3381 non-null float64 \n", 430 | " 2 last_openRaw 3381 non-null float64 \n", 431 | " 3 last_maxRaw 3381 non-null float64 \n", 432 | " 4 last_minRaw 3381 non-null float64 \n", 433 | "dtypes: datetime64[ns](1), float64(4)\n", 434 | "memory usage: 132.2 KB\n" 435 | ] 436 | } 437 | ], 438 | "source": [ 439 | "data.info()" 440 | ] 441 | }, 442 | { 443 | "cell_type": "code", 444 | "execution_count": 11, 445 | "id": "0836bfb3", 446 | "metadata": {}, 447 | "outputs": [ 448 | { 449 | "data": { 450 | "text/html": [ 451 | "
\n", 452 | "\n", 465 | "\n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | "
rowDatelast_closeRawlast_openRawlast_maxRawlast_minRaw
02023-01-0216576.24804716618.40625016625.90039116550.994141
12023-01-0116618.40625016537.54296916621.89843816499.666016
22022-12-3116537.42773416607.19921916635.91210916487.267578
32022-12-3016607.19921916636.41601616644.35351616360.028320
42022-12-2916636.41601616546.18554716659.05664116496.562500
\n", 519 | "
" 520 | ], 521 | "text/plain": [ 522 | " rowDate last_closeRaw last_openRaw last_maxRaw last_minRaw\n", 523 | "0 2023-01-02 16576.248047 16618.406250 16625.900391 16550.994141\n", 524 | "1 2023-01-01 16618.406250 16537.542969 16621.898438 16499.666016\n", 525 | "2 2022-12-31 16537.427734 16607.199219 16635.912109 16487.267578\n", 526 | "3 2022-12-30 16607.199219 16636.416016 16644.353516 16360.028320\n", 527 | "4 2022-12-29 16636.416016 16546.185547 16659.056641 16496.562500" 528 | ] 529 | }, 530 | "execution_count": 11, 531 | "metadata": {}, 532 | "output_type": "execute_result" 533 | } 534 | ], 535 | "source": [ 536 | "data.head()" 537 | ] 538 | }, 539 | { 540 | "cell_type": "code", 541 | "execution_count": 12, 542 | "id": "6a8ec0d6", 543 | "metadata": {}, 544 | "outputs": [], 545 | "source": [ 546 | "data.rename(\n", 547 | " columns={\n", 548 | " 'last_closeRaw': 'close',\n", 549 | " 'last_openRaw': 'open',\n", 550 | " 'last_maxRaw': 'high',\n", 551 | " 'last_minRaw': 'low',\n", 552 | " 'rowDate': 'Date'\n", 553 | " },\n", 554 | " inplace=True\n", 555 | ")" 556 | ] 557 | }, 558 | { 559 | "cell_type": "code", 560 | "execution_count": 13, 561 | "id": "3533749a", 562 | "metadata": {}, 563 | "outputs": [ 564 | { 565 | "data": { 566 | "text/html": [ 567 | "
\n", 568 | "\n", 581 | "\n", 582 | " \n", 583 | " \n", 584 | " \n", 585 | " \n", 586 | " \n", 587 | " \n", 588 | " \n", 589 | " \n", 590 | " \n", 591 | " \n", 592 | " \n", 593 | " \n", 594 | " \n", 595 | " \n", 596 | " \n", 597 | " \n", 598 | " \n", 599 | " \n", 600 | " \n", 601 | " \n", 602 | " \n", 603 | " \n", 604 | " \n", 605 | " \n", 606 | " \n", 607 | " \n", 608 | " \n", 609 | " \n", 610 | " \n", 611 | " \n", 612 | " \n", 613 | " \n", 614 | " \n", 615 | " \n", 616 | " \n", 617 | " \n", 618 | " \n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | "
Datecloseopenhighlow
02023-01-0216576.24804716618.40625016625.90039116550.994141
12023-01-0116618.40625016537.54296916621.89843816499.666016
22022-12-3116537.42773416607.19921916635.91210916487.267578
32022-12-3016607.19921916636.41601616644.35351616360.028320
42022-12-2916636.41601616546.18554716659.05664116496.562500
\n", 635 | "
" 636 | ], 637 | "text/plain": [ 638 | " Date close open high low\n", 639 | "0 2023-01-02 16576.248047 16618.406250 16625.900391 16550.994141\n", 640 | "1 2023-01-01 16618.406250 16537.542969 16621.898438 16499.666016\n", 641 | "2 2022-12-31 16537.427734 16607.199219 16635.912109 16487.267578\n", 642 | "3 2022-12-30 16607.199219 16636.416016 16644.353516 16360.028320\n", 643 | "4 2022-12-29 16636.416016 16546.185547 16659.056641 16496.562500" 644 | ] 645 | }, 646 | "execution_count": 13, 647 | "metadata": {}, 648 | "output_type": "execute_result" 649 | } 650 | ], 651 | "source": [ 652 | "data.head()" 653 | ] 654 | }, 655 | { 656 | "cell_type": "code", 657 | "execution_count": 14, 658 | "id": "0521e3df", 659 | "metadata": {}, 660 | "outputs": [], 661 | "source": [ 662 | "data.set_index('Date', inplace=True)" 663 | ] 664 | }, 665 | { 666 | "cell_type": "code", 667 | "execution_count": 15, 668 | "id": "1fde8512", 669 | "metadata": {}, 670 | "outputs": [], 671 | "source": [ 672 | "btc_prices = pd.DataFrame(data['close']).rename(columns={ 'close': 'Price' })" 673 | ] 674 | }, 675 | { 676 | "cell_type": "code", 677 | "execution_count": 16, 678 | "id": "3b31619a", 679 | "metadata": {}, 680 | "outputs": [ 681 | { 682 | "data": { 683 | "text/html": [ 684 | "
\n", 685 | "\n", 698 | "\n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | " \n", 707 | " \n", 708 | " \n", 709 | " \n", 710 | " \n", 711 | " \n", 712 | " \n", 713 | " \n", 714 | " \n", 715 | " \n", 716 | " \n", 717 | " \n", 718 | " \n", 719 | " \n", 720 | " \n", 721 | " \n", 722 | " \n", 723 | " \n", 724 | " \n", 725 | " \n", 726 | " \n", 727 | " \n", 728 | " \n", 729 | " \n", 730 | " \n", 731 | "
Price
Date
2023-01-0216576.248047
2023-01-0116618.406250
2022-12-3116537.427734
2022-12-3016607.199219
2022-12-2916636.416016
\n", 732 | "
" 733 | ], 734 | "text/plain": [ 735 | " Price\n", 736 | "Date \n", 737 | "2023-01-02 16576.248047\n", 738 | "2023-01-01 16618.406250\n", 739 | "2022-12-31 16537.427734\n", 740 | "2022-12-30 16607.199219\n", 741 | "2022-12-29 16636.416016" 742 | ] 743 | }, 744 | "execution_count": 16, 745 | "metadata": {}, 746 | "output_type": "execute_result" 747 | } 748 | ], 749 | "source": [ 750 | "btc_prices.head()" 751 | ] 752 | }, 753 | { 754 | "cell_type": "code", 755 | "execution_count": 17, 756 | "id": "d33fd9d5", 757 | "metadata": {}, 758 | "outputs": [ 759 | { 760 | "data": { 761 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAn0AAAG7CAYAAAC2KiP5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAB2OElEQVR4nO3dd5xcZdn/8c81ZVvabnpIISGEEjoEpEsvooKKioWiKD6KXZ9HUH+CWMAGVlREpFiwgIiAhA7SSegkQCokIb1usm3K/fvjnJk9Ozsz26bsznzfr9e+duY+9zlzn5nN7pXrbuacQ0REREQqW6jcDRARERGR4lPQJyIiIlIFFPSJiIiIVAEFfSIiIiJVQEGfiIiISBVQ0CciIiJSBRT0yZBgZueZmQt8NZvZC2b2WTOL9OH86SVobp+Y2bvM7CUza/Pb2Jij3qUZ70HczN4ws9+b2eSMuteb2fLA8+n++bsU8T4K/h6b2cfNbJGZdZjZlkJdtxjMrMnMLjez1/zPcpOZzTWzk/t5vS+a2Xt7Wfd4M/ujmS0xs1b/+6/NbHyWunVm9iMzW+3XfcLMjs5S78tm9m+/njOzS3vRjl3MrMWvv2sv6p9nZh/vzT32lZmdZGb/MbON/ufxupn9wMya+nGt/f1/P6N7UXekmX3LzB73X3uL//iMHPWP9I+3mtkaM7vSzOoz6nzSzO4ys1VmtsPMXjaz/zWzmox6Z5rZLf7vhVb/Z/FyMxvR13uWyqSgT4aa9wOHAe8DngZ+AXyrF+fd6Z+3unhN6zs/YP0TsAo4Ca+NzT2cdqRf71jg+8BpwJ1mFvz3/B3gPYHn04FLgKIFfRT4PTaznYBrgMeB44ATCnHdYjCzqcAzwPl4bT4F+DiwHbjbzC7ux2W/CPQq6AP+BxgDfNd/7cuBdwNPmtnwjLq/Bz6J9+/mnXif11wz2z+j3ieB8cBtfWjz1cDWPtQ/D+99Kigz+zowF2gDPgGcDPzGf71n/M+rL/bH+/fTY9AHTAM+AzwMfBT4IPA68E8zuzCjnfsC9wLr8D6LbwIfA67PuOa3gDXAF/x6f8X7N/6njHpfBRLA1/F+Dn4NfBq4N+P3g1Qr55y+9DXov/B+WTtg14zyB4Gtec6LAlbu9udp387+fX28F3Uv9etGMso/4ZfvmefcY/w6J5T7nvvw3rzdb/NxPdQzoKbMbX0I2AjMyHLsKiAJHNPHay4H/tjLuuOylB2d+bMF7OeXfSxQFgFeA27POD8UOO6AS3tow4eBtXjBard/q3net0cL/Fkc67/fV2U5NgPYBDzYx2tm/f2To+4woCFL+f3Amxll/wQWAdFA2Tn+ax3Yw+f7Lb/eLj3US10v778jfVXHlyJ/GeqeAUaa2Xi/C9OZ2WfM7Idm9hbQDjTm6nr0u02e9btCNpvZw2Z2eOB4g98ltMzvYlxmZt/ozf+azWySmd1oZhvMrN3MXjSzjwaOX4r3hx3g9377HurHe7DN/x4NXDvdvWtmx+AFx+D9jz/VPXxMoH5P70Pee/HrdHuPzWy53+14lpkt9Lum5pnZkfluyMyuxwsIAO73r3t9xjU/bmavAh142U7M7BTzuitbzWyrmd1mZrtnXPshM3vUr/u8X/c5M3ubmUXM7PvmdWlu8t/HYT209W14AeoVzrllWapcDGwGvpZx3n5m9k+/CzDVFXdx6h7x/kPwkcDndX2uNjjn1mcpfsb/Huz6fzcQw8sUpc6NAzcDJ5tZbaA8mev1MpnXZXolXqZpSy/PeQjvfTsicI8PBY4fYmb3mdl2/+fmfjM7pBeX/j+8wK5bdtX/fK4AjvE/t9RrRczsa2a2wLyu4PVmdreZ7WFm5wF/8KsuCrR1erYXd87tcM61ZDk0D9gp8JpRvGzc35xzsUC9v+H9TJ8euGavPt8+/BxIlepxLJTIIDcDrztjO9Dgl30D7xfdBUAYr4unGzP7MfAVvO6uS/CyA4fidc88bl7X61xgNl5Xykv+8f+H183zlVyN8gOFh4EmvK6WFXhdPTeZWYNz7hrgWuBl4O943XJ30hnA5RM2M/CCvNn+9V/xr5XNs8CFwK+Az9P5R2BBL9+H3txLPkcBu+O9b2147+UdZjbdObclxznfAeYDP/fb/iwQ/IN2LF6X27fxusaWm9kpeO/hA3hdasOBy4BHzWx/59yqwPm7Aj8Cvof3s/ND4Hb/K4KX2dnTr7MOL5DI5Xj/++3ZDjrn2szsXuBdZhZ2ziX84OUhYDHwJWAlMAvY1z/tPcBdwAt4GV4y7r833u5/Xxgo2wtYliUoeQWowXtfXunj64D3/r3qnLvJD5J64zPAH/H+jX7KL9sG6W7Ph/F+Rs/Dy1RdBDxsZoc6517IdkH/3+zbgX8557L+u8f7nH6AN2TgKb/sZuAM4KfAfUAdXqZ0Et7P1Hfxul7fj/dZQd+HMRwNvBp4PtN/nS7/bv2flyV4/7bzeTvev9XXe1EPuv4cSLUqd6pRX/rqzRedv/h3x/uj3IT3hyIB3ObXme7XeZaMLt3A+dP957v6516Z5zXP9s85OqP8G3j/Ex+f59zP+ucek1F+H14QEQ60wwHn9eI9uNSvm/m1EJiZUfd6YHng+TFk6d7t5fvQ23vp8h77ZcvxslxNgbI5fr0P93C/J+R43eVACzAxo3weXldZJFA2Ay+zdWWg7CG/LNgt9m7/te7LuOateEFSvnb+2j+3Nk+dK/w64/3nj+AFz926ATPus1fdu1nOHYEXYCzIeD/uAZ7M814fleVY3u5dvKC+HZid8XPQ7+5d4B94GcPGQNlIvAzerXmuN8F/7cvz1Knz61ztPz/Of/75POf0+p5ynH+Bf/5HAmWH+2WnZKn/KHB/nuvtC7QCv+vhdSfj/Ru9tz/t1lflfal7V4aaV/H+YG/CGzT+J7oPBL/NOed6uM4JeBOZ8mWpTgHewM/6pb7w/nBG8bJhuRwNrHLOPZRR/kdgHD3/Lz6fQ4GDgbcBHwB2APeY2YR+XKs378NA7+UJ59zmwPOX/O/T+tDOTE8659aknvjZyAOBvzqvuxJId+c9Rme2I+V159zSwPNUBmZuRr1XgSnmp1YLwcwagCOAP7ns3YADvX4E+AveH/yzgu9HEV6rBvgt3vi5BQW89NHAHS6QCXbObcPL0mV+lgN1El7w9bsCXxdID6/4OXCjcy5z4kV/rjcJ+BewBPhynnrD/XpxvMkhIurelSHnPXjdK83AGy57F05vul3G+N9X5qkzHm9cVSzH8TE5ysHr/s3WjjWB4/01P/CH/Gkze8R/rS+TMW6sF3rzPgz0XjYFnzjn2v0Yqq43Dcwhsz1NeBM6crVz54yyzRnPO/KUR/C6IHMFT6n3bjrehIhspuNlZjYCE/EC7Xzveb+YN9b0Brxg/jTn3IsZVTbT/b2Azs9wU5Zj+XwR773/uXUuNZQaZjHCzEY453qajZ5Nvp+5fEuubMQbQjA9T53UsRX+9zHAJudca9+a2DMzOxgvUH0Ab8JVUOpnLdv9jCZLN7uZjcGb7WvAybneW/OWfPk33mz9tzvnCv6zJkOTMn0y1LzsnJvnnHstR8AH3v/ae7LB/55vcPNGYBleVi3b17/znLsJ7497pomB4wXhnFuLdz/79lQ3i968DyW7lz7I/Iw3+2W52lnMNt7vf393toNmVgecCDzsnEvgtTVJcQbW/wZvPONZzrn7sxx/BZjhZxuDZuMFuIv7+Hqz8d7fVXj3tRlv7Ch4wyz+28frpeT7mcsMzNP8/ww9DJzov+/ZpD6nB/zvG4DRlrE23kCZ2T54mePngfe5rpM1wMvUteONswyeV4cXrC3IKB/pX28M3jCN4BjVYL0oXvf4HOAdzrmXstWT6qSgT6rVfXh/eC/IU+duYCqw3Q80M7825Dn3YbxuwSMyyj+MN8amYF1hfnfPWPIP9G/3v2f+YevN+1Cye+kv59wOvIkf7zezcKrczHbGGzv1UBFf+0m8MVgXmdmMLFUux8vc/Miv3+LX/2gPgUY73T+vnMzsJ3jZpI85527LUe3feEMT3h84L4IXKN7jnGvPcV4uV+BNqgl+/cA/9lG6Z7cy5brHh4F3WGBRYf/xu+j5s/wxXmD0/cwD/ufzNeAR51xqEsc9eJmzfG3N9e8nKzObhZeRWwq8M1sW0TnXgfc75gPWdYH5M4FaAhOD/CD9Trwxqic557IG536m90944xTP8H82RdLUvStVyTm3xMyuAr7s/zG5HW9CwyF4sxD/ivfL82N4S4b8BG8mZQ3erLt34/1SzTUm63q8hVRvNbNv4HXlfQQv4/MpP+PTX28zswTef9p2Bv7Xb/tv8pzzOl735MfNbBPeH7HXevk+FPNeCun/4f1hvMPMrsabvfttvMWCf1Lk1/4o3rI4T5rZD/EmlTTirZH2XuBbzrkHAvW/ihfYPOH/bK3Ey+7s75z7nF9nAXCUmb0Tr1tzg3NuebYXN7Ov4XXvX4e3rEhwvOl659wSAOfcc2b2V+CnfkZoGd7ivTPwPtPgNefgdYWmkgOzzexM//FdzrkW59yrdJ2RSmApk6dyBScBC4DPmNkH8TJfzc651/Bmb78T79/eD/CyuF/D6zq+LN8FnXP3mdklwLf9ttyIlx08EG8G8Fa8SVqp+g+a2S3AleYt2vwAXmB8NHCnP5Y19R+bC83sBrwhHy/6gVsX5u2Cci/e74pL8N63YJXnAsH1pcCTwN/M7Fd47/ePgH845+YHzrkFbxzoF4BhGZ/vEte5VMuv8AL67wE7MuqtVDevlH0mib701ZsvejF7js7Zu5/Ic/70jPL/AV7EC4I24WURDgscr8P7xfxqoM4zflmkhzZPAm7C6z5q91/noxl1BjJ7Nwm8hZe9OSSj7vUEZu/6ZZ/CyzzEyZgV24v3oTf30u09JscMVHq32G++2btZZ7XiTb55Am/83Fa8gey7Z9R5iIwZo7l+dsixIHaO1x6Nl+V6HW9c2Wa8LNKpOeof4H92W/z2vgp8LXB8D7zu0Ra/Ddfnee2HMn42gl/XZ9Stx1tTb43fzqcy3+PAz1Cua07P05bUz0FvZu9OxFuaptk/56HAsbfhZaK3401Wup+Mn/Mern0KXnfoZv9ndhFeQDU6S90I3qz81/G6udf77do9UOcSvG7sRL73gM6Z8r167/CCyyf8z2It3rIxDRl18l3vvEC95Xnq5f33pq/q+DLnejP8SURERESGMo3pExEREakCCvpEREREqoCCPhEREZEqoKBPREREpApU3ZItY8eOddOnTy93M0RERER6NH/+/A3OuXGFuFbVBX3Tp09n3rx55W6GiIiISI/M7I1CXato3btmtruZPR/42mZmXzSz0WZ2r5kt8r83+fXNzH5uZovN7EUzOzBwrXP9+ovM7NxA+UFm9pJ/zs8LuSm6iIiISCUpWtDnvL1R93fO7Q8chLfA6D/xVkS/3zk3C2+xzYv8U04FZvlfFwC/BjCz0XiLYr4Nb5eAS1KBol/nk4HzTinW/YiIiIgMZaWayHE83lYxbwCnAzf45TcAZ/iPTwdudJ4ngUZ/T9GTgXudc5ucc5vxtrc5xT820jn3pPNWmL4xcC0RERERCShV0HcW8Bf/8QTn3Gr/8Rpggv94MrAicM5Kvyxf+cos5d2Y2QVmNs/M5q1fn29PehEREZHKVPSgz8xq8Dan/3vmMT9DV/R94Jxz1zjn5jjn5owbV5AJMCIiIiJDSilm754KPOucW+s/X2tmk5xzq/0u2nV++SpgauC8KX7ZKrwNrIPlD/nlU7LU77dkMsnKlSvZsWPHQC5T9aLRKOPHj2fkyJHlboqIiIj4ShH0fYjOrl2A24FzgSv87/8KlH/WzG7Gm7Sx1Q8M5wLfD0zeOAm42Dm3yZ8RfCjwFHAO8IuBNHTDhg2YGbvvvjuhkNat7g/nHK2traxa5cXfCvxEREQGh6JGNmY2DDgRuDVQfAVwopktAk7wnwPcBSwFFgO/Az4D4JzbBHwHeMb/uswvw69zrX/OEuA/A2nvli1bmDBhggK+ATAzGhoamDx5MuvWrev5BBERESmJomb6nHM7gDEZZRvxZvNm1nXAhTmucx1wXZbyecDeBWkskEgkiEajhbpcVauvrycWi5W7GSIiIuJTSiuD1ncuDL2PIiIig4uCPhEREZEqoKBPREREpAoo6KtS5513Hu985zvL3QwREREpEQV9FeC8887DzDAzotEou+yyC1/96lfzrjf4s5/9jD/+8Y8lbKWIiIiUUynW6ZMSOOGEE7jpppuIxWL897//5ROf+AQ7duzg17/+dZd68XiccDjMqFGjytRSERGRruKJJG3xJMNrFZYUkzJ9FaK2tpaJEycydepUPvzhD/ORj3yE2267jUsvvZS9996b66+/npkzZ1JbW8uOHTu6de865/jJT37CrFmzqK2tZcqUKVx88cXp46tWreKss86iqamJpqYmTjvtNBYtWlSOWxURkQpz0a0vsfclc/FWb5NiUUidx7f//QoL3tpW0tecvdNILnnXXgO+TnCdvGXLlvHnP/+Zv//979TU1FBXV9et/te//nV+/etfc+WVV3L00Uezfv16nnvuOQBaWlo49thjOfzww3n44Yepqanhxz/+MSeccAILFy6koaFhwO0VEZHqdcuzKwFoiyWprwmXuTWVS0FfBXr66af585//zPHHe2tgd3R0cNNNNzFhwoSs9bdv385VV13FT3/6Uz7+8Y8DsOuuu3LYYYcBcPPNN+Oc4w9/+EN6/b3f/va3jB8/njvuuIMPfOADJbgrERGpVA3RMDs6EmxvjyvoKyIFfXkUIuNWKnfffTfDhw8nHo8Ti8U4/fTT+cUvfsHVV1/NlClTcgZ8AAsWLKC9vT0dJGaaP38+y5YtY8SIEV3KW1paWLJkSUHvQ0REqk99jRf07WiPM25EbbmbU7EU9FWIo48+mmuuuYZoNMpOO+3UZTu5YcOGDejayWSS/fffn5tvvrnbsdGjRw/o2iIiIiG/FymWSJa5JZVNQV+FaGhoYNddd+3XuXvuuSe1tbXcf//9zJo1q9vxAw88kL/85S+MHTuWxsbGAbZURESkq1TQF09qIkcxafauMGLECL7whS9w8cUX84c//IElS5bw9NNPp5d7+chHPsKECRM4/fTTefjhh1m2bBmPPPIIX/nKVzSDV0REBizkb9eeUNBXVMr0CQCXX345TU1NfOc732HlypVMmDCBc845B/CyiI888ggXXXQR73//+9m6dSs77bQTxx57LE1NTWVuuYiIDEUtHXFicceohmh6kqAyfcVl1bYmzpw5c9y8efOyHlu4cCF77rlniVtUufR+iohILkdc8QCrtrSy/IrT0o9v+fRhHLSzxooHmdl859ycQlxL3bsiIiJScqu2tKYfh/xoJJaorkRUqSnoExERkbIyvO7dL//1+fI2pMIp6BMREZGySk3keGtrW3kbUuEU9ImIiEhZpSZySHEp6MtQbRNbiiWZ1AKbIiLSO8GYL6kZvEWjoC+grq6OjRs3KvAbAOccHR0drFq1asA7gYiISHWY3FiffpzQ3+Ci0Tp9AVOmTGHlypWsX7++3E0Z0iKRCKNGjWLs2LHlboqIiAwBw2o6w5FE0hENl7ExFUxBX0A0GmXGjBnlboaIiEhVCWb3ksr0FY26d0VERKSsgsOqtBVb8SjoExERkbIKBnqaB1g8CvpERESkrIIbcWgiR/Eo6BMREalSP7nnNRa8ta3czeiyTIu6d4tHQZ+IiEgV6ogn+cUDiznj6sfK3ZQukzc0kaN4FPSJiIhUoVRw1REv7yC6S29/hceXbEw/V6aveBT0iYiIVKHBklC7/vHlXZ4r6CseBX0iIiJVqJzdqPl2vlL3bvEo6BMREalC5ZwlG8+Tzct3TAZGQZ+IiEgVcmUcyhdL5H7xpIK+olHQJyIiUoXK2Y0aS+R+ba3TVzwK+kRERKpQeYO+3Jk+TeQoHgV9IiIiVaicsVX+7t0SNqTKKOgTERGpQuXM9H33zoU5j6l7t3gU9ImIiFShcgZ9d764Oucxde8Wj4I+ERGRCpdIOi7/z0LWNbelywZrbKV1+opHQZ+IiEiFe2rpRn778FK+9o8X02WDdWkUZfqKR0GfiIhIhUvFUR2BCRTBjNpgCADfd+AUYHC0pVIp6BMREakSwZ7TYGxV6l0wLjh6ly7P//CxgznrkKmAJnIUU1GDPjNrNLN/mNmrZrbQzA4zs9Fmdq+ZLfK/N/l1zcx+bmaLzexFMzswcJ1z/fqLzOzcQPlBZvaSf87PzcyKeT8iIiJDUeqvY9egz2V9XArRcNc/1+NH1BLyG7lpR0dJ21JNip3p+xlwt3NuD2A/YCFwEXC/c24WcL//HOBUYJb/dQHwawAzGw1cArwNOAS4JBUo+nU+GTjvlCLfj4iIyJCTCrEcncGdCwR6pR5Hl3RQEw5RG/HCkEgoRHNbDIAv3Px8SdtSTYoW9JnZKOBo4PcAzrkO59wW4HTgBr/aDcAZ/uPTgRud50mg0cwmAScD9zrnNjnnNgP3Aqf4x0Y655503k/ujYFriYiISEqWTF9rR+f4vlJ3qSadI9g3FwmbJnCUQDEzfTOA9cAfzOw5M7vWzIYBE5xzqQV61gAT/MeTgRWB81f6ZfnKV2Yp78bMLjCzeWY2b/369QO8LRERkaHF/KgvGFZd89+l6celnjzhHITM0u2JhIxjdh8PwIXHzixpW6pJMYO+CHAg8Gvn3AHADjq7cgHwM3RF/0lzzl3jnJvjnJszbty4Yr+ciIjIoNI5pq/zT+5OjXXpxyXv3k06QkY6AgiHzHsO/OrBJSVtSzUpZtC3EljpnHvKf/4PvCBwrd81i/99nX98FTA1cP4Uvyxf+ZQs5SIiIhKQmiSRqxe3HGP6QoH+3Wg4hOZiFl/Rgj7n3BpghZnt7hcdDywAbgdSM3DPBf7lP74dOMefxXsosNXvBp4LnGRmTf4EjpOAuf6xbWZ2qD9r95zAtURERMSXzvQFyuKJwESOMozpwzonloRDXQM+p2VbiiJS5Ot/DviTmdUAS4GP4QWafzOz84E3gA/4de8C3gEsBlr8ujjnNpnZd4Bn/HqXOec2+Y8/A1wP1AP/8b9EREQkID171w+mVm5u6bL/bTkmUYTM0pnHSEbQF0s4aiLK/BVaUYM+59zzwJwsh47PUtcBF+a4znXAdVnK5wF7D6yVIiIi1eHZN7cQSyQ58gcPdilP+hN573xxNTuPaWDvyaOK2o6kc4S6zN7t2vG437fvYeF3tApboRU70yciIiJlFszj3btgbbfjqe7dC//8LADLrzitqO3xgr6us3eDWmOJor5+tdI2bCIiIhUu2H27bltb3uOlkHRgZunu5swxfVIcCvpEREQqXHCbtZYsWbRSB33O79799ul7Ux8Nd8v0SXGoe1dERKTCBSfDPvRq900KSr9OnzeR4+xDd+bsQ3cu6WtXM2X6REREKlx7vDO79/TyTd2OJ51j8brmkrUncyJHNrFEMn8F6TMFfSIiIhXuG/98Oe/xNza2cMKVj5SoNZ1j+vL5zh0LStSa6qGgT0REpAK1diR49s3NAKze2n3yRtCqLS2laBLgjee75dmVbG2N5a13/8J1eY9L3ynoExERqUBf/tvzvPfqx9mwvb3HuqUc0vfMci8Q3d4ez1uvTcu2FJyCPhERkQr0/IotALTHkxy6y+i8deMlHD/X29fauKOjyC2pPgr6REREKlDcT99FQsZ+Uxrz1o0lSpfqC2l5lrJR0CciIlKBUsuwhMx6DOriya7ZN+eKFwSG8kzgOHH2hKK9rijoExERqUipblSHI5HM36X62pquy7W8vnZ70doVzhN5HLfH+KK9rijoExERqUipyRnOQSxjpkbmDhj3ZcyULWYPbL5MXxETjIKCPhERkYqU6rJNOtdt8kR9TTjvubWR/McHIt/6fOreLS4FfSIiIhWoLZYK+jondaS0dORfDiVUxOgg35Zv40bUMrxWO8QWi4I+ERGRCpZMOuIZEzkSScenj5nJHz52cNZzitnNmvQv/osPHZD1eKn3Aa4mCqdFREQqmHOdgdTIugjb2rxFkb92yh5lWQA51ZYxw2qyH9fAvqJRpk9ERKSCJZ0jlkiyx8QR7D+tCYAPv20aAHXRMCOydKcWM+6688XVQO71+pLK9BWNgj4REZEKlnSOeNIRCRthP86qj3ZO1Nhl3LBu5ziKF3jd9OQbAIRzBH3BTF8x1wusRgr6REREKlhqIkckFEovl1IX7fzzny3jVopYK9fSLcHXVtKvsBT0iYiIVDDnL9kSCVk6wAsHpueGswRfyRJEfRu3t/dYR5M6CktBn4iISAVLZ/rCncFdMLmXNdNXpLYEu2t3aqzvsX4pgs9qoqBPRESkgiXTmb4Q9y5YC4DRGehl7s4BxeveDV5378mj+lRfBk5Bn4iISAULTuRICU6WyD6hojjRVl+XY1Gmr7AU9ImIiFQw5yCe8CZypAK8YDdrtgkVxYq1+jpGT2v2FZaCPhERkQrmnLcPbyRkfOmEWUDXDFq2TF/xxvT1sX6y5zrSewr6REREKliwe9f8rF4w4ZZtTF+xulVTmbtcu3EAnHnQlKK3o1op6BMREalg3kQO1yW4C8ZSNZHOUODy9+7T7Xghpbp3P33MzJx1fnTmvnzrnbMBBX2FpqBPRESkgiUd3uzdcOfizMExfamg76yDp9LUEPWPF6ktftCXazcOADMj6rdJY/oKS0GfiIhIBXN+9240bOn1+YIZtJqwFwp4Xb9+UFikUX2p1821G0dKOB2cFqUZVUtBn4iISAVLLc4cDlkg09d5PJXpM/O+Mo8XUipzl21B6KBswakMnII+ERGRChZcnNnSwVTn8VSmL2SQPxTru5OuephfPrCosy3+bNxsW78FhbJMOJGBU9AnIiJSwdKzd0Ods3eD3bepTF/IsmcCB+L1tdv58T2vd2kLQLiH6CMdnCrqKygFfSIiIhUsvThzONTZbRoIpqKBCMyK3K2amr1rPY3pC6UyfQr6CklBn4iISIVpiyXSj71Mn7c4cyrAiyW7Z/piCdc5pq8AbYgnuq+snM70qXu3LBT0iYiIVJgVm1rSj+NJR9JBJGxE/f13Y/HOgKzWD/riiSRG9yVd+qstni3o877nW7IFOjOOfd22TfJT0CciIlJh6qLh9ON4wgucouFQZ6YvkIXrzPQl0zM5ChFqtQeyjSmd3bv5z822R7AMnII+ERGRChMcC9fhZ9zCwe7dRPd1+mJJl569W4hYK1uWrnMih7p3y0FBn4iISIWJB6KljoSXcYuEjDHDvT1vx42oTR9PBYLxRDIwwWLg0VYsX9DX45i+rvWlMCLlboCIiIgUVjDLFot7jyMh47BdxvCLDx3AibMnpI/XpMf0uXSwVZBMX6L7RXo7ezd1XGP6CkuZPhERkQoTDwRc7f74vUg4hJnxrv126jLmLz2mL+nSEzkKEWvFkp3dyinpxZl76N7N3Ibt6WWbeGPjjoE3qsop0yciIlJhumb6vEgrNXM3U01w9m460zfwqC/VhmBXbm8XZw75x9c1t7G1tYEP/PYJABZ/71QiPZ0sORX1nTOz5Wb2kpk9b2bz/LLRZnavmS3yvzf55WZmPzezxWb2opkdGLjOuX79RWZ2bqD8IP/6i/1zC72DjIiIyJATT3bOzl3b3AZAOJT9T35NYEZv4Ub0dc4QDr5sau/d3nbvnn/DvC5rDm5tjRWgZdWrFOHysc65/Z1zc/znFwH3O+dmAff7zwFOBWb5XxcAvwYvSAQuAd4GHAJckgoU/TqfDJx3SvFvR0REZHALZvp++/BSoOdMXyzhOpdsKeDs3S6Zvixl2YQCx4OTUhKa2DEg5ciRng7c4D++ATgjUH6j8zwJNJrZJOBk4F7n3Cbn3GbgXuAU/9hI59yTzstD3xi4loiISNWKZxmUl2scXSrTF08GFmcuQK4vtVRMKPC66UCwl2P6AOa/sTn9WDHfwBQ76HPAPWY238wu8MsmOOdW+4/XAKkpRJOBFYFzV/pl+cpXZikXERGpatlmvUZydO92LtnSuQ3bmq1tA27Dn556E4Dmtni6LNWsngZjBWPCz//lufRjzeYdmGIHfUc65w7E67q90MyODh70M3RF/wTN7AIzm2dm89avX1/slxMRESmrbJm+XN276a3ZEsn0RIsv/+2FAbdhwsg6AA6ZMTpd1tt1+nLRun0DU9Sgzzm3yv++Dvgn3pi8tX7XLP73dX71VcDUwOlT/LJ85VOylGdrxzXOuTnOuTnjxo0b6G2JiIgMaolk931vc3WpRtLdu471ze0Dfu01W9s48cqHWbO1FYDRDTWBdvWuezfX2L0styV9ULSgz8yGmdmI1GPgJOBl4HYgNQP3XOBf/uPbgXP8WbyHAlv9buC5wElm1uRP4DgJmOsf22Zmh/qzds8JXEtERKRqxbMsjBzNsdRJxA/A4glHbSSctU5fvP+3j7No3XZue/4toOv4wGQvZ++2xbJHd5rIMTDFXKdvAvBP/4ONAH92zt1tZs8AfzOz84E3gA/49e8C3gEsBlqAjwE45zaZ2XeAZ/x6lznnNvmPPwNcD9QD//G/REREqlq2sW85J3JEOpdsOXkvb5j9Gfvv1O/XXrGptcvzYJyWalekh0xfrq5ode8OTNGCPufcUmC/LOUbgeOzlDvgwhzXug64Lkv5PGDvATdWRESkgvRlTF+tH/Q1NkQxM3Ye01DQwfbBa7X6a+7V1+TPKL59t3HMHDeMJeu77sKR1ESOAdGOHCIiIhVmW1v3RYxH1kWz1m1sqOEH79uHo2Z5Y97DZgXZhi0luLtHa4cf9EXzB31mxlGzxnUL+tS9OzAK+kRERCrMN/75creyXccPz1n/gwdPSz82G1hGbcywGjbu6Eg/D8Zpvc30eed1b4MmcgyMNrATERGpAr3dqTQcsoKuh5d0jrmvrGHp+u20+Jm+ht4EfTmuJf2nTJ+IiEiF+5+3z+x13ZDZgIKrzNhy1ZZWPnXTfACmNNUDUNeLWcLZ2qCgb2CU6RMREakgbX4XatC+U0b1+vyBBn2ZWcLX125PP1652ZvZG+ph9m5vry19o6BPRESkgqza0tqtrC8xVjjU/4kcx/zoQTa3dJ9E0h/Z4s5/zF/ZvVB6TUGfiIhIBck2M7a34/nACxD7m1FbvrGlX+dlk60Fqf18+6qlI95zpSqgoE9ERKSCZAuWQn0J+kID697NZ9roBg6ZPrrnivRu3F9vPLN8E7O/NZfHFm8oyPWGMgV9IiIiFSTbcit96t4d4Ji+fGojIcYMr+m5IjC8Lvtc045439ZteXSRF+w9sWRjn86rRAr6REREKkgqYDto56Z0WZ8yfVbYJVtSDt1lNEnnej2J4z0HTM5avqO9b1217X6QmNp5pJrpHRAREakgqYDt7EN3Tu+324eYj1CocIsgjxtR2+V50vU+AJ0xdljW8u19DPq2tnoLRdf1sAtINVDQJyIiUkE6El7EFgpZehHkvmT6aiJh2uPdl33pi30mj+KWTx/O7hNGpMuSzgtIc2wB3Gs7+jgpIxUEf++uhazYVLiJJkORgj4REZEK8tk/PwfA9rY4w2q8cXF9CfomjKhlzba2Pr9uPNGZHtx78igO2rmJDdvb02XOOa97tw9tue/Lb+9W1tfu3eDr3b9wbZ/OrTQK+kRERCrI4nXeYsgd8QSNDVGgb927w+sitLT3PdP35b+9kH6c8PuHX13TnC5LOm+SSV8WZo74daeOrk+Xbe9j24JzUkYPr81dsQoo6BMREalASQeTG71gqS/j4MJmJPoxe/f2F95KP45nmQjiZfr6NpN4UmMdO49p4LLT9+bLJ+4G9D3TF2xLtU/mqO67FxERqVBJ59KTF/qyzEkh1ulLLRvzpRN2C7QHEs6lJ5f0Rm0kzMP/eyzH7j4+PZu3rxM5gveSbTmbaqKgT0REpAIlnaPGz2zFEn0I+sz6NXs3GpihkcquTW7q7JZ1zuGc69PuIEGpLF1f1+kLZvqyZSCriYI+ERGRCpR0nYFYXwKlcIh+de+OH1EXeG3v/OAs4M7Zu/0L+lIZwr5mIZNdgr4CrUUzRCnoExERqUCJpCMa7numL9zPxZknjOycJBFPeOcHg81kP8b0dWmXf2Lq2r0VDPRifTy30ijoExERqUDOOU7bZxIAc3q53y2Qnl3b1/FvwbF6qaCxLRYM+vo+ezdru/qY6UskYcwwb+u3vgaMlSb7xnYiIiIypCWScPiuY1l+xWl9Oi/V/ZpwjhC9D9AsUDfVPZwIZNkWrt4G9G3NwKDU8i19HZeXSCbTE1rUvSsiIiIV4+S9JgDwsSOn9+v8VEatr128Ty/flH48aZQ3vu/cw6fzngMmd8kC9mX2bpd2Wf52rdrSisuSBUy4zkkg1Z7pU9AnIiJSQYbXRpncWM/Iumi/zk8FVwNZteXid+wJwIi6KFd9cP909yr0baHooEiebuef3beII654gBseX87dL6/muTc3p4+1xxLK9PnUvSsiIlJBHAPLZvlzP/o1gxfgB+/bJ2/AOdDZu5ndu4+8vp6r7nsdgEcXb+C+hesA0t3aa7a1sfuEESxYvU0TOcrdABERESkg1/9sGvTcjdqTDx48Le/x/nbvmhlm3SdyLFm/Pf24PWNpmmTSsXpLG9PHDgPUvaugT0REpIIknev3ZAkIrIdXwIWMg1fq7+LM4HXx/uKBxXztHy+myzbt6Eg/bot13Zd3W1uMjkSSCSPrMFP3roI+ERGRCuIYWKYvFfT1t3u3x+sPoHGpYPav81aky3a0dwZ6mU1OLRlTHw0TDYW0I0e5GyAiIiKF4xx9WGilu/5274ZDxqfevksvrt+vZgGdkzmCgjFkZjyZ2hGkLhqiI5HkAX+8X7VS0CciIlJBvExf/yOrhhpvpmtrR6KHmoHXdI5E0lEbzh5WBDNw/V2cGUjPws11bcsId1Nj/Goj3nmvrW3u92tXAgV9IiIiFcQ5N6BMX0ONt7DH9vZ4r89JZQUjOYK+4Ki+gYw3HDeic6u3VVtaga5bzGWO2WuPpYI+hTugoE9ERKSiOBhQ/+7wWi/o29GHoC+eDvqyv3AwG5czLuyFEXWdK809sHAt0HV/30xtfvdubTTE7Ekj04tGVyut0yciIlJJBjimr8bPivVlTbsOP9sWDfUc0Q0k0xcJXD/Vvo5E7qCvM9MXZoG/DdzWlhijGvq3cPVQp0yfiIhIBXG4AY3pS53al0WeU+vfRXNk+oLNGVDQF7h+KouXN+gLTORI2bCjvd+vP9Qp6BMREakgA5+923md3lq5uQXIN6avMILB7A/vfg3o2r1bX9N1oseLK7cCnRM5MutXGwV9IiIiFcQNcEeOVMiYufNFLi+s2MK7f/kYkDvTFwxDB7JAcrYFo4NB3LCarqPWfnb/IsCbyHHCnhOAvo1VrDQK+kRERCqIw3VbuqQvOrt3eye4DVokx5i+YBA6kP1vgzN1Z08aCXQN+moDS7p86a/PB8pDnH/kDK9+nu7gSqegT0REpIIMNNOXPrWXsVlwL91cs3eDpQPZ//apZZvSj1OZyI5EkiN2HcPY4bVdMoH/fG5V+nFdJBzYXq7fLz/kKegTERGpIEk3sMWZUxMtetu9G8zuRXsxpq8Q+9+euvfE9D6789/YzGOLNxIO5b52bTSUXipm7itruN9f7qXaKOgTERGpKANbnNn6OJEjmN3Ltk0adJ2xO5CA9NbPHM4P3rcPo+qjtMa67hgSMsu5dVxNOJRuw01PvsH5N8zrdxuGMgV9IiIiFWTg3bveyb3thA1O3siV6Qu2ZyC7Yxw4rYkPHjyNumiY1o4Ezo9MP/K2aYTM0otEZ4qEQ126oauVgj4REZEK4u292//zOzN9vQv7glm8XGP6gnabMKJf7QqqrwnTFkums5HjR9QRCpEz05fZzmqloE9ERKSCeHvvDnz27itvbetV/eDYv5yzd/3vV3/kQE6cPaHfbUupj4bpSCTTM3FDBmGzrJNEfvL+/by2ZQSk2ZZ/qXQK+kRERIaoz/75Wc741WNdygac6fNDtNQadwD3vLKGh19fn7V+cAWUXOv0pQK9I2aO7X/DAlJbxaW2WTPLPabvuD3GA15QGJQ5JrAaFD3oM7OwmT1nZnf4z2eY2VNmttjM/mpmNX55rf98sX98euAaF/vlr5nZyYHyU/yyxWZ2UbHvRUREZDC548XVPL9iS5eyAe/IkSUyuOCm+Zx73dNZ6ycCM2Zz7cjx/945m6e+fnzB9rxNDc9L+FlGM6MmEsoayIX9QDSUMabvsMvvL0hbhpJSZPq+ACwMPP8BcJVzbldgM3C+X34+sNkvv8qvh5nNBs4C9gJOAa72A8kw8CvgVGA28CG/roiISNVyMKBUX1+7hoOTJ3LN3o2EQ0wYWdfvNmVKjc9LZfZSz19atbVb3agfxWZm+ra1Vd/OHEUN+sxsCnAacK3/3IDjgH/4VW4AzvAfn+4/xz9+vF//dOBm51y7c24ZsBg4xP9a7Jxb6pzrAG7264qIiFQtb0xf//U1Xnz2jS3pxzUDmJnbF9Yt6INX1zRnrZuatavZu8XP9P0U+D8glfsdA2xxzqXC65XAZP/xZGAFgH98q18/XZ5xTq7ybszsAjObZ2bz1q/PPiZBRESkUgxkTF9fYyNHz5m+Qku9TGox5pAZx+4+LmvdVJuyLTb93Jubi9PAQapoQZ+ZvRNY55ybX6zX6C3n3DXOuTnOuTnjxmX/oRARERmqgsurDHRMX1/P3h7oJu3NjhyFkNm9awYHzxidva4f9AX36E15z9WP80iOCSqVqJifzhHAu81sOV7X63HAz4BGM4v4daYAqc3xVgFTAfzjo4CNwfKMc3KVi4iIVJXgpNV8a9X1Rl+zhNvbO4O+3qzTVwidmb7OMX01PQScY4bXAvC+A6d0KX9zU0vhGzhIFS3oc85d7Jyb4pybjjcR4wHn3EeAB4Ez/WrnAv/yH9/uP8c//oDz/utyO3CWP7t3BjALeBp4Bpjlzwau8V/j9mLdj4iIyGB167Mr04+fWLqRZ9/c0u9rBRcxPvT79/OrBxfnrR/MoOVap6/Qso3p66lrefSwGl759sn86Mx9u5RX06LN5Vin72vAl81sMd6Yvd/75b8HxvjlXwYuAnDOvQL8DVgA3A1c6JxL+OP+PgvMxZsd/De/roiISFX533+8WLBrBUOgNdva+NHc1/LWD87ezbVOX6GlArXUYsyhkOVcLiZoWG2k29ItVRTzEem5ysA55x4CHvIfL8WbeZtZpw14f47zvwd8L0v5XcBdBWyqiIhIVcsXBDnn0lm2lGB3cm8Cr0JINaGlI+4/77l7N+e1CtWoIUA7coiIiFSI1ISOgWx1lq+7M9t4wVgi2L1b2jF9598wD/Duu7/jCdW9KyIiIkNOzO/u3G/KqKJcP54l6Et06d4t7Zi+ra0xr10J1+8sYxXFfAr6REREKkWHn3Ur1iLJLR3dtzmLBYK+Ui2AnJmdSzrX7yxjZnd1JVPQJyIiUiHa/b1nayPhfl8jyxrGaWu2tnUrC+69WyqZ8V0i6bIuvpzL5Mb6nNeqZL0K+sys3sx2L3ZjREREpP8KkekL7rCRqTXWPdMXTzhO2HM8r3z75H6/Zl91z/TlD1Yzzf3S0QVu0dDQ40+Fmb0LeB5vuRTMbH8z03p4IiIig8zTyzYN+Br5gqdsEzniSUdtJMyw2pIsCAJ0H4eXdF1D1S+dsBvH7j6O/zsle75qeKCt1TSRozef0KV4S6w8BOCce95fJFlEREQGkf/z1+tbtmFHUa6fLehLJF3JxvKldMv0JV2XreguPHZmjxM7PnTIVP7y9ArqotUz0q03dxpzzm3NKBvYHi8iIiIyIMksAVgqFnJ96evMkO/MXEu2lGr7tZTMoC+Rcb+9mcl73uFe/ipR+iGJZdObTN8rZvZhIGxms4DPA48Xt1kiIiKST2agA2D+UsMDiPnyBozZXjOR7P/M2f7KfLmGmnCfJnIApOLCbPdUqXqT6fscsBfQDvwZ2Ap8sYhtEhERkR5ky7ods/s4AI7fs/+LM+d/ze5psXjSES7RnrspmcusTBpVzzG7jWfnMQ3M/WLvJmmksoUDyYoONT1m+pxzLcA3/C8REREZBDKDvnXNbey100j+8/Ia5kxv6vd1G2pyhwapvW67liVLtuduSjDTd9npe3HaPpMIhYyH//fYPlzDu0i24LlS9Wb27r1m1hh43mRmc4vaKhEREckrs1vyY394Jr1jRngAM1InjqrLeSxbF2q8jBM5JjfWc85h0wn14/VTbc4WyFaq3uRjxzrntqSeOOc2A+OL1iIRERHpUSIjWFm0bjuJpCNk9CsICpo6uj5rebZt2OKJ0o/pC/uZxdHDavp9jdSyLdvb4wVp01DQm6AvaWbTUk/MbGc0e1dERKSsMjN9HfEk8aQjUoDxdakJId1eM8eSLf3d97a/avzXG8hyK8PrvKCvuU1BX9A3gEfN7CYz+yPwCHBxcZslIiIi+WRbsqVQa+bl6h1u6Uhw4xPLu7x2PJksw+xd7/VyBae9EQ2HqI+GaW6LFapZg15vJnLcbWYHAof6RV90zm0obrNEREQkn8yu1pnjhhWsqzXXFS6+9SUAmhpqeNd+O5FMOpKOgmQX+2Ogm2mMrI8o0wdgZnv43w8EpgFv+V/T/DIREREpk8yu1rHDa0kkk+nxbgORuSRKptQ4uFTgWerFmVObrg006BtRF6W5XZk+gC8DFwA/yXLMAccVpUUiIiLSo8yZtG3pMX3Fy/SlXHzrS5x18FTi/rp9pZ69m5pZMJDuXYD6aJjWjkQBGjQ05Az6nHMXmFkI+KZz7rEStklERER60G0mrXOF2wc3yyUOnNbIs29uST9v94NMoORj+lJ3PtBMXyhkVNGKLfkncjjnksAvS9QWERER6aXMiRxJRwFn73aXud9tPOnSy8aUOug7cFoTB0xr5Bun7Tmg64Qt+4SYStWbvXfvN7P3Abe6atqrREREZBDLXLIlWcBMX7YxfZlF8USSWKp7t8RLttTXhPnnZ44Y8HUioVC6i7oa9OZT+hTwd6DdzLaZWbOZbStyu0RERCSPzJ0kOjN9Aw/6Ljt9r25lzwW6dgFiCZeeTBIt9Zi+AgmFoIpivp6DPufcCOdcyDlX45wb6T8fWYrGiYiISHaZEzlaO+L8+4W32LC9fcDXPnzmWF79zincdmFnNi1zDOFLq7akA8+ST+QokHDIumVMK1m+JVtmmdm/zOxlM/uzmU0uZcNEREQkt8wlW5ZvbAFgW4HWnauLhtl/amPO4x+/fh7PLN8ElH7JlkIJmWXdZaRS5cv0XQfcAbwPeA74RUlaJCIiIj0aDMHKm5u8QLNcizMPVDhk3TKmlSzfpzTCOfc759xrzrkfAdNL1CYRERHpQSro+9Mn3sZp+0wqSxtS3bs1kSEa9FVZpi/f7N06MzuAzpnb9cHnzrlni904ERERyS41Fi0csgGvV9dfa7a1ATC5sb48DRigUEhBX8pq4MrA8zWB59qRQ0REpIxSwUo4ZN3W0CuV1pi3m8WIut6sADf4hK26unfz7chxbCkbIiIiIr2XCvpCZhRz8uzf/+cwouEQZ/yq++ZcLf4evEO2e1eZPhERERnsHl20AfB2w7j7lTXp8vMOn17Q1zl4+uicxza1xACIlnhx5kIJhYwqivl6tTiziIiIDDLXProM8DJ9bbHOFYa/cPyskrXhhRVbgKEb9IVtcMyCLpWh+SmJiIgI0H07tlAJFkpecNnJXZ7XDtHu3WqbyJFvceaTzezMLOVnmtmJxW2WiIiI9EY80XUfsVLsjtFQE+GgnZvSz4dqpi+ioC/tW8DDWcofAi4rSmtERESkT2IZe/CGSzSTN5Xdq4uGtA3bEJEv6Kt1zq3PLHTObQCGFa9JIiIi0luZM2dLtXpLXTQMwNSmhtK8YBGEzEgq0wfASDPrNrvXzKLA0FyFUUREpEIc4s+qPXBaY5fyUmXdUuvbbdjeXpLXK4bMTN/l/1nILfNXlrFFxZUv6LsV+J2ZpbN6ZjYc+I1/TERERMqkNhrigGmNWEZqr1jdu3tMHNHl+UOveZ2Bm/1lW4aiUGAbtmTS8duHl/KVv79Q5lYVT76g75vAWuANM5tvZvOBZcB6/5iIiIiUSdK5rDtxFGv27m0XHsH/vH0mt114RFGuXw7hkLGjPc6itc1dMn7PLN9UxlYVT76g7y7n3EXAVOA8/2uac+4i59zQDetFREQqQCLpSjZpA7wxfBedugf7T20E4PfnzinZaxdL2F+c+cSrHqGlI5Euf/9vnihjq4on344c4wCcc63AS6VpjoiIiPRGMgmhMq6UcvCM3Dt1DBUbt3ekH7fHE3lqVoZ8Qd8oM3tvroPOOY3rExERKZOEc0TLGPXVRcJle+1CueXZzkkb7bFknpqVIW/QB7wTyJY7dmgyh4iISNkkkp1j+h74yts57ifZltYtnmh4aK7Nl0trrLozfW845z7e3wubWR3wCFDrv84/nHOXmNkM4GZgDDAfONs512FmtcCNwEHARuCDzrnl/rUuBs4HEsDnnXNz/fJTgJ8BYeBa59wV/W2viIjIUJJ0Lr08yy7jhpf89TNnDQ91Ty+rzMkbQfnywgP9NNuB45xz+wH7A6eY2aHAD4CrnHO7Apvxgjn875v98qv8epjZbOAsYC/gFOBqMwubWRj4FXAqMBv4kF9XRESk4mVO5PjNRw/irIOnlrFFQ9s3b3u53E0ounxB39mZBWY21noZ2jvPdv9p1P9ywHHAP/zyG4Az/Men+8/xjx/vv9bpwM3OuXbn3DJgMXCI/7XYObfUOdeBlz08vTdtExERGeoSSddleZZT9p7IFe/bt4wtGnoaanKPS6zEnTryBX3DzewhM7vVzA4ws5eBl4G1frdqj/yM3PPAOuBeYAmwxTkX96usBCb7jycDKwD841vxuoDT5Rnn5CrP1o4LzGyemc1bv77bznIiIiJDTtKVdsmWSnTN2bmXndnREc95bKjKN6bvl8DX8SZ0PACc6px70sz2AP4C3N3TxZ1zCWB/M2sE/gnsMeAW94Nz7hrgGoA5c+ZUXuguIiJVJ+nKu2QLwGG7jGHPSSPL24gBaGyI5jy2vT3OiLrcx4eifEFfxDl3D4CZXeacexLAOfdqXwdvOue2mNmDwGFAo5lF/GzeFGCVX20V3kLQK/09f0fhTehIlacEz8lVLiIiUtGSyew7cpTSXy44tKyvP1DZ9imuCYfoSCTZ3hb3IpEKku//CMEFa1ozjvWYLTOzcX6GDzOrB04EFgIPAmf61c4F/uU/vt1/jn/8Aeec88vPMrNaf+bvLOBp4BlglpnNMLMavMket/fULhERkUrQkUgSDZc51TfERbIEfWcd4uWTmturq3t3PzPbhjeLt95/jP+8rhfXngTc4M+yDQF/c87dYWYLgJvN7LvAc8Dv/fq/B24ys8XAJrwgDufcK2b2N2ABEAcu9LuNMbPPAnPxlmy5zjn3Sm9vXEREZCjb0hLL2z0pPYtkCZqnNjUAeJm+CpMz6HPODWipbefci8ABWcqX4s28zSxvA96f41rfA76Xpfwu4K6BtFNERGSo2bSjg+3tccaNqC13U4a0SaO657Cmjq4HvDF9lUZ5YRERkSFm1WZv1NWuZViUuZLURcMsuOzkLmVTKjjTp6BPRERkiIklvWH3NRH9GR+ohpoIl52+F6ftMwmAyY2Vm+nLN6ZPREREBqFY3Av6NJGjMM45bDrnHDadXwEd/nvbUoHr9OmnRUREZIiJ+7tFKOgrvNQyLolkDxWHIP20iIiIDDEdiVSmTztyFFpqFZeEq7y9HBT0iYiIDDHxhDJ9xWJmhKz69t4VERGRQSiW0Ji+YgqHTJk+ERERKb+OuLp3i8nMSCroExERkXJb39wOwJjhWpy5GMJm6t4VERGR8tu4o4No2BhZp5XXiiEcMs3eFRERkfJriyWoj4YxU/duMYQMde+KiIhI+c1/YzPbKnCbsMEiHNKYPhERERkEXlq1tdxNqGghMxIa0yciIiJS2UIVmunTCFAREZEhZtyIWk7Yc3y5m1Gxwsr0iYiIyGAQTyS1MHMRafauiIiIDAqxhFPQV0Rm4Cqwe1c/MSIiIkNMhzJ9RaVt2ERERKTsnHPEEklqtAVb0WhMn4iIiJRdIulwDmX6iqhSZ+/qJ0ZERGQIiSW8YCQa0Z/wYlGmT0RERMquI+5NK1Wmr3jMoDcxXzLpdbUPFfqJERERGUI6/CBDY/qKJxwykr2I+j79p/m86xePlqBFhaHFmUVERIaQVGZJmb7i6e3s3bmvrAVga2uMUfXRYjdrwPQTIyIiMoQo6Cu+nvbedc51yQQuWb+9FM0aMGX6REREhgjnHG//0UOAJnIUUzhk5Er0LV7XzAlXPsJp+05Kl21obi9RywZGPzEiIiJDRHt86EwaGMpCRs5M3ytvbQPgzhdXp8s27egoSbsGSpk+ERGRIaItlkg/joQ0kaNYQpZ9TN9NT77Bv194q1v5RgV9IiIiUkitgaCvcQhMHBiqwiFLL40T9P9uezlr/Tc3thS7SQWh7l0REZEhorWjM+g7bOaYMraksvVl7926aIg129qK3KLCUNAnIiIyRAQzfWbq3i0WM+vV4swAsyeNHDJbtinoExERGSKCY/qkeNpiCV5YsQXXQzB31KyxXlZwiGzZpqBPRERkiGjt8MaZ/eG8g8vcksr29LJNADy/YkvOOqfsNZFvvXM24ZARDwR9z765mfsWrC12E/tFEzlERESGiFT37tjhtWVuSXUIZXShj6yLsK0tDsAvP3wAkXDIC/pinZM+3nv14wAsv+K00jW0l5TpExERGSJSQV99jf58F1Njgzczujba9X0OduOG/SVzQtY10zeY6adGRERkiGjzZ+/WRcNlbkllu+K9+wDdF2iOBZ6nJtJEQjZkJnKoe1dERGSISGf6FPQVVWpf41TQd9tzq9jS0kE80X3tvnDIWL21jTc3tjB1dH1J29lXCvpERESGiM7uXQV9xZTquv36P19it/EjuPW5VTnrzn9jM5tbYhz9owd54ZKTStXEflHQJyIiMkSklmypiyjoK6ZU0Pfyqm28vGpb3rqbW2Lpx+ub23t1/U//cT4NNRF+8oH9+t/IftCYPhERkSEinnCEDELad7eowj28v588akbWc2JZun+z+c/La7jl2ZX9attAKNMnIiIyRMSTjkhY+Zpii4Ryv8cj6yJ847TZ3cobasI9LtKcSDr+/cJbA25ff+knR0REZIiIJ5JElOUrunyZvo8eunOX518+cTcAYolkj0u3XHXv63zxr8+nn5d6J4+iBX1mNtXMHjSzBWb2ipl9wS8fbWb3mtki/3uTX25m9nMzW2xmL5rZgYFrnevXX2Rm5wbKDzKzl/xzfm7aiFBERCpYPOl67HqUgcv3Hn/1pN27PP/88bP47LG7Eks4Esn83bt3vbS6y/NtrbEcNYujmJm+OPAV59xs4FDgQjObDVwE3O+cmwXc7z8HOBWY5X9dAPwavCARuAR4G3AIcEkqUPTrfDJw3ilFvB8REZGySiRdejkRKZ7htblHv2UbT1kTCZFIOu5dsC5d5pwjlkgy/aI7+e3DSwDYkhHkxXoIEgutaD85zrnVzrln/cfNwEJgMnA6cINf7QbgDP/x6cCNzvMk0Ghmk4CTgXudc5ucc5uBe4FT/GMjnXNPOm9H5BsD1xIREak4yvSVxsxxw/pUv87fueM3fnAH8I3bXqbZ37Lt8v+8CsBp+0zqcl6JY77SjOkzs+nAAcBTwATnXCq/uQaY4D+eDKwInLbSL8tXvjJLebbXv8DM5pnZvPXr1w/sZkRERMpEY/pKo6+jxYZlyQz++ak32dEe71Y+qj7KSbO90CdeKZm+FDMbDtwCfNE512WxGz9DV/RRjM65a5xzc5xzc8aNG1fslxMRESmYWCLJRbe8yKotrSSU6RuUhtVk7w5+a0trl+c72uMMr41w8l4TgdJn+oq6ZIuZRfECvj855271i9ea2STn3Gq/izbVAb4KmBo4fYpftgo4JqP8Ib98Spb6IiIiFePJpRu5+ZkVzH1lDUfNGqcxfYPQ+JG1Wcs/eM2T6cen/PQRXl3TDEAk7AXuFZPp82fS/h5Y6Jy7MnDodiA1A/dc4F+B8nP8WbyHAlv9buC5wElm1uRP4DgJmOsf22Zmh/qvdU7gWiIi3LdgLQd+515a/U3qRYaisN/VuLklxu0vvMWwWu3GMdgctssYfvz+/LtrpAI+gJD/mSZdhSzZAhwBnA0cZ2bP+1/vAK4ATjSzRcAJ/nOAu4ClwGLgd8BnAJxzm4DvAM/4X5f5Zfh1rvXPWQL8p4j3IyJDzE/ufZ1NOzpYtK6558oig9DWlhgfvvapLmUHTmvKUVuK4RNHzmDSqDouf+8+/OvCI7LWMTPOPKiz8/GCo3fJeb2jZo1Nj8vsaV2/Qita965z7lEg18CD47PUd8CFOa51HXBdlvJ5wN4DaKaIVLCmhigAW0u8FpZIoby6pvu+rzuP6dvMUhmYb75zNhedukefdkJp9H/3ZPPj9+/H8yu2AKVfnFnbsIlIxUr9ki71/6ZFCuXpZZu6lY0eljugkMK5+4tHMaree6/7uvVdNLCN29LvvyO9tl9bLEFdNJzO9CnoExEpEH+sNO2xEk+REymQn9z7erey4bUK+kphj4kj+3zOf//vWLa1xdKZvP2mNnZZzLku6o3HDCnoExEprLD/v+32uCZySOWoj2oix2A1dXQDAGOG1fKt0Ctc9u69stZTpk9EpMBSv1g1e1eGqhF1kfSuDimu+MvbygBNHFXHku+/I+fx1IzsUgd9WuxHRCpWODCORmQoqgmMJXvPAd6mU/n2hZWhIVymTJ+CPhGpWKlfrK0a0ydDUHNbjI07OtLPLz51D279zOEcoCVbhrx00JexTt+9C9Zy5A8eYMP29qK8roI+EalYyvTJUPbsm1u6PB9ZH9UafRUi9bvp1w8t6VL+iwcWsXJzK8s37CjK6yroE5GKlVqqRUGfDEU72ruO5avTBI6KkQr6Hl+ysUt5KvHXnPHZF4qCPhGpWM+v2Ax0Bn3f/vcrPL5kQzmbJNJrmoBUucKhrntX3LdgLdMvupOXVm0FYHubgj4RkV5bvG47Kza1AtAaS7CtLcYfHlvOOb9/uswtE+mdNn+pofccMJl37jupzK2RQgoGfe3xBJ+4cV6X4zc/8yZ/efpNXIH35tUUIBGpSCdc+XD6cVssycbt3oB4rXEmQ0Uq03fpu/dK7wwhlSG1ZAvA4Zc/0O34Y4s38tjijcwcN7ygr6tMn4hUvNZYgkTSm8GbLPD/nEWKpT3u/czWRfWnupKlZmiPHV7DjR8/pMuxD/z2iYK+ln6SRKTitcUSxBJesKdteGWouPvlNUDXtfqkMmT7NbS5JcbRu40r6uuqe1dEKl4skSTuB32Z62KJDEbrm9vTg/rNrIfaMtTk+zU0/5sn8PJb23j7buNIJh3hHxTudfXfBxGpaMNqwsQSjniqe1epPhkC1jcXZ3FeGRyyDTP514VHADBmeC1v9zN+oVBhA34FfSJS0UYPr2H+G5t5zl/oVpk+GQq2tcUAKPDffBkkGhu8iTl7Tx4JwH5TRrH35FFFf11174pIRUvN1r3sjgVA/m4VkcFge3ucL/31eQBu/+yR5W2MFMWkUfXc+6WjmTCqjrOvfYpL3r1XSV5XQZ+IVJxgF259jX7NydDy0GvrWL21DUBLtVSwWRNGAPCvEgb26t4VkYoT7MJt0Lp8MsQEh502DaspX0Ok4ijoE5GKkwj81Wyo6R70bW2JlbI5In0SXLh3eK0y1VI4CvpEpOIEg75xI2q7HX90sfbflcFLk42kWBT0iUjFSf3RnDF2GHVZunfHDFeXmQxeMX8njh+euW+ZWyKVRkGfiFSc1ESOsw/dOeuG5UqkyGAWS3hB35G7ji1zS6TSKOgTkYqT6t4Nh4wbnnij2/HP/Gl+qZsk0mupoC+q7dekwPQTJSIVJ9W9m2s1+82ayCGD2La2OAB1Uf2JlsLST5SIVJxUpi+i7QxkCPrZ/YuoiYQYUac1+qSwFPSJSMVJd++aZZ29C9qDVwanH979Kh3xJB3+ZA6RQlLQJyIVJ+n/vQyFjJsvOJRDZozuVieW1B9VGXyufmhJuZsgFUxBn4hUnNSYvnAIZo4bzkWn7pE+dsKe4706yvTJIDbvmyeUuwlSgRT0iUjFSQV0IX9ng9SyLQdMa+TQXcZ0qSOF8+qabVx57+vlbsaQtXD1tvTjscOzD0sQGQgFfSJScZIuNZEj5D/3yo3OQPDWZ1eVo2kV7cxfP8HP719EWyxR7qYMSZ//y3MA/N8pu5e5JVKpFPSJSMWJJzq7d6Fz0kbIjEjYC/ouuf2VsrStkrV0eEuNKIvaP2F/tvlnjtm1zC2RSqWgT0QqTirTl8rq1dd4W7FNHFWXLpPCS8V69yxYw/SL7mT11tbyNmiIqY2EOHq3ceVuhlQwBX0iUnGCO3IA7DulkR+duS+Xv3cfrd1XAl/66wtA1zFqyaRj1ZbqCwJfW9PMlfe8lnU7wExtsSR1Ef1ZluLRT5eIVJxsO3K8f85URtRFc+7SIYVXEw6nH1//+HKOuOIBXl/bXMYWld651z3Nzx9Y3KtdYNriCeqi4R7rifSXgj4RqTjJPDtyBMs2bm8vWZuq0fb2ePrxdY8tA2DZhh3lak5JvbWllUcXbWDNtjYAnl+xma/+/QWeXLox5zltsYS2XpOi0k+XiFSc1G4G4Szj98KBoO+g797H4nXVlXkqpf/54/z045Wbva7dapjZ++bGFg6/4gE++vun0mUfv34e/5i/krOueTLrOdvaYqxrbmdyY0OpmilVSEGfiFScJeu3AzB+ZPe1zsIZ2b9VW9q6PG9ui7Fma9cyKZzbnqv8pXKWbtie89iELD+T65rb2PfSe3DOW0tSpFgU9IlIxVm7rZ1wyNhl7PBuxzK7fGv8dV2a22IseGsbZ/76CY77yUOlaGZVevC19eVuQtG1dHTNZp62z6T042N3H9+t/r9fWJ1+vN/UxqK1SyRS7gaIiBTa+uZ2Rg+ryTppY9roYV2et8e9P9D7XHpPl/JE0nXLCkp+tZEQ7fHOPY0bG6IAvZq5Wkn+Gchm/vMzh7PXTqP4z8urSbrOoQe3PruSw2aOYdKoet7Y6I1z/ORRMxhVHy1Lm6U6KNMnIhVnzba2nNtYzd5pJAcGutDaYokuEw5Strd1L5P8Jo2q6/J8S0uMdc1tdCSSXcqv9yd1VKp7F6wFYK+dRnLAtCZqIiGWXn4as8YPpzWW4Jb5K/ny317gEzfMY+HqbbywYguH7jKab5w2u8wtl0qnoE9EKs4rb21j5rhhOY9//R17ph+3xhK0Z5lcsK2t5yU2pFNzW4zlG1u6lR/yvft5edVWAEbWeZ1Ll/57AS0d8fQOHpUoEjLu/PxRXcrqa8K0xRLMfWUN4P2cnvqz//LCyq1MG60JHFJ8CvpEpOLsaI8zcWRdzuNzpo/msYuOA6C1I0lbPNmtjoK+vnlrS+7JL48u8pYp2XvyqHTZ0vU7mP2tuRxxxQPpLvZKkOrK/syx3bdSa48leWzxRu7xM4FBu47vPv5UpNCKFvSZ2XVmts7MXg6UjTaze81skf+9yS83M/u5mS02sxfN7MDAOef69ReZ2bmB8oPM7CX/nJ+baW8lEfH+6Lb3YpHb4bVe1umNTTuyLiPSrO7dPtna6gXJP/3g/jz41WO6HHt1jbczx5Sm+nTZO3/xKACrtrSy+zfv5okludevG6j1ze1Mv+hOXlixpWivkZLqyq7NsrPGa2ubu3V1n7H/Thw1ayznH7lL0dsmUsxM3/XAKRllFwH3O+dmAff7zwFOBWb5XxcAvwYvSAQuAd4GHAJckgoU/TqfDJyX+VoiUoViCUfS0eMit/V+UPjbh5fS2pGle7dVmb6+SAV9M8cNZ8bYrl3rC1dvIxwy1mzLvRj2h373JOubi7NY9sHfuw+A03/12IDWCVyztY3pF93Jc29uzlknNZElW9AXlMpEf+74Wdx0/ts0aUhKomhBn3PuEWBTRvHpwA3+4xuAMwLlNzrPk0CjmU0CTgbudc5tcs5tBu4FTvGPjXTOPem8XPqNgWuJSBVLrbFXG8mf6avx/yhPaapnaZZdInrK9DnnWLGp+xi2apUK+kbWexnUn521f/rY8o0tjBtey5ydm7qdN3pYTfqz+O6dC4rezrMDCyb3ZNmGHZx81SPpYPTRxRsA+MHdr+bcWaQ9ljvou/DYmYA3S/exi47jni8dzcxx6taV0in1ki0TnHOpBYnWABP8x5OBFYF6K/2yfOUrs5RnZWYX4GUQmTZt2gCaLyKD3c/uXwTA2BE1Pdbdf2ojI+oiWbN6+cb0NbfF+PxfnuPB19Zzx+eO7DJWrVqlgr7UkiM7NdZ3OT68LsKFx+7KGftPJpZMcvxPHgbg3587kkkj69jl63excXtHwdu1cnPXwPyZ5Zv97c563uP2xieW89raZg7+3n1MaarnA3OmAvDk0k0c++OHePRrxxJLuC6ZzVSA2NjQ/efvf0/eg/89eY/0890mjOjXPYn0V9kmcvgZupIs3uScu8Y5N8c5N2fcuHGleEkRKZOaiNdNdsb+Of8fmFYbCdERT2bt3v3Vg0uyri/35sYW9rn0nvQiw6ndP6pdKnAeUecFfR0Zk2MWr9tOOGRMG9PQZWmXMf56ivtNGdVtXcVE0nHtf5dmXVKnNzZsb+cb//SGlafWDARYu613O67sNKozcF25uZUr7329y/Ejf/Agx/74oS5l37nDy1bOmqAMngw+pQ761vpds/jf1/nlq4CpgXpT/LJ85VOylItIldvaGmPmuGH0Zm5XbTRMWzyZ3kHhd+fM4Tcf9eaRbdjezoLV27qdsyRji61bntWvHoDbX3gL6NzmLvX2N/nB1g/et0+6bkNNhHDIOGLXMemMW20kTEfGLN77F67lu3cuZO9L5nL6Lx/tc5vmfPc+Hn7dC87nfeMEZvkzZDe39G68Zk/jQlNeXrWV6RfdyYlXPswTS70JKeq2lcGo1N27twPnAlf43/8VKP+smd2MN2ljq3NutZnNBb4fmLxxEnCxc26TmW0zs0OBp4BzgF+U8kZEZHDa1hpnZC93NXjEDwimjW6gJhLixNkTuhxfuLqZvXbK6LrNSP6lrlHNYolktzFuh+0yhl9++ABOmj0Rh+s2xnLBZScTCXUGVTWRULd1+4IZvhdWbmV9czvjRmRfdLsnkXCIH5y5L++9+nHmLd/Em5taePd+O+U9pzXLpI/Rw2o4etZYbnv+rXRZaibyonXefwg+ceQMomGtiCaDT9GCPjP7C3AMMNbMVuLNwr0C+JuZnQ+8AXzAr34X8A5gMdACfAzAD+6+Azzj17vMOZeaHPIZvBnC9cB//C8RqXJbW2OMGd7zeL6gu15aTSLZvSv3tudWsa01xsePnJEuW7xO3bmZUpNngsyMd+6bO6jKDAJrIiG2tHbtEo5lLG9y9u+f4u4vHt1je97c2MLi9c3URkJMbqzns8d5a+Y1+ePsvnvnQgDesfdEIjmCs454ku/f9Wq38s8euytnH7Yz33733iSd44Dv3NutzhkH9Dy0QKQcihb0Oec+lOPQ8VnqOuDCHNe5DrguS/k8YO+BtFFEKs/W1hi75NmNI6ipIcrmlljWgA+82ZqPLt7AaftOYoK/xMb37lqYPn7B0btwzSNLaW6LpceyVYs3Nu7gpife4OAZo9NrHl5z9kH9vl5NONRtHGBqBvW158zhEzfO49U1zT1Owmhui3H0jx5MPz9zzhTee6A3GqipoetntK65vduEk5RfPbg4a7kZRMMhRjV4weKVH9iP9niSg6c3set4TcyQwU35ZxGpKFtaOmjsZffulR/cP2v5T96/X5fnG7Z3Xz9uwWUnM3qYlzna59J7eGNj9iU8KtVP7nmdax9dxqdums9Ty7wOmP0Dexr3VY0/qSaeSHL5fxaycPW2dNB38PTR6Xp3vLg61yUAWJ2RdRwzrDPrOyrj56IlywSelNQs8JSLT/Vm3X7w4Kldyt974BQ+dMg0BXwyJCjoE5GK0R5PsK0tztjhvRv31ZAjY/S+g6Z0eb7gLW9Cx4srtwDw4bdNo6EmQiQw2/TtP3qox9friCf5zJ/m8/ra5l61bzALbp22ZP12IiFjXC/f92xG1UdZvrGFT//pWX778FJO/dl/aYslqAmHGNUQ5bn/dyIA3w9kWrPJDOSCS6eYGZe+a3b6eb6Fmveb4o3lvOqD+/GlE3bjU2+fyfIrTqOhptRD4UUKR0GfiFSMfS+9B4CG2t79Ya4JLKA7rCZ3l+HKza0AvPuXjwGdGaOeJgJkem1NM3e9tIaTrnqEt7a09uncwWLD9na+ePNzPLO8c1eKZet30NhQ06sZ07nM9Lvk7w3sS/vbR5amty1rGlbDAdMa2dLSQTJHd3x7PMEZv/I+o338tRP3m9LYpc55R8zgpvMPAfIHfU3Dathvyijec8AUvnDCrP7dlMggo6BPRCpGagus7b3cNzcY9B26y5ic9f75XNdlWVKL8Y4fWcen3t77PVNjyc4xaw+8ui5PzcHrUzfN57bn32LTjs6FlBes3sao+oFlwKI9bFsGcOZBU0g6WJajK/2mJ95IP77yA/ux/IrTmBhYEzAltQXf83n24m2PJXvc1UVkqFHQJyIVYe4ra9KPzztieq/OCf5R//mHDuhy7D2BGZhvbmphXWBB35mBiSJfOmE3dp8wAjPY2hLj+RVbus06Tdka2Plj6xDd2zfVxQ2w104j04+XrB/YmMYPzJnKuYftnLdOau27J5duzDqLOriW9qw8u13M8sffXf6f7rNzU55YupH2HJ+jyFCloE9EKsL1jy0H4FvvnN1twH4uwf1Rh2V0CX/vPXvzx/PfRo2/pMc2P3v4yaNmcNDOnRML6qJhzjhgMs7Bfpfdwxm/eoxv/POlrK8X3O5tqE78iCU6I6vgXrrfPG3PAV03Gg7x7dP35qmvH889Xzqa5Vec1q1OaiePb/zzZU648mGeWLIR5xzt8QSJpEuPM7zz80fmfa1RDVGmjq5nSlP2mbupPZVH9HKYgMhQoaBPRIa05rYYs75xV3onhHMPn97rc2vydCk21EQ4ctbYdPftD+/2skJzAjNJU+ozdm7427yV3erEEkm+cPPzABy6y2geeHVdl7FpiaTjhseXd1ugeLCpD0x++dqpe/Dbsw/i/CNncHYPWbremjCyLueetKllc1I+9LsneXzJRnb/5t3M/Ppd/Pie16mJhLovqJ3FrPEjGFHXGdS1xxNc/dBitrfH+eOTXjfx5e/dJ9fpIkOS/hsjIkPa1Q8tSWefRtRF0tuA9UZNL3ZNSK0Jd48/wSDbGnFnHzadS//t7bna2BBlS0uMZNJ12Uv2meWb0o9PnD2RJ5cuYEtrLL3sy2f+NJ+5r6ylPZ7ggqNn9voeSi21S8WUpnoaaiKcvNdETt5rYlFe66oP7semHZ3Z0Wzv/UeufarL852yjOHLJmRGYIgln//Lc8x9ZS0/vPs1AHYZN4ypoxv60WqRwUuZPhEZ0lJdcQAj+7hAcirTF8kTKGbO6q3Lkh0Mh4zD/IkgFxztZQbbMvaR3bDdm/hwx+eOZKy/Y8jzKzb737cw95W1/bqHUgqOa7zuvIOL/nrvOWAK5wd2QwnKtR3bpFHZu2wzhQySgUGANZmTNrJPEBYZ0hT0iciQ1hyYqRvsruuN1Ji+d++fe+mVzOxS07DsW7z9/rw5PPy/x6SXYrn56RW8tsZbj++5Nzez2F+bb/yIWg6c5o2FSy17csv8zu7g1liC1o4ED7y6FhcISnItU1JKqdnR00Y35OyCLbYjdvWC66cuPp7Xv3tquvzpb3ibPWUunpxLOGQkko7t7XFeWLGFf7/wVpfj+br+RYYqde+KyJAWXO8u13ZquUTCIeZ984S8Ez9O23cSF93aOTFj1vjhWes11ETYeUyEXcZ6xy+7YwEjaiN86cTduOyOBel6I+uj1EXDTBpVx0srt/Kfl1Zz05NvUB8N0xpLsK01zg1PLOeKLDNL37XfTlzx3n3Y0R5nw/YOZgdmz5ZCqmv3a6fsUdLXDbrm7Dls2tFBKGTUBDK040fUZZ38kUsoZCSd4yPXPsULWZZu+VZgEWeRSqGgT0SGrGTS8camFiY31rNqSyvxfmTDetq9Y0RdlFe/cwqf/fNzfPnE3XpcgPgjh05LB3nN7fEuAV80bOnM4eqtbaze2sajizd4571tGv95eQ1X3fd6zmv/+4W3umSkvnnannziqN6vEzhQrf5uF/U15cuCDauNdJlp/fEjZlAb7Xt7Qmbemn/rO5d+uedLR/Pnp97k7buP4/CZYwvSXpHBRPlrERmyNrV00BFPpndfiCeLs65aXTTMtefO6VVmLd+CvsHlTjKdOWdKlzFmAJ8+Zibfe8/e6eefP77rzhC/eXhJj+0ppNSCzINp3OG33jW7X5nHsHlDA7YFhgfsNmEEl757L47dfXwhmygyaCjoE5Eh6dk3N/NdP4u2q9/lGs8TVJXSbRceweEzO3f4+OWHvYWfv/3uvdJlnzmmc4buV07cjT0mjkwvN3PIjNH868Ij+Nope/DBOVM5Za+J/OajB/LlE3fj7i8exSvfPplPHjWj2z6zxbZwjbcHcb6Fj4eKUMjYsL293M0QKSl174rIkHPPK2u44Kb56een7TuJXz64mIY8++eW0v5TG/nOGXtz/E8eBrwMUuZ4s/89eXfeP2cqO49uSC/t8qmjd+H8I2cQDSwlEwmH+M3ZB6Wf7zHRyzYOr43S0pEgnkgS6cXSMwPV0hHnh3e/RkNNuNeLXw9moQHsEywyVCnTJyJDTjDgA9h9wgi+etJuXHtu8ZcR6a2Z44bzxMXH8ZUTd8s6+cPMmDF2WJe1/MysS8CXz9TR3tIkf/B3Iim21P7Du08c+lk+6Lpcy8HTm7jp/EPK2BqR0lCmT0SGlK0tXfesTQVOnz1uVo4zymfSqHo+d3xx2nX8HhMA+MNjy/jk0cWbzNEWS3Daz/+b7kq+5uw5RXutUrr12VXpx3//n8PL2BKR0lHQJyJDyip/iZZL3jWbSMg486DerctWaUY1RDlslzFFm7zy30XrOfv3T/uLGHtlkxvrcy6KLCKDn4I+ERlStrd7sy1njR/BkbOqe1mNhpowa5tjPVfsI+ccZ//+aaAz4AP4+jv2LPhriUjpKOgTkUHp/b95nPcdOIWzDpnWpXyjP+NyeB9336hEdTXhoszg/fu8lV2ez/vmCTQ11PRpX+Oh4jx/xrRINdBvTREZdNZta+OZ5Zt5Zvlm3r77uC77qf7d37Jswkh1M9ZHwyxdv4Nk0nWZEDJQL7+1FYAnLj6u13vZDjXnHrYzNzzxBpcGltERqXSavSsig0ZHPMnW1hh/eXpFuuzyuzq3I1uxqYUHXl3HjLHDKjYY6YuHX18PwAOvriv4tUfVRyv6Pf726Xv3ads2kUqgoE9EBo3dvvkf9vv2PV22Irv9hbdIJh2L123nqB8+CMAnS7j12GB2xXv3AeATN87jvgVrC3bdlo4EwwbJmociUjgK+kRkUOiId52Fetq+k9KPj/rhg1xw07zOY/tMQuC4PTq3C/vEjfNIJB1L12/nR3NfZcP2dra3x/PuOvGrBxfzxyffALzJGw+8upbfPbKUf8xfSUOtRv+IVBr9qxaRQWHl5pYuz39x1gF874y92f+ye9PLtAA89/9OZFTD0N8RohAsY1eJexes4X/++CwAv3qwc1/e2z97BPtOaexS9+6X1/Cjua8B8NBr67lvYddM4YcyJtCIyNCnoE9EBoU3NnpB32/PPogDpzURChkj6roHd03DakrdtEFt9qSRLFjt7Yn70qqt1ERC3bKm51z3NF85cTc+eujOtMeT3PTEG/zywcXp48GA770HTmbOzqP58NsU9IlUGgV9IjIoPLF0IwBzdm5izHBvZm7mEiG/+ehB3c6rdnd+/kh2dCTY+5K5XbJ7QVtaYvy/f73C//vXK13K95w0kt0nDCeedHz/vfswMkuQLSKVQ0GfiJTd2m1tXPPIUgBG58jk3fG5I9l78qhSNmtIMDOG10aY0lTPys1eN/hFp+7B7hNHcOzu4/npfa/z0/sWMXV0PSs2dXaT//B9+/Ku/XaiXhM2RKqGgj4RKbtbnvXW3nvvAZO7jVMDGDeiVgFfD+783FG85+rHWLphB/tOGcXhM73dSr54wm588YTdAHh1zTZaOhLsaI9z1Kxx5WyuiJSBgj4RKZpE0vGVvz3Pis2t/O1Th3Xrrn3k9fXMGDuMvz7jrcv3gzP37XaNBZedTChLIChdjWqIcs+Xjmbl5lamjx2Wtc4eE0eWuFUiMpgo6BORovn9o0u57fm3AJj59bs4YtcxXP+xQ9jRHqc2Euac657uUj8a7r6KVEONfk31ViQcyhnwiYjot6mIFFwi6Vjf3M4vHljcpfyxxRuZ9Y3/APDVk3brcuy7Z+xdsvaJiFQjBX0iUnA/uPvV9MSMXcYNY+n6Hd3q/Pgeb9eNxoYof//UYcyaMKKkbRQRqTYK+kSk4G564o3046s/ciCj6qPURcJsa4vx6ppmXly5Jb28yLXnzFHAJyJSAgr6RKSg3tzYQmsskX6++4QR6Rm5TcNq2HnMME6aPSEd9M2ZPros7RQRqTYK+kSkYL72jxf56zxvJu6158zh0Jljsi7BYma8Y5+JRELa/ltEpFQU9IlIQTy2eEM64HvPAZM5YfaEvPWv/oh21xARKSUFfSKS1+J124mEjImj6qiNhLpl7pJJx1/nreBb/3qZumiIv15wGPtNbSxPY0VEJCcFfRn+u2g9Z//+aSY31vPYRceVuzkiRfPmxhZeX9vcLSPXEU8SDhkvr9rKN297mZdWbU0fO2LXMVz1wf259dlV/H3eCjoSSbbsiNHcHmdyYz3/+PRhTBpVX+pbERGRXjDnXLnbUFJz5sxx8+bNy3l8n0vm0tweTz+/78tHs+v4Eby4cgvv/uVjNDZEOWhaE4vWbWefyaM4ctZYzjp4atZxSyKD0aYdHdzw+HJ+dv8iAP7xP4exdMMOWtrjPPPGZu5bsJb2eDJd/6Cdm5j/xuas1zpgWiOzJ43kkBmjOW2fSUSyLK4sIiL9Z2bznXNzCnItBX2ebW0x9r30nqznzBg7jGUbuq8zlvKLDx3Au/bbCYBYwvtjmW1ngZ4450gkHSEzQiEFkZXKOccfHlvOtrYYnzxqF5au38GUpnpCISMcMiIhoy4a7td1V25u5allm0gkk0RCIdY1txMOwdpt7bTHE2xtjfP0so2s3dae9RrDasJMGFXH6IYaDt1lDB88eCpTRzfQHk/w0Gvr+dRN82lsiPKpo2fyqaN30c+piEiRKegbgFxB3x8eW8a3/70AgH9deASn/+qxLsdH1kX45jtnc9LsCWza0cGEkXXURcPM+e69bG6JMf+bJ/DfRRv44l+fpzYSYsLIOnYdP5ydGuv4xJG7sFNjPa+vbebNTS3sP7WRnRo7u8CWrt/ODY8v518vvMWWlhjgZVCioRCvrW1meG2ESNhIJB2NDVEaaiLUR8OMGV7DkbuOZeroBmKJJJNG1RM2o2lYlBF10SK+i5Ul9W/AzHDOFSxrm0w6ks4RDhlmxsOvr+exxRv476INLFy9rcfzT5o9gbEjatneFscM2mNJNu5oJ5ZwxJNJ4gnv+vGkoyOeZGtrjOa2eNZrhUPGqPooo+qjNDVEOXXvSXz8yBn89ZkV3PLsSiaMrOXCY3dl+phhDKvVqA8RkcFCQV+AmZ0C/AwIA9c6567IVz9b0PfXZ97ka7e8BHhdXXOmj+aNjTt4a0sbH/rdk5x72M5c+u69sgYDf3tmBf93y4scuetYnly6kbgfmB06YwwvrdrKqi2tWdsxvDZCyKAtnqTD70o7fOYYDpjWyMOvr6c+GmZLS4wZY4cxvC5CIukIm7F+ezurNrcyvC7Cy6u2kszx8U1urOfg6U3sP7WRhaubaY0lmDCylhljh+Nw3Pj4G0wYVcceE0fQFksQSzgmjqxj+tgGks5RHw0zpamBiaPqGDu8FuccztEts5NMOjbu6GBrawej6msYO7xmQMGTc472eJIVm1pIOgiHIGTG8LoII+ui1IRDvcouxRNeEPT0sk3c+dJqwiHjkBmjGVEX5fU1zazZ1sZjizewZlsbzkEkZDQNq2Hzjg4Adh0/nBljh1EXDRNPOjZub2fTjg6a2+JEwkY0HPLbAq0dCToS3ufYEU8SS3hBWEeis4s0GjZiCUc0bOzUWM+xu4+nI5HkrS2tNNZHmdLUQNOwGhLJJOu2tXPto8sIh4ymhmg6CEs6x+TGeqLhENFwKJ0VDIWM2nCI2miYWeOHM31sAyPqogyvjbBTYz3hkBENG7WRvmcPRUSkvBT0+cwsDLwOnAisBJ4BPuScW5DrnF323Nd99de38vqaZt7a0kZLLM7Lq7YRDhm/+vCBnLL3xD634/L/LOS3D3tbTo2qj/LCJSelj819ZQ23PruShB8cfejgaWxri7FycysdiSQ14RAj6yK898Apfd4ofcWmFv794lvMnjSSjniSHR1x4gnHE0s2snJzK0vWb2fjjg5G1kUYURdl7bY24oEocVhNmFjCUV8TJmSw2c8yZjNhZC1bW2NMGlVPNGy8uamFaCjEjo54l8AzHDLqo2FaYwnGj6ilwX8NhyOe8ALHYN3aaIhYIklrR5K2WIKWjOtlioSMxoYo9TVhwmakqiaSXtd4LOGI+QFfLiGD0cNq2GunUewxcQQO2LC9nUjIC4weWbSeddvamTiqjua2OPU1IZoaapgwso4RtRESznuN9ljSC5BrwtRGwtSEQ0QjRk04TE0kRG3EC8ziSUd7LMHEUXV86JBpveq6XbetjXEjajVWVESkyino85nZYcClzrmT/ecXAzjnLs91Tu2kWW7SuT9lzLAaxgyvYfyIOuZMb+KCo3ehoaZ/3Vrt8QS7f/NuAD59zEy+dsoe/bpOoSWTjtXb2pg0so5QyIglkmzY3k4i6byu4IyMWVsswdL1O6iJhGjtSPDSqq08uXQj89/YzIE7N1EXCbG5JUY8mcSA6WOHMawmwrgRtTQ2RHlrSxvb22O0dCSojYRZ39xOayyeHt9YG/G+G97rxpJe4FQTCVEXDVEfjdBQE/aDKK+L3Pn30dwep7ktxva2OFtaY7S0e8Fh6hbMvKxXJBwiGjYaG2poaogycWQdB88YTWtHgtZYgpDB2OG1NDbUlOpjEBER6TcFfT4zOxM4xTn3Cf/52cDbnHOfzah3AXABwJRpOx+08PXFDC/wuKVNOzrYuL2dmeOGa3C7iIiIFEQhg76qWF/BOXeNc26Oc27OhHFjCx7wgdddOGvCCAV8IiIiMigN9aBvFTA18HyKXyYiIiIiAUM96HsGmGVmM8ysBjgLuL3MbRIREREZdIb0glzOubiZfRaYi7dky3XOuVfK3CwRERGRQWdIB30Azrm7gLvK3Q4RERGRwWyod++KiIiISC8o6BMRERGpAgr6RERERKqAgj4RERGRKqCgT0RERKQKKOgTERERqQIK+kRERESqgII+ERERkSqgoE9ERESkCijoExEREakCCvpEREREqoA558rdhpIys2bgtX6ePhbY0M9zRwFbh9i5ut/SvO5Azh3o+UPxnnW/vaf7Lc3rDvT8oXjPut/SnDsWGOOcG9HP87tyzlXVFzCvTOdeMwTP1f0O8jZX4z3rfnW/g+1+q/Gedb+lu9+B3HPml7p3S+ffQ/DcgdD9lubcQpxfjtfVZ1yacwdC91u684fiPet+S3NuQVVj9+4859ycUp87FOl+K1+13bPut7JV2/1C9d1zNd4vQKHuOVKIiwwx15Tp3KFI91v5qu2edb+VrdruF6rvnnW/A1B1mT4RERGRaqQxfSIiIiJVQEGfiIiISBWo6qDPzKaa2YNmtsDMXjGzL/jlo83sXjNb5H9v8sv3MLMnzKzdzL6a5XphM3vOzO4o9b30RiHv18yWm9lLZvZ8aqDpYFPg+200s3+Y2atmttDMDivHPfWkUPdsZrv7n23qa5uZfbFMt5VTgT/jL/nXeNnM/mJmdeW4p3wKfL9f8O/1lcH42ab0454/YmYv+r+fHjez/QLXOsXMXjOzxWZ2UbnuKZ8C3+91ZrbOzF4u1/30pFD3m+s6g1EB77nOzJ42sxf863y7xxcv1NovQ/ELmAQc6D8eAbwOzAZ+CFzkl18E/MB/PB44GPge8NUs1/sy8GfgjnLfW7HvF1gOjC33PZXwfm8APuE/rgEay31/xb7nwDXDwBpg53LfX7HuF5gMLAPq/ed/A84r9/0V8X73Bl4GGvAm9N0H7Fru+yvQPR8ONPmPTwWeCvwcLwF28f8NvwDMLvf9Fet+/edHAwcCL5f7vkrw+Wa9Trnvr8j3bMBw/3EUeAo4NN9rV3Wmzzm32jn3rP+4GViI98v/dLw/8vjfz/DrrHPOPQPEMq9lZlOA04Bri9/y/ink/Q4FhbpfMxuF98vz9369DufclhLcQp8V6TM+HljinHujWO3urwLfbwSoN7MIXjD0VnFb33cFvN898f5wtDjn4sDDwHuLfwd91497ftw5t9kvfxKY4j8+BFjsnFvqnOsAbvavMagU8H5xzj0CbCpNy/unUPeb5zqDTgHv2TnntvvlUf8r7+zcqg76gsxsOnAAXqQ8wTm32j+0BpjQi0v8FPg/IFmM9hVaAe7XAfeY2Xwzu6A4rSycAd7vDGA98Afzuu+vNbNhRWtsgRTgM045C/hLYVtXeAO5X+fcKuDHwJvAamCrc+6e4rV24Ab4+b4MHGVmY8ysAXgHMLVYbS2Uftzz+cB//MeTgRWBYysZpEFBygDvd8gp1P1mXGdQG+g9mzes7HlgHXCvcy7vPSvoA8xsOHAL8EXn3LbgMeflTfNGzmb2TmCdc25+8VpZOAO9X9+RzrkD8VLNF5rZ0YVvaWEU4H4jeF0kv3bOHQDswEu9D1oF+owxsxrg3cDfC97IAirAv+EmvP9lzwB2AoaZ2UeL1NwBG+j9OucWAj8A7gHuBp4HEkVpbIH09Z7N7Fi8P5BfK1kjC0j326kv95vvOoNNIe7ZOZdwzu2Pl/07xMz2zveaVR/0mVkU703/k3PuVr94rZlN8o9Pwoug8zkCeLeZLcfrMjjOzP5YpCYPSIHuN5UZwTm3DvgnXtfJoFOg+10JrAz8D+ofeEHgoFSoz9h3KvCsc25t4VtaGAW63xOAZc659c65GHAr3jiaQaeA/4Z/75w7yDl3NLAZb1zRoNTXezazffGG2pzunNvoF6+iazZzil826BTofoeMQt1vjusMSoX+jJ035OhB4JR8r1vVQZ+ZGd44rYXOuSsDh24HzvUfnwv8K991nHMXO+emOOem43WFPeCcG3RZgkLdr5kNM7MRqcfASXjdRYNKAT/fNcAKM9vdLzoeWFDg5hZEoe454EMM4q7dAt7vm8ChZtbgX/N4vHE2g0ohP18zG+9/n4Y3nu/PhW1tYfT1nv37uRU42zkXDGSfAWaZ2Qw/g32Wf41BpYD3OyQU6n7zXGfQKeA9jzOzRv9xPXAi8GreF3eDYCZLub6AI/HSpy/idW88jze2ZQxwP7AIb1bbaL/+RLyszzZgi/94ZMY1j2Hwzt4tyP3izX57wf96BfhGue+t2J8vsD8wz7/WbfgzqQbbV4HveRiwERhV7vsq0f1+G+8X5svATUBtue+vyPf7X7z/vLwAHF/ueyvgPV+Ll7lM1Z0XuNY78DKaS6ic31v57vcveGNUY/5nf365769Y95vrOuW+vyLf877Ac/51Xga+1dNraxs2ERERkSpQ1d27IiIiItVCQZ+IiIhIFVDQJyIiIlIFFPSJiIiIVAEFfSIiIiJVQEGfiEgOZpYws+fN7BUze8HMvmJmeX9vmtl0M/twqdooItJbCvpERHJrdc7t75zbC2/h01OBS3o4ZzqgoE9EBh2t0ycikoOZbXfODQ883wVvZ4exwM54izgP8w9/1jn3uJk9CewJLANuAH4OXIG3cHst8Cvn3G9LdhMiIj4FfSIiOWQGfX7ZFmB3oBlIOufazGwW8Bfn3BwzOwb4qnPunX79C4Dxzrnvmlkt8BjwfufcshLeiogIkXI3QERkiIoCvzSz/YEEsFuOeicB+5rZmf7zUcAsvEygiEjJKOgTEeklv3s3AazDG9u3FtgPb3x0W67TgM855+aWpJEiIjloIoeISC+Y2TjgN8AvnTcuZhSw2jmXBM4Gwn7VZmBE4NS5wKfNLOpfZzczG4aISIkp0yciklu9mT2P15Ubx5u4caV/7GrgFjM7B7gb2OGXvwgkzOwF4HrgZ3gzep81MwPWA2eUpvkiIp00kUNERESkCqh7V0RERKQKKOgTERERqQIK+kRERESqgII+ERERkSqgoE9ERESkCijoExEREakCCvpEREREqsD/Bw3NNKGVroKNAAAAAElFTkSuQmCC", 762 | "text/plain": [ 763 | "
" 764 | ] 765 | }, 766 | "metadata": { 767 | "needs_background": "light" 768 | }, 769 | "output_type": "display_data" 770 | } 771 | ], 772 | "source": [ 773 | "# Visualize closing prices\n", 774 | "import matplotlib.pyplot as plt\n", 775 | "\n", 776 | "btc_prices.plot(figsize=(10, 7))\n", 777 | "plt.ylabel('BTC Price')\n", 778 | "plt.title('Price of Bitcoin from Oct 2014 to Oct 2022', fontsize=16)\n", 779 | "plt.legend(fontsize=14);" 780 | ] 781 | }, 782 | { 783 | "cell_type": "code", 784 | "execution_count": 18, 785 | "id": "f51d9e86", 786 | "metadata": {}, 787 | "outputs": [ 788 | { 789 | "data": { 790 | "text/html": [ 791 | "
\n", 792 | "\n", 805 | "\n", 806 | " \n", 807 | " \n", 808 | " \n", 809 | " \n", 810 | " \n", 811 | " \n", 812 | " \n", 813 | " \n", 814 | " \n", 815 | " \n", 816 | " \n", 817 | " \n", 818 | " \n", 819 | " \n", 820 | " \n", 821 | " \n", 822 | " \n", 823 | " \n", 824 | " \n", 825 | " \n", 826 | " \n", 827 | " \n", 828 | " \n", 829 | " \n", 830 | " \n", 831 | " \n", 832 | " \n", 833 | " \n", 834 | " \n", 835 | " \n", 836 | " \n", 837 | " \n", 838 | "
Price
Date
2013-10-01140.300003
2013-10-02123.000000
2013-10-03130.990005
2013-10-04136.820007
2013-10-05136.699997
\n", 839 | "
" 840 | ], 841 | "text/plain": [ 842 | " Price\n", 843 | "Date \n", 844 | "2013-10-01 140.300003\n", 845 | "2013-10-02 123.000000\n", 846 | "2013-10-03 130.990005\n", 847 | "2013-10-04 136.820007\n", 848 | "2013-10-05 136.699997" 849 | ] 850 | }, 851 | "execution_count": 18, 852 | "metadata": {}, 853 | "output_type": "execute_result" 854 | } 855 | ], 856 | "source": [ 857 | "# Sort ascending order of prices\n", 858 | "btc_prices.sort_values('Date', inplace=True)\n", 859 | "btc_prices.head()" 860 | ] 861 | }, 862 | { 863 | "cell_type": "code", 864 | "execution_count": 19, 865 | "id": "ae9f158d", 866 | "metadata": {}, 867 | "outputs": [], 868 | "source": [ 869 | "timesteps = btc_prices.index.to_numpy()\n", 870 | "prices = btc_prices['Price'].to_numpy()" 871 | ] 872 | }, 873 | { 874 | "cell_type": "code", 875 | "execution_count": 20, 876 | "id": "c671c9ec", 877 | "metadata": {}, 878 | "outputs": [ 879 | { 880 | "data": { 881 | "text/plain": [ 882 | "(array(['2013-10-01T00:00:00.000000000', '2013-10-02T00:00:00.000000000',\n", 883 | " '2013-10-03T00:00:00.000000000', ...,\n", 884 | " '2022-12-31T00:00:00.000000000', '2023-01-01T00:00:00.000000000',\n", 885 | " '2023-01-02T00:00:00.000000000'], dtype='datetime64[ns]'),\n", 886 | " array([ 140.30000305, 123. , 130.99000549, ...,\n", 887 | " 16537.42773438, 16618.40625 , 16576.24804688]))" 888 | ] 889 | }, 890 | "execution_count": 20, 891 | "metadata": {}, 892 | "output_type": "execute_result" 893 | } 894 | ], 895 | "source": [ 896 | "timesteps, prices" 897 | ] 898 | }, 899 | { 900 | "cell_type": "code", 901 | "execution_count": 21, 902 | "id": "81af47f3", 903 | "metadata": {}, 904 | "outputs": [ 905 | { 906 | "data": { 907 | "text/plain": [ 908 | "((2704,), (677,), (2704,), (677,))" 909 | ] 910 | }, 911 | "execution_count": 21, 912 | "metadata": {}, 913 | "output_type": "execute_result" 914 | } 915 | ], 916 | "source": [ 917 | "# Create sequential splits at a specified point (80% train, 20% test)\n", 918 | "split_size = int(.8 * len(prices))\n", 919 | "\n", 920 | "X_train, y_train = timesteps[:split_size], prices[:split_size]\n", 921 | "X_test, y_test = timesteps[split_size:], prices[split_size:]\n", 922 | "\n", 923 | "X_train.shape, X_test.shape, y_train.shape, y_test.shape" 924 | ] 925 | }, 926 | { 927 | "cell_type": "code", 928 | "execution_count": 22, 929 | "id": "ee13c3f9", 930 | "metadata": {}, 931 | "outputs": [], 932 | "source": [ 933 | "# Create a helper plotting function\n", 934 | "def plot_time_series(timesteps, prices, format='.', start=0, end=None, label=None):\n", 935 | " plt.plot(timesteps[start:end], prices[start:end], format, label=label)\n", 936 | " plt.xlabel('Date')\n", 937 | " plt.ylabel('Price')\n", 938 | " if label:\n", 939 | " plt.legend(fontsize=14)\n", 940 | "\n", 941 | " # Display a grid for easier measurement readings\n", 942 | " plt.grid(True)" 943 | ] 944 | }, 945 | { 946 | "cell_type": "code", 947 | "execution_count": 23, 948 | "id": "6241d337", 949 | "metadata": { 950 | "scrolled": false 951 | }, 952 | "outputs": [ 953 | { 954 | "data": { 955 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEICAYAAACeSMncAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABMhUlEQVR4nO2deXxU1dn4v89kgbCHxbAkJKCAIAqSGKCKRlEUpEWLC26AG7Zqf/rW1r1u1b7W1qq0vu4UrYL6olZfBBHRFFBDICCyyyIh7AhhCVuWOb8/7p1kZjIzmQmzZJLn+/nMZ+597rnnPDOE+8w551nEGIOiKIqi1AdHrBVQFEVR4hc1IoqiKEq9USOiKIqi1Bs1IoqiKEq9USOiKIqi1Bs1IoqiKEq9iZgREZE+IvKd2+ugiNwtIu1FZK6IrLffU+32IiKTRWSDiHwvIoPc+ppgt18vIhPc5NkissK+Z7KISKQ+j6IoilIbiUaciIgkANuAwcAdwD5jzNMicj+Qaoy5T0RGAb8BRtntXjDGDBaR9sASIAcwQBGQbYwpFZFC4P8Bi4BZwGRjzOxAunTs2NFkZWVF5HMePnyYli1bRqRv1UF1iEcdYj2+6hAeHYqKin4yxnTyedEYE/EXMAL42j5eB3Sxj7sA6+zjV4Br3O5ZZ1+/BnjFTf6KLesCrHWTe7Tz98rOzjaR4quvvopY36qD6hCPOsR6fNUhPDoAS4yfZ2q09kTGAdPt4zRjzA77eCeQZh93A0rc7tlqywLJt/qQK4qiKFEiMdIDiEgy8AvgAe9rxhgjIhFfTxORScAkgLS0NPLz8yMyTllZWcT6Vh1Uh3jUIdbjqw5R0MHfFCVcL2AM8LnbuS5nRRDVQXVoSDrEenzVITw6EOPlrGuoWcoC+ARweVhNAD52k4+3vbSGAAeMtew1BxghIqm2J9cIYI597aCIDLG9ssa79aUoiqJEgYguZ4lIS+Ai4DY38dPA+yJyM1AMXGXLZ2F5Zm0AjgA3Ahhj9onIH4HFdrsnjDH77OPbgalACjDbfimKoihRIqJGxBhzGOjgJdsLDPfR1mC5//rqZwowxYd8CdA/LMoCTqeTrVu3cvjw4Xrd37ZtW9asWRMudVSHBq5DUlISJ510Em3atInoOIrSkIn4xno88dNPPyEi9OnTB4cj9JW+Q4cO0bp16whopjo0NB2MMRw9epRt27YBqCFRLEoKYfMCyBoGGbmx1iYqqBFxY//+/WRlZdXLgChNCxGhRYsWdOvWje3bt6sRUSwD8uYvoKocEpJhwidNwpDo09KNqqoqkpKSYq2GEkekpKRQUVERazWUhsDmBVB5HEyV9b55Qaw1igpqRLzQ9FtKKOjfi1JNSgfAaZ847fPGjxoRRVGUcHB0L4j9SBWHdd4EUCOi1GLixImMHj06pHvy8vK48847I6RRYO68807y8vJiMraiVJPSwTIe4oCEZtbmehNAN9bjmLqWUiZMmMDUqVND7veFF15wZQEImg8//DBu9pM2b95Mjx49WLx4MTk5ObFWR2kMlBTCZ/eD0wkicEqtKIZGixqROGbHjh3VxzNnzuTWW29l/fr1tGrVCrA2fd2pqKgI6kHftm3bkHVp3759yPcoSqNh8wLLKwunVbBi7aewYV6T8NDS5aw4pnPnztWvdu3aAVaCyc6dO3Ps2DHatWvH9OnTueCCC0hJSeGVV15h7969XHPNNaSnp5OSksJpp53GP//5T49+vZez8vLyuP3223nwwQfp2LEjJ510Er/73e9wOp0ebdyXs7KysnjyySe57bbbaNOmDenp6fzlL3/xGOeHH37gvPPOo3nz5vTp04dZs2bRqlWrgLOnqqoqfve735Gamkpqaip33303VVVVHm0+++wzLr74YlJTU2nfvj0XX3yxR+Bhjx49ADjrrLMQkeqlsMWLFzNixAg6duxImzZtOOecc/j222/r/odQlKxhllsvrtUB02Q8tNSIRICi4lJe/GoDRcWlsVaFBx54gNtvv53Vq1dz2WWXcezYMQYNGsTMmTNZtWoVd911F7fddhvz5s0L2M8777xDYmIi33zzDf/4xz94/vnnee+99wLe89xzz3H66aezdOlS7rvvPu69997qh7LT6eTyyy8nMTGRgoICpk6dyuOPP87x48cD9vnss8/y2muv8corr/Dtt99SVVXFO++849Hm8OHD3H777RQWFpKfn0/btm35+c9/Tnl5OQCFhYWAZWx27NjBhx9+CFgBijfccAMLFiygsLCQgQMHMmrUKPbubRobpMoJkJFrzToyh7oJnXDsYMxUiha6nBVmiopLue71AsornSQnOnjnliFkZ6bGTJ/f/OY3XHHFFR6y3//+99XHkyZN4ssvv2T69OkMH+5/Hbdfv3488cQTAPTu3ZvXXnuNefPmcc011/i9Z8SIEdWzk9/85jdMnjyZefPmMXToUObOncu6dev4/PPP6dbNKgPz3HPPcfbZZwf8PM8//zz33nsvV11lpVx74YUXmDNnjkebsWPHekSs//Of/6RNmzYUFhZyzjnn0KmTVaCtQ4cOdO7cufq+Cy64wKOfv//973zwwQfMnj2b66+/PqBeigLAvs2e5zoTUUKlYNNeyiudOA1UVDop2BTbX7HeG8dVVVU89dRTnHHGGXTo0IFWrVrx4YcfsmXLloD9nHHGGR7nXbt2Zffu3fW+Z+3atXTt2rXagIC1vBQoW8CBAwfYsWMHQ4fW/NpzOBwMHjzYo93GjRu56aabOPnkk2nTpg1paWk4nc46P+Pu3bu57bbb6N27N23btqV169bs3r27zvsUpTpa/dB2T3lis9joE0V0JhJmhvTsQHKig4pKJ0mJDob0jG3AkXdN5b/+9a88++yzvPDCC5x++um0atWKBx98sE6D4L0hLyIeeyLhuiccjB49mi5duvDKK6/QrVs3EhMT6devX/Vylj8mTJjArl27eO6558jKyqJZs2YMHz68zvuURsCJ5ryq3lj3olOfE9etgaNGJMxkZ6byzi1DKNi0lyE9O8R0KcsXCxcu5Oc//zk33HADYCUS/OGHH6o35qPFqaeeyvbt29m+fTtdu3YFYMmSJQGNTNu2benSpQsFBQXVS0/GGAoLC+nSpQsAe/fuZe3atfz1r3/lwgsvBGDp0qVUVlZW95OcnAxQa0N+4cKFTJ48mUsvvRSAXbt2eXjAKY2UkkKYeilUVUBCEkz8NHRDkjUMHAng9TdF54FhU7OhostZESA7M5U7zj+lwRkQsPYz5s2bx8KFC1m7di133nknP/74Y9T1uOiii+jTpw8TJkxg+fLlFBQU8Nvf/pbExMSA8S933XUXzzzzDDNmzGDdunXcfffdHg/61NRUOnbsyNSpU9mwYQP/+c9/+NWvfkViYs3vpZNOOomUlBTmzJnDrl27OHDgAGB9N2+//TarV69m8eLFjBs3rtrgKI2Y5dPsWYSx3pdPq18/Ti8DgjSJqHU1Ik2Mhx9+mNzcXEaOHMm5555Ly5Ytue6666Kuh8Ph4KOPPuL48ePk5uYyYcIEHnroIUSE5s2b+73vnnvu4cYbb+SWW25h8ODBOJ1OD/0dDgfvvfceq1aton///txxxx388Y9/pFmzmrXpxMREJk+ezOuvv07Xrl0ZM2YMAFOmTKGsrIzs7GzGjRvHTTfdRFZWVsS+A6WBULYn8HkwbF5gJV50R2gSUeu6nNVIuOKKKzDGcOjQIcCK0/AVdZ6amlrt0uoP7ziN/Pz8kNts3ry51j3ebXr37s38+fOrz5cvX05FRQWnnHKKX90SExN57rnneO655/y2ueCCC1i0aJFHPZGysjKPNrfccgu33HKLh2zAgAEsWrTIQ+Za9lMaMa06BT4PBl/JFrv/rNEHGoIaESWGfPTRR7Rs2ZJevXqxefNmfvvb3zJgwAAGDRoUa9WUpsSAa2HZOzV7IgOuDe3+kkL49LdeQgdc+Fi4NGzQqBFRYsahQ4e47777KCkpITU1lby8PJ577jlNr65El4xcGPkXWPMx9B0T+uxh+fTaS1lE3guxoaBGRIkZ48ePZ/z48bFWQ2mquNx6UzpYyROryqH4W0jrF5ohKdvlW/7Fo3Dj7PDo2oBRI6IoStPDvZStCBin9aoqtwxLKEakVZpv+Z4fwqNrAyei3lki0k5EZojIWhFZIyJDRaS9iMwVkfX2e6rdVkRksohsEJHvRWSQWz8T7PbrRWSCmzxbRFbY90wWXQdRFCUYXMGBpspO3+4ASbCSKIbqUdV5gG958zYnrmccEGkX3xeAz4wxpwIDgDXA/cA8Y0wvYJ59DjAS6GW/JgEvAYhIe+BRYDCQCzzqMjx2m1vd7rskwp9HUZTGgCvrriRAQiJkDIauA+GSp0PfE/EXC1K62ZrxNHIiZkREpC1wLvAGgDGm3BizHxgDvGk3exO4zD4eA7xlLAqAdiLSBbgYmGuM2WeMKQXmApfY19oYYwqM5cv6lltfiqIo/nFl3c0eD1WVUPw1bCuC2feG/uDPGkZNCng3jNPadG/kRHJPpAewB/iniAwAioC7gDRjjCvEeCfgWlDsBpS43b/VlgWSb/Uhr4WITMKa3ZCWluYz7gGstBquOIv6UFVVdUL3hwPVIfo6HDt2zOffVFlZmd+/tWgRax1iPX5dOpz242o64qypAlJ1nB+/fIstmUdCGqNHxuV0L6mJvxKs2lQ//biKVfn5Df57OBEiaUQSgUHAb4wxi0TkBWqWrgAwxhgRCa0Oaz0wxrwKvAqQk5Nj/NXjXrNmjUeAWqi4px+PFapD9HVo3rw5Z555Zi15fn5+zGu/x1qHWI8fUIeSQlha4iESEXpeMJ6eoS5p5eXBkvNg/p/hoJXJV4BOnTqRl5fXsL+HEySSeyJbga3GGFcI8Awso7LLXorCfnelj90GZLjdn27LAsnTfciVGDN58mRNF6I0bFxJFw96pW7vMsjadK/PXkbORLjyTXC4Za/+YU6j3xeJmBExxuwESkTElQt5OLAa+ARweVhNAD62jz8BxtteWkOAA/ay1xxghIik2hvqI4A59rWDIjLE9soa79ZXk0BEar3atGlTfTxx4sR69/3YY4/Rv3//8ClbByLCjBkzojae0sSpTrroxfalMO+PloEJ9eHvijvJOKtG5qxo9PsikY4T+Q3wjogkA5uAG7EM1/sicjNQDFxlt50FjAI2AEfsthhj9onIH4HFdrsnjDH77OPbgalACjDbfjUZ3LPXzpw5k1tvvZX169fTqlUrAFJSUmKlmqI0cPxFA9ir665svsEua7nHndTKWRfxFfuYElEXX2PMd8aYHGPMGcaYy4wxpcaYvcaY4caYXsaYC10GwfbKusMYc7Ix5nRjzBK3fqYYY06xX/90ky8xxvS377nT+Mo42Ijp3Llz9ctVDyQtLa1aNn/+fLKzs2nevDk9evTgoYce8iiw9OGHH3LGGWeQkpJC+/btOe+889i1a1d1vfNVq1ZVz2q8Ey6688wzz9C5c2datWrF+PHjayU7XLx4MSNGjKBjx460adOGc845p7rWOlC99HXllVciItXnGzduZMyYMXTu3JmWLVtW14ZXlBOmWTAxHCGEnbnHnXikPHGEnosrztBU8JGgpBAWPBvTtdA5c+Zw3XXXceedd7Jq1SqmTJnCjBkzePDBBwHYuXMn48aNY8KECaxZs4b58+dXZ6y9+uqrueeee+jTpw87duxgx44dXH311T7Hef/993n44Yd5/PHHWbp0KX369OHFF1/0aHPo0CFuuOEGFixYQGFhIQMHDmTUqFHs3Wv51y9ebE0yX3vtNXbs2FF9XlZWxsiRI5k7dy7Lly9n7Nix/PKXv2Tt2rUR+c6UpkFRcSl7l30ceH4gCTDgmuA73b3WcumthRN2rQ5Rw/hC056EG/dpbUKy5Yseg3TQTz31FL///e+58cYbATj55JP585//zPXXX89f/vIXtm/fTkVFBVdccQWZmZkAHnsgrVq1IjExkc6dOwcc5/nnn2fChAncdtttADz00EN88cUXHoWuXFUIXfz973/ngw8+YPbs2Vx//fV06mSl3m7Xrp3HeAMGDGDAgJpo4Iceeoj/+7//Y8aMGTz88MP1+VqUJk5RcSnXvV5APvusn9B+V7W8EyoGYO6jsOJ9/9cL/gf6PxOKmnGFzkTCjfu01pWHJwYUFRXx1FNP0apVq+rXtddey+HDh9m5cycDBgzgwgsvpH///owdO5aXXnqJPXtCL8azZs0ahg4d6iE766yzPM53797NbbfdRu/evWnbti2tW7dm9+7dbNmyJWDfhw8f5t5776Vfv36kpqbSqlUrlixZUud9iuKPgk17Ka90Uk4QFSuD/b+75pPA14/tD66fOEVnIuHGlU7BNROJUWUzp9PJo48+ypVXXlnrWqdOnUhISODzzz+noKCAzz//nDfeeIMHHniA//znPx6//sPBhAkT2LVrF8899xxZWVk0a9aM4cOHe+zP+OJ3v/sdn332GX/961/p1asXLVq0YPz48XXepyj+GNKzA8mJDtaYTDKo40eTr0JTvuj7C/j6ef/Xm7cLVr24RI1IuHGlU9i8wDIgMapsNmjQINauXRuwSqCIMHToUIYOHcojjzzCaaedxnvvvceAAQNITk6mqqruKX3fvn0pKCjgpptuqpYtWbLEo83ChQuZPHkyl156KQC7du3y8CwDSEpKqjXewoULGT9+PGPHjgWsyPCNGzfSu3fvOvVSFF9kZ6YycWgW3393FheVL0UC1f3YuTy4Ti963Io32TAXupwJm/Lx2FzvEt4fZQ0NNSKRICM35mUxH3nkEUaPHk1mZiZXXXUViYmJrFy5ksLCQp555hkKCgr44osvuPjii0lLS2PZsmWUlJTQr18/wPKYKi4uZunSpXTv3p3WrVt71Cl3cddddzF+/HjOOuss8vLymDFjBkuWLKF9+/bVbXr37s3bb7/N4MGDq5eokpM9lxOysrKYN28e5513Hs2aNSM1NZXevXvz0UcfMWbMGJKSknj88cc5duxYZL84pVEzbdEWChd8xjvJb+DEiUgAHyx/dUK8KSmENf9nrT5s+RY69YY9bs4fR36CICc18YjuiTRSLr74Yj799FO++uorcnNzyc3N5emnn6Z79+6AlSfs66+/ZvTo0fTq1Yt77rmHP/zhD1x//fUAjB07llGjRjF8+HA6derE9Om+A6auvvpqHnvsMR566CHOPPNMVqxYwR133OHRZsqUKZSVlZGdnc24ceO46aabakW0P/vss3z11VdkZGRUpxD529/+xkknncSwYcMYOXIkQ4YMYdiw2CwPKo2D2St3MMSxhiQqSKzLg9dfnRBvlk+HymM1+6AdTva83ndMvXSNF6SJhVaQk5NjvJdbXKxZs4a+ffvWu++mljNKdbDw93fTmPMlxcv43jpMW7SFGf/+gPeTnyDBnon4JKEZTJxZ94qCK32KK/rdkQQ3zrLcel3ldnMmNrjvIVREpMgYk+Prmi5nKYrSZLh2cHc6lp6MKahjGjLk18EtSW9eYKWSd+G09/XS+ll1RtL61V/ZOEGNiKIoTYoRLdfX5Gr3x87vg+ssa5hdXtclcFqeWhu+9IwVa8TonoiiKE2Ktc0HUCGJgSPWg93HyMiFPiM9ZYd2NohYsWihRkRRlCZDUXEpl31SwbhjD7DZ2dmPIZHQlqE6eLnRZw0Dh9siT7DxJnGKGhFFUZoMroj1ImdvXqu61E8rEzh40J2SQvj2H24CgeMHatKmmCqYfS9tDjTefG9qRLxoat5qyomhfy/xhStiPUHgQ8dFFP/sT9Atu3bDQzuD63DzAnC6BRY6EgCp2WAHqCqn3f6VJ6R3Q0Y31t1ISEigoqKiViCcovjj6NGjJCUl1d1QaRBkZ6byzi1DKNi0lyE9O5CVORK4w0qi6D77OHN83Z2VFMKBEkhIgqoKcDhg1LPWUtiyt2vcfhOS2d8uegXeoo0aETfatWvHrl276NatGw6HTtIU/xhjOHr0KNu2bSMtLcigNKVBkJ2ZSrZjPWz+GPZ0sFxxT70UUnt4xHYExD1btyPBaj/gmhq34ImfWkWtEBhwDQc3Honwp4odakTc6NixI1u3bmXdunX1uv/YsWM0b948zFqpDg1Vh6SkJNLS0mjTJpgCR0pDYe3iLzh59rUkOssRDIjDCi6c8EndxsPF5gVQddyqIeI00DbdM67EO/XRxvxwfoQGhRoRNxwOR3VakPqQn59fnbIjVqgODUcHpeFRVFxK/ifvc5ejAhGDAcQ4ofI45P835D0QXJBhSoeaIlTGCduWWrOTGOfMiwW6ZqMoSpPhg6Vb+bqyLxUkUmnco9adsPErK4VJMBVJj+7F4/G59lNreSuG1UxjhRoRRVGaDAIsNb15vOIGVjh74CTB7aqx9jiWT6u7o6xhkNiMmhzAxndg4ZKp8NoFnLbyT43WwETUiIjIZhFZISLficgSW9ZeROaKyHr7PdWWi4hMFpENIvK9iAxy62eC3X69iExwk2fb/W+w760rL6eiKE2YXw5KJzdxA48nvckAxyYc+KqZE8RjxFU3KGeitZ8iCbWL0L11Ocy8C7YV0fGnRTDlkkZpSKIxEznfGDPQLQPk/cA8Y0wvYJ59DjAS6GW/JgEvgWV0gEeBwUAu8KjL8NhtbnW775LIfxxFUeKV7MxU/nHaOpKlEoevWiKSAJ2DLCKVkQujn7ey/V7wkGVUXHsicx+FTV/WdAtW4OHXL5z4h2hgxGI5awzwpn38JnCZm/wtY1EAtBORLsDFwFxjzD5jTCkwF7jEvtbGGFNgrIivt9z6UhRF8clJrZM9jUe3bDj7bitViXHCrHusZahALJkK/7rces/IhWH3eG6qf/++7/sO7fAtj2MibUQM8LmIFInIJFuWZoxxfZM7AZeTfTegxO3erbYskHyrD7miKIpf1qaNphI7AaMjCS55Gpq3sSPPDTgr4dP/8r/0NPdRa5lq45fWuy+D0z7L47Q6r0EwQYxxRqRdfM8xxmwTkZOAuSLikUDGGGNEJOJ5I2wDNgkgLS2N/Pz8iIxTVlYWsb5VB9UhHnWI9fjeOuRvqWDl2pW8kwSCYJyG75cuBVoyEINgZ4k3Tn76+GFW9X/Qo682B9YycNkLNe2AQ//5Bz+tWsL+dv052PZUALo0G0hvvqm+71DzbuzoPoYdZVkQo+8jUv8WETUixpht9vtuEfkIa09jl4h0McbssJekdtvNtwEZbren27JtQJ6XPN+Wp/to70uPV4FXwapsGKkKY/FevUx1UB0a2/juOhQVl/L2598ySdaQQCUJAk6qGNT+sLUc9eP/wP5iwDIQnZpV1tb93ddwL0QiQJuyTbQp21RTOyQj127nhiOJPtf+N30i+UHrIFL/FhFbzhKRliLS2nUMjABWAp8ALg+rCcDH9vEnwHjbS2sIcMBe9poDjBCRVHtDfQQwx752UESG2F5Z4936UhRF8aBg014qnYZWHCYBMAYrYv3YQWvp6qDXb1BfS08/ra8tM1W1a4d47X20PrK57n2WOCWSM5E04CPb6zYRmGaM+UxEFgPvi8jNQDFwld1+FjAK2AAcAW4EMMbsE5E/Aovtdk8YY/bZx7cDU4EUYLb9UhRFqUVqCyux6oWOZYBbQcIfPrP2RNwzMp96qe8UKB17wU9+0iKJo8bF98zxsK3I83rB/wSfViWOiJgRMcZsAmr5yhlj9gLDfcgNcIefvqYAU3zIlwCNNz2moihho/SIlVXXeJeiMsZ6+Cck15S0Pftu352cfResnen7mrMCdq22lrNyJkL+n6BsV831yqMn/BkaIhqxrihKk+DQ0QoA/llllbM12PEbQ26vCR70jvfwZtfqwIMse6vmeMA1ntcObvft8VVSCAuejdtARE3AqChKk+DbTXsBeNc5HCrgqpbLGHTxhJolJu/Mu75YU8e2a+su1rtXxUMBy3V4+XTPMUoK4Z+jrFmMIwlunBV3SRx1JqIoSpOgWaL1uBskP9Beyvig9fW19yjqmhW06BhgBIe13AVW/i1nZe0me7zK5H79gmVAwHqPw4h2nYkoitIk6JXWmsriRUxPfpJEqnDu+xBKTq/55V9SaGXxraqwqhVO/LT2rGDfRv8DdDuzpn3ZHt9tKo97nu/d4Hm+c0XwH6iBoDMRRVGaBK2bJfLLhAUkU0mCGBKp8MzYu3yaXdI2QDbfxGb+B3B3CW51ksel6q389id73pPgVVp5/5a42xtRI6IoSpNg1Y6DdOSAp9BjxuCdjtHrvKTQ/wO+fU/PpbEB13jcX33kPZNJSPbqyNROJ9/AUSOiKEqTYGT/LvxEW0+h+4xhwDVWWnfEevf2rtq8wPc+B0BKqud5Ri7k3Fi7XaJXyWb31PFgZRH2ljVw1IgoitIkuHZwd046ZyKVkmS593qnfc/ItdK6D/+D9e69H3LsoP/OfUW3D7jGMgrudPJKfHLcu8+IpxIMO2pEFEVpMnToO4wFve7FSCJg4LP7PZeofKV1d7Hze/8dl/5YW5aRC5f+DSTBMg2+Zje+Ah91OUtRFKXhUVRcynWvF1C0ZgNOp9OqHeKrpK0/Op/h/9o3L/jeL8mZCDd9xo89bqiZ3bhqkcx91G7k9hj2ro4YB6iLr6IoTYKCTXspr3TyLX25MyERB1U4QnloHz/g/5prBuFrBpORy5bMI/R0GZCZdizJxi9rtx3y67gLNlQjoihKk2BIzw4kJzpYXtmbG50P82zuIboNHBHCQztA7XVJgJQOfi+3ObAWFhT5r3joIs6WskCNiKIoTYTszFTeuWUIBZv2MqTnz+iWmeq7YUmh9TDPGuZpYLxrr/e8AKqOwZZvraWxz+6HtH61jVJJIQO/exhMJXVunAeKQ2mgqBFRFKXJkJ2ZSrY/4wGBo9aP7sXav3Ba7+0zoeitmhTyVcd9L2ktn46YiuAULFls6RBHS1q6sa4oiuIiUNR61jA7wlys97I9VjGqasT3/srWxbVl/nBWWEka4wg1IoqiKNUEiFrftdot2NDUTgvfLtP3DGJ/SWgquNcgiQPUiCiKorjwF7VeUgiz7rFnHsYyJoe9kizWChy0adYmRCUCbOA3QNSIKIqiuPAXtb55ATjdlq7EAX1Get57tNR3rMiwe0JUIr6i1nVjXVEUxR1fxalSOuDxcB96J1z0OOxZU5O+3VRZeyje9xZ/Hdr4R/eHqnFM0ZmIoihKXRzda80+AJCawMP0s7wa+liK2jA3tLGKv7aCEuOEiBsREUkQkWUiMtM+7yEii0Rkg4i8JyLJtryZfb7Bvp7l1scDtnydiFzsJr/Elm0Qkfsj/VkURYk/NpRW8eJXGygqLg3uBld1wyVTa6ocZg0Dh2vhxsCyaZbce7/DO5YE4JSLfI+TECAmpK4yvA2IaCxn3QWsAVzf9p+B54wx74rIy8DNwEv2e6kx5hQRGWe3u1pE+gHjgNOArsAXItLb7utF4CJgK7BYRD4xxni5TCiK0lQpKi7lmcXHqDTrSE508M4tQ+qOE3nzF3YFQqc1+0hoBhM+gTOvhyX/pHpjffk0WPovt5vFjiXxYuxr7N36Ax1Lv/OUVx2v3dZFwDK8DYuIzkREJB24FHjdPhfgAmCG3eRN4DL7eIx9jn19uN1+DPCuMea4MeZHYAOQa782GGM2GWPKgXfttoqiKICdL8sJTgPlFU4KNvl4yLuzeYEdJ+K0zo3TMiibF8DxMruR2MWkBJzOmnsd/muBHGx3eu208IHYUhB82xgT6eWs54F7qf4XoQOw3xjjcrbeCnSzj7sBJQD29QN2+2q51z3+5IqiKACktqipHOj0OvdJ1rAaA+F+548LYcX7WJvrBroPtdx/E5sBDmupa9SzfiPN97frb/fr45Hr8KHT0X2B9WxARGw5S0RGA7uNMUUikhepcYLUZRIwCSAtLY38/PyIjFNWVhaxvlUH1SEedYj1+Es32tHntlGYsXAVS1eu5dT2CZyS6ntm0KXnjWRufo9m5XsRwCBUbFmMHatu+Wht+pJlbS+G0x+j3f6V7G/Xn4NlWeDns5YlpFe3PWnXV7Q8srWmL2c5uPcNUF7GD9MeYEfXi332Vx8i9W8RyT2Rs4FfiMgooDnWnsgLQDsRSbRnG+nANrv9NiAD2CoiiUBbYK+b3IX7Pf7kHhhjXgVeBcjJyTF5eXkn/OF8kZ+fT6T6Vh1Uh3jUIdbjb0/Zwoz1K6rPl+5xsmyPk6SESqZPGlp7f6SkEBa8bi9pWYgjkeTuZ8EmK3W7a44yKGEdjH4uKD3y8/MZlPcr62Tuo/D18x59VY/ldtzn8CL65P13UP0Hq0Mk/i0itpxljHnAGJNujMnC2hj/0hhzHfAVcIXdbALgckP4xD7Hvv6lMcbY8nG291YPoBdQCCwGetneXsn2GJ9E6vMoihJ/lB4pryUzQHmV4YOlW2vfsHmBlXzR4wYntM/y0Xs9ggKXTK07HbyL1p1D7z8GxCLY8D7gXRF5ElgGvGHL3wD+JSIbgH1YRgFjzCoReR9YDVQCdxhjZT0TkTuBOUACMMUYsyqqn0RRlAbNkJ4dSACqfFzzmVzEO6gQLCOCsfY0XDMUSYAB14amjHtBqmBIahla/zEiKkbEGJMP5NvHm7A8q7zbHAOu9HP/U8BTPuSzgFlhVFVRlEaGuLK3u5GUIPxyUHrtxhs+99GDgc4DYeRA+PS3llFx1OPRueytwNeTW0F5Wc158cLQx4gBGrGuKEqjpWDTXqq8DIgAV+Zk+I4XObTTd0c7v3OLAbHjREKtQhiw4JRA+x6eotSs0PqPEWpEFEVptAzp2YEEr3UrA/Tv2tb3DX7rrYu11CViByCGUJvdRUr7wNebtfY8T4+PwlRqRBRFadR4b38LvjfcAf/p3I+XWeVvnXYU+yVPh159cHeAZBoJyXaUvBs7vw+t/xihRkRRlEbLh0u3UuW9T06goEM/HlfbltREshvjO71JALpsnwP7Nvm+2C7TSjt/5nhPed/4SMARlBERkd4iMk9EVtrnZ4jIw5FVTVEU5cTw54TrdybSeaBvebcca7YgCfVayuq2NUD0wSkXWLOatH7gSLJkjiTrPA4IdibyGvAAUAFgjPke2wVXURSloTJ2UHotV94EhzCkZwffN/ibYTRraSVhvOAh6z3EpSyHz2SLdg4ul6uwe+ErZ1XoG/cxIlg/tRbGmEIrH2I1lf4aK4qiNASyM1Pp0VrYdKhmTnJ61zb+M/m6Ur47vR5vRW/5n6UEweFWPWlx3K2cbufT4bTLrfFcBimlAzW+yE445md/poER7EzkJxE5GXt2KCJXADsippWiKEoYKCou9TAgAN9tPeC/tkhGLnTqW1tuquDT/4Ivn7JSxfsqgxuAku6/rMniKwlw6d+ssrnuM5qje/EIgfz2HyGPEwuCNSJ3AK8Ap4rINuBu4NeRUkpRFCUc+Ev9HjAl/BE/14zTMiZV5fVbanIkAGK/+yBrmOc144yLJa2gjIhds+NCoBNwqjHmHGPM5ohqpiiKcoIM6dmBRK9NkeREh/89EYCscwJ3Wo+N9Xb7V9r7Hcb/fkdGrpVO3pFYUwwr1FiUGBDUnoiI/Al4xhiz3z5PBe4xxqiHlqIoDZbszFSu75vMsoMp7D9SwfHKKi4b2M3/nkhJIaz+t/8OW3SEa6aHvLFeXU+kqjywEcqZaL2v+dhy8Q01FiUGBLuxPtIY86DrxBhTaqd4VyOiKEqDpai4lLfXllPprHHpfXn+Jrp3aMm1g7vXvsFXFl93ju6DXatDfrgfbHuq5dW1eYHnZro3JYVWUGNVORR/a7n5NnBDEuyeSIKIVCd+EZEUIFAiGEVRlJjz4dKtVDpry2ev9OMXlDUMEpL8d2icVhLG+mx4Z+TW3kz3xlWe90T2XqJMsEbkHWCeiNwsIjcDc6mph64oitIg8Rds2KGln4j1jFwY+RcCPhpNFSyffqKq+cZVnreeQY2xIKjlLGPMn0Xke2C4LfqjMWZO5NRSFEU5ccYOSufdRVu8M8Hz40+H/d90dC91F5yqR0GqYMjIDW7ZqwERdFJ8Y8xsYHYEdVEURYkKaW2a+7/oWtKq8pMaBQm9IFUoZOTGhfFwEXA5S0QW2u+HROSg2+uQiMRHOKWiKE2WD5durTULAcjrc5L/mzJyYeKncOqlvq+3zYirh3ykCWhEjDHn2O+tjTFt3F6tjTFtoqOioihK6BQVl/Le4i0+r/lNwOgiIxda+TE0h7ZHJ5J8yVT41+XWewOmzuUsEUkAVhljTo2CPoqiKGHBV1VDsBKLBAw2dFG2x7fcVdUwUrORkkL4+nlY+6l1vvFL690VQ9LAqNM7yxhTBawTER9O1YqiKA2TIT07kORd1hAYM7Cr/2BDD/xtnkvkvKZKCmHq6BoD4qKu+uwxJFgX31RglV1T5BPXK9ANItJcRApFZLmIrBKRx215DxFZJCIbROQ9EUm25c3s8w329Sy3vh6w5etE5GI3+SW2bIOI3B/yp1cUpdGSnZnK9ElDOT89gcz2LWjVLIHLBnbl+XFnBtdBqzTf8kjuiSyfDr7SxicGcASIMcF6Z/2hHn0fBy4wxpSJSBKwUERmA78FnjPGvCsiLwM3Ay/Z76XGmFNEZBzwZ+BqEemHVbvkNKAr8IWI9LbHeBG4CNgKLBaRT4wxAWpQKorSlMjOTOXsbkl8u/QYxyuczPx+B7k9OviOVvdmwDWw7J3aD/X+v4yMsoDf2U+nPhEc88SoyzuruYjcDVwJnAp8bYz5j+sV6F5jUWafJtkvA1wAzLDlbwKX2cdjqAlgnAEMF6uAyRjgXWPMcWPMj8AGINd+bbCTQ5YD79ptFUVRqlm7r4rjFU4MUOk0PPLxSv+p4N3JyLXK1p58ATUp2h3QPII+Rf5qljRruH5MdS1nvQnkACuAkcCzoXQuIgki8h2wGyvKfSOw3xjjqviyFehmH3cDSgDs6weADu5yr3v8yRVFUao5tX0CCY6avRGnMYFTwbuTkQt5D9ipUAQSEiMbRb5hrm/5N39vsLVF6lrO6meMOR1ARN4AQvoU9qb8QBFpB3yENZuJOiIyCZgEkJaWRn5+fkTGKSsri1jfqoPqEI86xHr8DaVVLN95lBHdk5hTXInTQKJAs/3F5OdvDaqPNgfWMtBpEAzGafhu6VIObjwSkh7Bfg9nbfmOFljzHkPN/MeYKn788i22ZIY2bn10CJW6jEh1OktjTKVXedygMcbsF5GvgKFAOxFJtGcb6cA2u9k2IAPYKiKJQFtgr5vchfs9/uTe478KvAqQk5Nj8vLy6vU56iI/P59I9a06qA7xqEMsxy8qLuWZLwoorxSSE5388bLTKT1SzpCeHYL0zrJZUISrbK3gZFD7wzAsLyRdgv4efuwOxVvtsWoQoOcF4+l5Ahv6kfq3qGs5a4B7lDpwRrAR6yLSyZ6BuLL+XgSsAb4CrrCbTQA+to8/sc+xr39pjDG2fJztvdUD6IU1I1oM9LK9vZKxNt8DeowpitJ0+HDpVsrtFL7llU5WbT/AHeefEpoBgSgnRfT3Q71+P+CjQcCZiDHGTx3HoOgCvGkHKzqA940xM0VkNfCuiDwJLAPesNu/AfxLRDYA+7CMAsaYVSLyPrAaqATusJfJEJE7gTlAAjDFGLPqBPRVFKURsefQ8YDnQRPNpIiVx/xfi2SA4wkQdALGUDHGfA/Ucsg2xmzC8qzylh/D8gLz1ddTwFM+5LOAWSesrKIojY6OrZsFPA+JaCVFPHM8bCvyccFAShBR9jEg2GBDRVGUuGLsoHSSEwQBkhOEsYPSY61S3eRMhNOvqi0Xh52ivuGhRkRRlEZJdmYqj/2iP6d1cPDYL/qHvhcSK8a+BqNfgG7Z4Eiy92KaNdgCVRFbzlIURYklRcWlPPZ/qyivdPLY/62iT+fW8WNIciZar5LCBl+gSmciiqI0Sry9sz5cGlxciBIaOhNRFKVR4p2FKkIFbSNHSSG8+QurwmJCsuUh1gBnIzoTURSlUdKmWWLA8wbP5gWWATFV1vvmBbHWyCdqRBRFaZSs2nEw4HmDJ6pBjvUnzkyzoihKcIzs34UF63/yOI8rohnkeAKoEVEUpVHiqhkybf4qrj33tOBqiDQ0ohXkeALocpaiKI2WPp1bc2r7BPp0bh1rVRotakQURWmUFBWXct3rBXywvoLrXi8IrhBVPDD3UZh8pvXeAFAjoihKo+SV/2zkmF3RsKLSGXwhqobMB7fC18/Dvk3W+1uXx1ojNSKKojQ+pi3awuerd9UIRBjSs2EmMAyakkJY8b6nbNOXsGRqTNRxoUZEUZRGx3uLt3icVznjLtSwNh/d5lu+5mPf8iihRkRRlEZHs8Taj7a4Tnsy91FrCcsXLTpGVxcv1MVXUZRGR7sWybVkcTkXWTLVmmnsXOG/zZGf/F+LAmpEFEVpdHgbDIH4qCfizpKpMPOuutsltbD2S2IUT6LLWYqiNDpO8qpieFG/tPhJA+9i2VtBNBJY+ylMHW0ZkhigRkRRlEbHLwelk5zoQIBEB9x23smxVil0WgeTpsVYr6rjsHx6pDXyiRoRRVEaHdmZqTz289M4p1dHrj81Of5mIQBn32UlXwyWst3W+5Kp8K/Lo+b6q3siiqI0OoqKS3liplXVcJHAz4tL48+QZORC98FQ/E3w93xwa00sycYvrfeciWFXzZ2IzUREJENEvhKR1SKySkTusuXtRWSuiKy331NtuYjIZBHZICLfi8ggt74m2O3Xi8gEN3m2iKyw75ksIhKpz6MoSvxQsGkv5ZVOnAYqncRvtPq274Jve3Rf7WDEoPZVToxILmdVAvcYY/oBQ4A7RKQfcD8wzxjTC5hnnwOMBHrZr0nAS2AZHeBRYDCQCzzqMjx2m1vd7rskgp9HUZQ4YUjPDiQnOkgQa08kLqPV5z4KlUeCb+9rY72qPHz6+CFiy1nGmB3ADvv4kIisAboBY4A8u9mbQD5wny1/yxhjgAIRaSciXey2c40x+wBEZC5wiYjkA22MMQW2/C3gMmB2pD6ToigNn6LiUgo27eWR0adReqScZvuL428pC2DNJ6G1d1bWlh07EB5dAhCVPRERyQLOBBYBabaBAdgJpNnH3YASt9u22rJA8q0+5IqiNFFcmXuPVTgR4KysVEak1Xlbw6TvL6wkiydCYkpYVAk4RKQHEJFWwAfA3caYg+7bFsYYIyIRDyQVkUlYS2SkpaWRn58fkXHKysoi1rfqEB865G+pYMmuSnLSEslpf7zJfg+xGn/mxnKOVTgBy/m1cHMpizcbYB6npIbg6RRm6vU9JJ3PzxLfIKnyEK6npvWwFAwGAQ+5UDvI8of2w9lhjxupf4uIGhERScIyIO8YYz60xbtEpIsxZoe9XGX7pbENyHC7Pd2WbaNm+cslz7fl6T7a18IY8yrwKkBOTo7Jy8vz1eyEyc/PJ1J9qw4NX4dpi7YwdbWVnmLl3nLo14zHRkdXB29i/W8R7fG3p2xhxnrPFCEGofBQW265PCdqenhT7+9h70iPzXIB6Hk+smctHNruKXd7B6B9T/pc+9/0OVEd6iCS3lkCvAGsMcb8ze3SJ4DLw2oC8LGbfLztpTUEOGAve80BRohIqr2hPgKYY187KCJD7LHGu/WlKFHHO3Ps/K0VMdKk6fLvZb6TLG7aUxZlTcJASSGs+rC2fP9muOrNuu8/djDsKvkikt5ZZwM3ABeIyHf2axTwNHCRiKwHLrTPAWYBm4ANwGvA7QD2hvofgcX26wnXJrvd5nX7no3oproSQ9LaNPc4b9dcY3mjzdqdh3zKe3RqFWVNwsDmBeB01pb3/YUVQ5J5duD7j+2PSiqUSHpnLcRrduXGcB/tDXCHn76mAFN8yJcA/U9ATUUJG7eddzJfrdtNRZUhKUEY1SMp1io1OSqqfDx0gV/FY9qTrGGQ2AwqjwMGWnaEgdfBRY9b1y98DN64yP/9xlgb8xVHoe8YICsiamrEuqKEiezMVG4+uwefrdrJJad15pQWu+q+SQkbT89aw9GK2kakRQLx6eKbkQsTPrFmJFnDamfpzciF0S/AzLsBY6VIufRvMPv3UGUvpa791Hrf+CU9Mn4JEdgTUSOiKGFi2qItvDzfKhz08vxNFJ6UQOsecZhuI075bNVOn/K8jDh+zGXkBk7xnjMR0vrVGBqwZiAYMFUeTbuXfAQlvw57ynhdtFWUMDHl6x89zpfuruKa1wooKi6NkUZNi0tO61xLNjC9LVed2sxH60ZERi4Mu8d6Xz4NnP4cOgx88WjYh1cjoihhoKi4lI27a3sAlVc647ssaxxxkQ8j0jqlqe1L1ZE+sPgbK51KGFEjoihhoGDTXr/lV+OyLGsc8sp/NtaSjewfTE2ORsSAa/xeqjYvBS+HdUg1IooSBlJ91PR20b9r2yhq0nRZtd0zT1TblESuHdw9RtrEiIxc6pyNGB85tk4ANSKKEgZKj/jPlhromhI+Dh713Ato3SyON9TrS0kh/ua+1dKsc8M6pBoRRQkDQ3p2oHmS7/9OcZmGPM54etYaDh339EZq3byp7YdgeWn5onVXnJIEPS+A8R+FdUg1IooSBrIzU3lk9Gk+r/3r283RVaYJ4su9NzmxCT7esoZBgtvSqjisWJJ71rDgvBlhNyCgRkRRwob3mryLWSt2+JQr4WNgRrtasqvPamL7IWDtiUz8FHJuhJyb4KY5ES+P2wQXDRUlMvjzwiqvMhTFY43vOKJXWmuP84v6pTW9TXUXdQUohhmdiShKmAjkhRW3Nb7jhPW7PBMvntyxZYw0aXqoEVGUMFBUXMoTM1f5ve79kFPCR1FxKf/+bruH7EM/KeGV8KNGRFHCQMGmvZRX+s4g67oeTxQVl/LiVxviImXLBz4yAuw/orVcooXuiShKGBjSswPJiY7q0qzeZLRvEWWN6o+rTnl5pZNEh3BlTga/HJTeYPd0vlpTO1vyyfFYPyRO0ZmIooSB7MxU3rllCNcO7u4zXvjyM9N9SBsmrlmV01hOAe8s2sI1r37LQx+tiPnMxNcMaU/Z8Vrtnrz89Giq1aRRI6IoYSI7M5Vu7VIQLyvikPiKWh/SswMOr8/gMiZXv/ptzAzJtEVbuOKlb/jLnHVc8fI3TFtklSPu1s5zlte5dbMGO2tqjKgRUZQwMW3RFj5ftZMEh9T6jxUot1ZDIzsz1a++lVXGZ6LDSFNUXMqDH62odqM2Bh60Z0Yj+3tm7/1/F/aOun5NGd0TUZQwMG3RFh78aEX1ed/OrVlj1/t22g+8v81dh9PAVdnp3D+qb6xUrZOi4lL2lPmfOS0tif5MZNJbi33K73iniJ0HPZez/AV9KpFBZyKKEgbeW7zF43zDntq1RX4qK2ff4XJenr+Jp2etiZZqIePL28mdnw6VR3VJq6i4lL2HfXtbeRsQgD2HasuUyBExIyIiU0Rkt4isdJO1F5G5IrLefk+15SIik0Vkg4h8LyKD3O6ZYLdfLyIT3OTZIrLCvmeyiPdKtKJED2/33ipn4Coi7xc13DiGDUHEtFz/+qKoGcI//HtF3Y3c6Ni6kVcybGBEciYyFbjES3Y/MM8Y0wuYZ58DjAR62a9JwEtgGR3gUWAwkAs86jI8dptb3e7zHktRooZ3sr/2LQPvgVQ5/ceUxJp9h+t2AjhaUcXL8zcx8PE5Qfc7/o1F9H5oFnl/+SrgTMbbA2vLviNBjwEwdlD8eMI1BiJmRIwx84F9XuIxwJv28ZvAZW7yt4xFAdBORLoAFwNzjTH7jDGlwFzgEvtaG2NMgTHGAG+59aUoUcc72d9vL+pDxwA/iJMTav7rNbTAvroMoDv7j1Zy0bP5dbYb/8Yi5q//ifIqw+a9Rxj70je1Pu+0RVsY84+FjHv1W/46Zx1Xv/It0xZt4cK+aUHrc1ZWqnpmRZlo74mkGWNcKU13Aq6/jm5AiVu7rbYskHyrD7mixITCHz0j0rfsPUznVv7/e7kWu9zdVq96pfaDNRac4pXMsC7W7zlcpxFcsP6nWjJ3Ly+XY8LyrQeoqDIYoNJpePCjFUHNjFzcP7LhOiw0VsT6IR+hzkWygJnGmP72+X5jTDu366XGmFQRmQk8bYxZaMvnAfcBeUBzY8yTtvwPwFEg325/oS0fBtxnjBntR49JWMtkpKWlZb/77rvh/7BAWVkZrVrFNlJWdYi+DhtKq3hy0TEPWVoL4fwuTt7d6H+rbmK/ZKau9nxA9k518ODglLDpVp/vIX9LRS29giVZ4NWLa5IflpWVsbMipdb3A5Ao8Lrd9sH5h9ke2qqVBw7gwcHNOSU1oda1pvb3GAkdzj///CJjTI6va9F28d0lIl2MMTvsJandtnwbkOHWLt2WbcMyJO7yfFue7qO9T4wxrwKvAuTk5Ji8vDx/TU+I/Px8ItW36tBwdZj70QrA0zvr8pwedDy+nYQfj1HlZ/vjTR8P6uJDJqx61+d7eOaF+UD9jEi5gZvnHGbjf19aPf6fPz/ss22lgdY9BpCdmcr+ebOB4PaJEh3g7sdw2cCuPD/uTL/tm9rfY7R1iPZy1ieAy8NqAvCxm3y87aU1BDhgL3vNAUaISKq9oT4CmGNfOygiQ2yvrPFufSlKVPGey/ft3Jr7R/Vl7b4qvwbE130AxysjtzIQDGc9OZfVO04s43CVgbvfXQbAK98dw086MQAe/mgFT89aw5FAjbzodVJr/nT56Qzr1ZE/XX56QAOiRJ6IzUREZDrWLKKjiGzF8rJ6GnhfRG4GioGr7OazgFHABuAIcCOAMWafiPwRcEUaPWGMcW3W347lAZYCzLZfihJ1vOuI3DA0C4BT2ycAoWeTvfvdZdUPxvFvLKJg0166tkvh2asGRnTT+Kwn5wYMMjy3V0fAyq1V4TQEWgn/93fbeX7cmXy7s8p/I2DdrkPsCjGu48nLTyc7M7XpFp1qYETMiBhjrvFzabiPtga4w08/U4ApPuRLgP4noqOihINX53umAXm7YDPXDu7OKakJpLZIpPRIZUj9zfzeegC7PJoANu89wpUvf8P//upnETEkd7+7zK8BuW5wd48svi9+tYG/zFlXZ5+X/WNhnW2cBlKSglsQcQgR+/xK/dG0J4pyAhQVl7J5r+eO8A+7a6LVWySHbkQqndYDePWOgx5yp7FmAeF+iE5btKVWUScXKUkOnvLKiDukZ4eg+v1ua3DpR07u1Ipt+z033gW47dyeFG0ppWTfES4b2K1Bp4ppyqgRUZQT4M+za0dtJ7glT2ibklTrARkM32094DOl/Itfruel/A1c2DctbHsBs1fu8HvNV6Gt7MxUEsTa+wgVofZe0Hwv99+UJAdv3zJEZxxxgubOUpR6Mm3RFgo3146NOCO9Zo8kKaH+/8V8PaOPVDgpO17Fv7/bzvg3FtW7b3dO69LG77X0VN/FtP54Wej1Oto0T+S2c3vW2a7KadSAxBE6E1GUejLl6x99yt0D3ob27MByt2Wdfl1an7D3k4v5638i6/5PPWS9OrVk7j15IfVz8Lj/5ba/XT3Qp9y1qf23uev4KcBmvDvtWyZz/6i+dO/Qkhe/Wu93hhbscpnSMNCZiKLUk6PltR++rZslePyKbp2SVH0swMDuvn9ht00Jz++59XsOB5WGxB1fy2a5Wal88OvAm9jXDu7Okocvqna37dwmcOLDS07rXH1fVYC1sLduHhyU3krDQI2IotQTX7/AkxI9I6aH9OxA8yQHCQLNkhyMHZROs8Taj+37LulLknc5wXqy8SffwX3+OOw1Ezm3V0feD8EL6trB3fnXzYN58brsWhURXVw2sGv1xvi0RVvY6cett7Nm4I071IgoSj045+l5HPex6Zzm9RB01V7/7Yg+vGNvFk+7dahHm1+d25NrB3fnzO7twqJbm+bBz2qKiktreWZ9v61+RZ2yM1P531/9jGsHdycpwbImDqFWQGCgjfwXr8+u19hK7NA9EUUJkWmLtrDVz3r+IB+/3rMzPTPLZmdaS0UFm/YypGeH6mv3jezL2Je+OWH9DhytpKi4tM6ZhHscijtlx0IPkHTh+qxjB6VTsGkvzfYX1woKPK1Lm1oJGZsnOnjnVvXIikfUiChKkExbtIXZK3ew44BvA+IQ+GWQtSy8DYtL9sGvf8Ydbxd5LPekt2vOvsPlHKlw4sDy2kpwCJ3bNKOsvIqyY5VUuhXBMtQdT+LPgAD87OSOQX2GQLg+X35+7eJbs1furCV75OenqQGJU9SIKEodFBWXcuubi9l3xP8v9JbJCbx18+ATfhBmZ6ZS8NCFFBWX1pqp+MO7vjtAaovANUEWbvBtQCDyG9vb9h/1OHeApjCJY9SIKEoAiopL61xicgiseiK8hTV9zVT8ce3g7kz+4geP2csjH69g1fYD9KCqOg22yzAdOlqBv+q9vTq19H0hjHRrl0KxW7XCjPa+Y1GU+ECNiKIE4I63i+pskxAmr6oTYd8RT0+xSie8s8hKT//XotlM/FkWL8/fFLCP9HbNQ44xqQ9/u3ogV7z0DQbLvdhfLIoSH6gRUZQABJNhtnli7J0cExz+85Acq3TWaUAGpLfl4zvPiYRqtcjOTGWGD8cCJT5RI6Iofpi2aIvP1CPeXDc4M+K61EVKUgJHQ6jJ4Y13jfhIE8pyndKwUSOiKDbu9TTO7dWRRT/uq+MOaN8iqUFkl70qJ6PO2YY/0ts1141tpd7Efh6uKA2AUx781KOexvz1P/kMJvTmtQlnRVKtoLl/VN96bYoPTG/LwvtrlfhRlKDRmYjS5DnrybkEYS8ASE4QyqsMCWJlsm1ISzJz78njnKfn+Q2EBEh0CE+M6a8zDyVsqBFRmjTnPD0vYElYd9LbNeeFawY16A3hhfcP5+53l5H/wx7yenfiss4HyMvLi7VaSiNGjYjSJCgqLuWDpVsRqC71WldNcXccUL3s0xCNhzvueary8/Njp4jSJFAjcgK4lg5EYNgpHVn242EOfWbVdxCBMQO6hq36nBIcRcWlTJyyiEPHq+i0cC77j5Tj7bTkip8IhScvD70Ik6I0BeLeiIjIJcALQALwujHm6WiM2/uhWZTbfvnG1C7xaQz8+7vtHhlSB6a35d9uvvhFxaWMe+WbWg85F/26tG5w6+6Roqi4lGtf/ZbjPmIdBtqVAoOt2e0i2FmGO8kJwvRJQxn36rdUVJnqWt+6h6AovolrIyIiCcCLwEXAVmCxiHxijFkdyXH7P/JZtQEJhe+2HqhViS4Qq3ccql9W188+pV2LJMblZPCfH/awZmdNJT2HwKRhPX26pfpLytcqOYGVbmk96tq8dekQLkI1HvUl0QE/PDUKgHcnDW3Qex+K0lCIayMC5AIbjDGbAETkXWAMEHYjEtSDswGx/0iFz7gBp4GX528KKaagrLwqJOMXjyQnSLUBAQ2GU5Rgifc4kW5Aidv5VlsWVuLNgCihMTC9rYcBURQleMSY0JdlGgoicgVwiTHmFvv8BmCwMeZOr3aTgEkAaWlp2e+++25I40z8LNhyo66UcrFEdfDWoWUCvHhR5LPTelNWVkarVq2iPm5D0iHW46sO4dHh/PPPLzLG5Pi6Fu/LWduADLfzdFvmgTHmVeBVgJycHBOq33x6gf+ZSKvkBN68eXB1FbdbLrfcQE958NOgA9gSHZCc4KDKwMj+nVm17QDr94RWJ7uGWD+84UR0cG1sZ2em+tyjObdXx6DqXeTn58c8PkJ1iP34qkPkdYh3I7IY6CUiPbCMxzjg2nAPsvD+4T6XtNwfaN5V3Db86VKPoK9oufpe8NQsNh2qPbvs1CqZ0iPlQRk2dy+y/o98Rll5Va02Lq8lXxv04fpjjXRxJEVRTpy4NiLGmEoRuROYg+XiO8UYsyoSY9Unv1AsYkQeObtFWH9trAxzsSVFURoXcW1EAIwxs4BZsdZDURSlKRLv3lmKoihKDFEjoiiKotQbNSKKoihKvVEjoiiKotQbNSKKoihKvYnriPX6ICJ7gOIIdd8RqJ3BMLqoDqpDQ9Ih1uOrDuHRIdMY08nXhSZnRCKJiCzxlxpAdVAdmqIOsR5fdYi8DrqcpSiKotQbNSKKoihKvVEjEl5ejbUCqA4uVAeLWOsQ6/FBdXARER10T0RRFEWpNzoTURRFUeqNGpEAiEiGiHwlIqtFZJWI3GXL24vIXBFZb7+n2vJTReRbETkuIr/z0V+CiCwTkZmx0EFENovIChH5TkSWxEiHdiIyQ0TWisgaERkaTR1EpI/9+V2vgyJydwy+h/+y+1gpItNFpHkMdLjLHn9VBL+D60Tke/vv7hsRGeDW1yUisk5ENojI/cGMHwEdpojIbhFZGez44dTBXz9R1qG5iBSKyHK7n8dD+S4wxujLzwvoAgyyj1sDPwD9gGeA+235/cCf7eOTgLOAp4Df+ejvt8A0YGYsdAA2Ax1j+T0AbwK32MfJQLtY/FvYbRKAnVg+8FHTAauE849Ain3+PjAxyjr0B1YCLbCyeX8BnBKB8X8GpNrHI4FFbt/9RqCn/XewHOgXoe/Apw72+bnAIGBlhP9P+PsefPYTZR0EaGUfJwGLgCFBfxehfHFN/QV8DFwErAO6uP1DrvNq9xi1H57pwDzgAkIwImHWYTP1MCLh0gFoi/XwlFj+W7hdGwF8HYPvoRtQArTHeoDPBEZEWYcrgTfczv8A3Bup8W15KrDNPh4KzHG79gDwQCS/A28d3GRZhGhEwq2Ddz+x0gHrR8VSrDLjQY2ry1lBIiJZwJlYVjrNGLPDvrQTSAuii+eBe4Egi+ZGRAcDfC4iRWLVnY+2Dj2APcA/xVrWe11EQi5+HobvwcU4YHqo45+oDsaYbcBfgS3ADuCAMebzaOqANQsZJiIdRKQFMArPUtORGP9mYLZ97DKkLrbaspA4QR3CQrh08OonqjqItdT+HbAbmGuMCVoHNSJBICKtgA+Au40xB92vGct8B3RxE5HRwG5jTFGsdLA5xxgzCGsqe4eInBtlHRKxlg5eMsacCRzGmm5HUwdXP8nAL4D/DWX8cOhgr1GPwTKqXYGWInJ9NHUwxqwB/gx8DnwGfAfUroMcpvFF5HysB9d9wY7RlHQI1E80dDDGVBljBmKtmOSKSP9gx1cjUgcikoT1D/SOMeZDW7xLRLrY17tgWe9AnA38QkQ2A+8CF4jI21HWwfULGGPMbuAjIDfKOmwFtrr9ypmBZVSiqYOLkcBSY8yuYMcPow4XAj8aY/YYYyqAD7HWq6OpA8aYN4wx2caYc4FSrDX1sI8vImcArwNjjDF7bfE2PGc+6bYsKMKkwwkRLh389BNVHVwYY/YDXwFB18VWIxIAERHgDWCNMeZvbpc+ASbYxxOw1iL9Yox5wBiTbozJwlpC+dIYE9Qvz3DpICItRaS16xhrPyAoj5Qwfg87gRIR6WOLhgOro6mDG9cQ4lJWGHXYAgwRkRZ2n8OBNVHWARE5yX7vDvwSy+kjrOPbfX8I3GCMcTdSi4FeItLDnhWOs/uokzDqUG/CpUOAfqKpQycRaWcfp2Dtq6wNWpFgN0+a4gs4B2sq+D3WdP87rLXjDlib5OuxvFra2+07Y/3aPgjst4/bePWZR2jeWWHRAcsLZrn9WgU8FIvvARgILLH7+je2t0iUdWgJ7AXaxurvAXgc6z/qSuBfQLMY6LAAy4gvB4ZHaPzXsWY5rrZL3PoahTX72Uhk/x4D6TAda1+qwv5ubo6mDv76ibIOZwDL7H5WAo+E8v9CI9YVRVGUeqPLWYqiKEq9USOiKIqi1Bs1IoqiKEq9USOiKIqi1Bs1IoqiKEq9USOiKBFERKrEyha8ys6Seo+IBPx/JyJZInJttHRUlBNBjYiiRJajxpiBxpjTsIK4RgKP1nFPFqBGRIkLNE5EUSKIiJQZY1q5nffEitbuCGRiBRq6klDeaYz5RkQKgL5YGY/fBCYDT2MFqjYDXjTGvBK1D6EoAVAjoigRxNuI2LL9QB/gEOA0xhwTkV7AdGNMjojkYaVtH223nwScZIx5UkSaAV8DVxpjfoziR1EUnyTGWgFFacIkAf8QkYFYGXR7+2k3AjhDRK6wz9sCvbBmKooSU9SIKEoUsZezqrAyqz4K7AIGYO1PHvN3G/AbY8ycqCipKCGgG+uKEiVEpBPwMvAPY60jtwV2GGOcwA1YJWPBWuZq7XbrHODXdtpvRKS31KOYl6JEAp2JKEpkSbErxiUBlVgb6a603f8DfCAi47EKQx225d8DVSKyHJgKvIDlsbXUTv+9B7gsOuorSmB0Y11RFEWpN7qcpSiKotQbNSKKoihKvVEjoiiKotQbNSKKoihKvVEjoiiKotQbNSKKoihKvVEjoiiKotQbNSKKoihKvfn/CZ4BTvvqQoUAAAAASUVORK5CYII=", 956 | "text/plain": [ 957 | "
" 958 | ] 959 | }, 960 | "metadata": { 961 | "needs_background": "light" 962 | }, 963 | "output_type": "display_data" 964 | } 965 | ], 966 | "source": [ 967 | "plot_time_series(X_train, y_train, label='Training data')\n", 968 | "plot_time_series(X_test, y_test, label='Test data')" 969 | ] 970 | }, 971 | { 972 | "cell_type": "markdown", 973 | "id": "b4522dff", 974 | "metadata": {}, 975 | "source": [ 976 | "### Create windowed train & test sets" 977 | ] 978 | }, 979 | { 980 | "cell_type": "markdown", 981 | "id": "4c73c30c", 982 | "metadata": {}, 983 | "source": [ 984 | "**Phase 01 - Horizon 1; Window 7**" 985 | ] 986 | }, 987 | { 988 | "cell_type": "code", 989 | "execution_count": 24, 990 | "id": "e0a1fafc", 991 | "metadata": {}, 992 | "outputs": [], 993 | "source": [ 994 | "# predict 1 step at a time\n", 995 | "HORIZON = 1\n", 996 | "\n", 997 | "# use a week worth of timesteps to predict the horizon\n", 998 | "WINDOW_SIZE = 7" 999 | ] 1000 | }, 1001 | { 1002 | "cell_type": "code", 1003 | "execution_count": 25, 1004 | "id": "bf5b8858", 1005 | "metadata": {}, 1006 | "outputs": [], 1007 | "source": [ 1008 | "# Create function to label windowed data\n", 1009 | "def get_labelled_windows(x, horizon=1):\n", 1010 | " return x[:, :-horizon], x[:, -horizon:]" 1011 | ] 1012 | }, 1013 | { 1014 | "cell_type": "code", 1015 | "execution_count": 26, 1016 | "id": "243dfc20", 1017 | "metadata": {}, 1018 | "outputs": [], 1019 | "source": [ 1020 | "# Create a function to make windows across entire time series\n", 1021 | "def make_windows(x, window_size=7, horizon=1):\n", 1022 | " # Create a window of specific window_size\n", 1023 | " window_step = np.expand_dims(np.arange(window_size + horizon), axis=0)\n", 1024 | " \n", 1025 | " # Create a 2D array of multiple window steps\n", 1026 | " window_indexes = window_step + np.expand_dims(np.arange(len(x) - (window_size + horizon - 1)), axis=0).T\n", 1027 | " \n", 1028 | " windowed_array = x[window_indexes]\n", 1029 | " \n", 1030 | " windows, labels = get_labelled_windows(windowed_array, horizon)\n", 1031 | " return windows, labels" 1032 | ] 1033 | }, 1034 | { 1035 | "cell_type": "code", 1036 | "execution_count": 27, 1037 | "id": "2609aa60", 1038 | "metadata": {}, 1039 | "outputs": [], 1040 | "source": [ 1041 | "full_windows, full_labels = make_windows(prices, WINDOW_SIZE, HORIZON)" 1042 | ] 1043 | }, 1044 | { 1045 | "cell_type": "code", 1046 | "execution_count": 28, 1047 | "id": "807180e8", 1048 | "metadata": {}, 1049 | "outputs": [ 1050 | { 1051 | "data": { 1052 | "text/plain": [ 1053 | "(array([[ 140.30000305, 123. , 130.99000549, ...,\n", 1054 | " 136.69999695, 137.80000305, 135.80000305],\n", 1055 | " [ 123. , 130.99000549, 136.82000732, ...,\n", 1056 | " 137.80000305, 135.80000305, 136.49000549],\n", 1057 | " [ 130.99000549, 136.82000732, 136.69999695, ...,\n", 1058 | " 135.80000305, 136.49000549, 139.5 ],\n", 1059 | " ...,\n", 1060 | " [16837.23632812, 16831.79296875, 16918.1171875 , ...,\n", 1061 | " 16546.20703125, 16636.41601562, 16607.19921875],\n", 1062 | " [16831.79296875, 16918.1171875 , 16706.07421875, ...,\n", 1063 | " 16636.41601562, 16607.19921875, 16537.42773438],\n", 1064 | " [16918.1171875 , 16706.07421875, 16546.20703125, ...,\n", 1065 | " 16607.19921875, 16537.42773438, 16618.40625 ]]),\n", 1066 | " array([[ 136.49000549],\n", 1067 | " [ 139.5 ],\n", 1068 | " [ 140.41000366],\n", 1069 | " ...,\n", 1070 | " [16537.42773438],\n", 1071 | " [16618.40625 ],\n", 1072 | " [16576.24804688]]))" 1073 | ] 1074 | }, 1075 | "execution_count": 28, 1076 | "metadata": {}, 1077 | "output_type": "execute_result" 1078 | } 1079 | ], 1080 | "source": [ 1081 | "full_windows, full_labels" 1082 | ] 1083 | }, 1084 | { 1085 | "cell_type": "code", 1086 | "execution_count": 29, 1087 | "id": "41306e57", 1088 | "metadata": {}, 1089 | "outputs": [], 1090 | "source": [ 1091 | "def make_train_test_splits(windows, labels, test_split=.2):\n", 1092 | " split_size = int(len(windows) * (1 - test_split)) # 80%\n", 1093 | " train_windows = windows[:split_size]\n", 1094 | " train_labels = labels[:split_size]\n", 1095 | " test_windows = windows[split_size:]\n", 1096 | " test_labels = labels[split_size:]\n", 1097 | " return train_windows, test_windows, train_labels, test_labels" 1098 | ] 1099 | }, 1100 | { 1101 | "cell_type": "code", 1102 | "execution_count": 30, 1103 | "id": "910333fe", 1104 | "metadata": {}, 1105 | "outputs": [ 1106 | { 1107 | "data": { 1108 | "text/plain": [ 1109 | "(2699, 675, 2699, 675)" 1110 | ] 1111 | }, 1112 | "execution_count": 30, 1113 | "metadata": {}, 1114 | "output_type": "execute_result" 1115 | } 1116 | ], 1117 | "source": [ 1118 | "train_windows, test_windows, train_labels, test_labels = make_train_test_splits(full_windows, full_labels)\n", 1119 | "len(train_windows), len(test_windows), len(train_labels), len(test_labels)" 1120 | ] 1121 | }, 1122 | { 1123 | "cell_type": "markdown", 1124 | "id": "38502fa8", 1125 | "metadata": {}, 1126 | "source": [ 1127 | "### Attempt model training" 1128 | ] 1129 | }, 1130 | { 1131 | "cell_type": "code", 1132 | "execution_count": 29, 1133 | "id": "137689b1", 1134 | "metadata": { 1135 | "scrolled": false 1136 | }, 1137 | "outputs": [], 1138 | "source": [ 1139 | "model = tf.keras.Sequential([\n", 1140 | " tf.keras.layers.Input(shape=(WINDOW_SIZE)),\n", 1141 | " \n", 1142 | " # Expand dimensions to align with required input shape\n", 1143 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1)),\n", 1144 | " tf.keras.layers.RNN(LTSCell(32), time_major=True),\n", 1145 | " tf.keras.layers.Dense(HORIZON, activation='linear')\n", 1146 | "])" 1147 | ] 1148 | }, 1149 | { 1150 | "cell_type": "code", 1151 | "execution_count": 30, 1152 | "id": "619c2326", 1153 | "metadata": {}, 1154 | "outputs": [ 1155 | { 1156 | "name": "stdout", 1157 | "output_type": "stream", 1158 | "text": [ 1159 | "Model: \"sequential\"\n", 1160 | "_________________________________________________________________\n", 1161 | " Layer (type) Output Shape Param # \n", 1162 | "=================================================================\n", 1163 | " lambda (Lambda) (None, 1, 7) 0 \n", 1164 | " \n", 1165 | " rnn (RNN) (1, 32) 5040 \n", 1166 | " \n", 1167 | " dense (Dense) (1, 1) 33 \n", 1168 | " \n", 1169 | "=================================================================\n", 1170 | "Total params: 5,073\n", 1171 | "Trainable params: 5,073\n", 1172 | "Non-trainable params: 0\n", 1173 | "_________________________________________________________________\n" 1174 | ] 1175 | } 1176 | ], 1177 | "source": [ 1178 | "model.summary()" 1179 | ] 1180 | }, 1181 | { 1182 | "cell_type": "code", 1183 | "execution_count": 44, 1184 | "id": "9d818069", 1185 | "metadata": { 1186 | "scrolled": false 1187 | }, 1188 | "outputs": [ 1189 | { 1190 | "name": "stdout", 1191 | "output_type": "stream", 1192 | "text": [ 1193 | "Epoch 1/10\n", 1194 | "83/83 [==============================] - 10s 59ms/step - loss: nan - mae: nan - mse: nan\n", 1195 | "Epoch 2/10\n", 1196 | "83/83 [==============================] - 5s 57ms/step - loss: nan - mae: nan - mse: nan\n", 1197 | "Epoch 3/10\n", 1198 | "83/83 [==============================] - 5s 55ms/step - loss: nan - mae: nan - mse: nan\n", 1199 | "Epoch 4/10\n", 1200 | "83/83 [==============================] - 4s 54ms/step - loss: nan - mae: nan - mse: nan\n", 1201 | "Epoch 5/10\n", 1202 | "83/83 [==============================] - 5s 55ms/step - loss: nan - mae: nan - mse: nan\n", 1203 | "Epoch 6/10\n", 1204 | "83/83 [==============================] - 5s 60ms/step - loss: nan - mae: nan - mse: nan\n", 1205 | "Epoch 7/10\n", 1206 | "83/83 [==============================] - 5s 56ms/step - loss: nan - mae: nan - mse: nan\n", 1207 | "Epoch 8/10\n", 1208 | "83/83 [==============================] - 5s 56ms/step - loss: nan - mae: nan - mse: nan\n", 1209 | "Epoch 9/10\n", 1210 | "83/83 [==============================] - 5s 55ms/step - loss: nan - mae: nan - mse: nan\n", 1211 | "Epoch 10/10\n", 1212 | "83/83 [==============================] - 5s 56ms/step - loss: nan - mae: nan - mse: nan\n" 1213 | ] 1214 | } 1215 | ], 1216 | "source": [ 1217 | "model.compile(\n", 1218 | " optimizer = tf.keras.optimizers.Adam(\n", 1219 | " learning_rate = .01,\n", 1220 | " clipvalue=.5\n", 1221 | " ),\n", 1222 | " loss = tf.keras.losses.MAE,\n", 1223 | " metrics = ['mae', 'mse']\n", 1224 | ")\n", 1225 | "\n", 1226 | "history_1 = model.fit(\n", 1227 | " train_windows,\n", 1228 | " train_labels,\n", 1229 | " epochs=10,\n", 1230 | ")" 1231 | ] 1232 | }, 1233 | { 1234 | "cell_type": "markdown", 1235 | "id": "7e385c6d", 1236 | "metadata": {}, 1237 | "source": [ 1238 | "Gradient explosion occurs - Add LSTM layer" 1239 | ] 1240 | }, 1241 | { 1242 | "cell_type": "markdown", 1243 | "id": "bc9ff514", 1244 | "metadata": {}, 1245 | "source": [ 1246 | "### Experiments\n", 1247 | "train for less number of epochs for faster experimentation" 1248 | ] 1249 | }, 1250 | { 1251 | "attachments": {}, 1252 | "cell_type": "markdown", 1253 | "id": "ae0c934f", 1254 | "metadata": {}, 1255 | "source": [ 1256 | "**Model 2 - LTS, LSTM - activation tanh**" 1257 | ] 1258 | }, 1259 | { 1260 | "cell_type": "code", 1261 | "execution_count": 30, 1262 | "id": "2839a801", 1263 | "metadata": {}, 1264 | "outputs": [], 1265 | "source": [ 1266 | "model_2 = tf.keras.Sequential([\n", 1267 | " tf.keras.layers.Input(shape=(WINDOW_SIZE)),\n", 1268 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1)),\n", 1269 | " tf.keras.layers.RNN(LTSCell(128), time_major=True, return_sequences=True),\n", 1270 | " tf.keras.layers.LSTM(128),\n", 1271 | " tf.keras.layers.Dense(HORIZON, activation='linear')\n", 1272 | "])" 1273 | ] 1274 | }, 1275 | { 1276 | "cell_type": "code", 1277 | "execution_count": 31, 1278 | "id": "4e98bf5f", 1279 | "metadata": {}, 1280 | "outputs": [], 1281 | "source": [ 1282 | "model_2.compile(\n", 1283 | " optimizer = tf.keras.optimizers.Adam(learning_rate = .01),\n", 1284 | " loss = tf.keras.losses.MAE,\n", 1285 | " metrics = ['mae', 'mse']\n", 1286 | ")" 1287 | ] 1288 | }, 1289 | { 1290 | "cell_type": "code", 1291 | "execution_count": 32, 1292 | "id": "dc333126", 1293 | "metadata": { 1294 | "scrolled": true 1295 | }, 1296 | "outputs": [ 1297 | { 1298 | "name": "stdout", 1299 | "output_type": "stream", 1300 | "text": [ 1301 | "Epoch 1/10\n", 1302 | "83/83 [==============================] - 21s 146ms/step - loss: 4372.9141 - mae: 4372.9141 - mse: 43273584.0000\n", 1303 | "Epoch 2/10\n", 1304 | "83/83 [==============================] - 11s 139ms/step - loss: 4287.7246 - mae: 4287.7246 - mse: 42528044.0000\n", 1305 | "Epoch 3/10\n", 1306 | "83/83 [==============================] - 12s 145ms/step - loss: 4207.5068 - mae: 4207.5068 - mse: 41833844.0000\n", 1307 | "Epoch 4/10\n", 1308 | "83/83 [==============================] - 12s 145ms/step - loss: 4145.5225 - mae: 4145.5225 - mse: 41230492.0000\n", 1309 | "Epoch 5/10\n", 1310 | "83/83 [==============================] - 12s 139ms/step - loss: 4098.0391 - mae: 4098.0391 - mse: 40693164.0000\n", 1311 | "Epoch 6/10\n", 1312 | "83/83 [==============================] - 11s 137ms/step - loss: 4059.2590 - mae: 4059.2590 - mse: 40201160.0000\n", 1313 | "Epoch 7/10\n", 1314 | "83/83 [==============================] - 11s 135ms/step - loss: 4031.4204 - mae: 4031.4204 - mse: 39782460.0000\n", 1315 | "Epoch 8/10\n", 1316 | "83/83 [==============================] - 12s 139ms/step - loss: 4010.4553 - mae: 4010.4553 - mse: 39411092.0000\n", 1317 | "Epoch 9/10\n", 1318 | "83/83 [==============================] - 11s 139ms/step - loss: 3991.2244 - mae: 3991.2244 - mse: 39043848.0000\n", 1319 | "Epoch 10/10\n", 1320 | "83/83 [==============================] - 11s 137ms/step - loss: 3974.9739 - mae: 3974.9739 - mse: 38709372.0000\n" 1321 | ] 1322 | } 1323 | ], 1324 | "source": [ 1325 | "history_2 = model_2.fit(\n", 1326 | " train_windows,\n", 1327 | " train_labels,\n", 1328 | " epochs=10,\n", 1329 | ")" 1330 | ] 1331 | }, 1332 | { 1333 | "attachments": {}, 1334 | "cell_type": "markdown", 1335 | "id": "b8ad165f", 1336 | "metadata": {}, 1337 | "source": [ 1338 | "**Model 2 - LTS, LSTM - activation relu**" 1339 | ] 1340 | }, 1341 | { 1342 | "cell_type": "code", 1343 | "execution_count": 40, 1344 | "id": "20d1440c", 1345 | "metadata": {}, 1346 | "outputs": [], 1347 | "source": [ 1348 | "model_3 = tf.keras.Sequential([\n", 1349 | " tf.keras.layers.Input(shape=(WINDOW_SIZE)),\n", 1350 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1)),\n", 1351 | " tf.keras.layers.RNN(LTSCell(128), time_major=True, return_sequences=True),\n", 1352 | " tf.keras.layers.LSTM(128, activation='relu'),\n", 1353 | " tf.keras.layers.Dense(HORIZON, activation='linear')\n", 1354 | "])" 1355 | ] 1356 | }, 1357 | { 1358 | "cell_type": "code", 1359 | "execution_count": 41, 1360 | "id": "858624b8", 1361 | "metadata": {}, 1362 | "outputs": [], 1363 | "source": [ 1364 | "model_3.compile(\n", 1365 | " optimizer = tf.keras.optimizers.Adam(learning_rate = .01),\n", 1366 | " loss = tf.keras.losses.MAE,\n", 1367 | " metrics = ['mae', 'mse']\n", 1368 | ")" 1369 | ] 1370 | }, 1371 | { 1372 | "cell_type": "code", 1373 | "execution_count": 35, 1374 | "id": "25792eac", 1375 | "metadata": {}, 1376 | "outputs": [ 1377 | { 1378 | "name": "stdout", 1379 | "output_type": "stream", 1380 | "text": [ 1381 | "Epoch 1/10\n", 1382 | "83/83 [==============================] - 18s 125ms/step - loss: 4036.5979 - mae: 4036.5979 - mse: 34951756.0000\n", 1383 | "Epoch 2/10\n", 1384 | "83/83 [==============================] - 10s 126ms/step - loss: 3887.4932 - mae: 3887.4932 - mse: 30614040.0000\n", 1385 | "Epoch 3/10\n", 1386 | "83/83 [==============================] - 11s 131ms/step - loss: 3888.0662 - mae: 3888.0662 - mse: 29839728.0000\n", 1387 | "Epoch 4/10\n", 1388 | "83/83 [==============================] - 13s 155ms/step - loss: 3893.5405 - mae: 3893.5405 - mse: 31972696.0000\n", 1389 | "Epoch 5/10\n", 1390 | "83/83 [==============================] - 10s 125ms/step - loss: 3885.5898 - mae: 3885.5898 - mse: 29777146.0000\n", 1391 | "Epoch 6/10\n", 1392 | "83/83 [==============================] - 11s 134ms/step - loss: 3882.2156 - mae: 3882.2156 - mse: 29110770.0000\n", 1393 | "Epoch 7/10\n", 1394 | "83/83 [==============================] - 11s 135ms/step - loss: 3891.8828 - mae: 3891.8828 - mse: 33197134.0000\n", 1395 | "Epoch 8/10\n", 1396 | "83/83 [==============================] - 11s 127ms/step - loss: 3889.1423 - mae: 3889.1423 - mse: 31033480.0000\n", 1397 | "Epoch 9/10\n", 1398 | "83/83 [==============================] - 10s 125ms/step - loss: 3886.7629 - mae: 3886.7629 - mse: 31257466.0000\n", 1399 | "Epoch 10/10\n", 1400 | "83/83 [==============================] - 10s 122ms/step - loss: 3893.5042 - mae: 3893.5042 - mse: 30507088.0000\n" 1401 | ] 1402 | } 1403 | ], 1404 | "source": [ 1405 | "history_3 = model_3.fit(\n", 1406 | " train_windows,\n", 1407 | " train_labels,\n", 1408 | " epochs=10,\n", 1409 | ")" 1410 | ] 1411 | }, 1412 | { 1413 | "cell_type": "markdown", 1414 | "id": "358ea3da", 1415 | "metadata": {}, 1416 | "source": [ 1417 | "LSTM with ReLU activation performs better - moving forward with ReLU" 1418 | ] 1419 | }, 1420 | { 1421 | "attachments": {}, 1422 | "cell_type": "markdown", 1423 | "id": "3abba21e", 1424 | "metadata": {}, 1425 | "source": [ 1426 | "**Model 3 - Two LTS layers, LSTM**" 1427 | ] 1428 | }, 1429 | { 1430 | "cell_type": "code", 1431 | "execution_count": 39, 1432 | "id": "20f5adfc", 1433 | "metadata": {}, 1434 | "outputs": [], 1435 | "source": [ 1436 | "model_4 = tf.keras.Sequential([\n", 1437 | " tf.keras.layers.Input(shape=(WINDOW_SIZE)),\n", 1438 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1)),\n", 1439 | " tf.keras.layers.RNN(LTSCell(128), time_major=True, return_sequences=True),\n", 1440 | " tf.keras.layers.RNN(LTSCell(128), time_major=True, return_sequences=True),\n", 1441 | " tf.keras.layers.LSTM(128, activation='relu'),\n", 1442 | " tf.keras.layers.Dense(HORIZON, activation='linear')\n", 1443 | "])" 1444 | ] 1445 | }, 1446 | { 1447 | "cell_type": "code", 1448 | "execution_count": 42, 1449 | "id": "2666705e", 1450 | "metadata": {}, 1451 | "outputs": [], 1452 | "source": [ 1453 | "model_4.compile(\n", 1454 | " optimizer = tf.keras.optimizers.Adam(learning_rate = .01),\n", 1455 | " loss = tf.keras.losses.MAE,\n", 1456 | " metrics = ['mae', 'mse']\n", 1457 | ")" 1458 | ] 1459 | }, 1460 | { 1461 | "cell_type": "code", 1462 | "execution_count": 43, 1463 | "id": "6db49b7c", 1464 | "metadata": {}, 1465 | "outputs": [ 1466 | { 1467 | "name": "stdout", 1468 | "output_type": "stream", 1469 | "text": [ 1470 | "Epoch 1/10\n", 1471 | "83/83 [==============================] - 36s 274ms/step - loss: 4009.9414 - mae: 4009.9414 - mse: 34135700.0000\n", 1472 | "Epoch 2/10\n", 1473 | "83/83 [==============================] - 18s 218ms/step - loss: 3893.3445 - mae: 3893.3445 - mse: 32565966.0000\n", 1474 | "Epoch 3/10\n", 1475 | "83/83 [==============================] - 18s 214ms/step - loss: 3888.0906 - mae: 3888.0906 - mse: 30409908.0000\n", 1476 | "Epoch 4/10\n", 1477 | "83/83 [==============================] - 18s 215ms/step - loss: 3884.8523 - mae: 3884.8523 - mse: 28430382.0000\n", 1478 | "Epoch 5/10\n", 1479 | "83/83 [==============================] - 18s 213ms/step - loss: 3890.0063 - mae: 3890.0063 - mse: 30857058.0000\n", 1480 | "Epoch 6/10\n", 1481 | "83/83 [==============================] - 18s 215ms/step - loss: 3886.7402 - mae: 3886.7402 - mse: 30942880.0000\n", 1482 | "Epoch 7/10\n", 1483 | "83/83 [==============================] - 18s 222ms/step - loss: 3888.2368 - mae: 3888.2368 - mse: 28527480.0000\n", 1484 | "Epoch 8/10\n", 1485 | "83/83 [==============================] - 18s 213ms/step - loss: 3891.5120 - mae: 3891.5120 - mse: 32891158.0000\n", 1486 | "Epoch 9/10\n", 1487 | "83/83 [==============================] - 18s 215ms/step - loss: 3891.8335 - mae: 3891.8335 - mse: 32143486.0000\n", 1488 | "Epoch 10/10\n", 1489 | "83/83 [==============================] - 18s 214ms/step - loss: 3898.8528 - mae: 3898.8528 - mse: 30118692.0000\n" 1490 | ] 1491 | } 1492 | ], 1493 | "source": [ 1494 | "history_4 = model_4.fit(\n", 1495 | " train_windows,\n", 1496 | " train_labels,\n", 1497 | " epochs=10,\n", 1498 | ")" 1499 | ] 1500 | }, 1501 | { 1502 | "cell_type": "markdown", 1503 | "id": "472408fb", 1504 | "metadata": {}, 1505 | "source": [ 1506 | "Seems like more LTC cells do not make a difference" 1507 | ] 1508 | }, 1509 | { 1510 | "attachments": {}, 1511 | "cell_type": "markdown", 1512 | "id": "94577949", 1513 | "metadata": {}, 1514 | "source": [ 1515 | "**Model 4 - LTS, Two LSTM layers**" 1516 | ] 1517 | }, 1518 | { 1519 | "cell_type": "code", 1520 | "execution_count": 44, 1521 | "id": "5e2081e8", 1522 | "metadata": {}, 1523 | "outputs": [], 1524 | "source": [ 1525 | "model_5 = tf.keras.Sequential([\n", 1526 | " tf.keras.layers.Input(shape=(WINDOW_SIZE)),\n", 1527 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1)),\n", 1528 | " tf.keras.layers.RNN(LTSCell(128), time_major=True, return_sequences=True),\n", 1529 | " tf.keras.layers.LSTM(128, activation='relu', time_major=True, return_sequences=True),\n", 1530 | " tf.keras.layers.LSTM(128, activation='relu'),\n", 1531 | " tf.keras.layers.Dense(HORIZON, activation='linear')\n", 1532 | "])" 1533 | ] 1534 | }, 1535 | { 1536 | "cell_type": "code", 1537 | "execution_count": 45, 1538 | "id": "c0187aaa", 1539 | "metadata": {}, 1540 | "outputs": [], 1541 | "source": [ 1542 | "model_5.compile(\n", 1543 | " optimizer = tf.keras.optimizers.Adam(learning_rate = .01),\n", 1544 | " loss = tf.keras.losses.MAE,\n", 1545 | " metrics = ['mae', 'mse']\n", 1546 | ")" 1547 | ] 1548 | }, 1549 | { 1550 | "cell_type": "code", 1551 | "execution_count": 46, 1552 | "id": "ce50778d", 1553 | "metadata": {}, 1554 | "outputs": [ 1555 | { 1556 | "name": "stdout", 1557 | "output_type": "stream", 1558 | "text": [ 1559 | "Epoch 1/10\n", 1560 | "83/83 [==============================] - 18s 113ms/step - loss: 5180.6592 - mae: 5180.6592 - mse: 360904064.0000\n", 1561 | "Epoch 2/10\n", 1562 | "83/83 [==============================] - 9s 112ms/step - loss: 4230.2700 - mae: 4230.2700 - mse: 40868064.0000\n", 1563 | "Epoch 3/10\n", 1564 | "83/83 [==============================] - 9s 115ms/step - loss: 4019.8645 - mae: 4019.8645 - mse: 35986044.0000\n", 1565 | "Epoch 4/10\n", 1566 | "83/83 [==============================] - 11s 129ms/step - loss: 3926.8398 - mae: 3926.8398 - mse: 34356900.0000\n", 1567 | "Epoch 5/10\n", 1568 | "83/83 [==============================] - 10s 121ms/step - loss: 3920.2937 - mae: 3920.2937 - mse: 32970250.0000\n", 1569 | "Epoch 6/10\n", 1570 | "83/83 [==============================] - 10s 115ms/step - loss: 3899.7246 - mae: 3899.7246 - mse: 31049208.0000\n", 1571 | "Epoch 7/10\n", 1572 | "83/83 [==============================] - 10s 117ms/step - loss: 3899.9729 - mae: 3899.9729 - mse: 33573732.0000\n", 1573 | "Epoch 8/10\n", 1574 | "83/83 [==============================] - 10s 115ms/step - loss: 3896.9729 - mae: 3896.9729 - mse: 29563204.0000\n", 1575 | "Epoch 9/10\n", 1576 | "83/83 [==============================] - 10s 115ms/step - loss: 3903.6147 - mae: 3903.6147 - mse: 32524350.0000\n", 1577 | "Epoch 10/10\n", 1578 | "83/83 [==============================] - 10s 122ms/step - loss: 3890.1138 - mae: 3890.1138 - mse: 32594174.0000\n" 1579 | ] 1580 | } 1581 | ], 1582 | "source": [ 1583 | "history_5 = model_5.fit(\n", 1584 | " train_windows,\n", 1585 | " train_labels,\n", 1586 | " epochs=10,\n", 1587 | ")" 1588 | ] 1589 | }, 1590 | { 1591 | "cell_type": "markdown", 1592 | "id": "fd7d73df", 1593 | "metadata": {}, 1594 | "source": [ 1595 | "More LSTM layers do not make a difference either" 1596 | ] 1597 | }, 1598 | { 1599 | "attachments": {}, 1600 | "cell_type": "markdown", 1601 | "id": "2a44f074", 1602 | "metadata": {}, 1603 | "source": [ 1604 | "**Model 5 - Two LTS layers, Two LSTM layers**" 1605 | ] 1606 | }, 1607 | { 1608 | "cell_type": "code", 1609 | "execution_count": 47, 1610 | "id": "0839dd65", 1611 | "metadata": {}, 1612 | "outputs": [], 1613 | "source": [ 1614 | "model_6 = tf.keras.Sequential([\n", 1615 | " tf.keras.layers.Input(shape=(WINDOW_SIZE)),\n", 1616 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1)),\n", 1617 | " tf.keras.layers.RNN(LTSCell(128), time_major=True, return_sequences=True),\n", 1618 | " tf.keras.layers.RNN(LTSCell(128), time_major=True, return_sequences=True),\n", 1619 | " tf.keras.layers.LSTM(128, activation='relu', time_major=True, return_sequences=True),\n", 1620 | " tf.keras.layers.LSTM(128, activation='relu'),\n", 1621 | " tf.keras.layers.Dense(HORIZON, activation='linear')\n", 1622 | "])" 1623 | ] 1624 | }, 1625 | { 1626 | "cell_type": "code", 1627 | "execution_count": 48, 1628 | "id": "42777e90", 1629 | "metadata": {}, 1630 | "outputs": [], 1631 | "source": [ 1632 | "model_6.compile(\n", 1633 | " optimizer = tf.keras.optimizers.Adam(learning_rate = .01),\n", 1634 | " loss = tf.keras.losses.MAE,\n", 1635 | " metrics = ['mae', 'mse']\n", 1636 | ")" 1637 | ] 1638 | }, 1639 | { 1640 | "cell_type": "code", 1641 | "execution_count": 49, 1642 | "id": "cd583a98", 1643 | "metadata": {}, 1644 | "outputs": [ 1645 | { 1646 | "name": "stdout", 1647 | "output_type": "stream", 1648 | "text": [ 1649 | "Epoch 1/10\n", 1650 | "83/83 [==============================] - 33s 233ms/step - loss: 5846.9629 - mae: 5846.9629 - mse: 505566240.0000\n", 1651 | "Epoch 2/10\n", 1652 | "83/83 [==============================] - 18s 222ms/step - loss: 4698.0308 - mae: 4698.0308 - mse: 205116160.0000\n", 1653 | "Epoch 3/10\n", 1654 | "83/83 [==============================] - 19s 224ms/step - loss: 4638.6548 - mae: 4638.6548 - mse: 65713860.0000\n", 1655 | "Epoch 4/10\n", 1656 | "83/83 [==============================] - 20s 237ms/step - loss: 4103.3467 - mae: 4103.3467 - mse: 37764920.0000\n", 1657 | "Epoch 5/10\n", 1658 | "83/83 [==============================] - 20s 237ms/step - loss: 4125.0063 - mae: 4125.0063 - mse: 38572448.0000\n", 1659 | "Epoch 6/10\n", 1660 | "83/83 [==============================] - 19s 226ms/step - loss: 4011.9453 - mae: 4011.9453 - mse: 36266252.0000\n", 1661 | "Epoch 7/10\n", 1662 | "83/83 [==============================] - 18s 221ms/step - loss: 3994.3425 - mae: 3994.3425 - mse: 35959420.0000\n", 1663 | "Epoch 8/10\n", 1664 | "83/83 [==============================] - 19s 231ms/step - loss: 3927.7908 - mae: 3927.7908 - mse: 32880758.0000\n", 1665 | "Epoch 9/10\n", 1666 | "83/83 [==============================] - 20s 236ms/step - loss: 3909.2239 - mae: 3909.2239 - mse: 33311376.0000\n", 1667 | "Epoch 10/10\n", 1668 | "83/83 [==============================] - 18s 221ms/step - loss: 3901.2561 - mae: 3901.2561 - mse: 32431908.0000\n" 1669 | ] 1670 | } 1671 | ], 1672 | "source": [ 1673 | "history_6 = model_6.fit(\n", 1674 | " train_windows,\n", 1675 | " train_labels,\n", 1676 | " epochs=10,\n", 1677 | ")" 1678 | ] 1679 | }, 1680 | { 1681 | "cell_type": "markdown", 1682 | "id": "9c005ee9", 1683 | "metadata": {}, 1684 | "source": [ 1685 | "More of both layers do not make a difference either - return to base" 1686 | ] 1687 | }, 1688 | { 1689 | "cell_type": "markdown", 1690 | "id": "e6d92723", 1691 | "metadata": {}, 1692 | "source": [ 1693 | "### Tune hyperparameters\n", 1694 | "\n", 1695 | "Epochs and learning rate can be tuned by keras callbacks. Therefore, only batch size, optimization function, and hidden units are required to tune" 1696 | ] 1697 | }, 1698 | { 1699 | "cell_type": "markdown", 1700 | "id": "76d52892", 1701 | "metadata": {}, 1702 | "source": [ 1703 | "**Update hidden units (multiples of 32 are preferred)**" 1704 | ] 1705 | }, 1706 | { 1707 | "cell_type": "code", 1708 | "execution_count": 59, 1709 | "id": "176bc013", 1710 | "metadata": {}, 1711 | "outputs": [], 1712 | "source": [ 1713 | "model_7 = tf.keras.Sequential([\n", 1714 | " tf.keras.layers.Input(shape=(WINDOW_SIZE)),\n", 1715 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1)),\n", 1716 | " tf.keras.layers.RNN(LTSCell(32), time_major=True, return_sequences=True),\n", 1717 | " tf.keras.layers.LSTM(32, activation='relu'),\n", 1718 | " tf.keras.layers.Dense(HORIZON, activation='linear')\n", 1719 | "])" 1720 | ] 1721 | }, 1722 | { 1723 | "cell_type": "code", 1724 | "execution_count": 60, 1725 | "id": "d8251252", 1726 | "metadata": {}, 1727 | "outputs": [], 1728 | "source": [ 1729 | "model_7.compile(\n", 1730 | " optimizer = tf.keras.optimizers.Adam(learning_rate = .01),\n", 1731 | " loss = tf.keras.losses.MAE,\n", 1732 | " metrics = ['mae', 'mse']\n", 1733 | ")" 1734 | ] 1735 | }, 1736 | { 1737 | "cell_type": "code", 1738 | "execution_count": 61, 1739 | "id": "3b4b66f7", 1740 | "metadata": {}, 1741 | "outputs": [ 1742 | { 1743 | "name": "stdout", 1744 | "output_type": "stream", 1745 | "text": [ 1746 | "Epoch 1/10\n", 1747 | "83/83 [==============================] - 13s 58ms/step - loss: 4224.1118 - mae: 4224.1118 - mse: 40617880.0000\n", 1748 | "Epoch 2/10\n", 1749 | "83/83 [==============================] - 5s 59ms/step - loss: 3896.9988 - mae: 3896.9988 - mse: 32355542.0000\n", 1750 | "Epoch 3/10\n", 1751 | "83/83 [==============================] - 5s 58ms/step - loss: 3868.4753 - mae: 3868.4753 - mse: 29847568.0000\n", 1752 | "Epoch 4/10\n", 1753 | "83/83 [==============================] - 5s 56ms/step - loss: 3895.0820 - mae: 3895.0820 - mse: 34043076.0000\n", 1754 | "Epoch 5/10\n", 1755 | "83/83 [==============================] - 5s 56ms/step - loss: 3896.3674 - mae: 3896.3674 - mse: 30391228.0000\n", 1756 | "Epoch 6/10\n", 1757 | "83/83 [==============================] - 5s 56ms/step - loss: 3891.5137 - mae: 3891.5137 - mse: 32587410.0000\n", 1758 | "Epoch 7/10\n", 1759 | "83/83 [==============================] - 5s 57ms/step - loss: 3887.9116 - mae: 3887.9116 - mse: 30818276.0000\n", 1760 | "Epoch 8/10\n", 1761 | "83/83 [==============================] - 5s 56ms/step - loss: 3887.3694 - mae: 3887.3694 - mse: 31876788.0000\n", 1762 | "Epoch 9/10\n", 1763 | "83/83 [==============================] - 5s 59ms/step - loss: 3890.2361 - mae: 3890.2361 - mse: 29963236.0000\n", 1764 | "Epoch 10/10\n", 1765 | "83/83 [==============================] - 5s 64ms/step - loss: 3890.6362 - mae: 3890.6362 - mse: 33269162.0000\n" 1766 | ] 1767 | } 1768 | ], 1769 | "source": [ 1770 | "history_7 = model_7.fit(\n", 1771 | " train_windows,\n", 1772 | " train_labels,\n", 1773 | " epochs=10,\n", 1774 | ")" 1775 | ] 1776 | }, 1777 | { 1778 | "cell_type": "markdown", 1779 | "id": "26f4908b", 1780 | "metadata": {}, 1781 | "source": [ 1782 | "128 and 32 is very similar - moving forward with 32 as it trains faster" 1783 | ] 1784 | }, 1785 | { 1786 | "cell_type": "markdown", 1787 | "id": "d396f6ed", 1788 | "metadata": {}, 1789 | "source": [ 1790 | "**Update batch size**" 1791 | ] 1792 | }, 1793 | { 1794 | "cell_type": "code", 1795 | "execution_count": 62, 1796 | "id": "690b1e19", 1797 | "metadata": {}, 1798 | "outputs": [], 1799 | "source": [ 1800 | "model_8 = tf.keras.Sequential([\n", 1801 | " tf.keras.layers.Input(shape=(WINDOW_SIZE)),\n", 1802 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1)),\n", 1803 | " tf.keras.layers.RNN(LTSCell(32), time_major=True, return_sequences=True),\n", 1804 | " tf.keras.layers.LSTM(32, activation='relu'),\n", 1805 | " tf.keras.layers.Dense(HORIZON, activation='linear')\n", 1806 | "])" 1807 | ] 1808 | }, 1809 | { 1810 | "cell_type": "code", 1811 | "execution_count": 63, 1812 | "id": "c81e3b39", 1813 | "metadata": {}, 1814 | "outputs": [], 1815 | "source": [ 1816 | "model_8.compile(\n", 1817 | " optimizer = tf.keras.optimizers.Adam(learning_rate = .01),\n", 1818 | " loss = tf.keras.losses.MAE,\n", 1819 | " metrics = ['mae', 'mse']\n", 1820 | ")" 1821 | ] 1822 | }, 1823 | { 1824 | "cell_type": "code", 1825 | "execution_count": 64, 1826 | "id": "da5ecf0d", 1827 | "metadata": {}, 1828 | "outputs": [ 1829 | { 1830 | "name": "stdout", 1831 | "output_type": "stream", 1832 | "text": [ 1833 | "Epoch 1/10\n", 1834 | "21/21 [==============================] - 20s 228ms/step - loss: 4415.4756 - mae: 4415.4756 - mse: 43638328.0000\n", 1835 | "Epoch 2/10\n", 1836 | "21/21 [==============================] - 4s 210ms/step - loss: 4337.7793 - mae: 4337.7793 - mse: 42962548.0000\n", 1837 | "Epoch 3/10\n", 1838 | "21/21 [==============================] - 4s 207ms/step - loss: nan - mae: nan - mse: nan \n", 1839 | "Epoch 4/10\n", 1840 | "21/21 [==============================] - 4s 207ms/step - loss: nan - mae: nan - mse: nan\n", 1841 | "Epoch 5/10\n", 1842 | "21/21 [==============================] - 4s 193ms/step - loss: nan - mae: nan - mse: nan\n", 1843 | "Epoch 6/10\n", 1844 | "21/21 [==============================] - 5s 220ms/step - loss: nan - mae: nan - mse: nan\n", 1845 | "Epoch 7/10\n", 1846 | "21/21 [==============================] - 5s 222ms/step - loss: nan - mae: nan - mse: nan\n", 1847 | "Epoch 8/10\n", 1848 | "21/21 [==============================] - 4s 211ms/step - loss: nan - mae: nan - mse: nan\n", 1849 | "Epoch 9/10\n", 1850 | "21/21 [==============================] - 4s 198ms/step - loss: nan - mae: nan - mse: nan\n", 1851 | "Epoch 10/10\n", 1852 | "21/21 [==============================] - 4s 204ms/step - loss: nan - mae: nan - mse: nan\n" 1853 | ] 1854 | } 1855 | ], 1856 | "source": [ 1857 | "history_8 = model_8.fit(\n", 1858 | " train_windows,\n", 1859 | " train_labels,\n", 1860 | " batch_size=128,\n", 1861 | " epochs=10,\n", 1862 | ")" 1863 | ] 1864 | }, 1865 | { 1866 | "cell_type": "markdown", 1867 | "id": "43e430ea", 1868 | "metadata": {}, 1869 | "source": [ 1870 | "Issues with batch size 128, default 32 is better" 1871 | ] 1872 | }, 1873 | { 1874 | "cell_type": "markdown", 1875 | "id": "7e825e99", 1876 | "metadata": {}, 1877 | "source": [ 1878 | "**Update window size**" 1879 | ] 1880 | }, 1881 | { 1882 | "cell_type": "code", 1883 | "execution_count": 49, 1884 | "id": "d38ce0d5", 1885 | "metadata": {}, 1886 | "outputs": [], 1887 | "source": [ 1888 | "HORIZON = 1\n", 1889 | "WINDOW_SIZE = 30" 1890 | ] 1891 | }, 1892 | { 1893 | "cell_type": "code", 1894 | "execution_count": 50, 1895 | "id": "5c7aca0c", 1896 | "metadata": {}, 1897 | "outputs": [ 1898 | { 1899 | "data": { 1900 | "text/plain": [ 1901 | "(2680, 671, 2680, 671)" 1902 | ] 1903 | }, 1904 | "execution_count": 50, 1905 | "metadata": {}, 1906 | "output_type": "execute_result" 1907 | } 1908 | ], 1909 | "source": [ 1910 | "full_windows, full_labels = make_windows(prices, WINDOW_SIZE, HORIZON)\n", 1911 | "train_windows, test_windows, train_labels, test_labels = make_train_test_splits(full_windows, full_labels)\n", 1912 | "len(train_windows), len(test_windows), len(train_labels), len(test_labels)" 1913 | ] 1914 | }, 1915 | { 1916 | "cell_type": "code", 1917 | "execution_count": 51, 1918 | "id": "f097793e", 1919 | "metadata": {}, 1920 | "outputs": [], 1921 | "source": [ 1922 | "model_9 = tf.keras.Sequential([\n", 1923 | " tf.keras.layers.Input(shape=(WINDOW_SIZE)),\n", 1924 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1)),\n", 1925 | " tf.keras.layers.RNN(LTSCell(32), time_major=True, return_sequences=True),\n", 1926 | " tf.keras.layers.LSTM(32, activation='relu'),\n", 1927 | " tf.keras.layers.Dense(HORIZON, activation='linear')\n", 1928 | "])" 1929 | ] 1930 | }, 1931 | { 1932 | "cell_type": "code", 1933 | "execution_count": 52, 1934 | "id": "438e1a7a", 1935 | "metadata": {}, 1936 | "outputs": [], 1937 | "source": [ 1938 | "model_9.compile(\n", 1939 | " optimizer = tf.keras.optimizers.Adam(learning_rate = .01),\n", 1940 | " loss = tf.keras.losses.MAE,\n", 1941 | " metrics = ['mae', 'mse']\n", 1942 | ")" 1943 | ] 1944 | }, 1945 | { 1946 | "cell_type": "code", 1947 | "execution_count": 53, 1948 | "id": "cd2bc27b", 1949 | "metadata": { 1950 | "scrolled": false 1951 | }, 1952 | "outputs": [ 1953 | { 1954 | "name": "stdout", 1955 | "output_type": "stream", 1956 | "text": [ 1957 | "Epoch 1/10\n", 1958 | "84/84 [==============================] - 12s 56ms/step - loss: 4994.0220 - mae: 4994.0220 - mse: 74743080.0000\n", 1959 | "Epoch 2/10\n", 1960 | "84/84 [==============================] - 5s 56ms/step - loss: 4632.9243 - mae: 4632.9243 - mse: 60828820.0000\n", 1961 | "Epoch 3/10\n", 1962 | "84/84 [==============================] - 5s 55ms/step - loss: 4623.9263 - mae: 4623.9263 - mse: 60410628.0000\n", 1963 | "Epoch 4/10\n", 1964 | "84/84 [==============================] - 5s 61ms/step - loss: 4637.5337 - mae: 4637.5337 - mse: 59740820.0000\n", 1965 | "Epoch 5/10\n", 1966 | "84/84 [==============================] - 5s 57ms/step - loss: 4633.1724 - mae: 4633.1724 - mse: 61724464.0000\n", 1967 | "Epoch 6/10\n", 1968 | "84/84 [==============================] - 5s 56ms/step - loss: 4622.2690 - mae: 4622.2690 - mse: 60010080.0000\n", 1969 | "Epoch 7/10\n", 1970 | "84/84 [==============================] - 5s 58ms/step - loss: 4635.4961 - mae: 4635.4961 - mse: 60839452.0000\n", 1971 | "Epoch 8/10\n", 1972 | "84/84 [==============================] - 5s 55ms/step - loss: 4624.0151 - mae: 4624.0151 - mse: 58873680.0000\n", 1973 | "Epoch 9/10\n", 1974 | "84/84 [==============================] - 5s 57ms/step - loss: 4623.7148 - mae: 4623.7148 - mse: 62052088.0000\n", 1975 | "Epoch 10/10\n", 1976 | "84/84 [==============================] - 5s 62ms/step - loss: 4622.7373 - mae: 4622.7373 - mse: 58343204.0000\n" 1977 | ] 1978 | } 1979 | ], 1980 | "source": [ 1981 | "history_9 = model_9.fit(\n", 1982 | " train_windows,\n", 1983 | " train_labels,\n", 1984 | " epochs=10,\n", 1985 | ")" 1986 | ] 1987 | }, 1988 | { 1989 | "cell_type": "markdown", 1990 | "id": "ef87f97a", 1991 | "metadata": {}, 1992 | "source": [ 1993 | "Window size of 7 is better" 1994 | ] 1995 | }, 1996 | { 1997 | "cell_type": "markdown", 1998 | "id": "9d7c9c8c", 1999 | "metadata": {}, 2000 | "source": [ 2001 | "### Complete training" 2002 | ] 2003 | }, 2004 | { 2005 | "cell_type": "markdown", 2006 | "id": "c25830cf", 2007 | "metadata": {}, 2008 | "source": [ 2009 | "**Create callback functions**" 2010 | ] 2011 | }, 2012 | { 2013 | "cell_type": "code", 2014 | "execution_count": 54, 2015 | "id": "f66c8a13", 2016 | "metadata": {}, 2017 | "outputs": [], 2018 | "source": [ 2019 | "# Model checkpoint with a specific filename\n", 2020 | "def create_model_checkpoint(model_name, save_path='model_checkpoints', monitor_dataset_loss=False):\n", 2021 | " return tf.keras.callbacks.ModelCheckpoint(\n", 2022 | " filepath=os.path.join(save_path, model_name),\n", 2023 | " verbose=0,\n", 2024 | " save_best_only=True,\n", 2025 | " monitor='loss' if monitor_dataset_loss else 'val_loss',\n", 2026 | " )\n", 2027 | "\n", 2028 | "# Create a tensorboard callback\n", 2029 | "def create_tensorboard_callback(dir_name, experiment_name):\n", 2030 | " log_dir = dir_name + '/' + experiment_name + '/' + datetime.datetime.now().strftime('%Y%m%d-%H%M%S')\n", 2031 | " tensorboard_callback = tf.keras.callbacks.TensorBoard(\n", 2032 | " log_dir=log_dir\n", 2033 | " )\n", 2034 | "\n", 2035 | " print(f'Saving TensorBoard log files to: {log_dir}')\n", 2036 | " return tensorboard_callback\n", 2037 | "\n", 2038 | "# Create early stopping callback\n", 2039 | "early_stopping = tf.keras.callbacks.EarlyStopping(\n", 2040 | " patience=200,\n", 2041 | " restore_best_weights=True\n", 2042 | ")\n", 2043 | "\n", 2044 | "# create reduce lr on plateau callback\n", 2045 | "reduce_lr_plateau = tf.keras.callbacks.ReduceLROnPlateau(\n", 2046 | " patience=100,\n", 2047 | " verbose=1\n", 2048 | ")" 2049 | ] 2050 | }, 2051 | { 2052 | "cell_type": "code", 2053 | "execution_count": 55, 2054 | "id": "c1d606ac", 2055 | "metadata": {}, 2056 | "outputs": [], 2057 | "source": [ 2058 | "HORIZON = 1\n", 2059 | "WINDOW_SIZE = 7" 2060 | ] 2061 | }, 2062 | { 2063 | "cell_type": "code", 2064 | "execution_count": 56, 2065 | "id": "75b67110", 2066 | "metadata": {}, 2067 | "outputs": [ 2068 | { 2069 | "data": { 2070 | "text/plain": [ 2071 | "(2699, 675, 2699, 675)" 2072 | ] 2073 | }, 2074 | "execution_count": 56, 2075 | "metadata": {}, 2076 | "output_type": "execute_result" 2077 | } 2078 | ], 2079 | "source": [ 2080 | "full_windows, full_labels = make_windows(prices, WINDOW_SIZE, HORIZON)\n", 2081 | "train_windows, test_windows, train_labels, test_labels = make_train_test_splits(full_windows, full_labels)\n", 2082 | "len(train_windows), len(test_windows), len(train_labels), len(test_labels)" 2083 | ] 2084 | }, 2085 | { 2086 | "cell_type": "code", 2087 | "execution_count": 57, 2088 | "id": "52aa8607", 2089 | "metadata": {}, 2090 | "outputs": [], 2091 | "source": [ 2092 | "model_10 = tf.keras.Sequential([\n", 2093 | " tf.keras.layers.Input(shape=(WINDOW_SIZE)),\n", 2094 | " tf.keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=1)),\n", 2095 | " tf.keras.layers.RNN(LTSCell(32), time_major=True, return_sequences=True),\n", 2096 | " tf.keras.layers.LSTM(32, activation='relu'),\n", 2097 | " tf.keras.layers.Dense(HORIZON, activation='linear')\n", 2098 | "], name='model_10')" 2099 | ] 2100 | }, 2101 | { 2102 | "cell_type": "code", 2103 | "execution_count": 58, 2104 | "id": "91d7d1d7", 2105 | "metadata": {}, 2106 | "outputs": [], 2107 | "source": [ 2108 | "model_10.compile(\n", 2109 | " optimizer = tf.keras.optimizers.Adam(learning_rate = .01),\n", 2110 | " loss = tf.keras.losses.MAE,\n", 2111 | " metrics = ['mae', 'mse']\n", 2112 | ")" 2113 | ] 2114 | }, 2115 | { 2116 | "cell_type": "code", 2117 | "execution_count": null, 2118 | "id": "a60ea72d", 2119 | "metadata": {}, 2120 | "outputs": [], 2121 | "source": [ 2122 | "history_10 = model_10.fit(\n", 2123 | " train_windows,\n", 2124 | " train_labels,\n", 2125 | " epochs=5000,\n", 2126 | " callbacks=[\n", 2127 | " create_model_checkpoint(model_name=model_10.name),\n", 2128 | " create_tensorboard_callback(\n", 2129 | " dir_name='tensorboard_logs/tensorboard_logs_model_10',\n", 2130 | " experiment_name='model_10'\n", 2131 | " ),\n", 2132 | " early_stopping,\n", 2133 | " reduce_lr_plateau,\n", 2134 | " \n", 2135 | " ]\n", 2136 | ")" 2137 | ] 2138 | } 2139 | ], 2140 | "metadata": { 2141 | "kernelspec": { 2142 | "display_name": "Python 3 (ipykernel)", 2143 | "language": "python", 2144 | "name": "python3" 2145 | }, 2146 | "language_info": { 2147 | "codemirror_mode": { 2148 | "name": "ipython", 2149 | "version": 3 2150 | }, 2151 | "file_extension": ".py", 2152 | "mimetype": "text/x-python", 2153 | "name": "python", 2154 | "nbconvert_exporter": "python", 2155 | "pygments_lexer": "ipython3", 2156 | "version": "3.7.4" 2157 | }, 2158 | "vscode": { 2159 | "interpreter": { 2160 | "hash": "ed21d5474e4d34402bc17b7e6678523875881abed00fabc0c331683181e10ba6" 2161 | } 2162 | } 2163 | }, 2164 | "nbformat": 4, 2165 | "nbformat_minor": 5 2166 | } 2167 | --------------------------------------------------------------------------------