66 |
67 | # Frequentist View: Maximum Likelihood
68 | # Let's now embark on the path of obtaining the free parameters, $\mathbf{w}$, of our model. We will begin using a *frequentist*, or
69 | # *maximum likelihood*, approach. This approach assumes that we first obtain observation training data, $\mathbf{t}$, and that the *best*
70 | # value of $\mathbf{w}$, is that which maximizes the likelihood function, $p(\mathbf{t}|\mathbf{w})$.
71 | #
72 | # Under the Gaussian noise condition it can be shown that the maximum likelihood function for the training data is
73 | #
74 | # $p(\mathbf{t}|\mathbf{X},\mathbf{w},\sigma^2) = \prod_{n=1}^N ND(t_n|\mathbf{w}^T\phi(\mathbf{x}_n),\sigma^2)$
75 | #
76 | # $=\frac{N}{2}\ln\frac{1}{\sigma^2} -\frac{N}{2}\ln(2\pi) - \frac{1}{2\sigma^2}\sum_{n=1}^N
77 | # \{t_n -\mathbf{w}^T\phi(\mathbf{x}_n)\}^2$
78 | #
79 | # where $\mathbf{X}=\{\mathbf{x}_1,\ldots,\mathbf{x}_N\}$ is the input value set for the corresponding $N$ oberved output values contained in the vector
80 | # $\mathbf{t}$, and $ND(\mu,\sigma^2)$ is the Normal Distribution (Gaussian). (I used ND instead of the standard N to avoid confusion
81 | # with the product limit).
82 | #
83 | # Taking the logarithm of the maximum likelihood and setting the derivative with respect to $\mathbf{w}$ equal to zero, one can obtain
84 | # the maximum likelikhood parameters given by the normal equations:
85 | # $\mathbf{w}_{ML} = \left(\mathbf{\Phi}^T\mathbf{\Phi}\right)^{-1}\mathbf{\Phi}^T\mathbf{t}$
86 | # where $\Phi$ is the $N \times M$ design matrix with elements $\Phi_{n,j}=\phi_j(\mathbf{x}_n)$, and $\mathbf{t}$ is the $N \times K$
87 | # matrix of training set target values (for $K=1$, it is simply a column vector). Note that $\mathbf{\Phi}^T$ is a $M \times N$ matrix, so that $\mathbf{w}_{ML}=\left(\mathbf{\Phi}^T \mathbf{\Phi}\right)^{-1}\mathbf{\Phi}^T\mathbf{t}$ is
88 | # $(M \times N)\times(N \times M)\times(M\times N)\times(N \times K) = M \times K$, where $M$ is the number of free parameters and $K$ is the number of predicted
89 | # target values for a given input.
90 | #
91 | #
92 | # Note that the only term in the likelihood function that depends on $\mathbf{w}$ is the last term. Thus, maximizing the likelihood
93 | # function with respect to $\mathbf{w}$ __under the assumption of Gaussian noise__ is equivalent to minimizing a
94 | # sum-of-squares error function.
95 | #
96 | #
97 | # The quantity, $\mathbf{\Phi}^\dagger=\left(\mathbf{\Phi}^T\mathbf{\Phi}\right)^{-1}\mathbf{\Phi}^T$ is known as the
98 | # Moore-Penrose pseudo-inverse of $\Phi$. When $\Phi^T\Phi$ is invertible, the pseudo-inverse is
99 | # equivalent to the inverse. When this condition fails, the pseudo-inverse can be found with techniques such as singular value decomposition.
100 | #
101 |
102 | #
103 |
104 | # Example 1
105 | # (a) Linear Data
106 | # Let's generate data of the for $y = m*x + b + \epsilon $ where $\epsilon$ is a random Gaussian component with zero mean. Given this data, let's apply the maximum likelihood
107 | # solution to find values for the parameters $m$ and $b$. Given that we know our data is linear, we chose basis functions $\phi_0(x)=1$ and $\phi_1(x)=x$. Thus, our
108 | # our model will be $y=\theta_0\phi_0(x) + \theta_1\phi_1(x)$, where presumabely the solution should yield $\theta_0 \approx b$ and $\theta_1 \approx
109 | # m$
110 | #
111 |
112 | #
113 |
114 | import numpy as np
115 | from matplotlib import pyplot as plt
116 | #in order to compare between examples, set a seed in random
117 | seed = 123456789
118 | np.random.seed(seed)
119 | def y(x,m,b,mu=0,sigma=1): return m*x + b + np.random.normal(mu,sigma,1)[0]
120 | #training data, with N data points
121 | N = 101
122 | M = 2
123 | t = np.empty(N)
124 | domain_bound = 1.0/N
125 | domain = np.empty(N)
126 | for i in range(N): domain[i] = i*domain_bound
127 | for i in range(N): t[i] = y(x=domain[i],m=4.89,b=0.57)
128 | #design matrix, phi, N X M
129 | phi = np.array([np.ones(N),domain]).T
130 | #find the solution
131 | #in this case case phi.T X phi is invertible so do the folloing:
132 | temp1 = np.linalg.inv(np.dot(phi.T,phi)) #inverse of phi.T X phi
133 | temp2 = np.dot(temp1, phi.T)
134 | w1 = np.dot(temp2,t) #solution
135 | print 'w1=',w1
136 | #assuming that phi.T X phi was not invertible we could find the pseudo inverse using the pinv function
137 | #we expect to obtain the same solution
138 | phi_pi = np.linalg.pinv(phi)
139 | w2 = np.dot(phi_pi,t)
140 | print 'w2=',w2
141 | #compute the model predicted values for the training data domain
142 | predicted_t = [w2[0]+w2[1]*x for x in domain]
143 | plt.plot(domain,t)
144 | plt.plot(domain,predicted_t)
145 | plt.show()
146 |
147 | #
148 |
149 | # (b) Trigonometric Data
150 | # One common misconception regarding linear regression is that the basis functions are required to be linear. This is not the case. Indeed, the basis
151 | # functions need not even be polynomials. They must be linearly independent, i.e. orthoganol. It is only the dependence on the model parameters, $\mathbf{w}_{ML}$, that must be linear.
152 | # Note that an example of a nonlinear parameter model would be $y=\exp(a)\sin(x)$ where $a$ is the free parameter.
153 | # Let's generate trigonometric data of the form $y = a + b\sin(x) + c\cos(x) + \epsilon $ where again $\epsilon$ is a random Gaussian component with zero mean.
154 | # Here we chose basis functions $\phi_0=1$, $\phi_1(x)=sin(x)$ and $\phi_2(x)=cos(x)$. If you're wondering if we're cheating a bit here, the answer is yes.
155 | # In reality, we may not know ahead of time what the appropriate basis functions for the observed data should be. The appropraite choice may be suggested
156 | # by the data, knowledge of the problem, and other machine learing techniques (hopefully to be discussed later). In this example, our
157 | # our model will be $y=\theta_0\phi_0(x) + \theta_1\phi_1(x) + + \theta_2\phi_2(x)$, where presumabely the solution should yield $\theta_0 \approx a$,
158 | # $\theta_1 \approx b$ and $\theta_2 \approx c$.
159 | #
160 |
161 | #
162 |
163 | import numpy as np
164 | import math
165 | from matplotlib import pyplot as plt
166 | #in order to compare between examples, set a seed in random
167 | seed = 123456789
168 | np.random.seed(seed)
169 | def y(x,a,b,c,mu=0,sigma=1): return a + b*math.sin(x) + c*math.cos(x) + np.random.normal(mu,sigma,1)[0]
170 | #training data, with N data points
171 | N = 101
172 | M = 3
173 | t = np.empty(N)
174 | domain = np.empty(N)
175 | domain_bound = 4.0*math.pi/N
176 | for i in range(N): domain[i] = i * domain_bound
177 | for i in range (N): t[i] = y(x=domain[i],a=1.85,b=0.57,c=4.37)
178 | #design matrix, phi, N X M
179 | c1 = [math.sin(x) for x in domain]
180 | c2 = [math.cos(x) for x in domain]
181 | phi = np.array([np.ones(N),c1,c2]).T
182 | #find the solution
183 | #in this case case phi.T X phi is invertible so do the folloing:
184 | temp1 = np.linalg.inv(np.dot(phi.T,phi)) #inverse of phi.T X phi
185 | temp2 = np.dot(temp1, phi.T)
186 | w1 = np.dot(temp2,t) #solution
187 | print 'w1=',w1
188 | #assuming that phi.T X phi was not invertible we could find the pseudo inverse using the pinv function
189 | #we expect to obtain the same solution
190 | phi_pi = np.linalg.pinv(phi)
191 | w2 = np.dot(phi_pi,t)
192 | print 'w2=',w2
193 | #compute the model predicted values for the training data domain
194 | predicted_t = [w2[0]+w2[1]*math.sin(x)+w2[2]*math.cos(x) for x in domain]
195 | plt.plot(domain,t)
196 | plt.plot(domain, predicted_t)
197 | plt.show()
198 |
199 | #
200 |
201 | # Stochastic Gradient Descent
202 | # In cases where the training data set is very large or data is received in a stream, a direct solution using the normal equations
203 | # may not be possible. An alternative approach is the stochastic gradient descent algorithm. If the total error function, $TE$, is the sum of
204 | # a given error function, $E$, evaluated at each of the $N$ trianing inputs, $TE = \sum_{i=1}^N E(\mathbf{x}_i)$ then the stochastic gradient descent
205 | # algorithm is
206 | # $\mathbf{w}^{\tau + 1} = \mathbf{w}^\tau - \eta \bigtriangledown E_\tau$
207 | # where ${\tau}$ is the iteration number and $\eta$ is a learning rate parameter. For this type of total error function, the order of evaluation
208 | # does not change the result. If the error function is the sum-of-squares function, then the
209 | # algorithm is
210 | # $\mathbf{w}^{\tau + 1} = \mathbf{w}^\tau + \eta \left(t_n - \mathbf{w}^{(\tau)T}\phi_n\right)\phi_n$
211 | #
212 |
213 | #
214 |
215 | # Example 2
216 | # Repeat the evaluation in example 1(b) using the gradient descent approach. Is the answer the same when using the same training data?
217 |
218 | #
219 |
220 | import numpy as np
221 | import math
222 | from matplotlib import pyplot as plt
223 | #in order to compare between examples, set a seed in random
224 | seed = 123456789
225 | np.random.seed(seed)
226 | def y(x,a,b,c,mu=0,sigma=1): return a + b*math.sin(x) + c*math.cos(x) + np.random.normal(mu,sigma,1)[0]
227 | N = 101
228 | M = 3
229 | w = np.zeros((M,1))
230 | phi = np.empty((M,1))
231 | eta = 0.25
232 | #create arrays to store the values as they are generated so they can be plotted at the end
233 | x = np.empty(N)
234 | t = np.empty(N)
235 | domain_bound = 4.0*math.pi/N
236 | for i in range(N):
237 | x[i] = i * domain_bound
238 | t[i] = y(x[i],a=1.85,b=0.57,c=4.37)
239 | phi = np.array([[1],[math.sin(x[i])],[math.cos(x[i])]])
240 | w = w + eta*(t[i]-np.dot(w.T,phi))*phi #the learning model
241 |
242 | print w.T
243 | #compute the model predicted values for the training data domain
244 | predicted_t = [w[0]+w[1]*math.sin(item)+w[2]*math.cos(item) for item in x]
245 | plt.plot(x,t)
246 | plt.plot(x,predicted_t)
247 | plt.show()
248 |
249 | #
250 |
251 | # Over and Under Fitting
252 | # This section describes some of the techniques used to account for over fitting or under fitting a given model when using a
253 | # maximum likelihood approach.
254 | # Regularized Least Squares
255 | # Regularization attempts to address the problem of overfitting a given model to the training data and/or eliminating parameters that are not important
256 | # based on the data. The general total error function with a regularization term is given by
257 | # $E_D(\mathbf{w}) + \lambda E_W(\mathbf{w})$
258 | # where $\lambda$ is the regularization coefficient and $E_W$ is the regularization term. A commonly used regularization term is the sum-of-squares of the
259 | # model parameter elements
260 | # $E_W(\mathbf{w}) = \frac1{2}\mathbf{w}^T\mathbf{w}$
261 | # known as the weight decay regularizer. This regularization terms leads to the optimal solution, assuming a linear regression model with
262 | # Gaussian noise on the training data, of
263 | # $\mathbf{w} = \left(\lambda \mathbf{I} + \mathbf{\Phi}^T \mathbf{\Phi}\right)^{-1} \mathbf{\Phi}^T\mathbf{t}$
264 | #
265 |
266 | #
267 |
268 | # Example 4
269 | # In this example, we use the same training data as example 1(a), except that here, the model is erroneously chosen to be a 7th order polynomial.
270 | # This example is somewhat contrived, but it illustrates the point that an overfit model can be corrected to some extend using regularization.
271 |
272 | #
273 |
274 | import numpy as np
275 | from matplotlib import pyplot as plt
276 | #in order to compare between examples, set a seed in random
277 | seed = 123456789
278 | np.random.seed(seed)
279 | def y(x,m,b,mu=0,sigma=1): return m*x + b + np.random.normal(mu,sigma,1)[0]
280 | def el_pow(x,pow):
281 | temp = x
282 | for i in range(pow-1):
283 | temp = temp * x
284 | return temp
285 | def prediction(params, x):
286 | pred = 0
287 | for i in range(len(params)):pred += params[i]*math.pow(x,i)
288 | return pred
289 | #training data, with N data points
290 | N = 101
291 | M = 8
292 | t = np.empty(N)
293 | domain = np.empty(N)
294 | domain_bound = 1.0/N
295 | for i in range(N): domain[i] = i*domain_bound
296 | for i in range(N): t[i] = y(x=domain[i],m=4.89,b=0.57)
297 | #find the solution without using regularization
298 | #design matrix, phi, N X M
299 | phi = np.array([np.ones(N),domain, el_pow(domain,2),el_pow(domain,3),el_pow(domain,4),el_pow(domain,5),el_pow(domain,6),el_pow(domain,7)]).T
300 | temp1 = np.linalg.inv(np.dot(phi.T,phi)) #inverse of phi.T X phi
301 | temp2 = np.dot(temp1, phi.T)
302 | w1 = np.dot(temp2,t) #solution
303 | print 'w1=',w1
304 | predicted_t = [prediction(w1,x) for x in domain]
305 |
306 | #find the regularized solution
307 | lam = 0.1
308 | temp1 = np.linalg.inv(lam*np.eye(M)+np.dot(phi.T,phi))
309 | temp2 = np.dot(temp1,phi.T)
310 | w2 = np.dot(temp2,t)
311 | print 'w2=',w2
312 | predicted_t_reg = [prediction(w2,x) for x in domain]
313 |
314 | #add some plots
315 | plt.plot(domain,t)
316 | plt.plot(domain,predicted_t)
317 | plt.plot(domain,predicted_t_reg)
318 | plt.legend(("training","un-regularized","regularized"), loc='lower right')
319 | plt.show()
320 |
321 | #
322 |
323 | # Locally Weighted Linear Regression
324 | # In attempting to predict a target value, $y$, for some new input $x$ and given a training set $\mathbf{t}$, locally weighted linear regression weights
325 | # training inputs relative to their distance from the new input $x$. The adjusted regression model is one of finding $\theta$ to minimize
326 | # $\sum_i w^{(i)} \left(y^{(i)} - \theta^Tx^{(i)}\right)$
327 | # where
328 | #
329 | #
330 | # $w^{(i)} = \exp \left( -\frac{\left( x^{(i)} - x \right)^2}{\left( 2\tau^2\right)} \right)$
331 | # The parameter $\tau$, called the **bandwidth** controls how the weight of a training input, $x^{(i)}$, falls off with distance from the new input, $x$. Also, notice that the weights,
332 | # $w^{(i)}$, require that the training data is available for each evaluation of a new input $x$, making it a so-called non-parametric model.
333 | #
334 | # The solution is obtained as
335 | # $\mathbf{w}_{ML} = \left(\mathbf{\Phi}^T\mathbf{W}\mathbf{\Phi}\right)^{-1}\mathbf{\Phi}^T\mathbf{W}\mathbf{t}$
336 | # where $\mathbf{W}$ is the $N \times N$ weighting matrix with elements $w_{i,j} = \left[w^{(i)}\right]^2$ for $i=j$ and 0 otherwise.
337 | #
338 | # Example 5
339 | # Here we apply locally weighted regression to training data that is generated from a second order polynomial with Gaussian
340 | # noise.
341 |
342 | #
343 |
344 | import numpy as np
345 | from matplotlib import pyplot as plt
346 | #in order to compare between examples, set a seed in random
347 | seed = 123456789
348 | np.random.seed(seed)
349 | def y(x,coefs,inc_ran = True,mu=0,sigma=0.2):
350 | ans = 0
351 | for i in range(len(coefs)): ans += coefs[i]*math.pow(x,i)
352 | if inc_ran: return ans + np.random.normal(mu,sigma,1)[0]
353 | else: return ans
354 | #training data, with N = 101 data points
355 | N = 101
356 | M = 2
357 | t = np.empty(N)
358 | domain = np.empty(N)
359 | domain_bound = 3.0/N
360 | for i in range(N): domain[i] = i*domain_bound
361 | for i in range(N): t[i] = y(x=domain[i],coefs=[1.75, 0.25, -1.0])
362 |
363 | #first find the standard linear regression fit using model y = phi_0 + phi_1 * x
364 | #design matrix, phi, N X M
365 | phi = np.array([np.ones(N),domain]).T
366 | temp1 = np.linalg.inv(np.dot(phi.T,phi)) #inverse of phi.T X phi
367 | temp2 = np.dot(temp1, phi.T)
368 | w1 = np.dot(temp2,t) #solution
369 | predicted_t = [y(x,w1,inc_ran=False) for x in domain]
370 |
371 | #now construct the weighted locally weighted solution
372 | def wt(x,x_i,tau=1): return math.exp(-(x_i-x)*(x_i-x)/(2*tau*tau))
373 | def lws(x, wt_func, td_x,td_y, Phi):
374 | _M = Phi.shape[1]
375 | _N = Phi.shape[0]
376 | _W = np.zeros((_N,_N))
377 | for i in range(_N): _W[i,i] = wt_func(x,td_x[i])
378 | temp1 = np.linalg.inv(np.dot(np.dot(Phi.T,_W),Phi)) #inverse of phi.T X phi
379 | temp2 = np.dot(np.dot(temp1,Phi.T),_W)
380 | w1 = np.dot(temp2,t) #local solution for parameters
381 | return y(x,w1,inc_ran=False)
382 | predicted_local = [lws(x, wt, domain, t, phi) for x in domain]
383 | plt.plot(domain,t)
384 | plt.plot(domain,predicted_t)
385 | plt.plot(domain,predicted_local)
386 | plt.legend(('training','regression', 'local regression'),loc='lower left')
387 |
388 | #
389 |
390 | # Bayesian View
391 | # Here we consider a fully Bayesian approach to the regression problem. First note, that our fundamental **model** assumptions still hold. Namely,
392 | # we assume that the observation we are trying to predict is modeled by
393 | # $t = y(\mathbf{x},\mathbf{w}) + \epsilon$
394 | #
395 | # where $\epsilon$ is a Gaussian distributed noise component. Additionally, the assumption of a linear dependence on the model parameters, $\mathbf{w}$ also still holds:
396 | # $y(\mathbf{x},\mathbf{w}) = \sum_{j=0}^{M-1} w_j\phi_j(\mathbf{x}) = \mathbf{w}^T\mathbf{\phi}(\mathbf{x})$
397 | #
398 | # Recall that the Bayesian approach implies that the *best* model parameters, $\mathbf{w}$, are those that maximize the **posterior** probability, which from Bayes' Theorem is given by:
399 | # $p(\mathbf{w}|\mathbf{t}) = \frac{p(\mathbf{t}|\mathbf{w})p(\mathbf{w})}{p(\mathbf{t})}$
400 | #
401 | # Prior Model
402 | # The first step in formulating a Bayesian model is to construct a model for the *prior* probability model, $p(\mathbf{w})$. **NOTE:** In general any probability distribution model could be chosen to
403 | # model the prior. The appropriate choice is dependent on different factors including prior knowledge of the problem and mathematical convenience. Often, the prior is chosen to be the *conjugate prior* of the
404 | # likelihood function. This is a choice of mathematical convenience becuase it implies that the *posterior* can be derived analytically. It is often a reasonable choice, as is this case here for
405 | # the problem of linear regression. Thus, given the Gaussian distribution used for the likelihood function above, we assume a Gaussian distribution for our *prior* of the form:
406 | # $p(\mathbf{w}) = ND(\mathbf{w}|\mathbf{m}_0,\mathbf{S}_0)$
407 | # where $\mathbf{m}_0$ is the prior mean and $\mathbf{S}_0$ is the prior covariance. We will *assume* $\mathbf{m}_0 = 0$ and an infinitely broad prior so that $\mathbf{S}_0=\alpha^{-1}\mathbf{I}$ where $\alpha$
408 | # is a precision parameter that we will have to choose. Given these choices of *prior* and *likelihood* functions the *posterior* probability is given by
409 | # $p(\mathbf{w}|\mathbf{t}) = N\left(\mathbf{w}|\mathbf{m}_N, \mathbf{S}_N\right)$
410 | # where
411 | # $\mathbf{m}_N = \beta \mathbf{S}_N \mathbf{\Phi}^T \mathbf{t}$
412 | # $\mathbf{S}_N^{-1} = \alpha\mathbf{I} + \beta \mathbf{\Phi}^T\mathbf{\Phi}$
413 | #
414 | # where $\beta = 1/\sigma^2$ is the inverse variance of the random noise component associated with the target variable, $t = y(\mathbf{x},\mathbf{w}) + ND(0, \sigma^2)$.
415 | # Finally, the log of the *posterior* is seen to be the sum of the log likelihood **and** the log of the prior:
416 | # $\ln p(\mathbf{w}|\mathbf{t}) = -\frac{\beta}{2}\left[\sum_{n=1}^N\\{t_n - \mathbf{w}^T \phi(\mathbf{x}_n)\\}^2\right] - \frac{\alpha}{2}\mathbf{w}^T\mathbf{w} + constant$
417 | #
418 | # **NOTE:** Maximizing the posterior function with respect to $\mathbf{w}$ __under the assumption of Gaussian noise and a Gaussian prior__ is equivalent to the least-squares error solution
419 | # with the addition of a regulariztion term $\lambda = \alpha/\beta$.
420 | #
421 | # Rather than find a point estimate for $\mathbf{w}$ by maximizing the posterior and thereby make point predictions for the target variable $t$, it is more instructive to use the posterior
422 | # to formulate a *predictive distribution* for $t$. For **our model assumptions** this is given by
423 | # $p(t|\mathbf{t},\alpha,\beta) = \int p(t|\mathbf{w},\beta) p(\mathbf{w}|\mathbf{t}, \alpha, \beta) d\mathbf{w} =
424 | # ND(t|\mathbf{m}_N^T\phi(\mathbf{x}), \frac{1}{\beta} + \phi(\mathbf{x})^T \mathbf{S}_N \phi(\mathbf{x})$
425 | # where a point estimate of $t$ is given my the mean $\mu = \mathbf{m}_N^T\phi(\mathbf{x})$ and an estimate of the uncertainty is given by the standard deviation
426 | # $\sigma_N^2(\mathbf{x}) = \frac{1}{\beta} + \phi(\mathbf{x})^T \mathbf{S}_N \phi(\mathbf{x})$
427 | #
428 | # There is one final issue with completing the Bayesian model, namely the determination of $\alpha$ and $\beta$. This can be done in a fully Bayesian manner by developing prior models but
429 | # this tends to make the equations intractible. Instead the so called *evidence function* approach is used. See section 3.5 of Bishop for more detail.
430 |
431 | #
432 |
433 | # Example 6
434 | # Use the data provided in example 5 to formulate a Bayesian model. Here we know that $\beta = (1/0.2) = 5$. For this example, we will simply choose $\alpha = .4$ so that $\alpha/\beta = 0.08$ (See Bishop 3.5 for methods
435 | # on determining $\alpha$ and $\beta$.
436 |
437 | #
438 |
439 | import numpy as np
440 | from matplotlib import pyplot as plt
441 | #in order to compare between examples, set a seed in random
442 | seed = 123456789
443 | np.random.seed(seed)
444 | alpha = 0.4
445 | beta = 5.0
446 | def y(x,coefs, mu=0, sigma=1.0/beta):
447 | ans = 0
448 | for i in range(len(coefs)): ans += coefs[i]*math.pow(x,i)
449 | return ans + np.random.normal(mu,sigma,1)[0]
450 |
451 | #training data, with N = 101 data points
452 | N = 101
453 | M = 4
454 | t = np.empty(N)
455 | domain = np.empty(N)
456 | domain_bound = 3.0/N
457 | for i in range(N): domain[i] = i*domain_bound
458 | for i in range(N): t[i] = y(x=domain[i],coefs=[1.75, 0.25, -1.0])
459 |
460 | #Let's assume that we want to fit a 3rd order polynomial to the data even though we know its a second order
461 | #polynomial. Given the Bayesain approach, we should see the so that the unecessary terms are damped out. We have
462 | #y = phi_0 + phi_1 * x + phi_2 x^2 + phi_3 x^4
463 | #design matrix, phi, N X M where N = 101 and M = 4
464 | d2 = domain * domain
465 | phi = np.array([np.ones(N),domain, d2, d2 * domain]).T
466 | alphaI = alpha * np.eye(M)
467 | SN = np.linalg.inv(alphaI + beta * np.dot(phi.T,phi)) #posterior variance
468 | mN = beta * np.dot(np.dot(SN, phi.T), t)
469 | point_estimates = [np.dot(mN, phi[i]) for i in range(N)]
470 | uncertain_t = [1.0/beta + np.dot(np.dot(phi[i].T, SN), phi[i]) for i in range(N)]
471 | plt.plot(domain,t)
472 | plt.errorbar(domain,point_estimates, uncertain_t, ecolor = "red")
473 | plt.legend(('training','Bayes'),loc='lower left')
474 |
475 | #
476 |
477 |
478 |
--------------------------------------------------------------------------------
/05_Decision_Trees.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "metadata": {
3 | "name": "05_Decision_Trees"
4 | },
5 | "nbformat": 3,
6 | "nbformat_minor": 0,
7 | "worksheets": [
8 | {
9 | "cells": [
10 | {
11 | "cell_type": "markdown",
12 | "metadata": {},
13 | "source": [
14 | "Decision Trees
\n",
15 | "Introduction
\n",
16 | "Decision trees divide the input space into some number of cuboid regions. **Every** input within a given region is assigned the *same output*. A decision tree model can be represented by a binary tree where \n",
17 | "\n",
18 | "* Each tree node tests one attribute from the input vector $X$\n",
19 | "\n",
20 | "* Each tree branch selects one value (or range) for the given attribute\n",
21 | "\n",
22 | "* Each leaf node predicts an output, Y\n",
23 | "\n",
24 | "Decision tree models cab be applied to both regression and classification problems, however the focus here will be on classification trees. Note also, that decision trees can represent **any** boolean function. The tree will have $2^N$ leaf nodes where $N$ is the number of boolean variables, however there are $2^{2^N}$ possible trees given $N$ variables. For other decision trees (i.e. for non-Boolean variables) the optimal number of leaves is unknown and the input vector selection attribute may be repeated at multiple nodes. \n",
25 | "\n",
26 | "The algorithms presented here are the so-called *CART* (classification and regression trees) methods. Another popular method, is the C5.0 (formally ID3) method presented by Quinlan. The resulting tree models will be binary decision trees (i.e. there are exactly 2 edges from each node except for leaf nodes)"
27 | ]
28 | },
29 | {
30 | "cell_type": "markdown",
31 | "metadata": {},
32 | "source": [
33 | "Classification Trees
\n",
34 | "Assume our training data consists of $N$ input vectors each with $p$ elements and an associateed single output from a set of $K$ classes, i.e. each input is of the form $X_i = \\\\{x_{i1}, x_{i2}, \\ldots , x_{ip}\\\\}$ and with an associated output $y_i \\in \\\\{c_1, c_2, \\ldots, c_K\\\\}$ where $i \\in \\\\{1 \\ldots N\\\\}$. Assume the decision tree model has $M$ leaf nodes, i.e. the input space is divided into $M$ regions, denoted $R_m$. Every input in the region $R_m$ will be assigned to class $k(m)$ where \n",
35 | "\n",
36 | "$k(m) = \\max_k p^e_{mk} = \\max_k\\frac{1}{N_m} \\sum_{x_i \\in R_m} I(y_i = k)$\n",
37 | "\n",
38 | "where $I$ returns 0 if $y_i=k$ and 0 otherwise, $p^e$ is the estimate of the probability of the observation being in class $k$ given the training data, and \n",
39 | "\n",
40 | "$N_m = num\\\\{x_i \\in R_m\\\\}$\n",
41 | "\n",
42 | "is the number of inputs from the training set in region $R_m$. NOTE: I would prefer to use $\\hat{p}$ instead of $p^e$ but there appears to be a bug in IPython Notebook."
43 | ]
44 | },
45 | {
46 | "cell_type": "markdown",
47 | "metadata": {},
48 | "source": [
49 | "Growing The Tree
\n",
50 | "The standard approach to creating a decision tree model is to use a greedy algorithm. The tree is started with a single root node that splits the data into two regions based on the selection of a single input feature, $j$, and an associated splitting value, $s$. That is, $j$ is the choice of feature and $s$ is the value of that feature used to split the training data. From there new nodes are added by successively repeating this process for the training data subregions resulting from the previous split. This process is continued until some stopping criteria is reached. From there, the tree pruning is applied as a means to prevent overfitting the data. \n",
51 | "
\n",
52 | "\n",
53 | "Split Criteria
\n",
54 | "The first task is to define how to select the split variable, $j$, and value $s$. This is done by minimizing an impurity measure, $Q_m(T)$, summed over the two nodes resulting from the split. Some of the more impurity measures for classification problems are\n",
55 | "\n",
56 | "Misclassification error: $\\frac{1}{N_m} \\sum_{i\\in R_m} I \\left( y_i \\ne k(m) \\right) = 1 - p^e_{mk}(m)$\n",
57 | "\n",
58 | "Gini Index: $\\sum_{k \\ne k'} p^e_{mk} p^e_{mk'} = \\sum_{k=1}^{K} p^e_{mk} \\left(1 - p^e_{mk} \\right)$\n",
59 | "\n",
60 | "Cross-entropy: $- \\sum_{k=1}^K p^e_{mk} \\log p^e_{mk}$\n",
61 | "\n",
62 | "Given a choice of $Q_m$, it is a straightforward matter of iterating over each feature, $j$, and split value $s$ to determine the optimal choice. Note that even if a feature is continuous, implying an infinite space of possible split values, the training data set, from which $s$ must be selected, is finite.\n",
63 | "\n",
64 | "The stopping criteria is generally chosen as some number, $D$, of training values associated with the leaf nodes. That is, if any region $R_m$ contains greater than $D$ inputs from the training data, continue the splitting process.\n",
65 | "
"
66 | ]
67 | },
68 | {
69 | "cell_type": "markdown",
70 | "metadata": {},
71 | "source": [
72 | "**TODO** \n",
73 | "Pruning
\n",
74 | "Given a starting tree, $T_0$, pruning is done by minimizing the cost complexity function\n",
75 | "\n",
76 | "$C_{\\lambda}(T) = \\sum_{m=1}^{|T|} N_m Q_m(T) + \\lambda |T|$\n",
77 | "\n",
78 | "where $T \\subset T_0$, $|T|$ is the number of leaf nodes in $T$, and $\\lambda$ is a tuning parameter.\n",
79 | "\n",
80 | "Reduced-Error Pruning 1. Split data inot training and validation set 2. Create tree that classifies training set data Prune until further pruning is harmful 1. Evaluate impact on validation data of pruning each possible node 2. Greedily remove the node that most improves the validation accuracy\n",
81 | "\n",
82 | "Rule Post-Pruning 1. Convert tree to equivalent set of rules 2. Prune each rule independently of others 3. Sort final rules into desired sequence for use"
83 | ]
84 | },
85 | {
86 | "cell_type": "markdown",
87 | "metadata": {},
88 | "source": [
89 | "Example 1
\n",
90 | "Uses artificially generated data in 4 regions."
91 | ]
92 | },
93 | {
94 | "cell_type": "code",
95 | "collapsed": false,
96 | "input": [
97 | "sys.path.append('pysrc')\n",
98 | "import decision_trees as dt\n",
99 | "import networkx as nx\n",
100 | "import numpy as np\n",
101 | "\n",
102 | "inputs = np.zeros((16, 2))\n",
103 | "outputs = []\n",
104 | "row = 0\n",
105 | "for x in range(4):\n",
106 | " for y in range(4):\n",
107 | " inputs[row][0] = x\n",
108 | " inputs[row][1] = y\n",
109 | " row += 1\n",
110 | "\n",
111 | "for row in inputs:\n",
112 | " if (row[0] > 1 and row[1] < 2) or (row[0] < 2 and row[1] > 1):outputs.append(1)\n",
113 | " else: outputs.append(0)\n",
114 | " \n",
115 | "clazz = [0,1]\n",
116 | "meta = ['x','y']\n",
117 | "tree = dt.build_tree(inputs, outputs, clazz, meta)\n",
118 | "dt.draw_tree(tree)\n",
119 | "data = np.zeros((16,4))\n",
120 | "for r in range(16):\n",
121 | " data[r][0] = r\n",
122 | " data[r][1] = inputs[r][0]\n",
123 | " data[r][2] = inputs[r][1]\n",
124 | " data[r][3] = outputs[r]\n",
125 | "#print data"
126 | ],
127 | "language": "python",
128 | "metadata": {},
129 | "outputs": [
130 | {
131 | "output_type": "display_data",
132 | "png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD9CAYAAABQvqc9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VHXa//H3zCSTMikkBBKBwEJQSkgQWYoKSFGKBduu\niiBSXcVVV9ZH3FV/us8+7K7lYddHFFAXEQGVJqirBEnoRZCSBoTegoSE9D7l+/sjZEwQkklykpnk\n3K/rmusyMyfnfM/XMx9OZu5zH4NSSiGEEEIXjO4egBBCiKYjoS+EEDoioS+EEDoioS+EEDoioS+E\nEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoi\noS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+E\nEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoioS+EEDoi\noS+EEDoioS+EEDri5e4BCOFuBQUFrFy5kp0JCezbtYtLubmYTCY6XHcdfQcPZvioUYwePRqTyeTu\noQrRYAallHL3IIRwh9zcXF6bNYtPP/2UoUYjI4qKuAloCziAk8Beg4G1AQFc8PHhhVdf5amnn5bw\nF82ahL7QpfXr1zN13DjuKSri5bIy2tey/F7geYsFx/XXs3jVKrp06dIUwxRCcxL6QneWLlnCC088\nwdKSEobX4fccwD+NRuYEB/P9tm307NmzsYYoRKOR0Be6Eh8fz4SxY4kvLqa+kb3EYOBPoaHsO3SI\nNm3aaDo+IRqbhL7Qjfz8fGKiovggK4tRDVzXf5nNnBk5ki++/lqTsQnRVKRkU+jG7Nde4/bCQmfg\nLwduASzAMBd+fxnQCQgA0srL+TEhgQ0bNlx12VOnTjFs2DAsFgs9evQgPj6+xnXPmjWLsLAwwsLC\neOmll1zdJSHqTEJftAgZGRk1vl5SUsLCjz7i5dJS53OtgZmAKxGbCjwJLAUygECgdXExc//xj6su\nP27cOPr27Ut2djazZ8/mN7/5DVlZWVdddsGCBaxdu5akpCSSkpL4+uuvWbBggQujEqIelBAe5M03\n31QPPvhgteeeeeYZ9dxzz/1i2aKiIrV48WI1bNgwFR0dXeN6v/jiCzUyMFAp+MXjQ1BDr/J81cef\nQI2v8vNxUGZQQWazyszMrLattLQ05ePjowoLC53PDRkyRM2fP/+qY7v55pvVhx9+6Px54cKFauDA\ngbXOlRD1IWf6wqM89thjrFu3jry8PABsNhtffPEFjz/+uHOZnTt3Mn36dDp06MCSJUuYPn06e/fu\ndb4+Y8YMQkJCqj0mTpzIloICbqznuA4Cvav83AXwAW4wm/nxxx+rLZuamkqXLl2wWCzO53r37k1q\naurV133wIL17/7z22NjYay4rRENJ6AuPEhERweDBg1mxYgUA69ato02bNvTp04fly5fTvXt3Jk+e\nTFRUFCkpKcTFxTFu3Dh8fHyc63j//ffJycmp9rglOpq1wIF6jqsQCL7iuSAgqqSEAweqr7WwsJDg\n4OpLBwUFUVBQcPV1X7F8UFAQhYWF9RypEDWT0Bce5/HHH2fJkiUALFmyhMceewyA9PR0zp8/T58+\nfYiNjaVt27YurzMnJ4fWDRhTAJB3xXN5QFu7nZzMzOrLBgSQn59f7bnc3FyCgoKuvu4rls/LyyMg\nIKABoxXi2iT0hce59957SUpKIiUlhf/85z+MHz8egOeff5709HSGDx/O7NmziYyMZObMmb84037y\nyScJDAys9kg6dYohQMxVtmdwYUzRQGKVn48D5UAIYPL2rr5sdDQnTpyodraemJhIdHT01dcdHV1t\nHxITE+nVq5cLoxKiHtz9pYIQVzNt2jQVExOjRowYcc1l0tLS1J/+9CfVoUOHGpdTSqkH7rhDLbvi\ny1k7qBJQ80ANAVUKqvwaX+SmggoCtRVUIahxlx+T/P3V/Pnz1WuvvaaGDh3q3N7AgQPVCy+8oEpK\nStSqVatUq1atVFZWllJKqZMnTyqDwaBOnz6tlFJq/vz5qkePHio9PV2dO3dO9ezZUy1YsECDWRTi\nlyT0hUfaunWrMhgMatGiRbUu63A41K5du2pc5m+zZ6vnvb2rBfnHoAxXPCZXeT0A1LYqPy8D1RGU\nBdR9oHJA9QoMVHv27FFTpkxRr7zyinN7p06dUkOHDlV+fn6qe/fuKj4+3vnali1bVOfOnZXNZnM+\n9+KLL6rQ0FAVGhqqZs2aVY8ZE8I1ckWu8Ehnz56le/fuZGRkaPL59o8//shDQ4dytKgIrXpkHgZu\nCwzkbFYWAwYMICEhgZCQkFp/b/bs2bRt25bp06drNBIhXCehLzyOw+Fg5syZFBYW8tFHH2m23l93\n68brR45wt0bre85sxvLss/ztrbc0WqMQjU9CX3iUoqIiwsPD6dy5M+vWraN9+9qaHrtu9erVvDxx\nIvuLivBt4LpSgGEWCwfS0jQdoxCNTUJf6Mrdw4fTecsW3rXb672OEmCg2cy0t97imWef1W5wQjQB\nKdkUupGamkrMgAGsCgjgryYT9TnbKQbGentjiorCarOReUWNvhCeTkJf6MLOnTtZsWIFPj4+vDl3\nLis6dWKcnx9Xb4F2dQeAWywW2tx1F088+yz5+fksXLiQM2fONNawhdCchL5o0ZRSxMXFERcXB8Ad\nd9zB+PHj2ZWcTPspU+jl58dfTSZ+qmEdB4AnfH0ZGRDA83PnsnT1aqZOnUq3bt0oKSlh8eLFHDp0\nqEn2R4iGks/0RYtls9lYs2YNKSkpmEwm7r33XmJjY6stk5yczHv/+798/sUXdDSbuclmI7ysDIfB\nwAk/P/Y6HNh9fPjds8/yxFNPVWv94HA4+Pbbb/nxxx8xGAyMGTOG/v37N/VuClEnEvqiRSotLeXz\nzz/n1KlT+Pj48PDDD9d4M/OysjKSk5PZv38/ly5dwmQy0aFDB/r27UvXrl0xGq/+R7FSim3btjlv\nknLrrbdy++23YzC40txBiKYnoS9anPz8fJYsWcLFixcJDAxk/PjxRERENOo2Dxw4wFdffYXD4SA2\nNpZ7770Xk0mry8CE0I6EvmhRLl68yJIlS8jPzycsLIwJEybQqlWrJtn2sWPHWL58OeXl5XTp0oWH\nH364WstnITyBhL5oMU6dOsXnn39OaWkpHTt2ZNy4cfj5+TXpGM6fP8+yZcsoLCwkIiKC8ePHExgY\n2KRjEKImEvqiRUhNTWX16tXY7XZ69OjBAw88gPcVLY+bSk5ODkuWLOHSpUsEBwczYcIE2rRp45ax\nCHElCX3R7O3cudNZktm/f39Gjx59zS9em0pxcTHLli3j3Llz+Pn5MW7cODp27OjWMQkBEvqiGVNK\nsX79enbu3AlU1ODfcsstHlM5Y7VaWblyJWlpaXh5efHggw/So0cPdw9L6JyEvmiWXKnB9wRSyy88\njYS+aHbqWoPvblLLLzyJhL5oVtxRg68VqeUXnkBCXzQb7qzB14rU8gt3k9AXzYIn1OBrRWr5hTtJ\n6AuP50k1+FqRWn7hLhL6wqN5Yg2+VqSWX7iDhL7wSJ5eg68VqeUXTU1CX3ic5lKDrxWp5RdNSUJf\neJTmVoOvFanlF01FQl94jOZcg68VqeUXjU1CX3iEllCDrxWp5ReNSUJfuF1LqsHXitTyi8YioS/c\nqiXW4GtFavlFY5DQF27TkmvwtSK1/EJrEvqiyemlBl8rUssvtCShL5qU3mrwtSK1/EIrEvqiyei1\nBl8rV9byDxo0iBEjRshfSKJOJPRFk5AafO1ILb9oCAl90eikBl97Ussv6ktCXzQqqcFvPFLLL+pD\nQl80GqnBb3xSyy/qSkJfNAqpwW86Ussv6kJCX2hKavDdQ2r5hask9IVmpAbfvaSWX7hCQl9oQmrw\nPYNSiq1bt5KQkABILb/4JQl90WBSg+95pJZfXIuEvmgQqcH3XFLLL65GQl/Um9Tge77z58+zdOlS\nioqKpJZfABL6op6kBr/5kFp+UZWEvqgzqcFvfqSWX1SS0Bcukxr85k1q+QVI6AsXSQ1+yyC1/EJC\nX9RKavBbFqnl1zcJfeGklOLSpUuEhYU5n5Ma/Jarplr+qx0LomWQb98EUPFn/7p165g3bx4nT54E\nKmrwP/roIy5evEhYWBhTp06VwG9BbrzxRh599FHMZjNJSUksXbqUsrIyADZv3sy8efNITU118yiF\n1uRMX2Cz2Vi9ejUHDx4EwMfHh9tvv534+HipwdeBK2v5o6OjnbdkNBgMjBo1ioEDB7p5lEIrEvo6\nV1JSwueff87p06edz2VmZnLixAl69+5Nnz59pAZfBypr+Y8cOcKxY8eIiYnB39/f+fott9zCHXfc\nIZ/7twDy8Y6O5eXlsXDhwmqBf+7cOVJTUykpKaGgoIC7775bAl8HQkJCGDNmDOfOnaOkpIT9+/eT\nl5fnfH3Hjh2sWrUKm83mxlEKLUjo61RGRgYfffQRmZmZzueOHz/OsWPHAIiKiqJNmzYsX75c3ug6\nkJ2dzZdffklMTAxhYWFYrVYSExPJyspyLpOSksLSpUspLS1140hFQ0no69DJkydZuHAhBQUFQMWX\nuAcPHuTs2bMYjUZ69OhBZGQkAKdPn2bbtm3uHK5oZEopvvzyS4qKijAajURHR9OuXTscDgepqamk\np6c7lz158iQff/wx+fn5bhyxaAj5TF9nkpOTWbNmDXa7Haj4EjclJYXc3Fy8vLyIjo4mJCTEuXz3\n7t158MEH5SOeFi47O5slS5aQnZ3tfO706dPOSq6OHTtWuzYjODiY8ePH07Zt2yYfq2gYCX2dUEqx\nc+dO1q9f73yurKyMpKQkioqK8PHxISYmhoCAAOfr/fr1Y8yYMdJXRyeKiopYtmxZtTP7CxcukJaW\nhlKK8PBwunfv7vwy19fXl3HjxtGpUyd3DVnUg4S+DjgcDuLi4vjhhx+czxUVFZGUlERZWRn+/v7E\nxsbi6+vrfH3EiBEMGjRIqjV0pry8nBUrVnD06FHnc9nZ2aSmpmK32wkJCaFXr17Oi7hMJhMPPPAA\n0dHR7hqyqCMJ/WbK4XBw/Phx9u7dy5kzZ3A4HISEhNCnT59qAX5lDT5Abm4uKSkp2Gw2goODiYmJ\nwcvLCwCj0cjYsWO58cYb3bJfwv0cDgfffPMN+/btcz5XUFBAcnIy5eXlBAQEEBsbi9lsBq5ey5+T\nk8PevXtJTU2luLgYHx8funXrRt++feUCPzeT0G9msrKy+HD+fOa/8w6GkhL6Go10KSnBqBSZPj7s\n9/LiRFkZv33wQaY98wyHDh36RQ3+oUOHcDgctGnThh49ejg/vjGbzTz88MNERUW5a/eEh1BKsXnz\nZjZt2uR8rrS0lKSkJIqLi/H19SU2NrZaLf/AgQMpLS1l3ptvsnPvXm708yO2tJRAq5USLy9SfX3Z\nW1ZGt65dmTFrFg8//LDzHw7RdCT0mwmlFMuWLmXmjBncbbXydGkpN11j2YvAR0Yj/zKZ6BoTw9BR\nozCbzZw7d85Zktm+fXu6du3q/PgmICCA8ePHc9111zXNDolmYd++fXzzzTc4HA6goj1zcnIy+fn5\neHt706tXL4KDg8nJyeG7lSvxzc7mxdJSfgP4XmV9NuA/wL8sFrIjIli0YgV9+vRpwj0SEvrNgN1u\n53ePP86uNWtYVFTEr138vRxghpcXmy0WBg4f7qzMiIqKcpZkArRu3ZoJEyZUq9oRotKRI0dYsWIF\nVqsV+LnENysrC6PRSGBgIJu/+46XbTZmKoUrt19XwBLgj35+vPX++zw+aVIj7oGoSkLfwymlmDZh\nAqfWrGFtcTEBtf/KL7xhMPAPb2+i+/UjJiaG8PBw52uRkZGMGzeu2p/pQlwpPT2dZcuWUVRUBFQc\nl0ePHuXgwYOcTE7mPw4Hg+qx3sPAHf7+vPHBBzw6frymYxZXJ7V4Hu7TxYvZs3YtX9Uz8AFmKcVU\nq5Ws06er1VV369aNiRMnSuCLWrVv356pU6cSGhoKVHx5GxkZyelDh1hZz8AH6A58V1zMs088wfHj\nxzUbr7g2OdP3YBkZGcR07UpcYSEN/dSzHOjt7U3Pu+8mJjaWX//619x5551Sgy/qpGot/39WrqTf\n4cO8f/lCv4aYYzTy1U03sXH3bikTbmTyjvdg8+fO5X6r1Rn4y4FbAAswzIXfXwZ0AgKAh4G/Wa3s\nTkhg+PDh3HXXXdUC/9VXXyUmJgZvb2/+8pe/1LruWbNmERYWRlhYGC+99FId90w0VxaLhccff5zW\nrVuTlpbGG5cDvyHH5v3ARIeDc4cOsWvXrl8sW5djc+PGjQwbNoxWrVrRuXPnOu2bXkjou0lGRkaN\nr9vtdj547z2evnxTC4DWwEzAlYhNBZ4ElgIZgD8Vb0y/8nKMRuMvzqauv/563nrrLe66665az7QW\nLFjA2rVrSUpKIikpia+//poFCxa4MCrRHNR2bJrNZjJ/+onHDAYCLz/X0GPzGeCp4mLef/vtXyxf\nl2MzICCAadOm8dZbb7kwEn2S0G+g48eP07p1a/bv3w9U3JCiTZs2bNmy5RfLFhcX8+mnnzJ8+HBG\njBhR43oPHTqEv9VK1VuPjwB+A7hSVLkUGAsMouLs66/AamBsWRnfx8X9YvmJEycyevRoAgMDqe0T\nv08++YQXXniBdu3a0a5dO1544QUWLVrkwqhEU2qsYxMg/quv+M3lah7Q5ti8Uyk2XL55S1V1OTb7\n9evH+PHj5Sy/BhL6DRQVFcUbb7zBhAkTKCkpYfLkyUyePJkhQ4Y4l9m5cyfTp0+nQ4cOLFmyhOnT\np7N3717n6zNmzCAkJKTaY8CAAZwuLKS+18UeBHpX+bkL4ANEOBzs3by5nmu9vO6DB+nd++e1x8bG\nym31PFBjHZshISHsSE3luXqO61rHZglgLSvj/Pnz9VyzcIWEvgamTZtG165d6d+/PxkZGcyePRuA\n5cuX0717dyZPnkxUVBQpKSnExcUxbtw4fHx8nL///vvvk5OTU+3x4osvMgs4UM8xFQLBVzwXBIRC\ntSt067XuwkKCg39ee1BQEIWFhQ1ap2gcjXFsHjp0iDa+viTVc0zXOjYLga4+Pg0+PkXNJPQ1Mm3a\nNFJTU3nmmWecbYjT09M5f/68sx9OXdrQ2u12vBowngAg74rn8oDAy+tuiICAgGr91PPy8qp15xSe\npVGOzQZUfdV0bJpo+PEpaiahr4HCwkL+8Ic/MG3aNF577TVycnIAeP7550lPT2f48OHMnj2byMhI\nZs6cyYED1c/fn3zySQIDA6s93nzzTWYDMVfZnisFbdFAYpWfj1NRthkIhLRqVePv1vZlWXR0dLV9\nSExMpFevXi6MSjS1xjg2u3fvzk/FxZofmzcAWZcbB16LlHM2nIS+Bp577jn69+/PBx98wF133cWT\nTz7pfC0wMJDp06ezfft2Nm/ejK+vL/fccw+33367c5n58+dTUFBQ7fHdd9/RLziY5CrbcQClgPXy\nf5dd/u+rGQ98DWwDioBXgQeBQ8CNAwfy+uuvM2zYz8V1NpuN0tJS7HY7VquV0tJSZ7+VU6dOYTQa\nOXPmDFDxxdqcOXM4f/486enpzJkzh0lyGb1Haoxjs6CggOsjIvisyna0ODYdwLnSUr744ot6H5tK\nKUpLS7FarSilKCsro7y8vEFz2OIo0SBr1qxRHTp0UDk5OUoppQoLC1XXrl3VsmXLrvk7DodD7dq1\nq8b15ubmqkCzWeWCUpcfH4MyXPGYXOX1AFDbqvy8DFRHUBZQ94HKATU2IEAtXrxYTZkyRb3yyivO\n7T3++OPKYDBUe3zyySdKKaW2bNmiOnfurGw2m3P5F198UYWGhqrQ0FA1a9ashkyhaCSNdWwqpdRj\nDzyg3tX42PwG1K0xMQ06Njdu3Ohcxmg0KoPBoIYNG9aQaWxx5IpcD/bw3Xcz6NtveUaj/0Vngd5+\nfpy5eJHBgweTkJDgUpO12bNn07ZtW6ZPn67JOETzl5CQwHP33ktSYaFLH+m4YmxAAPf+61/MnTtX\njs1GJKHvwbZu3crE0aNJKS7GosH6Zvj44D1pEu/Mn6/B2oSeKaXo1bkzs0+f5j4N1rcXGBMYyKkL\nF6QXVCOTz/Q92ODBgxl85528VKWErr42AV/5+/P63//e4HUJYTAYmLd4MU/7+XGpgesqByZZLMx5\n7z0J/CYgoe/h3vngA74ODubfDahaOAKM9/fnw6VLpWe+0MyQIUN4dNo0HvT3p7ie67ADk3196TZk\nCOMnTNByeOIaJPQ9XEhICOu3buX10FD+ZjJR1wrmLcAwf3/+5//+jzFjxjTGEIWO/eOf/6TjnXcy\n0t+fs3X83Rzgt35+ZPTpw6erVkk5ZhOR0G8GbrjhBrbv28d/evWiv7c3P1Bx56Ga/ATMMJn4bUAA\nHyxfzuSpU5tgpEJvTCYTi774gtEvvEBvb2/eo6KdQk2swOdAd29vrnv0Ub5JSMDPz6/xBysAML3+\n+uuvu3sQonZBQUHYTSZ+slr5v6wslnl7k2m1UkxFfXMuFR/jfAW8abHwR5MJW3Q09z/6KA8//LC8\nqUSjMRgMBIWEUOpwsCY7m7+UlHDaaCTbbsdARS3+eSq+V/rYy4tpPj7svu46bhs7lkcfe4zrr7/e\nrePXG6neaSZSUlJYuXIlFouF3//+92zZsoX4devYu3Urp8+dw+FwEBocTJ8BAxg4bBi//e1viY+P\nJzExkZ49e/LQQw+5exdEC1VeXs67775LQUEBDzzwAIGBgaxcsYIfN20iJTmZ4tJSfMxmunfrRt/b\nbuO+Bx4gODiYhQsX4u3tzTPPPENQUJC7d0M3JPSbAZvNxty5c8nNzeWee+6hb9++Lv1efn4+7777\nLlarlSlTptCxY8dGHqnQo02bNrFp0ybat2/PtGnTXP5sfsWKFaSmptK7d2/uv//+Rh6lqCSf6TcD\nu3btIjc3l/DwcPr0cf3GiUFBQdxyyy0AxMXF1dqLXIi6ys/PZ/v27QCMGjWqTl/G3n777ZhMJhIT\nE6WdchOS0PdwRUVFbN26FYCRI0fW+Z62t956K4GBgaSnp5OcnFz7LwhRBwkJCVitVnr27FnnvyRD\nQkIYOHAgICclTUlC38Nt3LiRsrIyrr/+eqKiour8+2azmeHDhwOwYcMGrNZrtcESom7Onz/PgQMH\nMJlM1Zq01cXgwYPx9/fn9OnTHD58WOMRiquR0PdgFy9eZO/evRiNRkaOHFnv9fTu3ZuIiAjy8/PZ\nuXOnhiMUeqWUYv369QAMGDCA0NDQeq3H19fX2VHz+++/l176TUBC34OtX78epRR9+/alTZs29V6P\n0Whk1KhRAGzbtk3uciUaLC0tjVOnTuHv71/t9ov1UXl8Z2dns3v3bo1GKK5FQt9DHTt2jGPHjuHj\n48PQoUMbvL7OnTvTrVs3ysvLSUhIaPgAhW7Z7XbnWf7QoUPx9fVt0Pqq/iW7efNmiovr29RBuEJC\n3wM5HA7nm2rIkCFYLFr02IQ77rgDo9HI/v37uXDhgibrFPqzZ88esrOzCQsLc7l8uDZdu3YlKiqK\n0tJSNm/erMk6xdVJ6Hugffv2cfHiRUJCQhgwYIBm6w0LC6Nfv37Oz2OlWkLUVUlJiTOUR44ciclk\n0mS9BoOBkSNHYjAY2LNnD1lZWZqsV/yShL6HKSsrY+PGjUBFHbOXV0Nuj/5Lt912G76+vpw4cYKj\nR49qum7R8m3evJmSkhK6dOmiefuE8PBwbrrpJhwOB99//72m6xY/k9D3MFu3bqWoqIjIyEh69uyp\n+fr9/f257bbbgIoviqVaQrjq0qVL7N69G4PBUOcLsVw1bNgwzGYzaWlpnDx5UvP1Cwl9j5Kbm8uu\nXbuAul/dWBf9+/cnNDSUrKws9u7d2yjbEC3P999/j8PhoE+fPoSHhzfKNgICAhg8eDBQccFW5Q3Q\nhXYk9D3Ihg0bsNlsxMTE0KFDh0bbjslk4o477gAq+qaUlpY22rZEy3Dy5EkOHz5c7WK/xjJw4ECC\ng4O5cOECiYmJjbotPZLQ9xBnz54lJSUFLy+vel/dWBfdu3enU6dOFBcXs2XLlkbfnmi+HA4HcXFx\nAAwaNIiAgIBG3Z63t7fzPZCQkEB5eXmjbk9vJPQ9gFLK+aa6+eabCQ4ObvRtVn4uC/DDDz+QnZ3d\n6NsUzVNSUhIXLlwgODiYm2++uUm22atXL9q3b09BQYGzoZvQhoS+B0hNTeXcuXNYLBYGDRrUZNtt\n164dvXv3xm63s2HDhibbrmg+ysvLiY+PB2DEiBF4e3s3yXarnpTs2LGD/Pz8JtmuHkjou5nNZnMG\n7vDhw/Hx8WnS7Ve+kQ8ePMiZM2eadNvC8+3YsYOCggLat29PTExMk267Y8eOREdHY7Vanf/wiIaT\n0Hez+vbK14r03BfX0pBe+VqRnvvak9B3o4b2yteK9NwXV9OQXvlakZ772pPQd6OG9srXivTcF1fS\nole+VqTnvrYk9N1Eq175WpGe+6KSVr3ytSI997Uloe8mWvXK14r03BeVtOyVrxXpua8dCX030LpX\nvlak577Qule+VqTnvnYk9JtYY/XK14r03Ne3xuiVrxXpua8NCf0m1li98rUiPff1q7F65WtFeu5r\nQ0K/CTV2r3ytSM99fWrMXvlakZ77DSeh34Qau1e+VqTnvv40Ra98rUjP/YaR0G8iTdUrXyvSc19f\nmqJXvlak537DSOg3kabqla8V6bmvH03ZK18r0nO//iT0m0BT98rXivTcb/maule+VqTnfv1J6Dcy\nd/TK14r03G/53NErXyvSc79+JPQbmbt65WtFeu63XO7qla8V6blfPxL6jcjdvfK1Ij33W6bt27e7\nrVe+VqTnft1J6Dcid/fK14r03G958vPz2bFjB9A8qslqIj3360ZCv5F4Sq98rUjP/ZYlPj7e7b3y\ntSI99+umeSeRB/OUXvlakZ77Lcf58+dJTEz0iF75WpGe+66T0G8EntYrXyvSc7/5q1pN5gm98rVy\nZc99m80MT4qgAAAUdElEQVTm5hF5Lgn9RuBpvfK1Ij33m7/Dhw9z+vRpj+qVr5WqPff37Nnj7uF4\nLAl9jXlqr3ytSM/95stutzublHlSr3ytSM9910joa8jTe+VrRXruN0+7d+/22F75WpGe+7WT0NeQ\np/fK14r03G9+iouLPbpXvlak537tJPQ10lx65WtFeu43L5s3b6a0tNSje+VrRXru10xCv55KS0tZ\ntGiRM/CaS698rVyt535OTg4rVqzgyJEjbh6d2LBhAxs3bqS8vJysrCz27NnTLHrla6Vqz/0TJ05g\nt9vZtWsXX375pbuH5nYt+3S0EW3bto1Tp05x6tQp2rVrx4kTJ/D19dXNmwoqeu7v2bOHCxcu8P77\n75Obm4vdbicjI4OoqKgW+xGCp8vKymLHjh04HA727t2L1WrFbrfTt29fj++Vr5XKnvsbNmxg8eLF\nhISEkJOTA1SUHnfp0sXNI3QfOdOvh5ycnGp16hs2bGDXrl2UlZXRqlUrN46saRkMBiIiIvjhhx/4\n6quvKCsrA5Abr7hZ5Q1RAM6dO0dcXBwHDhzQXdB16tSJI0eOsGnTJg4dOuR8fv369bq+8YqEfj3E\nx8c7byGYn5/PxYsXnW0W3n33XbZu3driLw45evQo8+bN4+DBg1gsFqxWK6dPn3a+vmnTJkpKStw4\nQn06ceIEaWlpQMWFWMeOHQMgNDSUlStX8tlnn7X4LzcLCgpYs2YNH3/8MSEhIQDOj3gA3d94RUK/\njipviFKp8k0VGRmJj48PZWVl7Nq1q8XfVzYlJYXMzEwAZ5uJ9PR0Z9AXFxc7ew+JplG1ZBggIyOD\nwsJCfH19iYyMBCAtLa3FNyXLycnhwIEDKKVo27YtQUFBlJeXc/bsWecy8fHxur3xioR+HVS9hB0q\n2i3k5+djNpurNa1qzm2UXTV8+HBn//XAwEAiIiJwOBycOHHCuYzceKVpJSYmOq+bsNvtzpuGd+7c\n2fmXaLt27ZptG2VXdezYsVoxReVJydmzZ50fQRYWFur2xisS+nVQeUMUoFrAde7c2fmlZdu2bZt1\nG2VXBQcHO9stw89zkJmZSV5eHoDceKUJVb0hCvwccEFBQdW+vNVLoUFlu2WoOFbbtm1b7R9CqLjx\nSuWxqicS+i6yWq3Van7PnTtHaWkpAQEBREREOJ8fNWpUs2+j7Kpbb73VeU9VHx8f50cIx48fdy5z\n8ODBap/1i8axfft2Zy+ksrIy50cZVTu89uzZk06dOrllfE0tNDS02gWSXbp0wWg0cuHCBQoKCoCK\n97QeW4noI53qQSlFfn4+ly5dorS0lB9++MF5VmC1Wp13kIqKinKeObWUNsquMpvNjBgxwvlz5fca\n+fn5ZGRkOJ+Pi4vDZrORk5NDdnZ2i/+SuymUlZVx6dIl8vPzyc3Ndd4QBeDkyZPY7XbatGnjvCdz\nS2qj7KohQ4bg7+8PVHTh7NChA1D9pKTyxiuFhYVcunRJF8UHUqdfRV5eHos/+YRvv/iCH5OSKCkr\nw9topMRmI8xiIaJ9e3r07YvNZsNms9G6dWtndUBLa6Psqt69e/PDDz9w4cIFTCYTnTt35vDhw5w4\ncQKj0Ujyvn2cP3aMPzz9NObLfwFZlSImKorBI0cybcYMunXr5ua98HxKKRISEvh0wQJ279zJiZ9+\nws/LC6vDgbfJRLvWrekYHU2Xrl25cOECRqOxWolmS2qj7CpfX1+GDh3Kt99+C1R81v/TTz+Rm5vL\nTz/9xIULFziZmMjcN9+k2GrF12Si2GajfevW9OvXj0emTePuu+9ucVfXG5Q0TqGsrIy/vvoq782d\nyyijkUeKiugHtAMMgBU4BGwB3vP2JsNgILJ7d0aNGuU8k+jXrx933XWX2/bBnU6cOMHixYuBinDa\ntm0bafv2UVRQwFSluMfhoA8QdHn5AmA/sM7Li397efHrAQN4b9EifvWrX7lnBzxcQkICT0+ahFdO\nDtMLCxkCRAOVtzE/D/wIfOrtzbd2O+HXXceAwYOd/5j6+/vz7LPPtriumq6w2+3MmzfPWaZ67tw5\nNick8NOZM9xqMjGlvJwBQCcq3ut24AiwA1gYGMgZs5m333uPhx56qMV8F6L70D906BC/GTOGGzIz\nmVtcTPtallfAemCyyUTH6GjuGDvW+aZqqV01XfHZZ5+RlpZGanIy67/+mudtNl5Sitpipgz4l8nE\nWz4+zHnvPSZOmtQEo20erFYrM2fMYM2yZcwrLuYuKoKpJj8BvzMa2WOxcN/48YSHh3PnnXfSv3//\nJhixZzpy5AjLli0jPz+fNUuXEn7pEovtdlzpQLQTmGqx0HPIEBYtX+78DqtZUzqWnJysIoKD1UcG\ng3KAUnV45IEa4+Wlojt3Vps2bXL3rrhdZmamGnv33SrC21sdqONcKlApoH7l76/e/de/3L0rHsFq\ntaoHRo9WY/z9VU495vMTUK3MZjVr1ixls9ncvTtu5XA41Jw5c1Qbi0XNNhiUvY5zWQJqiq+vujk2\nVhUUFLh7dxpMt6Gfk5OjOrZpo5bW4w1V+SgHNdrbWz09bZq7d8ftEhISVFsfH3W0AfN5ElQHf3+1\nbt06d++O273wzDNqpL+/KmvAfC4HdV1wsMrKynL37rhVaWmp6tmpk3rLYKj3XDpATfX1VQ+MHq0c\nDoe7d6lBdBv6kx95RD3p41Pvg6DykQ2qvZ+f2rhxo7t3yW0KCgrUr9q2Vf9p4FwqUN+DimzdWuXm\n5rp7t9xm+/btKsLPT2VqMJ/P+fioR++/39275FYvv/iiutffv85/zV/5KAXV02JRny1b5u5dahBd\nhn5iYqJq5++v8jV4UylQq0Hd2LVrsz8DqK/Z//3f6lE/P03mUoGa7OurXv3Tn9y9W24zsFcvtUyj\nuSwC1dHfX+3evdvdu+UW6enpKsTXV53XaD53gmofGqrKy8vdvWv11mxDv6ysTE2ZMkV16tRJBQYG\nqhtvvFF99913Nf7OnDlzVEREhDJ7e6ubDIY6/emcB+o5UB1BBYCKAvUHUFmg7KC6Wixq586dtY77\n0qVL6r777lMWi0V16tRJLfOQs4a6zmdycrIaOXKkCgsLU4DaX8c3T03zmQrqulatXHpjvfvuu6pv\n377Kx8dHTZo0Scspqbe6zuWiRYtU3759VVBQkAoPD1eBXl6aHZsK1BtGo5r8yCO1jtsT51Kpus/n\nZ599prp166aCgoKUv5+f6mw0qnQN53NQYKBatWpVreP21Pd6s704y2az0bFjR7Zs2UJ+fj7/8z//\nw0MPPXTNqz/j4uJ44403iI+Px8/LCx+leM3FbZUDI6go24yjouRwJxAG7KbiCrfJxcV8tmhRret6\n+umn8fX15eLFiyxdupSnnnqKgwcPujiSxlPX+TSbzTzyyCPMnDkTgBvrsK3a5rMn0MnhcOkep+3b\nt+fVV19lypQpdRhB46rrXJaUlPDOO+9w6dIlHrjnHgJsNv7p4rZqm0uAyQ4HK778stYmgJ44l1D3\n+bz11lvZsmULeXl5dAgJoZvDwUwXt+XKfE4pKOCzDz+sdV2e+l5vtmf6VxMbG6tWr1591dfGjRun\nXn75ZXX06FHV0WJRCaAiXPyX/0NQ4Zf/VL7WMhtB3RIdXeP4CgsLldlsVkePHnU+N3HiRPXSSy9p\nOg9aqWk+K/35z39W1PEs35X5nOntrf7+t7+5PNZXXnnFo85Or+TKXCql1LCbblJPgLpHw7lUoK4P\nCFApKSkujdXT51Ip1+YzPz9f+Xl5qccun6lrNZ+HQf2qTZsat+3J7/Vme6Z/pYyMDI4cOUJ0dPRV\nXz948CC9e/cmOTmZWJOJWCADyHFh3RuAMYB/Dcv8HdiRmkpISMgvHmPHjgUq6oW9vLzo2rWr8/d6\n9+5NamqqazvZhGqbz0pp+/fXWjt+JVfmM95q5f+99lqN81mVUqqOo2g6rs4lQMqRI5wGerm4blfm\n8m7gVGEhAwYMaPZzCa7N57Zt22jXrh0lNhtngTdcXLcr8/lH4FRmZrN9r7eI64utVivjx49n0qRJ\n3HDDDVddprCwkODgYC5evEigw1Ht6tCQWtafDfy6lmX+A/gYDM5bsl1rDEFBQdWeCwwMdDaA8hSu\nzGel4qKiOq/flfl8Dfhk0CDWuNgQy1OvlqzLXALklZSQCixzcf2uzOU3wBSLhVvfeYepU6fWuk5P\nnUtwfT4HDRrE2rVreeW++/AuKOC/gHdcWL+r89nG15eUw4eveftJT36vN/szfYfDwWOPPYavry9z\n58695nIBAQHk5+dX3OjEaKSyoWqgC9toTcWl7jUpBXxq6dFROYaq8vLyCAx0ZRRNw9X5rGSux30D\nXJ3PuqzbE89O6zqXa9aswWq38zngapccV+YSoNRgwGw2u7ROT5xLqMexaTaD0chfgcUubsOV+VRA\nmcNR4z0zPPm93qxDXynF1KlTyczMZNWqVTXeiDs6OpoDBw7QrVs3DipFIhBO7Wf5ALdT8aVOcQ3L\njATKbDYCAwN/8ajsyXPDDTdgs9mcd9uCii5/vXq5+sd846rLfFbq0rMndY0IV+bzVWBtQkKN81mV\np52d1nUu161bxxNPPEHPTp2oy91bXZnLMcAXhYU8+eSTzXIuoX7H5g033MCh0lLKqfnjmqpcmc/h\nQGF5OZGRkc3zve6+rxMa7ne/+50aOHCgKiwsvOrrBoNBbd68WSml1Lp161RERIRKSkpSfl5eahCo\nP1X5cuZxUJOu8cVNGah+oEZf/hLHfrl8azaoby8vMxfU1HHjah3zI488osaNG6eKiorU1q1bVXBw\nsDp48KCm81JfdZlPpZQqKSlR8+fPV1Bx4UqphvM5KihIrVmzptYx22w2VVJSol566SX12GOPqdLS\nUo9oO1CXuYyPj1ehoaFq69atasbUqertK+aroXNZBMrP21uVlJTUOGZPnUul6jafS5YsUWfOnFFK\nKXVdcLDqB+oZDedzFagxt95a65g99b3ebEP/1KlTymAwKD8/PxUQEOB8VNbCnjlzRgUFBans7Gzn\n78yZM0eFh4crLy8vdQsVbRQq/2ePAPVRDd/Y512uAIjk59rdP1JxRa4CNSAwUK1du7bWcWdnZ1er\n3f3ss88abY7qoq7zefLkSWUwGJTBYFCAMoDqrNF8poMK8fNTeXl5tY77tddec46j8vGXv/ylUeeq\nNnWdy2HDhilvb28VEBCg/Pz8lBHUnRoemwtB3XXbbbWO2xPnUqm6z+fLL7+sOnTooCwWiwoMDFS3\nGAyqRMP5vN9iUfPnzat13J76Xm+2oV+bJUuWqD//+c9XfS0+Pl5FBwQoW5V/3XuC8+e6PnaB6hgW\n5jFnRY2hpvn8w1NPqf/y9q52ttSQ+XzFZFJPTZ7cxHvYdGqaS7vdrrped53arNFc2kH1CQhQ3377\nbRPvZdOpaT5TU1NVhJ+fswSzofN58vIJSXNuvNZiQ78mDodDDevfX71pMtXrf3zVRzmoGy0W9fHC\nhe7eLbc5e/asCrNYVGID51KBOgiqtb+/OnnypLt3y20+W7ZM9fT3r/ZxWX0f7xiN6pbevZXdbnf3\nbrnNo/ffr57XoM+WA9Qd/v5qtgf89dMQugx9pZQ6ceKECrNY1A8NPBD+6O2t7rztNt323an0748+\nUjEWi8ptwFwWgLrJYlHvz53r7t1xK4fDoe4fNUr93mxuUJOwfZf/AU1LS3P3LrlVVlaWuq5VK+dn\n8vV9/K/JpPp2766sVqu7d6lBdBv6Sin11VdfqbZ+fmpXPf/Vf9XbW/Xo1EllZma6e1fczuFwqKen\nTVMD/f3VpXrMZy6oIf7+atqECbr/B1Spis+DY6Oi1Itmc537vytQe0FF+PmplStWuHtXPML27dtV\nmL9/vYN/rtGoIsPCWsRfoLoOfaWU+vrrr1WYxaL+6uVV7Yvdmh6nQN1usagBvXqpjIwMd++Cx7Db\n7erF555THfz969RmeT0VnSCfe/JJXX8McaWsrCw1qE8fNdRiUcddnEsrqL+bTKq1v79LbR/0ZPv2\n7eq6Vq3UTLO51rYVlY8MUL/x81M9OnVSJ06ccPcuaEL3oa9Uxbf/owcPVp0tFvWG0ahOXz6Tr/o/\nvxzUDiruoNPK11f97b//u9n/mddY4uPjVefwcHVbYKD6/PJZ/JVvpjxQK0ANDwxUHcPCVFxcnLuH\n7ZFsNpt66x//UCF+fmqin5/aBr/owOkAdRbU2waDirJY1IiBA1vEGWljyMzMVI+MHauu8/NT/89k\nUkev8l63UdE19vc+PirU11e9+Ic/1Fru2pzo/h65Ve3Zs4f33n6bb7/7DoPNRnezGTOQCxwsLuZX\nERFMeOIJpj7xBG3btnX3cD2a1WplzZo1LHj7bXYdOEA7s5nIyxfUpDscnC0tpX9sLE/88Y88+OCD\nLl8tqldZWVl8/O9/s3jePI6np9PD359Qg4FyIK28HLvJxOiRI5nxwgsMHDjQIy+w8iQpKSm8/89/\nsnb1akpLSujp64svFW1ZUouLiWjdmkcmTeKJGTOIjIx093A1JaF/FUopzp07x7Fjx7BarQQFBREd\nHe0Rl1A3RzabjcOHD5ORkYFSivDwcHr06IFXLW0rxNUVFhaSmppKXl4e3t7eREVFERkZKUFfTxcu\nXCAtLY2ysjIsFgvR0dG0atXK3cNqNBL6QgihI826944QQoi6kdAXQggdkdAXQggdkdAXQggdkdAX\nQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggd\nkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAX\nQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggd\nkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAXQggdkdAX\nQggd+f+smSJTnTuXvgAAAABJRU5ErkJggg==\n"
133 | }
134 | ],
135 | "prompt_number": 3
136 | },
137 | {
138 | "cell_type": "markdown",
139 | "metadata": {},
140 | "source": [
141 | "Example 2
\n",
142 | "Uses data provided from the \"The Elements of Statistical Learning\" website on South African heart disease [here](http://www-stat.stanford.edu/~tibs/ElemStatLearn/)."
143 | ]
144 | },
145 | {
146 | "cell_type": "code",
147 | "collapsed": false,
148 | "input": [
149 | "sys.path.append('pysrc')\n",
150 | "import decision_trees as dt\n",
151 | "import numpy as np\n",
152 | "import networkx as nx\n",
153 | "from matplotlib import pyplot as plt\n",
154 | "\n",
155 | "p = 'data/SAheart.data'\n",
156 | "f = open(p,'r')\n",
157 | "all_lines = f.readlines()\n",
158 | "train_cnt = int(0.75 * len(all_lines))\n",
159 | "lines = all_lines[0:train_cnt]\n",
160 | "col_cnt = len(lines[0].split(','))\n",
161 | "row_cnt = len(lines)\n",
162 | "outputs = [int(line.split(',')[col_cnt-1][0]) for line in lines]\n",
163 | "inputs = np.zeros((row_cnt, col_cnt-1))\n",
164 | "for row in range(row_cnt):\n",
165 | " line = lines[row].split(',')[0:col_cnt-1]\n",
166 | " inputs[row] = [float(v) for v in line] \n",
167 | "clazz = [0,1]\n",
168 | "\n",
169 | "tree = dt.build_tree(inputs, outputs, clazz, meta=['sbp','tob','ldl','adip','famhist','typea','obes','alc','age'], max_rm=5)\n",
170 | "#dt.draw_tree(tree)\n",
171 | "\n",
172 | "#compare the tree precition to training values, since we haven't pruned this should be very accurate\n",
173 | "diff = []\n",
174 | "for row in range(train_cnt):\n",
175 | " p = dt.decide(tree, inputs[row])\n",
176 | " if p == outputs[row]: diff.append(0)\n",
177 | " else: diff.append(1)\n",
178 | " \n",
179 | "misses = sum(diff)\n",
180 | "print \"In the training data, there were {0} miss classifications for {1} inputs, a rate of {2}%\".format(misses, train_cnt, 100*misses/float(train_cnt))\n",
181 | "x = range(train_cnt)\n",
182 | "f, axarr = plt.subplots(2,1)\n",
183 | "f.subplots_adjust(right=1.5)\n",
184 | "f.subplots_adjust(top=1.5)\n",
185 | "\n",
186 | "#plot training comparison\n",
187 | "ax1 = axarr[0]\n",
188 | "ax1.scatter(x,diff)\n",
189 | "\n",
190 | "#compare the tree prediction to actual values not used in training set\n",
191 | "test_lines = all_lines[train_cnt+1:len(all_lines)-1]\n",
192 | "actual_out = [int(line.split(',')[col_cnt-1][0]) for line in test_lines]\n",
193 | "row_cnt = len(test_lines)\n",
194 | "test_in = np.zeros((row_cnt, col_cnt-1))\n",
195 | "for row in range(row_cnt):\n",
196 | " line = test_lines[row].split(',')[0:col_cnt-1]\n",
197 | " test_in[row] = [float(v) for v in line]\n",
198 | "\n",
199 | "diff = []\n",
200 | "for row in range(len(test_in)):\n",
201 | " p = dt.decide(tree, test_in[row])\n",
202 | " if p == actual_out[row]: diff.append(0)\n",
203 | " else: diff.append(1)\n",
204 | "misses = sum(diff) \n",
205 | "print \"In the hold out data, there were {0} miss classifications for {1} inputs, a rate of {2}%\".format(misses, len(test_in), 100*misses/float(len(test_in)))\n",
206 | "\n",
207 | "x = range(len(diff))\n",
208 | "ax2 = axarr[1]\n",
209 | "ax2.scatter(x,diff)\n"
210 | ],
211 | "language": "python",
212 | "metadata": {},
213 | "outputs": [
214 | {
215 | "output_type": "stream",
216 | "stream": "stdout",
217 | "text": [
218 | "In the training data, there were 95 miss classifications for 345 inputs, a rate of 27.5362318841%\n",
219 | "In the hold out data, there were 27 miss classifications for 114 inputs, a rate of 23.6842105263%"
220 | ]
221 | },
222 | {
223 | "output_type": "stream",
224 | "stream": "stdout",
225 | "text": [
226 | "\n"
227 | ]
228 | },
229 | {
230 | "output_type": "pyout",
231 | "prompt_number": 8,
232 | "text": [
233 | ""
234 | ]
235 | },
236 | {
237 | "output_type": "display_data",
238 | "png": "iVBORw0KGgoAAAANSUhEUgAAAoMAAAGqCAYAAABwJU7tAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl8VOW9P/DPmcwksySTnZANQkhIAgkhsmndoohaFayo\nLdp6vUCRcmutva21trWCbRX0elsrvb9Sq9ZWK2pdsBVSRQ3gAhEIWzZCSMhODAkBsk0y8/39MSeT\nzGQmCckAlfN5v155ycx5zjnf8zzf58k3ZxYVEREQERERkSbpzncARERERHT+sBgkIiIi0jAWg0RE\nREQaxmKQiIiISMNYDBIRERFpGItBIiIiIg0bUzG4dOlSxMTEICsry+v2l19+GdnZ2Zg+fTouvfRS\n7N+/fyynIyIiIiI/G1MxuGTJEuTl5fncnpycjG3btmH//v14+OGHcc8994zldERERETkZ2MqBi+/\n/HKEh4f73H7JJZcgNDQUADB37lzU1taO5XRERERE5Gf6c3Wi5557DjfccMOg5xVFOVchEBEREV1w\nxvo/kzsnHyD56KOP8Pzzz2Pt2rVet4uI5n8eeeSR8x7Dv8MP+4H9wH5gP7Af2A/sh5H/+MNZvzO4\nf/9+LF++HHl5eUO+pExERERE595ZvTNYXV2NRYsW4aWXXkJKSsrZPBURERERjcKY7gzecccd2Lp1\nK5qbm5GYmIjVq1ejp6cHALBixQo8+uijaG1txcqVKwEABoMBBQUFY4/6ApSbm3u+Q/i3wH5wYj84\nsR+c2A9O7Acn9oMT+8F/FPHXC86jDUBR/PaaNxEREZGW+KOO4v+BhIiIiEjDWAwSERERaRiLQSIi\nIiINYzFIREREpGEsBomIiIg0jMUgERERkYaxGCQiIiLSMBaDRERERBrGYpCIiIhIw1gMEhEREWkY\ni0EiIiIiDWMxSERERKRhLAaJiIiINIzFIBEREZGGsRgkIiIi0jAWg0REREQaNqZicOnSpYiJiUFW\nVpbPNvfddx9SU1ORnZ2NwsLCsZyOiIiIiPxMP5adlyxZgu9973v4j//4D6/bN23ahMOHD6O8vBw7\nd+7EypUrsWPHjrGc8oLW1dWF0tJShIeHY+LEiYO2t7e349ChQxg3bhwURUFlZSXsdjtEBDExMZgy\nZQp0Ou/1/dGjR3HixAlMmjQJVVVVsFgsSE5OhqIorjbNzc2orq5GcnIywsLCAAAdHR0oKyvDyZMn\nAcDV3uFwICYmBnFxcThy5AjGjx+P2NhYt3NWV1ejpaUFaWlpMJlMrhjS0tIQFBSEyspK1NbWIjAw\nEFOmTEFERIRrXxFBeXk5GhoaYDKZkJaWhtDQ0EH9EB8ff8b93NraisrKSkycOBGRkZE+23nGMGXK\nFBw7dgy9vb1ITExERUXFiGNoa2tDRUUFEhMT0dHR4eoHo9HoatPd3Y3S0lJYrVZMmjRpyOM1Njai\nvr4eqampCAkJ8drm5MmTOHz4MOLj4xETE+OXGDzzwWw2Iz093RXD6dOnUV5e7jUfPPshOjp60Pbj\nx4+jvLwcNpsNiYmJPmM4dOgQoqKikJCQMGheHDt2DKmpqQgODvbZfz09PSgtLYXRaERKSorbPPDk\nbV54am9vR2lpKU6dOuWaF+Xl5Whvb3fNTYfDgdLSUjQ3N7vmqYggLi7OawyeYzHc+jBQfX09jh07\nhuTkZNTV1UGn0w1aH3yNhYigsrISp0+fds3VPgNjMJlMKC8vh8PhgMPhQFRU1BnNC2/rQ3V1NfR6\nPVJTU3Hy5ElXDIGBgThy5Ag6OjqQnp4OvV7vypO+x4D3eTFcTgLu+RAZGYny8nJ0dXXB4XAgIiIC\n6enpbjEkJSWhoqLCbSxEBIcOHRq0PvjKyd7eXpSWliIwMBCpqamu8R/JWu1tLGprazF58mTXOulp\nYD/Y7XZ88cUXSEtLg9lsHnKcfBnpvPCcm8PlZGlpKbq6unzOCzpLZIwqKyslMzPT67YVK1bIhg0b\nXI/T0tKksbHRrY0fQrgglJaWSkzMJAkJmSZGY6SsWPF9cTgcru27du2S8PA4sVqzJCAgRHQ6swBm\nAcIEGCcGw3i59NL50tHR4XZch8MhS5f+lxiNUWKxpEpAQKhYLKliMsXIbbfdJb29vSIi8vzzL4rJ\nFC5W63SxWCLlH//4p+zfv18iIxMkICBaAKsAQeo5wwWIloCAKAkICJaQkEwJCgqTxx570nXOe+/9\nkRiNkRISMk2ioibILbfcKUZjlISETJXx45PlxhtvE70+VACTKEqyGI1h8vrrb4iIiM1mk+uuu0X0\n+ggBTKLTJYvFEilbtmxx64egoHB5+OFfnlE/v/XW22I2R4jVOl1MpnD5619f9trOMwZFSRKdLlSC\nguLEaIyXgIAQ9bqHj2Hz5s1isURJSEiW6HRWMRjCJSRkqsTGTpby8nIRETly5IjEx6dKSEiGGI3R\nctdd97iN/0BPPvlbCQoKE6s1S0JCxsn27dsHtfnoo48kJCRarNbpYjSGyYoV/zXmGAbng3NsQkKi\nZfv27fLpp5+K1Rqjjk1/Pnj2Q19Mf/zjc27bN2x4VYKCrKIooQIkisEQ6TWGqKhEsVozxWiMkG9+\n8z/d5kVAQIhYrVkSFhYrO3fu9Np/DQ0NkpKSLcHBqWIyxcqNN94uPT09Xtt6mxeedu3aJWFh40Wn\ni3LNC53OIjpdpGtuzplzpaSnzxSDYZwARgFMAoQKECsGw7hBMXiOxcKFXx9yfRjo0UcfF6PROb4B\nAaFiMk0UsznRbX3wNRZ2u10WL14iRuM4CQlJl8TENKmqqhIR9zVKr7eKTmcSIFiACAEiRa8fN6J5\n4Wt90OtD1PVlkpqjkRISki4JCVNk/vybxWQaL8HBUyQpaZrk5t4gJlOcBAenSFraRdLU1OR1XgyX\nk575YDCEq9cVql6XVfT6JImP74/BbE6SgACrBAdPdY1FR0eHXHnlV8VsTnRbH3zlZHNzs0ydOluC\ngyeL2Rwv8+YtkM7OzhGt1d7GIiAgWKzW6RIcHCVbtmwZdI0D+0GnCxG9PlSs1kyJiIiXwsJCr+M0\nlJHOC8+5OVxOmkwRoijhPucFeeePOuqsFoM33XSTfPLJJ67H8+bNk127drkHwGJQRESysi4RRVkn\ngAhwQiyWLHn77bdd2+PjUwXYIMAOAeIFiBNgoQBLBbAL0CNBQbfKgw/+3O24r732mlgsMwQ4qbZ/\nWD1Hu5jNl8mzzz4rR48eFZMpUoASddtnYjZHSHJylgC3CHCVALECJAvwNQHuEKBHjeMf6j61YjbH\ny+7du+Wf//ynWCwZArSo2+4RRUlTYxBRlDtEUaapi+0+tc0eMZsj5Pjx47JmzZMSFHS5ANECVKjb\nP5Lg4CiJi+vrBxHgmFgsSV6LIW9OnDghZnOEAAXq/gfFZIqQurq6QW0Hx7Ba7b8eAUYeQ0dHh1gs\nkQJsF+A1AbJd/aDT/UZmzrxSRETmzp0nOt0T6jFPicUyS15+eXChum/fPjGbYwWoVttulrCw8WK3\n211tenp6xGodJ8D7aptStQAZWwwpKdke+dAfQ2hojEREeM+Hwf0gAhwSkylKjhw5IiIix44dE5Mp\nQoBLBOiPwWye6SWG59Xtx0VRrAPmRYIADeq2N2TcuCSvY7JgwWIxGH4sgEOATjGbr5Hf/vbpQe18\nzYtTp065tXPOTc954T43dbopoig3CJAmQKQA1wrQH4PROM8tBs+xUJRoUZRnfK4PfXbs2CFmc6La\nD/cKcLcrBqPxdnnwwZ8PORYvvPCCWCwXC3BaAJGAgF/J5Zd/VUQGrlGH1HkxVc2Hr53RvPC2PgAp\n6npQIMALAsxxxaAoC0Snu0SATgEcoijzRKe7SoBuARxiMNwv11yzYNC8GC4nB+dDqxpDrgA3C9Df\nD8BNA2K4RIDfuY3F7bffKSaTZz/4zsnFi5eKwXCvOv42MZluksWL7xx2re4zeCzc18nu7m5XW4fD\nMaAfPhTnOv6F2v6vMnHiVK/zxJczmxee/TBcTg49L8g7f9RRZ/0DJM44+3m75btq1SrXT35+/tkO\n6d/S4cPFEFmsPgpFV9dXUVRUBACw2Wyor68AcDuAYgCXAWgDcBrAN+B866ce3d1fx65dRW7HLSoq\nQnv7TQBCABQB+Ka6xYyOjptRWFiEw4cPIzBwGoB0ddvF0OkiUFVVCiAAwDQAFwOoAtAL4OsA2gGc\nAHCTuk88dLrLUFxcjOLiYths1wMIV7dFQOQ2NQZAJBwilwNIBjBdbZMDvT4BlZWV2LOnGN3d2QBm\nqm0AIBd2uwENDX39AADjIHI1iouLR9THR48eRUDAeACz1WemITAwDRUVFYPaDo6hWL1uB4CRx1Bf\nXw8gGM4xKwKwwNUPDsdilJU5x6u0tBgOR9/4B6O9/SYcOFA06HglJSUICPgKgET1mevR2dmFlpYW\nV5svvvgCNpsAuEZ9Rg8gdMwxVFYWoz8fBsdw8qT3fBjcDwCQisDAbBw6dAgAcOTIEQQGJsOZY/0x\ndHQs8BJD/3aR0+ifF1cBGK9uuwXNzXXo6OgY1IcHDhSjp+cbABQARnR03ILduwf3ta95UVtb62rT\nPzc954X73HQ49BCZqPZbMIBGdbszhq6uRW4xeI6FSAdE7lAfu68PAxUXF0NRctV+KAZwpyuGrq7b\nsGtX0ZBjsX9/MdrbFwKwAADs9sUoLi5S+6NvjSqDc16Uw5kPt+FM5oW39QHoi3m2GvfXXDGIhMHh\n+AYAIwAFIlY4HHcACASgoKdnMQ4cOOh1XgyVk33686FajaEGQDSA/n4AwgfE0Nev/WOxb18ZOjs9\n+8F3Tu7bNzAHDejsvBWFhQeHXav7DB6L/nXS4QhEY2Ojq+3Jkydx6lRfPxQDmA8gSt36DVRXl8Lh\ncHgOk09nNi88+2G4nBx6XpBTfn6+W93kD2e1GIyPj0dNTY3rcW1trdf3kQy8qNzc3LMZ0r+tSZPS\noChvqo9Ow2h8D+npzskWGBiImJgkABsBpAH4DM6JEwzgTQACwI7AwDeRk5Phdtz09HRYLHlw/pJK\nB/C6uqULZvO7mD49HZMnT4bNVgTgsLptNxyO40hMnALn4lYGYCeACQAMAN5Szx0C4F/qPsfgcHyC\ntLQ09T0+7wM4qW47AUV5W40BcBayn8K5aPYtzAfQ01ODiRMnIjs7DUFBBwHsgXOBBoBPEBBgw7hx\nff0AAMehKPmufhrOhAkT0NvbAGCv+kwZbLYyJCcnD2o7OIY09boDAIw8Bud7lE7B2X/pADa7+kFR\n3kRKinO/1NQ06HR9498Bi2Uzpk3LGHS8tLQ02O2fAWhQn/kAQUGBbu+3jIqKgsEgALaqzzjg7POx\nxTBhQhr682FwDMHB3vNhcD8AQCVstv1ISUkBACQlJcFmqwAwEc6cdsZgNm/yEkP/dmce9s2LbQC+\nULe9i4iI8TCZTIP6cOrUNOj1ffPGBpPpH8jOHjx+vuZFQkKCq03/3PScF+5zU1HsUJQa9fpPAYgZ\nsN0Go/Edtxg8x0JRzADeUB+7rw8DpaWlQaSvH9IA/N0Vg9H4NnJyMoYci2nT0mA2vwugEwCg072J\ntDTnefrXqFQ458VkOPNhI85kXnhbH4CP4cynvWrc/3DFAJyETvd3AN0AAEU5rT7uASAwGN7A1Knp\nXufFUDnZpz8fEtX9EwAcB9DfD0DbgBjSBo1FZmYKjEbPfvCdk5mZaTAY+sa/FybTRmRlDb9W9xk8\nFv3rpE7XjZiYGFdbq9UKi6WvH9IBfACgVd36FhISfL/X3Jszmxee/TBcTg49L8gpNzfX78XgWX2Z\n+N1335WvftX5EsNnn30mc+fOHdTGDyFcEIqKitT3Ql0kJtN4+c//XOn2nqAdO3aI1RojoaGzJCAg\nVAICgqX//XsJYjBMkNmzr5TTp0+7HdfhcMiddy4TkylWgoOniU4XIhZLppjNCXLTTV93vQ/lD394\nVkymCAkNnS1mc6S8+eZbsmfPHgkLGy8BAePU85jUc0YIECcBAeNFpwsWq3WmGI2R8sgjv3adc/ny\n74nJFCNW60USEREvN9xwq5hMsWK15khU1AS55pqFYjBECGARRUkXozFcXnnF+f7Srq4uyc29QQyG\naAEsotOli9kcKZs3b3brB6MxSh54wP1l8eG8+urrrus0mSLkT396wWs7zxgUZYrodFYxGieKyZQ0\n4LqHj+Gdd/4hZnOkWK2zRKcLFYMhWqzWHImOniilpaUiInLo0CGJiZkkVusMMZvj5Otfv9vtpd+B\nfvWrtWI0Rkpo6CwJDo6WDz/8cFCb9957TyyWKLWfIuTuu5eNOYbB+RAsOl26WCxR8uGHH8rWrVsl\nJGSces7+fPDsh9DQ2WI0Rsgzz/w/t+0vvvhXCQoKVd8zOFkCA2O8xhAeHiehoTPFZBont956h9u8\n0OvDJDR0llitMW5vURmorq5OkpKmSkhIplgsE+WaaxaKzWbz2tbbvPC0Y8cOCQmJVt8z6JwXitL3\nnkHn3Jwx4xJJTp4uBsN4dQ4Zxfm+tIkSGBg/KAbPsbjuupslMtL3+jDQQw+tEqMxSqzWGaLTWcVk\nShWLZbLb+uBrLHp7e+WWW+4UszlerNZsiY2dLBUVFSLivkbp9eHq+5b73jM4XgIC4kY0L3ytDwZD\nuNo3aaLTWSUwMEas1mwZPz5ZrrjierFYJkhISJYkJk6Riy+eJxZLkoSETJPJk7OkoaHB67wYLic9\n8yEwMFoUpe89g5ECRIheP8UtBrM5VXS6YAkJyXGNxenTp9WYkt3WB1852dTUJKmpMyQkZKpYLJPk\nssuulY6OjhGt1d7GIiAgREJDZ4vFEiWbN28edI0D+8EZU7iEhs6UsLBY+fzzz72O01BGOi885+Zw\nOWk0hqvvGfQ+L8g7f9RRinqgUbnjjjuwdetWNDc3IyYmBqtXr0ZPTw8AYMWKFQCAe++9F3l5ebBY\nLHjhhRdw0UUXuR1DUZRBLyVr1enTp1FUVITw8HBMmTJl0Pa2tjaUlJQgJiYGiqLg0KFDsNvtcDgc\nSEhIQGZmJgICAgbtJ+qn3Nra2pCcnIzKykpYLBZkZGS4vWxfX1+Po0ePIiUlxfXpwlOnTuHAgQM4\nfvw4dDodAgICICJwOByIi4tDQkICKioqEBsbO+gTjocPH8bx48cxbdo0WCwWVwzTpk2D2WxGaWkp\njh49CqPRiIyMDLe/Zh0OB4qKilBTUwOLxYLMzEzXJ38H9sNwn7r1pqmpCRUVFUhKSvL56UJvMWRk\nZKChoQF2ux0TJkzA4cOHRxxDc3MzysvLkZiYiPb2dlc/WCwWV5uOjg4cPHgQVqsVaWlpQ36Krrq6\nGvX19UhLS0N4eLjXNi0tLTh06BDi4+ORmJjolxg88yE0NBTTpk1zxdDa2oqysjKv+TCwHyZMmOD1\nVYLGxkaUlJSgu7sbSUlJPmMoLi5GZGQkUlJSBs2LxsZGpKen+/yEI+D8pO6BAwdgNBoxbdq0Ifva\n27zw1NbWhgMHDqCtrc01L8rKytDW1uaam3a7HQcOHEBjY6Pbp4knTJjgNQbPsWhvbx9yfRioqqoK\njY2NmDx5MmpqahAQEDBoffA1FiKC0tJSnD59GpmZmW53VweuUX1zuKenB729vYiPjz+jeeFtfais\nrERgYCDS09PR1tbmisFoNKK4uBidnZ3IzMxEYGAgioqKYLPZkJmZ6frEs7d5MVxOAu75EB0djZKS\nEnR0dMBut7u+Pm1gDAM/Tdw3Fna7HQcPHhy0PvjKSZvNhgMHDiAwMBDTpk2DTqcb8VrtbSxqamow\nZcoUn9+QMLAf7HY7mpqaMHXqVFit1iHHyZeRzgvPuTlcTjrf2tTuc17QYP6oo8ZUDPoDi0EiIiKi\n0fFHHcX/AwkRERGRhrEYJCIiItIwFoNEREREGsZikIiIiEjDWAwSERERaRiLQSIiIiINYzFIRERE\npGEsBomIiIg0jMUgERERkYaxGCQiIiLSMBaDRERERBrGYpCIiIhIw1gMEhEREWkYi0EiIiIiDWMx\nSERERKRhLAaJiIiINIzFIBEREZGGjakYzMvLQ3p6OlJTU7F27dpB25ubm3H99ddjxowZyMzMxJ//\n/OexnI6IiIiI/EwRERnNjna7HWlpadiyZQvi4+Mxe/ZsvPLKK8jIyHC1WbVqFbq7u/H444+jubkZ\naWlpOHbsGPR6fX8AioJRhkBERESkaf6oo0Z9Z7CgoAApKSlISkqCwWDA4sWLsXHjRrc2sbGxOHny\nJADg5MmTiIyMdCsEiYiIiOj8GnVlVldXh8TERNfjhIQE7Ny5063N8uXLcfXVVyMuLg6nTp3Ca6+9\n5vVYq1atcv07NzcXubm5ow2LiIiI6IKVn5+P/Px8vx5z1MWgoijDtnnssccwY8YM5Ofno6KiAvPn\nz8e+ffsQEhLi1m5gMUhERERE3nneNFu9evWYjznql4nj4+NRU1PjelxTU4OEhAS3Np9++iluv/12\nAMDkyZMxadIklJWVjfaURERERORnoy4GZ82ahfLyclRVVcFms+HVV1/FwoUL3dqkp6djy5YtAIBj\nx46hrKwMycnJY4uYiIiIiPxm1C8T6/V6rFu3Dtdddx3sdjuWLVuGjIwMrF+/HgCwYsUK/PSnP8WS\nJUuQnZ0Nh8OBJ554AhEREX4LnoiIiIjGZtRfLeO3APjVMkRERESjcl6/WoaIiIiIvvxYDBIRERFp\nGItBIiIiIg1jMUhERESkYSwGiYiIiDSMxSARERGRhrEYJCIiItIwFoNEREREGsZikIiIiEjDWAwS\nERERaRiLQSIiIiINYzFIREREpGEsBomIiIg0jMUgERERkYaxGCQiIiLSMBaDRERERBrGYpCIiIhI\nw8ZUDObl5SE9PR2pqalYu3at1zb5+fnIyclBZmYmcnNzx3I6IiIiIvIzRURkNDva7XakpaVhy5Yt\niI+Px+zZs/HKK68gIyPD1ebEiRO49NJL8a9//QsJCQlobm5GVFSUewCKglGGQERERKRp/qijRn1n\nsKCgACkpKUhKSoLBYMDixYuxceNGtzZ/+9vfcOuttyIhIQEABhWCRERERHR+6Ue7Y11dHRITE12P\nExISsHPnTrc25eXl6OnpwVVXXYVTp07h+9//Pu66665Bx1q1apXr37m5uXw5mYiIiMiL/Px85Ofn\n+/WYoy4GFUUZtk1PTw/27NmDDz74AB0dHbjkkktw8cUXIzU11a3dwGKQiIiIiLzzvGm2evXqMR9z\n1MVgfHw8ampqXI9rampcLwf3SUxMRFRUFEwmE0wmE6644grs27dvUDFIREREROfHqN8zOGvWLJSX\nl6Oqqgo2mw2vvvoqFi5c6Nbm5ptvxscffwy73Y6Ojg7s3LkTU6dOHXPQREREROQfo74zqNfrsW7d\nOlx33XWw2+1YtmwZMjIysH79egDAihUrkJ6ejuuvvx7Tp0+HTqfD8uXLWQwSERER/RsZ9VfL+C0A\nfrUMERER0aic16+WISIiIqIvPxaDRERERBrGYpCIiIhIw1gMEhEREWkYi0EiIiIiDWMxSERERKRh\nLAaJiIiINIzFIBEREZGGsRgkIiIi0jAWg0REREQaxmKQiIiISMNYDBIRERFpGItBIiIiIg1jMUhE\nRESkYSwGiYiIiDSMxSARERGRhrEYJCIiItKwMRWDeXl5SE9PR2pqKtauXeuz3eeffw69Xo8333xz\nLKcjIiIiIj8bdTFot9tx7733Ii8vD8XFxXjllVdQUlLitd2DDz6I66+/HiIypmCJiIiIyL9GXQwW\nFBQgJSUFSUlJMBgMWLx4MTZu3Dio3TPPPIPbbrsN0dHRYwqUiIiIiPxPP9od6+rqkJiY6HqckJCA\nnTt3DmqzceNGfPjhh/j888+hKIrXY61atcr179zcXOTm5o42LCIiIqILVn5+PvLz8/16zFEXg74K\nu4Huv/9+rFmzBoqiQER8vkw8sBgkIiIiIu88b5qtXr16zMccdTEYHx+Pmpoa1+OamhokJCS4tdm9\nezcWL14MAGhubsbmzZthMBiwcOHC0Z6WiIiIiPxIkVF+qqO3txdpaWn44IMPEBcXhzlz5uCVV15B\nRkaG1/ZLlizBggULsGjRIvcA1LuGRERERHRm/FFHjfrOoF6vx7p163DdddfBbrdj2bJlyMjIwPr1\n6wEAK1asGFNgRERERHT2jfrOoN8C4J1BIiIiolHxRx3F/wMJERERkYaxGCQiIiLSMBaDRERERBrG\nYpCIiIhIw1gMEhEREWkYi0EiIiIiDWMxSERERKRhLAaJiIiINIzFIBEREZGGsRgkIiIi0jAWg0RE\nREQaxmKQiIiISMNYDBIRERFpGItBIiIiIg1jMUhERESkYSwGiYiIiDRszMVgXl4e0tPTkZqairVr\n1w7a/vLLLyM7OxvTp0/HpZdeiv3794/1lERERETkJ4qIyGh3ttvtSEtLw5YtWxAfH4/Zs2fjlVde\nQUZGhqvNZ599hqlTpyI0NBR5eXlYtWoVduzY0R+AomAMIRARERFplj/qqDHdGSwoKEBKSgqSkpJg\nMBiwePFibNy40a3NJZdcgtDQUADA3LlzUVtbO5ZTEhEREZEf6ceyc11dHRITE12PExISsHPnTp/t\nn3vuOdxwww2Dnl+1apXr37m5ucjNzR1LWEREREQXpPz8fOTn5/v1mGMqBhVFGXHbjz76CM8//zw+\n+eSTQdsGFoNERERE5J3nTbPVq1eP+ZhjKgbj4+NRU1PjelxTU4OEhIRB7fbv34/ly5cjLy8P4eHh\nYzklEREREfnRmN4zOGvWLJSXl6Oqqgo2mw2vvvoqFi5c6NamuroaixYtwksvvYSUlJQxBUtERERE\n/jWmO4N6vR7r1q3DddddB7vdjmXLliEjIwPr168HAKxYsQKPPvooWltbsXLlSgCAwWBAQUHB2CMn\nIiIiojEb01fL+CUAfrUMERER0aic96+WISIiIqIvNxaDRERERBrGYpCIiIhIw1gMEhEREWkYi0Ei\nIiIiDWMxSERERKRhLAaJiIiINIzFIBEREZGGsRgkIiIi0jAWg0REREQaxmKQiIiISMNYDBIRERFp\nGItBIiK3zxf7AAAgAElEQVQiIg1jMUhERESkYSwGiYiIiDSMxSARERGRhrEYJCIiItKwMRWDeXl5\nSE9PR2pqKtauXeu1zX333YfU1FRkZ2ejsLBwLKcjIiIiIj/Tj3ZHu92Oe++9F1u2bEF8fDxmz56N\nhQsXIiMjw9Vm06ZNOHz4MMrLy7Fz506sXLkSO3bs8EvgF5LOzk48+eT/4uDBw0hKGodjx1qxe/ce\n2O29ABS1lYLY2HjMmZOFY8daAQDf+c5/Yu7cufjiiy/w+OP/g/z87ejs7AIAiAgAQKdToNcHYebM\n6YiKCsPRo01ITIxCW1sHdu3aje7ubgAKdDodUlOnYNq0SSgsLEFlZRVEHBg/Ph6LFn0V3/3uSgQE\nBLhibmlpwdq1T6GqqgGTJ8eioqIWBw8WweGwA1BgMAQiOzsLsbGR2LmzEK2tJ10x7NixBydPtmPu\n3Jl48MH7cfDgQfzpTy+houIIRJz7G40WXHbZHDz44H9j69Zt+MtfXkN19VG3fqiubsD+/Qdc/SSi\nQFHE1V+e/eBwOP+bnJyM7OwpOHiwAuXl5RBxuO0DOLzG8M47WxAZGQKDQY+GhhYkJ8eOKAYASEpK\nxkUXpaO6uglGowFhYcHYsWMPmpuPQwRQFEFk5DhcfPEMtLS0YdeuPejp6VGPIXA4HNDpFAAKAgL0\nmDp1KiZPjseuXQdRU1MLQNTzOmPQ6YDExImYNWsqysqqUVpagsTEpDHFICJu+aDT6fDnP7+Iv/3t\nTdTU1EJRgISERMyenTkoHzz7oaTkCMrKygZsV6AoOqSmpmD58m8hLS0NTzzxW+zc+bnXGObOnY66\nuibs27cf0dHjXfPCuT0Cn3++D/X1DQC850N4eCTmzp2Bjo4uFBTscs2Dvn7o28fbvHA47K6xGDg3\nb7/9Jtx++6148snfoKqqAddeexkWLLgJa9Y8hfz87dDrjbj22ivw4IM/REhICACgpqYGjz/+FD79\ndMegGDzHIiwsEhdfPN3r+uDcBxBRoNfrkZU1DRMnxqKysgExMWHo7RXs2FHgWh+GGovAQBNmzZqB\n8PBgfPbZbrS0tLrmRXR0LC6+eDpaWk6ju7sHS5fegauuugptbW1Yu/YpHD5cM6J54W19mD07Bz/+\n8fdRUVGBl19+C8HBQQgJMeOzz3ajvb0Lc+bMQHCwGZ99tgtdXT2YM2cGjMZA7NixGydPnvQ6L4bL\nSc986O7uwYkTp3H77Tdi0aJF6O7uxlNP/RbvvvueWwy1tc2D1mqTKQRz5+YgMNDgWh+GysmQkHBc\nckkOurtt2LnTmYMjWau9jUVXlw0TJ8Zg166DqK4+6pYPANz6oa7uOAICdIiODkVBwV40Nh4btE4q\nSv86ONZ54Tk3h8vJu+++HZde+hXXvDCbQ3DDDVfjRz/6AYxGI+jsUKQvC87QZ599htWrVyMvLw8A\nsGbNGgDAT37yE1eb73znO7jqqqvwjW98AwCQnp6OrVu3IiYmpj8ARcEoQ7gg2O12XHrptdi3Lwxd\nXdkAngAQAyAEQCuANgBhAC4HkAjgGQC/AKDAbF6LN974K7797ftQX2+BSCeAJgAG9Ri9AI4D+BmA\n59Tj3ADgVwAS1HbNALoA3AugBMB29XEkgIsAXIegoD/jttvS8dJLfwIAtLe3IytrLurqvgKbzQpg\nvRrvOHXfZgAPA3hJjb9tQAw29fHPAdQhKOj3AILR3d2u7m8EcArAg1CUUphMf4XDYUVXV4tHP/wO\nQDiAKACn1XOaBsTg2Q9Nao//J4AWAG+rj2MG9EOnzxiAcejo+C6Ax9UYJp1BDN8A0APgVQCrADwP\noFuNo0eNYRKAywA8BiBOPc5x9biRA/LhJIAH1HEqUh+HeInhZgCBAF5QH/sjBvd8SEiIw9NPb0BX\nV5Maw7Vqf3jmg2c/vAIgAEAsnL84mtQxugfABAQG/hKK0onu7r6x8IwhA8D/qtfsOS82A6gEcMLH\nWMTAmUc3A3hU3bcvhr5+GG5eeJubVyIo6P8hMLAe3d0LYLPNhsn0ewQENKK9PRYiQQC+jYCAD5Ce\nfgS7d29Da2srpk6didbWCDgNjMFzLDIAzITv9SFczYUWAD9V+6FJ7dNfqsfoHsFYHFf3/z/1cSv6\n50W6GsOTcOZgGMzmx/GXv6zDz372GKqqstDdHYfh54W39eHnAJoRFPRbKIoVXV0PDojhFICfAPiN\nmg/tAH6k9oVV3d/XvBguJwfmwy8BLAOQCrP5f7BmzQ/w5pub8PHH9ejtPT4ghikA5nqMRSeA++C+\nPgyVk0YAi+HMwQSMfK32NRa7AXwOoMMjH4579MMf4Zz/bwJoVPPFc50UuM/NscwLz7k5XE5mIiho\nDRSlDV1dcep5F8Ng2Ig5c2zYtm0zdDq+u82TP+qoUReDf//73/Gvf/0Lzz77LADgpZdews6dO/HM\nM8+42ixYsAAPPfQQvvKVrwAArrnmGqxduxYzZ87sD0BR8Mgjj7ge5+bmIjc3dzQhfSkVFBRg3ry7\ncfr0QQA/gHPh2wegCs4JbAFQCuBjAHcCuALASnXvZ5GZ+UccOTIOHR1bAcwBMBnOBeAjOBeMh9X/\nfgXAUQBrAZQB+ATOYuErcE78x+FcdHLV5/cC2AXnOwlOIzAwDg0NVYiIiMDrr7+OpUv/iNOn34dz\nEY0BUAegL4b/gnMhSPcSQwyAp9XzAMB4ANfDuQAVqNf/LwDT1O3BAG7y0g/BAI6o1zEPQLxHDJ79\nMBXOxe1PAGYAyIGzMB3YD0PFcABAMZy/nM8khnYAfwNwKZwL/cB+uGhADOVqPzUMiGHgWFTBmQ+z\n4cyTYLXfIr3E0ARnsTsfQJIfY+jPB4MhFoAdPT19MVQB8JUPnv2Qrh5724AYQuAsIgDgW+p5PvcR\nwz3qWHnOi6UAQocYi758qIOz0CjwiMEzH3zNi76xGJiTCoA/w1l0fKZexzoAb6jX0aBeoyAkZC7e\neOPX2LdvH3760wL09HiLwXMshlofStS4rodzzPv6oQHAOwBeO4OxeBrOIu4mDJ4XfTFEwfnLHQDe\nRnLyw/jiCytOnRrpvPC2PtygHm8igBc9YnhZjeFu9M/NLwD8N4aeF1UYOic986EUzvUBAAoRHn4T\nbDYD2ts9Y/A2FrsxeH0YKicb4CyG38eZrdW+xiIC3vPBsx9uBfA1OAsxX+uk59wcy7zw7IfhchIA\nHgRwCM55dBRAEIBeWCwZ2L79NeTk5EDr8vPzkZ+f73q8evXqMReDo36ZWFGU4RsBgwL0tt+qVatG\nG8aXXnd3N3S6YDj/Ou+G8y/GEPXfAudEsML5y6Zb/XefUHR32yBiUffvgXNITep+PWp7m/pc4IBz\nmOGcgBhwzG71v0FqDH1/gZmg0wXCZrO5YnbfRw/nAtAXg1V93lsMNo9r6HuJ1gjnQt7ssb3XRz+E\nq+372njG4NkPfdv7YtZ56YehYui7hjONYWA/efbDwBj06B+bvhiA/rHoywer2md29cfgJQbPsfFX\nDO750Nt7ekAMQ+WDt5jMHjEM7G+dus1XDN3qvz3nhd2jH3zlg1E9r2cMI50X3uYm1P8OvA4MiMHk\naqMoVthsNnR322C3+4rBcyyGWh9C0J8PA/vBorY7k7Hom7ve5kVfDKEDri9UXRfOZF54Wx/6OLzE\nYIXzD5yBc7MWw8+L4XLSMx/c19aeHhsUJdRLDN7Gwtv6MFROmtQ2Z7pW+xoLX/ng2Q995xhqnfTn\nvPDsh+FyEuo5LQP6AQACoNNZXL+DtM7zptnq1avHfMyAVaOsxE6ePIl33nkH3/rWtwAA7777LkJD\nQ3HZZZe52mzbtg0RERHIzMwEADz22GO47777EBwc7GqzevVqTReD48aNw7PPPoOOjkY4HFPgvDty\nCs6XIpoA7ITzdr4Dzr/mHoXzr7cKmM334xe/+D4++ODPsNkmwDm59sJ5FysGzgn1Bpx/zb2vPp8D\n4Ck4J2w8nH/Bvw/nHaQuAPvh/MvzJJwvOQRBr38YF11kwfe/vxKKoiAmJgb/938/R3e3Hs6/TF8C\ncBhAXwxvALgY/X9NvjUghjD1cTqAHQgI2AiD4TTs9gI4F4YkABvV6/8Aev0HCAgAHA7PfvgTnL8Q\nUuBctHbD+dekr34oArADzjuRAQA+BFAI52LZ1w++YwgM3Ine3svhvINgV/d5bgQx7IHzr3aj2u+X\nD+iHQ+o+BgA1cN6J/F/1+EnqsXeoY9GXD/+A8yXDJjjvMuwGUO0RQwmcvzCsAP6uxuGPGPrzYeZM\nC2bPvhjl5VVwOArgvCtg8JEPnv3wLpx5ZoXzzkUZnHcJEgG0IjBwPYAd6vtHvcWQpI5DE9znRbYa\nxwl1bH3lw170v8QWOiCGvn4Ybl54m5s6BAQ8i4CAjyFiBiAICnoOwDaIJAP4FEAsFOUvCA3dgief\nfByJiYl4/vmfoKcnxEsMnmPRrJ73aQxeH47BOZ/a4MzbLLUf/qGO9VPqdYxkLN5U93kbzjtAleif\nF30x/AJAKoAGmM3fww9/uASff/4uurp6IRKH4eeFt/UhA8Ae6PWvIzBwuzrX+mJ4C8CVcN45SlKv\n8Ur1PFPgvEvobV5UYeic9MyHX6rnOw2z+V4sWXIjGhqqcOJEtNovfTE0qv09cK3+B5x3wAauD7+E\n75z8DM67Yv8D55o00rXa11icgvNu8uce+fC5Rz/8Cs67qAfgLFD3Y/A6uRfuc3Ms88Jzbg6Xk10w\nGP4ARfkEImFw3mGMhKI8g7i4Mvzyl7+AXj/qe1gXLH/UUaN+mbi3txdpaWn44IMPEBcXhzlz5uCV\nV14Z9AGSdevWYdOmTdixYwfuv//+QR8g0fp7BgGgvr4e3/nOD1FaehixseFoajqBmpp6OBw9cL5p\nPgCKosBqDUFGRhKam9uhKAoeeGAF7rrrWzhw4AC+850HcOBAEXp6ugDooCg6iAgUxYGAACMSE+MQ\nHR2GhobjGDcuFG1tHaiqqobd7vxLS1ECER0dieTkeBw6VIGWlpMAFAQHW3D99fOwbt0TsFr7/3or\nLS3Fd7/7IGprGxAfH4GjRxvR2NgEh8P513nfOceNC0dxcRm6uuxITIxDVFQoiovL0N3tQErKJDzx\nxM/x6aef47nn/orm5jY1ZoFeH4SMjFT85jeP4o03/okNG15Ha+tpAP39UFf3BerqGiHigIgdihKg\nvmFa4FyQ3fvB+ZeqHhER4ZgyJREVFdVobm6Fw+FQtwMiAWp/DI7h3Xc/QEiICXq9AS0tbYiLCx9R\nDDqdHmFhoUhPn4iGhhMIDNTDajWjuLgMHR0dcC6IgNlsRkbGJLS0nMbRo7VwOHoBOCCig/OOkgPO\nD4oEISYmGklJsSgpKUVbW4frDd59MQABCA119lNVVT2amloRGhoyphgUReeWD4GBgXjggYfx97+/\niba2DgAKQkOtyMhIGpQPnv1w5EgNmppa3PpeUQIRFRWB5cu/idmzZ+DHP/4lKiqqvMYwdWoyGhqO\no66uEcHBFte8EHEgOjoMJSWlOHmyw2s+AIDRaEJ6+iS0t3fjyJEq1zwAAgbki/d54exrx6C5uWDB\ntVi5cgkeeGA1amsbMG/eZViy5A7cf//DOHiwGDpdAGbPzsH69U9h0qRJAIAdO3bge997CKWl5W4x\neI5FdXUdjEYjpk6d5HV96MtZEefcGz9+HBITY1Bf34yIiGDYbHaUlx9xrQ9DjUVAgBETJ8YjLMyK\nkpJSdHZ2uq7TYrFg6tRJOHGiC3a7HStWfBP33rsSR44cwcqVD+Do0doRzQtv68OkSROxZs1D2L+/\nGH/5yxsICtLDYjGjpKQUPT1AUlICQkLMKCk5hN5eIDk5ESaTCaWlh9Dd3el1XgyXk5750N3tQEdH\nJ2677UasWvVTNDc3Y+XK/8bWrR+7YrBag9HU1DJorTYYTEhJmYDAwCDX+uArJxUFCAoyIi1tErq7\ne3D4cCXsdtuI1mpvY9HT04Px48NRVlaO1tbTXtYDnasfjh07BZ0OiIy0oqSkDCdPdrh+D/etk841\np38ejGVeeM7NoXIyPDwM3/zmIixYcB1+8IOHUVZ2GAZDIC67bC7Wr/9fjB8/3vcvUg07r+8ZBIDN\nmzfj/vvvh91ux7Jly/DQQw9h/fr1AIAVK1YAAO69917k5eXBYrHghRdewEUXXeQeAItBIiIiolE5\n78WgP7AYJCIiIhodf9RR/Iw2ERERkYaxGCQiIiLSMBaDRERERBrGYpCIiIhIw1gMEhEREWkYi0Ei\nIiIiDWMxSERERKRhLAaJiIiINIzFIBEREZGGsRgkIiIi0jAWg0REREQaxmKQiIiISMNYDBIRERFp\nGItBIiIiIg1jMUhERESkYSwGiYiIiDSMxeC/ifz8/PMdwr8F9oMT+8GJ/eDEfnBiPzixH5zYD/4z\n6mKwpaUF8+fPx5QpU3DttdfixIkTg9rU1NTgqquuwrRp05CZmYnf/e53Ywr2QsakdmI/OLEfnNgP\nTuwHJ/aDE/vBif3gP6MuBtesWYP58+fj0KFDmDdvHtasWTOojcFgwG9+8xsUFRVhx44d+P3vf4+S\nkpIxBUxERERE/jPqYvCdd97B3XffDQC4++678fbbbw9qM378eMyYMQMAEBwcjIyMDNTX14/2lERE\nRETkZ4qIyGh2DA8PR2trKwBARBAREeF67E1VVRWuvPJKFBUVITg4uD8ARRnN6YmIiIgIzjpsLPRD\nbZw/fz4aGxsHPf/rX//a7bGiKEMWdadPn8Ztt92Gp59+2q0QBMZ+AUREREQ0ekMWg++//77PbTEx\nMWhsbMT48ePR0NCAcePGeW3X09ODW2+9Fd/61rfwta99bWzREhEREZFfjfo9gwsXLsSLL74IAHjx\nxRe9FnoigmXLlmHq1Km4//77Rx8lEREREZ0Vo37PYEtLC77+9a+juroaSUlJeO211xAWFob6+nos\nX74c7777Lj7++GNcccUVmD59uutl5McffxzXX3+9Xy+CiIiIiEZn1HcGIyIisGXLFhw6dAjvvfce\nwsLCAABxcXF49913AQCXXXYZHA4H9u7di8LCQhQWFroKwVWrViEhIQE5OTnIycnB5s2bXcd+/PHH\nkZqaivT0dLz33ntjub4vhby8PKSnpyM1NRVr16493+GcU0lJSZg+fTpycnIwZ84cACP7Dssvu6VL\nlyImJgZZWVmu54a67gt1TnjrBy2uDb6+k1VrOeGrH7SWE11dXZg7dy5mzJiBqVOn4qGHHgKgvXzw\n1Q9ay4c+drsdOTk5WLBgAQA/54OcJ6tWrZKnnnpq0PNFRUWSnZ0tNptNKisrZfLkyWK3289DhOdG\nb2+vTJ48WSorK8Vms0l2drYUFxef77DOmaSkJDl+/Ljbcw888ICsXbtWRETWrFkjDz744PkI7aza\ntm2b7NmzRzIzM13P+bruC3lOeOsHLa4NDQ0NUlhYKCIip06dkilTpkhxcbHmcsJXP2gxJ9rb20VE\npKenR+bOnSvbt2/XXD6IeO8HLeaDiMhTTz0ld955pyxYsEBE/Ps747z+7+jEyyvUGzduxB133AGD\nwYCkpCSkpKSgoKDgPER3bhQUFCAlJQVJSUkwGAxYvHgxNm7ceL7DOqc882Ak32H5ZXf55ZcjPDzc\n7Tlf130hzwlv/QBob23w9p2sdXV1mssJX/0AaC8nzGYzAMBms8FutyM8PFxz+QB47wdAe/lQW1uL\nTZs24dvf/rbr2v2ZD+e1GHzmmWeQnZ2NZcuWuW5v1tfXIyEhwdUmISHBtRhciOrq6pCYmOh6fKFf\nrydFUXDNNddg1qxZePbZZwEAx44dQ0xMDADnp9aPHTt2PkM8Z3xdt9bmBKDttaGqqgqFhYWYO3eu\npnOirx8uvvhiANrLCYfDgRkzZiAmJsb10rkW88FbPwDay4cf/OAHePLJJ6HT9Zdt/syHs1oMzp8/\nH1lZWYN+3nnnHaxcuRKVlZXYu3cvYmNj8cMf/tDncS7kL6a+kK9tJD755BMUFhZi8+bN+P3vf4/t\n27e7bR/uOywvVMNd94XcJ1peG06fPo1bb70VTz/9NEJCQty2aSknPL+bVos5odPpsHfvXtTW1mLb\ntm346KOP3LZrJR88+yE/P19z+fDPf/4T48aNQ05Ojs/vZh5rPgz5PYNjNdT3FA707W9/2/WGyPj4\neNTU1Li21dbWIj4+/qzE9+/A83pramrcKvoLXWxsLAAgOjoat9xyCwoKCkb8HZYXGl/XrbU5MXC8\ntbQ29H0n61133eX6qi4t5oS376bVak4AQGhoKG688Ubs3r1bk/nQp68fdu3ahdzcXNfzWsiHTz/9\nFO+88w42bdqErq4unDx5EnfddZdf8+G8vUzc0NDg+vdbb73l+jThwoULsWHDBthsNlRWVqK8vNz1\nKdML0axZs1BeXo6qqirYbDa8+uqrWLhw4fkO65zo6OjAqVOnAADt7e147733kJWVNaLvsLwQ+bpu\nrc0JLa4N4uM7WbWWE776QWs50dzc7Hrps7OzE++//z5ycnI0lw+++mHg/xlNC/nw2GOPoaamBpWV\nldiwYQOuvvpq/PWvf/VvPpyVj7yMwF133SVZWVkyffp0ufnmm6WxsdG17de//rVMnjxZ0tLSJC8v\n73yFeM5s2rRJpkyZIpMnT5bHHnvsfIdzzhw5ckSys7MlOztbpk2b5rr248ePy7x58yQ1NVXmz58v\nra2t5zlS/1u8eLHExsaKwWCQhIQEef7554e87gt1Tnj2w3PPPafJtWH79u2iKIpkZ2fLjBkzZMaM\nGbJ582bN5YS3fti0aZPmcmL//v2Sk5Mj2dnZkpWVJU888YSIDL02aqkftJYPA+Xn57s+TezPfBj1\nl04TERER0Zffef00MRERERGdXywGiYiIiDSMxSARERGRhrEYJCIiItIwFoNEREREGsZikIiIiEjD\nWAwSERERaRiLQSIiIiINYzFIREREpGEsBomIiIg0jMUgERERkYaxGCQiIiLSMBaDRERERBrGYpCI\niIhIw1gMEhEREWkYi0EiIiIiDWMxSERERKRhLAaJiIiINIzFIBEREZGGjakYXLp0KWJiYpCVleV1\n+8svv4zs7GxMnz4dl156Kfbv3z+W0xERERGRn42pGFyyZAny8vJ8bk9OTsa2bduwf/9+PPzww7jn\nnnvGcjoiIiIi8rMxFYOXX345wsPDfW6/5JJLEBoaCgCYO3cuamtrx3I6IiIiIvIz/bk60XPPPYcb\nbrhh0POKopyrEIiIiIguOCIypv3PyQdIPvroIzz//PNYu3at1+0iwh8//jzyyCPnPYYL7Yd9yj79\nMvywT9mnX4Yf9ql/f/zhrN8Z3L9/P5YvX468vLwhX1ImIiIionPvrN4ZrK6uxqJFi/DSSy8hJSXl\nbJ6KiIiIiEZhTHcG77jjDmzduhXNzc1ITEzE6tWr0dPTAwBYsWIFHn30UbS2tmLlypUAAIPBgIKC\ngrFHTUPKzc093yFccNin/sc+9T/2qf+xT/2PffrvRxF/veA82gAUxW+veRMRERFpiT/qKP4fSIiI\niIg0jMUgERERkYaxGCQiIiLSMBaDRERERBrGYpCIiIhIw1gMEhEREWkYi0EiIiIiDWMxSERERKRh\nLAaJiIiINIzFIBEREZGGsRgkIiIi0jAWg0REREQaxmKQiIiISMNYDBIRERFpGItBIiIiIg1jMUhE\nRESkYWMqBpcuXYqYmBhkZWX5bHPfffchNTUV2dnZKCwsHMvpiIiIiMjPxlQMLlmyBHl5eT63b9q0\nCYcPH0Z5eTn++Mc/YuXKlWM5HY1QXV0dCgsL0d7e7nV7V1cX9u3bh6qqKp/HqK2txd69e9HR0eF1\ne2dnJ/bu3Yvq6uoRx9XS0oI9e/agpaVlxPucC+3t7SgsLER9ff35DuW8qK+vHzJfPIkIysrKUFRU\nBLvdPqJ9+nLu6NGjPtvU1NRg79696Ozs9LrdM+ccDgdKSkpQUlICh8MBAKiurh7yGJ7sdjuKi4tR\nWlrqOsb50Nvbi4MHD+LQoUMQEYgIKioqsH//fthstvMW10i0tbVhz549+OKLL7xuFxEcOnQIRUVF\n6O3tBTD8GnU2+CNfaOz68qWpqWnE+5zpGuVNbW0tCgsLXb/TWltbsWfPHjQ3NwMY2Rrl6d/1d9qo\nyBhVVlZKZmam120rVqyQDRs2uB6npaVJY2OjWxs/hEAD/OxnqyUoKFys1iwJD4+T3bt3u20vLy+X\n2NjJEhIyVYzGKFm69L/E4XC4tfnRj34mRmOEWK2ZEhmZIPv27XPbXlJSIuPGJUlIyDQxGiNl5cof\nDDqGpw0bXhWTKVys1uliNkfIa6/93T8XPEYFBQUSFhYrVmuWGI3h8sgjvz7fIZ1TjzzyazEanfkS\nFhYrBQUFQ7bv7OyUK6/8qpjNiWKxJEtW1sXS0tIy5D5lZWUyfnyyK+eWL//eoHy5//4HXTkXFTVB\nDhw44La9qKhIoqMnunLunnvuk1mzrhSLZaJYLBNl9uxcWb78XjEaIyUkZJqMG5ckJSUlQ8bV2toq\n2dlfEYtlkpjNiXL55ddJR0fHkPucDU1NTZKWdpEEB6eIyRQn1177NVm06JtiMsVIcHCaTJw4VWpq\nas55XCPx3nvvSXBwlFit08VoDJM//OFZt+1dXV1y9dU3idmcIBZLskybNkf++79/MuQadTacOnVK\n5sy5ypUvM2deIStWfN+VL9HRE6WoqOisx6F1W7ZsccuXdev+MOw+Z7pGefOTn/xCzblMiYiIl9/+\n9rdiNkeI1TpdTKZwWbv2yWHXKE+vvfZ3t2Ns2PDaGcflL/6oo85qMXjTTTfJJ5984no8b9482bVr\nl3sALAb9Ztu2bWI2JwlwTAARYIPEx09xazNz5pWi0/2vuv2kWCw58tpr/Un8/vvvi8WSIkCz2ubP\nkpyc5XaMzMyLRVF+r25vFYslUzZu3OgzrmPHjonJFCHAXnWfPWI2R0hzc7N/O2AUxo9PFuB1Na5G\nMVHGFaEAACAASURBVJsnyKeffnq+wzonPv30UzGbJwjQqF7/6zJ+fPKQ+/z856vFZPqaAD0C2CUw\ncIXcffd3htwnO/tSUZTfqedoE4tlurzxxhuu7Xl5eWKxpAlwXG3znKSm5rgdIz19lijKenV7i+j1\n48Vg+KYAvQL0isFwhRgMUwQ4IYCIovxeMjMvHjKuZcu+K0FB3xbALkCPGI23ykMP/WKYXvO/RYu+\nJQbDDwRwCNAtBsNUMRjmCtAhgEMCAh6Rq69eeM7jGk5nZ6cEB0cJsFUdl3IxmaLk8OHDrjarVv1K\nTKYFAtgEcIhe/zUJCIgbco06G+6//8diNN7pyhe9/irR6ycL0KLmy3rJyJh91uPQsu7ubjVfPlLH\nvkJMpnFSVlbmc5/RrFGePvroI7FYkgVoUo/xrCiKSYCd6uMiUZRQUZSnfa5Rnpqbm8VkChdgj7rP\nXjGZIqSpqemMYvMXf9RRZ/0DJM44+ymKMqjNqlWrXD/5+flnO6QLVnFxMYB5AMapz9yO+vrD6Onp\ncbUpKyuGw7FYfRSCjo4bUVRU5HaM3t5rAUSqzyxGVVWx2ziWlxdB5A71URi6ur7qdgxPlZWVCAyc\nBCBbfSYHen3ikC9TnwtdXV1oaqr+/+3deVxU9f4/8NfADMwZVgEBBRSVgWFR1MA9Qw01U+u6FFZq\nbpnGLc1K697fTbslkmlaWpllWRnara5WKrkklal43Rc0ceErIunFJXeW4f37w+M0I4s2M16yeT0f\njx555nw+7/mcz3nzmTfnjEcA/dVXQqDRdFHn8c8vLy8PGk0XACHqK/1x8uRRXLlypcY+27bl4fLl\nAQC0ANxQVvYgtm+v+dwDQH5+HkSu5ZwvLl/uZZMve/fuRXl5DwAB6itpOHzYNubhw9Yx6qGiwh/l\n5WkA3AG4o7y8EcrLewPwAwCIpCE/v/Zxbd+eh9LSB3D12zJaXLkyEFu21N7nVti5M089Fg0AD5SX\nh6G8fAAABYAGZnNarT9fdaW4uBgiCoDO6itR8PBohQMHDlja/JYvOgAaVFQ0Q2VlF9S2Rt0KW7fm\n4cqVgbiWLxUVkaio6AWgHoCr+XLo0B9vjv9MfvnlF1RWegBIUV9pCg+PO/Dzzz/X2MeeNaq6GJWV\ndwOor76SBJEAAG3U7TiIlFl9plVdo65XUFAAna4RgFbqK4nw8GiCw4cP3/S4HJGTk2NTNznDLS0G\nw8LCUFhYaNk+duwYwsLCqrSzPqiUlJRbOaQ/NZPJBI1mHYBT6ivLEBraBDqdztImKioGGs2X6tZF\nGAzZMJlMlv0xMTHQatcCOKu+8iUaNTLZFPFNm5qg0Xyhbp2HoqyyiXG9yMhIlJUdAXCtyNqD8vKj\naNy4sf0H6wR6vR5BQWEAvlZfKQHwfa3H8mcSExMDke9x9bgB4GsEBYVDr9fX2CcxMQZ6/TIAZgAC\nne5LtGhR+3w1bWqdcxegKN8iNjbWst9kMkGnWw3gV/WVL9G4sW3Mxo1jAFyLcQ5a7a/Q6T4HUAmg\nElrtMeh0KwGcBwBoNF+iWbNY1KZ58xh4ePwbgAAwQ69filat/vfnPj4+Blrtl+o4yqHTFUOnWwrg\n6geeu/uXf8icDA0NBXARwEb1lQKUle2A0Wi0tLmaL0sBVAAQuLsXwM0tB7WtUbdCYmIMPD2X4rd8\nKYRW+y2Ac2qLLxEZ+ceb4z+TkJAQuLmVAvhJfeUoysq22uTL9exZo6qL4ea2DsAZ9ZWd0GhOA7j2\nF1oPQKPxsPpMq7pGXa9x48YoLz8KYI/6Sh7Kyo4gMjLypsfliJSUFKcXg7f0NvHy5cvlnnvuERGR\njRs3Stu2bau0ccIQyMrV7/sFiZ9fkvj5hUpubq7N/v3790v9+o3F17eVKEoDeeihETbfjaisrJS/\n/vUZUZRg8fO7Q/z9G1T5Ts+ePXskKChCfH1bi6KEVvu9w+t99NEnoigB4ueXLIoSIIsWZTnvoB2w\nYcMG8fUNET+/JNHrg2TSpP/9bcK6NGnSPyz54usbcsNb5BcvXpR27bqJl1dT8faOFZPpjhve7s/L\ny5OgoEaWfBk8+LEqOff44+MsORcQECbbt2+3ibFz504JDAxXY4TIo4+OlhYt2ou3d7R4exulZcuO\nMnjwKFGUUPH1bS1BQRGyZ8+eWsd16tQpiYtLFm9vk3h5NZM2bbrIhQsXbjBjzldcXCxNmzYXH594\n8fKKlLvuukd69RogBkO4+Pq2kPDwaCkoKPifj+tmLF++XAyGQPHzSxa9PkBmz55rs//SpUvSsWOq\neHk1EW/vWImObiVPPPF0rWvUrXDu3Dlp2bKjeHsbxds7Wpo3bydDh44WRQkRX9/WEhgYLrt27brl\n43B1K1euFC+vIHW9DZAZM964YZ/fu0ZVZ9y4iaIo9cXPL0n8/RtIRkamzefRSy+9XOsaVZ1Fi7Js\nYnz00Se/e1zO4ow6SqMGssugQYPw/fffo6SkBCEhIZgyZYrlcv/o0aMBAOnp6cjOzoaXlxc++OAD\ntG7d2iaGRqOpciuZHHPkyBGcOHECsbGx8PPzq7L/4sWL2Lt3L/z8/BAdHV3trftDhw6hpKQEsbGx\n8PX1rbL/woUL2Lt3LwICAmr9zc7aL7/8giNHjqBp06YICQm5cYf/kbNnz2L//v0ICQlBkyZN6no4\n/3PX8sVkMsHf3/+G7c1mM/bs2QOz2YzmzZvf1FWdaznn7++P6OjoatscPHgQp06dQlxcHHx8fKrs\nP3/+PPLy8hAYGIioqChUVFRg9+7d0Gg0SEhIgFarRX5+Pk6fPo34+Hh4e3vfcFzl5eXYvXs33N3d\nkZCQAHd39xv2uRVKS0uxZ88eeHh4ID4+HhqNBvv27cPFixeRkJAARVHqZFw3o6SkBPn5+WjUqFG1\nd34qKyuxZ88eVFRUICEhAR4eHjdco26Fa39jW0TQvHlzaLVaS87dbL6Q406dOoUDBw4gIiIC4eHh\nN9Xn965R1Tl8+DBOnjyJuLg4+Pr64uTJkzh06BAiIyPRoEGDm1qjrnfixAkcPnwYTZo0Ua+U1w1n\n1FEOFYPOwGKQiIiIyD7OqKP4L5AQERERuTAWg0REREQujMUgERERkQtjMUhERETkwlgMEhEREbkw\nFoNERERELozFIBEREZELYzFIRERE5MJYDBIRERG5MBaDRERERC6MxSARERGRC2MxSEREROTCWAwS\nERERuTAWg0REREQujMUgERERkQtjMUhERETkwlgMEhEREbkwh4rB7OxsmEwmGI1GZGZmVtlfUlKC\nnj17omXLlkhISMCHH37oyNsRERERkZNpRETs6Wg2mxETE4M1a9YgLCwMycnJyMrKQmxsrKXN5MmT\nUVpaioyMDJSUlCAmJgYnTpyAVqv9bQAaDewcAhEREZFLc0YdZfeVwc2bNyMqKgqRkZHQ6XRIS0vD\nsmXLbNo0aNAA586dAwCcO3cOgYGBNoUgEREREdUtuyuzoqIiREREWLbDw8ORm5tr02bUqFHo2rUr\nGjZsiPPnz+Ozzz6rNtbkyZMtf05JSUFKSoq9wyIiIiL608rJyUFOTo5TY9pdDGo0mhu2mTp1Klq2\nbImcnBwcOnQIqamp2LlzJ3x8fGzaWReDRERERFS96y+aTZkyxeGYdt8mDgsLQ2FhoWW7sLAQ4eHh\nNm02bNiAgQMHAgCaNWuGJk2a4Oeff7b3LYmIiIjIyewuBpOSkpCfn4+CggKUlZVhyZIl6Nu3r00b\nk8mENWvWAABOnDiBn3/+GU2bNnVsxERERETkNHbfJtZqtZgzZw569OgBs9mMESNGIDY2FvPmzQMA\njB49Gi+88AKGDRuGxMREVFZW4tVXX0VAQIDTBk9EREREjrH70TJOGwAfLUNERERklzp9tAwRERER\n3f5YDBIRERG5MBaDRERERC6MxSARERGRC2MxSEREROTCWAwSERERuTAWg0REREQujMUgERERkQtj\nMUhERETkwlgMEhEREbkwFoNERERELozFIBEREZELYzFIRERE5MJYDBIRERG5MBaDRERERC6MxSAR\nERGRC2MxSEREROTCHCoGs7OzYTKZYDQakZmZWW2bnJwctGrVCgkJCUhJSXHk7YiIiIjIyTQiIvZ0\nNJvNiImJwZo1axAWFobk5GRkZWUhNjbW0ubs2bPo2LEjvv32W4SHh6OkpARBQUG2A9BoYOcQiIiI\niFyaM+oou68Mbt68GVFRUYiMjIROp0NaWhqWLVtm0+bTTz9F//79ER4eDgBVCkEiIiIiqltaezsW\nFRUhIiLCsh0eHo7c3FybNvn5+SgvL0eXLl1w/vx5PPXUUxg8eHCVWJMnT7b8OSUlhbeTiYiIiKqR\nk5ODnJwcp8a0uxjUaDQ3bFNeXo5t27Zh7dq1uHTpEtq3b4927drBaDTatLMuBomIiIioetdfNJsy\nZYrDMe0uBsPCwlBYWGjZLiwstNwOviYiIgJBQUFQFAWKoqBz587YuXNnlWKQiIiIiOqG3d8ZTEpK\nQn5+PgoKClBWVoYlS5agb9++Nm3uu+8+rF+/HmazGZcuXUJubi7i4uIcHjQREREROYfdVwa1Wi3m\nzJmDHj16wGw2Y8SIEYiNjcW8efMAAKNHj4bJZELPnj3RokULuLm5YdSoUSwGiYiIiP5A7H60jNMG\nwEfLEBEREdmlTh8tQ0RERES3PxaDRERERC6MxSARERGRC2MxSEREROTCWAwSERERuTAWg0REREQu\njMUgERERkQtjMUhERETkwlgMEhEREbkwFoNERERELozFIBEREZELYzFIRERE5MJYDBIRERG5MBaD\nRERERC6MxSARERGRC2MxSEREROTCWAwSERERuTCHisHs7GyYTCYYjUZkZmbW2O4///kPtFotvvzy\nS0fejoiIiIiczO5i0Gw2Iz09HdnZ2cjLy0NWVhb27dtXbbuJEyeiZ8+eEBGHBktEREREzmV3Mbh5\n82ZERUUhMjISOp0OaWlpWLZsWZV2b775JgYMGID69es7NFAiIiIicj6tvR2LiooQERFh2Q4PD0du\nbm6VNsuWLcN3332H//znP9BoNNXGmjx5suXPKSkpSElJsXdYRERERH9aOTk5yMnJcWpMu4vBmgo7\na+PGjcO0adOg0WggIjXeJrYuBomIiIioetdfNJsyZYrDMe0uBsPCwlBYWGjZLiwsRHh4uE2brVu3\nIi0tDQBQUlKClStXQqfToW/fvva+LRERERE5kUbs/FsdFRUViImJwdq1a9GwYUO0adMGWVlZiI2N\nrbb9sGHD0KdPH/Tr1892AOpVQyIiIiL6fZxRR9l9ZVCr1WLOnDno0aMHzGYzRowYgdjYWMybNw8A\nMHr0aIcGRkRERES3nt1XBp02AF4ZJCIiIrKLM+oo/gskRERERC6MxSARERGRC2MxSEREROTCWAwS\nERERuTAWg0REREQujMUgERERkQtjMUhERETkwlgMEhEREbkwFoNERERELozFIBEREZELYzFIRERE\n5MJYDBIRERG5MBaDRERERC6MxSARERGRC2MxSEREROTCWAwSERERuTCHi8Hs7GyYTCYYjUZkZmZW\n2b9o0SIkJiaiRYsW6NixI3bt2uXoWxIRERGRk2hEROztbDabERMTgzVr1iAsLAzJycnIyspCbGys\npc3GjRsRFxcHPz8/ZGdnY/Lkydi0adNvA9Bo4MAQiIiIiFyWM+ooh64Mbt68GVFRUYiMjIROp0Na\nWhqWLVtm06Z9+/bw8/MDALRt2xbHjh1z5C2JiIiIyIm0jnQuKipCRESEZTs8PBy5ubk1tn///ffR\nq1evKq9PnjzZ8ueUlBSkpKQ4MiwiIiKiP6WcnBzk5OQ4NaZDxaBGo7nptuvWrcOCBQvw008/Vdln\nXQwSERERUfWuv2g2ZcoUh2M6VAyGhYWhsLDQsl1YWIjw8PAq7Xbt2oVRo0YhOzsb9erVc+QtiYiI\niMiJHPrOYFJSEvLz81FQUICysjIsWbIEffv2tWlz9OhR9OvXD5988gmioqIcGiwREREROZdDVwa1\nWi3mzJmDHj16wGw2Y8SIEYiNjcW8efMAAKNHj8ZLL72EM2fOYMyYMQAAnU6HzZs3Oz5yIiIiInKY\nQ4+WccoA+GgZIiIiIrvU+aNliIiIiOj2xmKQiIiIyIWxGCQiIiJyYSwGiYiIiFwYi0EiIiIiF8Zi\nkIiIiMiFsRgkIiIicmEsBomIiIhcGItBIiIiIhfGYpCIiIjIhbEYJCIiInJhLAaJiIiIXBiLQSIi\nIiIXxmKQiIiIyIWxGCQiIiJyYSwGiYiIiFwYi0EiIiIiF+ZQMZidnQ2TyQSj0YjMzMxq2zz55JMw\nGo1ITEzE9u3bHXk7IiIiInIyrb0dzWYz0tPTsWbNGoSFhSE5ORl9+/ZFbGyspc2KFStw8OBB5Ofn\nIzc3F2PGjMGmTZucMnC6ORUVFZg9+03k5u5CQkIUnn32aSiKYtNm+/btmDPnPZjNZjz22BB06NDB\nZn95eTlmzpyNrVv3IjExBs88Mx6enp42bbZs2YK33loAEcGYMcMQERGBzMyZOHHiNO67rzvS0h60\naS8i+PTTLHz99Ro0aBCESZMm4MiRI3jnnYVwc9MgPX0kgoODMX36LJw8eQb9+/dC7973Yvr0mdi9\nOx9t2jTHk0+mIytrMVasyEF4eDCef/4Z7N+/H/PnfwKdzh1//etjqFevHl59dRZOnz6HBx/sg9TU\nVGRmzsC+fYfRvn1L/PWvT0Crtf0xyMnJwYIFWfD01GHcuDEwGAyYPn02fv31Ih566H7cddddyMyc\ngZ9/LkCnTndg7NjHsWDBB1i7dgOaNAnDpEnPYMuWLVi48DMYDHqMHz8W7u7umDlzDs6du4QhQwag\nffv2yMycgfz8o0hJaYvHHhuJefPm4/vvN8NobISJEydgw4YN+PjjL+Dra8DTT6fDbDbj9dffwqVL\nVzB06ANISkrCtGmv4ciRInTr1gHDhw/DW2+9g/XrtyImJhITJ07A999/j08/XQo/Py88++xTuHTp\nEmbNehulpeUYPnwQUlJSquTLm2/OxcaNOxAb2xQTJ07AqlWr8Nln3yAgwBfPPTcOjRo1sulTUlKC\njIzXcOzYSdx7bxekpT2IN96Yg82bd6N5cyOeffZp6PV6mz5bt27F3Lnvo7JS8PjjQ9GuXTub/WVl\nZZg5cza2bctDy5YmTJgwrkrO3ciVK1cwY8Ys7Nz5M+64Ix7jxz+JJUs+wzfffIewsPqYNGkCgoOD\nbfoUFhbi1Vdn4dSpXzFw4L3o2bMnpk+fiT17DqJt2xZ48sl0fPLJImRn/4CIiBA8//wz2Lt3L957\nbxE8PHR46qnR8PHxwfTps3HmzHkMGnQfunbtiszMGdi//wg6dGiF9PSx+PDDhVi9+idERjbApEnP\noF69erUey/nz55GZOQMHDvwfOndOxuOPP4b589/HunWbEBUVgYkTJyA3NxcfffQveHkpmDAhHQAw\nY8YcXLx4GUOHPoA2bdogM3MGDh4sRJcu7TBq1Ai88867+OGH/yA6ujEmTpyAH3/8EYsW/Rt+fl6Y\nMOGvKC0txaxZb+Py5VIMHz4ILVu2xLRpr6GgoBipqR0xYsRwaDQam7F+/fXXyMpahnr1fPDss08h\nMjKy1mOrbo3Kzs7Gv/61HIGBfnjuuXEoKSmxWaOioqIwbdoMFBX9F717d8WDDz6A119/w2aNWrp0\nGZYu/RbBwfUwceLTaNiwYa3jKC0txYwZs7Bjx360bh2H8eOfxOeff1Hna1R6+lh89NHHWLVqPRo1\nCsXzzz+DXbt23ZZr1KhRI6vky/Lly52+Rp05cwZvvvkuysvNGDXqEXTq1KnWc1+d69eoyMhIZGbO\nRHFxCfr0uRsPPTSoyrEsXrwEy5atQkhIACZNmoDQ0NDf/b51Ruy0YcMG6dGjh2U7IyNDMjIybNqM\nHj1aFi9ebNmOiYmRX375xaaNA0OgG6isrJT77hskBsPdAswXvb6ftG3bVSoqKixttmzZIgZDkAAZ\nAswQRakv3333nU2Mnj37iaL0FGC+KEpf6dy5p5jNZkubDRs2qDFeFWC6KEqA1KvXULTacQLME4Mh\nRqZNe81mbC+/PE0MhlgB5olW+6QEBDQQRakvwGsCZIqiBIifXwNxd39GgHdEUZpJZGS86PV/UcfR\nXaKiWojB0FyAd0WnGyuBgWGiKMECzBRgqihKPfH1DRF394kCvC2KEimNG8eJXj9QgPliMHSR/v0f\nsRnX8uXLxWAIFWCWaDT/FEWpJ97eweLm9oIAc0VRIiQiwiSenoPUGHdKdHRLMRiSBZgvHh7DJTS0\nsShKAwHeEI1mihgMAeLlFSQazT8EmCOK0lAaNjSKp+cQNUZ7iY5uJQZDewHmi6fnYGnYsKkYDOEC\nzBGN5h/i5XUtxhQB3hBFCZXQ0Gbi4TFcjZGsxrhTPdeDJDw8SgyGRgLMFTe3F8TbO1AMhkDRaP4p\nwCwxGEJl+fLlNsffv/8jYjB0UWMMlEaNYsRgaCLA2+LuPlHq1WsoRUVFlvZnz56VsDCj6HRjBXhX\nFCVBoqJaiqJ0V2PcLx06pNrkXG5urpovmQK8JgZDffn+++9tcu7uu/uKovRSz3Vv6dLlXpucuxGz\n2Sx33tlTFKWvGqOnmi/xAswTnS5dGjRoJmfOnLH0KS4uloCAMHF3f1bNlyZqzvVT5/huMRoTxcsr\nUc25xyUoKFwUJUSA10WjeUUUpZ74+ASLm9vzArwlitJIGjWKFb3+QTXGXWq+3CHAu+LhMVKaNk2Q\nCxcu1HgspaWlEh/fRjw9H1ZjdFTPdVs1X4ZKaGikKEpDAd4UjeZFq3x5UYA3RVEaSIMGUeLpOVSN\n0VaN0VGN8bCEhTUTgyFCzZe/i5dXoBrjJQFmi14fLCEhTcXDY6QA74rBcIeMG/eczVjfe2+BGAyN\nBXhL3NyeF3//BlJYWFjjsVW3RjVubBKDoamac8+Kr299UZRAyxql1wdKYGCE6HTp6voSJ1FRiTZr\nVJMm8WIwRAvwjmi1T0v9+o3k5MmTteZLly73iqL0VmP0kqZNE/4Qa5TR2FIMhlZqvjwmwcERoii3\n5xr1xBNP28z7hx8udPoa5eNTXwyGQAGmCjBTDIZgWb16dY3nvjrXr1GKEigBAWGi1T6p5lysvPzy\nNJs+06a9JgZDjJov4yQ4uLGUlJT8rve1lzPqKLsj/Otf/5KRI0datj/++GNJT0+3adO7d2/56aef\nLNvdunWTLVu22A4AkBdffNHy37p16+wdEl2nsLBQ9PogAS4LIAJUiLe3SXJzcy1tBg4cKsAsdb8I\nsFDuuquPZX9+fr66aJSq+8vFy6uJ7Nixw9Kmd+80Ad62ivGQuLn1s9reLz4+wTZj8/IKEOCQpY2b\nWzMBFlj1+Yu4uT1stb1YgIYCVKjblwTQCVBoFSNSgE+t+twjGs1Iq+35otFECmBWty+Kp2c9OX78\nuGVcSUldBfjCqk830WjSrbZniUYTI0Clul0igFaA0+p2pbi5NRDgG6s+nQR4xmp7qmg0za1iHFOP\n5awlhkYTLMAaqz5tBfh/Vtt/Eze3ZKsYBwXwFOC8um0WjSZQgB+t+rQW4GWr7c8lKamr5diPHz8u\nnp711Lm9FsNfgM2WPjrdaJtf+hYuXCheXn2tYm4QwFeAK5ac8/KKkq1bt1r63H//wwK8adXnfenW\n7X7L/v3796sfMmXq/jLx8mose/bsuenc37Fjh3h5NRGgXI1xWZ2fAsv7Ggx/kffff9/SZ/r06Wqh\n89vPAhBhlXMX1XNdbJVzjQRYYtUnVTSax6223xKNpplVzp1RY5RYzrW3d1f5/PPPazyWVatWiY9P\nktW5PqHGOGOVL6ECZFu9bwcBnrfanixubq2tYhQI4CHAOasYQQKss+qTJMBkq+3nxM2tvdX2f0Wr\n9ZTy8nLLWMPCTAJstLTRasfKP//5co3HVt0aBfgJsM0SQ6OJF9s1aoS4ufWy2l4ngL9Yr1GAlwB7\nLW30+kfkjTfeqHEce/bsES+vxlY5VyqAXup+jTqnnusTVjHC5fZco06JVusppaWllnlv3DhBnL1G\nXR33dKsYi6Rjx3tuuGZYq7pGDRE3N+t17qB4eQXY9PHxCRbgZ0sbRUmTt95663e9781at26dTd3k\njGLQ7tvE118ereXK4w37TZ482d5hUC1KS0vh5uYJwEN9xR1ubt4oKyuztLl8uRSAr1UvP5SWll0X\nQwGgs4rhdYMYOohY39LzQ0VFGayVl9v2EXGrJoa/1bYHAAWAu2U/INfF0FwXQ1slhogBv31VVg83\nN0+bY7ly5fpjqRoD8AagsdoGAC/1/xpcTXnbYwH8r9v2tYqhU8dksMS4yjqGOwA/mxgi1jG06n/X\nvgJw7Rhri1HdufYEcO3cuak/v7/FMJur9rk6jms8rP67GuNmcu7Klepy7trypIWbm8Emxo1cjeGF\n3/LFA0AlbPPFr8q5N5uvPxaDVQwtruacj1WM63Ouurz1xm/nw0ON8Vu+AH61HltpaSk0Gmfkiw9s\n88Udv+VLdTG01cSw3u8NEYHZbLZ81aKszPbcms1+6s9Uzcd2/RpV9Ty5VxmXiPW4ruXsb2vU9WuD\n2XzjOXZzM+C3nNNVM466WKOu5Yt3LeO4XdYoLwAaVFRUwMPj6phKS69fCxxfo66WNTXHuBnVf6bZ\nxrz+M62iwrZPZWXtOeeIlJQUm9vnU6ZMcTyovVXkxo0bbW4TT506VaZNs71sOnr0aMnKyrJs8zbx\n/5bZbJbExA7i4TFGgI2i1f5NGjUyyaVLlyxtvvnmG1GUMAG+FuBbMRiaycKFH1n2l5eXS2xskuh0\nTwqwUXS656RJkwS5cuWKpc3nn3+hXupfLsBK0esbiqenvwDzBVgvitJNRox4wmZsQ4c+rt5K/EmA\nd8XT01cUJVKuXt34RvT6UPU3wA/UGJ3Ex6eBaLXPq+N4Qnx9w0RR7hVgg2g0b4mnp58oSpQACad1\njgAADyRJREFUqwT4SvT6+mqMjwX4QRSlrXh7h4q7+z8E2CgeHqOkdes7bW4/zp37tnpraI0A/xZP\nz0Dx9AxQf5v/XhSltXh5BYu7+z8F2CCeno+Kn1+E6PUPCLBB3NxmiKIEiMGQIMB3Anwunp71xNMz\nUK5eQcoRRWkuBkN9cXPLVGM8LP7+EeptwA3i5jZNDIZA9dZQjgBLxMPDX/T6+gJ8LsB3oiixYjAE\nipvbDDXGQDXGowJsEHf3f4qXV30xGNoI8L0An4qnp6/o9cEC/FuANWIwxMrcuW/b5Evr1neKh8dj\nAmwUd/d/iLd3iBgMHQT4QYCPxWAIkt27d1v6FBYWio9PsGg0bwmwQfT6XuLnFy463RNqzj0vkZFx\ncvnyZUufpUuXqrcjvxEgWwyGJrJo0aeW/WVlZRIT01p0uvFqjAkSFZVoc0XhRq5cuSJNmiSIVvuc\neq6fVPOlp5pz74iPT7D83//9n6VPXl6eemtooQA/iqK0Fx+fUNFq/6bGeFz8/MLVW88bRKOZo+Zc\ntACrBVgqnp5Bar58ouZckpovUwTYKJ6eI9R86a/GeF38/RvIiRMnajyWs2fPSnBwpLi7T1XP9RD1\nXKep+TJdzbkWcvUq2WdqzgUJ8JkA60RR4sVgCBI3t+kCbBAPjwfVGEPUfJkqBkOQGAyt1ZzLEk9P\nP/H0rC9Xr0KtFYMhRhQlUDSa19Vz3V96937AZqyTJv1DDIZ2ar58IgZDkOzcubPGY6tujfLxCRWD\noZN6xWihmrcNLWuUXt9I9Pp6ArwjwE+i13cXX98wmzXKx6eBKEoXAdYLsEC8vIIkPz+/xnGUlpZK\nVFSiaLUT1Bjj1Xyp+zXKzy9c9Pr71RhviF7vL4piuu3WKL3+AenZs5/NvP/tb1OcvkZdjdFAgK8E\nWCUGg1Hef/+DGs99da5fo/T6cDXn3hXgJ1GU7vLoo4/b9Bkx4glRlG5qzs0XL68gOXTo0O96X3s5\no46yO0J5ebk0bdpUjhw5IqWlpZKYmCh5eXk2bZYvXy733HP18uzGjRulbdu2VQfAYvCWOn36tAwa\nNEKMxiTp0yfN5vte13z22b8kMbGzJCR0lPfeW1Bl/3//+18ZOHCoGI1Jcv/9D0txcXGVNosWfSot\nWtwpzZt3ko8//kS2bt0qnTrdIzExbeS55/4uZWVlNu1LS0tlwoQXJCamjXTqdI9s27ZNPvxwoTRv\n3klatLhTsrIWy6ZNm6RDhx5iMrWVF16YLEePHpX77ntIjMYkefDBYXL8+HF56qnnJDo6WTp3vld2\n794t7777niQkdJTExM7yxRdfyPr166Vdu1QxmdrKiy++LAUFBdK794NiNCbJww+PtPnOmMjV7zDN\nnfu2xMd3kNatU+Srr76SnJwcadPmbomNbScvv5wphw8flnvuGSBGY5IMGTJaiouL5bHHnpTo6GS5\n++77Zf/+/TJz5hsSH99B7riji6xcuVJWr14tycndJC6uvbz66kw5ePCgdO/eT4zGJBk+/AkpLi6W\n4cOfEKMxSbp37ycHDx6UV1+dKXFx7SU5uZusXr1aVq5cKXfc0UXi4zvIzJlvyP79++Xuu++X6Ohk\neeyxJ6W4uFiGDBktRmOS3HPPADl8+LC8/HKmxMa2kzZt7pacnBz56quvpHXrFImP7yBz574tlZWV\nNsd/5swZefjhkWI0Jknv3g9KQUGBvPjiy2IytZV27VJl/fr1Vc797t27pXPneyU6Olmeeuo5OX78\nuDz44DAxGpPkvvsesrkNf01W1mJLvnz44cIq+0+ePCkDBgwRozFJ+vUbXGuxVJPi4mK5//6HxWhM\nkoEDh0pRUZGMHz9JYmLaSOfO91ZbpGzcuFHat+8uJlNb+fvfX5KjR49Knz5pYjQmyaBBI+T48eOS\nnv6MREcnS0pKb9m9e7e89dY8SUjoKK1a3SX//ve/5YcffrDky0svZUhBQYHce+8DYjQmySOPPCbF\nxcUyZsx4iY5Olm7d7pP9+/ff8FiOHDkiPXr0l+joZHn00TFSXFwsI0emS3R0sqSm/kUOHDggr702\nS+Li2ktSUlf59ttv5dtvv5WkpK4SF9deZsyYJQcOHJDU1L9IdHSyjByZLsXFxfLoo2MkOjpZevbs\nL4cOHZKMjOkSG9tOkpO7ydq1a+Wbb76x5Mvs2XNk37590q3bfRIdnSxjxoy3+cVS5OqH9UsvZVhy\n7ocffrjhsV2/Rh09elT+/veXxGRqK+3bd5eNGzdWWaN27twpnTvfKzExbWT8+ElSVFRks0YVFhbK\npEn/kJiYNtKxY0/ZvHnzDcdx4sQJ6ddvsBiNSTJgwBA5duzYH2KNKi4ulrFjn5bo6GTp0qWP7N27\n97Zdoy5evFglX27FGvXFF19Y8uXdd9+74bmvzvVr1LZt2yyfaRMmvFDll9OysjJ57rm/W/LF+qsx\nt5oz6iiNGsguK1euxLhx42A2mzFixAg8//zzmDdvHgBg9OjRAID09HRkZ2fDy8sLH3zwAVq3bm0T\nQ6PRVLmVTEREREQ35ow6yqFi0BlYDBIRERHZxxl1FP8FEiIiIiIXxmKQiIiIyIWxGCQiIiJyYSwG\niYiIiFwYi0EiIiIiF8ZikIiIiMiFsRgkIiIicmEsBomIiIhcGItBIiIiIhfGYpCIiIjIhbEYJCIi\nInJhLAaJiIiIXBiLQSIiIiIXxmKQiIiIyIWxGCQiIiJyYSwGiYiIiFwYi8E/oZycnLoewp8O59T5\nOKfOxzl1Ps6p83FO/3jsLgZPnz6N1NRUREdHo3v37jh79myVNoWFhejSpQvi4+ORkJCAN954w6HB\n0s3hD5rzcU6dj3PqfJxT5+OcOh/n9I/H7mJw2rRpSE1NxYEDB9CtWzdMmzatShudTofXX38de/fu\nxaZNmzB37lzs27fPoQETERERkfPYXQx+9dVXGDp0KABg6NChWLp0aZU2oaGhaNmyJQDA29sbsbGx\nOH78uL1vSUREREROphERsadjvXr1cObMGQCAiCAgIMCyXZ2CggLcdddd2Lt3L7y9vX8bgEZjz9sT\nEREREa7WYY7Q1rYzNTUVv/zyS5XXX3nlFZttjUZTa1F34cIFDBgwALNnz7YpBAHHD4CIiIiI7Fdr\nMbh69eoa94WEhOCXX35BaGgoiouLERwcXG278vJy9O/fH4888gjuv/9+x0ZLRERERE5l93cG+/bt\ni4ULFwIAFi5cWG2hJyIYMWIE4uLiMG7cOPtHSURERES3hN3fGTx9+jQeeOABHD16FJGRkfjss8/g\n7++P48ePY9SoUVi+fDnWr1+Pzp07o0WLFpbbyBkZGejZs6dTD4KIiIiI7GP3lcGAgACsWbMGBw4c\nwKpVq+Dv7w8AaNiwIZYvXw4A6NSpEyorK7Fjxw5s374d27dvtxSCzz77LGJjY5GYmIh+/frh119/\ntcTOyMiA0WiEyWTCqlWrHDk+l5OdnQ2TyQSj0YjMzMy6Hs5tqabnY97MszWpdmazGa1atUKfPn0A\ncE4ddfbsWQwYMACxsbGIi4tDbm4u59RBGRkZiI+PR/PmzfHQQw+htLSUc/o7DB8+HCEhIWjevLnl\ntdrmj5/3N1bdnDq7hqqzf4Gke/fu2Lt3L3bu3Ino6GhkZGQAAPLy8rBkyRLk5eUhOzsbY8eORWVl\nZV0N87ZiNpuRnp6O7Oxs5OXlISsri891tENNz8e8mWdrUu1mz56NuLg4y50CzqljnnrqKfTq1Qv7\n9u3Drl27YDKZOKcOKCgowPz587Ft2zbs3r0bZrMZixcv5pz+DsOGDUN2drbNazXNHz/vb051c+rs\nGqrOisHU1FS4uV19+7Zt2+LYsWMAgGXLlmHQoEHQ6XSIjIxEVFQUNm/eXFfDvK1s3rwZUVFRiIyM\nhE6nQ1paGpYtW1bXw7rtVPd8zKKiopt6tibV7NixY1ixYgVGjhxpeYoA59R+v/76K3788UcMHz4c\nAKDVauHn58c5dYCvry90Oh0uXbqEiooKXLp0CQ0bNuSc/g533nkn6tWrZ/NaTfPHz/ubU92cOruG\n+kP828QLFixAr169AADHjx9HeHi4ZV94eDiKiorqami3laKiIkRERFi2OXeOKygowPbt29G2bVuc\nOHECISEhAK7+bfoTJ07U8ehuL+PHj8f06dMtCxgAzqkDjhw5gvr162PYsGFo3bo1Ro0ahYsXL3JO\nHRAQEIAJEyagUaNGaNiwIfz9/ZGamso5dVBN88fPe+dwRg11S4vB1NRUNG/evMp/X3/9taXNK6+8\nAg8PDzz00EM1xuGDqW8O58m5Lly4gP79+2P27Nnw8fGx2XejZ2uSrW+++QbBwcFo1apVjc8W5Zz+\nPhUVFdi2bRvGjh2Lbdu2wcvLq8rtS87p73Po0CHMmjULBQUFOH78OC5cuIBPPvnEpg3n1DE3mj/O\n7e/jrBqq1ucMOqq25xQCwIcffogVK1Zg7dq1ltfCwsJQWFho2T527BjCwsJu2Rj/TK6fu8LCQpvf\nEOjmXXs+5uDBgy2PTbrZZ2tSVRs2bMBXX32FFStW4MqVKzh37hwGDx7MOXVAeHg4wsPDkZycDAAY\nMGAAMjIyEBoayjm105YtW9ChQwcEBgYCAPr164eNGzdyTh1U0885P+8d48waqs5uE2dnZ2P69OlY\ntmwZ9Hq95fW+ffti8eLFKCsrw5EjR5Cfn482bdrU1TBvK0lJScjPz0dBQQHKysqwZMkS9O3bt66H\nddup6fmYN/NsTare1KlTUVhYiCNHjmDx4sXo2rUrPv74Y86pA0JDQxEREYEDBw4AANasWYP4+Hj0\n6dOHc2onk8mETZs24fLlyxARrFmzBnFxcZxTB9X0c87Pe/s5vYaSOhIVFSWNGjWSli1bSsuWLWXM\nmDGWfa+88oo0a9ZMYmJiJDs7u66GeFtasWKFREdHS7NmzWTq1Kl1PZzb0o8//igajUYSExMt+bly\n5Uo5deqUdOvWTYxGo6SmpsqZM2fqeqi3pZycHOnTp4+ICOfUQTt27JCkpCRp0aKF/OUvf5GzZ89y\nTh2UmZkpcXFxkpCQIEOGDJGysjLO6e+QlpYmDRo0EJ1OJ+Hh4bJgwYJa54+f9zd2/Zy+//77Tq+h\n7H7oNBERERHd/v4Qf5uYiIiIiOoGi0EiIiIiF8ZikIiIiMiFsRgkIiIicmEsBomIiIhcGItBIiIi\nIhf2/wE9dvlhFBepTgAAAABJRU5ErkJggg==\n"
239 | }
240 | ],
241 | "prompt_number": 8
242 | },
243 | {
244 | "cell_type": "code",
245 | "collapsed": false,
246 | "input": [],
247 | "language": "python",
248 | "metadata": {},
249 | "outputs": [],
250 | "prompt_number": 2
251 | }
252 | ],
253 | "metadata": {}
254 | }
255 | ]
256 | }
--------------------------------------------------------------------------------
![](\"files/images/XOR2.png\"/)
\n",
225 | "\n",
226 | "\"The feedforward neural network for the 2 input Exclusive Or problem\""
227 | ]
228 | },
229 | {
230 | "cell_type": "code",
231 | "collapsed": false,
232 | "input": [
233 | "import networkx as nx\n",
234 | "import sys\n",
235 | "from math import tanh\n",
236 | "sys.path.append('pysrc')\n",
237 | "import neural_networks as nn\n",
238 | "from functools import partial\n",
239 | "\n",
240 | "DG = nx.DiGraph()\n",
241 | "DG.add_nodes_from(['X1','X2','H0','H1','H2','O1'])\n",
242 | "\n",
243 | "#set the activation functions\n",
244 | "DG.node['H0']['af']=1.0\n",
245 | "for node in ['H1','H2','O1']:\n",
246 | " DG.node[node]['af']=tanh\n",
247 | " \n",
248 | "#set the derivatives of the activation functions for all nodes except the output, this is done below\n",
249 | "def zero(x):\n",
250 | " return 0\n",
251 | "for node in ['X1','X2','H0']: #the inputs and bias terms have zero derivatives\n",
252 | " DG.node[node]['daf'] = zero\n",
253 | "def dtanh(x):\n",
254 | " return 1.0 - tanh(x) * tanh(x)\n",
255 | "for node in ['H1','H2']:\n",
256 | " DG.node[node]['daf']=dtanh\n",
257 | " \n",
258 | "#create the edges\n",
259 | "for source in ['X1','X2']:\n",
260 | " for target in ['H1','H2']:\n",
261 | " DG.add_weighted_edges_from([(source,target,1.0)])\n",
262 | "for source in ['H0','H1','H2']:\n",
263 | " DG.add_weighted_edges_from([(source, 'O1', 1.0)])\n",
264 | " \n",
265 | "#set the input values\n",
266 | "DG.node['X1']['af']=0\n",
267 | "DG.node['X2']['af']=1\n",
268 | "\n",
269 | "#given these inputs, the correct output should be 1\n",
270 | "#we'll use a partial function so we can assign the correct 'daf' value\n",
271 | "#dynamically when we iteratively train the network\n",
272 | "def dout(x, t):\n",
273 | " if x<0:\n",
274 | " xx = 0\n",
275 | " else:\n",
276 | " xx = 1\n",
277 | " return xx - t\n",
278 | "DG.node['O1']['daf']=partial(dout, t=1)\n",
279 | "\n",
280 | "nn.forward_prop(DG)\n",
281 | "\n",
282 | "nn.error_back_prop(DG)\n",
283 | "\n",
284 | "for node in ['X1','X2','H0','H1','H2','O1']:\n",
285 | " print \"node {0} has output {1} and error {2}\".format(node, DG.node[node]['o'], DG.node[node]['e'])"
286 | ],
287 | "language": "python",
288 | "metadata": {},
289 | "outputs": [
290 | {
291 | "output_type": "stream",
292 | "stream": "stdout",
293 | "text": [
294 | "node X1 has output 0 and error 0.0\n",
295 | "node X2 has output 1 and error 0.0\n",
296 | "node H0 has output 1.0 and error 0.0\n",
297 | "node H1 has output 0.761594155956 and error 0.0\n",
298 | "node H2 has output 0.761594155956 and error 0.0\n",
299 | "node O1 has output 0.987217029728 and error 0.0\n"
300 | ]
301 | }
302 | ],
303 | "prompt_number": 5
304 | },
305 | {
306 | "cell_type": "code",
307 | "collapsed": false,
308 | "input": [],
309 | "language": "python",
310 | "metadata": {},
311 | "outputs": []
312 | }
313 | ],
314 | "metadata": {}
315 | }
316 | ]
317 | }
318 |
--------------------------------------------------------------------------------
/pysrc/ipynbs_src/02_Linear_Regression.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | #