├── Estimating Shannon Entropy.ipynb
└── Mutual Entropy for SCA  with bounds.ipynb


/Estimating Shannon Entropy.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 1,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "import numpy as np\n",
 10 |     "import matplotlib.pyplot as plt\n",
 11 |     "import pandas as pd\n",
 12 |     "from collections import Counter #for counting elements in a histogram\n",
 13 |     "from scipy.stats.kde import gaussian_kde\n",
 14 |     "import statsmodels.api as sm\n",
 15 |     "from statsmodels.nonparametric.bandwidths import bw_silverman, bw_scott, select_bandwidth\n",
 16 |     "from IPython.display import display\n",
 17 |     "from scipy.integrate import quad\n",
 18 |     "from scipy.stats import norm, entropy\n",
 19 |     "from scipy.stats import rayleigh\n",
 20 |     "from scipy.stats import binom\n",
 21 |     "import seaborn as sns\n",
 22 |     "import scipy.stats as ss\n",
 23 |     "from sklearn.neighbors import KernelDensity"
 24 |    ]
 25 |   },
 26 |   {
 27 |    "cell_type": "markdown",
 28 |    "metadata": {},
 29 |    "source": [
 30 |     "## Estimation of Shannon entropy\n",
 31 |     "\n",
 32 |     "When we want to calculate for a data set, for which we do not know the analytical formula,  we can no longer **calculate** the entropy, but only **estimate** entropy. \n",
 33 |     "The first step then, is  to estimate a pdf, as described above\n",
 34 |     "\n",
 35 |     "<b> To explore the concepts in this notebook: </b>:\n",
 36 |     "* compare the entropy values of the estimated set with the analytical value of the entropy in the cell output;\n",
 37 |     "* run the cell a couple of time and see how the estimate changes as a function of the random number generated;\n",
 38 |     "* change the number of points availabe for estimating the pdf, by changing the value of the variable ```samples``` and observe the difference in entropy\n",
 39 |     "\n"
 40 |    ]
 41 |   },
 42 |   {
 43 |    "cell_type": "code",
 44 |    "execution_count": 3,
 45 |    "metadata": {},
 46 |    "outputs": [
 47 |     {
 48 |      "name": "stdout",
 49 |      "output_type": "stream",
 50 |      "text": [
 51 |       "Entropy value =  3.0470955851806423\n",
 52 |       "H(X) (true):     3.101851507400887\n",
 53 |       "H(X) (crude):    3.0985811531540732\n",
 54 |       "H(X) (KDE):      3.0067320240140405\n",
 55 |       "H(X) (hist):     2.9984540194989875\n"
 56 |      ]
 57 |     }
 58 |    ],
 59 |    "source": [
 60 |     "def entropy_estimate_true(x, m, s):\n",
 61 |     "    \"\"\"  \n",
 62 |     "    Estimate Shannon Entropy of the dataset x, using\n",
 63 |     "    the analytical formula of the true distribution is known\n",
 64 |     "    \n",
 65 |     "    x = input data\n",
 66 |     "    m = true mean \n",
 67 |     "    s = true standard deviation \n",
 68 |     "    \n",
 69 |     "    returns H(X) - estimate Shannon Entropy\n",
 70 |     "    \"\"\"\n",
 71 |     "    p_x=ss.norm.pdf(x,m,s)\n",
 72 |     "    return -np.mean(np.where(p_x != 0, np.log2(p_x), 0))\n",
 73 |     "\n",
 74 |     "def entropy_estimate_crude(x):\n",
 75 |     "    \"\"\" \n",
 76 |     "    Estimate the Shannon Entropy for dataset x\n",
 77 |     "    using estimated mean and standard deviation\n",
 78 |     "          \n",
 79 |     "    x = input data\n",
 80 |     "    \n",
 81 |     "    returns H(X) - estimate Shannon Entropy\n",
 82 |     "    \"\"\"\n",
 83 |     "    (m,s)=[np.mean(x),np.std(x)]\n",
 84 |     "    p_x=ss.norm.pdf(x,m,s)\n",
 85 |     "    return -np.mean(np.where(p_x != 0, np.log2(p_x), 0))\n",
 86 |     "  \n",
 87 |     "def entropy_estimate_kde(x, bandwidth=0.4):\n",
 88 |     "    \"\"\" \n",
 89 |     "    Estimate the Shannon Entropy for dataset x\n",
 90 |     "    using the KDE for density estimation\n",
 91 |     "          \n",
 92 |     "    x = input data\n",
 93 |     "    \n",
 94 |     "    returns H(X) - estimate Shannon Entropy\n",
 95 |     "    \"\"\"\n",
 96 |     "    kde= KernelDensity(bandwidth, kernel='gaussian').fit(x[:, np.newaxis])\n",
 97 |     "    p_x = np.exp(kde.score_samples(x[:, np.newaxis]))\n",
 98 |     "   \n",
 99 |     "    return -np.mean(np.where(p_x > 0, np.log2(p_x), 0))\n",
100 |     "   \n",
101 |     "def entropy_estimate_hist(xs_syntetic, b='auto'):\n",
102 |     "    \"\"\" \n",
103 |     "    Estimate the Shannon Entropy for dataset x\n",
104 |     "    using the histogram method\n",
105 |     "          \n",
106 |     "    x = input data\n",
107 |     "    \n",
108 |     "    returns H(X) - estimate Shannon Entropy\n",
109 |     "    \"\"\"\n",
110 |     "\n",
111 |     "    ys_freq,ys_hist=np.histogram(xs_syntetic,bins=b)\n",
112 |     "    bin_size=ys_hist[1]-ys_hist[0]\n",
113 |     "    p_y=ys_freq/len(xs_syntetic)\n",
114 |     "    acc=0\n",
115 |     "    if bin_size > 0:\n",
116 |     "        for p in p_y:\n",
117 |     "            if p/bin_size > 0:\n",
118 |     "                acc += p * np.log2(p/bin_size)\n",
119 |     "    return(-acc)      \n",
120 |     "    #return -np.sum(np.where(p_y/bin_size > 0, p_y * np.log2(p_y/bin_size), 0))\n",
121 |     "    \n",
122 |     "\n",
123 |     "def entropy_normal(f,m,s):\n",
124 |     "    \"\"\" \n",
125 |     "    Calculates the Shannon Entropy for variable x, \n",
126 |     "    when we known the formula for the probability density function \n",
127 |     "          \n",
128 |     "    f = formula for probability density\n",
129 |     "    m = first order moment\n",
130 |     "    s = second order moment\n",
131 |     "    \n",
132 |     "    returns H(X) - Shannon Entropy\n",
133 |     "    \"\"\"\n",
134 |     "    g= lambda x:f(x,m,s)\n",
135 |     "    def g_log(x):\n",
136 |     "        temp=g(x)\n",
137 |     "        if temp==0:\n",
138 |     "            return 0\n",
139 |     "        else:\n",
140 |     "            return temp*(np.log2(temp))\n",
141 |     "    i,err=quad(g_log, -np.inf,np.inf)\n",
142 |     "    return -i\n",
143 |     "\n",
144 |     "\n",
145 |     "m=-2 \n",
146 |     "s=2\n",
147 |     "\n",
148 |     "samples=150\n",
149 |     "xs_syntetic=ss.norm.rvs(m,s,samples)\n",
150 |     "print('Entropy value =  ' +str(entropy_normal(norm.pdf,m,s)))\n",
151 |     "print('H(X) (true):    ',entropy_estimate_true(xs_syntetic, m,s))\n",
152 |     "print('H(X) (crude):   ',entropy_estimate_crude(xs_syntetic))\n",
153 |     "print('H(X) (KDE):     ',entropy_estimate_kde(xs_syntetic, bandwidth=0.4))\n",
154 |     "print('H(X) (hist):    ',entropy_estimate_hist(xs_syntetic))\n"
155 |    ]
156 |   },
157 |   {
158 |    "cell_type": "markdown",
159 |    "metadata": {},
160 |    "source": [
161 |     "## Precision of estimation\n",
162 |     "\n",
163 |     "In this section we explore the precision of the estimation as a function of the number of samples we have available.\n",
164 |     "We expect that the more points we have in the dataset, the closer the estimated values will be to the true value. The raeson being that estimated mean and standard deviation will be to the true mean and standard deviation. \n",
165 |     "\n",
166 |     "<b> To explore the concepts in this section: </b>:\n",
167 |     "* change the parameters of the probability density, by changing the values of ```m```, the mean, and ```s```  standard density values;\n",
168 |     "* run the cell a couple of time and see how the estimate changes as a function of the generated dataset;\n",
169 |     "* consider the following questions: Are our assumptions correct? The more points we have the more precise the estimation becomes?  How does the standard deviation value influence the precision of the estimate? Which of the three estimations methods seems to perform best?\n",
170 |     "\n"
171 |    ]
172 |   },
173 |   {
174 |    "cell_type": "code",
175 |    "execution_count": 8,
176 |    "metadata": {},
177 |    "outputs": [
178 |     {
179 |      "data": {
180 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3zV1f348de5K3vvvdlCgAioqIgLBalQEYuj2qp11NWf9Vu/X7WW1ra2ahUH1qI4EffEgQytigJhz5uELEJIbva8+57fHzeJAZJwA7lZnOfjkQe5n/m+13jP56z3EVJKFEVRlFOXZqADUBRFUQaWKggURVFOcaogUBRFOcWpgkBRFOUUpwoCRVGUU5xuoAPorcjISJmamjrQYSiKogwpW7ZsqZZSRnW1b8gVBKmpqeTm5g50GIqiKEOKEKKku32qaUhRFOUUpwoCRVGUU5wqCBRFUU5xXi8IhBBaIcQ2IcSnPRxzhRBCCiFyvB2PoiiKcqT+qBHcBezrbqcQIgi4E9jYD7EoiqIoR/FqQSCESARmA8t6OOzPwD8AizdjURRFUbrm7RrBk8B9gKurnUKIiUCSlLLbZqO2424WQuQKIXKrqqq8EKaiKMqpy2sFgRBiDmCSUm7pZr8G+Bfw/453LSnlC1LKHCllTlRUl/Mhho7ybbDrXXA5BzoSRVEUwLs1grOAuUKIYmAlMFMI8Xqn/UHAOODrtmOmAR8P6w7jyr3Uv/ozCj/+DSw7H8q6LCMVRVH6ldcKAinl/VLKRCllKnAVsE5KeU2n/Q1SykgpZWrbMT8Cc6WUw3PacONh5BsLuCMyiJ8lxvN/0oRp+YXw8R3QUj3Q0SmKcgrr93kEQojFQoi5/X3fAWVtghULWEsL2/Uazko4i8/9fZmTkswLhR9jeXoybPqPai5SFGVAiKG2VGVOTo4cUrmGnA54cyH2A+uZNyqbIGcSf8t+DG2ihce3PM7a0rXEo+V3lRVcFJyJmP0EJE0Z6KgVRRlmhBBbpJRdNr2rmcXeJCWs+h0UrOH9M66jxFLFRcW/ZNWSXex9q4G/nf4Pll20jMCwDO6NieJ6XT37XrsUPrwNmtXoqP5itdSzbccrNDeWDXQoijIgVI3Am759HNYupuWsO7m09r+MMIwhe/UCYlKDqSptwsdfx3nXjCL5tHDey3+PZ7Y9Tb21nsubW7mz2UHkjP+FnF+DdsgliR30XA4rudtfZFXee3xlraBJo8EgJdM1QcxKOp9zJ9+Gf3D8QIepKH2mpxqBKgi8Zec78P6NcNoCnkufyNKdz/OP0GUUft7Cooen4nRI1r6yl+qDzYycFsvZV2Zh1Zn5945/s2LfG/hIFzfV1nKtbzKGSx+DlDMH+h0NedLpJG//u6za/RqftRRRqdXg75Jc4BvL2Ykz2F6xmdXNB6jSCHxdknO0wcxKvpCzJ9+Gb1DMQIc/vElJfXUe+4pWs78il30Nheyz16MDTvdPYGr8WeSMWUhI5IiBjnTIUgUB8KdP9rC3vNELER1rtHUn/1f7f+QZRvGn8PvY77uYINdpzNl9HRoJW0f5AiBckuRKB8kVDmx6yEs2UBesxSoqqdS9Q7N2J3F2F/9TW4NOTmNF0I3UayP65T0MG1IS49yKnlXkGUopNmjQSskYqz8hjinI1p+RUm4nuFWPVW/jYLQ/ttAfserXkOdTSZ1W4OdyMcHiT4hzMibt5Vg1IQP9roY26STUlY+/3IqLAhq0Jsr0Fip1P7VURzskcQ5/nLgoMFiwaARCSjJtkOiIwuA6jTrNTJq1p0YB7eOqJdS1kejoSTyw4PITukZPBYFqc+hjCfZS7q1bTKUujsfDHqJC/xESO8kt8whulRTG//SRS42gJE5PTYiWkSU2Tjtg43CElsKEaJLlb2l27sWke4u7YzTkmPdyd+0t7PC5ii8CfoZTqP90PYlwFBAkP6BUn8f6QAFAltWHs1omYHbMJ6YmkKRKCxq06Oxmoqq30RCcio/DDw6djtkwHv8Yf2wh32PTr2WvTxWN2u8JdH3LeEsAIc4cKrRzVaFwHBppJ9y5E1+5A7umiFptNaUGO3u17i99ISXxDkGCLZiJ1TFEVkcSXB1ARE0TwS02QFId6kdNnIvDcRUURB7mW/8qHGI9OrmOkTZBvD0ancymWjODFm3kgL7fvuDjaiLcuRktO2nRHuSwpgFbYyTxh9PRNx4CTqwg6MkpUyPoF02VsOwCcFjgxjUUapzM/2g+C0cu5OL6RfzwwQHmjC7C8dUHRN58M8GzZyO0WgAcdicbPy5i+5pSgsJ9Of+Xo0kYEYbD5eCdvHd4duvTNNmbmN/YxB2aGMIvfQzSzh7gNzy4WOqK+GbTEj499A3faWw4hCBd6pgdM5VLJv0WWRPHrs/zKMlvQaIhtC6PZFc+o2ZPIPTSWZh37qLs028pOmCjInwCFr8oNLhISPEhc0Yq1bq1fFX4NmtbS2nSCIKdLs7XhTMr7RKmTL4FnX/4QH8EA8phaaCocC37D33H3pq97DdXsh8bzRr3l75OSjLQM1YXxQRbApmtsQTWBNBQ3EhdpZkmTQgt/nG0BsRi9QnruK7GZSeosYSQxiLCnCYi0wJpibexM7ictUFl7NJacQmBr8vFRKlnanAGU1MuYPSYK9AGDO5MBNLawsGitewq/Zrd1bvZbamkvAVGHkohtSaVMGs6Ln0aDn0QAGPjypjxx+tO6F6qaag/2Fpg+aVQnQfXr4KESdy17i42Vmzks/mf8dXjBWh1GiZv+SeWXbtASnyysoi65x4Cz5uBEO6n1sMF9ax9ZR8NVWbGz0xk2uUZ6A1aGqwNPL9jKSv3v4mvy8UttXUsSroA/cV/hVO4U9PZUsWmzc+yqvgL1sgmWjQaoqXgkrDTmJ19M0khU9n33SH2rC2ixaxBb2sirmozWZlakhfNwS8np+Ozb+dqbaXp628oXbWB4oNQGZ6N1TcMrXCRlOFH5rmplPM5qwvfZX1rGS0aQZjTxQX6CGalz2HyxJvQ+od1E/HwYG06TP6BL9lXvpH9dXnss1aRJ5xY2770faVkhPAl2xXNaeZ4YhpiwORP/WEzja06WvxiaPGPxebzU41Kq3ERGq4nIiWU8KQQwuMCcNhdVBQ2cHifiZoKKy7p/m/la64mpLGIkOYSIiIl9rhm9oeb+Cq8ih0BdgCCnC4mSwPTwkYyJfVCMkfNQwxkYW23UFX6PbuL17Cragd7WsrZ77QRfzickZVpxDan4kMaZt8kpMb9gOjvqCMywExsahAJOWnETB2N1qA/odurgsDbnA5462rIXw1XvQkjZ7HNtI3rPr+OOybewYLoq1nx8EbO+nkGvvfNI/SKK/CfmE3VU0uwlZTgl51N1D33EDDVPX/AbnXywwcH2PV1GaEx/pz/y9HEprv/hylsKOSfGx/lu8MbSLE7uLfRzLlT7kZMuw10hoH8FPqNtDSxb/tyPs1/jy9sVVTptARKuCAwndnjfsnkzLmUGxvY/WU+JXktSARhdUaS7PsZecl4wn8+D124Z18IzuZmmtaspfjzzRQf1mGKzMZuCEancZI6KoiMc5Iptn3M6sIP+NpyCLMQRDidXKiPYlbGZUycdBMa3yHcfCQlLTX57C/8in2HN7GvsZB9tjoKteBsK0CDXJIxIoBJrdFkNqUQVBeHtUZPQ4OkWRNKi38sdkNQxyV1GiehIYLwxGAiMqMJjw8kPC6AoHBfhEZ0FwkOu5Oq0mZ3wbC/iooD9Zgt7uM1ThvBTSUENxYRpm9AE1VPQWQF6yNr2RLpRApBuNPJFHyZGjaGqemzSBwxB+EX6p3PzWGj6VAue4q+YrdpK7ubD7LHZUbbYGBsWQIpdamE2tKx+aR11H40Ljth2gaiY7TEj40j8ewxBMb2XXyqIPAmKeGze2HzMrj0MZhyE1JKrv38Wsqby/l03qfsWV3Jpk+LWHRbMhULLiPukUcI/fl8pN1O/QcfUP3sczgqKwk46yyi7rkHv3FjASjbX8vaV/fRUmdl4kUpTJmThlbvfuL6tuxb/rnxrxQ1l3GG2cx9rhAyL34MMs4byE/Dexw2Du55m8/2vM6q1hKK9Dp0UnKObxyzRy7gnHHX4mjWsO+7g+xZV0KLWbif/k0byUoTJC2ag//UqQjNiU+dcdbXU//lVxSv3kZJtT9Vkdk49P4YtE7SxoWSNj2BfPMHrC78kP9aDmMVgmiHk4t8Yrg44zImTLwR4Rvchx9KH3G5sDdXUFO9j8raAkyNxZTWF7KvqYT9jiZKOnXiRrhggiOIiXVpxDamomuMpLVBR6Pdnxa/GBx6/45j9dgJCXQRFudP5IhYItMjCY8PICDU55ha2ImQUtJcZ3UXDPm1HN5rorba0anWUOWuNVjL0YfUUhZewn9jGtgS78KmF8Q5HEwRAUyNGMeUjNnEZM0Cn6Dj3LULTgfWip0Yi75i9+HN7G4qZperhTqbllFlAWRVphLbmo5GpNEckIxL635g83M1ExlsJzYjhMSpmcRMSEar9d7ULlUQeNP3S+CrB+HMO+GiPwOwpmQN93x9Dw+f8TDzs+bz5p824hdk4Pxx1Ry66y5S332348sewGWxULfiTWpeeAFnfT1BF19M1F134pOejs3s4Lt389n3/WHC4wO44PoxRCW7/1jtLjtvG9/m2S1P0eows6CpidujzyR01j8gJHFAPo4+5XJSV/AlX25fxqqGfWw3uDvIJ+tCmZ02m4sm/oYgn1BK99Sw+6sDlOY1u5/+a/eRbNlL1qzxRCyYj84LGWsd1dXUfbGawq92crAhlOrI8Th1vvjqHGRMjCTlzDj2Nr3Nlwc+4TtrBXYhiHM4udgnhlmZ8xiTfb33CwUpkeY6mmsLMdUaqawvwtRUhqm1EpOljkpHEyaXFRMuarQa5FFfzokODVMbksmqzyC4KQFnSwjNVj+a9eE4dX4dx+mlhRBfG2HRvkRkRBE9Nonw5BD8gw198oXfGw6bk6rSJg4XNnB492EqS5oxW91frhqnleCmEkKaSvDxMVETUsSGmCq2JUnqAwWpdjtTNcFMjZrA6ZmXEZpxPhgCjryBy4mzaj9FRWvZVf4je+oL2OVspERqSarUMLo8lpT6NAKc6bT6pmL2jwZASCehhhZi4gwkTEggYVoWQRH+R4fvVaog8JY9H8A718PYefDzl0Cjwe6yM/+j+WiFlnfnvkt9uYW3/rKJcxeNJGb7+1Q//zwjt25B4+t7zOWczc3UvrSc2pdfxmWxEDLvcqJuvx19fDzFu6pZ//p+LE12Jl+ayuRLUjqeHuot9Ty7dQnv5L+Lv8vJbY2tLJx4G/oz7wSdTz9/KCdJSswHf2D9ludZVbWFDQaBQwgyNX7MSTyPSyfdRlxICs11FvZ+e4g964tpNQv0tkbiK34kM1WSeNVsAs46q6Mj3tvsFRXUfvYlB9buo8wcSU3EOFxaA/4GB5k5sSSdEcmOmpV8WfQpP1hNOIQgye7gYt84ZmXNZ8SEXyJ8e/kkamvBUV9Gda0RU90BTI2lVLZUYDJXYbI1YnKaMeGgUqvB3EUtKERCtDAQI4JJtMQT2xBLUFMEupZgnGZ/LDZ/mkVox9MrgMHZQrDeTGiEnoiUMKLGJRM5OoGAkMH7NyalpKnWQmVhI+X7TBzeX0VtrUTiLqD8zFUENxQR4CyjKbCErZHF7EqWlEVIRjidTNWFMiVqImaHmd11Rnbb69in1RBeq2FEuS8jTSlEWDJw6FJoDErtKCB9sBIZ5iRuRDiJUzKIGRGJTt8/f4/dUQWBN5T+CK/MhfiJcN1HoHd/sb+1/y3+svEvPD3zaWYkzeCHDw+wbXUpNzx6FtX3/w5bYREZn63q8dKO2lpq/v1v6la8CUDYol8Q8Zvf4PAJ4tu38sjbVElUchDnXz+aiPjAjvMK6gr4xw+L+aFqG2k2O7+3+3L2hY9B1gXe+xz6iKNyLxtzn2NV+X9Zo3Ni1miIQc+lMVOYPfFWRsZMwOV0UbKnlt1rCjmY14REEF67l6TW3WRedBoRV/4cfWzsgL4P28GD1Hz6JQXf5HPIEU9t+GikRkegr4OsaQkkTAljq+kNvij8jI22KpxCkGp3cLFvArNGzCdz3FVgbaK5rghTbR6VDSWYmg9hajVRaavHZG/BJG2YNFCj1eI66olbJyEaLdFaP6INwUQbIokSKYSZE/CrD4F6P+z10NIMLXY9FuEP4qeCQric+DoaCNCYCQkRRCQFEzk6kZjJmfhHDsJmrRPgsDkxlTRRkV/LoV3lmMrMWOzuL2mt00pQYzFBrSXYDCXsDzvA3sQW/C0wsiKKlKYMtM5UmoLSaPGPdX920kWon5XoBD8SJiWTMD6B4Ejffq8NHY8qCPpadQG8eAH4hcONa6BtJEKLvYVL37+UtJA0ll+8HIDXH/yBkGh/5t6ZTcGFF+E7biyJ//qXR7exl5dT9eyzNHzwIRpfX8Kvv57wG66nuMDMNyuMWM0Opl6WTvaFyWjaOtmklHxT9g2PbVhMiaWK6a1mfh8ygfRLnoCwFO98Hp5y2qGlCporsTQcoqqhkMN1B1h36Du+EGZqdFqC0HBR2DhmT7iRycnnohEammrdT/97vy6h1QwGawNxFT+SkWQj8arLCDznHIRu8M2rsBYWUv3JlxR8V8whkUxd2EgQGkICnIyYnkTMpCA2lr/G6qLP2WyrwSUE0Q4HzRoNrV08xQdLQbTGhxh9INE+YUT7xxAVmEiELoMAUtDV+GA71EpTZTONdTZazAKzyw/Z6Yse6cLHWo+fq5lAg53AYC3BUf6EJocTmhVP6MhU9CGBx9x7OJNS0lRj4fCBBsp3lVORV0Ndo+anWkOrCYfeH7ve/bnohYOoSIgbFUnCpBRi0kIw+A6+v7+jqYKgLzVXuQsBazPc+BWEp3fsem77cyzdsZQ3Ln2D8VHjqSxu5N2/5zLzulGMGB9CXk4OUXffReQtt/TqltbCQqqeWkLTl1+iDQkh4je/weeyn/Pt+yUUbqsiNj2Y8385htCYn9oc7U47K/a+yr+3P4fZaeWqZjO3jLmekLN/31F76RNSgrkOmk3QXElrQxlVDcVUNR2kutWEyVJDta2JKqeZKhxUabVUabU0deoUM0g4Nyid2WOv5eysuRi0BlxOF8W7atizrpiDeY1Iifvpv3knmReOJfzKBRgSE/rufXiRlBJrXh6mj1ZTsLGMcn0GDaGZAISHOBkxPZXIib5sKHmV7RWbCPcJIzownujgFKLCMgkzpOFjDsRS66ShuIr6sjoaq8w0N7totetxcWSTg8HagK+1Fn+NmUB/SXCYD8HxQYSmxhA2OhnflCQ0PoO3OWcwsNucVJU0Ur6viord5Rj89SRMTCIuK4KwWP8eRzcNVqog6Cu2VnjlMqjcDb/8FJJO79hVba7m0vcv5eyEs3l8xuMAfPduPrvWl/Grf07HadxNyS8WkfjccwTNPLGRPebde6j6179o+f57dDExRNx6G1UpZ/HtOwdw2l1Mm5fB+BmJR/yR1lpqeWbjo7xX/BlBTie3W3UsmPl3dCMv7flmdjM0V0KzCdlUQUvjQUyNB6luOkSVuYoqax1V9haqXBaqNcL9Ba/T0tLFk6wBQZTGh0h9INE+oUT6RREVGE9UcBJRIemcFjeZYIO72aGx2sze78vZ1+npP/7wBjLiLcRfOYeg82ci9Cc2jnowkFJi2b2bio/WULClksN+I2gKTgUgKtxFSk4SrdVNNFY00VRvo9mswSmP/KLX25vxtdTgZ63D3+AgKFhDcHQAIUnhhGXG45+Rgj4ubkh/TkrfUwVBX3A54e3rYP8qWPgajL7siN1//uHPvJ//Ph9d/hHJwclIl+SV/91AVHIQs28bT93Kt6h4+GEy165Bn3ByT7ItGzdR9cQTmHfswJCSgv/Nd5BbnkDpnloSRoQy87rRBEf6HXGOsdbIP/57P5sa8sm02fi9XyZnjLmKxsYyqprKqGqpoNpcjcnWQLWj1f30rnM/vVdrtV12OPqhIVLrR5QhiCifCKICookMTCA6JIXI4BSi/KOI8o8i2BDcY3up0+miZGcNu9e7n/6REFG7h8TG7WScP5qIhVdiSBngZi0vkC4X5m3bKP9oHQd21lEROIrmwES0DjN+lhr3l72jkUB/SVC4DyHxwYSmRhOQnowhJRldTMxJDYdVTi2qIOgLX9wPPz4Hs/4O0249YldRQxHzPprHlSOv5H+n/i8A5fn1fPD4Vi789RhGnB5LxeLFNHz8CSM2b+q7MdTr11P1ryex5udjGD2axsvuYPNODUg464pMxkyPP+JeUkrWFa/msQ1/oszRhMElsXVRxQ0QOqJ0/kQaQojyiyQqIJaooCQiQ1KIDogl0j+SKL8oAvWBJ/RenE4X9RWt1JQ3YypuIm9DGWazxMdaT9zhDWRENRO38DKCLroQjeEUmSTncNC6eTNNW3fiExeDT2oyhuRktBERg67TURmaVEFwsn5cCl/8AabeCpf8/Zjdd6+/mx8P/8iqeauI8HNnB/3mTSP7Nxzmhn9Ox+Cro/hq93LNqW+83qehSaeTxlWrqFryNPayMmTOOewfsYjD5U6Sx4Zz3jWjCQw7sj3Y5rTxzp5XqWgsJSo4haigeKL83E/vUX5R+Ov7ZnyzlJLWBhs1h5qpPtRMTWkj1SX11FfbOib9COkkvHYvifVbSZ8xkoirFuKTkdEn91cU5Scq++jJ2PeJuzYwag5c/Mgxu7eZtrG2dC2/zf5tRyHgcro4sNVEymmRGHx17s5Co5GQuZcdc/7JElotIXPnEjxrFvXvvUfVc88xKvdWIs69gf3GHFb+eSNnX5nFiKmxHU+WBq2Bq8ff2Kdx2G1Oastb3F/6pY1UF9VQW2HBavvpadbHWk9g8yESWw4R2FpBWIgkPDmM4EUzCL7kN13OrVAUxftUQdCTg5vhvRshYTLM/w9ojuy0k1LyeO7jRPlFce2Yazu2H8qrx9xkJ+t096xCR3k5ruZmfEaM9FqowmAg7Be/IOTyy6l9/XW0/1lGoP0T8s64kzUvOziwrYoZV4/CP/jkmlqkS9JYY6amrIXqg41UFVRRU95CUzNAe94XK4Et5YQ3lxPYUk5ooJ2IhCCCJqTgk5mFT9ZFGNLSTplmH0UZ7FRB0J3aQnhzIQTFwi9WguHY5pJ1pevYUbWDh894+IjmlPzcSvS+WlLGumsIFmMeAD4jvb+6ksbPj8ibbiJs4UJqXnwJ/1f/RmnUmRTyM97Mr+Pcq0eTOTnao2tZWuzuJ/yDTVTnm6gubaS+3oXD1dZBKV34masIbCknsrmcEB8z4fH+hI+MwzcrE5+sMzCkp6snfUUZ5FRB0JXWWnj9CpAuuPo9CDw2V43dZefJrU+SHpLOzzJ/1rHd6XBRuK2K9AlR6AzuGoQ1zwiAT1b/LbOnDQ4m+p67Cb/masKe/zcRnzzK3hFX8+V/nBRsDGXGdafhG6jviLm+spXqsiaqjJVUF9VSW+3AbP/pz0Nvbyag+RCxLYcI1rUSEWUgIj2SgBHp+Iy4CJ/0dDQBAd2FoyjKIKYKgqPZLfDmL6ChDH75MURmdnnYB/kfUNxYzNMzn0an+eljPLi3Fmurg8ycn566LUYj+qQktIH9/0Wpi4oi9sEHCL/hemKXPMvOHTsodF3Cob3riUs0UGuy0NiqQ9K2YpTLQUBrBSHNh0ikgfAILRGpoYSMTMU3axo+mZlog04gQ6OiKIOW1wsCIYQWyAUOSSnnHLXvFuB2wAk0AzdLKfd6O6ZuuVzwwW/g4I+w4GVIntblYa32Vp7b/hyToidxbuK5R+zL31KJj7+OpNE/5bu3GvP6pVmoJ4bERBL/8Tei8vPJf/JlttWmU94cQmDLIVKctYSFCCKTgogYlYDfiDH4ZM5FG+qlXO2Kogwq/VEjuAvYB3SVsWqFlPJ5ACHEXOAJYFY/xNS1NX+EvR/ChX92ZxTtxit7XqHGUsOSmUuOGOPtsDkp2l5NVk402rYc7i6LBVtxMcGzBu5tdeaTlcW4Zx8h02jEWVePT9Z56CIiBjosRVEGkFcLAiFEIjAbeAT43dH7pZSNnV4GAAM3qWHTf2DDEjj9Rjjzjm4PqzZXs3zPci5MuZDxUeOP2Feyuwa71Unm6TEd26wFB8Dlwmek90YMnQjfQRaPoigDx9s1gieB+4BuG5WFELfjLiQMwMxujrkZuBkgOTm576M0fg6f3wcjZsGsR6GHmZxLty/F7rRz16S7jtmXn1uJX7CBhBE/rVdrNbo7in0HuGlIURSlO15LVCKEmAOYpJRbejpOSvmslDID+B/ggW6OeUFKmSOlzInq69WmDm2Fd38FsePhipdA233ZWNRQxHv577Fg5AJSgo/MfWOzOCjeVUPmxKiOlNDgHjEk/PzQJyX1bdyKoih9xJsZq84C5gohioGVwEwhRE/5FVYCl3sxnmPVlcCKheAfCYvePnZZuqM8tfUpfLQ+/Gb8b47ZV7yzGqfddUSzELjnEPhkZfXbalmKoii95bWCQEp5v5QyUUqZClwFrJNSXtP5GCFEVqeXs4F8b8VzDHMdvLEAnFa45l0Iiunx8O2m7awtXcuvxv2qI5VEZ/m5JgLDfIhLD+nYJqXEun+/ahZSFGVQ6/d5BEKIxUCulPJj4LdCiAsAO1AH/LJfgnBYYeU1UFcE134AUT13nHaXSqKdpcVO6Z4axp935FoAjqoqnPX1Xk0toSiKcrL6pSCQUn4NfN32+0Odth/b4+ptLhd8dDuUfAfzl0Hq9OOesq50HdurtvPHM/7YZWbOwu1VuJySrKOahaz9mFpCURTlRJ16q1qs/wvsegfOfwjGLzju4Z1TSVye2XUXRkFuJcGRvkQlHzk4qj21hO8IVRAoijJ4HbcgEEIECOFe/VoIMUIIMVcIMTTXwNvyMnz7OEz6JUw/ZlpDl9pTSdw96e4jUkm0a220UWasJysn5pgFRCxGI7rYWDVDV1GUQc2TGsF/AV8hRAKwFrgBeNmbQXlF/lfw6e8g8wKY/USPcwXadU4lMSNpRpfHFG4zIV3HNgvB4EgtoSiKcjyeFARCStkKzAeellLOA8Z4NywvcFghMcedQ6iHuQKdtVxvSTYAACAASURBVKeS+F3O77pdLjA/10RYXADh8UcOPZU2G9bCQnxVR7GiKIOcRwWBEOIM4GpgVdu2oZe1dPQc+NWX4ONZ5szOqSQmRE3o8pjmOivlBfVk5UQfU1BYi4rBbh90qSUURVGO5klBcDdwP/CBlHKPECIdWO/dsLykF4uAP7/j+W5TSbQ7sNUEErJyumgWylOpJRRFGRqO+2QvpfwG+EYIEdD2uhC409uBDaSihiLezXuXBSOOTSXRWX5uJZFJgYTGHDuk1Go0IvR6DKmpXoxUURTl5HkyaugMIcRe3KmkEUJMEEI85/XIBtCSrUvw0fpwy4Rbuj2msdpMZVFjl7UBcKeWMGRmIvRDc4CVoiinDk+ahp4ELgZqAKSUO4BzvBnUQNpu2s6a0jXcMO6GLlNJtMvPrQTodv1fq9Go5g8oijIkeDShTEp58KhNTi/EMuDaU0lE+kVy3Zjrejw2P9dEbHowwZF+x+xz1NXhMJlUR7GiKEOCJwXBQSHEmYAUQhiEEPfS1kw03Kw76E4lcVv2bV2mkmhXV9FCTVkzmZO7bhZSqSUURRlKPCkI2tcVTgDKgOy218OKw+XgyS1PkhaSxrzM7pepBHdtANFDs1DHiCFVI1AUZfDzZNRQNe45BMPa+/nvU9xYzJLzlnSZSqKdlJKC3EoSskIJCPXp8hiL0Yg2IgJdZKS3wlUURekzxy0IhBDL6WItYSnlr7wS0QDwJJVEu5pDzdRVtDJ+ZvcrjlmNeWr+gKIoQ4YnM4Q/7fS7LzAPKPdOOAPjlb3uVBJPzXyq21QS7fI3mxAaQcakrpfMlE4n1vx8wn7xC2+EqiiK0uc8aRp6r/NrIcSbwBqvRdTPqs3VLN/dcyqJdlJKCrZUkjQqDL9AQ5fH2EpKkVarGjGkKMqQcSLrEWQByX0dyEBpTyVx58TjT5Y2FTfRWG0hs5tJZKBSSyiKMvR40kfQhLuPQLT9WwH8j5fj6hedU0mkhqQe9/j83Eo0OkF6dvedwBajEbRaDBkZfRipoiiK93jSNORZus4hyJNUEu2kyz1aKGVsBD7+3aeNsBrzMKSlovHpekSRoijKYNNtQSCEmNTTiVLKrX0fTv9pTyVxe/btPaaSaHf4QD0tDbZucwu1sxqN+GVn91WYiqIoXtdTjeDxHvZJYGYfx9JvpJQ8seUJj1JJtMvPNaHTa0g5rftCw9nUhP3QIUIXLuyrUBVFUbyu24JASnlefwbSn9YdXMc20zYeOuOhHlNJtHM5XRzYaiJ1fCQG3+7LTmt+PqA6ihVFGVo8WmlMCDEO9/KUvu3bpJSveisob+pNKol2h4z1mJvsHjULAWroqKIoQ4on6xH8EXi67ec84B/AXE9vIITQCiG2CSE+7WLf74QQe4UQO4UQa4UQ3a8C00faU0ncPenuHlNJdJafW4neV0vyuPAej7MYjWhCQtDF9FxgKIqiDCaezCO4AjgfqJBS3gBMAHozJOYuus9Wug3IkVKOB97FXch4Tau9laU7ljIpehLnJXnW8uV0uCjcXkV6dhQ6vbbHY63GPHxHjDju7GRFUZTBxJOCwCyldAEOIUQwYALSPbm4ECIRmA0s62q/lHK9lLK17eWPQKIn1z1Rr+x9hWpzNfdMvsfjL+uDe2uxtjq6zTTaTrpcWPPyVLOQoihDjidtI7lCiFDgP8AWoBnY5OH1nwTuAzyZi/Br4HMPr9trnVNJZEd7PrwzP7cSnwAdSaN7bhayl5fjamlRaxAoijLkeDKh7La2X58XQnwBBEspdx7vPCHEHMAkpdwihJhxnGOvAXKAc7vZfzNwM0By8ollt1ixbwU2p82jVBLt7DYnRTuqyTo9Bq2u58pTe0exWoNAUZShxpMUEx8BbwEfSSmLe3Hts4C5QohLcY82ChZCvC6lvOao618A/B9wrpTS2tWFpJQvAC8A5OTkHJMS2xO3TriVaXHTPEol0a5kVw12q5OsnJ6bhaAttYQQ+GRmnkh4iqIoA8aTPoIngOnAXiHEO0KIK4QQvsc7SUp5v5QyUUqZClwFrOuiEJgI/BuYK6U09T58z+m1eqbETenVOQW5lfgFG4gfEXbcY63GPAzJyWj8jz8vQVEUZTA5bkEgpfymrXkoHfdT+ZW4O4xPiBBisRCiffjpP4FA4B0hxHYhxMcnet2+ZrM4KN5dQ+akaDSa43csW41G1VGsKMqQ5OmEMj/gMmAhMAl4pTc3kVJ+DXzd9vtDnbZf0Jvr9KeiHdU47S6PmoVcZjO2khKCL5vTD5EpiqL0LU/6CN4CpgJfAM8CX7cNJx3WCnIrCQzzITY95LjHWgsKQErVUawoypDkSY1gObBISun0djCDhaXFTuneWsbPTEJ42CwEKrWEoihDkyfDR7/oj0AGk8LtVbic0qNmIQCLMQ+Nvz/6hAQvR6YoitL3TmSpymGvILeS4Cg/opI9W5PHun8/PiNGIDTq41QUZehR31xHaW20Uba/jqycaI/SUEgpsajUEoqiDGGeZB99TwgxWwhxShQaB7aakJLjppxu56isxNXQoFJLKIoyZHny5b4UWATkCyH+LoQY5eWYBlR+biXh8QFEJAR6dLxKLaEoylDnyYSyNVLKq3HPHygGvhJCbBBC3CCE6H4V9yGouc7C4YIGjzuJwd1RDOAzQtUIFEUZmjxq7hFCRADXAzfiXkPgKdwFw1dei2wAFGxxT5jOnOz5wjJWoxF9fDzaIM86lhVFUQYbTyaUvQ+MAl4DLpNSHm7b9ZYQItebwfW3/FwTUclBhMZ4ni/ImqdSSyiKMrR5MqHsGSnluq52SClz+jieAdNQZcZU3MgZ8zM8Psdls2EtLCLw/PO9GJmiKIp3eVIQbBBC/A53BlIJfAcslVJavBpZPyvYUglw3JXIOrMdOABOp+ooVhRlSPOkIHgVaMK9eD3AL3A3Ey3wVlADIX+zidj0EIIj/Dw+x6JSSyiKMgx4UhCMlFJO6PR6vRBih7cCGgi1h1uoOdTM9CuzenWe1ZiH8PHBcIKrpimKogwGnowa2iaEmNb+QggxFfjeeyH1v4LcShC9axaCtjUIMjMROo+yeSuKogxKnhQEU3H3ExQLIYqBH4BzhRC7hBDHXbt4sJNSkp9rImFEKAEhPr06V6WWUBRlOPDkUXaW16MYQNVlzdRXtpJ9QVKvznNUV+OsrsZXpZZQFGWI8yQNdYkQYgJwdtumb6WUw6aPoCC3Eo1GkD4xqlfnWfPaZhSrGoGiKEOcJ0nn7gLeAKLbfl4XQtzh7cD6Q3uzUOLocPwCDb06V6WWUBRluPCkaejXwFQpZQuAEOJR3P0ET/d41hBQWdxIU42FKXPSen2u1WhEFxWFLjzcC5EpiqL0H086iwXQeZlKZ9u2Ia9gswmNTpCW3btmIQCLSi2hKMow4emaxRuFEB+0vb4ceNF7IfUPl0uSv6WSlLER+Pj1bvindDiw5RcQcN0ZXopOURSl/3jSWfyEEOJr3CkmBHCDlHKbtwPztsMF9bQ22Mg63fNMo+1sxcVIu12lllAUZVjosSBoW5Vsp5RyHLC1f0LqHwW5JnQGDamnRfb6XJVaQlGU4aTHPgIppQvYIYQ44RwKQgitEGKbEOLTLvadI4TYKoRwCCGuONF79JbL6aJgq4nU8ZHofbS9Pt9qzAOdDp+03ncyK4qiDDaeNI7HAXuEEJuAlvaNUsq5Ht7jLmAfENzFvlLcC97c6+G1+kSZsQ5Ls93jdYmPZjUa8UlPRxh6N+RUURRlMPKkIPjTiV5cCJEIzAYeAX539H4pZXHbca4TvceJyM81YfDVkjz2xIZ+WvLy8M8ZNksxKIpyivOkILhUSvk/nTe0zSX4xoNznwTuA05qHUchxM3AzQDJJ5np02l3UbitivTsKHT63jcLORsacBw+rFJLKIoybHgyj+DCLrZdcryThBBzAJOUckuvozqKlPIFKWWOlDInKqr3Y/47K91Xi83sIPMERguBSi2hKMrw022NQAhxK3AbkH5UltEgYIMH1z4LmCuEuBTwBYKFEK9LKa85mYBPVv7mSnwD9CSOCjuh839KLaEKAkVRhoeemoZWAJ8DfwP+0Gl7k5Sy9ngXllLeD9wPIISYAdw70IWA3eakaGc1I6bEoNV6Uhk6ltVoRBsaii765GomiqIog0W334ZSygYpZbGU8hdAGWDHvWZx4EkOJ10shJjb9vvpQogy3Mte/lsIsedEr+uJkl01OKzOEx4tBD+llhBiWGTZUBRFOX5nsRDit8DDQCXQPrpHAuM9vYmU8mvg67bfH+q0fTOQ6Ol1TlZ+biX+wQbis0JP6HzpcmHNyyd0Qb9NeVAURfE6T0YN3Y173eIabwfjTTazg5LdNYydHo9Gc2JP8/aDB5Fms0otoSjKsOJJQ/lBoMHbgXhb0c5qnHYXmSfTLNSeWkJ1FCuKMox4UiMoBL4WQqwCrO0bpZRPeC0qLzD4akmbEElsWlcTnD1jNeaBRoNPZkYfRqYoijKwPCkIStt+DG0/Q1LahCjSJpzcSB9rnhFDSgoaP78+ikpRFGXgeZKG+k8AQoiA9lXKTlUWYx6+Y8YMdBiKoih9ypM1i88QQuzFnTgOIcQEIcRzXo9skHG1tGAvLVWpJRRFGXY86Sx+ErgYqAGQUu4AzvFmUIORNT8fUKklFEUZfjyaXiulPHjUJmeXBw5jKrWEoijDlSedxQeFEGcCUghhAO6krZnoVGI1GtEEBKBPiB/oUBRFUfqUJzWCW4DbgQTcqSay216fUlRqCUVRhitPRg1VA1f3QyyDlpQSqzGP4DmzBzoURVGUPndiKThPMY7Dh3E1NanUEoqiDEuqIPCASi2hKMpwpgoCD1g7RgxlDXAkiqIofa+nFcqOWWy+s6GWa+hkWPOM6BMT0QYGDnQoiqIofa6nzuL2BedHAqcDH7e9vgz4rzeDGmwsxjw1kUxRlGGr24KgU46h1cAkKWVT2+uHgXf6JbpBwGW1YisqIvjiiwY6FEVRFK/wpI8gGbB1em0DUr0SzSBkLSgAl0t1FCuKMmx5MrP4NWCTEOID3EtUzgNe9WpUg0hHR7FKNqcoyjDlyYSyR4QQnwNnt226QUq5zbthDR5WoxHh64shOXmgQ1EURfEKT4eP+gONUsqngDIhRJoXYxpULHlGfLKyEFrtQIeiKIriFZ6sR/BH4H+A+9s26YHXvRnUYCGlxLrfqJqFFEUZ1jypEcwD5gItAFLKcn4aWjqsOaurcdbV4as6ihVFGcY8KQhsUkqJu6MYIURAb24ghNAKIbYJIT7tYp+PEOItIUSBEGKjECK1N9f2to41CNQcAkVRhjFPRg29LYT4NxAqhLgJ+BWwrBf3uAv3+gXBXez7NVAnpcwUQlwFPAos7MW1vcrakWNIpZZQBp7dbqesrAyLxTLQoSiDmK+vL4mJiej1eo/P8WTU0GNCiAuBRtyzjB+SUn7lycWFEInAbOARoKuUFT8DHm77/V3gGSGEaKuBDDhrnhFdTAy6sLCBDkVRKCsrIygoiNTUVLUuhtIlKSU1NTWUlZWRlub5mB5POosflVJ+JaX8vZTyXinlV0KIRz28/pPAfYCrm/0JwEEAKaUDaAAiuojhZiFErhAit6qqysNbnzx3agnVUawMDhaLhYiICFUIKN0SQhAREdHrWqMnfQQXdrHtEg8CmgOYpJRbejqsi23H1AaklC9IKXOklDlRUVHHu3WfkHY71gMH1BoEyqCiCgHleE7kb6Sn7KO3ArcB6UKInZ12BQHfe3Dts4C5QohLAV8gWAjxupTymk7HlAFJuOcm6IAQoLaX78ErrEVFYLer1BKKogx7PdUIVuDONPpx27/tP5OP+jLvkpTyfillopQyFbgKWNfFeR8Dv2z7/Yq2YwZH/4BKLaEoR6ipqSE7O5vs7GxiY2NJSEggOzub0NBQxowZ06+xbN++nc8++6zj9ccff8zf//73E7pWamoq1dXVJx1TX11nIHRbEEgpG6SUxVLKX0gpSwAz7mabQCHECedbEEIsFkLMbXv5IhAhhCjA3Zn8hxO9bl+z5hlBr8enFx0uijKcRUREsH37drZv384tt9zCPffc0/Fao+n7Na4cDke3+44uCObOncsf/jBovj6GnOOOGhJCXAY8AcQDJiAF93DQsZ7eREr5NfB12+8PddpuARb0JuD+YjEa8cnIQPRiCJai9Jc/fbKHveWNfXrNMfHB/PEyj/+3PoLT6eSmm25iw4YNJCQk8NFHH+Hn58eBAwe4/fbbqaqqwt/fn//85z+MGjWKkpISfvWrX1FVVUVUVBTLly8nOTmZ66+/nvDwcLZt28akSZNYvHgxd9xxB7t27cLhcPDwww9zySWX8NBDD2E2m/nuu++4//77MZvN5Obm8swzz1BZWcktt9xCYWEhAEuXLuXMM8/k8ssv5+DBg1gsFu666y5uvvnmbt/P0qVLKSoq4h//+AcAL7/8Mlu2bOHpp58+7nWKi4uZM2cOu3fvBuCxxx6jubmZhx9+uNvPY6B5Uoz/BZgG5Ekp04Dz8ayPYEizGvPwVc1CiuKR/Px8br/9dvbs2UNoaCjvvfceADfffDNPP/00W7Zs4bHHHuO2224D4Le//S3XXXcdO3fu5Oqrr+bOO+/suFZeXh5r1qzh8ccf55FHHmHmzJls3ryZ9evX8/vf/x673c7ixYtZuHAh27dvZ+HCI6ce3XnnnZx77rns2LGDrVu3Mnasu3B76aWX2LJlC7m5uSxZsoSamppu388VV1zB+++/3/H6rbfe6rhPb65ztO4+j4HmyYQyu5SyRgihEUJopJTrezF8dEhy1NXhqKxUHcXKoHWiT+7ekpaWRnZ2NgCTJ0+muLiY5uZmNmzYwIIFP1X6rVYrAD/88EPHF+21117Lfffd13HMggUL0LYleVy9ejUff/wxjz32GOAeQltaWtpjLOvWrePVV92Z8rVaLSEhIQAsWbKEDz74AICDBw+Sn59PRMQxo9UBiIqKIj09nR9//JGsrCyMRiNnnXVWr6/TWU+fx0DzpCCoF0IE4l6e8g0hhAnovvFuGLDm5QMqtYSieMrHx6fjd61Wi9lsxuVyERoayvbt2497fuchjwEBP2WxkVLy3nvvMfKo/xc3btzYq/i+/vpr1qxZww8//IC/vz8zZsw47lj7hQsX8vbbbzNq1CjmzZuHEMKj6+h0Olyun6ZOte/vzefR3zxpGvoZ7o7ie4AvgAO4Rw8NW+2pJVTTkKKcuODgYNLS0njnHffKtlJKduzYAcCZZ57JypUrAXjjjTeYPn16l9e4+OKLefrpp2kfTLhtm3splKCgIJqamro85/zzz2fp0qWAu++isbGRhoYGwsLC8Pf3Z//+/fz444/HjX/+/Pl8+OGHvPnmmx3NQp5cJyYmBpPJRE1NDVarlU8//fS4n8dAO25BIKVskVI6ca9J8AnuFNSDYoint1iM+9GGh6ONjBzoUBRlSHvjjTd48cUXmTBhAmPHjuWjjz4C3M0ry5cvZ/z48bz22ms89dRTXZ7/4IMPYrfbGT9+POPGjePBBx8E4LzzzmPv3r1kZ2fz1ltvHXHOU089xfr16znttNOYPHkye/bsYdasWTgcDsaPH8+DDz7ItGnTjht7WFgYY8aMoaSkhClTpgB4dB29Xs9DDz3E1KlTmTNnzhGdwd19HgNNHG/YvhDiN8Bi3LUCF+7ZwFJKme798I6Vk5Mjc3NzvXqPoisWoAkKJGX5cq/eR1F6Y9++fYwePXqgw1CGgK7+VoQQW6SUOV0d70kfwb3AWCnl0Jwp0UvS6cSan0/YVVcNdCiKoij9wpM+ggNAq7cDGSxsJaVIqxWfQTC2V1EUpT94UiO4H9gghNgIdIx1klLe2f0pQ5c1T3UUK4pyavGkIPg3sA7YRffppIcNi9EIWi2GjIyBDkVRFKVfeFIQOKSUXS0qMyxZjXkY0lLRdBoXrSiKMpx50kewvm1hmDghRHj7j9cjGyBWo1EtVq8oyinFk4JgEW39BMCWth/vjt8cIM6mJuyHDqkZxYrShe7SUGdnZ2Oz2QY6vA4Oh4PQ0NBenXPNNdfw4YcfHrN948aN3HPPPQAsW7aMu+++G4Bnn32WN954A3DnHqqoqOg454YbbsDYNil1qPBkzeJTJg+zNb89tYTqKFaUo7WnoQZ4+OGHCQwM5N577z3iGCklUkqvpKUeiPtMnTqVqVOnHrP99ttv7/j9pZdeYtKkScTGxgKwfAjOP+pphbKZUsp1Qoj5Xe2XUr7f1fah7KfUEqpGoAxyn/8BKnb17TVjT4NLer+4S0FBAZdffjnTp09n48aNfPjhh0yYMIH6+noAVq5cyZo1a1i2bBmVlZXceuutlJaWotFoWLJkyTGzc5ctW8aqVatobW2luLiYa6+9lgceeOCY+3z66aesX7+eRx99FCklc+fO5a9//WvHde655x6++eYbIiIiWLlyJRERETz//PO8+OKL2Gw2RowYwauvvoqfnx8AX375JY8//jiVlZU89dRTXHLJJaxZs4ZnnnnmmNrCAw88QGRkJHFxcR0ZUP38/Ni0aRMzZ87kmWeeITs7m88//5zFixdjtVrJysripZdeIiAggN///vesWrUKnU7HJZdcwqOPDmwez56K03Pb/r2si585Xo5rQFiMRjTBwejaSnZFUTyzd+9efv3rX7Nt2zYSEhK6Pe7OO+/kvvvuIzc3l7fffpsbb7yxy+M2bdrEypUr2bp1KytWrOioiXS+j5SSBx54gPXr17Nt2za+//77jrw+DQ0NTJs2ja1bt3LGGWfw5z//GXBnNt28eTM7duwgIyODl19+ueOeBw8e5JtvvuGTTz7h5ptv9igz6MKFCzvSXGzfvh2DwdCxz2Qy8fe//521a9eydetWxo8fz1NPPUVlZSWfffYZe/bsYefOndx///3HvY+3dVsjkFL+se3XxVLKos77hBDDsrnIaszDd8QItUC4MvidwJO7N2VkZHD66acf97g1a9Yc0X5eV1eH2WzueCpvd/HFFxMWFgbA5ZdfznfffcesWbOOuM/GjRuZOXMmkW05wRYtWsR///tfZs2ahU6n60j3fM0117Bo0SIAdu7cyUMPPUR9fT1NTU3MmfPTM+2VV16JRqNh5MiRJCUlkd/WVHyiNmzYwN69eznzzDMBsNlsTJ8+nfDwcDQaDTfddBOzZ88+IoaB4snw0feASUdtexeY3PfhDBzpcmHNyyPk8ssHOhRFGXI6p47WaDR0zmHWOU2zlJJNmzYd8eTclaMfxtpfH52iurfnX3fddXz++eeMGzeOZcuWHZE9tLtzTpSUklmzZvHaa68dsy83N5evvvqKlStXsnTpUlavXn1S9zpZ3TYNCSFGCSF+DoQIIeZ3+rke8O23CPuJvbwcV0uL6ihWlJOk0WgICwsjPz8fl8vVsYgLwAUXXMCzzz7b8bq73PyrV6+mvr6e1tZWPvroo45FYTqbNm0a69evp6amBofDwcqVKzn3XHeLtt1u71j4ZsWKFR1prltaWoiNjcVut7NixYojrvfOO+8gpSQvL4+DBw+SlZXl0fvtLiX2mWeeyTfffNOxZGZLSwv5+fk0NTXR2NjInDlz+Ne//tWRWnsg9VQjGIm7LyCUI9cfaAJu8mZQA0F1FCtK33n00UeZNWsWycnJjBkzpqO9/dlnn+XWW29l+fLlOBwOzjvvvCMKhnbTp09n0aJFHDhwgGuvvZbs7GwKCgqOOCYxMZHFixczY8YMpJRcdtllzJ49G4fDQUhICFu3buWvf/0r4eHhHamqFy9ezJQpU0hOTmbcuHFH1FYyMzM555xzMJlMvPDCC8ettbS74YYbuPHGGzs6i9vFxMTw4osvsnDhwo7htX/961/x8/Nj/vz5WK1WXC4XTzzxRO8+XC/wJA31GVLKH/opnuPyVhrqqueeo/rpZxiZuxlNp+qnogwWp0oa6mXLlrF7926efPLJgQ5lyOptGmpPBuHOE0IECyH0Qoi1QohqIcQ1fRHsYGI15qFPTlKFgKIopxxPOosvklLeJ4SYB5QBC4D1uFcqGzZUaglFGRy6G1KqeI8nNQJ927+XAm9KKWu9GM+AcJnN2EpKVGoJRVFOSZ4UBJ8IIfYDOcBaIUQUYDnOOQghfIUQm4QQO4QQe4QQf+rimJS25qadQoivhRCJvX8LJ89aUABSqhFDiqKckjxZvP4PwBlAjpTSjnu1sp95cG0rMFNKOQHIBmYJIY5e6fkx4FUp5Xjc6yL/rTfB9xU1YkhRlFNZT/MI7uv08gIppRNAStkCHHd1MunW3PZS3/Zz9BClMcDatt/X41kB0+csxjyEvz/6xAGpkCiKogyonmoEnVdvPzoZxixPLi6E0AohtsP/b+/c46qq8v7//ooIkgoi1mimIlneQLTS1PRBrdTGez5laZI2mRPeelLLJg2dyZ+N5jTQU/x0vOU1lURHrdAEjTRBEpBMBJVRs5LRgQQVFdbzx97neMDDHUI46/16ndfZe+211/6uddbZ370u+7O4AOxWSh0qFCUReNrcHgE0FJEmdtKZKCKHReRwRkZGaS5dJnJTUnBt2xapYiVDjaam8/PPPzN69Gh8fHzo0KEDTz31FCdOnCh3eunp6XTq1KkSLSyZgIAAyjIFfdWqVUyePNnuMYt8hG0+Dh8+zNSpxrNydHQ0Bw4csMYPCwvjk08+Ka/pVUZxs4akiG17+3YxWxH+IuIBbBWRTkqpZJsoM4APzbeV9wM/AjftpLMUWArGewSluXZpUUqRm5JCwwEDKjNZjabWoZRixIgRBAYGsnHjRsB4M/iXX37hgQduja/l5eXh5ORUXWb+pjbY3uQtPPzwwzz8sDFdPzo6mgYNGlgdxqRJk6rcpvJQnCNQRWzb2y8WpVSmiERjtCSSbcLPAyMBRKQB8LRSKqssaVeUmxcukJeVpQeKNTWK92LfvSsodAAAHY1JREFU4/il45WaZjvPdrzR7Y0ij0dFReHs7FzgZubv7w8YN7x58+ZZZZl37drF4MGDSU42/u6LFy8mOzub4OBg4uPjmTBhAm5ublbpBzBu3m+++SbR0dHk5uYSFBTEK6+8UsCG9PR0Bg4cSPfu3Tly5IhVStrNzY3WrVszYcIEIiMjmTx5Mu3atWPSpElcuXIFHx8fVqxYYRWyW7t2LVOnTuXXX39lxYoVdOvWjdjYWKZPn24VwVu5ciUPmuOGZ8+eZeDAgZw+fZrnn3+ed94xNDkbNGhAdnZ2ARujo6NZvHgxH374IWFhYTg5ObF27VpCQ0P56quvrOs4nDx5kqCgIDIyMnBzc2PZsmW0a9eOzZs3M2/ePJycnHB3d2f//v3l/UlLTXF9IZ1F5FcRuQz4mduWfd+SEhaRpmZLABGpDzwOHC8Ux0tELDbMBlaUKxcVQA8UazSlIzk5mYceKlprMjY2lnfffZdjx44Vm8748eMJCQnh4MGCggXLly/H3d2duLg44uLiWLZsGadPn77t/JSUFCZOnEhSUhKNGjXio48+sh5zdXUlJiaG0aNHM27cON577z2SkpLw9fVl3rxbExdzcnI4cOAAH330ERMmTACgXbt27N+/nyNHjjB//nzeeuutAnlbt24dCQkJbN68uVRdS61bt2bSpEm89tprJCQk0Lt37wLHJ06cSGhoKPHx8SxevJhXX30VMGQwvvzySxITE9m+fXuJ16kMipOhrmi7qhmwWkScMBzOJqXUDhGZDxxWSm0HAoD/JyIKo2soqMjUqohrpiNweUC3CDQ1h+Ke3KuLbt264e1dvEJ9VlYWmZmZVnG4F154gc8//xwwhOaSkpLYsmWLNW5qauptad53331WEbqxY8cSEhJiXSnt2WeftXudwMBAqyw1wHPPPQdAnz59+PXXX62y1IGBgaSmpiIi3Lhxwxr/iSeeoEkTY/hy5MiRxMTEWLt/ykN2djYHDhwoYJNFj6lXr168+OKLPPPMM4wcaXddsEqnNG8WlwulVBLQxU74XJvtLRiS1tVGbsoJ6jZvhlOjRtVphkZzx9OxY0frTdoethLRdevWJT8/37pvEXdTShUp76yUIjQ0lAEljNcVJxd9VyklYuylMWfOHPr27cvWrVtJT08nICCgVNcsD/n5+Xh4eNhVXw0LC+PQoUPs3LkTf39/EhISrE6oqnD4aTJaWkKjKR39+vUjNzeXZcuWWcPi4uLYt2/fbXHvueceLly4wMWLF8nNzbWuHObh4YG7uzsxMTEA1gXgwViM5uOPP7Y+iZ84cYKcnJzb0j5z5oy1W2nDhg0FxhksuLu707hxY77++msA1qxZY20dAFY10piYGNzd3XF3dycrK8u6uprtymUAu3fv5tKlS1y9epWIiAi7stj2KEqiulGjRnh7e7N582bAcIKJiYkAnDx5ku7duzN//ny8vLw4e/Zsqa5VERzaEeRfv07u6dNaWkKjKQUiwtatW9m9ezc+Pj507NiR4OBgmjdvfltcZ2dn5s6dS/fu3Rk8eDDt2rWzHlu5ciVBQUH06NGjwMpkf/jDH+jQoQNdu3alU6dOvPLKK9y8edskQtq3b8/q1avx8/Pj0qVL/PGPf7Rr7+rVq5k5cyZ+fn4kJCQwd661M4LGjRvTs2dPJk2axPLlywGYNWsWs2fPplevXuTl5RVI67HHHrPKYT/99NOl7hYaMmQIW7duxd/f3+qULKxbt47ly5fTuXNnOnbsyLZt2wCYOXMmvr6+dOrUiT59+tC5c+dSXasilChDfadRmTLU144f5/TwEdy75H0aPfVUpaSp0VQVjiJDXRzp6ekFZiNp7FMVMtS1FsuMId0i0Gg0joxDO4JrKSeQevWo16pVdZui0WhKQevWrXVroApwaEeQm5KCy/33I3WrbPKURqPR3PE4tCO4diJFdwtpNBqHx2Edwc2LF8nL+LeWltBoNA6PwzqCXFMxUUtLaDQaR8dhHcE1PWNIoykzDRo0sG7v2rWLtm3bcubMGYKDg7n33nvx9/enbdu2jBw5soDmUEBAAA8++CD+/v74+/szatSoCtmRmZlZQGPo/PnzFU7TQkRERIl6SbUNh3UEuSkncGrqRV1Pz+o2RaOpcXz11VdMmTKFL774gpYtWwJYxdVSU1N59tln6devH7brh1hE2xISEoqVqigNhR1B8+bNK5ymhfI4AnsvvtUkHHa6jJaW0NRkfl6wgNwfKleG2qV9O35no7hZFF9//TUvv/wyu3btwsfHx26cZ599lp07d7J+/XqmTZtWquvn5OQwZcoUjh49ys2bNwkODmbYsGF8//33jB8/nuvXr5Ofn094eDhz5szh5MmT+Pv788QTTxAUFGR90WzVqlVERESQl5dHcnIyr7/+OtevX2fNmjW4uLiwa9cuPD09WbZsGUuXLuX69evcf//9rFmzhoSEBLZv386+ffv4y1/+Qnh4OJcvX7YrZx0QEEDPnj355ptvGDp0KK+//nqZyvtOwiFbBOrmTXLT0nS3kEZTRnJzcxk2bBgREREFZCPs0bVrV44fv+WsxowZY+0amjlz5m3x3333Xfr160dcXBxRUVHMnDmTnJwcwsLCmDZtGgkJCRw+fJgWLVqwcOFCfHx8SEhIYNGiRbellZyczPr164mNjeVPf/oTbm5uHDlyhB49elhXCBs5ciRxcXEkJibSvn17li9fTs+ePRk6dCiLFi0iISEBHx+fYuWsMzMz2bdvX412AuCgLYLr//oX6vp1XPWMIU0NpTRP7lWBs7MzPXv2ZPny5fz9738vNm5h+Zp169YVq9ETGRnJ9u3bWbx4MWAolp45c4YePXrw7rvvcu7cOUaOHEnbtm1LtLNv3740bNiQhg0b4u7uzpAhQwDw9fUlKSkJMJzF22+/TWZmJtnZ2XZVT0uSs7bIXtd0HLJFoKUlNJryUadOHTZt2kRcXBwLFiwoNu6RI0fKpI2klCI8PNw6jnDmzBnat2/P888/z/bt26lfvz4DBgxg7969Jabl4uJSwGbLfp06daz9+S+++CIffvghR48e5Z133rFKZZeF0spe3+k4pCO4lnIC6talXps21W2KRlPjcHNzY8eOHVb1THuEh4cTGRlpXQCmNAwYMIDQ0FBrS+LIkSMAnDp1ijZt2jB16lSGDh1KUlJSkfLOZeHy5cs0a9aMGzduFJDDtk27JDnr2oJDdg3lpqTg4u1NnXr1qtsUjaZG4unpyRdffEGfPn3w8vIC4G9/+xtr164lJyeHTp06sXfvXpo2bWo9Z8yYMVbZaS8vL/bs2VMgzTlz5jB9+nT8/PxQStG6dWt27NjBp59+ytq1a3F2duZ3v/sdc+fOxdPTk169etGpUycGDRpEUFDZFzf885//TPfu3WnVqhW+vr7Wm//o0aN5+eWXCQkJYcuWLaxevdo6WNymTRtWrlxZ3mK7Y3FIGerUfv1w6/oQ9y6+fZBJo7lT0TLUmtKiZahLIO/XX7l5/ictLaHRaDQmDucItLSERqPRFMThHIGWltBoNJqCOJwjyE05gZO7O3Xvvru6TdFoNJo7Agd0BMYaBCJS3aZoNBrNHUGVOQIRcRWRWBFJFJHvRWSenTgtRSRKRI6ISJKIVOkK8io/n2upqbpbSKPRaGyoyhZBLtBPKdUZ8AcGisijheK8DWxSSnUBRgMfUYXcOHcOdeWKlpbQaMqJrQw1wKpVq5g8eTIAYWFhVh0fe0RHR3PgwIEqte+34oMPPuDKlSvW/aeeeorMzMwKp5uQkMCuXbsqnE5ZqTJHoAyyzV1n81P4pQUFNDK33YHzVWUP6IFijaYqmTRpEuPGjSvyeGU6gry8vEpJp7wUdgS7du3Cw8OjwumWxxFUhgR2lb5ZLCJOQDxwP/C/SqlDhaIEA5EiMgW4C3i8iHQmAhMBq/Z5echNOQEiuNx/f7nT0GjuBL7edIJ/n80uOWIZ8LqvAb2fKX9rOTg4mAYNGjBjxgxCQkIICwujbt26dOjQgYULFxIWFoaTkxNr164lNDSUli1bMmHCBDIyMmjatCkrV66kZcuWnDx5kjFjxpCXl8egQYNYsmQJ2dnZREdHM2/ePJo1a0ZCQgLHjh1j+PDhnD17lmvXrjFt2jQmTpwIGC2XoKAg9uzZQ+PGjVmwYAGzZs3izJkzfPDBBwwdOvQ2+xctWsSmTZvIzc1lxIgRzJs3j5ycHJ555hnOnTtHXl4ec+bM4ZdffuH8+fP07dsXLy8voqKiaN26NYcPHyY7O5uBAwfy2GOP8e2339K5c2fGjx/PO++8w4ULF1i3bh3dunUjNjaW6dOnc/XqVerXr8/KlSvx9vZm7ty5XL16lZiYGGbPns0TTzzBhAkTOHXqFG5ubixduhQ/Pz+Cg4M5f/486enpeHl5sX79+nL/blDFjkAplQf4i4gHsFVEOimlkm2iPAesUkq9LyI9gDVmnPxC6SwFloLxZnF57clNSaFeq1bUMV9z12g0ZePq1av4+/tb9y9dumT3prpw4UJOnz6Ni4sLmZmZeHh4MGnSJKujABgyZAjjxo0jMDCQFStWMHXqVCIiIpg2bRrTpk3jueeeIywsrEC6sbGxJCcn4+3tDcCKFSvw9PTk6tWrPPLIIzz99NM0adKEnJwcAgICeO+99xgxYgRvv/02u3fv5tixYwQGBt5mc2RkJKmpqcTGxqKUYujQoezfv5+MjAyaN2/Ozp07AUON1N3dnSVLlhAVFWWV17AlLS2NzZs3s3TpUh555BHWr19PTEwM27dvZ8GCBVYJ7/3791O3bl327NnDW2+9RXh4OPPnz+fw4cN8+OGHAEyZMoUuXboQERHB3r17GTduHAkJCQDEx8cTExNjle2oCL+J1pBSKlNEooGBgK0jeMkMQyl1UERcAS/gQlXYce1ECq7t9Cv6mppPRZ7cK0L9+vWtNyIwxgjsSb74+fkxZswYhg8fzvDhw+2mdfDgQT777DMAXnjhBWbNmmUNj4iIAOD555+3Og6Abt26WZ0AQEhICFu3bgXg7NmzpKam0qRJE+rVq8fAgQMBQ3raxcUFZ2dnfH19SU9Pv82WyMhIIiMj6dKlCwDZ2dmkpqbSu3dvZsyYwRtvvMHgwYPp3bt3iWXk7e2Nr68vAB07dqR///6ISIFrZ2VlERgYSGpqKiLCjRs37KYVExNDeHg4AP369ePixYtkZWUBMHTo0EpxAlC1s4aami0BRKQ+RrdP4SWVzgD9zTjtAVcggyogPyeHG2fOamkJjeY3YOfOnQQFBREfH89DDz1Uqn7s0kzptpV9jo6OZs+ePRw8eJDExES6dOlilZJ2dna2pleUDLUtSilmz55tlcBOS0vjpZde4oEHHiA+Ph5fX19mz57N/PnzS7SxNBLYc+bMoW/fviQnJ/PPf/6zSAlse1pwlnxVpgR2Vc4aagZEiUgSEAfsVkrtEJH5ImJpl70OvCwiicAG4EVVRSp4uWlpoJSWltBoqpj8/HzOnj1L3759+etf/2pd+KWwdHTPnj3ZuHEjYCxa89hjjwHw6KOPWp+CLcftkZWVRePGjXFzc+P48eN8++235bZ5wIABrFixguxsY9zlxx9/5MKFC5w/fx43NzfGjh3LjBkz+O677wAqLIOdlZXFvffeCxitKguF0+3Tp49VIjs6OhovLy8aNWpEZVNlXUNKqSSgi53wuTbbx4BeVWWDLXrGkEbz25CXl8fYsWPJyspCKcVrr72Gh4cHQ4YMYdSoUWzbto3Q0FBCQkKYMGECixYtsg4WgzEjZ+zYsbz//vv8/ve/x93d3e51Bg4cSFhYGH5+fjz44IM8+mjh2eml58knn+SHH36gR48egDHYvHbtWtLS0pg5cyZ16tTB2dmZjz/+GICJEycyaNAgmjVrRlRUVJmvN2vWLAIDA1myZAn9+vWzhvft25eFCxfi7+/P7NmzCQ4OZvz48fj5+eHm5sbq1avLncficBgZ6stffUXmZ1tpERqC1HG4F6o1tQBHkaG+cuUK9evXR0TYuHEjGzZsYNu2bdVtVo2irDLUDrMwTcP+/WnYv391m6HRaEogPj6eyZMno5TCw8ODFStWVLdJtR6HcQQajaZm0Lt3bxITE6vbDIdC95FoNDWImtaVq/ntKU8d0Y5Ao6khuLq6cvHiRe0MNEWilOLixYu4urqW6TzdNaTR1BBatGjBuXPnyMiokldtNLUEV1dXWrRoUaZztCPQaGoIzs7OBd6q1WgqC901pNFoNA6OdgQajUbj4GhHoNFoNA5OjXuzWEQygH+V41Qv4N+VbE5NRZdFQXR5FESXxy1qU1m0Uko1tXegxjmC8iIih4t6vdrR0GVREF0eBdHlcQtHKQvdNaTRaDQOjnYEGo1G4+A4kiNYWt0G3EHosiiILo+C6PK4hUOUhcOMEWg0Go3GPo7UItBoNBqNHbQj0Gg0Ggen1jsCERkoIikikiYib1a3PVWBiNwnIlEi8oOIfC8i08xwTxHZLSKp5ndjM1xEJMQskyQR6WqTVqAZP1VEAqsrT5WBiDiJyBER2WHue4vIITNvn4pIPTPcxdxPM4+3tkljthmeIiIDqicnFUdEPERki4gcN+tJD0etHyLymvk/SRaRDSLi6sh1AzBkS2vrB3ACTgJtgHpAItChuu2qgnw2A7qa2w2BE0AH4K/Am2b4m8B75vZTwOeAAI8Ch8xwT+CU+d3Y3G5c3fmrQLn8D7Ae2GHubwJGm9thwB/N7VeBMHN7NPCpud3BrDMugLdZl5yqO1/lLIvVwB/M7XqAhyPWD+Be4DRQ36ZOvOjIdUMpVetbBN2ANKXUKaXUdWAjMKyabap0lFI/KaW+M7cvAz9gVPhhGDcAzO/h5vYw4BNl8C3gISLNgAHAbqXUJaXUf4DdwMDfMCuVhoi0AH4P/MPcF6AfsMWMUrg8LOW0Behvxh8GbFRK5SqlTgNpGHWqRiEijYA+wHIApdR1pVQmjls/6gL1RaQu4Ab8hIPWDQu13RHcC5y12T9nhtVazKZrF+AQcI9S6icwnAVwtxmtqHKpTeX1ATALyDf3mwCZSqmb5r5t3qz5No9nmfFrS3m0ATKAlWZX2T9E5C4csH4opX4EFgNnMBxAFhCP49YNoPY7ArETVmvny4pIAyAcmK6U+rW4qHbCVDHhNQoRGQxcUErF2wbbiapKOFYrygPjCbgr8LFSqguQg9EVVBS1tjzMcZBhGN05zYG7gEF2ojpK3QBqvyM4B9xns98COF9NtlQpIuKM4QTWKaU+M4N/MZv0mN8XzPCiyqW2lFcvYKiIpGN0B/bDaCF4mN0BUDBv1nybx92BS9Se8jgHnFNKHTL3t2A4BkesH48Dp5VSGUqpG8BnQE8ct24Atd8RxAFtzRkB9TAGe7ZXs02VjtlnuRz4QSm1xObQdsAysyMQ2GYTPs6cHfIokGV2DXwJPCkijc0npyfNsBqFUmq2UqqFUqo1xm++Vyk1BogCRpnRCpeHpZxGmfGVGT7anDniDbQFYn+jbFQaSqmfgbMi8qAZ1B84hmPWjzPAoyLiZv5vLGXhkHXDSnWPVlf1B2MGxAmMUf0/Vbc9VZTHxzCapUlAgvl5CqMv8ysg1fz2NOML8L9mmRwFHrZJawLGwFcaML6681YJZRPArVlDbTD+rGnAZsDFDHc199PM421szv+TWU4pwKDqzk8FysEfOGzWkQiMWT8OWT+AecBxIBlYgzHzx2HrhlJKS0xoNBqNo1Pbu4Y0Go1GUwLaEWg0Go2Dox2BRqPRODjaEWg0Go2Dox2BRqPRODjaEdQSRESJyPs2+zNEJLiS0l4lIqNKjlnh6/y3qYwZVdXXKsGOdBHxqk4b7CEiARYlVTvHNphKoa/9BnZ4iMirNvvNRWRLceeUIe3hItLBZn++iDxeGWlrikY7gtpDLjDyTruBiYhTGaK/BLyqlOpbVfbURkTkd0BPpZSfUupvpTynbsmxisQDQ5UTAKXUeaVUZT0oDMdQ9rSkPVcptaeS0tYUgXYEtYebGOur3vZEWPiJXkSyze8AEdknIptE5ISILBSRMSISKyJHRcTHJpnHReRrM95g83wnEVkkInHm0+grNulGich6jBeSCtvznJl+soi8Z4bNxXgxLkxEFhWK30xE9otIgnlObzP8YxE5LIa2/Dyb+OkiskBEDprHu4rIlyJyUkQm2di4X0S2isgxEQkTkdv+DyIy1iyPBBH5/2aencwyTTbzUZYyLyovT5r2ficim8XQjbKsp3FcRGKAkYWvYxIJ3G2m2VtE/EXkW/M32Sq31hmINstlHzCtkL13icgK87c8IiLDzPCONvlPEpG2wELAxwxbJCKtRSTZjP+iiESIyD9F5LSITBaR/zHT/FZEPM14L5vXShSRcDHe9O0JDAUWmWn72JajiPQ30zlq2upi83vPM8vuqIi0K6KcNEVR3W+06U/lfIBsoBGQjqGHMgMINo+tAkbZxjW/A4BMjPUMXIAfgXnmsWnABzbnf4Hx4NAWQ2fFFZgIvG3GccF4c9XbTDcH8LZjZ3OM1/ybYoih7QWGm8eisXmL1eac1zHfCsdYY6Khue1pExYN+Jn76dzSk/8bxtu0Dc1rXrDJ+zWMN0qdMCSVR9mc7wW0B/4JOJvhHwHjgIcw5Jgt9nnYsbmoMr8tL+a19gN3meFvAHPNMj5rlrlgaObvsHOt1kCyzX4S8F/m9nyb3zEa+KiI+rMAGGvJD8bb+HcBocAYM7weUN/O9az7GNr+aTblnQVMsvktppvbTWzO/wswpYhyW4Uh7WApiwfM8E9s0kq3Of9V4B/V/X+saR/dIqhFKENx9BNgahlOi1PGega5GK/LR5rhRzH+4BY2KaXylVKpGAuStMPQmhknIgkYstdNMG5aALHK0GkvzCNAtDJEv24C6zC08ou1ERgvxpiHrzLWXAB4RkS+A44AHbHpUuCWptRRjIVVLiulMoBrIuJhY+MppVQesAGjRWJLf4ybfpyZx/4YjuMU0EZEQkVkIFCc0mtp8vKoafs35nUCgVYYZXxaKZWqjLvc2pISFxF3DMe0zwxaTcHy/bSIU58E3jSvH41x420JHATeEpE3gFZKqaulyGOUTXlnYThTKFinOonRwjwKjMH4/YrjQYyyOFFEvixCi/EUrLeaUlCRfkLNnckHwHfASpuwm5jdgCIiGE92FnJttvNt9vMpWD8Ka5FYpHinKKUKCI+JSABGi8Ae9uR7i0UptV9E+mAsNLPG7Dr6GqPV84hS6j8isgrj5mXBNh+F82jJl708FbZ1tVJq9m2ZEOmMsVBLEPAMhgaPLXbLvIi8/AejhfFcoWv427GpohT3uzytlEopFP6DiBwy7f1SRP6A4QiLozR1ahVGSzBRRF7EaKEVR0n1xnKNPPR9rczoFkEtQyl1CaML4SWb4HSMJ1swtNidy5H0f4tIHTHGDdpgCG19CfxRDAlsROQBMRY8KY5DwH+JiJcYA8nPAfuKO0FEWmF06SzDUFntitENlgNkicg92NeUL4luYijT1gGeBWIKHf8KGCUid5t2eIpIKzEG5OsopcKBOaY9hUnHTpkXkZdvgV4icr8Zx01EHsAQRvOWW2M1BRyFPZRSWcB/LGMPwAuUUL4mXwJTTKeFiHQxv9sAp5RSIRitLD/gMkbXT0VoCPxk1p0xNuFFpX0caG0pI0qfL00p0J6zdvI+MNlmfxmwTURiMW5uRT0VFkcKxh/vHow+32si8g+MZvh35g0kg1tL/NlFKfWTiMzGkP0VYJdSaltx52A8Lc4UkRsYYyHjlFKnReQI8D3GE+o35cjTQYyBT1+MPvqthWw9JiJvA5Gms7iB0QK4irHal+VB6rYWA0WXub28ZJhPxRssA6AYYy8nRGQisFNE/o3hqDqVIl+BGIPubhhlM74U5/wZozWZZP6W6cBgDAc51rT3Z2C+UuqSiHxjDhB/jqFUWlbmYDwU/Aujy8hy898ILBORqdyShcasb+OBzWLMeIrDWFtYUwlo9VGNQ2J2X81QSg2ubls0mupGdw1pNBqNg6NbBBqNRuPg6BaBRqPRODjaEWg0Go2Dox2BRqPRODjaEWg0Go2Dox2BRqPRODj/ByCVw2LKrOuEAAAAAElFTkSuQmCC\n",
181 |       "text/plain": [
182 |        "<Figure size 432x288 with 1 Axes>"
183 |       ]
184 |      },
185 |      "metadata": {
186 |       "needs_background": "light"
187 |      },
188 |      "output_type": "display_data"
189 |     }
190 |    ],
191 |    "source": [
192 |     "m=-2\n",
193 |     "s=5\n",
194 |     "\n",
195 |     "min_points = 50 #the minimum points in the set to start estimation\n",
196 |     "max_points = 10000 # the maximum number of points \n",
197 |     "step = 1000\n",
198 |     "samples = list(range(min_points,max_points,step))\n",
199 |     "\n",
200 |     "hx_theory = np.zeros(len(samples))+ entropy_normal(norm.pdf,m,s)\n",
201 |     "hx_true = np.zeros(len(samples))\n",
202 |     "hx_crude = np.zeros(len(samples))\n",
203 |     "hx_kde = np.zeros(len(samples))\n",
204 |     "hx_hist = np.zeros(len(samples))\n",
205 |     "\n",
206 |     "for i,value in enumerate(samples):\n",
207 |     "    #xs_syntetic=ss.norm.rvs(m,s,value, random_state=1234)\n",
208 |     "    xs_syntetic=ss.norm.rvs(m,s,value)\n",
209 |     "    hx_true[i]=entropy_estimate_true(xs_syntetic, m,s)\n",
210 |     "    hx_crude[i]=entropy_estimate_crude(xs_syntetic)\n",
211 |     "    hx_kde[i] = entropy_estimate_kde(xs_syntetic, bandwidth=0.4)\n",
212 |     "    hx_hist[i]= entropy_estimate_hist(xs_syntetic)\n",
213 |     "    \n",
214 |     "\n",
215 |     "\n",
216 |     "plt.plot(samples, hx_theory, label='Theoretical value')\n",
217 |     "plt.plot(samples, hx_true, label='True probabilities')\n",
218 |     "plt.plot(samples, hx_crude, label='Crude probabilities')\n",
219 |     "plt.plot(samples, hx_kde, label='KDE estimator')\n",
220 |     "plt.plot(samples, hx_hist, label='Histogram estimator')\n",
221 |     "plt.xlabel('Number of samples used for estimation')\n",
222 |     "plt.ylabel('Estimated entropy values')\n",
223 |     "plt.legend()\n",
224 |     "plt.show()"
225 |    ]
226 |   },
227 |   {
228 |    "cell_type": "code",
229 |    "execution_count": null,
230 |    "metadata": {},
231 |    "outputs": [],
232 |    "source": []
233 |   }
234 |  ],
235 |  "metadata": {
236 |   "kernelspec": {
237 |    "display_name": "Python 3",
238 |    "language": "python",
239 |    "name": "python3"
240 |   },
241 |   "language_info": {
242 |    "codemirror_mode": {
243 |     "name": "ipython",
244 |     "version": 3
245 |    },
246 |    "file_extension": ".py",
247 |    "mimetype": "text/x-python",
248 |    "name": "python",
249 |    "nbconvert_exporter": "python",
250 |    "pygments_lexer": "ipython3",
251 |    "version": "3.7.3"
252 |   }
253 |  },
254 |  "nbformat": 4,
255 |  "nbformat_minor": 4
256 | }
257 | 


--------------------------------------------------------------------------------
/Mutual Entropy for SCA  with bounds.ipynb:
--------------------------------------------------------------------------------
  1 | {
  2 |  "cells": [
  3 |   {
  4 |    "cell_type": "code",
  5 |    "execution_count": 1,
  6 |    "metadata": {},
  7 |    "outputs": [],
  8 |    "source": [
  9 |     "import numpy as np\n",
 10 |     "import matplotlib.pyplot as plt\n",
 11 |     "from IPython.display import display\n",
 12 |     "from scipy.stats import norm, entropy\n",
 13 |     "import scipy.stats as ss\n",
 14 |     "from sklearn.model_selection import KFold"
 15 |    ]
 16 |   },
 17 |   {
 18 |    "cell_type": "markdown",
 19 |    "metadata": {},
 20 |    "source": [
 21 |     "Ack: this notebook has been written by Ileana Buhan and Olivier Bronchain. Discussions with Kostas Papagiannopoulos were instrumentantal for deriving the formula for mutual information applied to SCA."
 22 |    ]
 23 |   },
 24 |   {
 25 |    "cell_type": "markdown",
 26 |    "metadata": {},
 27 |    "source": [
 28 |     "## Modeling the True Leakage\n",
 29 |     "\n",
 30 |     "\n",
 31 |     "The notation used here is chosen to match the notation used in the paper of Bronchain et al. https://eprint.iacr.org/2019/132.pdf\n",
 32 |     "\n",
 33 |     "Simulations settings and notations:\n",
 34 |     "* Random variable letters are denoted with capital letters and their realisation with lower cases.\n",
 35 |     "* We are interested in the leakage $L$ associated to an intermediate (sensible) variable $K$. In the following, we take $K$ as a 2-bit binary variable, so $K$ can take 4 values.\n",
 36 |     "* We assume the discrete variable $K$ follows a uniform distribution.  \n",
 37 |     "* For a given value of $K$ denoted $k$, the probability to observe $l$ is mathematically written as $p(l|k)$ and in code is refered to as$p_l_k$. In our case, the samples are generated according to four conditional probabilities, one for each value of $k$.\n",
 38 |     "* We assume normal distribution, with a given mean and std. This means that each $p(l|k)$ follows a Gaussian distribution. This distribution is called the \"True distribution\" because it does not suffer from estimation issues.\n",
 39 |     "* Leakage samples are generated according to these \"True distributions\". \n"
 40 |    ]
 41 |   },
 42 |   {
 43 |    "cell_type": "code",
 44 |    "execution_count": 2,
 45 |    "metadata": {},
 46 |    "outputs": [],
 47 |    "source": [
 48 |     "number_bits=2\n",
 49 |     "number_classes=2**number_bits #the total number of values taken by the target intermediate\n",
 50 |     "\n",
 51 |     "#setting the probability of the key\n",
 52 |     "p_k= np.ones(number_classes, dtype=np.float64)\n",
 53 |     "p_k/=number_classes\n",
 54 |     "\n",
 55 |     "\n",
 56 |     "#defining the values for p(l|k), for the 4 different values of k\n",
 57 |     "norm_params = np.array([[1, 4],[0,2], [0, 5], [-1, 4]])\n",
 58 |     "#norm_params = np.array([[0, 0.1],[1, .1],[2, .1], [3, .1]])"
 59 |    ]
 60 |   },
 61 |   {
 62 |    "cell_type": "markdown",
 63 |    "metadata": {},
 64 |    "source": [
 65 |     "## Lets measure!\n",
 66 |     "\n",
 67 |     "To simulate the measurement process, we sample data from the \"True distributions\".\n"
 68 |    ]
 69 |   },
 70 |   {
 71 |    "cell_type": "code",
 72 |    "execution_count": 3,
 73 |    "metadata": {},
 74 |    "outputs": [],
 75 |    "source": [
 76 |     "def measure_data(p_k,norm_params,number_samples=100):\n",
 77 |     "    \"\"\" \n",
 78 |     "    p_k = discrete probability distribution if K\n",
 79 |     "    norm_params = parameters of the true distributions\n",
 80 |     "    number_samples = number of samples for each class of K. This corresponds to n^k_p\n",
 81 |     "\n",
 82 |     "    returns leakage samples correponding to the target intermediate K and norm_params\n",
 83 |     "    \"\"\"\n",
 84 |     "    data = {}\n",
 85 |     "    for k, _ in enumerate(p_k):\n",
 86 |     "        data[k]= ss.norm.rvs(*(norm_params[k]),size=number_samples)\n",
 87 |     "    return data\n",
 88 |     "\n",
 89 |     "true_data=measure_data(p_k, norm_params)"
 90 |    ]
 91 |   },
 92 |   {
 93 |    "cell_type": "markdown",
 94 |    "metadata": {},
 95 |    "source": [
 96 |     "### Visualize the histogram of the measured data"
 97 |    ]
 98 |   },
 99 |   {
100 |    "cell_type": "code",
101 |    "execution_count": 4,
102 |    "metadata": {},
103 |    "outputs": [
104 |     {
105 |      "data": {
106 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAdq0lEQVR4nO3df7SVVb3v8ffHDUIjNROwg2y8YOI44i/SjWhHuWaS2PWG5i9IExIvnZSGt24Nf4ybHb3V0U4O6pSaVKZSiqanYhhK4Y/qeNTYKjsBj7oljiyg2oK/StkCfu8f69m0XKy9WWs/z3rW3pvPa4w19vPMZ8655uMem69zzueZUxGBmZlZb+3W6AaYmVn/5kBiZmapOJCYmVkqDiRmZpaKA4mZmaUyqNENaIThw4fHmDFjGt0MM7N+5YknnngpIkaUp++SgWTMmDG0trY2uhlmZv2KpP+qlO6hLTMzS8WBxMzMUnEgMTOzVHbJORIzs2pt2bKFQqHA5s2bG92U3AwdOpTm5mYGDx5cVX4HEjOzHhQKBfbcc0/GjBmDpEY3p+4igo0bN1IoFBg7dmxVZTy0ZWbWg82bNzNs2LBdIogASGLYsGE19cAcSMzMdmJXCSJdar1fBxIzM0vFgcTMzFLxZLtZdx76596V+9Dl2bbDrI9zIDEzq8G8Xz2XaX2fm3JQVfmWLl3KrbfeyoIFC1J93/33388ll1zCtm3buPDCC7nssstS1Qce2jIz6xfa2tr4wAc+kKqObdu2cfHFF3PfffexatUq7rjjDlatWpW6bQ4kZmb9QFcg6ezsZNasWVxxxRVERE11/O53v+PAAw/kgAMOYPfdd2f69On8/Oc/T902D22ZmfUDbW1t7Lvvvpx88slceOGFnHfeeduvHX/88bz++us7lPnGN77BSSedtP183bp1jB49evt5c3Mzjz/+eOq2OZCYmfVxW7ZsYc2aNcyYMYObbrqJY4899h3Xf/vb31ZVT6UeTBbvyDiQmJn1catWrWLixIls2rSJpqamHa5X2yNpbm5m7dq1288LhQL77bdf6vY5kJiZ9XFtbW188IMf5LzzzuP000/nwQcf5H3ve9/269X2SCZOnMjzzz/PH/7wB0aNGsXChQu5/fbbU7fPgcTMrAbVPq6bpba2NiZNmsRBBx3Etddey9lnn83SpUurXp23y6BBg/jOd77DySefzLZt27jgggs45JBDUrdPtc76p/oyaSrwLaAJ+H5EXFN2fTLwTeBwYHpE3J2kfwiYV5L175PrP5N0C/DfgVeTa7MiYnlP7WhpaQlvtWs71dsXEnvDLzH2Wc888wwHH3xwo5uRu0r3LemJiGgpz5tbj0RSE3A9MAUoAMskLYqI0oeYXwRmAV8oLRsRDwETknr2AdqBX5Zk+WJX0DEzs3zlObR1NNAeEasBJC0EpgHbA0lErEmuvd1DPWcC90XEG/VrqpmZVSvPFxJHAWtLzgtJWq2mA3eUpX1V0u8lzZM0pFIhSXMktUpq7ejo6MXXmplZJXkGkkoPK9c0QSNpJHAYsKQk+XKKcyYTgX2ASyuVjYj5EdESES0jRoyo5WvNzKwHeQaSAjC65LwZWF9jHWcDP42ILV0JEbEhijqBH1IcQjMzs5zkGUiWAeMkjZW0O8UhqkU11jGDsmGtpJeCiq9nngasyKCtZmZWpdwCSURsBeZSHJZ6BrgrIlZKulrSxwAkTZRUAM4CbpK0squ8pDEUezS/Lqv6x5KeBp4GhgNfqfe9mJnZ3+T6QmJELAYWl6VdWXK8jOKQV6Wya6gwOR8RJ2bbSjMzq4XfbDczq0XWL6pW+TJqVhtbXXDBBdx7773su+++rFiRzUyA9yMxM+sHstjYCmDWrFncf//9GbTobxxIzMz6gSw2tgKYPHky++yzT6Zt89CWmVk/kMXGVvXiQGJm1sdltbFVvTiQmJn1cVltbFUvDiRmZn1cVhtb1YsDiZlZLRqwd0xWG1sBzJgxg4cffpiXXnqJ5uZmrrrqKmbPnp2qfQ4kZmZ93HXXXbf9+NRTT+XUU0/tdV133FG+eHp6fvzXzMxScSAxM7NUHEjMzCwVBxIzM0vFgcTMzFJxIDEzs1QcSMzMLBUHEjMzS8UvJJqZ1eCG5TdkWt9FEy6qKl8WG1utXbuW888/nz/+8Y/stttuzJkzh0suuaTX9XVxIDEz6wey2Nhq0KBBXHfddRx55JG8/vrrHHXUUUyZMoXx48enqzdV6RpJmgp8C2gCvh8R15Rdnwx8EzgcmB4Rd5dc2wY8nZy+GBEfS9LHAguBfYAngU9GxFv1vhezTPV2+9YGrPtkjdHW1sanPvUpOjs7+fSnP81+++3HV7/6VSRVXcfIkSMZOXIkAHvuuScHH3ww69at6z+BRFITcD0wBSgAyyQtiohVJdleBGYBX6hQxZsRMaFC+rXAvIhYKOm7wGzgxkwbb2bWYFlvbLVmzRqeeuopJk2alLptefZIjgbaI2I1gKSFwDRgeyCJiDXJtberqVDFUHwi8Ikk6Vbgn3AgMbMBJOuNrf7yl79wxhln8M1vfpO99tordfvyDCSjgLUl5wWgllA4VFIrsBW4JiJ+BgwDXomIrSV1jqpUWNIcYA7A/vvvX2PTzcwaJ8uNrbZs2cIZZ5zBueeey8c//vFM2pdnIKk0kFfLzvX7R8R6SQcAD0p6Gnit2jojYj4wH6ClpaWW7zUza6isNraKCGbPns3BBx/M5z//+czal2cgKQCjS86bgfXVFo6I9cnP1ZIeBj4A3APsLWlQ0iupqU4zs1pV+7hulrLa2OqRRx5hwYIFHHbYYUyYUJxy/trXvsZHP/rRVO3LM5AsA8YlT1mtA6bzt7mNHkl6L/BGRHRKGg78A/D1iAhJDwFnUnxyaybw87q03sysQbLa2Oq4444jIvsBmdzebE96DHOBJcAzwF0RsVLS1ZK6HuWdKKkAnAXcJGllUvxgoFVSG/AQxTmSrkn6S4HPS2qnOGfyg7zuyczMcn6PJCIWA4vL0q4sOV5GcXiqvNx/AId1U+dqik+EmZlZA3itLTMzS8WBxMzMUnEgMTOzVBxIzMwsFQcSMzNLxYHEzMxS8X4kZmY16Pj2dzKtb8Rn51aVL4uNrTZv3szkyZPp7Oxk69atnHnmmVx11VW9rq+LA4mZWT+QxcZWQ4YM4cEHH2SPPfZgy5YtHHfccZxyyikcc8wxqer10JaZWT/QFUg6OzuZNWsWV1xxRc3LnUhijz32AIqrAG/ZsqWmjbG64x6JmVk/kNXGVtu2beOoo46ivb2diy++uN9tbGVmZr2Q5cZWTU1NLF++nFdeeYXTTz+dFStWcOihh6ZqnwOJ9TlZT2Z2qXZS06yvyXJjqy577703J5xwAvfff78DiZnZQJfVxlYdHR0MHjyYvffemzfffJOlS5dy6aWXpm6fA4mZWQ0a0bPNamOrDRs2MHPmTLZt28bbb7/N2Wef3eu9TUo5kJiZ9XFZbWx1+OGH89RTT2XVrO38+K+ZmaXiQGJmZqk4kJiZ7UQ99jnvy2q931wDiaSpkp6V1C7psgrXJ0t6UtJWSWeWpE+Q9KiklZJ+L+mckmu3SPqDpOXJZ0Je92NmA9/QoUPZuHHjLhNMIoKNGzcydOjQqsvkNtkuqQm4HpgCFIBlkhZFxKqSbC8Cs4AvlBV/Azg/Ip6XtB/whKQlEfFKcv2LEXF3fe/AzHZFzc3NFAoFOjo6Gt2U3AwdOpTm5uaq8+f51NbRQHtErAaQtBCYBmwPJBGxJrn2dmnBiHiu5Hi9pD8DI4BXMDOro8GDBzN27NhGN6NPy3NoaxSwtuS8kKTVRNLRwO7ACyXJX02GvOZJGpKumWZmVos8A0mlJSZrGnSUNBJYAHwqIrp6LZcDfw9MBPYBKr6mKWmOpFZJrbtSF9XMrN7yDCQFYHTJeTOwvtrCkvYCfgH834h4rCs9IjZEUSfwQ4pDaDuIiPkR0RIRLSNGjOjVDZiZ2Y7yDCTLgHGSxkraHZgOLKqmYJL/p8BtEfGTsmsjk58CTgNWZNpqMzPrUW6BJCK2AnOBJcAzwF0RsVLS1ZI+BiBpoqQCcBZwk6SVSfGzgcnArAqP+f5Y0tPA08Bw4Ct53ZOZmeW81lZELAYWl6VdWXK8jOKQV3m5HwE/6qbOEzNuppmZ1cBvtpuZWSoOJGZmloqXkbddw5rfwkM77iBnZum5R2JmZqk4kJiZWSoOJGZmlooDiZmZpeJAYmZmqfipLbMUOu5dnnmdI0713mzWv7hHYmZmqTiQmJlZKg4kZmaWSs2BRNK7k/3XzczMdh5IJO0m6ROSfpHslf6fwAZJKyX9i6Rx9W+mmZn1VdX0SB4C3k9xS9u/i4jREbEvcDzwGHCNpPPq2EYzM+vDqnn896SI2FKeGBGbgHuAeyQNzrxlZmbWL+y0R9IVRCTtsPNg11xJpUBjZma7hlom20dJmtF1ImlfYGn2TTIzs/6kljfbPw0skfQCEMAPgUvr0iozM+s3qnlq6zZJ/xs4FrgYmA/cCJwWEb+o5cskTZX0rKR2SZdVuD5Z0pOStko6s+zaTEnPJ5+ZJelHSXo6qfNfJamWNpmZWTrV9EhuBY4ALkh+jgGWAedJWhERd1fzRcl8yvXAFKAALJO0KCJWlWR7EZgFfKGs7D7Al4EWir2hJ5KyL1MManMoPkG2GJgK3FdNm8yyVHj5zUzqaV+9cYe0Yw8YlkndZvWw00ASEQ8AD3SdSxoEjKcYVI4BqgokwNFAe0SsTupZCEwDtgeSiFiTXHu7rOzJwK+SJ8WQ9CtgqqSHgb0i4tEk/TbgNBxIzMxys9NAImn/CsmvAL8Gfl1y/ZWIeK2HqkYBa0vOC8CkKttZqeyo5FOokL4DSXMo9lzYf/9Kt2RmZr1R7dBWdwJQ8vMW4LYe8laau4gqvr+nslXXGRHzKc7v0NLSUu33mpnZTlQztPWhjL6rAIwuOW8G1tdQ9oSysg8n6c29rNPMzDJQzdDWrUAbsBxoi4gdZwKrswwYJ2kssA6YDnyiyrJLgK9Jem9y/hHg8ojYJOl1SccAjwPnA9/uZftsF/BohYnsNIZmNMFu1p9V80Ji13DVTOABSS9IulfSVySdVe0XRcRWYC7FoPAMcFdErJR0taSPAUiaKKkAnAXcJGllUnYT8P8oBqNlwNVdE+/AZ4DvA+3AC3ii3cwsV2mf2poE/KTaL4uIxRQf0S1Nu7LkeBnvHKoqzXczcHOF9Fbg0GrbYGZm2apmaEsRsX1yOulZ/D75LKiUx8zMdh1VLSMv6bPljwFL2l3SickcysxuypqZ2QBXzeO/Uym+1X5HMlH+CjAUaAJ+CcyLiOX1a6KZmfVl1cyRbJb0XeC9wNeB4cCbEfFKvRtnZmZ9X1XLyEfE28CJEbElIjY4iJiZWZda9iNZLunLkmopY2ZmA1wt+5GMBg4DPiPpcZIntyKi6sd/zcxs4Kk6kETE2QCShgCHUAwqNb1HYmZmA0/Vw1SSTpH0GMWeyGXAcxHxhZ0UMzOzAa6W+Y4bgP9DcQ+S+cC/lO7hbmZmu6Za5kj+FBGPJMdLJT1KcaHEO7JvlpmZ9Re19EjWJAs17p6cbwFer0ObzMysH6klkATwcWCtpH+nuNruw5LG1aVlZmbWL9Ty1NYMAElDKa62e0Ty+b6kAyJidE/lzcxsYKpljgQoLpkCtCYfMzPbxfktdTMzS6XmHolZf9Vx73JvjWtWB+6RmJlZKg4kZmaWSq5DW5KmAt+iuCnW9yPimrLrQ4DbgKOAjcA5EbFG0rnAF0uyHg4cGRHLJT0MjAS6xiw+EhF/ru+dmOXr0dUbK6Y/tvW51HV/bspBqeuwXVtuPRJJTcD1wCnAeGCGpPFl2WYDL0fEgcA84FqAiPhxREyIiAnAJ4E1Zbsyntt13UHEzCxfeQ5tHQ20R8TqiHgLWAhMK8szDbg1Ob4b+LAkleWZgZdlMTPrM/IMJKOAtSXnhSStYp6I2Aq8Cgwry3MOOwaSH0paLulLFQIPAJLmSGqV1NrR0dHbezAzszJ5BpJK/8BHLXkkTQLeiIgVJdfPjYjDgOOTzycrfXlEzI+IlohoGTFiRG0tNzOzbuUZSAoUd1ns0gys7y6PpEHAe4BNJdenU9YbiYh1yc/XgdspDqGZmVlO8gwky4BxksYmKwhPBxaV5VkEzEyOzwQejIgASPaKP4vi3ApJ2iBJw5PjwcCpwArMzCw3uT3+GxFbJc0FllB8/PfmiFgp6WqgNSIWAT8AFkhqp9gTmV5SxWSgEBGrS9KGAEuSINIELAW+l8PtmJlZItf3SCJiMbC4LO3KkuPNFHsdlco+THF3xtK0v1J858Ss4Z7Vpp1nqkLbbu1V5y28ducOaUfudU4m7TCrlt9sNzOzVBxIzMwsFQcSMzNLxYHEzMxScSAxM7NUHEjMzCwVBxIzM0vFgcTMzFJxIDEzs1QcSMzMLBUHEjMzSyXXtbbMrP6erLD+Vk9uWF6+d1w2LppwUV3qtb7HgcT6nzW/bXQL+ozm156ouUxhL69zatny0JaZmaXiQGJmZqk4kJiZWSoOJGZmlooDiZmZpeKntqzfW7b5T1XnfU1b69gSs11Trj0SSVMlPSupXdJlFa4PkXRncv1xSWOS9DGS3pS0PPl8t6TMUZKeTsr8qyTld0dmZpZbIJHUBFwPnAKMB2ZIGl+WbTbwckQcCMwDri259kJETEg+/1iSfiMwBxiXfKbW6x7MzGxHefZIjgbaI2J1RLwFLASmleWZBtyaHN8NfLinHoakkcBeEfFoRARwG3Ba9k03M7Pu5BlIRgFrS84LSVrFPBGxFXgV6Fq/YaykpyT9WtLxJfkLO6kTAElzJLVKau3o6Eh3J2Zmtl2ek+2VehZRZZ4NwP4RsVHSUcDPJB1SZZ3FxIj5wHyAlpaWinmsb3hs9cYerze/+uY7zj2Bns6jL/T837u3OjueA+BzUw6qS/3Wd+QZSArA6JLzZmB9N3kKkgYB7wE2JcNWnQAR8YSkF4CDkvzNO6nT6qjj299pdBPMrMHyDCTLgHGSxgLrgOnAJ8ryLAJmAo8CZwIPRkRIGkExoGyTdADFSfXVEbFJ0uuSjgEeB84Hvp3T/Riw7I/LMq9zQ+cbPV7/qzZl/p1m1nu5BZKI2CppLrAEaAJujoiVkq4GWiNiEfADYIGkdmATxWADMBm4WtJWYBvwjxHR9a/JZ4BbgHcB9yUfMzPLSa4vJEbEYmBxWdqVJcebgbMqlLsHuKebOluBQ7NtqZmZVctLpJiZWSpeIsVqNu9Xz20/fvfLPc9nmNnA50BiZr1y/AO/7/H6yCFvA9Dxn9Vv5Tvis3NTtckaw4HEGmavzg2NbsIuqTfb84K36LXueY7EzMxScY+kn7ph+Q0N++4nX/vbm9DH95DPzHYN7pGYmVkqDiRmZpaKh7bM+pgj/v2lzOtsO2545nWadXGPxMzMUnEgMTOzVBxIzMwsFQcSMzNLxYHEzMxS8VNbZruALJ4EGzuk57W1bNflHomZmaXiHkmNGrk0iZlZX+RAYmZ19djqjTvPlGgv2eumGp+bclCtzbE68NCWmZmlkmsgkTRV0rOS2iVdVuH6EEl3JtcflzQmSZ8i6QlJTyc/Tywp83BS5/Lks29+d2RmZrkNbUlqAq4HpgAFYJmkRRGxqiTbbODliDhQ0nTgWuAc4CXgf0bEekmHAkuAUSXlzo2I1lxuJCdjfvJ45nWuOWtS5nWameXZIzkaaI+I1RHxFrAQmFaWZxpwa3J8N/BhSYqIpyJifZK+EhgqaUgurTYzsx7lGUhGAWtLzgu8s1fxjjwRsRV4FSjf8PkM4KmI6CxJ+2EyrPUlSar05ZLmSGqV1NrR0ZHmPszMrESeT21V+gc+askj6RCKw10fKbl+bkSsk7QncA/wSeC2HSqJmA/MB2hpaSn/3gGn8PIbO6Q9+kL1T8+YmVUrzx5JARhdct4MrO8uj6RBwHuATcl5M/BT4PyIeKGrQESsS36+DtxOcQjNzMxykmcgWQaMkzRW0u7AdGBRWZ5FwMzk+EzgwYgISXsDvwAuj4hHujJLGiRpeHI8GDgVWFHn+zAzsxK5DW1FxFZJcyk+cdUE3BwRKyVdDbRGxCLgB8ACSe0UeyLTk+JzgQOBL0n6UpL2EeCvwJIkiDQBS4Hv5XVP/c3xD3itJDPLXq5vtkfEYmBxWdqVJcebgbMqlPsK8JVuqj0qyzaamVlt/Ga7mZml4kBiZmapOJCYmVkqDiRmZpaKA4mZmaXiQGJmZqk4kJiZWSreIdHM6mJDZ+2LTDz52p015b9hefmarn3XRRMuanQT6sY9EjMzS8WBxMzMUnEgMTOzVDxHYmb9Vj332Dn2/f1n/qXRHEjsHfbq3NCrcq8NGZlxS8ysv/DQlpmZpeIeiZlZDm5YfkOjm1C3R5DdIzEzs1TcI2mgniYKB738Ro4tMTPrPfdIzMwsFQcSMzNLJddAImmqpGcltUu6rML1IZLuTK4/LmlMybXLk/RnJZ1cbZ1mZlZfuQUSSU3A9cApwHhghqTxZdlmAy9HxIHAPODapOx4YDpwCDAVuEFSU5V1mplZHeXZIzkaaI+I1RHxFrAQmFaWZxpwa3J8N/BhSUrSF0ZEZ0T8AWhP6qumTjMzq6M8n9oaBawtOS8Ak7rLExFbJb0KDEvSHysrOyo53lmdAEiaA8xJTv8i6dle3EO9DQdeAvhRgxtSJ9vvbwDzPaZx4911qbZGw4GXBuLf4MVc3HXY29/hf6uUmGcgUYW0qDJPd+mVelTldRYTI+YD83tqYKNJao2Ilka3o14G+v2B73EgGOj3B9nfY55DWwVgdMl5M7C+uzySBgHvATb1ULaaOs3MrI7yDCTLgHGSxkraneLk+aKyPIuAmcnxmcCDERFJ+vTkqa6xwDjgd1XWaWZmdZTb0FYy5zEXWAI0ATdHxEpJVwOtEbEI+AGwQFI7xZ7I9KTsSkl3AauArcDFEbENoFKded1THfTpobcMDPT7A9/jQDDQ7w8yvkcV/4ffzMysd/xmu5mZpeJAYmZmqTiQNJiksyStlPS2pJayaxWXhenPJP2TpHWSliefjza6TVnYFZbqkbRG0tPJ76210e3JgqSbJf1Z0oqStH0k/UrS88nP9zayjWl1c4+Z/h06kDTeCuDjwG9KE7tbFib/5tXFvIiYkHwWN7oxae1iS/V8KPm9DZT3LG6h+PdV6jLggYgYBzyQnPdnt7DjPUKGf4cOJA0WEc9ERKW37LtbFsb6Hi/V009FxG8oPiFaqnSppluB03JtVMa6ucdMOZD0XZWWlBnVTd7+Zq6k3ydd7n49bJAYyL+rUgH8UtITyZJDA9X7ImIDQPJz3wa3p14y+zt0IMmBpKWSVlT49PR/rdUsKdMn7eR+bwTeD0wANgDXNbSx2ei3v6sa/UNEHElxCO9iSZMb3SDrtUz/Dr3Vbg4i4qReFOu3y79Ue7+SvgfcW+fm5KHf/q5qERHrk59/lvRTikN6v+m5VL/0J0kjI2KDpJHAnxvdoKxFxJ+6jrP4O3SPpO/qblmYfi35w+xyOsWHDfq7Ab9Uj6R3S9qz6xj4CAPjd1dJ6VJNM4GfN7AtdZH136F7JA0m6XTg28AI4BeSlkfEyT0tC9PPfV3SBIpDP2uATze2Oel1t/xPg5uVtfcBPy1uD8Qg4PaIuL+xTUpP0h3ACcBwSQXgy8A1wF2SZgMvAmc1roXpdXOPJ2T5d+glUszMLBUPbZmZWSoOJGZmlooDiZmZpeJAYmZmqTiQmJlZKg4kZmaWigOJmZml4kBi1kdIepekX3dtFyDpJEkLJO0u6TeS/AKx9UkOJGZ9xwXAv5WsYHAE0JYsTf8AcE7DWmbWAwcSs5xJWijpTkmPS/ovSf8juXQu71zX6QhgeXL8s+S6WZ/jQGKWvyOA1RExiWJw+HKy2OMBEbGmLF9bcrwCmJhrK82q5EBiliNJ7wKGA1clSauA9yZpr5TkGwzsFREdAMlw11tdK/Ca9SUOJGb5OhR4PiI2J+dHUux1vAkMLck3HnimrOwQYDNmfYwDiVm+jgD2lzQ02dfjKmBeRLwMNEkaWpKva34EScOAjojYknuLzXbCgcQsX0cAPwYeprgh1o0R8Uhy7ZfAcSX52krKfQhYnFMbzWri/UjMciTpN8D/iohnK1z7APD5iPhkhWv/BlxeqZxZo7lHYpav9wPPV7oQEU8BD3W9kNgleaLrZw4i1le5R2JmZqm4R2JmZqk4kJiZWSoOJGZmlooDiZmZpeJAYmZmqTiQmJlZKg4kZmaWyv8H8YfyMlIECYkAAAAASUVORK5CYII=\n",
107 |       "text/plain": [
108 |        "<Figure size 432x288 with 1 Axes>"
109 |       ]
110 |      },
111 |      "metadata": {
112 |       "needs_background": "light"
113 |      },
114 |      "output_type": "display_data"
115 |     }
116 |    ],
117 |    "source": [
118 |     "def visualize_distribution_points(data_dictionary, b='auto'):\n",
119 |     "    \"\"\"\n",
120 |     "    plot the histogram of the elements in the lists\n",
121 |     "    as a function of the key values\n",
122 |     "    \n",
123 |     "    data_dictionary =  dictionary of lists, the key is the value of the keys and values are the samples l\n",
124 |     "                    = k: [l_0,l_1,....l_n]\n",
125 |     "    \n",
126 |     "    \"\"\"\n",
127 |     "    for k in data_dictionary:\n",
128 |     "\n",
129 |     "        plt.hist(data_dictionary[k], bins=b,density=True,alpha = 0.5,label='$k = %d$'%k)\n",
130 |     "    plt.xlabel(\"$p(l)$\")\n",
131 |     "    plt.ylabel(\"$pr(l|k)$\")\n",
132 |     "    plt.legend()\n",
133 |     "    plt.show()\n",
134 |     "\n",
135 |     "visualize_distribution_points(true_data)"
136 |    ]
137 |   },
138 |   {
139 |    "cell_type": "markdown",
140 |    "metadata": {},
141 |    "source": [
142 |     "## Calculating Mutual Information for SCA"
143 |    ]
144 |   },
145 |   {
146 |    "cell_type": "markdown",
147 |    "metadata": {},
148 |    "source": [
149 |     "### Notations:\n",
150 |     "* $H(K)$ is the entropy of $K$. If uniform, this corresponds to the number of bits in $K$.\n",
151 |     "* $H(K|L)$ is the conditional entropy. It is the entropy of $K$ given leakage $L$.\n",
152 |     "* $I(K;L)$ is the mutual information between $K$ and $L$ and it represents the amount of information the two variables provide about eachother.  These terms are related such that\n",
153 |     "$$H(K|L) = H(K) - I(K;L).$$\n",
154 |     "\n",
155 |     "Calculating these terms requires to know the analytical form for the distribution of both $K$ and $L$.\n",
156 |     "\n",
157 |     "### Interest for Side Channel\n",
158 |     "In side-channel context, it is interesting to learning $H(K|L)$ or equivalently $I(K;L)$ (since $H(K)$ is typically known in cryptographic contextm as $K$ is uniformly distributed). \n",
159 |     "\n",
160 |     "Informally,  $I(K;L)$ is the loss of entropy of $K$ given a leakage samples $L$. The data complexity of an attack can be approximated as the number of traces required to reduce the conditional entropy $H(K|L)$ to zero. Roughly, this gives:\n",
161 |     "$$N_{attack} \\propto \\frac{H(K)}{I(K;L)}.$$\n",
162 |     "\n",
163 |     "### Deriving an expression for $I(K;L)$ in the context of SCA\n",
164 |     "\n",
165 |     "As written above, the classic formula is  $I(K,L)=H(K)-H(K|L).$ A problem  for directly applying the formula is that we do not know the value of $H(K|L)$. Let see why. The classic formula for conditional entropy is: $$H(K|L)=\\sum_{k,l} \\color{red}{p(l)}\\color{red}{p(k|l)}\\log_2\\color{red}{p(k|l)},$$ where the distributions marked in red are unknown. In the SCA scenario, we can only observe the following distributions:\n",
166 |     "* $p(k)$ the distribution of the secret variable, typically uniform.\n",
167 |     "* $p(l|k)$ can only be sampled from power or EM traces in a pratical context. Its estimation is next denoted as $\\hat{p}(l|k)$. In a simulated setting, the analytical expression of this can be known and so $p(l|k) = \\hat{p}(l|k)$. \n",
168 |     "\n",
169 |     "So how do we compute  mutual information ? An ingenuous solution is known, fear not! By applying Bayes law, we know that: $\\color{red}{p(k|l)}\\color{red}{p(l)}=p(l|k)p(k)$. This is better! We can now re-write the formula above as: $$H(K|L)=\\sum_{k,l} p(k)p(l|k)log_2\\color{red}{p(k|l)},$$ and we are left with only one distribution that we cannot observe namely $\\color{red}{p(k|l)}$. \n",
170 |     "\n",
171 |     "However, by applying Bayes formula again, we can write $\\color{red}{p(k|l)}= \\frac{p(l|k)p(k)}{\\color{red}{p(l)}}$and now, by applying the formula for computing marginal probabilities we have: $\\color{red}{p(l)}=\\sum_{k^{*}}p(l|k^{*})p(k^{*}).$ When the variable $K$ is uniformly distributed, we can write: $p(k|l)=\\frac{p(l|k)}{\\sum{k^*}p(l|k^*)}$\n",
172 |     "\n",
173 |     "Finally, by combining all the pieces of the puzzle, we now have:\n",
174 |     "\n",
175 |     "$$H(K|L)=\\sum_{k,l} p(k)p(l|k)\\log_2 \\frac{p(l|k)p(k)}{\\sum_{k^*}{p(l|k^*)} p(k^*)}.$$ \n",
176 |     "\n",
177 |     "### Computation of $I(K;L)$\n",
178 |     "In order to compute the previous expression in a side-channel context. One has access to the following:\n",
179 |     "* $n^k_p$ observed leakage from $p(l|k)$ where the $i$-th observation is denoted as $l^{k}_i$.\n",
180 |     "* $p(k)$ is typically known and uniformed.\n",
181 |     "* $\\hat{p}(k|l)$ a model estimating the leakage distribution. This is used to compute the last term in the previous equation.\n",
182 |     "\n",
183 |     "Then, the previous expression is computed by sampling with:\n",
184 |     "$$\\hat{H}(K|L) = \\sum_{k} p(k) \\cdot \\color{blue}{\\sum_{i=1}^{n^k_p} \\frac{1}{n^k_p}} \\cdot \\log_2 \\color{green}{\\frac{\\hat{p}(l^k_i|k)p(k)}{\\sum_{k^*}{\\hat{p}(l^k_i|k^*)} p(k^*)}}$$\n",
185 |     "There, the blue part corresponds to the true leakage distribution that is sampled. The green term corresponds to the modeled distribution.\n",
186 |     "\n",
187 |     "Lets go and implement the formula to compute $\\hat{I}(K;L)$."
188 |    ]
189 |   },
190 |   {
191 |    "cell_type": "code",
192 |    "execution_count": 5,
193 |    "metadata": {},
194 |    "outputs": [
195 |     {
196 |      "name": "stdout",
197 |      "output_type": "stream",
198 |      "text": [
199 |       "MI is 0.145023\n"
200 |      ]
201 |     }
202 |    ],
203 |    "source": [
204 |     "def information(p_k,data,model):\n",
205 |     "    \"\"\"\n",
206 |     "    implements I(K;L)\n",
207 |     "    \n",
208 |     "    p_k = the distribution of the sensitive variable K\n",
209 |     "    data = the samples we 'measured'. It its the n^k_p samples from p(l|k)\n",
210 |     "    model = the estimated model \\hat{p}(l|k).\n",
211 |     "\n",
212 |     "    returns an estimated of mutual information\n",
213 |     "    \"\"\"\n",
214 |     "    N_k = len(p_k)              #N_k is the number of possible values for $K$\n",
215 |     "    acc = entropy(p_k,base=2)   #we initialize the value with H(K)\n",
216 |     "    for k in range(N_k):\n",
217 |     "        l = data[k]\n",
218 |     "        p_l_k = np.zeros((N_k,len(l)))\n",
219 |     "        for k_star in range(N_k):\n",
220 |     "            p_l_k[k_star,:] = ss.norm.pdf(l,*(model[k_star]))\n",
221 |     "        p_l=np.sum(p_k*(p_l_k).T,axis=1)\n",
222 |     "        p_k_l =  p_k[k]*p_l_k[k,:]/ p_l\n",
223 |     "        acc += p_k[k] * np.mean(np.log2(p_k_l))\n",
224 |     "    return acc\n",
225 |     "import statsmodels.api as sm\n",
226 |     "\n",
227 |     "data=measure_data(p_k, norm_params, number_samples=10000)\n",
228 |     "MI = information(p_k,data,norm_params)\n",
229 |     "print(\"MI is %f\"%(MI))"
230 |    ]
231 |   },
232 |   {
233 |    "cell_type": "markdown",
234 |    "metadata": {},
235 |    "source": [
236 |     "## Bounds on Mutual Information\n",
237 |     "\n",
238 |     "The following information graphs allows to compute all the information metrics according to what we set to the inputs:\n",
239 |     "\n",
240 |     "* 𝑀𝐼 : the model and the data should perfectly match ($p(l|k) = \\hat{p}(l|k)$). Therefore, not feasible on real measurements because you never know perfectly the model\n",
241 |     "* 𝑃𝐼 : the model $\\hat{p}(l|k)$ is estimated $n^k_t$ samples which are different than the $n^k_p$ samples. From a fixed size data set, PI is so computed with cross-validation.\n",
242 |     "* 𝐻𝐼 : the model $\\hat{p}(l|k)$ is estimated on data than the $n^k_p$ samples.\n",
243 |     "\n",
244 |     "What's the relation between all these for  𝑛𝑡←∞ :\n",
245 |     "\n",
246 |     "* 𝐻𝐼  is an average upper bound to  𝑀𝐼 if the model estimation is non parametric ($\\hat{p}(k|l)$ converges to $p(k|l)$ whate ever $p(k|l)$).\n",
247 |     "* 𝑃𝐼  is an average lower bound to  𝑀𝐼 . The two matches perfectly if the model is non parametric or if no modeling error is introduced (ie. we assume leakage is Gaussian and the real leakage is perfectly Gaussian).\n",
248 |     "\n",
249 |     "Therefore, if the models used are non parametrics, both  𝐻𝐼  and  𝑃𝐼  converges to  𝑀𝐼  and  𝑃𝐼  is a lower bound to  𝑀𝐼  and  𝐻𝐼  is the upper bound."
250 |    ]
251 |   },
252 |   {
253 |    "cell_type": "code",
254 |    "execution_count": 6,
255 |    "metadata": {},
256 |    "outputs": [
257 |     {
258 |      "name": "stdout",
259 |      "output_type": "stream",
260 |      "text": [
261 |       "PI is 0.133190\n",
262 |       "HI is 0.135488\n",
263 |       "MI is 0.145023\n"
264 |      ]
265 |     }
266 |    ],
267 |    "source": [
268 |     "data_build_model=measure_data(p_k, norm_params,number_samples=1000)\n",
269 |     "data_fresh=measure_data(p_k, norm_params,number_samples=1000)\n",
270 |     "\n",
271 |     "\n",
272 |     "esti_models = [None for i in p_k]\n",
273 |     "for k in range(len(p_k)):\n",
274 |     "    esti_models[k] = [np.mean(data_build_model[k]),np.std(data_build_model[k])]\n",
275 |     "\n",
276 |     "PI = information(p_k,data_fresh,esti_models)\n",
277 |     "HI = information(p_k,data_build_model,esti_models)\n",
278 |     "print(\"PI is %f\"%(PI))\n",
279 |     "print(\"HI is %f\"%(HI))\n",
280 |     "print(\"MI is %f\"%(MI))"
281 |    ]
282 |   },
283 |   {
284 |    "cell_type": "markdown",
285 |    "metadata": {},
286 |    "source": [
287 |     "## Convergence plots for PI and HI\n",
288 |     "\n",
289 |     "Next, we show convergence of PI and HI. This allows to observe the convergence of the used estimator for given data."
290 |    ]
291 |   },
292 |   {
293 |    "cell_type": "code",
294 |    "execution_count": 7,
295 |    "metadata": {},
296 |    "outputs": [],
297 |    "source": [
298 |     "N_s= 1000\n",
299 |     "n_fold = 10\n",
300 |     "N_av = 50\n",
301 |     "\n",
302 |     "def keep_measures(p_k,data_orig,I):\n",
303 |     "    \"\"\" p_k: discrete probability distribution if K\n",
304 |     "        data_orig: set from which we select samples\n",
305 |     "        I: the indexes of the measures to keep\n",
306 |     "\n",
307 |     "        returns a slice of data_orig\n",
308 |     "    \"\"\"\n",
309 |     "    data = [data_orig[k][I] for k in range(len(p_k))]\n",
310 |     "    return data\n",
311 |     "\n",
312 |     "### Generate data\n",
313 |     "data=measure_data(p_k, norm_params,number_samples=100000)\n",
314 |     "\n",
315 |     "# data are exactly the one from the model, \n",
316 |     "# this can not be done in real settings because norm_params are unkown\n",
317 |     "MI = information(p_k,data,norm_params)"
318 |    ]
319 |   },
320 |   {
321 |    "cell_type": "code",
322 |    "execution_count": 9,
323 |    "metadata": {},
324 |    "outputs": [
325 |     {
326 |      "data": {
327 |       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAELCAYAAADHksFtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO29e3xcVb3+/16dJNO0SUqbNr0khbY0KaSFQBuIwdKiAoIXFK9wFOgRDt5AUDlH8ByVr8eDeD1cFEFRkIugcgDhJ4igvRgNgRIJtIUmtAltektJesmtaZNZvz92ZpgmM3uvvfeszM7Mevrar8ys51mf/Zm1npnVmbX3WkJKiYGBgYGBQTJMSHcCBgYGBgbBhhkoDAwMDAxsYQYKAwMDAwNbmIHCwMDAwMAWZqAwMDAwMLCFGSgMDAwMDGyRk+4EUo3p06fLefPmAXD48GHy8vJc1Veto6Jz0iTj3ZYHBbrz8xt/vPrBjstmTwTZD6paL31ux/lpk5deeuktKeWMhKSUMqOOZcuWyShWr14t3UK1jorOSZOMd1seFOjOz2/88eoHOy6bPRFkP6hqvfS5HeenTYD1MsnnqpAZdsNddXW1XL9+PQD79u1j6tSpruqr1lHROWmS8W7LgwLd+fmNP179YMdlsyeC7AdVrZc+t+P8tIkQ4iUpZXUiLqPnKLq7u7XVUdE5aZLxbsuDAt35+Y0/Xv1gx2WzJ4LsB1Wtlz6343S1d1oHCiHEeUKIzUKIN4QQ19voPiaEkEKIhKNdMmzdutV1Tqp1VHROmmS82/KgQHd+fuOPVz/YcdnsiSD7QVXrpc/tOF3tnbbJbCFECPgpcA7QDrwohHhCSrlphK4Q+BLQMPZZGhgYGCTGkSNHaG9v59ChQwn5KVOm8Nprr9nGsNN44VTOOXHiRMrKysjNzbXVxSOdVz2dDrwhpdwKIIR4GPgQsGmE7r+B7wPXuT1B9OonHXVUdE6aZLzb8qBAd35+449XP9hx2eyJdPuhvb2dwsJC5s2bhxBilHZgYIBwOGwbz07jhXM6p5SSzs5O2tvbmT9/vm1u8UjnQFEKbI973g7UxAuEEKcCc6WU/58QIulAIYS4ErgSYM6cOaxZswaA2bNns2/fPpqamgAoLi5m8eLFrFu3DoCcnByWL19OY2MjBw8eBOCEE05gy5YtbN9upVZeXk44HGbDhg0AlJSUUFFRwdatW2lrayMcDlNbW8v69evp6ekBoKamhvb2drZv305bWxuLFi0iFAqxaZM1Bs6aNYv58+fHYuTn51NTU0NDQwP9/f1EIhFmz55Na2sru3fvBqCyspJIJBJ7baWlpZSVldHQYH3RKigooLq6mvr6egYGBgBYvnw5zc3NdHR0ALBkyRIGBgZoaWkBYO7cucycOZPo5H9RURFLly6lrq6OwcFBAFasWMHGjRvp7OwEoKqqiu7u7thX3Hnz5jFt2jQaGxuJRCIcOHCAqqoq1q5da10tIQQrV66kqamJffv2AbB06VK6urpoa2sDYMGCBRQWFjr20549e2J1qqur2bNnj2M/1dXVARAOh1m8eHHCftqxYwdAwn6aPn16rM1H9hNAbW0tra2t7Ny5k7a2NiorKxkaGmLz5s2j+ikSifDWW28l7afdu3fT1taWsJ8mT54cyyO+nw4fPkxbW5urfgKYOnWqtn7y8n6K76dk76eR/RR9/0TfT/X19Y795Pf9tG/fvlh7TJ8+nTlz5sTyzM3NJTc3l76+PgCEEITD4aPmDQoKCujv72doaCj2egcGBjh8+DAAeXl55OTk0NfXh5SSoaEhJk2adFSMwsJCBgcHY3UmTZoUex69OCkUCsVeeygUIj8/P5ZnXl4ehw4dOqqfqqsdftVPdjmU7gP4OHB33PNLgNvjnk8A1gDzhp+vAaqd4vq6PDYSCcTlkOZSSD3xzeWxY49Mvjx206ZNttqDBw86xrPTeOFUzill4tyxuTw2nZPZ7cDcuOdlwM6454XAEmCNEKINeAfwhNsJbWXs3AnveQ9FGzdqCW9gYGCQahQUFBz1/MEHH+Sqq64C4MYbb+SHP/xhSs6TzoHiRaBcCDFfCJEHXAQ8ESWllAeklNOllPOklPOA54ELpJTrVU/g6nriggJobWXx974Hw1/R/MZ20iTj3ZYHBbrz8xvfS33VOjr9YMdlsyeC7AewfvLxo/HCJZorSQXSNlBIKQeBq4BngNeA30kpNwohvi2EuCAV56iqqlIXFxXBffcRbm+HL385JbGdNMl4t+VBge78/Mb3Ul+1jk4/2HHZ7Ikg+wGsuQM/Gi+c0+S5V6T1Pgop5VNSygop5fFSyv8ZLvumlPKJBNqz3HybAFi7dq27hM48k20XXwx33w1PjErBdWwnTTLebXlQoDs/v/G91Feto9MPdlw2eyLIfoCxueGuv7+fU045JXZ84xvfUM7PDTJuUcB4SA/Lk7SuWsWxr70GV1wBr74KM2d6ju2kSca7LQ8KdOfnN76X+qp1dPrBjstmTwTKD9deCy+/fFRR/tAQOPz8ZKcJV1bCHXfY18/P5+W48955552xK8pSiYxewsPT73V5efDAA9DdDZdfDkmMoRLbSZOMd1seFOjOz298L/VV6+j0gx2XzZ4Ish8yDRm9KKAv3HYbXHMN3HknfPaz/uMZGBhkFF577TVOPPHEtOZQUFAQuz8C4N5772X9+vX85Cc/4cYbb6SgoIDrrht9C1qi3LN2UcDojUGe6lx1FZxzDnzlK9Dc7Cm2kyYZ77Y8KNCdn9/4vvyQAp1XP9hx2eyJIPsBiN1451XjhYveHJhqZPQcRfQOU091JkyAe+6Bk06CT38a/v53iFsbRSW2kyYZ77Y8KNCdn9/4vvyQAp1XP9hx2eyJIPsBiN197VWjwvWMuJT/X/7lXygsLASs+yhShYz+RuEbpaVw113w4ovwne+kOxsDAwOD9CDZLdvj9YhfwuPAgQOjblN3QsI6l1wiZSgkZX29q9hOmmS82/KgQHd+fuOnzA8edV79YMdlsyfS7QenJTwGBwcd49lpvHAq55RyfC3hoR1dXV2pqXP77VBWZv0ENfxVTyW2kyYZ77Y8KNCdn9/4KfODR51XP9hx2eyJIPsBiC2s6VXjhVM5pxdk9EARXeXRd50pU+C++2DrVmtyWzG2kyYZ77Y8KNCdn9/4KfODR51XP9hx2eyJIPsBiK3u6lXjhVM5pxdk9ECRUqxYAf/+7/CLXzjetW1gYGCQScjogWLBggWprfPtb0NVFVxxBQuHryzwc/5kvNvyoEB3fn7jp9wPLnVe/WDHZbMnguwHsPZ98KPxwqmc0wsyeqAoVPgwd1UnHIYHH4SDB5l57bXW0uQ+zp+Md1seFOjOz2/8lPvBpc6rH+y4bPZEkP0AY7N6bCgU4pRTTmHJkiV8/OMfj91HMXL5cb/I6IFCyw01ixfDnXcSamiAE06AW2+FJBNI5oa7YMU3N9yNPbL5hrvoDnNeNSpcdK2nDRs2kJeXx09/+lPl/NwgowcKbVi1ihfvuQfe+U5rMbDqavjHP9KdlYGBQRbjzDPPZMuWLVpiZ/RAUVxcrK3OpJNPhqeegv/7P+jstAaNK66At95SjpWMd1seFOjOz298nX5Q0Xn1gx2XzZ4Ish9gbDcuGhwc5Omnn+akk05Szs8NMnpRwEgkwoQJ7sZC1TpH6Xp6rInu//1fawOkm2+Gyy8nAraxkp3LbXlQoDs/v/HHzA8eNXa88UTqY/v1Q/zCetf+6Vpe3v2yXVXXqJpVxa3n3ZqQk1IihCAUCsUGhzPPPJMf/vCHhMPhUYsFjoRZFDAO69at01bnKF1BAXz/+9Z69EuWwJVXwhln0PjLX3o6l9vyoEB3fn7jj5kfPGrseOOJ1MfW6Qfwv9bTkcNHknLRQSA6R/Hyyy9z++23a7uPIqMXBRxzLF4Ma9ZY+1lcdx3LPvc5a/Oj//5v66Y9AwODjMQt590yqqy7u9vxKik7jcoOeWOFjP5GkZPjfhxUrZNUJwRccgls3syuD38YfvITWLTIuqx2xM98yWK4LQ8KdOfnN35a/OBCY8cbT6Q+tk4/ZBoyeo4iEHjpJfj8560VaM86y9raMM2bnRgYGPhHEDYu8gozRxGHxsZGbXVUdI2NjbBsGdTXWzvlNTXBySfDDTdAb2/SGG7LgwLd+fmNHwg/eOSNJ1IfW6cfAHp7e31pvHAq5/SCjB4oDh48qK2Oii6mCYWs7VQ3b7Z+lrr5ZqisZPJDD0GCm2qSxfbyesYSuvPzGz8wfvDAG0+kPrZOP4B1hZQfjRdO5ZxekNEDReAwYwb86lfwt7/BjBks+uEP4bjj4Fvfgj170p2dgYGBQWIk26hivB7xGxd1d3crbeIRD9U6KjpbTSQi+556SsoPflBKkDIvT8rPfEbKV19NWs/L6xlL6M7Pb/xA+8GBN55IfWy/fjAbF2UI9nj4X7pqHRWdrUYIdlZUWEuWb95s3dX90ENw0klMOO88+NOfRl0l5eX1jCV05+c3fqD94MAn47LZE0H2A8CRI8nvg1DReOFUzukFGT1QbN++XVsdFZ2TJsZXVMBPfwrbt8NNNxF6/XU4/3zr5r1f/CI2j+Hl9YwldOfnN/648YMLLps9EWQ/gBkoDHShuBhuuIHnH3rI2lEvL8+6y/vYY+Fb3yI34NteGhgYjC2EEFxyySWx54ODg8yYMYMPfOADANx7771cddVVvs+T0QNFeXm5tjoqOidNMn5hZaV1dVRjI6xeDbW18O1vc8bFF8Pll8OGDUo5jjW8tPdYxh+vfrDjdLe5X+jML8h+AAiHw740KtzkyZPZsGFDbNnxv/3tb5SWlirnqIqMHihUOsprHb8msONj5UJYN+kNz2Mc+tSnYvMYnHtuwnmMdMJLe49l/PHqBztOd5v7hc78guwHsP6370ejyp1//vn88Y9/BOB3v/sdF198sXKOqsjogWKDh/95q9ZR0TlpkvEJyysqaLj00tg8Bhs2WPMYixdb92WsWQNpXhvGS3uPZfzx6gc7Tneb+4XO/ILsB4BDhw750qhyF110EQ8//DCHDh3ilVdeoaamRjlHVWTnwiXjGcPzGHz1q/C738Ett1jPASZMgMpKqKmxjtNPtwaSLF2fxsBgLHHWWUc/HxrK5+KL4QtfgL4+eN/7Rte56KIcPvc5axubj33saO7JJ9XOe/LJJ9PW1sZDDz3Eueee6yl3J6T1E0QIcR5wKxAC7pZS3jyC/wpwBTAI7AU+I6V8UzV+SUmJ65xU66jonDTJeKXyvDz49Ket46234IUXrKOhAR57DKJLnE+aZO3AFx04amqgrMz6WSvF8NLeYxl/vPrBjtPd5n6hM78g+wHUfnqaMCH55kRuFom84IILuO666/jTn/6kZxmPZDdY6D6wBoctwAIgD2gCKkdo3gVMGn78eeC3TnHjb7g7cuSI0s0n8VCto6Jz0iTj3ZaPQiQiZXOzlA88IOXVV0t5+unWDX3WjIaUs2dL+aEPSXnTTVL+5S9SHjigFtcBXtp7LOOPVz/Ycbrb3C905pduPzjdcBeJRBzj2WlUuMmTJ0sppdy+fbu85ZZbZCQSkatXr5bvf//7pZRS3nPPPfKLX/ziqPrj6Ya704E3pJRbpZSHgYeBD8ULpJSrpZR9w0+fB8rcnKCurs51Uqp1VHROmmS82/JREALKy+FTn4LbbrO+ZRw8aP297TZ497th40b4+tfhPe+BY46xfqK6/HK4+26L87BmjJf2Hsv449UPdpzuNvcLnfkF2Q+A7Q5zKho3XFlZGddcc43SOb0gnT89lQLxd6+0A3azMJcDT2vNKJMRDls/PZ1+Olx9tVXW2Wktf97QYB2PP26tRQXWRkvveId1aW5trfWTldl8ycAgUEg0MJx11lmcNTxhsmrVKlatWuX7PGnbj0II8XHgvVLKK4afXwKcLqW8OoH208BVwEop5UAC/krgSoA5c+Yse/DBBwFrJcVTTz2VpqYmwNoYffHixbHtDHNycli+fDmNjY2xVSFDoRBz5syJ3YFZXl5OOByOXe1QUlJCRUUFzzzzDJMnTyYcDlNbW8v69etjnVZTU0N7ezvNzc1MnjyZRYsWEQqF2LRpEwCzZs1i/vz5PPfcc0yePJn8/HxqampoaGigv7+f3t5ezj77bFpbW9m9ezcAlZWVsfoApaWllJWV0dDQAEBBQQHV1dXU19czMGA10fLly2lubqajowOAJUuWMDAwQEtLCwBz585l5syZRPfvKCosZGlBAc2//jUFr75K0caNTG5rQ0iJFIK+444j58wz6TnpJLaUlNA3dy7zFixg2rRpNDY20tvbS1lZGVVVVaxduza2r+/KlStpampi3759ACxdupSuri7a2toAWLBgAYWFhY799Mwzz8QuT6yurmbPnj2O/RT9H2C0Xm5u7qh+2rFjB0DCfurs7Izd7TqynwBqa2tpbW1ly5YtTJ48mcrKSoaGhti8efOofurt7WXmzJlJ+6m1tZXJkycn7KedO3fGts0sKipi6dKl1NXVceDAASZPnsyKFSvYuHEjnZ2dAFRVVdHd3c3WrVsBmDdvXqyfAKZOnaqtn7y8n+L7Kdn7aWQ/vfjii0yePDn2fqqvr3fsJ7/vp9WrV5ObmwvA9OnTqaioiPG5ubnk5ubS12f9CBKJRJgyZcpRO9UVFBTQ399/1BaoeXl5sS1M8/LyyMnJoa+vj6GhIfLy8pg0adJRMQoLCzlw4EBs7+5JkyYxODjI4cOHGRoaIj8/n1AoFHvtoVCI/Pz8owaV9vZ2+vv7Y/1UXV1NYWFh0v0o0jlQ1AI3SinfO/z8BgAp5XdH6M4GbscaJDqc4gZu46LxjgMHrEny+nrreP552L/f4qZOtb5p1NbCGWfAaaeZbx0GWQOzcdHY4EWgXAgxXwiRB1wEPBEvEEKcCtwFXKAySIyElwFDtY6KzkmTjHdbrhVTpsA558A3vwlPP239XLVpk3VV1Uc/at3XceONluaYY2DmTGvQuOQSq/yBB6wBpqPD982Bfl//ePWDHRf0/xTpzC/IfoDM2rgobXMUUspBIcRVwDNYV0D9Skq5UQjxbazZ9yeAHwAFwO+HLzXbJqW8QPUcXiZ2VOv4naiy492WjykmTLC2cj3xRPjMZ6yyAwegoYGtjzzCAilhyxZYt270PuGFhXD88UcfCxdaf8vKrA2ebOD39Y9XP9hxgfCEDXTmFwQ/RH+2S4Sgblzk5VektN5HIaV8CnhqRNk34x6fPeZJGbjHlClw7rlsy8tjQfxdR4cOQVubNXC88Yb1d8sW667yJ56A+JUuc3OtTZzmzYP580f/nTlzTF+SgYETJk6cSGdnJ8XFxUr3TAQBUko6OzuZOHGiq3ppm6PQhfg5iv7+fvLz813VV62jonPSJOPdlgcFrvIbGoL29rcHjy1boLXVGlhaW2Hv3qP1EycSOe44JixYkHgwKS52vIlwvPrBjssoT4xxbL9+OHLkCO3t7UmX2rD7tqGi8cKpnHPixImUlZXFJuWjsJujyOi1Hdrb212vEKlaR0XnpEnGuy0PClzlFwpZ3yCOO866r2MkenutQSM6cLS10ffqqxTs2WNNqA9fmRNDQYEVq6TE2nI2wdHR18dx1dXWoOLwM5fb16TTD3ZcRnlijGP7/XzIzc1l/vz5SbUtLS2O8e00XjiVc3pBRg8UO3bscN1oqnVUdE6aZLzb8qAgpflNnmzdBLh4caxo/Zo1sevDOXBg1EDCm29ak+b//Kf1jSR6ddYwjos+EAKmTRs9mBQVRe9djx0Tt2+H0tJR5cBRzyfv2AFz5lhLq0SPcPio55HWVmvl33hN3HHw1VdhYMAa9KJHOAxCGE9oiK3z80FVa6fxwulq74weKAwyGFOmQFWVdSTDkSPWVVp798LevWxcs4bFJSWx57Hj9dfhb3+zVt8V4qhjdiRiLao4ojz2M9fw4+mDg9Y8y+HDbx+Dg0els8jhJS1LVBgKQUEBtbm51jeh+EGkoIBF3d3WjZIFBdbgmpdn5Zub6+2vmyMU0rJmmEHwkNEDxaJFTm9N73VUdE6aZLzb8qBAd36u4+fmwqxZ1gFMO/FEmD3bVYi9u3YxW6FOZyJdJGINVsMDx57t25k5derRg8nAQOxx165dTJs4EXp6Rh1DHR1WvN5eq2z3bujpoeTgQfj7361BzsOyK74RCh09eEycaC1EmZ8PkyZRm5NjXTYdVzbqiC8Ph61+y821Br3o4wTPT5w+3fpmGS2LDuiK0Pn5oKq103jhdL0HM3qgCCn+Du2ljorOSZOMd1seFOjOz2/8MffDhAnWB9/wXeFiaMiaQ0mCwY6OpHxPRweTEnCdHR3WiqZSvj3oDA5aA1Siv3Zc9O/Q0NtaN8eRI1YOfX2xQxw4YH2ra28/qpy+vlHfuNwi4XVw8d92En0Dins+Hay+cdDFH8cMDlrf3Jy+aeXkUHjokDVI2sSf1NtrfVOM54cHvnB3t/WNN36wHD5yenqs/zREy4YHSF3vwYwbKDZvfntd+P378zjmGPjEJ+zXhF+1yjreegvOPdeqE4/Pfx4++Unr3rLo9rTR2GBtDfHBD1rn/uxn364X1fzXf8HZZ8PLL8O1147mb7rJukftH/+w1umLjw3WlhOnnAL337+LJ58c/WFx112waJG1fv2PfjT69d1/P8ydC7/9LfzsZ6P5Rx6B6dPh3nutYySeesr6z94dd1hbYIzEmjXW3299q5vXXjs6v/x86z49gP/+b/jLX46uW1wM//d/1uMbbrDuzYtHWZl1zx7AF75wmLfeOpqvqICf/9x6fOWV0Nx8NH/KKVb7AXzqU5KRe8/X1sJ3h9cC+OhHrc+0eCxY0Btb/ur882F4VYQYPvABuO462LRpE5/4xOi+ifdeIm+pem/Nmi3cccfo+Oeeu4evf72Ezc2Cz352InD0ZY/JvBfFTTfBGe9523sjEfXec8/Bd74zmnfy3heuqecTn6hN7D0Z4ZF7e5me38u9903g3t9Ptr4VSQkyAhHJU//vRSaFBrjjybn8rm6OVS4lRCR9fb387cuPMXRkgP9dfRpPby637hGQEikjhEOHueeCHzA0dITbXvgQ9e1LkIffXhE1X+zlR2fdiIwM8dOmK9nQuQQYXrlawvSJu/ja4i8TGRriZ83fYGtPJZFIBCGsFOZMbOGzx15FJDLEndt+wq5DC4m/hnRu/stcNPfLRAT8ou1+9h221jSVw1965hXU8/7jvo4E7ml+hN7B4uGaQ8AQC45Zx7uP/Q5SwL0bnuJIxLraSgKTj8C/9t3FdViNfhZrQAgGJ00ip9ryiZvPvZF7YYxExg0UBgYG6ojICAODR+g9PMiRSB5SRogMf9BGkLy2dxuH2g/wyp5J7Ds0dxT/yMa/MfVgB42vl7LtwKnDvMVJGeHnW3/OU4/fRUvDUpr3vpuIlERkJBbn7Mf+AzG5k7c2vJ+u/Rcghz+oo//mbbqSoZxu+vas4vChD8c4pIQcmNjzNeuFyK8CE0FgHQC5/cwvvt96XLgQwrOOfvGTDvCOE/9uPd51AQzMO5ov6uYvy1utx309sPvwUfTG4iM8+8Hhxa2fiMCI/2S8PguePX/4yf8BB4/m35gLz0XvFDsA9B3Nb10Az60cfrILiPtPzrRIPpz1fjhjqvVN7p65ICVHhiJaPtQz+j6K119/nRNOOMFVfdU6KjonTTLebXlQoDs/v/HHox8GI4N0D3TTuKmRGWUzOHDoAAcHDnJgwPq7pX0LBccU0D/Yz6HBQ/Qf6ad/cPg4kvhvvG4w4u/nn0QIiRDhnDB5oTxyyGFSeBJ5oTzCoTDhnDDhkMXlhfLIDeUSEiFyJuTEjtCE4edixPMoP6zv6upidslsQiJEaELoqL/RenZlO3fs5Ni5xzJBTIgdAnHU8wliAkK8XbbtzW3MnzcfIQQhEbLVtm5tpXxhecLYUV1LSwuLKhYhEAghYn8niAk0b27mhBNOOIqL1t38+uaE60z5eY/Y3UeR0QPFwMCA6w3UVeuo6Jw0yXi35UGB7vz8xh9LP0gpOTR4iAMDBzhw6AAHBg6wt3svhyJHl8V/8O/v30/3kW7r+XB57xG1tXtyJ+SSn5tPfk5+wr8TcyYeXTb8OP5DO9ER/cBPdOROyI198Ed14VCYUNyubTo9EWQ/qGrtNF44P22StTfc1dfXv33dfYrrqOicNMl4t+VBge78/Mb36oczV5zJ3r697OnZQ0dvB3t697CnZw97evewt28v+w/t583dbyLyxVEDwJHIEcf4BXkFTAlPoShchBgQlJWUcdyU4ygKF8XKp0ycws7WndRU1cSeR/mXX3iZc9917lEfzkGCTk+kyw+qdfx+RnjhdLV3Rg8UBgZ26B7o5o2uN9jVs+voQSBuIGjf186BtQes38VHIBwKUzK5hGMmHoOMSI4tOJZFxYuYEp7ClIlTRv3dumkrK9+xMlZWFC466gN+TfwNhSOw5tAazqoczW0ObQ7sIGGQOcjogcLLOjCqdVR0TppkvNvyoEB3fl7iDwwOsHXfVpo7m3lu93M8+MSDtHS10NzZzK6eXaP0k3InMXPyTGYWzOT4qcezMG8hJy04KVYW/7coXBRbV6ehoYGaGrsNGqFhXwNVs5LfIGj3+ownUh9b5+eDqtZLn9txuto7o+coDLIDERlh+4HtNHc2v310WX/b9rcRkW/fiDZj0gwqiiuoKK6gfFo55cXllBaWxj78J+dNTuMrMTBIH+zmKGLXFGfKsWzZMhnF888/L91CtY6KzkmTjHdbHhTozC8Sicg/rv2jXNe2Tt790t3ya89+TV748IVyyR1L5MTvTJTcSOyY/D+T5al3nio/+ftPym/89Rvy/qb7ZUN7g/zz3/7s+rxB8IMdl82e8Btb5+eDqtZLn9txftoEax+ghJ+rGf3TU//Iu6NSWEdF56RJxrstDwpSkV9XfxctndZPQ9GfiFq6WmjpbKH78Nv7BudOyOX4acdTPq2c844/z/qGUFxORXEFswtmJ1xqeU3LGtf5BMEPdlw2eEJXbJ2fD6paL31ux+lq74weKAyCiyNDR1j35jrq2+vfHhA6W+jsf/uupQliAvOOmUf5tHjMswwAACAASURBVHLeOfed0AXvr3k/5cXlHDvlWHImGPsaGIwFMnqOwtxHMbZwyq/vSB9/3vJnHnv9MZ7c/CT7Dll7SpQWlsbmDOK/Gcw/Zj7hnLfjBfm6eZ1+sOPGuyfSGdvcR3E07OYoJniKOE7Q2tqqrY6KzkmTjHdbHhQkym9f/z7ub7qfj/z2I8z4wQwu/O2FPLH5CT5Q8QEe/cSjHLz+IO1faeevl/2Vuz54F18946tcsOgCTph+wlGDRLL4fvNLVR2dfrDjxqMnghJbpx9UtV763I7T1d4ZPVDs3r1bWx0VnZMmGe+2PCiI5rezeyd3vHgH59x/DiU/LOHSxy+lYUcDq6pW8ewlz9JxXQf3XXgfF554IYXhQtfx/eano45OP9hx48UTQYyt0w+qWi99bsfpam/zI69BStDS2cJD2x7i+ruvp2FHAwAVxRV8tfarXHjChZxWehoTREb/v8TAIGOR0QNFZWWltjoqOidNMt5tebqwae8mHnr1IR57/TE27t0IwLLZy/jOu77DhSdeyInTT3Tc6N0N/L7+8eoHOy5onhgJnfkF2Q+qWi99bsfpau+MHiiGhoa01VHROWmS8W7LxxJ7e/fy0IaHuK/pPl7a9RITxARWHLeCW5fdSu20Wk4rP03buf2+/vHqBzsuCJ6wg878guwHVa2XPrfjdLV3Rv8WsHnzZm11VHROmmS823LdGBgc4JFNj3DBQxcw58dzuOZP1yCR3PLeW9j5lZ2svmw1X6r5Er071FY69Qq/r3+8+sGOS5cnVKEzvyD7QVXrpc/tOF3tndHfKAy8Q0rJ8+3Pc1/TfTy88WH2H9rPnMI5fOUdX+GSqktYUrIk3SkaGBiMETJ6oCgtLdVWR0XnpEnGuy1PJVr3tfLAKw9w3yv38UbXG+Tn5POREz/CZVWX8e7577ZdqVR3fn7jj1c/2HFj4Qk/0JlfkP2gqvXS53acrvbO6Bvu+vv7Xa+mqFpHReekSca7LfeLA4cO8MimR7jvlftY9+Y6AN41711cWnUpHz3xo8qXsOrKL1Xxx6sf7Djdbe4XOvMLsh9UtV763I7z0yZZe8NdQ0ODtjoqOidNMt5tuRcMRgZ5uuVpLv6/i5n1o1lc8eQV7OnZw/+8+39ou6aNv172V1adssrVfQ6pzE9H/PHqBztOd5v7hc78guwHVa2XPrfjdLV3Rv/0ZJAYGzo2cOFvL+SNrjeYlj+Ny0+9nEurLuW0Oael9HJWAwODzEBGDxQFBQXa6qjonDTJeLflbvCH1//Apx/7NIV5hTzy8Uf44KIPkhfK8x0XUpOfzvjj1Q92nO429wud+QXZD6paL31ux+lqb6U5CiHER4DvASWAGD6klLJIS1Y+YDYuSgwpJd+t+y7/9df/onpONY9f9DhzCuekOy0DA4OAIBVzFN8HLpBSTpFSFkkpC4M4SIxEfX29tjoqOidNMt5tuRP6jvTxL4/+C//51//k4pMuZu2qtVoGCa/5jVX88eoHO053m/uFzvyC7AdVrZc+t+N0tbfqQLFHSvlaqk8uhDhPCLFZCPGGEOL6BHxYCPHbYb5BCDHPTfyBgQHXOanWUdE5aZLxbsvt0H6wnRX3rOC3G37Lze+5mQcufID8XD1XoXjJbyzjj1c/2HG629wvdOYXZD+oar30uR2nq71V5yjWCyF+CzwOxDKRUj7q9cRCiBDwU+AcoB14UQjxhJRyU5zscmCflHKhEOIirJ+/Pun1nNmG59uf58LfXkjP4R7+cNEf+OCiD6Y7JQMDg3EI1TmKexIUSynlZzyfWIha4EYp5XuHn98wHPS7cZpnhjX1QogcYDcwQ9okHT9HMTg4SE6Ou/l61ToqOidNMt5teSLc33Q///bkv1FaVMoTFz3B4pLFSvX8wEt7j2X88eoHO053m/uFzvyC7AdVrZc+t+P8tInvOQop5b8mODwPEsMoBbbHPW8fLkuokVIOAgeAYtUTNDc3u05KtY6KzkmTjHdbHo+hyBD/8ex/cOnjl3LG3DN44YoXxmSQAG/tPZbxx6sf7Djdbe4XOvMLsh9UtV763I7T1d5KQ48Qogy4HXgnIIE64BopZbuPcye6YH/kNwUVDUKIK4ErAebMmcOaNWsAOHz4MLNnz6apqQmA4uJiFi9ezLp11t3HOTk5LF++nMbGRg4ePAhYqy+Gw2G2b7fGsPLycsLhMBs2bACgpKSEiooKtm7dSkdHB+FwmNraWtavX09PTw8ANTU1tLe3xzSLFi0iFAqxaZP1q9qsWbOYP39+jM/Pz6empoaGhgb6+/vp6enh+OOPp7W1NbYRSWVlJTt27KCjowOwbtUvKyuL3WBTUFBA+ZJyzv/l+dS/Vc+H53yY31z0G1q3tPJqx6sALFmyhIGBAVpaWgCYO3cuM2fOJPoNrKioiKVLl1JXV8fg4CAAK1asYOPGjXR2WntZV1VV0d3dzdatWwGYN28e06ZNo7GxkZ6eHo4cOUJVVRVr165FSokQgpUrV9LU1MS+fdbWp0uXLqWrq4u2tjYAFixYQGFhoWM/tbW1xV5/dXU1e/bsceynuro6AMLhMAMDA/T19Y3qpx07dgAk7Kfdu3fHzjmynwBqa2tpbW2N9WVlZSVDQ0Oxxdni+6mnp4e+vj6qq6upr6+P/Z68fPlympubYzES9dOuXbtiecT30/79++no6HDVTwBTp07V1k9e3k/x/ZTs/TSyn6LtFX0/RSdy7frJzfspUT9t27YtVsfp/dTX10dlZaXt+2lwcJCCggLX76c333wzlkd8P/X09FBQUOC6n6qrE36ReBtSSscDeBb4V6yBJQdYBTyrUtcmZi3wTNzzG4AbRmieAWqHH+cAbzH8c1myY9myZTKK1atXS7dQraOic9Ik492WSyll81vN8oSfnCBzvp0j73zxTsfcdMBLe49l/PHqBztOd5v7hc78guwHVa2XPrfj/LQJsF4m+7xORsijP7BfVilzcwx/8G8F5gN5QBOweITmi8Cdw48vAn7nFDd+oNi7d6/rxlKto6Jz0iTj3ZY/u+VZOfXmqbL4e8VyTesax7x0wUt7j2X88eoHO053m/uFzvyC7AdVrZc+t+P8tIndQKF6eexbQohPCyFCw8engU7FugkhrTmHq4a/Nbw2PAhsFEJ8WwhxwbDsl0CxEOIN4CvAqEto7TBeL4d0U/6Ll37BeQ+cR2lRKS/+24usnLfSMS9dMJfH+tME6XLIVMFcHutdEyQ/qA4UnwE+gXXV0S7gY8NlviClfEpKWSGlPF5K+T/DZd+UUj4x/PiQlPLjUsqFUsrTpZRb3cSP/naoo46KzkmTjFctX79zPZ//4+c55/hz+Mdn/sH8qfMdc9IJL+09lvHHqx/sON1t7hc68wuyH1S1XvrcjtPV3kqT2VLKbcAFjkKDMUP/kX4ueewSZhXM4jcf+Y2rVV4NDAwM3MB2oBBC/IeU8vtCiNtJcLWRlPJL2jJLAebOnautjorOSZOMVym//rnref2t13n2kmeZmj/VMZexgJf2Hsv449UPdpzuNvcLnfkF2Q+qWi99bsfpam+nbxTRZTvG5Sp7M2fO1FZHReekScY7lT+39Tlue+E2rj79as5ecLZjHmMFL+09lvHHqx/sON1t7hc68wuyH1S1XvrcjtPV3rZzFFLKJ4cf9kkpfx1/AH1aMkohvKwiq1pHReekScbble8/tJ9//cO/sqh4ETeffbNzomMI3av2+o0/Xv1gxwV9pWSd+QXZD6paL31ux+lqb9XJ7BsUyww046qnrmJX9y7uv/B+JuVOSnc6BgYGWQCnOYrzgfcBpUKI2+KoImBQZ2KpQFGR+5XQVeuo6Jw0yfhk5Q09DTz46oPcuPJGTis9zTnJMYaX9h7L+OPVD3ac7jb3C535BdkPqlovfW7H6Wpv20UBhRBVwCnAt4FvxlHdwGop5T4tWflApm5ctKt7F0t+toTjpx7P3z/zd3JDuelOycDAIIPgeVFAKWXT8HzEwhFzFI8GcZAYiej6MTrqqOicNMn4keVSSi5/4nJ6B3q5/8L7AztIeGnvsYw/Xv1gx+luc7/QmV+Q/aCq9dLndpyu9lZdj3aeEOK7QCUwMVoopVygJasUIboQl446KjonTTJ+ZPldL93F0288zZcWfolF0xcp5ZcOeGnvsYw/Xv1gx+luc7/QmV+Q/aCq9dLndpyu9ladzL4H+BnWvMS7gPuA+7VkZHAUWjpb+Oqfv8o5C87hQ3M+lO50DAwMshCqGxe9JKVcJoR4VUp50nDZ36SUZ2rP0CXi5ygikQgTJqiOhbiqo6Jz0iTjo+WDkUHOvOdMXn/rdTZ8fgOzC2a7fj1jCS/tPZbxx6sf7Djdbe4XOvMLsh9UtV763I7z0ya+Ny4CDgkhJgAtQoirhBAXAiWeshlDbNy4UVsdFZ2TJhkfLf9e3fd4vv157njfHZQWlXp6PWMJ3fn5jT9e/WDHZbMnguwHVa2XPrfjdLW36kBxLTAJ+BKwDLgEuExLRilEdHMQHXVUdE6aZHxnZyeNuxq5ce2NfHLxJ7n4pItd5ZYu6M7Pb/zx6gc7Lps9EWQ/qGq99Lkdp6u9VRcFfHH4YQ/WBkYGGnE4cphLHruEkskl3PH+O9KdjoGBQbYj2UYV8ugNhKqBx4BG4JXooVJ3rI/4jYu6uroctuoYDdU6KjonTTL+849/XnIj8k8tf/KUW7qgOz+/8cerH+y4bPZEkP2gqvXS53acnzYhBRsXPYh15dNHgQ/GHYFGd3e3tjoqOidNIn5162p+9vLP+EL1F3jvwvd6yi1d0J2f3/jj0Q9OXDZ7Ish+UNV66XM7Tld7qw4Ue6WUT0gpW6WUb0YPLRmlENENy3XUUdE5aUbyBw4d4LLHL6Msv4zvn/N9z7mlC7rz8xt/vPlBhctmTwTZD6paL31ux+lqb9Ub7r4lhLgb+AsQ22tPSvmolqyyFF/605fY2b2T2065jcl5k9OdjoGBgQGgPlD8K3ACkAtEhsskEOiBYt68edrqqOicNPH8o689yn1N9/GNFd/gffPf5yu3dEF3fn7jjyc/qHLZ7Ikg+0FV66XP7Thd7a06UFTJ4RvtxhOmTZumrY6KzkkT5Xf37ObKJ69k2exlfGPFN+jv7feVW7qgOz+/8ceLH9xw2eyJIPtBVeulz+04Xe2tOkfxvBCiUksGGtHY2KitjorOSdPY2IiUkiueuILeI28v+JesnpfXM5bQnZ/f+OPBD265bPZEkP2gqvXS53acrvZW/UaxHLhMCNGKNUchACmlPFlLVlmEuxvv5o8tf+SW997CiTNOTHc6BgYGBqOgOlCcpzULTZg6daq2Oio6J01PXg9ffubLvGf+e7i65mrHel5ez1hCd35+4wfdD3a88UTqY+v0g6rWS5/bcbra23FRwOE1nl6RUi7RkkGKMV42LhqKDLHy3pVs6NjAq59/lblT5qY7JQMDgyyGr0UBpZQRoEkIcWzKM9OMtWvXaqujorPT/OAfP+Dv2//OT973k1GDRLJ6Xl7PWEJ3fn7jB9kPTrzxROpj6/SDqtZLn9txutpb9aen2cBGIcQLQG+0UEp5gZasUgSnb0t+6qjokmna9rfxzdXfZOX0lXzqpE8p1/PyesYSuvPzGz+oflDhjSdSH1unH1S1XvrcjtPV3qoDxf/TcnbNEEJoq6OiS6b5/t+tu66/uPCLCTXJ6nl5PWMJ3fn5jR9UP6jwxhOpj63TD6paL31ux+lqb6WNi4YTmAmcNvz0BSllh5aMfCLocxS7uncx/9b5XFp1KT//4M/TnY6BgYEBkIKNi4QQnwBeAD4OfAJoEEJ8LHUp6kFTU5O2Oiq6RJof1/+YI5Ej/Mc7/yNpDLflQYHu/PzGD6IfVHnjidTH1ukHVa2XPrfjdLW36k9P/wmcFv0WIYSYATwHPKIlqxRh37592uqo6EZquvq7+Nn6n3HRkotYOG0ha15Z4yq2l9czltCdn9/4QfODG954IvWxdfpBVeulz+04Xe2temf2hBE/NXW6qGswjNsabqP3SC83LL8h3akYGBgYKENpjkII8QPgZOCh4aJPYt1b8TWNuXlC/BzFwYMHKSoqclVftY6KLl7TPdDNcbccx4rjVvD4RY/bxnBbHhTozs9v/CD5wS1vPJH62Dr9oKr10ud2nJ828TxHIYQIA0gp/x24C2uwqAJ+7meQEEJME0I8K4RoGf476nZCIcQpQoh6IcRGIcQrQohPuj1PV1eX69xU66jo4jV3rr+TfYf28fUzv+4Yw215UKA7P7/xg+QHt7zxROpj6/SDqtZLn9txutrb6eejegAhxP1SykellF+RUn5ZSvmYz/NeD/xFSlmOtcfF9Qk0fcClUsrFWEuI3CKEOMbNSdra2lwnplpHRRfVHBo8xI/qf8TZC87m9NLTHWO4LQ8KdOfnN35Q/OCFN55IfWydflDVeulzO05XeztNZucJIS4DzhBCfGQk6WPjog8BZw0//jWwBjjqG4qUsjnu8U4hRAcwA9jv8Zxpw6/++Sv29O7hoeUPOYsNDAwMAgangeJzwKeAYxi9R7afjYtmSil3AUgpdwkhSuzEQojTgTxgi5uTLFiwwHViqnVUdAsWLODI0BG+//fvU1tWy1nzzlKK4bY8KNCdn9/4QfCDV954IvWxdfpBVeulz+04Xe2tOpl9uZTyl64CC/EcMCsB9Z/Ar6WUx8Rp90kpEy57KISYjfWN4zIp5fNJNFcCVwLMmTNn2YMPPgjAzJkzmTVrVuza4uLiYhYvXsy6desAyMnJYfny5TQ2NnLw4EEAKioq6O/vZ/v27QCUl5cTDofZsGEDACUlJVRUVLB27VpCoRDhcJja2lrWr19PT08PADU1NbS3t7Nt2zae3fss39v8Pe4//37K+soAmDVrFvPnz6euro5QKER+fj41NTU0NDTQ39/P0NAQy5cvp7W1ld27dwNQWVnJwYMHaW9vB6C0tJSysjIaGhoAKCgooLq6mvr6egYGrN1qly9fTnNzMx0d1gVrS5YsYWBggJaWFgDmzp3LzJkziU7+FxUVsXTpUurq6hgcHARgxYoVbNy4kc7OTgCqqqro7u6O7c07b948pk2bRmNjI0NDQ0yfPp2qqirWrl2LlBIhBCtXrqSpqSl26d7SpUvp6uqKfU1esGABhYWFjv30/PPPc+jQIQCqq6vZs2ePYz/V1dUBEA6HOeGEE9iyZcuoftqxYwcAixYtIhQKsWnTplg/TZ06lddeew1gVD8B1NbW0trayo4dOwiFQlRWVjI0NMTmzZtH9dPQ0BBTpkxJ2k+7du0iFAol7Kf8/Hyam5tH9dPAwAChUMhVP4G1yqiufvLyforvp2Tvp5H99OqrrxIKhWLvp/r6esd+8vt+euWVV2Kvzen9NHHiRN7xjnfYvp/mz5+PEML1++mFF16gr69vVD8NDQ1RXl7uup+qq6spLCxMOpmNlFLpAM4A/gW4NHqo1k0QazMwe/jxbGBzEl0R0Ah8XDX2smXLZBSrV6+WbqFaR0X33F+fk4tuXyRP/tnJMhKJKMdwWx4U6M7Pb/x0+8FJY8cbT6Q+tk4/qGq99Lkd56dNgPUyyeeq0g13Qoj7geOBl4Gh6BgD3KdSPwGeAC4Dbh7++4cE58wDHgPuk1L+3uN50oq6t+rY3LmZhz/6cODX5DEwMDBIBtU7s6uByuFRJxW4GfidEOJyYBvW0iAIIaqBz0kpr8BaKmQFUCyEWDVcb5WU8mXVkxQXF7tOTLWOk05KycM7HqZ8Wjkfq0y82kmyGG7LgwLd+fmNn04/qGjseOOJ1MfW6QdVrZc+t+N0tbfqHMXvgS/J4QnoICP+hrtIJMKECe5uIFet46R7uuVp3veb9/HLC37JZ079jKsYbsuDAt35+Y2fTj+oaOx444nUx9bpB1Wtlz634/y0ie9FAYHpwCYhxDNCiCeih6dsxhDRyRsddZx0N9XdREm4hE+f/GnXMdyWBwW68/MbP51+UNHY8cYTqY+t0w+qWi99bsfpam/Vn55u1HL2DMW6N9dRt62OqxdeTV4oL93pGBgYGPiC0kAhpQz2fotJkJOjOg66r2Onu+lvN1EyuYQPlX3IUwy35UGB7vz8xk+XH1Q1drzxROpj6/SDqtZLn9txutrbdo5CCNGNdXXTKAqQUsrArUaW7o2L1u9cz2m/OI3vvue7XL880cokBgYGBsGD5zkKKWWhlLIowVEYxEFiJKI3F+mok0z33brvMiU8hS+c9gXHWMl4t+VBge78/MZPhx/caOx444nUx9bpB1Wtlz6343S1d3Avl0gBoncd6qiTSLdp7yYefe1Rrj79aorCRY6xkvFuy4MC3fn5jT/WfnCrseONJ1IfW6cfVLVe+tyO09XeGT1QjDVurruZSbmTuOYd16Q7FQMDA4OUQek+ivGE+DmKnp4eCgoKXNVXrTNSt3XfVipur+BLNV/ix+/9sVKsZLzb8qBAd35+44+lH7xo7HjjidTH1ukHVa2XPrfj/LRJKu6jGJfYs2ePtjojdT/4+w8ITQjx1dqvKsdKxrstDwp05+c3/lj6wYvGjjeeSH1snX5Q1XrpcztOV3tn9EARXbFSR5143c7unfzq5V+xqmoVpUWlyrGS8W7LgwLd+fmNP1Z+8Kqx440nUh9bpx9UtV763I7T1d4ZPVCMFX5c/2OGIkN8bXngthA3MDAw8I2MHijKy8u11YnqOvs6uXP9nVx80sUsmLogocbtudyWBwW68/Mbfyz84EdjxxtPpD62Tj+oar30uR2nq70zeqAIh8Pa6kR1tzbcSu+RXq5/5+ib65xiJePdlgcFuvPzG38s/OBHY8cbT6Q+tk4/qGq99Lkdp6u9M3qgiO6ipaPOhg0bODhwkNtfuJ0LT7iQxSWLXcdKxrstDwp05+c3vm4/+NXY8cYTqY+t0w+qWi99bsfpau+MHih042cv/oz9h/bz9TO/nu5UDAwMDLQhoweKkpISbXWKiov48fM/5tzjz6V6TuJtZp1iJePdlgcFuvPzG1+nH1R0Xv1gx2WzJ4LsB1Wtlz6343S1d0bfcDc4OOh6NUXVOrfW38q1f76WNZetYeW8lZ5iJePdlgcFuvPzG1+nH1R0Xv1gx2WzJ4LsB1Wtlz634/y0SdbecFdXV6elzuGhw9y09ibeOfedrDhuhedYyXi35UGB7vz8xtflB1WdVz/YcdnsiSD7QVXrpc/tOF3tHdz/igQYv3n1N3QMdHDPmfcghEh3OgYGBgZakdHfKHRd/vbY649Rml/K+QvP9xXLXB47tvHN5bFjD3N5rHdNkPyQ0XMUujD3f+ey8riVPPCRB7Sex8DAwGCskLVzFF4GDKc6Hb0dtB9sZ8bgDN+xkvFuy4MC3fn5ja/DD250Xv1gx2WzJ4LsB1Wtlz6343S1d0YPFD09PSmv889d/wTguLzjfMdKxrstDwp05+c3vg4/uNF59YMdl82eCLIfVLVe+tyO09XeGT1Q6EDjLmurwYUFC9OciYGBgcHYIKPnKPr7+8nPz3dV36nOx3//cf6565+8+m+vOsZ2ipWMd1seFOjOz298HX5wo/PqBzsumz0RZD+oar30uR3np02ydo6ivb095XUadzWydPZSpdhOmmS82/KgQHd+fuPr8IMbnVc/2HHZ7Ikg+0FV66XP7Thd7Z3RA8WOHTtSWmdf/z627tvK0tlLlWI7aZLxbsuDAt35+Y2faj+41Xn1gx2XzZ4Ish9UtV763I7T1d4ZPVCkGi/vfhmApbOXpjkTAwMDg7FDRt+ZvWjRopTWiU5knzrrVAYnDfo+fzLebXlQoDs/v/FT7Qe3Oq9+sOOy2RNB9oOq1kuf23G62jujv1GEQqGU1mnc3cjcornMmDxDKbaTJhnvtjwo0J2f3/ip9oNbnVc/2HHZ7Ikg+0FV66XP7Thd7Z3RA8WmTZtSWuelnS/FfnZSie2kSca7LQ8KdOfnN36q/eBW59UPdlw2eyLIflDVeulzO05Xe2f0QJFKdA9009zZbOYnDAwMsg5pGSiEENOEEM8KIVqG/0610RYJIXYIIX7i9jyzZs1ynVuyOk17mpDI2EChEttJk4x3Wx4U6M7Pb/xU+sGLzqsf7Lhs9kSQ/aCq9dLndpyu9k7LDXdCiO8DXVLKm4UQ1wNTpZRfS6K9FZgxrL/KKXb8DXcDAwOuV1NMVue2htu45k/XsOMrO5hTOEcptpMmGe+2PCjQnZ/f+Kn0gxedVz/YcdnsiSD7QVXrpc/tOD9tEsQb7j4E/Hr48a+BDycSCSGWATOBP3s5SX19fcrqNO5qZFbBLOYUzlGO7aRJxrstDwp05+c3fir94EXn1Q92XDZ7Ish+UNV66XM7Tld7p2ugmCml3AUw/HfURq9CiAnAj4B/H+PcEiJ6R7aBgYFBtkHbfRRCiOeARD+Y/adiiC8AT0kptzvtIieEuBK4EmDOnDmsWbMmxu3bt4+mpiYAiouLWbx4MevWrQMgJyeH5cuX09jYyMGDBwHIzc1ly5YtbN++HYDy8nIioQgbOzZyysRT2LRpExUVFfT29rJmzRrC4TC1tbWsX78+tnJjTU0N7e3tMc2iRYsIhUKxKxJmzZrF/PnzY3x+fj41NTU0NDTQ399Pb28vAwMDtLa2snv3bgAqKysRQsReW2lpKWVlZTQ0NABQUFBAdXU19fX1DAwMALB8+XKam5vp6OgAYMmSJQwMDNDS0gLA3LlzmTlzZmxp4qKiIpYuXUpdXR2Dg9Z9IitWrGDjxo10dnYCUFVVRXd3N1u3bgVg3rx5TJs2jcbGRnp7e2lqaqKqqoq1a9cipUQIwcqVK2lqamLfvn0ALF26lK6uLtra2gBYsGABhYWFjv00MDAQe/3V1dXs2bPnqH4Kh8Ns2LABsDaZr6ioiG0NGQ6Hyc/PT9hP0btZE/VTOByOnXNkPwHU1tbS2toa68vKykqGhobYGhpx7QAAD0RJREFUvHnzqH7q7e1l/fr1SfspGiNRP+Xm5sbyiO+naB03/QQwdepUbf3k9H5y6qdk76eR/RR97dH3U/R/03b95Pf9NDg4GKvj9H6K1rN7P4VCIbZt2+b6/XT48OFYHvH91Nvby7Zt21z3U3V1wl+cYkjXHMVm4Cwp5S4hxGxgjZRy0QjNg8CZQAQoAPKAO6SU19vF1rFx0Qs7XqDm7hoe/cSjXHjihSmNbWBgYBAEBHGO4gngsuHHlwF/GCmQUn5KSnmslHIecB1wn9MgMRLR/x34rRO9Izv+pyeV2E6aZLzb8qBAd35+46fKD151Xv1gx2WzJ4LsB1Wtlz6343S1d7oGipuBc4QQLcA5w88RQlQLIe5O1UmiXzv91mnc1ci0/GkcO+VYV7GdNMl4t+VBge78/MZPlR+86rz6wY7LZk8E2Q+qWi99bsfpau+0rPUkpewE3pOgfD1wRYLye4F7tSeWBNGJbKe5EgMDA4NMREZvXJSK66QPDx2m8LuFXFtzLd8753tJdSqxVHlzzbye+OY+irGHuY/C3EcReLS2tvqus2nvJg4PHR51aaxKbCdNMt5teVCgOz+/8VPhBz86r36w47LZE0H2g6rWS5/bcbraO6MHiuilcH7qJJrIVo3tpEnGuy0PCnTn5zd+KvzgR+fVD3ZcNnsiyH5Q1XrpcztOV3tn9ECRCry08yUK8wo5ftrx6U7FwMDAIC3I6IGisrLSd53G3Y2cOvtUJogJtjov50/Guy0PCnTn5zd+KvzgR+fVD3ZcNnsiyH5Q1XrpcztOV3tn9EAxNDTkq85gZJCm3U0snTV66Q6V2E6aZLzb8qBAd35+4/v1g1+dVz/YcdnsiSD7QVXrpc/tOF3tndEDRXQZBa91Nr+1mf7B/oRrPKnEdtIk492WBwW68/Mb368f/Oq8+sGOy2ZPBNkPqlovfW7H6WrvjB4o/CLZRLaBgYFBNiGjB4rS0lJfdRp3NZKfk8+i6aM3LFeJ7aRJxrstDwp05+c3vl8/+NV59YMdl82eCLIfVLVe+tyO09XeGX3DXX9/P/n5+a7qx9dZee9Kjgwd4R+X/8NWpxLLDe+2PCjQnZ/f+H794Ffn1Q92XDZ7Ish+UNV66XM7zk+bZO0Nd34W/YrICP/c9c+kPzuZRQFHwywK6E8TpEXgUgWzKKB3TZD8kNEDhR9s6dpC9+FuMz9hYGCQ9cjogaKgoMBzHaeJbJXYTppkvNvyoEB3fn7j+/FDKnRe/WDHZbMnguwHVa2XPrfjdLV3Rs9R+MHXnv0atzTcQvcN3eSF8lKQmYGBgUFwkbVzFH42T2/c3chJJSclHST8bpxux4/1xumpgu78/Mb344dU6Lz6wY7LZk8E2Q+qWi99bsfpau+MHiiie9a6rSOljO1B4Se2kyYZ77Y8KNCdn9/4Xv2QKp1XP9hx2eyJIPtBVeulz+04Xe2d0QOFV2w7sI2u/i4zkW1gYGBAhs9RDA4OkpPjbhO/wcFBnmx5ko/87iM0XNHA6aWnJ9U5xXbSJOPdlgcFuvPzG9+rH1Tq6PSDHZfNngiyH1S1XvrcjvPTJlk7R9Hc3Oypzku7XiIkQpxUcpKv2E6aZLzb8qBAd35+43v1Q6p0Xv1gx2WzJ4LsB1Wtlz6343S1d0YPFB0dHZ7qNO5qpHJGJfm5ye9wVIntpEnGuy0PCnTn5ze+Vz+kSufVD3ZcNnsiyH5Q1XrpcztOV3tn9EDhBVJKXtr1kpmfMDAwMBhGRg8US5YscV1nxoIZdPR2OA4UKrGdNMl4t+VBge78/Mb3Ul+1jk4/2HHZ7Ikg+0FV66XP7Thd7Z3RA4WXS8Ve2vkS4Ly0uLk8djTM5bH+NEG6HDJVMJfHetcEyQ8ZPVC0tLS4rrNm8xoEgqqZVb5jO2mS8W7LgwLd+fmN76W+ah2dfrDjstkTQfaDqtZLn9txuto7owcKL2jpaaGiuILCcGG6UzEwMDAIBDJ6oJg7d67rOlv7t7JszrKUxHbSJOPdlgcFuvPzG99LfdU6Ov1gx2WzJ4LsB1Wtlz6343S1d0YPFDNnznSl39u7l119u1g6y/mKJ5XYTppkvNvyoEB3fn7je6mvWkenH+y4bPZEkP2gqvXS53acrvbO6IHC7Sqy/9z9T0Btj2yV2E6aZLzb8qBAd35+43upr1pHpx/suGz2RJD9oKr10ud2nK72zuiBwi2ie1CcOvvUNGdiYGBgEBxk9EBRVFTkSt+4q5HSSaUcM/GYlMR20iTj3ZYHBbrz8xvfS33VOjr9YMdlsyeC7AdVrZc+t+N0tXdGLwroFgtvW8ips0/l9x//fYqzMjAwMAg2snZRwLq6OmXt/kP72bJvC9MGpqUstpMmGe+2PCjQnZ/f+F7qq9bR6Qc7Lps9EWQ/qGq99Lkdp6u90zJQCCGmCSGeFUK0DP+dmkR3rBDiz0KI14QQm4QQ89ycZ3Bw0FVe//ve/+W0Y05LWWwnTTLebXlQoDs/v/G91Feto9MPdlw2eyLIflDVeulzO05Xe6frG8X1wF+klOXAX4afJ8J9wA+klCcCpwPalqI8ZuIxXPuOa1lYsFDXKQwMDAzGJdIyRyGE2AycJaXcJYSYDayRUi4aoakEfi6lXO4mdvwcRSQSYcIEd2Ohah0VnZMmGe+2PCjQnZ/f+OPVD3ZcNnsiyH5Q1XrpczvOT5sEcY5ippRyF8Dw35IEmgpgvxDiUSHEP4UQPxBChNycZOPGja4TU62jonPSJOPdlgcFuvPzG3+8+sGOy2ZPBNkPqlovfW7H6WpvbXsoCiGeA2YloP5TMUQOcCZwKrAN+C2wCvhlgnNdCVwJMGfOHNasWQPA4cOHKSsro6mpCYDi4mIWL17MunXrrBPk5LB8+XIaGxs5ePAgAENDQ2zZsoXt27cDUF5eTjgcZsOGDQCUlJRQUVHBm2++SWdnJ+FwmNraWtavX09PTw8ANTU1tLe3xzSLFi0iFAqxadMmAGbNmsX8+fNjfH5+PjU1NTQ0NNDf309PTw8VFRW0traye/duACorK9m9ezednZ0AlJaWUlZWRkNDAwAFBQVUV1dTX18fW0Fy+fLlNDc3xzYzWbJkCQMDA7GFw+bOncvMmTNjN+kUFRWxdOlS6urqYr91rlixgo0bN8bOW1VVRXd3N1u3bgVg3rx5TJs2jcbGRnp6eohEIlRVVbF27VqklAghWLlyJU1NTezbtw+ApUuX0tXVRVtbGwALFiygsLDQsZ+2b98ey6O6upo9e/Y49lN0ci8cDjMwMJCwn3bs2AGQsJ/27t0b89PIfgKora2ltbU11peVlZUMDQ2xefPmUf3U09PDwMBA0n6KxkjUTx0dHbE84vtp//79dHZ2uuongKlTp2rrJy/vp/h+SvZ+GtlP0faKvp/q6+sd+8nv+2nHjh2xOk7vp76+Pk466STb99Pg4CBTpkxx/X5qb2+PxYjvp56eHqZMmeK6n6qrE36ReBtSyjE/gM3A7OHHs4HNCTTvwPpJKvr8EuCnTrGXLVsmo1i9erV0C9U6KjonTTLebXlQoDs/v/HHqx/suGz2RJD9oKr10ud2nJ82AdbLZJ/ZyQidB/AD4Prhx9cD30+gCQFNwIzh5/cAX3SKHT9QdHV1uW4s1ToqOidNMt5teVCgOz+/8cerH+y4bPZEkP2gqvXS53acnzaxGyjSNUdxM3COEKIFOGf4OUKIaiHE3QBSyiHgOuAvQohXAQH8ws1Juru7XSemWkdF56RJxrstDwp05+c3/nj1gx2XzZ4Ish9UtV763I7T1d5pGSiklJ1SyvdIKcuH/3YNl6+XUl4Rp3tWSnmylPIkKeUqKeVhN+eJ/u6no46KzkmTjHdbHhTozs9v/PHqBzsumz0RZD+oar30uR2nq72De12dgYGBgUEgkHFrPQkh9gJvDj+dAhxwGUK1jorOSZOMT1Y+HXhLIbd0wUt7j2X88eoHOy6bPRFkP6hqvfS5HefHD8dJKWckZJJNXmTCgXXDnpY6KjonTTLepjzpZFMQDi/tPZbxx6sfjCfGnx9S4Ykg+SHTf3p6UmMdFZ2TJhnvJe8gQHfefuOPVz+4ySNo0Jl3kP2gqvXa52Pqh4z76SmTIYRYL5PcYm+QnTCeMIiHLj9k+jeKTMPP052AQeBgPGEQDy1+MN8oDAwMDAxsYb5RGBgYGBjYwgwUBgYGBga2MAPFOIYQYoEQ4pdCiEfSnYtBMCCE+LAQ4hdCiD8IIc5Ndz4G6YUQ4kQhxJ1CiEeEEJ/3GscMFAGDEOJXQogOIcSGEeXnCSE2CyHeEEJcDyCl3CqlvDw9mRqMFVx64nEp5b9hLcn/yTSka6AZLv3wmpTyc8AnAM9XQ5mBIni4FzgvvmB4w6afAucDlcDFwzsAGmQH7sW9J/5rmDfIPNyLCz8IIS4A6rC2nfYEM1AEDFLKdUDXiOLTgTeGv0EcBh4GPjTmyRmkBW48ISx8D3haStk41rka6Ifbzwgp5RNSyjOAT3k9pxkoxgdKge1xz9uBUiFEsRDiTuBUIcQN6UnNIE1I6AngauBs4GNCiM+lIzGDtCDZZ8RZQojbhBB3AU95Da5tK1SDlEIkKJNSyk7AfBhkJ5J54jbgtrFOxiDtSOaHNcAav8HNN4rxgXZgbtzzMmBnmnIxCAaMJwziodUPZqAYH3gRKBdCzBdC5AEXAU+kOSeD9MJ4wiAeWv1gBoqAQQjxEFAPLBJCtAshLpdSDgJXAc8ArwG/k1JuTGeeBmMH4wmDeKTDD2atJwMDAwMDW5hvFAYGBgYGtjADhYGBgYGBLcxAYWBgYGBgCzNQGBgYGBjYwgwUBgYGBga2MAOFgYGBgYEtzEBhYJBGCCFWCSHmpDsPAwM7mIHCwCC9WAWYgcIg0DADhYFBiiGEmCeEeG14p7mNQog/CyHyE+g+hrWZzINCiJcTaQwMggAzUBgY6EE58FMp5WJgP/DRkQIp5SPAeuBTUspTpJT9Y5yjgYESzEBhYKAHrVLKl4cfvwTMS2MuBga+YAYKAwM9GIh7PITZ+8VgHMMMFAYG6UU3UJjuJAwM7GAGCgOD9OJe4E4zmW0QZJhlxg0MDAwMbGG+URgYGBgY2MJMsBkYjAGEED8F3jmi+FYp5T3pyMfAwA3MT08GBgYGBrYwPz0ZGBgYGNjCDBQGBgYGBrYwA4WBgYGBgS3MQGFgYGBgYAszUBgYGBgY2MIMFAYGBgYGtvj/AUvFI4yhzSxaAAAAAElFTkSuQmCC\n",
328 |       "text/plain": [
329 |        "<Figure size 432x288 with 1 Axes>"
330 |       ]
331 |      },
332 |      "metadata": {
333 |       "needs_background": "light"
334 |      },
335 |      "output_type": "display_data"
336 |     }
337 |    ],
338 |    "source": [
339 |     "# Estimating PI and HI for various sampling efforts\n",
340 |     "N_HI = np.logspace(np.log10(5),np.log10(N_s),20,dtype=int)\n",
341 |     "N_PI = np.logspace(np.log10(5),np.log10(N_s*(1-1/n_fold)),20,dtype=int)\n",
342 |     "PI = np.zeros(len(N_HI))\n",
343 |     "HI = np.zeros(len(N_PI))\n",
344 |     "kf = KFold(n_splits=n_fold)\n",
345 |     "\n",
346 |     "\n",
347 |     "for av in range(N_av):\n",
348 |     "    # fresh data to compute the HI and PI on\n",
349 |     "    data = measure_data(p_k, norm_params,number_samples=N_HI[-1])\n",
350 |     "    \n",
351 |     "    #HI\n",
352 |     "    for i,n_p in enumerate(N_HI):\n",
353 |     "        # Keep the n first data samples\n",
354 |     "        data_n = keep_measures(p_k=p_k,data_orig=data,I=np.arange(n_p))\n",
355 |     "        \n",
356 |     "        #estimate model with n samples\n",
357 |     "        esti_models = [None for i in p_k]\n",
358 |     "        for k in range(len(p_k)):\n",
359 |     "            esti_models[k] = [np.mean(data_n[k]),np.std(data_n[k])]\n",
360 |     "        HI[i] += information(p_k,data_n,esti_models)\n",
361 |     "\n",
362 |     "    #PI\n",
363 |     "    PI_x = np.zeros(PI.shape)\n",
364 |     "    for train_index, test_index in kf.split(range(N_s)):\n",
365 |     "        data_model = keep_measures(p_k=p_k,data_orig=data,I=train_index)\n",
366 |     "        data_p = keep_measures(p_k=p_k,data_orig=data,I=test_index)\n",
367 |     "\n",
368 |     "        for i,n in enumerate(N_PI):\n",
369 |     "            data_model_nt = keep_measures(p_k=p_k,data_orig=data_model,I=range(n))\n",
370 |     "    \n",
371 |     "            #generate fresh samples\n",
372 |     "            esti_models = [None for i in p_k]\n",
373 |     "            for k in range(len(p_k)):\n",
374 |     "                esti_models[k] = [np.mean(data_model_nt[k]),np.std(data_model_nt[k])]\n",
375 |     "            PI_x[i] += information(p_k,data_p,esti_models)\n",
376 |     "    PI += PI_x/n_fold\n",
377 |     "HI /= N_av\n",
378 |     "PI /= N_av\n",
379 |     "\n",
380 |     "#visualize plots\n",
381 |     "plt.figure()\n",
382 |     "plt.semilogx(N_HI,HI,label=\"HI\",color=\"r\")\n",
383 |     "plt.semilogx(N_PI,PI,label=\"PI\",color=\"g\")\n",
384 |     "plt.axhline(MI,label=\"MI\",color=\"b\",ls=\"--\")\n",
385 |     "plt.grid(True,which=\"both\",ls=\"--\")\n",
386 |     "plt.xlabel(\"n_t\")\n",
387 |     "plt.ylabel(\"Information\")\n",
388 |     "plt.legend()\n",
389 |     "plt.show()"
390 |    ]
391 |   },
392 |   {
393 |    "cell_type": "code",
394 |    "execution_count": null,
395 |    "metadata": {},
396 |    "outputs": [],
397 |    "source": []
398 |   }
399 |  ],
400 |  "metadata": {
401 |   "kernelspec": {
402 |    "display_name": "Python 3",
403 |    "language": "python",
404 |    "name": "python3"
405 |   },
406 |   "language_info": {
407 |    "codemirror_mode": {
408 |     "name": "ipython",
409 |     "version": 3
410 |    },
411 |    "file_extension": ".py",
412 |    "mimetype": "text/x-python",
413 |    "name": "python",
414 |    "nbconvert_exporter": "python",
415 |    "pygments_lexer": "ipython3",
416 |    "version": "3.7.3"
417 |   }
418 |  },
419 |  "nbformat": 4,
420 |  "nbformat_minor": 4
421 | }
422 | 


--------------------------------------------------------------------------------