├── README.md └── codes ├── Ising_2D ├── assets │ ├── Z.png │ ├── Z2.png │ ├── gen1.png │ ├── gen2.png │ ├── gen3.png │ ├── gen4.png │ ├── gradients.png │ ├── image-20190430141119642.png │ ├── image-20190430143433180.png │ ├── image-20190430144239877.png │ ├── image-20190430144449213.png │ ├── image-20190430145634644.png │ ├── image-20190430150813001.png │ ├── image-20190430152031663.png │ ├── init.png │ ├── init2.png │ └── px.png ├── kacward.py └── tensor_contraction_simple.ipynb ├── MPS_Born_machine ├── .ipynb_checkpoints │ └── mps_tutorial-checkpoint.ipynb ├── assets │ ├── Z.png │ ├── Z2.png │ ├── gen1.png │ ├── gen2.png │ ├── gen3.png │ ├── gen4.png │ ├── gradients.png │ ├── image-20190430141119642.png │ ├── image-20190430143433180.png │ ├── image-20190430144239877.png │ ├── image-20190430144449213.png │ ├── image-20190430145634644.png │ ├── image-20190430150813001.png │ ├── image-20190430152031663.png │ ├── init.png │ ├── init2.png │ └── px.png ├── imgs │ ├── L.png │ ├── L0.png │ ├── Z.png │ ├── born_machine.png │ ├── cond_prob.png │ ├── cp.png │ ├── joint_prob.png │ ├── mnist_mps.png │ ├── mps.png │ ├── mps1.png │ ├── mps2.png │ ├── mps_left.png │ ├── psi_prime.png │ ├── rank_one.png │ ├── recon0.png │ ├── recon1.png │ ├── supunsup.png │ ├── tensor.png │ ├── tensor_diagram.png │ ├── tensor_networks.png │ ├── training.png │ ├── tucker.png │ └── two_qubits.png ├── mnist784_bin_1000.npy └── mps_tutorial.ipynb ├── VAE ├── .ipynb_checkpoints │ └── REINFORCE_VS_Reparametrization-checkpoint.ipynb └── REINFORCE_VS_Reparametrization.ipynb ├── VAN ├── .ipynb_checkpoints │ └── SK_variational-checkpoint.ipynb └── SK_variational.ipynb └── neural_networks ├── .ipynb_checkpoints ├── MLP_sin-checkpoint.ipynb ├── grad_matrix_products-checkpoint.ipynb ├── logistic_regression-checkpoint.ipynb ├── lr_mlp_conv-checkpoint.ipynb └── tanh-checkpoint.ipynb ├── MLP_sin.ipynb ├── grad_matrix_products.ipynb ├── logistic_regression.ipynb ├── lr_mlp_conv.ipynb └── tanh.ipynb /README.md: -------------------------------------------------------------------------------- 1 | This repo contains demo codes for my lectures in 2 | * Lectures at the ITP, CAS http://www.itp.cas.cn/xshd/yjsgjkc/201903/t20190311_5253828.html 3 | * Lectures at summer school of *The fifth conference on statistical physics and complex systems* http://spcsc2019.ustc.edu.cn/summerschool2019 4 | -------------------------------------------------------------------------------- /codes/Ising_2D/assets/Z.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/Z.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/Z2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/Z2.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/gen1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/gen1.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/gen2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/gen2.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/gen3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/gen3.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/gen4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/gen4.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/gradients.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/gradients.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/image-20190430141119642.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/image-20190430141119642.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/image-20190430143433180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/image-20190430143433180.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/image-20190430144239877.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/image-20190430144239877.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/image-20190430144449213.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/image-20190430144449213.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/image-20190430145634644.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/image-20190430145634644.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/image-20190430150813001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/image-20190430150813001.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/image-20190430152031663.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/image-20190430152031663.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/init.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/init.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/init2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/init2.png -------------------------------------------------------------------------------- /codes/Ising_2D/assets/px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panzhang83/lectures/723042252171ab9d252c4921bd199f97910f8dce/codes/Ising_2D/assets/px.png -------------------------------------------------------------------------------- /codes/Ising_2D/kacward.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | ''' 4 | Kac-Ward exact Ising 5 | See Theorem 1 of https://arxiv.org/abs/1011.3494 6 | ''' 7 | 8 | phi = np.array([[0., np.pi/2, -np.pi/2, np.nan ], 9 | [-np.pi/2, 0.0, np.nan, np.pi/2], 10 | [np.pi/2, np.nan, 0.0, -np.pi/2], 11 | [np.nan, -np.pi/2, np.pi/2, 0] 12 | ]) 13 | 14 | def logcosh(x): 15 | xp = np.abs(x) 16 | if (xp< 12): 17 | return np.log( np.cosh(x) ) 18 | else: 19 | return xp - np.log(2.) 20 | 21 | def neighborsite(i, n, L): 22 | """ 23 | The coordinate system is geometrically left->right, down -> up 24 | y| 25 | | 26 | | 27 | |________ x 28 | (0,0) 29 | So as a definition, l means x-1, r means x+1, u means y+1, and d means y-1 30 | """ 31 | x = i%L 32 | y = i//L # y denotes 33 | site = None 34 | # ludr : 35 | if (n==0): 36 | if (x-1>=0): 37 | site = (x-1) + y*L 38 | elif (n==1): 39 | if (y+1=0): 43 | site = x + (y-1)*L 44 | elif (n==3): 45 | if (x+1" 149 | ] 150 | }, 151 | "execution_count": 3, 152 | "metadata": {}, 153 | "output_type": "execute_result" 154 | }, 155 | { 156 | "data": { 157 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VNX9//HXmckOSSAhQFiSCYtFICRA2BJZ3AAFERWruECgFbdq3WutLXy1334t2vqre61CEFEQBERcSlUoSlgMm4ZdIEAgQEggK9nP74+ZjEnInsncmcnn+XjkMZO7fnJh3vfMnTPnKq01Qggh3J/J6AKEEEI4hgS6EEJ4CAl0IYTwEBLoQgjhISTQhRDCQ0igCyGEh5BAF0IIDyGBLoQQHkICXQghPISXM3fWqVMnbbFYnLlLIYRwe9u3bz+ntQ5raDmnBrrFYiElJcWZuxRCCLenlDrWmOXkkosQQngICXQhhPAQEuhCCOEhnHoNXRijtLSU9PR0ioqKjC5FCFEPPz8/evTogbe3d7PWl0BvA9LT0wkMDMRisaCUMrocIUQttNZkZWWRnp5OVFRUs7Yhl1zagKKiIkJDQyXMhXBhSilCQ0Nb9E66wUBXSi1QSp1VSqVWmRailPqPUuqQ7bFjsysQTiFhLoTra+nrtDEt9CRgYo1pTwNfa637Al/bfm81n/2QwZKtjeqGKYQQbVaDga613ghk15h8I7DI9nwRMNXBdVXz+Y8ZzP/yAIUlZa25G9GKzGYzsbGxDBw4kBtuuIELFy4AkJaWhr+/P7Gxsfaf9957D7B+Ee3cuXOAteXy+OOP27f30ksvMW/ePADmzZtH9+7d7es//bS1fVFSUsIjjzxC79696du3LzfeeCPp6ekN1gRw8OBBrr/+evr06cPll1/OL3/5S86cOcOGDRsIDg6uVu9XX33VqsdOiMZq7jX0LlrrDADbY2fHlXSpxAQLORdLWb3zVGvuRrQif39/du3aRWpqKiEhIbz++uv2eb1792bXrl32nxkzZlyyvq+vLytXrrQHfE2PPvqoff0XXngBgGeeeYa8vDwOHjzIoUOHmDp1KjfffDOVN0avq6aioiImTZrE/fffz08//cS+ffu4//77yczMBGD06NHV6r3mmmsceqyEaK5W/1BUKTVHKZWilEqpfEE0VVxkRwZ2DyIp+aj9xSjc16hRozh58mST1vHy8mLOnDm8/PLLjVq+sLCQhQsX8vLLL2M2mwGYNWsWvr6+fPPNN/XW9MEHHzBq1ChuuOEG+/wrr7ySgQMHNqlmIZytud0WzyilwrXWGUqpcOBsXQtqrd8G3gaIi4trVhorpUiMj+KJ5btJPpxFQp9Ozata8D+f7mHvqVyHbrN/tyDm3jCgUcuWl5fz9ddf86tf/co+7fDhw8TGxtp/f/XVVxk9evQl6z744IMMGjSIp5566pJ5L7/8Mu+//z4Af/3rXwkPDyciIoKgoKBqy8XFxbFnzx6uvvrqOmtKTU1l6NChdf4N3377bbV6P/74Y3r37t3Qny5Eq2tuoK8BZgIv2B4/cVhFdZg8KJz/+3wfCzelSaC7oYsXLxIbG0taWhpDhw7l2muvtc+rvOTSkKCgIGbMmMErr7yCv79/tXmPPvooTzzxhP333bt319pjQGttn15fTfUZPXo0a9eubdSyQjhTg4GulPoQGAd0UkqlA3OxBvlHSqlfAceBW1uzSAA/bzN3jojg1fU/cSyrgMjQdq29S4/U2Ja0o1Ver87JyWHy5Mm8/vrrPPzww03eziOPPMKQIUOYNWtWvcv16dOHY8eOkZeXR2BgoH36jh077JdS6qppwIAB/Pe//21ybUIYrTG9XKZrrcO11t5a6x5a63e11lla66u11n1tjzV7wbSKO0dGYlaK9zZLF0Z3FRwczCuvvMJLL71EaWlpk9cPCQnhl7/8Je+++269y7Vr146ZM2fy2GOPUV5eDsB7771HYWEhV111Vb013XHHHSQnJ/PZZ5/Zl/nyyy/58ccfm1yvEM7kVt8U7RLkx/XR4Xz0/QkKiqULo7saPHgwMTExLF26FPj5GnrlzyuvvFLv+o8//nidvV2q+r//+z/8/Py47LLL6Nu3L8uXL2fVqlW1XoqpWpO/vz9r167l1VdfpW/fvvTv35+kpCQ6d7Z25qq8hl75s2LFimYcBSEcTzmz10hcXJxu6Q0udh4/z01vJPPcjQOYMcrimMI83L59+7j88suNLkMI0Qi1vV6VUtu11nENretWLXSAwREdienZgaTkNCoqpAujEEJUcrtAB5gVb+FIZgHf/tTw224hhGgr3DLQr48OJyzQl4WbjhpdihBCuAy3DHQfLxN3jYhkw4FMjmTmG12OEEK4BLcMdIA7RkTgYzZJF0YhhLBx20APC/Rlckw4y1NOkFvU9P7MQgjhadw20AFmxUdRUFLOipT0hhcWhmrrw+cmJSUxffr0atPOnTtHWFgYxcXFjT6Ob731lv34uIPExMRm99P/y1/+Uu33+Ph4R5QEWL9xvHHjRodtrylee+01Fi5c2Dob11o77Wfo0KHa0W5+Y5MeM/8bXV5e4fBte4q9e/caXYJu166d/fmMGTP0n//8Z6211kePHtUDBgyodZ3IyEidmZmptdba19dXWywW++8vvviinjt3rtZa67lz5+oXX3zxkvUff/xxPXv2bF1WVqa11nrBggV62LBhuqKiot6aLl68qPv06aPXrFljn//NN9/oH3/8Ua9fv15PmjSp3r91/fr1eubMmdWm5eTk6NDQUF1QUGCf9uabb+rZs2fXu62qSktLG71sa2pKHTNnztTLly9v1n6q/vs4UlZWlh4xYkSrbLsxCgoKdGxsbJ3za3u9Aim6ERnr1i10gFkJFo5lFbL+QJ0DPgoX0xaHzw0KCmLMmDF8+umn9mlLly61t9qfe+45hg0bxsCBA5kzZ459mOhx48bxzDPPMHbsWP7xj38wb948XnrpJQD+9a9/MWzYMGJiYrjlllsoLCwErK3ihx9+mPj4eHr16lWthTx//nyio6OJiYmxv5M5fPgwEydOZOjQoYwePZr9+/dfUv+8efOYM2cO48ePZ8aMGZSXl/Pkk08ybNgwBg0axD//+U/A2kD8zW9+Q//+/Zk0aRJnz/78uqz6jislJYVx48YBkJ+fz6xZs4iOjmbQoEF8/PHHPP300/bB0+68804A2rdvb9/Hk08+ycCBA4mOjmbZsmUAbNiwgXHjxjFt2jT69evHnXfeWetw2ytWrGDixJ9vwmaxWHjmmWcYNWoUcXFx7NixgwkTJtC7d2/eeust+3Ivvvii/e+dO3euffrUqVMZOnQoAwYM4O2337ZPb9++PX/4wx+IiYlh5MiRnDlzBoCAgAAsFgvbtm27pLaWau5oiy5jwoCudA3yIyk5jasv72J0Oa7vi6fhtIPHJOkaDde90KhF2/LwudOnT+eDDz7gtttu49SpUxw8eJArr7wSgN/85jf86U9/AuDuu+9m7dq19hPKhQsX7IOFVV5mArj55pu55557AHj22Wd59913eeihhwDIyMjgu+++Y//+/UyZMoVp06bxxRdfsHr1arZu3UpAQADZ2dYhmObMmcNbb71F37592bp1Kw888ECtJ73t27fz3Xff4e/vz9tvv01wcDDff/89xcXFJCQkMH78eHbu3MmBAwf48ccfOXPmDP3792f27Nn1Hpfnn3+e4OBg+1g558+f55ZbbuG1116rdRTOlStXsmvXLnbv3s25c+cYNmwYY8aMAWDnzp3s2bOHbt26kZCQwKZNm7jiiiuqrb9p0yamTZtWbVrPnj3ZvHkzjz76KImJiWzatImioiIGDBjAfffdx7p16zh06BDbtm1Da82UKVPYuHEjY8aMYcGCBYSEhHDx4kWGDRvGLbfcQmhoKAUFBYwcOZL//d//5amnnuJf//oXzz77LGD9f/jtt98yfPjweo9NU7l9oHubTdw9KpIX/32AQ2fy6NslsOGVhNO1heFzR4wYQXFxMfn5+WRnZ9tD/69//SsTJkxg8uTJPPDAA+Tm5vLRRx8xbdo0+7uH9evXM3/+fAoLC8nOzmbAgAH2QL/ttttqrSM1NZVnn32WCxcukJ+fz4QJE+zzpk6dislkon///vaW4VdffcWsWbMICAgArAOd5efnk5yczK23/jxgal3X9KdMmWI/7uvWreOHH36wt/5zcnI4dOgQGzduZPr06ZjNZrp163bJQGi1+eqrr+zj+gB07Fj/Pee/++47+z66dOnC2LFj+f777wkKCmL48OH06NEDwP5vWzPQMzIyCAsLu+RvA4iOjiY/P5/AwEACAwPx8/PjwoULrFu3jnXr1jF48GDA+q7i0KFDjBkzhldeeYVVq1YBcOLECQ4dOkRoaCg+Pj5MnjwZgKFDh/Kf//zHvr/OnTvX+k6opdw+0AGmD4/gH18fIik5jf+9KdroclxbI1vSjtYWhs/dunUrYH3rn5SURFJSUrX5/v7+TJw4kVWrVrF06VL75aOioiIeeOABUlJS6NmzJ/PmzaOoqMi+Xrt2tQ8VnZiYyOrVq4mJiSEpKYkNGzbY5/n6+tqfV152qHoyq1RRUUGHDh0adUKtWofWmldffbXaSQTg888/r/PO9V5eXlRUVNj/5qrbasrd7mu7jFKp6t9tNpspK7t0ED9/f/9q+6+6nslkqrYNk8lEWVkZWmt+//vfc++991Zbb8OGDXz11Vds3ryZgIAAxo0bZ9+2t7e3/e+qWUtRUdEljRJHcPtr6AAh7XyYGtuNlTtOklMoXRhdWVsfPnf69On8/e9/58yZM4wcORL4Odw6depEfn5+o3uF5OXlER4eTmlpKUuWLGlw+fHjx7NgwQL7tfbs7GyCgoKIiopi+fLlgDUsd+/e3eC2JkyYwJtvvmn/Nzx48CAFBQWMGTOGpUuXUl5eTkZGBuvXr7evY7FY2L59O2C9TFW1rtdee83++/nz5wFrINb2f2TMmDEsW7aM8vJyMjMz2bhxY5MuXVx++eX89NNPjV6+8u9dsGAB+fnWLzKePHmSs2fPkpOTQ8eOHQkICGD//v1s2bKlUds7ePBgq9zS0CMCHSAxPoqLpeV8lHLC6FJEA9ry8Lnjx4/n1KlT3HbbbfY6OnTowD333EN0dDRTp05l2LBhjdrW888/z4gRI7j22mvp169fg8tPnDiRKVOmEBcXR2xsrP3D1SVLlvDuu+8SExPDgAED+OSThm9A9utf/5r+/fszZMgQBg4cyL333ktZWRk33XQTffv2JTo6mvvvv5+xY8fa15k7dy6//e1vGT16tP1SE1iv/58/f56BAwcSExNjPwnMmTOHQYMG2T8UrXTTTTcxaNAgYmJiuOqqq5g/fz5du3Zt1DEDmDRpUrV3M40xfvx47rjjDkaNGkV0dDTTpk0jLy+PiRMnUlZWxqBBg/jjH/9oP0k3ZNOmTa1yc3G3Gz63Prf9czMnL1zkv09eidnU+Ldwnk6GzxWiuiuuuIK1a9fSoUMHp+97586d/P3vf2fx4sW1zm9Tw+fWZ1aChfTzF/lq3xmjSxFCuLC//e1vHD9+3JB9nzt3jueff75Vtu0RH4pWuubyLnTv4M/CTUeZMKDxb8GEEG3LiBEjDNt3Y3tTNYdHtdC9zCZmjIpky5Fs9mXkGl2OEEI4lUcFOsBtw3ri521iUXKa0aUIIYRTeVygdwjw4eYhPVi18yTZBSVGlyOEEE7jcYEOkBhvobisgqXfG/OhhxBCGMEjA/2yLoEk9All8eZjlJVXGF2OoP6hat1NUlISp06dqnP+n/70p0YNqVvT6tWr2bt3b4u3I9oujwx0sI6VnpFTxL/3SBdGV1D5NfvU1FRCQkJ4/fXXW3V/tX3l21HqC/Ty8nKee+65Zn1ppGagN3c7ou3y2EC/sl9nIkICSEqWG0m7mprD59Y2LGlaWhr9+vVj5syZDBo0iGnTptm/st7YoWY//fRTRowYweDBg7nmmmvsg1TNmzePmTNnMn78eCwWCytXruSpp54iOjqaiRMn2r9uvn37dsaOHcvQoUOZMGECGRkZrFixgpSUFO68805iY2O5ePEiFouF5557jiuuuILly5fbb+qQkpJi/zZpdHS0/ZuhtQ17m5yczJo1a3jyySeJjY3l8OHD1W4O8fXXXzN48GCio6OZPXu2fQAti8XC3LlzGTJkCNHR0a0y4JNwHx7VD70qs0kxY1Qkf/5sH6kncxjYPdjoklzCX7f9lf3Zjn3R9wvpx++G/65Ry9YcqrauYUkjIiI4cOAA7777LgkJCcyePZs33niDJ554otFDzZ4/f54tW7aglOKdd95h/vz5/O1vfwOsww2sX7+evXv3MmrUKD7++GPmz5/PTTfdxGeffcakSZN46KGH+OSTTwgLC2PZsmX84Q9/YMGCBbz22mu89NJLxMX9/MU9Pz8/vvvuO8A67gtYh0itHPTqySeftI/BXdewt1OmTGHy5MmXDO1aVFREYmIiX3/9NZdddhkzZszgzTff5JFHHgGsY8Ds2LGDN954g5deeol33nmnKf98woN4bAsd4JfDehLgY2bhpjSjS2nzKoeqDQ0NJTs72/7liqrDkg4ZMoT9+/dz6NAhwDpGdUJCAgB33XWXPTDXr1/PiBEjiI6O5ptvvmHPnj32/VQdajY9PZ0JEyYQHR3Niy++WG256667Dm9vb6KjoykvL7eHbXR0NGlpaRw4cIDU1FSuvfZaYmNj+fOf/1zt9nU11TXELcBHH33Ejh07eOEF60iXqampjB49mujoaJYsWVKtrtocOHCAqKgoLrvsMgBmzpxZ7fZpN998M2AdojUtLa3ebQnP5rEtdIAgP2+mDe3B0m0n+P31/ejU3rfhlTxcY1vSjlbXULV1DUualpZ2ySBaSqkmDTX70EMP8dhjjzFlyhQ2bNhQ7eYQVYdLrTrMadXhUgcMGMDmzZsb9ffVNcTtnj17mDt3Lhs3brQPSFXfsLe1aWi8pcq/pa7hYkXb4dEtdICZ8RZKyiv4cKt0YXQFNYeqrWtYUoDjx4/bA/XDDz/kiiuuaNJQszk5OXTv3h2ARYsWNanOX/ziF2RmZtr3X1paam9JBwYGkpeX1+A2cnJyuP3223nvvfeq3VChrmFv69puv379SEtLsw/5unjx4mqjGApRqUWBrpR6VCm1RymVqpT6UCnl56jCHKV3WHvGXhbG4i3HKCmTLoyuoOpQtXUNSwrWcasXLVrEoEGDyM7O5v7772/SULPz5s3j1ltvZfTo0XTq1KlJNfr4+LBixQp+97vfERMTQ2xsLMnJyYC1hX3ffffZPxSty+rVqzl27Bj33HOP/cNRqHvY29tvv50XX3yRwYMHc/jwYft0Pz8/Fi5cyK233kp0dDQmk4n77ruvSX+PaBuaPXyuUqo78B3QX2t9USn1EfC51jqprnVae/jcuqw/cJZZC7/nH7fHcmNsd6fv32juOHxuWloakydPJjU11ehShHAqI4fP9QL8lVJeQABQ97ctDDS2bxi9OrUjScZ3EUJ4sGYHutb6JPAScBzIAHK01uscVZgjmUyKmfEWdh6/wK4T7vsNxbbEYrFI61yIJmp2oCulOgI3AlFAN6CdUuquWpabo5RKUUqlZGZmNr/SFrplaA8Cfb1I2tQ2v2jkzDtTCSGap6Wv05ZccrkGOKq1ztRalwIrgfiaC2mt39Zax2mt46p+0u9s7X29uDWuJ5/9mMHZ3KKGV/Agfn5+ZGVlSagL4cK01mRlZeHn1/y+JS3ph34cGKmUCgAuAlcDzv/EswlmjIpkYfJR3t96nMeuvczocpymR48epKenY+Q7JCFEw/z8/OjRo0ez1292oGuttyqlVgA7gDJgJ/B2sytxAkundlz1i858sPUYD17ZG18vc8MreQBvb2+ioqKMLkMI0cpa1MtFaz1Xa91Paz1Qa3231rrYUYW1llkJUZzLL2Ht7gyjSxFCCIfy+G+K1pTQJ5Q+nduTlJwm15SFEB6lzQW6UorEeAs/nsxhx/HzRpcjhBAO0+YCHeDmId0J8vNigYzCKITwIG0y0AN8vLh9eARfpp4mI6fusTiEEMKdtMlAB7h7ZCRaa97fcszoUoQQwiHabKD3DAng2v5d+GDrcYpKy40uRwghWqzNBjpAYnwU5wtLWbPLJccUE0KIJmnTgT6yVwj9ugayULowCiE8QJsOdKUUsxIs7MvIZevRbKPLEUKIFmnTgQ5wY2x3OgZ4kyRdGIUQbq7NB7qft5npwyNYt/c06ecLjS5HCCGarc0HOsBdIyNRSrF4s3RhFEK4Lwl0oFsHfyYO6MqH245TWFJmdDlCCNEsEug2sxIs5BaVsXqndGEUQrgnCXSboZEdGdg9iKTko9KFUQjhliTQbayjMEZx8Ew+yYezjC5HCCGaTAK9ihtiwunU3oeFbfRG0kII9yaBXoWvl5k7hkfw9f6zHMsqMLocIYRoEgn0Gu4cGYlZKd6TLoxCCDcjgV5DlyA/Jg0K56PvT5BfLF0YhRDuQwK9FonxFvKKy1i5I93oUoQQotEk0GsxOKIjMT07kJScRkWFdGEUQrgHCfQ6zE6wcCSzgI2HMo0uRQghGkUCvQ7XDQwnLNCXpOQ0o0sRQohGkUCvg4+XibtGRLLhQCZHMvONLkcIIRokgV6PO0ZE4GM2sUha6UIINyCBXo+wQF8mx4SzYns6uUWlRpcjhBD1kkBvwKz4KApKylmRIl0YhRCuTQK9AdE9gomL7MiizWmUSxdGIYQLk0BvhMQEC8eyCtlw4KzRpQghRJ0k0BthwoCuhAf7SRdGIYRLa1GgK6U6KKVWKKX2K6X2KaVGOaowV+JtNnHXyEi+PXSOQ2fyjC5HCCFq1dIW+j+AL7XW/YAYYF/LS3JN04dH4Otlkla6EMJlNTvQlVJBwBjgXQCtdYnW+oKjCnM1Ie18mBrbnZU7TpJTKF0YhRCupyUt9F5AJrBQKbVTKfWOUqpdzYWUUnOUUilKqZTMTPceF2VmvIWLpeUsSzludClCCHGJlgS6FzAEeFNrPRgoAJ6uuZDW+m2tdZzWOi4sLKwFuzNe/25BjIgKYVHyMenCKIRwOS0J9HQgXWu91fb7CqwB79FmJURx8sJF/rP3jNGlCCFENc0OdK31aeCEUuoXtklXA3sdUpULu+byznTv4E9SstxIWgjhWlray+UhYIlS6gcgFvhLy0tybV5mEzNGRbLlSDb7MnKNLkcIIexaFOha61226+ODtNZTtdbnHVWYK7t9WAT+3maSNqUZXYoQQtjJN0WbITjAm5uGdGf1rpNkF5QYXY4QQgAS6M2WGG+huKyCpd9LF0YhhGuQQG+my7oEckWfTizefIzS8gqjyxFCCAn0lkiMt5CRU8S6PdKFUQhhPAn0FriyX2ciQgKkC6MQwiVIoLeA2aSYGW/h+7TzpJ7MMbocIUQbJ4HeQrfG9aCdj5mF0oVRCGEwCfQWCvLzZtrQHny6+xTn8ouNLkcI0YZJoDvAjHgLJeUVfLBVujAKIYwjge4AvcPaM/ayMN7fcoySMunCKIQwhgS6g8xKsHA2r5gvUjOMLkUI0UZJoDvImL5h9OrUTj4cFUIYRgLdQUy2Loy7Tlxg5/E2MUaZEMLFSKA70C1DexDo6yU3khZCGEIC3YHa+3pxa1xPPvshgzO5RUaXI4RoYyTQHWxmfCTlWrNEujAKIZxMAt3BIkPbcXW/znyw9RjFZeVGlyOEaEMk0FtBYnwU5/JLWLtbujAKIZxHAr0VJPQJpW/n9iQlp6G1NrocIUQbIYHeCpRSJCZY+PFkDtuPSRdGIYRzSKC3kpsGdyfIz4uF0oVRCOEkEuitJMDHi9uHR/Bl6mkyci4aXY4Qog2QQG9Fd4+MRGvN4s3HjC5FCNEGSKC3op4hAVzbvwsfbjtOUal0YRRCtC4J9FY2KyGK84WlrNl1yuhShBAeTgK9lY2ICqFf10AWbDoqXRiFEK1KAr2VKaWYlWBh/+k8th7NNrocIYQHk0B3ghtju9MxwJskGStdCNGKJNCdwM/bzPThEazbe5oT2YVGlyOE8FAS6E5y18hIlFK8v0W6MAohWkeLA10pZVZK7VRKrXVEQZ6qWwd/Jg7syofbjlNYUmZ0OUIID+SIFvpvgX0O2I7HmxVvIbeojFU7TxpdihDCA7Uo0JVSPYBJwDuOKcezDY3syMDuQSRtklEYhRCO19IW+v8DngIq6lpAKTVHKZWilErJzMxs4e7cm1KKWfFRHDqbz6afsowuRwjhYZod6EqpycBZrfX2+pbTWr+ttY7TWseFhYU1d3ceY3JMOJ3a+5CUfNToUoQQHqYlLfQEYIpSKg1YClyllHrfIVV5MF8vM3cMj+Dr/Wc5llVgdDlCCA/S7EDXWv9ea91Da20Bbge+0Vrf5bDKPNhdIyMxK8WiZOnCKIRwHOmHboDOQX5MGhTO8pQT5BdLF0YhhGM4JNC11hu01pMdsa22IjHeQl5xGSt3pBtdihDCQ0gL3SCDIzoS27MDSZvSqKiQLoxCiJaTQDfQrAQLR84VsPFQ2+7OKYRwDAl0A103MJzOgb4kyY2khRAOIIFuIB8vE3eNjGTDgUwOZ+YbXY4Qws1JoBts+vAIfMwm3pNWuhCihSTQDRYW6MsNMd1YsT2d3KJSo8sRQrgxCXQXkBhvoaCknOUp0oVRCNF8EuguILpHMHGRHVmUnEa5dGEUQjSTBLqLmJUQxfHsQtbvP2t0KUIINyWB7iLGD+hCeLCfdGEUQjSbBLqL8DZbuzB+99M5Dp3JM7ocIYQbkkB3IdOHR+DrZWKhtNKFEM3gHoF+4QRcPG90Fa0upJ0PU2O7s3JHOjmF0oVRCNE07hHoq++H10fCgS+MrqTVJSZYKCqtYFnKcaNLEUK4GfcI9PF/hnad4MPbYeUcKMw2uqJWc3l4ECN7hbAo+Rhl5XXeqlUIIS7hHoHeLRbuWQ9jn4bUj+GNkbD/M6OrajWJ8VGcvHCRr/ZJF0YhROO5R6ADePnAlb+3Bnu7zrD0Dvj41x7ZWr+2fxe6d/CXG0kLIZrEfQK9UvggmLMexj0De1bB68Nh36dGV+VQZpNiZnw3NqU9AAATUklEQVQkW45ksy8j1+hyhBBuwv0CHcDsDeN+B3M2QGA4LLsLVsyGgiyjK3OY2+Ii8Pc2k7QpzehShBBuwj0DvVLXaLjnG7jyWdi7xtpa3/uJ0VU5RHCANzcN6c7qXSfJLigxuhwhhBtw70AHa2t97JNw738huDt8NAOWJ0LBOaMra7FZ8RaKyyr4cJt0YRRCNMz9A71SlwHw66/hqj9ae8C8Ptx6jd2N9e0SyBV9OvH+lmOUShdGIUQDPCfQwdpaH/MEzPkvdIiwttQ/mgH57nsT5sR4Cxk5Razbc8boUoQQLs6zAr1Sl/7wq6/g6rnWb5e+Ptzaf12731jjV/XrTGRoAAs3SRdGIUT9PDPQAcxeMPoxuPdbCImy9oJZdhfkuVdL12RSzBhlIeXYeX5MzzG6HCGEC/PcQK/UuR/MXgfX/A8c+g+8MQJ+WO5WrfVb43rQzscsY6ULIerl+YEO1tb6FY/Afd9BaB9Y+WtYeifknTa6skYJ8vNm2tAefLr7FJl5xUaXI4RwUW0j0CuFXQaz/20d7Ovw1/D6CNi9zC1a6zPiLZSUSxdGIUTd2lagA5jMEP+QtbUe9gtYNQc+nA65GUZXVq/eYe0Z94swFm85RkmZdGEUQlyq7QV6pU59YdYXMOEvcGS99dr6rg9durWeGG8hM6+YL1Jd++QjhDBGswNdKdVTKbVeKbVPKbVHKfVbRxbmFCYzjHoQ7k+GsMth9X3wwW2Qe8roymo1pm8YvTq1Y6GM7yKEqEVLWuhlwONa68uBkcCDSqn+jinLyUJ7w6zPYeILcHSj9e5IO5e4XGvdZFIkJljYdeICO497/i35hBBN0+xA11pnaK132J7nAfuA7o4qzOlMZhh5P9y/yTqMwCcPwJJbIeek0ZVVc/OQHgT6ekkXRiHEJRxyDV0pZQEGA1sdsT1DhfaGxM/guvlwbJP17kg7FrtMa729rxe3xvXksx8yOJNbZHQ5QggX0uJAV0q1Bz4GHtFaX3I3BqXUHKVUilIqJTPTTcZUMZlgxL3W1nrXQbDmN/D+LZCTbnRlAMyMj6Rca5ZsOWZ0KUIIF9KiQFdKeWMN8yVa65W1LaO1fltrHae1jgsLC2vJ7pwvpBfM/BSufwmOb7FeW9++yPDWemRoO67u15klW49TXFZuaC1CCNfRkl4uCngX2Ke1/rvjSnIxJhMMv8faWu8WC58+DItvggsnDC0rMT6KrIIS1u6WLoxCCKuWtNATgLuBq5RSu2w/1zuoLtcTEgUz1sCkv8GJbfDGKEhZaFhrPaFPKH07t2dh8lG0i1zfF0IYqyW9XL7TWiut9SCtdazt53NHFudyTCYY9mt4YDN0HwxrH4HFU+G8869lK2Xtwph6Mpftx6QLoxCiLX9TtCU6Rlpb65P/H6SnwJvx8P07UOHcr+TfNLg7QX5eLJQujEIIJNCbTymIm2VtrfcYBp89Du9NgfNpTishwMeL6cMj+DL1NKcuXHTafoUQrkkCvaU6RMDdq+CGV+DULngjHrb9y2mt9btHRaK15n3pwihEmyeB7ghKwdCZ1tZ6xEj4/AlYdANkH2n1XffoGMD4/l35cNtxikqlC6MQbZkEuiN16Al3fQxTXoPTP8CbCbD1n63eWk9MsHC+sJRPdrnWMAVCCOeSQHc0pWDI3fDAFohMgC+egkWTIetwq+1yRFQI/boGsnBTmnRhFKINk0BvLcHd4c7lcOMbcDrV2lrf8martNaVUsxOiGL/6Ty2HMl2+PaFEO5BAr01KQWD74QHt0DUGPjyaUi6vlVa61Niu9ExwJuk5KMO37YQwj1IoDtDUDe4YxlMfQvO7rX2W9/8OlQ47kNMP28z04dH8J+9ZziRXeiw7Qoh3IcEurMoBbHT4YGt0Gsc/PsZWHgdnDvksF3cPSoSpRSLpQujEG2SBLqzBYXD9KVw09uQeQDeugKSX3VIaz082J+JA7uydNtxCkvKHFCsEMKdSKAbQSmIuQ0e3Aq9r4J1z8KCCZB5sMWbnhVvIbeojFU7pQujEG2NBLqRArvC7R/Aze9A1k/W1vqmf7SotT40siPR3YN54fP9PLhkB4u3HOOns/nSnVGINkA584UeFxenU1JSnLY/t5J3Bj57DPavhe5Drd0dO/dr1qYOncnjzf8eZsvhLE7lWG9T1znQl5G9QonvHcqo3qFEhARgHdJeCOHqlFLbtdZxDS4nge5CtIY9K+GzJ6AkH8b9HuIfBrNXMzenOZ5dyObDWSQfzmLzkSwy84oB6Bbsx8jeoYzqZQ34Hh0DHPmXCCEcSALdneWftY7euG8NdBsCU9+Azpe3eLNaaw5nFrD5SBZbbAGfXVACQERIgD3cR/UOpUuQX4v3J4RwDAl0T7BnlTXYi/Ng7O8g4ZFmt9ZrU1GhOXg2j82Hs9h8OIutR7PJuVgKQK9O7RjZ23qJZmSvUDq193XYfoUQTSOB7ikKzllHb9yzCsJjra31LgNaZVflFZp9GbnWgD+Sxbaj2eQXW7s/Xtalvb0FPyIqlI7tfFqlBiHEpSTQPc2e1dbWelGOtbV+xSNg9m7VXZaVV5B66ueA//5oNhdLy1EK+nUNsn7A2iuU4b1CCPJr3VqEaMsk0D1RQRZ88SSkfgxdB8HUN6HrQKftvqSsgh/SL9gDfvux8xSXVWBSMLB7MKN6hTKydyjDLCG093XcpSEh2joJdE+2d421i+PFCzDmSRj9WKu31mtTVFrOzuMX7B+y7jxxntJyjdmkiOkRbP2AtVcnhkZ2xN/H7PT6hPAUEuierjDbOtb6j8uha7S133r4IENLulhSzvZj59l85BybD2exOz2H8gqNj9lEbM8O9m6SgyM64OctAS9EY0mgtxX71sLaR+FiNox+AkY/Dl6u8YFlfnEZ36dl27tIpp7MoUKDr5eJoZEd7R+yDurRAR8v+dKyEHWRQG9LCrOtY63/sAy6DLT2hAmPMbqqS+RcLGXb0Wz7Nfh9GbkA+HubGRYVYg/4gd2C8DJLwAtRSQK9Ldr/Oax9xNrVMXY6hPaBwG7WER4DbT++7Y2u0u58QQlbj2bZv8l66Gw+AIG+XtUC/vLwIMwmGaZAtF0S6G1VYTas+6P1W6bFuZfO9w2yBntQ+KVhXzmtfWcwOf8ad2ZeMVuOZNk/ZD1yrgCAYH9vRkSF2L/FelnnQEwS8KINkUAXUJwPeRmQewryTkPeKcjNqPKYYZ2ua4zuqMzQvsvPYR/UzToypP0EYHv0DWzV8k/nFNk/YN18JIsT2RcBCGnnw8helS34TvQOaycDjQmP5lGB/qdNf2J35m6CfIII8g2yPtZ87hNEsG9wtel+XjIeSYMqyqEg0xb8NcI+99TP04tzLl3XJ9AW8LWEfeVju84OG64g/XyhfZiCzUeyyLCNJBkW6PvzODS9QokMlZEkhWfxqEBftGcRuzN3k1uSS25xrv0xrzSv3vV8TD51nwB8a5wEakyXk0ENJQXW1rw95Gs85p22PlbUuFOSMllDveYlnqBu1R99A603/mgkrTXHsgrZfOTngK8cSTI82M8e7jKSpPAEHhXodSmvKCe/NN8e8jklOZeEfm5J7c9bcjKo+U5ATgY2FRVQeK6W0K+8vGObVnTh0nW929Ud9pXX99t3rbO1bx9J8vA56zX4I9n2kSR7hvhXacF3omtwG/33EW7LKYGulJoI/AMwA+9orV+ob3lXuobuzJNBrSeAtnwyKCmsEvC2yzzVWv+2eRWlNVZU1g9sq4V+1Q91bdP8gqnQ2EeSTD6cxdYjWeQWWd89VI4kOaqXdSTJsEAZSVK4tlYPdKWUGTgIXAukA98D07XWe+tax5UCvSWqngxySnKqB76DTgbBPsENXi7y6JNBRQUUZtX4ALdmq/8UXDx/6breATVa+F2pCAwnvawDOy/48+0Zb746obhgvUJD387tie4RjJ+3GW+Twttswstswses8DKb8Dab8DYr26MJL7PCx/ZYdZ6XyYSPl8LLZF3O/tzLVGW7P29HumKKxmpsoLfk06rhwE9a6yO2HS4FbgTqDHRPYTaZCfYNJtg3mJ70bNK65RXl5JXk1R74tZwAzhSe4dD5Q+SW5JJfml/vtn3NvvaQD/AOwKRMmJW52qPJdOm0Sx5N9c9vzDJmZW7SvupcxscLc5gFU+detc8vL8FccA5TwTnM+Wcx5Z/FnHcGU/5pzHmnMZ3YginvNKbyEiKACKz/SbVSlHUM5by5EydLgjm5vx2lWlGmTZRpKKuwPlZgohwTZShKMFGBsv2YqNDWeRpFec15/DyvwjZfoyjXJvt8rRTKZMZkMqNMpirPzZjMtuNi/vl3s8mMyWzCZPLCbDZjNpsxmb2s88xmvMzW5bzMXphMXnh5mTCbvezTvb28bdO88fKyLu/l5YWXl22e2Ww9+VSeuEwKH68aJzGTwmxS8qGzi2pJoHcHTlT5PR0Y0bJyPJ/ZZKaDXwc6+HVo8rplFWXkl+TXezKofMdwsewi5bqcCl1BuS6npKLE+ntFRbXpFbqi2vO6lqltHZfnBXQEOnbFhAmTUphR1kcNJjRmXYxJn8akK1D2N6s13rVe8i5WU1ucNXYaUGVfDS9b+/oayrD+NLeG2ibqyofat6Sr/Vbb6q4Z9IpL/lWrzHGO5+JfI27AuFbdR0sCvfZ/z5oLKTUHmAMQERHRgt0JL5NXs08Gjqa1rv1kUPWxov6TQkMnD0dso2YtdS1T26VHXUcE2JfVGtCgNVpXWJ+jba8C27TKZaosrytPhrrCviyV61f+Xrld22PV7VTfbtX9VV9PV1RQoW3TtaZCV9Zq/berXF5XWB8rbPVoe+3Wfek6tg/2kqjll0bTtf1WZaKqdbmGt1bfyUfbptV+xbmlHUVqX9/fr/W/pd2SQE+HatcbegCnai6ktX4beBus19BbsD/hQpRSmJUZM2a8kZtbCOEKWjIC0vdAX6VUlFLKB7gdWOOYsoQQQjRVs1voWusypdRvgH9j7ba4QGu9x2GVCSGEaJIWfSdba/058LmDahFCCNECMui0EEJ4CAl0IYTwEBLoQgjhISTQhRDCQ0igCyGEh3Dq8LlKqUzgWDNX7wScc2A5jiJ1NY3U1TRSV9N4al2RWuuwhhZyaqC3hFIqpTGjjTmb1NU0UlfTSF1N09brkksuQgjhISTQhRDCQ7hToL9tdAF1kLqaRupqGqmradp0XW5zDV0IIUT93KmFLoQQoh4uFehKqQVKqbNKqdQ65iul1CtKqZ+UUj8opYa4SF3jlFI5Sqldtp8/Oamunkqp9UqpfUqpPUqp39ayjNOPWSPrcvoxU0r5KaW2KaV22+r6n1qW8VVKLbMdr61KKYuL1JWolMqscrx+3dp1Vdm3WSm1Uym1tpZ5Tj9ejazLkOOllEpTSv1o2+clN1Bu9dejtt3BxBV+gDHAECC1jvnXA19gvdnISGCri9Q1DlhrwPEKB4bYngdivWl3f6OPWSPrcvoxsx2D9rbn3sBWYGSNZR4A3rI9vx1Y5iJ1JQKvOfv/mG3fjwEf1PbvZcTxamRdhhwvIA3oVM/8Vn09ulQLXWu9EciuZ5Ebgfe01Ragg1Iq3AXqMoTWOkNrvcP2PA/Yh/Ver1U5/Zg1si6nsx2Dyjtte9t+an6IdCOwyPZ8BXC1auU7IjeyLkMopXoAk4B36ljE6cerkXW5qlZ9PbpUoDdCbTemNjwobEbZ3jJ/oZQa4Oyd297qDsbauqvK0GNWT11gwDGzvU3fBZwF/qO1rvN4aa3LgBwg1AXqArjF9jZ9hVKqZy3zW8P/A54C6roruCHHqxF1gTHHSwPrlFLblfV+yjW16uvR3QK9UTemNsAOrF/NjQFeBVY7c+dKqfbAx8AjWuvcmrNrWcUpx6yBugw5Zlrrcq11LNZ74A5XSg2ssYghx6sRdX0KWLTWg4Cv+LlV3GqUUpOBs1rr7fUtVsu0Vj1ejazL6cfLJkFrPQS4DnhQKTWmxvxWPV7uFuiNujG1s2mtcyvfMmvrXZy8lVKdnLFvpZQ31tBcorVeWcsihhyzhuoy8pjZ9nkB2ABMrDHLfryUUl5AME683FZXXVrrLK11se3XfwFDnVBOAjBFKZUGLAWuUkq9X2MZI45Xg3UZdLzQWp+yPZ4FVgHDayzSqq9Hdwv0NcAM2yfFI4EcrXWG0UUppbpWXjdUSg3HelyznLBfBbwL7NNa/72OxZx+zBpTlxHHTCkVppTqYHvuD1wD7K+x2Bpgpu35NOAbbfs0y8i6alxnnYL1c4lWpbX+vda6h9bagvUDz2+01nfVWMzpx6sxdRlxvJRS7ZRSgZXPgfFAzZ5xrfp6bNE9RR1NKfUh1t4PnZRS6cBcrB8QobV+C+v9S68HfgIKgVkuUtc04H6lVBlwEbi9tf9T2yQAdwM/2q6/AjwDRFSpzYhj1pi6jDhm4cAipZQZ6wnkI631WqXUc0CK1noN1hPRYqXUT1hbmre3ck2NrethpdQUoMxWV6IT6qqVCxyvxtRlxPHqAqyytVO8gA+01l8qpe4D57we5ZuiQgjhIdztkosQQog6SKALIYSHkEAXQggPIYEuhBAeQgJdCCE8hAS6EEJ4CAl0IYTwEBLoQgjhIf4/6hUoxKYp8n8AAAAASUVORK5CYII=\n", 158 | "text/plain": [ 159 | "
" 160 | ] 161 | }, 162 | "metadata": {}, 163 | "output_type": "display_data" 164 | } 165 | ], 166 | "source": [ 167 | "%matplotlib inline\n", 168 | "import matplotlib.pyplot as plt\n", 169 | "\n", 170 | "plt.plot(np.log10(Ns),vars1)\n", 171 | "plt.plot(np.log10(Ns),vars3)\n", 172 | "plt.plot(np.log10(Ns),vars2)\n", 173 | "\n", 174 | "#plt.plot(vars1)\n", 175 | "#plt.plot(vars2)\n", 176 | "plt.legend(['REINFORCE', 'REINFORCE+Variance reduction (mean)','Reparametrization'])" 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": {}, 182 | "source": [ 183 | "Variance of the estimates using reparameterization trick is one order of magnitude smaller than the estimates from the first method!" 184 | ] 185 | } 186 | ], 187 | "metadata": { 188 | "kernelspec": { 189 | "display_name": "Python 3", 190 | "language": "python", 191 | "name": "python3" 192 | }, 193 | "language_info": { 194 | "codemirror_mode": { 195 | "name": "ipython", 196 | "version": 3 197 | }, 198 | "file_extension": ".py", 199 | "mimetype": "text/x-python", 200 | "name": "python", 201 | "nbconvert_exporter": "python", 202 | "pygments_lexer": "ipython3", 203 | "version": "3.6.8" 204 | } 205 | }, 206 | "nbformat": 4, 207 | "nbformat_minor": 1 208 | } 209 | -------------------------------------------------------------------------------- /codes/VAE/REINFORCE_VS_Reparametrization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Reparameterization Trick\n", 8 | "Following https://gokererdogan.github.io/2016/07/01/reparameterization-trick/" 9 | ] 10 | }, 11 | { 12 | "cell_type": "markdown", 13 | "metadata": {}, 14 | "source": [ 15 | "Considering a normal distribution with mean $\\mu$ and unit variance, $q_{\\mu}(x) = N(\\mu,1)$, and we want to adjust $\\theta$ for optimizing the second moment\n", 16 | "\n", 17 | "$\n", 18 | "\\hat\\mu=\\arg\\min_{\\mu} \\mathcal L(\\mu)=\\arg\\min_{\\mu}\\mathbb E_q[x^2]\n", 19 | "$\n", 20 | "\n", 21 | "Apparently, analytically we have $\\nabla_\\mu\\mathcal L=\\nabla_\\mu(\\mu^2+1)=2\\mu$.\n", 22 | "\n", 23 | "But we want to compare numerical computation of this quantity using both REINFORCE and the Reparametrization method.\n", 24 | "\n", 25 | "For the REINFORMCE algorithm, we have \n", 26 | "\n", 27 | "$\\nabla_\\mu\\mathcal L=\\mathbb E_q(x^2\\nabla_\\mu\\log q(x))=\\mathbb E_q(x^2(x-\\mu))=\\mathbb E_q(x^3-x^2\\mu))=\\mu^3+3\\mu-\\mu(\\mu^2+1)=2\\mu$\n", 28 | "\n", 29 | "One way to keep the expectation of gradients, while reduce the variance of the gradients is add a *baseline* which is not a function of $x$. A very simple approach is use the mean such that\n", 30 | "\n", 31 | "$\\nabla_\\mu\\mathcal L=\\mathbb E_q([x^2-\\mathbb E_q(x^2)]\\nabla_\\mu\\log q(x))=\\mathbb E_q([x^2-\\mathbb E_q(x^2)](x-\\mu))=2\\mu$\n", 32 | "\n", 33 | "For the reparametrization,\n", 34 | "$\n", 35 | "x = \\mu + \\epsilon, \\quad \\epsilon \\sim \\mathcal{N}(0,1)\n", 36 | "$\n", 37 | "\n", 38 | "Then we have\n", 39 | "$\\nabla_\\mu\\mathcal L=\\nabla_\\mu\\mathbb E_{\\mathcal N(0,1)} (\\mu+\\epsilon)^2=\\mathbb E_{\\mathcal N(0,1)}\\nabla_\\mu (\\mu+\\epsilon)^2=\\mathbb E_{\\mathcal N(0,1)}[2(\\mu+\\epsilon)]=2\\mu$\n", 40 | "\n", 41 | "Let have some impression on the gradient estimate given by three methods by comparing them on a single instance:" 42 | ] 43 | }, 44 | { 45 | "cell_type": "code", 46 | "execution_count": 9, 47 | "metadata": {}, 48 | "outputs": [ 49 | { 50 | "name": "stdout", 51 | "output_type": "stream", 52 | "text": [ 53 | "grad of REINFORCE= 4.097396172279478\n", 54 | "grad of REINFORCE + baseline= 4.01904525710015\n", 55 | "grad of Reparametrization = 4.031061372628896\n" 56 | ] 57 | } 58 | ], 59 | "source": [ 60 | "import numpy as np\n", 61 | "N = 1000\n", 62 | "theta = 2.0\n", 63 | "eps = np.random.randn(N)\n", 64 | "x = theta + eps\n", 65 | "\n", 66 | "grad1 = lambda x: np.sum(np.square(x)*(x-theta)) / x.size\n", 67 | "grad2 = lambda eps: np.sum(2*(theta + eps)) / x.size\n", 68 | "grad3 = lambda x: np.sum((np.square(x)-np.sum(np.square(x))/x.size)*(x-theta)) / x.size\n", 69 | "\n", 70 | "print(\"grad of REINFORCE=\",grad1(x))\n", 71 | "print(\"grad of REINFORCE + baseline=\",grad3(x))\n", 72 | "print(\"grad of Reparametrization = \",grad2(eps))\n" 73 | ] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "metadata": {}, 78 | "source": [ 79 | "Let us check the variance for different sample sizes." 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 8, 85 | "metadata": {}, 86 | "outputs": [ 87 | { 88 | "name": "stdout", 89 | "output_type": "stream", 90 | "text": [ 91 | "[4.55637839 4.10093377 4.03974593 3.98094517 4.00103464]\n", 92 | "[4.11529199 4.02050707 3.99739092 3.99746569 3.99969491]\n", 93 | "[3.89359207 3.98636754 4.01466151 3.99127621 4.00268074]\n", 94 | "[1.54686336e+01 9.41211946e-01 1.28511151e-01 8.85848125e-03\n", 95 | " 8.06813959e-04]\n", 96 | "[3.18476310e-01 3.68889932e-02 3.64861942e-03 2.92998741e-04\n", 97 | " 2.77811917e-05]\n", 98 | "[7.63648618e+00 4.21657403e-01 5.56574377e-02 3.59091375e-03\n", 99 | " 3.74900223e-04]\n" 100 | ] 101 | } 102 | ], 103 | "source": [ 104 | "Ns = [10, 100, 1000, 10000, 100000]\n", 105 | "reps = 100\n", 106 | "\n", 107 | "means1 = np.zeros(len(Ns))\n", 108 | "vars1 = np.zeros(len(Ns))\n", 109 | "means2 = np.zeros(len(Ns))\n", 110 | "means3 = np.zeros(len(Ns))\n", 111 | "vars2 = np.zeros(len(Ns))\n", 112 | "vars3 = np.zeros(len(Ns))\n", 113 | "\n", 114 | "est1 = np.zeros(reps)\n", 115 | "est2 = np.zeros(reps)\n", 116 | "est3 = np.zeros(reps)\n", 117 | "for i, N in enumerate(Ns):\n", 118 | " for r in range(reps):\n", 119 | " x = np.random.randn(N) + theta\n", 120 | " est1[r] = grad1(x)\n", 121 | " est3[r] = grad3(x)\n", 122 | " eps = np.random.randn(N)\n", 123 | " est2[r] = grad2(eps)\n", 124 | " means1[i] = np.mean(est1)\n", 125 | " means2[i] = np.mean(est2)\n", 126 | " means3[i] = np.mean(est3)\n", 127 | " vars1[i] = np.var(est1)\n", 128 | " vars2[i] = np.var(est2)\n", 129 | " vars3[i] = np.var(est3)\n", 130 | " \n", 131 | "print(means1)\n", 132 | "print(means2)\n", 133 | "print(means3)\n", 134 | "print\n", 135 | "print (vars1)\n", 136 | "print (vars2)\n", 137 | "print (vars3)" 138 | ] 139 | }, 140 | { 141 | "cell_type": "code", 142 | "execution_count": 3, 143 | "metadata": {}, 144 | "outputs": [ 145 | { 146 | "data": { 147 | "text/plain": [ 148 | "" 149 | ] 150 | }, 151 | "execution_count": 3, 152 | "metadata": {}, 153 | "output_type": "execute_result" 154 | }, 155 | { 156 | "data": { 157 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VNX9//HXmckOSSAhQFiSCYtFICRA2BJZ3AAFERWruECgFbdq3WutLXy1334t2vqre61CEFEQBERcSlUoSlgMm4ZdIEAgQEggK9nP74+ZjEnInsncmcnn+XjkMZO7fnJh3vfMnTPnKq01Qggh3J/J6AKEEEI4hgS6EEJ4CAl0IYTwEBLoQgjhISTQhRDCQ0igCyGEh5BAF0IIDyGBLoQQHkICXQghPISXM3fWqVMnbbFYnLlLIYRwe9u3bz+ntQ5raDmnBrrFYiElJcWZuxRCCLenlDrWmOXkkosQQngICXQhhPAQEuhCCOEhnHoNXRijtLSU9PR0ioqKjC5FCFEPPz8/evTogbe3d7PWl0BvA9LT0wkMDMRisaCUMrocIUQttNZkZWWRnp5OVFRUs7Yhl1zagKKiIkJDQyXMhXBhSilCQ0Nb9E66wUBXSi1QSp1VSqVWmRailPqPUuqQ7bFjsysQTiFhLoTra+nrtDEt9CRgYo1pTwNfa637Al/bfm81n/2QwZKtjeqGKYQQbVaDga613ghk15h8I7DI9nwRMNXBdVXz+Y8ZzP/yAIUlZa25G9GKzGYzsbGxDBw4kBtuuIELFy4AkJaWhr+/P7Gxsfaf9957D7B+Ee3cuXOAteXy+OOP27f30ksvMW/ePADmzZtH9+7d7es//bS1fVFSUsIjjzxC79696du3LzfeeCPp6ekN1gRw8OBBrr/+evr06cPll1/OL3/5S86cOcOGDRsIDg6uVu9XX33VqsdOiMZq7jX0LlrrDADbY2fHlXSpxAQLORdLWb3zVGvuRrQif39/du3aRWpqKiEhIbz++uv2eb1792bXrl32nxkzZlyyvq+vLytXrrQHfE2PPvqoff0XXngBgGeeeYa8vDwOHjzIoUOHmDp1KjfffDOVN0avq6aioiImTZrE/fffz08//cS+ffu4//77yczMBGD06NHV6r3mmmsceqyEaK5W/1BUKTVHKZWilEqpfEE0VVxkRwZ2DyIp+aj9xSjc16hRozh58mST1vHy8mLOnDm8/PLLjVq+sLCQhQsX8vLLL2M2mwGYNWsWvr6+fPPNN/XW9MEHHzBq1ChuuOEG+/wrr7ySgQMHNqlmIZytud0WzyilwrXWGUqpcOBsXQtqrd8G3gaIi4trVhorpUiMj+KJ5btJPpxFQp9Ozata8D+f7mHvqVyHbrN/tyDm3jCgUcuWl5fz9ddf86tf/co+7fDhw8TGxtp/f/XVVxk9evQl6z744IMMGjSIp5566pJ5L7/8Mu+//z4Af/3rXwkPDyciIoKgoKBqy8XFxbFnzx6uvvrqOmtKTU1l6NChdf4N3377bbV6P/74Y3r37t3Qny5Eq2tuoK8BZgIv2B4/cVhFdZg8KJz/+3wfCzelSaC7oYsXLxIbG0taWhpDhw7l2muvtc+rvOTSkKCgIGbMmMErr7yCv79/tXmPPvooTzzxhP333bt319pjQGttn15fTfUZPXo0a9eubdSyQjhTg4GulPoQGAd0UkqlA3OxBvlHSqlfAceBW1uzSAA/bzN3jojg1fU/cSyrgMjQdq29S4/U2Ja0o1Ver87JyWHy5Mm8/vrrPPzww03eziOPPMKQIUOYNWtWvcv16dOHY8eOkZeXR2BgoH36jh077JdS6qppwIAB/Pe//21ybUIYrTG9XKZrrcO11t5a6x5a63e11lla66u11n1tjzV7wbSKO0dGYlaK9zZLF0Z3FRwczCuvvMJLL71EaWlpk9cPCQnhl7/8Je+++269y7Vr146ZM2fy2GOPUV5eDsB7771HYWEhV111Vb013XHHHSQnJ/PZZ5/Zl/nyyy/58ccfm1yvEM7kVt8U7RLkx/XR4Xz0/QkKiqULo7saPHgwMTExLF26FPj5GnrlzyuvvFLv+o8//nidvV2q+r//+z/8/Py47LLL6Nu3L8uXL2fVqlW1XoqpWpO/vz9r167l1VdfpW/fvvTv35+kpCQ6d7Z25qq8hl75s2LFimYcBSEcTzmz10hcXJxu6Q0udh4/z01vJPPcjQOYMcrimMI83L59+7j88suNLkMI0Qi1vV6VUtu11nENretWLXSAwREdienZgaTkNCoqpAujEEJUcrtAB5gVb+FIZgHf/tTw224hhGgr3DLQr48OJyzQl4WbjhpdihBCuAy3DHQfLxN3jYhkw4FMjmTmG12OEEK4BLcMdIA7RkTgYzZJF0YhhLBx20APC/Rlckw4y1NOkFvU9P7MQgjhadw20AFmxUdRUFLOipT0hhcWhmrrw+cmJSUxffr0atPOnTtHWFgYxcXFjT6Ob731lv34uIPExMRm99P/y1/+Uu33+Ph4R5QEWL9xvHHjRodtrylee+01Fi5c2Dob11o77Wfo0KHa0W5+Y5MeM/8bXV5e4fBte4q9e/caXYJu166d/fmMGTP0n//8Z6211kePHtUDBgyodZ3IyEidmZmptdba19dXWywW++8vvviinjt3rtZa67lz5+oXX3zxkvUff/xxPXv2bF1WVqa11nrBggV62LBhuqKiot6aLl68qPv06aPXrFljn//NN9/oH3/8Ua9fv15PmjSp3r91/fr1eubMmdWm5eTk6NDQUF1QUGCf9uabb+rZs2fXu62qSktLG71sa2pKHTNnztTLly9v1n6q/vs4UlZWlh4xYkSrbLsxCgoKdGxsbJ3za3u9Aim6ERnr1i10gFkJFo5lFbL+QJ0DPgoX0xaHzw0KCmLMmDF8+umn9mlLly61t9qfe+45hg0bxsCBA5kzZ459mOhx48bxzDPPMHbsWP7xj38wb948XnrpJQD+9a9/MWzYMGJiYrjlllsoLCwErK3ihx9+mPj4eHr16lWthTx//nyio6OJiYmxv5M5fPgwEydOZOjQoYwePZr9+/dfUv+8efOYM2cO48ePZ8aMGZSXl/Pkk08ybNgwBg0axD//+U/A2kD8zW9+Q//+/Zk0aRJnz/78uqz6jislJYVx48YBkJ+fz6xZs4iOjmbQoEF8/PHHPP300/bB0+68804A2rdvb9/Hk08+ycCBA4mOjmbZsmUAbNiwgXHjxjFt2jT69evHnXfeWetw2ytWrGDixJ9vwmaxWHjmmWcYNWoUcXFx7NixgwkTJtC7d2/eeust+3Ivvvii/e+dO3euffrUqVMZOnQoAwYM4O2337ZPb9++PX/4wx+IiYlh5MiRnDlzBoCAgAAsFgvbtm27pLaWau5oiy5jwoCudA3yIyk5jasv72J0Oa7vi6fhtIPHJOkaDde90KhF2/LwudOnT+eDDz7gtttu49SpUxw8eJArr7wSgN/85jf86U9/AuDuu+9m7dq19hPKhQsX7IOFVV5mArj55pu55557AHj22Wd59913eeihhwDIyMjgu+++Y//+/UyZMoVp06bxxRdfsHr1arZu3UpAQADZ2dYhmObMmcNbb71F37592bp1Kw888ECtJ73t27fz3Xff4e/vz9tvv01wcDDff/89xcXFJCQkMH78eHbu3MmBAwf48ccfOXPmDP3792f27Nn1Hpfnn3+e4OBg+1g558+f55ZbbuG1116rdRTOlStXsmvXLnbv3s25c+cYNmwYY8aMAWDnzp3s2bOHbt26kZCQwKZNm7jiiiuqrb9p0yamTZtWbVrPnj3ZvHkzjz76KImJiWzatImioiIGDBjAfffdx7p16zh06BDbtm1Da82UKVPYuHEjY8aMYcGCBYSEhHDx4kWGDRvGLbfcQmhoKAUFBYwcOZL//d//5amnnuJf//oXzz77LGD9f/jtt98yfPjweo9NU7l9oHubTdw9KpIX/32AQ2fy6NslsOGVhNO1heFzR4wYQXFxMfn5+WRnZ9tD/69//SsTJkxg8uTJPPDAA+Tm5vLRRx8xbdo0+7uH9evXM3/+fAoLC8nOzmbAgAH2QL/ttttqrSM1NZVnn32WCxcukJ+fz4QJE+zzpk6dislkon///vaW4VdffcWsWbMICAgArAOd5efnk5yczK23/jxgal3X9KdMmWI/7uvWreOHH36wt/5zcnI4dOgQGzduZPr06ZjNZrp163bJQGi1+eqrr+zj+gB07Fj/Pee/++47+z66dOnC2LFj+f777wkKCmL48OH06NEDwP5vWzPQMzIyCAsLu+RvA4iOjiY/P5/AwEACAwPx8/PjwoULrFu3jnXr1jF48GDA+q7i0KFDjBkzhldeeYVVq1YBcOLECQ4dOkRoaCg+Pj5MnjwZgKFDh/Kf//zHvr/OnTvX+k6opdw+0AGmD4/gH18fIik5jf+9KdroclxbI1vSjtYWhs/dunUrYH3rn5SURFJSUrX5/v7+TJw4kVWrVrF06VL75aOioiIeeOABUlJS6NmzJ/PmzaOoqMi+Xrt2tQ8VnZiYyOrVq4mJiSEpKYkNGzbY5/n6+tqfV152qHoyq1RRUUGHDh0adUKtWofWmldffbXaSQTg888/r/PO9V5eXlRUVNj/5qrbasrd7mu7jFKp6t9tNpspK7t0ED9/f/9q+6+6nslkqrYNk8lEWVkZWmt+//vfc++991Zbb8OGDXz11Vds3ryZgIAAxo0bZ9+2t7e3/e+qWUtRUdEljRJHcPtr6AAh7XyYGtuNlTtOklMoXRhdWVsfPnf69On8/e9/58yZM4wcORL4Odw6depEfn5+o3uF5OXlER4eTmlpKUuWLGlw+fHjx7NgwQL7tfbs7GyCgoKIiopi+fLlgDUsd+/e3eC2JkyYwJtvvmn/Nzx48CAFBQWMGTOGpUuXUl5eTkZGBuvXr7evY7FY2L59O2C9TFW1rtdee83++/nz5wFrINb2f2TMmDEsW7aM8vJyMjMz2bhxY5MuXVx++eX89NNPjV6+8u9dsGAB+fnWLzKePHmSs2fPkpOTQ8eOHQkICGD//v1s2bKlUds7ePBgq9zS0CMCHSAxPoqLpeV8lHLC6FJEA9ry8Lnjx4/n1KlT3HbbbfY6OnTowD333EN0dDRTp05l2LBhjdrW888/z4gRI7j22mvp169fg8tPnDiRKVOmEBcXR2xsrP3D1SVLlvDuu+8SExPDgAED+OSThm9A9utf/5r+/fszZMgQBg4cyL333ktZWRk33XQTffv2JTo6mvvvv5+xY8fa15k7dy6//e1vGT16tP1SE1iv/58/f56BAwcSExNjPwnMmTOHQYMG2T8UrXTTTTcxaNAgYmJiuOqqq5g/fz5du3Zt1DEDmDRpUrV3M40xfvx47rjjDkaNGkV0dDTTpk0jLy+PiRMnUlZWxqBBg/jjH/9oP0k3ZNOmTa1yc3G3Gz63Prf9czMnL1zkv09eidnU+Ldwnk6GzxWiuiuuuIK1a9fSoUMHp+97586d/P3vf2fx4sW1zm9Tw+fWZ1aChfTzF/lq3xmjSxFCuLC//e1vHD9+3JB9nzt3jueff75Vtu0RH4pWuubyLnTv4M/CTUeZMKDxb8GEEG3LiBEjDNt3Y3tTNYdHtdC9zCZmjIpky5Fs9mXkGl2OEEI4lUcFOsBtw3ri521iUXKa0aUIIYRTeVygdwjw4eYhPVi18yTZBSVGlyOEEE7jcYEOkBhvobisgqXfG/OhhxBCGMEjA/2yLoEk9All8eZjlJVXGF2OoP6hat1NUlISp06dqnP+n/70p0YNqVvT6tWr2bt3b4u3I9oujwx0sI6VnpFTxL/3SBdGV1D5NfvU1FRCQkJ4/fXXW3V/tX3l21HqC/Ty8nKee+65Zn1ppGagN3c7ou3y2EC/sl9nIkICSEqWG0m7mprD59Y2LGlaWhr9+vVj5syZDBo0iGnTptm/st7YoWY//fRTRowYweDBg7nmmmvsg1TNmzePmTNnMn78eCwWCytXruSpp54iOjqaiRMn2r9uvn37dsaOHcvQoUOZMGECGRkZrFixgpSUFO68805iY2O5ePEiFouF5557jiuuuILly5fbb+qQkpJi/zZpdHS0/ZuhtQ17m5yczJo1a3jyySeJjY3l8OHD1W4O8fXXXzN48GCio6OZPXu2fQAti8XC3LlzGTJkCNHR0a0y4JNwHx7VD70qs0kxY1Qkf/5sH6kncxjYPdjoklzCX7f9lf3Zjn3R9wvpx++G/65Ry9YcqrauYUkjIiI4cOAA7777LgkJCcyePZs33niDJ554otFDzZ4/f54tW7aglOKdd95h/vz5/O1vfwOsww2sX7+evXv3MmrUKD7++GPmz5/PTTfdxGeffcakSZN46KGH+OSTTwgLC2PZsmX84Q9/YMGCBbz22mu89NJLxMX9/MU9Pz8/vvvuO8A67gtYh0itHPTqySeftI/BXdewt1OmTGHy5MmXDO1aVFREYmIiX3/9NZdddhkzZszgzTff5JFHHgGsY8Ds2LGDN954g5deeol33nmnKf98woN4bAsd4JfDehLgY2bhpjSjS2nzKoeqDQ0NJTs72/7liqrDkg4ZMoT9+/dz6NAhwDpGdUJCAgB33XWXPTDXr1/PiBEjiI6O5ptvvmHPnj32/VQdajY9PZ0JEyYQHR3Niy++WG256667Dm9vb6KjoykvL7eHbXR0NGlpaRw4cIDU1FSuvfZaYmNj+fOf/1zt9nU11TXELcBHH33Ejh07eOEF60iXqampjB49mujoaJYsWVKtrtocOHCAqKgoLrvsMgBmzpxZ7fZpN998M2AdojUtLa3ebQnP5rEtdIAgP2+mDe3B0m0n+P31/ejU3rfhlTxcY1vSjlbXULV1DUualpZ2ySBaSqkmDTX70EMP8dhjjzFlyhQ2bNhQ7eYQVYdLrTrMadXhUgcMGMDmzZsb9ffVNcTtnj17mDt3Lhs3brQPSFXfsLe1aWi8pcq/pa7hYkXb4dEtdICZ8RZKyiv4cKt0YXQFNYeqrWtYUoDjx4/bA/XDDz/kiiuuaNJQszk5OXTv3h2ARYsWNanOX/ziF2RmZtr3X1paam9JBwYGkpeX1+A2cnJyuP3223nvvfeq3VChrmFv69puv379SEtLsw/5unjx4mqjGApRqUWBrpR6VCm1RymVqpT6UCnl56jCHKV3WHvGXhbG4i3HKCmTLoyuoOpQtXUNSwrWcasXLVrEoEGDyM7O5v7772/SULPz5s3j1ltvZfTo0XTq1KlJNfr4+LBixQp+97vfERMTQ2xsLMnJyYC1hX3ffffZPxSty+rVqzl27Bj33HOP/cNRqHvY29tvv50XX3yRwYMHc/jwYft0Pz8/Fi5cyK233kp0dDQmk4n77ruvSX+PaBuaPXyuUqo78B3QX2t9USn1EfC51jqprnVae/jcuqw/cJZZC7/nH7fHcmNsd6fv32juOHxuWloakydPJjU11ehShHAqI4fP9QL8lVJeQABQ97ctDDS2bxi9OrUjScZ3EUJ4sGYHutb6JPAScBzIAHK01uscVZgjmUyKmfEWdh6/wK4T7vsNxbbEYrFI61yIJmp2oCulOgI3AlFAN6CdUuquWpabo5RKUUqlZGZmNr/SFrplaA8Cfb1I2tQ2v2jkzDtTCSGap6Wv05ZccrkGOKq1ztRalwIrgfiaC2mt39Zax2mt46p+0u9s7X29uDWuJ5/9mMHZ3KKGV/Agfn5+ZGVlSagL4cK01mRlZeHn1/y+JS3ph34cGKmUCgAuAlcDzv/EswlmjIpkYfJR3t96nMeuvczocpymR48epKenY+Q7JCFEw/z8/OjRo0ez1292oGuttyqlVgA7gDJgJ/B2sytxAkundlz1i858sPUYD17ZG18vc8MreQBvb2+ioqKMLkMI0cpa1MtFaz1Xa91Paz1Qa3231rrYUYW1llkJUZzLL2Ht7gyjSxFCCIfy+G+K1pTQJ5Q+nduTlJwm15SFEB6lzQW6UorEeAs/nsxhx/HzRpcjhBAO0+YCHeDmId0J8vNigYzCKITwIG0y0AN8vLh9eARfpp4mI6fusTiEEMKdtMlAB7h7ZCRaa97fcszoUoQQwiHabKD3DAng2v5d+GDrcYpKy40uRwghWqzNBjpAYnwU5wtLWbPLJccUE0KIJmnTgT6yVwj9ugayULowCiE8QJsOdKUUsxIs7MvIZevRbKPLEUKIFmnTgQ5wY2x3OgZ4kyRdGIUQbq7NB7qft5npwyNYt/c06ecLjS5HCCGarc0HOsBdIyNRSrF4s3RhFEK4Lwl0oFsHfyYO6MqH245TWFJmdDlCCNEsEug2sxIs5BaVsXqndGEUQrgnCXSboZEdGdg9iKTko9KFUQjhliTQbayjMEZx8Ew+yYezjC5HCCGaTAK9ihtiwunU3oeFbfRG0kII9yaBXoWvl5k7hkfw9f6zHMsqMLocIYRoEgn0Gu4cGYlZKd6TLoxCCDcjgV5DlyA/Jg0K56PvT5BfLF0YhRDuQwK9FonxFvKKy1i5I93oUoQQotEk0GsxOKIjMT07kJScRkWFdGEUQrgHCfQ6zE6wcCSzgI2HMo0uRQghGkUCvQ7XDQwnLNCXpOQ0o0sRQohGkUCvg4+XibtGRLLhQCZHMvONLkcIIRokgV6PO0ZE4GM2sUha6UIINyCBXo+wQF8mx4SzYns6uUWlRpcjhBD1kkBvwKz4KApKylmRIl0YhRCuTQK9AdE9gomL7MiizWmUSxdGIYQLk0BvhMQEC8eyCtlw4KzRpQghRJ0k0BthwoCuhAf7SRdGIYRLa1GgK6U6KKVWKKX2K6X2KaVGOaowV+JtNnHXyEi+PXSOQ2fyjC5HCCFq1dIW+j+AL7XW/YAYYF/LS3JN04dH4Otlkla6EMJlNTvQlVJBwBjgXQCtdYnW+oKjCnM1Ie18mBrbnZU7TpJTKF0YhRCupyUt9F5AJrBQKbVTKfWOUqpdzYWUUnOUUilKqZTMTPceF2VmvIWLpeUsSzludClCCHGJlgS6FzAEeFNrPRgoAJ6uuZDW+m2tdZzWOi4sLKwFuzNe/25BjIgKYVHyMenCKIRwOS0J9HQgXWu91fb7CqwB79FmJURx8sJF/rP3jNGlCCFENc0OdK31aeCEUuoXtklXA3sdUpULu+byznTv4E9SstxIWgjhWlray+UhYIlS6gcgFvhLy0tybV5mEzNGRbLlSDb7MnKNLkcIIexaFOha61226+ODtNZTtdbnHVWYK7t9WAT+3maSNqUZXYoQQtjJN0WbITjAm5uGdGf1rpNkF5QYXY4QQgAS6M2WGG+huKyCpd9LF0YhhGuQQG+my7oEckWfTizefIzS8gqjyxFCCAn0lkiMt5CRU8S6PdKFUQhhPAn0FriyX2ciQgKkC6MQwiVIoLeA2aSYGW/h+7TzpJ7MMbocIUQbJ4HeQrfG9aCdj5mF0oVRCGEwCfQWCvLzZtrQHny6+xTn8ouNLkcI0YZJoDvAjHgLJeUVfLBVujAKIYwjge4AvcPaM/ayMN7fcoySMunCKIQwhgS6g8xKsHA2r5gvUjOMLkUI0UZJoDvImL5h9OrUTj4cFUIYRgLdQUy2Loy7Tlxg5/E2MUaZEMLFSKA70C1DexDo6yU3khZCGEIC3YHa+3pxa1xPPvshgzO5RUaXI4RoYyTQHWxmfCTlWrNEujAKIZxMAt3BIkPbcXW/znyw9RjFZeVGlyOEaEMk0FtBYnwU5/JLWLtbujAKIZxHAr0VJPQJpW/n9iQlp6G1NrocIUQbIYHeCpRSJCZY+PFkDtuPSRdGIYRzSKC3kpsGdyfIz4uF0oVRCOEkEuitJMDHi9uHR/Bl6mkyci4aXY4Qog2QQG9Fd4+MRGvN4s3HjC5FCNEGSKC3op4hAVzbvwsfbjtOUal0YRRCtC4J9FY2KyGK84WlrNl1yuhShBAeTgK9lY2ICqFf10AWbDoqXRiFEK1KAr2VKaWYlWBh/+k8th7NNrocIYQHk0B3ghtju9MxwJskGStdCNGKJNCdwM/bzPThEazbe5oT2YVGlyOE8FAS6E5y18hIlFK8v0W6MAohWkeLA10pZVZK7VRKrXVEQZ6qWwd/Jg7syofbjlNYUmZ0OUIID+SIFvpvgX0O2I7HmxVvIbeojFU7TxpdihDCA7Uo0JVSPYBJwDuOKcezDY3syMDuQSRtklEYhRCO19IW+v8DngIq6lpAKTVHKZWilErJzMxs4e7cm1KKWfFRHDqbz6afsowuRwjhYZod6EqpycBZrfX2+pbTWr+ttY7TWseFhYU1d3ceY3JMOJ3a+5CUfNToUoQQHqYlLfQEYIpSKg1YClyllHrfIVV5MF8vM3cMj+Dr/Wc5llVgdDlCCA/S7EDXWv9ea91Da20Bbge+0Vrf5bDKPNhdIyMxK8WiZOnCKIRwHOmHboDOQX5MGhTO8pQT5BdLF0YhhGM4JNC11hu01pMdsa22IjHeQl5xGSt3pBtdihDCQ0gL3SCDIzoS27MDSZvSqKiQLoxCiJaTQDfQrAQLR84VsPFQ2+7OKYRwDAl0A103MJzOgb4kyY2khRAOIIFuIB8vE3eNjGTDgUwOZ+YbXY4Qws1JoBts+vAIfMwm3pNWuhCihSTQDRYW6MsNMd1YsT2d3KJSo8sRQrgxCXQXkBhvoaCknOUp0oVRCNF8EuguILpHMHGRHVmUnEa5dGEUQjSTBLqLmJUQxfHsQtbvP2t0KUIINyWB7iLGD+hCeLCfdGEUQjSbBLqL8DZbuzB+99M5Dp3JM7ocIYQbkkB3IdOHR+DrZWKhtNKFEM3gHoF+4QRcPG90Fa0upJ0PU2O7s3JHOjmF0oVRCNE07hHoq++H10fCgS+MrqTVJSZYKCqtYFnKcaNLEUK4GfcI9PF/hnad4MPbYeUcKMw2uqJWc3l4ECN7hbAo+Rhl5XXeqlUIIS7hHoHeLRbuWQ9jn4bUj+GNkbD/M6OrajWJ8VGcvHCRr/ZJF0YhROO5R6ADePnAlb+3Bnu7zrD0Dvj41x7ZWr+2fxe6d/CXG0kLIZrEfQK9UvggmLMexj0De1bB68Nh36dGV+VQZpNiZnw3NqU9AAATUklEQVQkW45ksy8j1+hyhBBuwv0CHcDsDeN+B3M2QGA4LLsLVsyGgiyjK3OY2+Ii8Pc2k7QpzehShBBuwj0DvVLXaLjnG7jyWdi7xtpa3/uJ0VU5RHCANzcN6c7qXSfJLigxuhwhhBtw70AHa2t97JNw738huDt8NAOWJ0LBOaMra7FZ8RaKyyr4cJt0YRRCNMz9A71SlwHw66/hqj9ae8C8Ptx6jd2N9e0SyBV9OvH+lmOUShdGIUQDPCfQwdpaH/MEzPkvdIiwttQ/mgH57nsT5sR4Cxk5Razbc8boUoQQLs6zAr1Sl/7wq6/g6rnWb5e+Ptzaf12731jjV/XrTGRoAAs3SRdGIUT9PDPQAcxeMPoxuPdbCImy9oJZdhfkuVdL12RSzBhlIeXYeX5MzzG6HCGEC/PcQK/UuR/MXgfX/A8c+g+8MQJ+WO5WrfVb43rQzscsY6ULIerl+YEO1tb6FY/Afd9BaB9Y+WtYeifknTa6skYJ8vNm2tAefLr7FJl5xUaXI4RwUW0j0CuFXQaz/20d7Ovw1/D6CNi9zC1a6zPiLZSUSxdGIUTd2lagA5jMEP+QtbUe9gtYNQc+nA65GUZXVq/eYe0Z94swFm85RkmZdGEUQlyq7QV6pU59YdYXMOEvcGS99dr6rg9durWeGG8hM6+YL1Jd++QjhDBGswNdKdVTKbVeKbVPKbVHKfVbRxbmFCYzjHoQ7k+GsMth9X3wwW2Qe8roymo1pm8YvTq1Y6GM7yKEqEVLWuhlwONa68uBkcCDSqn+jinLyUJ7w6zPYeILcHSj9e5IO5e4XGvdZFIkJljYdeICO497/i35hBBN0+xA11pnaK132J7nAfuA7o4qzOlMZhh5P9y/yTqMwCcPwJJbIeek0ZVVc/OQHgT6ekkXRiHEJRxyDV0pZQEGA1sdsT1DhfaGxM/guvlwbJP17kg7FrtMa729rxe3xvXksx8yOJNbZHQ5QggX0uJAV0q1Bz4GHtFaX3I3BqXUHKVUilIqJTPTTcZUMZlgxL3W1nrXQbDmN/D+LZCTbnRlAMyMj6Rca5ZsOWZ0KUIIF9KiQFdKeWMN8yVa65W1LaO1fltrHae1jgsLC2vJ7pwvpBfM/BSufwmOb7FeW9++yPDWemRoO67u15klW49TXFZuaC1CCNfRkl4uCngX2Ke1/rvjSnIxJhMMv8faWu8WC58+DItvggsnDC0rMT6KrIIS1u6WLoxCCKuWtNATgLuBq5RSu2w/1zuoLtcTEgUz1sCkv8GJbfDGKEhZaFhrPaFPKH07t2dh8lG0i1zfF0IYqyW9XL7TWiut9SCtdazt53NHFudyTCYY9mt4YDN0HwxrH4HFU+G8869lK2Xtwph6Mpftx6QLoxCiLX9TtCU6Rlpb65P/H6SnwJvx8P07UOHcr+TfNLg7QX5eLJQujEIIJNCbTymIm2VtrfcYBp89Du9NgfNpTishwMeL6cMj+DL1NKcuXHTafoUQrkkCvaU6RMDdq+CGV+DULngjHrb9y2mt9btHRaK15n3pwihEmyeB7ghKwdCZ1tZ6xEj4/AlYdANkH2n1XffoGMD4/l35cNtxikqlC6MQbZkEuiN16Al3fQxTXoPTP8CbCbD1n63eWk9MsHC+sJRPdrnWMAVCCOeSQHc0pWDI3fDAFohMgC+egkWTIetwq+1yRFQI/boGsnBTmnRhFKINk0BvLcHd4c7lcOMbcDrV2lrf8martNaVUsxOiGL/6Ty2HMl2+PaFEO5BAr01KQWD74QHt0DUGPjyaUi6vlVa61Niu9ExwJuk5KMO37YQwj1IoDtDUDe4YxlMfQvO7rX2W9/8OlQ47kNMP28z04dH8J+9ZziRXeiw7Qoh3IcEurMoBbHT4YGt0Gsc/PsZWHgdnDvksF3cPSoSpRSLpQujEG2SBLqzBYXD9KVw09uQeQDeugKSX3VIaz082J+JA7uydNtxCkvKHFCsEMKdSKAbQSmIuQ0e3Aq9r4J1z8KCCZB5sMWbnhVvIbeojFU7pQujEG2NBLqRArvC7R/Aze9A1k/W1vqmf7SotT40siPR3YN54fP9PLhkB4u3HOOns/nSnVGINkA584UeFxenU1JSnLY/t5J3Bj57DPavhe5Drd0dO/dr1qYOncnjzf8eZsvhLE7lWG9T1znQl5G9QonvHcqo3qFEhARgHdJeCOHqlFLbtdZxDS4nge5CtIY9K+GzJ6AkH8b9HuIfBrNXMzenOZ5dyObDWSQfzmLzkSwy84oB6Bbsx8jeoYzqZQ34Hh0DHPmXCCEcSALdneWftY7euG8NdBsCU9+Azpe3eLNaaw5nFrD5SBZbbAGfXVACQERIgD3cR/UOpUuQX4v3J4RwDAl0T7BnlTXYi/Ng7O8g4ZFmt9ZrU1GhOXg2j82Hs9h8OIutR7PJuVgKQK9O7RjZ23qJZmSvUDq193XYfoUQTSOB7ikKzllHb9yzCsJjra31LgNaZVflFZp9GbnWgD+Sxbaj2eQXW7s/Xtalvb0FPyIqlI7tfFqlBiHEpSTQPc2e1dbWelGOtbV+xSNg9m7VXZaVV5B66ueA//5oNhdLy1EK+nUNsn7A2iuU4b1CCPJr3VqEaMsk0D1RQRZ88SSkfgxdB8HUN6HrQKftvqSsgh/SL9gDfvux8xSXVWBSMLB7MKN6hTKydyjDLCG093XcpSEh2joJdE+2d421i+PFCzDmSRj9WKu31mtTVFrOzuMX7B+y7jxxntJyjdmkiOkRbP2AtVcnhkZ2xN/H7PT6hPAUEuierjDbOtb6j8uha7S133r4IENLulhSzvZj59l85BybD2exOz2H8gqNj9lEbM8O9m6SgyM64OctAS9EY0mgtxX71sLaR+FiNox+AkY/Dl6u8YFlfnEZ36dl27tIpp7MoUKDr5eJoZEd7R+yDurRAR8v+dKyEHWRQG9LCrOtY63/sAy6DLT2hAmPMbqqS+RcLGXb0Wz7Nfh9GbkA+HubGRYVYg/4gd2C8DJLwAtRSQK9Ldr/Oax9xNrVMXY6hPaBwG7WER4DbT++7Y2u0u58QQlbj2bZv8l66Gw+AIG+XtUC/vLwIMwmGaZAtF0S6G1VYTas+6P1W6bFuZfO9w2yBntQ+KVhXzmtfWcwOf8ad2ZeMVuOZNk/ZD1yrgCAYH9vRkSF2L/FelnnQEwS8KINkUAXUJwPeRmQewryTkPeKcjNqPKYYZ2ua4zuqMzQvsvPYR/UzToypP0EYHv0DWzV8k/nFNk/YN18JIsT2RcBCGnnw8helS34TvQOaycDjQmP5lGB/qdNf2J35m6CfIII8g2yPtZ87hNEsG9wtel+XjIeSYMqyqEg0xb8NcI+99TP04tzLl3XJ9AW8LWEfeVju84OG64g/XyhfZiCzUeyyLCNJBkW6PvzODS9QokMlZEkhWfxqEBftGcRuzN3k1uSS25xrv0xrzSv3vV8TD51nwB8a5wEakyXk0ENJQXW1rw95Gs85p22PlbUuFOSMllDveYlnqBu1R99A603/mgkrTXHsgrZfOTngK8cSTI82M8e7jKSpPAEHhXodSmvKCe/NN8e8jklOZeEfm5J7c9bcjKo+U5ATgY2FRVQeK6W0K+8vGObVnTh0nW929Ud9pXX99t3rbO1bx9J8vA56zX4I9n2kSR7hvhXacF3omtwG/33EW7LKYGulJoI/AMwA+9orV+ob3lXuobuzJNBrSeAtnwyKCmsEvC2yzzVWv+2eRWlNVZU1g9sq4V+1Q91bdP8gqnQ2EeSTD6cxdYjWeQWWd89VI4kOaqXdSTJsEAZSVK4tlYPdKWUGTgIXAukA98D07XWe+tax5UCvSWqngxySnKqB76DTgbBPsENXi7y6JNBRQUUZtX4ALdmq/8UXDx/6breATVa+F2pCAwnvawDOy/48+0Zb746obhgvUJD387tie4RjJ+3GW+Twttswstswses8DKb8Dab8DYr26MJL7PCx/ZYdZ6XyYSPl8LLZF3O/tzLVGW7P29HumKKxmpsoLfk06rhwE9a6yO2HS4FbgTqDHRPYTaZCfYNJtg3mJ70bNK65RXl5JXk1R74tZwAzhSe4dD5Q+SW5JJfml/vtn3NvvaQD/AOwKRMmJW52qPJdOm0Sx5N9c9vzDJmZW7SvupcxscLc5gFU+detc8vL8FccA5TwTnM+Wcx5Z/FnHcGU/5pzHmnMZ3YginvNKbyEiKACKz/SbVSlHUM5by5EydLgjm5vx2lWlGmTZRpKKuwPlZgohwTZShKMFGBsv2YqNDWeRpFec15/DyvwjZfoyjXJvt8rRTKZMZkMqNMpirPzZjMtuNi/vl3s8mMyWzCZPLCbDZjNpsxmb2s88xmvMzW5bzMXphMXnh5mTCbvezTvb28bdO88fKyLu/l5YWXl22e2Ww9+VSeuEwKH68aJzGTwmxS8qGzi2pJoHcHTlT5PR0Y0bJyPJ/ZZKaDXwc6+HVo8rplFWXkl+TXezKofMdwsewi5bqcCl1BuS6npKLE+ntFRbXpFbqi2vO6lqltHZfnBXQEOnbFhAmTUphR1kcNJjRmXYxJn8akK1D2N6s13rVe8i5WU1ucNXYaUGVfDS9b+/oayrD+NLeG2ibqyofat6Sr/Vbb6q4Z9IpL/lWrzHGO5+JfI27AuFbdR0sCvfZ/z5oLKTUHmAMQERHRgt0JL5NXs08Gjqa1rv1kUPWxov6TQkMnD0dso2YtdS1T26VHXUcE2JfVGtCgNVpXWJ+jba8C27TKZaosrytPhrrCviyV61f+Xrld22PV7VTfbtX9VV9PV1RQoW3TtaZCV9Zq/berXF5XWB8rbPVoe+3Wfek6tg/2kqjll0bTtf1WZaKqdbmGt1bfyUfbptV+xbmlHUVqX9/fr/W/pd2SQE+HatcbegCnai6ktX4beBus19BbsD/hQpRSmJUZM2a8kZtbCOEKWjIC0vdAX6VUlFLKB7gdWOOYsoQQQjRVs1voWusypdRvgH9j7ba4QGu9x2GVCSGEaJIWfSdba/058LmDahFCCNECMui0EEJ4CAl0IYTwEBLoQgjhISTQhRDCQ0igCyGEh3Dq8LlKqUzgWDNX7wScc2A5jiJ1NY3U1TRSV9N4al2RWuuwhhZyaqC3hFIqpTGjjTmb1NU0UlfTSF1N09brkksuQgjhISTQhRDCQ7hToL9tdAF1kLqaRupqGqmradp0XW5zDV0IIUT93KmFLoQQoh4uFehKqQVKqbNKqdQ65iul1CtKqZ+UUj8opYa4SF3jlFI5Sqldtp8/Oamunkqp9UqpfUqpPUqp39ayjNOPWSPrcvoxU0r5KaW2KaV22+r6n1qW8VVKLbMdr61KKYuL1JWolMqscrx+3dp1Vdm3WSm1Uym1tpZ5Tj9ejazLkOOllEpTSv1o2+clN1Bu9dejtt3BxBV+gDHAECC1jvnXA19gvdnISGCri9Q1DlhrwPEKB4bYngdivWl3f6OPWSPrcvoxsx2D9rbn3sBWYGSNZR4A3rI9vx1Y5iJ1JQKvOfv/mG3fjwEf1PbvZcTxamRdhhwvIA3oVM/8Vn09ulQLXWu9EciuZ5Ebgfe01Ragg1Iq3AXqMoTWOkNrvcP2PA/Yh/Ver1U5/Zg1si6nsx2Dyjtte9t+an6IdCOwyPZ8BXC1auU7IjeyLkMopXoAk4B36ljE6cerkXW5qlZ9PbpUoDdCbTemNjwobEbZ3jJ/oZQa4Oyd297qDsbauqvK0GNWT11gwDGzvU3fBZwF/qO1rvN4aa3LgBwg1AXqArjF9jZ9hVKqZy3zW8P/A54C6roruCHHqxF1gTHHSwPrlFLblfV+yjW16uvR3QK9UTemNsAOrF/NjQFeBVY7c+dKqfbAx8AjWuvcmrNrWcUpx6yBugw5Zlrrcq11LNZ74A5XSg2ssYghx6sRdX0KWLTWg4Cv+LlV3GqUUpOBs1rr7fUtVsu0Vj1ejazL6cfLJkFrPQS4DnhQKTWmxvxWPV7uFuiNujG1s2mtcyvfMmvrXZy8lVKdnLFvpZQ31tBcorVeWcsihhyzhuoy8pjZ9nkB2ABMrDHLfryUUl5AME683FZXXVrrLK11se3XfwFDnVBOAjBFKZUGLAWuUkq9X2MZI45Xg3UZdLzQWp+yPZ4FVgHDayzSqq9Hdwv0NcAM2yfFI4EcrXWG0UUppbpWXjdUSg3HelyznLBfBbwL7NNa/72OxZx+zBpTlxHHTCkVppTqYHvuD1wD7K+x2Bpgpu35NOAbbfs0y8i6alxnnYL1c4lWpbX+vda6h9bagvUDz2+01nfVWMzpx6sxdRlxvJRS7ZRSgZXPgfFAzZ5xrfp6bNE9RR1NKfUh1t4PnZRS6cBcrB8QobV+C+v9S68HfgIKgVkuUtc04H6lVBlwEbi9tf9T2yQAdwM/2q6/AjwDRFSpzYhj1pi6jDhm4cAipZQZ6wnkI631WqXUc0CK1noN1hPRYqXUT1hbmre3ck2NrethpdQUoMxWV6IT6qqVCxyvxtRlxPHqAqyytVO8gA+01l8qpe4D57we5ZuiQgjhIdztkosQQog6SKALIYSHkEAXQggPIYEuhBAeQgJdCCE8hAS6EEJ4CAl0IYTwEBLoQgjhIf4/6hUoxKYp8n8AAAAASUVORK5CYII=\n", 158 | "text/plain": [ 159 | "
" 160 | ] 161 | }, 162 | "metadata": {}, 163 | "output_type": "display_data" 164 | } 165 | ], 166 | "source": [ 167 | "%matplotlib inline\n", 168 | "import matplotlib.pyplot as plt\n", 169 | "\n", 170 | "plt.plot(np.log10(Ns),vars1)\n", 171 | "plt.plot(np.log10(Ns),vars3)\n", 172 | "plt.plot(np.log10(Ns),vars2)\n", 173 | "\n", 174 | "#plt.plot(vars1)\n", 175 | "#plt.plot(vars2)\n", 176 | "plt.legend(['REINFORCE', 'REINFORCE+Variance reduction (mean)','Reparametrization'])" 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": {}, 182 | "source": [ 183 | "Variance of the estimates using reparameterization trick is one order of magnitude smaller than the estimates from the first method!" 184 | ] 185 | } 186 | ], 187 | "metadata": { 188 | "kernelspec": { 189 | "display_name": "Python 3", 190 | "language": "python", 191 | "name": "python3" 192 | }, 193 | "language_info": { 194 | "codemirror_mode": { 195 | "name": "ipython", 196 | "version": 3 197 | }, 198 | "file_extension": ".py", 199 | "mimetype": "text/x-python", 200 | "name": "python", 201 | "nbconvert_exporter": "python", 202 | "pygments_lexer": "ipython3", 203 | "version": "3.6.8" 204 | } 205 | }, 206 | "nbformat": 4, 207 | "nbformat_minor": 1 208 | } 209 | -------------------------------------------------------------------------------- /codes/VAN/.ipynb_checkpoints/SK_variational-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Variational methods for Sherrington-Kirkpatrick model\n", 8 | "Pan Zhang\n", 9 | "Institute of Theoretical Physics, Chinese Academy of Sciences\n", 10 | "\n", 11 | "We compuare variational free energy given by the variational mean-field method and the Variational Autoregressive Network (Physical Review Letters 122, 080602), for the Sherrington-Kirkpatrick spin glass model." 12 | ] 13 | }, 14 | { 15 | "cell_type": "markdown", 16 | "metadata": {}, 17 | "source": [ 18 | "## SK model\n", 19 | "$n$ denotes number of variables, $J_{ij}=\\frac{1}{\\sqrt{n}}\\mathcal N(0,1)$ is the couplings matrix and $\\beta$ is the inverse temperature. One needs to notice that diagonal terms of the couplings matrix must be zero, i.e.\n", 20 | "$J_{ii}=0$, and the coupling matrix $\\mathbf J$ is symmetic.\n", 21 | "\n", 22 | "The following code generates a small instance of the SK model" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 203, 28 | "metadata": {}, 29 | "outputs": [], 30 | "source": [ 31 | "import torch,math\n", 32 | "import numpy as np\n", 33 | "from torch import nn\n", 34 | "from scipy.special import logsumexp\n", 35 | "import sys\n", 36 | "device=torch.device('cpu')\n", 37 | "device=torch.device('cuda:0')\n", 38 | "n=20 # number of spins\n", 39 | "beta=0.5 # inverse temperature\n", 40 | "seed=1\n", 41 | "torch.manual_seed(seed)\n", 42 | "J=torch.randn(n,n,device=device)/math.sqrt(n)\n", 43 | "J = torch.triu(J,diagonal=1) # take the upper triangular matrix\n", 44 | "J = J+J.t() # make the coupling matrix symmetric\n", 45 | "J_np = J.cpu().numpy()\n", 46 | "J.requires_grad=False" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "## Exact enumerations for small systems\n", 54 | "Whent the number of spins is small, e.g. $n\\leq 20$, we can compute exactly the free energy, energy and entropy by enumerating all $2^n$ configurations." 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 204, 60 | "metadata": { 61 | "scrolled": true 62 | }, 63 | "outputs": [], 64 | "source": [ 65 | "def cfg_id_to_sample(cfg_id):\n", 66 | " return np.array( [((cfg_id >> i) & 1) * 2 - 1 for i in range(n-1,-1,-1)])\n", 67 | "\n", 68 | "def list_energy(print_step=float('inf')):\n", 69 | " samples=[]\n", 70 | " energy_arr = []\n", 71 | " for cfg_id in range(1 << n):\n", 72 | " if (cfg_id + 1) % print_step == 0:\n", 73 | " sys.stdout.write(\"\\rEnumerating all configurations: %d / 100\"%(int(cfg_id /(1 << n)*100)+1))\n", 74 | " sample = cfg_id_to_sample(cfg_id)\n", 75 | " energy_arr.append(sample.dot(J_np).dot(sample)/2.0)\n", 76 | " samples.append(sample)\n", 77 | " \n", 78 | " cfg_id_arr = np.arange(1 << (n), dtype=int)\n", 79 | " energy_arr = np.array(energy_arr)\n", 80 | " energy_arr *= -1.0\n", 81 | " samples = np.array(samples)\n", 82 | "\n", 83 | " return cfg_id_arr, energy_arr\n", 84 | " \n", 85 | "def f_exact():\n", 86 | " if(n>20):\n", 87 | " return 0,0,0\n", 88 | " step=int((1<> i) & 1) * 2 - 1 for i in range(n-1,-1,-1)])\n", 67 | "\n", 68 | "def list_energy(print_step=float('inf')):\n", 69 | " samples=[]\n", 70 | " energy_arr = []\n", 71 | " for cfg_id in range(1 << n):\n", 72 | " if (cfg_id + 1) % print_step == 0:\n", 73 | " sys.stdout.write(\"\\rEnumerating all configurations: %d / 100\"%(int(cfg_id /(1 << n)*100)+1))\n", 74 | " sample = cfg_id_to_sample(cfg_id)\n", 75 | " energy_arr.append(sample.dot(J_np).dot(sample)/2.0)\n", 76 | " samples.append(sample)\n", 77 | " \n", 78 | " cfg_id_arr = np.arange(1 << (n), dtype=int)\n", 79 | " energy_arr = np.array(energy_arr)\n", 80 | " energy_arr *= -1.0\n", 81 | " samples = np.array(samples)\n", 82 | "\n", 83 | " return cfg_id_arr, energy_arr\n", 84 | " \n", 85 | "def f_exact():\n", 86 | " if(n>20):\n", 87 | " return 0,0,0\n", 88 | " step=int((1<)\n" 50 | ] 51 | } 52 | ], 53 | "source": [ 54 | "lnZ = 0.0\n", 55 | "M = T.clone() # M will be the converged envioment tensor\n", 56 | "print(math.log(M.norm()))\n", 57 | "Niter = 20 # number of iterations in RG. That is, 2^Niter matrices will be contracted finally.\n", 58 | "for i in range(Niter): # after i steps, there are totally 2^i matrices contracted.\n", 59 | " s = M.norm() # This is the normalization of a matrix contracted of 2^i T.\n", 60 | " lnZ = lnZ + torch.log(s)/2**i # Notice that we can only record a density of logarithm of the results, for contraction of infinite matrices.\n", 61 | " M = M/s\n", 62 | " M = M@M\n", 63 | "lnZ = lnZ + torch.trace(M)/(2**Niter) # trace(M) is the trace of contraction of all tensors.\n", 64 | "print(lnZ)\n", 65 | "lnZ.backward()\n", 66 | "RG_grad = T.grad.clone()" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "### Autograd of the leading eigenvalue of the transfer matrix\n", 74 | "In the last section, we have evaluated $\\ln Z$ using RG. Actually, this can be evaluated analytically as\n", 75 | "$$\\ln Z =\n", 76 | "\\frac{1}{t}\\lim_{t\\to\\infty}\\ln \\mathrm{tr}(T^t)=\\frac{1}{t}\\lim_{t\\to\\infty}\\ln\\sum_{i=1}^N \\lambda_i^t=\n", 77 | "\\ln\\lambda_\\mathrm{max},$$\n", 78 | "where $\\lambda_{\\mathrm{max}}$ is the leading eigenvalue. Thus we can do back propagation directly on $\\lambda_{\\mathrm{max}}$." 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 6, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "w, v = torch.symeig(T, eigenvectors=True) # W is an arry of eigenvalues, v is a matrix with each row storing an eigenvector\n", 88 | "T.grad.zero_()\n", 89 | "lambda_max = torch.log(w[-1])\n", 90 | "lambda_max.backward()\n", 91 | "eigenvalue_grad = ((T.grad + T.grad.t())/2) # need to symmetrize since it is an upper triangular matrix" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "### Using cavity of the matrix product\n", 99 | "Using the property of the environment matrix $M$, we have $Z=\\mathrm{tr}\\left[\\underbrace{T\\times T \\times \\cdots \\times T}_{\\mathrm{\\infty}}\\right]=\\mathrm{Tr}(M\\times T \\times M)$. Then the gradient with respect to a specific $T$ can be written as\n", 100 | "$$\\nabla_T \\ln Z = \\frac{M\\times M}{\\mathrm{tr}(M\\times T\\times M)}.$$\n" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 46, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "impurity_grad = (M@M).t()/torch.trace(M@T@M)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "metadata": {}, 115 | "source": [ 116 | "### Autograd of $\\ln Z$ using vector environment\n", 117 | "Here comes the trick: Infinite tensor product with a Periodic Boundary Condition (PBC) is equivalent to that with a Open Boundary Condition (OBC), which is written as\n", 118 | "$\\ln Z=\\ln \\left[u\\times\\underbrace{T\\times T \\times \\cdots \\times T}_{\\mathrm{\\infty}}\\times u\\right]=\\ln(m\\times T\\times m)$, where $m$ is the environment for the OBC tensor product, a vector. Obviously, $m$ is the leading eigenvector of the transfer matrix $T$ hence is normalized, and the gradient can be written as $$\\nabla_T\\ln Z=\\exp(-\\lambda_\\mathrm{max}) m\\otimes m.$$" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": 47, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "v=v[:,-1] # the leading eigenvector of the transfer matrix $T$\n", 128 | "eigenvector_grad=v[:,None]@v[None,:]/torch.exp(lambda_max) # outer product of the leading eigenvector and its transpose" 129 | ] 130 | }, 131 | { 132 | "cell_type": "markdown", 133 | "metadata": {}, 134 | "source": [ 135 | "### Comparing these gradients" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 48, 141 | "metadata": {}, 142 | "outputs": [ 143 | { 144 | "name": "stdout", 145 | "output_type": "stream", 146 | "text": [ 147 | "2.168404344971009e-18\n", 148 | "3.469446951953614e-18\n", 149 | "4.336808689942018e-18\n" 150 | ] 151 | } 152 | ], 153 | "source": [ 154 | "# comparing RG_grad, eigenvector_grad, eigenvalue_grad and impurity_grad\n", 155 | "print ((impurity_grad-RG_grad).abs().max().item())\n", 156 | "print ((impurity_grad-eigenvalue_grad).abs().max().item())\n", 157 | "print ((impurity_grad-eigenvector_grad).abs().max().item())" 158 | ] 159 | } 160 | ], 161 | "metadata": { 162 | "kernelspec": { 163 | "display_name": "Python 3", 164 | "language": "python", 165 | "name": "python3" 166 | }, 167 | "language_info": { 168 | "codemirror_mode": { 169 | "name": "ipython", 170 | "version": 3 171 | }, 172 | "file_extension": ".py", 173 | "mimetype": "text/x-python", 174 | "name": "python", 175 | "nbconvert_exporter": "python", 176 | "pygments_lexer": "ipython3", 177 | "version": "3.6.8" 178 | } 179 | }, 180 | "nbformat": 4, 181 | "nbformat_minor": 2 182 | } 183 | -------------------------------------------------------------------------------- /codes/neural_networks/.ipynb_checkpoints/logistic_regression-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import time\n", 10 | "import torch\n", 11 | "import torchvision\n", 12 | "import torch.nn as nn\n", 13 | "import torch.nn.functional as F\n", 14 | "import torch.optim as optim\n", 15 | "from torchvision import datasets, transforms\n", 16 | "%matplotlib inline\n", 17 | "import matplotlib.pyplot as plt" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": null, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "def show_imgs(imgs,l1=4,l2=5,s1=6,s2=6):\n", 27 | " \"\"\" Plot images \"\"\"\n", 28 | " plt.rcParams['figure.figsize']=(s1,s2)\n", 29 | " imgs=imgs.cpu().reshape([-1,28,28])\n", 30 | " g, ax = plt.subplots(l1,l2)\n", 31 | " for i in range(l1): \n", 32 | " for j in range(l2):\n", 33 | " a=i*l2+j\n", 34 | " if(a>=imgs.shape[0]):\n", 35 | " break\n", 36 | " ax[i][j].imshow(imgs[a,:,:],cmap='summer')\n", 37 | " ax[i][j].set_xticks([])\n", 38 | " ax[i][j].set_yticks([])\n", 39 | " plt.show()\n", 40 | "#show_imgs(data,2,10,10,2)" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": null, 46 | "metadata": { 47 | "scrolled": true 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "batch_size=100\n", 52 | "test_batch_size=100\n", 53 | "epochs=1000\n", 54 | "device=torch.device(\"cpu\")\n", 55 | "#device=torch.device(\"cuda:0\")\n", 56 | "kwargs = {'num_workers': 1, 'pin_memory': True} if device != torch.device('cpu') else {}\n", 57 | "train_loader = torch.utils.data.DataLoader( \n", 58 | " datasets.MNIST('../../data', train=True,download=True, \n", 59 | " transform=transforms.Compose([ transforms.ToTensor(),\n", 60 | " transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=batch_size,\n", 61 | " shuffle=False, **kwargs)\n", 62 | "test_loader = torch.utils.data.DataLoader(datasets.MNIST('../data', train=False,\n", 63 | " transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,),\n", 64 | " (0.3081,)) ])), batch_size=test_batch_size, shuffle=False, **kwargs)\n", 65 | "train_data=[]\n", 66 | "test_data=[]\n", 67 | "for batch_idx, (data, target) in enumerate(train_loader):\n", 68 | " data, target = data.to(device), target.to(device)\n", 69 | " train_data.append((data,target))\n", 70 | "for batch_idx, (data, target) in enumerate(test_loader):\n", 71 | " data, target = data.to(device), target.to(device)\n", 72 | " test_data.append((data,target))\n", 73 | "#print(train_data[0])\n", 74 | "\n", 75 | "\n", 76 | " " 77 | ] 78 | }, 79 | { 80 | "cell_type": "code", 81 | "execution_count": null, 82 | "metadata": {}, 83 | "outputs": [], 84 | "source": [ 85 | "show_imgs(train_data[0][0],2,10,10,2)" 86 | ] 87 | }, 88 | { 89 | "cell_type": "code", 90 | "execution_count": null, 91 | "metadata": {}, 92 | "outputs": [], 93 | "source": [ 94 | "nin=784\n", 95 | "nout=10\n", 96 | "W=torch.randn(nin,nout,device=device) \n", 97 | "W.requires_grad=True\n", 98 | "bias = torch.randn(nout,device=device)\n", 99 | "bias.requres_grad=True\n", 100 | "print(train_data[0][0].shape)\n", 101 | "lr=0.001\n", 102 | "optimizer = torch.optim.Adam([W,bias], lr=lr, betas=(0.9, 0.999))\n", 103 | "print_bin = epochs/100\n", 104 | "for epoch in range(1, epochs + 1):\n", 105 | " optimizer.zero_grad()\n", 106 | " t1=time.time()\n", 107 | " for batch_idx, (data, target) in enumerate(train_data):\n", 108 | "# loss = F.cross_entropy(data.view(data.shape[0],-1)@W+bias.expand(data.shape[0],nout),target)\n", 109 | " output=F.log_softmax(data.view(data.shape[0],-1)@W+bias.expand(data.shape[0],nout), dim=1)\n", 110 | " loss = F.nll_loss(output, target)\n", 111 | " loss.backward()\n", 112 | " optimizer.step()\n", 113 | " test_loss = 0\n", 114 | " correct = 0\n", 115 | " for data, target in test_data:\n", 116 | " output=F.log_softmax(data.view(data.shape[0],-1)@W+bias.expand(data.shape[0],nout), dim=1)\n", 117 | " test_loss += F.nll_loss(output, target, reduction='sum').item()\n", 118 | " pred = output.argmax(dim=1, keepdim=True)\n", 119 | " correct += pred.eq(target.view_as(pred)).sum().item()\n", 120 | " test_loss /= 10000\n", 121 | " correct /= 10000\n", 122 | " if(epoch % print_bin==0):\n", 123 | " print(\"#%d loss=%.3f test_loss=%.3f accuracy=%.3f time=%.3f\"%(epoch,loss,test_loss,correct,time.time()-t1))\n", 124 | " " 125 | ] 126 | }, 127 | { 128 | "cell_type": "code", 129 | "execution_count": null, 130 | "metadata": {}, 131 | "outputs": [], 132 | "source": [] 133 | } 134 | ], 135 | "metadata": { 136 | "kernelspec": { 137 | "display_name": "Python 3", 138 | "language": "python", 139 | "name": "python3" 140 | }, 141 | "language_info": { 142 | "codemirror_mode": { 143 | "name": "ipython", 144 | "version": 3 145 | }, 146 | "file_extension": ".py", 147 | "mimetype": "text/x-python", 148 | "name": "python", 149 | "nbconvert_exporter": "python", 150 | "pygments_lexer": "ipython3", 151 | "version": "3.6.8" 152 | } 153 | }, 154 | "nbformat": 4, 155 | "nbformat_minor": 2 156 | } 157 | -------------------------------------------------------------------------------- /codes/neural_networks/.ipynb_checkpoints/lr_mlp_conv-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": { 7 | "slideshow": { 8 | "slide_type": "" 9 | } 10 | }, 11 | "outputs": [], 12 | "source": [ 13 | "import time\n", 14 | "import torch\n", 15 | "import torchvision\n", 16 | "import torch.nn as nn\n", 17 | "import torch.nn.functional as F\n", 18 | "import torch.optim as optim\n", 19 | "from torchvision import datasets, transforms\n", 20 | "\n", 21 | "batch_size=1000\n", 22 | "test_batch_size=1000\n", 23 | "epochs=1000\n", 24 | "device=torch.device(\"cpu\")\n", 25 | "device=torch.device(\"cuda:0\")\n", 26 | "data_path='../../../data'\n", 27 | "kwargs = {'num_workers': 1, 'pin_memory': True} if device != torch.device('cpu') else {}\n", 28 | "train_loader = torch.utils.data.DataLoader( \n", 29 | " datasets.MNIST(data_path, train=True,download=True, \n", 30 | " transform=transforms.Compose([ transforms.ToTensor(),\n", 31 | " transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=batch_size,\n", 32 | " shuffle=False, **kwargs)\n", 33 | "test_loader = torch.utils.data.DataLoader(datasets.MNIST(data_path, train=False,\n", 34 | " transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,),\n", 35 | " (0.3081,)) ])), batch_size=test_batch_size, shuffle=False, **kwargs)\n", 36 | "train_data=[]\n", 37 | "test_data=[]\n", 38 | "for batch_idx, (data, target) in enumerate(train_loader):\n", 39 | " data, target = data.to(device), target.to(device)\n", 40 | " train_data.append((data,target))\n", 41 | "for batch_idx, (data, target) in enumerate(test_loader):\n", 42 | " data, target = data.to(device), target.to(device)\n", 43 | " test_data.append((data,target))" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 20, 49 | "metadata": { 50 | "scrolled": true, 51 | "slideshow": { 52 | "slide_type": "slide" 53 | } 54 | }, 55 | "outputs": [ 56 | { 57 | "name": "stdout", 58 | "output_type": "stream", 59 | "text": [ 60 | "#10 loss=1.225 test_loss=1.346 accuracy=0.661 time=0.140\n", 61 | "#20 loss=0.684 test_loss=0.710 accuracy=0.812 time=0.132\n", 62 | "#30 loss=0.323 test_loss=0.417 accuracy=0.860 time=0.144\n", 63 | "#40 loss=0.418 test_loss=0.540 accuracy=0.836 time=0.136\n", 64 | "#50 loss=0.209 test_loss=0.271 accuracy=0.891 time=0.132\n", 65 | "#60 loss=0.133 test_loss=0.153 accuracy=0.954 time=0.120\n", 66 | "#70 loss=0.158 test_loss=0.215 accuracy=0.935 time=0.133\n", 67 | "#80 loss=0.115 test_loss=0.121 accuracy=0.962 time=0.106\n", 68 | "#90 loss=0.359 test_loss=0.466 accuracy=0.870 time=0.131\n", 69 | "#100 loss=0.147 test_loss=0.150 accuracy=0.954 time=0.135\n", 70 | "#110 loss=0.115 test_loss=0.101 accuracy=0.969 time=0.133\n", 71 | "#120 loss=0.544 test_loss=0.625 accuracy=0.871 time=0.104\n", 72 | "#130 loss=0.127 test_loss=0.136 accuracy=0.961 time=0.140\n", 73 | "#140 loss=0.332 test_loss=0.390 accuracy=0.891 time=0.145\n", 74 | "#150 loss=0.120 test_loss=0.099 accuracy=0.971 time=0.131\n", 75 | "#160 loss=0.116 test_loss=0.097 accuracy=0.972 time=0.133\n", 76 | "#170 loss=0.114 test_loss=0.093 accuracy=0.973 time=0.137\n", 77 | "#180 loss=0.232 test_loss=0.251 accuracy=0.932 time=0.110\n", 78 | "#190 loss=0.144 test_loss=0.132 accuracy=0.964 time=0.090\n", 79 | "#200 loss=0.144 test_loss=0.120 accuracy=0.967 time=0.091\n", 80 | "#210 loss=0.124 test_loss=0.097 accuracy=0.974 time=0.112\n", 81 | "#220 loss=0.119 test_loss=0.096 accuracy=0.975 time=0.129\n", 82 | "#230 loss=0.118 test_loss=0.095 accuracy=0.976 time=0.108\n", 83 | "#240 loss=0.140 test_loss=0.166 accuracy=0.959 time=0.105\n", 84 | "#250 loss=0.202 test_loss=0.233 accuracy=0.947 time=0.098\n", 85 | "#260 loss=0.161 test_loss=0.133 accuracy=0.969 time=0.098\n", 86 | "#270 loss=0.125 test_loss=0.091 accuracy=0.977 time=0.088\n", 87 | "#280 loss=0.119 test_loss=0.092 accuracy=0.977 time=0.087\n", 88 | "#290 loss=0.113 test_loss=0.094 accuracy=0.978 time=0.087\n", 89 | "#300 loss=0.107 test_loss=0.095 accuracy=0.977 time=0.081\n", 90 | "#310 loss=0.102 test_loss=0.097 accuracy=0.978 time=0.102\n", 91 | "#320 loss=0.110 test_loss=0.117 accuracy=0.974 time=0.089\n", 92 | "#330 loss=0.258 test_loss=0.218 accuracy=0.953 time=0.113\n", 93 | "#340 loss=0.406 test_loss=0.480 accuracy=0.917 time=0.137\n", 94 | "#350 loss=0.154 test_loss=0.109 accuracy=0.978 time=0.106\n", 95 | "#360 loss=0.137 test_loss=0.107 accuracy=0.979 time=0.129\n", 96 | "#370 loss=0.128 test_loss=0.107 accuracy=0.979 time=0.134\n", 97 | "#380 loss=0.123 test_loss=0.108 accuracy=0.979 time=0.125\n", 98 | "#390 loss=0.112 test_loss=0.111 accuracy=0.979 time=0.121\n", 99 | "#400 loss=0.101 test_loss=0.111 accuracy=0.978 time=0.133\n", 100 | "#410 loss=0.101 test_loss=0.113 accuracy=0.979 time=0.126\n" 101 | ] 102 | }, 103 | { 104 | "ename": "KeyboardInterrupt", 105 | "evalue": "", 106 | "output_type": "error", 107 | "traceback": [ 108 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 109 | "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", 110 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0moutput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcross_entropy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 111 | "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 487\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 488\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 490\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 491\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 112 | "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 93\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 113 | "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 487\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 488\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 490\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 491\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 114 | "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mweak_script_method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 115 | "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m 1350\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1351\u001b[0m \u001b[0;31m# fused op is marginally faster\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1352\u001b[0;31m \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unwrap_optional\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1353\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1354\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 116 | "\u001b[0;31mKeyboardInterrupt\u001b[0m: " 117 | ] 118 | } 119 | ], 120 | "source": [ 121 | "n_in=784\n", 122 | "n_out=10\n", 123 | "n_hidden=500\n", 124 | "model = \"lr\" # logistic regression\n", 125 | "#model = \"fc2\" # 2-layer MLP\n", 126 | "#model = \"conv\" # convolution\n", 127 | "if (model == \"lr\"):\n", 128 | " net = nn.Sequential(nn.Linear(n_in,n_out))\n", 129 | "elif(model == \"fc2\"):\n", 130 | " net = nn.Sequential(nn.Linear(n_in,n_hidden),nn.Sigmoid(),nn.Linear(n_hidden,n_out))\n", 131 | "elif(model == \"conv\"):\n", 132 | " net = nn.Sequential(nn.Conv2d(1,20,5),nn.ReLU(),nn.MaxPool2d(2),nn.Conv2d(20,50,5),nn.ReLU(),nn.MaxPool2d(2))\n", 133 | " net2 = nn.Sequential(nn.Linear(4*4*50,n_hidden),nn.ReLU(),nn.Linear(n_hidden,n_out))\n", 134 | "else:\n", 135 | " print(\"Wrong model\")\n", 136 | "net = net.to(device)\n", 137 | "net2 = net2.to(device)\n", 138 | "optimizer = torch.optim.Adam(list(net.parameters()), lr=0.001, betas=(0.9, 0.999))\n", 139 | "print_bin = epochs/100\n", 140 | "for epoch in range(1, epochs + 1):\n", 141 | " optimizer.zero_grad()\n", 142 | " t1=time.time()\n", 143 | " for batch_idx, (data, target) in enumerate(train_data):\n", 144 | " if(model == 'conv'):\n", 145 | " output=net(data).view(output.shape[0],-1)\n", 146 | " output = net2(output)\n", 147 | " else:\n", 148 | " output=net(data.view(data.shape[0],-1))\n", 149 | "\n", 150 | " loss = F.cross_entropy(output, target)\n", 151 | " loss.backward()\n", 152 | " optimizer.step()\n", 153 | " test_loss = 0\n", 154 | " correct = 0\n", 155 | " for data, target in test_data:\n", 156 | " if(model == 'conv'):\n", 157 | " output=net(data).view(output.shape[0],-1)\n", 158 | " output = net2(output)\n", 159 | " else:\n", 160 | " output=net(data.view(data.shape[0],-1))\n", 161 | " output=F.log_softmax(output, dim=1)\n", 162 | " test_loss += F.nll_loss(output, target, reduction='sum').item()\n", 163 | " pred = output.argmax(dim=1, keepdim=True)\n", 164 | " correct += pred.eq(target.view_as(pred)).sum().item()\n", 165 | " test_loss /= 10000\n", 166 | " correct /= 10000\n", 167 | " if(epoch % print_bin==0):\n", 168 | " print(\"#%d loss=%.3f test_loss=%.3f accuracy=%.3f time=%.3f\"%(epoch,loss,test_loss,correct,time.time()-t1))\n", 169 | " " 170 | ] 171 | }, 172 | { 173 | "cell_type": "code", 174 | "execution_count": null, 175 | "metadata": {}, 176 | "outputs": [], 177 | "source": [] 178 | } 179 | ], 180 | "metadata": { 181 | "kernelspec": { 182 | "display_name": "Python 3", 183 | "language": "python", 184 | "name": "python3" 185 | }, 186 | "language_info": { 187 | "codemirror_mode": { 188 | "name": "ipython", 189 | "version": 3 190 | }, 191 | "file_extension": ".py", 192 | "mimetype": "text/x-python", 193 | "name": "python", 194 | "nbconvert_exporter": "python", 195 | "pygments_lexer": "ipython3", 196 | "version": "3.6.8" 197 | } 198 | }, 199 | "nbformat": 4, 200 | "nbformat_minor": 2 201 | } 202 | -------------------------------------------------------------------------------- /codes/neural_networks/grad_matrix_products.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Different ways of computing gradient of trace of infinite matrix products\n", 8 | "$\\nabla_T \\ln Z = \\nabla _T \\ln \\mathrm{tr}\\left[\\underbrace{T\\times T \\times \\cdots \\times T}_{\\mathrm{\\infty}}\\right]$\n", 9 | "We present several methods for computing the gradients with:\n", 10 | "- Autograd of $\\ln Z$ that is computed using the environment matrix and RG\n", 11 | "- Autograd of the leading eigenvalue of the transfer matrix\n", 12 | "- Autograd of $\\ln Z$ that is computed using the environment vector and power method\n", 13 | "- Using cavity of the matrix product\n", 14 | "- Using the leading eigenvector" 15 | ] 16 | }, 17 | { 18 | "cell_type": "code", 19 | "execution_count": 3, 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "import torch,math\n", 24 | "torch.manual_seed(1)\n", 25 | "N = 10 # size of the matrix\n", 26 | "A = torch.randn(N, N, dtype=torch.float64) \n", 27 | "T = A@A.t() # a symmetric positive definite transfer matrix\n", 28 | "T.requires_grad=True" 29 | ] 30 | }, 31 | { 32 | "cell_type": "markdown", 33 | "metadata": {}, 34 | "source": [ 35 | "### Autograd of $\\ln Z$ using environment matrix\n", 36 | "Consider a infinite-length chain as product of matrices $T$. Analogous to real-space RG for the one-dimensional Ising model, we merge all pairs of consequtive matrices at each RG step, resulting to a chain with half length (although still infinite), then truncate the spectrum of merged matrices to original dimension for a low-rank approximation. By keep doing this RG step, merged matrices will finally converge to environments (for each tensor $T$) $M$ which also indicates that $Z=M\\times T \\times M$, and $$ \\ln Z=\\ln(M\\times T \\times M).$$" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 4, 42 | "metadata": {}, 43 | "outputs": [ 44 | { 45 | "name": "stdout", 46 | "output_type": "stream", 47 | "text": [ 48 | "4.042987293864085\n", 49 | "tensor(3.8794, dtype=torch.float64, grad_fn=)\n" 50 | ] 51 | } 52 | ], 53 | "source": [ 54 | "lnZ = 0.0\n", 55 | "M = T.clone() # M will be the converged envioment tensor\n", 56 | "print(math.log(M.norm()))\n", 57 | "Niter = 20 # number of iterations in RG. That is, 2^Niter matrices will be contracted finally.\n", 58 | "for i in range(Niter): # after i steps, there are totally 2^i matrices contracted.\n", 59 | " s = M.norm() # This is the normalization of a matrix contracted of 2^i T.\n", 60 | " lnZ = lnZ + torch.log(s)/2**i # Notice that we can only record a density of logarithm of the results, for contraction of infinite matrices.\n", 61 | " M = M/s\n", 62 | " M = M@M\n", 63 | "lnZ = lnZ + torch.trace(M)/(2**Niter) # trace(M) is the trace of contraction of all tensors.\n", 64 | "print(lnZ)\n", 65 | "lnZ.backward()\n", 66 | "RG_grad = T.grad.clone()" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "### Autograd of the leading eigenvalue of the transfer matrix\n", 74 | "In the last section, we have evaluated $\\ln Z$ using RG. Actually, this can be evaluated analytically as\n", 75 | "$$\\ln Z =\n", 76 | "\\frac{1}{t}\\lim_{t\\to\\infty}\\ln \\mathrm{tr}(T^t)=\\frac{1}{t}\\lim_{t\\to\\infty}\\ln\\sum_{i=1}^N \\lambda_i^t=\n", 77 | "\\ln\\lambda_\\mathrm{max},$$\n", 78 | "where $\\lambda_{\\mathrm{max}}$ is the leading eigenvalue. Thus we can do back propagation directly on $\\lambda_{\\mathrm{max}}$." 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": 5, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "w, v = torch.symeig(T, eigenvectors=True) # W is an arry of eigenvalues, v is a matrix with each row storing an eigenvector\n", 88 | "T.grad.zero_()\n", 89 | "lambda_max = torch.log(w[-1])\n", 90 | "lambda_max.backward()\n", 91 | "eigenvalue_grad = ((T.grad + T.grad.t())/2) # need to symmetrize since it is an upper triangular matrix" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "### Using cavity of the matrix product\n", 99 | "Using the property of the environment matrix $M$, we have $Z=\\mathrm{tr}\\left[\\underbrace{T\\times T \\times \\cdots \\times T}_{\\mathrm{\\infty}}\\right]=\\mathrm{Tr}(M\\times T \\times M)$. Then the gradient with respect to a specific $T$ can be written as\n", 100 | "$$\\nabla_T \\ln Z = \\frac{M\\times M}{\\mathrm{tr}(M\\times T\\times M)}.$$\n" 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 6, 106 | "metadata": {}, 107 | "outputs": [], 108 | "source": [ 109 | "impurity_grad = (M@M).t()/torch.trace(M@T@M)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "metadata": {}, 115 | "source": [ 116 | "### Autograd of $\\ln Z$ using vector environment\n", 117 | "Here comes the trick: Infinite tensor product with a Periodic Boundary Condition (PBC) is equivalent to that with a Open Boundary Condition (OBC), which is written as\n", 118 | "$\\ln Z=\\ln \\left[u\\times\\underbrace{T\\times T \\times \\cdots \\times T}_{\\mathrm{\\infty}}\\times u\\right]=\\ln(m\\times T\\times m)$, where $m$ is the environment for the OBC tensor product, a vector. Obviously, $m$ is the leading eigenvector of the transfer matrix $T$ hence is normalized, and the gradient can be written as $$\\nabla_T\\ln Z=\\exp(-\\lambda_\\mathrm{max}) m\\otimes m.$$" 119 | ] 120 | }, 121 | { 122 | "cell_type": "code", 123 | "execution_count": 7, 124 | "metadata": {}, 125 | "outputs": [], 126 | "source": [ 127 | "v=v[:,-1] # the leading eigenvector of the transfer matrix $T$\n", 128 | "eigenvector_grad=v[:,None]@v[None,:]/torch.exp(lambda_max) # outer product of the leading eigenvector and its transpose" 129 | ] 130 | }, 131 | { 132 | "cell_type": "markdown", 133 | "metadata": {}, 134 | "source": [ 135 | "### Comparing these gradients" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 8, 141 | "metadata": {}, 142 | "outputs": [ 143 | { 144 | "name": "stdout", 145 | "output_type": "stream", 146 | "text": [ 147 | "1.734723475976807e-18\n", 148 | "1.6046192152785466e-17\n", 149 | "1.6046192152785466e-17\n" 150 | ] 151 | } 152 | ], 153 | "source": [ 154 | "# comparing RG_grad, eigenvector_grad, eigenvalue_grad and impurity_grad\n", 155 | "print ((impurity_grad-RG_grad).abs().max().item())\n", 156 | "print ((impurity_grad-eigenvalue_grad).abs().max().item())\n", 157 | "print ((impurity_grad-eigenvector_grad).abs().max().item())" 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "execution_count": null, 163 | "metadata": {}, 164 | "outputs": [], 165 | "source": [] 166 | } 167 | ], 168 | "metadata": { 169 | "kernelspec": { 170 | "display_name": "Python 3", 171 | "language": "python", 172 | "name": "python3" 173 | }, 174 | "language_info": { 175 | "codemirror_mode": { 176 | "name": "ipython", 177 | "version": 3 178 | }, 179 | "file_extension": ".py", 180 | "mimetype": "text/x-python", 181 | "name": "python", 182 | "nbconvert_exporter": "python", 183 | "pygments_lexer": "ipython3", 184 | "version": "3.6.8" 185 | } 186 | }, 187 | "nbformat": 4, 188 | "nbformat_minor": 2 189 | } 190 | -------------------------------------------------------------------------------- /codes/neural_networks/logistic_regression.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 174, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import time\n", 10 | "import torch\n", 11 | "import torchvision\n", 12 | "import torch.nn as nn\n", 13 | "import torch.nn.functional as F\n", 14 | "import torch.optim as optim\n", 15 | "from torchvision import datasets, transforms\n", 16 | "%matplotlib inline\n", 17 | "import matplotlib.pyplot as plt" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 175, 23 | "metadata": {}, 24 | "outputs": [], 25 | "source": [ 26 | "def show_imgs(imgs,l1=4,l2=5,s1=6,s2=6):\n", 27 | " \"\"\" Plot images \"\"\"\n", 28 | " plt.rcParams['figure.figsize']=(s1,s2)\n", 29 | " imgs=imgs.cpu().reshape([-1,28,28])\n", 30 | " g, ax = plt.subplots(l1,l2)\n", 31 | " for i in range(l1): \n", 32 | " for j in range(l2):\n", 33 | " a=i*l2+j\n", 34 | " if(a>=imgs.shape[0]):\n", 35 | " break\n", 36 | " ax[i][j].imshow(imgs[a,:,:],cmap='summer')\n", 37 | " ax[i][j].set_xticks([])\n", 38 | " ax[i][j].set_yticks([])\n", 39 | " plt.show()\n", 40 | "#show_imgs(data,2,10,10,2)" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 183, 46 | "metadata": { 47 | "scrolled": true 48 | }, 49 | "outputs": [], 50 | "source": [ 51 | "batch_size=10000\n", 52 | "test_batch_size=100\n", 53 | "epochs=1000\n", 54 | "device=torch.device(\"cpu\")\n", 55 | "device=torch.device(\"cuda:0\")\n", 56 | "data_path='../../../data'\n", 57 | "kwargs = {'num_workers': 1, 'pin_memory': True} if device != torch.device('cpu') else {}\n", 58 | "train_loader = torch.utils.data.DataLoader( \n", 59 | " datasets.MNIST(data_path, train=True,download=True, \n", 60 | " transform=transforms.Compose([ transforms.ToTensor(),\n", 61 | " transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=batch_size,\n", 62 | " shuffle=False, **kwargs)\n", 63 | "test_loader = torch.utils.data.DataLoader(datasets.MNIST(data_path, train=False,\n", 64 | " transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,),\n", 65 | " (0.3081,)) ])), batch_size=test_batch_size, shuffle=False, **kwargs)\n", 66 | "train_data=[]\n", 67 | "test_data=[]\n", 68 | "for batch_idx, (data, target) in enumerate(train_loader):\n", 69 | " data, target = data.to(device), target.to(device)\n", 70 | " train_data.append((data,target))\n", 71 | "for batch_idx, (data, target) in enumerate(test_loader):\n", 72 | " data, target = data.to(device), target.to(device)\n", 73 | " test_data.append((data,target))\n", 74 | "#print(train_data[0])\n", 75 | "\n", 76 | "\n", 77 | " " 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 179, 83 | "metadata": {}, 84 | "outputs": [ 85 | { 86 | "data": { 87 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkMAAAB/CAYAAADsHsZkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXeYFFXWxt8hZ5AgOSk5gyhBFBQRUEBFRVHB9BnXHNewa05rWhWzrrrqGgARFQMoRlQQFRURBCQpoICApBkJ/f1xzvHW9PT0zDDdXVVd7+95eM6lu6bn1lR11a0T3pMTi8VACCGEEBJVyvg9AUIIIYQQP+FiiBBCCCGRhoshQgghhEQaLoYIIYQQEmm4GCKEEEJIpOFiiBBCCCGRhoshQgghhEQaLoYIIYQQEmm4GCKEEEJIpClXko1zqlSKoVb1dM0lvWzYhNjW3Jxkm4R6/wBg1dq1sVisXrJNQr2PxTiGAPcx8PC7CCDk+8jz9C+yfR9DvX9Asb6LQAkXQ6hVHThz5G7PyVcee6XobcK8fwBww2PLitwmzPtYnGMIcB+DDr+LQpj3keepI9v3Mcz7BxTvuwiGyQghhBAScbgYIoQQQkik4WKIEEIIIZGGiyFCCCGERJqSJVCTlLBPI+C8/WQ8tqvY/34j9oGZwFer/JkXISTc3DcUuKCXjOf+JnbY/4BlG/ybEyHp4L2TASuDO/iZ0n8ePUOEEEIIiTS+e4bKlgFqViz4unlOqpQX27au2L9NAe46VMajO4vN3QHc/omMb/ggbVMtNd0aiJ02Bqih+xyLiR3TReyItkCdOzI/t0wzcC+xz2vFZv+ngQVrfZtOyrj2QLE3HASU0ceWAU+L/XCpHzMiyaheEahWQcaHtxZbr6rYez4D8nb4M6+S0qKW2JO6ALv0mtJer5nt6maHZ6hNHbHlywIHNpPxQ8PE2j4XxuT5Yo+fIPbPnamfXyopXxbo21TGtw4Uu/+T/s0nSNw7RGzfpi6ikgoyshhqVhOoUFbGdoD76clcqxJwdIeiP+PnP8TePxQ4qr2MN+WJ/ebXYN9o9mssduJxYmtWcougTX+K/VMvunWqAL31b/TVSn0vQ1/cA5u7OUz6Ib2/a99GYr9Ymd7fkylO6Sb2yn5ivRfnWBEXapI5Wuwh9sr9xfZpAnTaM/G2DasBF7yVmXmVljVbxX60TB6osoGOelxO0VSCYzuKLZMDNFINQPueFfUds7/JI7p4uuht4I+81M011dSsCLx/soxXbxbboJobR5HbDxF7dk+x23cC7/2Uus9nmIwQQgghkSatnqFuDcVOHyvekN3BVv7XThe7+U/g+e9kvGqT2PW5wQuxWHivR0PgOQ0FNaxWcLuF68T+a4bYF48BZpwmY9vn2z5O3zy9DGghtnWd9HqGyuQALfUJvXlNsUVq3gec5hqmqOR74Hn36dVEwiwA0F+9hB09XpPLpopdqd+7fs2A576V8cyfMzPH3aGdhosu6g2cqPtXWY9TTg6wYqOMzUtr4aVRHYGHvpDx/IBdX+LZonPPhnCYcZuGhw5rnbrPtIKVJ78GZixP3eemkwbVnI2yZ6h3E7Hl1YXzyXLg5e9T9/n0DBFCCCEk0qT1OXa5PqWs21Y8z5A9XW7IBQ5qKWPLl3k2hYlSmeDR4WJHd0q+XQ/1nlkS54fLnIemS/20TK1Q7KnpszQ/5TesDpzRQ8bmWQj6k3dhHKKJ4Ofvl//1+WulpBkAfg3409xxeo7eNwSoW0XGOeqq+2ApUE9fu3NQ/p/LyXHvWWJqELBrzR2aY2D7V71CwW0XrgMGPyfj8prXOP9vYutWcX+PoFNL97lrA3/nkUqmaT5IvGfoty3i2QFckYI3R8/yUs27GXZywu42T8CBzYFrtNhktF47ft9W+PajO7vcvsXrxZqnOlWkdTFkO3f5VGBYGxl/vVrs/UPddnP0tUHPit3yp3PPX9grnTNMPftoYrBVpnhPZEvyfv1HVxFnIQf7u6zPBQ7WhWCmvwNlMvQLnxjhxgt/z8zvTAf9mgFPHSnj+MX+nTOCG7Iop/7gnprY/7gu3KuUlwRcALjpI7GfLAcq6iLh5WPFHrq3+6zZAUyAP6qd2P/rUfg2dkEd9KwLk7Wqk955pRMLyzerWfC9fRu5h42gnpOJeFhDlK/Oz//69p3Jw0VWqTv3XLGNPA3X7bOCeN4WhiWHVyrv7zxSyWPDJR0DADpoP/lPkoQtrz5ACnsA4IzXxX6zOrVzYpiMEEIIIZEmI+mer84Hpi+RsSUpdtUQ0Ok9gLs+lbElAQLA96qeeubrmZhh6fFqCAH5dYTeWiRjcwf2b+GSo5/4SuyaLWK/We1cvoerN61Hw/SqUls4rn6CBO904NWVmrY4M78zHZzcLf9TJyBhJSC1+hepxpKkvR46QMISx42Xsbfs+DgtafZ6hACRu3gmgPtpJdjxLN3gpByunCbWvEKAS5wOI+ZhfnoOcP2A/O9dP0BSDwBg3KxMzqp07Ngl1nuMisPgVmL3qFzwPZNoCYt+lJeeDYHPV/g9i9SwdbvH45VkFWL31eY13X0xXUUq9AwRQgghJNJkrBA4XuBqY64bWzLtS3PFFqUmGjTa1AEuVxE3yx1Zq56eVZuBZ+bIeLN6vqb8KP+Kwsp/L+0LnDgxdfONxxIUK6f5bDDPk5XVA8Avm9L7O9OBJdWe1t2dqxs0P+7mj/yZU3G56WCJvwPuyczKx6+dnliIzhId47ngLefRDBJnvCb2zH3ETlXv46LfJfm2MOpXTe+8MsFNHxb0DEWF4zVR/gw97omuZ/98P3PzKQ07dgEb9btonvS9a/s3n1Rx08FiO9cHftA8tm9+LbhdVS12MBHbKuWBz7WwZ8K89MyNniFCCCGERBrfJOKu/1DsPo0khwZwZcpTQ5JHUlH/encd6rwr1iJk7KtiZ68svcclUYVIKmkblyth+Vqpxiro6lcFflSxyU0BlsSPx/o/WVsVLw9oLsb7SzI3n5Lwz/5irz7AyVW8o7lsV74rdtt2t73F5Q/d251/Vhlp3q/JcVU+QcHyZ67/oGQ/16dpyqfiC3+Vm/s7jYxgIpp/7we0Us9J+QSP+FaxvD0kf5QNucDHWtlpldhhpqleQywKtGMXcN6bMk7kXb5nsNhjtVXXyk3p783m22LIkqXPeB346iwZP64Jne8vcaWPD6oLP4j9nbqrRpBXB+OIF8UGuVdaUaSiX1iNisAQTWS0hF1vAq6Vbm/IRWiw/fHqP1lvnPtmZn4+xcH0Z87dV2ws5hZBR75YcHu7oVgDXZOKAIAJqvZqaulh5AKV6qiaQHOoc1yPsk9XpF9zKx0Ut19X0LGHjzGqf2YPy16sx2WifbWQ79/fBd5cKGPvgp+kn056rZykD5CWYvDArMT3yMv6irVej8YtGejCwDAZIYQQQiKN752UFv8OnKIhpaeOEDumq3sasCc4K1VeFaCE23s07JOT41a5qfAI+e3mrp2gJBVw6rYWLjlExSGb1AAqqDCfua3L5LinsJm/iM3T8Ey5MsCXIRI9A4Aj27muycYny4GT9dzdGFAPlx0Xr5KydWLfUxOGT+0udkRbp/JqiuixGGAP3c9pT0CvBEaQMSHCDrpP1/UvqGZcJqdgwYaF2U6dDOwMSVgl2+hUH3jteBnvbprAxyri99iXqZmT39Qp5LocNEzU9aQuwJN6T49XCu/TBLhKCznu+Uxs7couLGb6v3bff3R2Wqcsc0z/ryCEEEIICS6+e4YA1yHdWjPccygwUOPDt2rnYutufsvHwC9/ZHZ+8VhCmwlCxWLAawtS9/nxMf85KZYdj8c8OPaA/MgwV37txXJlbNVuomhbtwPz1sj4P9ozaPZK5yX7VRPkfr5YbOVy4elFlixp+qf1we87ZsnSa7aKrVcFWHKRjBPlWZhXxPItGlYD1urPvp7CczxdlC8LdNfvpR2zhirpsG2H27/PVLxuSCvnQTLsyXZke+C+z2Vsf0eSOcwDnaw3VzIvul2nh7YG3lqY0qn5woi2fs+geJjEwRMj3DXGjs8ivcf3bCT/AOAI3a/GNdx31a5Xp01O+3T/IhCLIWOu6g2MGg8M1z+Qhc7O6im2dR1g0H8zPzcvlfXiaSGI37YAL31fus+0yjSvRoipdl/1buk+uyjOnSJ2mSq99i2kqma5vm/9fX7QBdDnRSSZmt5LPQ3L/LR+9+bpB6ZzkUj76vZPMjuX3cES1C1Z+o3RQG0NmVl/LqsKe3qO6yf44jFiG1YDXpybmbmWBvsuDmkFvBK3cL1BK1enLwFmaOjEQsHTT3ahQcOaz942sOA5Hwbl4kQLhAO1aWlYFKjn/goMeFrGVoBhif+5hRyD07VSKb5pclh5f6nYsFSTWUNk69e4fZe7/pygOnnr9fpy92DXSNcWRTk5bvFkYf0Vl4gd8LSk1KQThskIIYQQEmkC5RkyNuQCz2rilPVPKqdPOwc2Bwa0kLH1gfKbvJ27n9htHqFrVeX38r6uf87dmli2OUMJq3ekydMxsGX+/09Mk4JoKrEQaHw/LsB5UhaEJNQHADPVe1fvzuTbmQfBtL92xYLtySuvHqEbDhJ7eV/3nvUEfEBlDzbkOu/kmyeI7VzfhcBMMsA8RUe0dRID76qEwh0zgPVxyfJz0tg3cHdIVFo/sr1Y6xBuYe0gs2yD2FuKqepuulLZ4hlaHteTrXxZoLmG7e1vEyTO0giAzfvmj4Cnvk687flvAo8Ol3GfJgXft8io6bal2ysE0DNECCGEkIgTKM+QJege0wHYt7GMy8Ut1+atAT5altl5FcXuJE+b58F6mlln8MkLgKNfSs28gsqkECTiTh0j1tv52nKjTAoiG7F8OK93Iag5Q2XLADepR8jE2rZsF5E9wM3b8hZ6NgbGDZWxCaYuXAecozlz9hRaQ3tB9W3qpCIseXXaGPf7V6gHt+W/U7M/qeIRLUO2J3Uvlr930duZm0+mGJzAixtmdsRlhecAqFjWl6kUi8l6XX9FC6JWbCx827pVCubqjZ4AzI3rfvBzBoul6BkihBBCSKTx3TPUti5wnsZ4La7doFrB7Uz8bNUm/7vaWzzTSj6PbAdc+Fbxf/7iPsA/NEfIutw//63YsZNSMkVSSupoNYP3XLPu7pnK4fIDq9gJA2f2cB6hrSoPcdbrrrdhb/Uum6jk0NauT+CNWmH21JyCT7AmK/D2IvkHAKM7iz2hs9vu4ndSsx+pJiyyFV4s98ty9KYvKVnrjFO7A/cNSf28/MRyE+14tqsLXNRbxlYBHCRMhiIZdr87tqPzwFo+0MulrMguLRlfDNlCxy4u5+3ntFwSMVvVi603SSr1fHYXuz9agmKDasD96n43nZ11WkLYuwkwRl3tpuDcpIZLMntHL9x2o81mbPHYpjbw+Qp/51IYVhZaJoG2yacBnXMqGdzK7xkUn38OcOOy6uO+fH8nT2F91rxYku1tWixQXIXpF77Lb4OMJYxbIvHenr/DhXoztcbCmUhMLYp+zYFrVNdskOrLtbwveZjFpBFMUfyewQX1orZpCX5hpfhhwRb3jWsAlwR0AV5crEfiOT1FkgYADn7Gv/l4YZiMEEIIIZEmI56h+tVcSee4w8S2q1v49jN/Bu78VMbmKvQ7NJaMsjluxXu09lYxV3vrBE+nn65wyZr/fD/98wsK5klL5HUJAt0auM7Ydr5Z2fWDs4KvNp0K9trD7xkUn9WbnUCiJZZ2re/et07lVnDx6nxgqZYkR6Hn2PdaPu89pkHc7XFDCybTXrE/sCmv8J8xD1IPFezzygiY5MrDmkhu19qwE4uFVwndJAH+T4UxY3A94zKZJJ0MeoYIIYQQEmnS4hmyeO6jw8R2a5j8idNyMe5Wb9A7i0uWPJdprK/RF9p5fd9G7j3Liapf1b22TvusvKgJYiVJts5G+jSV1g9Bo1algsn71gfvsqmZn48fWKfvZD2fgsKBT0nxAgD00FL537a4vD2T/g/r03RpeUw9I8ND0s7Byzk9S7b9b1uA13+UsV1fw54rFE+NisARer5bP8+wYJIU1mP0uW+B6wIWFUnZYqiXqkhe3hfYT6s4GtcofHur/rh/JnCrJkdvCUmVjrn1Rqoe0Fn7OAXpeO6b6dy1i9alf25BJlnDRRIMrD/gQj1X99oD2FsfZNZs8WdOhbEpzynVmyWOeVqF9MNaoH2StAS/OWWyS/Y+uWvR2y9e7+4fH2sI9LGv3LmbbYxSDbq8na4fZNh4Sh9+TRdscgAKoeJhmIwQQgghkSZlnqGj1H13VPuC781bA7yhLkxT1bSQ2IbcgtuHBetHdv0HrmSXFMR6RB3b0d95FMX8tS5k26+Zv3PxG/PWPjECuGWgjM9/U2wY+loR17+q80P+zqMo5qxyujmzVErl5oOc+vurWkQzTfvDTZ4vyfNRwQoA2tdzcgFh47aP89sgQs8QIYQQQiJNyjxD1g/ILCGGJUsHMWnay+rNQP+n/J5FMLD+Qsd3cnIDJmZ46mSxYcnxI8EnTz0ej87Obwlw/AS/ZxAN6BkihBBCSKTxvTcZISR4mGjoqPEuZ8jKnS0/jrlDhJBsgYshQkih/JHnEqfNEkJItsEwGSGEEEIiTU4sVvymXzk5OWsALEvfdNJK81gsVi/ZBiHfPyD797HI/QO4jyEg289TIPv3keepku37GPL9A4p7HEuyGCKEEEIIyTYYJiOEEEJIpOFiiBBCCCGRhoshQgghhEQaLoYIIYQQEmm4GCKEEEJIpOFiiBBCCCGRhoshQgghhEQaLoYIIYQQEmm4GCKEEEJIpOFiiBBCCCGRpkRd63OqVIqhVvV0zSW9bNiE2NbcnGSbhHr/AGDV2rVF9pkJ8z4W4xgC3MfAw+8igJDvI8/Tv8j2fQz1/gHF+i4CJVwMoVZ14MyRuz0nX3nslaK3CfP+AcANjxXdTC/M+1icYwhwH4MOv4tCmPeR56kj2/cxzPsHFO+7iJIuhggh+WhTR+zbJ4ktq4Hn5vf6Mx9CCCElhzlDhBBCCIk09AwRsps8cBhwXEcZ164s9o0f/ZsPISS72WsPsbcNFHtUe7FdHgbmr/VnTtkCPUOEEEIIiTS+eYY6aG73sDbAmfvI+IuVYr9e5bb79+di/9yZubkRkoj61cS+cpzY3k2AWEzGc38Te/prmZ8XIST76dvU5Sau2Sr2wS/E/rrFnzllE/QMEUIIISTSZNwzdFZPsXcdKrZaBffe3rXFHt/JvWbeoveXpH9uUcWOwXH6d8/dAezTUMbVK4o9sTPwwVIZ/7Kp8M9avVns5PnA7JUpn6pvtKnjztlejd3rV70n1vZ13dbMzisV5KjKyAtHA4e1lnGHB8X+/Ic/cyIlY0xXsYfuDXRrIOO2ddz7n/8sdvgLYjfmZm5uflO1AvDByTJuVEPs/k8CSzf4N6eScHgbsRNGAY/MlvE1et3Zut2fOWUjGV8Mjf9e7I0DxHoXQ4mwkMRx48VOXZyOWUWbf/YXe1nf5NsNaVX8z7yqHzBvjYxfmJvfLl1fsvkFgdpV3ELBiy0WwrxYr6xXgf2bue+jHesnvvJnTiQ5dauIfWKE2OFtxW7IBT5dIWO72Q9oAfRrJuPPThdri91soVF1oF7V/K+t3yb2oJbAPo1kvGCd2HXbMje33aWVLmZfPlbsh0uBS9+R8a6YL1PKahgmI4QQQkikybhn6HddkV/3gdi7BwNVyst4+UaxzWq67WtVEmtPqlHwDDWvJdae2Ed3Bs7pmX+bKQvFnvpq6X/fyPaFv2dhn29/K3ybBWuBtnVlbMerewOg054yvuVg/YxfxYbJM2Siiv8b6cJJxsiXJBwYdszVvnAd0FhV9+OfsrOVS/sCFcrKuL2ewyd2ce9buXLHgHlSLJG2hV4r/jVD7J0z3DXWaFcXmHWGjO18Nm/wjR+md56ppFN9sRfs566RRps6+e8bAHD7J2I71HPf3V/Uk2vHPKhUKgc8MVzG3+l1c9T47PIImRyJyZNcfaB4+LxcO13sbR+nfz70DBFCCCEk0vhWWm+JYGf3BLpqwt8feYVvP25W+ufkJ4fsJXZke/EEAUBNTV5O9DDQu0nqfvfg58TaU+OP69x75jVYlSRp2oslXH93TsEntRGa1zAlRMKElpjarCbwpnrjzn5D7C9Zllz84BeSXwI4L0k20b+F81b2by72qPZAfJfKmOcL11qLOub9TWwQcm0G7Q101wKHlzUH86p3C99+/lonUXLtgWJP7S42TJ6hg1uIPb1HwffydgDPfavbtRT7937ufTumT88RG/RCh5sOAnrpNb71/WKT3R/DRu+mwL2DZbyfFqTEYvm/e4D8HQC5N6UiCpIM3xWob/4IuEa/oFYFkYiguzVLiiU+dtaL876NC26z6U+xz3/rqupe+E5s7o7UzWXx7/ltaRimlQ/ehVCeakQ9HqJk3E810dTOyaUbgIs1eTHbFkHGrF/ceJS6rq98t/gL4SDQsLpUxQFOrdeoWQmoqiF5C5t8uRLo0bDwzyuj21UtotAjk5QrAyzS7+qLc4v3MxPmibXFUCW98teoGPyb7PUDxF6+v3vtGV3UrFF9nbs+c2P7zr6jocS6Vd179ncIKhX1uJzU1VXvZlNFpyX+Pz7cPXDZsXl1vks7GNtN7LEdxPZu4tYA6dIcZJiMEEIIIZHGd8/QhHnAJ8tlPHWM2M71C253sybhHvNyZuaVDuroqvi2gcBp6qa2ZMcv1fNz+wynZrxNQ1SWWB5UKpQF7h8q47FdC77f50mxc1YVfC9oHNFOrLmozW07fh6QGwFND/OY2FPYiLbAo7P9m09xsTDz4yOApjWK3t7CXWu3uqdVS9586gixTTyfYzIRQeD9JUD3R2RcXJ2ZvLin6fqaIH9CZ5eyEFTMm2cFJcs2AtdoYq3Xa9lKQ5pXHyDWigC2bAeu13BgKj3q6eAK9X5Vq+C0hLKJyaPFtq/riqEOe77gdgu1yOYQDXk2qQG0164V36xOz9zoGSKEEEJIpPHdM3RiF6CreoIsuTER5j0KM//QeP3pPYAHNCHcVv+b//RnTqXhIF21j+kCnNIt/3vbdwEXvCXj+QF6qk5GrUrAAc0Sv7d+W/LY/YW9xXq9EpdNTd3cMkV8AmNYcvXsiTqRV8i8IldOAz7XvKgFng7flkx7YS+xXo+QCReOeSV1cy0tu+Pd+EmftL/X72JHfcpuXSfx9kFiwg9ih5g6ej3g9kNkfO4UsTUrAvdoQq4pNpvX/ZaPgYe/yMxcS8uhe4udsRz4KgSe9JKyzePJnLyg+D/3R554cdMJPUOEEEIIiTQZ9wy10wzySceLbVVbqiOK4rUSrCKDgAlJXtlPPCcAcNHbYt9fCryzSMZBj2EnwkohLcerbHxtMsTDYLlOO0MiFLYz5mT7rYpol7730bKC21/cx43P30+sVwzuUm1v0uQesdlaheYn9iSdSGrCzr8xk8TOKMK73CSBV8meXtP9VJputqt3bMeu5NsFkTmaI2L91TrUc+XzgzRX7N4hBaU8btA8oQdmpn+OpaWfSj3Yedz5ocTbmfSFda3/PokYbhCxnMScHNcuxSob964NnKI5p3YdXq2VZqMnpv/6mfHFkCVBtdSbRnEWQoC78Zz/ZurnlA6shPXKfk4LxBLGwrgA8mJl14kWQUaFssCUE2RsTUxfV32hSfOBub+mb367S//mLkxmSq92Q13rUfW10t0DmjntJGOLhjt//sOpck8YJfb4CcCykDSHDAu24LSHD8D15rKbYbJF0B6Vnbr9gc3zv/fpCqctFXasZLtS3BV/U8DL6gHREALySwBYsvtE7V2ZA6fH9qRKeLwaInX4k1Rb7gcNYy7xXCcsBeHuwcAeqvBvoV8LxT8YEh0+C8/GYsAlek+/VK0tgAC5VgKZlUJgmIwQQgghkSbjnqFJmgx3xTSxdwwq+LSSiIbV0jendHCVlnfGYukRSvSTV/QYmpdv30auPDkRPRvlt9f1d4q41lPpty2pn2dxMdXslh6RvpVasvvsN2IXrXMK3Sb+dkQ7YK3Oe+pPYu/+TGzNisD0k904LJgbOz6ROqg89qVYO/825gInTJTx6s1F//zZPZ3KrWFJxqPGF+8zwoD1MGsblzD99qKC29at4roC9NGwzXh9Qvcmnmeaoryq5sW761OxKwIuSeLFpFbs3M3b4YoXrhsg9qzXgXesHF2TyU0GYvHviY9l0FinHvbqFd39wHvNMakIP6Qs6BkihBBCSKTxrbT+fk1qW7gOqFU5/3vlygDjDpNxjRA9VXuZpcl+PRu7fdmmnqFpi/2ZU6qwnIzDVSyrWU33ZG5ibiM7uKed+NSiMjkuXmxx4oHP+NeRuV9TsdYrBwAeV4+D9W6qXw2461AZ21PZpjzgZX1itti99bJ6ZJjLx3hvidgw5AuFxSNkTJyX3xaX4ZrrZd3bAZdcbCKEYfcKWZ5QkxpA36aJt3lkGPCllnBbW5LalZ1EgbUEMkHDU9LcHyoRZfWR3fL5EqUqTlkIDP9fxqaUMjqqnIzlznoT3O14mMfHmz/zkrZh6ad/k6sOCIdnqKOKnfZu4goWXjrWvW9RBz88Q77rDL2V4ADm5Lgvn12sLGm1ea3g3VR6NQG+1guK9U0ZqguFC3oB/9B9sETaXo9L88RsYfnGgirZby1yvXWs0mq/BP3XrGHmZX1dyCzTdEnQEy++geUrxwG94uZ/xIvAh0tl3FtvNp+c6t63UGAY9YaMbwOY6J4KXtVqVu/iz3SxLPQWVCqXB/bUhw67YfbWc9OqrACgkiaVW9JqIjruKT3bvPzna9dM2QoHlq4v3ZxLw4vaa26k9qlKtF4P2yLeaBCX/uG9L1i49trphf+86Sd9d25q55VuPv8Z6JSg08StH2d+LgbDZIQQQgiJNL57hhJRoWx+9zUgisYAsDMAOhkNtazzDS0db1bDdTR/7luxpn46bpbzDFXTzte148KC2crz+rcwl+67Y8XGlzADzhPoB7X0yTgnx3VNNswj2aKWS/S7VI/1h0tdUvX/RrrPsG3MMxRmFv/u9wxSy60DxcbrSAHAhwm0pIJAZfXwXK/XkeFtnV5bIqwE3UJcO3YVlDB5QsvPH5kdTKWwFA7GAAAGyUlEQVTjRtWBUzXMfnScR+irlcA36rG0bcxTFnZ+8fRaK47sQZg72nfWEGGi76If0DNECCGEkEgTSM/QzQcVfM2EtIKwEv7qLLGW3H3lNOcRisd6VgHAu1p+PTdkqqGlxZICLVEzkWfox3WZm09hxGKJ8xEASe62vIQuGutefrGThViiORUHPCV2Y27apkl2kwplge6aY2PJ+rEYcKEqwy8MwDmYCMtvMrXlvJ2SMAy4886UsvN2uH5qdq2cf57zYFqPskvUuxnUnogD9wJujLsPWO7MuFnAke1kbJ4hPxJuU4Elg+ckEbBNRv8WYsMgnhmP9Smz7+IHS13OrR/QM0QIIYSQSJMWz1AdLbM2QagX5jrhwWRYLs6ZPQu+ZyV3QcBkAazlxv1D5Z+XhZpr0bo2sEwrra56V+wfIVzFN6wOnNFDxlbxYG1GisJKY7smqB4wr5F1E/cDe6q+vC9whJZcW3VYN51z9Qpu+7HaPycnx4kuXq/VZ9nWf6xiIH3HJcNadZzUxXlXjBfmutw2v6QdisL6r5kXaOTLwJwkeT6WH3SHdnZvXMOJmo4aLzaoHiHrveW9no54Qax51htUK5hTujRgFcbFxU65klbDlVdBxrP1XvlsIZGJoNKuLnC63k/W6Ln58Bf+Hse0XOrsRDYtjzZ1nKKv3SwW6WJhn0bOhXuFKvt6tYXuVjXRlZ7EMr+5Tcv/rPlh94bAIS3zb2M9ZKYsdKXVi0KYjGqln2+f5BLe9rijeD9bX3/2Eg0VHtyy4DY/6MLqEx+TV+04bt3ubpwzThOb7CLl1Rl6K0t6WMVzWOtwNLpMhCmLPz5c7DEd3HsXa2hs3KzgLoIMOwc3aOg1WV+/SuWA8SrhcbjqYeXtdL2egpgs7WWQLvxqVnQJ7W9omb8tAIa1caruFl1aE9JGuhbeW6WaVtaj7OHZibe3v8HDh4s1ZfGTJ6VnfqnGZBzeGQM0VufHleokyGQfskQwTEYIIYSQSJMWz9AD2kHXOtP3aQp8cIqMzQ1mK+IDmrknOCMWc6GY6z4QG8S+XtYDJ5v59xCx5hUC3HFdoAmnlggHuDLgK/Z3KtPeEBMg4SVL+DOhOz/5cqXY0RPdnM1d7+UZ7VP2nT6Zf73aiS5mA79udkJvyYT6woI9eXo9Qos11HR/iLxdVlxgMg+PDXepCN+sFmuJ0Zfv7/qPzdTQ8zlTkofVgsRfie1wHjHzhljS9H1DgfUqXWISASY+GDZWacTDxAbv9qjgP6+pJXup7EjX+sDV2vPS7oeHPit2bUg8Y/8aJLZxdQlRAy764zf0DBFCCCEk0qTFM/S59q76TPtzPfsN8NAwGVuM02wi1ucCHR5Mx8xISbG+WqM6utdMWuBrfSr1lpFbTLh7ghYXxqY84KiXZBwkz8qUH10bgijy504gd3v+1wbtFb6cIRMkvLRv/td/XOfa5ISJduPE3nSw2Mv6OqG6Ia3yb/vafCcKGoZeVfF4xRMtsXbaGLEHeCQ5Tp0s9vUFmZlXunlwVv7/3z0YGHd4/tc25TmP5s0fifWzFL0kHKKFCyd1Ebtth/85QvGktVbEvpQVywHV4kJhdrMc3dm9ZjfVQf9N56xISZimFRwvzgWO75T/vWQLHi9WMWaKzBN/AGb+nJr5kdQyR0OA1kC3WoXCtw0qpvh+XMf8rz8wK3h9DUvCP6bnt9nIDx69oGP0+FmStKn6P/iFqyzLNmxRFL84Civm9PA2YwWAsZMKqv37DcNkhBBCCIk0GVERydsB3FlIR/ITJmZiBmR3sW7Vp04GXlOXtJXIW2LniLZue2/X5ekaYpuv24UliTPK3KLu906aMF9cLamg0HHP/NIcgOtCb+cjCS5WpFChrPPwzdYCB7v+3PtZ5udFSk7l8i5UbVIIEzU0NilAuoEGPUOEEEIIiTRZoC9LMkHeDskbApw1oiAxEBVM+qLPE/7OY3cZ2wUYqknFpvx+nyadLlib+GdIcLCS+X/NkH8kvJzSDTh3Xxl/qkVVY1/1bz5FQc8QIYQQQiINPUOEkKxh6k8uT8E6s9MjREjm2K+x2KsPcBIAj2veXl4AxZMNLoYIIVnDez8B5W70exaERJdZqnze9B5/51FSGCYjhBBCSKTJiSVryx2/cU7OGgA+9hcvFc1jsVjSjksh3z8g+/exyP0DuI8hINvPUyD795HnqZLt+xjy/QOKexxLshgihBBCCMk2GCYjhBBCSKThYogQQgghkYaLIUIIIYREGi6GCCGEEBJpuBgihBBCSKThYogQQgghkYaLIUIIIYREGi6GCCGEEBJpuBgihBBCSKT5f4efLVfyp9j3AAAAAElFTkSuQmCC\n", 88 | "text/plain": [ 89 | "
" 90 | ] 91 | }, 92 | "metadata": {}, 93 | "output_type": "display_data" 94 | } 95 | ], 96 | "source": [ 97 | "show_imgs(train_data[0][0],2,10,10,2)" 98 | ] 99 | }, 100 | { 101 | "cell_type": "code", 102 | "execution_count": 184, 103 | "metadata": {}, 104 | "outputs": [ 105 | { 106 | "name": "stdout", 107 | "output_type": "stream", 108 | "text": [ 109 | "torch.Size([10000, 1, 28, 28])\n", 110 | "#10 loss=26.888 test_loss=27.740 accuracy=0.136 time=0.036\n", 111 | "#20 loss=19.086 test_loss=19.920 accuracy=0.230 time=0.036\n", 112 | "#30 loss=13.808 test_loss=14.583 accuracy=0.330 time=0.030\n", 113 | "#40 loss=10.418 test_loss=11.069 accuracy=0.420 time=0.029\n", 114 | "#50 loss=8.243 test_loss=8.756 accuracy=0.497 time=0.030\n", 115 | "#60 loss=6.787 test_loss=7.193 accuracy=0.557 time=0.037\n", 116 | "#70 loss=5.767 test_loss=6.108 accuracy=0.608 time=0.035\n", 117 | "#80 loss=5.024 test_loss=5.324 accuracy=0.646 time=0.033\n", 118 | "#90 loss=4.460 test_loss=4.735 accuracy=0.678 time=0.036\n", 119 | "#100 loss=4.021 test_loss=4.278 accuracy=0.702 time=0.036\n", 120 | "#110 loss=3.671 test_loss=3.917 accuracy=0.723 time=0.029\n", 121 | "#120 loss=3.384 test_loss=3.624 accuracy=0.738 time=0.029\n", 122 | "#130 loss=3.146 test_loss=3.379 accuracy=0.752 time=0.029\n", 123 | "#140 loss=2.947 test_loss=3.172 accuracy=0.764 time=0.029\n", 124 | "#150 loss=2.778 test_loss=2.995 accuracy=0.773 time=0.029\n", 125 | "#160 loss=2.633 test_loss=2.842 accuracy=0.783 time=0.029\n", 126 | "#170 loss=2.505 test_loss=2.708 accuracy=0.792 time=0.030\n", 127 | "#180 loss=2.392 test_loss=2.589 accuracy=0.799 time=0.029\n", 128 | "#190 loss=2.290 test_loss=2.482 accuracy=0.805 time=0.029\n", 129 | "#200 loss=2.199 test_loss=2.386 accuracy=0.810 time=0.030\n", 130 | "#210 loss=2.116 test_loss=2.298 accuracy=0.816 time=0.030\n", 131 | "#220 loss=2.041 test_loss=2.218 accuracy=0.820 time=0.030\n", 132 | "#230 loss=1.972 test_loss=2.144 accuracy=0.823 time=0.030\n", 133 | "#240 loss=1.908 test_loss=2.076 accuracy=0.829 time=0.029\n", 134 | "#250 loss=1.849 test_loss=2.012 accuracy=0.833 time=0.029\n", 135 | "#260 loss=1.795 test_loss=1.954 accuracy=0.836 time=0.030\n", 136 | "#270 loss=1.744 test_loss=1.899 accuracy=0.839 time=0.030\n", 137 | "#280 loss=1.697 test_loss=1.848 accuracy=0.841 time=0.029\n", 138 | "#290 loss=1.652 test_loss=1.800 accuracy=0.844 time=0.030\n", 139 | "#300 loss=1.610 test_loss=1.755 accuracy=0.847 time=0.035\n", 140 | "#310 loss=1.570 test_loss=1.713 accuracy=0.850 time=0.035\n", 141 | "#320 loss=1.531 test_loss=1.672 accuracy=0.852 time=0.028\n", 142 | "#330 loss=1.495 test_loss=1.634 accuracy=0.854 time=0.029\n", 143 | "#340 loss=1.460 test_loss=1.597 accuracy=0.856 time=0.029\n", 144 | "#350 loss=1.427 test_loss=1.562 accuracy=0.857 time=0.036\n", 145 | "#360 loss=1.395 test_loss=1.528 accuracy=0.858 time=0.043\n", 146 | "#370 loss=1.364 test_loss=1.496 accuracy=0.859 time=0.036\n", 147 | "#380 loss=1.335 test_loss=1.465 accuracy=0.861 time=0.037\n", 148 | "#390 loss=1.306 test_loss=1.435 accuracy=0.863 time=0.030\n", 149 | "#400 loss=1.279 test_loss=1.406 accuracy=0.864 time=0.030\n", 150 | "#410 loss=1.252 test_loss=1.379 accuracy=0.866 time=0.031\n", 151 | "#420 loss=1.227 test_loss=1.352 accuracy=0.867 time=0.031\n", 152 | "#430 loss=1.203 test_loss=1.326 accuracy=0.867 time=0.030\n", 153 | "#440 loss=1.179 test_loss=1.301 accuracy=0.869 time=0.030\n", 154 | "#450 loss=1.156 test_loss=1.277 accuracy=0.870 time=0.029\n", 155 | "#460 loss=1.134 test_loss=1.254 accuracy=0.871 time=0.029\n", 156 | "#470 loss=1.113 test_loss=1.232 accuracy=0.872 time=0.033\n", 157 | "#480 loss=1.093 test_loss=1.210 accuracy=0.874 time=0.029\n", 158 | "#490 loss=1.073 test_loss=1.190 accuracy=0.875 time=0.030\n", 159 | "#500 loss=1.054 test_loss=1.170 accuracy=0.875 time=0.029\n", 160 | "#510 loss=1.035 test_loss=1.150 accuracy=0.876 time=0.030\n", 161 | "#520 loss=1.018 test_loss=1.133 accuracy=0.876 time=0.029\n", 162 | "#530 loss=0.999 test_loss=1.119 accuracy=0.876 time=0.030\n", 163 | "#540 loss=1.012 test_loss=1.120 accuracy=0.874 time=0.029\n", 164 | "#550 loss=0.969 test_loss=1.092 accuracy=0.878 time=0.029\n", 165 | "#560 loss=0.965 test_loss=1.077 accuracy=0.880 time=0.029\n", 166 | "#570 loss=0.943 test_loss=1.049 accuracy=0.879 time=0.029\n", 167 | "#580 loss=0.927 test_loss=1.041 accuracy=0.880 time=0.029\n", 168 | "#590 loss=0.914 test_loss=1.027 accuracy=0.881 time=0.041\n", 169 | "#600 loss=0.901 test_loss=1.007 accuracy=0.881 time=0.035\n", 170 | "#610 loss=0.885 test_loss=1.001 accuracy=0.881 time=0.038\n", 171 | "#620 loss=0.874 test_loss=0.981 accuracy=0.881 time=0.037\n", 172 | "#630 loss=0.861 test_loss=0.970 accuracy=0.882 time=0.037\n", 173 | "#640 loss=0.850 test_loss=0.960 accuracy=0.883 time=0.036\n", 174 | "#650 loss=0.844 test_loss=0.942 accuracy=0.882 time=0.040\n", 175 | "#660 loss=0.832 test_loss=0.947 accuracy=0.882 time=0.036\n", 176 | "#670 loss=0.825 test_loss=0.920 accuracy=0.883 time=0.034\n", 177 | "#680 loss=0.805 test_loss=0.921 accuracy=0.882 time=0.029\n", 178 | "#690 loss=0.796 test_loss=0.903 accuracy=0.885 time=0.035\n", 179 | "#700 loss=0.800 test_loss=0.891 accuracy=0.884 time=0.035\n", 180 | "#710 loss=0.779 test_loss=0.893 accuracy=0.882 time=0.031\n", 181 | "#720 loss=0.763 test_loss=0.866 accuracy=0.884 time=0.036\n", 182 | "#730 loss=0.762 test_loss=0.858 accuracy=0.885 time=0.042\n", 183 | "#740 loss=0.750 test_loss=0.860 accuracy=0.884 time=0.036\n", 184 | "#750 loss=0.739 test_loss=0.839 accuracy=0.885 time=0.037\n", 185 | "#760 loss=0.729 test_loss=0.829 accuracy=0.886 time=0.036\n", 186 | "#770 loss=0.734 test_loss=0.832 accuracy=0.885 time=0.038\n", 187 | "#780 loss=0.717 test_loss=0.818 accuracy=0.886 time=0.036\n", 188 | "#790 loss=0.710 test_loss=0.808 accuracy=0.885 time=0.033\n", 189 | "#800 loss=0.702 test_loss=0.806 accuracy=0.885 time=0.029\n", 190 | "#810 loss=0.691 test_loss=0.785 accuracy=0.886 time=0.030\n", 191 | "#820 loss=0.685 test_loss=0.787 accuracy=0.885 time=0.029\n", 192 | "#830 loss=0.693 test_loss=0.775 accuracy=0.888 time=0.029\n", 193 | "#840 loss=0.678 test_loss=0.788 accuracy=0.884 time=0.031\n", 194 | "#850 loss=0.673 test_loss=0.758 accuracy=0.890 time=0.045\n", 195 | "#860 loss=0.655 test_loss=0.760 accuracy=0.886 time=0.038\n", 196 | "#870 loss=0.648 test_loss=0.748 accuracy=0.887 time=0.048\n", 197 | "#880 loss=0.654 test_loss=0.735 accuracy=0.891 time=0.046\n", 198 | "#890 loss=0.630 test_loss=0.736 accuracy=0.887 time=0.037\n", 199 | "#900 loss=0.626 test_loss=0.725 accuracy=0.888 time=0.038\n", 200 | "#910 loss=0.631 test_loss=0.716 accuracy=0.891 time=0.035\n", 201 | "#920 loss=0.612 test_loss=0.718 accuracy=0.889 time=0.035\n", 202 | "#930 loss=0.612 test_loss=0.703 accuracy=0.889 time=0.035\n", 203 | "#940 loss=0.619 test_loss=0.705 accuracy=0.890 time=0.029\n", 204 | "#950 loss=0.603 test_loss=0.701 accuracy=0.890 time=0.036\n", 205 | "#960 loss=0.598 test_loss=0.690 accuracy=0.890 time=0.035\n", 206 | "#970 loss=0.588 test_loss=0.683 accuracy=0.893 time=0.029\n", 207 | "#980 loss=0.580 test_loss=0.676 accuracy=0.892 time=0.043\n", 208 | "#990 loss=0.589 test_loss=0.667 accuracy=0.892 time=0.035\n", 209 | "#1000 loss=0.576 test_loss=0.682 accuracy=0.891 time=0.035\n" 210 | ] 211 | } 212 | ], 213 | "source": [ 214 | "nin=784\n", 215 | "nout=10\n", 216 | "W=torch.randn(nin,nout,device=device) \n", 217 | "W.requires_grad=True\n", 218 | "bias = torch.randn(nout,device=device)\n", 219 | "bias.requres_grad=True\n", 220 | "print(train_data[0][0].shape)\n", 221 | "lr=0.001\n", 222 | "optimizer = torch.optim.Adam([W,bias], lr=lr, betas=(0.9, 0.999))\n", 223 | "print_bin = epochs/100\n", 224 | "for epoch in range(1, epochs + 1):\n", 225 | " optimizer.zero_grad()\n", 226 | " t1=time.time()\n", 227 | " for batch_idx, (data, target) in enumerate(train_data):\n", 228 | "# loss = F.cross_entropy(data.view(data.shape[0],-1)@W+bias.expand(data.shape[0],nout),target)\n", 229 | " output=F.log_softmax(data.view(data.shape[0],-1)@W+bias.expand(data.shape[0],nout), dim=1)\n", 230 | " loss = F.nll_loss(output, target)\n", 231 | " loss.backward()\n", 232 | " optimizer.step()\n", 233 | " test_loss = 0\n", 234 | " correct = 0\n", 235 | " for data, target in test_data:\n", 236 | " output=F.log_softmax(data.view(data.shape[0],-1)@W+bias.expand(data.shape[0],nout), dim=1)\n", 237 | " test_loss += F.nll_loss(output, target, reduction='sum').item()\n", 238 | " pred = output.argmax(dim=1, keepdim=True)\n", 239 | " correct += pred.eq(target.view_as(pred)).sum().item()\n", 240 | " test_loss /= 10000\n", 241 | " correct /= 10000\n", 242 | " if(epoch % print_bin==0):\n", 243 | " print(\"#%d loss=%.3f test_loss=%.3f accuracy=%.3f time=%.3f\"%(epoch,loss,test_loss,correct,time.time()-t1))\n", 244 | " " 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": null, 250 | "metadata": {}, 251 | "outputs": [], 252 | "source": [] 253 | } 254 | ], 255 | "metadata": { 256 | "kernelspec": { 257 | "display_name": "Python 3", 258 | "language": "python", 259 | "name": "python3" 260 | }, 261 | "language_info": { 262 | "codemirror_mode": { 263 | "name": "ipython", 264 | "version": 3 265 | }, 266 | "file_extension": ".py", 267 | "mimetype": "text/x-python", 268 | "name": "python", 269 | "nbconvert_exporter": "python", 270 | "pygments_lexer": "ipython3", 271 | "version": "3.6.8" 272 | } 273 | }, 274 | "nbformat": 4, 275 | "nbformat_minor": 2 276 | } 277 | -------------------------------------------------------------------------------- /codes/neural_networks/lr_mlp_conv.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 21, 6 | "metadata": { 7 | "slideshow": { 8 | "slide_type": "" 9 | } 10 | }, 11 | "outputs": [], 12 | "source": [ 13 | "import time\n", 14 | "import torch\n", 15 | "import torchvision\n", 16 | "import torch.nn as nn\n", 17 | "import torch.nn.functional as F\n", 18 | "import torch.optim as optim\n", 19 | "from torchvision import datasets, transforms\n", 20 | "\n", 21 | "batch_size=1000\n", 22 | "test_batch_size=1000\n", 23 | "epochs=1000\n", 24 | "device=torch.device(\"cpu\")\n", 25 | "device=torch.device(\"cuda:0\")\n", 26 | "data_path='../../../data'\n", 27 | "kwargs = {'num_workers': 1, 'pin_memory': True} if device != torch.device('cpu') else {}\n", 28 | "train_loader = torch.utils.data.DataLoader( \n", 29 | " datasets.MNIST(data_path, train=True,download=True, \n", 30 | " transform=transforms.Compose([ transforms.ToTensor(),\n", 31 | " transforms.Normalize((0.1307,), (0.3081,)) ])), batch_size=batch_size,\n", 32 | " shuffle=False, **kwargs)\n", 33 | "test_loader = torch.utils.data.DataLoader(datasets.MNIST(data_path, train=False,\n", 34 | " transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,),\n", 35 | " (0.3081,)) ])), batch_size=test_batch_size, shuffle=False, **kwargs)\n", 36 | "train_data=[]\n", 37 | "test_data=[]\n", 38 | "for batch_idx, (data, target) in enumerate(train_loader):\n", 39 | " data, target = data.to(device), target.to(device)\n", 40 | " train_data.append((data,target))\n", 41 | "for batch_idx, (data, target) in enumerate(test_loader):\n", 42 | " data, target = data.to(device), target.to(device)\n", 43 | " test_data.append((data,target))" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 24, 49 | "metadata": { 50 | "scrolled": false, 51 | "slideshow": { 52 | "slide_type": "slide" 53 | } 54 | }, 55 | "outputs": [ 56 | { 57 | "name": "stdout", 58 | "output_type": "stream", 59 | "text": [ 60 | "#10 loss=0.231 test_loss=0.266 accuracy=0.923 time=0.419\n", 61 | "#20 loss=0.227 test_loss=0.257 accuracy=0.922 time=0.418\n", 62 | "#30 loss=0.492 test_loss=0.577 accuracy=0.828 time=0.418\n", 63 | "#40 loss=0.124 test_loss=0.097 accuracy=0.972 time=0.418\n", 64 | "#50 loss=0.108 test_loss=0.076 accuracy=0.977 time=0.418\n", 65 | "#60 loss=0.191 test_loss=0.168 accuracy=0.948 time=0.418\n", 66 | "#70 loss=0.122 test_loss=0.072 accuracy=0.978 time=0.419\n", 67 | "#80 loss=0.120 test_loss=0.075 accuracy=0.978 time=0.418\n", 68 | "#90 loss=0.224 test_loss=0.203 accuracy=0.938 time=0.418\n", 69 | "#100 loss=0.097 test_loss=0.058 accuracy=0.981 time=0.418\n", 70 | "#110 loss=0.094 test_loss=0.059 accuracy=0.983 time=0.418\n", 71 | "#120 loss=0.099 test_loss=0.051 accuracy=0.985 time=0.418\n", 72 | "#130 loss=0.206 test_loss=0.182 accuracy=0.949 time=0.418\n", 73 | "#140 loss=0.097 test_loss=0.050 accuracy=0.984 time=0.417\n", 74 | "#150 loss=0.322 test_loss=0.418 accuracy=0.905 time=0.417\n", 75 | "#160 loss=0.096 test_loss=0.053 accuracy=0.985 time=0.417\n", 76 | "#170 loss=0.100 test_loss=0.051 accuracy=0.985 time=0.418\n", 77 | "#180 loss=0.112 test_loss=0.065 accuracy=0.981 time=0.417\n", 78 | "#190 loss=0.097 test_loss=0.048 accuracy=0.987 time=0.417\n", 79 | "#200 loss=0.098 test_loss=0.048 accuracy=0.987 time=0.417\n", 80 | "#210 loss=0.172 test_loss=0.181 accuracy=0.958 time=0.417\n", 81 | "#220 loss=0.097 test_loss=0.053 accuracy=0.984 time=0.417\n", 82 | "#230 loss=0.101 test_loss=0.051 accuracy=0.985 time=0.418\n", 83 | "#240 loss=0.105 test_loss=0.058 accuracy=0.985 time=0.418\n", 84 | "#250 loss=0.103 test_loss=0.055 accuracy=0.985 time=0.417\n", 85 | "#260 loss=0.101 test_loss=0.053 accuracy=0.985 time=0.417\n", 86 | "#270 loss=0.103 test_loss=0.054 accuracy=0.986 time=0.417\n", 87 | "#280 loss=0.106 test_loss=0.056 accuracy=0.986 time=0.417\n", 88 | "#290 loss=0.115 test_loss=0.066 accuracy=0.984 time=0.418\n", 89 | "#300 loss=0.121 test_loss=0.064 accuracy=0.984 time=0.417\n", 90 | "#310 loss=0.135 test_loss=0.071 accuracy=0.982 time=0.417\n", 91 | "#320 loss=0.117 test_loss=0.063 accuracy=0.984 time=0.417\n", 92 | "#330 loss=0.114 test_loss=0.065 accuracy=0.985 time=0.417\n" 93 | ] 94 | }, 95 | { 96 | "ename": "KeyboardInterrupt", 97 | "evalue": "", 98 | "output_type": "error", 99 | "traceback": [ 100 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 101 | "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", 102 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'conv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 27\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 103 | "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 487\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 488\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 490\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 491\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 104 | "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 93\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 105 | "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 487\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 488\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 490\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 491\u001b[0m \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 106 | "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0mweak_script_method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 107 | "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m 1350\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1351\u001b[0m \u001b[0;31m# fused op is marginally faster\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1352\u001b[0;31m \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_unwrap_optional\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1353\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1354\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 108 | "\u001b[0;31mKeyboardInterrupt\u001b[0m: " 109 | ] 110 | } 111 | ], 112 | "source": [ 113 | "n_in=784\n", 114 | "n_out=10\n", 115 | "n_hidden=500\n", 116 | "model = \"lr\" # logistic regression\n", 117 | "model = \"fc2\" # 2-layer MLP\n", 118 | "model = \"conv\" # convolution\n", 119 | "if (model == \"lr\"):\n", 120 | " net = nn.Sequential(nn.Linear(n_in,n_out))\n", 121 | "elif(model == \"fc2\"):\n", 122 | " net = nn.Sequential(nn.Linear(n_in,n_hidden),nn.Sigmoid(),nn.Linear(n_hidden,n_out))\n", 123 | "elif(model == \"conv\"):\n", 124 | " net = nn.Sequential(nn.Conv2d(1,20,5),nn.ReLU(),nn.MaxPool2d(2),nn.Conv2d(20,50,5),nn.ReLU(),nn.MaxPool2d(2))\n", 125 | " net2 = nn.Sequential(nn.Linear(4*4*50,n_hidden),nn.ReLU(),nn.Linear(n_hidden,n_out))\n", 126 | "else:\n", 127 | " print(\"Wrong model\")\n", 128 | "net = net.to(device)\n", 129 | "net2 = net2.to(device)\n", 130 | "optimizer = torch.optim.Adam(list(net.parameters()), lr=0.001, betas=(0.9, 0.999))\n", 131 | "print_bin = epochs/100\n", 132 | "for epoch in range(1, epochs + 1):\n", 133 | " optimizer.zero_grad()\n", 134 | " t1=time.time()\n", 135 | " for batch_idx, (data, target) in enumerate(train_data):\n", 136 | " if(model == 'conv'):\n", 137 | " output=net(data).view(output.shape[0],-1)\n", 138 | " output = net2(output)\n", 139 | " else:\n", 140 | " output=net(data.view(data.shape[0],-1))\n", 141 | "\n", 142 | " loss = F.cross_entropy(output, target)\n", 143 | " loss.backward()\n", 144 | " optimizer.step()\n", 145 | " test_loss = 0\n", 146 | " correct = 0\n", 147 | " for data, target in test_data:\n", 148 | " if(model == 'conv'):\n", 149 | " output=net(data).view(output.shape[0],-1)\n", 150 | " output = net2(output)\n", 151 | " else:\n", 152 | " output=net(data.view(data.shape[0],-1))\n", 153 | " output=F.log_softmax(output, dim=1)\n", 154 | " test_loss += F.nll_loss(output, target, reduction='sum').item()\n", 155 | " pred = output.argmax(dim=1, keepdim=True)\n", 156 | " correct += pred.eq(target.view_as(pred)).sum().item()\n", 157 | " test_loss /= 10000\n", 158 | " correct /= 10000\n", 159 | " if(epoch % print_bin==0):\n", 160 | " print(\"#%d loss=%.3f test_loss=%.3f accuracy=%.3f time=%.3f\"%(epoch,loss,test_loss,correct,time.time()-t1))\n", 161 | " " 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": null, 167 | "metadata": {}, 168 | "outputs": [], 169 | "source": [] 170 | } 171 | ], 172 | "metadata": { 173 | "kernelspec": { 174 | "display_name": "Python 3", 175 | "language": "python", 176 | "name": "python3" 177 | }, 178 | "language_info": { 179 | "codemirror_mode": { 180 | "name": "ipython", 181 | "version": 3 182 | }, 183 | "file_extension": ".py", 184 | "mimetype": "text/x-python", 185 | "name": "python", 186 | "nbconvert_exporter": "python", 187 | "pygments_lexer": "ipython3", 188 | "version": "3.6.8" 189 | } 190 | }, 191 | "nbformat": 4, 192 | "nbformat_minor": 2 193 | } 194 | -------------------------------------------------------------------------------- /codes/neural_networks/tanh.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 34, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "data": { 10 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzsnXd8VfX5+N+fc+7M3mQRMlghEIJEhiKigoLioFrFUQfiRNvaaavf2tpl7c9Wq2irdStSq7VSGYqioMiGsGdCIJPsdZM7z+f3xw2QQEISuBiIn/frdbj3ns84zyXJec7neZ7P8wgpJQqFQqFQHEbrbQEUCoVCcWahFINCoVAo2qEUg0KhUCjaoRSDQqFQKNqhFINCoVAo2qEUg0KhUCjaoRSDQqFQKNqhFINCoVAo2qEUg0KhUCjaYeptAU6GmJgYmZqa2ttiKBQKxVnFhg0bqqSUsV31OysVQ2pqKuvXr+9tMRQKheKsQghxoDv9lClJoVAoFO1QikGhUCgU7VCKQaFQKBTtOCt9DB3h8XgoLi7G6XT2tiinBZvNRnJyMmazubdFUSgUfZyAKAYhxCvAdKBCSjm8g3YBPANcDjQDt0spN7a23QY82tr1d1LK109GhuLiYkJDQ0lNTcV/ub6DlJLq6mqKi4tJS0vrbXEUCkUfJ1CmpNeAqSdonwYMaj3uBl4AEEJEAY8BY4ExwGNCiMiTEcDpdBIdHd3nlAKAEILo6Og+uxpSKBRnFgFRDFLKFUDNCbpcDbwh/awGIoQQCcBlwFIpZY2UshZYyokVzAnpi0rhMH35uykUijOLb8rHkAQUtflc3Hqus/MKhUJxRmMYEpfXwOX14fYauLwGHp+Bxyfx+AzcPgNv63uvIfEeeZV4DQOfIfEaEuPwq5T4DP/hfw+G9Lcb0v9eSsnt56cRFWw5rd/tm1IMHT3uyhOcP34CIe7Gb4YiJSUlcJIpFIpvBYYhaXR6qW12U9fioaHFQ4PTQ0OLlyaXh0anl0anl2a3F4fLh8Ptpdnto8Xto8Xjf3V5fbg8Bk6vD4+vw1vVaUUIuConsc8ohmKgf5vPyUBp6/lJx5z/oqMJpJQvAi8C5ObmfvM/EYVCcUbi8vo4VO+itL6F8nonhxqcVDS6qGh0UdXootrhosbhpsbhxjjBnUMICLGYCLaaCLLqBFtM2C060SEW7GYdu1nHataxmTVsZh2rScNi0rCa2r7XMOsaJk1gNmmYNQ2TLo6cM+kCU+s5kybQhP+cLgSa5n/Vdf95/zmOvBfimzMpf1OKYQHwgBBiPn5Hc72UskwI8THwhzYO50uBX3xDMp0WlixZwg9+8AN8Ph+zZ8/m4YcfbtdeV1fHvHnzuP/++wEoLCxk+vTpbNu2rTfEVSjOCpweHwWVDgqqmiisclBQ5aCoppmimhYONTqRx9zw7WaduDArsSFW0mKCyU2NIirIQkSQmcjW13C7/wi1mQm1mbCbdTRN+fIgcOGq7+B/8o8RQhTjjzQyA0gp/w4swh+qug9/uOodrW01QojfAutap3pcSnkiJ/YZjc/nY86cOSxdupTk5GTOPfdcrrrqKoYNG3akT11dHc8///wRxaBQKNpT0eBka0k920sb2F5az55DTRyodrR72o8Ps5ESHcT5A2NIjrSTFGknMdxOfLiN+HAbIdY+s0WrVwjI/56U8sYu2iUwp5O2V4BXAiFHb7N27VoGDhxIeno6ADNnzuTDDz9spxgefvhh8vPzycnJYcqUKcyZMwefz8ddd93F119/TVJSEh9++CF2u723voZC8Y3hMyQ7yxpYXVDNpoN1bDpYS2n90bDstJhghsaHcuXIRAbFhZARG0JqTBBBFnXjP530yf/d3/xvOztKGwI657DEMB67MuuEfUpKSujf/6grJTk5mTVr1rTr88QTT7Bt2zby8vIAvylp7969vPPOO7z00ktcf/31vP/++9xyyy0BlV+hOFMoqmnmi90VLN9Txdr91TQ4vQAkRdgZnRrFnf0jyE4OJzMhTD359xLqfz2AyGMNnXTPWZSWlkZOTg4Ao0ePprCwMNCiKRS9hpSS7aUNLNpaxic7DrGvogmA/lF2Lh+RwLj0aMalRxMfbutlSRWH6ZOKoasn+9NFcnIyRUVHt2UUFxeTmJjI3LlzeemllwB4/vnnjxtntVqPvNd1nZaWltMvrEJxmtlf5eD9DcV8uLmEopoWdE0wLj2KG8ekMGlILOkxwWrj5hlKn1QMvcW5557L3r172b9/P0lJScyfP5958+aRlZXFnDl+F0t1dTWNjY29LKlCcXpwenws3FLGvLUH2XCgFk3A+QNjePCiQUwZ1o/I0xx/rwgMSjEEEJPJxHPPPcdll12Gz+dj1qxZZGW1X71ER0dz/vnnM3z4cKZNm3ZEYSgUZzOHGpy8saqQd9YWUeNwkx4bzMPThjJjVBL9wpSJ6GxDdGQXP9PJzc2Vx5b23LlzJ5mZmb0k0TfDt+E7Ks4uimqaeWF5Pu+tL8ZrGEzO7Mdt56VyXkbfTGh5tiOE2CClzO2qn1oxKBSKHlNe7+SvS/fw3sZidCG4LjeZeydmkBId1NuiKQKAUgwKhaLbNDg9/P2LfF5ZuR/DgFvHD+CeiRkqoqiPoRSDQqHoEiklCzaX8ruFO6lsdHFNTiI/vnQI/aPUCqEvohSDQqE4IfurHDz6362s3FfNyORwXr4tl+zkiN4WS3EaUYpBoVB0iJSSN1cf4A+LdmLWNX57dRY3jR2ArhLN9XmUYlAoFMdRXu/kp+9t5su9VVw4OJYnr8tWYaffIpRiUCiOwWj20LKjhqBRcQj92/d0/NXeKh58ZyNOj8HvrhnOzWNTVOjptwylGBSKY6j9YB8tW6vwlDuImJ7e2+J8YxiG5IXl+Tz1yW4GxoXwwi2jyYgN6W2xFL2A1tsC9DVmzZpFXFwcw4cP77D9cD2GwxQWFnbaV/HN07K9ipatVZhi7TR9VYJjU0Vvi/SN0Oz2cu9bG/jzx7uZnp3If+ecr5TCtxilGALM7bffzpIlSzptP1YxKM4cjBYvtf/Nx5wQTL/vj8KSFk7t+3txlzT1tminlYpGJzNfXM2nOw/xq+nDeGZmjqp38C0nIIpBCDFVCLFbCLFPCPFwB+1/FULktR57hBB1bdp8bdoWBEKe3mTixIlERUV12t62UM9Pf/pTgCOFerKysrj00ktVdtVeon7xfowmN5HXDkKYdaJvHooebKb6jR0YrTUD+hr7KhqZMfdr9h5q4qVbc5k1IU35ExSn7mMQQujAXGAKUAysE0IskFLuONxHSvlQm/4PAqPaTNEipcw5VTnasfhhKN8a0CmJHwHTnjjlaVShnjMTV2E9jrXlhFyYjCU5FAA9xELkdwdR9c9tOPfUEpQd28tSBpYtxXV87+W1mHWNf90zTu1NUBwhECuGMcA+KWWBlNINzAeuPkH/G4F3AnDdPoMq1NP7NG+sQFh1wientDtvTQtHmDXchYGtCNjbbDhQy80vrSHMbuKD+89TSkHRjkAYEpOAojafi4GxHXUUQgwA0oBlbU7bhBDrAS/whJTyv6csUQCe7AOJKtRz5uPcV4c1PRxh1tudF7qGpX8orgN9RzGsLqhm1mvr6Bdm4+3ZY0mMUPXFFe0JxIqhI4NkZ7m8ZwLvSSl9bc6ltKaBvQl4WgiR0eFFhLhbCLFeCLG+srLy1CT+hpkzZw55eXnk5eUxZMgQVajnDMNb3YKvxoltUGSH7ZbUMDylTRius9/PsOFADXe8uo7ECDv/unucUgqKDgmEYigG+rf5nAyUdtJ3JseYkaSUpa2vBcAXtPc/tO33opQyV0qZGxt75tp6b7zxRsaPH8/u3btJTk7m5ZdfbtfetlDPYeezondx7vPHQlgHdWxOsaaGgwT3wbNboe8sa+COV9fRL8zKO3eNI07tZFZ0QiBMSeuAQUKINKAE/83/pmM7CSGGAJHAqjbnIoFmKaVLCBEDnA88GQCZeo133unafTJv3rx2n7dt23bk/U9+8pOAy6Q4Ma69tejhVkwxHT89W1JCQYCrsKHTVcWZzoFqB7e+spYgi4k37xxLbKi160GKby2nrBiklF4hxAPAx4AOvCKl3C6EeBxYL6U8HIJ6IzBfti8Zlwn8Qwhh4F+9PNE2mkmhON1IQ+LMr8ee1XnFMc1mwhwfjPss9TNUNbn43str8fgM5t0zXqXKVnRJQHaxSCkXAYuOOferYz7/uoNxXwMjAiGDQnEyeEqakC1ebJ2YkQ5jSQ2jecMhpE+eVfmTXF4f97y5gUMNTubfPY5B/UJ7WyTFWYDa+az4VuPcWwuANePEisGaGo50G3jKzp5d0FJKfvH+VjYcqOUv1+cwKuXsNIMpvnmUYlB8q3HurcOcEIweYjlhP0tqGOD3M5wtvLA8n/9sKuFHUwZzRXZCb4ujOItQikHxrcVw+XAfbMDaDYeyKdyKHmE9a/wMn++q4Mklu7lqZCIPXjywt8VRnGUoxaD41uIqrAef7NK/cBhrahiuwnrax0+ceRTXNvPDf+WRmRDGk9dlq9xHih6jFIPiW4uroB50gbXVTNQVltRwjEYPvhrnaZbs5HF5fcyZt8lfW+Hmc7Ads5NboegOSjEovrV4ihsxJwQflwajMyz9/RE9Z3Ia7j8s3Mnmojr+/N1sUmOCe1scxVmKUgwBpKioiIsuuojMzEyysrJ45plnjuujCvWcGUgpcZc4sCR1vxjN4Q1w3uozM5fVkm3lvL7qALPOT2PqcOVsVpw8SjEEEJPJxFNPPcXOnTtZvXo1c+fOZceO9vv1VKGeMwNfjRPp9GLugWLQrDpaqAVv1ZlnSqpodPKL/2xheFIYD08b2tviKM5ylGIIIAkJCZxzzjkAhIaGkpmZSUlJSbs+qlDPmcFhc5AlsWflK03RtjNuxSCl5GfvbaHZ7ePpG3KwmNSfteLU6JP1+/609k/sqtkV0DmHRg3l52N+3u3+hYWFbNq0ibFj22cgV4V6zgzcJU2gC8zxPbPDm6LtOPfUnCapTo631xzki92V/PrKYQyMUzubFaeOerQ4DTQ1NXHttdfy9NNPExbWdcSLKtTzzeMpacIcH4zo4dO1KcaO0eg5Y1Jw769y8PuFO7lgUAy3jk/tbXEUfYQ+uWLoyZN9oPF4PFx77bXcfPPNfOc73wFUoZ4zDb/juYmgETE9HmuK9qeq9lY7e2yGCjSGIXn4/S2YdcH/++5INE3tV1AEBrViCCBSSu68804yMzP50Y9+dOS8KtRzZuGrdSFbeuZ4PsyZFJk0f10Ra/bX8MgVmfRTtRUUAUQphgCycuVK3nzzTZYtW0ZOTg45OTksWtQu6awq1HMG4C7xK+aehKoe5siKoZcjkw41OPnjop2MT4/m+tz+XQ9QKHpAnzQl9RYTJkzoVroEVaind/GcpOMZQLOa0ELNvbpikFLyf//dhttn8MfvjFApLxQBR60YFN863CVNmPsF9djxfBhTtB1vVe8pho+3H+KTHYd4aMpgtbtZcVoIiGIQQkwVQuwWQuwTQjzcQfvtQohKIURe6zG7TdttQoi9rcdtgZBHoegMKSWekiYsSScf1mmKtuOt7h1TUovbx28/2sHQ+FBmT0jrFRkUfZ9TNiUJIXRgLjAFKAbWCSEWdFCi819SygeOGRsFPAbkAhLY0Dq29lTlUig6wlfrwmg+OcfzYUwxNowNbgyXD836zSape/6LfZTUtfCvu8dh0tWCX3F6CMRv1hhgn5SyQErpBuYDV3dz7GXAUillTasyWApMDYBMCkWHHNnxfCqKIfqYyCQpoakS6oqOHl7XKct6LIVVDv6xvIBrchIZmx4d8PkVisMEwvmcBBS1+VwMjO2g37VCiInAHuAhKWVRJ2OTAiCTQtEhntIm0E7O8XwYU7jf2etdMheL7xN8FftYWRJBg8e/H0UDRkWXk5AYA7FDIGU8DL0CojNOSfbHP9qBWRf84vLMU5pHoeiKQKwYOgqJODY0539AqpQyG/gUeL0HY/0dhbhbCLFeCLG+srLypIVVfLvxlDZhjrMjzD381ZcS8pfB/JsxvZELgPfgQTDbWeGZyLrq/lTo6VSYMihwJfHf8tE0hQ+D6n2w9P/g2XNg7jj48ilo6bml9LOdh1i2q4IfTh6s9iwoTjuBUAzFQNtA6mSgtG0HKWW1lPLw2volYHR3x7aZ40UpZa6UMjc2NjYAYiu+jXjKHJgTemBGMnyw+V/w9wvgzRlQtBZt9PVodok360H2DvslG3fWMmrqlcz6+3xmvfAOM3//NG5DY1Fhf4z7VsMPt8LUP0FQFHz2OPwlCxY/7Dc5dUdmn8HvF+0kPSaY285LPbkvrlD0gEAohnXAICFEmhDCAswEFrTtIIRomxz+KmBn6/uPgUuFEJFCiEjg0tZzCkXA8Tk8+BrcmBO6aUba/yX8YyJ8cDf43HDVc/DQNrj8SUz9wnGVN/HxC8/QL30QE2+ZdWRYTP8BTL7zfop2bGXVe/MgIgXG3Qt3LIJ7v4LMK2HdS/BcLnz+B3A7TijG/LUHKah08IvLM1XmVMU3win/lkkpvcAD+G/oO4F3pZTbhRCPCyGuau32fSHEdiHEZuD7wO2tY2uA3+JXLuuAx1vPnZU4nU7GjBnDyJEjycrK4rHHHjuujyrU03t4yv034C79Cw2l8O6t8Pp0cDbAd1+D+1fDOd8Dk9+PoEdaaS7x/6pe+dDPMZnN7abIuvASsiZNZvUH71K4ZdPRhvgR8J1/wPc3wdDpsPxP8GwubPtPx6I4Pfz1072MTYticmbcyX1xhaKHBOTxQ0q5SEo5WEqZIaX8feu5X0kpF7S+/4WUMktKOVJKeZGUclebsa9IKQe2Hq8GQp7ewmq1smzZMjZv3kxeXh5Llixh9erV7fqoQj29h6esVTF0tmKQEra8C8+Pgz2fwEWPwgNrIWsGaO3/VOqdlVixc8nt9xMeF9/hdJfMupeIuHi+/vfbxzdGpMB1L8MdSyAkFt67A/59Oziq23V74Yt8ahxuHr1imNrhrPjGUOvSACKEICTEb7/2eDx4PJ7j/phVoZ7ew1PmQAsxo4dajm90VPtXCf+5C2KHwn0r4cKfgtne4VyFBf56GukDR3V6PbPVRs5l0ynbs4uKwoKOOw0YD7OXwSW/gp0ftSolvzW1pK6Fl7/az4xRSYxIDu/Zl1UoToE+mSup/A9/wLUzsIV6rJlDif/lL7vs5/P5GD16NPv27WPOnDmqUM8ZhKfc0fFqoWgd/Ps2cFTC5N/AeQ+C1vnGtZrSEvYXbGJwUg6+GtcJA6yHXXgxX77zGls+XcLk2fd33Ek3wQU/hkGXwgf3wrzr4fwf8HTN1QjgJ5cN6dkXVShOEbViCDC6rpOXl0dxcTFr165tlyCvM1ShntOP9Ek8h45RDFLC6r/Dq9NAM8Gdn8CEH55QKQBs+WwJLYZ/o5yv7sQb2ewhoQwZfwE7v/oct7OLlWD8CJj9GeTOgpXPcN32+7k/N4SkiI5XLQrF6aJPrhi682R/uomIiGDSpEksWbKE5cuXq0I9vYy3qhm88qjj2dMCHz4A296DIZfDNc+DPbLredxuti//jJRzchB1Gr56d5djRk6Zxo4Vy9i1cjnZl3Sxsd9sg+l/5ZWieGaWP8W5e+6AonnQf0x3vqZCERDUiiGAVFZWUldXB0BLSwuffvopQ4cOVYV6zgCORCQlhPijjl6dBtveh4v/D254u1tKAWDvmpU4GxsYOeVy9AgrvvquU18kDBpKbEoqmz9Z3K207FuK63j8wHDeO+d1NGsIvHYF5M3rcpxCESiUYgggZWVlXHTRRWRnZ3PuuecyZcoUpk+f3q6PKtTTO3jKHP4aDO6d8OJFULUXZs6DiT85LuLoRGz+dDER8QmkDM9GD7d2aUoCf1BC9uRpVBTmcyh/b5f9/98ne4gMMjPjsslw1zJIGQf/vQ8+fsS/4U6hOM30SVNSb5Gdnc2mTZu67KcK9XzzeMocmMM8iDevhZB+8L0PoN+wHs1RXVJEya4dTLz5DoSmoYdbce3tXnqLzAsuYsXbr7Jl2cfEDxzcab/VBdWs2FPJI5dnEmozA1Fwy3/g41/CquegZj9c+xJYVB0GxelDrRgUfR8p8Rw4hLl+hd/Be9eyHisFgH1rVwEwdMKFAOjhFnyNbqSva/OQNSiI9NFj2LduNUYnT/1SSv6ydA/9wqx8b/yAow26GS7/M0x7EvYshlcvh8byHsuvUHQXpRgUfRufF99/H8bntPgjkm77HwTHnNRU+RvW0C99EKFR/vF6hBUk+Bq7l2I7I3csLQ31lO/b02H76oIa1u6v4b4LM7CZO4iMGnsPzHzHbwZ76RKo2Hl8H4UiACjFoOi7uJpg/k14Nq4EwHzprE43rHWFo66Wsn17yMg9Gh2kh/ujyboTmQSQljMaTdfJX7+mw/ZnPttDXKiVmWNSOp9kyFSYtRgML7x8GRQs7/6XUCi6iVIMir5J4yF47XLYtxRP5kMAmBNPvjhP/oa1ICUDc8cdOWc6rBi64YAGsAWHkJw5nH0dKIY1BdWsLqjh3s5WC21JGAmzP4WwRHjrWn/2V4UigCjFoOh7VOyCf05ujTx6B495ROepMLpJ/oY1hMXGEZOSeuTc0RVD96u1ZeSOpaakiNqyknbnn/lsL7GhVm4ae4LVQlsi+sOsJf6IpQ/uhuVP+jfsKRQBQCkGRd9i/5fw8qXgc/nTXA+Z6i/OcwqrBY/LycEteWSMHtsu95Ww6QiL3jPFMNpvisrfsPbIuXWFNXydX809E9O7Xi20xR7hj1jKngmf/x4WPAA+T/fHKxSdoBSDou+w+V/+Yjqh8X5TS+IopNfAU9GMJfHkwzsPbN2M1+MmY3T7vFdCCPQIS7dNSQDhcfHEpKSSv+GoOelvn+0lJsTCzWMHnGBkJ5gsMOPvMPFnsOktf54lZ33P51Eo2qAUg+LsR0r4/I9+k0rKOLjzY39aa8BzqBl8smdV244hf/0aLPYgkodlHdemh1vxNnTP+XyYgbljKdm1g5bGBrYU1/Hl3ipmX5CO3dKD1UJbhICLH/EXEtq/Al6ZCnUHT24uhQKlGE4LPp+PUaNGHbfrGVShnoDjcfpTZS9/Akbe5DettElv4Sn1J7szn+SKwTB8FGxcS9qoXHST+bj27u5+bkvG6LFIw2D/pvU8/3k+YTYTN3fXt3Aizvke3Pwe1Jf4w1mLN5z6nIpvJQFRDEKIqUKI3UKIfUKIhzto/5EQYocQYosQ4jMhxIA2bT4hRF7rseDYsWcjzzzzDJmZmR22qUI9AaSpEt64Grb+25/z6Jrn/aaVNnjKHAiLjin65MJUy/ftobm+jozcsR226+FWjCY30mt0e85+6QMJjowib+VKlmwv5/bzUlt3OQeAjItg9lJ/WO5rl3daGU6hOBGnrBiEEDowF5gGDANuFEIcu610E5ArpcwG3gOebNPWIqXMaT2u4iynuLiYhQsXMnv27A7bVaGeAFG2BV6cBGWb4bpX/TmPOqhw5i5twpwQjNBOrvpZ4eaNIASpI8/psN0UfniTW/fNSULTSMsZTcn2zQSZBLefn3ZSsnVK7BD/7u6EHH9luGW/A6P7ikuhCESupDHAPillAYAQYj5wNbDjcAcp5edt+q8GTmsVmi/f3UNVUVNA54zpH8IF13ee4+YwP/zhD3nyySc7zaCqCvUEgB0f+gva2CP9m70SO66iJg2Jp9RB0OiTr5V8YEse8RmDsIeEdtiuRxwNWTVF2ro9b0h6FtrnS7kpQyMq2L/KaahqYdPSgxza34CmCzRNEBRuJWtiIslDIntW2jM4Bm5bAAt/BCv+7N8lPeMfYD15X4vi20MgTElJQFGbz8WcsKYVdwKL23y2CSHWCyFWCyGu6WyQEOLu1n7rKysrT03i08RHH31EXFwco0eP7tE4Vainm/i8sPQxfwnOfsPhrs87VQoA3hon0u3DcpKhqk5HE2X7dpOa3fk19HD/Tb2nfoaPa0ORwBhLJfWVzXz66g7e+tVqdnxVii3YhNVuQjdrlOypZcHTecz/7Vp2fl3WrbTdRzBZ/Q7pqU/A7kXw0sX+vR0KRRcEYsXQ0WNMh7+9QohbgFzgwjanU6SUpUKIdGCZEGKrlDL/uAmlfBF4ESA3N/eEfx3debI/HaxcuZIFCxawaNEinE4nDQ0N3HLLLYwfP14V6jlVmirh/Vn+qJvRd8C0P/lvfCfgqOP55BRD0fYtSMNgwAkVQ8/SYgBUN7n419Ya7ohM4tDWPN7bmIrX4yP74mRyLkkhJPLo9/J6fOxdV8GWz4tY9sZO9m+u5OJbM7EFd9MnIQSMuw/ihvnNSi9e5PfFDDvrrbaK00ggVgzFQP82n5OB0mM7CSEmA48AV0kpjzxeSSlLW18LgC+Azv8Kz3D++Mc/UlxcTGFhIfPnz+fiiy/mrbfeUoV6TpUDq+DFC+HgGrh6Llz5dJdKAVoVgyYw9ws6uctu2YTZZidh0NBO+2g2E8Las01ur686gMtrMDAzh4r9e9BMHm54dAwTrhvUTikAmMw6meclcP0vz2XCdwdxYGs17/5hHYcKG3r2ZdIvhHtWQOxgePd7/toOXjfS48NweXu2ElH0eQKxYlgHDBJCpAElwEzgprYdhBCjgH8AU6WUFW3ORwLNUkqXECIGOJ/2juk+R9tCPdOmTWPOnDm9LdKZi+GDL5+CL/4IEQP8NZkTc7o93F3qwNwvCGE6ueefwi2b6J81At104j8TPdyKt5umpGa3lzdWFXJlSgxV2+sAyehLdSLiTqy8hBCMvKQ//dLD+Pilbfzn/23gstnDSc+J7ea3AcKT4Y7F8PEjGF//k6Y8M41Nk5EeCbpACzJjzQgn7OIUzF3Io+jbnLJikFJ6hRAPAB8DOvCKlHK7EOJxYL2UcgHwZyAE+HerA+1gawRSJvAPIYSBf/XyhJRyR4cXOsuYNGkSkyZN6rBNFerpBvXFfgdz4Zcw4rtwxV/AFtajKTylTdiGRJ3U5evKy6g/VM7oy6/usq8eYcXX0D3F8O/1xTQ7PGQ3uTGHJuNqsFJ5YAftraudE58Wzg2PjOGj5zaz5MVtTJk1jEG5/bo1FgCTFUesex0WAAAgAElEQVT8T6nXrsSo1bCbVmPJGYoRnoWvwU3L9ipaNldiHxFD2JQBmGOVgvg2EpAKblLKRcCiY879qs37yZ2M+xoYEQgZFH0EKWHzfFj8c39q6avnQs7NHYaiHqbR3cje2r3srd1LeXM5tc5a3PUt3N80nYWOpez+6g2i7FEMjhzM4MjBpIWnYdZObKM/sNVfiW9Adsdhqm0xhVtpKe06Cs7rM3jpywKu04Lx1Hm46me5fP1uNge2dF31ry22YDNX/SCHhXO3sPTl7fg8BkPHJ3RrbPPWKmr/vQdLSjjhE4OwrlkKu34LmVfB9L/im55O05fFNH1dhnNnDRFXDyS4J4pH0SdQpT0VZw6N5bDwx7DrI0gZ73eSRqUf162qpYq1ZWtZW76WdeXrONh4NP2DSZiIsEVwXrPfVbVR28b28nyqWqrwGP4Ec8HmYCb1n8SlAy7l/KTzserH+ysKN28iLDaOyITELsXWwy0YTR6k1zih2WrxtnJMFS6SHFZGXTaAfmlhDMgeRcHGddRXlBMeF9/ltQ5jsZmY/uBIFj2/hc9e34nQBEPGnni8t9ZJ7ft7MSeHEHt3tl/WYYvg62f9SfieX4U+/a+ET72SkPOSqJm/i9r39uAqqCPimoFoJ5uyQ3HWoRSDovcxDNj4Giz9NXidMOW3MH4OaEdvRPvr9/PZwc/4vOhztlRuASDUHEpufC4zBs04shroF9QPIQQNyw7SsOsAT13/LJrNhMfwcKD+ALtrd7O2fC2fHviUhQULCbeGc+uwW7lp6E2EWPzRS4bPx8Ftmxly3gXd2jtwJDKpwY0pquO9DFJKXv58H1c4rUQmBDNmun9T2+GIpwNb8siePLVH/21mi84Vc7L56Dm/crDYdNJGduxzkD5JzfzdICXRNw49qsA0HSb8EAZNgQ/ugX/dAkOno097kpjZI2j47CCNyw7iLmki5vasHu3VUJy9KMWg6F3Kt8LCn0DRaki9AKY/DTED/U2OchbvX8yi/YvYVbMLgKzoLB7IeYAJyRMYGjkUXev4KdZT2oQebUOz+X/FzZqZgZEDGRg5kCvSr+DRcY+ypmwN83fN59lNz/La9te4Pet2bs+6ncp9+bhbmhkwonsBckc2udW5OlUMqwtq6FfgJMgwMfn2THSz/8YclZhMaHQshVs29lgxgD9q6fL7RvDh03l8/NJ2pj84kuQhkcf1a1h2EPeBBqJuGNJxepB+Wf59IavmwhdPwNyxiIsfIfzi2VhTw6h+eycVc/OIuS0LS/+ON/sp+g5KMSh6B0eVP1XDxtfBFgFXPw85N9HsbWHpvg9ZkL+AdeXrkEhGxIzgZ+f+jCkDphAf3D1zi7uoEUtqeKftZs3MhKQJTEiawPbq7fx98995dtOzLCpYxG11ExFCI2XEyG5d6+iKoXMH9Nsf72WEW2fEJcnEDTjqRBdCMCA7h71rv8bw+dD0nptrLDYTVz4wkg/+spFFz2/hmh+NancNb1ULjcsOEnROHEGjTrALXDf7Vw/Drvab9JY8DOtfxTb1D8TdN56q17ZT+eIWom4Ygn34ydXNVpwdqOyqim8WtwO+/Av87RzY9CaMvRf54AbWJwzh0ZX/x6R3J/Hoykcpc5RxX859LJyxkHlXzON7w77XbaXgrXfhq3djSenek21WdBbPXvwsL0x+AYfXwepVi5HxIZjs3TObHN793FnIan5lE+YdjWDWGHP58XmRUkeeg8vhoDz/5Hcl20LMXPX9HGwhZv737GZqyx1H2ppWl4EmCJ/WzZxMUWlwy/sw8x3wueGtazEvvY24ay2Y44OpfnsnjV+VdD2P4qxFKQbFN4PXBWtfgr+Ngs9+AynjKLv9Q/6RkMoVS27hjo/v4NODnzItbRqvT32dhTMWct/I+0gJ63k6avcB/+Yva0rPwlsnJE1g/iVvEltvJS/4ILM/mU1VS1WX4zSrf5Ob0Uldhjf/t4eBXp3sySkd7lhOGT4ShPAn7DsFgiOsXPWDHISABc/k0VjjxHD7cKwvxz48pmelTYWAoZfDnDUw5XEoXov+1oXExs7FPtBG/UcF1C3IRxpqY1xfRCkGxenF1QRfPwfPjIRFP6E5Op0Flz/G7NgwLvv8fp7Le46E4AT+MOEPLPvuMn5z3m84p985PUsYdwzug41g0jAn9LwGQ/WefISEaybfwY7qHcz8aCbbq7d3OU4Ps3RoSqpucuHaWIPPIhg/NbXDsfbQMOLTB/Y4bLUjIuKCuPL7ObhbvPzvb3nUry5DOn2EdDOc9ThMVjj/B/CDLXDBjxF7PyKqaCohcdto+rqU6jd3YLi8pyy34sxCKYYAk5qayogRI8jJySE3N/e49m9NoZ76EvjscXh6ON5PHmFldBK/HHMtkyzVPLLzVYobi7lv5H0suXYJL1/2MldmXEmQOTCbqdwHGrAkh5zUjucDmzdhsQdxzQW38fq01xFCcNvi21hetPyE4/yK4fgVw9sf7iHZq5E1uT9ma+f+g9SR51C2bzdOx6lnBY7tH8oVc7JpqHZSsaQQU3wQlgE9Wz0dhz0CLvkV/GAz4vwHiHD+ngjT33HurKLi6ZV4q5pPWW7FmYNSDKeBzz//nLy8PNavX39cW58u1GMYULAc3r0N39MjWL9uLr9PTOWSwVncKyr4om4Xl6ddwetTX2fxdxZzX859JIWcKBFvz5EeA3dp00ndCKWUFG7ZSMrwbHSTiWHRw5h/xXwGRgzkh5//kCX7l3Q6Vg+zHqcYnB4vNasrcFkEky4/fj9GWwZkj0IaBkXbtvRY7o5IHBTJtBnpBBuS3Q0ePC5fQOYlJA6m/AYe2kbIpaOICfsrvtpmKv6yHOf/3oDmmsBcR9GrqKikb5i2hXqmTJnCnDlzjhTq+frrr0lKSuLDDz/Ebj+5imO9Qm0hbP4Xnry3WOesYFlYBJ+mp1NtuLCKJi5MvJDL0y/ngqQLsOg9sHOfBO7SJvBJrN10PLeltqyUhsoKxlx93ZFz0fZo/nnpP3lg2QP8bMXPcHgcXDv42uPG6uH+FYM05JGiQO8vyifWI0iakojexeolYdBQLHY7hVs2MmjseT2WvSNCyh00WzR2lbdQ/dxmrnww54Srlh5hj4CJP8F23veJW7OQ6iWSqpUphK39JaHDnYhzbob0Sf5IJ8VZR59UDJ+/9iIVBwoCOmfcgHQuuv3uLvsJIbj00ksRQnDPPfdw993tx/SZQj11B2HHAqq2v8eq+t2ssAfxVWQwTcRh121MSL6AS1OmcF5INuayKtxbimlY/Breigr/UV2N4XD4j5YWhKaByYQwmTBFRWHq1w9TXBzWwYOwjxiBdeBAhLnrm8xhx7Olh45n4Ijz99g0GCGWEF6Y/AIPffEQv171a3zSx/VDrm/XRw+1gCExmj3oIRaklOxZUUqYDtOvHNjltXWTif5ZIyncvAkp5Sn5WMBfUa5lWzWhExK5OC6YpS9v53/P5nHFnJFY7Sa8lZW0bN2Gc/t2PMXFeCsr8FRUIJtbkD4f0utFmExowcFowcHo4eGYYmMxxcVhju+HuX8Klv7JmBMTMZ8/g7jRXmrnradhz824tm4nasfd6EH403tnzYAB5yslcRbRJxVDb7Jy5UoSExOpqKhgypQpDB06lIkTJ55wzFlRqMfnhdKNNO9eRF7+YtY4y1hlt7PTaoGYaAY6w7i9ZTijHDEkVPrwfXQQ1/7fUFJf324aLSzMf4OJjsacnIwWHIRms4OU/huSy4W3phpXfj6Or77CaPbbroXVSvC4cYROnUroxRehh3e8R8F9sAE9ytazCJxWDmzZSES/BCL6HR8WazfZefaiZ3noi4f43erfYTfZuTLjyiPtbesy6CEWPl1VTJxDEpwbjambqSRSs0eRv341deWlRCacmonNuasGDEnQqH4MSggGJJ++soP3fr6YnF0voR1sDY3VNEzx/TDHxmFNS0MLDgGTjtBNSK8Hw9GM4XDgq6vDtXcv3qoq8LUxS5lMWPr3x5KejiUtDVvGMJyFWRySrxEe8RVBm+ciNrwGtnAYOAWGTPOvJILVPogzmT6pGLrzZH+6SEz059aJi4tjxowZrF27lq1bt559hXoMHxzaTm3BMvIKP2VT7R42meGAz0xiFQyoDOWahgh+XGsltLgO0VwLfAlAS2wM1rR0wqZNxZKaiiVlgP/pMikJLaj7DmYpJZ6DB2nZuo2WvDwal31G0/LllJnNhE2ZQvRds7FlZrbr7zrYiDW9841tneHzeijavpVhF17SaR+zbuapSU8x59M5PLryUWwmG1MGTAFAC2ut5NZa+3nVwv2ECcmM73Zey+FYDteVLty8MSCKQQ+3oIdKql97Dd54gxHOSLZm3cW69Du55LpGokdnYsvM7NnPxOfDW1mJp6gId1Ex7sJC3Pv349pfQNOKFeDxoIXEYzvndmrdE6n1DsZi34jNvQ/rxi+wbHkPoQHx2X4FkToB+o/xl2lVnDH0ScXQWzgcDgzDIDQ0FIfDwSeffMKvfvUrpk6deqTuQnV19ZlXqEdKaDpE7YGv2HXwC3ZXbqOgqoyaeg1rnUZylSSlysTEakGw42hReT3ch3VwMtbci7AOGoh10CCsAweiR0QERCwhBJYBA7AMGED49Cvo98gvcW7dSsPChdT9+z0aFi0ieOIFxD74IPYRI/DVuzAa3D3evwBQunsnHpfzhGU8Aay6lb9d/DfuWXoPP1vxM567+DnOTzofPexo7edte6uJrPYgB4USHt51QaHDRMQnEN4vnsItmxg19cquB3SC9Bo499YiTJXsu+QHGPX1BJ17LiO+ex2pqdksfmUvS/dGMfWiDBJ7oBQAhK5jjo/HHB9P0Lnntr+ux4P74EFce/fi3L0X94GNSH0YLucUmnaace85ADIIS1wo1tAWLObXsYa9iCXUhyV9EFpqLiSNhsRzIC5TmZ56EaUYAsihQ4eYMWMGAF6vl5tuuompU9vnv+ntQj2yuYbq0g0Ulq6hqHArlUX7cVQ2YjRAaJ0goUYyvBZGe4PwmIPwmoLwhISjJ6dijEyhKTYBLbYfWnQsIiiYFs1ftF43aZjRsZT4sNU1EBplwx5qPmVbeVuEENizs7FnZxNz//3UvjOfmjfeoPD6G4i88UZCLr0NoNs7nttSsGk9mm6if1Z2l32DzEHMnTyXWUtm8dAXD/HqZa8yLDIThD+R3qJlhdiAq6/v/mrhMKkjR7N9+ad43W5MlpNz1Nd9sALp1mleMZ+g3Fxi7pqNvdVUGQ5c93A4i17Yyod/3cQFMwczfGJgIsOE2Yw1IwNrRgZhrb/3vgYXdYsKaDFNwTJ4Mpq1FF/Zalr2bqOhxAG07jURdZhDPsES8hGWUC+WUIklKRHzwKGYB+egJQzzV5+LGNAuuaLi9CACUdJPCDEVeAZ/oZ5/SimfOKbdCrwBjAaqgRuklIWtbb8A7gR8wPellB93db3c3Fx5bCjozp07yWxjVuiLdOc7Gs4Gqit3UFG0marCHdSXFNJUWYurVkKTDZMzhCBXCHZvCIYegsccisccjNscitsWgtcShlezIQMQyaybNcJi7MT2DyE2JZS41DD6pYWh64GLkvY1NVH5zN+offtt7Lm3YkoeT9JvJyB6eI1XHrqXsJhYrnvkt90eU9lcyS2LbsHpc/LmtDcx/a0UmRbOu1+W4o638fNfn9/Tr8P+vA3854+PMePhx0gfdW7XA9rgramh/LHHcJdGYM64hKhrQwkeO7rDvq5mD0tf2cGBbdUMHtuPC64f3P060ieBp7KZhqUHaNlSBbrAPjyGoJxIMKpwF+zHVZCPp/AArvw9uA8UIV3tQ39Ndh/mYB/mYANTVCjmfrGYE5MwJQ7AlDIIU2omIjIFgmNBU1H4nSGE2CClPH6D1bH9TlUxCCF0YA8wBX/953XAjW0rsQkh7geypZT3CiFmAjOklDcIIYYB7wBjgETgU2CwlPKEQdffGsUgJYbhxedx4fO42LV7H7XrFuCoaaSlzom72Ye3RcNw60ivFSktCGkBYcGnmfGazPh0Mz6hAQZIHyDbHGAyaVhsZizBNqxBdmwhwdiCgwgKDyc0JorQqAiswRYsVhNmm47JrKGbNXSThhAgDb9t3+sxcDu9uFt8tDS6aap10ljtpK6ihcqDjTha8whZ7CZShkWRmh1Dek5swMInW7Zuo/LlXRhNddgH1xL3o4cQXZTkPExNaTGvPnQvF8+6l1GXTe/RdffX7+fWxbcSagnlpaLfUF7rYXW5k3PvGcaYUd2vr3AYr8fD87NvIvP8C5ly9wPdHteSl0fxD36Ir7aW0KufwpISR+xdJ179GIZk/aJC1i8qJCjUzKRbhpI64vQ6hT2VzTjWlONYfwjp9KKFW9DS7XgTwBPixu1y4na24KmtxVNdhbu6Gl9NFUZNJb66WoxGBzS7wCfRJGjSQDMkupSYdB8Ws4HFrmEJMmMNsWMJD8MaFYE5OhY9KhY9uh96dDxaTCJ6TAIiJBpMpzeE+kyiu4ohEKakMcA+KWVB64XnA1cDbUt0Xg38uvX9e8Bzwm9juBqYL6V0AfuFEPta51sVALkChjQMpGFgGD4Mwwc+A6P13NFDIqX/FSmRkjav/kPC4fsxQgK0NbOINu9E+6ZWDCd4NkdjIQLL4YksEsMskRgY0ochffikB59swWO48BhuPD4nLl8zTl8zBu11rhdwNgKVHX93ITRCoqKJiE8gIj6BmOQU4gcOJjY1HbOl+/bz5gY3Zfl1HNhazYFt1ezbUIHFbmLIuHiyLkgkOjGk23N1hCV1MJqtFlOog5pXXsG5ZQuJf3kKc9wJsom2sm/dagAyRo/p8XXTwtOYe8lcZn8ym+3OXSQ4UqgL1U5KKQCYzGbSRp5D/oY1TDbu94fxngApJbVvz+PQn/6EOT6e/i+9Re0HddiGRnd5LU0TjJmeRuqIaD57fScL525hYG4c465OJzyAJT2llDRUHqJs3x4qCguoLS2hwVFBaEMoCY504uvS0IUJabiodxZT6TxIjaucOncFbsN5dCJdh4gw6K4LywNUAVX1CFmHbuxGNyS6IdEMA13K1sPAJCS6kJg0gUkT6LrApLc+CJnMmCwmdLMZs8WCbrGiW62YrFZ0mxXdake3WtFtdky2IHSbHf3Iqx1hC0K3B6NbrWjWYDR7EJrFhjBbuvz59haBUAxJQFGbz8XA2M76tNaIrgeiW8+vPmZsYLfCtqHxYCUm0X65LNr82/bs8aZx0a6XhsBvOevGE6/o4BIngVmzMDR89JG5xBHpuz+5sOlo4Rb0CCt6pMVfsyDWDJEmvD4PHmcL7pZmWhobcNTX0VxfR0PFIWoPlbF37Sq2fua39Gm6TlxaBmk5uaSfcy790jJO+EseFGYhY1QcGaPikIakLL+ebStK2P5lCVs/LyZjVCxjr04nMr7n+Y0AnHtqAYi591qCz42k7FePUXjDTFJe/ifW9BPvOs7fsJa41AzCYrpWIh2RHZvNkxOfpPzlPWQIQeaFJ5mXqJWMc8exZ81Kygv2kjBwSKf9pM/Hod//ntp57xBy0UUkPvFHWnY4gDpsQ7sf5RM3IIzrf3Eu65cUkvfJQQo2VjLsgkRyp6USHNF95d+W5oZ6CvM2ULBpPQe35tHS6N9foukmIvrFE5GQSPjwBLTIcJqCDIKbLViqbSQdCiaxLuPIPFqYGT3ahh5hQYuwoIWYwK6BTcOwSAxh4BUeDK8Hr9uN1+3G43LhdfsPT3Mz7oZ63LXVuOtr8Dga8TY7cDud+NxuPF4vXp8PtyHxGhIffpu2gcCHBh7QPBrCKdDwIYQbTXjRaEEIHU1oaGgIoSHQ0I55FUJDQxxpP/pZIKSGEML/mcPth9+39kEAGprw30QEgv6XpzH0mu+e1M+luwRCMXR0VzrWPtVZn+6M9U8gxN3A3QApKT3PuOmf2MAnO0r4JY+/aNvH+3aCdSCeaP/q/xn6f7AIgdAEQtMQmo5m0tF0/yE0cbRfu3nEcXMLITA12kj50/FRM1K2Lkd8BtIrkR4D6fFhuHxIpxejxYfP4cZo8uBrdOOrdeGtceLObwCvceQ65sQQrOnhhKUlYR01DM16/K9HY00V5fl7Kd+3h6JtW1j1/jusem8eIVHRZF04meEXTelwH0C7/y5NkDgogsRBEbQ0DmLLF8Vs/rSIgs1VZJ6XwLir07H3cB+Cc3cNWqgZc0IwlquuwjpoEAfvupsDN91M/5dexD6i49LizfV1lO7ZyfhrZ/boescyqf8kPmxyYzUJtoXOZ4b8v5N2vKeNykVoGvnr13SqGKTbTenDD9OwaDFRd84i7sc/Rmgazt3F6FE2TDE92zmvmzXGXpnO8IlJrFtYyPYvS9nxZSlpOTFkTUwieXDkkR3dneF1u9m3bhXbvviUA1vzQEqCwiNIG5VL4uChxGcMJiZlALrpxL4MX6MbT5kDT1kTnjIH3mon7j0NGA5Pp2OEWcNk1jCbNOwmK0KzgeZfeUsJ0pAYVgOfyYcRamD4Wlf5hgQDhJQIKRDSv5r3354PPwCeWVR4us74e6oEQjEUA/3bfE4GSjvpUyyEMOEPjqjp5lgApJQvAi+C38dwMoKGpfTNouZ+BQRoOsIMdPOeIA2Jt8bp/wMsdeAqbKBpVSlNX5aASWAbHIV9eDT2YdFHKqGFRsUQGhXDoHPHA0efDHd9vYK1//03az74Fykjchj3nRvoP6zjm3Fb7KEWxl6ZzogLk9mwuJBtK0rYv7mSC28cQsY53XuClz6Jc08d9qzoIzcvW2YmqW+/xcFZd3LwtttJnvscwePHHze2YOM6kJKM3HHd+0/rhC9WFqO5zWCC5bs/Iz4inruzT24/jT0klOShWeSvX8OEmbce1240N1P84PdxrFxJ3E9/QvSddwIgPT5c+XUE5fY7aaUUHG5l0k1DGDWlP9tWlLLr6zLyN1YSEmllwIgYUodHkzQksp1vqKWxgQ0L/8vmTxbhdDQRFhvHuO/cwMDcccSlpvfYXKKHWtBDLdgGt1/1uBrcNB9y4KxqwV3rwtvgwt3kweVowdPixud2IVsM8ErwCYQh0Iyj1xa09a6197Z5MfAJHz5hYGg+fEJiaAZSk/5Dl0gdpAZCB0wCYQJ0gdA1hFmg6xrCrKHpGprp8KGjmzR0s/+h0GRqfdVN6GYd3dT20NB1/3tN09BNJn9/XWAymdE0DZNJJ+kbiMoKhGJYBwwSQqQBJcBM4KZj+iwAbsPvO7gOWCallEKIBcA8IcRf8DufBwFrAyCTohsITWCOsWOOscMIf61g6TFwHWzAuaOalq1VOHdUU2fRCBoVR8j4RMzHmHqCwsIZNvFihk28mMbqKrYv/4y8Txby7m9+QXLmcM777k3dCwENs3DBDYMZNiGRz17fyZIXtzEoN46JNw7pMlrGXdSAdHqPM59YBgxgwLx5FM2eTdG999H/H/8geFx7K+e+9WsIjY4lLvXE5qau+HrxflJbAzmuiruCZzc9S6w9lhmDZpzUfBm54/jijZeoO1TebgVmOJ0U3T+H5rVrSfj974i49mjeJldhA9JjYBsSdUrfBSA8Nojzrx3I2KvSyN9YSf7GCnavKWf7ihKEJoiMDyIqXqOxahUlO5fj9bhJHzWO7ClTSRuZ02klOiklPq+Bx+nD7fThavbgavbiavbidHhwNvmPFocbZ6OHliYPzY0uWprcGJ0sGAx8uEwtOE0OXGYHblMLMsiHbgOzXcMSZMJutxIUbCU42E5ocBChISGEhYQQERpKRFAYIdYQ7CZ7QMOrz2ZOWTG0+gweAD7Gb3B/RUq5XQjxOLBeSrkAeBl4s9W5XINfedDa7138jmovMKeriCTF6UWYNWwZEdgyIgi/Ih13USOONWU4NhzCsaYc68AIwi4d0OEmstDoGMZ95wZGT7+GrZ99zNoP3+Pdx3/JkPEXMOm2uwiJ7PqGFZ0UwrU/H83GJQdYv7CQigONXH5fNlGJnfsenLtqQRPYBh1vVzf3iyPljdc5eOutFN1/Pyn//CdB5/jNcR63iwNbNjH8osmndEPYU1BLaJWbltQgqPMya8Ct7AjK5zerfkO0PZqJySdOidIRGblj+eKNl8hfv4bRV1wNgOF2U/z979O8Zg2Jf3qC8KuuajfGfaABBFhTTzHFdhtMZp0hY+MZMjYen8egZG8tpXtr2bduBds/X4w0WtDMQ7CEjqN0fzSlL9ajaSv+f3v3HR5llTZw+HemZdI76SEECL2HZgNFxI4ioLgiVnbRde0NXSvuomtZP1dRRMquDbtY6IKgiBKagIHQQnohvU0/3x8zgQRSSZkknPu6uKa97ztPQpJnTnsOeqPW2UeuAYTAYXNgtzmw22Sjm/sIvcTuYaZKX0GpppBSXRFVIeWY9OVU6cvx8NYR4O9DYIAf3YKCCAvsRrh3GN28ehPqFUqQMQiN6JiDup1Fq6xjaG9nzXTVU7jza7RXWKnYlkP55kwcFVaM/YLwnxR3WguiJpvFwrYVn/PrV5+g1ek5b8bNDJ14eZO7FrIPFbNy4V5sZjsX39qf+KGhdR6X+/oONJ46QmfX3zKx5edz7KaZ2AoKiF26FM+BAzi8/Ve+eul5rpv73IlyFGfi1Re3ojtaweX3D8G4JBn/K+PRjA7g1lW3klqayqJLFjE4tPFW06mWPXQ3nr5+TH/6n0ibjcz776ds7TrCn3uWwOnTTzs+f/Fe7CVmwu+ve+1CayjISGftu/8hc/8+ovr05/wbZ2PwCqOs0ERlidnZCqiyYTXbXTP1nK0ErVaDVudaCGnUojEICm35ZFrTOGo6zMHK/Rw2p1CpLcOhseOp86Snf0/iA+KJ948nzi+OWL9YYnxjMOqatuWqcrr2nK6q1FBcXMwdd9zB3r17EUKwePFixtbo2y4uLubDDz/krrvuApzVVa+88kr27t3rrpCbROutx298DD5jIyn/OZOyTRnk/r6woagAACAASURBVN8OfM6Lxu/iWDR1FIrTGQyMnTqDvueNY/17C/hh8dsc3bGNS+9+AC+/xusZRfQKYPrjiax8ew8r397DOVN6MeyS2hMPbCVmrNkVje5nrAsNJXbpEo7dNJP0O+6g+0cfcmDLZjy8vIkZ0PhYSH0KCqvQpFZQFuZBz4RgMnUCe6kZX703b138FjO/n8mcdXNYPGkxfYLqn2FUl16jxvLrF59QVnCcspdfpWztOsLmPl5nUpAOiSWtFK9BdSfPlpJSsnfjWn5Y/A46g4FL/vI3Bo67+ESSD21ktbnJZmJX/i6ScpLYmbeTPdl7qLI5a4IFGYMYGDmQm4JvoG9gXxKCEojyiVKf+t1IJYZWdu+993LppZfy2WefYbFYqKysvbNV9UY91Ymhs9F4aPG7KBbv0RGUrDxK+aYMqvbkE3ht79MGC6sFhkdy3dzn2L12JRv/+y7/e+QeLr/noSaNPfgEGrn2weGsX5bMli8OYa60Mnpy/ImuH9MB58YwTZmeqY+IIHbJYlJvmMGR2bNJCfNh4PiJjc6Sacjny5PRS8G4q50x1dywJ8QzhEWTFjFr5Sxmr53NkkuXEO/f9LGM/udfyNbPP2bb/HlEfLOakLvvJujm0wejAWz5lUiTHUP35pcDaYy5spJ1i95k/88/EjtwCJff8xDeAQ1/vx3Swf7C/fyc+TNbsrawO383VocVrdDSJ6gPU3pPYWi3oQwJGUK4d7jq2+9gVGJoRaWlpWzatImlS5cCYDAYMJxS76arbNSj9dYTNDUB7+FhFH15kOOL9+JzXhT+l8bVuaWmEIKhl1xOZEJfvv33i3z6/JOMn3UHwy+7uo6r16YzaJl4+wAMnjq2rzqGpcrG+dcnIDQC0/4itAEe6Lo1bUGWITaWmHfeZtM9c7BbPeg/pvllK6qZTTbK9hRT5qvhvBHOqrpaPwP2kpPlHKJ8onj3kne5ZdUt3LnmTpZduoxo3+gmXT8wIoqw4G6kpPxBn+umEPLX+utqWdLKXF9f640vABTn5vDli89SlJXJudfPZNQ1U9HUMyum0lrJL9m/sDF9I5szNlNgKgCgX1A/bux7I6MiRjG823B8DC1bzKi0vS6ZGIq/OYwlq6JVr2mI9Cbgqp4NHnPkyBFCQ0O59dZb2b17NyNGjOD111/H2/tkP3yX2ajHxSPen7C/Daf4+yOU/5SJ+WgJQTP6Omc61aFbXDw3zf83K//zChuWLqQoO4sLZ91Z7yyWahqNYPyf+uDhqWPn2jRsVgfjpvTElFKEdzOnZ3oOGkTeoH74pB3D+u//IN9e0OTyGTV9/VUKRgd0n3DyD73Wz4A1s/a+zT38e7Bw4kJuW30bt66+lXcnvkucf1yj1y/bsIFuu/eRGx0KN97Q4NdoPlaK8NQ1e/1CQzIPJPP1y/OQdjvT/j6vzhZeqaWUH9N/ZM2xNfyS9Qtmuxlfgy/nRZ7H+dHnMzZyLCGeau+FzkZ14rUim83Gjh07mDNnDjt37sTb25v58+c3el6n2KinAUKvIXByL4Jn9sNWaCLvjZ1UJRfUe7zB6MlVDzzOiCuvZdfqb/nqX89jMTW+B4UQgrFTepJ4RRzJW7LZ894+sDnwHt28lcYFGWnk5WXTf+Q5VP70EznPz6O5kzAcDknallyOG2DyxSfHN6q7kk69Xp+gPrw36T0sdguzVs3iQOGBBq9ftW8fmQ88SI/waPQeHuzdvKHB4y1pZXjE+ja6CK2p9m/ZxKfPz8XDy4sZ816plRQqrZV8d+Q7/rr+r4xbPo65P80luSCZqQlTWXTJIn68/kdeGvcSV/W8SiWFTqpLthga+2TfVqKjo4mOjmb0aOdc+alTpzJ//nzefPPNzrdRzxnwHBBCWJQPBf9LpuC/f+A3KQ7fcdF1ftLVaLSMn3k7geGRrF+8gM/mPcmUx57F6NNwN4MQzvo+dosdz63ZmPwN6MObV9dn78Z1aLRaRtxzH+WevhS8uwhDbCzBt9/W5Gv8sOoonhaJ7/mh6HUnWztaP4Nz5bnJjvCs/evVN6gvSy5dwuw1s7l19a0suHgBQ0KHnHZta3Y2GX+ZgzYwgLgFC0j4/EMO/LKZi26Zjd54+owcR5UNW14lXkNaZ+B599qVrHvvLaL69GPyQ0/i6euHzWFja/ZWvjn8DRvSN1Blq6KbVzdu7Hsjk+ImMTBkoBos7kLU/2QrCg8PJyYmhgMHnJ8G169fT//+/bn77rvZtWsXu3btok+fPh1vo55WpAswEvrnwXgOCqF0VSpFyw8gbY56jx8y8TKuuu8xco8c5pPnHqeypLjR9xBCMLRfEN5awZ7cKn779miT47PbbPyx6Qfih4/Eyz+A0Pvvx/eyS8n7178oXb2middwsHdNGnk6BzdMqT3TSFu9k1upuc5z4/3jWXbZMgI8Arh99e18c/ib2tcuLyf9z3/BUVVFzNtvo+/WjYHjL8ZqqiLl15/rvKYlvXp8oeUDz9tWfM66RW8SPyyR6554ngxbLi9ve5mJn01kzro5/JT5E1fGX8mSSUtYO3UtD498mMGhg1VS6GLU/2Yre+ONN/jTn/7E4MGD2bVrF3Pnzq31es2Neh5++GE3Rdm2NAYtQTP64jexO5W78slftAdHZf11bnqPPodrH32KouwsPn76UcoKGq8FU7E1G42vAf+RYSR9l8ruH9IbPQcgdfd2KkuKGTDeuSWn0GiI/Oc/8Rw6lKxHHqFyx85Gr/HL2mPoTQ48hwfj71l7csGJndxKLXWdCjgHpN+//H0Ghw5m7k9zeSXpFewOO9JqJfPe+zAfPkzU6//GmJDgPL7vAALCI9i7cW2d1zO7FrYZYs48MUgp+fmT99n0wRLiR4+h6oqezFwziykrpvBB8gcMChnEa+NfY8P0DTw19ikSwxNVMujCumRXkjsNHTqUUxffnerDDz+s9bjmGoaHHnqoTeJqb0II/CbEogs2UvhpCnkLdhNyywB0wXUPjsYNGc7UJ57ni/nPsPzZx5j+1D/qrXZqPV6FOaUIv4tjGX9hDOZKOz99chCjt54+o+sv4CelJOnbL/EOCKTH0JOLwDRGI9FvvUnqjBlkzJlD948+wiO+7nURVrOdnauOkamzc9fkhNNe1/q7Wgwl9ScGcM7df2fiO7z020ss3beUP47v4/F1vlh//pmIF+bhc+7J2VJCCAZeeAk/fbSMnMMHCe/Zu9a1LGml6MO8T9Szai4pJZs+XELSii+o7OfPvJAvsGy30iewD4+OfJTL4y8nyNjyMhtK56FSvtKmvIZ2I/SOQdjLreS9tRtzWmm9x0b17c/UJ5/HVFbG8mcepyQvt87jKn7JAq3Ae3QEGq2Gibf3J6pPAD8sSyZ1T/2tjdTdO8j4Yy+jp1yP9pRZSLqgIGLffRe0WtLvvBNbft0bVCStPYbG7MA2wJ/Y4NNXfTfWlVSTXqPniTFP8Nw5z5Hw2Xas364h+4Zx+E05vb7S0EuuwOjrx+aPltV63rmwreyM1y+kFKbw6it3k7TiC/bHlrG6zzGm9pnGJ1d+wmdXf8ZN/W9SSeEspBKD0uY8evjT7a4hCA8t+Qv3ULmnnl2BgIhefZj65DzMleUsf/YxinNzar1uKzRRsS0Xz4EhaF2luXV6LZf/ZTDB0T6sWriXrIOnj1NIh4PNHy7FPyycwRMm1fne1WscbIWFpP/5L9jLa087NVVY2bEmjUM6OzddeXprAUDotWi8dA12JZ1q/HYzV/9kYc/YcO6N+4lZK2exNXtrrZlNHl5ejLl2Oml7dnHs910nvx95lUizvVnrFzLKMli0ZxFTvr6W+S/eAdvSKB3gx/S7n+SH6T/w+OjH6RfctcvLKA3rUomhM9Z9aqrO/rXpQ73odtcQDFE+FH6wn9KN6fV+TeE9ezPtyRewmkwsf/oRCjKd4wfS7qDw4/0gwH9SXK1zDJ46rrpnCH7BRr57czf5abUH+Pdv2UT+saOcO/2mBlc6ew4aRNRrr2JKSSH9ztk4Kk6uh9m0PAWHxUFhvCdDY+rfRkzr74G9uPEWA0DxF1+S8+xz+Iwbx9SFa3ju3OfJKs/izjV3MmvVLDZlbMLmcO4hMmTi5fiGhLL5o6UnvncnBp4bGF9wSAfJBcm8testpn0zjcu+uIzXt79OnyTBgFQ/+k28hGf+/gET4yZi0J4921wq9esyicFoNFJQUNDp/4DWRUpJQUEBxjqmKnYmWh8DoXcMwnNIKKWrUil8PxlHVV0bJ0FYfC+mP/1PHA4Hy595jLzUI5SuS8OSVkbglN7ogk7/Xnj6Grjqb0MxeOlY8X+7KMpx/lG326z8/Mn7hHbvQd9zGq906jt+PFEvv0zV77+T/pc5OKqqOLo7n4O/5fKLh5UZl/Rq+Ov098Be0nhiKPnmG7KfeALvsWOJ+r/X0ej1XNv7Wr6/7nvmjp5LVnkWd6+/mws/uZBntjzDlrytDL5mMrlHDpGy1TlDyZJVjjBoai1sK7OUsSN3Bx8mf8gDGx9g3PJxTP92Om/vfhsvnRcPDLufp4umE5JiYdQ107js9ntUSQqlli5TXdVqtZKRkYHJZKrnrM7NaDQSHR2NXn/mdX06Cikl5ZszKVmVijbAg+AZfev9xFuYlcmn857A3x7EeYHX4pUYRtDUurtxqhXnVvLFy9sRGsHk+4ZxdMc6Nv73XaY89gw9hjVaWPKEkm+/I+uRR9CPOZ/NwX8ix2pjbRSsfXA82gYWkhV9eZCqPceJfOr0jYGqlX7/PZkPPYxXYiIx77yNpo4SKBa7hc2Zm1mTuoaN6RuptFUiJFz7czQe6MmaGsXMXZPQIFg8YhXFpmLyq/LJrTw5NhPpHcnI8JGMihjFuZHn4qf1YeUbr3Dwty2ce/1Mxky5vsnfD6XzO+uqq+r1enr0aLjCptIxCCHwvSAaQ5wfhR/uJ2/BbrxHhuF7YQy6gNotgcDwSK6b/gQlK49Sai2kOMBMEA0nhoAwLybfN4yvX9/Fpy98TmXh58QNGU7c0OaVo/a/8gqw21i7bD9VehPrDUXcccGYBpMCgDbAA0elDYfFflrVWSklhe+9R94rr+I5YjgxC96qMykAGLQGJsROYELsBEw2E7/n/05KUQpHPJIwfHUI/7VZBBt82RLyO0WmIgKMAfTw70F8QDwJgQkkBCYQ7n1yllZZ4XGW/+tRco8eZvzNd57Y50FRTtVlEoPS+XjE+hH2t2GUrE6lIimXiqRcvIZ1wxDtg8ZLD1JStjEDa3YFXpFBJBev4cC7WzielcoFN91abzE3cG74c/40f1a88hUabRCjp9x1Rt0laf4jyO3mS0zaKp7N2cTAOxc1es6JtQwlZjShJ1dlS4uF7GefpeTzL/C7/DIi/vEPNE3sHjTqjIyKGMWoiFHQ/yZ2BKxg2/ufYowxMOW8G7h51P0Nnp998ABfvzwPi8nE5IeepFfi6AaPV85uLUoMQoggYDkQB6QC06WURaccMxRYAPgBduAFKeVy12tLgXFAievwW6SUu1DOGhovPYHX9sb3wljKNqZTkZRDZdLJrhBtkJGgGX3wHBTKZY6heP3vPbZ/9xUZyXsZN/P2eveVLsrOZO3C+fgE+mPwu4Hv3tzPuBslfcc2va7Sno0ZbPo4hZA+AbxGb14t+oXsWTdjf+ABAmfcUG/hPW1AdWKwoHclBlNKCjlPP0PVzp2E3DWHkL/+tdl7Idc0/LKr4agZ0iA5eTOJI6fVmfhM5eVs/eIjdq76Dt+QEKY+OY+QmO5n/L7K2aFFYwxCiJeAQinlfCHEY0CglPLRU45JAKSU8qAQIhLYDvSTUha7EsO3UsrPmvO+dY0xKF2DtDmc3TCVVhxmO4Yon9PKeO/fsokf319MecFxeo0cw6AJkwiMiMI/NIycwwfZveY7DvyyGb2nFzOeewkP71DWvrePzJRi+p4TwQU3JKCvY2Ohmn7fkMHm5Sn0GBLCukA7q/7IZfPtg6h49u9UbPkFj/79iHjqKTxdxQ9rsh2vIuflJAKnJWDs48PxN9+k8L//RevjQ9hTf8f/iita5XtVvPIoZT+m89nRVwiOjWHIJVfQ7/zx2K1WirIzydz/B7999SmmygoGjp/IBX+6BU/f1i3LrXQuTR1jaGliOACMl1JmCyEigI1Syga3qRJC7AamuhLFUlRiUM6A1WJmx3df8+tXn2J1VWYVGg3S4cDg6Un/Cy5ixOXXEBDubCE47A62fZdK0spU/EI8GTohhj5jwjGcslo4P72MHauPcSgpj/ihoQyaGs/4V39k5tjuPH3VAKSUlK1aRe4/52PLy8Nz2DB8L56A74QJ6MLCnO9VaSbnX3sQ8gDlGxfhKCkhYNpUQh94AF1g4xsKNVX+4r3YS83kDy5g5+pvyU89cuJ7UK374GGMu+k2Qrur8Tel/RJDsZQyoMbjIillvT/5QohRwDJggJTS4UoMYwEzsB54TErZ6Dw/lRiUaubKSo6nH6MoO5Oi7Ez8QkLpd954DJ51V1xNTy5k61eHyTtWhsFTR/zQEPRGHRqNoCingrR9heiNWgaPj2bklT2YtzKZ//5yjB8fHk904Mlr2ssrKHr/fUrXrMb8R/Jp7+N92SvYj+/FI7KQwBk31NmyaAkpJdkv/IqxTxBB0xKcjw/u59C2rXgHBBIYEUVgZBQBYRFqKqpyQqslBiHEOqCuAjRPAMuamhiqWxTALCnl1hrP5QAGYCFwWEr5XD3nzwZmA8TGxo44duxYw1+ZotRDSknOkVJ+/yGdzJQiHHaJwyExGHUMHBfFoHFReHjpKSg3c96LG7hsUDivTq//D7slI5OKzZtOLoYTAkteArqwAEJvO/P9pBtiL7WQ/Y9f8b8qHt9zo9rkPZSup9Wmq0opL27gTXKFEBE1upLy6jnOD/gOeLI6Kbiune26axZCLAHqrSAnpVyIM3mQmJjY+RZfKB2GEIKInv5E9PRv8LilW1Ix2ezcNb7h/T0M0VEYZsyo9dzxZfuwF7XdmhpLlrNchyFCbZOptL6WrnxeAcxy3Z8FfH3qAUIIA/Al8F8p5aenvBbhuhXANcDeU89XFHcoM1lZuiWVSf3D6dWt+QXqtP4e2BqpsNoSVldi0EeeXshPUVqqpYlhPjBRCHEQmOh6jBAiUQhRPeF7OnABcIsQYpfrX3W7/AMhxB5gDxACzGthPIrSKt7fmkaZycZdF57ZboDaAA9klXORW1uwZpWjDTaecaltRWlIi36qpJQFwIQ6nk8C7nDdfx94v57zL2rJ+ytKWzBZ7bz301HO7x3C4Oj6i+U1ROvvWstQbEbTrXlbjzaFJasCQ5TqRlLaRpcpoqcorWX5tnSOl5uZ08jYQkN0JzbsaVqV1eZwmGzYC02qG0lpMyoxKEoNZpudBRsPMzIukLHxwWd8nRMthjZIDNYs5+wnfaRqMShtQyUGRanhk23p5JSauHdCQovm/9fsSmptJ2YkqcSgtBGVGBTFxWyz89bGw4zoHsi5vc68tQAgdBo0Pvpm7eTWVNacCjQ++hM72ClKa1OJQVFcPk3KILvExL0TerfKamGtvwe2NmgxWLMr0Ier8QWl7ajEoCiAxeZgwcbDDIsN4PzeIa1yzabu5NYc0i6x5laij1CJQWk7KjEoCvDp9nQyi6tarbUAoPU3tHpisBVUgc2hEoPSplRiUM56JqudN9YfYnhsAOMSQlvturoAD6TJjsNc977WZ8Ka7VrxrLqSlDakEoNy1nt/6zFySk08PKlvq1YibYuZSdbsStAI9G2waE5RqqnEoJzVykxW3txwiPN7hzC2Z8tmIp3q5FqG1puZZM0uR9/N87TNixSlNamfLuWstvinVIoqrTx0SYP7S52RtljkZs2pQK8qqiptTCUG5axVVGFh0eYjTBoQxpCYM6uJ1BCtnwEErTZl1VFpde4jrcYXlDamEoNy1npr4yHKLTYebIPWAlQvcjO02hiDJdtVCkPNSFLamEoMylkpvbCSZVuOMXV4NAlhzd9voal0wUbnFNNWYFWJQWknKjEoZ6UXV+1HqxFt1lqopgvxbNXEoEphKO1BJQblrLMjrYhvf8/mzgviCfc3tul76YI9cZRZW2Utg3PgWbUWlLbXosQghAgSQqwVQhx03QbWc5y9xu5tK2o830MI8avr/OWubUAVpc1IKXnhu2RCfDz48wXxbf5+uhBn4rEVtGz/Z2cpDFUjSWkfLW0xPAasl1L2Bta7HtelSko51PXv6hrPvwi85jq/CLi9hfEoSoNW7s1h+7EiHrwkAW+Ptt8WUxfsCdDi7iRnKQypWgxKu2hpYpgMLHPdXwZc09QThXOJ6UXAZ2dyvqI0V5XFzgvfJdMnzJfpiTHt8p4nEsPxlrUYVCkMpT21NDGESSmzAVy33eo5ziiESBJCbBVCVP/xDwaKpZTVna8ZQFQL41GUei348TCZxVU8O3kAWk3rlb5oiMZDi8ZX3+IWgzWrArSqFIbSPhptSwsh1gHhdbz0RDPeJ1ZKmSWEiAd+EELsAUrrOE42EMdsYDZAbGxsM95aUeBYQQVv/3iYq4dEMqYFW3aeCV2wJ7bjLUsMlsxy9OHeqhSG0i4aTQxSyovre00IkSuEiJBSZgshIoC8eq6R5bo9IoTYCAwDPgcChBA6V6shGshqII6FwEKAxMTEehOIotTl+W//QK8RPHFFv3Z/b12wJ6aUojM+X0qJJbMcr0Gts0+EojSmpR8/VgCzXPdnAV+feoAQIlAI4eG6HwKcC/whpZTABmBqQ+crSkv9sD+Xdcl5/G1Cb8L82nZ6al10IUYcZRYcZvsZnW8vMiOrbOijVI0kpX20NDHMByYKIQ4CE12PEUIkCiEWuY7pByQJIXbjTATzpZR/uF57FHhACHEI55jDey2MR1FqqTDbeOrrffQM9ebWc3u4JYaWzkyyZJYBYFCJQWknLZqvJ6UsACbU8XwScIfr/hZgUD3nHwFGtSQGRWnIq2tTyCiq4pM/j8Xgpv75monBENn8P+7WzHLnwLOakaS0EzWSpXRZu9KLWfLzUW4aE8uoHkFui6Oli9wsmeXow7zUwLPSbtRPmtIlWWwOHvv8d7r5Gnn00r5ujUXjoUPjoz+jmUlSSqyZ5Rii2q7Qn6Kcqu2XfiqKG7zz42H255Tx7s2J+Br17g7HOWX1DMYY7EVmHJU29FGqG0lpP6rFoHQ5ezNLeH39Qa4cHMHE/mHuDgeorrLa/K4kS6ZzxbNqMSjtSSUGpUsxWe3ct3wXwT4G5l0z0N3hnKALNuIoteCwNG/KqjWrHDRq4FlpXyoxKF3Ki6v2cyivnJenDSHAq+MU69WFVM9Mal6r4cTAs179qirtR/20KV3G5oP5LPk5lVvOieP83qHuDqeWk8X0mj7O4Bx4LlML25R2pxKD0iXkl5l58JPd9Az1dvsspLrogqunrDY9MdhLzDgqbGphm9Lu1KwkpdOzOyT3fryTkiory24bhadB6+6QTqMxNn/KqjXDVWpbJQalnanEoHR6/16XwpbDBbw0dTD9IvzcHU699BHeJ/7YN4Uloxw0YFCb8yjtTHUlKZ3axgN5vPHDIaaNiG63zXfOlEd3P6y5FTiqmrb/s/lwMYZoX4S+47WAlK5NJQal00o9XsF9y3fRN9yX5yZ3nKmp9THE+YMES1pdW5HU5qi0Yskow6NXQDtEpii1qcSgdEollVZuW7YNAbwzc0SHHFc4lSHWFzRgTm08MZiPlIAEY+/AdohMUWpTYwxKp2O1O7j7wx2kF1bywR1j6B7cOfrgNQYt+kifJiUG06FihEHrTCaK0s5Ui0HpVKSUPPvNPn46dJx/XDvIrVVTz4RHdz8s6WVIm6PB48wHi/CI90do1a+o0v7UT53SqbzxwyHe35rGn8fFM62DDzbXxRDnDzYHlqz6ZyfZCk3YCkx49FbjC4p7tCgxCCGChBBrhRAHXbendYgKIS4UQuyq8c8khLjG9dpSIcTRGq8NbUk8Ste2bEsqr65N4brh0Tw6qeMtYmsKjzjndFpLA91JpkPO/aHV+ILiLi1tMTwGrJdS9gbWux7XIqXcIKUcKqUcClwEVAJrahzycPXrUspdLYxH6aK+2pnJ0yv2MbF/GC9eNwiNRrg7pDOi9TWgDTY2OM5gPliM1s+ALtSzHSNTlJNamhgmA8tc95cB1zRy/FRgpZSysoXvq5xFVuzO4sFPdzM2Ppg3ZgxD18n73T26+2E5VoKU8rTXpENiPlyMR68AhOicyU/p/Fr6GxYmpcwGcN12a+T4G4CPTnnuBSHE70KI14QQHvWdKISYLYRIEkIk5efntyxqpdP4bHsG9328kxGxgbw7KxFjF1js5RHnj6PCVmd5DGtWOY5Km+pGUtyq0cQghFgnhNhbx7/JzXkjIUQEMAhYXePpx4G+wEggCHi0vvOllAullIlSysTQ0I5VOVNpGx//lsbDn+1mbM9glt42Eh+PrjG72tDAOIPpoHN8QS1sU9yp0d80KeXF9b0mhMgVQkRIKbNdf/jzGrjUdOBLKaW1xrWzXXfNQoglwENNjFvpwqSUvLnhEC+vSWFcQijvzBzRJVoK1XShnmi8dJgPF+M9MvzE8/YKK+U/ZWGI80Pr23H2klDOPi3tSloBzHLdnwV83cCxMzilG8mVTBDOztRrgL0tjEfp5Kx2B49/sYeX16RwzdBIFt7ctZICgBACz8GhVO7Op2p/4YnnS749gqPKRuA1vdwYnaK0PDHMByYKIQ4CE12PEUIkCiEWVR8khIgDYoAfTzn/AyHEHmAPEALMa2E8SidWUmXl9mVJfLwtnXsu6sVr1w/FQ9e1kkI1/8t7oI/wpvCj/VjzKzEdKKRyZx6+46PVNp6K24m6ZkZ0dImJiTIpKcndYSit6I+sUuZ8sJ3MoipeuHYg14+MdXdIbc5WZCLvPzvReOuRFgfCoCHsb8MRus4960rpuIQQ26WUiY0dp34CFbf7rMoE7QAACZ5JREFUfHsGUxb8jMlqZ/mfx5wVSQFAF2gk6MZ+2I5XYS8xE3hdgkoKSofQNaZ5KJ1SSZWVZ1bs48udmYyJD+KNGcMJ9a13xnKXZOwZQNCMfsgqGx7dO+4mQ8rZRSUGxS22HDrOQ5/uJrfMzL0TenPPRb06/cK1M+U1KMTdIShKLSoxKO2quNLCi6sO8NFvacSHePP5nHMYGqPm7CtKR6ISg9IupJR8sSOTf3yfTHGVlTvO68GDl/TpFBvsKMrZRiUGpc39eqSAf67cz670YobFBvC/awbRP1L1pytKR6USg9Jm/sgq5ZU1B1i/P49wPyMvTR3M1OHRnbYyqqKcLVRiUFrd9mNFvLnhED/sz8PXqOORS/tw27k9utwKZkXpqlRiUFqF1e5g1d4clm1JJelYEYFeeh6cmMDN58Th76l3d3iKojSDSgxKi6Qer+DzHRks35ZOXpmZ2CAv/n5lf24YGYN3F6mGqihnG/WbqzRbfpmZ1fty+GpnJknHitAIOL93KPOv6874hG5qDEFROjmVGJRGSSk5cryCjQfyWb0vh22phUgJPUO9efTSvlw7LIpwf6O7w1QUpZWoxKDUKb/MzK9HC/jlcAGbDuaTXujcbSwhzIe/XdSbywdFkBDmo7afVJQuSCUGBavdwcHccnalF7MjrYgdaUUcya8AwNugZUx8MLMv6Mn4hFBigrzcHK2iKG1NJYaziJSSnFITB3PLSckt42BuOfuyS0jJKcdidwAQ5G1gWEwA00bEMLZnMAMj/c7aGkaKcrZqUWIQQkwDngH6AaOklHVukiCEuBR4HdACi6SU1Rv69AA+xrnf8w5gppTS0pKYzmZSSkpNNnJKTGSVVJFRWElGURVphZUcPV7BsYJKqqz2E8cHexvoF+HHrefG0T/Sj6ExAcQGeanuIUU5y7W0xbAXmAK8U98BQggt8CbOHd4ygG1CiBVSyj+AF4HXpJQfCyHeBm4HFrQwpi5DSkmV1U5JlZWSKitFFVaKKy0UVlooLLdQUGHheLmZvDIz+WVm8kpNVFjsta5h0GqIDvQkLsSbc3qG0CPEi17dfEkI8yHY5+wqca0oStO0KDFIKZOBxj5hjgIOSSmPuI79GJgshEgGLgJudB23DGfro0MmBodDYnNIbA6H89YusdkdWB0Sq82BzeHAbHNgtUssNgcWmwOzze66dWCy2p3/XPerrHaqLHYqLc7bCouNCrONcrOdcrOVcpONMpMNm6P+HfZ8jTpCfDwI9fVgQKQf4/uEEunvSbi/kcgAI9GBXoT6eKjpo4qiNEt7jDFEAek1HmcAo4FgoFhKaavxfFRbBjL3yz38eqQAKcEhJXYpcThc9x3yxK3NIXE4Tr5udTho7R1QDToNnnotXgbnP28PHd4GHVEBenyNvvh46PA16vD31OPnqcfPqCfQS0+Al4FAbz1B3oYuux+yoiju1WhiEEKsA8LreOkJKeXXTXiPuj6uygaery+O2cBsgNjYM9v6MSrAk74RfmiEQCNw3Qq0GtBqqu87b3Ua132NQK8RaDUatBrQaTXoNM7XdVoNeq1Ap9Fg0GnQazUYdAKDVotB53zOqNfgoXM+Nuo0GPVajHotWvUpXlGUDqrRxCClvLiF75EBxNR4HA1kAceBACGEztVqqH6+vjgWAgsBEhMTz+jz+90X9jqT0xRFUc4q7TEPcRvQWwjRQwhhAG4AVkgpJbABmOo6bhbQlBaIoiiK0oZalBiEENcKITKAscB3QojVrucjhRDfA7haA38FVgPJwCdSyn2uSzwKPCCEOIRzzOG9lsSjKIqitJyQrT2q2g4SExNlUlKdSyYURVGUegghtkspExs7Ti1pVRRFUWpRiUFRFEWpRSUGRVEUpRaVGBRFUZRaVGJQFEVRaumUs5KEEPnAMXfHcYoQnIv2OoPOFCt0rng7U6zQueLtTLFCx4y3u5QytLGDOmVi6IiEEElNmQbWEXSmWKFzxduZYoXOFW9nihU6X7w1qa4kRVEUpRaVGBRFUZRaVGJoPQvdHUAzdKZYoXPF25lihc4Vb2eKFTpfvCeoMQZFURSlFtViUBRFUWpRiaGVCSHuEUIcEELsE0K85O54GiOEeEgIIYUQIe6OpT5CiH8JIfYLIX4XQnwphAhwd0x1EUJc6vq/PySEeMzd8dRHCBEjhNgghEh2/Zze6+6YGiOE0AohdgohvnV3LI0RQgQIIT5z/cwmCyHGujum5lKJoRUJIS4EJgODpZQDgJfdHFKDhBAxwEQgzd2xNGItMFBKORhIAR53czynEUJogTeBy4D+wAwhRH/3RlUvG/CglLIfMAa4uwPHWu1enGX7O4PXgVVSyr7AEDpP3CeoxNC65gDzpZRmACllnpvjacxrwCM0sKVqRyClXFNjb/CtOHf762hGAYeklEeklBbgY5wfEjocKWW2lHKH634Zzj9cbbrfeksIIaKBK4BF7o6lMUIIP+ACXHvLSCktUspi90bVfCoxtK4E4HwhxK9CiB+FECPdHVB9hBBXA5lSyt3ujqWZbgNWujuIOkQB6TUeZ9CB/9hWE0LEAcOAX90bSYP+jfMDjMPdgTRBPJAPLHF1fS0SQni7O6jmanTPZ6U2IcQ6ILyOl57A+f0MxNk8Hwl8IoSIl26a+tVIrHOBS9o3ovo1FKuU8mvXMU/g7Ab5oD1jayJRx3MduiUmhPABPgfuk1KWujueugghrgTypJTbhRDj3R1PE+iA4cA9UspfhRCvA48Bf3dvWM2jEkMzSSkvru81IcQc4AtXIvhNCOHAWS8lv73iq6m+WIUQg4AewG4hBDi7ZnYIIUZJKXPaMcQTGvq+AgghZgFXAhPclWgbkQHE1HgcDWS5KZZGCSH0OJPCB1LKL9wdTwPOBa4WQlwOGAE/IcT7Usqb3BxXfTKADClldQvsM5yJoVNRXUmt6yvgIgAhRAJgoOMV0UJKuUdK2U1KGSeljMP5wzzcXUmhMUKIS3HuD361lLLS3fHUYxvQWwjRQwhhAG4AVrg5pjoJ56eB94BkKeWr7o6nIVLKx6WU0a6f0xuAHzpwUsD1O5QuhOjjemoC8IcbQzojqsXQuhYDi4UQewELMKuDfrrtbP4DeABrXS2crVLKv7g3pNqklDYhxF+B1YAWWCyl3OfmsOpzLjAT2COE2OV6bq6U8ns3xtSV3AN84PqAcAS41c3xNJta+awoiqLUorqSFEVRlFpUYlAURVFqUYlBURRFqUUlBkVRFKUWlRgURVGUWlRiUBRFUWpRiUFRFEWpRSUGRVEUpZb/B8p6rqM0TbujAAAAAElFTkSuQmCC\n", 11 | "text/plain": [ 12 | "
" 13 | ] 14 | }, 15 | "metadata": {}, 16 | "output_type": "display_data" 17 | } 18 | ], 19 | "source": [ 20 | "# Following HIPS autograd https://github.com/HIPS/autograd/blob/master/examples/tanh.py\n", 21 | "# Following https://github.com/QuantumBFS/SSSS/blob/master/1_deep_learning/tanh.py\n", 22 | "import torch\n", 23 | "%matplotlib inline\n", 24 | "import matplotlib.pyplot as plt\n", 25 | "x = torch.linspace(-7, 7, 100, requires_grad=True)\n", 26 | "for i in range(7):\n", 27 | " if (i==0):\n", 28 | " y = torch.tanh(x/2)\n", 29 | " else:\n", 30 | " y, = torch.autograd.grad(y, x, grad_outputs=torch.ones(y.shape[0]), create_graph=True)\n", 31 | " plt.plot(x.detach().numpy(), y.detach().numpy(), '-', label='$%g$-th'%(i))\n", 32 | "plt.legend(loc=2);plt.show()" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": null, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [] 41 | } 42 | ], 43 | "metadata": { 44 | "kernelspec": { 45 | "display_name": "Python 3", 46 | "language": "python", 47 | "name": "python3" 48 | }, 49 | "language_info": { 50 | "codemirror_mode": { 51 | "name": "ipython", 52 | "version": 3 53 | }, 54 | "file_extension": ".py", 55 | "mimetype": "text/x-python", 56 | "name": "python", 57 | "nbconvert_exporter": "python", 58 | "pygments_lexer": "ipython3", 59 | "version": "3.6.8" 60 | } 61 | }, 62 | "nbformat": 4, 63 | "nbformat_minor": 2 64 | } 65 | --------------------------------------------------------------------------------