├── .ipynb_checkpoints ├── batch_norm-checkpoint.ipynb └── chinese_restaurant_process-checkpoint.ipynb ├── README.md ├── batch_norm.ipynb ├── bn.png ├── chinese_restaurant_process.ipynb ├── vb_exponential.py └── vb_normal_gamma.py /.ipynb_checkpoints/chinese_restaurant_process-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Demos on Chinese Restaurant Process\n", 8 | "#### compare theoretical and emprical results for $E(K)$ and $P(K=k)$\n", 9 | "## 中国餐馆过程的演示\n", 10 | "\n", 11 | "Prof Richard Xu, 徐亦达教授\n", 12 | "\n", 13 | "University of Technology Sydney (UTS) 悉尼科技大学\n", 14 | "\n", 15 | "Yida.Xu@uts.edu.au\n", 16 | "\n", 17 | "2018-05-01" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "metadata": {}, 23 | "source": [ 24 | "#### this is my implementation of unsigned Stirling number of the first kind using the following relations:\n", 25 | "##### 这是用递归的方式写的第一种Stirling值的取得方法\n", 26 | "$s(n + 1, k) = n s(n, k) + s(n, k - 1)$\n", 27 | "\n", 28 | "be aware that this can be quite slow when $n$ is large" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 1, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "def stirling(n,k):\n", 38 | "\n", 39 | " if n<=0:\n", 40 | " return 1\n", 41 | " \n", 42 | " elif k<=0:\n", 43 | " return 0\n", 44 | " \n", 45 | " elif (n == k):\n", 46 | " return 1\n", 47 | " \n", 48 | " elif n!=0 and n==k:\n", 49 | " return 1\n", 50 | " \n", 51 | " elif k >n:\n", 52 | " return 0\n", 53 | " \n", 54 | " else:\n", 55 | " return (n-1) * stirling(n-1,k)+ stirling(n-1,k-1)" 56 | ] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": {}, 61 | "source": [ 62 | "this is to test to see if \"unsigned Stirling number of first kind\" works\n" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 2, 68 | "metadata": {}, 69 | "outputs": [ 70 | { 71 | "name": "stdout", 72 | "output_type": "stream", 73 | "text": [ 74 | "str( 1 , 1 )= 1\n", 75 | "str( 1 , 2 )= 0\n", 76 | "str( 1 , 3 )= 0\n", 77 | "str( 1 , 4 )= 0\n", 78 | "\n", 79 | "\n", 80 | "str( 2 , 1 )= 1\n", 81 | "str( 2 , 2 )= 1\n", 82 | "str( 2 , 3 )= 0\n", 83 | "str( 2 , 4 )= 0\n", 84 | "\n", 85 | "\n", 86 | "str( 3 , 1 )= 2\n", 87 | "str( 3 , 2 )= 3\n", 88 | "str( 3 , 3 )= 1\n", 89 | "str( 3 , 4 )= 0\n", 90 | "\n", 91 | "\n", 92 | "str( 4 , 1 )= 6\n", 93 | "str( 4 , 2 )= 11\n", 94 | "str( 4 , 3 )= 6\n", 95 | "str( 4 , 4 )= 1\n", 96 | "\n", 97 | "\n" 98 | ] 99 | } 100 | ], 101 | "source": [ 102 | "for n in range(1,5):\n", 103 | " for k in range(1,5):\n", 104 | " print \"str(\", n, \",\", k, \")=\", stirling(n,k)\n", 105 | " print \"\\n\"" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "The function below samples from the joint distribution **directly** $\\Pr(z_1, z_2, \\dots z_N | \\alpha)$\n", 113 | "by using $(z_1, z_2,\\dots, z_n) \\sim \\big( \\Pr(z_1)\\Pr(z_2|z_1)\\dots \\Pr(z_n | z_1, \\dots z_{n-1}) \\equiv \\Pr(z_1, z_2, \\dots z_{n}) \\big)$\n", 114 | "这个函数是直接从联合分布里采样" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 3, 120 | "metadata": {}, 121 | "outputs": [], 122 | "source": [ 123 | "import numpy as np\n", 124 | "\n", 125 | "def Draw_CRP_Direct_Sample(N = 10, alpha = 3, T = 50, VERBOSE = False):\n", 126 | "\n", 127 | " Z_table = np.zeros((T,N))\n", 128 | " \n", 129 | " for t in range(T):\n", 130 | " \n", 131 | " Z = np.zeros(N,dtype=int)\n", 132 | "\n", 133 | " for i in range(N):\n", 134 | "\n", 135 | " if i == 0:\n", 136 | " Z[i] = 1\n", 137 | " else:\n", 138 | " if VERBOSE:\n", 139 | " print Z\n", 140 | " unique, counts = np.unique(Z, return_counts=True)\n", 141 | "\n", 142 | " # remove the zeros unsigned tables\n", 143 | " if unique[0] == 0:\n", 144 | " unique = np.delete(unique,0)\n", 145 | " counts = np.delete(counts,0)\n", 146 | "\n", 147 | " #if VERBOSE:\n", 148 | " # print \"unique,counts,alpha\", unique,counts,alpha\n", 149 | "\n", 150 | " # added alpha to the end of the counts (weights) array\n", 151 | " counts = np.append(counts,alpha)\n", 152 | "\n", 153 | " # also the new table index will be the max of table seen so far\n", 154 | " unique = np.append(unique,max(unique)+1)\n", 155 | "\n", 156 | " #print \"np.append(counts,alpha)\",counts\n", 157 | "\n", 158 | " #if VERBOSE:\n", 159 | " # print sum(counts)\n", 160 | " u = np.random.uniform()*sum(counts)\n", 161 | "\n", 162 | " a_counts = np.cumsum(counts)\n", 163 | "\n", 164 | " if VERBOSE:\n", 165 | " print counts, u, a_counts > u\n", 166 | "\n", 167 | " # first index where accumuated sum is greater than random variable\n", 168 | " index = np.argmax(a_counts > u)\n", 169 | "\n", 170 | " #print \"index\", index\n", 171 | "\n", 172 | " Z[i] = unique[index]\n", 173 | "\n", 174 | " if VERBOSE:\n", 175 | " print Z\n", 176 | " print(\"\\n\\n\") \n", 177 | " \n", 178 | " \n", 179 | " Z_table[t,:] = Z\n", 180 | " return Z_table" 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": {}, 186 | "source": [ 187 | "This function below samples from the joint distribution $\\Pr(z_1, z_2, \\dots z_N | \\alpha)$ using **Gibbs sampling**\n", 188 | "这个函数用吉布斯采样从联合分布$\\Pr(z_1, z_2, \\dots z_N | \\alpha)$中采样" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 4, 194 | "metadata": { 195 | "scrolled": true 196 | }, 197 | "outputs": [], 198 | "source": [ 199 | "import numpy as np\n", 200 | "\n", 201 | "def Draw_CRP_Gibbs_Sample(N = 10, alpha = 3, T = 50, burn_in = 10, VERBOSE = False):\n", 202 | " \n", 203 | " Z = np.ones(N,dtype=int)\n", 204 | " Z_table = np.zeros((T,N))\n", 205 | "\n", 206 | " for t in range(T+burn_in):\n", 207 | "\n", 208 | " for i in range(N):\n", 209 | "\n", 210 | " if VERBOSE:\n", 211 | " print Z\n", 212 | "\n", 213 | " # remove current table assignment\n", 214 | " Z[i] = 0\n", 215 | "\n", 216 | " if VERBOSE:\n", 217 | " print Z\n", 218 | "\n", 219 | "\n", 220 | " unique, counts = np.unique(Z, return_counts=True)\n", 221 | "\n", 222 | " # remove the zeros in unassigned tables\n", 223 | " if unique[0] == 0:\n", 224 | " unique = np.delete(unique,0)\n", 225 | " counts = np.delete(counts,0)\n", 226 | "\n", 227 | " if VERBOSE:\n", 228 | " print \"unique,counts,alpha\", unique,counts,alpha\n", 229 | "\n", 230 | " # added alpha to the end of the counts (weights) array\n", 231 | " counts = np.append(counts,alpha)\n", 232 | "\n", 233 | " # also the new table index will be the max of table seen so far\n", 234 | " unique = np.append(unique,max(unique)+1)\n", 235 | "\n", 236 | " #print \"np.append(counts,alpha)\",counts\n", 237 | "\n", 238 | " #if VERBOSE:\n", 239 | " # print sum(counts)\n", 240 | " u = np.random.uniform()*sum(counts)\n", 241 | "\n", 242 | " a_counts = np.cumsum(counts)\n", 243 | "\n", 244 | " if VERBOSE:\n", 245 | " print counts, u, a_counts > u\n", 246 | "\n", 247 | " # first index where accumuated sum is greater than random variable\n", 248 | " index = np.argmax(a_counts > u)\n", 249 | "\n", 250 | " #print \"index\", index\n", 251 | "\n", 252 | " Z[i] = unique[index]\n", 253 | "\n", 254 | " if VERBOSE:\n", 255 | " print Z\n", 256 | " print(\"\\n\") \n", 257 | "\n", 258 | " old_table = np.unique(Z)\n", 259 | " new_table = np.array(range(1,len(old_table)+1))\n", 260 | "\n", 261 | " for k in range(len(old_table)):\n", 262 | " Z[Z == old_table[k]]=new_table[k]\n", 263 | "\n", 264 | " if t >= burn_in:\n", 265 | " Z_table[t-burn_in,:] = Z\n", 266 | "\n", 267 | " if VERBOSE:\n", 268 | " print Z\n", 269 | " print(\"\\n\\n\\n\") \n", 270 | "\n", 271 | " if VERBOSE:\n", 272 | " print Z_table\n", 273 | " \n", 274 | " return Z_table\n", 275 | "\n", 276 | "\n", 277 | " \n", 278 | "\n" 279 | ] 280 | }, 281 | { 282 | "cell_type": "markdown", 283 | "metadata": {}, 284 | "source": [ 285 | "#### in the following, we compare sample $\\widehat{E}(K)$ vs true $E(K)$, and sample $\\widehat{P}(K = k)$ vs true$P(K= k)$\n", 286 | "$K$ is the number of tables\n", 287 | "\n", 288 | "在以下的函数里，我们对$K$的样本期望值和$K$理论期望值进行比较; 我们还对$P(K=k)$的样本值和$P(K=k)$理论期望值进行比较; " 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": 5, 294 | "metadata": {}, 295 | "outputs": [ 296 | { 297 | "name": "stdout", 298 | "output_type": "stream", 299 | "text": [ 300 | "sample E(K) = 2.2864 theorical E(K) = 2.2833333333333337\n", 301 | "\n", 302 | "\n", 303 | "sample Pr(K = 1 ) = 0.2004 ; theorical Pr(K = 1 ) = 0.2\n", 304 | "sample Pr(K = 2 ) = 0.4169 ; theorical Pr(K = 2 ) = 0.416666666667\n", 305 | "sample Pr(K = 3 ) = 0.2892 ; theorical Pr(K = 3 ) = 0.291666666667\n", 306 | "sample Pr(K = 4 ) = 0.0829 ; theorical Pr(K = 4 ) = 0.0833333333333\n", 307 | "sample Pr(K = 5 ) = 0.0106 ; theorical Pr(K = 5 ) = 0.00833333333333\n" 308 | ] 309 | } 310 | ], 311 | "source": [ 312 | "from scipy.special import digamma, gamma, gammaln\n", 313 | "from math import log,exp\n", 314 | "\n", 315 | "alpha = 1\n", 316 | "N = 5\n", 317 | "T = 10000\n", 318 | "VERBOSE = False\n", 319 | "burn_in = int(T/10)\n", 320 | "\n", 321 | "# ------------------------------------------\n", 322 | "# call either of the following function\n", 323 | "# 您可以选择运行下面的其中一个函数\n", 324 | "# ------------------------------------------\n", 325 | "\n", 326 | "#Z_table = Draw_CRP_Direct_Sample(N, alpha , T , VERBOSE)\n", 327 | "\n", 328 | "Z_table = Draw_CRP_Gibbs_Sample(N , alpha , T , burn_in , VERBOSE)\n", 329 | "\n", 330 | "\n", 331 | "\n", 332 | "\n", 333 | "table_numbers = np.zeros(T,dtype=int)\n", 334 | "\n", 335 | "for t in range(T):\n", 336 | " unique, count = np.unique(Z_table[t,:], return_counts=True)\n", 337 | " table_numbers[t] = len(count)\n", 338 | "\n", 339 | "\n", 340 | "# ------------------------------------------\n", 341 | "# we compare sample E(K) vs true E(K)\n", 342 | "# 我们对 K 的样本期望值和理论期望值进行比较\n", 343 | "# ------------------------------------------ \n", 344 | "\n", 345 | "exp_average = np.mean(table_numbers) \n", 346 | "theory_average = alpha * (digamma(alpha + N) - digamma(alpha))\n", 347 | "print \"sample E(K) = \", exp_average, \" theorical E(K) = \", theory_average\n", 348 | "print \"\\n\"\n", 349 | "\n", 350 | "# ------------------------------------------\n", 351 | "# We compared Pr(K = k) between sample and true\n", 352 | "# 我们对 Pr(K=k) 的样本期望值和理论期望值进行比较\n", 353 | "# ------------------------------------------ \n", 354 | "\n", 355 | "unique, count = np.unique(table_numbers, return_counts=True)\n", 356 | "\n", 357 | "for t in range(len(unique)):\n", 358 | " k = unique[t]\n", 359 | " exp_prob = count[t]* 1.000 / T\n", 360 | " \n", 361 | " # ------------------------------------------\n", 362 | " # to avoid overflow, we use a little trick: \n", 363 | " # 为防止变量接近0，或变成极值，我们会用以下的小技巧:\n", 364 | " # a*b/c = exp(log(a)+log(b)-log(c))\n", 365 | " # ------------------------------------------ \n", 366 | " \n", 367 | " theory_prob = log(stirling(N,k))+k*log(alpha)+gammaln(alpha)- gammaln(alpha + N)\n", 368 | " theory_prob = exp(theory_prob)\n", 369 | " \n", 370 | " print \"sample Pr(K = \", unique[t],\") = \", exp_prob, \"; theorical Pr(K = \", unique[t],\") = \", theory_prob" 371 | ] 372 | }, 373 | { 374 | "cell_type": "code", 375 | "execution_count": null, 376 | "metadata": {}, 377 | "outputs": [], 378 | "source": [] 379 | }, 380 | { 381 | "cell_type": "code", 382 | "execution_count": null, 383 | "metadata": {}, 384 | "outputs": [], 385 | "source": [] 386 | } 387 | ], 388 | "metadata": { 389 | "kernelspec": { 390 | "display_name": "Python 2", 391 | "language": "python", 392 | "name": "python2" 393 | }, 394 | "language_info": { 395 | "codemirror_mode": { 396 | "name": "ipython", 397 | "version": 2 398 | }, 399 | "file_extension": ".py", 400 | "mimetype": "text/x-python", 401 | "name": "python", 402 | "nbconvert_exporter": "python", 403 | "pygments_lexer": "ipython2", 404 | "version": "2.7.14" 405 | } 406 | }, 407 | "nbformat": 4, 408 | "nbformat_minor": 2 409 | } 410 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python_machine_learning 2 | this is all the python code to accompany my machine learning notes 3 | -------------------------------------------------------------------------------- /bn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roboticcam/python_machine_learning/d49a9299759f73f19b7c3b3b5751a5f69d990eea/bn.png -------------------------------------------------------------------------------- /chinese_restaurant_process.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Demos on Chinese Restaurant Process\n", 8 | "#### compare theoretical and emprical results for $E(K)$ and $P(K=k)$\n", 9 | "## 中国餐馆过程的演示\n", 10 | "\n", 11 | "Prof Richard Xu, 徐亦达教授\n", 12 | "\n", 13 | "University of Technology Sydney (UTS) 悉尼科技大学\n", 14 | "\n", 15 | "Yida.Xu@uts.edu.au\n", 16 | "\n", 17 | "2018-05-01" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "metadata": {}, 23 | "source": [ 24 | "#### this is my implementation of unsigned Stirling number of the first kind using the following relations:\n", 25 | "##### 这是用递归的方式写的第一种Stirling值的取得方法\n", 26 | "$s(n + 1, k) = n s(n, k) + s(n, k - 1)$\n", 27 | "\n", 28 | "be aware that this can be quite slow when $n$ is large" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 1, 34 | "metadata": {}, 35 | "outputs": [], 36 | "source": [ 37 | "def stirling(n,k):\n", 38 | "\n", 39 | " if n<=0:\n", 40 | " return 1\n", 41 | " \n", 42 | " elif k<=0:\n", 43 | " return 0\n", 44 | " \n", 45 | " elif (n == k):\n", 46 | " return 1\n", 47 | " \n", 48 | " elif n!=0 and n==k:\n", 49 | " return 1\n", 50 | " \n", 51 | " elif k >n:\n", 52 | " return 0\n", 53 | " \n", 54 | " else:\n", 55 | " return (n-1) * stirling(n-1,k)+ stirling(n-1,k-1)" 56 | ] 57 | }, 58 | { 59 | "cell_type": "markdown", 60 | "metadata": {}, 61 | "source": [ 62 | "this is to test to see if \"unsigned Stirling number of first kind\" works\n" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 2, 68 | "metadata": {}, 69 | "outputs": [ 70 | { 71 | "name": "stdout", 72 | "output_type": "stream", 73 | "text": [ 74 | "str( 1 , 1 )= 1\n", 75 | "str( 1 , 2 )= 0\n", 76 | "str( 1 , 3 )= 0\n", 77 | "str( 1 , 4 )= 0\n", 78 | "\n", 79 | "\n", 80 | "str( 2 , 1 )= 1\n", 81 | "str( 2 , 2 )= 1\n", 82 | "str( 2 , 3 )= 0\n", 83 | "str( 2 , 4 )= 0\n", 84 | "\n", 85 | "\n", 86 | "str( 3 , 1 )= 2\n", 87 | "str( 3 , 2 )= 3\n", 88 | "str( 3 , 3 )= 1\n", 89 | "str( 3 , 4 )= 0\n", 90 | "\n", 91 | "\n", 92 | "str( 4 , 1 )= 6\n", 93 | "str( 4 , 2 )= 11\n", 94 | "str( 4 , 3 )= 6\n", 95 | "str( 4 , 4 )= 1\n", 96 | "\n", 97 | "\n" 98 | ] 99 | } 100 | ], 101 | "source": [ 102 | "for n in range(1,5):\n", 103 | " for k in range(1,5):\n", 104 | " print \"str(\", n, \",\", k, \")=\", stirling(n,k)\n", 105 | " print \"\\n\"" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "The function below samples from the joint distribution **directly** $\\Pr(z_1, z_2, \\dots z_N | \\alpha)$\n", 113 | "by using $(z_1, z_2,\\dots, z_n) \\sim \\big( \\Pr(z_1)\\Pr(z_2|z_1)\\dots \\Pr(z_n | z_1, \\dots z_{n-1}) \\equiv \\Pr(z_1, z_2, \\dots z_{n}) \\big)$\n", 114 | "这个函数是直接从联合分布里采样" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 3, 120 | "metadata": {}, 121 | "outputs": [], 122 | "source": [ 123 | "import numpy as np\n", 124 | "\n", 125 | "def Draw_CRP_Direct_Sample(N = 10, alpha = 3, T = 50, VERBOSE = False):\n", 126 | "\n", 127 | " Z_table = np.zeros((T,N))\n", 128 | " \n", 129 | " for t in range(T):\n", 130 | " \n", 131 | " Z = np.zeros(N,dtype=int)\n", 132 | "\n", 133 | " for i in range(N):\n", 134 | "\n", 135 | " if i == 0:\n", 136 | " Z[i] = 1\n", 137 | " else:\n", 138 | " if VERBOSE:\n", 139 | " print Z\n", 140 | " unique, counts = np.unique(Z, return_counts=True)\n", 141 | "\n", 142 | " # remove the zeros unsigned tables\n", 143 | " if unique[0] == 0:\n", 144 | " unique = np.delete(unique,0)\n", 145 | " counts = np.delete(counts,0)\n", 146 | "\n", 147 | " #if VERBOSE:\n", 148 | " # print \"unique,counts,alpha\", unique,counts,alpha\n", 149 | "\n", 150 | " # added alpha to the end of the counts (weights) array\n", 151 | " counts = np.append(counts,alpha)\n", 152 | "\n", 153 | " # also the new table index will be the max of table seen so far\n", 154 | " unique = np.append(unique,max(unique)+1)\n", 155 | "\n", 156 | " #print \"np.append(counts,alpha)\",counts\n", 157 | "\n", 158 | " #if VERBOSE:\n", 159 | " # print sum(counts)\n", 160 | " u = np.random.uniform()*sum(counts)\n", 161 | "\n", 162 | " a_counts = np.cumsum(counts)\n", 163 | "\n", 164 | " if VERBOSE:\n", 165 | " print counts, u, a_counts > u\n", 166 | "\n", 167 | " # first index where accumuated sum is greater than random variable\n", 168 | " index = np.argmax(a_counts > u)\n", 169 | "\n", 170 | " #print \"index\", index\n", 171 | "\n", 172 | " Z[i] = unique[index]\n", 173 | "\n", 174 | " if VERBOSE:\n", 175 | " print Z\n", 176 | " print(\"\\n\\n\") \n", 177 | " \n", 178 | " \n", 179 | " Z_table[t,:] = Z\n", 180 | " return Z_table" 181 | ] 182 | }, 183 | { 184 | "cell_type": "markdown", 185 | "metadata": {}, 186 | "source": [ 187 | "This function below samples from the joint distribution $\\Pr(z_1, z_2, \\dots z_N | \\alpha)$ using **Gibbs sampling**\n", 188 | "这个函数用吉布斯采样从联合分布$\\Pr(z_1, z_2, \\dots z_N | \\alpha)$中采样" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 4, 194 | "metadata": { 195 | "scrolled": true 196 | }, 197 | "outputs": [], 198 | "source": [ 199 | "import numpy as np\n", 200 | "\n", 201 | "def Draw_CRP_Gibbs_Sample(N = 10, alpha = 3, T = 50, burn_in = 10, VERBOSE = False):\n", 202 | " \n", 203 | " Z = np.ones(N,dtype=int)\n", 204 | " Z_table = np.zeros((T,N))\n", 205 | "\n", 206 | " for t in range(T+burn_in):\n", 207 | "\n", 208 | " for i in range(N):\n", 209 | "\n", 210 | " if VERBOSE:\n", 211 | " print Z\n", 212 | "\n", 213 | " # remove current table assignment\n", 214 | " Z[i] = 0\n", 215 | "\n", 216 | " if VERBOSE:\n", 217 | " print Z\n", 218 | "\n", 219 | "\n", 220 | " unique, counts = np.unique(Z, return_counts=True)\n", 221 | "\n", 222 | " # remove the zeros in unassigned tables\n", 223 | " if unique[0] == 0:\n", 224 | " unique = np.delete(unique,0)\n", 225 | " counts = np.delete(counts,0)\n", 226 | "\n", 227 | " if VERBOSE:\n", 228 | " print \"unique,counts,alpha\", unique,counts,alpha\n", 229 | "\n", 230 | " # added alpha to the end of the counts (weights) array\n", 231 | " counts = np.append(counts,alpha)\n", 232 | "\n", 233 | " # also the new table index will be the max of table seen so far\n", 234 | " unique = np.append(unique,max(unique)+1)\n", 235 | "\n", 236 | " #print \"np.append(counts,alpha)\",counts\n", 237 | "\n", 238 | " #if VERBOSE:\n", 239 | " # print sum(counts)\n", 240 | " u = np.random.uniform()*sum(counts)\n", 241 | "\n", 242 | " a_counts = np.cumsum(counts)\n", 243 | "\n", 244 | " if VERBOSE:\n", 245 | " print counts, u, a_counts > u\n", 246 | "\n", 247 | " # first index where accumuated sum is greater than random variable\n", 248 | " index = np.argmax(a_counts > u)\n", 249 | "\n", 250 | " #print \"index\", index\n", 251 | "\n", 252 | " Z[i] = unique[index]\n", 253 | "\n", 254 | " if VERBOSE:\n", 255 | " print Z\n", 256 | " print(\"\\n\") \n", 257 | "\n", 258 | " old_table = np.unique(Z)\n", 259 | " new_table = np.array(range(1,len(old_table)+1))\n", 260 | "\n", 261 | " for k in range(len(old_table)):\n", 262 | " Z[Z == old_table[k]]=new_table[k]\n", 263 | "\n", 264 | " if t >= burn_in:\n", 265 | " Z_table[t-burn_in,:] = Z\n", 266 | "\n", 267 | " if VERBOSE:\n", 268 | " print Z\n", 269 | " print(\"\\n\\n\\n\") \n", 270 | "\n", 271 | " if VERBOSE:\n", 272 | " print Z_table\n", 273 | " \n", 274 | " return Z_table\n", 275 | "\n", 276 | "\n", 277 | " \n", 278 | "\n" 279 | ] 280 | }, 281 | { 282 | "cell_type": "markdown", 283 | "metadata": {}, 284 | "source": [ 285 | "#### in the following, we compare sample $\\widehat{E}(K)$ vs true $E(K)$, and sample $\\widehat{P}(K = k)$ vs true$P(K= k)$\n", 286 | "$K$ is the number of tables\n", 287 | "\n", 288 | "在以下的函数里，我们对$K$的样本期望值和$K$理论期望值进行比较; 我们还对$P(K=k)$的样本值和$P(K=k)$理论期望值进行比较; " 289 | ] 290 | }, 291 | { 292 | "cell_type": "code", 293 | "execution_count": 6, 294 | "metadata": {}, 295 | "outputs": [ 296 | { 297 | "name": "stdout", 298 | "output_type": "stream", 299 | "text": [ 300 | "sample E(K) = 2.2766 theorical E(K) = 2.2833333333333337\n", 301 | "\n", 302 | "\n", 303 | "sample Pr(K = 1 ) = 0.2039 ; theorical Pr(K = 1 ) = 0.2\n", 304 | "sample Pr(K = 2 ) = 0.4143 ; theorical Pr(K = 2 ) = 0.416666666667\n", 305 | "sample Pr(K = 3 ) = 0.2912 ; theorical Pr(K = 3 ) = 0.291666666667\n", 306 | "sample Pr(K = 4 ) = 0.0825 ; theorical Pr(K = 4 ) = 0.0833333333333\n", 307 | "sample Pr(K = 5 ) = 0.0081 ; theorical Pr(K = 5 ) = 0.00833333333333\n" 308 | ] 309 | } 310 | ], 311 | "source": [ 312 | "from scipy.special import digamma, gamma, gammaln\n", 313 | "from math import log,exp\n", 314 | "\n", 315 | "alpha = 1\n", 316 | "N = 5\n", 317 | "T = 10000\n", 318 | "VERBOSE = False\n", 319 | "burn_in = int(T/10)\n", 320 | "\n", 321 | "# ------------------------------------------\n", 322 | "# call either of the following function\n", 323 | "# 您可以选择运行下面的其中一个函数\n", 324 | "# ------------------------------------------\n", 325 | "\n", 326 | "#Z_table = Draw_CRP_Direct_Sample(N, alpha , T , VERBOSE)\n", 327 | "\n", 328 | "Z_table = Draw_CRP_Gibbs_Sample(N , alpha , T , burn_in , VERBOSE)\n", 329 | "\n", 330 | "\n", 331 | "\n", 332 | "\n", 333 | "table_numbers = np.zeros(T,dtype=int)\n", 334 | "\n", 335 | "for t in range(T):\n", 336 | " unique, count = np.unique(Z_table[t,:], return_counts=True)\n", 337 | " table_numbers[t] = len(count)\n", 338 | "\n", 339 | "\n", 340 | "# ------------------------------------------\n", 341 | "# we compare sample E(K) vs true E(K)\n", 342 | "# 我们对 K 的样本期望值和理论期望值进行比较\n", 343 | "# ------------------------------------------ \n", 344 | "\n", 345 | "exp_average = np.mean(table_numbers) \n", 346 | "theory_average = alpha * (digamma(alpha + N) - digamma(alpha))\n", 347 | "print \"sample E(K) = \", exp_average, \" theorical E(K) = \", theory_average\n", 348 | "print \"\\n\"\n", 349 | "\n", 350 | "# ------------------------------------------\n", 351 | "# We compared Pr(K = k) between sample and true\n", 352 | "# 我们对 Pr(K=k) 的样本期望值和理论期望值进行比较\n", 353 | "# ------------------------------------------ \n", 354 | "\n", 355 | "unique, count = np.unique(table_numbers, return_counts=True)\n", 356 | "\n", 357 | "for t in range(len(unique)):\n", 358 | " k = unique[t]\n", 359 | " exp_prob = count[t]* 1.000 / T\n", 360 | " \n", 361 | " # ------------------------------------------\n", 362 | " # to avoid overflow, we use a little trick: \n", 363 | " # 为防止变量接近0，或变成极值，我们会用以下的小技巧:\n", 364 | " # a*b/c = exp(log(a)+log(b)-log(c))\n", 365 | " # ------------------------------------------ \n", 366 | " \n", 367 | " theory_prob = log(stirling(N,k))+k*log(alpha)+gammaln(alpha)- gammaln(alpha + N)\n", 368 | " theory_prob = exp(theory_prob)\n", 369 | " \n", 370 | " print \"sample Pr(K = \", unique[t],\") = \", exp_prob, \"; theorical Pr(K = \", unique[t],\") = \", theory_prob" 371 | ] 372 | }, 373 | { 374 | "cell_type": "code", 375 | "execution_count": null, 376 | "metadata": {}, 377 | "outputs": [], 378 | "source": [] 379 | }, 380 | { 381 | "cell_type": "code", 382 | "execution_count": null, 383 | "metadata": {}, 384 | "outputs": [], 385 | "source": [] 386 | } 387 | ], 388 | "metadata": { 389 | "kernelspec": { 390 | "display_name": "Python 2", 391 | "language": "python", 392 | "name": "python2" 393 | }, 394 | "language_info": { 395 | "codemirror_mode": { 396 | "name": "ipython", 397 | "version": 2 398 | }, 399 | "file_extension": ".py", 400 | "mimetype": "text/x-python", 401 | "name": "python", 402 | "nbconvert_exporter": "python", 403 | "pygments_lexer": "ipython2", 404 | "version": "2.7.14" 405 | } 406 | }, 407 | "nbformat": 4, 408 | "nbformat_minor": 2 409 | } 410 | -------------------------------------------------------------------------------- /vb_exponential.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from matplotlib.widgets import Button 4 | from scipy.stats import norm 5 | 6 | class VariationalInference2DGaussian: 7 | def __init__(self, mu, Sigma): 8 | self.mu = mu 9 | self.Sigma = Sigma 10 | self.q_z1 = {'mean': -2, 'var': 1} 11 | self.q_z2 = {'mean': -1.5, 'var': 3} 12 | 13 | def update_q_z1(self): 14 | 15 | # Update q(z1) using the formula: μ1 + (σ12/σ22) * (E[z2] - μ2) 16 | sigma_12 = self.Sigma[0, 1] 17 | sigma_22 = self.Sigma[1, 1] 18 | 19 | # E[z2] is the mean of q(z2) 20 | E_z2 = self.q_z2['mean'] 21 | 22 | # Calculate the updated mean for q(z1) 23 | updated_mean = self.mu[0] + (sigma_12 / sigma_22) * (E_z2 - self.mu[1]) 24 | 25 | # Update the variance (this remains the same as before) 26 | updated_var = self.Sigma[0, 0] - (sigma_12**2 / sigma_22) 27 | 28 | # Set the new mean and variance for q(z1) 29 | self.q_z1['mean'] = updated_mean 30 | self.q_z1['var'] = updated_var 31 | 32 | 33 | 34 | def update_q_z2(self): 35 | # Update q(z2) using the formula: μ2 + (σ21/σ11) * (E[z1] - μ1) 36 | sigma_21 = self.Sigma[1, 0] 37 | sigma_11 = self.Sigma[0, 0] 38 | 39 | # E[z1] is the mean of q(z1) 40 | E_z1 = self.q_z1['mean'] 41 | 42 | # Calculate the updated mean for q(z2) 43 | updated_mean = self.mu[1] + (sigma_21 / sigma_11) * (E_z1 - self.mu[0]) 44 | 45 | # Update the variance 46 | updated_var = self.Sigma[1, 1] - (sigma_21**2 / sigma_11) 47 | 48 | # Set the new mean and variance for q(z2) 49 | self.q_z2['mean'] = updated_mean 50 | self.q_z2['var'] = updated_var 51 | 52 | def iterate(self): 53 | self.update_q_z1() 54 | self.update_q_z2() 55 | 56 | # Example usage 57 | mu = np.array([1, 2]) 58 | Sigma = np.array([[2, 0.5], [0.5, 1]]) 59 | 60 | vi = VariationalInference2DGaussian(mu, Sigma) 61 | 62 | # Set up the plot 63 | fig, ax = plt.subplots(figsize=(10, 8)) 64 | plt.subplots_adjust(bottom=0.2) 65 | x = np.linspace(-5, 5, 100) 66 | y = np.linspace(-5, 5, 100) 67 | X, Y = np.meshgrid(x, y) 68 | 69 | def multivariate_gaussian(pos, mu, Sigma): 70 | n = mu.shape[0] 71 | Sigma_det = np.linalg.det(Sigma) 72 | Sigma_inv = np.linalg.inv(Sigma) 73 | N = np.sqrt((2*np.pi)**n * Sigma_det) 74 | fac = np.einsum('...k,kl,...l->...', pos-mu, Sigma_inv, pos-mu) 75 | return np.exp(-fac / 2) / N 76 | 77 | # True distribution 78 | pos = np.empty(X.shape + (2,)) 79 | pos[:, :, 0] = X 80 | pos[:, :, 1] = Y 81 | Z_true = multivariate_gaussian(pos, mu, Sigma) 82 | 83 | # Initial variational distribution 84 | Z_q = np.zeros_like(Z_true) 85 | 86 | im = ax.imshow(Z_true, extent=[-5, 5, -5, 5], origin='lower', cmap='viridis', alpha=0.5) 87 | contour_true = ax.contour(X, Y, Z_true, colors='k', alpha=0.5) 88 | contour_q = ax.contour(X, Y, Z_q, colors='r', alpha=0.5) 89 | 90 | ax.set_xlabel('z1') 91 | ax.set_ylabel('z2') 92 | ax.set_title('True Distribution (black) vs Variational Distribution (red)') 93 | 94 | # Add marginal distributions 95 | ax_marg_x = ax.inset_axes([0, 1.05, 1, 0.2], sharex=ax) 96 | ax_marg_y = ax.inset_axes([1.05, 0, 0.2, 1], sharey=ax) 97 | 98 | line_q1, = ax_marg_x.plot([], [], 'r-', lw=2, label='q(z1)') 99 | line_q2, = ax_marg_y.plot([], [], 'r-', lw=2, label='q(z2)') 100 | 101 | ax_marg_x.set_ylim(0, 0.5) 102 | ax_marg_y.set_xlim(0, 0.5) 103 | 104 | ax_marg_x.axis('off') 105 | ax_marg_y.axis('off') 106 | 107 | def update_plot(event): 108 | vi.iterate() 109 | global contour_q, line_q1, line_q2 110 | 111 | # Update variational distribution 112 | Z_q = multivariate_gaussian(pos, np.array([vi.q_z1['mean'], vi.q_z2['mean']]), 113 | np.diag([vi.q_z1['var'], vi.q_z2['var']])) 114 | 115 | # Clear previous contours and plot new ones 116 | for coll in contour_q.collections: 117 | coll.remove() 118 | 119 | contour_q = ax.contour(X, Y, Z_q, colors='r', alpha=0.5) 120 | 121 | # Update marginal distributions 122 | z1_range = np.linspace(-5, 5, 100) 123 | z2_range = np.linspace(-5, 5, 100) 124 | 125 | q1_pdf = norm.pdf(z1_range, vi.q_z1['mean'], np.sqrt(vi.q_z1['var'])) 126 | q2_pdf = norm.pdf(z2_range, vi.q_z2['mean'], np.sqrt(vi.q_z2['var'])) 127 | 128 | line_q1.set_data(z1_range, q1_pdf) 129 | line_q2.set_data(q2_pdf, z2_range) 130 | 131 | ax_marg_x.clear() 132 | ax_marg_y.clear() 133 | ax_marg_x.plot(z1_range, q1_pdf, 'r-', lw=2) 134 | ax_marg_y.plot(q2_pdf, z2_range, 'r-', lw=2) 135 | ax_marg_x.set_ylim(0, max(q1_pdf) * 1.1) 136 | ax_marg_y.set_xlim(0, max(q2_pdf) * 1.1) 137 | ax_marg_x.axis('off') 138 | ax_marg_y.axis('off') 139 | 140 | fig.canvas.draw_idle() 141 | 142 | # Add a button for iteration 143 | ax_button = plt.axes([0.81, 0.05, 0.1, 0.075]) 144 | button = Button(ax_button, 'Iterate') 145 | button.on_clicked(update_plot) 146 | 147 | plt.show() -------------------------------------------------------------------------------- /vb_normal_gamma.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from matplotlib.widgets import Button 4 | from scipy.stats import norm, gamma 5 | 6 | def true_posterior(mu, tau, data, mu0, lambda0, a0, b0): 7 | N = len(data) 8 | x_bar = np.mean(data) 9 | 10 | lambda_n = lambda0 + N 11 | mu_n = (lambda0 * mu0 + N * x_bar) / lambda_n 12 | a_n = a0 + N / 2 13 | b_n = b0 + 0.5 * (np.sum((data - x_bar)**2) + 14 | N * lambda0 / lambda_n * (x_bar - mu0)**2) 15 | return norm.pdf(mu, mu_n, 1/np.sqrt(lambda_n*tau)) * gamma.pdf(tau, a_n, scale=1/b_n) 16 | 17 | class VariationalInferenceNormalGamma: 18 | def __init__(self, data, mu0, lambda0, a0, b0): 19 | self.data = data 20 | self.N = len(data) 21 | self.x_bar = np.mean(data) 22 | self.mu0 = mu0 23 | self.lambda0 = lambda0 24 | self.a0 = a0 25 | self.b0 = b0 26 | 27 | # Initialize variational parameters - far from expected valueså 28 | self.lambda_n = 0.1 # Much smaller than lambda0 + N 29 | self.mu_n = 10.0 # Far from the expected mean 30 | self.a_n = 0.5 # Much smaller than a0 + N/2 31 | self.b_n = 10.0 # Much larger than expected 32 | 33 | 34 | def update(self): 35 | self.lambda_n = self.lambda0 + self.N 36 | self.mu_n = (self.lambda0 * self.mu0 + self.N * self.x_bar) / self.lambda_n 37 | self.a_n = self.a0 + self.N / 2 38 | E_tau = self.a_n / self.b_n 39 | E_mu2 = 1 / (E_tau * self.lambda_n) + self.mu_n**2 40 | self.b_n = self.b0 + 0.5 * (np.sum(self.data**2) + 41 | self.lambda0 * self.mu0**2 + 42 | (self.lambda_n * E_mu2) - 43 | 2 * (self.N * self.x_bar + self.lambda0 * self.mu0) * self.mu_n) 44 | 45 | def plot(self, ax_joint, ax_marg_x, ax_marg_y): 46 | # Plot joint distribution 47 | mu_range = np.linspace(self.mu_n - 3/np.sqrt(self.lambda_n), self.mu_n + 3/np.sqrt(self.lambda_n), 100) 48 | tau_range = np.linspace(0, self.a_n/self.b_n*3, 100) 49 | Mu, Tau = np.meshgrid(mu_range, tau_range) 50 | 51 | # Variational distribution 52 | Z_var = np.exp(norm.logpdf(Mu, self.mu_n, 1/np.sqrt(self.lambda_n*Tau)) + 53 | gamma.logpdf(Tau, self.a_n, scale=1/self.b_n)) 54 | 55 | # True posterior 56 | Z_true = true_posterior(Mu, Tau, self.data, self.mu0, self.lambda0, self.a0, self.b0) 57 | 58 | # Plot both distributions 59 | ax_joint.contourf(Mu, Tau, Z_var, levels=20, alpha=0.5, cmap='Blues') 60 | ax_joint.contour(Mu, Tau, Z_true, levels=20, colors='r', alpha=0.5) 61 | ax_joint.set_xlabel('μ') 62 | ax_joint.set_ylabel('τ') 63 | ax_joint.set_title('q(μ,τ) (blue) vs True Posterior (red)') 64 | 65 | # Plot q(μ) 66 | ax_marg_x.plot(mu_range, norm.pdf(mu_range, self.mu_n, 1/np.sqrt(self.lambda_n * self.a_n/self.b_n)), 'b-', label='q(μ)') 67 | ax_marg_x.plot(mu_range, norm.pdf(mu_range, self.mu_n, 1/np.sqrt(self.lambda_n * (self.a_n-1)/self.b_n)), 'r--', label='True p(μ)') 68 | ax_marg_x.set_xlabel('μ') 69 | ax_marg_x.set_ylabel('Density') 70 | ax_marg_x.set_title('q(μ) vs True p(μ)') 71 | ax_marg_x.legend() 72 | 73 | # Plot q(τ) 74 | ax_marg_y.plot(gamma.pdf(tau_range, self.a_n, scale=1/self.b_n), tau_range, 'b-', label='q(τ)') 75 | ax_marg_y.plot(gamma.pdf(tau_range, self.a_n-1, scale=1/self.b_n), tau_range, 'r--', label='True p(τ)') 76 | ax_marg_y.set_xlabel('Density') 77 | ax_marg_y.set_ylabel('τ') 78 | ax_marg_y.set_title('q(τ) vs True p(τ)') 79 | ax_marg_y.legend() 80 | 81 | # Generate some example data 82 | np.random.seed(0) 83 | true_mu, true_tau = 1, 2 84 | data = np.random.normal(true_mu, 1/np.sqrt(true_tau), 5) 85 | 86 | # Initialize the variational inference object 87 | vi = VariationalInferenceNormalGamma(data, mu0=0, lambda0=1, a0=1, b0=1) 88 | 89 | # Set up the plot 90 | fig = plt.figure(figsize=(12, 8)) 91 | gs = fig.add_gridspec(2, 2, width_ratios=(4, 1), height_ratios=(1, 4), 92 | left=0.1, right=0.9, bottom=0.1, top=0.9, 93 | wspace=0.05, hspace=0.05) 94 | 95 | ax_joint = fig.add_subplot(gs[1, 0]) 96 | ax_marg_x = fig.add_subplot(gs[0, 0], sharex=ax_joint) 97 | ax_marg_y = fig.add_subplot(gs[1, 1], sharey=ax_joint) 98 | 99 | plt.setp(ax_marg_x.get_xticklabels(), visible=False) 100 | plt.setp(ax_marg_y.get_yticklabels(), visible=False) 101 | 102 | # Create a button for updating 103 | ax_button = plt.axes([0.4, 0.02, 0.2, 0.075]) 104 | button = Button(ax_button, 'Update') 105 | 106 | # Define the update function 107 | def update(event): 108 | vi.update() 109 | ax_joint.clear() 110 | ax_marg_x.clear() 111 | ax_marg_y.clear() 112 | vi.plot(ax_joint, ax_marg_x, ax_marg_y) 113 | fig.canvas.draw() 114 | 115 | button.on_clicked(update) 116 | 117 | # Initial plot 118 | vi.plot(ax_joint, ax_marg_x, ax_marg_y) 119 | 120 | plt.show() --------------------------------------------------------------------------------