├── LA4DS_TOC.pdf ├── LA4DS_ch02.ipynb ├── LA4DS_ch03.ipynb ├── LA4DS_ch04.ipynb ├── LA4DS_ch05.ipynb ├── LA4DS_ch06.ipynb ├── LA4DS_ch07.ipynb ├── LA4DS_ch08.ipynb ├── LA4DS_ch09.ipynb ├── LA4DS_ch10.ipynb ├── LA4DS_ch11.ipynb ├── LA4DS_ch12.ipynb ├── LA4DS_ch13.ipynb ├── LA4DS_ch14.ipynb ├── LA4DS_ch15.ipynb ├── README.md └── pyFiles ├── LA4DS_ch02.py ├── LA4DS_ch03.py ├── LA4DS_ch04.py ├── LA4DS_ch05.py ├── LA4DS_ch06.py ├── LA4DS_ch07.py ├── LA4DS_ch08.py ├── LA4DS_ch09.py ├── LA4DS_ch10.py ├── LA4DS_ch11.py ├── LA4DS_ch12.py ├── LA4DS_ch13.py ├── LA4DS_ch14.py └── LA4DS_ch15.py /LA4DS_TOC.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikexcohen/LinAlg4DataScience/4155de62076e3d826ef4020741fe822c4bff4cfa/LA4DS_TOC.pdf -------------------------------------------------------------------------------- /LA4DS_ch04.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "LA4DS_ch04.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "mount_file_id": "1CNEC5tpXXVD9_kPZZ72mm-uy6dwQQfhF", 10 | "authorship_tag": "ABX9TyOCDwyXDHmxkg4K+oZIczNv" 11 | }, 12 | "kernelspec": { 13 | "name": "python3", 14 | "display_name": "Python 3" 15 | }, 16 | "language_info": { 17 | "name": "python" 18 | } 19 | }, 20 | "cells": [ 21 | { 22 | "cell_type": "markdown", 23 | "source": [ 24 | "# Practical Linear Algebra for Data Science\n", 25 | "## Mike X Cohen (sincxpress.com)\n", 26 | "### https://www.oreilly.com/library/view/practical-linear-algebra/9781098120603/\n", 27 | "\n", 28 | "#### Code for chapter 4" 29 | ], 30 | "metadata": { 31 | "id": "SbGFWGzkd44U" 32 | } 33 | }, 34 | { 35 | "cell_type": "code", 36 | "source": [ 37 | "import numpy as np\n", 38 | "import matplotlib.pyplot as plt\n", 39 | "\n", 40 | "# NOTE: these lines define global figure properties used for publication.\n", 41 | "import matplotlib_inline.backend_inline\n", 42 | "matplotlib_inline.backend_inline.set_matplotlib_formats('svg') # print figures in svg format\n", 43 | "plt.rcParams.update({'font.size':14}) # set global font size" 44 | ], 45 | "metadata": { 46 | "id": "IvinJbZP_CLv" 47 | }, 48 | "execution_count": null, 49 | "outputs": [] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "source": [ 54 | "\n", 55 | "N = 30\n", 56 | "\n", 57 | "# correlated random variables\n", 58 | "x = np.linspace(0,10,N) + np.random.randn(N)\n", 59 | "y = x + np.random.randn(N)\n", 60 | "\n", 61 | "\n", 62 | "# set up figure\n", 63 | "_,axs = plt.subplots(2,2,figsize=(6,6))\n", 64 | "\n", 65 | "# positive correlation\n", 66 | "axs[0,0].plot(x,y,'ko')\n", 67 | "axs[0,0].set_title('Positive correlation',fontweight='bold')\n", 68 | "axs[0,0].set_xlabel('Variable x')\n", 69 | "axs[0,0].set_ylabel('Variable y')\n", 70 | "axs[0,0].set_xticks([])\n", 71 | "axs[0,0].set_yticks([])\n", 72 | "axs[0,0].axis('square')\n", 73 | "\n", 74 | "\n", 75 | "# negative correlation\n", 76 | "axs[0,1].plot(x,-y,'ko')\n", 77 | "axs[0,1].set_title('Negative correlation',fontweight='bold')\n", 78 | "axs[0,1].set_xlabel('Variable x')\n", 79 | "axs[0,1].set_ylabel('Variable y')\n", 80 | "axs[0,1].set_xticks([])\n", 81 | "axs[0,1].set_yticks([])\n", 82 | "axs[0,1].axis('square')\n", 83 | "\n", 84 | "\n", 85 | "# zero correlation, part 1\n", 86 | "axs[1,0].plot(np.random.randn(N),np.random.randn(N),'ko')\n", 87 | "axs[1,0].set_title('Zero correlation',fontweight='bold')\n", 88 | "axs[1,0].set_xlabel('Variable x')\n", 89 | "axs[1,0].set_ylabel('Variable y')\n", 90 | "axs[1,0].set_xticks([])\n", 91 | "axs[1,0].set_yticks([])\n", 92 | "axs[1,0].axis('square')\n", 93 | "\n", 94 | "\n", 95 | "# zero correlation, part 2\n", 96 | "x = np.cos(np.linspace(0,2*np.pi,N)) + np.random.randn(N)/20\n", 97 | "y = np.sin(np.linspace(0,2*np.pi,N)) + np.random.randn(N)/20\n", 98 | "axs[1,1].plot(x,y,'ko')\n", 99 | "axs[1,1].set_title('Zero correlation',fontweight='bold')\n", 100 | "axs[1,1].set_xlabel('Variable x')\n", 101 | "axs[1,1].set_ylabel('Variable y')\n", 102 | "axs[1,1].set_xticks([])\n", 103 | "axs[1,1].set_yticks([])\n", 104 | "axs[1,1].axis('square')\n", 105 | "\n", 106 | "\n", 107 | "plt.tight_layout()\n", 108 | "plt.savefig('Figure_04_01.png',dpi=300) # write out the fig to a file\n", 109 | "plt.show()" 110 | ], 111 | "metadata": { 112 | "id": "OU4BBQmyUdOE" 113 | }, 114 | "execution_count": null, 115 | "outputs": [] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "source": [ 120 | "" 121 | ], 122 | "metadata": { 123 | "id": "Vg5BHVwt0gQG" 124 | }, 125 | "execution_count": null, 126 | "outputs": [] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "source": [ 131 | "### Note: The code for k-means is in Exercise 7 below." 132 | ], 133 | "metadata": { 134 | "id": "-puIEx1q0gtR" 135 | }, 136 | "execution_count": null, 137 | "outputs": [] 138 | }, 139 | { 140 | "cell_type": "code", 141 | "source": [ 142 | "" 143 | ], 144 | "metadata": { 145 | "id": "klzkPQ8D0gwc" 146 | }, 147 | "execution_count": null, 148 | "outputs": [] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "source": [ 153 | "# Exercise 1" 154 | ], 155 | "metadata": { 156 | "id": "nuvWclOc0g1J" 157 | } 158 | }, 159 | { 160 | "cell_type": "code", 161 | "source": [ 162 | "# the function\n", 163 | "def corrAndCosine(x,y):\n", 164 | "\n", 165 | " # compute cosine similarity\n", 166 | " num = np.dot(x,y) # numerator\n", 167 | " den = np.linalg.norm(x) * np.linalg.norm(y) # denominator\n", 168 | " cos = num / den\n", 169 | "\n", 170 | " # compute correlation (similar to above but mean-centered!)\n", 171 | " xm = x-np.mean(x)\n", 172 | " ym = y-np.mean(y)\n", 173 | " num = np.dot(xm,ym) # numerator\n", 174 | " den = np.linalg.norm(xm) * np.linalg.norm(ym) # denominator\n", 175 | " cor = num / den\n", 176 | "\n", 177 | " return cor,cos\n", 178 | "\n", 179 | "\n", 180 | "# test it\n", 181 | "a = np.random.randn(15)\n", 182 | "b = np.random.randn(15)\n", 183 | "\n", 184 | "# compute the correlation and cosine\n", 185 | "r,c = corrAndCosine(a,b)\n", 186 | "\n", 187 | "# confirm that the correlation matches with np.corrcoef\n", 188 | "print(r,np.corrcoef(a,b)[0,1])" 189 | ], 190 | "metadata": { 191 | "id": "l9cyAOCn_lt-" 192 | }, 193 | "execution_count": null, 194 | "outputs": [] 195 | }, 196 | { 197 | "cell_type": "code", 198 | "source": [ 199 | "# compare r and c without mean-centering\n", 200 | "a = np.random.randn(15) + 10 # note the offset!\n", 201 | "b = np.random.randn(15)\n", 202 | "\n", 203 | "# mean-center\n", 204 | "aNoMean = a - np.mean(a)\n", 205 | "bNoMean = b - np.mean(b)\n", 206 | "\n", 207 | "\n", 208 | "# show the results with and without mean-centering\n", 209 | "print('Without mean-centering (should differ):')\n", 210 | "print( np.round(corrAndCosine(a,b),4) )\n", 211 | "print(' ')\n", 212 | "\n", 213 | "print('With mean-centering (should be the same):')\n", 214 | "print( np.round(corrAndCosine(aNoMean,bNoMean),4) )\n", 215 | "\n", 216 | "# NOTE: In the printing code above, I rounded to 4 significant digits just for visual clarity." 217 | ], 218 | "metadata": { 219 | "id": "TCHVd-TBCOc-" 220 | }, 221 | "execution_count": null, 222 | "outputs": [] 223 | }, 224 | { 225 | "cell_type": "markdown", 226 | "source": [ 227 | "## Exercise 2" 228 | ], 229 | "metadata": { 230 | "id": "XfDHd5pgIG6I" 231 | } 232 | }, 233 | { 234 | "cell_type": "code", 235 | "source": [ 236 | "# create the variables\n", 237 | "a = np.arange(4,dtype=float)\n", 238 | "offsets = np.arange(-50,51)\n", 239 | "\n", 240 | "# initialize the results\n", 241 | "results = np.zeros((len(offsets),2))\n", 242 | "\n", 243 | "# run the simulation!\n", 244 | "for i in range(len(offsets)):\n", 245 | " results[i,:] = corrAndCosine(a,a+offsets[i])\n", 246 | "\n", 247 | "\n", 248 | "# plot the results!\n", 249 | "plt.figure(figsize=(8,4))\n", 250 | "h = plt.plot(offsets,results)\n", 251 | "h[0].set_color('k')\n", 252 | "h[0].set_marker('o')\n", 253 | "h[1].set_color([.7,.7,.7])\n", 254 | "h[1].set_marker('s')\n", 255 | "\n", 256 | "plt.xlabel('Mean offset')\n", 257 | "plt.ylabel('r or c')\n", 258 | "plt.legend(['Pearson','Cosine sim.'])\n", 259 | "plt.savefig('Figure_04_02.png',dpi=300) # write out the fig to a file\n", 260 | "plt.show()" 261 | ], 262 | "metadata": { 263 | "id": "GL26OrcAp6Hm" 264 | }, 265 | "execution_count": null, 266 | "outputs": [] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "source": [ 271 | "" 272 | ], 273 | "metadata": { 274 | "id": "L5YGtmh4q1G6" 275 | }, 276 | "execution_count": null, 277 | "outputs": [] 278 | }, 279 | { 280 | "cell_type": "markdown", 281 | "source": [ 282 | "## Exercise 3" 283 | ], 284 | "metadata": { 285 | "id": "VrJ4GQmzI0RU" 286 | } 287 | }, 288 | { 289 | "cell_type": "code", 290 | "source": [ 291 | "# import the function\n", 292 | "from scipy.stats import pearsonr\n", 293 | "\n", 294 | "# inspect the source code\n", 295 | "??pearsonr" 296 | ], 297 | "metadata": { 298 | "id": "cHjKrhM1DOdD" 299 | }, 300 | "execution_count": null, 301 | "outputs": [] 302 | }, 303 | { 304 | "cell_type": "code", 305 | "source": [ 306 | "" 307 | ], 308 | "metadata": { 309 | "id": "ZXMHlxZu00Fs" 310 | }, 311 | "execution_count": null, 312 | "outputs": [] 313 | }, 314 | { 315 | "cell_type": "markdown", 316 | "source": [ 317 | "## Exercise 4" 318 | ], 319 | "metadata": { 320 | "id": "-mp0Tn-6Mx7E" 321 | } 322 | }, 323 | { 324 | "cell_type": "code", 325 | "metadata": { 326 | "id": "oo6WdWnsdcne" 327 | }, 328 | "source": [ 329 | "# a bare-bones correlation function\n", 330 | "def rho(x,y):\n", 331 | " xm = x-np.mean(x)\n", 332 | " ym = y-np.mean(y)\n", 333 | " n = np.dot(xm,ym)\n", 334 | " d = np.linalg.norm(xm) * np.linalg.norm(ym)\n", 335 | " return n/d\n", 336 | "\n", 337 | "\n", 338 | "# import the time library\n", 339 | "import time\n", 340 | "\n", 341 | "# experiment parameters\n", 342 | "numIters = 1000\n", 343 | "varLength = 500\n", 344 | "\n", 345 | "# clock my custom-written function\n", 346 | "tic = time.time()\n", 347 | "for i in range(numIters):\n", 348 | " x = np.random.randn(varLength,2)\n", 349 | " rho(x[:,0],x[:,1])\n", 350 | "t1 = time.time() - tic\n", 351 | "\n", 352 | "\n", 353 | "# now for numpy's corrcoef function\n", 354 | "tic = time.time()\n", 355 | "for i in range(numIters):\n", 356 | " x = np.random.randn(varLength,2)\n", 357 | " pearsonr(x[:,0],x[:,1])\n", 358 | "t2 = time.time() - tic\n", 359 | "\n", 360 | "\n", 361 | "# print the results!\n", 362 | "# Note: time() returns seconds, so I multiply by 1000 for ms\n", 363 | "print(f'My function took {t1*1000:.2f} ms')\n", 364 | "print(f' pearsonr took {t2*1000:.2f} ms')" 365 | ], 366 | "execution_count": null, 367 | "outputs": [] 368 | }, 369 | { 370 | "cell_type": "code", 371 | "source": [ 372 | "" 373 | ], 374 | "metadata": { 375 | "id": "gy0-joJm03P4" 376 | }, 377 | "execution_count": null, 378 | "outputs": [] 379 | }, 380 | { 381 | "cell_type": "markdown", 382 | "source": [ 383 | "# Exercise 5" 384 | ], 385 | "metadata": { 386 | "id": "bvHwvToaN1tv" 387 | } 388 | }, 389 | { 390 | "cell_type": "code", 391 | "source": [ 392 | "# create the kernel (in the book figure I used +1.5)\n", 393 | "kernel = np.array([-1,1])\n", 394 | "\n", 395 | "# and the \"signal\" (a plateau)\n", 396 | "signal = np.zeros(30)\n", 397 | "signal[10:20] = 1\n", 398 | "\n", 399 | "\n", 400 | "# plot them\n", 401 | "_,axs = plt.subplots(1,2,figsize=(12,4))\n", 402 | "axs[0].plot(kernel,'ks-')\n", 403 | "axs[0].set_title('Kernel')\n", 404 | "axs[0].set_xlim([-15,15])\n", 405 | "\n", 406 | "axs[1].plot(signal,'ks-')\n", 407 | "axs[1].set_title('Time series signal')\n", 408 | "\n", 409 | "plt.savefig('Figure_04_04ab.png',dpi=300)\n", 410 | "plt.show()" 411 | ], 412 | "metadata": { 413 | "id": "hF-G_i8IN3uU" 414 | }, 415 | "execution_count": null, 416 | "outputs": [] 417 | }, 418 | { 419 | "cell_type": "code", 420 | "source": [ 421 | "# initialize the feature map as zeros\n", 422 | "featureMap = np.zeros(len(signal))\n", 423 | "\n", 424 | "# loop over the signal and do template-matching (via dot products!)\n", 425 | "for t in range(1,len(signal)-1):\n", 426 | " featureMap[t] = np.dot(kernel,signal[t-1:t+1])\n", 427 | "\n", 428 | "\n", 429 | "# plot the result\n", 430 | "_,axs = plt.subplots(1,2,figsize=(12,4))\n", 431 | "axs[0].plot(kernel,'ks-')\n", 432 | "axs[0].set_title('Kernel')\n", 433 | "axs[0].set_xlim([-15,15])\n", 434 | "\n", 435 | "\n", 436 | "axs[1].plot(signal,'ks-',label='Signal',linewidth=3)\n", 437 | "markers,stemlines,_ = axs[1].stem(range(len(featureMap)),featureMap,\n", 438 | " basefmt=' ',linefmt='',markerfmt='o',\n", 439 | " label='Edge detection')\n", 440 | "\n", 441 | "plt.setp(stemlines,'color',[.7,.7,.7])\n", 442 | "plt.setp(markers,'color',[.7,.7,.7])\n", 443 | "\n", 444 | "axs[1].legend()\n", 445 | "plt.savefig('Figure_04_04c.png',dpi=300)\n", 446 | "plt.show()" 447 | ], 448 | "metadata": { 449 | "id": "VU4mT29IN2EE" 450 | }, 451 | "execution_count": null, 452 | "outputs": [] 453 | }, 454 | { 455 | "cell_type": "code", 456 | "source": [ 457 | "" 458 | ], 459 | "metadata": { 460 | "id": "FHS9GLbFN2Hd" 461 | }, 462 | "execution_count": null, 463 | "outputs": [] 464 | }, 465 | { 466 | "cell_type": "markdown", 467 | "source": [ 468 | "# Exercise 6" 469 | ], 470 | "metadata": { 471 | "id": "MboACnUwN2J-" 472 | } 473 | }, 474 | { 475 | "cell_type": "code", 476 | "source": [ 477 | "# define the kernel (a sorta-kinda Gaussian)\n", 478 | "kernel = np.array([0,.1,.3,.8,1,.8,.3,.1,0])\n", 479 | "kernel = kernel / np.sum(kernel)\n", 480 | "\n", 481 | "# some handy length parameters\n", 482 | "Nkernel = len(kernel)\n", 483 | "halfKrn = Nkernel//2\n", 484 | "\n", 485 | "\n", 486 | "# and the signal\n", 487 | "Nsignal = 100\n", 488 | "timeseries = np.random.randn(Nsignal)\n", 489 | "\n", 490 | "\n", 491 | "# plot them\n", 492 | "_,axs = plt.subplots(1,2,figsize=(12,4))\n", 493 | "axs[0].plot(kernel,'ks-')\n", 494 | "axs[0].set_title('Kernel')\n", 495 | "axs[0].set_xlim([-1,Nsignal])\n", 496 | "\n", 497 | "axs[1].plot(timeseries,'ks-')\n", 498 | "axs[1].set_title('Time series signal')\n", 499 | "\n", 500 | "plt.savefig('Figure_04_06ab.png',dpi=300)\n", 501 | "plt.show()" 502 | ], 503 | "metadata": { 504 | "id": "LyLAL7L0RBDt" 505 | }, 506 | "execution_count": null, 507 | "outputs": [] 508 | }, 509 | { 510 | "cell_type": "code", 511 | "source": [ 512 | "# make a copy of the signal for filtering\n", 513 | "filtsig = timeseries.copy()\n", 514 | "\n", 515 | "# loop over the signal time points\n", 516 | "for t in range(halfKrn+1,Nsignal-halfKrn):\n", 517 | " filtsig[t] = np.dot(kernel,timeseries[t-halfKrn-1:t+halfKrn])\n", 518 | "\n", 519 | "\n", 520 | "# and plot\n", 521 | "_,axs = plt.subplots(1,2,figsize=(12,4))\n", 522 | "axs[0].plot(kernel,'ks-')\n", 523 | "axs[0].set_title('Kernel')\n", 524 | "axs[0].set_xlim([-1,Nsignal])\n", 525 | "\n", 526 | "axs[1].plot(timeseries,color='k',label='Original',linewidth=1)\n", 527 | "axs[1].plot(filtsig,'--',color=[.6,.6,.6],label='Smoothed',linewidth=2)\n", 528 | "axs[1].legend()\n", 529 | "\n", 530 | "plt.savefig('Figure_04_06c.png',dpi=300)\n", 531 | "plt.show()" 532 | ], 533 | "metadata": { 534 | "id": "1HDFchK6RBGm" 535 | }, 536 | "execution_count": null, 537 | "outputs": [] 538 | }, 539 | { 540 | "cell_type": "markdown", 541 | "source": [ 542 | "# Exercise 7" 543 | ], 544 | "metadata": { 545 | "id": "fpe9_N2_RBJZ" 546 | } 547 | }, 548 | { 549 | "cell_type": "code", 550 | "source": [ 551 | "# define the kernel (a sorta-kinda Gaussian)\n", 552 | "kernel = np.array([0,.1,.3,.8,-1,.8,.3,.1,0])\n", 553 | "kernel /= np.sum(kernel)\n", 554 | "kernel -= np.mean(kernel)\n", 555 | "\n", 556 | "# plot them\n", 557 | "_,axs = plt.subplots(1,2,figsize=(12,4))\n", 558 | "axs[0].plot(kernel,'s-')\n", 559 | "axs[0].set_title('Kernel')\n", 560 | "axs[0].set_xlim([-1,Nsignal])\n", 561 | "\n", 562 | "axs[1].plot(timeseries,'s-')\n", 563 | "axs[1].set_title('Time series signal')\n", 564 | "plt.show()\n", 565 | "\n", 566 | "\n", 567 | "\n", 568 | "# loop over the signal time points\n", 569 | "filtsig2 = timeseries.copy()\n", 570 | "for t in range(halfKrn+1,Nsignal-halfKrn):\n", 571 | " filtsig2[t] = np.dot(kernel,timeseries[t-halfKrn-1:t+halfKrn])\n", 572 | "\n", 573 | "plt.plot(timeseries,color='k',label='Original',linewidth=1)\n", 574 | "plt.plot(filtsig2,color=[.9,.2,.7],label='Sharpened',linewidth=1)\n", 575 | "plt.legend()\n", 576 | "plt.show()" 577 | ], 578 | "metadata": { 579 | "id": "poWxOslmcbAz" 580 | }, 581 | "execution_count": null, 582 | "outputs": [] 583 | }, 584 | { 585 | "cell_type": "code", 586 | "source": [ 587 | "" 588 | ], 589 | "metadata": { 590 | "id": "t9Ud0MCwcbGf" 591 | }, 592 | "execution_count": null, 593 | "outputs": [] 594 | }, 595 | { 596 | "cell_type": "markdown", 597 | "source": [ 598 | "# Exercise 8" 599 | ], 600 | "metadata": { 601 | "id": "7w7jf9jXcbPL" 602 | } 603 | }, 604 | { 605 | "cell_type": "code", 606 | "source": [ 607 | "## Create data\n", 608 | "nPerClust = 50\n", 609 | "\n", 610 | "# blur around centroid (std units)\n", 611 | "blur = 1\n", 612 | "\n", 613 | "# XY centroid locations\n", 614 | "A = [ 1, 1 ]\n", 615 | "B = [ -3, 1 ]\n", 616 | "C = [ 3, 3 ]\n", 617 | "\n", 618 | "# generate data\n", 619 | "a = [ A[0]+np.random.randn(nPerClust)*blur , A[1]+np.random.randn(nPerClust)*blur ]\n", 620 | "b = [ B[0]+np.random.randn(nPerClust)*blur , B[1]+np.random.randn(nPerClust)*blur ]\n", 621 | "c = [ C[0]+np.random.randn(nPerClust)*blur , C[1]+np.random.randn(nPerClust)*blur ]\n", 622 | "\n", 623 | "# concatanate into a matrix\n", 624 | "data = np.transpose( np.concatenate((a,b,c),axis=1) )\n", 625 | "\n", 626 | "\n", 627 | "# plot data\n", 628 | "plt.plot(data[:,0],data[:,1],'ko',markerfacecolor='w')\n", 629 | "plt.title('Raw (preclustered) data')\n", 630 | "plt.xticks([])\n", 631 | "plt.yticks([])\n", 632 | "\n", 633 | "plt.show()" 634 | ], 635 | "metadata": { 636 | "id": "z5hEt8nigBL1" 637 | }, 638 | "execution_count": null, 639 | "outputs": [] 640 | }, 641 | { 642 | "cell_type": "code", 643 | "source": [ 644 | "## initialize random cluster centroids\n", 645 | "k = 3 # extract three clusters\n", 646 | "\n", 647 | "# random cluster centers (randomly sampled data points)\n", 648 | "ridx = np.random.choice(range(len(data)),k,replace=False)\n", 649 | "centroids = data[ridx,:]\n", 650 | "\n", 651 | "\n", 652 | "# setup the figure\n", 653 | "fig,axs = plt.subplots(2,2,figsize=(6,6))\n", 654 | "axs = axs.flatten()\n", 655 | "lineColors = [ [0,0,0],[.4,.4,.4],[.8,.8,.8] ]#'rbm'\n", 656 | "\n", 657 | "\n", 658 | "# plot data with initial random cluster centroids\n", 659 | "axs[0].plot(data[:,0],data[:,1],'ko',markerfacecolor='w')\n", 660 | "axs[0].plot(centroids[:,0],centroids[:,1],'ko')\n", 661 | "axs[0].set_title('Iteration 0')\n", 662 | "axs[0].set_xticks([])\n", 663 | "axs[0].set_yticks([])\n", 664 | "\n", 665 | "\n", 666 | "\n", 667 | "# loop over iterations\n", 668 | "for iteri in range(3):\n", 669 | " \n", 670 | " # step 1: compute distances\n", 671 | " dists = np.zeros((data.shape[0],k))\n", 672 | " for ci in range(k):\n", 673 | " dists[:,ci] = np.sum((data-centroids[ci,:])**2,axis=1)\n", 674 | " \n", 675 | " # step 2: assign to group based on minimum distance\n", 676 | " groupidx = np.argmin(dists,axis=1)\n", 677 | " \n", 678 | " # step 3: recompute centers\n", 679 | " for ki in range(k):\n", 680 | " centroids[ki,:] = [ np.mean(data[groupidx==ki,0]), np.mean(data[groupidx==ki,1]) ]\n", 681 | " \n", 682 | "\n", 683 | " # plot data points\n", 684 | " for i in range(len(data)):\n", 685 | " axs[iteri+1].plot([ data[i,0],centroids[groupidx[i],0] ],[ data[i,1],centroids[groupidx[i],1] ],color=lineColors[groupidx[i]])\n", 686 | " axs[iteri+1].plot(centroids[:,0],centroids[:,1],'ko')\n", 687 | " axs[iteri+1].set_title(f'Iteration {iteri+1}')\n", 688 | " axs[iteri+1].set_xticks([])\n", 689 | " axs[iteri+1].set_yticks([])\n", 690 | "\n", 691 | "\n", 692 | "plt.savefig('Figure_04_03.png',dpi=300)\n", 693 | "plt.show()" 694 | ], 695 | "metadata": { 696 | "id": "2nKemxTUgHMM" 697 | }, 698 | "execution_count": null, 699 | "outputs": [] 700 | }, 701 | { 702 | "cell_type": "code", 703 | "source": [ 704 | "" 705 | ], 706 | "metadata": { 707 | "id": "kUMVrvJ2gBPY" 708 | }, 709 | "execution_count": null, 710 | "outputs": [] 711 | } 712 | ] 713 | } 714 | -------------------------------------------------------------------------------- /LA4DS_ch09.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "LA4DS_ch09.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "authorship_tag": "ABX9TyNJwYt3gWKX2Ggt6ivOaiDi" 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | } 18 | }, 19 | "cells": [ 20 | { 21 | "cell_type": "markdown", 22 | "source": [ 23 | "# Practical Linear Algebra for Data Science\n", 24 | "## Mike X Cohen (sincxpress.com)\n", 25 | "### https://www.oreilly.com/library/view/practical-linear-algebra/9781098120603/\n", 26 | "\n", 27 | "#### Code for chapter 9" 28 | ], 29 | "metadata": { 30 | "id": "SbGFWGzkd44U" 31 | } 32 | }, 33 | { 34 | "cell_type": "code", 35 | "source": [ 36 | "import numpy as np\n", 37 | "import matplotlib.pyplot as plt\n", 38 | "\n", 39 | "# used to create non-regular subplots\n", 40 | "import matplotlib.gridspec as gridspec\n", 41 | "\n", 42 | "\n", 43 | "# NOTE: these lines define global figure properties used for publication.\n", 44 | "import matplotlib_inline.backend_inline\n", 45 | "matplotlib_inline.backend_inline.set_matplotlib_formats('svg') # display figures in vector format\n", 46 | "plt.rcParams.update({'font.size':14}) # set global font size" 47 | ], 48 | "metadata": { 49 | "id": "GVWZlfT-nThD" 50 | }, 51 | "execution_count": null, 52 | "outputs": [] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "source": [ 57 | "" 58 | ], 59 | "metadata": { 60 | "id": "_lzwhI-9nY7O" 61 | }, 62 | "execution_count": null, 63 | "outputs": [] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "source": [ 68 | "# Orthogonal matrices" 69 | ], 70 | "metadata": { 71 | "id": "ng1qvfR9nY9y" 72 | } 73 | }, 74 | { 75 | "cell_type": "code", 76 | "source": [ 77 | "# specify the matrices\n", 78 | "Q1 = np.array([ [1,-1],[1,1] ]) / np.sqrt(2)\n", 79 | "Q2 = np.array([ [1,2,2],[2,1,-2],[-2,2,-1] ]) / 3\n", 80 | "\n", 81 | "# should be I (to within rounding error...)\n", 82 | "print( np.round(Q1.T @ Q1,8) ), print(' ')\n", 83 | "\n", 84 | "print( np.round(Q2.T @ Q2,8) )" 85 | ], 86 | "metadata": { 87 | "id": "S9Aw-h1hncgn" 88 | }, 89 | "execution_count": null, 90 | "outputs": [] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "source": [ 95 | "" 96 | ], 97 | "metadata": { 98 | "id": "cE0U9IyUncjH" 99 | }, 100 | "execution_count": null, 101 | "outputs": [] 102 | }, 103 | { 104 | "cell_type": "markdown", 105 | "source": [ 106 | "# QR decomposition" 107 | ], 108 | "metadata": { 109 | "id": "UlONi5SqnclZ" 110 | } 111 | }, 112 | { 113 | "cell_type": "code", 114 | "source": [ 115 | "# create a random matrix\n", 116 | "A = np.random.randn(6,6)\n", 117 | "\n", 118 | "# QR decomposition\n", 119 | "Q,R = np.linalg.qr(A)\n", 120 | "\n", 121 | "\n", 122 | "\n", 123 | "# show the matrices\n", 124 | "fig = plt.figure(figsize=(10,6))\n", 125 | "axs = [0]*5\n", 126 | "c = 1.5 # color limits\n", 127 | "\n", 128 | "gs1 = gridspec.GridSpec(2,6)\n", 129 | "axs[0] = plt.subplot(gs1[0,:2])\n", 130 | "axs[0].imshow(A,vmin=-c,vmax=c,cmap='gray')\n", 131 | "axs[0].set_title('A',fontweight='bold')\n", 132 | "\n", 133 | "axs[1] = plt.subplot(gs1[0,2:4])\n", 134 | "axs[1].imshow(Q,vmin=-c,vmax=c,cmap='gray')\n", 135 | "axs[1].set_title('Q',fontweight='bold')\n", 136 | "\n", 137 | "axs[2] = plt.subplot(gs1[0,4:6])\n", 138 | "axs[2].imshow(R,vmin=-c,vmax=c,cmap='gray')\n", 139 | "axs[2].set_title('R',fontweight='bold')\n", 140 | "\n", 141 | "axs[3] = plt.subplot(gs1[1,1:3])\n", 142 | "axs[3].imshow(A - Q@R,vmin=-c,vmax=c,cmap='gray')\n", 143 | "axs[3].set_title('A - QR',fontweight='bold')\n", 144 | "\n", 145 | "axs[4] = plt.subplot(gs1[1,3:5])\n", 146 | "axs[4].imshow(Q.T@Q,cmap='gray')\n", 147 | "axs[4].set_title(r'Q$^T$Q',fontweight='bold')\n", 148 | "\n", 149 | "# remove ticks from all axes\n", 150 | "for a in axs:\n", 151 | " a.set_xticks([])\n", 152 | " a.set_yticks([])\n", 153 | "\n", 154 | "plt.tight_layout()\n", 155 | "plt.savefig('Figure_09_01.png',dpi=300)\n", 156 | "plt.show()" 157 | ], 158 | "metadata": { 159 | "id": "XiirvzAbxDIM" 160 | }, 161 | "execution_count": null, 162 | "outputs": [] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "source": [ 167 | "" 168 | ], 169 | "metadata": { 170 | "id": "FijwWkd6xDLW" 171 | }, 172 | "execution_count": null, 173 | "outputs": [] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "source": [ 178 | "# QR and matrix sizes\n", 179 | "\n", 180 | "M = 4\n", 181 | "N = 14\n", 182 | "\n", 183 | "A = np.random.randn(M,N)\n", 184 | "Q,R = np.linalg.qr(A)\n", 185 | "\n", 186 | "# print the results\n", 187 | "print(f'Size of A (M,N): {A.shape}')\n", 188 | "print(f'Size of Q (M,N): {Q.shape}')\n", 189 | "print(f'Size of R (M,N): {R.shape}')" 190 | ], 191 | "metadata": { 192 | "id": "qVYlldRKxDQM", 193 | "colab": { 194 | "base_uri": "https://localhost:8080/" 195 | }, 196 | "executionInfo": { 197 | "status": "ok", 198 | "timestamp": 1651655526791, 199 | "user_tz": -60, 200 | "elapsed": 281, 201 | "user": { 202 | "displayName": "Mike X Cohen", 203 | "userId": "13901636194183843661" 204 | } 205 | }, 206 | "outputId": "6c7eabf9-48b8-4f3e-e8fc-5d30a369ea7b" 207 | }, 208 | "execution_count": null, 209 | "outputs": [ 210 | { 211 | "output_type": "stream", 212 | "name": "stdout", 213 | "text": [ 214 | "Size of A (M,N): (4, 14)\n", 215 | "Size of Q (M,N): (4, 4)\n", 216 | "Size of R (M,N): (4, 14)\n" 217 | ] 218 | } 219 | ] 220 | }, 221 | { 222 | "cell_type": "code", 223 | "source": [ 224 | "# illustration of full Q from M>N A\n", 225 | "\n", 226 | "A = np.array([ [1,-1] ]).T\n", 227 | "\n", 228 | "Q,R = np.linalg.qr(A,'complete')\n", 229 | "Q*np.sqrt(2)" 230 | ], 231 | "metadata": { 232 | "id": "1VoFE25FnZAd" 233 | }, 234 | "execution_count": null, 235 | "outputs": [] 236 | }, 237 | { 238 | "cell_type": "code", 239 | "source": [ 240 | "" 241 | ], 242 | "metadata": { 243 | "id": "MvATXzy6gxLF" 244 | }, 245 | "execution_count": null, 246 | "outputs": [] 247 | }, 248 | { 249 | "cell_type": "code", 250 | "source": [ 251 | "" 252 | ], 253 | "metadata": { 254 | "id": "_zZV-8T16w_j" 255 | }, 256 | "execution_count": null, 257 | "outputs": [] 258 | }, 259 | { 260 | "cell_type": "markdown", 261 | "source": [ 262 | "# Exercise 1" 263 | ], 264 | "metadata": { 265 | "id": "pa6VmNmtgxC6" 266 | } 267 | }, 268 | { 269 | "cell_type": "code", 270 | "source": [ 271 | "# compute matrices\n", 272 | "Q = np.linalg.qr( np.random.randn(5,5) )[0]\n", 273 | "Qt = Q.T\n", 274 | "Qi = np.linalg.inv( Q )\n", 275 | "\n", 276 | "# QtQ\n", 277 | "print(np.round( Qt@Q,8 )), print(' ')\n", 278 | "\n", 279 | "# QQt\n", 280 | "print(np.round( Q@Qt,8 )), print(' ')\n", 281 | "\n", 282 | "# Q^-1 Q\n", 283 | "print(np.round( Qi@Q,8 )), print(' ')\n", 284 | "\n", 285 | "# QQ^-1\n", 286 | "print(np.round( Q@Qi,8 ))\n" 287 | ], 288 | "metadata": { 289 | "id": "2IeYpplUgyl6" 290 | }, 291 | "execution_count": null, 292 | "outputs": [] 293 | }, 294 | { 295 | "cell_type": "code", 296 | "source": [ 297 | "" 298 | ], 299 | "metadata": { 300 | "id": "FIL-5EB1gxAL" 301 | }, 302 | "execution_count": null, 303 | "outputs": [] 304 | }, 305 | { 306 | "cell_type": "markdown", 307 | "source": [ 308 | "# Exercise 2" 309 | ], 310 | "metadata": { 311 | "id": "aTOAHGupgw9E" 312 | } 313 | }, 314 | { 315 | "cell_type": "code", 316 | "source": [ 317 | "# create the matrix \n", 318 | "m = 4\n", 319 | "n = 4\n", 320 | "A = np.random.randn(m,n)\n", 321 | "\n", 322 | "# initialize\n", 323 | "Q = np.zeros((m,n))\n", 324 | "\n", 325 | "\n", 326 | "# the GS algo\n", 327 | "for i in range(n):\n", 328 | " \n", 329 | " # initialize\n", 330 | " Q[:,i] = A[:,i]\n", 331 | " \n", 332 | " # orthogonalize\n", 333 | " a = A[:,i] # convenience\n", 334 | " for j in range(i): # only to earlier cols\n", 335 | " q = Q[:,j] # convenience\n", 336 | " Q[:,i]=Q[:,i]-np.dot(a,q)/np.dot(q,q)*q\n", 337 | " \n", 338 | " # normalize\n", 339 | " Q[:,i] = Q[:,i] / np.linalg.norm(Q[:,i])\n", 340 | "\n", 341 | " \n", 342 | "# \"real\" QR decomposition for comparison\n", 343 | "Q2,R = np.linalg.qr(A)\n", 344 | "\n", 345 | "\n", 346 | "# note the possible sign differences.\n", 347 | "# seemingly non-zero columns will be 0 when adding\n", 348 | "print( np.round( Q-Q2 ,10) ), print(' ')\n", 349 | "print( np.round( Q+Q2 ,10) )" 350 | ], 351 | "metadata": { 352 | "id": "rDE2e0ykiFQF" 353 | }, 354 | "execution_count": null, 355 | "outputs": [] 356 | }, 357 | { 358 | "cell_type": "code", 359 | "source": [ 360 | "" 361 | ], 362 | "metadata": { 363 | "id": "oh6UxjEigw1p" 364 | }, 365 | "execution_count": null, 366 | "outputs": [] 367 | }, 368 | { 369 | "cell_type": "markdown", 370 | "source": [ 371 | "# Exercise 3" 372 | ], 373 | "metadata": { 374 | "id": "6RwxfkYygwrX" 375 | } 376 | }, 377 | { 378 | "cell_type": "code", 379 | "source": [ 380 | "# create an orthogonal matrix, call it U (to avoid confusing with Q)\n", 381 | "U = np.linalg.qr( np.random.randn(6,6) )[0]\n", 382 | "\n", 383 | "\n", 384 | "# part 2: modulate the column norms\n", 385 | "for i in range(U.shape[0]):\n", 386 | " U[:,i] = U[:,i]*(10+i)\n", 387 | "\n", 388 | "\n", 389 | "# part 3: Change one matrix element\n", 390 | "U[0,3] = 0 # this is q_{1,4}\n", 391 | "\n", 392 | "\n", 393 | "# QR decomp\n", 394 | "q,r = np.linalg.qr(U)\n", 395 | "\n", 396 | "# show R and, for part 2, Q'Q\n", 397 | "print( np.round(r,3) ), print(' ')\n", 398 | "# print( np.round(Q.T@Q,4))" 399 | ], 400 | "metadata": { 401 | "id": "VOo3ORO5nZDf" 402 | }, 403 | "execution_count": null, 404 | "outputs": [] 405 | }, 406 | { 407 | "cell_type": "code", 408 | "source": [ 409 | "" 410 | ], 411 | "metadata": { 412 | "id": "VyjtADNlnZGc" 413 | }, 414 | "execution_count": null, 415 | "outputs": [] 416 | }, 417 | { 418 | "cell_type": "markdown", 419 | "source": [ 420 | "# Exercise 4" 421 | ], 422 | "metadata": { 423 | "id": "1qVzJvY8nZI_" 424 | } 425 | }, 426 | { 427 | "cell_type": "code", 428 | "source": [ 429 | "# a function to compute the inverse\n", 430 | "def oldSchoolInv(A):\n", 431 | "\n", 432 | " # matrix size\n", 433 | " m = A.shape[0]\n", 434 | "\n", 435 | "\n", 436 | " # abort if non-square\n", 437 | " if not np.diff(A.shape)[0]==0:\n", 438 | " raise Exception('Matrix must be square.')\n", 439 | " \n", 440 | " # abort if singular\n", 441 | " if np.linalg.matrix_rank(A)tol] = 1/s[s>tol]\n", 519 | "\n", 520 | "# reconstruct\n", 521 | "S = np.zeros_like(A)\n", 522 | "np.fill_diagonal(S,sInv)\n", 523 | "Apinv = Vt.T @ S @ U.T\n", 524 | "\n", 525 | "# compare to pinv()\n", 526 | "ApinvNp = np.linalg.pinv(A)\n", 527 | "\n", 528 | "print(np.round( ApinvNp - Apinv ,5))" 529 | ], 530 | "metadata": { 531 | "id": "hk3pvwmhnAKm" 532 | }, 533 | "execution_count": null, 534 | "outputs": [] 535 | }, 536 | { 537 | "cell_type": "code", 538 | "source": [ 539 | "# check the source code for pinv\n", 540 | "??np.linalg.pinv" 541 | ], 542 | "metadata": { 543 | "id": "qJSkysn3nAM7" 544 | }, 545 | "execution_count": null, 546 | "outputs": [] 547 | }, 548 | { 549 | "cell_type": "code", 550 | "source": [ 551 | "" 552 | ], 553 | "metadata": { 554 | "id": "Sakx5KbTnAPJ" 555 | }, 556 | "execution_count": null, 557 | "outputs": [] 558 | }, 559 | { 560 | "cell_type": "markdown", 561 | "source": [ 562 | "# Exercise 7" 563 | ], 564 | "metadata": { 565 | "id": "TBj-JcJSY-9D" 566 | } 567 | }, 568 | { 569 | "cell_type": "code", 570 | "source": [ 571 | "# left-inverse\n", 572 | "A = np.random.randn(6,4)\n", 573 | "\n", 574 | "# explicit left inverse\n", 575 | "Linv = np.linalg.inv(A.T@A)@A.T\n", 576 | "\n", 577 | "# pinv\n", 578 | "Apinv = np.linalg.pinv(A)\n", 579 | "\n", 580 | "# compare\n", 581 | "print(np.round( Linv - Apinv ,5))" 582 | ], 583 | "metadata": { 584 | "id": "ytwaaeWqyg1q" 585 | }, 586 | "execution_count": null, 587 | "outputs": [] 588 | }, 589 | { 590 | "cell_type": "code", 591 | "source": [ 592 | "# right-inverse\n", 593 | "A = np.random.randn(4,6)\n", 594 | "\n", 595 | "# explicit right inverse\n", 596 | "Rinv = A.T@np.linalg.inv(A@A.T)\n", 597 | "\n", 598 | "# pinv\n", 599 | "Apinv = np.linalg.pinv(A)\n", 600 | "\n", 601 | "# compare\n", 602 | "print(np.round( Rinv - Apinv ,5))" 603 | ], 604 | "metadata": { 605 | "id": "hjxHGXdEyg4b" 606 | }, 607 | "execution_count": null, 608 | "outputs": [] 609 | }, 610 | { 611 | "cell_type": "code", 612 | "source": [ 613 | "" 614 | ], 615 | "metadata": { 616 | "id": "BbJ_reOPyg7K" 617 | }, 618 | "execution_count": null, 619 | "outputs": [] 620 | }, 621 | { 622 | "cell_type": "markdown", 623 | "source": [ 624 | "# Exercise 8" 625 | ], 626 | "metadata": { 627 | "id": "0dE5WAVqyg90" 628 | } 629 | }, 630 | { 631 | "cell_type": "code", 632 | "source": [ 633 | "# the matrix (from chapter 12)\n", 634 | "M = np.array([ [-1,1],\n", 635 | " [-1,2] ])\n", 636 | "\n", 637 | "# its eigendecomposition\n", 638 | "evals,evecs = np.linalg.eig(M)\n", 639 | "l = evals[1] # extract lambda1 for convenience\n", 640 | "v = evecs[:,[1]] # extract v1 for convenience\n", 641 | "\n", 642 | "LHS = M@v\n", 643 | "RHS = l*v\n", 644 | "\n", 645 | "# print out the two sides (as row vectors for visual convenience)\n", 646 | "print(LHS.T)\n", 647 | "print(RHS.T)" 648 | ], 649 | "metadata": { 650 | "id": "yzUaxMLSyhAL" 651 | }, 652 | "execution_count": null, 653 | "outputs": [] 654 | }, 655 | { 656 | "cell_type": "code", 657 | "source": [ 658 | "# pinv(v)\n", 659 | "vPinv = np.linalg.pinv(v)\n", 660 | "\n", 661 | "# check\n", 662 | "vPinv@v" 663 | ], 664 | "metadata": { 665 | "id": "J5pJMWCTUcv8" 666 | }, 667 | "execution_count": null, 668 | "outputs": [] 669 | }, 670 | { 671 | "cell_type": "code", 672 | "source": [ 673 | "# first equation\n", 674 | "LHS = vPinv @ M @ v\n", 675 | "RHS = l * vPinv @ v\n", 676 | "\n", 677 | "# these results are scalars (quadratic form)\n", 678 | "print(LHS)\n", 679 | "print(RHS)" 680 | ], 681 | "metadata": { 682 | "id": "Muh1-R6yUcy6" 683 | }, 684 | "execution_count": null, 685 | "outputs": [] 686 | }, 687 | { 688 | "cell_type": "code", 689 | "source": [ 690 | "# second equation\n", 691 | "LHS = M @ v @ vPinv\n", 692 | "RHS = l * v @ vPinv\n", 693 | "\n", 694 | "# these results are matrices\n", 695 | "print(LHS), print(' ')\n", 696 | "print(RHS)" 697 | ], 698 | "metadata": { 699 | "id": "6d6kordeUc11" 700 | }, 701 | "execution_count": null, 702 | "outputs": [] 703 | }, 704 | { 705 | "cell_type": "code", 706 | "source": [ 707 | "" 708 | ], 709 | "metadata": { 710 | "id": "sSw_ibhMUdBH" 711 | }, 712 | "execution_count": null, 713 | "outputs": [] 714 | } 715 | ] 716 | } 717 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LinAlg4DataScience 2 | Code that accompanies the book "Linear Algebra for Data Science" 3 | 4 | Check out the table of contents ("_TOC.pdf" file) 5 | 6 | Link to the book on amazon: 7 | https://www.amazon.com/Practical-Linear-Algebra-Data-Science/dp/1098120612/ 8 | 9 | E-version viewable for O'Reilly members: 10 | https://www.oreilly.com/library/view/practical-linear-algebra/9781098120603/ 11 | 12 | 13 | This repository contains all the Python code that accompanies the book. The code probably isn't useful without the book. 14 | 15 | 16 | ### All of the exercises have accompanying YT videos! 17 | The videos provide more explanation and context for the problems, their solutions, and the code. Check it out! 18 | https://www.youtube.com/watch?v=Vpei9S9mFyM&list=PLn0OLiymPak3REyB3XNqqqsRAhZ3LSEH8 19 | -------------------------------------------------------------------------------- /pyFiles/LA4DS_ch02.py: -------------------------------------------------------------------------------- 1 | # import libraries 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | 5 | 6 | # NOTE: these lines define global figure properties used for publication. 7 | import matplotlib_inline.backend_inline 8 | matplotlib_inline.backend_inline.set_matplotlib_formats('svg') # display figures in vector format 9 | plt.rcParams.update({'font.size':14}) # set global font size 10 | 11 | # a vector as a Python list datatype 12 | asList = [1,2,3] 13 | 14 | # same numbers, but as a dimensionless numpy array 15 | asArray = np.array([1,2,3]) 16 | 17 | # again same numbers, but now endowed with orientations 18 | rowVec = np.array([ [1,2,3] ]) # row 19 | colVec = np.array([ [1],[2],[3] ]) # column 20 | 21 | # Note the use of spacing when defining the vectors; that is not necessary but makes the code more readable. 22 | 23 | # Check the sizes of the variables 24 | print(f'asList: {np.shape(asList)}') # using np's shape function 25 | print(f'asArray: {asArray.shape}') # using a method associated with numpy objects 26 | print(f'rowVec: {rowVec.shape}') 27 | print(f'colVec: {colVec.shape}') 28 | 29 | # create a vector 30 | v = np.array([-1,2]) 31 | 32 | # plot that vector (and a dot for the tail) 33 | plt.arrow(0,0,v[0],v[1],head_width=.5,width=.1) 34 | plt.plot(0,0,'ko',markerfacecolor='k',markersize=7) 35 | 36 | # add axis lines 37 | plt.plot([-3,3],[0,0],'--',color=[.8,.8,.8],zorder=-1) 38 | plt.plot([0,0],[-3,3],'--',color=[.8,.8,.8],zorder=-1) 39 | 40 | # make the plot look nicer 41 | plt.axis('square') 42 | plt.axis([-3,3,-3,3]) 43 | plt.xlabel('$v_0$') 44 | plt.ylabel('$v_1$') 45 | plt.title('Vector v in standard position') 46 | plt.show() 47 | 48 | # A range of starting positions 49 | 50 | startPos = [ 51 | [0,0], 52 | [-1,-1], 53 | [1.5,-2] 54 | ] 55 | 56 | 57 | # create a new figure 58 | fig = plt.figure(figsize=(6,6)) 59 | 60 | for s in startPos: 61 | 62 | # plot that vector (and a dot for the tail) 63 | # note that plt.arrow automatically adds an offset to the third/fourth inputs 64 | plt.arrow(s[0],s[1],v[0],v[1],head_width=.5,width=.1,color='black') 65 | plt.plot(s[0],s[1],'ko',markerfacecolor='k',markersize=7) 66 | 67 | # indicate the vector in its standard position 68 | if s==[0,0]: 69 | plt.text(v[0]+.1,v[1]+.2,'"Standard pos."') 70 | 71 | 72 | # add axis lines 73 | plt.plot([-3,3],[0,0],'--',color=[.8,.8,.8],zorder=-1) 74 | plt.plot([0,0],[-3,3],'--',color=[.8,.8,.8],zorder=-1) 75 | 76 | # make the plot look nicer 77 | plt.axis('square') 78 | plt.axis([-3,3,-3,3]) 79 | plt.xlabel('$v_0$') 80 | plt.ylabel('$v_1$') 81 | plt.title('Vector $\mathbf{v}$ in various locations') 82 | plt.savefig('Figure_02_01.png',dpi=300) # write out the fig to a file 83 | plt.show() 84 | 85 | # Using 2D vectors here instead of 3D vectors in the book to facilitate visualization 86 | v = np.array([1,2]) 87 | w = np.array([4,-6]) 88 | vPlusW = v+w 89 | 90 | # print out all three vectors 91 | print(v) 92 | print(w) 93 | print(vPlusW) 94 | 95 | # Note that here we don't need to worry about vector orientation (row vs. column), 96 | # so for simplicity the vectors are created orientationless. 97 | 98 | # Where's the code to generate Figure 3-2? 99 | # It's exercise 0! 100 | 101 | # Same v and w as above for comparison 102 | vMinusW = v-w 103 | 104 | # print out all three vectors 105 | print(v) 106 | print(w) 107 | print(vMinusW) 108 | 109 | # Code for Figure 3-2 is part of Exercise 0. 110 | 111 | # a scalar 112 | s = -1/2 113 | 114 | # a vector 115 | b = np.array([3,4]) 116 | 117 | # print them 118 | print(b*s) 119 | 120 | # Question: Does vector b need to be a numpy array? What happens if it's a list? 121 | 122 | # Scalar-vector addition 123 | s = 3.5 124 | 125 | print(v) 126 | print(s+v) 127 | 128 | # plot 129 | plt.plot([0,b[0]],[0,b[1]],'m--',linewidth=3,label='b') 130 | plt.plot([0,s*b[0]],[0,s*b[1]],'k:',linewidth=3,label='sb') 131 | 132 | plt.grid() 133 | plt.axis('square') 134 | plt.axis([-6,6,-6,6]) 135 | plt.legend() 136 | plt.show() 137 | 138 | # Effects of different scalars 139 | 140 | # a list of scalars: 141 | scalars = [ 1, 2, 1/3, 0, -2/3 ] 142 | 143 | baseVector = np.array([ .75,1 ]) 144 | 145 | # create a figure 146 | fig,axs = plt.subplots(1,len(scalars),figsize=(12,3)) 147 | i = 0 # axis counter 148 | 149 | for s in scalars: 150 | 151 | # compute the scaled vector 152 | v = s*baseVector 153 | 154 | # plot it 155 | axs[i].arrow(0,0,baseVector[0],baseVector[1],head_width=.3,width=.1,color='k',length_includes_head=True) 156 | axs[i].arrow(.1,0,v[0],v[1],head_width=.3,width=.1,color=[.75,.75,.75],length_includes_head=True) 157 | axs[i].grid(linestyle='--') 158 | axs[i].axis('square') 159 | axs[i].axis([-2.5,2.5,-2.5,2.5]) 160 | axs[i].set(xticks=np.arange(-2,3), yticks=np.arange(-2,3)) 161 | axs[i].set_title(f'$\sigma$ = {s:.2f}') 162 | i+=1 # update axis counter 163 | 164 | plt.tight_layout() 165 | plt.savefig('Figure_02_03.png',dpi=300) 166 | plt.show() 167 | 168 | 169 | # Row vector 170 | r = np.array([ [1,2,3] ]) 171 | 172 | # orientationless array 173 | l = np.array([1,2,3]) 174 | 175 | # print out the vector, its transpose, and its double-transpose 176 | print(r), print(' ') 177 | 178 | # Transpose the row vector 179 | print(r.T), print(' ') 180 | 181 | # double-transpose 182 | print(r.T.T) 183 | 184 | # Same for the orientationless array 185 | print(l), print(' ') 186 | print(l.T), print(' ') 187 | print(l.T.T) 188 | 189 | 190 | 191 | # just some random vectors... 192 | v = np.array([ 0,1,2 ]) 193 | w = np.array([ 3,5,8 ]) 194 | u = np.array([ 13,21,34 ]) 195 | 196 | # two ways to comptue 197 | res1 = np.dot( v, w+u ) 198 | res2 = np.dot( v,w ) + np.dot( v,u ) 199 | 200 | # show that they are equivalent 201 | res1,res2 202 | 203 | 204 | 205 | # The vectors 206 | v = np.array([1,2]) 207 | w = np.array([4,-6]) 208 | vPlusW = v+w 209 | 210 | 211 | # now plot all three vectors 212 | plt.figure(figsize=(6,6)) 213 | 214 | a1 = plt.arrow(0,0,v[0],v[1],head_width=.3,width=.1,color='k',length_includes_head=True) 215 | a2 = plt.arrow(v[0],v[1],w[0],w[1],head_width=.3,width=.1,color=[.5,.5,.5],length_includes_head=True) 216 | a3 = plt.arrow(0,0,vPlusW[0],vPlusW[1],head_width=.3,width=.1,color=[.8,.8,.8],length_includes_head=True) 217 | 218 | 219 | # make the plot look a bit nicer 220 | plt.grid(linestyle='--',linewidth=.5) 221 | plt.axis('square') 222 | plt.axis([-6,6,-6,6]) 223 | plt.legend([a1,a2,a3],['v','w','v+w']) 224 | plt.title('Vectors $\mathbf{v}$, $\mathbf{w}$, and $\mathbf{v+w}$') 225 | plt.savefig('Figure_02_02a.png',dpi=300) # write out the fig to a file 226 | plt.show() 227 | 228 | # vector difference 229 | vMinusW = v-w 230 | 231 | 232 | # now plot all three vectors 233 | plt.figure(figsize=(6,6)) 234 | 235 | a1 = plt.arrow(0,0,v[0],v[1],head_width=.3,width=.1,color='k',length_includes_head=True) 236 | a2 = plt.arrow(0,0,w[0],w[1],head_width=.3,width=.1,color=[.5,.5,.5],length_includes_head=True) 237 | a3 = plt.arrow(w[0],w[1],vMinusW[0],vMinusW[1],head_width=.3,width=.1,color=[.8,.8,.8],length_includes_head=True) 238 | 239 | 240 | # make the plot look a bit nicer 241 | plt.grid(linestyle='--',linewidth=.5) 242 | plt.axis('square') 243 | plt.axis([-6,6,-6,6]) 244 | plt.legend([a1,a2,a3],['v','w','v-w']) 245 | plt.title('Vectors $\mathbf{v}$, $\mathbf{w}$, and $\mathbf{v-w}$') 246 | plt.savefig('Figure_02_02b.png',dpi=300) 247 | plt.show() 248 | 249 | 250 | 251 | 252 | # the function 253 | def normOfVect(v): 254 | return np.sqrt(np.sum(v**2)) 255 | 256 | # test it on a unit-norm vector 257 | w = np.array([0,0,1]) 258 | print( normOfVect(w) ) 259 | 260 | # non-unit-norm vector, and confirm using np.linalg.norm 261 | w = np.array([1,2,3]) 262 | print( normOfVect(w),np.linalg.norm(w) ) 263 | 264 | 265 | 266 | 267 | # define function 268 | def createUnitVector(v): 269 | # get vector norm 270 | mu = np.linalg.norm(v) 271 | # return unit vector 272 | return v / mu 273 | 274 | 275 | # test on a unit vector 276 | w = np.array([0,1,0]) 277 | print( createUnitVector(w) ) 278 | 279 | # test on a non-unit vector that is easy to confirm 280 | w = np.array([0,3,0]) 281 | print( createUnitVector(w) ) 282 | 283 | # test on a non-unit vector 284 | w = np.array([13,-5,7]) 285 | uw = createUnitVector(w) 286 | print( uw ), print(' ') 287 | # confirm the vectors' norms 288 | print( np.linalg.norm(w),np.linalg.norm(uw) ) 289 | 290 | # what happens with the zeros vector? 291 | print('\n\n\n') # just some spaces 292 | createUnitVector( np.zeros((4,1)) ) 293 | 294 | 295 | 296 | # define the function 297 | def createMagVector(v,mag): 298 | # get vector norm 299 | mu = np.linalg.norm(v) 300 | # return unit vector 301 | return mag * v / mu 302 | 303 | # test on a vector that is easy to confirm 304 | w = np.array([1,0,0]) 305 | mw = createMagVector(w,4) 306 | print( mw ) 307 | 308 | # confirm the vectors' norms 309 | print( np.linalg.norm(w),np.linalg.norm(mw) ) 310 | 311 | 312 | 313 | 314 | # the row vector to transpose 315 | v = np.array([[1,2,3]]) 316 | 317 | # initialize the column vector 318 | vt = np.zeros((3,1)) 319 | 320 | # direct implementation of the formula using a for loop 321 | for i in range(v.shape[1]): 322 | vt[i,0] = v[0,i] 323 | 324 | # confirm! 325 | print(v), print(' ') 326 | print(vt) 327 | 328 | # Note about data types: The two vectors actually have different data types 329 | # (ints vs. floats). That happened because I defined v using ints while the default type 330 | # for np.zeros is float. You can match data types in several ways, including: 331 | # (1) write 3. instead of 3 when creating v; (2) use dtype=np.float as an optional input. 332 | 333 | 334 | 335 | # some vector 336 | c = np.random.randn(5) 337 | 338 | # squared norm as dot product with itself 339 | sqrNrm1 = np.dot(c,c) 340 | 341 | # squared norm via our function from exercise 1 342 | sqrNrm2 = normOfVect(c)**2 343 | 344 | # print both to confirm they're the same 345 | print( sqrNrm1 ) 346 | print( sqrNrm2 ) 347 | 348 | 349 | 350 | # dimensionality 351 | n = 11 352 | 353 | # some random column vectors 354 | a = np.random.randn(n,1) 355 | b = np.random.randn(n,1) 356 | 357 | # dot products both ways 358 | atb = np.sum(a*b) 359 | bta = np.sum(b*a) 360 | 361 | # they're equal if their difference is 0 362 | atb - bta 363 | 364 | # For an extra challenge, see what happens when you use np.dot() to compute the dot products. 365 | 366 | # the vectors a and b 367 | a = np.array([1,2]) 368 | b = np.array([1.5,.5]) 369 | 370 | # compute beta 371 | beta = np.dot(a,b) / np.dot(a,a) 372 | 373 | # compute the projection vector (not explicitly used in the plot) 374 | projvect = b - beta*a 375 | 376 | 377 | # draw the figure 378 | plt.figure(figsize=(4,4)) 379 | 380 | # vectors 381 | plt.arrow(0,0,a[0],a[1],head_width=.2,width=.02,color='k',length_includes_head=True) 382 | plt.arrow(0,0,b[0],b[1],head_width=.2,width=.02,color='k',length_includes_head=True) 383 | 384 | # projection vector 385 | plt.plot([b[0],beta*a[0]],[b[1],beta*a[1]],'k--') 386 | 387 | # projection on a 388 | plt.plot(beta*a[0],beta*a[1],'ko',markerfacecolor='w',markersize=13) 389 | 390 | # make the plot look nicer 391 | plt.plot([-1,2.5],[0,0],'--',color='gray',linewidth=.5) 392 | plt.plot([0,0],[-1,2.5],'--',color='gray',linewidth=.5) 393 | 394 | # add labels 395 | plt.text(a[0]+.1,a[1],'a',fontweight='bold',fontsize=18) 396 | plt.text(b[0],b[1]-.3,'b',fontweight='bold',fontsize=18) 397 | plt.text(beta*a[0]-.35,beta*a[1],r'$\beta$',fontweight='bold',fontsize=18) 398 | plt.text((b[0]+beta*a[0])/2,(b[1]+beta*a[1])/2+.1,r'(b-$\beta$a)',fontweight='bold',fontsize=18) 399 | 400 | # some finishing touches 401 | plt.axis('square') 402 | plt.axis([-1,2.5,-1,2.5]) 403 | plt.show() 404 | 405 | 406 | 407 | # generate random R2 vectors (note: no orientation here! we don't need it for this exercise) 408 | t = np.random.randn(2) 409 | r = np.random.randn(2) 410 | 411 | # the decomposition 412 | t_para = r * (np.dot(t,r) / np.dot(r,r)) 413 | t_perp = t - t_para 414 | 415 | # confirm that the two components sum to the target 416 | print(t) 417 | print( t_para+t_perp ) 418 | 419 | # confirm orthogonality (dot product must be zero!) 420 | print( np.dot(t_para,t_perp) ) 421 | # Note about this result: Due to numerical precision errors, 422 | # you might get a result of something like 10^-17, which can be interpretd as zero. 423 | 424 | 425 | 426 | # draw them! 427 | plt.figure(figsize=(6,6)) 428 | 429 | # draw main vectors 430 | plt.plot([0,t[0]],[0,t[1]],color='k',linewidth=3,label=r'$\mathbf{t}$') 431 | plt.plot([0,r[0]],[0,r[1]],color=[.7,.7,.7],linewidth=3,label=r'$\mathbf{r}$') 432 | 433 | # draw decomposed vector components 434 | plt.plot([0,t_para[0]],[0,t_para[1]],'k--',linewidth=3,label=r'$\mathbf{t}_{\|}$') 435 | plt.plot([0,t_perp[0]],[0,t_perp[1]],'k:',linewidth=3,label=r'$\mathbf{t}_{\perp}$') 436 | 437 | plt.axis('equal') 438 | plt.legend() 439 | plt.savefig('Figure_02_08.png',dpi=300) 440 | plt.show() 441 | 442 | # Replace t_para in the previous exercise with the line below: 443 | t_para = r * (np.dot(t,r) / np.dot(t,t)) 444 | -------------------------------------------------------------------------------- /pyFiles/LA4DS_ch03.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | # NOTE: these lines define global figure properties used for publication. 5 | import matplotlib_inline.backend_inline 6 | matplotlib_inline.backend_inline.set_matplotlib_formats('svg') # print figures in svg format 7 | plt.rcParams.update({'font.size':14}) # set global font size 8 | 9 | 10 | 11 | # the scalars 12 | l1 = 1 13 | l2 = 2 14 | l3 = -3 15 | 16 | # the vectors 17 | v1 = np.array([4,5,1]) 18 | v2 = np.array([-4,0,-4]) 19 | v3 = np.array([1,3,2]) 20 | 21 | # linear weighted combination 22 | l1*v1 + l2*v2 + l3*v3 23 | 24 | 25 | 26 | 27 | # points (in Cartesian coordinates) 28 | p = (3,1) 29 | q = (-6,2) 30 | 31 | plt.figure(figsize=(6,6)) 32 | 33 | # draw points 34 | plt.plot(p[0],p[1],'ko',markerfacecolor='k',markersize=10,label='Point p') 35 | plt.plot(q[0],q[1],'ks',markerfacecolor='k',markersize=10,label='Point q') 36 | 37 | # draw basis vectors 38 | plt.plot([0,0],[0,1],'k',linewidth=3,label='Basis S') 39 | plt.plot([0,1],[0,0],'k',linewidth=3) 40 | 41 | plt.plot([0,3],[0,1],'k--',linewidth=3,label='Basis T') 42 | plt.plot([0,-3],[0,1],'k--',linewidth=3) 43 | 44 | 45 | plt.axis('square') 46 | plt.grid(linestyle='--',color=[.8,.8,.8]) 47 | plt.xlim([-7,7]) 48 | plt.ylim([-7,7]) 49 | plt.legend() 50 | plt.savefig('Figure_03_04.png',dpi=300) 51 | plt.show() 52 | 53 | 54 | 55 | ## Note: Make sure you run the code earlier to create variables l1, l2, etc. 56 | 57 | # organized into lists 58 | scalars = [ l1,l2,l3 ] 59 | vectors = [ v1,v2,v3 ] 60 | 61 | # initialize the linear combination 62 | linCombo = np.zeros(len(v1)) 63 | 64 | # implement linear weighted combination using zip() 65 | for s,v in zip(scalars,vectors): 66 | linCombo += s*v 67 | 68 | # confirm it's the same answer as above 69 | linCombo 70 | 71 | 72 | 73 | # Whether it works or throws an error depends on how you set up the code. 74 | # Using zip() as above works, because zip() will use only the minimum-matching items. 75 | # Re-writing the code using indexing will cause an error, as in the code below. 76 | 77 | # make the scalars longer 78 | scalars = [ l1,l2,l3,5 ] 79 | vectors = [ v1,v2,v3 ] 80 | 81 | linCombo = np.zeros(len(v1)) 82 | 83 | for i in range(len(scalars)): 84 | linCombo += scalars[i]*vectors[i] 85 | 86 | 87 | 88 | # the vector set (just one vector) 89 | A = np.array([ 1,3 ]) 90 | 91 | # x-axis range 92 | xlim = [-4,4] 93 | 94 | # random scalars in that range 95 | scalars = np.random.uniform(low=xlim[0],high=xlim[1],size=100) 96 | 97 | 98 | 99 | # create a figure, etc 100 | plt.figure(figsize=(6,6)) 101 | 102 | # loop over the random scalars 103 | for s in scalars: 104 | 105 | # create point p 106 | p = A*s 107 | 108 | # plot it 109 | plt.plot(p[0],p[1],'ko') 110 | 111 | 112 | plt.xlim(xlim) 113 | plt.ylim(xlim) 114 | plt.grid() 115 | plt.text(-4.5,4.5,'A)',fontweight='bold',fontsize=18) 116 | plt.savefig('Figure_03_07a.png',dpi=300) 117 | plt.show() 118 | 119 | import plotly.graph_objects as go 120 | 121 | # two vectors in R3 122 | v1 = np.array([ 3,5,1 ]) 123 | v2 = np.array([ 0,2,2 ]) 124 | 125 | ## uncomment the lines below for the second part 126 | # v1 = np.array([ 3.0,5.0,1.0 ]) 127 | # v2 = np.array([ 1.5,2.5,0.5 ]) 128 | 129 | 130 | # random scalars in the x-axis range 131 | scalars = np.random.uniform(low=xlim[0],high=xlim[1],size=(100,2)) 132 | 133 | 134 | 135 | # create random points 136 | points = np.zeros((100,3)) 137 | for i in range(len(scalars)): 138 | 139 | # define this point as a random weighted combination of the two vectors 140 | points[i,:] = v1*scalars[i,0] + v2*scalars[i,1] 141 | 142 | 143 | # draw the dots in the plane 144 | fig = go.Figure( data=[go.Scatter3d(x=points[:,0], y=points[:,1], z=points[:,2], 145 | mode='markers', marker=dict(size=6,color='black') )]) 146 | 147 | fig.update_layout(margin=dict(l=0,r=0,b=0,t=0)) 148 | plt.savefig('Figure_03_07b.png',dpi=300) 149 | fig.show() 150 | 151 | 152 | -------------------------------------------------------------------------------- /pyFiles/LA4DS_ch04.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | # NOTE: these lines define global figure properties used for publication. 5 | import matplotlib_inline.backend_inline 6 | matplotlib_inline.backend_inline.set_matplotlib_formats('svg') # print figures in svg format 7 | plt.rcParams.update({'font.size':14}) # set global font size 8 | 9 | 10 | N = 30 11 | 12 | # correlated random variables 13 | x = np.linspace(0,10,N) + np.random.randn(N) 14 | y = x + np.random.randn(N) 15 | 16 | 17 | # set up figure 18 | _,axs = plt.subplots(2,2,figsize=(6,6)) 19 | 20 | # positive correlation 21 | axs[0,0].plot(x,y,'ko') 22 | axs[0,0].set_title('Positive correlation',fontweight='bold') 23 | axs[0,0].set_xlabel('Variable x') 24 | axs[0,0].set_ylabel('Variable y') 25 | axs[0,0].set_xticks([]) 26 | axs[0,0].set_yticks([]) 27 | axs[0,0].axis('square') 28 | 29 | 30 | # negative correlation 31 | axs[0,1].plot(x,-y,'ko') 32 | axs[0,1].set_title('Negative correlation',fontweight='bold') 33 | axs[0,1].set_xlabel('Variable x') 34 | axs[0,1].set_ylabel('Variable y') 35 | axs[0,1].set_xticks([]) 36 | axs[0,1].set_yticks([]) 37 | axs[0,1].axis('square') 38 | 39 | 40 | # zero correlation, part 1 41 | axs[1,0].plot(np.random.randn(N),np.random.randn(N),'ko') 42 | axs[1,0].set_title('Zero correlation',fontweight='bold') 43 | axs[1,0].set_xlabel('Variable x') 44 | axs[1,0].set_ylabel('Variable y') 45 | axs[1,0].set_xticks([]) 46 | axs[1,0].set_yticks([]) 47 | axs[1,0].axis('square') 48 | 49 | 50 | # zero correlation, part 2 51 | x = np.cos(np.linspace(0,2*np.pi,N)) + np.random.randn(N)/20 52 | y = np.sin(np.linspace(0,2*np.pi,N)) + np.random.randn(N)/20 53 | axs[1,1].plot(x,y,'ko') 54 | axs[1,1].set_title('Zero correlation',fontweight='bold') 55 | axs[1,1].set_xlabel('Variable x') 56 | axs[1,1].set_ylabel('Variable y') 57 | axs[1,1].set_xticks([]) 58 | axs[1,1].set_yticks([]) 59 | axs[1,1].axis('square') 60 | 61 | 62 | plt.tight_layout() 63 | plt.savefig('Figure_04_01.png',dpi=300) # write out the fig to a file 64 | plt.show() 65 | 66 | 67 | 68 | ### Note: The code for k-means is in Exercise 7 below. 69 | 70 | 71 | 72 | # the function 73 | def corrAndCosine(x,y): 74 | 75 | # compute cosine similarity 76 | num = np.dot(x,y) # numerator 77 | den = np.linalg.norm(x) * np.linalg.norm(y) # denominator 78 | cos = num / den 79 | 80 | # compute correlation (similar to above but mean-centered!) 81 | xm = x-np.mean(x) 82 | ym = y-np.mean(y) 83 | num = np.dot(xm,ym) # numerator 84 | den = np.linalg.norm(xm) * np.linalg.norm(ym) # denominator 85 | cor = num / den 86 | 87 | return cor,cos 88 | 89 | 90 | # test it 91 | a = np.random.randn(15) 92 | b = np.random.randn(15) 93 | 94 | # compute the correlation and cosine 95 | r,c = corrAndCosine(a,b) 96 | 97 | # confirm that the correlation matches with np.corrcoef 98 | print(r,np.corrcoef(a,b)[0,1]) 99 | 100 | # compare r and c without mean-centering 101 | a = np.random.randn(15) + 10 # note the offset! 102 | b = np.random.randn(15) 103 | 104 | # mean-center 105 | aNoMean = a - np.mean(a) 106 | bNoMean = b - np.mean(b) 107 | 108 | 109 | # show the results with and without mean-centering 110 | print('Without mean-centering (should differ):') 111 | print( np.round(corrAndCosine(a,b),4) ) 112 | print(' ') 113 | 114 | print('With mean-centering (should be the same):') 115 | print( np.round(corrAndCosine(aNoMean,bNoMean),4) ) 116 | 117 | # NOTE: In the printing code above, I rounded to 4 significant digits just for visual clarity. 118 | 119 | # create the variables 120 | a = np.arange(4,dtype=float) 121 | offsets = np.arange(-50,51) 122 | 123 | # initialize the results 124 | results = np.zeros((len(offsets),2)) 125 | 126 | # run the simulation! 127 | for i in range(len(offsets)): 128 | results[i,:] = corrAndCosine(a,a+offsets[i]) 129 | 130 | 131 | # plot the results! 132 | plt.figure(figsize=(8,4)) 133 | h = plt.plot(offsets,results) 134 | h[0].set_color('k') 135 | h[0].set_marker('o') 136 | h[1].set_color([.7,.7,.7]) 137 | h[1].set_marker('s') 138 | 139 | plt.xlabel('Mean offset') 140 | plt.ylabel('r or c') 141 | plt.legend(['Pearson','Cosine sim.']) 142 | plt.savefig('Figure_04_02.png',dpi=300) # write out the fig to a file 143 | plt.show() 144 | 145 | 146 | 147 | # import the function 148 | from scipy.stats import pearsonr 149 | 150 | # inspect the source code 151 | ??pearsonr 152 | 153 | 154 | 155 | # a bare-bones correlation function 156 | def rho(x,y): 157 | xm = x-np.mean(x) 158 | ym = y-np.mean(y) 159 | n = np.dot(xm,ym) 160 | d = np.linalg.norm(xm) * np.linalg.norm(ym) 161 | return n/d 162 | 163 | 164 | # import the time library 165 | import time 166 | 167 | # experiment parameters 168 | numIters = 1000 169 | varLength = 500 170 | 171 | # clock my custom-written function 172 | tic = time.time() 173 | for i in range(numIters): 174 | x = np.random.randn(varLength,2) 175 | rho(x[:,0],x[:,1]) 176 | t1 = time.time() - tic 177 | 178 | 179 | # now for numpy's corrcoef function 180 | tic = time.time() 181 | for i in range(numIters): 182 | x = np.random.randn(varLength,2) 183 | pearsonr(x[:,0],x[:,1]) 184 | t2 = time.time() - tic 185 | 186 | 187 | # print the results! 188 | # Note: time() returns seconds, so I multiply by 1000 for ms 189 | print(f'My function took {t1*1000:.2f} ms') 190 | print(f' pearsonr took {t2*1000:.2f} ms') 191 | 192 | 193 | 194 | # create the kernel 195 | kernel = np.array([-1,1]) 196 | 197 | # and the "signal" (a plateau) 198 | signal = np.zeros(30) 199 | signal[10:20] = 1 200 | 201 | 202 | # plot them 203 | _,axs = plt.subplots(1,2,figsize=(12,4)) 204 | axs[0].plot(kernel,'ks-') 205 | axs[0].set_title('Kernel') 206 | axs[0].set_xlim([-15,15]) 207 | 208 | axs[1].plot(signal,'ks-') 209 | axs[1].set_title('Time series signal') 210 | 211 | plt.savefig('Figure_04_04ab.png',dpi=300) 212 | plt.show() 213 | 214 | # initialize the feature map as zeros 215 | featureMap = np.zeros(len(signal)) 216 | 217 | # loop over the signal and do template-matching (via dot products!) 218 | for t in range(1,len(signal)-1): 219 | featureMap[t] = np.dot(kernel,signal[t-1:t+1]) 220 | 221 | 222 | # plot the result 223 | _,axs = plt.subplots(1,2,figsize=(12,4)) 224 | axs[0].plot(kernel,'ks-') 225 | axs[0].set_title('Kernel') 226 | axs[0].set_xlim([-15,15]) 227 | 228 | 229 | axs[1].plot(signal,'ks-',label='Signal',linewidth=3) 230 | markers,stemlines,_ = axs[1].stem(range(len(featureMap)),featureMap, 231 | basefmt=' ',linefmt='',markerfmt='o', 232 | label='Edge detection') 233 | 234 | plt.setp(stemlines,'color',[.7,.7,.7]) 235 | plt.setp(markers,'color',[.7,.7,.7]) 236 | 237 | axs[1].legend() 238 | plt.savefig('Figure_04_04c.png',dpi=300) 239 | plt.show() 240 | 241 | 242 | 243 | # define the kernel (a sorta-kinda Gaussian) 244 | kernel = np.array([0,.1,.3,.8,1,.8,.3,.1,0]) 245 | kernel = kernel / np.sum(kernel) 246 | 247 | # some handy length parameters 248 | Nkernel = len(kernel) 249 | halfKrn = Nkernel//2 250 | 251 | 252 | # and the signal 253 | Nsignal = 100 254 | timeseries = np.random.randn(Nsignal) 255 | 256 | 257 | # plot them 258 | _,axs = plt.subplots(1,2,figsize=(12,4)) 259 | axs[0].plot(kernel,'ks-') 260 | axs[0].set_title('Kernel') 261 | axs[0].set_xlim([-1,Nsignal]) 262 | 263 | axs[1].plot(timeseries,'ks-') 264 | axs[1].set_title('Time series signal') 265 | 266 | plt.savefig('Figure_04_06ab.png',dpi=300) 267 | plt.show() 268 | 269 | # make a copy of the signal for filtering 270 | filtsig = timeseries.copy() 271 | 272 | # loop over the signal time points 273 | for t in range(halfKrn+1,Nsignal-halfKrn): 274 | filtsig[t] = np.dot(kernel,timeseries[t-halfKrn-1:t+halfKrn]) 275 | 276 | 277 | # and plot 278 | _,axs = plt.subplots(1,2,figsize=(12,4)) 279 | axs[0].plot(kernel,'ks-') 280 | axs[0].set_title('Kernel') 281 | axs[0].set_xlim([-1,Nsignal]) 282 | 283 | axs[1].plot(timeseries,color='k',label='Original',linewidth=1) 284 | axs[1].plot(filtsig,'--',color=[.6,.6,.6],label='Smoothed',linewidth=2) 285 | axs[1].legend() 286 | 287 | plt.savefig('Figure_04_06c.png',dpi=300) 288 | plt.show() 289 | 290 | # define the kernel (a sorta-kinda Gaussian) 291 | kernel = np.array([0,.1,.3,.8,-1,.8,.3,.1,0]) 292 | kernel /= np.sum(kernel) 293 | kernel -= np.mean(kernel) 294 | 295 | # plot them 296 | _,axs = plt.subplots(1,2,figsize=(12,4)) 297 | axs[0].plot(kernel,'s-') 298 | axs[0].set_title('Kernel') 299 | axs[0].set_xlim([-1,Nsignal]) 300 | 301 | axs[1].plot(timeseries,'s-') 302 | axs[1].set_title('Time series signal') 303 | plt.show() 304 | 305 | 306 | 307 | # loop over the signal time points 308 | filtsig2 = timeseries.copy() 309 | for t in range(halfKrn+1,Nsignal-halfKrn): 310 | filtsig2[t] = np.dot(kernel,timeseries[t-halfKrn-1:t+halfKrn]) 311 | 312 | plt.plot(timeseries,color='k',label='Original',linewidth=1) 313 | plt.plot(filtsig2,color=[.9,.2,.7],label='Sharpened',linewidth=1) 314 | plt.legend() 315 | plt.show() 316 | 317 | 318 | 319 | ## Create data 320 | nPerClust = 50 321 | 322 | # blur around centroid (std units) 323 | blur = 1 324 | 325 | # XY centroid locations 326 | A = [ 1, 1 ] 327 | B = [ -3, 1 ] 328 | C = [ 3, 3 ] 329 | 330 | # generate data 331 | a = [ A[0]+np.random.randn(nPerClust)*blur , A[1]+np.random.randn(nPerClust)*blur ] 332 | b = [ B[0]+np.random.randn(nPerClust)*blur , B[1]+np.random.randn(nPerClust)*blur ] 333 | c = [ C[0]+np.random.randn(nPerClust)*blur , C[1]+np.random.randn(nPerClust)*blur ] 334 | 335 | # concatanate into a matrix 336 | data = np.transpose( np.concatenate((a,b,c),axis=1) ) 337 | 338 | 339 | # plot data 340 | plt.plot(data[:,0],data[:,1],'ko',markerfacecolor='w') 341 | plt.title('Raw (preclustered) data') 342 | plt.xticks([]) 343 | plt.yticks([]) 344 | 345 | plt.show() 346 | 347 | ## initialize random cluster centroids 348 | k = 3 # extract three clusters 349 | 350 | # random cluster centers (randomly sampled data points) 351 | ridx = np.random.choice(range(len(data)),k,replace=False) 352 | centroids = data[ridx,:] 353 | 354 | 355 | # setup the figure 356 | fig,axs = plt.subplots(2,2,figsize=(6,6)) 357 | axs = axs.flatten() 358 | lineColors = [ [0,0,0],[.4,.4,.4],[.8,.8,.8] ]#'rbm' 359 | 360 | 361 | # plot data with initial random cluster centroids 362 | axs[0].plot(data[:,0],data[:,1],'ko',markerfacecolor='w') 363 | axs[0].plot(centroids[:,0],centroids[:,1],'ko') 364 | axs[0].set_title('Iteration 0') 365 | axs[0].set_xticks([]) 366 | axs[0].set_yticks([]) 367 | 368 | 369 | 370 | # loop over iterations 371 | for iteri in range(3): 372 | 373 | # step 1: compute distances 374 | dists = np.zeros((data.shape[0],k)) 375 | for ci in range(k): 376 | dists[:,ci] = np.sum((data-centroids[ci,:])**2,axis=1) 377 | 378 | # step 2: assign to group based on minimum distance 379 | groupidx = np.argmin(dists,axis=1) 380 | 381 | # step 3: recompute centers 382 | for ki in range(k): 383 | centroids[ki,:] = [ np.mean(data[groupidx==ki,0]), np.mean(data[groupidx==ki,1]) ] 384 | 385 | 386 | # plot data points 387 | for i in range(len(data)): 388 | axs[iteri+1].plot([ data[i,0],centroids[groupidx[i],0] ],[ data[i,1],centroids[groupidx[i],1] ],color=lineColors[groupidx[i]]) 389 | axs[iteri+1].plot(centroids[:,0],centroids[:,1],'ko') 390 | axs[iteri+1].set_title(f'Iteration {iteri+1}') 391 | axs[iteri+1].set_xticks([]) 392 | axs[iteri+1].set_yticks([]) 393 | 394 | 395 | plt.savefig('Figure_04_03.png',dpi=300) 396 | plt.show() 397 | 398 | 399 | -------------------------------------------------------------------------------- /pyFiles/LA4DS_ch05.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | # for null spaces 5 | import scipy.linalg 6 | 7 | # a pretty-looking matrix from scipy 8 | from scipy.linalg import toeplitz 9 | 10 | 11 | # NOTE: these lines define global figure properties used for publication. 12 | import matplotlib_inline.backend_inline 13 | matplotlib_inline.backend_inline.set_matplotlib_formats('svg') # print figures in svg format 14 | plt.rcParams.update({'font.size':14}) # set global font size 15 | 16 | v = np.array([[1,2,3]]).T # col vector 17 | w = np.array([[10,20]]) # row vector 18 | v + w 19 | 20 | # create some matrices 21 | A = np.random.randn(3,4) 22 | B = np.random.randn(100,100) 23 | C = -toeplitz(np.arange(8),np.arange(10)) 24 | 25 | 26 | # and show them as images 27 | fig,axs = plt.subplots(1,3,figsize=(10,3)) 28 | 29 | axs[0].imshow(A,cmap='gray') 30 | axs[1].imshow(B,cmap='gray') 31 | axs[2].imshow(C,cmap='gray') 32 | 33 | for i in range(3): axs[i].axis('off') 34 | plt.tight_layout() 35 | plt.savefig('Figure_05_01.png',dpi=300) 36 | plt.show() 37 | 38 | 39 | 40 | # create a matrix 41 | A = np.reshape(np.arange(1,10),(3,3)) 42 | print(A) 43 | 44 | # get the n-th row 45 | 46 | print( A[1,:] ) 47 | 48 | # note that to extract only one row, you don't need the column indices. 49 | print( A[1] ) 50 | # But that's potentially confusing, so I recommend avoiding that notation. 51 | 52 | # get the n-th column 53 | print( A[:,1] ) 54 | # Note that it prints out as a "row" even thought it's a column of the matrix 55 | 56 | # multiple rows 57 | A[0:2,:] 58 | 59 | # multiple columns 60 | A[:,1:] 61 | 62 | ## extracting a submatrix (multiple rows and cols) 63 | 64 | # The goal here is to extract a submatrix from matrix A. Here's A: 65 | # [[1 2 3] 66 | # [4 5 6] 67 | # [7 8 9]] 68 | 69 | # And we want rows 0-1 and columns 0-1, thus: 70 | # [[1 2] 71 | # [4 5]] 72 | 73 | 74 | # seems like this should work... 75 | print( A[0:2,1:2] ) 76 | print(' ') 77 | 78 | # but this does (remember x:y:z slices from x to y-1 in steps of z) 79 | print( A[0:2:1,0:2:1] ) 80 | 81 | # This cell has the example shown in the book. 82 | 83 | # the full matrix 84 | A = np.arange(60).reshape(6,10) 85 | 86 | # a block of it 87 | sub = A[1:4:1,0:5:1] 88 | 89 | 90 | # print them out 91 | print('Original matrix:\n') 92 | print(A) 93 | 94 | print('\n\nSubmatrix:\n') 95 | print(sub) 96 | 97 | 98 | 99 | ## create some matrices 100 | 101 | # square 102 | M1 = np.random.permutation(16).reshape(4,4) 103 | 104 | # upper-triangular square 105 | M2 = np.triu(np.random.randint(10,20,(3,3))) 106 | 107 | # lower-triangular rectangular 108 | M3 = np.tril(np.random.randint(8,16,(3,5))) 109 | 110 | # diagonal 111 | M4 = np.diag( np.random.randint(0,6,size=8) ) 112 | 113 | # identity 114 | M5 = np.eye(4,dtype=int) 115 | 116 | # zeros 117 | M6 = np.zeros((4,5),dtype=int) 118 | 119 | matrices = [ M1,M2,M3,M4,M5,M6 ] 120 | matLabels = [ 'Square','Upper-triangular','Lower-triangular','Diagonal','Identity','Zeros' ] 121 | 122 | 123 | _,axs = plt.subplots(2,3,figsize=(12,6)) 124 | axs = axs.flatten() 125 | 126 | for mi,M in enumerate(matrices): 127 | axs[mi].imshow(M,cmap='gray',origin='upper', 128 | vmin=np.min(M),vmax=np.max(M)) 129 | axs[mi].set(xticks=[],yticks=[]) 130 | axs[mi].set_title(matLabels[mi]) 131 | 132 | # text labels 133 | for (j,i),num in np.ndenumerate(M): 134 | axs[mi].text(i,j,num,color=[.8,.8,.8],ha='center',va='center',fontweight='bold') 135 | 136 | 137 | 138 | plt.savefig('Figure_05_02.png',dpi=300) 139 | plt.tight_layout() 140 | plt.show() 141 | 142 | 143 | 144 | # matrix size parameters (called 'shape' in Python lingo) 145 | Mrows = 4 # shape 0 146 | Ncols = 6 # shape 1 147 | 148 | # create the matrix! 149 | A = np.random.randn(Mrows,Ncols) 150 | 151 | # print out the matrix (rounding to facilitate visual inspection) 152 | np.round(A,3) 153 | 154 | # Extract the triangular part of a dense matrix 155 | 156 | M = 4 157 | N = 6 158 | A = np.random.randn(M,N) 159 | 160 | # upper triangular 161 | print('Upper triangular:\n') 162 | print(np.triu(A)) 163 | 164 | # lower triangular 165 | print('\n\nLower triangular:\n') 166 | print(np.tril(A)) 167 | 168 | # Diagonal 169 | 170 | # input a matrix to get the diagonal elements 171 | A = np.random.randn(5,5) 172 | d = np.diag(A) 173 | print('Input a matrix:\n',d) 174 | 175 | # OR input a vector to create a diagonal matrix! 176 | v = np.arange(1,6) 177 | D = np.diag(v) 178 | print('\n\nInput a vector:\n',D) 179 | 180 | # Identity and zeros matrices 181 | 182 | # Note that you only specify one input 183 | n = 4 184 | I = np.eye(n) 185 | print(f'The {n}x{n} identity matrix:\n',I) 186 | 187 | 188 | # Zeros matrix 189 | 190 | # Important: All shape parameters are given as one input (a tuple or list), 191 | # unlike np.random.randn() 192 | n = 4 193 | m = 5 194 | I = np.zeros((n,m)) 195 | print(f'The {n}x{m} zeros matrix:\n',I) 196 | 197 | 198 | 199 | A = np.array([ [2,3,4], 200 | [1,2,4] ]) 201 | 202 | B = np.array([ [ 0, 3,1], 203 | [-1,-4,2] ]) 204 | 205 | print(A+B) 206 | 207 | 208 | 209 | # Not shifting; broadcasting scalar addition 210 | 3 + np.eye(2) 211 | 212 | # This is shifting: 213 | 214 | # the matrix 215 | A = np.array([ [4,5, 1], 216 | [0,1,11], 217 | [4,9, 7] ]) 218 | 219 | # the scalar 220 | s = 6 221 | 222 | print('Original matrix:') 223 | print(A), print(' ') 224 | 225 | # as in the previous cell, this is broadcasting addition, not shifting 226 | print('Broadcasting addition:') 227 | print(A + s), print(' ') 228 | 229 | # This is shifting 230 | print('Shifting:') 231 | print( A + s*np.eye(len(A)) ) 232 | 233 | 234 | 235 | print(A), print(' ') 236 | print(s*A) 237 | 238 | 239 | 240 | # two random matrices 241 | A = np.random.randn(3,4) 242 | B = np.random.randn(3,4) 243 | 244 | # this is Hadamard multiplication 245 | A*B 246 | 247 | # and so is this 248 | np.multiply(A,B) 249 | 250 | # this one is NOT Hadamard multiplication 251 | A@B 252 | 253 | 254 | 255 | # Create a few matrices 256 | A = np.random.randn(3,6) 257 | B = np.random.randn(6,4) 258 | C = np.random.randn(6,4) 259 | 260 | # try some multiplications, and print out the shape of the product matrix 261 | print( (A@B).shape ) 262 | print( np.dot(A,B).shape ) # same as above 263 | print( (B@C).shape ) 264 | print( (A@C).shape ) 265 | 266 | # Note/reminder: 267 | 268 | # This is Hadamard (element-wise) multiplication: 269 | print( np.multiply(B,C) ), print(' ') 270 | 271 | # This is matrix multiplication 272 | print( np.dot(B,C.T) ) 273 | 274 | # demonstration: 275 | # np.dot(B,C.T)-B@C.T 276 | 277 | 278 | 279 | # some matrix 280 | M = np.array([ [2,3],[2,1] ]) 281 | x = np.array([ [1,1.5] ]).T # transposed into a column vector! 282 | Mx = M@x 283 | 284 | 285 | plt.figure(figsize=(6,6)) 286 | 287 | plt.plot([0,x[0,0]],[0,x[1,0]],'k',linewidth=4,label='x') 288 | plt.plot([0,Mx[0,0]],[0,Mx[1,0]],'--',linewidth=3,color=[.7,.7,.7],label='Mx') 289 | plt.xlim([-7,7]) 290 | plt.ylim([-7,7]) 291 | plt.legend() 292 | plt.grid() 293 | plt.savefig('Figure_05_05a.png',dpi=300) 294 | plt.show() 295 | 296 | # some matrix 297 | M = np.array([ [2,3],[2,1] ]) 298 | v = np.array([ [1.5,1] ]).T # transposed into a column vector! 299 | Mv = M@v 300 | 301 | 302 | plt.figure(figsize=(6,6)) 303 | 304 | plt.plot([0,v[0,0]],[0,v[1,0]],'k',linewidth=4,label='v') 305 | plt.plot([0,Mv[0,0]],[0,Mv[1,0]],'--',linewidth=3,color=[.7,.7,.7],label='Mv') 306 | plt.xlim([-7,7]) 307 | plt.ylim([-7,7]) 308 | plt.legend() 309 | plt.grid() 310 | plt.savefig('Figure_05_05b.png',dpi=300) 311 | plt.show() 312 | 313 | 314 | 315 | 316 | 317 | # A matrix to transpose 318 | A = np.array([ [3,4,5],[1,2,3] ]) 319 | 320 | A_T1 = A.T # as method 321 | A_T2 = np.transpose(A) # as function 322 | 323 | # double-transpose 324 | A_TT = A_T1.T 325 | 326 | 327 | # print them 328 | print( A_T1 ), print(' ') 329 | print( A_T2 ), print(' ') 330 | print( A_TT ) 331 | 332 | 333 | 334 | 335 | 336 | # indexing 337 | 338 | A = np.arange(12).reshape(3,4) 339 | print(A) 340 | 341 | # find the element in the 2nd row, 4th column 342 | ri = 1 343 | ci = 3 344 | 345 | print(f'The matrix element at index ({ri+1},{ci+1}) is {A[ri,ci]}') 346 | 347 | # Create the matrix 348 | C = np.arange(100).reshape((10,10)) 349 | 350 | # extract submatrix 351 | C_1 = C[0:5:1,0:5:1] 352 | 353 | # here's what the matrices look like 354 | print(C), print(' ') 355 | print(C_1) 356 | 357 | 358 | 359 | # visualize the matrices as maps 360 | _,axs = plt.subplots(1,2,figsize=(10,5)) 361 | 362 | axs[0].imshow(C,cmap='gray',origin='upper',vmin=0,vmax=np.max(C)) 363 | axs[0].plot([4.5,4.5],[-.5,9.5],'w--') 364 | axs[0].plot([-.5,9.5],[4.5,4.5],'w--') 365 | axs[0].set_title('Original matrix') 366 | # text labels 367 | for (j,i),num in np.ndenumerate(C): 368 | axs[0].text(i,j,num,color=[.8,.8,.8],ha='center',va='center') 369 | 370 | 371 | axs[1].imshow(C_1,cmap='gray',origin='upper',vmin=0,vmax=np.max(C)) 372 | axs[1].set_title('Submatrix') 373 | # text labels 374 | for (j,i),num in np.ndenumerate(C_1): 375 | axs[1].text(i,j,num,color=[.8,.8,.8],ha='center',va='center') 376 | 377 | 378 | plt.savefig('Figure_05_06.png',dpi=300) 379 | plt.show() 380 | 381 | # cut it into blocks 382 | C_1 = C[0:5:1,0:5:1] 383 | C_2 = C[0:5:1,5:10:1] 384 | C_3 = C[5:10:1,0:5:1] 385 | C_4 = C[5:10:1,5:10:1] 386 | 387 | # rearrange the blocks 388 | newMatrix = np.vstack( (np.hstack((C_4,C_3)), 389 | np.hstack((C_2,C_1))) ) 390 | 391 | 392 | # visualize the matrices 393 | _,axs = plt.subplots(1,2,figsize=(10,5)) 394 | 395 | axs[0].imshow(C,cmap='gray',origin='upper',vmin=0,vmax=np.max(C)) 396 | axs[0].plot([4.5,4.5],[-.5,9.5],'w--') 397 | axs[0].plot([-.5,9.5],[4.5,4.5],'w--') 398 | axs[0].set_title('Original matrix') 399 | # text labels 400 | for (j,i),num in np.ndenumerate(C): 401 | axs[0].text(i,j,num,color=[.8,.8,.8],ha='center',va='center') 402 | 403 | 404 | axs[1].imshow(newMatrix,cmap='gray',origin='upper',vmin=0,vmax=np.max(C)) 405 | axs[1].plot([4.5,4.5],[-.5,9.5],'w--') 406 | axs[1].plot([-.5,9.5],[4.5,4.5],'w--') 407 | axs[1].set_title('Block-shifted') 408 | # text labels 409 | for (j,i),num in np.ndenumerate(newMatrix): 410 | axs[1].text(i,j,num,color=[.8,.8,.8],ha='center',va='center') 411 | 412 | plt.savefig('Figure_05_07.png',dpi=300) 413 | plt.show() 414 | 415 | def addMatrices(A,B): 416 | 417 | # check that both matrices have the same size 418 | if A.shape != B.shape: 419 | raise('Matrices must be the same size!') 420 | 421 | # initialize sum matrix 422 | C = np.zeros(A.shape) 423 | 424 | # sum! 425 | for i in range(A.shape[0]): 426 | for j in range(A.shape[1]): 427 | C[i,j] = A[i,j] + B[i,j] 428 | 429 | return C 430 | 431 | 432 | # test the function 433 | M1 = np.zeros((6,4)) 434 | M2 = np.ones((6,4)) 435 | 436 | addMatrices(M1,M2) 437 | 438 | 439 | # create random matrices and a scalar 440 | A = np.random.randn(3,4) 441 | B = np.random.randn(3,4) 442 | s = np.random.randn() 443 | 444 | # equations shown in the text 445 | expr1 = s*(A+B) 446 | expr2 = s*A + s*B 447 | expr3 = A*s + B*s 448 | 449 | 450 | # There are a few ways to test for 3-way equality. 451 | # My choice below is that if x=y=z, then 2x-y-z=0. 452 | 453 | # print out, rounded to 8 digits after the decimal point 454 | print(np.round(2*expr1 - expr2 - expr3,8)) 455 | 456 | 457 | 458 | # generate two matrices 459 | m = 4 460 | n = 6 461 | A = np.random.randn(m,n); 462 | B = np.random.randn(n,m) 463 | 464 | # build up the product matrix element-wise 465 | C1 = np.zeros((m,m)) 466 | for rowi in range(m): 467 | for coli in range(m): 468 | C1[rowi,coli] = np.dot( A[rowi,:],B[:,coli] ) 469 | 470 | 471 | 472 | # implement matrix multiplication directly 473 | C2 = A@B 474 | 475 | # compare the results (using isclose(); results should be a matrix of TRUEs) 476 | np.isclose( C1,C2 ) 477 | 478 | 479 | 480 | # Create the matrices 481 | L = np.random.randn(2,6) 482 | I = np.random.randn(6,3) 483 | V = np.random.randn(3,5) 484 | E = np.random.randn(5,2) 485 | 486 | # multiplications indicated in the instructions 487 | res1 = ( L@I@V@E ).T 488 | # res2 = L.T @ I.T @ V.T @ E.T 489 | res3 = E.T @ V.T @ I.T @ L.T 490 | 491 | # show that res1 and res3 are the same (within rounding error tolerance) 492 | print(res1-res3) 493 | 494 | 495 | 496 | def isMatrixSymmetric(S): 497 | 498 | # difference between matrix and its transpose 499 | D = S-S.T 500 | 501 | # check whether sum of squared errors (SSE) is smaller than a threshold 502 | sse = np.sum(D**2) 503 | 504 | # output TRUE if sse is tiny; FALSE means the matrix is asymmetric 505 | return sse<10**-15 506 | 507 | # note: There are many other ways you could solve this. 508 | # If you want to explore different methods, consider np.all() or np.isclose() 509 | 510 | # create symmetric and nonsymmetric matrices 511 | A = np.random.randn(4,4) 512 | AtA = A.T@A 513 | 514 | # test! 515 | print(isMatrixSymmetric(A)) 516 | print(isMatrixSymmetric(AtA)) 517 | 518 | 519 | 520 | # create symmetric and nonsymmetric matrices 521 | A = np.random.randn(4,4) 522 | AtA = (A + A.T) / 2 # additive method! 523 | 524 | # test! 525 | print(isMatrixSymmetric(A)) 526 | print(isMatrixSymmetric(AtA)) 527 | 528 | 529 | 530 | import plotly.graph_objects as go 531 | 532 | # As a matrix with two columns in R3, instead of two separate vectors 533 | A = np.array( [ [3,0], 534 | [5,2], 535 | [1,2] ] ) 536 | 537 | # uncomment the line below 538 | # A = np.array( [ [3,1.5], 539 | # [5,2.5], 540 | # [1, .5] ] ) 541 | 542 | 543 | xlim = [-4,4] 544 | scalars = np.random.uniform(low=xlim[0],high=xlim[1],size=(100,2)) 545 | 546 | # create random points 547 | points = np.zeros((100,3)) 548 | for i in range(len(scalars)): 549 | points[i,:] = A@scalars[i] 550 | 551 | # draw the dots in the figure 552 | fig = go.Figure( data=[go.Scatter3d(x=points[:,0], y=points[:,1], z=points[:,2], mode='markers')]) 553 | fig.show() 554 | 555 | 556 | 557 | n = 4 558 | 559 | # create "base" matrices 560 | O = np.ones((n,n)) 561 | D = np.diag(np.arange(1,n+1)**2) 562 | S = np.sqrt(D) 563 | 564 | # pre- and post-multiply 565 | pre = D@O 566 | pst = O@D 567 | 568 | # and both 569 | both = S@O@S 570 | 571 | 572 | 573 | # print out the "base" matrices 574 | print('Ones matrix:') 575 | print(O), print(' ') 576 | 577 | print('Diagonal matrix:') 578 | print(D), print(' ') 579 | 580 | print('Sqrt-diagonal matrix:') 581 | print(S), print(' ') 582 | 583 | 584 | 585 | print('Pre-multiply by diagonal:') 586 | print(pre), print(' ') 587 | 588 | print('Post-multiply by diagonal:') 589 | print(pst), print(' ') 590 | 591 | print('Pre- and post-multiply by sqrt-diagonal:') 592 | print(both) 593 | 594 | 595 | 596 | # Create two diagonal matrices 597 | N = 5 598 | D1 = np.diag( np.random.randn(N) ) 599 | D2 = np.diag( np.random.randn(N) ) 600 | 601 | # two forms of multiplication 602 | hadamard = D1*D2 603 | standard = D1@D2 604 | 605 | # compare them 606 | hadamard - standard 607 | 608 | 609 | 610 | 611 | -------------------------------------------------------------------------------- /pyFiles/LA4DS_ch06.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | # for null spaces 5 | import scipy.linalg 6 | 7 | # a pretty-looking matrix from scipy 8 | from scipy.linalg import toeplitz 9 | 10 | 11 | # NOTE: these lines define global figure properties used for publication. 12 | import matplotlib_inline.backend_inline 13 | matplotlib_inline.backend_inline.set_matplotlib_formats('svg') # print figures in svg format 14 | plt.rcParams.update({'font.size':14}) # set global font size 15 | 16 | 17 | 18 | # some matrix (just one lonely little column) 19 | A = np.array([ [1],[3] ]) 20 | 21 | 22 | # The column space contains an infinite number of points, but for computational convenience, 23 | # we can draw the line using only two points 24 | xlim = [-5,5] 25 | colspace_p1 = xlim[0]*A 26 | colspace_p2 = xlim[1]*A 27 | 28 | plt.figure(figsize=(6,6)) 29 | 30 | plt.plot([0,A[0,0]],[0,A[1,0]],'k',linewidth=4,label='A') 31 | plt.plot([colspace_p1[0,0],colspace_p2[0,0]],[colspace_p1[1,0],colspace_p2[1,0]], 32 | '--',linewidth=3,color=[.7,.7,.7],label='C(A)') 33 | plt.xlim(xlim) 34 | plt.ylim(xlim) 35 | plt.legend() 36 | plt.grid() 37 | plt.savefig('Figure_06_01.png',dpi=300) 38 | plt.show() 39 | 40 | # some matrix 41 | A1 = np.array([ [1,1],[3,2] ]) 42 | A2 = np.array([ [1,2],[3,6] ]) 43 | 44 | 45 | 46 | # some other plotting specifications 47 | xlim = [-6,6] 48 | color = [ [0,0,0],[.7,.7,.7] ] 49 | 50 | 51 | # make the plot 52 | _,axs = plt.subplots(1,2,figsize=(12,6)) 53 | 54 | # loop over columns 55 | for i in range(2): 56 | axs[0].plot([0,A1[0,i]],[0,A1[1,i]],color=color[i],linewidth=4) 57 | axs[1].plot([0,A2[0,i]],[0,A2[1,i]],color=color[i],linewidth=4,zorder=-i) 58 | 59 | # set some axis properties 60 | axs[i].set_xlim(xlim) 61 | axs[i].set_ylim(xlim) 62 | axs[i].grid() 63 | axs[i].text(xlim[0]-.7,xlim[1]+.5,f'{"AB"[i]})',fontweight='bold',fontsize=16) 64 | 65 | # set the legends and subplot letters 66 | for i in [0,1]: axs[i].legend([f'A{i+1}$_{{[:,0]}}$',f'A{i+1}$_{{[:,1]}}$']) 67 | 68 | plt.savefig('Figure_06_02.png',dpi=300) 69 | plt.show() 70 | 71 | 72 | 73 | # a matrix with two columns in R3 74 | A = np.array( [ [3,0], 75 | [5,2], 76 | [1,2] ] ) 77 | 78 | 79 | # create a 3D graph 80 | ax = plt.figure(figsize=(6,6)).add_subplot(111, projection='3d') 81 | 82 | # draw plane corresponding to the column space 83 | xx, yy = np.meshgrid(np.linspace(-5,5,10),np.linspace(-5,5,10)) 84 | cp = np.cross(A[:,0],A[:,1]) 85 | z1 = (-cp[0]*xx - cp[1]*yy)/cp[2] 86 | ax.plot_surface(xx,yy,z1,alpha=.2) 87 | 88 | 89 | ## plot the two vectors from matrix S 90 | ax.plot([0, A[0,0]],[0, A[1,0]],[0, A[2,0]],color=color[0],linewidth=4) 91 | ax.plot([0, A[0,1]],[0, A[1,1]],[0, A[2,1]],color=color[1],linewidth=4) 92 | 93 | plt.savefig('Figure_06_03.png',dpi=300) 94 | plt.show() 95 | 96 | 97 | 98 | # The two matrices 99 | A = np.array([ [1,-1],[-2,2] ]) 100 | B = np.array([ [1,-1],[-2,3] ]) 101 | 102 | # null spaces 103 | print( scipy.linalg.null_space(A) ) 104 | print(' ') 105 | 106 | print( scipy.linalg.null_space(B) ) 107 | 108 | # Using matrix A above 109 | 110 | nullvect = scipy.linalg.null_space(A) 111 | 112 | 113 | 114 | # some other plotting specifications 115 | xlim = [-3,3] 116 | color = [ [0,0,0],[.7,.7,.7] ] 117 | 118 | 119 | # make the plot 120 | plt.figure(figsize=(6,6)) 121 | 122 | # plot the rows 123 | for i in range(2): 124 | plt.plot([0,A[i,0]],[0,A[i,1]],color=color[i],linewidth=4,label='A$_{{[%g,:]}}$'%i) 125 | 126 | # plot the nullspace vector 127 | plt.plot([0,nullvect[0,0]],[0,nullvect[1,0]],'--',color=[.4,.4,.4], 128 | linewidth=4,label='y') 129 | 130 | # plot the rest of the nullspace 131 | plt.plot([xlim[0]*nullvect[0,0],xlim[1]*nullvect[0,0]], 132 | [xlim[0]*nullvect[1,0],xlim[1]*nullvect[1,0]], 133 | ':',color=[.4,.4,.4],label='N(A)') 134 | 135 | # set some axis properties 136 | plt.xlim(xlim) 137 | plt.ylim(xlim) 138 | plt.grid() 139 | plt.legend() 140 | 141 | plt.savefig('Figure_06_04.png',dpi=300) 142 | plt.show() 143 | 144 | 145 | 146 | 147 | 148 | # experiment simulations 149 | scalingVals = np.linspace(0,50,40) # range of scaling parameters (0 to 50 in 40 steps) 150 | nExperiments = 10 151 | 152 | 153 | # initialize output 154 | matrixNorms = np.zeros((len(scalingVals),nExperiments)) 155 | 156 | # run experiment! 157 | for si in range(len(scalingVals)): 158 | for expi in range(nExperiments): 159 | 160 | # generate a random scaled matrix 161 | R = np.random.randn(10,10) * scalingVals[si] 162 | 163 | # store its norm 164 | matrixNorms[si,expi] = np.linalg.norm(R,'fro') 165 | 166 | 167 | # plot the results! 168 | plt.plot(scalingVals,np.mean(matrixNorms,axis=1),'ko-') 169 | plt.xlabel('Matrix scalar') 170 | plt.ylabel('Matrix Frobenius norm') 171 | plt.savefig('Figure_06_07.png',dpi=300) 172 | plt.show() 173 | 174 | # check that norm=0 for zeros matrix 175 | print(matrixNorms[0,:]) 176 | 177 | 178 | 179 | # Function to compute Euclidean distance 180 | 181 | def EuclideanDistance(M1,M2): 182 | 183 | # matrix difference 184 | D = M1-M2 185 | 186 | # matrix distance 187 | return np.sqrt(np.sum(D**2)) 188 | 189 | # optimization code 190 | 191 | # create two matrices 192 | N = 7 193 | A = np.random.randn(N,N) 194 | B = np.random.randn(N,N) 195 | 196 | # optimization 197 | numIters = 0 198 | s = 1 199 | while EuclideanDistance(s*A,s*B)>1: 200 | s *= .9 201 | numIters += 1 202 | 203 | # report the results. Note that my loop code scales once more after criteria is reached, 204 | # so I subtract one from numIters and undo the final s scaling. 205 | print(f'Number of iterations: {numIters-1}') 206 | print(f'Final value of scalar: {s/.9:.3f}') 207 | print(f'Final Euclidean distance: {EuclideanDistance(s/.9*A,s/.9*B):.3f}') 208 | 209 | # The code below isn't part of the exercise, but I was curious to repeat the optimization 210 | # 1000 times to see the distribution of numIters 211 | 212 | nIters = np.zeros(1000) 213 | 214 | for i in range(1000): 215 | # create two matrices 216 | A = np.random.randn(N,N) 217 | B = np.random.randn(N,N) 218 | 219 | numIters,s = 0,1 220 | while EuclideanDistance(s*A,s*B)>1: 221 | s *= .9 222 | numIters += 1 223 | nIters[i] = numIters-1 224 | 225 | plt.hist(nIters) 226 | plt.xlabel('Number of iterations') 227 | plt.ylabel('Count'); 228 | 229 | 230 | 231 | # Create a matrix 232 | M = 50 233 | A = np.random.randn(M,M) 234 | 235 | # trace method 236 | norm1 = np.sqrt(np.sum(np.diag(A.T@A))) 237 | 238 | # Euclidean norm method 239 | norm2 = np.sqrt(np.sum(A**2)) 240 | 241 | # if they're equal, their difference should be (very close to) zero 242 | norm1-norm2 243 | 244 | 245 | 246 | # size of the matrix 247 | N = 10 248 | 249 | shifting = np.linspace(0,1,30) 250 | 251 | # original matrix 252 | A = np.random.randn(N,N) 253 | normA = np.linalg.norm(A,'fro') 254 | 255 | # initialize results matrices 256 | shiftingResults = np.zeros( (len(shifting),3) ) 257 | resultsNames = [ 'Change in norm (%)','Corr. with original','Frobenius distance' ] 258 | 259 | 260 | 261 | for si in range(len(shifting)): 262 | 263 | # shift the matrix 264 | As = A + shifting[si]*normA*np.eye(N) 265 | 266 | # get the new norm and transform to %-change 267 | normShift = np.linalg.norm(As,'fro') 268 | shiftingResults[si,0] = 100 * (normShift-normA)/normA 269 | 270 | # compute correlation 271 | shiftingResults[si,1] = np.corrcoef(A.flatten(),As.flatten())[0,1] 272 | 273 | # Frobenius distance 274 | shiftingResults[si,2] = EuclideanDistance(A,As) 275 | 276 | 277 | 278 | 279 | ## plotting! 280 | _,axs = plt.subplots(1,3,figsize=(12,4)) 281 | 282 | for i in range(3): 283 | 284 | # plot the results 285 | axs[i].plot(shifting,shiftingResults[:,i],'ks-') 286 | axs[i].set_xlabel('Shifting (prop. of norm)') 287 | axs[i].set_ylabel(resultsNames[i]) 288 | 289 | plt.tight_layout() 290 | plt.savefig('Figure_06_06.png',dpi=300) 291 | plt.show() 292 | 293 | 294 | 295 | # Make a matrix with specified size and rank 296 | 297 | M = 5 298 | N = 8 299 | r = 3 300 | 301 | A = np.random.randn(M,r) @ np.random.randn(r,N) 302 | 303 | print(A.shape) 304 | print(np.linalg.matrix_rank(A)) 305 | 306 | 307 | 308 | # summed matrix has rank-0 309 | 310 | A = np.diag([ 1,0,0,0,0]) 311 | B = np.diag([-1,0,0,0,0]) 312 | C = A+B 313 | 314 | # print out their ranks 315 | np.linalg.matrix_rank(A),np.linalg.matrix_rank(B),np.linalg.matrix_rank(C) 316 | 317 | # summed matrix has rank-1 318 | 319 | A = np.diag([1,0,0,0,0]) 320 | B = np.zeros(A.shape) 321 | B[0,1] = 10 322 | C = A+B 323 | 324 | # print out their ranks 325 | np.linalg.matrix_rank(A),np.linalg.matrix_rank(B),np.linalg.matrix_rank(C) 326 | 327 | # summed matrix has rank-2 328 | 329 | A = np.diag([1,0,0,0,0]) 330 | B = np.diag([0,1,0,0,0]) 331 | C = A+B 332 | 333 | # print out their ranks 334 | np.linalg.matrix_rank(A),np.linalg.matrix_rank(B),np.linalg.matrix_rank(C) 335 | 336 | # random matrices have maximum possible rank! 337 | A = np.random.randn(5,1) @ np.random.randn(1,5) 338 | B = np.random.randn(5,1) @ np.random.randn(1,5) 339 | C = A+B 340 | 341 | # print out their ranks 342 | np.linalg.matrix_rank(A),np.linalg.matrix_rank(B),np.linalg.matrix_rank(C) 343 | 344 | 345 | 346 | # The function 347 | def makeAmatrix(M,r): 348 | return np.random.randn(M,r) @ np.random.randn(r,M) 349 | 350 | 351 | # parameters 352 | matSize = 20 # matrix size (square) 353 | rs = range(2,16) # range of ranks 354 | 355 | # initialize results matrix 356 | Ranks = np.zeros((len(rs),len(rs),2)) 357 | 358 | # run the simulation 359 | for i in range(len(rs)): 360 | for j in range(len(rs)): 361 | 362 | # create the matrices 363 | S = makeAmatrix(matSize,rs[i]) + makeAmatrix(matSize,rs[j]) 364 | M = makeAmatrix(matSize,rs[i]) @ makeAmatrix(matSize,rs[j]) 365 | 366 | # compute their ranks 367 | Ranks[i,j,0] = np.linalg.matrix_rank(S) 368 | Ranks[i,j,1] = np.linalg.matrix_rank(M) 369 | 370 | 371 | 372 | ## visualization 373 | fig,axs = plt.subplots(1,2,figsize=(10,6)) 374 | s = '+@' # symbols for title 375 | 376 | for i in range(2): 377 | 378 | # draw heatmat 379 | h = axs[i].imshow(Ranks[:,:,i],vmin=np.min(rs),vmax=np.max(rs),origin='lower', 380 | extent=(rs[0],rs[-1],rs[0],rs[-1]),cmap='gray') 381 | 382 | # add colorbar and other niceties 383 | fig.colorbar(h,ax=axs[i],fraction=.045) 384 | axs[i].set_xlabel('Rank of A') 385 | axs[i].set_ylabel('Rank of B') 386 | axs[i].set_title(f'Rank of A{s[i]}B') 387 | 388 | 389 | plt.savefig('Figure_06_09.png',dpi=300) 390 | plt.tight_layout() 391 | plt.show() 392 | 393 | 394 | 395 | # matrix sizes and rank 396 | M = 15 397 | N = 8 398 | r = 4 399 | 400 | # compute the four matrices 401 | A = np.random.randn(M,r) @ np.random.randn(r,N) 402 | At = A.T 403 | AtA = A.T@A 404 | AAt = A@A.T 405 | 406 | # print their ranks 407 | print( 408 | np.linalg.matrix_rank(A), 409 | np.linalg.matrix_rank(At), 410 | np.linalg.matrix_rank(AtA), 411 | np.linalg.matrix_rank(AAt) 412 | ) 413 | 414 | 415 | 416 | # function to run algorithm 417 | def is_V_inColA(A,v): 418 | 419 | # check sizes 420 | if A.shape[0]!=v.shape[0]: 421 | raise Exception('Size mismatch! A and v must have the same column dimensionality!.') 422 | 423 | # compute ranks 424 | rankA = np.linalg.matrix_rank(A) 425 | rankAv = np.linalg.matrix_rank( np.hstack((A,v)) ) 426 | 427 | # function outputs TRUE if v \in C(A) 428 | return rankA==rankAv 429 | 430 | 431 | # create matrix and vector 432 | A = np.random.randn(4,3) 433 | v = np.random.randn(4,1) 434 | 435 | # test! 436 | is_V_inColA(A,v) 437 | 438 | 439 | 440 | # matrix sizes 441 | ns = np.arange(3,31) 442 | 443 | # iteration 444 | iters = 100 445 | 446 | # initialize 447 | dets = np.zeros((len(ns),iters)) 448 | 449 | # loop over matrix sizes 450 | for ni in range(len(ns)): 451 | for i in range(iters): 452 | 453 | # step 1 454 | A = np.random.randn(ns[ni],ns[ni]) 455 | 456 | # step 2 457 | A[:,0] = A[:,1] 458 | 459 | # step 3 460 | dets[ni,i]=np.abs(np.linalg.det(A)) 461 | 462 | 463 | # note: the number of elements in a square matrix is the columns squared 464 | 465 | 466 | # plotting 467 | plt.figure(figsize=(6,4)) 468 | plt.plot(ns**2,np.log(np.mean(dets,axis=1)),'ks-',linewidth=3) 469 | plt.xlabel('Number of elements in the matrix') 470 | plt.ylabel('Log determinant') 471 | plt.title('Empirical determinants of singular matrices') 472 | plt.savefig('Figure_06_10.png',dpi=300) 473 | plt.show() 474 | 475 | 476 | -------------------------------------------------------------------------------- /pyFiles/LA4DS_ch07.py: -------------------------------------------------------------------------------- 1 | # import libraries 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | 5 | # setup animation 6 | import matplotlib.animation as animation 7 | from matplotlib import rc 8 | rc('animation', html='jshtml') 9 | 10 | 11 | # to read an image from a url (io) and convert it to grayscale (color) 12 | from skimage import io,color 13 | # convolution 14 | from scipy.signal import convolve2d 15 | 16 | 17 | import pandas as pd 18 | 19 | 20 | # NOTE: these lines define global figure properties used for publication. 21 | import matplotlib_inline.backend_inline 22 | matplotlib_inline.backend_inline.set_matplotlib_formats('svg') # display figures in vector format 23 | plt.rcParams.update({'font.size':14}) # set global font size 24 | 25 | 26 | 27 | # information about the data 28 | #https://archive.ics.uci.edu/ml/datasets/Communities+and+Crime 29 | 30 | # raw data file 31 | #https://archive.ics.uci.edu/ml/machine-learning-databases/communities/communities.data 32 | 33 | # dataset citation (see also above website for more): 34 | # Redmond, M. A. and A. Baveja: A Data-Driven Software Tool for Enabling Cooperative Information Sharing Among Police Departments. European Journal of Operational Research 141 (2002) 660-678. 35 | 36 | # read the data into a pandas dataframe 37 | url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/communities/communities.data' 38 | data = pd.read_csv(url,sep=',',header=None) 39 | 40 | # attach column labels (don't worry, I didn't type this all in by hand, lol) 41 | data.columns = [ 'state', 'county', 'community', 'communityname', 'fold', 'population', 'householdsize', 'racepctblack', 'racePctWhite', 42 | 'racePctAsian', 'racePctHisp', 'agePct12t21', 'agePct12t29', 'agePct16t24', 'agePct65up', 'numbUrban', 'pctUrban', 'medIncome', 'pctWWage', 43 | 'pctWFarmSelf', 'pctWInvInc', 'pctWSocSec', 'pctWPubAsst', 'pctWRetire', 'medFamInc', 'perCapInc', 'whitePerCap', 'blackPerCap', 'indianPerCap', 44 | 'AsianPerCap', 'OtherPerCap', 'HispPerCap', 'NumUnderPov', 'PctPopUnderPov', 'PctLess9thGrade', 'PctNotHSGrad', 'PctBSorMore', 'PctUnemployed', 'PctEmploy', 45 | 'PctEmplManu', 'PctEmplProfServ', 'PctOccupManu', 'PctOccupMgmtProf', 'MalePctDivorce', 'MalePctNevMarr', 'FemalePctDiv', 'TotalPctDiv', 'PersPerFam', 'PctFam2Par', 46 | 'PctKids2Par', 'PctYoungKids2Par', 'PctTeen2Par', 'PctWorkMomYoungKids', 'PctWorkMom', 'NumIlleg', 'PctIlleg', 'NumImmig', 'PctImmigRecent', 'PctImmigRec5', 47 | 'PctImmigRec8', 'PctImmigRec10', 'PctRecentImmig', 'PctRecImmig5', 'PctRecImmig8', 'PctRecImmig10', 'PctSpeakEnglOnly', 'PctNotSpeakEnglWell', 'PctLargHouseFam', 'PctLargHouseOccup', 48 | 'PersPerOccupHous', 'PersPerOwnOccHous', 'PersPerRentOccHous', 'PctPersOwnOccup', 'PctPersDenseHous', 'PctHousLess3BR', 'MedNumBR', 'HousVacant', 'PctHousOccup', 'PctHousOwnOcc', 49 | 'PctVacantBoarded', 'PctVacMore6Mos', 'MedYrHousBuilt', 'PctHousNoPhone', 'PctWOFullPlumb', 'OwnOccLowQuart', 'OwnOccMedVal', 'OwnOccHiQuart', 'RentLowQ', 'RentMedian', 50 | 'RentHighQ', 'MedRent', 'MedRentPctHousInc', 'MedOwnCostPctInc', 'MedOwnCostPctIncNoMtg', 'NumInShelters', 'NumStreet', 'PctForeignBorn', 'PctBornSameState', 'PctSameHouse85', 51 | 'PctSameCity85', 'PctSameState85', 'LemasSwornFT', 'LemasSwFTPerPop', 'LemasSwFTFieldOps', 'LemasSwFTFieldPerPop', 'LemasTotalReq', 'LemasTotReqPerPop', 'PolicReqPerOffic', 'PolicPerPop', 52 | 'RacialMatchCommPol', 'PctPolicWhite', 'PctPolicBlack', 'PctPolicHisp', 'PctPolicAsian', 'PctPolicMinor', 'OfficAssgnDrugUnits', 'NumKindsDrugsSeiz', 'PolicAveOTWorked', 'LandArea', 53 | 'PopDens', 'PctUsePubTrans', 'PolicCars', 'PolicOperBudg', 'LemasPctPolicOnPatr', 'LemasGangUnitDeploy', 'LemasPctOfficDrugUn', 'PolicBudgPerPop', 'ViolentCrimesPerPop', 54 | ] 55 | 56 | # have a look at the data 57 | data 58 | 59 | # extract only the numeric data 60 | numberDataset = data._get_numeric_data() 61 | 62 | # drop a few additional columns, and convert to a numpy array 63 | dataMat = numberDataset.drop(['state','fold'],axis=1).values 64 | dataMat 65 | 66 | # compute the mean of each data feature 67 | datamean = np.mean(dataMat,axis=0) 68 | 69 | # mean-center the data using broadcasting 70 | dataMatM = dataMat - datamean 71 | 72 | # confirm that any given feature has mean=0 (or very close...) 73 | print(np.mean(dataMatM[:,0])) 74 | 75 | 76 | # Now to compute the covariance matrix 77 | covMat = dataMatM.T @ dataMatM # data matrix times its transpose 78 | covMat /= (dataMatM.shape[0]-1) # divide by N-1 79 | 80 | # dynamic color scaling 81 | clim = np.max(np.abs(covMat)) * .2 82 | 83 | # and show it 84 | plt.figure(figsize=(6,6)) 85 | plt.imshow(covMat,vmin=-clim,vmax=clim,cmap='gray') 86 | plt.colorbar() 87 | plt.title('Data covariance matrix') 88 | plt.savefig('Figure_07_01.png',dpi=300) 89 | plt.show() 90 | 91 | 92 | 93 | # Pure rotation matrix 94 | 95 | # angle to rotate by 96 | th = np.pi/5 97 | 98 | # transformation matrix 99 | T = np.array([ 100 | [ np.cos(th),np.sin(th)], 101 | [-np.sin(th),np.cos(th)] 102 | ]) 103 | 104 | 105 | # original dots are a vertical line 106 | x = np.linspace(-1,1,20) 107 | origPoints = np.vstack( (np.zeros(x.shape),x) ) 108 | 109 | 110 | # apply the transformation 111 | transformedPoints = T @ origPoints 112 | 113 | 114 | # plot the points 115 | plt.figure(figsize=(6,6)) 116 | plt.plot(origPoints[0,:],origPoints[1,:],'ko',label='Original') 117 | plt.plot(transformedPoints[0,:],transformedPoints[1,:],'s',color=[.7,.7,.7],label='Transformed') 118 | 119 | plt.axis('square') 120 | plt.xlim([-1.2,1.2]) 121 | plt.ylim([-1.2,1.2]) 122 | plt.legend() 123 | plt.title(f'Rotation by {np.rad2deg(th):.0f} degrees.') 124 | plt.savefig('Figure_07_02.png',dpi=300) 125 | plt.show() 126 | 127 | 128 | 129 | # function to update the axis on each iteration 130 | def aframe(ph): 131 | 132 | # create the transformation matrix 133 | T = np.array([ 134 | [ 1, 1-ph ], 135 | [ 0, 1 ] 136 | ]) 137 | 138 | # apply the transformation to the points using matrix multiplication 139 | P = T@points 140 | 141 | # update the dots 142 | plth.set_xdata(P[0,:]) 143 | plth.set_ydata(P[1,:]) 144 | 145 | # export the plot handles 146 | return plth 147 | 148 | 149 | # define XY points 150 | theta = np.linspace(0,2*np.pi,100) 151 | points = np.vstack((np.sin(theta),np.cos(theta))) 152 | 153 | 154 | # setup figure 155 | fig,ax = plt.subplots(1,figsize=(12,6)) 156 | plth, = ax.plot(np.cos(x),np.sin(x),'ko') 157 | ax.set_aspect('equal') 158 | ax.set_xlim([-2,2]) 159 | ax.set_ylim([-2,2]) 160 | 161 | # define values for transformation (note: clip off the final point for a smooth animation loop) 162 | phi = np.linspace(-1,1-1/40,40)**2 163 | 164 | # run animation! 165 | animation.FuncAnimation(fig, aframe, phi, interval=100, repeat=True) 166 | 167 | 168 | 169 | # image 170 | imgN = 20 171 | image = np.random.randn(imgN,imgN) 172 | 173 | # convolution kernel 174 | kernelN = 7 175 | Y,X = np.meshgrid(np.linspace(-3,3,kernelN),np.linspace(-3,3,kernelN)) 176 | kernel = np.exp( -(X**2+Y**2)/7 ) 177 | kernel = kernel / np.sum(kernel) # normalize 178 | 179 | # now for the convolution 180 | halfKr = kernelN//2 181 | convoutput = np.zeros((imgN+kernelN-1,imgN+kernelN-1)) 182 | 183 | imagePad = np.zeros(convoutput.shape) 184 | imagePad[halfKr:-halfKr:1,halfKr:-halfKr:1] = image 185 | 186 | 187 | # double for-loop over rows and columns (width and height of picture) 188 | for rowi in range(halfKr,imgN+halfKr): 189 | for coli in range(halfKr,imgN+halfKr): 190 | 191 | # cut out a piece of the image 192 | pieceOfImg = imagePad[rowi-halfKr:rowi+halfKr+1:1,coli-halfKr:coli+halfKr+1:1] 193 | 194 | # dot product: element-wise multiply and sum 195 | dotprod = np.sum( pieceOfImg*kernel ) 196 | 197 | # store the result for this pixel 198 | convoutput[rowi,coli] = dotprod 199 | 200 | 201 | # trim off edges 202 | convoutput = convoutput[halfKr:-halfKr:1,halfKr:-halfKr:1] 203 | 204 | # using scipy 205 | convoutput2 = convolve2d(image,kernel,mode='same') 206 | 207 | fig,ax = plt.subplots(2,2,figsize=(8,8)) 208 | 209 | ax[0,0].imshow(image) 210 | ax[0,0].set_title('Image') 211 | 212 | ax[0,1].imshow(kernel) 213 | ax[0,1].set_title('Convolution kernel') 214 | 215 | ax[1,0].imshow(convoutput) 216 | ax[1,0].set_title('Manual convolution') 217 | 218 | ax[1,1].imshow(convoutput2) 219 | ax[1,1].set_title("Scipy's convolution") 220 | 221 | # for i in ax.flatten(): i.axis('off') 222 | 223 | plt.savefig('Figure_07_04b.png',dpi=300) 224 | plt.show() 225 | 226 | 227 | 228 | # read a pic from the web 229 | bathtub = io.imread('https://upload.wikimedia.org/wikipedia/commons/6/61/De_nieuwe_vleugel_van_het_Stedelijk_Museum_Amsterdam.jpg') 230 | 231 | # check the size 232 | print(bathtub.shape) 233 | 234 | # let's see what the famous Bathtub Museum looks like 235 | fig = plt.figure(figsize=(10,6)) 236 | plt.imshow(bathtub) 237 | plt.savefig('Figure_07_05a.png',dpi=300) 238 | plt.show() 239 | 240 | # transform image to 2D for convenience (not necessary for convolution!) 241 | bathtub2d = color.rgb2gray(bathtub) 242 | 243 | # check the size again 244 | print(bathtub2d.shape) 245 | 246 | # convolution kernel 247 | kernelN = 29 # a bit bigger than in the previous example... feel free to change this parameter! 248 | Y,X = np.meshgrid(np.linspace(-3,3,kernelN),np.linspace(-3,3,kernelN)) 249 | kernel = np.exp( -(X**2+Y**2)/20 ) 250 | kernel = kernel / np.sum(kernel) # normalize the kernel to integrate to 1, which preserves the numerical scale of the image. 251 | 252 | 253 | # smoothing via Gaussian convolution 254 | smooth_bathtub = convolve2d(bathtub2d,kernel,mode='same') 255 | 256 | fig = plt.figure(figsize=(10,6)) 257 | plt.imshow(smooth_bathtub,cmap='gray') 258 | plt.savefig('Figure_07_05b.png',dpi=300) 259 | plt.show() 260 | 261 | 262 | 263 | # Diagonal matrix of inverse standard deviations 264 | variances = np.diag(covMat) # variances are the diagonals of a covariance 265 | standard_devs = np.sqrt( variances ) 266 | S = np.diag( 1/standard_devs ) 267 | 268 | # you can also do this in one line: 269 | #S = np.diag( 1/np.sqrt(np.diag(covMat)) ) 270 | 271 | 272 | # compute the correlation matrix 273 | corrMat = S @ covMat @ S 274 | 275 | 276 | # and show the matrices 277 | fig,axs = plt.subplots(1,2,figsize=(13,6)) 278 | h1 = axs[0].imshow(covMat,vmin=-clim,vmax=clim,cmap='gray') 279 | axs[0].set_title('Data covariance matrix',fontweight='bold') 280 | 281 | h2 = axs[1].imshow(corrMat,vmin=-.5,vmax=.5,cmap='gray') 282 | axs[1].set_title('Data correlation matrix',fontweight='bold') 283 | 284 | fig.colorbar(h1,ax=axs[0],fraction=.045) 285 | fig.colorbar(h2,ax=axs[1],fraction=.045) 286 | 287 | plt.tight_layout() 288 | plt.savefig('Figure_07_06.png',dpi=300) 289 | plt.show() 290 | 291 | # a bit of code to explore specific pairs of correlations 292 | 293 | # here list two indices into the correlation matrix (row, col) 294 | i,j = 43,17 295 | 296 | # the printed tuple will show the correlation and the pairs of variables 297 | corrMat[i,j], data.columns[i], data.columns[j] 298 | 299 | 300 | 301 | # numpy's correlation function (note transposing the matrix!) 302 | corrMat_np = np.corrcoef(dataMat.T) 303 | 304 | 305 | # and show it 306 | fig,axs = plt.subplots(1,3,figsize=(13,6)) 307 | h1 = axs[0].imshow(corrMat,vmin=-.5,vmax=.5,cmap='gray') 308 | axs[0].set_title('My correlation matrix',fontweight='bold') 309 | 310 | h2 = axs[1].imshow(corrMat_np,vmin=-.5,vmax=.5,cmap='gray') 311 | axs[1].set_title("Numpy's correlation matrix",fontweight='bold') 312 | 313 | h3 = axs[2].imshow(corrMat_np-corrMat,vmin=-.0005,vmax=.0005,cmap='gray') 314 | axs[2].set_title('Difference matrix',fontweight='bold') 315 | 316 | fig.colorbar(h1,ax=axs[0],fraction=.045) 317 | fig.colorbar(h2,ax=axs[1],fraction=.045) 318 | fig.colorbar(h3,ax=axs[2],fraction=.045) 319 | 320 | plt.tight_layout() 321 | plt.savefig('Figure_07_07.png',dpi=300) 322 | plt.show() 323 | 324 | ??np.corrcoef 325 | 326 | ??np.cov 327 | 328 | 329 | 330 | # Transformation matrix 331 | T = np.array([ 332 | [1,.5], 333 | [0,.5] 334 | ]) 335 | 336 | 337 | # define the set of points (a circle) 338 | theta = np.linspace(0,2*np.pi-2*np.pi/20,20) 339 | origPoints = np.vstack( (np.cos(theta),np.sin(theta)) ) 340 | 341 | # apply transformation 342 | transformedPoints = T @ origPoints 343 | 344 | 345 | # plot the points 346 | plt.figure(figsize=(6,6)) 347 | plt.plot(origPoints[0,:],origPoints[1,:],'ko',label='Original') 348 | plt.plot(transformedPoints[0,:],transformedPoints[1,:],'s', 349 | color=[.7,.7,.7],label='Transformed') 350 | 351 | plt.axis('square') 352 | plt.xlim([-2,2]) 353 | plt.ylim([-2,2]) 354 | plt.legend() 355 | plt.savefig('Figure_07_08.png',dpi=300) 356 | plt.show() 357 | 358 | 359 | 360 | # function to draw the plots 361 | def aframe(ph): 362 | 363 | # create the transformation matrix 364 | T = np.array([ [ 1-ph/3,0 ], 365 | [ 0,ph ] ]) 366 | 367 | # apply the transformation to the points using matrix multiplication 368 | P1 = T@Y1 369 | P2 = T@Y2 370 | 371 | # update the lower/upper lines 372 | plth1.set_xdata(P1[0,:]) 373 | plth1.set_ydata(P1[1,:]) 374 | 375 | plth2.set_xdata(P2[0,:]) 376 | plth2.set_ydata(P2[1,:]) 377 | 378 | # export the plot handles 379 | return (plth1,plth2) 380 | 381 | # define XY points 382 | th = np.linspace(0,2*np.pi,100) # th = theta (angles) 383 | Y1 = np.vstack((th,np.cos(th))) 384 | Y2 = np.vstack((th,np.sin(th))) 385 | 386 | 387 | # setup figure 388 | fig,ax = plt.subplots(1,figsize=(12,6)) 389 | 390 | plth1, = ax.plot(Y1[0,:],Y1[1,:],'ko') 391 | plth2, = ax.plot(Y2[0,:],Y2[1,:],'s',color=[.7,.7,.7]) 392 | ax.set_ylim([-2,2]) 393 | 394 | 395 | # define phases and run animation 396 | phi = 1-np.linspace(-1,1-1/40,40)**2 397 | animation.FuncAnimation(fig, aframe, phi, interval=50, repeat=True) 398 | 399 | 400 | 401 | # initialize smoothed image 402 | smooth_bathtub = np.zeros(bathtub.shape) 403 | 404 | # smooth each layer individually 405 | for i in range(smooth_bathtub.shape[2]): 406 | smooth_bathtub[:,:,i] = convolve2d(bathtub[:,:,i],kernel,mode='same') 407 | 408 | 409 | fig = plt.figure(figsize=(10,6)) 410 | plt.imshow(smooth_bathtub.astype(np.uint8)) 411 | plt.show() 412 | 413 | # check data types 414 | print( smooth_bathtub.dtype ) 415 | print( smooth_bathtub.astype(np.uint8).dtype ) 416 | 417 | # layer-specific kernel widths 418 | kernelN = 31 419 | kernelWidths = [.5,5,50] 420 | 421 | 422 | # initialize smoothed image 423 | smooth_bathtub = np.zeros(bathtub.shape) 424 | 425 | # to show the kernels 426 | _,axs = plt.subplots(1,3,figsize=(12,6)) 427 | 428 | # smooth each layer individually 429 | for i in range(smooth_bathtub.shape[2]): 430 | 431 | # create kernel 432 | Y,X = np.meshgrid(np.linspace(-3,3,kernelN),np.linspace(-3,3,kernelN)) 433 | kernel = np.exp( -(X**2+Y**2) / kernelWidths[i] ) 434 | kernel = kernel / np.sum(kernel) # normalize 435 | 436 | # visualize the kernels 437 | axs[i].imshow(kernel,cmap='gray') 438 | axs[i].set_title(f'Width: {kernelWidths[i]} ({"RGB"[i]} channel)') 439 | 440 | # now run convolution 441 | smooth_bathtub[:,:,i] = convolve2d(bathtub[:,:,i],kernel,mode='same') 442 | 443 | plt.savefig('Figure_07_10.png',dpi=300) 444 | plt.show() # close the kernels figure 445 | 446 | 447 | # show the smoothed image 448 | fig = plt.figure(figsize=(10,6)) 449 | plt.imshow(smooth_bathtub.astype(np.uint8)) 450 | plt.show() 451 | 452 | 453 | 454 | # Create two feature-detection kernels 455 | 456 | # vertical kernel 457 | VK = np.array([ [1,0,-1], 458 | [1,0,-1], 459 | [1,0,-1] ]) 460 | 461 | # horizontal kernel 462 | HK = np.array([ [ 1, 1, 1], 463 | [ 0, 0, 0], 464 | [-1,-1,-1] ]) 465 | 466 | fig,ax = plt.subplots(2,2,figsize=(16,8)) 467 | 468 | ax[0,0].imshow(VK,cmap='gray') 469 | ax[0,0].set_title('Vertical kernel') 470 | ax[0,0].set_yticks(range(3)) 471 | 472 | ax[0,1].imshow(HK,cmap='gray') 473 | ax[0,1].set_title('Horizontal kernel') 474 | ax[0,1].set_yticks(range(3)) 475 | 476 | # run convolution and show the result 477 | convres = convolve2d(bathtub2d,VK,mode='same') 478 | ax[1,0].imshow(convres,cmap='gray',vmin=0,vmax=.01) 479 | ax[1,0].axis('off') 480 | 481 | convres = convolve2d(bathtub2d,HK,mode='same') 482 | ax[1,1].imshow(convres,cmap='gray',vmin=0,vmax=.01) 483 | ax[1,1].axis('off') 484 | 485 | plt.savefig('Figure_07_11.png',dpi=300) 486 | plt.show() 487 | 488 | 489 | -------------------------------------------------------------------------------- /pyFiles/LA4DS_ch08.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | # NOTE: these lines define global figure properties used for publication. 5 | import matplotlib_inline.backend_inline 6 | matplotlib_inline.backend_inline.set_matplotlib_formats('svg') # display figures in vector format 7 | plt.rcParams.update({'font.size':14}) # set global font size 8 | 9 | 10 | 11 | # a matrix 12 | A = np.array([ [1,4],[2,7] ]) 13 | 14 | # its inverse 15 | Ainv = np.linalg.inv(A) 16 | 17 | # confirm that it produces the identity matrix 18 | A@Ainv 19 | 20 | # The matrices visualized 21 | 22 | fig,axs = plt.subplots(1,3,figsize=(10,6)) 23 | 24 | 25 | # the matrix 26 | axs[0].imshow(A,cmap='gray') 27 | axs[0].set_title('The matrix') 28 | for (j,i),num in np.ndenumerate(A): 29 | axs[0].text(i,j,num,color=[.8,.8,.8],ha='center',va='center',fontsize=28) 30 | 31 | # its inverse 32 | axs[1].imshow(Ainv,cmap='gray') 33 | axs[1].set_title('Its inverse') 34 | for (j,i),num in np.ndenumerate(Ainv): 35 | axs[1].text(i,j,num,color=[.8,.8,.8],ha='center',va='center',fontsize=28) 36 | 37 | # their product 38 | AAi = A@Ainv 39 | axs[2].imshow(AAi,cmap='gray') 40 | axs[2].set_title('Their product') 41 | for (j,i),num in np.ndenumerate(AAi): 42 | axs[2].text(i,j,num,color=[.8,.8,.8],ha='center',va='center',fontsize=28) 43 | 44 | 45 | # common properties 46 | for i in range(3): 47 | axs[i].set_xticks([]) 48 | axs[i].set_yticks([]) 49 | 50 | plt.tight_layout() 51 | plt.savefig('Figure_08_01.png',dpi=300) 52 | plt.show() 53 | 54 | 55 | 56 | # the full inverse is two-sided 57 | print( A@Ainv ), print(' ') 58 | print( Ainv@A ) 59 | 60 | # reminder to use the correct operator: 61 | A*Ainv # Hadamard multiplication! 62 | 63 | # try again with a singular matrix 64 | A = np.array([ [1,4],[2,8] ]) 65 | 66 | # its inverse 67 | Ainv = np.linalg.inv(A) 68 | 69 | # does it produce the identity matrix? 70 | A@Ainv 71 | 72 | 73 | 74 | D = np.diag( np.arange(1,6) ) 75 | Dinv = np.linalg.inv(D) 76 | 77 | print('The diagonal matrix:') 78 | print(D), print(' ') 79 | 80 | print('Its inverse:') 81 | print(Dinv), print(' ') 82 | 83 | print('Their product:') 84 | print(D@Dinv) 85 | 86 | 87 | 88 | # making an invertible square matrix from a tall full column-rank matrix 89 | 90 | # here's a tall matrix. 91 | T = np.random.randint(-10,11,size=(40,4)) 92 | 93 | # confirm that it has its maximum possible rank (full column-rank) 94 | print( f'This matrix has rank={np.linalg.matrix_rank(T)}\n\n' ) 95 | 96 | # next, create a square full-rank matrix 97 | TtT = T.T@T 98 | 99 | # check whether it has an inverse 100 | TtT_inv = np.linalg.inv(TtT) 101 | print( np.round(TtT_inv@TtT,4) ) 102 | 103 | # finish creating the left-inverse 104 | 105 | # our left-inverse 106 | L = TtT_inv @ T.T 107 | 108 | # confirm that it works 109 | print( np.round( L@T,6 ) ), print(' ') 110 | 111 | # but it's one-sided! 112 | print( np.round( T@L,6 ) ) 113 | 114 | 115 | # visualize! of course :) 116 | 117 | fig,axs = plt.subplots(2,2,figsize=(10,10)) 118 | 119 | axs[0,0].imshow(T,cmap='gray') 120 | axs[0,0].set_title('Tall matrix') 121 | 122 | axs[0,1].imshow(L,cmap='gray') 123 | axs[0,1].set_title('Left inverse') 124 | 125 | axs[1,0].imshow(L@T,cmap='gray') 126 | axs[1,0].set_title('L@T') 127 | 128 | axs[1,1].imshow(T@L,cmap='gray') 129 | axs[1,1].set_title('T@L') 130 | 131 | for a in axs.flatten(): 132 | a.set_xticks([]) 133 | a.set_yticks([]) 134 | 135 | plt.tight_layout() 136 | plt.savefig('Figure_08_04.png',dpi=300) 137 | plt.show() 138 | 139 | 140 | 141 | 142 | 143 | # The same singular matrix as before 144 | A = np.array([ [1,4],[2,8] ]) 145 | 146 | # its inverse 147 | Apinv = np.linalg.pinv(A) 148 | print(Apinv*85), print(' ') 149 | 150 | # does it produce the identity matrix? 151 | A@Apinv 152 | 153 | # an exmple with random numbers 154 | A = np.random.randn(7,5) @ np.random.randn(5,7) 155 | print(f'The rank of this matrix is {np.linalg.matrix_rank(A)}.\n') 156 | 157 | Apinv = np.linalg.pinv(A) 158 | plt.imshow(A@Apinv) 159 | plt.title('The matrix times its pinv') 160 | plt.colorbar() 161 | plt.show() 162 | 163 | 164 | 165 | n = 5 166 | 167 | # the matrix 168 | A = np.random.randn(n,n) 169 | 170 | # its inverse, and its inverse's inverse 171 | Ai = np.linalg.inv(A) 172 | Aii = np.linalg.inv(Ai) 173 | 174 | # equal the original matrix within tolerance 175 | np.round( A-Aii ,10) 176 | 177 | 178 | 179 | # create matrix 180 | m = 4 181 | A = np.random.randn(m,m) 182 | 183 | # initialize 184 | M = np.zeros((m,m)) # minors matrix 185 | G = np.zeros((m,m)) # grid matrix 186 | 187 | # compute minors matrix 188 | for i in range(m): 189 | for j in range(m): 190 | 191 | # select rows and cols 192 | rows = [True]*m 193 | rows[i] = False 194 | 195 | cols = [True]*m 196 | cols[j] = False 197 | 198 | # compute the minors 199 | M[i,j]=np.linalg.det(A[rows,:][:,cols]) 200 | 201 | # compute Grid 202 | G[i,j] = (-1)**(i+j) 203 | 204 | 205 | # compute cofactors matrix 206 | C = M * G 207 | 208 | # compute adjugate matrix 209 | Ainv = C.T / np.linalg.det(A) 210 | 211 | # 'regular' inverse function 212 | AinvI = np.linalg.inv(A) 213 | 214 | # compare against inv() 215 | np.round( AinvI-Ainv ,8) 216 | 217 | # plot them 218 | 219 | fig,axs = plt.subplots(2,3,figsize=(14,7)) 220 | 221 | axs[0,0].imshow(M,cmap='gray') 222 | axs[0,0].set_title('Minors matrix') 223 | 224 | axs[0,1].imshow(G,cmap='gray') 225 | axs[0,1].set_title('Grid matrix') 226 | 227 | axs[0,2].imshow(C,cmap='gray') 228 | axs[0,2].set_title('Cofactors matrix') 229 | 230 | axs[1,0].imshow(Ainv,cmap='gray') 231 | axs[1,0].set_title('Adjugate (inverse)') 232 | 233 | axs[1,1].imshow(AinvI,cmap='gray') 234 | axs[1,1].set_title('np.linalg.inv') 235 | 236 | axs[1,2].imshow(A@Ainv,cmap='gray') 237 | axs[1,2].set_title('A@Ainv') 238 | 239 | for a in axs.flatten(): 240 | a.set_xticks([]) 241 | a.set_yticks([]) 242 | 243 | 244 | plt.savefig('Figure_08_03.png',dpi=300) 245 | plt.show() 246 | 247 | 248 | 249 | # Start from the code for the left-inverse, and swap as necessary. 250 | 251 | # here's a wide matrix. 252 | W = np.random.randint(-10,11,size=(4,40)) 253 | 254 | # confirm that it has its maximum possible rank (full column-rank) 255 | print( f'This matrix has rank={np.linalg.matrix_rank(T)}\n\n' ) 256 | 257 | # next, create a square full-rank matrix 258 | WWt = W@W.T 259 | 260 | # check whether it has an inverse 261 | WWt_inv = np.linalg.inv(WWt) 262 | print( np.round(WWt_inv@WWt,4) ) 263 | 264 | # finish creating the right-inverse 265 | 266 | # our left-inverse 267 | R = W.T @ WWt_inv 268 | 269 | # confirm that it works 270 | print( np.round( W@R,6 ) ), print(' ') 271 | 272 | # but it's one-sided! 273 | print( np.round( R@W,6 ) ) 274 | 275 | 276 | # visualize! of course :) 277 | 278 | fig,axs = plt.subplots(2,2,figsize=(10,10)) 279 | 280 | axs[0,0].imshow(W,cmap='gray') 281 | axs[0,0].set_title('Wide matrix') 282 | 283 | axs[0,1].imshow(R,cmap='gray') 284 | axs[0,1].set_title('Right inverse') 285 | 286 | axs[1,0].imshow(R@W,cmap='gray') 287 | axs[1,0].set_title('R@W') 288 | 289 | axs[1,1].imshow(W@R,cmap='gray') 290 | axs[1,1].set_title('W@R') 291 | 292 | for a in axs.flatten(): 293 | a.set_xticks([]) 294 | a.set_yticks([]) 295 | 296 | plt.tight_layout() 297 | plt.show() 298 | 299 | 300 | 301 | # Full inverse case 302 | M = 4 303 | 304 | A = np.random.randn(M,M) 305 | 306 | Ainv = np.linalg.inv(A) 307 | Apinv = np.linalg.pinv(A) 308 | 309 | np.round( Ainv-Apinv,10 ) 310 | 311 | # left inverse case 312 | M,N = 14,4 313 | 314 | A = np.random.randn(M,N) 315 | 316 | ALeft = np.linalg.inv(A.T@A) @ A.T 317 | Apinv = np.linalg.pinv(A) 318 | 319 | np.round( ALeft-Apinv,10 ) 320 | 321 | # right inverse case 322 | M,N = 4,14 323 | 324 | A = np.random.randn(M,N) 325 | 326 | ARight = A.T @ np.linalg.inv(A@A.T) 327 | Apinv = np.linalg.pinv(A) 328 | 329 | np.round( ARight-Apinv,10 ) 330 | 331 | 332 | 333 | # create the matrices 334 | N = 4 335 | A = np.random.randn(N,N) 336 | B = np.random.randn(N,N) 337 | 338 | # compute the three specified options 339 | op1 = np.linalg.inv(A@B) 340 | op2 = np.linalg.inv(A) @ np.linalg.inv(B) 341 | op3 = np.linalg.inv(B) @ np.linalg.inv(A) 342 | 343 | # compute distances 344 | dist12 = np.sqrt(np.sum( (op1-op2)**2 )) 345 | dist13 = np.sqrt(np.sum( (op1-op3)**2 )) 346 | 347 | # print results! 348 | print(f'Distance between (AB)^-1 and (A^-1)(B^-1) is {dist12:.8f}') 349 | print(f'Distance between (AB)^-1 and (B^-1)(A^-1) is {dist13:.8f}') 350 | 351 | 352 | 353 | # create the matrices 354 | M,N = 14,4 355 | T = np.random.randn(M,N) 356 | 357 | # compute the three specified options 358 | op1 = np.linalg.inv(T.T@T) 359 | op2 = np.linalg.inv(T) @ np.linalg.inv(T.T) 360 | 361 | # The answer is No, it doesn't work, because a tall matrix has no inverse. 362 | 363 | 364 | 365 | # Transformation matrix 366 | T = np.array([ 367 | [1,.5], 368 | [0,.5] 369 | ]) 370 | 371 | # Compute its inverse 372 | Ti = np.linalg.inv(T) 373 | 374 | 375 | # define the set of points (a circle) 376 | theta = np.linspace(0,2*np.pi-2*np.pi/20,20) 377 | origPoints = np.vstack( (np.cos(theta),np.sin(theta)) ) 378 | 379 | # apply transformation 380 | transformedPoints = T @ origPoints 381 | 382 | # undo the transformation via the inverse of the transform 383 | backTransformed = Ti @ transformedPoints 384 | 385 | 386 | # plot the points 387 | plt.figure(figsize=(6,6)) 388 | plt.plot(origPoints[0,:],origPoints[1,:],'ko',label='Original') 389 | plt.plot(transformedPoints[0,:],transformedPoints[1,:],'s', 390 | color=[.7,.7,.7],label='Transformed') 391 | plt.plot(backTransformed[0,:],backTransformed[1,:],'rx',markersize=15, 392 | color=[.7,.7,.7],label='Inverse-transformed') 393 | 394 | plt.axis('square') 395 | plt.xlim([-2,2]) 396 | plt.ylim([-2,2]) 397 | plt.legend() 398 | plt.savefig('Figure_08_06.png',dpi=300) 399 | plt.show() 400 | 401 | 402 | 403 | # a function to create a Hilbert matrix 404 | def hilbmat(k): 405 | H = np.zeros((k,k)) 406 | for i in range(k): 407 | for j in range(k): 408 | 409 | # note: the math formula has denominator: i+j-1 410 | # but with 0-based indexing, this is: (i+1)+(j+1)-1 411 | # which can be shortened to: i+j+1 412 | 413 | H[i,j] = 1 / (i+j+1) 414 | return H 415 | 416 | 417 | 418 | # The double for-loop above is a direct implementation of the math. 419 | # The function below gives the same result but without the loops. 420 | def hilbmat(k): 421 | k = np.arange(1,k+1).reshape(1,-1) # reshape to a row vector (instead of a 1D array) 422 | return 1 / (k.T+k-1) # outer product and element-wise division 423 | 424 | print( hilbmat(5) ), print(' ') 425 | 426 | # you can confirm the accuracy of your function against the scipy Hilbert-matrix function: 427 | from scipy.linalg import hilbert 428 | print( hilbert(5) ) 429 | 430 | # create a 5x5 Hilbert matrix and show it, its inverse, and their product 431 | H = hilbmat(5) 432 | Hi = np.linalg.inv(H) 433 | 434 | fig,axs = plt.subplots(1,3,figsize=(12,6)) 435 | h = [0,0,0] 436 | 437 | # the matrix 438 | h[0] = axs[0].imshow(H,cmap='gray') 439 | axs[0].set_title('Hilbert matrix') 440 | 441 | # its inverse 442 | h[1] = axs[1].imshow(Hi,cmap='gray') 443 | axs[1].set_title('Inv(hilbert)') 444 | 445 | # their product 446 | h[2] = axs[2].imshow(H@Hi,cmap='gray') 447 | axs[2].set_title('Their product') 448 | 449 | 450 | for i in range(2): 451 | fig.colorbar(h[i],ax=axs[i],fraction=.045) 452 | axs[i].set_xticks([]) 453 | axs[i].set_yticks([]) 454 | 455 | plt.tight_layout() 456 | plt.savefig('Figure_08_05.png',dpi=300) 457 | plt.show() 458 | 459 | 460 | 461 | 462 | matSizes = np.arange(3,13) 463 | 464 | identityError = np.zeros((len(matSizes),2)) 465 | condNumbers = np.zeros((len(matSizes),2)) 466 | 467 | 468 | for i,k in enumerate(matSizes): 469 | 470 | ### for the Hilbert matrix 471 | H = hilbmat(k) # the matrix 472 | Hi = np.linalg.inv(H) # its inverse 473 | HHi = H@Hi # should be identity 474 | err = HHi - np.eye(k) # difference from true identity 475 | identityError[i,0] = np.sqrt(np.sum(err**2)) # Euclidean distance 476 | condNumbers[i,0] = np.linalg.cond(H) # condition number 477 | 478 | 479 | ### repeat for a random matrix 480 | H = np.random.randn(k,k) # the matrix 481 | Hi = np.linalg.inv(H) # its inverse 482 | HHi = H@Hi # should be identity 483 | err = HHi - np.eye(k) # difference from true identity 484 | identityError[i,1] = np.sqrt(np.sum(err**2)) # Euclidean distance 485 | condNumbers[i,1] = np.linalg.cond(H) # condition number 486 | 487 | 488 | 489 | # now plot 490 | fig,axs = plt.subplots(1,2,figsize=(14,5)) 491 | 492 | ## plot the Euclidean distance to the identity matrix 493 | h = axs[0].plot(matSizes,np.log(identityError),'s-',markersize=12) 494 | h[0].set_color('k') # adjust the individual line colors and shapes 495 | h[0].set_marker('o') 496 | h[1].set_color('gray') 497 | 498 | axs[0].legend(['Hilbert','Random']) 499 | axs[0].set_xlabel('Matrix size') 500 | axs[0].set_ylabel('Log Euclidan distance') 501 | axs[0].set_title('Distance to identity matrix') 502 | 503 | 504 | 505 | ## plot the condition numbers 506 | h = axs[1].plot(matSizes,np.log(condNumbers),'s-',markersize=12) 507 | h[0].set_color('k') # adjust the individual line colors and shapes 508 | h[0].set_marker('o') 509 | h[1].set_color('gray') 510 | 511 | axs[1].legend(['Hilbert','Random']) 512 | axs[1].set_xlabel('Matrix size') 513 | axs[1].set_ylabel('Log Kappa') 514 | axs[1].set_title('Matrix condition number') 515 | 516 | plt.savefig('Figure_08_07.png',dpi=300) 517 | plt.show() 518 | 519 | ## interesting to see the "identity" matrix 520 | H = hilbmat(k) 521 | Hi = np.linalg.inv(H) 522 | HHi = H@Hi 523 | 524 | plt.imshow(HHi,vmin=0,vmax=.1) 525 | plt.title('Should be identity') 526 | plt.colorbar(); 527 | 528 | 529 | -------------------------------------------------------------------------------- /pyFiles/LA4DS_ch09.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | 4 | # used to create non-regular subplots 5 | import matplotlib.gridspec as gridspec 6 | 7 | 8 | # NOTE: these lines define global figure properties used for publication. 9 | import matplotlib_inline.backend_inline 10 | matplotlib_inline.backend_inline.set_matplotlib_formats('svg') # display figures in vector format 11 | plt.rcParams.update({'font.size':14}) # set global font size 12 | 13 | 14 | 15 | # specify the matrices 16 | Q1 = np.array([ [1,-1],[1,1] ]) / np.sqrt(2) 17 | Q2 = np.array([ [1,2,2],[2,1,-2],[-2,2,-1] ]) / 3 18 | 19 | # should be I (to within rounding error...) 20 | print( np.round(Q1.T @ Q1,8) ), print(' ') 21 | 22 | print( np.round(Q2.T @ Q2,8) ) 23 | 24 | 25 | 26 | # create a random matrix 27 | A = np.random.randn(6,6) 28 | 29 | # QR decomposition 30 | Q,R = np.linalg.qr(A) 31 | 32 | 33 | 34 | # show the matrices 35 | fig = plt.figure(figsize=(10,6)) 36 | axs = [0]*5 37 | c = 1.5 # color limits 38 | 39 | gs1 = gridspec.GridSpec(2,6) 40 | axs[0] = plt.subplot(gs1[0,:2]) 41 | axs[0].imshow(A,vmin=-c,vmax=c,cmap='gray') 42 | axs[0].set_title('A',fontweight='bold') 43 | 44 | axs[1] = plt.subplot(gs1[0,2:4]) 45 | axs[1].imshow(Q,vmin=-c,vmax=c,cmap='gray') 46 | axs[1].set_title('Q',fontweight='bold') 47 | 48 | axs[2] = plt.subplot(gs1[0,4:6]) 49 | axs[2].imshow(R,vmin=-c,vmax=c,cmap='gray') 50 | axs[2].set_title('R',fontweight='bold') 51 | 52 | axs[3] = plt.subplot(gs1[1,1:3]) 53 | axs[3].imshow(A - Q@R,vmin=-c,vmax=c,cmap='gray') 54 | axs[3].set_title('A - QR',fontweight='bold') 55 | 56 | axs[4] = plt.subplot(gs1[1,3:5]) 57 | axs[4].imshow(Q.T@Q,cmap='gray') 58 | axs[4].set_title(r'Q$^T$Q',fontweight='bold') 59 | 60 | # remove ticks from all axes 61 | for a in axs: 62 | a.set_xticks([]) 63 | a.set_yticks([]) 64 | 65 | plt.tight_layout() 66 | plt.savefig('Figure_09_01.png',dpi=300) 67 | plt.show() 68 | 69 | 70 | 71 | # QR and matrix sizes 72 | 73 | M = 4 74 | N = 14 75 | 76 | A = np.random.randn(M,N) 77 | Q,R = np.linalg.qr(A) 78 | 79 | # print the results 80 | print(f'Size of A (M,N): {A.shape}') 81 | print(f'Size of Q (M,N): {Q.shape}') 82 | print(f'Size of R (M,N): {R.shape}') 83 | 84 | # illustration of full Q from M>N A 85 | 86 | A = np.array([ [1,-1] ]).T 87 | 88 | Q,R = np.linalg.qr(A,'complete') 89 | Q*np.sqrt(2) 90 | 91 | 92 | 93 | 94 | 95 | # compute matrices 96 | Q = np.linalg.qr( np.random.randn(5,5) )[0] 97 | Qt = Q.T 98 | Qi = np.linalg.inv( Q ) 99 | 100 | # QtQ 101 | print(np.round( Qt@Q,8 )), print(' ') 102 | 103 | # QQt 104 | print(np.round( Q@Qt,8 )), print(' ') 105 | 106 | # Q^-1 Q 107 | print(np.round( Qi@Q,8 )), print(' ') 108 | 109 | # QQ^-1 110 | print(np.round( Q@Qi,8 )) 111 | 112 | 113 | 114 | 115 | # create the matrix 116 | m = 4 117 | n = 4 118 | A = np.random.randn(m,n) 119 | 120 | # initialize 121 | Q = np.zeros((m,n)) 122 | 123 | 124 | # the GS algo 125 | for i in range(n): 126 | 127 | # initialize 128 | Q[:,i] = A[:,i] 129 | 130 | # orthogonalize 131 | a = A[:,i] # convenience 132 | for j in range(i): # only to earlier cols 133 | q = Q[:,j] # convenience 134 | Q[:,i]=Q[:,i]-np.dot(a,q)/np.dot(q,q)*q 135 | 136 | # normalize 137 | Q[:,i] = Q[:,i] / np.linalg.norm(Q[:,i]) 138 | 139 | 140 | # "real" QR decomposition for comparison 141 | Q2,R = np.linalg.qr(A) 142 | 143 | 144 | # note the possible sign differences. 145 | # seemingly non-zero columns will be 0 when adding 146 | print( np.round( Q-Q2 ,10) ), print(' ') 147 | print( np.round( Q+Q2 ,10) ) 148 | 149 | 150 | 151 | # create an orthogonal matrix, call it U (to avoid confusing with Q) 152 | U = np.linalg.qr( np.random.randn(6,6) )[0] 153 | 154 | 155 | # part 2: modulate the column norms 156 | for i in range(U.shape[0]): 157 | U[:,i] = U[:,i]*(10+i) 158 | 159 | 160 | # part 3: Change one matrix element 161 | U[0,3] = 0 # this is q_{1,4} 162 | 163 | 164 | # QR decomp 165 | q,r = np.linalg.qr(U) 166 | 167 | # show R and, for part 2, Q'Q 168 | print( np.round(r,3) ), print(' ') 169 | # print( np.round(Q.T@Q,4)) 170 | 171 | 172 | 173 | # a function to compute the inverse 174 | def oldSchoolInv(A): 175 | 176 | # matrix size 177 | m = A.shape[0] 178 | 179 | 180 | # abort if non-square 181 | if not np.diff(A.shape)[0]==0: 182 | raise Exception('Matrix must be square.') 183 | 184 | # abort if singular 185 | if np.linalg.matrix_rank(A)tol] = 1/s[s>tol] 260 | 261 | # reconstruct 262 | S = np.zeros_like(A) 263 | np.fill_diagonal(S,sInv) 264 | Apinv = Vt.T @ S @ U.T 265 | 266 | # compare to pinv() 267 | ApinvNp = np.linalg.pinv(A) 268 | 269 | print(np.round( ApinvNp - Apinv ,5)) 270 | 271 | # check the source code for pinv 272 | ??np.linalg.pinv() 273 | 274 | 275 | 276 | # left-inverse 277 | A = np.random.randn(6,4) 278 | 279 | # explicit left inverse 280 | Linv = np.linalg.inv(A.T@A)@A.T 281 | 282 | # pinv 283 | Apinv = np.linalg.pinv(A) 284 | 285 | # compare 286 | print(np.round( Linv - Apinv ,5)) 287 | 288 | # right-inverse 289 | A = np.random.randn(4,6) 290 | 291 | # explicit right inverse 292 | Rinv = A.T@np.linalg.inv(A@A.T) 293 | 294 | # pinv 295 | Apinv = np.linalg.pinv(A) 296 | 297 | # compare 298 | print(np.round( Rinv - Apinv ,5)) 299 | 300 | 301 | 302 | # the matrix (from chapter 12) 303 | M = np.array([ [-1,1], 304 | [-1,2] ]) 305 | 306 | # its eigendecomposition 307 | evals,evecs = np.linalg.eig(M) 308 | l = evals[1] # extract lambda1 for convenience 309 | v = evecs[:,[1]] # extract v1 for convenience 310 | 311 | LHS = M@v 312 | RHS = l*v 313 | 314 | # print out the two sides (as row vectors for visual convenience) 315 | print(LHS.T) 316 | print(RHS.T) 317 | 318 | # pinv(v) 319 | vPinv = np.linalg.pinv(v) 320 | 321 | # check 322 | vPinv@v 323 | 324 | # first equation 325 | LHS = vPinv @ M @ v 326 | RHS = l * vPinv @ v 327 | 328 | # these results are scalars (quadratic form) 329 | print(LHS) 330 | print(RHS) 331 | 332 | # second equation 333 | LHS = M @ v @ vPinv 334 | RHS = l * v @ vPinv 335 | 336 | # these results are matrices 337 | print(LHS), print(' ') 338 | print(RHS) 339 | 340 | 341 | --------------------------------------------------------------------------------