├── Getting Data.ipynb ├── Modern Python Idioms.ipynb ├── Optimizing Python.ipynb ├── README.md ├── Session 4.ipynb ├── example.csv ├── example.json ├── hanoi.py ├── nyct_ene.xml ├── parallel python.ipynb └── thousand_signs.csv /Getting Data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "Getting Data" 4 | }, 5 | "nbformat": 3, 6 | "nbformat_minor": 0, 7 | "worksheets": [ 8 | { 9 | "cells": [ 10 | { 11 | "cell_type": "heading", 12 | "level": 2, 13 | "metadata": {}, 14 | "source": [ 15 | "XML" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "Simple example from http://software-carpentry.org/3_0/xml.html\n", 23 | "\n", 24 | " \n", 25 | " \n", 26 | " 87.97\n", 27 | " \n", 28 | "\n", 29 | "XML files are best viewed as trees" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "metadata": {}, 35 | "source": [ 36 | "Very common structured data format, supporting hierarchical, nested data with metadata." 37 | ] 38 | }, 39 | { 40 | "cell_type": "markdown", 41 | "metadata": {}, 42 | "source": [ 43 | "Recommended libraries:\n", 44 | "\n", 45 | "- `lxml` (Performance)\n", 46 | "- `untangle` (Simplicity)" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "`$ pip install untangle`" 54 | ] 55 | }, 56 | { 57 | "cell_type": "heading", 58 | "level": 3, 59 | "metadata": {}, 60 | "source": [ 61 | "untangle" 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "Example data is an XML file with information about outages on escalators and elevators in the New York subway system" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "collapsed": false, 74 | "input": [ 75 | "xml_file = \"nyct_ene.xml\"" 76 | ], 77 | "language": "python", 78 | "metadata": {}, 79 | "outputs": [], 80 | "prompt_number": 1 81 | }, 82 | { 83 | "cell_type": "code", 84 | "collapsed": false, 85 | "input": [ 86 | "import untangle" 87 | ], 88 | "language": "python", 89 | "metadata": {}, 90 | "outputs": [], 91 | "prompt_number": 2 92 | }, 93 | { 94 | "cell_type": "code", 95 | "collapsed": false, 96 | "input": [ 97 | "doc = untangle.parse(xml_file)" 98 | ], 99 | "language": "python", 100 | "metadata": {}, 101 | "outputs": [], 102 | "prompt_number": 3 103 | }, 104 | { 105 | "cell_type": "code", 106 | "collapsed": false, 107 | "input": [ 108 | "doc.get_elements()" 109 | ], 110 | "language": "python", 111 | "metadata": {}, 112 | "outputs": [ 113 | { 114 | "output_type": "pyout", 115 | "prompt_number": 4, 116 | "text": [ 117 | "[Element(name = NYCOutages, attributes = {}, cdata = )]" 118 | ] 119 | } 120 | ], 121 | "prompt_number": 4 122 | }, 123 | { 124 | "cell_type": "code", 125 | "collapsed": false, 126 | "input": [ 127 | "doc." 128 | ], 129 | "language": "python", 130 | "metadata": {}, 131 | "outputs": [ 132 | { 133 | "ename": "SyntaxError", 134 | "evalue": "invalid syntax (, line 1)", 135 | "output_type": "pyerr", 136 | "traceback": [ 137 | "\u001b[0;36m File \u001b[0;32m\"\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m doc.\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n" 138 | ] 139 | } 140 | ], 141 | "prompt_number": 5 142 | }, 143 | { 144 | "cell_type": "code", 145 | "collapsed": false, 146 | "input": [ 147 | "outages = doc.NYCOutages.outage" 148 | ], 149 | "language": "python", 150 | "metadata": {}, 151 | "outputs": [], 152 | "prompt_number": 6 153 | }, 154 | { 155 | "cell_type": "code", 156 | "collapsed": false, 157 | "input": [ 158 | "len(outages)" 159 | ], 160 | "language": "python", 161 | "metadata": {}, 162 | "outputs": [ 163 | { 164 | "output_type": "pyout", 165 | "prompt_number": 7, 166 | "text": [ 167 | "38" 168 | ] 169 | } 170 | ], 171 | "prompt_number": 7 172 | }, 173 | { 174 | "cell_type": "code", 175 | "collapsed": false, 176 | "input": [ 177 | "outage = outages[5]" 178 | ], 179 | "language": "python", 180 | "metadata": {}, 181 | "outputs": [], 182 | "prompt_number": 8 183 | }, 184 | { 185 | "cell_type": "code", 186 | "collapsed": false, 187 | "input": [ 188 | "outage.get_elements()" 189 | ], 190 | "language": "python", 191 | "metadata": {}, 192 | "outputs": [ 193 | { 194 | "output_type": "pyout", 195 | "prompt_number": 9, 196 | "text": [ 197 | "[Element(name = station, attributes = {}, cdata = 181 ST STATION),\n", 198 | " Element(name = borough, attributes = {}, cdata = MN),\n", 199 | " Element(name = trainno, attributes = {}, cdata = 1),\n", 200 | " Element(name = equipment, attributes = {}, cdata = EL110),\n", 201 | " Element(name = equipmenttype, attributes = {}, cdata = EL),\n", 202 | " Element(name = serving, attributes = {}, cdata = LOWER MEZZANINE TO UPPER MEZZANINE),\n", 203 | " Element(name = ADA, attributes = {}, cdata = N),\n", 204 | " Element(name = outagedate, attributes = {}, cdata = 12/27/2012 8:07:00 AM),\n", 205 | " Element(name = estimatedreturntoservice, attributes = {}, cdata = 12/29/2012 12:00:00 AM),\n", 206 | " Element(name = reason, attributes = {}, cdata = REPAIR),\n", 207 | " Element(name = isupcomingoutage, attributes = {}, cdata = N),\n", 208 | " Element(name = ismaintenanceoutage, attributes = {}, cdata = N)]" 209 | ] 210 | } 211 | ], 212 | "prompt_number": 9 213 | }, 214 | { 215 | "cell_type": "code", 216 | "collapsed": false, 217 | "input": [ 218 | "outage.estimatedreturntoservice.cdata" 219 | ], 220 | "language": "python", 221 | "metadata": {}, 222 | "outputs": [ 223 | { 224 | "output_type": "pyout", 225 | "prompt_number": 10, 226 | "text": [ 227 | "u'12/29/2012 12:00:00 AM'" 228 | ] 229 | } 230 | ], 231 | "prompt_number": 10 232 | }, 233 | { 234 | "cell_type": "markdown", 235 | "metadata": {}, 236 | "source": [ 237 | "##CSV" 238 | ] 239 | }, 240 | { 241 | "cell_type": "markdown", 242 | "metadata": {}, 243 | "source": [ 244 | "Means a file where data just have some sort of separator" 245 | ] 246 | }, 247 | { 248 | "cell_type": "markdown", 249 | "metadata": {}, 250 | "source": [ 251 | "Example, small sample of http://a841-dotweb01.nyc.gov/datafeeds/ParkingReg/signs.CSV (88 MB)" 252 | ] 253 | }, 254 | { 255 | "cell_type": "code", 256 | "collapsed": false, 257 | "input": [ 258 | "%%bash\n", 259 | "head thousand_signs.csv" 260 | ], 261 | "language": "python", 262 | "metadata": {}, 263 | "outputs": [ 264 | { 265 | "output_type": "stream", 266 | "stream": "stdout", 267 | "text": [ 268 | "B,P-004958,1,0000 , ,Curb Line\r\n", 269 | "B,P-004958,2,0009 , ,Property Line\r\n", 270 | "B,P-004958,3,0030 , ,NIGHT REGULATION (MOON & STARS SYMBOLS) NO PARKING (SANITATION BROOM SYMBOL) MIDNIGHT TO 3AM TUES & FRI <--> (SUPERSEDED BY SP-841C) \r\n", 271 | "B,P-004958,4,0030 , ,1 HOUR PARKING 9AM-7PM EXCEPT SUNDAY \r\n", 272 | "B,P-004958,5,0208 , ,NIGHT REGULATION (MOON & STARS SYMBOLS) NO PARKING (SANITATION BROOM SYMBOL) MIDNIGHT TO 3AM TUES & FRI <--> (SUPERSEDED BY SP-841C) \r\n", 273 | "B,P-004958,6,0208 , ,1 HOUR PARKING 9AM-7PM EXCEPT SUNDAY \r\n", 274 | "B,P-004958,7,0218 , ,Property Line\r\n", 275 | "B,P-004958,8,0232 , ,Curb Line\r\n", 276 | "B,P-009318,1,0000 , ,Curb Line\r\n", 277 | "B,P-009318,2,0014 , ,Building Line\r\n" 278 | ] 279 | } 280 | ], 281 | "prompt_number": 11 282 | }, 283 | { 284 | "cell_type": "markdown", 285 | "metadata": {}, 286 | "source": [ 287 | "Builtin module `csv`\n", 288 | "\n", 289 | "Primary documentation: http://docs.python.org/2.7/library/csv.html" 290 | ] 291 | }, 292 | { 293 | "cell_type": "code", 294 | "collapsed": false, 295 | "input": [ 296 | "# Simple reading\n", 297 | "\n", 298 | "f = open(\"thousand_signs.csv\", \"r\")\n", 299 | "for _ in range(3):\n", 300 | " print(repr(f.readline()))" 301 | ], 302 | "language": "python", 303 | "metadata": {}, 304 | "outputs": [ 305 | { 306 | "output_type": "stream", 307 | "stream": "stdout", 308 | "text": [ 309 | "'B,P-004958,1,0000 , ,Curb Line\\r\\n'\n", 310 | "'B,P-004958,2,0009 , ,Property Line\\r\\n'\n", 311 | "'B,P-004958,3,0030 , ,NIGHT REGULATION (MOON & STARS SYMBOLS) NO PARKING (SANITATION BROOM SYMBOL) MIDNIGHT TO 3AM TUES & FRI <--> (SUPERSEDED BY SP-841C) \\r\\n'\n" 312 | ] 313 | } 314 | ], 315 | "prompt_number": 12 316 | }, 317 | { 318 | "cell_type": "code", 319 | "collapsed": false, 320 | "input": [ 321 | "import csv" 322 | ], 323 | "language": "python", 324 | "metadata": {}, 325 | "outputs": [], 326 | "prompt_number": 13 327 | }, 328 | { 329 | "cell_type": "code", 330 | "collapsed": false, 331 | "input": [ 332 | "# Using a CSV reader\n", 333 | "\n", 334 | "f.seek(0)\n", 335 | "\n", 336 | "reader = csv.reader(f)\n", 337 | "reader" 338 | ], 339 | "language": "python", 340 | "metadata": {}, 341 | "outputs": [ 342 | { 343 | "output_type": "pyout", 344 | "prompt_number": 14, 345 | "text": [ 346 | "<_csv.reader at 0x1084d54b0>" 347 | ] 348 | } 349 | ], 350 | "prompt_number": 14 351 | }, 352 | { 353 | "cell_type": "code", 354 | "collapsed": false, 355 | "input": [ 356 | "for _ in range(3):\n", 357 | " print(repr(reader.next()))" 358 | ], 359 | "language": "python", 360 | "metadata": {}, 361 | "outputs": [ 362 | { 363 | "output_type": "stream", 364 | "stream": "stdout", 365 | "text": [ 366 | "['B', 'P-004958', '1', '0000 ', ' ', 'Curb Line']\n", 367 | "['B', 'P-004958', '2', '0009 ', ' ', 'Property Line']\n", 368 | "['B', 'P-004958', '3', '0030 ', ' ', 'NIGHT REGULATION (MOON & STARS SYMBOLS) NO PARKING (SANITATION BROOM SYMBOL) MIDNIGHT TO 3AM TUES & FRI <--> (SUPERSEDED BY SP-841C) ']\n" 369 | ] 370 | } 371 | ], 372 | "prompt_number": 15 373 | }, 374 | { 375 | "cell_type": "code", 376 | "collapsed": false, 377 | "input": [ 378 | "f.close()" 379 | ], 380 | "language": "python", 381 | "metadata": {}, 382 | "outputs": [], 383 | "prompt_number": 16 384 | }, 385 | { 386 | "cell_type": "markdown", 387 | "metadata": {}, 388 | "source": [ 389 | "Rows are split in to lists based on the seperator on the fly" 390 | ] 391 | }, 392 | { 393 | "cell_type": "markdown", 394 | "metadata": {}, 395 | "source": [ 396 | "### Dialects" 397 | ] 398 | }, 399 | { 400 | "cell_type": "markdown", 401 | "metadata": {}, 402 | "source": [ 403 | "There's no clear standard for the which characters are used for separation of columns and rows.\n", 404 | "\n", 405 | "The specific setting of formatting is called a **dialect**\n", 406 | "\n", 407 | "The default in `csv` is to use `,` for columns and `\\r\\n` for rows." 408 | ] 409 | }, 410 | { 411 | "cell_type": "code", 412 | "collapsed": false, 413 | "input": [ 414 | "csv.list_dialects()" 415 | ], 416 | "language": "python", 417 | "metadata": {}, 418 | "outputs": [ 419 | { 420 | "output_type": "pyout", 421 | "prompt_number": 17, 422 | "text": [ 423 | "['excel-tab', 'excel']" 424 | ] 425 | } 426 | ], 427 | "prompt_number": 17 428 | }, 429 | { 430 | "cell_type": "markdown", 431 | "metadata": {}, 432 | "source": [ 433 | "`excel-tab` uses `\\t` for column separations" 434 | ] 435 | }, 436 | { 437 | "cell_type": "markdown", 438 | "metadata": {}, 439 | "source": [ 440 | "If we want to separate fields which internally uses `,`, we could for example use `;`\n", 441 | "as a separator." 442 | ] 443 | }, 444 | { 445 | "cell_type": "code", 446 | "collapsed": false, 447 | "input": [ 448 | "%%bash\n", 449 | "head example.csv" 450 | ], 451 | "language": "python", 452 | "metadata": {}, 453 | "outputs": [ 454 | { 455 | "output_type": "stream", 456 | "stream": "stdout", 457 | "text": [ 458 | "chapter; pages with footnotes; pages with references\n", 459 | "1; 5,6; 1,2,4,5\n", 460 | "2; 8; 7,8,9\n", 461 | "3; 11,12; 10,12,13,14\n", 462 | "4; 16,19; 16,17,20" 463 | ] 464 | } 465 | ], 466 | "prompt_number": 18 467 | }, 468 | { 469 | "cell_type": "code", 470 | "collapsed": false, 471 | "input": [ 472 | "csv.register_dialect(\"semicolon\", delimiter=\";\", skipinitialspace=True)\n", 473 | "csv.list_dialects()" 474 | ], 475 | "language": "python", 476 | "metadata": {}, 477 | "outputs": [ 478 | { 479 | "output_type": "pyout", 480 | "prompt_number": 19, 481 | "text": [ 482 | "['excel-tab', 'excel', 'semicolon']" 483 | ] 484 | } 485 | ], 486 | "prompt_number": 19 487 | }, 488 | { 489 | "cell_type": "code", 490 | "collapsed": false, 491 | "input": [ 492 | "f = open(\"example.csv\", \"r\")\n", 493 | "\n", 494 | "reader = csv.reader(f, \"semicolon\")\n", 495 | "\n", 496 | "for line in reader:\n", 497 | " print(line)\n", 498 | "\n", 499 | "f.close()" 500 | ], 501 | "language": "python", 502 | "metadata": {}, 503 | "outputs": [ 504 | { 505 | "output_type": "stream", 506 | "stream": "stdout", 507 | "text": [ 508 | "['chapter', 'pages with footnotes', 'pages with references']\n", 509 | "['1', '5,6', '1,2,4,5']\n", 510 | "['2', '8', '7,8,9']\n", 511 | "['3', '11,12', '10,12,13,14']\n", 512 | "['4', '16,19', '16,17,20']\n" 513 | ] 514 | } 515 | ], 516 | "prompt_number": 20 517 | }, 518 | { 519 | "cell_type": "markdown", 520 | "metadata": {}, 521 | "source": [ 522 | "### Automatically detecting dialect" 523 | ] 524 | }, 525 | { 526 | "cell_type": "code", 527 | "collapsed": false, 528 | "input": [ 529 | "sniffer = csv.Sniffer()" 530 | ], 531 | "language": "python", 532 | "metadata": {}, 533 | "outputs": [], 534 | "prompt_number": 21 535 | }, 536 | { 537 | "cell_type": "markdown", 538 | "metadata": {}, 539 | "source": [ 540 | "`csv.Sniffer` analyzes text for patterns, to find the dialect" 541 | ] 542 | }, 543 | { 544 | "cell_type": "code", 545 | "collapsed": false, 546 | "input": [ 547 | "f = open(\"example.csv\", \"r\")" 548 | ], 549 | "language": "python", 550 | "metadata": {}, 551 | "outputs": [], 552 | "prompt_number": 22 553 | }, 554 | { 555 | "cell_type": "code", 556 | "collapsed": false, 557 | "input": [ 558 | "sample = f.read()\n", 559 | "\n", 560 | "dialect = sniffer.sniff(sample)\n", 561 | "\n", 562 | "f.seek(0)\n", 563 | "reader = csv.reader(f, dialect)\n", 564 | "for line in reader:\n", 565 | " print(line)\n", 566 | "\n", 567 | "f.seek(0)" 568 | ], 569 | "language": "python", 570 | "metadata": {}, 571 | "outputs": [ 572 | { 573 | "output_type": "stream", 574 | "stream": "stdout", 575 | "text": [ 576 | "['chapter', 'pages with footnotes', 'pages with references']\n", 577 | "['1', '5,6', '1,2,4,5']\n", 578 | "['2', '8', '7,8,9']\n", 579 | "['3', '11,12', '10,12,13,14']\n", 580 | "['4', '16,19', '16,17,20']\n" 581 | ] 582 | } 583 | ], 584 | "prompt_number": 23 585 | }, 586 | { 587 | "cell_type": "markdown", 588 | "metadata": {}, 589 | "source": [ 590 | "`csv` can be very large, reading all of it to find the dialect can be time\n", 591 | "consuming and not necessary" 592 | ] 593 | }, 594 | { 595 | "cell_type": "code", 596 | "collapsed": false, 597 | "input": [ 598 | "sample = f.read(7) # Read 7 bytes\n", 599 | "\n", 600 | "print(repr(sample))\n", 601 | "print(\"\")\n", 602 | "\n", 603 | "f.seek(0)\n", 604 | "\n", 605 | "dialect = sniffer.sniff(sample)\n", 606 | "\n", 607 | "reader = csv.reader(f, dialect)\n", 608 | "for line in reader:\n", 609 | " print(line)\n", 610 | "\n", 611 | "f.seek(0)" 612 | ], 613 | "language": "python", 614 | "metadata": {}, 615 | "outputs": [ 616 | { 617 | "output_type": "stream", 618 | "stream": "stdout", 619 | "text": [ 620 | "'chapter'\n", 621 | "\n", 622 | "['chap', 'er; pages wi', 'h foo', 'no', 'es; pages wi', 'h references']\n", 623 | "['1; 5,6; 1,2,4,5']\n", 624 | "['2; 8; 7,8,9']\n", 625 | "['3; 11,12; 10,12,13,14']\n", 626 | "['4; 16,19; 16,17,20']\n" 627 | ] 628 | } 629 | ], 630 | "prompt_number": 24 631 | }, 632 | { 633 | "cell_type": "markdown", 634 | "metadata": {}, 635 | "source": [ 636 | "Undersampling can of course yield erronous dialects" 637 | ] 638 | }, 639 | { 640 | "cell_type": "code", 641 | "collapsed": false, 642 | "input": [ 643 | "sample = f.read(80)\n", 644 | "\n", 645 | "print(repr(sample))\n", 646 | "print(\"\")\n", 647 | "\n", 648 | "f.seek(0)\n", 649 | "\n", 650 | "dialect = sniffer.sniff(sample)\n", 651 | "\n", 652 | "reader = csv.reader(f, dialect)\n", 653 | "for line in reader:\n", 654 | " print(line)\n", 655 | "\n", 656 | "f.seek(0)" 657 | ], 658 | "language": "python", 659 | "metadata": {}, 660 | "outputs": [ 661 | { 662 | "output_type": "stream", 663 | "stream": "stdout", 664 | "text": [ 665 | "'chapter; pages with footnotes; pages with references\\n1; 5,6; 1,2,4,5\\n2; 8; 7,8,9'\n", 666 | "\n", 667 | "['chapter', 'pages with footnotes', 'pages with references']\n", 668 | "['1', '5,6', '1,2,4,5']\n", 669 | "['2', '8', '7,8,9']\n", 670 | "['3', '11,12', '10,12,13,14']\n", 671 | "['4', '16,19', '16,17,20']\n" 672 | ] 673 | } 674 | ], 675 | "prompt_number": 25 676 | }, 677 | { 678 | "cell_type": "markdown", 679 | "metadata": {}, 680 | "source": [ 681 | "For larger files, around 1024 bytes is quite good, you want to have a few lines in the samples" 682 | ] 683 | }, 684 | { 685 | "cell_type": "code", 686 | "collapsed": false, 687 | "input": [ 688 | "dialect.delimiter, dialect.lineterminator, dialect.skipinitialspace" 689 | ], 690 | "language": "python", 691 | "metadata": {}, 692 | "outputs": [ 693 | { 694 | "output_type": "pyout", 695 | "prompt_number": 26, 696 | "text": [ 697 | "(';', '\\r\\n', True)" 698 | ] 699 | } 700 | ], 701 | "prompt_number": 26 702 | }, 703 | { 704 | "cell_type": "code", 705 | "collapsed": false, 706 | "input": [ 707 | "sniffer.has_header(sample)" 708 | ], 709 | "language": "python", 710 | "metadata": {}, 711 | "outputs": [ 712 | { 713 | "output_type": "pyout", 714 | "prompt_number": 27, 715 | "text": [ 716 | "True" 717 | ] 718 | } 719 | ], 720 | "prompt_number": 27 721 | }, 722 | { 723 | "cell_type": "code", 724 | "collapsed": false, 725 | "input": [ 726 | "f.close()" 727 | ], 728 | "language": "python", 729 | "metadata": {}, 730 | "outputs": [], 731 | "prompt_number": 28 732 | }, 733 | { 734 | "cell_type": "code", 735 | "collapsed": false, 736 | "input": [ 737 | "f = open(\"thousand_signs.csv\", \"r\")\n", 738 | "sample = f.read(100)\n", 739 | "f.close\n", 740 | "\n", 741 | "sniffer.has_header(sample)" 742 | ], 743 | "language": "python", 744 | "metadata": {}, 745 | "outputs": [ 746 | { 747 | "output_type": "pyout", 748 | "prompt_number": 29, 749 | "text": [ 750 | "False" 751 | ] 752 | } 753 | ], 754 | "prompt_number": 29 755 | }, 756 | { 757 | "cell_type": "markdown", 758 | "metadata": {}, 759 | "source": [ 760 | "## JSON" 761 | ] 762 | }, 763 | { 764 | "cell_type": "markdown", 765 | "metadata": {}, 766 | "source": [ 767 | "Builtin module `json`\n", 768 | "\n", 769 | "JavaScript Object Notation, human readable serialization of data.\n", 770 | "\n", 771 | "Mostly known from sending data between server and client in web applications\n", 772 | "\n", 773 | "In Python nomenclature, one can say it consists of lists and dictionaries,\n", 774 | "which can be populated by strings, integers and floats" 775 | ] 776 | }, 777 | { 778 | "cell_type": "code", 779 | "collapsed": false, 780 | "input": [ 781 | "%%bash\n", 782 | "# Example from wikipeda page on JSON\n", 783 | "cat example.json" 784 | ], 785 | "language": "python", 786 | "metadata": {}, 787 | "outputs": [ 788 | { 789 | "output_type": "stream", 790 | "stream": "stdout", 791 | "text": [ 792 | "{\n", 793 | " \"firstName\": \"John\",\n", 794 | " \"lastName\": \"Smith\",\n", 795 | " \"age\": 25,\n", 796 | " \"address\": {\n", 797 | " \"streetAddress\": \"21 2nd Street\",\n", 798 | " \"city\": \"New York\",\n", 799 | " \"state\": \"NY\",\n", 800 | " \"postalCode\": \"10021\"\n", 801 | " },\n", 802 | " \"phoneNumber\": [\n", 803 | " {\n", 804 | " \"type\": \"home\",\n", 805 | " \"number\": \"212 555-1234\"\n", 806 | " },\n", 807 | " {\n", 808 | " \"type\": \"fax\",\n", 809 | " \"number\": \"646 555-4567\"\n", 810 | " }\n", 811 | " ]\n", 812 | "}" 813 | ] 814 | } 815 | ], 816 | "prompt_number": 30 817 | }, 818 | { 819 | "cell_type": "code", 820 | "collapsed": false, 821 | "input": [ 822 | "import json" 823 | ], 824 | "language": "python", 825 | "metadata": {}, 826 | "outputs": [], 827 | "prompt_number": 31 828 | }, 829 | { 830 | "cell_type": "code", 831 | "collapsed": false, 832 | "input": [ 833 | "f = open(\"example.json\", \"r\")\n", 834 | "\n", 835 | "json_data = json.load(f)\n", 836 | "\n", 837 | "f.close()\n", 838 | "\n", 839 | "json_data" 840 | ], 841 | "language": "python", 842 | "metadata": {}, 843 | "outputs": [ 844 | { 845 | "output_type": "pyout", 846 | "prompt_number": 32, 847 | "text": [ 848 | "{u'address': {u'city': u'New York',\n", 849 | " u'postalCode': u'10021',\n", 850 | " u'state': u'NY',\n", 851 | " u'streetAddress': u'21 2nd Street'},\n", 852 | " u'age': 25,\n", 853 | " u'firstName': u'John',\n", 854 | " u'lastName': u'Smith',\n", 855 | " u'phoneNumber': [{u'number': u'212 555-1234', u'type': u'home'},\n", 856 | " {u'number': u'646 555-4567', u'type': u'fax'}]}" 857 | ] 858 | } 859 | ], 860 | "prompt_number": 32 861 | }, 862 | { 863 | "cell_type": "markdown", 864 | "metadata": {}, 865 | "source": [ 866 | "## YAML" 867 | ] 868 | }, 869 | { 870 | "cell_type": "markdown", 871 | "metadata": {}, 872 | "source": [ 873 | "Package `pyyaml`, install by `pip install pyyaml`" 874 | ] 875 | }, 876 | { 877 | "cell_type": "heading", 878 | "level": 2, 879 | "metadata": {}, 880 | "source": [ 881 | "Databases" 882 | ] 883 | }, 884 | { 885 | "cell_type": "markdown", 886 | "metadata": {}, 887 | "source": [ 888 | "Drivers for popular none-SQL databases:\n", 889 | "\n", 890 | "- pycouchdb\n", 891 | "- pymongo" 892 | ] 893 | }, 894 | { 895 | "cell_type": "heading", 896 | "level": 2, 897 | "metadata": {}, 898 | "source": [ 899 | "Web services " 900 | ] 901 | }, 902 | { 903 | "cell_type": "markdown", 904 | "metadata": {}, 905 | "source": [ 906 | "A common way to make data available is in the form of a web service.\n", 907 | "\n", 908 | "To get data one sends a `GET` command to a server, which interprets\n", 909 | "the `URI` used to send the `GET` command, and sends back what one\n", 910 | "asked for." 911 | ] 912 | }, 913 | { 914 | "cell_type": "markdown", 915 | "metadata": {}, 916 | "source": [ 917 | "Python comes with several http libraries, but the recommendation is to install the package `requests`\n", 918 | "\n", 919 | "( `pip install requests` )\n", 920 | "\n", 921 | "Documentation on http://docs.python-requests.org/" 922 | ] 923 | }, 924 | { 925 | "cell_type": "code", 926 | "collapsed": false, 927 | "input": [ 928 | "import requests" 929 | ], 930 | "language": "python", 931 | "metadata": {}, 932 | "outputs": [], 933 | "prompt_number": 33 934 | }, 935 | { 936 | "cell_type": "code", 937 | "collapsed": false, 938 | "input": [ 939 | "r = requests.get(\"http://ws.spotify.com/search/1/track.json\", params={\"q\": \"kaizers orchestra\"})" 940 | ], 941 | "language": "python", 942 | "metadata": {}, 943 | "outputs": [], 944 | "prompt_number": 34 945 | }, 946 | { 947 | "cell_type": "code", 948 | "collapsed": false, 949 | "input": [ 950 | "r.url" 951 | ], 952 | "language": "python", 953 | "metadata": {}, 954 | "outputs": [ 955 | { 956 | "output_type": "pyout", 957 | "prompt_number": 35, 958 | "text": [ 959 | "u'http://ws.spotify.com/search/1/track.json?q=kaizers+orchestra'" 960 | ] 961 | } 962 | ], 963 | "prompt_number": 35 964 | }, 965 | { 966 | "cell_type": "code", 967 | "collapsed": false, 968 | "input": [ 969 | "r.status_code" 970 | ], 971 | "language": "python", 972 | "metadata": {}, 973 | "outputs": [ 974 | { 975 | "output_type": "pyout", 976 | "prompt_number": 36, 977 | "text": [ 978 | "200" 979 | ] 980 | } 981 | ], 982 | "prompt_number": 36 983 | }, 984 | { 985 | "cell_type": "code", 986 | "collapsed": false, 987 | "input": [ 988 | "r.text[:2000]" 989 | ], 990 | "language": "python", 991 | "metadata": {}, 992 | "outputs": [ 993 | { 994 | "output_type": "pyout", 995 | "prompt_number": 37, 996 | "text": [ 997 | "u'{\"info\": {\"num_results\": 417, \"limit\": 100, \"offset\": 0, \"query\": \"kaizers orchestra\", \"type\": \"track\", \"page\": 1}, \"tracks\": [{\"album\": {\"released\": \"2012\", \"href\": \"spotify:album:5AN6A9IR1g1xRgY0RoKOsT\", \"name\": \"Hjerteknuser\", \"availability\": {\"territories\": \"NO\"}}, \"name\": \"Hjerteknuser\", \"popularity\": \"0.66621\", \"external-ids\": [{\"type\": \"isrc\", \"id\": \"NOHDL1002070\"}], \"length\": 199.407, \"href\": \"spotify:track:6dKWi7apHjn2W7Ojncv4Wu\", \"artists\": [{\"href\": \"spotify:artist:1s1DnVoBDfp3jxjjew8cBR\", \"name\": \"Kaizers Orchestra\"}], \"track-number\": \"1\"}, {\"album\": {\"released\": \"2012\", \"href\": \"spotify:album:2oZ0PnxiH9LaoUAFzlPSGK\", \"name\": \"Siste dans\", \"availability\": {\"territories\": \"NO\"}}, \"name\": \"Siste dans\", \"popularity\": \"0.64841\", \"external-ids\": [{\"type\": \"isrc\", \"id\": \"NOHDL1202060\"}], \"length\": 217.891, \"href\": \"spotify:track:0z26fQRDfSwxxuyKYrjZn3\", \"artists\": [{\"href\": \"spotify:artist:1s1DnVoBDfp3jxjjew8cBR\", \"name\": \"Kaizers Orchestra\"}], \"track-number\": \"1\"}, {\"album\": {\"released\": \"2010\", \"href\": \"spotify:album:6jbtJwRmuezfOSXyJy3tRZ\", \"name\": \"Violeta Violeta Volume I\", \"availability\": {\"territories\": \"NO\"}}, \"name\": \"Hjerteknuser\", \"popularity\": \"0.63483\", \"external-ids\": [{\"type\": \"isrc\", \"id\": \"NOHDL1002070\"}], \"length\": 200.322, \"href\": \"spotify:track:3NThq9BqYtKYBfsHIogjM6\", \"artists\": [{\"href\": \"spotify:artist:1s1DnVoBDfp3jxjjew8cBR\", \"name\": \"Kaizers Orchestra\"}], \"track-number\": \"7\"}, {\"album\": {\"released\": \"2012\", \"href\": \"spotify:album:5E9Kg0KC7H0CWOVongiKRe\", \"name\": \"Violeta Violeta Volume III\", \"availability\": {\"territories\": \"NO\"}}, \"name\": \"Begravelsespolka\", \"popularity\": \"0.61493\", \"external-ids\": [{\"type\": \"isrc\", \"id\": \"NOHDL1202020\"}], \"length\": 426.58, \"href\": \"spotify:track:4k2VAoUhJx7lxMscgY8USe\", \"artists\": [{\"href\": \"spotify:artist:1s1DnVoBDfp3jxjjew8cBR\", \"name\": \"Kaizers Orchestra\"}], \"track-number\": \"1\"}, {\"album\": {\"released\": \"2012\", \"href\": \"spotify:album:0hoeWFBKo9kGoKMhKOuKRY\", \"name\": \"V\\\\u00e5re Demoner\", \"availabilit'" 998 | ] 999 | } 1000 | ], 1001 | "prompt_number": 37 1002 | }, 1003 | { 1004 | "cell_type": "code", 1005 | "collapsed": false, 1006 | "input": [ 1007 | "json_data = r.json()" 1008 | ], 1009 | "language": "python", 1010 | "metadata": {}, 1011 | "outputs": [], 1012 | "prompt_number": 38 1013 | }, 1014 | { 1015 | "cell_type": "code", 1016 | "collapsed": false, 1017 | "input": [ 1018 | "len(json_data[\"tracks\"])" 1019 | ], 1020 | "language": "python", 1021 | "metadata": {}, 1022 | "outputs": [ 1023 | { 1024 | "output_type": "pyout", 1025 | "prompt_number": 39, 1026 | "text": [ 1027 | "100" 1028 | ] 1029 | } 1030 | ], 1031 | "prompt_number": 39 1032 | }, 1033 | { 1034 | "cell_type": "code", 1035 | "collapsed": false, 1036 | "input": [ 1037 | "json_data[\"tracks\"][1]" 1038 | ], 1039 | "language": "python", 1040 | "metadata": {}, 1041 | "outputs": [ 1042 | { 1043 | "output_type": "pyout", 1044 | "prompt_number": 40, 1045 | "text": [ 1046 | "{u'album': {u'availability': {u'territories': u'NO'},\n", 1047 | " u'href': u'spotify:album:2oZ0PnxiH9LaoUAFzlPSGK',\n", 1048 | " u'name': u'Siste dans',\n", 1049 | " u'released': u'2012'},\n", 1050 | " u'artists': [{u'href': u'spotify:artist:1s1DnVoBDfp3jxjjew8cBR',\n", 1051 | " u'name': u'Kaizers Orchestra'}],\n", 1052 | " u'external-ids': [{u'id': u'NOHDL1202060', u'type': u'isrc'}],\n", 1053 | " u'href': u'spotify:track:0z26fQRDfSwxxuyKYrjZn3',\n", 1054 | " u'length': 217.891,\n", 1055 | " u'name': u'Siste dans',\n", 1056 | " u'popularity': u'0.64841',\n", 1057 | " u'track-number': u'1'}" 1058 | ] 1059 | } 1060 | ], 1061 | "prompt_number": 40 1062 | }, 1063 | { 1064 | "cell_type": "code", 1065 | "collapsed": false, 1066 | "input": [ 1067 | "r.headers" 1068 | ], 1069 | "language": "python", 1070 | "metadata": {}, 1071 | "outputs": [ 1072 | { 1073 | "output_type": "pyout", 1074 | "prompt_number": 41, 1075 | "text": [ 1076 | "{'access-control-allow-origin': '*',\n", 1077 | " 'age': '10058',\n", 1078 | " 'content-length': '58592',\n", 1079 | " 'content-type': 'application/json; charset=utf-8',\n", 1080 | " 'date': 'Tue, 22 Jan 2013 12:17:59 GMT',\n", 1081 | " 'expires': 'Wed, 23 Jan 2013 09:30:21 GMT',\n", 1082 | " 'last-modified': 'Mon, 21 Jan 2013 23:35:15 GMT',\n", 1083 | " 'server': 'lighttpd smisk/1.1.6',\n", 1084 | " 'vary': 'Accept-Charset',\n", 1085 | " 'via': '1.1 varnish',\n", 1086 | " 'x-varnish': '766974840 766737315'}" 1087 | ] 1088 | } 1089 | ], 1090 | "prompt_number": 41 1091 | }, 1092 | { 1093 | "cell_type": "code", 1094 | "collapsed": false, 1095 | "input": [ 1096 | "logo = requests.get(\"http://www.scilifelab.se/images/logo_header.png\")" 1097 | ], 1098 | "language": "python", 1099 | "metadata": {}, 1100 | "outputs": [], 1101 | "prompt_number": 42 1102 | }, 1103 | { 1104 | "cell_type": "code", 1105 | "collapsed": false, 1106 | "input": [ 1107 | "logo.headers" 1108 | ], 1109 | "language": "python", 1110 | "metadata": {}, 1111 | "outputs": [ 1112 | { 1113 | "output_type": "pyout", 1114 | "prompt_number": 43, 1115 | "text": [ 1116 | "{'accept-ranges': 'bytes',\n", 1117 | " 'connection': 'close',\n", 1118 | " 'content-length': '14922',\n", 1119 | " 'content-type': 'image/png',\n", 1120 | " 'date': 'Tue, 22 Jan 2013 12:13:37 GMT',\n", 1121 | " 'etag': '\"172816a-3a4a-4aab46ed30140\"',\n", 1122 | " 'last-modified': 'Wed, 17 Aug 2011 14:37:17 GMT',\n", 1123 | " 'server': 'Apache/2.2.3 (CentOS)'}" 1124 | ] 1125 | } 1126 | ], 1127 | "prompt_number": 43 1128 | }, 1129 | { 1130 | "cell_type": "code", 1131 | "collapsed": false, 1132 | "input": [ 1133 | "logo.content[:100]" 1134 | ], 1135 | "language": "python", 1136 | "metadata": {}, 1137 | "outputs": [ 1138 | { 1139 | "output_type": "pyout", 1140 | "prompt_number": 44, 1141 | "text": [ 1142 | "'\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIHDR\\x00\\x00\\x00\\xfa\\x00\\x00\\x00P\\x08\\x06\\x00\\x00\\x000kh\\xb5\\x00\\x00 \\x00IDATx\\x9c\\xed\\x9dw\\\\TW\\xf6\\xc0\\xcf\\xbd\\xefMo\\x94\\x01\\xa4\\t(\\xa2\\x82X@\\x8d1Q\\x83-\\xd1\\x98\\xc4$\\x0b\\xeb\\xc6\\x18uctw\\x13\\xb3\\xae\\xeeoM4e\\xdcMq\\x93\\x8dIL\\xaf\\x9ah'" 1143 | ] 1144 | } 1145 | ], 1146 | "prompt_number": 44 1147 | }, 1148 | { 1149 | "cell_type": "code", 1150 | "collapsed": false, 1151 | "input": [ 1152 | "from IPython.core.display import Image" 1153 | ], 1154 | "language": "python", 1155 | "metadata": {}, 1156 | "outputs": [], 1157 | "prompt_number": 45 1158 | }, 1159 | { 1160 | "cell_type": "code", 1161 | "collapsed": false, 1162 | "input": [ 1163 | "Image(logo.content)" 1164 | ], 1165 | "language": "python", 1166 | "metadata": {}, 1167 | "outputs": [ 1168 | { 1169 | "output_type": "pyout", 1170 | "png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAABQCAYAAAAwa2i1AAAgAElEQVR4nO2dd1xUV/bAz73vTW+U\nAaQJKKKCWECNMVGDLdGYxCQL68YYdWN0dxOzru5vTTRl3E1xk41JTK+aaEwCamKJXdEo2BHUAaT3\nNgwwTJ95753fHwgLiArEvnw/Hz8Jb96de+68d24595xzAXrooYc7HnKzBejh9iBRnyj2cmeG8gwX\nwTJ0iNNtiSCU9OZ5h4gQRgBgLISKj/QPnPhRX69Jppstbw9tYW+2AD3c2uTXJWryK9OHW52776tH\n93i3vTFKAKdGQA4QEQghANQNwIvrvRVRv/Yxhjputsw9XErPiN5Dh1RUoDyz4Y17ahvPzhLAPpmK\n7H4c7wZBoADIAEECBMQAlAMC4nq1OPi1AcMe/Kgfmeq82bL3cCk9it7DJRwv+iKstObwmy40TkVe\nZJewyjyWkTdSwiISkDo5k5bnHYE8afACQVatloa98NDQ1esIIfzNlr2HjulR9B7akJypG1VryXvL\nzVsClDL/71Riv50iiWcFS+ROAABBJGaFRqOHk9gibK7qeykjS586KOEHQoa7b7bsPVyeLik6Ioqe\neeYZr/r6ek+5XM5SSrG2ttber1+/unfffddECMHrJWgP1xcEJDsyXxhjNud9woDY5ut19/yx4QvT\nr/RMEzGRiYd4oee5dw1EJADAbN26VbZ+/XqFQqHwCA0N5fz9/YsXLFhwXTrMqyo6IpLPP/9cvX37\n9qFWq/VukUgU7Xa7/bAJHgBsHh4exfX19SdiY2NPv/POO4WEkDbC7tq1y5/jOMe0adPqr0cjevit\nIEkvTRqhr1j3qYgo+JjwhXPDve45fx0qIrGxseqGhga1y+Xi6+vrucjISF6lUnX6C3ieJ6WlpbSi\nokLk6elJ4+LiTN99913jdZD1mqDT6WhNTY2vv7+/5uDBg2qZTKa2WCxah8MR4O3tHZiTkxOiVqv7\n9u7dOzs+Pv75J554ovZ6yHFFqzsi0gULFow7evTo791u94NWq9Xu5+d3gBBy/OTJk/UAgKNGjfIs\nKSnpw7Ls8+np6e5hw4Yd/utf/7ozMjLy3Pz58xvnzZvnuXHjxhciIyMPAsBP16MR14rkwjVSW0Pm\nABsYPBjk0c1TTgwyt8DSTo5YbiDIsxznFoObZcRy1qHgiX7S8M9v6e0mfcW+4IySH1YDUu9Qnwf/\ncJ2UHD777DN27969Y/Ly8h5xu90KSqlEr9dzAOAWi8VXLe9yuYDnebGPjw8bGRnpYBjGyrJsEiLu\nv1XtA5GRkfK0tLRniouLx9TV1QXX19f72u12TXV1NdN8DyEEKKWNHh4e120pfVlFz8jIUMydO3fB\nrl27/iGRSDTR0dEfBAUFffzJJ5+UEUK45vuOHTsGAABPP/10SHFxcbzVal20fv36eVqt9lRSUtLR\nurq6vpWVlQ8FBASkXa9GXCu8zWIZlWhj3Q7bEAJCuEuo7WsRDHLCiZDA1aanCDw4CQtSXir2rZSI\n5XkEIJd6BZUDwC2r6FiDyt1Vy+dZXQV3hXs99hptePIkwKzrUldlZSXv7++fXlVV1SAWi31CQ0Pv\nKSkpeSQzMzPcarVetTzLstC7d+/skJCQbb6+vsdsNluNSqUqAADhugh8Daivr3cWFxdvd7lcaf37\n9x9eUVExs7q62qv1PYQQYBiGk8lk120J1GEPkpqaKtu8efM/1qxZ83eO4yQPPfTQ6+vWrXuHEGK5\n0pchoujvf//7xP3797+fkZHRj2VZhyAIIpVKZV60aNGfdTrdD9enGdcGRCQHi9ZKwHaSFWSe3jaL\n9V6Xo/YZG1c1jjJiuNxTIEAAwQkMqis08vDVLC/fzTmNBTzrzU2LBQchulv0RURytODjabnV2z5h\nQGQe5j9nZmTo4zesQ87NzZV8++23g1NTU984dOjQRI7jrnj/gw8++Mu4ceNeefDBB89HRUW5bpCY\n1wxEFCckJEzKzMz8QK/XhzVfp5TCyJEj97/55psz4uLibszUHRGZBQsW/Omnn376h9FolE+ZMuUH\nnU73ydWUHACAEOJGxL3//Oc/l9TW1n5VVlbm0/yZ1Wq9+tzsJnPRqNTs8GEBgOJ96W+c5Ij7XU6w\nTgW+I1UnIIATRIxncZjP/ct29n36Bx2QW1Sx21JRcdD7eOWZOW6sDvRSxH3hEzI290bW369fPycA\nnPzwww9fKikpCc/NzQ293L0DBw7Mnz59+rJ58+adu12Nf4QQFyImz5s3b0N+fv4yh8Nxw3a9aOs/\nEJG89957cT///PNig8Eg9/b2tkVFRW3o27dvp3sZQgj3yiuv7ImJidkglUq55u+12+3M1cpeTxBR\n9vDDD4fPnz+/LyJKOltu4tBlOWF+D+gYUFTjJctAAgg8ILJ2rXrwB7Z1TyfeLkqu0yEtbDzzUL39\nwhTKK+1qSe9f/5Kgtd0MWZ599tmMyMjI3SzLdvjbyeVyftiwYdvnzZuXc7sqeSscfn5+RzUajflG\nVkrb/a3ctWvXPKPRGAgA4OHhUTxjxowMQrr28hJCnHPnzl07dOjQNAAAjuOI1Wq9aYq+f//+wCee\neOKFnJycb0+dOvXt008//dyuXbu8rl6yiVDpnCwRFZ+hzKUdMAIPYlZd5KvsvydOR64897zmINl7\n7l8Dd559bv6usy/OzajaGnb1Mk08/bcyD6M9/zEkNpmE1RrFjCIvKenmGLQIIY7o6OjTGo2mw1mj\nRqMxBwUFnSaE3PbutYQQwc/Pr0oul99Qu03L1B0R6UcffTQyKytrIs/zhBACUqm0KDY21tidL54+\nffq53NzctdnZ2VF2u51aLBbRtRO78xw5ckT18ccfv/LTTz8943A4CCEE8vPzYwFAqdfr3+zMWs83\nARw/v6MqAL4OABCaTRsICAAIcpFnhYoTlV7XhrQDEcnp4m+m5leffcktWIYgELRVVKVcqNj2fP+A\naRfgKsZDQ/2xu0yOorsZIgEpq65mFb0qbpTsHSGTyUq8vLxMRqNR3f4ziURSZ7fbS26GXNcDt9tt\nJYTc0NlT6xFdfOjQobiq6qqWkQ4RrdBNiyYhhJ80adK+yMjIDAAgVquVXnQUuKEcOXKkz9mzZ8c0\nr4cQEUwmkyQ7O3v81q1bgzr1JQcJRwlTInDE2dp8SQABkQe5SGswStgbbBwyexUbDyy2C9WjAEAG\nyMnNjuJJhfWpz1RgpexKJUtTUVZr1T/A8WYvAlJgGGlZmGRAww0SvEMyMzMtVqvV3tFnJpPJUlBQ\ncFUb0e3CqlWrsKSk5IbOnloUfffu3Z41NTVDXE5Xy6uMiAoA6PaUe+jQoXkDBw48JBaLhZtljHM4\nHG6JRHKJEgqC4BKLxZ32QmKp1EYo5Vr3VAgEGCIGq8tgrLEW3tAHl117wtvFWUMoZQGABQJioJQF\nu7N2QEXONvmVyrr7neptthXeTSglBCU8Q+XVXl43uqNqi8FgcLjd7g5l8PDwcA8ZMuSOcbE1Go3I\n8/wNHfRaFD0/P99PEITA5r8REVwuV1BWVlbn3ZbaQQjBAQMGHPX19bUaDAbpbxW2O4wbN65g8ODB\nO5RKZcto4efn1zBw4MAtixcv7vR0VSrxcAIAj62GdAIIlDDgFmwuy5nDN9RIpBT3rxUzHgU8x4GA\nTkBwAgoIcpF3RmzE5CuOfhX1qbEOvq4vS0RAKHECuMtWrAi9qYqu0Wh4uVzeYWepVCo5tVp9SzrE\n3C60rNH37dvnU1paqm39YU1NTb9Vq1bdg4g/ddfzyMvL63SfPn0KTpw40e0O47cQFxfnWLVq1X+K\ni4tL3G73RIlEIvTp0+fnOXPmdKlNBKlAELC1NiMACAjAECkKcGPtRF+9G9Qw8cmh72K908PiqhnM\nEOpWSPyP+MkHriUk9LKhoomJyNRb/zaIF3gVoQRYkLhFrKdRp7u5uwUSiQQ7sqhf9BpDk8l0u1vb\nbyotil5TU8PabLY2++pms1mak5Pzh6SkpCMAUNWdCiildQEBAZlyuVxCCCEAl/U7uW4sXry4Ljk5\n+cvCwsJNSqUS4uPj666dy+TNef90OiK8+iru2p9nLtZyg+4hyDsFmfnXoWGziwBmX7bcpElFquR8\nUzQFToSCAEh4u4cyvPLGSd51EHt0/LfSothRUVEOo9HorKmpaXPD0aNHp0VEROQg4uudcZppz5w5\nc1wGg+FbQojUarWSpKSkayB214mLi+MAwHBTKr9OXBwBMy/+6xR1WK7mBVcYUACCFBColXeb6q6f\nlD3cCrQo+oABA2ozMjKMABDa+ga32y3esmXLcwAgbN++/T9djUC7uAd/GBHJHeDscNtTVn06iEOH\nFwAFHt2glmhNEoX6jrFo99AxLcY4rVZbjYgd7gUbDAblhg0bFq1evfrfixcvHoaI7R1trkqPkt98\nEJG4eVNvQeCVgAQAKAiCqw547pYNuunh2tAyos+aNat+7969yVlZWQ+azeZLnFtsNpt83759zxQU\nFEyorKz8+r333tv217/+9XxXvea6CiKSCRMmBBBCAiZMmCCXSCRsQUGBu76+vjIiIsKg0+k6vf+r\n1+vFn376qcZkMnk+8MADddcr9vdm0eyncIVOleGJ3ZdHh5QSAAos8IAmg7ng6qFjV6n36NF3pWXO\nPEbjJUde3pub2m+h62pOO7cLa9askWZkZHjW19cH+Pj4aFQqFaakpDjz8vIMiYmJRcOHX5/sOojI\nPPzww721Wq2vVCqV8jzvsNlsDSzLmtasWWPoip2pRdEJIe6NGzfuSUlJKTabzeEd3SwIAuTl5fUx\nGAwv5eTkPHrs2LGfly1btjU3N1eflJR0zbY/EJHMnTvXr7q6etSkSZNGIWJ4YGBgXX5+frXL5WIp\npeEHDhwI1ev1BdOnT083mUwnhg4dmvruu+/aAYA89dRTASqVKkiv16sIIWofHx/v8vJy7ZIlS7SV\nlZUhDQ0N/mVlZWsA4MtrJfONIDn5Pjb0vlfZyjIDEVfXi0Erl5vtegkQT63FWeX/S8YijZ9naCoA\nFHRUvqwMRCAwvSghIkQAAThUin0t/f3j7QBvdFuuvLrjKvRiHpfbGX9CWIbYq8Wb0+YVquXLTnmI\nhhYP75tw280YdDqdsqioaHhjY2P0+vXr+2dlZQ2MioqSAEB2QUFBRWBgoLq+vj74T3/6k2HcuHFp\nUVFRZ8LCwjL/7//+r1udJiEELJamFdTq1aslhYWFIxcsWDAJALT19fUyQRD86urqgnJzc1GlUhU/\n9NBD+kceeeT0iBEjjr/00ktX9cpsY2V//PHH8/fs2bMmMTFxeUNDw2WdLkwmkzQtLS1Wr9cPHjBg\nQHzv3r1/efrppzf8+c9/vvBbejdEJM8880zgrFmzHsrIyJheWlo6KiwsLC8mJubz4ODg49nZ2TVV\nVVXCfffdFxATE/NcamrqjOzs7MeUSmUlAPz8+uuvf75s2bLsxYsXj0lPT386Jycn0mw2qyilIrPZ\nzAIARUSKiFBcXLyvu3LeDBITkRH8Xxube27fvSZHiTcFKmeNKn8n36AGUu6J6PAUBJ44jbY/w2UU\n3eHIE1xco5wwFJDjAVDgZKxH2TfvD/1Na3RbmYznJLRYJPKsFRGJhoDrHsHF/c5ozsd6UpmyW6/7\nXqWefHR08OgOPd9uJRBR/OKLL47evXv33KqqqvsqKyt78TzPDh48OCc8PHylt7f3kU2bNtmmTZum\njI2NHXb27NlFx44dm1NaWlrSp0+fIwkJCT/9+OOPuwkhnfZLQERwu90YGRnpfu+99/ySk5OX2Gy2\nSKVSecDhcBzx9vaua2hokHp6eob179//d+fPnx//yy+/PKTVausNBkP6/Pnzvxw7duz2J5988rKZ\ndtooOiHEXVlZ+UVRUVHo/v37/8jz/GW94hARHA6HKD09fbBer4+OiIiYVlxcvGnp0qWbV65cmdU6\nOUUnG8u89NJLI06fPv1KXl7eRKfLyd4z+p6N995774p//etf+tb3Hjx4sAoR/5aQkKDeuHHj40aj\nMfjQoUMLXS7XSLVavcThcOz29fXNDwgImHjs2LH5+fn5oe2q4wHgln/p2oMgRhdfy0lY5ZAGR944\nCgQIEQHyLiCUBURq4fn2Kxkkp4s3xJis+VHl/M+MW7AOBqHJX58QFpy8LeD+pz6bMmnGaonACEQQ\neMowAAJPCAAAZRApMoIAhLAo5o3W7EaTtRC8VTGVkwe/qieE4JAhQ6wAcKC5xtTS1E0i3DfIYMv5\np10onucyGx612Eq2phv2rRzqMzHnBv5kXQIR6TvvvDN1z549b6WlpfVrvh4REVH84osvLvrd7363\nu/laVlYWAEDu+vXrz3/22WcfHzlyZFxBQUG/wMDAaX/605/eqqmp+dTX17fTHahUKm10uVx+27Zt\n+6dUKtXGxMSsePPNNw+3u+3ITz/9tG3nzp1Pbt26dVlVVZW/wWCIKysrG5aZmTn6zTff/M+LL75Y\n1NH3XxKP7u/vb1i6dOlr5eXlkpKSkgSz2XxVjza32030en10YWFhlMlkuv/3v//9t2fPnv1x8ODB\nnbLQIyJduHDh5CNHjryenp4+DADgvvvu+0Wn0y0ZN25ch9MSQojpjTfe2Pbzzz8/4Ha7FYIgQEpK\nyl1yuXz53Llz5z7xxBMnEfHMa6+9Vvvpp5+uLC8vbx+tdltlwE1IIHxiYvyvAJDiF3PfOgBuuclR\nPF8AgVAQAwACoQyPlG2zLj6U+8Gkktozr9ldxoGEigSed8qbUmUQIEBFRltWQoM9bxoAMghAmhx7\ngQhNeg4UBcCmLO6ISJAHq4syIpOvOuwFQkiHKacujtwnj+Z8sbik4YDcTWrGmp1FcwpKfvAvNh5b\nEuI9qtPbgTeSiRMn+lRXV//p/PnzLUpOCMGYmJhtjz/++KGOyjz55JOZ77///vtnz54dajKZNOXl\n5T5btmx5xdPT04KIX3RmHU0IQZ7ng1955ZX3GhsbNQ899NCfFy1adLajex999NEGRPwEEZ3btm37\nd1VVlWdJSYlHaWnpsyzLBuzcufPFKVOmXGhfrsNUUv/+979LVq5c+XxWVtax1NTUv+fm5va5mrAA\nADabjZ48efLurKys2Pr6+qmvvPLK+ytWrDhwNYv7smXLpm/evHl1RUVFIABATEzM+UcfffSFyyl5\nMz4+Pr9GRUXlp6enD26+dvjw4Xuio6PvBYBNhBAOEb8/c+bM5M2bN/+uM224lUlISOIBgAdIKs0o\nX/duZvnW8YLQ0A+F5jAC2jRXAYBkTGZtZ3+JLTXufovHxiEEGUCeAQYk0LqP4wWzggNO0bbXIwCE\nAqIAABTIxdekKVqPB29l1MGhoVMPwFXYvWFe7vgZ1i+KjLtGUIaX1duy708v+abWYMC/+PiQGxqP\n3Rnsdru0srLSu/W1iIiIqjFjxnx/pRDZ4cOHp0RFRZ1KTU2dAABQVVWl2rJly+KhQ4ceAoCsq9Ur\nCAKcOHFieGFhofGxxx6bdTklb4YQwiPiN0899ZTil19++WddXZ0KEeHXX3991O12a3fs2PHE1KlT\ny1qXuew22QsvvGBau3btF0OGDJn/6KOPfhcSEtLp/XOLxSI+cODAQxs2bPhk5syZs64UtbZx48bh\nO3fufLVZyVUqlSs2Nnb9888/f0mv1J558+aV3nvvvRtFIlFLR6JQKHhvb+/WIYBWRDzHsuwd5Ss9\nmHmyTMFqzwnCRQUkBAB5QBQQACCgvDqKE8xvIvD9RODVyFLvBpYqrK1z3wnAIyFSq5jxqRcx3iaW\n8W5o+ufVwBJNA0vVRhYUdkrEQEEMFFgQMR61od73fEiIz1UVVacjXLRm3BExq7iAiCBh5cRky3so\nt+HD0Tpd17dorzeLFi0qmzlz5vcRERG1IpEINRqNc8SIET9ERkZeMb3W6NGjjX379j3X+lpBQUFA\nSkpKVCerJjzPk4EDBybfc889xztVgBDXSy+9tH7YsGFbm68JggBHjx4d8+GHH/6loqKijY3tillg\nL66z9ycmJp4Ri8WbfXx85ly4cGGc2Wy+JGa4PTzPQ15eXj+32/3v6dOnixFxbft1OyIqHnzwwb+d\nP3++ZUSOioo6179//y3tU0ZfTr49e/Z8VVtbG7pr165HFQqFMH78+K/uv//+1OXLl7eWpU4qlbot\nFstNzXJzTfEDJ18plAAyHKHIAhBA5IERsQCAhOM3cWKx57e+RPGZQMWCTKR0uHnnaKMt609u3uoh\noAuU4sAib3nUf3jOWigQKqNMUycgECQ8Z3dqpIG0wVa6pNpychxLFACEA7U0eB9rE5/urJjp2UOr\nVB4hpxz2M4MJSCgQh6bBXvT72bOLjut0cFNDY9uTkJDAb9my5QsAqMjJyYlVKBQFgwcP/ikuLu6K\ngQyEEH7ChAkGQkiLuy7P8/T48ePeVyrXGqVSyQ8YMCD1iSeesM6cObNTZSIiIowTJ078NjMz88HK\nykoPgCZlv3Dhwh8//fTTQ4i4t3n7u1OHLCYkJNQBwOa33nor9fjx4xPPnj27wGQyxdbU1Fwx7hkA\noLi4uJfL5Xp10aJF9Yj4c/OaBRHpihUrRqSlpT3QnBSQYRgMDQ39dcmSJWV///vfO9XYyZMnV3z9\n9dcvsCy7TSaTOYcNG3Zs+PDhbbZzevXq5RCLxTc4+8t1R2BR4gDKCs3zdUIYEHiBABDUH0/Mzox/\nM0tH/uuMv/PcMhEv4Pzm6RWlkuIBwQ9s7KUaUgPQftZF8ELlzhHF9akBhFzsH5EFldj/YL9+Uzpt\nUY6LI47d5168ACh2CYBSSkVgc9YNrWZ2BwDcWooOAPDII4+YAeCHhQsX/vTBBx+4Nm3a1ClfAJFI\nZKeUAs83PQue50lVVVWnQ7PFYrGZZdlcAOj0e0oIQUQ8vWnTprNVVVVjmzuZ/Px8v3Pnzj154cKF\nVAAwA3TxNNV//OMfVQCwfseOHbt37tw58dixY0+dO3dussPhuOI0rLKyMmj//v3/+vrrr88DQPOU\nXJWbmzujpqamxUjm4eFh8fX13QMAXcq+8cc//tEAAD9f7nOlUsmJRKLbIpdbVxCJFS5woIAEgODF\n6ftFEhIS2ixVEhORkYjVvdDskAmAQEAElIhMRVXlF0ertnYUrED5bsOy2XZ3VR8RUQAADyyjbHRS\nZ15+fpKa0jXEbBaTlkzWmray1dgECtAIEjcRLNBYz1CRiwAnJQjAC9ZebrGtLyBmwS3qMfnBBx90\n6bBInudp++AbQej8K+fl5WWWyWSGbjig1d911117z507N8rpdLZ0LGlpaRO3bt0aAwCHALp5bPLU\nqVMNiPjj66+/fsTPz+8PxcXFz5w7d65DJ5tmsrOzB+zcuTMeEd8mhDjfeuutfnl5eaNb/xgqlaqm\nX79+Rdfa204ikfCE3FZG9quzAoB5nBEu2Ty4zOIkPh5Ist7ZCwhIAAUAQBCzCoOdre5wdC6THu1b\n78i9j6EsA0gAgQIiJ3PZ65bnMWkNCIQARQICAoIA0AgUBAAEgRAEgkQgCDx1ECJwbnsIQaIAwgIS\nBET0sNgNAclwkInrwgh2KxMQEKDuimK3x2KxWIuLi7tsoCSECJ9//nm6Wq02GQyGlqzL5eXlvsXF\nxePgtyh6cwUAUIqIq5577rlDlNKVer1+LMdxHY7uHMeRgoKC+JdffnkTIubMnj17aEFBQe9W3wci\nkahMo9H0RFJ1ghUAMBo7PxpmZq6gFleNBhApAQoEGI7nXaVxoXOcAHMvub+w6shdHGcNpqTpFaFU\nACmrOSuge4fJabAiiigAANNOT3kAIEJTIp6mXogFKiAnYTzdAABIOSpiRDwRHMcOrTh0R8yyTp06\npV26dOnI7pYnhIBEIrEolcpu5ZETiUQXtFpteWtF5ziOZmVlDW/+u9uK3kpIDgCOf/DBB3/88ccf\n30hJSUm4XNBLTk5ORExMzAgAKKaUDq6rq2uZ8DEMA0FBQWWzZs1qfOqpp36rWD20Q9xrplRU/oU3\nB1agIANCqEspC67taOqs0yHbyC2JQgAlQQoACIQhwFDxyWiPjz4MC7v9s7FeK1auXKlZu3btPL1e\nf/fFNXO3po4ajcYeExPj/Oqrr7pclud5o7e3dyUADG193WazBTT//zXb4li4cGHh/fffv2zQoEEH\nLzdNttls0tra2piDBw/KrVZrcLPhAqDJsKBSqWrhDpnK3Wq4+Bw5Iq8lQABAAALgQOA7TKX16kxg\nXFxDEMNS2rQ0QKCUAY53O7K4nbfkmvpGk5iYyIwdOzZar9frGhoaIqZMmfKjRCLp0rq+NVKplFOp\nVN1NxOr29PSsba93lZWVLVmdKACAXq9Xdif0tD0vv/xy4ezZs1+NiYkp7OhzQRDAYDAEbd26VZ2d\nnd0mtRTHcXDkyBF3V11nbz7kZiWZ6RIl5YeldletgiUyIECBIVK33VnboW/EnoL/Y+2uegUlrV4J\nAYAQQmRs9Q03diAiEQThutbbmZH44rHhERMmTPj922+//ZWnp+ebNput9OGHH14aFhaWzPN8t2bI\nF9NlCcHBwd1S9MOHD7szMjKs7Y2BFoulxauVBQD48ssvF3l6eiYDQEp3KmrNkiVLzmRmZm5PT09f\n2HrEbsZgMCiqqqpk7fe0KaUQFBREjEbjbZegAkEgUVFRAHDts+ckJ+tYVd8BIQEefqYAdffP5dLI\nw6RmU5GcRx4ABBCxCoevemCHxh+TuJgIgpuh/z3vExB4AMJKfRWhv3m51xUuJimlMpnsuig6IjLv\nvPPOPY888ogKAPYBQJtR+bPPPhOdPn06xmg0Dp88efLQjIyM2F69eiljY2MPaTSaVR4eHkcSEhJc\nixYtkkM33aovBrUQg8HQrfJr164FjUZzSSfR+uQbFgDg7Nmz90skEhaugaIDgMPX1/eQl5fXHIPB\ncElCSLvdzlBKUSaTtRGMYRgSHBysBAARANxWB+gJ6GKKxcevy4vI9zJorDb5MiOV/wQA27v7PWKW\n1RLCeArIASUCMFRSrZVGdTiia1whaJTnCy5Mi5MAABoFSURBVADNPksEeJ4HESsJctdbNNB0Lt0N\nw+l0MtfxSC9Jbm7u1JMnT/IAkAzQNLp/+eWXvgcPHkz47LPPppaXlw+sra31l8vl/KBBg/aHh4d/\nOGXKlJSEhIQ2vwOl3Z8UC4LAmkymbn2BTqejBw4cYH/99dc21729vW3V1dVNsgEAcBwn1NbWDkbE\nq3q8XQ1CCO/r65vt4+PTYcJBjUbjDA0NNfv5+bUx6HAcB5WVlYF5eXk3JS10dyBAAAgABy5RdZnf\n9Rlx2KDeFkf1ABB+W971BnOpB8e7FZQwQIkUBMS6QvupDiP4Jk9+2y1l1Bbh4jZcExQcfEOYQ1qu\n6ajM9cRsNovOnTt3XU762bp1q0dtbW3/6upqKwC4EFH0wgsvPPDFF19s2Lhx43tpaWkPVFdXh/j4\n+FgffvjhpVu3bn1i3bp1u9sr+W/FbDZLz5492602hoeHs/7+/ur2a3Rvb++WjpwCALjdbqivr4/e\nuHHjkN8k7UW0Wq2BEFLTvmJCCPTq1csUERFhUyqVbbzXBEEAm83W++DBg512G7z5IHCCHTzlfRX9\nw6deF0VnXWyU1VXhVV538Dd5kVEKal6wSgkwICAHMpGqvrfg06Gir1gBgoTVlPAc/99Zl0DAzZv7\nujlrn0u96K4vDMOI1Wr1dTkAJDMz07+0tDRSIpGYEVF4//33p2zfvv2jEydOjHc6nRQAIDAw0DR7\n9uyV69ev/9jH5+o+/t3BarXKCwsLr+pp2hFVVVXikpISr/ZrdG9v75a8BBQAABGFysrKkM2bN09E\nxN/cc9bW1nI8z1+SaUMikXD+/v7Z48ePt0kkkkwvL682o5TBYOidnp7e53of3SSXy2H48OFdqkMg\nHctECAUUXN6axu6faHM5EJEKYBvqcBvFDc7cTll0mcuE7qiVAd48usQIAiBS4AV3lcPRcUy+Tkc4\nucT/HAVqbnKYQwCgwINVYnc0TEpOvvZtBWhyG+3oularVYwaNeqanwuAiEx5efmICxcuhERHR5eW\nlZX1Xbt27Zvtzi4XJk2atGblypWrrxRy2tDQIBYEoVtTb0QEDw8P5d13363oTvnevXv719bWtjle\nTC6X20aMGNGSXIUCNG1tWSwW0YULFyZmZmZeqxH1kpfBz8+vMTAw8ERQUJArIiLiTGBgYJvc0vX1\n9ZqGhoYJAKC8RjIAAIDL5Wpz7hvDMDBkyJAuPRQBbSwC0Fau44BAgAIFq9Og4iTGKx6D1B2yyjb1\nbbAVPAhAXb7SUW2WOq8CADCtzByEAAgCaYlTbQOSGkuBDyUMC0CBYaQCCFx9ZGSHNwMAgLdm0DGW\nVRYiuKHlUEmeQp05+yHP3j92Kmy5K1RUVEg6ylUIAFBdXa2sqKjoda3rTEpKUlZUVEwWBIGLiYlp\n/Oabbybo9fo2Hp4hISHFcXFxVwxTBQDIz89XcVybE7vA7e58siWz2SypqqpSdWeQKy0tjaypqQls\nfa1fv375KpWqxeZGAQCaty6qqqoGf/PNN/f+1q22kJAQCcuy6vZTiYiIiOMxMTFnCSF8//790318\nfNqE/wmCQAoLC6esW7cupLt1IyLR6XRt5Hc6nWzr7RmbzQY+Pj5d+kGdbpMEoG3GnaYdZgQe7X5E\npA7urswdg6TCnDnJ7CqLEFEVp/b0aLvt+CqAwHMMIF5cIBFAROLCS41WyckHGTfXKOeBpwAAFEFQ\nSAMbk5LiL1u7q2Javq9y0F5e4HgEAQAIUGDBKZhCauz6mb9l+p6LOyRZVXuj08u+iWi+ptFoZCzL\ndmifcblcMrPZ3OdazDZbc/LkyTEpKSnjxWKxpVevXuo9e/YM5Tiuza5CRESE3sfHJ/tK35OYmMho\nNBqf9u97Y+NlMztdgtlsVrhcLi/oom8LIoqOHj16l81maxkcCSHQv3//7QsXLmzJ59Cs6BQAoKKi\nQnny5Mk5SUlJvl2prF3FpKqqKsjhcAS0vq5UKrnQ0NDE+Ph4IwDAU089Vd23b9+NGo2mzZT09OnT\n0SdPnvw9InbZKIeIohUrVozTarVteuXKykqpw+FgWt0HLperC9NPpIjUDxBl7ff9CBBwcdZAi8s8\n9FquXS/U7ulvtGTOJgxHWSpy8kLbXYokSCJO3iol9KKeX0wPJWE9L5Fh0CB/mZT11PK8g1IgIACx\nM4ysOjPz8h4Aw4cTd2+f8T9KWW0BD82rMAIUKDVas2adyF03trttqy/JDq9pSPu9zWpu6RxHjRrl\nKZPJOpy62mw2SVZW1siioqJuTW074sSJE71SUlIWVFdXayQSidvhcEiysrK07Wd+IpGoqrq643iA\nZvz9/dVOp7Nv62uUUpg6tfN2G0EQZBzHBUMXt+gOHz6sTU9Pv8/tdre8z9HR0VlDhgxJhP9um7RV\ndACA1NTUyZs2bXpOr9d31/ghPn/+/KTCwkL/lkoohdjY2F3Lli37uTnOnBDCLVmy5Ofhw4f/3Hpb\nwul0Mlu3bv3zs88+O62rFa9fv35ETU3NbKVS2abnt1qtng6Ho6U9iEjLy8uViNgpZU9NBInAW/sQ\nEZD2qkGABQFdyhrTmWkWS023O8jW5GKuJLdi199c7voYBsRAGZGbs5W0UfR4iKcMEckFwc00jeZu\nIEDFXvIB6vj4tu0qtJxVOAWTt4iKgBAAljA2q7Oo8WrnrfX96N4zvqroTyiROfHi9jIBCpzQGFZp\nPrQ8o2pzl9ubGI+MubFxioCcWwyiDICmtXJZWVmk0Wj06KgMz/OQlpZ279tvvz2xs8/sSiCiz6pV\nq145ffr0AwBN6/CAgACLj4+PHVopmiAIYLVaSX19/RWVr6GhISwrKyu29TWRSEQGDBhwueXcJR2s\nxWIRNTQ0DDt9+nSnOzNElHz77bfTS0tLWxJcqNVq24QJE15btmyZvrU/SrMxrqUhTqdTtHfv3gVb\ntmx5uLMVtubjjz8emJmZ+bvWiSXDw8PzZs+e/W5oaGgbS/uAAQPM06dP/8+gQYNOtb5eXFzsnZqa\nunz16tWdDhQoLi72PHDgwKNqtTpv0KBBLQkIk5KSRCqVKrhdoktSVlbW67vvvuvUjxo6McXHxduj\nUGiawraFACEEzO7SuFOlX8387LPfNr3U65OVWWc+mN9gyX6CMgwLSIASlrNh21xwYAAJLzgDgLgZ\nQAAECgLhRAKCZ3x8W/uItSFf6nAZVUBYEIAHSpV2VuR/1QAKoiNCv8BJ33hJ+q9FABcQvqm9wEKj\noyyuoGLHPwqq9/l1vnVI+//nm/uQwr1qie+B4f3nGwEAKisrPU+dOhVnt9svO7g0NjYq9Xr980lJ\nSdGdr+9Szpw547FgwYJlBw4cmON0OlkAAIZhqJeXlzEiIqKIUtpit0BEoJQGRkREXHZ2iYji77//\n/uHKyso2SzeO40S5ubkBHS2DJ02aBHK5vM2LhIhgNBoHZ2dnB7a//3K89957AzMyMuY2h6cyDMPH\nxsauX7VqVcuA2kzz9lobYerq6rTr1q1748MPP5zalfX6woULg9asWbP0yJEjLc71/v7+5Y899tgr\nc+fO/bUjj7fnnnsubcyYMS8MGDAgjWH++36eO3duaGJi4vvPPPPMxKvJoNPpxF999dUfKaVBfn5+\n37dOOV1bWzugoKAgtnUIISJCQ0PDcLlcPuxqbdLrdeJTJd8/7UTzwItL3EsgIAYAt6a84ejygWP/\nPacCT3XZMJeMyeyxgo8HFzg3v9/oynsTKCgBKQjIA6ViNyv2bjP6Zrt2BDi42ijCUGjK60YB0UWd\nXP2IifEFbbZpAgLu0gABLY8cEEKBoazRU9nL2Bm5gjWj63p7j3xRLen7NqViO1A3NIW+s6zFUfHX\nM6XfrUvO/NfE1NJVV9wayqjardh1dtnsovrUt8QyxcaYsGHHAAgiovLTTz/9fU5OzuirGaKOHDly\n75o1a97S6XRjk5OTpZ01XCEiWbVqlWzGjBl3Pfvss+9s2LDhL62TprhcLtGpU6ccd99990E/P782\nv0tOTs7gwsLCkR3VhYjk3XffnVJeXv7QpEmTtgcFBbXEDnAcRy9cuBC7efPm0Fb3M4cOHYoODw8f\nx/P8Jbpw9uzZoXv37n301KlTVx0svvnmm8Ddu3cvT0tLGw4AoNFoXKNGjVozYsSIFYSQSzpxAgAQ\nERGRUVBQ0KdXr14nNBqNy+l09i8uLu7dq1evmsmTJy+dMWPG1okTJzZezjUVEelrr73Wf9u2bUvT\n0tJmcxwHIpFIiImJOT9+/Ph/vfHGG5uvFmP+7LPP3qXX65empKQ8wHGcrNmwERYWVjh48OD/eHp6\n7p0/f35ZWVmZKz4+HuPi4qher5c+8sgjfWpra39HKY0dOXLkv5cuXXpkxYoV7LRp09jDhw+H/Prr\nry/s2LHjyeY90WakUilMmDBh38CBA1/q169fdmVlpV2n07maHugKkpkZyZZJcvw4i2F2nf3C3wkh\nGnLZ/oZAk5uoEwSBmj2kId8r1H03AUMvULvMaBeHuaeE+woHYTveB+MA4D5ysGgtAw3FUpCCB2E8\n+jkcVWMa7HmPOXlTJEtFpGm5T4AHO3jIIvd7eY1+6p7gGZUASTSzCHyKzKl/M1r0/8cQEWm23yAR\ngABjUktC3vBSRa119a2uiyM6Lq1ww7js6o3fcEJjCMuIgaGaZK2iz9wJka8XX+mZtCbXuEOdW7L3\nuUZX2Vw3b+5LAAhSAoLgAhaUJUppwC9Sid8BqUijb2w8Y+UFBpVSPxEj9vRyugzRVs70oNvdEKMQ\n9f5851b7h72lvRmLxRJQUFDwh1OnTi3Iycnp9DIgOjo6PzIycpNWqz1kt9vzvby86mfMmOHcv3+/\nYDQaISEhgW7fvl2Uk5OjkMvlWoPB0N9gMNxTVFT0YGVlZUh7oxmltDYsLOyRvLy803PmzPnn5s2b\nn2/OfkwIgfHjx++cOXPm3+fOnZtzMRaDzpw5U+nh4TG2tLT0r3369Pnl2Wef/XrlypX/+O677/7h\ncDhEAAASicRx//33rw0JCfmA4zhHbW3tKE9Pz1k2m+3YL7/8MqW+vv4uQggolUqjVCp1mM1mba9e\nverHjBnzksPh+DEpKekSpxxEZN95553wHTt2vJqcnDyDEAIhISHFkyZN+iogIOATnU7XoZs0AQAY\nNWrUMYZhUv38/N6Pi4tzFxUVRaekpEwtKSm539PT0yMwMHCfWCze1tjYmKZQKBrGjRvncjgcePDg\nQXGfPn28HQ7H3ZmZmfMyMjLuoZSCr69vTXR09JY//OEPX8yZM+d0ZxNJLF++PDA5OXlOXV3d1Lq6\nugF2u11jsVgYsVjMBwcHn/L19U1zOp0lIpHI5nK5lADgGxQUFOV0Oq1Wq/XNw4cPn8jMzFR88MEH\n03Jzc2MQcXRKSkqs0+ls3gxuDztkyJAcrVab2qtXYOq8V+b+4haORjntlSFSkSKs0VESZ+dMIwXk\nWQrs1f3vCQFBcBIAQDHrYZGKPbMokiwJ610lYZQ2p2DnRIyYMCwrtlhrVDzlgnjeFupwN/R38Y0q\nQCQMEcN/s71Q4NFMveSDdkSFzni+sHpvX+RtgywO88RGR+44QkVS0nrJSgjwghMYKnJ6SPseEDHy\nQ16a6C3Au/pmVf7wFSUYTFgGFEzwrgCvu+eODJvbpaOwEzGR8c7MH9doz5vl4BvGcoI9mBBBJKAD\nKGWACHIXpeIcRKwDRKSUVbJU6udwN6jkUuVZD/mQD9euKCzNyDw1SaWS+fE8PzwtLS3a6Wyxx3Ym\nxoEAAGEYhoaHhxuVSmU+IaQ0JCSkThAEp8vlIgqFgq2srFQ3NDT4siwbaDAYQquqquRcU86y9u8C\nI5FIqtVq9UyDwXBkw4YNfl9//fXCnJycmbW1tb1tNhsFaMr0GhYW9t2BAweqQkND1YMHD+7ndDrD\nKKVb1q1bt/li/sKAjz766JVDhw49aTKZFIgIDMNAaGjoeYVCYXO5XAFDhgz5JjAw8NvVq1f/wPP8\nsBEjRhzXarWfAkAtz/PDLly4MM3T09PX19d3i0aj2Wy320uDg4O5goICRiwWexJC7jMYDH/Q6/Ux\ncrm8UqvVnoyNjf1yzZo1e64UEEYAAJYvXz5y1qxZWQMGDGjx+klOTmaPHTvWe8eOHZPtdvv9Xl5e\nvhzHOXiet6lUKofD4RDV1NT4chwX3NjY6MmybJ2fn1+2QqE4ERAQsPvVV1890a9fvy6H7SEiWbx4\ncYDFYhlYXV0dU1xcHCGXy4MrKio0LpdL4eXlxYWEhNjcbnetIAgXCCGHAgICjn377bdGAIA1a9Z4\nJCUlTT948GCYzWZrBAATw3TsRnLRQUMpFovVgwfHVr73+YqdZvHuSWZrSRQHnJrnbIJA0MQQGQ+d\n6qo4ACoFAAdwvFWCQFQiRkEpUKBEhmJWISBxEYfbQgB4EJCnAnI8Ci4zQxQuhkpREFo/Kwo8cUjk\nYu2JaN9ZKTm1Pz3u5hv6Wdz1YkLQxIDEhUI790PKggB2kcA7vKQib7efenSivyrC93jJ++8RAH9K\nEbSyYT9GeS38S3BwcLeSfBzJ/krlEMpG8W5znNldO0RAdwjH25UcZxEjISJCKS8iCjcg10Aom6WW\nR+waEnD/3pmPv1ppMlnvKS8tHNvQ2Ci4XC4bAJgv93yuBM/zFAAkACAHAIZlWcbPz49QSqG6uhpd\nLldT9E5T2LMdAFwMw1zSkfA8L9Jqtda7775797Zt22oBmkbNFStWDDp69OhUSungiooKv8LCQk2/\nfv2EoKCgakQs8Pb2TuvTp8+vL7/8cn7r79uyZYvq+++/fyg/P3+yy+UKKSsrkwQHB7vlcnmpj4/P\npp9//nkXIaQ3IWTvyJEjrfPnz5/x9NNPNxslyUsvvRSQm5s7wuFwjFOpVDKlUtlosVi4qqoqtVwu\n93c4HF5FRUXlkZGRh/38/E7GxsbmL1iw4KpHXnUmNI957rnnPMxmc6DZbI7Mz88PGjhwoIevr6/M\n7XaTnJwc6/nz5yvGjh2bNWLEiLwxY8YYR4/+7UfvICJJSkoSnT59WjZgwADZ7t27pVlZWWKZTMaM\nGTPGBQBWk8nU+Pnnn7dZj+h0OqrT6STQ1Gu7Lv73cu1sfvAiAKCJiYluWZ91kkzHUUbW6OFaOCX3\n4rZKZ3c8mqvCllKJkEAhM5PJPKGnAAAhAGD2C8c6iwT1EMknxicJ/xXj0nqSkuJpQkKSgKgjSQcP\nyustOYKn/R5nfHyisGIFIa++eukoSAhCYmICzZQlSSL7xHO9xXF/yanZ9johKCcMAaUoZPXIXvNe\nDAgY3q2MJi2tTUQmMypJw3lLNfnV+xQKECt8VIMkpQ2nbFJWa1F5hpkB+tW1OoqJQNNvzUKTArpb\nXe+WCBfLUmheP7X9TLj4rzN1XNKVr169WhIZGanctm2b4uDBg1KpVMrOmDHDUVNTY5ZIJEadTtdh\n94+IdOXKlZra2lr1yZMnxdOnT7eNHj3afNddd5kJIThx4sTeGo3mxWHDhqUuX778u/YzXkQkS5cu\nVVZVVXn6+vqq8/Ly5HV1daxKpXIGBgYaAMD02WefWbpyyGKXfuCLZ5wzOp2Ojhw5koSEhKBer+fj\n4+OF2y209H+FRExkJOm/vtzoKHyRIIoJw4CYeLxpsfb959w4XU+mmJuATqdjw8LCNMOHDzdHRUXd\nkEjNOyxjYg/tqcIMxan0NW8bbfr5lLAMAQre8gEv9x5y11tRJOG2Cgfuofvccqdl9HBtcTdaJQDE\nB5BnCEEQkHeKxKq6SIi/IxIz9tA5bmi2kB5uPHXmArmdq1USygI0mQWdNkdtNSGXD2jp4c6jZ0S/\nw6HUTwXIKRCadpYoEXMKmZ+5/YENPdzZ9Cj6HQ4j2DVAGE3zbgBBtKllvQ03W64ebiw9in6HU9F4\nRuVGp4IhIgAgQFnWLDhqbrkzz3q4vvQo+h0Ox3OeILgUTenceaBUVuFkJDc0uWMPN58eRb/DkYhk\nAYC8suk0dBDE1CN7TPjCq3pS9XBn0aPodzRIzbZSNY8owqYIN7uK9cohBDqf46iHO4IeRb+DSUwE\nlgBoEN0sQR7ERGGkEo+sHov7/x49in4HM3bsWZFAWD9CKMPxLpCI1TkR2nsvXL1kD3caPYp+B2Mj\nRUqOt4YguACA8ippyOGKpNgOD9bo4c6mR9HvYBocZm+3YA/gwQESxrPaSxqeMnwB6Vmf/w/So+h3\nMBZrZRjPN3oj8qBS9En1EQ87c7Nl6uHm0KPodyinEEV23jjcKdSrWfCye0l7bw4OjupWookebn96\nglruUGRFO7xdbvNIwriJiu2dHOAZePBmy9TDzaNnRL9DqXNnxRlt2fdJ0L/cx2PIhyHax7qUH66H\nO4seRb8DqajYpjVaLjzOg0WuVcd+G6H+8+GeDED/2/Qo+h1GfDwy2aazj9Vbc6Yr2eAjkb3u/9jX\nl/T4tv+P05NK6o4CydmSrWMyyr74HICwfX0feG50+LO7brZUPdx8ekb0OwQdID1Tun5cZsW6rxA5\n8NNELxgd/pfdN1uuHm4Nekb0WxxEpAdy3vXXgOCMjVhSd2lq4GR2z/k0f6e7cLrJUfBXBkR2jSp8\n8dTo/+wjPT7tPVykR9FvYZKTkXVpX3zc7jIuIJRpkDLeJzjBXOASLGYAABmr8eSBRlsc5RPcfMMg\nCaM9Few1+m+jwv9y+mbL3sOtRY+i36IgovTXrLdmlpoOv8oJjcGUioGCGAgRWwkQK4BABcGlRcoD\nClCkVQz+boDPo1/0Cbin0+ep9fC/Q4+i34JcqNimzane90+TI2c2D3Y5RRkg8EAIAgICy7Ig8BK3\nmFGlsIx8d4jXiC0xJ+blkITOn9zRw/8WPZ5xtyBme4U/J9gjCRGbJETJAwEKQDgAWiNhNKUiRpLB\nSGVpSkmvVFnjotLYUOIGeOZmi93DLcz/A6Kq8os+JPH+AAAAAElFTkSuQmCC\n", 1171 | "prompt_number": 46, 1172 | "text": [ 1173 | "" 1174 | ] 1175 | } 1176 | ], 1177 | "prompt_number": 46 1178 | }, 1179 | { 1180 | "cell_type": "markdown", 1181 | "metadata": {}, 1182 | "source": [ 1183 | "## Task\n", 1184 | "\n", 1185 | "Make a module and script which fetches the XML formatted for the status of escalators in\n", 1186 | "the NYC subway system at http://www.grandcentral.org/developers/data/nyct/nyct_ene.xml,\n", 1187 | "calculate the fraction of those which have the reason \"Repair\", and prints this fraction.\n", 1188 | "\n", 1189 | "(Information about the data can be found at http://www.grandcentral.org/developers/download.html)\n", 1190 | "\n", 1191 | "The script should use the module, and should be installable by `python setup.py install`\n", 1192 | "\n", 1193 | " [lastname]/\n", 1194 | " [lastname]/\n", 1195 | " __init__.py\n", 1196 | " scripts/\n", 1197 | " getting_data.py\n", 1198 | " README.md\n", 1199 | " setup.py\n", 1200 | "\n", 1201 | "In `setup.py`, add the scripts to the call of the `setup` function.\n", 1202 | "For details, see http://peak.telecommunity.com/DevCenter/setuptools#basic-use\n", 1203 | "\n", 1204 | "This means that after `python setup.py install` has been run, it should be possible to simple type\n", 1205 | " \n", 1206 | " $ getting_data.py\n", 1207 | "\n", 1208 | "in the terminal and get the desired output." 1209 | ] 1210 | }, 1211 | { 1212 | "cell_type": "code", 1213 | "collapsed": false, 1214 | "input": [], 1215 | "language": "python", 1216 | "metadata": {}, 1217 | "outputs": [], 1218 | "prompt_number": 46 1219 | } 1220 | ], 1221 | "metadata": {} 1222 | } 1223 | ] 1224 | } -------------------------------------------------------------------------------- /Modern Python Idioms.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "Modern Python Idioms" 4 | }, 5 | "nbformat": 3, 6 | "nbformat_minor": 0, 7 | "worksheets": [ 8 | { 9 | "cells": [ 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "## Context Managers\n", 15 | "\n", 16 | "A _context manager_ is a way to specify particular runtime contexts in your python code.\n", 17 | "\n", 18 | "Most used example is file opening" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "collapsed": false, 24 | "input": [ 25 | "with open(\"example.json\") as file_handle:\n", 26 | " data = file_handle.read()" 27 | ], 28 | "language": "python", 29 | "metadata": {}, 30 | "outputs": [], 31 | "prompt_number": 2 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "metadata": {}, 36 | "source": [ 37 | "In the _context_ of the `with` block there is an open file object `file_handle`. Since having a file open takes limited system resources, it's always important to close open files." 38 | ] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "collapsed": false, 43 | "input": [ 44 | "file_handle = open(\"example.json\")\n", 45 | "data = file_handle.read()\n", 46 | "file_handle.close()" 47 | ], 48 | "language": "python", 49 | "metadata": {}, 50 | "outputs": [], 51 | "prompt_number": 9 52 | }, 53 | { 54 | "cell_type": "markdown", 55 | "metadata": {}, 56 | "source": [ 57 | "One should however always do" 58 | ] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "collapsed": false, 63 | "input": [ 64 | "file_handle = open(\"example.json\")\n", 65 | "try:\n", 66 | " data = file_handle.read()\n", 67 | "except IOError:\n", 68 | " pass\n", 69 | "except Exception:\n", 70 | " print(\"Got exception\")\n", 71 | "else:\n", 72 | " print(\"Got no excpetion\")\n", 73 | "finally:\n", 74 | " file_handle.close()" 75 | ], 76 | "language": "python", 77 | "metadata": {}, 78 | "outputs": [], 79 | "prompt_number": 7 80 | }, 81 | { 82 | "cell_type": "markdown", 83 | "metadata": {}, 84 | "source": [ 85 | "In the case using the managed context, parts of setting up things, and ensuring that things get properly torn down, are handled automatically when _entering_ and _exiting_ the context.\n", 86 | "\n", 87 | "This pattern is extremely common, therefore we have the `with` statement which uses what we call context managers to create contexts.\n", 88 | "\n", 89 | "A context manager is a class whose objects (called a _context guard_) has an `__enter__` method and an `__exit__` method " 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "collapsed": false, 95 | "input": [ 96 | "class ContextManager(object):\n", 97 | " def __enter__(self):\n", 98 | " pass\n", 99 | " \n", 100 | " def __exit__(self, type, value, traceback):\n", 101 | " pass" 102 | ], 103 | "language": "python", 104 | "metadata": {}, 105 | "outputs": [], 106 | "prompt_number": 13 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": {}, 111 | "source": [ 112 | "When using the `with` statement what actually happens is (equivalent to)" 113 | ] 114 | }, 115 | { 116 | "cell_type": "code", 117 | "collapsed": false, 118 | "input": [ 119 | "context_guard = ContextManager()\n", 120 | "value = context_guard.__enter__()\n", 121 | "exc = True\n", 122 | "try:\n", 123 | " try:\n", 124 | " var = value\n", 125 | " pass # Here the code in the context\n", 126 | " # would go.\n", 127 | "\n", 128 | " except:\n", 129 | " exc = False\n", 130 | " if not context_guard.__exit__(*sys.exc_info()):\n", 131 | " raise\n", 132 | "\n", 133 | "finally:\n", 134 | " if exc:\n", 135 | " context_guard.__exit__(None, None, None)" 136 | ], 137 | "language": "python", 138 | "metadata": {}, 139 | "outputs": [], 140 | "prompt_number": 20 141 | }, 142 | { 143 | "cell_type": "markdown", 144 | "metadata": {}, 145 | "source": [ 146 | "If `__enter__` returns a value, this can be caught by `... as var`\n", 147 | "\n", 148 | "One can suppress exceptions under certain conditions with the return value of the `__exit__` method\n", 149 | "\n", 150 | "Example: " 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "collapsed": false, 156 | "input": [ 157 | "import sys\n", 158 | "from StringIO import StringIO\n", 159 | "\n", 160 | "class redirect_stdout:\n", 161 | " def __init__(self, target):\n", 162 | " self.stdout = sys.stdout\n", 163 | " self.target = target\n", 164 | "\n", 165 | " def __enter__(self):\n", 166 | " sys.stdout = self.target\n", 167 | "\n", 168 | " def __exit__(self, type, value, tb):\n", 169 | " sys.stdout = self.stdout" 170 | ], 171 | "language": "python", 172 | "metadata": {}, 173 | "outputs": [], 174 | "prompt_number": 1 175 | }, 176 | { 177 | "cell_type": "code", 178 | "collapsed": false, 179 | "input": [ 180 | "out = StringIO()" 181 | ], 182 | "language": "python", 183 | "metadata": {}, 184 | "outputs": [], 185 | "prompt_number": 2 186 | }, 187 | { 188 | "cell_type": "code", 189 | "collapsed": false, 190 | "input": [ 191 | "with redirect_stdout(out):\n", 192 | " print(\"Prints to sys.stdout\")" 193 | ], 194 | "language": "python", 195 | "metadata": {}, 196 | "outputs": [], 197 | "prompt_number": 3 198 | }, 199 | { 200 | "cell_type": "code", 201 | "collapsed": false, 202 | "input": [ 203 | "print(\"This too\")" 204 | ], 205 | "language": "python", 206 | "metadata": {}, 207 | "outputs": [ 208 | { 209 | "output_type": "stream", 210 | "stream": "stdout", 211 | "text": [ 212 | "This too\n" 213 | ] 214 | } 215 | ], 216 | "prompt_number": 4 217 | }, 218 | { 219 | "cell_type": "code", 220 | "collapsed": false, 221 | "input": [ 222 | "out.getvalue()" 223 | ], 224 | "language": "python", 225 | "metadata": {}, 226 | "outputs": [ 227 | { 228 | "output_type": "pyout", 229 | "prompt_number": 5, 230 | "text": [ 231 | "'Prints to sys.stdout\\n'" 232 | ] 233 | } 234 | ], 235 | "prompt_number": 5 236 | }, 237 | { 238 | "cell_type": "markdown", 239 | "metadata": {}, 240 | "source": [ 241 | "## Decorators\n", 242 | "\n", 243 | "Say I have some functions defined in a module, among them these:" 244 | ] 245 | }, 246 | { 247 | "cell_type": "code", 248 | "collapsed": false, 249 | "input": [ 250 | "def my_add(x, y):\n", 251 | " return x + y\n", 252 | "\n", 253 | "def my_subtract(x, y):\n", 254 | " return x - y\n", 255 | "\n", 256 | "def my_multiply(x, y):\n", 257 | " return x * y" 258 | ], 259 | "language": "python", 260 | "metadata": {}, 261 | "outputs": [], 262 | "prompt_number": 7 263 | }, 264 | { 265 | "cell_type": "markdown", 266 | "metadata": {}, 267 | "source": [ 268 | "But I decide that whenever I'm calling this subset of functions, I want to print the parameters.\n", 269 | "That means I would have to modify each function this way:" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "collapsed": false, 275 | "input": [ 276 | "def my_add(x, y):\n", 277 | " print(x, y)\n", 278 | " return x + y\n", 279 | "\n", 280 | "my_add(6,7)" 281 | ], 282 | "language": "python", 283 | "metadata": {}, 284 | "outputs": [ 285 | { 286 | "output_type": "stream", 287 | "stream": "stdout", 288 | "text": [ 289 | "(6, 7)\n" 290 | ] 291 | }, 292 | { 293 | "output_type": "pyout", 294 | "prompt_number": 8, 295 | "text": [ 296 | "13" 297 | ] 298 | } 299 | ], 300 | "prompt_number": 8 301 | }, 302 | { 303 | "cell_type": "markdown", 304 | "metadata": {}, 305 | "source": [ 306 | "The printing functionality implementation would be the same for every function. This violates the principle of DRY: Don't Repeat Yourself.\n", 307 | "\n", 308 | "A better way would be to make the implementation of the argument printing once, and apply that implementation to each function." 309 | ] 310 | }, 311 | { 312 | "cell_type": "code", 313 | "collapsed": false, 314 | "input": [ 315 | "def print_args(function):\n", 316 | " def wrapper(*args, **kwargs):\n", 317 | " print(\"args: {}\".format(args))\n", 318 | " print(\"kwargs: {}\".format(kwargs))\n", 319 | "\n", 320 | " return function(*args, **kwargs)\n", 321 | " \n", 322 | " return wrapper" 323 | ], 324 | "language": "python", 325 | "metadata": {}, 326 | "outputs": [], 327 | "prompt_number": 9 328 | }, 329 | { 330 | "cell_type": "code", 331 | "collapsed": false, 332 | "input": [ 333 | "def my_add(x, y):\n", 334 | " return x + y\n", 335 | "\n", 336 | "my_add = print_args(my_add)\n", 337 | "\n", 338 | "my_add(3,4)" 339 | ], 340 | "language": "python", 341 | "metadata": {}, 342 | "outputs": [ 343 | { 344 | "output_type": "stream", 345 | "stream": "stdout", 346 | "text": [ 347 | "args: (3, 4)\n", 348 | "kwargs: {}\n" 349 | ] 350 | }, 351 | { 352 | "output_type": "pyout", 353 | "prompt_number": 10, 354 | "text": [ 355 | "7" 356 | ] 357 | } 358 | ], 359 | "prompt_number": 10 360 | }, 361 | { 362 | "cell_type": "markdown", 363 | "metadata": {}, 364 | "source": [ 365 | "There is a python shorthand for the definition and modification, which is a **decorator**." 366 | ] 367 | }, 368 | { 369 | "cell_type": "code", 370 | "collapsed": false, 371 | "input": [ 372 | "@print_args\n", 373 | "def my_subtract(x, y):\n", 374 | " return x - y\n", 375 | "\n", 376 | "my_subtract(5,2)" 377 | ], 378 | "language": "python", 379 | "metadata": {}, 380 | "outputs": [ 381 | { 382 | "output_type": "stream", 383 | "stream": "stdout", 384 | "text": [ 385 | "args: (5, 2)\n", 386 | "kwargs: {}\n" 387 | ] 388 | }, 389 | { 390 | "output_type": "pyout", 391 | "prompt_number": 11, 392 | "text": [ 393 | "3" 394 | ] 395 | } 396 | ], 397 | "prompt_number": 11 398 | }, 399 | { 400 | "cell_type": "code", 401 | "collapsed": false, 402 | "input": [ 403 | "my_add(1,2,3,5, a1=1, a2=4)" 404 | ], 405 | "language": "python", 406 | "metadata": {}, 407 | "outputs": [ 408 | { 409 | "ename": "TypeError", 410 | "evalue": "my_add() takes exactly 2 arguments (6 given)", 411 | "output_type": "pyerr", 412 | "traceback": [ 413 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", 414 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmy_add\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma1\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 415 | "\u001b[0;32m\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"kwargs: {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 416 | "\u001b[0;31mTypeError\u001b[0m: my_add() takes exactly 2 arguments (6 given)" 417 | ] 418 | }, 419 | { 420 | "output_type": "stream", 421 | "stream": "stdout", 422 | "text": [ 423 | "args: (1, 2, 3, 5)\n", 424 | "kwargs: {'a1': 1, 'a2': 4}\n" 425 | ] 426 | } 427 | ], 428 | "prompt_number": 12 429 | }, 430 | { 431 | "cell_type": "markdown", 432 | "metadata": {}, 433 | "source": [ 434 | "There are some built-in decorators in python, and many packages uses them for functionality.\n", 435 | "\n", 436 | "For example, in our production pipeline we use a `task` decorator from the package Celery to define available remotely executable \"tasks\" \n", 437 | "\n", 438 | "( https://github.com/SciLifeLab/bcbb/blob/master/nextgen/bcbio/distributed/tasks.py#L84 )\n", 439 | "\n", 440 | " ...\n", 441 | " \n", 442 | " from celery.task import task\n", 443 | " from bcbio.pipeline import lane\n", 444 | "\n", 445 | " ...\n", 446 | "\n", 447 | " @task\n", 448 | " def remove_contaminants(*args):\n", 449 | " return lane.remove_contaminants(*args)\n", 450 | "\n", 451 | " ...\n" 452 | ] 453 | }, 454 | { 455 | "cell_type": "markdown", 456 | "metadata": {}, 457 | "source": [ 458 | "### `@property` decorator\n", 459 | "\n", 460 | "You used to have a simple property of a class, but years later you realized this property would be more appropriate as returned value. But you don't want to break the functionality of the class for all the people who are already using your package." 461 | ] 462 | }, 463 | { 464 | "cell_type": "code", 465 | "collapsed": false, 466 | "input": [ 467 | "class CLS(object):\n", 468 | " def __init__(self):\n", 469 | " self.a = 1" 470 | ], 471 | "language": "python", 472 | "metadata": {}, 473 | "outputs": [], 474 | "prompt_number": 13 475 | }, 476 | { 477 | "cell_type": "code", 478 | "collapsed": false, 479 | "input": [ 480 | "obj = CLS()\n", 481 | "obj.a" 482 | ], 483 | "language": "python", 484 | "metadata": {}, 485 | "outputs": [ 486 | { 487 | "output_type": "pyout", 488 | "prompt_number": 14, 489 | "text": [ 490 | "1" 491 | ] 492 | } 493 | ], 494 | "prompt_number": 14 495 | }, 496 | { 497 | "cell_type": "code", 498 | "collapsed": false, 499 | "input": [ 500 | "class CLS(object):\n", 501 | " def __init__(self):\n", 502 | " self.b = 3.\n", 503 | " self.c = 2.\n", 504 | " \n", 505 | " @property\n", 506 | " def a(self):\n", 507 | " return self.b / self.c" 508 | ], 509 | "language": "python", 510 | "metadata": {}, 511 | "outputs": [], 512 | "prompt_number": 15 513 | }, 514 | { 515 | "cell_type": "code", 516 | "collapsed": false, 517 | "input": [ 518 | "obj = CLS()\n", 519 | "\n", 520 | "obj.a" 521 | ], 522 | "language": "python", 523 | "metadata": {}, 524 | "outputs": [ 525 | { 526 | "output_type": "pyout", 527 | "prompt_number": 16, 528 | "text": [ 529 | "1.5" 530 | ] 531 | } 532 | ], 533 | "prompt_number": 16 534 | }, 535 | { 536 | "cell_type": "code", 537 | "collapsed": false, 538 | "input": [ 539 | "obj.b = 4.\n", 540 | "\n", 541 | "obj.a" 542 | ], 543 | "language": "python", 544 | "metadata": {}, 545 | "outputs": [ 546 | { 547 | "output_type": "pyout", 548 | "prompt_number": 18, 549 | "text": [ 550 | "2.0" 551 | ] 552 | } 553 | ], 554 | "prompt_number": 18 555 | }, 556 | { 557 | "cell_type": "markdown", 558 | "metadata": {}, 559 | "source": [ 560 | "Property objects have `getter`, `setter` and `deleter` attributes which can be used as decorators." 561 | ] 562 | }, 563 | { 564 | "cell_type": "code", 565 | "collapsed": false, 566 | "input": [ 567 | "class CLS(object):\n", 568 | " def __init__(self):\n", 569 | " self.b = 3.\n", 570 | " self.c = 2.\n", 571 | " \n", 572 | " @property\n", 573 | " def a(self):\n", 574 | " return self.b / self.c\n", 575 | " \n", 576 | " @a.setter\n", 577 | " def a(self, value):\n", 578 | " self.c = 1.0\n", 579 | " self.b = value" 580 | ], 581 | "language": "python", 582 | "metadata": {}, 583 | "outputs": [], 584 | "prompt_number": 19 585 | }, 586 | { 587 | "cell_type": "code", 588 | "collapsed": false, 589 | "input": [ 590 | "obj = CLS()\n", 591 | "\n", 592 | "print(\"Initial b, c, a: \\n{}, {}, {}\".format(obj.b, obj.c, obj.a))\n", 593 | "\n", 594 | "obj.a = 5.0\n", 595 | "\n", 596 | "print(\"\")\n", 597 | "print(\"b, c, a after a being set: \\n{}, {}, {}\".format(obj.b, obj.c, obj.a))" 598 | ], 599 | "language": "python", 600 | "metadata": {}, 601 | "outputs": [ 602 | { 603 | "output_type": "stream", 604 | "stream": "stdout", 605 | "text": [ 606 | "Initial b, c, a: \n", 607 | "3.0, 2.0, 1.5\n", 608 | "\n", 609 | "b, c, a after a being set: \n", 610 | "5.0, 1.0, 5.0\n" 611 | ] 612 | } 613 | ], 614 | "prompt_number": 20 615 | }, 616 | { 617 | "cell_type": "markdown", 618 | "metadata": {}, 619 | "source": [ 620 | "### Class Decorators\n", 621 | "\n", 622 | "One can also define decorators that take a class and return a new class. E.g. to add some pattern of methods or standard fields. This can be a quicker (to implement) alternative to MetaClasses." 623 | ] 624 | }, 625 | { 626 | "cell_type": "markdown", 627 | "metadata": {}, 628 | "source": [ 629 | "## Object Orientation\n", 630 | "\n", 631 | "(Remember inheritance? See http://software-carpentry.org/4_0/oop/inherit.html if you need a refresher)" 632 | ] 633 | }, 634 | { 635 | "cell_type": "code", 636 | "collapsed": false, 637 | "input": [ 638 | "class A(object):\n", 639 | " a = []\n", 640 | "\n", 641 | "class B(A):\n", 642 | " def __init__(self, name):\n", 643 | " self.name = name\n", 644 | " self.a.append(self.name)\n", 645 | " \n", 646 | " def print_list(self):\n", 647 | " print(self.a)" 648 | ], 649 | "language": "python", 650 | "metadata": {}, 651 | "outputs": [], 652 | "prompt_number": 21 653 | }, 654 | { 655 | "cell_type": "code", 656 | "collapsed": false, 657 | "input": [ 658 | "b1 = B(\"1\")\n", 659 | "b2 = B(\"2\")" 660 | ], 661 | "language": "python", 662 | "metadata": {}, 663 | "outputs": [], 664 | "prompt_number": 22 665 | }, 666 | { 667 | "cell_type": "code", 668 | "collapsed": false, 669 | "input": [ 670 | "b1.print_list()" 671 | ], 672 | "language": "python", 673 | "metadata": {}, 674 | "outputs": [ 675 | { 676 | "output_type": "stream", 677 | "stream": "stdout", 678 | "text": [ 679 | "['1', '2']\n" 680 | ] 681 | } 682 | ], 683 | "prompt_number": 23 684 | }, 685 | { 686 | "cell_type": "code", 687 | "collapsed": false, 688 | "input": [ 689 | "b2.a is b1.a" 690 | ], 691 | "language": "python", 692 | "metadata": {}, 693 | "outputs": [ 694 | { 695 | "output_type": "pyout", 696 | "prompt_number": 25, 697 | "text": [ 698 | "True" 699 | ] 700 | } 701 | ], 702 | "prompt_number": 25 703 | }, 704 | { 705 | "cell_type": "code", 706 | "collapsed": false, 707 | "input": [ 708 | "b3 = B(\"3\")\n", 709 | "b1.print_list()" 710 | ], 711 | "language": "python", 712 | "metadata": {}, 713 | "outputs": [ 714 | { 715 | "output_type": "stream", 716 | "stream": "stdout", 717 | "text": [ 718 | "['1', '2', '3']\n" 719 | ] 720 | } 721 | ], 722 | "prompt_number": 26 723 | }, 724 | { 725 | "cell_type": "code", 726 | "collapsed": false, 727 | "input": [ 728 | "A.a" 729 | ], 730 | "language": "python", 731 | "metadata": {}, 732 | "outputs": [ 733 | { 734 | "output_type": "pyout", 735 | "prompt_number": 27, 736 | "text": [ 737 | "['1', '2', '3']" 738 | ] 739 | } 740 | ], 741 | "prompt_number": 27 742 | }, 743 | { 744 | "cell_type": "code", 745 | "collapsed": false, 746 | "input": [ 747 | "del b3" 748 | ], 749 | "language": "python", 750 | "metadata": {}, 751 | "outputs": [], 752 | "prompt_number": 28 753 | }, 754 | { 755 | "cell_type": "code", 756 | "collapsed": false, 757 | "input": [ 758 | "b1.print_list()" 759 | ], 760 | "language": "python", 761 | "metadata": {}, 762 | "outputs": [ 763 | { 764 | "output_type": "stream", 765 | "stream": "stdout", 766 | "text": [ 767 | "['1', '2', '3']\n" 768 | ] 769 | } 770 | ], 771 | "prompt_number": 29 772 | }, 773 | { 774 | "cell_type": "code", 775 | "collapsed": false, 776 | "input": [ 777 | "b3" 778 | ], 779 | "language": "python", 780 | "metadata": {}, 781 | "outputs": [ 782 | { 783 | "ename": "NameError", 784 | "evalue": "name 'b3' is not defined", 785 | "output_type": "pyerr", 786 | "traceback": [ 787 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", 788 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mb3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 789 | "\u001b[0;31mNameError\u001b[0m: name 'b3' is not defined" 790 | ] 791 | } 792 | ], 793 | "prompt_number": 30 794 | }, 795 | { 796 | "cell_type": "code", 797 | "collapsed": false, 798 | "input": [ 799 | "class A(object):\n", 800 | " a = []\n", 801 | "\n", 802 | "\n", 803 | "class B(A, object):\n", 804 | " def __init__(self, name):\n", 805 | " self.name = name\n", 806 | " self.a.append(self.name)\n", 807 | " \n", 808 | " def print_list(self):\n", 809 | " print(self.a)\n", 810 | " \n", 811 | " def __del__(self):\n", 812 | " self.a.remove(self.name)\n", 813 | "\n", 814 | "\n", 815 | "b1 = B(\"1\")\n", 816 | "b2 = B(\"2\")\n", 817 | "b3 = B(\"3\")" 818 | ], 819 | "language": "python", 820 | "metadata": {}, 821 | "outputs": [], 822 | "prompt_number": 31 823 | }, 824 | { 825 | "cell_type": "code", 826 | "collapsed": false, 827 | "input": [ 828 | "b1.print_list()" 829 | ], 830 | "language": "python", 831 | "metadata": {}, 832 | "outputs": [ 833 | { 834 | "output_type": "stream", 835 | "stream": "stdout", 836 | "text": [ 837 | "['1', '2', '3']\n" 838 | ] 839 | } 840 | ], 841 | "prompt_number": 32 842 | }, 843 | { 844 | "cell_type": "code", 845 | "collapsed": false, 846 | "input": [ 847 | "del b3" 848 | ], 849 | "language": "python", 850 | "metadata": {}, 851 | "outputs": [], 852 | "prompt_number": 33 853 | }, 854 | { 855 | "cell_type": "code", 856 | "collapsed": false, 857 | "input": [ 858 | "b1.print_list()" 859 | ], 860 | "language": "python", 861 | "metadata": {}, 862 | "outputs": [ 863 | { 864 | "output_type": "stream", 865 | "stream": "stdout", 866 | "text": [ 867 | "['1', '2']\n" 868 | ] 869 | } 870 | ], 871 | "prompt_number": 34 872 | }, 873 | { 874 | "cell_type": "markdown", 875 | "metadata": {}, 876 | "source": [ 877 | "Python has many method names starting and ending with `__` which makes classes integrate more with the standard Python syntax.\n", 878 | "\n", 879 | "These are normally called \"Magic methods\" or \"Dunder methods\".\n", 880 | "\n", 881 | "This way one can make class objects behave in a nice way with any operator.\n", 882 | "\n", 883 | "A complete list of these methods, with descriptions, is available at http://docs.python.org/2/reference/datamodel.html#special-method-names\n", 884 | "\n", 885 | "A few examples are\n", 886 | "\n", 887 | " __add__\n", 888 | " __eq__\n", 889 | " __and__\n", 890 | " __str__" 891 | ] 892 | }, 893 | { 894 | "cell_type": "code", 895 | "collapsed": false, 896 | "input": [ 897 | "class A(object):\n", 898 | " a = []\n", 899 | "\n", 900 | "\n", 901 | "class B(A):\n", 902 | " def __init__(self, name):\n", 903 | " self.name = name\n", 904 | " self.a.append(self.name)\n", 905 | " \n", 906 | " def print_list(self):\n", 907 | " print(self.a)\n", 908 | " \n", 909 | " def __del__(self):\n", 910 | " self.a.remove(self.name)\n", 911 | " \n", 912 | " def __add__(self, other):\n", 913 | " combined_name = \"\".join(sorted([self.name, other.name]))\n", 914 | " combined_b = B(combined_name)\n", 915 | " return combined_b\n", 916 | "\n", 917 | "b1 = B(\"1\")\n", 918 | "b2 = B(\"2\")" 919 | ], 920 | "language": "python", 921 | "metadata": {}, 922 | "outputs": [], 923 | "prompt_number": 38 924 | }, 925 | { 926 | "cell_type": "code", 927 | "collapsed": false, 928 | "input": [ 929 | "b12 = b2 + b1" 930 | ], 931 | "language": "python", 932 | "metadata": {}, 933 | "outputs": [], 934 | "prompt_number": 39 935 | }, 936 | { 937 | "cell_type": "code", 938 | "collapsed": false, 939 | "input": [ 940 | "b1.print_list()" 941 | ], 942 | "language": "python", 943 | "metadata": {}, 944 | "outputs": [ 945 | { 946 | "output_type": "stream", 947 | "stream": "stdout", 948 | "text": [ 949 | "['1', '2', '12']\n" 950 | ] 951 | } 952 | ], 953 | "prompt_number": 37 954 | }, 955 | { 956 | "cell_type": "markdown", 957 | "metadata": {}, 958 | "source": [ 959 | "It is usually a good design choice to make the `repr` of an object behave as the input of the constructur to create an equivalent object." 960 | ] 961 | }, 962 | { 963 | "cell_type": "markdown", 964 | "metadata": {}, 965 | "source": [ 966 | "# NumPy\n", 967 | "\n", 968 | "NumPy is the heart of most calculations when using Python for scientific computing.\n", 969 | "\n", 970 | "The most important part of it is that it provides a very efficient multidimensional array object" 971 | ] 972 | }, 973 | { 974 | "cell_type": "code", 975 | "collapsed": false, 976 | "input": [ 977 | "import numpy as np" 978 | ], 979 | "language": "python", 980 | "metadata": {}, 981 | "outputs": [], 982 | "prompt_number": 40 983 | }, 984 | { 985 | "cell_type": "markdown", 986 | "metadata": {}, 987 | "source": [ 988 | "The difference between an `array` and a `list` is that an `array` puts data in to contiguous memory blocks. While a `list` has a block of addresses referring to data.\n", 989 | "\n", 990 | "This means `lists` are dynamic, you can store anything of any size in them, and grow them and shrink them however you want.\n", 991 | "\n", 992 | "NumPy `array`s can only have one datatype per instance. They _can_ be grown and shrunk in size, but that is a (relatively) costly operation and should be avoided.\n", 993 | "\n", 994 | "To instantiate an array one can pass a list to its constructor. But there are also efficient methods to create commonly needed arrays." 995 | ] 996 | }, 997 | { 998 | "cell_type": "code", 999 | "collapsed": false, 1000 | "input": [ 1001 | "a = np.array([1., 2., 3.])" 1002 | ], 1003 | "language": "python", 1004 | "metadata": {}, 1005 | "outputs": [], 1006 | "prompt_number": 41 1007 | }, 1008 | { 1009 | "cell_type": "code", 1010 | "collapsed": false, 1011 | "input": [ 1012 | "a" 1013 | ], 1014 | "language": "python", 1015 | "metadata": {}, 1016 | "outputs": [ 1017 | { 1018 | "output_type": "pyout", 1019 | "prompt_number": 42, 1020 | "text": [ 1021 | "array([ 1., 2., 3.])" 1022 | ] 1023 | } 1024 | ], 1025 | "prompt_number": 42 1026 | }, 1027 | { 1028 | "cell_type": "markdown", 1029 | "metadata": {}, 1030 | "source": [ 1031 | "Using the efficient array data structure, with numpy one can do _broadcasted_ operations on the arrays" 1032 | ] 1033 | }, 1034 | { 1035 | "cell_type": "code", 1036 | "collapsed": false, 1037 | "input": [ 1038 | "a * 4.0" 1039 | ], 1040 | "language": "python", 1041 | "metadata": {}, 1042 | "outputs": [ 1043 | { 1044 | "output_type": "pyout", 1045 | "prompt_number": 44, 1046 | "text": [ 1047 | "array([ 4., 8., 12.])" 1048 | ] 1049 | } 1050 | ], 1051 | "prompt_number": 44 1052 | }, 1053 | { 1054 | "cell_type": "code", 1055 | "collapsed": false, 1056 | "input": [ 1057 | "type(a)" 1058 | ], 1059 | "language": "python", 1060 | "metadata": {}, 1061 | "outputs": [ 1062 | { 1063 | "output_type": "pyout", 1064 | "prompt_number": 45, 1065 | "text": [ 1066 | "numpy.ndarray" 1067 | ] 1068 | } 1069 | ], 1070 | "prompt_number": 45 1071 | }, 1072 | { 1073 | "cell_type": "markdown", 1074 | "metadata": {}, 1075 | "source": [ 1076 | "Arrays can have any number of dimensions, unlike a list which onle has one. (n-dimensional array)\n", 1077 | "\n", 1078 | "Instantiated by lists of lists" 1079 | ] 1080 | }, 1081 | { 1082 | "cell_type": "code", 1083 | "collapsed": false, 1084 | "input": [ 1085 | "b = np.array([[1, 4, 6], [5, 2, 2]])" 1086 | ], 1087 | "language": "python", 1088 | "metadata": {}, 1089 | "outputs": [], 1090 | "prompt_number": 46 1091 | }, 1092 | { 1093 | "cell_type": "code", 1094 | "collapsed": false, 1095 | "input": [ 1096 | "b" 1097 | ], 1098 | "language": "python", 1099 | "metadata": {}, 1100 | "outputs": [ 1101 | { 1102 | "output_type": "pyout", 1103 | "prompt_number": 47, 1104 | "text": [ 1105 | "array([[1, 4, 6],\n", 1106 | " [5, 2, 2]])" 1107 | ] 1108 | } 1109 | ], 1110 | "prompt_number": 47 1111 | }, 1112 | { 1113 | "cell_type": "markdown", 1114 | "metadata": {}, 1115 | "source": [ 1116 | "We can access items like on a list of lists" 1117 | ] 1118 | }, 1119 | { 1120 | "cell_type": "code", 1121 | "collapsed": false, 1122 | "input": [ 1123 | "b[1][0]" 1124 | ], 1125 | "language": "python", 1126 | "metadata": {}, 1127 | "outputs": [ 1128 | { 1129 | "output_type": "pyout", 1130 | "prompt_number": 49, 1131 | "text": [ 1132 | "5" 1133 | ] 1134 | } 1135 | ], 1136 | "prompt_number": 49 1137 | }, 1138 | { 1139 | "cell_type": "markdown", 1140 | "metadata": {}, 1141 | "source": [ 1142 | "But arrays also have their own multidimensional accessors " 1143 | ] 1144 | }, 1145 | { 1146 | "cell_type": "code", 1147 | "collapsed": false, 1148 | "input": [ 1149 | "b[1, 0]" 1150 | ], 1151 | "language": "python", 1152 | "metadata": {}, 1153 | "outputs": [ 1154 | { 1155 | "output_type": "pyout", 1156 | "prompt_number": 50, 1157 | "text": [ 1158 | "5" 1159 | ] 1160 | } 1161 | ], 1162 | "prompt_number": 50 1163 | }, 1164 | { 1165 | "cell_type": "markdown", 1166 | "metadata": {}, 1167 | "source": [ 1168 | "As well as multidimensional slicing" 1169 | ] 1170 | }, 1171 | { 1172 | "cell_type": "code", 1173 | "collapsed": false, 1174 | "input": [ 1175 | "b[:, 1:2]" 1176 | ], 1177 | "language": "python", 1178 | "metadata": {}, 1179 | "outputs": [ 1180 | { 1181 | "output_type": "pyout", 1182 | "prompt_number": 51, 1183 | "text": [ 1184 | "array([[4],\n", 1185 | " [2]])" 1186 | ] 1187 | } 1188 | ], 1189 | "prompt_number": 51 1190 | }, 1191 | { 1192 | "cell_type": "markdown", 1193 | "metadata": {}, 1194 | "source": [ 1195 | "Beware though that the ndarray slices are _views_ in to the array rather than copies!\n", 1196 | "\n", 1197 | "This means one can also broadcast on to what the view is referring to" 1198 | ] 1199 | }, 1200 | { 1201 | "cell_type": "code", 1202 | "collapsed": false, 1203 | "input": [ 1204 | "b[1:2, 1:2] = 0" 1205 | ], 1206 | "language": "python", 1207 | "metadata": {}, 1208 | "outputs": [], 1209 | "prompt_number": 58 1210 | }, 1211 | { 1212 | "cell_type": "code", 1213 | "collapsed": false, 1214 | "input": [ 1215 | "b" 1216 | ], 1217 | "language": "python", 1218 | "metadata": {}, 1219 | "outputs": [ 1220 | { 1221 | "output_type": "pyout", 1222 | "prompt_number": 59, 1223 | "text": [ 1224 | "array([[1, 0, 6],\n", 1225 | " [5, 0, 2]])" 1226 | ] 1227 | } 1228 | ], 1229 | "prompt_number": 59 1230 | }, 1231 | { 1232 | "cell_type": "code", 1233 | "collapsed": false, 1234 | "input": [ 1235 | "b.shape" 1236 | ], 1237 | "language": "python", 1238 | "metadata": {}, 1239 | "outputs": [ 1240 | { 1241 | "output_type": "pyout", 1242 | "prompt_number": 60, 1243 | "text": [ 1244 | "(2, 3)" 1245 | ] 1246 | } 1247 | ], 1248 | "prompt_number": 60 1249 | }, 1250 | { 1251 | "cell_type": "code", 1252 | "collapsed": false, 1253 | "input": [], 1254 | "language": "python", 1255 | "metadata": {}, 1256 | "outputs": [], 1257 | "prompt_number": 60 1258 | }, 1259 | { 1260 | "cell_type": "markdown", 1261 | "metadata": {}, 1262 | "source": [ 1263 | "Good overview of most things one would need to know about arrays to use them efficiently:\n", 1264 | "\n", 1265 | "- http://www.scipy.org/Tentative_NumPy_Tutorial\n", 1266 | "\n", 1267 | "When operations are performed in a _vectorized_ way, specialized C and Fortran methods will be used to perform the operation. Making them very quick and efficient." 1268 | ] 1269 | }, 1270 | { 1271 | "cell_type": "markdown", 1272 | "metadata": {}, 1273 | "source": [ 1274 | "## Assignment\n", 1275 | "\n", 1276 | "- Make a new file in your module called `session2.py`, and move the code from `__init__.py` in to it.\n", 1277 | "\n", 1278 | "- Change your `getting_data.py` to import in this fashion: `from lastname.session2 import ...`\n", 1279 | "\n", 1280 | "- Make a context manager which changes the current directory for the python script using it, and changes back to where you came from upon exit.\n", 1281 | "\n", 1282 | "(Hint, look up `os.chdir` here http://docs.python.org/2/library/os.html )\n", 1283 | "\n", 1284 | "Implement a class `CourseRepo` which takes a \"surname\" string in the constructor.\n", 1285 | "\n", 1286 | "The class should have an attribute \"required\" which is a list of these strings:\n", 1287 | "\n", 1288 | "- `\".git\"`\n", 1289 | "- `\"setup.py\"`\n", 1290 | "- `\"README.md\"`\n", 1291 | "- `\"scripts/getting_data.py\"`\n", 1292 | "- `\"scripts/check_repo.py\"`\n", 1293 | "- `\"lastname/__init__.py\"`\n", 1294 | "- `\"lastname/session3.py\"`\n", 1295 | "\n", 1296 | "Where `\"lastname\"` is the \"surname\" string you gave to the constructor\n", 1297 | "\n", 1298 | "`surname` should be a **property**, such that when it is set, the `required` attribute changes to reflect the new surname.\n", 1299 | "\n", 1300 | "Example:\n", 1301 | "\n", 1302 | " repo = CourseRepo(\"a\")\n", 1303 | " print(repo.required[-1])\n", 1304 | " # prints a/session3.py\n", 1305 | " \n", 1306 | " repo.surname = \"b\"\n", 1307 | " print(repo.required[-1])\n", 1308 | " # prints b/session3.py\n", 1309 | "\n", 1310 | "The class should have a method `check()` which shall return `True` if all the strings in that list are existing files or directories.\n", 1311 | "\n", 1312 | "(Hint: `os.path.exists`)\n", 1313 | "\n", 1314 | "The context manager and class should be implemented in a file `session3.py` in your `lastname` module.\n", 1315 | "\n", 1316 | "Then make a script, `scripts/check_repo.py`, which imports the context manager for changing current directory as well as the `CourseRepo` class from the module. Like this:\n", 1317 | "\n", 1318 | " from lastname.session3 import CourseRepo, repo_dir\n", 1319 | "\n", 1320 | "(where `repo_dir` is the name of the context manager)\n", 1321 | "\n", 1322 | "This script should take an argument which is the absolute path to a repository.\n", 1323 | "\n", 1324 | "(Hint: `sys.argv` or the built in `argparse` module)\n", 1325 | "\n", 1326 | "The script should change directory to this given directory using the context manager. It should make an instance of `CourseRepo` using the final part of the absolute path, and call the `check()` method. If `check()` returns `True` the script shall print \"PASS\", otherwise the script should print \"FAIL\".\n", 1327 | "\n", 1328 | "Example: If I call the script like\n", 1329 | "\n", 1330 | " $ check_repo.py /Home/user/a\n", 1331 | "\n", 1332 | "it should make a `CourseRepo` instance like in the example above (with `\"a\"`)\n", 1333 | "\n", 1334 | "Like the other script, this script should also be installed when running `python setup.py install`.\n", 1335 | "\n", 1336 | "Note: If your repo has a structure this script is not expecting, fixing it should be rather smooth using `git mv` for renaming/moving files and directories." 1337 | ] 1338 | }, 1339 | { 1340 | "cell_type": "code", 1341 | "collapsed": false, 1342 | "input": [], 1343 | "language": "python", 1344 | "metadata": {}, 1345 | "outputs": [] 1346 | } 1347 | ], 1348 | "metadata": {} 1349 | } 1350 | ] 1351 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Python for Scientific Programming 2 | ===================== 3 | 4 | This repository contains IPython notebooks which I used in stead of slides for a Python course at KTH/SciLifeLab. 5 | 6 | You can easily view them here: 7 | 8 | - Getting Data: http://nbviewer.ipython.org/urls/raw.github.com/vals/scilife-python-course/master/Getting%2520Data.ipynb 9 | - Modern Python Idioms: http://nbviewer.ipython.org/urls/raw.github.com/vals/scilife-python-course/master/Modern%20Python%20Idioms.ipynb 10 | - Various data containers (including Pandas): http://nbviewer.ipython.org/urls/raw.github.com/vals/scilife-python-course/master/Session%204.ipynb 11 | - Optimizing Python: http://nbviewer.ipython.org/urls/raw.github.com/vals/scilife-python-course/master/Optimizing%20Python.ipynb 12 | - Parallel Python: http://nbviewer.ipython.org/urls/raw.github.com/vals/scilife-python-course/master/parallel%20python.ipynb 13 | -------------------------------------------------------------------------------- /example.csv: -------------------------------------------------------------------------------- 1 | chapter; pages with footnotes; pages with references 2 | 1; 5,6; 1,2,4,5 3 | 2; 8; 7,8,9 4 | 3; 11,12; 10,12,13,14 5 | 4; 16,19; 16,17,20 -------------------------------------------------------------------------------- /example.json: -------------------------------------------------------------------------------- 1 | { 2 | "firstName": "John", 3 | "lastName": "Smith", 4 | "age": 25, 5 | "address": { 6 | "streetAddress": "21 2nd Street", 7 | "city": "New York", 8 | "state": "NY", 9 | "postalCode": "10021" 10 | }, 11 | "phoneNumber": [ 12 | { 13 | "type": "home", 14 | "number": "212 555-1234" 15 | }, 16 | { 17 | "type": "fax", 18 | "number": "646 555-4567" 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /hanoi.py: -------------------------------------------------------------------------------- 1 | """Solve 'Towers of Hanoi' 2 | """ 3 | import sys 4 | 5 | PRINT = False 6 | 7 | @profile 8 | def solve(g,n): 9 | X = [sum(g[0])] 10 | Y = [sum(g[1])] 11 | Z = [sum(g[2])] 12 | 13 | moved = 0 14 | for i in range(2**n - 1): 15 | tops = [a[0] for a in g] 16 | movable = False 17 | j = 0 18 | while not movable: 19 | max_legal = max([t for t in tops if g[j][0] % 2 != t % 2]) 20 | if g[j][0] != moved and g[j][0] > 0: 21 | num_zeros = len([z for z in tops if z == 0]) 22 | if g[j][0] < max_legal or num_zeros > 0: 23 | movable = True 24 | if not movable: 25 | j += 1 26 | 27 | moved = g[j][0] 28 | 29 | legal = False 30 | k = 2 31 | while not legal: 32 | if (tops[k] % 2 != g[j][0] % 2 and g[j][0] < tops[k]) or tops[k] == 0: 33 | legal = True 34 | if not legal: 35 | k -= 1 36 | 37 | g[k] = [g[j][0]] + g[k] 38 | g[j] = g[j][1::] 39 | 40 | if PRINT: 41 | print g 42 | 43 | S = [sum(s) for s in g] 44 | X += [S[0]] 45 | Y += [S[1]] 46 | Z += [S[2]] 47 | 48 | return (X,Y,Z) 49 | 50 | 51 | if __name__ == "__main__": 52 | n = int(sys.argv[-1]) 53 | game = [range(1,n+1)+[0], [0], [0]] 54 | 55 | X, Y, Z = solve(game,n) 56 | -------------------------------------------------------------------------------- /nyct_ene.xml: -------------------------------------------------------------------------------- 1 | 0149TH ST - 3RD AVENUE STATIONBX2/5EL129ELSTREET TO UPTOWN PLATFORMY12/29/2012 11:30:00 PM12/30/2012 5:00:00 AMMAINTENANCEYN14TH ST - UNION SQ STATIONMNL/Q/N/RES258XESSTREET TO MEZZANINE ALL TRAINSN12/22/2012 6:38:00 AM12/31/2012 12:00:00 AMREPAIRNN14TH ST STATIONMNA/C/EEL223ELMEZZANINE TO UPTOWN A,C&E PLATFORMY12/29/2012 11:26:00 PM12/30/2012 4:30:00 AMMAINTENANCEYN161 ST - YANKEE STADIUM STATIONBX4/B/DEL133ELMEZZANINE TO DOWNTOWNY12/27/2012 9:31:00 AM12/29/2012 12:00:00 AMREPAIRNN181 ST STATIONMNAEL119ELLOWER MEZZANINE TO STREETN12/26/2012 8:09:00 PM12/28/2012 8:09:00 PMREPAIRNN181 ST STATIONMN1EL110ELLOWER MEZZANINE TO UPPER MEZZANINEN12/27/2012 8:07:00 AM12/29/2012 12:00:00 AMREPAIRNN190TH ST STATIONMNAEL120ELLOWER MEZZANINE TO STREETN12/27/2012 4:58:00 AM12/29/2012 4:30:00 AMREPAIRNN191 ST STATIONMN1EL105ELLOWER MEZZANINE TO UPPER MEZZANINEN12/24/2012 1:10:00 PM12/28/2012 11:00:00 PMREPAIRNN207TH ST YARD / ELECT SHOPMNN/AEL154ELUNASSIGNEDN12/21/2012 6:54:00 AM01/13/2013 12:00:00 AMREPAIRNN233RD ST - WOODLAWN STATIONBX2/5/METRO-NORTHEL194ELMEZZANINE TO PLATFORMY12/29/2012 11:30:00 PM12/30/2012 5:00:00 AMMAINTENANCEYN233RD ST - WOODLAWN STATIONBX2/5/METRO-NORTHEL193ELMEZZANINE TO UPTOWNY12/29/2012 11:30:00 PM12/30/2012 5:00:00 AMMAINTENANCEYN42ND ST - GRAND CENTRAL STATIONMN4/5/6/7/S/METRO-NORTHES609XES3 LANDINGSN12/24/2012 3:30:00 PM01/31/2013 12:00:00 AMREPAIRNN42ND ST - PORT AUTHORITY BUS TERMMNA/C/EEL290XEL3 LANDINGSN12/26/2012 7:22:00 AM12/28/2012 10:17:00 PMREPAIRNN51ST ST STATIONMN6ES252ESMEZZANINE TO PLATFORMY12/27/2012 8:08:00 AM12/29/2012 12:00:00 AMREPAIRNN59TH ST - COLUMBUS CIR STATIONMN1/A/B/C/DEL276XELSTREET TO MEZZANINE ALL TRAINSY12/07/2012 6:57:00 AM01/31/2013 10:20:00 PMREPAIRNN59TH ST - LEXINGTON AVE STATIONMN4/5/6ES213ESNORTHBOUND EXPRESS PLATFORM TO NORTHBOUND LOCAL PLATFORMN12/29/2012 11:34:00 PM12/30/2012 4:30:00 AMMAINTENANCEYNBOWERY STATIONMNJ/ZES334ESCENTER ISLAND PLATFORM TO MEZZANINEN08/03/2012 4:15:00 AM03/12/2013 4:15:00 AMSTATION IS UNDER REHABILITATIONNNCLARK ST STATIONBKN2/3ES379XES60 WALL STREET ATRIUM TO MEZZANINE 2/3 TRAIN PASSAGEWAYN12/27/2012 8:07:00 AM12/29/2012 12:00:00 AMREPAIRNNCLARK ST STATIONBKN2/3ES378XES60 WALL STREET ATRIUM TO MEZZANINE 2/3 TRAIN PASSAGEWAYN12/27/2012 8:05:00 AM12/29/2012 12:00:00 AMREPAIRNNEAST BROADWAY STATIONMNFES329ESCENTER ISLAND PLATFORM TO MEZZANINEN08/15/2012 10:49:00 AM03/12/2013 10:49:00 AMSTATION IS UNDER REHABILITATIONNNFAR ROCKAWAY MOTT AVE STATIONQNSAEL497ELSTREET TO PLATFORMY12/27/2012 8:49:00 AM12/29/2012 12:00:00 AMINSPECTIONNNHOWARD BEACHQNSA/CEL490XEL3 LANDINGSY11/14/2012 9:03:00 AM01/31/2013 12:00:00 AMREPAIRNNHOWARD BEACHQNSA/CES498XESMEZZANINE TO AIR TRAINN12/23/2012 4:39:00 AM12/27/2012 12:00:00 AMREPAIRNNHOWARD BEACHQNSA/CEL491XELMEZZANINE TO AIR TRAINN11/12/2012 10:20:00 PM01/31/2013 10:20:00 PMREPAIRNNHOWARD BEACHQNSA/CEL492XELMEZZANINE TO AIR TRAINN11/25/2012 5:15:00 PM01/31/2013 4:30:00 AMREPAIRNNHOWARD BEACHQNSA/CEL493XELMEZZANINE TO AIR TRAINN11/17/2012 5:45:00 AM01/31/2013 10:20:00 PMREPAIRNNJACKSON HTS - ROOSEVELT AVE STATIONQNSE/F/M/RES452ESLOWER MEZZANINE TO UPPER MEZZANINEN12/27/2012 11:20:00 PM12/28/2012 5:00:00 AMMAINTENANCEYNJAY/LAWRENCE STREET CONNECTIONBKNA/C/F/RES357ESMEZZANINE TO PLATFORMN12/17/2012 11:57:00 AM12/28/2012 11:57:00 AMREPAIRNNLEXINGTON AVENUE (3RD AVENUE)QNSE/MES254XESPASSAGEWAY MEZZANINE TO PLATFORM E/MN12/25/2012 10:16:00 PM12/31/2012 10:16:00 PMREPAIRNNMYRTLE AVE / WYCKOFF AVE STATIONBKNL/MES333ESLOWER MEZZANINE TO UPPER MEZZANINEN12/27/2012 9:26:00 AM12/29/2012 12:00:00 AMREPAIRNNPARSONS BLVD STATIONQNSFES445ESCENTER ISLAND LOWER PLATFORM TO MEZZANINEY12/23/2012 11:35:00 AM12/28/2012 12:00:00 AMREPAIRNNROOSEVELT ISLAND STATIONQNSFES418ESSOUTHBOUND PLATFORM TO LOWER MEZZANINEY12/27/2012 11:30:00 PM12/28/2012 5:00:00 AMMAINTENANCEYNTIME SQUARE COMPLEXMN1/2/3/7/ES267XESSTREET TO MEZZANINE ALL TRAINSN12/23/2012 8:52:00 PM01/31/2013 12:00:00 AMREPAIRNNTIME SQUARE COMPLEXMN1/2/3/7/ES268XESSTREET TO MEZZANINE ALL TRAINSN12/12/2012 10:15:00 PM03/31/2013 10:30:00 PMREPAIRNNWEST 4TH ST - WASHINGTON SQ STATIONMNA/B/C/D/E/F/MEL334ELMIDDLE MEZZANINE TO DOWNTOWN PLATFORMSY12/27/2012 8:39:00 AM12/29/2012 12:00:00 AMREPAIRNNWHITEHALL ST - SOUTH FERRY STATIONMNRES312ESLOWER MEZZANINE TO UPPER MEZZANINEN08/06/2012 11:18:00 AM03/12/2013 11:18:00 AMSTATION IS UNDER REHABILITATIONNNWOODSIDE - 61 ST STATIONQNS7/LIRREL416XELMEZZANINE TO DOWNTOWNY12/26/2012 7:23:00 AM12/28/2012 10:17:00 PMREPAIRNNWORLDWIDE CENTER/50th STREETMNC/EES264XESMEZZANINE TO E TRAIN PLATFORMN12/05/2012 5:36:00 AM01/07/2013 10:20:00 PMREPAIRNN -------------------------------------------------------------------------------- /parallel python.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "parallel python" 4 | }, 5 | "nbformat": 3, 6 | "nbformat_minor": 0, 7 | "worksheets": [ 8 | { 9 | "cells": [ 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "# Parallel computing with Python\n", 15 | "\n", 16 | "## `multiprocessing`\n", 17 | "\n", 18 | "The built in module `multiprocessing` provides functionality to create _processes_ which runs given tasks.\n", 19 | "\n", 20 | "http://docs.python.org/2/library/multiprocessing.html\n", 21 | "\n", 22 | "All strategies for paralleliztion has a rathe large overhead compared to lower level languages such as C or FORTRAN.\n", 23 | "\n", 24 | "The way `multiprocessing` runs code in parallel is by launching subprocesses with a seperate interpretor for for each process. This means that in order to gain speed the computation we want to perform should be relatively substantial.\n", 25 | "\n", 26 | "(In case you are familiar with threads: It should be noted that Python has a `threading` module for working with threads, however, all threads will be run on a single CPU.)\n", 27 | "\n", 28 | "Byt using multiprocessing we can utilize the machines we are running code on more efficiently" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "collapsed": false, 34 | "input": [ 35 | "import multiprocessing" 36 | ], 37 | "language": "python", 38 | "metadata": {}, 39 | "outputs": [], 40 | "prompt_number": 1 41 | }, 42 | { 43 | "cell_type": "code", 44 | "collapsed": false, 45 | "input": [ 46 | "multiprocessing.cpu_count()" 47 | ], 48 | "language": "python", 49 | "metadata": {}, 50 | "outputs": [ 51 | { 52 | "output_type": "pyout", 53 | "prompt_number": 2, 54 | "text": [ 55 | "16" 56 | ] 57 | } 58 | ], 59 | "prompt_number": 2 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": {}, 64 | "source": [ 65 | "Before talking about some more advanced featuers, let's describe the most typical use pattern of `multiprocessing`.\n", 66 | "\n", 67 | "Note: `multiprocessing` _can_ be used in the IPython Notebook, but there are sometimes issues with printing from subprocesses. To make things clearer and avoid complications we shall run external scripts in stead." 68 | ] 69 | }, 70 | { 71 | "cell_type": "markdown", 72 | "metadata": {}, 73 | "source": [ 74 | "### `Process`\n", 75 | "Processes share nothing\n", 76 | "\n", 77 | "To spawn a process, initiate it with a target function and call the `.start()` method.\n", 78 | "\n", 79 | "This method will arrange things so that given code will be run in a seperate process from the parent process. To get the parent process to wait until a process has finished before moving on one need to call the `.join()` method." 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "collapsed": false, 85 | "input": [ 86 | "import os\n", 87 | "os.getpid()" 88 | ], 89 | "language": "python", 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "output_type": "pyout", 94 | "prompt_number": 4, 95 | "text": [ 96 | "83928" 97 | ] 98 | } 99 | ], 100 | "prompt_number": 4 101 | }, 102 | { 103 | "cell_type": "code", 104 | "collapsed": false, 105 | "input": [ 106 | "%%file mp.py\n", 107 | "from multiprocessing import Process\n", 108 | "import os\n", 109 | "\n", 110 | "def worker():\n", 111 | " print(\"Worker process {}\".format(os.getpid()))\n", 112 | "\n", 113 | "if __name__ == \"__main__\":\n", 114 | " proc1 = Process(target=worker)\n", 115 | " proc1.start()\n", 116 | " proc2 = Process(target=worker)\n", 117 | " proc2.start()" 118 | ], 119 | "language": "python", 120 | "metadata": {}, 121 | "outputs": [ 122 | { 123 | "output_type": "stream", 124 | "stream": "stdout", 125 | "text": [ 126 | "Overwriting mp.py\n" 127 | ] 128 | } 129 | ], 130 | "prompt_number": 7 131 | }, 132 | { 133 | "cell_type": "code", 134 | "collapsed": false, 135 | "input": [ 136 | "%%bash\n", 137 | "python mp.py" 138 | ], 139 | "language": "python", 140 | "metadata": {}, 141 | "outputs": [ 142 | { 143 | "output_type": "stream", 144 | "stream": "stdout", 145 | "text": [ 146 | "Worker process 87461\n", 147 | "Worker process 87462\n" 148 | ] 149 | } 150 | ], 151 | "prompt_number": 8 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "To get the target function to actually work on some input, you need to provide the arguments in the constructur of the `Process`." 158 | ] 159 | }, 160 | { 161 | "cell_type": "code", 162 | "collapsed": false, 163 | "input": [ 164 | "%%file mp.py\n", 165 | "from multiprocessing import Process\n", 166 | "import os\n", 167 | "\n", 168 | "def worker(arg):\n", 169 | " print(\"Worker process {}, argument was {}\".format(os.getpid(), arg))\n", 170 | "\n", 171 | "if __name__ == \"__main__\":\n", 172 | " proc1 = Process(target=worker, args=(10,))\n", 173 | " proc1.start()\n", 174 | " proc2 = Process(target=worker, args=(11,))\n", 175 | " proc2.start()" 176 | ], 177 | "language": "python", 178 | "metadata": {}, 179 | "outputs": [ 180 | { 181 | "output_type": "stream", 182 | "stream": "stdout", 183 | "text": [ 184 | "Overwriting mp.py\n" 185 | ] 186 | } 187 | ], 188 | "prompt_number": 9 189 | }, 190 | { 191 | "cell_type": "code", 192 | "collapsed": false, 193 | "input": [ 194 | "%%bash\n", 195 | "python mp.py" 196 | ], 197 | "language": "python", 198 | "metadata": {}, 199 | "outputs": [ 200 | { 201 | "output_type": "stream", 202 | "stream": "stdout", 203 | "text": [ 204 | "Worker process 87490, argument was 10\n", 205 | "Worker process 87491, argument was 11\n" 206 | ] 207 | } 208 | ], 209 | "prompt_number": 10 210 | }, 211 | { 212 | "cell_type": "markdown", 213 | "metadata": {}, 214 | "source": [ 215 | "Processes communicate over interprocess communication channel\n", 216 | "\n", 217 | "- `Queue`\n", 218 | "- `Pipe`" 219 | ] 220 | }, 221 | { 222 | "cell_type": "markdown", 223 | "metadata": {}, 224 | "source": [ 225 | "### `Pipe`\n", 226 | "\n", 227 | "Gives a pair of connection objects which are connected by a pipe. Uses `send` and `recv` methods on objects to comminucate between processes." 228 | ] 229 | }, 230 | { 231 | "cell_type": "markdown", 232 | "metadata": {}, 233 | "source": [ 234 | "### `Queue`\n", 235 | "\n", 236 | "Gives a thread and process safe queue shared between processes. Can contain any pickle-able object." 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "collapsed": false, 242 | "input": [ 243 | "%%file mp2.py\n", 244 | "from multiprocessing import Process, Queue\n", 245 | "import os\n", 246 | "\n", 247 | "def worker(tasks, results):\n", 248 | " t = tasks.get()\n", 249 | " result = t * 2\n", 250 | " results.put([os.getpid(), t, \"->\", result])\n", 251 | "\n", 252 | "if __name__ == \"__main__\":\n", 253 | " n = 20\n", 254 | " my_tasks = Queue()\n", 255 | " my_results = Queue()\n", 256 | " \n", 257 | " workers = [Process(target=worker, args=(my_tasks, my_results)) for i in range(n)]\n", 258 | " \n", 259 | " for proc in workers:\n", 260 | " proc.start()\n", 261 | " \n", 262 | " for i in range(n):\n", 263 | " my_tasks.put(i)\n", 264 | " \n", 265 | " for i in range(n):\n", 266 | " result = my_results.get()\n", 267 | " print(result)" 268 | ], 269 | "language": "python", 270 | "metadata": {}, 271 | "outputs": [ 272 | { 273 | "output_type": "stream", 274 | "stream": "stdout", 275 | "text": [ 276 | "Overwriting mp2.py\n" 277 | ] 278 | } 279 | ], 280 | "prompt_number": 11 281 | }, 282 | { 283 | "cell_type": "code", 284 | "collapsed": false, 285 | "input": [ 286 | "%%bash\n", 287 | "python mp2.py" 288 | ], 289 | "language": "python", 290 | "metadata": {}, 291 | "outputs": [ 292 | { 293 | "output_type": "stream", 294 | "stream": "stdout", 295 | "text": [ 296 | "[87680, 0, '->', 0]\n", 297 | "[87681, 1, '->', 2]\n", 298 | "[87682, 2, '->', 4]\n", 299 | "[87683, 3, '->', 6]\n", 300 | "[87685, 5, '->', 10]\n", 301 | "[87684, 4, '->', 8]\n", 302 | "[87687, 7, '->', 14]\n", 303 | "[87686, 6, '->', 12]\n", 304 | "[87688, 8, '->', 16]\n", 305 | "[87689, 9, '->', 18]\n", 306 | "[87691, 11, '->', 22]\n", 307 | "[87690, 10, '->', 20]\n", 308 | "[87693, 13, '->', 26]\n", 309 | "[87692, 12, '->', 24]\n", 310 | "[87694, 14, '->', 28]\n", 311 | "[87695, 15, '->', 30]\n", 312 | "[87696, 16, '->', 32]\n", 313 | "[87697, 17, '->', 34]\n", 314 | "[87699, 19, '->', 38]\n", 315 | "[87698, 18, '->', 36]\n" 316 | ] 317 | } 318 | ], 319 | "prompt_number": 12 320 | }, 321 | { 322 | "cell_type": "markdown", 323 | "metadata": {}, 324 | "source": [ 325 | "Because the processes are executed in parallel we can never know the order of results being put in the `Queue`." 326 | ] 327 | }, 328 | { 329 | "cell_type": "code", 330 | "collapsed": false, 331 | "input": [ 332 | "from multiprocessing import Queue" 333 | ], 334 | "language": "python", 335 | "metadata": {}, 336 | "outputs": [], 337 | "prompt_number": 13 338 | }, 339 | { 340 | "cell_type": "code", 341 | "collapsed": false, 342 | "input": [ 343 | "q = Queue()" 344 | ], 345 | "language": "python", 346 | "metadata": {}, 347 | "outputs": [], 348 | "prompt_number": 14 349 | }, 350 | { 351 | "cell_type": "code", 352 | "collapsed": false, 353 | "input": [ 354 | "q.get?" 355 | ], 356 | "language": "python", 357 | "metadata": {}, 358 | "outputs": [], 359 | "prompt_number": 15 360 | }, 361 | { 362 | "cell_type": "markdown", 363 | "metadata": {}, 364 | "source": [ 365 | "### `Manager`\n", 366 | "\n", 367 | "A special `Process` which holds Python objects so that other processes can manipulate them.\n", 368 | "When a managed object is manipulated somewhere, the manager will make sure the change is propagated to any other processes using the managed object." 369 | ] 370 | }, 371 | { 372 | "cell_type": "code", 373 | "collapsed": false, 374 | "input": [ 375 | "%%file mp3.py\n", 376 | "from multiprocessing import Process, Manager\n", 377 | "import os\n", 378 | "\n", 379 | "def worker(l):\n", 380 | " p = os.getpid()\n", 381 | " l[int(str(p)[-2:])] = p\n", 382 | "\n", 383 | "\n", 384 | "if __name__ == \"__main__\":\n", 385 | " n = 100\n", 386 | " manager = Manager()\n", 387 | " l = manager.list()\n", 388 | " l.extend([0] * n)\n", 389 | " \n", 390 | " processes = [Process(target=worker, args=(l,)) for i in range(20)]\n", 391 | " \n", 392 | " for proc in processes:\n", 393 | " proc.start()\n", 394 | " \n", 395 | " for proc in processes:\n", 396 | " proc.join()\n", 397 | " \n", 398 | " print(l)" 399 | ], 400 | "language": "python", 401 | "metadata": {}, 402 | "outputs": [ 403 | { 404 | "output_type": "stream", 405 | "stream": "stdout", 406 | "text": [ 407 | "Overwriting mp3.py\n" 408 | ] 409 | } 410 | ], 411 | "prompt_number": 16 412 | }, 413 | { 414 | "cell_type": "code", 415 | "collapsed": false, 416 | "input": [ 417 | "%%bash\n", 418 | "python mp3.py" 419 | ], 420 | "language": "python", 421 | "metadata": {}, 422 | "outputs": [ 423 | { 424 | "output_type": "stream", 425 | "stream": "stdout", 426 | "text": [ 427 | "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87824, 87825, 87826, 87827, 87828, 87829, 87830, 87831, 87832, 87833, 87834, 87835, 87836, 87837, 87838, 87839, 87840, 87841, 87842, 87843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n" 428 | ] 429 | } 430 | ], 431 | "prompt_number": 17 432 | }, 433 | { 434 | "cell_type": "markdown", 435 | "metadata": {}, 436 | "source": [ 437 | "### `Pool`\n", 438 | "\n", 439 | "The `Pool` class distributes work between workers and collects the results as a list. It is _extremely_ handy for quickly implementing some simple parallelization." 440 | ] 441 | }, 442 | { 443 | "cell_type": "code", 444 | "collapsed": false, 445 | "input": [ 446 | "%%file mp.py\n", 447 | "import multiprocessing\n", 448 | "import os\n", 449 | "\n", 450 | "\n", 451 | "def task(args):\n", 452 | " print \"Running process\", os.getpid(), \"with args\", args\n", 453 | " return os.getpid(), args\n", 454 | "\n", 455 | "\n", 456 | "if __name__ == \"__main__\":\n", 457 | " pool = multiprocessing.Pool(processes=4)\n", 458 | " result = pool.map(task, [1,2,3,4]*3)\n", 459 | " print(result)" 460 | ], 461 | "language": "python", 462 | "metadata": {}, 463 | "outputs": [ 464 | { 465 | "output_type": "stream", 466 | "stream": "stdout", 467 | "text": [ 468 | "Overwriting mp.py\n" 469 | ] 470 | } 471 | ], 472 | "prompt_number": 20 473 | }, 474 | { 475 | "cell_type": "code", 476 | "collapsed": false, 477 | "input": [ 478 | "%%bash\n", 479 | "python mp.py" 480 | ], 481 | "language": "python", 482 | "metadata": {}, 483 | "outputs": [ 484 | { 485 | "output_type": "stream", 486 | "stream": "stdout", 487 | "text": [ 488 | "Running process 88078 with args 4\n", 489 | "Running process 88077 with args 3\n", 490 | "Running process 88077 with args 3\n", 491 | "Running process 88077 with args 2\n", 492 | "Running process 88075 with args 1\n", 493 | "Running process 88075 with args 1\n", 494 | "Running process 88075 with args 4\n", 495 | "Running process 88075 with args 3\n", 496 | "Running process 88076 with args 2\n", 497 | "Running process 88076 with args 2\n", 498 | "Running process 88076 with args 1\n", 499 | "Running process 88076 with args 4\n", 500 | "[(88075, 1), (88076, 2), (88077, 3), (88078, 4), (88075, 1), (88076, 2), (88077, 3), (88075, 4), (88076, 1), (88077, 2), (88075, 3), (88076, 4)]\n" 501 | ] 502 | } 503 | ], 504 | "prompt_number": 21 505 | }, 506 | { 507 | "cell_type": "markdown", 508 | "metadata": {}, 509 | "source": [ 510 | "The method `.map()` works like the built in function `map()`, but will send data from the iterable to different processes. By default it will send one element at a time, but this can be changed with the `chunksize` parameter.\n", 511 | "\n", 512 | "A similar method called `.map_async()` usually performs better in parallel, but in that case one has to fetch the results using a `.get()` method of the returned value of `.map_async()` (which is an instance of the class `AsyncResult`)." 513 | ] 514 | }, 515 | { 516 | "cell_type": "code", 517 | "collapsed": false, 518 | "input": [ 519 | "%%file mp.py\n", 520 | "import multiprocessing\n", 521 | "import os\n", 522 | "\n", 523 | "def task(args):\n", 524 | " print \"Running process\", os.getpid(), \"with args\", args\n", 525 | " return os.getpid(), args\n", 526 | "\n", 527 | "\n", 528 | "if __name__ == \"__main__\":\n", 529 | " pool = multiprocessing.Pool(processes=4)\n", 530 | " result = pool.map_async(task, [1,2,3,4])\n", 531 | " print(result.get())" 532 | ], 533 | "language": "python", 534 | "metadata": {}, 535 | "outputs": [ 536 | { 537 | "output_type": "stream", 538 | "stream": "stdout", 539 | "text": [ 540 | "Overwriting mp.py\n" 541 | ] 542 | } 543 | ], 544 | "prompt_number": 22 545 | }, 546 | { 547 | "cell_type": "code", 548 | "collapsed": false, 549 | "input": [ 550 | "%%bash\n", 551 | "python mp.py" 552 | ], 553 | "language": "python", 554 | "metadata": {}, 555 | "outputs": [ 556 | { 557 | "output_type": "stream", 558 | "stream": "stdout", 559 | "text": [ 560 | "Running process 88108 with args 1\n", 561 | "Running process 88109 with args 2\n", 562 | "Running process 88110 with args 3\n", 563 | "Running process 88111 with args 4\n", 564 | "[(88108, 1), (88109, 2), (88110, 3), (88111, 4)]\n" 565 | ] 566 | } 567 | ], 568 | "prompt_number": 23 569 | }, 570 | { 571 | "cell_type": "markdown", 572 | "metadata": {}, 573 | "source": [ 574 | "## IPython Parallel\n", 575 | "\n", 576 | "http://ipython.org/ipython-doc/dev/parallel/\n", 577 | "\n", 578 | "The strategy for achieving parallelization in IPython Parallel differs from `multiprocessing` in that you need to start _engine_ processes and a _hub_ process before you want to run code.\n", 579 | "\n", 580 | "One then connect to the hub, which will aid in the distrubution of work across the engines.\n", 581 | "\n", 582 | "When IPython is installed, a program `ipcluster` is also installed which simplifies starting engines and hub. To start a cluster with 4 workers one would run \n", 583 | "\n", 584 | " $ ipcluster start -n 4\n", 585 | "\n", 586 | "\n", 587 | "In the IPython Notebook, clusters can also be started from the IPython Dashboard." 588 | ] 589 | }, 590 | { 591 | "cell_type": "code", 592 | "collapsed": false, 593 | "input": [ 594 | "from IPython.parallel import Client" 595 | ], 596 | "language": "python", 597 | "metadata": {}, 598 | "outputs": [], 599 | "prompt_number": 24 600 | }, 601 | { 602 | "cell_type": "markdown", 603 | "metadata": {}, 604 | "source": [ 605 | "Let's try initiating the client before we have started the cluster!" 606 | ] 607 | }, 608 | { 609 | "cell_type": "code", 610 | "collapsed": false, 611 | "input": [ 612 | "cli = Client()" 613 | ], 614 | "language": "python", 615 | "metadata": {}, 616 | "outputs": [ 617 | { 618 | "ename": "IOError", 619 | "evalue": "Connection file not found: u'/Users/vale/.ipython/profile_default/security/ipcontroller-client.json'", 620 | "output_type": "pyerr", 621 | "traceback": [ 622 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mIOError\u001b[0m Traceback (most recent call last)", 623 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcli\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mClient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 624 | "\u001b[0;32m/Users/vale/.virtualenvs/devel/lib/python2.7/site-packages/IPython/parallel/client/client.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, url_or_file, profile, profile_dir, ipython_dir, context, debug, exec_key, sshserver, sshkey, password, paramiko, timeout, **extra_args)\u001b[0m\n\u001b[1;32m 409\u001b[0m \u001b[0murl_or_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msecurity_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl_or_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 410\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl_or_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 411\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Connection file not found: %r\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0murl_or_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 412\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl_or_file\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 413\u001b[0m \u001b[0mcfg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 625 | "\u001b[0;31mIOError\u001b[0m: Connection file not found: u'/Users/vale/.ipython/profile_default/security/ipcontroller-client.json'" 626 | ] 627 | } 628 | ], 629 | "prompt_number": 25 630 | }, 631 | { 632 | "cell_type": "markdown", 633 | "metadata": {}, 634 | "source": [ 635 | "IPython keeps track of addresses and such of local engines in the `.ipython` directory in your home folder. But `Client()` can also take a URL or a path to a configuration file with information on how to connect to the cluster.\n", 636 | "\n", 637 | "Now start the cluster (either in terminal or on the Dashboard)" 638 | ] 639 | }, 640 | { 641 | "cell_type": "code", 642 | "collapsed": false, 643 | "input": [ 644 | "cli = Client()" 645 | ], 646 | "language": "python", 647 | "metadata": {}, 648 | "outputs": [], 649 | "prompt_number": 26 650 | }, 651 | { 652 | "cell_type": "code", 653 | "collapsed": false, 654 | "input": [ 655 | "cli.ids" 656 | ], 657 | "language": "python", 658 | "metadata": {}, 659 | "outputs": [ 660 | { 661 | "output_type": "pyout", 662 | "prompt_number": 27, 663 | "text": [ 664 | "[0, 1, 2, 3, 4, 5, 6, 7]" 665 | ] 666 | } 667 | ], 668 | "prompt_number": 27 669 | }, 670 | { 671 | "cell_type": "markdown", 672 | "metadata": {}, 673 | "source": [ 674 | "The `ids` field lists the engines in the cluster.\n", 675 | "\n", 676 | "Say we want to have a look at the pid like we have done for all the `multiprocessing` exampels." 677 | ] 678 | }, 679 | { 680 | "cell_type": "code", 681 | "collapsed": false, 682 | "input": [ 683 | "def get_pid():\n", 684 | " import os\n", 685 | " return os.getpid()" 686 | ], 687 | "language": "python", 688 | "metadata": {}, 689 | "outputs": [], 690 | "prompt_number": 28 691 | }, 692 | { 693 | "cell_type": "code", 694 | "collapsed": false, 695 | "input": [ 696 | "cli[0].apply_sync(get_pid)" 697 | ], 698 | "language": "python", 699 | "metadata": {}, 700 | "outputs": [ 701 | { 702 | "output_type": "pyout", 703 | "prompt_number": 29, 704 | "text": [ 705 | "88707" 706 | ] 707 | } 708 | ], 709 | "prompt_number": 29 710 | }, 711 | { 712 | "cell_type": "code", 713 | "collapsed": false, 714 | "input": [ 715 | "get_pid()" 716 | ], 717 | "language": "python", 718 | "metadata": {}, 719 | "outputs": [ 720 | { 721 | "output_type": "pyout", 722 | "prompt_number": 30, 723 | "text": [ 724 | "83928" 725 | ] 726 | } 727 | ], 728 | "prompt_number": 30 729 | }, 730 | { 731 | "cell_type": "markdown", 732 | "metadata": {}, 733 | "source": [ 734 | "We need to `import os` in the function we send to the engine. That engine was started without any imported functions. Every time we want the engines to have some data, we need to explicitly send it to them. Or instructions on how to get it in the case of `import`.\n", 735 | "\n", 736 | "There is a method for syncing import across engines." 737 | ] 738 | }, 739 | { 740 | "cell_type": "code", 741 | "collapsed": false, 742 | "input": [ 743 | "cli[:].apply_sync(get_pid)" 744 | ], 745 | "language": "python", 746 | "metadata": {}, 747 | "outputs": [ 748 | { 749 | "output_type": "pyout", 750 | "prompt_number": 33, 751 | "text": [ 752 | "[88707, 88708, 88706, 88709, 88710, 88711, 88712, 88713]" 753 | ] 754 | } 755 | ], 756 | "prompt_number": 33 757 | }, 758 | { 759 | "cell_type": "markdown", 760 | "metadata": {}, 761 | "source": [ 762 | "A collection of engines is referred to as an _engine pool_, the standard _view_ to an engine pool is called a _direct view_." 763 | ] 764 | }, 765 | { 766 | "cell_type": "code", 767 | "collapsed": false, 768 | "input": [ 769 | "dview = cli[:]" 770 | ], 771 | "language": "python", 772 | "metadata": {}, 773 | "outputs": [], 774 | "prompt_number": 34 775 | }, 776 | { 777 | "cell_type": "markdown", 778 | "metadata": {}, 779 | "source": [ 780 | "The direct view provides a decorator which can be used to make any function parallelized." 781 | ] 782 | }, 783 | { 784 | "cell_type": "code", 785 | "collapsed": false, 786 | "input": [ 787 | "@dview.parallel(block=True)\n", 788 | "def pstr(x):\n", 789 | " return str(x)" 790 | ], 791 | "language": "python", 792 | "metadata": {}, 793 | "outputs": [], 794 | "prompt_number": 35 795 | }, 796 | { 797 | "cell_type": "markdown", 798 | "metadata": {}, 799 | "source": [ 800 | "After the `.parallel` decorater have been used on a function, it is changed in such a way that arguments given to the function will be split up, sent to engines, then put together again after the calculation is done.\n", 801 | "\n", 802 | "The new `ParallelFunction` also has a `.map()` method which will run the function for each of the values in an iterable on different engines, and return the list of results when it's done." 803 | ] 804 | }, 805 | { 806 | "cell_type": "code", 807 | "collapsed": false, 808 | "input": [ 809 | "str(range(10))" 810 | ], 811 | "language": "python", 812 | "metadata": {}, 813 | "outputs": [ 814 | { 815 | "output_type": "pyout", 816 | "prompt_number": 36, 817 | "text": [ 818 | "'[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]'" 819 | ] 820 | } 821 | ], 822 | "prompt_number": 36 823 | }, 824 | { 825 | "cell_type": "code", 826 | "collapsed": false, 827 | "input": [ 828 | "pstr(range(10))" 829 | ], 830 | "language": "python", 831 | "metadata": {}, 832 | "outputs": [ 833 | { 834 | "output_type": "pyout", 835 | "prompt_number": 37, 836 | "text": [ 837 | "['[0, 1]', '[2, 3]', '[4]', '[5]', '[6]', '[7]', '[8]', '[9]']" 838 | ] 839 | } 840 | ], 841 | "prompt_number": 37 842 | }, 843 | { 844 | "cell_type": "code", 845 | "collapsed": false, 846 | "input": [ 847 | "pstr.map(range(10))" 848 | ], 849 | "language": "python", 850 | "metadata": {}, 851 | "outputs": [ 852 | { 853 | "output_type": "pyout", 854 | "prompt_number": 38, 855 | "text": [ 856 | "['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']" 857 | ] 858 | } 859 | ], 860 | "prompt_number": 38 861 | }, 862 | { 863 | "cell_type": "code", 864 | "collapsed": false, 865 | "input": [ 866 | "from numpy import random" 867 | ], 868 | "language": "python", 869 | "metadata": {}, 870 | "outputs": [], 871 | "prompt_number": 39 872 | }, 873 | { 874 | "cell_type": "code", 875 | "collapsed": false, 876 | "input": [ 877 | "@dview.parallel(block=True)\n", 878 | "def task(delay):\n", 879 | " import os, time\n", 880 | " \n", 881 | " t0 = time.time()\n", 882 | " pid = os.getpid()\n", 883 | " time.sleep(delay)\n", 884 | " t1 = time.time()\n", 885 | " \n", 886 | " return [pid, t0, t1]" 887 | ], 888 | "language": "python", 889 | "metadata": {}, 890 | "outputs": [], 891 | "prompt_number": 40 892 | }, 893 | { 894 | "cell_type": "code", 895 | "collapsed": false, 896 | "input": [ 897 | "delays = random.rand(4)" 898 | ], 899 | "language": "python", 900 | "metadata": {}, 901 | "outputs": [], 902 | "prompt_number": 43 903 | }, 904 | { 905 | "cell_type": "code", 906 | "collapsed": false, 907 | "input": [ 908 | "task.map(delays)" 909 | ], 910 | "language": "python", 911 | "metadata": {}, 912 | "outputs": [ 913 | { 914 | "output_type": "pyout", 915 | "prompt_number": 44, 916 | "text": [ 917 | "[[88707, 1361974304.03308, 1361974304.959194],\n", 918 | " [88708, 1361974304.034218, 1361974304.233133],\n", 919 | " [88706, 1361974304.035217, 1361974304.954711],\n", 920 | " [88709, 1361974304.036044, 1361974304.276678]]" 921 | ] 922 | } 923 | ], 924 | "prompt_number": 44 925 | }, 926 | { 927 | "cell_type": "code", 928 | "collapsed": false, 929 | "input": [ 930 | "def visualize_tasks(results):\n", 931 | " res = np.array(results)\n", 932 | " fig, ax = plt.subplots(figsize=(10, res.shape[1]))\n", 933 | " \n", 934 | " yticks = []\n", 935 | " yticklabels = []\n", 936 | " tmin = min(res[:,1])\n", 937 | " for n, pid in enumerate(np.unique(res[:,0])):\n", 938 | " yticks.append(n)\n", 939 | " yticklabels.append(\"%d\" % pid)\n", 940 | " for m in np.where(res[:,0] == pid)[0]:\n", 941 | " ax.add_patch(Rectangle((res[m,1] - tmin, n-0.25),\n", 942 | " res[m,2] - res[m,1], 0.5, color=\"green\", alpha=0.5))\n", 943 | " \n", 944 | " ax.set_ylim(-.5, n+.5)\n", 945 | " ax.set_xlim(0, max(res[:,2]) - tmin + 0.)\n", 946 | " ax.set_yticks(yticks)\n", 947 | " ax.set_yticklabels(yticklabels)\n", 948 | " ax.set_ylabel(\"PID\")\n", 949 | " ax.set_xlabel(\"seconds\")" 950 | ], 951 | "language": "python", 952 | "metadata": {}, 953 | "outputs": [], 954 | "prompt_number": 45 955 | }, 956 | { 957 | "cell_type": "code", 958 | "collapsed": false, 959 | "input": [ 960 | "delays = random.rand(100) / 4." 961 | ], 962 | "language": "python", 963 | "metadata": {}, 964 | "outputs": [], 965 | "prompt_number": 46 966 | }, 967 | { 968 | "cell_type": "code", 969 | "collapsed": false, 970 | "input": [ 971 | "result = task.map(delays)" 972 | ], 973 | "language": "python", 974 | "metadata": {}, 975 | "outputs": [], 976 | "prompt_number": 47 977 | }, 978 | { 979 | "cell_type": "code", 980 | "collapsed": false, 981 | "input": [ 982 | "visualize_tasks(result)" 983 | ], 984 | "language": "python", 985 | "metadata": {}, 986 | "outputs": [ 987 | { 988 | "output_type": "display_data", 989 | "png": "iVBORw0KGgoAAAANSUhEUgAAAmwAAADOCAYAAACD3l7pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9UVPed//HnCGSFrZhAtkVHJ6MJlED4MVSIMaadtvFg\n7CYGixuELPFgsmxim5xUAknt8fiju3ZPu9HYrm31eIzLohOIp4Y9pu4qBjj+gEEZAz3GLdJMDER7\nmm2gKpREmO8f+TobigKRO3JneD3O4Rwvcz/3835/uGfO23vvzNvi8/l8iIiIiIhpTRrvAERERERk\neCrYRERERExOBZuIiIiIyalgExERETG58PEOIFAsFst4hyAiIiIyasN9DjRkCzYYPnGRkaxdu5a1\na9eOdxgSpHT+yFjpHJpYRrrQpFuiIiIiIiangk1ERETE5FSwiVyH0+kc7xAkiOn8kbHSOSSfZQnV\nTgcWi0XPsImIiEhQGKluCekPHQC8eOhFLly6MN5hjNqZD8+QeHvieIcxaoGIdzzXINjW/1pCIYfr\nCbbcPk+8wZbbWEykXIczmnW4uk/cF+L40YM/ukmRiRmFfMF24dIF7LfaxzuMUTty7siEj3c81yDY\n1v9aQiGH6wm23D5PvMGW21hMpFyHM5p1uLqPt8t7U2IS8wrIM2wbN24kOTmZlJQU8vPz6evrw+12\nk5WVhcPhIDMzk6amJgAqKipwOBz+n7CwMFpaWgBYvXo1NpuNKVOmDDr+L37xC1JTU3E4HNx33328\n/fbbgUhDRERExBQML9i8Xi/bt2+nubmZ1tZW+vv7cblclJWVsWHDBjweD+vXr6e0tBSAgoICPB4P\nHo+H8vJyZs+eTWpqKgCLFy/G7XYPmaOgoICWlhY8Hg/f//73WbVqldFpiIiIiJiG4bdEo6OjiYiI\noKenh7CwMHp6epg+fTpxcXF0d3cD0NXVhdVqHTJ29+7d5OXl+bezsrKuOcdnr7hdunSJ22+/3eAs\nRERERMzD8IItJiaGVatWYbPZiIyMJDs7mwULFpCQkMD8+fMpKSlhYGCA48ePDxlbWVlJdXX1qObZ\nunUrL7/8MpcvX+bYsWPX3Gft2rWcOnMK72Qv9nQ79nT7WFITERERMURtbS21tbWj3t/wgq29vZ3N\nmzfj9XqZOnUqS5cupaKigp07d7JlyxZycnKoqqqiqKiIgwcP+sc1NjYSFRVFUlLSqOZ55plneOaZ\nZ9izZw9FRUW89dZbQ/ZZu3Yt3n1ePdwqIiIipuJ0Ogd91966deuG3d/wZ9hOnDjBvHnziI2NJTw8\nnCVLlnD06FHcbjc5OTkA5ObmDnk2zeVykZ+f/7nne+yxx2hubjYkdhEREREzMrxgS0xMpKGhgd7e\nXnw+HzU1NSQlJREfH09dXR0Ahw8fJiEhwT9mYGCAqqqqQc+vDefs2bP+f+/fv9//IQURERGRUGT4\nLdG0tDQKCwuZM2cOkyZNIiMjg+LiYubOncvKlSvp6+sjMjKSbdu2+cfU19djs9mw2+2DjlVaWsqe\nPXvo7e1l5syZPPXUU6xZs4af/exnHDp0iIiICP7mb/6GnTt3Gp2GiIiIiGmEfGsqdToILHU6MJ9Q\nyOF6gi03dTq4tomU63DU6UA+a6TWVCFfsImIiIiY3Uh1S0A6HYiIiIiIcVSwiYiIiJicCjYRERER\nk1PBJiIiImJyKthERERETE4Fm4iIiIjJqWATERERMTkVbCIiIiImp4JNRERExOQM7yVqNmZqTRVq\n7VhCLZ/hhFquNzOfYFm7YInzs24kZrPmada4xttY1iXY11TtuAYL+YLtwqUL2G+1j3cYABw5d8Q0\nsRgh1PIZTqjlejPzCZa1C5Y4P+tGYjZrnmaNa7yNZV2CfU29Xd7xDsFUAnJLdOPGjSQnJ5OSkkJ+\nfj59fX243W6ysrJwOBxkZmbS1NQEQEVFBQ6Hw/8TFhZGS0sLAKtXr8ZmszFlypRBx3/55ZdJTk4m\nLS2NBx98kHPnzgUiDRERERFTMLxg83q9bN++nebmZlpbW+nv78flclFWVsaGDRvweDysX7+e0tJS\nAAoKCvB4PHg8HsrLy5k9ezapqakALF68GLfbPWSOjIwMTp48ydtvv01ubq7/WCIiIiKhyPCCLTo6\nmoiICHp6erhy5Qo9PT1Mnz6duLg4uru7Aejq6sJqtQ4Zu3v3bvLy8vzbWVlZxMXFDdnP6XQyefJk\nAO699146OjqMTkNERETENAx/hi0mJoZVq1Zhs9mIjIwkOzubBQsWkJCQwPz58ykpKWFgYIDjx48P\nGVtZWUl1dfXnmm/Hjh0sWrTomq+tXbuWU2dO4Z3sxZ5ux55uv5GURERERAxVW1tLbW3tqPc3vGBr\nb29n8+bNeL1epk6dytKlS6moqGDnzp1s2bKFnJwcqqqqKCoq4uDBg/5xjY2NREVFkZSUNOq5/uM/\n/oPm5mY2bdp0zdfXrl2Ld583qB+6FBERkdDjdDpxOp3+7XXr1g27v+G3RE+cOMG8efOIjY0lPDyc\nJUuWcPToUdxuNzk5OQDk5uYOeTbN5XKRn58/6nkOHTrEP//zP1NdXU1ERIShOYiIiIiYieEFW2Ji\nIg0NDfT29uLz+aipqSEpKYn4+Hjq6uoAOHz4MAkJCf4xAwMDVFVVDXp+bTgej4d//Md/5D//8z+5\n/fbbjU5BRERExFQMvyWalpZGYWEhc+bMYdKkSWRkZFBcXMzcuXNZuXIlfX19REZGsm3bNv+Y+vp6\nbDYbdrt90LFKS0vZs2cPvb29zJw5k6eeeoo1a9ZQWlrK5cuXyc3NBeCOO+5g3759RqciIiIiYgoW\nn8/nG+8gAsFiseDz+dTpIIBCLZ/hhFqu6nQwVLDE+VnqdBD61Olg4nQ6uFq3XPf1UC/YRERERMxu\npLpFzd9FRERETE4Fm4iIiIjJqWATERERMTkVbCIiIiImp4JNRERExORUsImIiIiYnAo2EREREZNT\nwSYiIiJicoa3pjIbM3U6MKNg/ybszwq2XIIt3qs+b9yBznO81tHoeccjj9HOGaiOChPtm+xFxiLk\nC7YLly5gv9U+3mGY1pFzR0JmfYItl2CL96rPG3eg8xyvdTR63vHIY7Rz3khsoxnj7fJ+rmOKTGS6\nJSoiIiJicgEp2DZu3EhycjIpKSnk5+fT19eH2+0mKysLh8NBZmYmTU1NAFRUVOBwOPw/YWFhtLS0\nALB69WpsNhtTpkwZdPz6+noyMjKIiIhg7969gUhBRERExDQML9i8Xi/bt2+nubmZ1tZW+vv7cblc\nlJWVsWHDBjweD+vXr6e0tBSAgoICPB4PHo+H8vJyZs+eTWpqKgCLFy/G7XYPmeOOO+5g165d5Ofn\nGx2+iIiIiOkY/gxbdHQ0ERER9PT0EBYWRk9PD9OnTycuLo7u7m4Aurq6sFqtQ8bu3r2bvLw8/3ZW\nVtY157jjjjsAmDRJd3RFREQk9BlesMXExLBq1SpsNhuRkZFkZ2ezYMECEhISmD9/PiUlJQwMDHD8\n+PEhYysrK6murjYslrVr13LqzCm8k73Y0+3Y0+2GHVtERETkRtXW1lJbWzvq/Q0v2Nrb29m8eTNe\nr5epU6eydOlSKioq2LlzJ1u2bCEnJ4eqqiqKioo4ePCgf1xjYyNRUVEkJSUZFsvatWvx7vMG5Sfx\nREREJHQ5nU6cTqd/e926dcPub/g9xRMnTjBv3jxiY2MJDw9nyZIlHD16FLfbTU5ODgC5ublDnk1z\nuVw39EyaxWIxJG4RERERszK8YEtMTKShoYHe3l58Ph81NTUkJSURHx9PXV0dAIcPHyYhIcE/ZmBg\ngKqqqkHPr42Gz+fD5/MZGr+IiIiI2RhesKWlpVFYWMicOXNITU3F5/NRXFzML3/5S0pLS0lPT+cH\nP/gB27Zt84+pr6/HZrNht9sHHau0tJSZM2fS29vLzJkzWb9+PQBNTU3MnDmT119/neLiYlJSUoxO\nQ0RERMQ0LL4QvURlsVjw+XxqTTWCYG2PdC3BlkuwxXuVWlMFZl61phKZ2K7WLdd9PdQLNhERERGz\nG6lu0ReZiYiIiJicCjYRERERk1PBJiIiImJyKthERERETE4Fm4iIiIjJqWATERERMTkVbCIiIiIm\np4JNRERExOTCxzuAQFKXg/8TrN+qP1qByi/Y1m24eIMtl0DSWph7DUKhe4WZ1zfQJkLu49GlI6QL\ntguXLmC/1T7eYZjCkXNHQnotApVfsK3bcPEGWy6BpLUw9xqMV2xGzmvm9Q20iZC7t8t70+fULVER\nERERkwtIwbZx40aSk5NJSUkhPz+fvr4+3G43WVlZOBwOMjMzaWpqAqCiogKHw+H/CQsLo6WlBYDV\nq1djs9mYMmXKoOP39fXx2GOPER8fz9y5c3nvvfcCkYaIiIiIKRhesHm9XrZv305zczOtra309/fj\ncrkoKytjw4YNeDwe1q9fT2lpKQAFBQV4PB48Hg/l5eXMnj2b1NRUABYvXozb7R4yx44dO4iNjaWt\nrY3nn3+esrIyo9MQERERMQ3DC7bo6GgiIiLo6enhypUr9PT0MH36dOLi4uju7gagq6sLq9U6ZOzu\n3bvJy8vzb2dlZREXFzdkv+rqap544gkAvv3tb1NTU2N0GiIiIiKmYfiHDmJiYli1ahU2m43IyEiy\ns7NZsGABCQkJzJ8/n5KSEgYGBjh+/PiQsZWVlVRXV484R2dnJzNnzvw0gfBwpk6dyh//+EdiYmIG\n7XfKdQrvZC8A9nQ79nT7mPMTERERGava2lpqa2tHvb/hBVt7ezubN2/G6/UydepUli5dSkVFBTt3\n7mTLli3k5ORQVVVFUVERBw8e9I9rbGwkKiqKpKQkw2JJz0sP+U+qiIiISPBxOp04nU7/9rp164bd\n3/BboidOnGDevHnExsYSHh7OkiVLOHr0KG63m5ycHAByc3OHPJvmcrnIz88f1RxWq5Vz584BcOXK\nFbq7u4dcXRMREREJFYYXbImJiTQ0NNDb24vP56OmpoakpCTi4+Opq6sD4PDhwyQkJPjHDAwMUFVV\nNej5teE88sgj7Nq1C4DXX3+db37zm0anISIiImIaht8STUtLo7CwkDlz5jBp0iQyMjIoLi5m7ty5\nrFy5kr6+PiIjI9m2bZt/TH19PTabDbvdPuhYpaWl7Nmzh97eXmbOnMlTTz3FmjVrWLFiBX//939P\nfHw8sbGxuFwuo9MQERERMQ2Lz+fzXe/FV199lS1btnDmzBkAkpKS+O53v+v/hKaZWSwWyg6WqTXV\n/xfqrULUmupTak01OloLc6+BWlMFt4mQeyBaU1ksFoYpya5/hW3Xrl288sorvPzyyzgcDnw+Hx6P\nhxdeeAGLxUJhYaGhgQbCze7zJSIiIhII173Cdu+99+JyuZg1a9ag33u9Xh577DEaGxtvSoA3aqRK\nVURERMQsRqpbrvuhg4sXLw4p1gDsdjsXL140JjoRERERGdF1C7bJkydfd9Bwr4mIiIiIsa57SzQy\nMpK77rrrmoPa29vp6ekJaGBjpVuiIiIiEixu+EMH77zzTkACEhEREZHPZ9iv9QhmusImIiIiweKG\nr7B94QtfwGKxXPegf/rTn8YenYiIiIiMSFfYRERERMbZDV9h6+3t5Re/+AXt7e2kpKSwYsUKwsMN\n72QVUC8eetE0nQ4mwjc/j8TMa2Dm2EbLqBwCvRY3a61D4W961VhzCaW1GM5EyfOqQOQbzGsYiO4D\nZnLdCuyJJ57glltuYf78+bz55pucPn2aV1555WbGNmYXLl3Afqt9vMMA4Mi5I6aJZbyYeQ3MHNto\nGZVDoNfiZq11KPxNrxprLqG0FsOZKHleFYh8g3kNvV3e8Q4hoIb9lGhraysATz75JJmZmTctKBER\nERH5P9f94tzP3v78vLdCN27cSHJyMikpKeTn59PX14fb7SYrKwuHw0FmZiZNTU0AVFRU4HA4/D9h\nYWG0tLQAcPLkSVJSUoiPj+e5557zH/+9997jm9/8JmlpaXz961+ns7Pzc8UnIiIiEkyuW7C1tLQw\nZcoU/09ra6v/39HR0dc9oNfrZfv27TQ3N9Pa2kp/fz8ul4uysjI2bNiAx+Nh/fr1lJaWAlBQUIDH\n48Hj8VBeXs6sWbNITU0F4Omnn2bHjh20tbXR1tbGgQMHACgpKWH58uW8/fbbrFmzhpdeesnINRER\nERExlesWbP39/Vy8eNH/c+XKFf+/h/tKj+joaCIiIujp6eHKlSv09PQwffp04uLi6O7uBqCrqwur\n1Tpk7O7du1m2bBkA58+f5+LFi2RlZQFQWFjIvn37gE9v137jG98AwOl08sYbb9xg+iIiIiLmZ/jH\nPmNiYli1ahU2m43IyEiys7NZsGABCQkJzJ8/n5KSEgYGBjh+/PiQsZWVlVRXVwPQ2dnJjBkz/K9Z\nrVb/rc+0tDT27t3Ls88+y69+9SsuXrzIRx99xG233TboeKdcp/BO9gJgT7djT7cbna6IiIjI51Zb\nW0ttbe2o9ze8YGtvb2fz5s14vV6mTp3K0qVLqaioYOfOnWzZsoWcnByqqqooKiri4MGD/nGNjY1E\nRUWRlJQ04hw/+clP+M53vsOrr77KV7/6VaxWK2FhYUP2S89LD9pPu4iIiEjocjqdOJ1O//a6deuG\n3d/wgu3EiRPMmzeP2NhYAJYsWcLRo0dxu90cOnQIgNzcXJ588slB41wuF/n5+f5tq9VKR0eHf7uj\no8N/G3XatGns3bsXgEuXLrF3795hn6sTERERCWbXfYbtRiUmJtLQ0EBvby8+n4+amhqSkpKIj4+n\nrq4OgMOHD5OQkOAfMzAwQFVVFXl5ef7fTZs2jejoaBobG/H5fJSXl/Poo48C8L//+78MDAwAn34i\ndcWKFUanISIiImIahl9hS0tLo7CwkDlz5jBp0iQyMjIoLi5m7ty5rFy5kr6+PiIjI9m2bZt/TH19\nPTabDbvdPuhYW7duZfny5fT29rJo0SIWLlwIfHrf96WXXsJisfC1r32Nf/u3fzM6DRERERHTCOle\nomUHy9SaykTMvAZmjm201JpqfOa5GdSaanQmSp5XqTXVYMHemmqkXqIhXbCFaGoiIiISYkaqWwx/\nhk1EREREjKWCTURERMTkVLCJiIiImJwKNhERERGTU8EmIiIiYnIq2ERERERMTgWbiIiIiMmpYBMR\nERExOcNbU5nJ8n3LxzsE0zLzt1nfaGxmzul6gjFmI03k/G9m7sG8zsEc+1iZMfdAxHTod4d4cPaD\nhh7zLwV7FwQI8YLNfqt9vEMwrSPnjph2fW40NjPndD3BGLORJnL+NzP3YF7nYI59rMyYeyBi6vmk\nJ+B5eru8AT3+zaBboiIiIiImF5CCbePGjSQnJ5OSkkJ+fj59fX243W6ysrJwOBxkZmbS1NQEQEVF\nBQ6Hw/8TFhZGS0sLACdPniQlJYX4+Hiee+45//HPnj3LAw88gMPhIC0tjV//+teBSENERETEFAwv\n2LxeL9u3b6e5uZnW1lb6+/txuVyUlZWxYcMGPB4P69evp7S0FICCggI8Hg8ej4fy8nJmzZpFamoq\nAE8//TQ7duygra2NtrY2Dhw4AMAPf/hDHn/8cTweDy6Xi2eeecboNERERERMw/Bn2KKjo4mIiKCn\np4ewsDB6enqYPn06cXFxdHd3A9DV1YXVah0ydvfu3SxbtgyA8+fPc/HiRbKysgAoLCxk3759LFy4\nkGnTpo14LIDaV2v9/7an27Gn2w3MVEREROTG1NbWUltbO+r9DS/YYmJiWLVqFTabjcjISLKzs1mw\nYAEJCQnMnz+fkpISBgYGOH78+JCxlZWVVFdXA9DZ2cmMGTP8r1mtVjo7OwF46aWXuO+++/jpT3/K\n5cuXqampuWYszuVOo9MTERERGTOn04nT6fRvr1u3btj9Db8l2t7ezubNm/F6vXzwwQdcunSJiooK\nVqxYwZYtWzh37hybNm2iqKho0LjGxkaioqJISkoacY7vfe97PPnkk7z//vu8+eabPP7440anISIi\nImIahhdsJ06cYN68ecTGxhIeHs6SJUs4evQobrebnJwcAHJzc3G73YPGuVwu8vPz/dtWq5WOjg7/\ndkdHh/+K27Fjx/i7v/s7AObOncuf//xnPvzwQ6NTERERETEFwwu2xMREGhoa6O3txefzUVNTQ1JS\nEvHx8dTV1QFw+PBhEhIS/GMGBgaoqqoiLy/P/7tp06YRHR1NY2MjPp+P8vJyFi9e7J/j0KFDALzz\nzjv8+c9/5vbbbzc6FRERERFTMPwZtrS0NAoLC5kzZw6TJk0iIyOD4uJi5s6dy8qVK+nr6yMyMpJt\n27b5x9TX12Oz2bDb7YOOtXXrVpYvX05vby+LFi1i4cKFAPz4xz9mxYoVbNq0CYvFwq5du4xOQ0RE\nRMQ0LD6fzzfeQQSCxWLhiV89Md5hmJYZW55cpdZUE8dEzl+tqUYnmGMfKzPmrtZUgWOxWBiuJAvp\ngi1EUxMREZEQM1LdotZUIiIiIiangk1ERETE5FSwiYiIiJicCjYRERERk1PBJiIiImJyKthERERE\nTE4Fm4iIiIjJqWATERERMTnDW1OZyfJ9ywN6fDN+C/VnmT2+QDrz4RmAkM9/Iv+Nr2WirYcZ8jVD\nDEYJpVw+KxjyCmSMZsz/RjovhHTBZr/VHtDjHzl3JOBzjIXZ4wukI+eOAIE/B8bbRP4bX8tEWw8z\n5GuGGIwSSrl8VjDkFcgYzZi/t8v7ucfolqiIiIiIyQWkYNu4cSPJycmkpKSQn59PX18fbrebrKws\nHA4HmZmZNDU1AVBRUYHD4fD/hIWF0dLSAsDJkydJSUkhPj6e5557zn/8733ve/79v/zlL3PbbbcF\nIg0RERERUzC8YPN6vWzfvp3m5mZaW1vp7+/H5XJRVlbGhg0b8Hg8rF+/ntLSUgAKCgrweDx4PB7K\ny8uZNWsWqampADz99NPs2LGDtrY22traOHDgAAAvv/yyf8x3v/tdvv3tbxudhoiIiIhpGP4MW3R0\nNBEREfT09BAWFkZPTw/Tp08nLi6O7u5uALq6urBarUPG7t69m2XLlgFw/vx5Ll68SFZWFgCFhYXs\n27ePhQsXDhmzYcOGa8ZS+2qt/9/2dDv2dLsBGYqIiIiMTW1tLbW1taPe3/CCLSYmhlWrVmGz2YiM\njCQ7O5sFCxaQkJDA/PnzKSkpYWBggOPHjw8ZW1lZSXV1NQCdnZ3MmDHD/5rVaqWzs3PQ/u+99x5e\nr5dvfOMb14zFudxpXGIiIiIiBnE6nTidTv/2unXrht3f8Fui7e3tbN68Ga/XywcffMClS5eoqKhg\nxYoVbNmyhXPnzrFp0yaKiooGjWtsbCQqKoqkpKRRz+VyuVi6dCkWi8XoNERERERMw/CC7cSJE8yb\nN4/Y2FjCw8NZsmQJR48exe12k5OTA0Bubi5ut3vQOJfLRX5+vn/barXS0dHh3+7o6BhyG/W1117z\n30IVERERCVWGF2yJiYk0NDTQ29uLz+ejpqaGpKQk4uPjqaurA+Dw4cMkJCT4xwwMDFBVVUVeXp7/\nd9OmTSM6OprGxkZ8Ph/l5eU8+uij/tfPnDnDRx99xNy5c41OQURERMRUDH+GLS0tjcLCQubMmcOk\nSZPIyMiguLiYuXPnsnLlSvr6+oiMjGTbtm3+MfX19dhsNux2+6Bjbd26leXLl9Pb28uiRYsGfeBA\nV9dERERkorD4fD7feAcRCBaLhSd+9URA5zBju4vPMnt8gaTWVBPTRFsPM+RrhhiMEkq5fFYw5KXW\nVJ/WLcOVZCFdsIVoaiIiIhJiRqpb1JpKRERExORUsImIiIiYnAo2EREREZNTwSYiIiJicirYRERE\nRExOBZuIiIiIyalgExERETE5FWwiIiIiJmd4ayozefHQi1y4dOGmzWfGb1MeTxNxPYIt52CLdyzG\nkmsorVOw5hKscQ8nUDlNtK4BgWSmfEO6YLtw6QL2W+03bb4j547c1PnMbiKuR7DlHGzxjsVYcg2l\ndQrWXII17uEEKqdArlUo/h2GY6Z8dUtURERExOQCUrBt3LiR5ORkUlJSyM/Pp6+vD7fbTVZWFg6H\ng8zMTJqamgCoqKjA4XD4f8LCwmhpaQHg5MmTpKSkEB8fz3PPPTdojsrKSpKTk7nnnnsoKCgIRBoi\nIiIipmB4web1etm+fTvNzc20trbS39+Py+WirKyMDRs24PF4WL9+PaWlpQAUFBTg8XjweDyUl5cz\na9YsUlNTAXj66afZsWMHbW1ttLW1ceDAAQDa2tr40Y9+xLFjx/jNb37DK6+8YnQaIiIiIqZh+DNs\n0dHRRERE0NPTQ1hYGD09PUyfPp24uDi6u7sB6Orqwmq1Dhm7e/duli1bBsD58+e5ePEiWVlZABQW\nFrJv3z4WLlzI9u3b+c53vsPUqVMBuP32268ZyynXKbyTvQDY0+3Y0+0GZysiIiLy+XlPefGe8o56\nf8MLtpiYGFatWoXNZiMyMpLs7GwWLFhAQkIC8+fPp6SkhIGBAY4fPz5kbGVlJdXV1QB0dnYyY8YM\n/2tWq5XOzk7g0ytsFouF+fPn09/fz9q1a8nOzh5yvPS8dNM8LCgiIiJy1V9eSKrbVTfs/obfEm1v\nb2fz5s14vV4++OADLl26REVFBStWrGDLli2cO3eOTZs2UVRUNGhcY2MjUVFRJCUljTjHJ598wtmz\nZ6mrq2PPnj089dRT/qt3IiIiIqHG8ILtxIkTzJs3j9jYWMLDw1myZAlHjx7F7XaTk5MDQG5uLm63\ne9A4l8tFfn6+f9tqtdLR0eHf7ujo8F9xmzlzJg8//DBhYWHY7XYSEhI4e/as0amIiIiImILhBVti\nYiINDQ309vbi8/moqakhKSmJ+Ph46uo+vdx3+PBhEhIS/GMGBgaoqqoiLy/P/7tp06YRHR1NY2Mj\nPp+P8vJyFi9eDMCjjz5KbW0tAB9++CG//e1vmT17ttGpiIiIiJiC4c+wpaWlUVhYyJw5c5g0aRIZ\nGRkUFxczd+5cVq5cSV9fH5GRkWzbts0/pr6+HpvNht1uH3SsrVu3snz5cnp7e1m0aBELFy4EIDs7\nm//+7/9kH5imAAAG7ElEQVQmOTmZsLAwfvKTn3DbbbcZnYqIiIiIKVh8Pp9vvIMIBIvFQtnBMrWm\nGkcTcT2CLedgi3cs1JrqU8GaS7DGPRy1pjK/m5nvrpxdDFeShXTBFqKpyU1SW1uL0+kc7zAkSOn8\nkbHSOTSxjFS3qDWVyHVcfU5S5Ebo/JGx0jkkn6WCTURERMTkVLCJiIiImFxIP8MmIiIiEiyGK8kM\n/1oPswjROlREREQmIN0SFRERETE5FWwiIiIiJqeCTURERMTkgr5gO3DgAImJicTHx/Mv//Iv19zn\n2WefJT4+nrS0NDwez02OUMxupHOotraWqVOn4nA4cDgc/PCHPxyHKMWsioqK+NKXvkRKSsp199F7\nkFzPSOeP3n/EzxfErly54rvzzjt97777ru/jjz/2paWl+U6fPj1on/379/seeughn8/n8zU0NPju\nvffe8QhVTGo059Bbb73le/jhh8cpQjG7+vp6X3Nzs++ee+655ut6D5LhjHT+6P1HrgrqK2xut5u7\n7roLu91OREQEeXl5vPHGG4P2qa6u5oknngDg3nvvpauri9///vfjEa6Y0GjOIdCnjuX6HnjgAW67\n7bbrvq73IBnOSOcP6P1HPhXUBVtnZyczZ870b8+YMYPOzs4R9+no6LhpMYq5jeYcslgsHDt2jLS0\nNBYtWsTp06dvdpgSxPQeJGOh9x+5Kqi/h220X477l/870ZfqylWjORcyMjJ4//33iYqK4te//jWP\nPvoov/3tb29CdBIq9B4kN0rvP3JVUF9hs1qtvP/++/7t999/nxkzZgy7T0dHB1ar9abFKOY2mnNo\nypQpREVFAfDQQw/xySef8Mc//vGmxinBS+9BMhZ6/5GrgrpgmzNnDm1tbXi9Xj7++GNee+01Hnnk\nkUH7PPLII/z7v/87AA0NDdx666186UtfGo9wxYRGcw79/ve/918hcbvd+Hw+YmJixiNcCUJ6D5Kx\n0PuPXBXUt0TDw8P52c9+RnZ2Nv39/axYsYK7776bX/7ylwAUFxezaNEi3nzzTe666y7++q//mp07\nd45z1GImozmHXn/9dX7+858THh5OVFQULpdrnKMWM1m2bBl1dXV8+OGHzJw5k3Xr1vHJJ58Aeg+S\nkY10/uj9R64K2ebvIiIiIqEiqG+JioiIiEwEKthERERETE4Fm4iIiIjJqWATERERMTkVbCIiAVRb\nW8vDDz883mGISJBTwSYiIiJicirYRCSkXb58mW9961ukp6eTkpJCZWUlJ0+exOl0MmfOHBYuXMiF\nCxcAOHv2LA8++CDp6el85Stf4d133wXghRdeICUlhdTUVCorK4FPr5w5nU6WLl3K3XffzeOPP+6f\n88CBA9x999185Stf4Ve/+pX/93V1dTgcDhwOBxkZGVy6dOkmroSIBLOg/uJcEZGRHDhwAKvVyv79\n+wH405/+xEMPPUR1dTWxsbG89tprrF69mh07dlBQUMD3v/99Fi9ezMcff0x/fz979+7l7bffpqWl\nhT/84Q9kZmby1a9+FYBTp05x+vRppk2bxv3338+xY8fIyMjgH/7hH3jrrbe48847eeyxx/y9Q//1\nX/+VrVu3ct9999HT08Nf/dVfjdu6iEhw0RU2EQlpqampHDx4kBdffJEjR45w7tw5fvOb3/Dggw/i\ncDj4p3/6Jzo7O7l06RIffPABixcvBuCWW24hMjKSo0ePkp+fj8Vi4Ytf/CJf+9rXaGpqwmKxkJWV\nxfTp07FYLKSnp/Puu+9y5swZZs2axZ133gnA448/7m8tdP/99/P888/z05/+lI8++oiwsLBxWxcR\nCS66wiYiIS0+Ph6Px8P+/fv5wQ9+wNe//nWSk5M5duzYoP0uXrx43WP8ZUOYq1fMPnuFLCwsjCtX\nrvhfu9bYsrIy/vZv/5b9+/dz//3381//9V98+ctfvuHcRGTi0BU2EQlp58+fZ/LkyRQUFFBSUoLb\n7ebDDz+koaEBgE8++YTTp08zZcoUZsyYwRtvvAFAX18fvb29PPDAA7z22msMDAzwhz/8gfr6erKy\nsoYUcfBpIZeYmIjX6+V3v/sdAHv27PG/3t7eTnJyMqWlpWRmZvI///M/N2EFRCQU6AqbiIS01tZW\nXnjhBSZNmsQtt9zCz3/+c8LCwnj22Wfp7u7mypUrPP/88yQlJVFeXk5xcTFr1qwhIiKC119/nZyc\nHI4fP05aWhoWi4Uf//jHfPGLX+Sdd94ZcjUNPr3qtm3bNr71rW8RFRXFAw88wOXLlwF45ZVXeOut\nt5g0aRL33HMPDz300M1eDhEJUmr+LiIiImJyuiUqIiIiYnIq2ERERERMTgWbiIiIiMmpYBMREREx\nORVsIiIiIiangk1ERETE5P4fUPipv8DkdaQAAAAASUVORK5CYII=\n" 990 | } 991 | ], 992 | "prompt_number": 48 993 | }, 994 | { 995 | "cell_type": "markdown", 996 | "metadata": {}, 997 | "source": [ 998 | "There are more views to engine pools, one of them is the `load_balanced_view` which automatically helps utilize engines which normally would sit aroudn idly waiting for other engines to finish." 999 | ] 1000 | }, 1001 | { 1002 | "cell_type": "code", 1003 | "collapsed": false, 1004 | "input": [ 1005 | "lbview = cli.load_balanced_view()" 1006 | ], 1007 | "language": "python", 1008 | "metadata": {}, 1009 | "outputs": [], 1010 | "prompt_number": 49 1011 | }, 1012 | { 1013 | "cell_type": "code", 1014 | "collapsed": false, 1015 | "input": [ 1016 | "@lbview.parallel(block=True)\n", 1017 | "def lb_task(delay):\n", 1018 | " import os, time\n", 1019 | " \n", 1020 | " t0 = time.time()\n", 1021 | " pid = os.getpid()\n", 1022 | " time.sleep(delay)\n", 1023 | " t1 = time.time()\n", 1024 | " \n", 1025 | " return [pid, t0, t1]" 1026 | ], 1027 | "language": "python", 1028 | "metadata": {}, 1029 | "outputs": [], 1030 | "prompt_number": 50 1031 | }, 1032 | { 1033 | "cell_type": "code", 1034 | "collapsed": false, 1035 | "input": [ 1036 | "result = lb_task.map(delays)" 1037 | ], 1038 | "language": "python", 1039 | "metadata": {}, 1040 | "outputs": [], 1041 | "prompt_number": 51 1042 | }, 1043 | { 1044 | "cell_type": "code", 1045 | "collapsed": false, 1046 | "input": [ 1047 | "visualize_tasks(result)" 1048 | ], 1049 | "language": "python", 1050 | "metadata": {}, 1051 | "outputs": [ 1052 | { 1053 | "output_type": "display_data", 1054 | "png": "iVBORw0KGgoAAAANSUhEUgAAAmwAAADOCAYAAACD3l7pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9w1PWdx/HnksSaHAQlXrsQiIuaFBOTsCGJEWO7bWGC\n9BSDcGLiRQb0ckqrY8EFa8fjR+/gaFWkPdrCMOjlAmuiU8yNljt+mGSOkCyQxaSjXmPOFRNNp16F\nEpNGSfb+cHabkEB+8N3sd5fXYyYz+e5+P9/v+/35frK8+X6/+/1YfD6fDxERERExrQmhDkBERERE\nLk0Fm4iIiIjJqWATERERMTkVbCIiIiImFx3qAILFYrGEOgQRERGREbvU90AjtmCDSycucinr169n\n/fr1oQ5DwpjGkFwujaEry3AnmnRJVERERMTkVLCJiIiImJwKNpEhOByOUIcgYU5jSC6XxpD0Z4nU\nmQ4sFovuYRMREZGwMFzdEtFfOvBbd2gdHZ0doQ7DNKwTrWyZt+WK75dw7YfxiDtc+2Yo1olWgLDP\nYzhXSp4QnFzDbcxHwvEOtz43gj/nsbgiCraOzg5s19hCHYZpeM94AfVLuPbDeMQdrn0zFH8u4Z7H\ncK6UPCE4uYbbmI+E4x1ufW4Ef85jEZR72DZv3kxaWhrp6ekUFRXR09OD2+0mNzcXu91OTk4Ox48f\nB6C8vBy73R74iYqKoqmpCYCnn36apKQkJk2aNGD7v/zlL8nIyMBut3Pbbbfx1ltvBSMNEREREVMw\nvGDzer3s2rWLxsZGmpub6e3txeVysXbtWjZt2oTH42Hjxo04nU4AiouL8Xg8eDweysrKuOGGG8jI\nyABg0aJFuN3uQfsoLi6mqakJj8fDD3/4Q1avXm10GiIiIiKmYfgl0fj4eGJiYujq6iIqKoquri6m\nTZuG1Wrl7NmzAJw5c4bExMRBbffu3cuyZcsCy7m5uUPuo/8Zt87OTq677jqDsxARERExD8MLtilT\nprB69WqSkpKIjY2loKCA+fPnk5KSQn5+PmvWrKGvr49jx44NaltRUUFVVdWI9rNjxw6ee+45Pvvs\nM+rq6oZcx/+E6FPvnoI8sM22jTUtEREREcNUV1dTXV094vUNL9haW1vZtm0bXq+XyZMns3TpUsrL\ny9mzZw/bt2+nsLCQyspKVqxYwcGDBwPtGhoaiIuLIzU1dUT7efTRR3n00UfZt28fK1as4M033xy0\njr9g8+73XjE3NIqIiIj5ORyOAc/a27BhwyXXN/wethMnTjB37lwSEhKIjo5m8eLFHD16FLfbTWFh\nIQBLliwZdG+ay+WiqKho1Pu77777aGxsNCR2ERERETMyvGCbNWsW9fX1dHd34/P5OHz4MKmpqSQn\nJ1NTUwPAkSNHSElJCbTp6+ujsrJywP1rl/Lee+8Ffn/99dcDX1IQERERiUSGXxLNzMykpKSE7Oxs\nJkyYQFZWFqWlpeTl5bFq1Sp6enqIjY1l586dgTa1tbUkJSVhs9kGbMvpdLJv3z66u7uZMWMGDz/8\nMM888ww///nPOXToEDExMfz1X/81e/bsMToNEREREdMIyoNznU5n4LEdftnZ2TQ0NAy5vsPhGPKL\nA1u3bmXr1q2DXt+2bduo4rFOtF7Ww+oijf8J2Vd6v4RrP4xH3OHaN0Px5xLueQznSskTgpNruI35\nSDje4dbnRvDnPBaaS1REREQkxIarW4Iy04GIiIiIGEcFm4iIiIjJqWATERERMTkVbCIiIiImp4JN\nRERExORUsImIiIiYnAo2EREREZNTwSYiIiJicirYREREREwuKFNTmcXy/cuDun3rRCtb5m1h3aF1\ndHR2BHVfoeDPDxj3HIO570g/bpcSqmM6nn1+JR/foag/RiYU/eSfpkjHZbD+n1XypYgu2GzX2IK6\nff/cZx2dHUHfVyj0n9ttvHMM5r4j/bhdSqiO6Xj2+ZV8fIei/hiZUPSTf586LoNdKXOLjkZQLolu\n3ryZtLQ00tPTKSoqoqenB7fbTW5uLna7nZycHI4fPw5AeXk5drs98BMVFUVTUxMATz/9NElJSUya\nNGnA9p977jnS0tLIzMxk3rx5nD59OhhpiIiIiJiC4QWb1+tl165dNDY20tzcTG9vLy6Xi7Vr17Jp\n0yY8Hg8bN27E6XQCUFxcjMfjwePxUFZWxg033EBGRgYAixYtwu12D9pHVlYWJ0+e5K233mLJkiWB\nbYmIiIhEIsMLtvj4eGJiYujq6uL8+fN0dXUxbdo0rFYrZ8+eBeDMmTMkJiYOart3716WLVsWWM7N\nzcVqtQ5az+FwcPXVVwNw66230tbWZnQaIiIiIqZh+D1sU6ZMYfXq1SQlJREbG0tBQQHz588nJSWF\n/Px81qxZQ19fH8eOHRvUtqKigqqqqlHtb/fu3SxcuHDI96pfrA78bpttwzbbNqpti4iIiARDdXU1\n1dXVI17f8IKttbWVbdu24fV6mTx5MkuXLqW8vJw9e/awfft2CgsLqaysZMWKFRw8eDDQrqGhgbi4\nOFJTU0e8r3//93+nsbGR559/fsj3Hcsdl5uOiIiIiOEcDgcOhyOwvGHDhkuub/gl0RMnTjB37lwS\nEhKIjo5m8eLFHD16FLfbTWFhIQBLliwZdG+ay+WiqKhoxPs5dOgQ//zP/0xVVRUxMTGG5iAiIiJi\nJoYXbLNmzaK+vp7u7m58Ph+HDx8mNTWV5ORkampqADhy5AgpKSmBNn19fVRWVg64f+1SPB4P//AP\n/8B//Md/cN111xmdgoiIiIipGH5JNDMzk5KSErKzs5kwYQJZWVmUlpaSl5fHqlWr6OnpITY2lp07\ndwba1NbWkpSUhM1mG7Atp9PJvn376O7uZsaMGTz88MM888wzOJ1OPvvsM5YsWQLA9ddfz/79+41O\nRURERMQUgvLgXKfTOehRG9nZ2TQ0NAy5vsPhoK6ubtDrW7duZevWrYNe73/v26UE+8F7/qdUWyda\nI/Ihf/78/L+PZ47B3HekH7dLCdUxHc8+v5KP71DUHyMTin7y71PHZbD+n1XyJYvP5/OFOohgsFgs\nRGhqIiIiEmGGq1s0+buIiIiIyalgExERETE5FWwiIiIiJqeCTURERMTkVLCJiIiImJwKNhERERGT\nU8EmIiIiYnIq2ERERERMLigzHZjFukPr6OjsCHUYIWGdaGXLvC2B5bH2hX874dyXocqh/zEI1r4j\n4fiMVrByHo/jFSz+p8KHU8xGGu2YiNS/m3AcB+EYc6hEdMHW0dmB7RpbqMMIiQunOhlrX/i3E859\nGaoc+h+DYO07Eo7PaAUr5/E4XsHijz2cYjbSaMdEpP7dhOM4CMeYQ0WXREVERERMLigF2+bNm0lL\nSyM9PZ2ioiJ6enpwu93k5uZit9vJycnh+PHjAJSXl2O32wM/UVFRNDU1AfD000+TlJTEpEmTBmy/\ntraWrKwsYmJiePXVV4ORgoiIiIhpGF6web1edu3aRWNjI83NzfT29uJyuVi7di2bNm3C4/GwceNG\nnE4nAMXFxXg8HjweD2VlZdxwww1kZGQAsGjRItxu96B9XH/99bz00ksUFRUZHb6IiIiI6Rh+D1t8\nfDwxMTF0dXURFRVFV1cX06ZNw2q1cvbsWQDOnDlDYmLioLZ79+5l2bJlgeXc3Nwh93H99dcDMGGC\nruiKiIhI5DO8YJsyZQqrV68mKSmJ2NhYCgoKmD9/PikpKeTn57NmzRr6+vo4duzYoLYVFRVUVVUZ\nFssp1ym8V3sBsM22YZttM2zbIiIiImPlPeXFe8o74vUNL9haW1vZtm0bXq+XyZMns3TpUsrLy9mz\nZw/bt2+nsLCQyspKVqxYwcGDBwPtGhoaiIuLIzU11bBYZi+brW+eiIiIiOlceCKp5qWaS65v+DXF\nEydOMHfuXBISEoiOjmbx4sUcPXoUt9tNYWEhAEuWLBl0b5rL5RrTPWkWi8WQuEVERETMyvCCbdas\nWdTX19Pd3Y3P5+Pw4cOkpqaSnJxMTc2X1eORI0dISUkJtOnr66OysnLA/Wsj4fP58Pl8hsYvIiIi\nYjaGF2yZmZmUlJSQnZ1NRkYGPp+P0tJSfvWrX+F0Opk9ezY/+tGP2LlzZ6BNbW0tSUlJ2Gy2Adty\nOp3MmDGD7u5uZsyYwcaNGwE4fvw4M2bM4JVXXqG0tJT09HSj0xARERExjaDMdOB0OgOP7fDLzs6m\noaFhyPUdDgd1dXWDXt+6dStbt24d9HpOTg4ffvjhsHFYJ1oHPfH/SuGf7qP/8lj6wr+dcO7LUOXQ\n/xgEa9+RcHxGK1g5j8fxChZ/7OEUs5FGOyYi9e8mHMdBOMYcKhZfhF5TtFgsulwqIiIiYWG4ukUP\nMhMRERExORVsIiIiIiangk1ERETE5FSwiYiIiJicCjYRERERk1PBJiIiImJyKthERERETE4Fm4iI\niIjJBWWmA7NZd2gdHZ0dIY3BOtHKlnlbTBHLcPyxgjn6LhQu93j170OIzH4crzEdTn87Rgh1vqHe\n/1iNNW7/k/bDKdfLEa7H90IXfsZeCa6Igq2jswPbNbaQxuCfdsMMsQyn/xQh4RBvMFzu8bpwmpVI\n7MfxGtPh9LdjhFDnG+r9j9VY4/a3C6dcL0e4Ht8LXYlTWemSqIiIiIjJBaVg27x5M2lpaaSnp1NU\nVERPTw9ut5vc3Fzsdjs5OTkcP34cgPLycux2e+AnKiqKpqYmAJ5++mmSkpKYNGnSgO339PRw3333\nkZycTF5eHh988EEw0hARERExBcMLNq/Xy65du2hsbKS5uZne3l5cLhdr165l06ZNeDweNm7ciNPp\nBKC4uBiPx4PH46GsrIwbbriBjIwMABYtWoTb7R60j927d5OQkEBLSwtPPPEEa9euNToNEREREdMw\nvGCLj48nJiaGrq4uzp8/T1dXF9OmTcNqtXL27FkAzpw5Q2Ji4qC2e/fuZdmyZYHl3NxcrFbroPWq\nqqp48MEHAbj33ns5fPiw0WmIiIiImIbhXzqYMmUKq1evJikpidjYWAoKCpg/fz4pKSnk5+ezZs0a\n+vr6OHbs2KC2FRUVVFVVDbuP9vZ2ZsyY8WUC0dFMnjyZP/7xj0yZMmXAeuvXrwfg1LunIA9ss22X\nnZ+IiIjI5aqurqa6unrE6xtesLW2trJt2za8Xi+TJ09m6dKllJeXs2fPHrZv305hYSGVlZWsWLGC\ngwcPBto1NDQQFxdHamqqYbH4Czbvfm9YfxtGREREIovD4cDhcASWN2zYcMn1Db8keuLECebOnUtC\nQgLR0dEsXryYo0eP4na7KSwsBGDJkiWD7k1zuVwUFRWNaB+JiYmcPn0agPPnz3P27NlBZ9dERERE\nIoXhBdusWbOor6+nu7sbn8/H4cOHSU1NJTk5mZqaGgCOHDlCSkpKoE1fXx+VlZUD7l+7lLvvvpuX\nXnoJgFdeeYXvfOc7RqchIiIiYhqGXxLNzMykpKSE7OxsJkyYQFZWFqWlpeTl5bFq1Sp6enqIjY1l\n586dgTa1tbUkJSVhs9kGbMvpdLJv3z66u7uZMWMGDz/8MM888wwrV67k7/7u70hOTiYhIQGXy2V0\nGiIiIiKmccmC7cUXX2T79u28++67AKSmpvL9738/8A3Ni3E6nYHHdvhlZ2fT0NAw5PoOh4O6urpB\nr2/dupWtW7cOev0rX/kKFRUVl4yhP+tEa8ifiuyf/sQMsQzHH6v/d7PHGwyXe7z69+HlbMfMxmtM\nh9PfjhFCnW+o9z9WY43b3y6ccr0c4Xp8L3ThZ+yVwOLz+XxDvfHSSy+xbds2nnvuOex2Oz6fD4/H\nw5NPPsnjjz9OSUnJeMc6KhaLhYukJiIiImIqw9UtFy3Ybr31VlwuFzNnzhzwutfr5b777rvo2TKz\nUMEmIiIi4WK4uuWiXzo4d+7coGINwGazce7cOWOiExEREZFhXbRgu/rqqy/a6FLviYiIiIixLnpJ\nNDY2lptuumnIRq2trXR1dQU1sMulS6IiIiISLoarWy76LdF33nknKAGJiIiIyOhc9AxbuNMZNhER\nEQkXYz7DNnHiRCwWy0U3+qc//enyoxMRERGRYekMm4iIiEiIjfkMW3d3N7/85S9pbW0lPT2dlStX\nEh1t+ExWQbfu0Do6OjtCHUZQWSda2TJvyxWR61ipj4amfvmL8egL/9PZ1dfjM+5G09+R+rcQzDEX\njuPZf5zD0UUrsAcffJCrrrqK/Px83njjDd5++21eeOGF8YzNEB2dHdiusYU6jKDyTy9yJeQ6Vuqj\noalf/mI8+sK/D/W1Fwj+uBtNf0fq30Iwx1w4judwno7rkt8SbW5uBuChhx4iJydn3IISERERkb+4\n6INz+1/+HO2l0M2bN5OWlkZ6ejpFRUX09PTgdrvJzc3FbreTk5PD8ePHASgvL8dutwd+oqKiaGpq\nAuDkyZOkp6eTnJzM448/Htj+Bx98wHe+8x0yMzP51re+RXt7+6jiExEREQknFy3YmpqamDRpUuCn\nubk58Ht8fPxFN+j1etm1axeNjY00NzfT29uLy+Vi7dq1bNq0CY/Hw8aNG3E6nQAUFxfj8XjweDyU\nlZUxc+ZMMjIyAHjkkUfYvXs3LS0ttLS0cODAAQDWrFnD8uXLeeutt3jmmWd46qmnjOwTEREREVO5\naMHW29vLuXPnAj/nz58P/H6pR3rEx8cTExNDV1cX58+fp6uri2nTpmG1Wjl79iwAZ86cITExcVDb\nvXv3cv/99wPw8ccfc+7cOXJzcwEoKSlh//79wJeXa7/97W8D4HA4eO2118aYvoiIiIj5Gf61zylT\nprB69WqSkpKIjY2loKCA+fPnk5KSQn5+PmvWrKGvr49jx44NaltRUUFVVRUA7e3tTJ8+PfBeYmJi\n4NJnZmYmr776Ko899hi//vWvOXfuHJ9++inXXnvtgO2tX7+eU++ewnu1F9tsG7bZNqPTFRERERm1\n6upqqqurR7y+4QVba2sr27Ztw+v1MnnyZJYuXUp5eTl79uxh+/btFBYWUllZyYoVKzh48GCgXUND\nA3FxcaSmpg67j5/+9Kd873vf48UXX+Qb3/gGiYmJREVFDVpv/fr1ePd7w+obLCIiIhL5HA4HDocj\nsLxhw4ZLrm94wXbixAnmzp1LQkICAIsXL+bo0aO43W4OHToEwJIlS3jooYcGtHO5XBQVFQWWExMT\naWtrCyy3tbUFLqNOnTqVV199FYDOzk5effXVS95XJyIiIhLOLnoP21jNmjWL+vp6uru78fl8HD58\nmNTUVJKTk6mpqQHgyJEjpKSkBNr09fVRWVnJsmXLAq9NnTqV+Ph4Ghoa8Pl8lJWVcc899wDwf//3\nf/T19QFffiN15cqVRqchIiIiYhqGn2HLzMykpKSE7OxsJkyYQFZWFqWlpeTl5bFq1Sp6enqIjY1l\n586dgTa1tbUkJSVhs9kGbGvHjh0sX76c7u5uFi5cyIIFC4Avr/s+9dRTWCwWvvnNb/Kv//qvRqch\nIiIiYhpBmWvK6XQGHtvhl52dTUNDw5DrOxwO6urqBr0+Z86cwMN7+7v33nu59957RxSLdaI1rJ9s\nPBL+6UGuhFzHSn00NPXLX4xHX/j3ob4en3E3mv6O1L+FYI65cBzP/pjDkSZ/FxEREQmx4eoWw+9h\nExERERFjqWATERERMTkVbCIiIiImp4JNRERExORUsImIiIiYnAo2EREREZNTwSYiIiJicirYRERE\nREwuKDMdmM26Q+vo6OwIdRjjxjrRypZ5WwLLZs7fH6tZY/Q/FduMsY1F/7Fhpj43ahxEwvEy+98E\nhH8/m7GPjYjJjHmNxGjG04X/vl1JroiCraOzA9s1tlCHMW4unCbEzPn7YzVrjP74zBjbWPQfG2bq\nc6PGQSQcL7P/TUD497MZ+9iImMyY10iMZjyF0zRYRtMlURERERGTC0rBtnnzZtLS0khPT6eoqIie\nnh7cbje5ubnY7XZycnI4fvw4AOXl5djt9sBPVFQUTU1NAJw8eZL09HSSk5N5/PHHA9t/7733uOOO\nO7Db7WRmZvKb3/wmGGmIiIiImILhBZvX62XXrl00NjbS3NxMb28vLpeLtWvXsmnTJjweDxs3bsTp\ndAJQXFyMx+PB4/FQVlbGzJkzycjIAOCRRx5h9+7dtLS00NLSwoEDBwD48Y9/zAMPPIDH48HlcvHo\no48anYaIiIiIaRh+D1t8fDwxMTF0dXURFRVFV1cX06ZNw2q1cvbsWQDOnDlDYmLioLZ79+7l/vvv\nB+Djjz/m3Llz5ObmAlBSUsL+/ftZsGABU6dOHXZbAOvXrwfg1LunIA9ss20GZysiIiIyetXV1VRX\nV494fcMLtilTprB69WqSkpKIjY2loKCA+fPnk5KSQn5+PmvWrKGvr49jx44NaltRUUFVVRUA7e3t\nTJ8+PfBeYmIi7e3tADz11FPcdttt/OxnP+Ozzz7j8OHDQ8biL9i8+71hdQOmiIiIRDaHw4HD4Qgs\nb9iw4ZLrG35JtLW1lW3btuH1evnoo4/o7OykvLyclStXsn37dk6fPs3zzz/PihUrBrRraGggLi6O\n1NTUYffxgx/8gIceeogPP/yQN954gwceeMDoNERERERMw/CC7cSJE8ydO5eEhASio6NZvHgxR48e\nxe12U1hYCMCSJUtwu90D2rlcLoqKigLLiYmJtLW1BZbb2toCZ9zq6ur427/9WwDy8vL485//zCef\nfGJ0KiIiIiKmYHjBNmvWLOrr6+nu7sbn83H48GFSU1NJTk6mpqYGgCNHjpCSkhJo09fXR2VlJcuW\nLQu8NnXqVOLj42loaMDn81FWVsaiRYsC+zh06BAA77zzDn/+85+57rrrjE5FRERExBQMv4ctMzOT\nkpISsrOzmTBhAllZWZSWlpKXl8eqVavo6ekhNjaWnTt3BtrU1taSlJSEzWYbsK0dO3awfPlyuru7\nWbhwIQsWLADgJz/5CStXruT555/HYrHw0ksvGZ2GiIiIiGkEZaYDp9MZeGyHX3Z2Ng0NDUOu73A4\nqKurG/T6nDlzaG5uHvT6jTfeOKpvVlgnWq+opyP7p/nov2zW/P2xmjVGf3xmjG0s+o8NM/W5UeMg\nEo6X2f8mIPz72Yx9bERMZsxrJEYzni789+1KYvH5fL5QBxEMFouFCE1NREREIsxwdYumphIREREx\nORVsIiIiIiangk1ERETE5FSwiYiIiJicCjYRERERk1PBJiIiImJyKthERERETE4Fm4iIiIjJBWWm\nA7NYvn95qEO4KOtEK1vmbWHdoXV0dHYEdR9AUPcTLP37CAh6fw2133Drs7EKh5zDIUajBSPnSOzH\nUOfkf/r+aPYd6piHM5acwk3/fyPDQUQXbLZrbKEO4aL8U3B0dHYELc7+03wEcz/B0r+P/MYjj/E4\nNmYTDjmHQ4xGC0bOkdiPoc7Jv//R7DvUMQ9nLDmFm3Cavgt0SVRERETE9IJSsG3evJm0tDTS09Mp\nKiqip6cHt9tNbm4udrudnJwcjh8/DkB5eTl2uz3wExUVRVNTEwAnT54kPT2d5ORkHn/88cD2f/CD\nHwTW//rXv861114bjDRERERETMHwgs3r9bJr1y4aGxtpbm6mt7cXl8vF2rVr2bRpEx6Ph40bN+J0\nOgEoLi7G4/Hg8XgoKytj5syZZGRkAPDII4+we/duWlpaaGlp4cCBAwA899xzgTbf//73uffee41O\nQ0RERMQ0DL+HLT4+npiYGLq6uoiKiqKrq4tp06ZhtVo5e/YsAGfOnCExMXFQ271793L//fcD8PHH\nH3Pu3Dlyc3MBKCkpYf/+/SxYsGBQm02bNg0ZS/WL1YHfbbNt2GbbDMhQRERE5PJUV1dTXV094vUN\nL9imTJnC6tWrSUpKIjY2loKCAubPn09KSgr5+fmsWbOGvr4+jh07NqhtRUUFVVVVALS3tzN9+vTA\ne4mJibS3tw9Y/4MPPsDr9fLtb397yFgcyx3GJSYiIiJiEIfDgcPhCCxv2LDhkusbfkm0tbWVbdu2\n4fV6+eijj+js7KS8vJyVK1eyfft2Tp8+zfPPP8+KFSsGtGtoaCAuLo7U1NQR78vlcrF06VIsFovR\naYiIiIiYhuEF24kTJ5g7dy4JCQlER0ezePFijh49itvtprCwEIAlS5bgdrsHtHO5XBQVFQWWExMT\naWtrCyy3tbUNuoz68ssvBy6hioiIiEQqwwu2WbNmUV9fT3d3Nz6fj8OHD5OamkpycjI1NTUAHDly\nhJSUlECbvr4+KisrWbZsWeC1qVOnEh8fT0NDAz6fj7KyMu65557A+++++y6ffvopeXl5RqcgIiIi\nYiqG38OWmZlJSUkJ2dnZTJgwgaysLEpLS8nLy2PVqlX09PQQGxvLzp07A21qa2tJSkrCZrMN2NaO\nHTtYvnw53d3dLFy4cMAXDnR2TURERK4UQZnpwOl0Bh7b4ZednU1DQ8OQ6zscDurq6ga9PmfOHJqb\nm4ds84//+I/DxmHmpxj7p/2wTrQGLU7/PoK9n2Dp30f9Xwt2HuNxbMwmHHIOhxiNFoycI7EfQ52T\nf/+j2XeoYx7OWHIKN/3/bQkHFp/P5wt1EMFgsViI0NREREQkwgxXt2hqKhERERGTU8EmIiIiYnIq\n2ERERERMTgWbiIiIiMmpYBMRERExORVsIiIiIiangk1ERETE5FSwiYiIiJhcUGY6MJN1h9YFfu/o\n7AhhJMaxTrSyZd4W1h1aZ7qczBzbePL3g1+49ofRxzMSx8dIcvI/UT1Scr5cYx0Hoe7HUO//coQy\n9tEe7/6xXvhZeiWL+IKt/+CwXWMLXSAG8k8V0tHZYbqczBzbeLpwOpdw7Q+jj2ckjo+R5ORfJ1Jy\nvlxjHQeh7sdQ7/9yhDL20R7v/rFG8tRYo6VLoiIiIiImF5SCbfPmzaSlpZGenk5RURE9PT243W5y\nc3Ox2+3k5ORw/PhxAMrLy7Hb7YGfqKgompqaADh58iTp6ekkJyfz+OOPD9hHRUUFaWlp3HLLLRQX\nFwcjDRERERFTMLxg83q97Nq1i8bGRpqbm+nt7cXlcrF27Vo2bdqEx+Nh48aNOJ1OAIqLi/F4PHg8\nHsrKypgQgHzVAAAJsUlEQVQ5cyYZGRkAPPLII+zevZuWlhZaWlo4cOAAAC0tLWzZsoW6ujp++9vf\n8sILLxidhoiIiIhpGH4PW3x8PDExMXR1dREVFUVXVxfTpk3DarVy9uxZAM6cOUNiYuKgtnv37uX+\n++8H4OOPP+bcuXPk5uYCUFJSwv79+1mwYAG7du3ie9/7HpMnTwbguuuuGzKW9evXc+rdUwBYb7Fi\ny7cZna6IiIjIqFVXV1NdXT3i9Q0v2KZMmcLq1atJSkoiNjaWgoIC5s+fT0pKCvn5+axZs4a+vj6O\nHTs2qG1FRQVVVVUAtLe3M3369MB7iYmJtLe3A1+eYbNYLOTn59Pb28v69espKCgYtL3169fj3e81\nOkURERGRy+JwOHA4HIHlDRs2XHJ9wy+Jtra2sm3bNrxeLx999BGdnZ2Ul5ezcuVKtm/fzunTp3n+\n+edZsWLFgHYNDQ3ExcWRmpo67D6++OIL3nvvPWpqati3bx8PP/xw4OydiIiISKQxvGA7ceIEc+fO\nJSEhgejoaBYvXszRo0dxu90UFhYCsGTJEtxu94B2LpeLoqKiwHJiYiJtbW2B5ba2tsAZtxkzZnDX\nXXcRFRWFzWYjJSWF9957z+hUREREREzB8IJt1qxZ1NfX093djc/n4/Dhw6SmppKcnExNTQ0AR44c\nISUlJdCmr6+PyspKli1bFnht6tSpxMfH09DQgM/no6ysjEWLFgFwzz33BK77fvLJJ/zud7/jhhtu\nMDoVEREREVMw/B62zMxMSkpKyM7OZsKECWRlZVFaWkpeXh6rVq2ip6eH2NhYdu7cGWhTW1tLUlIS\nNpttwLZ27NjB8uXL6e7uZuHChSxYsACAgoIC/uu//ou0tDSioqL46U9/yrXXXmt0KiIiIiKmEJSZ\nDpxOZ+CxHX7Z2dk0NDQMub7D4aCurm7Q63PmzKG5uXnINs8++yzPPvvssLH4p7iAwU+fD1f+nKwT\nrabLycyxjaf+486/HI79YfTxjMTxMZKc/OtESs6Xa6zjINT9GOr9X45Qxj7a490/1gs/S69kFp/P\n5wt1EMFgsViI0NRkHFRXVw/49o7IaGkMyeXSGLqyDFe3aGoqkSGM5tk4IkPRGJLLpTEk/algExER\nETE5FWwiIiIiJhfR97CJiIiIhItLlWRB+ZaoGURoHSoiIiJXIF0SFRERETE5FWwiIiIiJqeCTURE\nRMTkwr5gO3DgALNmzSI5OZl/+Zd/GXKdxx57jOTkZDIzM/F4POMcoZjdcGOourqayZMnY7fbsdvt\n/PjHPw5BlGJGK1as4Gtf+xrp6ekXXUefP3Ipw40hff5IgC+MnT9/3nfjjTf63n//fd/nn3/uy8zM\n9L399tsD1nn99dd9d955p8/n8/nq6+t9t956ayhCFZMayRh68803fXfddVeIIhQzq62t9TU2Nvpu\nueWWId/X548MZ7gxpM8f8QvrM2xut5ubbroJm81GTEwMy5Yt47XXXhuwTlVVFQ8++CAAt956K2fO\nnOH3v/99KMIVExrJGAJ961iGdscdd3Dttdde9H19/shwhhtDoM8f+VJYF2zt7e3MmDEjsDx9+nTa\n29uHXaetrW3cYhRzG8kYslgs1NXVkZmZycKFC3n77bfHO0wJU/r8kculzx/xC+vnsI304bgX/u9E\nD9UVv5GMhaysLD788EPi4uL4zW9+wz333MPvfve7cYhOIoE+f+Ry6PNH/ML6DFtiYiIffvhhYPnD\nDz9k+vTpl1ynra2NxMTEcYtRzG0kY2jSpEnExcUBcOedd/LFF1/wxz/+cVzjlPCkzx+5XPr8Eb+w\nLtiys7NpaWnB6/Xy+eef8/LLL3P33XcPWOfuu+/m3/7t3wCor6/nmmuu4Wtf+1oowhUTGskY+v3v\nfx84S+J2u/H5fEyZMiUU4UqY0eePXC59/ohfWF8SjY6O5uc//zkFBQX09vaycuVKbr75Zn71q18B\nUFpaysKFC3njjTe46aab+Ku/+iv27NkT4qjFTEYyhl555RV+8YtfEB0dTVxcHC6XK8RRi1ncf//9\n1NTU8MknnzBjxgw2bNjAF198AejzR0ZmuDGkzx/xi9jJ30VEREQiRVhfEhURERG5EqhgExERETE5\nFWwiIiIiJqeCTURERMTkVLCJiARRdXU1d911V6jDEJEwp4JNRERExORUsIlIRPvss8/47ne/y+zZ\ns0lPT6eiooKTJ0/icDjIzs5mwYIFdHR0APDee+8xb948Zs+ezZw5c3j//fcBePLJJ0lPTycjI4OK\nigrgyzNnDoeDpUuXcvPNN/PAAw8E9nngwAFuvvlm5syZw69//evA6zU1Ndjtdux2O1lZWXR2do5j\nT4hIOAvrB+eKiAznwIEDJCYm8vrrrwPwpz/9iTvvvJOqqioSEhJ4+eWXefrpp9m9ezfFxcX88Ic/\nZNGiRXz++ef09vby6quv8tZbb9HU1MQf/vAHcnJy+MY3vgHAqVOnePvtt5k6dSq33347dXV1ZGVl\n8fd///e8+eab3Hjjjdx3332B+UOfffZZduzYwW233UZXVxdf+cpXQtYvIhJedIZNRCJaRkYGBw8e\nZN26dfz3f/83p0+f5re//S3z5s3DbrfzT//0T7S3t9PZ2clHH33EokWLALjqqquIjY3l6NGjFBUV\nYbFY+OpXv8o3v/lNjh8/jsViITc3l2nTpmGxWJg9ezbvv/8+7777LjNnzuTGG28E4IEHHghMLXT7\n7bfzxBNP8LOf/YxPP/2UqKiokPWLiIQXnWETkYiWnJyMx+Ph9ddf50c/+hHf+ta3SEtLo66ubsB6\n586du+g2LpwQxn/GrP8ZsqioKM6fPx94b6i2a9eu5W/+5m94/fXXuf322/nP//xPvv71r485NxG5\ncugMm4hEtI8//pirr76a4uJi1qxZg9vt5pNPPqG+vh6AL774grfffptJkyYxffp0XnvtNQB6enro\n7u7mjjvu4OWXX6avr48//OEP1NbWkpubO6iIgy8LuVmzZuH1evnf//1fAPbt2xd4v7W1lbS0NJxO\nJzk5OfzP//zPOPSAiEQCnWETkYjW3NzMk08+yYQJE7jqqqv4xS9+QVRUFI899hhnz57l/PnzPPHE\nE6SmplJWVkZpaSnPPPMMMTExvPLKKxQWFnLs2DEyMzOxWCz85Cc/4atf/SrvvPPOoLNp8OVZt507\nd/Ld736XuLg47rjjDj777DMAXnjhBd58800mTJjALbfcwp133jne3SEiYUqTv4uIiIiYnC6JioiI\niJicCjYRERERk1PBJiIiImJyKthERERETE4Fm4iIiIjJqWATERERMbn/BzTleJbh/gVBAAAAAElF\nTkSuQmCC\n" 1055 | } 1056 | ], 1057 | "prompt_number": 52 1058 | }, 1059 | { 1060 | "cell_type": "markdown", 1061 | "metadata": {}, 1062 | "source": [ 1063 | "(Load balancing visualization example from http://nbviewer.ipython.org/urls/raw.github.com/jrjohansson/scientific-python-lectures/master/Lecture-6B-HPC.ipynb )" 1064 | ] 1065 | }, 1066 | { 1067 | "cell_type": "markdown", 1068 | "metadata": {}, 1069 | "source": [ 1070 | "To make sure that engines have the same modules loaded, one can use the method `.sync_imports()` of an engine pool view." 1071 | ] 1072 | }, 1073 | { 1074 | "cell_type": "code", 1075 | "collapsed": false, 1076 | "input": [ 1077 | "with dview.sync_imports():\n", 1078 | " import numpy\n", 1079 | " import os" 1080 | ], 1081 | "language": "python", 1082 | "metadata": {}, 1083 | "outputs": [ 1084 | { 1085 | "output_type": "stream", 1086 | "stream": "stdout", 1087 | "text": [ 1088 | "importing numpy on engine(s)\n", 1089 | "importing os on engine(s)\n" 1090 | ] 1091 | } 1092 | ], 1093 | "prompt_number": 53 1094 | }, 1095 | { 1096 | "cell_type": "code", 1097 | "collapsed": false, 1098 | "input": [ 1099 | "%%px\n", 1100 | "os.getpid()" 1101 | ], 1102 | "language": "python", 1103 | "metadata": {}, 1104 | "outputs": [ 1105 | { 1106 | "output_type": "display_data", 1107 | "text": [ 1108 | "\u001b[0;31mOut[0:1]: \u001b[0m88707" 1109 | ] 1110 | }, 1111 | { 1112 | "output_type": "display_data", 1113 | "text": [ 1114 | "\u001b[0;31mOut[1:1]: \u001b[0m88708" 1115 | ] 1116 | }, 1117 | { 1118 | "output_type": "display_data", 1119 | "text": [ 1120 | "\u001b[0;31mOut[2:1]: \u001b[0m88706" 1121 | ] 1122 | }, 1123 | { 1124 | "output_type": "display_data", 1125 | "text": [ 1126 | "\u001b[0;31mOut[3:1]: \u001b[0m88709" 1127 | ] 1128 | }, 1129 | { 1130 | "output_type": "display_data", 1131 | "text": [ 1132 | "\u001b[0;31mOut[4:1]: \u001b[0m88710" 1133 | ] 1134 | }, 1135 | { 1136 | "output_type": "display_data", 1137 | "text": [ 1138 | "\u001b[0;31mOut[5:1]: \u001b[0m88711" 1139 | ] 1140 | }, 1141 | { 1142 | "output_type": "display_data", 1143 | "text": [ 1144 | "\u001b[0;31mOut[6:1]: \u001b[0m88712" 1145 | ] 1146 | }, 1147 | { 1148 | "output_type": "display_data", 1149 | "text": [ 1150 | "\u001b[0;31mOut[7:1]: \u001b[0m88713" 1151 | ] 1152 | } 1153 | ], 1154 | "prompt_number": 54 1155 | }, 1156 | { 1157 | "cell_type": "markdown", 1158 | "metadata": {}, 1159 | "source": [ 1160 | "Since the engines will not have the same namespace as any interpreter that might want to use them, if we want them to have the same data to work on one need to send that data to the engines.\n", 1161 | "\n", 1162 | "This is done with the `.push()` method of an engine pool view." 1163 | ] 1164 | }, 1165 | { 1166 | "cell_type": "code", 1167 | "collapsed": false, 1168 | "input": [ 1169 | "dview.push(dict(a=np.arange(5), b=np.zeros(5)), block=True)" 1170 | ], 1171 | "language": "python", 1172 | "metadata": {}, 1173 | "outputs": [ 1174 | { 1175 | "output_type": "pyout", 1176 | "prompt_number": 55, 1177 | "text": [ 1178 | "[None, None, None, None, None, None, None, None]" 1179 | ] 1180 | } 1181 | ], 1182 | "prompt_number": 55 1183 | }, 1184 | { 1185 | "cell_type": "code", 1186 | "collapsed": false, 1187 | "input": [ 1188 | "dview.pull(\"a\", block=True)" 1189 | ], 1190 | "language": "python", 1191 | "metadata": {}, 1192 | "outputs": [ 1193 | { 1194 | "output_type": "pyout", 1195 | "prompt_number": 56, 1196 | "text": [ 1197 | "[array([0, 1, 2, 3, 4]),\n", 1198 | " array([0, 1, 2, 3, 4]),\n", 1199 | " array([0, 1, 2, 3, 4]),\n", 1200 | " array([0, 1, 2, 3, 4]),\n", 1201 | " array([0, 1, 2, 3, 4]),\n", 1202 | " array([0, 1, 2, 3, 4]),\n", 1203 | " array([0, 1, 2, 3, 4]),\n", 1204 | " array([0, 1, 2, 3, 4])]" 1205 | ] 1206 | } 1207 | ], 1208 | "prompt_number": 56 1209 | }, 1210 | { 1211 | "cell_type": "markdown", 1212 | "metadata": {}, 1213 | "source": [ 1214 | "For the sake of convenience, pushing and pulling to the namespaces on the engines can be done through a dictionary like syntax" 1215 | ] 1216 | }, 1217 | { 1218 | "cell_type": "code", 1219 | "collapsed": false, 1220 | "input": [ 1221 | "dview[\"a\"]" 1222 | ], 1223 | "language": "python", 1224 | "metadata": {}, 1225 | "outputs": [ 1226 | { 1227 | "output_type": "pyout", 1228 | "prompt_number": 57, 1229 | "text": [ 1230 | "[array([0, 1, 2, 3, 4]),\n", 1231 | " array([0, 1, 2, 3, 4]),\n", 1232 | " array([0, 1, 2, 3, 4]),\n", 1233 | " array([0, 1, 2, 3, 4]),\n", 1234 | " array([0, 1, 2, 3, 4]),\n", 1235 | " array([0, 1, 2, 3, 4]),\n", 1236 | " array([0, 1, 2, 3, 4]),\n", 1237 | " array([0, 1, 2, 3, 4])]" 1238 | ] 1239 | } 1240 | ], 1241 | "prompt_number": 57 1242 | }, 1243 | { 1244 | "cell_type": "code", 1245 | "collapsed": false, 1246 | "input": [ 1247 | "dview[\"c\"] = \"See\"" 1248 | ], 1249 | "language": "python", 1250 | "metadata": {}, 1251 | "outputs": [], 1252 | "prompt_number": 58 1253 | }, 1254 | { 1255 | "cell_type": "code", 1256 | "collapsed": false, 1257 | "input": [ 1258 | "dview.pull(\"c\", block=True)" 1259 | ], 1260 | "language": "python", 1261 | "metadata": {}, 1262 | "outputs": [ 1263 | { 1264 | "output_type": "pyout", 1265 | "prompt_number": 59, 1266 | "text": [ 1267 | "['See', 'See', 'See', 'See', 'See', 'See', 'See', 'See']" 1268 | ] 1269 | } 1270 | ], 1271 | "prompt_number": 59 1272 | }, 1273 | { 1274 | "cell_type": "markdown", 1275 | "metadata": {}, 1276 | "source": [ 1277 | "Something which might be more useful is to push _different_ sections of data to various engines, just like when we applied the parallel function above.\n", 1278 | "\n", 1279 | "The direct view has the methods `.scatter()` and `.gather()`." 1280 | ] 1281 | }, 1282 | { 1283 | "cell_type": "code", 1284 | "collapsed": false, 1285 | "input": [ 1286 | "dview.scatter('x', np.arange(95), block=True)" 1287 | ], 1288 | "language": "python", 1289 | "metadata": {}, 1290 | "outputs": [], 1291 | "prompt_number": 60 1292 | }, 1293 | { 1294 | "cell_type": "markdown", 1295 | "metadata": {}, 1296 | "source": [ 1297 | "This will divide the array to four sections, and divide those out to different engines." 1298 | ] 1299 | }, 1300 | { 1301 | "cell_type": "code", 1302 | "collapsed": false, 1303 | "input": [ 1304 | "%%px\n", 1305 | "x" 1306 | ], 1307 | "language": "python", 1308 | "metadata": {}, 1309 | "outputs": [ 1310 | { 1311 | "output_type": "display_data", 1312 | "text": [ 1313 | "\u001b[0;31mOut[0:2]: \u001b[0marray([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])" 1314 | ] 1315 | }, 1316 | { 1317 | "output_type": "display_data", 1318 | "text": [ 1319 | "\u001b[0;31mOut[1:2]: \u001b[0marray([12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23])" 1320 | ] 1321 | }, 1322 | { 1323 | "output_type": "display_data", 1324 | "text": [ 1325 | "\u001b[0;31mOut[2:2]: \u001b[0marray([24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35])" 1326 | ] 1327 | }, 1328 | { 1329 | "output_type": "display_data", 1330 | "text": [ 1331 | "\u001b[0;31mOut[3:2]: \u001b[0marray([36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47])" 1332 | ] 1333 | }, 1334 | { 1335 | "output_type": "display_data", 1336 | "text": [ 1337 | "\u001b[0;31mOut[4:2]: \u001b[0marray([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59])" 1338 | ] 1339 | }, 1340 | { 1341 | "output_type": "display_data", 1342 | "text": [ 1343 | "\u001b[0;31mOut[5:2]: \u001b[0marray([60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71])" 1344 | ] 1345 | }, 1346 | { 1347 | "output_type": "display_data", 1348 | "text": [ 1349 | "\u001b[0;31mOut[6:2]: \u001b[0marray([72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83])" 1350 | ] 1351 | }, 1352 | { 1353 | "output_type": "display_data", 1354 | "text": [ 1355 | "\u001b[0;31mOut[7:2]: \u001b[0marray([84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94])" 1356 | ] 1357 | } 1358 | ], 1359 | "prompt_number": 61 1360 | }, 1361 | { 1362 | "cell_type": "code", 1363 | "collapsed": false, 1364 | "input": [ 1365 | "def x_mean():\n", 1366 | " return x.mean()" 1367 | ], 1368 | "language": "python", 1369 | "metadata": {}, 1370 | "outputs": [], 1371 | "prompt_number": 62 1372 | }, 1373 | { 1374 | "cell_type": "code", 1375 | "collapsed": false, 1376 | "input": [ 1377 | "dview.apply_sync(x_mean)" 1378 | ], 1379 | "language": "python", 1380 | "metadata": {}, 1381 | "outputs": [ 1382 | { 1383 | "output_type": "pyout", 1384 | "prompt_number": 63, 1385 | "text": [ 1386 | "[5.5, 17.5, 29.5, 41.5, 53.5, 65.5, 77.5, 89.0]" 1387 | ] 1388 | } 1389 | ], 1390 | "prompt_number": 63 1391 | }, 1392 | { 1393 | "cell_type": "code", 1394 | "collapsed": false, 1395 | "input": [ 1396 | "dview.gather(\"x\", targets=1, block=True)" 1397 | ], 1398 | "language": "python", 1399 | "metadata": {}, 1400 | "outputs": [ 1401 | { 1402 | "output_type": "pyout", 1403 | "prompt_number": 64, 1404 | "text": [ 1405 | "array([12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23])" 1406 | ] 1407 | } 1408 | ], 1409 | "prompt_number": 64 1410 | }, 1411 | { 1412 | "cell_type": "markdown", 1413 | "metadata": {}, 1414 | "source": [ 1415 | "An alternative to importing modules in the definition of a function, there is a `@require` decoratir in `IPython.parallel` which will cause the required module to be imported before the code in the function is executed." 1416 | ] 1417 | }, 1418 | { 1419 | "cell_type": "code", 1420 | "collapsed": false, 1421 | "input": [ 1422 | "from IPython.parallel import require" 1423 | ], 1424 | "language": "python", 1425 | "metadata": {}, 1426 | "outputs": [], 1427 | "prompt_number": 65 1428 | }, 1429 | { 1430 | "cell_type": "code", 1431 | "collapsed": false, 1432 | "input": [ 1433 | "@require(\"os\")\n", 1434 | "def get_pid():\n", 1435 | " return os.getpid()" 1436 | ], 1437 | "language": "python", 1438 | "metadata": {}, 1439 | "outputs": [], 1440 | "prompt_number": 66 1441 | }, 1442 | { 1443 | "cell_type": "code", 1444 | "collapsed": false, 1445 | "input": [ 1446 | "dview.apply_sync(get_pid)" 1447 | ], 1448 | "language": "python", 1449 | "metadata": {}, 1450 | "outputs": [ 1451 | { 1452 | "output_type": "pyout", 1453 | "prompt_number": 67, 1454 | "text": [ 1455 | "[88707, 88708, 88706, 88709, 88710, 88711, 88712, 88713]" 1456 | ] 1457 | } 1458 | ], 1459 | "prompt_number": 67 1460 | }, 1461 | { 1462 | "cell_type": "markdown", 1463 | "metadata": {}, 1464 | "source": [ 1465 | "## Assignment 7\n", 1466 | "\n", 1467 | "This is a very naive function for factoring integers" 1468 | ] 1469 | }, 1470 | { 1471 | "cell_type": "code", 1472 | "collapsed": false, 1473 | "input": [ 1474 | "def factorize(n):\n", 1475 | " if n < 2:\n", 1476 | " return []\n", 1477 | " factors = []\n", 1478 | " p = 2\n", 1479 | "\n", 1480 | " while True:\n", 1481 | " if n == 1:\n", 1482 | " return factors\n", 1483 | " r = n % p\n", 1484 | " if r == 0:\n", 1485 | " factors.append(p)\n", 1486 | " n = n / p\n", 1487 | " elif p * p >= n:\n", 1488 | " factors.append(n)\n", 1489 | " return factors\n", 1490 | " elif p > 2:\n", 1491 | " p += 2\n", 1492 | " else:\n", 1493 | " p += 1" 1494 | ], 1495 | "language": "python", 1496 | "metadata": {}, 1497 | "outputs": [] 1498 | }, 1499 | { 1500 | "cell_type": "markdown", 1501 | "metadata": {}, 1502 | "source": [ 1503 | "The task is to factorize all numbers from 2 to 500000, and count the number of _unique_ factors. Then count how many times a factor count occurs. That is, make a histogram of factor counts.\n", 1504 | "\n", 1505 | "For the sake of clarity, this is what the task would be for the numbers up to 10:\n", 1506 | "\n", 1507 | " number factors unique num_unique_factors\n", 1508 | " 2 [2] 1\n", 1509 | " 3 [3] 1\n", 1510 | " 4 [2, 2] [2] 1\n", 1511 | " 5 [5] 1\n", 1512 | " 6 [2, 3] 2\n", 1513 | " 7 [7] 1\n", 1514 | " 8 [2, 2, 2] [2] 1\n", 1515 | " 9 [3, 3] [3] 1\n", 1516 | " 10 [2, 5] 2\n", 1517 | "\n", 1518 | "Which gives us\n", 1519 | "\n", 1520 | " {1: 7, 2: 2}\n", 1521 | "\n", 1522 | "\n", 1523 | "### 1.\n", 1524 | "First implement this serially, that is, how you normally would. (It should take in the order of 15 seconds for 500000 numbers).\n", 1525 | "\n", 1526 | "\n", 1527 | "### 2.\n", 1528 | "Then make an implementation which uses `multiprocessing` for parallelization, (the parallelization strategy should be to factor some numbers on some processes, don't worry about the implementation of the factoring function).\n", 1529 | "\n", 1530 | "### 3.\n", 1531 | "Implement the same thing using `IPython.parallel`. (Remember that you need to start the `ipcluster` before running a script using `IPython.parallel`, this is something I have a tendency to forget.)\n", 1532 | "\n", 1533 | "For this assignment, put _everything_ in one script. Call the script `num_factors.py`. Put it in the `scripts` directory of your repository.\n", 1534 | "\n", 1535 | "The script should take a character as an argument to determining how it will be run.\n", 1536 | "So to run it serially, you run\n", 1537 | "\n", 1538 | " $ num_factors.py s\n", 1539 | "\n", 1540 | "With `multiprocessing`\n", 1541 | "\n", 1542 | " $ num_factors.py m\n", 1543 | "\n", 1544 | "and with `IPython.parallel`\n", 1545 | "\n", 1546 | " $ num_factors.py i\n", 1547 | "\n", 1548 | "The script should print a dictionary like in the example above" 1549 | ] 1550 | }, 1551 | { 1552 | "cell_type": "code", 1553 | "collapsed": false, 1554 | "input": [], 1555 | "language": "python", 1556 | "metadata": {}, 1557 | "outputs": [] 1558 | } 1559 | ], 1560 | "metadata": {} 1561 | } 1562 | ] 1563 | } --------------------------------------------------------------------------------