├── Data
├── Basics.xlsx
├── BubbleColumn
│ ├── Test_01.xlsx
│ ├── Test_02.xlsx
│ └── Test_03.xlsx
└── TensileTest
│ ├── AlMg3.txt
│ ├── HDPE.txt
│ └── Stahl.txt
├── DataManagement
└── 1_DataManagement.ipynb
├── DataVisualization
└── 2_Visualization.ipynb
├── JupyterHacks
├── .ipynb_checkpoints
│ └── Jupyter-Hacks-checkpoint.ipynb
├── Jupyter-Hacks.ipynb
├── README.md
├── hello_world.ipynb
└── hello_world.py
├── LICENSE
├── ML
├── 3_ML.ipynb
├── Bereiche-des-Machine-Learnings.png
├── HealthRiskIndex.png
└── deepmind_parkour.0.gif.mp4
└── README.md
/Data/Basics.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/astridwalle/python_jupyter_basics/27bbd1591fed3ad9b1def6b9eb31efe8fbef1f3a/Data/Basics.xlsx
--------------------------------------------------------------------------------
/Data/BubbleColumn/Test_01.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/astridwalle/python_jupyter_basics/27bbd1591fed3ad9b1def6b9eb31efe8fbef1f3a/Data/BubbleColumn/Test_01.xlsx
--------------------------------------------------------------------------------
/Data/BubbleColumn/Test_02.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/astridwalle/python_jupyter_basics/27bbd1591fed3ad9b1def6b9eb31efe8fbef1f3a/Data/BubbleColumn/Test_02.xlsx
--------------------------------------------------------------------------------
/Data/BubbleColumn/Test_03.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/astridwalle/python_jupyter_basics/27bbd1591fed3ad9b1def6b9eb31efe8fbef1f3a/Data/BubbleColumn/Test_03.xlsx
--------------------------------------------------------------------------------
/Data/TensileTest/AlMg3.txt:
--------------------------------------------------------------------------------
1 | Probenkennz. ALMg3
Datum 21.11.13
Geschwindigkeit 20 mm/min
Kraftmaximum 2590,2 N
Probenhoehe a0 0,99 mm
Probenbreite b0 13,01 mm
Laege l0 70 mm
Probenhe aU NACH ZUGVERSUCH 0,78 mm
Probenbreite bU NACH ZUGVERSUCH 11,41 mm
Laege lU NACH ZUGVERSUCH 79,1 mm
Standardweg Standardkraft
mm N
0,000916856 5,246944
0,002417175 9,83472
0,0041652 14,89712
0,006047685 20,38136
0,00800094 27,31576
0,00997542 35,48944
0,01196406 43,8476
0,013959765 51,88944
0,01594845 59,90488
0,0180786 68,65856
0,0199398 77,2804
0,02194965 87,2208
0,02395245 97,5824
0,02595525 107,7072
0,027951 118,1224
0,02994675 128,8272
0,03194955 140,112
0,03394515 151,9504
0,03594795 164,1848
0,03795075 176,6824
0,03995355 189,4968
0,0419634 202,6008
0,0439521 216,232
0,04594785 230,18
0,04794345 244,576
0,0499392 259,4472
0,0519633 274,74
0,0539589 290,4016
0,05595465 306,564
0,0579504 323,0168
0,05994615 339,812
0,0619419 357,0296
0,06394455 374,7216
0,0659403 392,8352
0,06795015 411,292
0,06995295 430,4872
0,07207605 451,2904
0,07394445 469,9576
0,07594725 490,26
0,07794285 511,0104
0,07994565 532,0512
0,08195565 553,54
0,0839583 575,3184
0,085947 597,44
0,08794275 619,6408
0,08994555 641,9472
0,09194835 665,044
0,09395115 689,9344
0,09594675 716,5904
0,09794955 743,6424
0,0999453 770,6952
0,1019481 798,0104
0,10394385 825,432
0,10594665 853,04
0,10794945 880,752
0,10994505 908,432
0,11195505 936,28
0,11395785 964,12
0,1159464 992,04
0,11794215 1019,968
0,119952 1048,096
0,1219548 1076,336
0,12395055 1104,472
0,1259604 1132,336
0,12795615 1160,024
0,12995175 1187,864
0,1319475 1215,656
0,13395735 1243,24
0,1359531 1270,872
0,13794885 1298,264
0,13995165 1325,264
0,14195445 1351,944
0,14395005 1378,336
0,1459458 1404,472
0,1479486 1430,336
0,1499514 1455,96
0,1519545 1481,168
0,153957 1506,112
0,155952 1530,576
0,1579485 1554,728
0,1599435 1578,464
0,1619535 1601,72
0,1639575 1624,288
0,1659525 1646,168
0,1679475 1667,32
0,1699515 1687,432
0,171954 1706,784
0,173949 1725,088
0,175953 1742,28
0,177948 1758,544
0,1799505 1773,864
0,181953 1788,256
0,183957 1801,656
0,1859445 1814,024
0,187941 1825,256
0,190071 1836,064
0,191946 1844,424
0,193956 1852,384
0,1959525 1859,32
0,1979475 1865,408
0,19995 1870,816
0,2019465 1875,768
0,203949 1880,224
0,205944 1884,312
0,207954 1888,216
0,210084 1891,984
0,211953 1895,096
0,2139555 1898,448
0,2159445 1902,008
0,2179335 1905,536
0,2199435 1908,728
0,2219535 1912,208
0,2239485 1914,184
0,225951 1915,688
0,22794 1916,688
0,23007 1918,592
0,231945 1920,304
0,233949 1922,648
0,2359515 1925,448
0,2379465 1929,056
0,239949 1933,064
0,241953 1936,704
0,243948 1935,832
0,2459445 1931,664
0,247947 1926,952
0,250077 1928,976
0,2519595 1930,456
0,2539545 1932,168
0,255951 1934,512
0,25794 1936,888
0,2599425 1938,208
0,261945 1939,712
0,2639475 1941,216
0,26595 1942,056
0,267954 1942,584
0,269949 1943,824
0,271944 1945,064
0,273948 1946,328
0,275943 1949,336
0,27573742 1952,312
0,28492982 1955,216
0,29406048 1956,904
0,30321858 1959,8
0,31234238 1963,44
0,32147304 1966,576
0,33063114 1968,736
0,33978924 1969,712
0,34894734 1971,928
0,358078 1976,672
0,3672018 1983,712
0,37636676 1986,744
0,38549056 1989,328
0,39464866 1990,784
0,40380676 1998,904
0,41297172 2004,336
0,42212982 2006,176
0,43128792 2008,472
0,44038428 2008,608
0,44950808 2011,376
0,45866618 2015,088
0,46783114 2018,832
0,47698924 2022,056
0,48614734 2024,64
0,4952437 2026,96
0,5050192 2029,224
0,5135256 2028,168
0,522718 2030,016
0,53184866 2028,432
0,54100676 2027,72
0,55013742 2029,12
0,55929552 2030,968
0,56841932 2032,544
0,57757742 2034,208
0,58670808 2036
0,59644928 2038,056
0,60502428 2039,376
0,61421668 2040,432
0,62334734 2041,272
0,63247114 2042,144
0,64162924 2043,224
0,6507599 2044,728
0,659918 2045,648
0,6690761 2046,016
0,67820676 2046,92
0,68794796 2048,576
0,69648866 2049,952
0,70565362 2051,136
0,71477742 2053,008
0,72393552 2055,752
0,73309362 2056,936
0,74228602 2059,12
0,75141668 2061,656
0,76054048 2066,4
0,76963684 2069,536
0,77937804 2073,232
0,78795304 2075,896
0,797118 2077,368
0,8062761 2078,952
0,8153999 2080,688
0,824558 2082,512
0,83368866 2085,592
0,84284676 2089,84
0,85200486 2090,344
0,86113552 2092,448
0,87091102 2096,824
0,87945172 2101,944
0,88858238 2107,872
0,89770618 2102,184
0,90686428 2098,936
0,91599494 2099,624
0,92515304 2104,528
0,93434544 2104,576
0,94346924 2114,072
0,9525999 2129,048
0,9623068 2135,4
0,97545056 2129,232
0,98464296 2128,496
0,99376676 2129,232
1,00295916 2125,248
1,01205552 2118,688
1,02117932 2121,296
1,03034428 2124,752
1,03950238 2128,92
1,04924358 2132,056
1,05781858 2131,312
1,06697668 2140,2
1,07610734 2149,904
1,08526544 2149,616
1,09438924 2142,68
1,1035542 2140,968
1,112678 2142,888
1,12180866 2144,496
1,1309942 2145,92
1,14015916 2147,264
1,15381742 2149,64
1,16297552 2151,224
1,17213362 2152,304
1,18126428 2152,096
1,19042238 2152,512
1,19961478 2154,256
1,20873858 2156,52
1,21783494 2157,92
1,22754184 2159,288
1,23615114 2159,92
1,24530924 2160,24
1,2544742 2160,976
1,263598 2162,168
1,27272866 2163,512
1,28188676 2164,672
1,29104486 2165,696
1,30020296 2167,176
1,30991672 2169,36
1,31845742 2170,888
1,32764982 2171,552
1,33678048 2172,368
1,34590428 2174,48
1,35500064 2177,96
1,36419304 2181,28
1,37335114 2185,496
1,38254354 2187,848
1,39166734 2189,848
1,4013468 2191,8
1,4099561 2192,984
1,4191142 2194,52
1,42821056 2196,048
1,43743726 2197,44
1,44652676 2198,92
1,45565742 2198,312
1,46484982 2199,792
1,47397362 2202,504
1,48310428 2205,72
1,49682428 2212,792
1,50598238 2221,144
1,51514048 2224,68
1,52427114 2214,712
1,53349784 2212,152
1,54258734 2213,576
1,551718 2218,408
1,5608418 2224,36
1,570583 2225,152
1,57916486 2218,192
1,58832296 2219,856
1,59744676 2228,32
1,60657742 2235,968
1,61570122 2236,256
1,62489362 2238,128
1,63405172 2243,952
1,64318238 2243,4
1,65234048 2236,864
1,66204738 2231,744
1,67062924 2232,72
1,67975304 2235,752
1,68891114 2236,752
1,6980761 2241,976
1,7071999 2239,944
1,716358 2242,216
1,7255504 2246,112
1,73464676 2248,304
1,74377056 2253,576
1,75293552 2261,088
1,76209362 2266,416
1,77581362 2257,72
1,78497172 2251,68
1,79410238 2253,312
1,80322618 2254,688
1,81235684 2255,816
1,82151494 2256,792
1,83070734 2257,792
1,83986544 2258,848
1,8489961 2260,04
1,858703 2261,088
1,86721626 2261,752
1,87640866 2261,912
1,88560106 2261,648
1,89472486 2262,624
1,90385552 2264,496
1,91304792 2266,128
1,92217172 2267,288
1,93129552 2268,552
1,94042618 2269,216
1,95013308 2269,768
1,95871494 2270,296
1,96790734 2271,112
1,97706544 2272,064
1,98618924 2273,296
1,9953199 2274,272
2,0044437 2274,832
2,0136018 2275,592
2,02276676 2276,592
2,0319523 2277,968
2,04163176 2279,232
2,05480296 2278,888
2,06389932 2279,024
2,07302998 2281,656
2,08222238 2283,928
2,09138048 2286,192
2,10053858 2288,8
2,10966238 2291,864
2,11879304 2294,08
2,12795114 2295,632
2,13710924 2296,208
2,1508361 2297,184
2,1599942 2298,848
2,1691523 2300,408
2,17828296 2301,616
2,18740676 2302,616
2,19653742 2304,176
2,20569552 2305,016
2,21481932 2305,256
2,22397742 2307,024
2,23773858 2302,064
2,24686238 2307,576
2,25595874 2325,56
2,26511684 2330,28
2,27430924 2334,84
2,28346734 2335,944
2,29259114 2338,528
2,3023392 2336,392
2,3154418 2336,16
2,32460676 2340,248
2,33376486 2335,792
2,34292296 2327,216
2,35204676 2332,944
2,36121172 2334,256
2,37033552 2326,848
2,37949362 2326,688
2,38862428 2326,296
2,39833118 2329,04
2,40690618 2333,072
2,41607114 2332,28
2,42526354 2327,56
2,43438734 2325,872
2,44354544 2331,016
2,4526418 2338,136
2,4617999 2341,032
2,470958 2341,144
2,48012296 2343,064
2,48928106 2344,856
2,49840486 2337,264
2,51209742 2331,28
2,52122122 2335,496
2,53041362 2347,36
2,53957858 2355,192
2,54873668 2361,624
2,55789478 2369,376
2,56701858 2372,544
2,57614924 2357,2
2,58527304 2346,496
2,594438 2350,08
2,6081923 2352,16
2,6173161 2352,952
2,62644676 2353,928
2,63560486 2354,88
2,64472866 2355,696
2,65385932 2356,224
2,66301742 2356,832
2,67217552 2357,328
2,68191672 2358,384
2,69052602 2359,096
2,69965668 2359,544
2,70875304 2360,048
2,71787684 2360,464
2,72703494 2360,152
2,73619304 2360,128
2,745358 2360,496
2,7545504 2361,52
2,7636399 2362,208
2,77277056 2363,184
2,78189436 2364,104
2,79564866 2365,48
2,80481362 2366,056
2,81397172 2366,4
2,82312982 2366,664
2,83225362 2367,088
2,84138428 2367,32
2,85050808 2367,56
2,85967304 2367,616
2,86883114 2368,272
2,88258544 2369,144
2,89170924 2369,904
2,9008399 2370,776
2,90993626 2371,856
2,91912866 2372,992
2,92828676 2373,416
2,93744486 2372,52
2,94660296 2373,28
2,95573362 2376,76
2,96941932 2380,032
2,97854998 2381,216
2,98774238 2381,664
2,99690048 2383,536
3,00602428 2385,488
3,01518924 2387,048
3,02431304 2388,336
3,0334437 2389,472
3,0426018 2390,128
3,0517599 2390,16
3,0655142 2391,128
3,0746723 2393,08
3,08380296 2394,848
3,09292676 2396,144
3,10205742 2397,04
3,11121552 2397,984
3,12037362 2398,464
3,12950428 2398,912
3,13866238 2398,856
3,15238238 2398,04
3,16151304 2397,936
3,17067114 2402,024
3,17982924 2410,752
3,18898734 2414,416
3,198118 2416,944
3,2072761 2420,08
3,2163999 2420,344
3,22556486 2421,376
3,23468866 2417,104
3,24841552 2410,696
3,25757362 2414,176
3,26673172 2423,536
3,27585552 2424,8
3,28502048 2423,088
3,29414428 2424,592
3,30326808 2429,576
3,31243304 2432,472
3,32217424 2432,184
3,33078354 2430,52
3,33990734 2429,312
3,349038 2429,736
3,3581618 2424,064
3,3673199 2423,144
3,37645056 2421,456
3,38564296 2426,224
3,39480106 2423,744
3,40392486 2427,488
3,41305552 2434,768
3,42677552 2438,88
3,43594048 2427,728
3,44509858 2420,848
3,45422238 2431,68
3,46341478 2432,84
3,47251114 2428,096
3,48166924 2416,176
3,4907999 2416,472
3,499958 2417,84
3,5091161 2420,896
3,5228361 2444,6
3,53200106 2448,664
3,54112486 2447,08
3,55022122 2450,216
3,55937932 2445,128
3,56857172 2441,992
3,57769552 2440,672
3,58686048 2439,616
3,59601858 2441,232
3,60971114 2442,68
3,61883494 2443,496
3,62799304 2443,256
3,63718544 2443,024
3,6463161 2443,968
3,6554399 2446,584
3,664598 2448,376
3,6737561 2446,976
3,68346986 2447,368
3,69201056 2449,904
3,70117552 2454,256
3,71033362 2451,008
3,71952602 2453,36
3,72864982 2454,336
3,73778048 2453,544
3,74690428 2455,416
3,75600064 2453,176
3,76519304 2449,88
3,77438544 2448,608
3,78810544 2447,872
3,7972361 2447,816
3,8063599 2448,344
3,81549056 2448,64
3,82464866 2448,4
3,83380676 2447,952
3,84299916 2448,08
3,85212296 2448,008
3,86125362 2446,656
3,87497362 2444,368
3,88406998 2444,392
3,89322808 2444,736
3,90242048 2444,336
3,91157858 2443,232
3,92070924 2441,28
3,92983304 2440,28
3,9389637 2440,096
3,9487049 2439,936
3,9572799 2439,856
3,9664723 2440,04
3,97560296 2440,2
3,98476106 2438,72
3,99391916 2437,248
4,00301552 2436,352
4,01213932 2436,032
4,02126998 2439,616
4,03046238 2441,232
4,03962048 2445,232
4,04932738 2449,504
4,05790238 2453,408
4,06703304 2454,856
4,07619114 2455,52
4,08534924 2461,4
4,09450734 2466,832
4,103638 2466,568
4,1127961 2463,848
4,12192676 2471,968
4,13108486 2475,136
4,14018122 2477,904
4,15396296 2480,992
4,16309362 2483,76
4,17221742 2484,576
4,18138238 2483,864
4,19054048 2486,16
4,19966428 2488,264
4,20879494 2486,68
4,21795304 2489,344
4,22711114 2491,112
4,240838 2492,168
4,2499961 2491,168
4,2591199 2491,168
4,26825056 2491,776
4,27744296 2491,64
4,28656676 2494,808
4,29572486 2482,648
4,30488296 2469,912
4,31463102 2461,32
4,32317172 2459,184
4,33226808 2460,528
4,34142618 2462,192
4,35058428 2463,432
4,35974238 2464,248
4,36893478 2465,376
4,37799684 2466,248
4,3871618 2466,488
4,3962856 2466,352
4,4060611 2466,88
4,4146018 2468,016
4,4237942 2469,048
4,43292486 2469,096
4,44204866 2470,992
4,45121362 2475,16
4,46033742 2479,96
4,46949552 2482,68
4,47865362 2483,76
4,48778428 2483,864
4,49697668 2482,52
4,50610048 2482,176
4,51523114 2482,888
4,52435494 2483,576
4,53351304 2484,6
4,54270544 2489,872
4,5518361 2493,464
4,5609599 2492,672
4,57009056 2490,928
4,57924866 2490,112
4,58837246 2489,824
4,59812052 2491,664
4,60669552 2495,256
4,61585362 2494,568
4,62501172 2494,912
4,63416982 2494,808
4,64326618 2488,952
4,65239684 2485,976
4,66155494 2487,104
4,67067874 2494,912
4,67990544 2502,056
4,68902924 2492,856
4,6981942 2491,696
4,7119142 2493,224
4,72101056 2492,592
4,73016866 2500,552
4,73932676 2499
4,74848486 2502,056
4,75764982 2495,44
4,76673932 2496,096
4,77590428 2502,976
4,78502808 2498,68
4,79874808 2499,08
4,80794048 2501,472
4,81709858 2502,744
4,82626354 2499,312
4,83538734 2499,792
4,8444837 2502,896
4,8536418 2502,744
4,8627999 2503,08
4,8725754 2505,536
4,88112296 2502,928
4,89028106 2500,024
4,89940486 2503,848
4,90853552 2504,432
4,91765932 2508,752
4,92678998 2507,96
4,93598238 2503,72
4,94514048 2504,272
4,95429858 2498,176
4,96401234 2497,864
4,97255304 2495,752
4,9816837 2493,616
4,9908761 2495,888
4,9999999 2499,08
5,0091923 2498,024
5,0183161 2496,464
5,02744676 2497,28
5,03657056 2498,52
5,04572866 2496,072
5,05544242 2496,304
5,06401742 2494,672
5,07317552 2492,672
5,08236792 2494,776
5,09149858 2506,12
5,10062238 2501,344
5,10971874 2489,952
5,11891114 2488,608
5,12806924 2490,008
5,1371999 2491,456
5,14638544 2493,272
5,1555161 2500,16
5,1646399 2508,096
5,17377056 2523,544
5,18292866 2530,848
5,19208676 2512,736
5,20121742 2496,76
5,21040982 2499,6
5,21953362 2516,16
5,22924052 2527,496
5,23778808 2525,6
5,24694618 2520,672
5,25610428 2513,416
5,26526924 2509,04
5,27442734 2510,44
5,28358544 2511,024
5,2926818 2511,128
5,3018399 2510,232
5,31097056 2511,6
5,32071176 2515,424
5,32932106 2522,704
5,33844486 2526,656
5,34760296 2526,496
5,35673362 2534,04
5,36582312 2529,264
5,37501552 2525,072
5,38418048 2529,16
5,39333858 2543,584
5,40249668 2539,6
5,41220358 2537,048
5,42075114 2535,568
5,42987494 2535,176
5,4390056 2531,088
5,4481637 2533,224
5,4573561 2530,64
5,4665142 2541,656
5,47564486 2544,744
5,48476866 2548,696
5,49389932 2529,768
5,50305742 2515,928
5,51681172 2503,16
5,52596982 2494,432
5,53510048 2499,84
5,54422428 2516,976
5,55338238 2528,976
5,56251304 2532,96
5,57170544 2542,424
5,58141234 2534,752
5,58998734 2544,664
5,5991523 2545,96
5,6082761 2555,104
5,61740676 2536,176
5,62653056 2522,304
5,63568866 2516,344
5,64484676 2509,624
5,65401172 2511,864
5,66371862 2514,712
5,67688982 2516,584
5,68602048 2512,312
5,69514428 2515,952
5,70427494 2518,776
5,71343304 2513,84
5,72262544 2518,192
5,73174924 2519,192
5,741463 2514,368
5,7500037 2519,856
5,7591618 2518,984
5,76832676 2517,376
5,77748486 2522,304
5,78660866 2518,192
5,79576676 2519,8
5,80493172 2526,128
5,81405552 2522,568
5,82318618 2531,432
5,83289308 2530,976
5,84150238 2526,896
5,85062618 2525,552
5,85981858 2526,92
5,8689904 2524,728
5,8780456 2529,296
5,8871694 2528,392
5,8963618 2527,976
5,9055542 2533,96
5,914678 2528,952
5,9244192 2532,696
5,9329942 2536,856
5,942118 2533,408
5,9512418 2539,632
5,9604342 2538,76
5,969558 2534,8
5,9786818 2529,792
5,9878742 2534,248
5,996998 2529,872
6,0061218 2536,28
6,015863 2537,968
6,024438 2536,384
6,0335618 2542,448
6,0427542 2536,096
6,051878 2537,464
6,0610018 2538,52
6,0701256 2531,824
6,079318 2538,76
6,0884418 2537,624
6,0976342 2540,024
6,1073068 2534,008
6,1158818 2535,224
6,1250742 2535,568
6,134198 2547,168
6,1433218 2540,896
6,1524456 2539,392
6,161638 2528,872
6,1707618 2538,656
6,1799542 2530,4
6,189078 2534,2
6,1987506 2526,208
6,2073256 2528,872
6,216518 2537,864
6,2256418 2549,328
6,2348342 2553,784
6,2440266 2554,608
6,2530818 2540,896
6,2622742 2536,672
6,271398 2537,464
6,2805218 2549,04
6,290263 2543,448
6,298838 2535,064
6,3080304 2533,088
6,3171542 2540,576
6,326278 2550,44
6,3354018 2554,368
6,3445256 2545,8
6,353718 2549,28
6,3628418 2541,368
6,3720342 2541,792
6,3817068 2530,48
6,3902818 2529,424
6,3994056 2531,8
6,408598 2532,088
6,4177218 2537,36
6,4269142 2532,064
6,436038 2540,76
6,4451618 2537,6
6,4543542 2539,336
6,463478 2542,424
6,4731506 2536,336
6,4863218 2538,176
6,4955142 2545,8
6,504638 2539,784
6,5138304 2547,224
6,5229542 2544,504
6,532078 2545,376
6,5412018 2549,016
6,5503942 2544,536
6,559518 2553,944
6,5687104 2547,088
6,5823618 2549,408
6,5915542 2556,848
6,6006094 2549,2
6,6098018 2559,584
6,6189256 2552,52
6,628118 2558,296
6,6373104 2554,608
6,6464342 2554,944
6,655558 2556,448
6,6692094 2553,392
6,6784018 2559,32
6,6875942 2545,904
6,6967866 2542,24
6,7059104 2537,384
6,7150342 2538,048
6,7240894 2540,976
6,7332818 2534,832
6,7424056 2540,416
6,751598 2539,024
6,765318 2539,048
6,7745104 2539,784
6,7836342 2546,088
6,7926894 2540,232
6,8018818 2548,064
6,8110742 2544,16
6,820198 2544,424
6,8293904 2549,304
6,839063 2539,944
6,847638 2542,448
6,8567618 2543,424
6,8659542 2556,24
6,875078 2561,568
6,8842018 2556,448
6,8933256 2551,76
6,902518 2547,376
6,9116418 2547,488
6,9213144 2541,952
6,929958 2544,216
6,9390818 2542,504
6,9482056 2547,04
6,957398 2545,088
6,9665218 2544,424
6,9757142 2548,248
6,984838 2540,944
6,9939618 2546,592
7,0031542 2542,528
7,0128954 2541,792
7,0214018 2546,512
7,0305942 2540,416
7,039718 2548,408
7,0488418 2541,792
7,0579656 2545,376
7,067158 2545,96
7,0762818 2543,632
7,0854742 2550,336
7,094598 2542,712
7,1043392 2550,072
7,1128456 2544,24
7,1219694 2546,144
7,1312304 2545,824
7,1403542 2540,944
7,149478 2548,328
7,1586704 2542,08
7,1677942 2549,912
7,176918 2552,096
7,1860418 2549,224
7,1957144 2544,768
7,204358 2545,824
7,2134818 2543,952
7,2226742 2548,488
7,231798 2542,24
7,2409218 2551,36
7,2501142 2543,824
7,259238 2551,28
7,2684304 2545,696
7,2775542 2547,144
7,2872268 2548,8
7,2958018 2542,552
7,3049256 2549,832
7,314118 2543,32
7,3233104 2546,8
7,3324342 2538,92
7,341558 2542,896
7,3506818 2543,848
7,3598742 2543,664
7,368998 2547,248
7,3786706 2542,16
7,3872456 2540,208
7,396438 2529,48
7,4056304 2533,96
7,4147542 2528,688
7,423878 2536,704
7,4330018 2530,528
7,4421256 2540,792
7,451318 2533,456
7,4605104 2540,104
7,470183 2536,784
7,4833542 2533,352
7,492478 2541,976
7,5016018 2536,36
7,5107256 2538,944
7,519918 2538,176
7,5290418 2533,672
7,5382342 2539,92
7,547358 2532,776
7,5564818 2543,32
7,5702018 2543,768
7,5793942 2546,592
7,588518 2551,44
7,5977104 2557,688
7,6068342 2557,688
7,615958 2561,192
7,6250818 2555,16
7,6342742 2562,168
7,643398 2561,984
7,657118 2563,856
7,6663104 2569,632
7,6754342 2571,976
7,6844894 2580,256
7,6936818 2581,104
7,7028742 2590,2
7,711998 2575,592
7,7217392 2559,64
7,7303142 2536,832
7,739438 2545,48
7,7486304 2536,672
7,7576856 2546,776
7,7668094 2540,208
7,7760018 2550,6
7,7851256 2543,136
7,794318 2549,728
7,8035104 2544,72
7,8126342 2548,488
7,8262856 2553,392
7,835478 2546,8
7,8446018 2556,632
7,8537942 2551,072
7,862918 2561,04
7,8721104 2556,632
7,8812342 2563,992
7,890358 2555,392
7,8994818 2562,672
7,9086742 2555,528
7,9223942 2553,312
7,9315866 2561,592
7,9407104 2555,08
7,9497656 2562,648
7,9588894 2554,816
7,9680818 2562,648
7,9772742 2561,088
7,986398 2564,2
7,9955218 2556,816
8,0047142 2553,968
8,0183656 2563,12
8,027558 2556,896
8,0366818 2564,232
8,0458742 2557,296
8,054998 2564,832
8,0641904 2557,768
8,0733142 2565,232
8,082438 2559,008
8,0915618 2566,416
8,1052818 2563,568
8,1144742 2563,176
8,123598 2562,832
8,1327218 2556,16
8,1418456 2552,6
8,151038 2550,04
8,1602304 2551,52
8,1693542 2554,184
8,1790954 2550,624
8,1876704 2553,864
8,1967942 2546,96
8,205918 2552,072
8,2150418 2546,984
8,2241656 2552,208
8,233358 2547,408
8,2424818 2552,968
8,2516742 2548,168
8,260798 2553,232
8,2704706 2549,144
8,2790456 2553,392
8,2881694 2555,08
8,2973618 2553,336
8,3065542 2557,792
8,315678 2552,704
8,3248704 2556,584
8,3339942 2553,128
8,343118 2556,184
8,3522418 2546,984
8,361983 2556,344
8,370558 2548,648
8,3796818 2556,664
8,3888742 2549,384
8,397998 2552,704
8,4071218 2550,256
8,4162456 2549,648
8,425438 2553,104
8,4345618 2548,544
8,4437542 2554,896
8,4534268 2547,144
8,4620018 2555,576
8,4711256 2549,096
8,480318 2557,584
8,4894418 2550,544
8,4986342 2558,192
8,507758 2549,064
8,5169504 2550,464
8,5260742 2538,6
8,535198 2545,056
8,5448706 2545,616
8,5534456 2551,992
8,562638 2543,848
8,5717618 2549,096
8,5809542 2543,688
8,590078 2550,784
8,5992018 2549,096
8,6083942 2552,968
8,617518 2556,792
8,6267104 2555,208
8,636383 2559,352
8,644958 2556,664
8,6541504 2559,112
8,6632056 2554,816
8,672398 2559,664
8,6815218 2552,92
8,6906456 2536,568
8,699838 2522,408
8,7090304 2517,4
8,7181542 2524,176
8,7278268 2519,008
8,7364018 2525,208
8,7455942 2519,672
8,754718 2526,52
8,7639104 2521,544
8,7730342 2527,576
8,782158 2523,704
8,7912818 2529,216
8,8004056 2527,024
8,809598 2530,928
8,8192706 2532,064
8,8279142 2530,904
8,8371066 2535,36
8,8461618 2531,664
8,8553542 2535,648
8,8644094 2531,32
8,8736018 2534,592
8,8827256 2531,952
8,891918 2533,04
8,9010418 2533,72
8,910783 2528,656
8,919358 2531,984
8,9284818 2529,608
8,9376742 2526,976
8,946798 2520,408
8,9559218 2521,808
8,9651142 2523,336
8,974238 2514,896
8,9834304 2513,08
8,9924856 2512,968
9,0022268 2506,304
9,0108018 2498,416
9,0199942 2488,952
9,029118 2483,624
9,0382418 2471,944
9,0474342 2461,848
9,056558 2452,168
9,0657504 2433,528
9,0748742 2426,064
9,083998 2405,208
9,0937392 2385,648
9,1023142 2368,168
9,111438 2344,512
9,1205618 2318,336
9,1296856 2290,544
9,138878 2258,272
9,1480018 2220,512
9,1571942 2175,792
9,166318 2119,368
--------------------------------------------------------------------------------
/Data/TensileTest/Stahl.txt:
--------------------------------------------------------------------------------
1 | Probenkennz. Baustahl
Datum 02.08.13
Geschwindigkeit 20 mm/min
Kraftmaximum 4251,6396 N
Probenhoehe a0 1 mm
Probenbreite b0 12,96 mm
Laenge l0 70 mm
Probenhoehe a0 NACH ZUGVERSUCH 0,8 mm
Probenbreite b0 NACH ZUGVERSUCH 8,84 mm
Laenge l0 NACH ZUGVERSUCH 85,84 mm
Standardweg Standardkraft
mm N
0 0
0,00052905 3,84582432
0,00187457 3,06590784
0,002924234 4,78711296
0,004022534 5,48635152
0,005145144 2,71628448
0,006279917 2,90454384
0,007418745 10,7306448
0,008561627 23,3170368
0,009704467 33,482928
0,010847366 42,8688864
0,01199834 52,6582752
0,013145277 63,5771712
0,014288177 74,4422928
0,015435113 85,441728
0,016578013 98,5932
0,017793841 116,585184
0,018867763 134,254032
0,0200147 153,133824
0,021161637 171,878976
0,022308574 190,973376
0,023447436 210,660192
0,024594373 231,207072
0,025741309 252,775584
0,026884123 275,4204
0,02803106 298,87224
0,029177997 323,076432
0,030324933 348,141504
0,031467833 374,175168
0,032610732 401,230464
0,033753632 429,145824
0,034900483 458,029776
0,036051543 487,666896
0,03719848 518,057184
0,038341293 549,254496
0,039484193 581,284944
0,04063113 614,06856
0,041769992 647,713056
0,042920966 682,271472
0,044067903 717,421488
0,045210716 753,056208
0,046357653 789,336384
0,04750459 826,10208
0,048643452 863,23824
0,049790389 900,83952
0,050933288 939,10992
0,052080139 977,88624
0,053227076 1017,12768
0,054374013 1056,85056
0,055516912 1096,97328
0,056659812 1137,56112
0,057806749 1178,32848
0,058953685 1219,47936
0,060100622 1260,89136
0,061247559 1302,44208
0,062394496 1343,83776
0,063537309 1385,22528
0,064680209 1427,06976
0,065827146 1469,50992
0,066974083 1512,6192
0,068116982 1556,05488
0,069263919 1599,59664
0,070410856 1643,40768
0,071553669 1687,56144
0,072696569 1732,23744
0,073839468 1777,19904
0,074986405 1822,5768
0,076133342 1868,34624
0,077280279 1914,31152
0,078423178 1960,40736
0,079570029 2006,87856
0,080716966 2053,7496
0,081859865 2101,06128
0,083006802 2148,6096
0,084153739 2196,4272
0,085296638 2244,3264
0,086447183 2291,9808
0,087590512 2339,25984
0,088732982 2386,58784
0,089876311 2434,13616
0,091023076 2481,84768
0,092169841 2529,6408
0,093316606 2577,132
0,09446423 2623,95408
0,0956067 2670,18864
0,096750029 2715,74592
0,097892499 2760,22608
0,099043559 2803,2864
0,100190324 2844,62496
0,101333653 2884,12752
0,102476123 2921,08416
0,103622888 2954,58912
0,104842668 2986,13568
0,105912982 3009,64464
0,107059747 3029,05728
0,108206512 3041,42784
0,109349841 3046,29936
0,110500901 3043,20672
0,111643371 3033,97776
0,1127867 3022,14576
0,11392917 3010,99104
0,115075935 3002,00688
0,116226995 2995,03824
0,117366029 2989,39152
0,118517089 2984,6016
0,119656123 2981,02752
0,120802888 2979,11808
0,17393 2966,098291
0,18648 2968,066483
0,19979 2968,840051
0,21305 2968,683379
0,22635 2968,164403
0,23975 2967,713971
0,2531 2967,518131
0,26645 2968,164403
0,27976 2969,231731
0,29306 2971,229299
0,30722 2973,550003
0,31977 2976,164467
0,33307 2978,494963
0,34642 2980,463155
0,35978 2981,755699
0,37313 2982,588019
0,38648 2982,588019
0,39974 2981,755699
0,41304 2980,296691
0,42639 2978,103283
0,4406 2974,881715
0,45315 2971,944115
0,46645 2969,261107
0,47976 2966,969779
0,49306 2965,324723
0,50641 2964,649075
0,51976 2964,678451
0,53312 2964,874291
0,54642 2965,745779
0,55977 2966,813107
0,57312 2967,909811
0,58638 2969,104435
0,59973 2970,847411
0,61304 2972,942899
0,62644 2975,038387
0,63979 2977,035955
0,65314 2978,818099
0,66645 2980,365235
0,67975 2981,843827
0,69306 2983,498675
0,70726 2985,329779
0,71981 2986,847539
0,73316 2988,335923
0,74642 2990,137651
0,75977 2991,851251
0,77307 2993,662771
0,78638 2995,885555
0,79978 2998,470643
0,81308 3001,143859
0,82643 3003,885619
0,84063 3006,666547
0,85314 3009,153715
0,86649 3011,445043
0,87975 3013,569907
0,89305 3015,214963
0,90645 3016,448755
0,9198 3017,222323
0,93315 3018,377779
0,94646 3019,797619
0,95976 3021,256627
0,97307 3023,254195
0,98642 3025,868659
1,00647 3029,971507
1,01982 3032,644723
1,03308 3035,552947
1,04643 3038,490547
1,09975 3050,426995
1,1131 3052,365811
1,12645 3053,785651
1,14646 3054,980275
1,15976 3056,008435
1,17307 3057,467443
1,18642 3059,171251
1,19977 3060,855475
1,21312 3062,206771
1,22638 3063,430771
1,23973 3064,370803
1,25393 3065,144371
1,26639 3066,113779
1,27974 3067,367155
1,29314 3068,953459
1,30649 3070,990195
1,31979 3073,506739
1,3331 3076,150579
1,3464 3078,862963
1,35966 3081,898483
1,37311 3084,963379
1,38646 3087,989107
1,39977 3090,867955
1,41982 3095,000179
1,43307 3097,546099
1,44638 3099,935347
1,45973 3102,324595
1,47313 3104,518003
1,48648 3105,967219
1,49979 3106,642867
1,497895 3120,476832
1,506652 3125,36376
1,515962 3130,72488
1,525307 3136,13496
1,534624 3141,9
1,543934 3147,5916
1,553314 3153,11184
1,562631 3158,2404
1,571941 3163,16088
1,581258 3167,83656
1,591198 3172,5612
1,599948 3176,6004
1,609328 3180,59064
1,618638 3184,74
1,62799 3188,7792
1,6373 3192,97752
1,646645 3197,09016
1,655927 3201,28848
1,665272 3205,19304
1,674617 3209,23224
1,684564 3213,22248
1,693314 3216,41712
1,702624 3219,91776
1,711941 3223,46736
1,721251 3227,26176
1,730631 3231,05616
1,739976 3235,09536
1,749328 3238,92648
1,758673 3242,67192
1,768543 3246,50304
1,777265 3250,05264
1,786575 3253,88376
1,79599 3257,60472
1,805307 3261,10536
1,814652 3264,5448
1,823962 3267,8496
1,833279 3271,19112
1,842624 3274,9488
1,851934 3278,81664
1,861874 3282,77016
1,875321 3288,05784
1,884631 3291,57072
1,893976 3294,91224
1,903293 3298,10688
1,912603 3301,5708
1,921955 3305,04696
1,9313 3308,47416
1,940645 3312,02376
1,950025 3315,12048
1,963997 3319,9308
1,973272 3323,79864
1,982617 3327,78888
1,991969 3331,66896
2,001279 3335,09616
2,010624 3338,76816
2,020004 3341,91384
2,029321 3344,86368
2,038631 3347,92368
2,047913 3351,19176
2,061948 3356,14896
2,071293 3359,57616
2,080645 3363,08904
2,08999 3366,84672
2,0993 3370,55544
2,108582 3374,34984
2,117927 3378,02184
2,127244 3381,52248
2,137212 3384,87624
2,145969 3387,9852
2,155314 3391,0452
2,164624 3393,90936
2,173941 3396,82248
2,183286 3400,07832
2,192631 3403,4688
2,201948 3406,85928
2,211328 3410,49456
2,220673 3414,00744
2,229983 3417,31224
2,243955 3422,23272
2,2533 3425,41512
2,262617 3428,32824
2,271962 3430,8252
2,281272 3433,44456
2,290652 3436,18632
2,299969 3438,97704
2,309279 3442,00032
2,319191 3445,62336
2,327976 3448,78128
2,337321 3452,0004
2,346666 3455,26848
2,355948 3458,41416
2,365293 3461,32728
2,374603 3464,35056
2,383955 3467,45952
2,3933 3470,4828
2,402645 3472,97976
2,41199 3475,52568
2,421307 3478,18176
2,430617 3481,0092
2,439962 3484,39968
2,449279 3488,14512
2,458624 3491,62128
2,467969 3494,76696
2,477349 3497,54544
2,487226 3500,49528
2,495941 3503,11464
2,505293 3505,69728
2,514568 3508,47576
2,523948 3511,1808
2,533293 3513,80016
2,542645 3516,46848
2,551955 3519,49176
2,5613 3522,51504
2,570617 3525,46488
2,580557 3528,57384
2,589307 3531,3156
2,598624 3534,17976
2,607969 3536,8848
2,617349 3539,50416
2,626631 3542,03784
2,635941 3544,54704
2,645251 3547,20312
2,654568 3550,03056
2,663983 3552,7356
2,673328 3555,44064
2,687328 3559,51656
2,696645 3562,13592
2,705955 3564,63288
2,7153 3567,25224
2,724617 3569,83488
2,733962 3572,2584
2,743307 3574,75536
2,752624 3577,42368
2,761969 3580,00632
2,775941 3583,83744
2,785286 3586,66488
2,794631 3589,56576
2,803976 3592,11168
2,813321 3594,44952
2,822638 3596,82408
2,831948 3599,37
2,841265 3601,83024
2,851205 3604,37616
2,859955 3606,78744
2,869335 3609,21096
2,878652 3611,38968
2,887962 3613,446
2,897279 3615,99192
2,906589 3618,7704
2,915934 3621,47544
2,925279 3624,05808
2,934631 3626,43264
2,944571 3628,78272
2,953321 3630,71664
2,962631 3633,05448
2,971913 3635,39232
2,981293 3637,57104
2,990638 3639,74976
2,999983 3642,05088
3,0093 3644,352
3,018645 3646,76328
3,027962 3649,39488
3,037832 3652,29576
3,046617 3654,59688
3,055962 3657,09384
3,065279 3659,39496
3,074624 3661,37784
3,084004 3663,06696
3,093321 3664,92744
3,102596 3667,02048
3,111941 3669,3216
3,121293 3671,85528
3,131233 3674,71944
3,139983 3677,10624
3,1493 3679,44408
3,158645 3681,70848
3,167927 3684,04632
3,1773 3686,22504
3,186617 3688,24464
3,195927 3690,252
3,205307 3692,1492
3,214624 3694,00968
3,224564 3696,15168
3,233251 3698,208
3,242596 3700,74168
3,251941 3703,12848
3,261286 3705,58872
3,270666 3707,96328
3,279983 3710,142
3,289328 3712,12488
3,29861 3714,34032
3,30792 3716,51904
3,317895 3718,57536
3,326645 3720,26448
3,33599 3722,00256
3,345307 3723,70392
3,354652 3725,47872
3,363969 3727,65744
3,373279 3730,1544
3,382624 3732,61464
3,391969 3734,87904
3,401321 3736,97208
3,410631 3738,86928
3,424638 3741,73344
3,433948 3743,8632
3,443293 3745,84608
3,452638 3747,62088
3,461955 3749,39568
3,4713 3751,17048
3,480645 3753,06768
3,489962 3755,16072
3,499272 3757,49856
3,508617 3759,64056
3,522624 3762,468
3,531969 3764,52432
3,541314 3766,45824
3,550659 3768,31872
3,559941 3770,09352
3,569258 3771,90504
3,578603 3773,67984
3,587948 3775,29552
3,597923 3777,10704
3,606638 3779,12664
3,61599 3781,06056
3,625265 3783,11688
3,634617 3785,22216
3,643962 3787,15608
3,653307 3789,01656
3,662617 3790,66896
3,671969 3792,2724
3,681279 3793,88808
3,690624 3795,54048
3,704631 3798,36792
3,713941 3800,30184
3,723356 3802,03992
3,732638 3803,97384
3,741983 3805,83432
3,751293 3807,3276
3,76061 3809,01672
3,769955 3810,91392
3,77986 3813,0192
3,788617 3814,5492
3,79799 3816,16488
3,807307 3817,7316
3,816617 3819,22488
3,825934 3820,63248
3,835279 3822,57864
3,844624 3824,7084
3,854004 3826,53216
3,863321 3828,1356
3,873226 3830,08176
3,881948 3831,61176
3,891258 3833,01936
3,900638 3834,72072
3,909955 3836,37312
3,9193 3838,02552
3,928645 3839,43312
3,93799 3841,01208
3,947335 3842,62776
3,956617 3844,40256
3,970624 3847,14432
3,979969 3848,91912
3,989279 3850,57152
3,998659 3852,0648
4,007976 3853,5948
4,017286 3855,08808
4,026631 3856,29984
4,035948 3857,7564
4,045258 3859,2864
4,059293 3861,74664
4,068673 3863,31336
4,07799 3864,89232
4,087265 3866,6304
4,096617 3868,2828
4,105927 3870,00864
4,115272 3871,55088
4,124624 3872,88504
4,133997 3874,13352
4,143314 3875,54112
4,152624 3876,99768
4,161941 3878,24616
4,171286 3879,86184
4,180631 3881,63664
4,189976 3883,32576
4,199293 3885,06384
4,208638 3886,80192
4,217955 3888,40536
4,23192 3890,58408
4,2413 3891,88152
4,250645 3893,0076
4,259962 3894,13368
4,269307 3895,30872
4,278624 3896,48376
4,287969 3897,85464
4,297279 3899,42136
4,307219 3901,35528
4,315941 3903,17904
4,325321 3904,70904
4,334666 3906,11664
4,343948 3907,53648
4,353293 3908,94408
4,362603 3910,40064
4,371955 3911,56344
4,3813 3912,70176
4,390645 3913,77888
4,40055 3914,86824
4,409307 3916,12896
4,418617 3917,57328
4,427962 3919,02984
4,437314 3920,43744
4,446659 3922,21224
4,455969 3923,95032
4,465314 3925,28448
4,474631 3926,53296
4,483941 3927,78144
4,493258 3928,90752
4,507293 3930,73128
4,516638 3932,01648
4,525955 3933,26496
4,5353 3934,47672
4,544645 3935,89656
4,553962 3937,26744
4,563237 3938,76072
4,572617 3940,0092
4,581962 3941,4168
4,591314 3942,66528
4,605314 3944,12184
4,614631 3945,2112
4,623941 3946,42296
4,633286 3947,67144
4,642631 3948,91992
4,651948 3950,33976
4,661328 3951,66168
4,670673 3952,71432
4,68055 3954,04848
4,68923 3955,29696
4,69861 3956,66784
4,70799 3957,95304
4,717335 3959,09136
4,726617 3960,09504
4,735997 3960,78048
4,745279 3961,62504
4,754589 3963,11832
4,763969 3964,66056
4,773881 3966,1416
4,782631 3967,43904
4,791976 3968,80992
4,801321 3969,77688
4,810638 3970,86624
4,819948 3971,99232
4,829265 3973,08168
4,838645 3974,04864
4,847955 3975,10128
4,857895 3976,34976
4,866617 3977,56152
4,875962 3978,61416
4,885279 3979,94832
4,894624 3981,23352
4,903969 3982,44528
4,913279 3983,62032
4,922631 3984,58728
4,931941 3985,62768
4,941286 3986,6436
4,951198 3987,73296
4,959948 3988,73664
4,969293 3989,78928
4,978638 3990,79296
4,987983 3991,8456
4,9973 3993,0084
5,00661 3994,09776
5,015927 3995,35848
5,025272 3996,60696
5,034617 3997,69632
5,044557 3998,66328
5,053314 3999,50784
5,062624 4000,40136
5,071969 4001,28264
5,081286 4002,372
5,090596 4003,66944
5,099976 4004,67312
5,109321 4005,72576
5,118673 4006,76616
5,127983 4008,13704
5,137328 4009,2264
5,146645 4010,31576
5,160645 4012,21296
5,169962 4013,17992
5,179307 4013,95104
5,188652 4014,6732
5,197969 4015,44432
5,207314 4016,28888
5,216624 4017,21912
5,225969 4018,39416
5,235286 4019,59368
5,244666 4020,76872
5,258638 4022,58024
5,267948 4023,6696
5,277293 4024,72224
5,28661 4025,6892
5,295955 4026,61944
5,3053 4027,26816
5,31468 4027,90464
5,32399 4028,71248
5,333307 4029,55704
5,342589 4030,32816
5,356589 4031,98056
5,365969 4033,31472
5,375314 4034,48976
5,384659 4035,57912
5,393976 4036,74192
5,403286 4037,63544
5,412568 4038,2352
5,421948 4039,16544
5,431888 4040,1324
5,440638 4040,65872
5,44999 4041,18504
5,459265 4042,10304
5,468645 4043,11896
5,477927 4044,1716
5,487272 4045,26096
5,496617 4046,46048
5,505969 4047,54984
5,515314 4048,40664
5,525219 4049,496
5,533969 4050,30384
5,543216 4051,06272
5,552596 4051,67472
5,561948 4052,51928
5,571328 4053,168
5,580638 4053,85344
5,590018 4054,698
5,5993 4055,70168
5,609205 4056,87672
5,617927 4058,0028
5,627307 4058,96976
5,636617 4059,9
5,645997 4060,49976
5,655342 4061,14848
5,664624 4061,76048
5,673934 4062,48264
5,683251 4063,3272
5,692631 4064,45328
5,701976 4065,5916
5,711321 4066,35048
5,725328 4067,56224
5,734638 4068,45576
5,74392 4069,37376
5,753265 4070,304
5,76261 4071,07512
5,771955 4071,84624
5,781335 4072,40928
5,790652 4072,92336
5,799934 4073,69448
5,809244 4074,66144
5,823279 4076,118
5,832624 4077,12168
5,842004 4078,0152
5,851321 4078,85976
5,860666 4079,58192
5,869948 4080,23064
5,879258 4080,9528
5,888638 4081,6872
5,897955 4082,56848
5,911955 4083,74352
5,921335 4084,30656
5,930645 4084,992
5,939927 4085,95896
5,949244 4087,0116
5,958624 4087,69704
5,967969 4088,5416
5,977314 4089,38616
5,987219 4090,15728
5,995941 4090,6836
6,005251 4091,40576
6,014603 4092,05448
6,023976 4092,69096
6,033293 4093,3764
6,042673 4094,26992
6,051983 4095,20016
6,0613 4096,08144
6,07117 4097,1708
6,079927 4098,01536
6,089307 4098,8232
6,098652 4099,4352
6,107962 4099,91256
6,117314 4100,52456
6,126659 4101,00192
6,135934 4101,65064
6,145286 4102,26264
6,154596 4102,94808
6,163976 4103,79264
6,177983 4105,32264
6,187293 4106,21616
6,196603 4107,01176
6,205955 4107,99096
6,215265 4108,74984
6,224645 4109,23944
6,233955 4109,68008
6,243307 4110,084
6,252652 4110,36552
6,266624 4110,92856
6,275934 4111,7364
6,285279 4112,86248
6,294659 4113,91512
6,304004 4114,96776
6,313286 4115,97144
6,322631 4116,6936
6,331983 4117,21992
6,34186 4117,90536
6,35061 4118,30928
6,35992 4118,99472
6,3693 4119,4476
6,37861 4119,80256
6,38799 4120,32888
6,3973 4120,89192
6,40661 4121,82216
6,41599 4122,78912
6,4253 4123,84176
6,43468 4124,5272
6,44868 4125,61656
6,45799 4126,09392
6,46723 4126,49784
6,47654 4127,23224
6,48592 4127,79528
6,4953 4128,27264
6,50468 4128,72552
6,51399 4129,44768
6,5233 4130,13312
6,53261 4130,73288
6,54661 4132,10376
6,55599 4132,9116
6,56537 4133,5236
6,57461 4133,92752
6,58392 4134,33144
6,5933 4134,77208
6,60261 4135,37184
6,61192 4136,02056
6,6213 4136,5836
6,6353 4137,55056
6,64468 4138,32168
6,65399 4138,92144
6,66323 4139,41104
6,67261 4140,1332
6,68199 4140,90432
6,6913 4141,43064
6,70061 4141,99368
6,71055 4142,36088
6,7193 4142,83824
6,72861 4143,36456
6,73799 4144,05
6,7473 4144,89456
6,75661 4145,62896
6,76592 4146,10632
6,7753 4146,66936
6,78461 4147,15896
6,79392 4147,75872
6,8033 4148,32176
6,81261 4149,09288
6,82661 4149,90072
6,83599 4150,34136
6,84537 4150,95336
6,85461 4151,67552
6,86392 4152,36096
6,8733 4152,96072
6,88261 4153,73184
6,89199 4154,17248
6,90137 4154,454
6,91061 4155,10272
6,92461 4155,82488
6,93392 4156,27776
6,9433 4156,7184
6,95261 4157,36712
6,96199 4157,80776
6,9713 4158,17496
6,98061 4158,8604
6,98999 4159,61928
6,99986 4160,3904
7,00868 4160,95344
7,01799 4161,60216
7,0273 4162,07952
7,03661 4162,36104
7,04599 4162,64256
7,0553 4162,92408
7,06461 4163,25456
7,07399 4163,73192
7,0833 4164,53976
7,09261 4165,15176
7,10199 4165,5924
7,11599 4166,59608
7,12523 4167,4896
7,13461 4168,21176
7,14399 4168,7748
7,1533 4169,142
7,16268 4169,42352
7,17199 4169,70504
7,1813 4169,86416
7,19117 4170,10896
7,19992 4170,63528
7,2093 4171,23504
7,21868 4172,00616
7,22799 4172,77728
7,2373 4173,46272
7,24668 4174,0992
7,25592 4174,55208
7,26523 4174,956
7,27461 4175,27424
7,28455 4175,71488
7,2933 4176,04536
7,30268 4176,2412
7,31199 4176,44928
7,3213 4176,8532
7,33061 4177,33056
7,33992 4177,94256
7,3493 4178,66472
7,35868 4179,38688
7,36799 4180,0356
7,3773 4180,31712
7,39123 4180,75776
7,40061 4181,08824
7,40992 4181,49216
7,4193 4181,8104
7,42868 4182,21432
7,43799 4182,58152
7,4473 4182,86304
7,45661 4183,4628
7,46592 4184,23392
7,47999 4185,2376
7,4893 4185,92304
7,49868 4186,36368
7,50799 4186,60848
7,5173 4186,81656
7,52661 4187,22048
7,53592 4187,502
7,5453 4187,69784
7,55524 4188,06504
7,56392 4188,50568
7,5733 4188,87288
7,58261 4189,35024
7,59192 4189,87656
7,6013 4190,64768
7,61068 4191,12504
7,61999 4191,45552
7,62937 4191,77376
7,63861 4192,01856
7,64799 4192,01856
7,6573 4192,30008
7,6713 4193,05896
7,68061 4193,54856
7,68999 4194,03816
7,6993 4194,44208
7,70868 4194,67464
7,71792 4195,07856
7,72723 4195,72728
7,73661 4196,2536
7,74599 4196,73096
7,7553 4197,13488
7,7693 4197,37968
7,77868 4197,4164
7,78799 4197,6612
7,79723 4198,3956
7,80661 4199,03208
7,81599 4199,436
7,82537 4199,96232
7,83468 4200,40296
7,84455 4200,68448
7,8533 4201,01496
7,86261 4201,36992
7,87192 4201,578
7,8813 4201,7004
7,89061 4201,85952
7,89999 4202,22672
7,9093 4202,54496
7,91868 4202,94888
7,92792 4203,5976
7,9373 4204,28304
7,94661 4204,68696
7,96061 4205,0052
7,96999 4205,25
7,9793 4205,40912
7,98861 4205,60496
7,99792 4205,89872
8,0073 4206,13128
8,01661 4206,4128
8,02599 4206,58416
8,03586 4207,18392
8,04461 4207,74696
8,05399 4208,31
8,0633 4208,87304
8,07261 4209,39936
8,08199 4209,52176
8,0913 4209,60744
8,10061 4209,68088
8,10999 4209,60744
8,1193 4209,80328
8,12917 4210,29288
8,13792 4210,64784
8,1473 4211,05176
8,15661 4211,50464
8,16599 4212,26352
8,1753 4212,78984
8,18461 4212,99792
8,19392 4213,15704
8,2033 4213,47528
8,21324 4213,7568
8,22199 4213,7568
8,2313 4213,84248
8,24061 4213,84248
8,24992 4213,96488
8,2593 4214,2464
8,26861 4214,93184
8,27799 4215,61728
8,2873 4216,05792
8,29668 4216,37616
8,31061 4216,82904
8,31992 4217,11056
8,3293 4217,35536
8,33861 4217,51448
8,34792 4217,4288
8,3573 4217,5512
8,36668 4217,58792
8,37599 4217,83272
8,3853 4218,19992
8,39461 4218,9588
8,40392 4219,60752
8,41799 4220,17056
8,4273 4220,29296
8,43661 4220,53776
8,44599 4220,66016
8,4553 4220,66016
8,46461 4220,90496
8,47392 4221,18648
8,4833 4221,18648
8,49268 4221,38232
8,50199 4221,78624
8,5113 4222,22688
8,52061 4222,67976
8,52999 4222,998
8,5393 4223,3652
8,54868 4223,68344
8,55799 4223,72016
8,5673 4223,92824
8,57661 4223,96496
8,58592 4224,12408
8,59586 4224,24648
8,60461 4224,49128
8,61868 4224,8952
8,62799 4225,17672
8,6373 4225,42152
8,64661 4225,94784
8,65592 4226,26608
8,6653 4226,5476
8,67461 4226,95152
8,68392 4227,07392
8,6933 4227,07392
8,70268 4227,14736
8,71255 4227,47784
8,7213 4227,60024
8,73537 4228,0776
8,74468 4228,48152
8,75399 4228,64064
8,7633 4228,6896
8,77261 4228,812
8,78192 4229,13024
8,79123 4229,6076
8,80054 4229,81568
8,81055 4230,01152
8,8193 4230,0972
8,82868 4230,18288
8,83799 4230,18288
8,8473 4230,342
8,85661 4230,74592
8,86599 4231,02744
8,8753 4231,3824
8,88461 4231,59048
8,89399 4231,59048
8,9033 4231,78632
8,91261 4231,90872
8,92192 4232,08008
8,9313 4232,1168
8,9453 4232,39832
8,95461 4232,64312
8,96392 4232,80224
8,9733 4233,12048
8,98261 4233,56112
8,99199 4233,80592
9,0013 4233,85488
9,01061 4233,92832
9,02055 4234,08744
9,0293 4234,17312
9,03861 4234,20984
9,04792 4234,45464
9,0573 4234,73616
9,06668 4234,89528
9,07599 4235,14008
9,0853 4235,4216
9,09461 4235,50728
9,10399 4235,70312
9,11386 4236,26616
9,12261 4236,79248
9,13199 4237,11072
9,1413 4237,074
9,15061 4237,00056
9,15999 4236,91488
9,1693 4236,75576
9,17861 4236,91488
9,18792 4237,11072
9,1973 4237,3188
9,20668 4237,75944
9,21655 4238,20008
9,2253 4238,37144
9,23461 4238,40816
9,24392 4238,604
9,25323 4238,8488
9,26261 4239,05688
9,27199 4238,89776
9,2813 4238,81208
9,29061 4238,77536
9,29999 4238,68968
9,3093 4238,77536
9,31861 4239,17928
9,33261 4239,86472
9,34199 4240,30536
9,3513 4240,58688
9,36061 4240,746
9,36992 4240,78272
9,3793 4240,8684
9,38861 4240,78272
9,39792 4240,70928
9,4073 4240,5012
9,41724 4240,46448
9,42599 4240,70928
9,4353 4240,9908
9,44461 4241,43144
9,45399 4241,95776
9,4633 4242,32496
9,47268 4242,44736
9,48199 4242,44736
9,4913 4242,48408
9,50061 4242,3984
9,50992 4242,3984
9,51986 4242,5208
9,52861 4242,55752
9,54268 4242,3984
9,55199 4242,55752
9,5613 4242,7656
9,57061 4242,96144
9,57992 4243,45104
9,5893 4244,01408
9,59868 4244,09976
9,60799 4244,09976
9,6173 4243,97736
9,62668 4243,89168
9,63599 4243,9284
9,6453 4243,9284
9,65461 4244,09976
9,66399 4244,418
9,6733 4244,6628
9,68268 4244,73624
9,69192 4244,9076
9,7013 4245,06672
9,71054 4245,34824
9,72055 4245,54408
9,7293 4245,66648
9,73861 4245,66648
9,74799 4245,50736
9,7573 4245,31152
9,76661 4245,31152
9,77599 4245,47064
9,7853 4246,03368
9,79461 4246,51104
9,80392 4246,63344
9,81337 4246,51104
9,8273 4246,75584
9,83661 4246,87824
9,84592 4246,79256
9,8553 4246,79256
9,86461 4246,71912
9,87399 4246,71912
9,8833 4246,59672
9,89261 4246,96392
9,90199 4247,12304
9,91186 4247,44128
9,92061 4247,68608
9,92999 4247,80848
9,93937 4247,80848
9,94861 4247,68608
9,95792 4247,77176
9,9673 4247,8452
9,97661 4247,88192
9,98592 4247,88192
9,9953 4247,93088
10,00468 4248,05328
10,01399 4248,05328
10,02393 4248,37152
10,03261 4248,68976
10,04199 4248,89784
10,0513 4249,05696
10,06061 4249,09368
10,06999 4248,97128
10,0793 4248,81216
10,08868 4248,77544
10,09792 4248,77544
10,10723 4248,86112
10,11668 4248,81216
10,12599 4249,09368
10,1353 4249,33848
10,14461 4249,54656
10,15399 4249,58328
10,16792 4249,7424
10,17723 4249,8648
10,18661 4249,95048
10,19599 4249,90152
10,2053 4249,65672
10,21468 4249,42416
10,22399 4249,26504
10,2333 4249,17936
10,24261 4249,26504
10,25255 4249,46088
10,2613 4249,95048
10,27068 4250,26872
10,27999 4250,46456
10,2893 4250,58696
10,29861 4250,58696
10,30792 4250,55024
10,3173 4250,58696
10,32668 4250,55024
10,33592 4250,39112
10,3453 4250,02392
10,35461 4249,77912
10,36392 4249,90152
10,3733 4250,42784
10,3873 4250,99088
10,39668 4251,07656
10,40599 4251,03984
10,4153 4250,91744
10,42461 4250,67264
10,43392 4250,63592
10,44323 4250,67264
10,45261 4250,83176
10,46192 4250,79504
10,4713 4250,67264
10,48061 4250,51352
10,48992 4250,58696
10,50392 4251,07656
10,5133 4251,23568
10,52268 4251,23568
10,53199 4251,23568
10,5413 4251,11328
10,55068 4251,03984
10,55999 4251,03984
10,56923 4251,15
10,57861 4251,03984
10,58792 4250,91744
10,59786 4250,91744
10,60668 4251,07656
10,62061 4251,07656
10,62992 4251,19896
10,63923 4251,48048
10,64861 4251,6396
10,65799 4251,60288
10,6673 4251,5172
10,67668 4251,48048
10,68599 4251,32136
10,6953 4251,07656
10,70461 4251,11328
10,71399 4251,19896
10,7233 4251,19896
10,7373 4251,3948
10,74661 4251,35808
10,75599 4251,23568
10,76523 4251,23568
10,77461 4251,3948
10,78399 4251,44376
10,7933 4251,35808
10,80261 4251,23568
10,81199 4251,03984
10,8213 4250,86848
10,83061 4250,83176
10,83999 4251,07656
10,8493 4251,19896
10,85924 4251,19896
10,86862 4251,2724
10,8773 4251,2724
10,88668 4251,11328
10,89592 4250,95416
10,90523 4250,95416
10,91461 4250,86848
10,92392 4250,70936
10,93337 4250,58696
10,94261 4250,63592
10,95199 4250,70936
10,96186 4250,70936
10,9753 4250,83176
10,98461 4250,91744
10,99399 4250,86848
11,0033 4250,79504
11,01261 4250,63592
11,02192 4250,42784
11,0313 4250,232
11,04061 4250,14632
11,04992 4250,02392
11,0593 4249,95048
11,06924 4249,95048
11,07792 4250,06064
11,09192 4250,232
11,10123 4250,14632
11,11061 4250,3544
11,11999 4250,26872
11,12937 4250,06064
11,13868 4249,77912
11,14792 4249,26504
11,1573 4248,93456
11,16661 4248,97128
11,17592 4249,21608
11,1853 4249,3752
11,19524 4249,33848
11,20399 4249,58328
11,2133 4249,70568
11,22261 4249,58328
11,23192 4249,42416
11,2413 4249,42416
11,25061 4249,26504
11,25999 4248,86112
11,2693 4248,4572
11,27861 4248,09
11,28792 4247,93088
11,29786 4247,93088
11,30668 4248,17568
11,31599 4248,40824
11,3253 4248,49392
11,33468 4248,4572
11,34399 4248,24912
11,3533 4248,28584
11,36261 4247,9676
11,37192 4247,93088
11,38123 4247,80848
11,39068 4247,52696
11,39999 4247,08632
11,4093 4246,79256
11,41861 4246,63344
11,43261 4246,75584
11,44192 4246,96392
11,4513 4247,12304
11,46068 4246,96392
11,46999 4246,75584
11,4793 4246,63344
11,48861 4246,4376
11,49792 4246,15608
11,5073 4246,03368
11,51661 4245,91128
11,52599 4245,54408
11,53593 4245,26256
11,54461 4245,22584
11,55392 4245,34824
11,56323 4245,59304
11,57261 4245,50736
11,58199 4245,47064
11,5913 4245,34824
11,60068 4245,18912
11,60999 4245,01776
11,6193 4244,82192
11,62854 4244,62608
11,63799 4244,418
11,64793 4244,22216
11,65668 4244,0508
11,67068 4243,89168
11,67999 4243,85496
11,68923 4243,9284
11,69861 4243,85496
11,70792 4243,61016
11,7173 4243,32864
11,72661 4243,16952
11,73599 4242,72888
11,7453 4242,36168
11,75461 4242,276
11,76392 4242,1536
11,77799 4242,32496
11,7873 4242,5208
11,79668 4242,3984
11,80599 4242,08016
11,81523 4241,87208
11,82461 4241,63952
11,83392 4241,358
11,8433 4240,90512
11,85261 4240,78272
11,86199 4240,6236
11,87592 4240,26864
11,8853 4240,21968
11,89461 4240,14624
11,90399 4240,06056
11,9133 4239,97488
11,92261 4239,77904
11,93199 4239,53424
11,9413 4239,13032
11,95061 4238,89776
11,96055 4238,7264
11,9693 4238,53056
11,97861 4238,20008
11,98792 4237,91856
11,9973 4237,84512
12,00661 4237,75944
12,01592 4237,84512
12,02523 4238,04096
12,03461 4238,00424
12,04399 4237,63704
12,05337 4237,1964
12,06268 4236,71904
12,07199 4236,22944
12,08592 4235,94792
12,0953 4235,94792
12,10461 4235,7888
12,11399 4235,58072
12,1233 4235,62968
12,13261 4235,544
12,14192 4235,26248
12,15123 4235,0544
12,16061 4234,85856
12,17055 4234,69944
12,1793 4234,33224
12,18868 4233,96504
12,19799 4233,5244
12,20737 4232,99808
12,21661 4232,71656
12,22592 4232,52072
12,2353 4232,59416
12,24468 4232,43504
12,25399 4232,52072
12,2633 4232,43504
12,27261 4232,1168
12,28192 4231,83528
12,29186 4231,6272
12,30061 4231,27224
12,30999 4230,86832
12,3193 4230,41544
12,32861 4230,01152
12,33799 4229,6076
12,3473 4229,28936
12,35661 4229,28936
12,36592 4229,16696
12,3753 4229,09352
12,38468 4228,88544
12,39399 4228,84872
12,4033 4228,6896
12,41261 4228,28568
12,42192 4228,00416
12,4313 4227,88176
12,44061 4227,35544
12,44999 4226,86584
12,4593 4226,51088
12,46861 4226,22936
12,47792 4226,058
12,4873 4225,98456
12,50137 4225,73976
12,51068 4225,45824
12,51999 4224,96864
12,5293 4224,6504
12,53861 4224,20976
12,54792 4223,87928
12,5573 4223,59776
12,56661 4223,27952
12,57648 4222,91232
12,5853 4222,55736
12,59468 4222,31256
12,60399 4221,90864
12,61323 4221,70056
12,62261 4221,5904
12,63192 4221,38232
12,6413 4221,1008
12,65068 4220,7336
12,65999 4220,29296
12,6693 4219,76664
12,67861 4219,28928
12,68799 4219,00776
12,6973 4218,92208
12,70661 4218,92208
12,71599 4218,60384
12,7253 4218,19992
12,73461 4217,796
12,74392 4217,51448
12,75386 4217,184
12,76261 4216,98816
12,77192 4216,54752
12,7813 4215,97224
12,79061 4215,45816
12,79999 4215,09096
12,8093 4214,72376
12,81861 4214,3688
12,82799 4214,124
12,8373 4213,8792
12,84668 4213,59768
12,85599 4213,10808
12,86999 4212,3492
12,87923 4211,8596
12,88861 4211,50464
12,89799 4211,17416
12,9073 4210,64784
12,91661 4210,17048
12,92592 4209,68088
12,9353 4209,32592
12,94461 4209,27696
12,95861 4208,922
12,96799 4208,39568
12,9773 4207,78368
12,98668 4207,18392
12,99592 4206,6576
13,0053 4206,25368
13,01461 4205,84976
13,02399 4205,32344
13,03386 4204,80936
13,04261 4204,56456
13,05199 4204,16064
13,0613 4203,91584
13,07061 4203,5976
13,07992 4203,19368
13,08923 4202,82648
13,09861 4202,30016
13,10799 4201,65144
13,11737 4200,89256
13,12668 4200,15816
13,13592 4199,64408
13,14992 4198,95864
13,1593 4198,3956
13,16868 4198,10184
13,17799 4197,86928
13,1873 4197,50208
13,19661 4197,06144
13,20592 4196,73096
13,21523 4196,16792
13,22461 4195,5192
13,23392 4194,79704
13,24792 4193,7444
13,2573 4192,89984
13,26661 4192,17768
13,27592 4191,68808
13,2853 4191,40656
13,29468 4191,0516
13,30399 4190,8068
13,3133 4190,36616
13,32261 4189,83984
13,33192 4189,22784
13,34592 4188,38328
13,3553 4187,66112
13,36461 4186,81656
13,37399 4185,88632
13,3833 4185,2376
13,39261 4184,51544
13,40192 4184,06256
13,4113 4183,54848
13,42124 4183,0956
13,42999 4182,53256
13,4393 4181,96952
13,44868 4181,36976
13,45799 4180,51296
13,4673 4179,59496
13,47661 4178,82384
13,48592 4178,016
13,4953 4177,20816
13,50468 4176,55944
13,51399 4175,95968
13,52799 4175,02944
13,5373 4174,55208
13,54661 4173,98904
13,55592 4173,3036
13,5653 4172,53248
13,57468 4171,76136
13,58399 4170,72096
13,5933 4169,79072
13,60317 4168,73808
13,61192 4168,08936
13,6213 4167,45288
13,63068 4166,64504
13,63992 4165,8372
13,6493 4165,06608
13,65861 4164,22152
13,66792 4163,37696
13,6773 4162,64256
13,68661 4161,83472
13,69655 4160,79432
13,7053 4159,86408
13,71468 4158,89712
13,72399 4157,8812
13,7333 4156,87752
13,74261 4156,03296
13,75192 4155,3108
13,7613 4154,29488
13,77068 4153,2912
13,77999 4152,44664
13,7893 4151,46744
13,80323 4149,69264
13,81261 4148,52984
13,82199 4147,5996
13,8313 4146,59592
13,84061 4145,50656
13,84992 4144,57632
13,8593 4143,60936
13,86861 4142,6424
13,87862 4141,62648
13,8873 4140,69624
13,89661 4139,53344
13,90592 4138,3584
13,9153 4137,35472
13,92461 4136,10624
13,93392 4134,6864
13,9433 4133,4012
13,95261 4132,18944
13,96199 4130,94096
13,97137 4129,6068
13,98068 4128,444
13,99461 4126,82832
14,00392 4125,77568
14,0133 4124,49048
14,02261 4123,27872
14,03199 4121,85888
14,04137 4120,20648
14,05068 4118,63976
14,05999 4117,1832
14,06986 4115,6532
14,07868 4114,31904
14,08799 4112,98488
14,0973 4111,69968
14,10668 4110,40224
14,11599 4109,03136
14,1253 4107,41568
14,13461 4105,88568
14,14392 4104,3924
14,1533 4102,94808
14,16317 4101,17328
14,17199 4099,47192
14,1813 4097,7828
14,19068 4095,92232
14,19992 4094,18424
14,2093 4092,61752
14,21861 4091,28336
14,22799 4089,8268
14,2373 4088,1744
14,24724 4086,3996
14,25592 4084,71048
14,26523 4082,85
14,27461 4080,9528
14,28399 4079,01888
14,2933 4076,99928
14,30261 4074,86952
14,31199 4072,76424
14,3213 4070,916
14,33061 4069,1412
14,34055 4067,31744
14,3493 4065,62832
14,35861 4063,73112
14,36799 4061,63808
14,3773 4059,4104
14,38661 4057,28064
14,39592 4055,10192
14,4053 4052,99664
14,41461 4050,86688
14,42392 4048,68816
14,43386 4046,17896
14,44261 4043,84112
14,45199 4041,71136
14,4613 4039,85088
14,47061 4037,83128
14,47999 4035,53016
14,4893 4033,19232
14,49868 4030,69536
14,50799 4028,30856
14,51786 4025,72592
14,52661 4023,63288
14,53592 4021,17264
14,5453 4018,50432
14,55461 4015,88496
14,56399 4013,42472
14,5733 4010,84208
14,58261 4008,25944
14,59192 4005,88488
14,6013 4003,26552
14,61061 4000,63392
14,62461 3996,64368
14,63399 3993,77952
14,6433 3990,91536
14,65268 3987,97776
14,66192 3985,30944
14,6713 3982,482
14,68061 3979,70352
14,68992 3976,96176
14,69993 3973,76712
14,70868 3970,90296
14,71799 3968,00208
14,7273 3965,2236
14,73661 3962,23704
14,74599 3959,12808
14,7553 3955,9824
14,76461 3952,83672
14,77392 3949,56864
14,78386 3946,05576
14,79261 3943,11816
14,80199 3939,96024
14,8113 3936,77784
14,82068 3933,59544
14,82999 3930,36408
14,8393 3927,05928
14,84868 3923,46072
14,85799 3919,9968
14,86786 3916,53288
14,87661 3913,30152
14,88592 3909,7152
14,8953 3906,0432
14,90461 3902,28552
14,91392 3898,6992
14,92323 3895,30872
14,93261 3891,8448
14,94199 3888,2952
14,9513 3884,41512
14,96068 3880,584
14,97461 3874,73328
14,98399 3870,70632
14,9933 3866,86296
15,00261 3862,99512
15,01199 3858,84576
15,0213 3854,80656
15,03061 3851,01216
15,04055 3846,81384
15,0493 3842,86032
15,05861 3838,83336
15,06792 3834,75744
15,0773 3830,48568
15,08661 3826,00584
15,09599 3821,61168
15,10537 3817,04616
15,11461 3812,33376
15,12392 3807,85392
15,13386 3803,166
15,14261 3798,93096
15,15199 3794,34096
15,1613 3789,73872
15,17061 3785,1732
15,17992 3780,4608
15,1893 3775,57704
15,19861 3770,69328
15,20799 3765,61368
15,2173 3760,41168
15,22724 3754,80576
15,23599 3749,83632
15,2453 3744,38952
15,25461 3739,11408
15,26399 3733,98552
15,2733 3728,78352
15,28268 3723,45912
15,29199 3717,9756
15,3013 3712,35744
15,31061 3706,55568
15,31992 3700,58256
15,32923 3694,536
15,3433 3685,38048
15,35261 3679,04016
15,36199 3672,46728
15,3713 3666,17592
15,38061 3659,76216
15,38992 3653,26272
15,3993 3646,84896
15,40868 3640,15368
15,41799 3633,01776
15,42737 3625,58808
15,4413 3614,41296
15,45061 3606,87312
15,45992 3599,1252
15,46923 3591,42624
15,47868 3583,11528
15,48799 3574,6452
15,4973 3566,05272
15,50661 3557,41128
15,51592 3548,50056
15,5253 3539,29608
15,5393 3525,09768
15,54861 3514,93848
15,55799 3504,48552
15,5673 3493,51848
15,57661 3482,22096
15,58592 3470,51952
15,5953 3458,37744
15,60461 3445,79472
15,61392 3432,44088
15,62799 3411,25344
15,6373 3396,45528
15,64661 3380,71464
15,65592 3363,98256
15,6653 3346,0632
15,67461 3327,1524
15,68392 3306,93192
15,6933 3285,38952
15,70324 3260,8728
15,71199 3238,03296
15,7213 3212,13312
15,73061 3184,42176
15,73999 3154,8132
15,7493 3122,78112
15,75868 3088,1664
15,76799 3050,93232
15,7773 3010,8708
15,78661 2967,43104
15,79592 2920,14792
15,80992 2840,14728
15,8193 2778,8004
15,82868 2709,41184
15,83792 2629,09296
--------------------------------------------------------------------------------
/DataManagement/1_DataManagement.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Workshop: Data Management with Python\n",
8 | "\n",
9 | "## Prerequisites\n",
10 | "### Installation:\n",
11 | "Minimum requirement is Python3.\n",
12 | "\n",
13 | "I personally do prefer the installation of the Anaconda Distribution. \n",
14 | "https://www.anaconda.com/products/individual\n",
15 | "\n",
16 | "Basically all packages you need to get started are included, as well as jupyter and spider editor. Also when installing new packages with conda\n",
17 | "```python\n",
18 | "conda install PACKAGE_NAME\n",
19 | "```\n",
20 | "the compatibility is taken care of."
21 | ]
22 | },
23 | {
24 | "cell_type": "code",
25 | "execution_count": null,
26 | "metadata": {},
27 | "outputs": [],
28 | "source": [
29 | "# in case a package is missing and you recognise when you want to import it, \n",
30 | "# you can do the installation also from within the notebook\n",
31 | "!pip install pandas\n",
32 | "#!conda install pandas"
33 | ]
34 | },
35 | {
36 | "cell_type": "code",
37 | "execution_count": null,
38 | "metadata": {},
39 | "outputs": [],
40 | "source": [
41 | "# For installation on Windows the command might need to look like:\n",
42 | "!{sys.executable} -m pip install pandas\n",
43 | "!{sys.executable} -m pip install xl..."
44 | ]
45 | },
46 | {
47 | "cell_type": "markdown",
48 | "metadata": {},
49 | "source": [
50 | "### Packages:\n",
51 | "For the data management tutorial we need packages for importing and exporting the data and for data wrangling. The most common package, which covers both is the **pandas module**. This\n",
52 | "Python open source library provides high-performance, easy-to-use data structures and data analysis tools.\n",
53 | "Another tool we need is the **numpy module**. This\n",
54 | "Python open source library makes handling of vectors, matrices and big multidimensional arrays easy.\n",
55 | "\n",
56 | "So after the installation of numpy and pandas package, you have to import them at the beginning of your python script or notebook to use their functionalities\n",
57 | "\n",
58 | "Good Practice tip: import as \"np\" and \"pd\" because it's faster to write \"pd\" than \"pandas\"\n",
59 | "```python\n",
60 | "import pandas as pd\n",
61 | "import numpy as np\n",
62 | "```\n",
63 | "\n",
64 | "### Data:\n",
65 | "At the very beginning we will look at a minimal Excel example Basics.xlsx to get to know some basic operations.\n",
66 | "\n",
67 | "Later on we will take a look at two examples of experimental data:\n",
68 | "- Tensile Test\n",
69 | "- Bubble Column\n",
70 | "\n",
71 | "### Plan B:\n",
72 | "If something went wrong with the installation or you cannot access the datafiles, you can view this notebook at nbviewer (passive) or at binder (interactive). Also you can view it directly on github.\n",
73 | "\n",
74 | "Links:\n",
75 | "- binder https://mybinder.org/v2/gh/astridwalle/python_jupyter_basics/HEAD?filepath=DataManagement%2F1_DataManagement.ipynb\n",
76 | "- nbviewer https://nbviewer.jupyter.org/github/astridwalle/python_jupyter_basics/blob/main/DataManagement/1_DataManagement.ipynb\n",
77 | "- github https://github.com/astridwalle/python_jupyter_basics\n",
78 | "\n",
79 | "## Now let's get started! What is this about? Why Python?\n",
80 | "### Separation of Data and Analysis!\n",
81 | "For example you have performed experimental analyses and now you have loads of raw data:\n",
82 | "\n",
83 | "There is actually no need to copy this data for analyzing and to store different analysis versions...\n",
84 | "\n",
85 | "Well-known szenarios are:\n",
86 | "\n",
87 | "- Multiple Excel files floating around on share drives\n",
88 | "- Lots of versions and local copies\n",
89 | "- Hard or even impossible to keep overview and trace changes\n",
90 | "- Changing the underlying data (add columns, change format, ...) can break everything\n",
91 | "\n",
92 | "\n",
93 | "### Typical Usecases for which Python has advantages over Excel, Matlab, ...\n",
94 | "For large datasets Excel becomes unhandy and slow. This is no problem for python due to the paradigm of separating the data from the analysis. If you want to try and store multiple approaches and analyses, this becomes a problem in Excel due to large file copies.\n",
95 | "With python your different analysis files stay small (just textfiles) and can also be version controlled (e.g. with git), which is great for collaboration.\n",
96 | "You don't end up with ..._final ..._final_final_\n",
97 | "\n",
98 | "With Python:\n",
99 | "- Import data and with that\n",
100 | "- Decouple Business Logic, Computations, Visualisation from the data\n",
101 | "- Access databases, xlsx tabels and more sources without changing them.\n",
102 | "- Export your analysis afterwards to every format you like (database, xlsx, ...)\n",
103 | "\n",
104 | "\n",
105 | "### Huge community of users and contributors\n",
106 | "You can google everything!"
107 | ]
108 | },
109 | {
110 | "cell_type": "markdown",
111 | "metadata": {},
112 | "source": [
113 | "## Jupyter Basics to get started:\n",
114 | "Check out the following short-cut and menu bar functionalities: \n",
115 | "- Markdown vs. Code\n",
116 | "- Coding Environment\n",
117 | "- auto-complete ```tab```\n",
118 | "- Running a cell: ```Run``` or ```Shift+Enter``` \n",
119 | "- ? in front of function call to get some help\n",
120 | "\n",
121 | "## Important for this course:\n",
122 | "The intention is to have an interactive working document, so\n",
123 | "- Add as many code cells as you like!\n",
124 | "- Try evrything!\n",
125 | "- Don't be shy, you cannot break anything!"
126 | ]
127 | },
128 | {
129 | "cell_type": "markdown",
130 | "metadata": {},
131 | "source": [
132 | "## Example files for this class\n",
133 | "Basic sample data: \n",
134 | "```\n",
135 | "Data/Basics.xlsx\n",
136 | "```\n",
137 | "\n",
138 | "Real data from testing:\n",
139 | "- Tensile Testing\n",
140 | "```\n",
141 | "Data/TensileTest/AIMg3.txt Data/TensileTest/HDPE.txt Data/TensileTest/Staht.txt\n",
142 | "```\n",
143 | "- Bubble colum\n",
144 | "```\n",
145 | "Data/BubbleColumn/Test_01.xlsx Data/BubbleColumn/Test_02.xlsx Data/BubbleColumn/Test_03.xlsx \n",
146 | "```\n",
147 | "\n",
148 | "## Importing basic libraries\n",
149 | "\n",
150 | "**pandas module**\n",
151 | "\n",
152 | "Python open source library providing high-performance, easy-to-use data structures and data analysis tools. Built-in great capabilities for data import and export! (csv, xlsx, ...)\n",
153 | "\n",
154 | "Good Practice tip: import as \"np\" and \"pd\" because it's faster to call \"pd\" than \"pandas\"\n",
155 | "\n",
156 | "**numpy module** (We don't need it today)\n",
157 | "\n",
158 | "Python open source library for easy handling of vectors, matrices, big multidimensional arrays and mathematical operations\n",
159 | "\n",
160 | "## General notes\n",
161 | "### Modules\n",
162 | "\n",
163 | "You can either import a complete module, or just single classes or functions of a module.\n",
164 | "\n",
165 | "```python\n",
166 | "import pandas as pd\n",
167 | "\n",
168 | "pd.read_csv(\"file.csv\")\n",
169 | "```\n",
170 | "\n",
171 | "```python\n",
172 | "import matplotlib.pyplot as plt\n",
173 | "\n",
174 | "plt.plot(x,y)\n",
175 | "```\n",
176 | "\n",
177 | "## Classes and functions\n",
178 | "Python is a multi-paradigm programming language. You can do object oriented programming, but also structured, sequential programming. You can create objects with classes or just use functions."
179 | ]
180 | },
181 | {
182 | "cell_type": "markdown",
183 | "metadata": {},
184 | "source": [
185 | "## Let's go!\n",
186 | "At first just one statement in a code cell to get a nice output. (This enables the print of multiple variables per coding cell.)"
187 | ]
188 | },
189 | {
190 | "cell_type": "code",
191 | "execution_count": null,
192 | "metadata": {},
193 | "outputs": [],
194 | "source": [
195 | "# make all print statements in a cell appear in output\n",
196 | "from IPython.core.interactiveshell import InteractiveShell\n",
197 | "InteractiveShell.ast_node_interactivity = \"all\""
198 | ]
199 | },
200 | {
201 | "cell_type": "markdown",
202 | "metadata": {},
203 | "source": [
204 | "## Now it is your turn!\n",
205 | "1. Import the modules\n",
206 | "2. The functions of these modules are then called with pd.XXX\n",
207 | "3. Check which functions are available and how to use them\n",
208 | "4. Try pd. tab/autocomplete to see all available options\n",
209 | "5. Try inserting a ? in front of the function name"
210 | ]
211 | },
212 | {
213 | "cell_type": "markdown",
214 | "metadata": {},
215 | "source": [
216 | "## Press ```+``` and get started :)"
217 | ]
218 | },
219 | {
220 | "cell_type": "code",
221 | "execution_count": null,
222 | "metadata": {},
223 | "outputs": [],
224 | "source": []
225 | },
226 | {
227 | "cell_type": "code",
228 | "execution_count": null,
229 | "metadata": {},
230 | "outputs": [],
231 | "source": []
232 | },
233 | {
234 | "cell_type": "code",
235 | "execution_count": null,
236 | "metadata": {},
237 | "outputs": [],
238 | "source": [
239 | "# import module\n",
240 | "import pandas as pd\n",
241 | "import numpy as np"
242 | ]
243 | },
244 | {
245 | "cell_type": "code",
246 | "execution_count": null,
247 | "metadata": {},
248 | "outputs": [],
249 | "source": []
250 | },
251 | {
252 | "cell_type": "code",
253 | "execution_count": null,
254 | "metadata": {},
255 | "outputs": [],
256 | "source": []
257 | },
258 | {
259 | "cell_type": "markdown",
260 | "metadata": {},
261 | "source": [
262 | "### Our first Testcases - Basics.xlsx / Tensile Testing\n",
263 | "If you have multiple tables, e.g. from different test runs, which you want to combine to one datasource for analyzing, that is pretty easy.\n",
264 | "\n",
265 | "We will do this for:\n",
266 | "1. One xlsx file with multiple sheets as raw data\n",
267 | "2. Multiple txt files as raw data\n",
268 | "\n",
269 | "Stitching together rowwise or columnwise is easy.\n",
270 | "\n",
271 | "### Exploratory Data Analysis\n",
272 | "We always start with an exploratory data analysis to ensure we know how our data looks like.\n",
273 | "\n",
274 | "### Our main datastructure: Pandas DataFrame\n",
275 | "For data analysis the most common module is pandas with the main tabular data structure DataFrame, which enables for endless operations."
276 | ]
277 | },
278 | {
279 | "cell_type": "code",
280 | "execution_count": null,
281 | "metadata": {},
282 | "outputs": [],
283 | "source": [
284 | "# We start with reading in an Excel File.\n",
285 | "# To check the usage of the function we can put the questionmark in front of the function.\n",
286 | "# \n",
287 | "#?pd.ExcelFile\n",
288 | "#??pd.ExcelFile"
289 | ]
290 | },
291 | {
292 | "cell_type": "code",
293 | "execution_count": null,
294 | "metadata": {},
295 | "outputs": [],
296 | "source": [
297 | "# 1. Read xlsx file - Variant 1 --> reads all the sheets in the file\n",
298 | "xls1=pd.ExcelFile(\"../Data/Basics.xlsx\")"
299 | ]
300 | },
301 | {
302 | "cell_type": "code",
303 | "execution_count": null,
304 | "metadata": {},
305 | "outputs": [],
306 | "source": [
307 | "xls1.sheet_names"
308 | ]
309 | },
310 | {
311 | "cell_type": "code",
312 | "execution_count": null,
313 | "metadata": {},
314 | "outputs": [],
315 | "source": [
316 | "# 2. Read xlsx file - Variant 2 --> reads just the first sheet!\n",
317 | "xls2=pd.read_excel(\"../Data/Basics.xlsx\")\n",
318 | "xls2"
319 | ]
320 | },
321 | {
322 | "cell_type": "code",
323 | "execution_count": null,
324 | "metadata": {},
325 | "outputs": [],
326 | "source": [
327 | "# We use Variant 1 --> Parse the sheets into dataframes \n",
328 | "df_test1=xls1.parse(\"Test1\")\n",
329 | "df_test2=xls1.parse(\"Test2\")\n",
330 | "df_test3=xls1.parse(\"Test3\")"
331 | ]
332 | },
333 | {
334 | "cell_type": "code",
335 | "execution_count": null,
336 | "metadata": {},
337 | "outputs": [],
338 | "source": [
339 | "# Check the results of the parsing on your own!\n",
340 | "# Always use autocompletion!"
341 | ]
342 | },
343 | {
344 | "cell_type": "code",
345 | "execution_count": null,
346 | "metadata": {},
347 | "outputs": [],
348 | "source": [
349 | "# show results: have a look at the resulting dataframes\n",
350 | "# --> check the additional index column, which was added automatically\n",
351 | "df_test1\n",
352 | "df_test2\n",
353 | "df_test3\n",
354 | "# show no of columns and rows with df.shape\n",
355 | "df_test1.shape\n",
356 | "df_test2.shape\n",
357 | "df_test3.shape"
358 | ]
359 | },
360 | {
361 | "cell_type": "markdown",
362 | "metadata": {},
363 | "source": [
364 | "### pd.concat - join dataframes\n",
365 | "\n",
366 | "**Task**\n",
367 | "\n",
368 | "At first we want to join the two dataframes df_test1 and df_test2 into one big DataFrame, which has \n",
369 | "1. all columns\n",
370 | "2. all rows\n",
371 | "Let's find out the caveats here...\n",
372 | "\n",
373 | "**hints**\n",
374 | "- have a look again at the dataframes:\n",
375 | " - just type the name of the df and shift+enter\n",
376 | " - have a look at number of rows and columns using df.shape\n",
377 | "- use **?** to find out how to use pd.concat\n",
378 | "- ask google"
379 | ]
380 | },
381 | {
382 | "cell_type": "code",
383 | "execution_count": null,
384 | "metadata": {},
385 | "outputs": [],
386 | "source": [
387 | "df_test1"
388 | ]
389 | },
390 | {
391 | "cell_type": "code",
392 | "execution_count": null,
393 | "metadata": {},
394 | "outputs": [],
395 | "source": [
396 | "df_test2"
397 | ]
398 | },
399 | {
400 | "cell_type": "code",
401 | "execution_count": null,
402 | "metadata": {},
403 | "outputs": [],
404 | "source": [
405 | "# 1. all columns\n",
406 | "df_all_cols=pd.concat([df_test1,df_test2],axis=1,ignore_index=False)"
407 | ]
408 | },
409 | {
410 | "cell_type": "code",
411 | "execution_count": null,
412 | "metadata": {},
413 | "outputs": [],
414 | "source": [
415 | "df_all_cols"
416 | ]
417 | },
418 | {
419 | "cell_type": "code",
420 | "execution_count": null,
421 | "metadata": {
422 | "scrolled": true
423 | },
424 | "outputs": [],
425 | "source": [
426 | "# show resulting dataframe\n",
427 | "df_all_rows=pd.concat([df_test1,df_test2],axis=0,ignore_index=True)\n",
428 | "df_all_rows"
429 | ]
430 | },
431 | {
432 | "cell_type": "markdown",
433 | "metadata": {},
434 | "source": [
435 | "### Your turn: try around with\n",
436 | "- ?pd.concat\n",
437 | "- Change the axis\n",
438 | "- Change ignore_index\n",
439 | "- Try other options with autocomplete..."
440 | ]
441 | },
442 | {
443 | "cell_type": "markdown",
444 | "metadata": {},
445 | "source": [
446 | "## Combine different testruns into one dataframe\n",
447 | "\n",
448 | "If you want to merge different \"cases\", present in different tables, e.g. each representing another test or another parameter varation. pd.concat can join them into one df, adding an additional index, a so called multiindex\n",
449 | "\n",
450 | "We will do this now with our test tables:"
451 | ]
452 | },
453 | {
454 | "cell_type": "code",
455 | "execution_count": null,
456 | "metadata": {},
457 | "outputs": [],
458 | "source": [
459 | "#df_all_Params=pd.concat([df_Param1,df_Param2,df_Param3],axis=0,ignore_index=True)\n",
460 | "df_all_Params=pd.concat([df_test1,df_test2],keys=[\"Test1\",\"Test2\"],axis=0,names=[\"Param\",\"Row_Index\"],ignore_index=False)"
461 | ]
462 | },
463 | {
464 | "cell_type": "code",
465 | "execution_count": null,
466 | "metadata": {},
467 | "outputs": [],
468 | "source": [
469 | "df_all_Params"
470 | ]
471 | },
472 | {
473 | "cell_type": "markdown",
474 | "metadata": {},
475 | "source": [
476 | "### Your turn: try out join / merge: \n",
477 | "Join and merge are also extremely interesting functionalities for dataframes. --> check it out now or later with\n",
478 | "```\n",
479 | "df_test1.join()\n",
480 | "df_test1.merge()\n",
481 | "```\n",
482 | "Ask google or add **?** to find out more. Use autocompletion!"
483 | ]
484 | },
485 | {
486 | "cell_type": "code",
487 | "execution_count": null,
488 | "metadata": {},
489 | "outputs": [],
490 | "source": []
491 | },
492 | {
493 | "cell_type": "markdown",
494 | "metadata": {},
495 | "source": [
496 | "## Now let's have a look at some real testing data\n",
497 | "```\n",
498 | "Data/TensileTest/AIMg3.txt\n",
499 | "```\n",
500 | "```\n",
501 | "Data/TensileTest/HDPE.txt\n",
502 | "```\n",
503 | "```\n",
504 | "Data/TensileTest/Stahl.tx`\n",
505 | "```\n",
506 | "\n",
507 | "### Your turn:\n",
508 | "\n",
509 | "- Read in all 3 files with the function pd.read_csv()\n",
510 | "- What are the caveats here?\n",
511 | "- Hint: read_csv can detect commas and whitespaces automatically, but here are tabs..."
512 | ]
513 | },
514 | {
515 | "cell_type": "code",
516 | "execution_count": null,
517 | "metadata": {},
518 | "outputs": [],
519 | "source": [
520 | "# 1. Read in multiple txt files\n",
521 | "# Try autocomplete- for everything! Paths, Variable names, functions, options...\n",
522 | "df_AlMg3=pd.read_csv(\"../Data/TensileTest/AlMg3.txt\", sep=\"\\t\", skiprows=0, header=None)\n",
523 | "\n",
524 | "# See the whole dataframe \n",
525 | "df_AlMg3\n",
526 | "\n",
527 | "# See the beginning or the end\n",
528 | "df_AlMg3.head(15)\n",
529 | "df_AlMg3.tail(10)\n",
530 | "# See a slice of the df\n",
531 | "df_AlMg3[5:15]"
532 | ]
533 | },
534 | {
535 | "cell_type": "code",
536 | "execution_count": null,
537 | "metadata": {},
538 | "outputs": [],
539 | "source": [
540 | "# This looks a bit messy, so let's do some data wrangling\n",
541 | "# Split it up into metadata and actual data\n",
542 | "df_AlMg3_meta=df_AlMg3[1:9]\n",
543 | "df_AlMg3_data=df_AlMg3[12::]\n",
544 | "df_AlMg3_meta\n",
545 | "df_AlMg3_data"
546 | ]
547 | },
548 | {
549 | "cell_type": "code",
550 | "execution_count": null,
551 | "metadata": {},
552 | "outputs": [],
553 | "source": [
554 | "# And some more wrangling\n",
555 | "# Set menaningful columnnames\n",
556 | "df_AlMg3_meta.columns=[\"key\",\"value\",\"unit\"]\n",
557 | "df_AlMg3_data.columns=[\"Standardweg [mm]\",\"Standardkraft [N]\",\"None\"]\n",
558 | "df_AlMg3_meta\n",
559 | "df_AlMg3_data"
560 | ]
561 | },
562 | {
563 | "cell_type": "code",
564 | "execution_count": null,
565 | "metadata": {},
566 | "outputs": [],
567 | "source": [
568 | "# and at last we will cut the last column from the data, as this is empyt anyway:\n",
569 | "df_AlMg3_data.drop(columns=[\"None\"]).reset_index()\n",
570 | "# ATTENTION! Make sure that the action was really APPLIED and not just executed!\n",
571 | "df_AlMg3_data\n",
572 | "# so either you write the df new, or set the inplace option or copy\n",
573 | "# df_AlMg3=df_AlMg3_data.drop(columns=[\"None\"]).reset_index()\n",
574 | "df_AlMg3_data=df_AlMg3_data.drop(columns=[\"None\"])\n",
575 | "df_AlMg3_data.reset_index(inplace=True)\n",
576 | "# And we drop the empty rows of the metadata df\n"
577 | ]
578 | },
579 | {
580 | "cell_type": "code",
581 | "execution_count": null,
582 | "metadata": {},
583 | "outputs": [],
584 | "source": [
585 | "df_AlMg3_data"
586 | ]
587 | },
588 | {
589 | "cell_type": "markdown",
590 | "metadata": {},
591 | "source": [
592 | "## Now that the data looks good, we can apply the transforming steps to the other datasets as well:\n",
593 | "### Option 1. Read one by one\n",
594 | "### Option 2. Read files in a loop"
595 | ]
596 | },
597 | {
598 | "cell_type": "code",
599 | "execution_count": null,
600 | "metadata": {},
601 | "outputs": [],
602 | "source": [
603 | "! ls -al ../Data/TensileTest/"
604 | ]
605 | },
606 | {
607 | "cell_type": "code",
608 | "execution_count": null,
609 | "metadata": {},
610 | "outputs": [],
611 | "source": [
612 | "import os\n",
613 | "# we can assign dynamic variable names with the loop index, but this is a bad idea. It is better \n",
614 | "# to write it into a so called dictionary. Later on we can access it by the name\n",
615 | "tests={}\n",
616 | "#for i in [\"HDPE\",\"Stahl\"]:\n",
617 | "for i in os.listdir(\"../Data/TensileTest/\"):\n",
618 | " i=i.strip(\".txt\")\n",
619 | " df=pd.read_csv(\"../Data/TensileTest/\"+i+\".txt\", sep=\"\\t\", skiprows=0, header=None)\n",
620 | " df_meta=df[1:9]\n",
621 | " df_data=df[12::]\n",
622 | " df_meta.columns=[\"key\",\"value\",\"unit\"]\n",
623 | " df_data.columns=[\"Standardweg [mm]\",\"Standardkraft [N]\",\"None\"]\n",
624 | " df_data=df_data.drop(columns=[\"None\"])\n",
625 | " df_data.reset_index(inplace=True)\n",
626 | " key_data=i+\"_data\"\n",
627 | " value_data=df_data\n",
628 | " key_meta=i+\"_meta\"\n",
629 | " value_meta=df_meta\n",
630 | " tests[key_data]=value_data\n",
631 | " tests[key_meta]=value_meta"
632 | ]
633 | },
634 | {
635 | "cell_type": "code",
636 | "execution_count": null,
637 | "metadata": {},
638 | "outputs": [],
639 | "source": [
640 | "# Now we check the keys of the dictionary\n",
641 | "tests[\"AlMg3_meta\"][\"key\"]"
642 | ]
643 | },
644 | {
645 | "cell_type": "code",
646 | "execution_count": null,
647 | "metadata": {},
648 | "outputs": [],
649 | "source": [
650 | "# And now we combine all the testing data into one dataframe, using just the data values of the dictionary.\n",
651 | "df_tensile_tests=pd.concat([tests[\"AlMg3_data\"],tests[\"Stahl_meta\"]],keys=[\"AlMg3\",\"Stahl\"],axis=0,\n",
652 | " names=[\"Material\",\"Row_Index\"],ignore_index=False)"
653 | ]
654 | },
655 | {
656 | "cell_type": "code",
657 | "execution_count": null,
658 | "metadata": {},
659 | "outputs": [],
660 | "source": [
661 | "# Let's check the result\n",
662 | "df_tensile_tests"
663 | ]
664 | },
665 | {
666 | "cell_type": "markdown",
667 | "metadata": {},
668 | "source": [
669 | "## Example 2: Data Exploration with large datafiles...\n",
670 | "We always start by looking at the data. Get to know your data\n",
671 | "\n",
672 | "**Does the data look ok?**\n",
673 | "\n",
674 | "**Does it look as we expected it to be?**\n",
675 | "\n",
676 | "In the following we will list a few commands, which are helpful for data exploration!\n",
677 | "\n",
678 | "For this example we will use some data from the bubblecolumn.\n"
679 | ]
680 | },
681 | {
682 | "cell_type": "code",
683 | "execution_count": null,
684 | "metadata": {},
685 | "outputs": [],
686 | "source": [
687 | "df_bub=pd.read_excel(\"../Data/BubbleColumn/Test_01.xlsx\")\n",
688 | "df_bub"
689 | ]
690 | },
691 | {
692 | "cell_type": "markdown",
693 | "metadata": {},
694 | "source": [
695 | "The column names to not look nice. Let's check the original fiel and adapt our function call.\n",
696 | "\n",
697 | "We need to give some extra options for reading in the files. --> Check the read_excel function with ?"
698 | ]
699 | },
700 | {
701 | "cell_type": "code",
702 | "execution_count": null,
703 | "metadata": {},
704 | "outputs": [],
705 | "source": [
706 | "?pd.read_excel"
707 | ]
708 | },
709 | {
710 | "cell_type": "code",
711 | "execution_count": null,
712 | "metadata": {},
713 | "outputs": [],
714 | "source": [
715 | "# we give the first 2 lines as header --> then a multiindex is created.\n",
716 | "df_bub=pd.read_excel(\"../Data/BubbleColumn/Test_01.xlsx\",header=[0,1])\n",
717 | "df_bub"
718 | ]
719 | },
720 | {
721 | "cell_type": "markdown",
722 | "metadata": {},
723 | "source": [
724 | "## Now we will do some data exploration\n",
725 | "- shape\n",
726 | "- describe\n",
727 | "- slicing\n",
728 | "- accessing columns / rows"
729 | ]
730 | },
731 | {
732 | "cell_type": "code",
733 | "execution_count": null,
734 | "metadata": {},
735 | "outputs": [],
736 | "source": [
737 | "df_bub.shape"
738 | ]
739 | },
740 | {
741 | "cell_type": "code",
742 | "execution_count": null,
743 | "metadata": {},
744 | "outputs": [],
745 | "source": [
746 | "df_bub.describe()"
747 | ]
748 | },
749 | {
750 | "cell_type": "code",
751 | "execution_count": null,
752 | "metadata": {},
753 | "outputs": [],
754 | "source": [
755 | "df_bub.columns"
756 | ]
757 | },
758 | {
759 | "cell_type": "code",
760 | "execution_count": null,
761 | "metadata": {},
762 | "outputs": [],
763 | "source": [
764 | "# To get the \"correct\" names for slicing and for looping\n",
765 | "df_bub.columns.values"
766 | ]
767 | },
768 | {
769 | "cell_type": "code",
770 | "execution_count": null,
771 | "metadata": {},
772 | "outputs": [],
773 | "source": [
774 | "# access specific columns --> there are multiple ways to do so.\n",
775 | "df_bub.cam0\n",
776 | "df_bub.cam0[\"Waddel Disk Diameter\"]\n",
777 | "df_bub[\"cam0\"][\"Waddel Disk Diameter\"]"
778 | ]
779 | },
780 | {
781 | "cell_type": "markdown",
782 | "metadata": {},
783 | "source": [
784 | "## Get an overview"
785 | ]
786 | },
787 | {
788 | "cell_type": "code",
789 | "execution_count": null,
790 | "metadata": {},
791 | "outputs": [],
792 | "source": [
793 | "df_bub.shape\n",
794 | "df_bub.head(5)\n",
795 | "df_bub.tail(5)\n",
796 | "df_bub.index\n",
797 | "len(df_bub)"
798 | ]
799 | },
800 | {
801 | "cell_type": "markdown",
802 | "metadata": {},
803 | "source": [
804 | "## Slicing"
805 | ]
806 | },
807 | {
808 | "cell_type": "code",
809 | "execution_count": null,
810 | "metadata": {},
811 | "outputs": [],
812 | "source": [
813 | "# 1 row\n",
814 | "df_bub[4:5]\n",
815 | "\n",
816 | "#multiple rows\n",
817 | "df_bub[10:20]\n",
818 | "\n",
819 | "#ignore last 100 rows\n",
820 | "df_bub[:-100]\n",
821 | "\n",
822 | "# 1 column\n",
823 | "df_bub[\"cam1\"][\"Bounding \\nleft\"]\n",
824 | "\n",
825 | "# multiple columns\n",
826 | "df_bub[df_bub.columns.values[:3]]"
827 | ]
828 | },
829 | {
830 | "cell_type": "markdown",
831 | "metadata": {},
832 | "source": [
833 | "## Filtering"
834 | ]
835 | },
836 | {
837 | "cell_type": "code",
838 | "execution_count": null,
839 | "metadata": {},
840 | "outputs": [],
841 | "source": [
842 | "df_bub[df_bub[\"cam1\"][\"Bounding \\nleft\"]>32]"
843 | ]
844 | },
845 | {
846 | "cell_type": "markdown",
847 | "metadata": {},
848 | "source": [
849 | "### Your task: \n",
850 | "Try filtering options with Boolean Algebra:\n",
851 | "- < / >\n",
852 | "- == / !=\n",
853 | "- multiple options combined with ```&```"
854 | ]
855 | },
856 | {
857 | "cell_type": "code",
858 | "execution_count": null,
859 | "metadata": {},
860 | "outputs": [],
861 | "source": []
862 | },
863 | {
864 | "cell_type": "markdown",
865 | "metadata": {},
866 | "source": [
867 | "## Some more filtering - groupby\n",
868 | "\n",
869 | "Filter data by categorical values\n",
870 | "\n",
871 | "Applies if you want to get single dataframes for specific groups.\n",
872 | "\n",
873 | "Example: RKI Covid Case Data - 1 row per day per Landkreis. To get all rows only for one Landkreis, you can use groupby."
874 | ]
875 | },
876 | {
877 | "cell_type": "code",
878 | "execution_count": null,
879 | "metadata": {},
880 | "outputs": [],
881 | "source": [
882 | "# you can also read the csv directly from url!\n",
883 | "df_rki=pd.read_csv(\"https://www.arcgis.com/sharing/rest/content/items/f10774f1c63e40168479a1feb6c7ca74/data\")\n",
884 | "df_rki"
885 | ]
886 | },
887 | {
888 | "cell_type": "code",
889 | "execution_count": null,
890 | "metadata": {},
891 | "outputs": [],
892 | "source": [
893 | "# Grouping works great to get separate DataFrames for different categories.\n",
894 | "df_grouped=df.groupby(\"Landkreis\")\n",
895 | "\n",
896 | "for name, dataframe in df_grouped:\n",
897 | " print(name, len(dataframe))\n"
898 | ]
899 | },
900 | {
901 | "cell_type": "markdown",
902 | "metadata": {},
903 | "source": [
904 | "## Min / Max / Mean"
905 | ]
906 | },
907 | {
908 | "cell_type": "code",
909 | "execution_count": null,
910 | "metadata": {},
911 | "outputs": [],
912 | "source": [
913 | "df_bub[\"cam1\"][\"Bounding \\nleft\"].min()\n",
914 | "df_bub[\"cam1\"][\"Bounding \\nleft\"].max()\n",
915 | "df_bub[\"cam1\"][\"Bounding \\nleft\"].mean()"
916 | ]
917 | },
918 | {
919 | "cell_type": "markdown",
920 | "metadata": {},
921 | "source": [
922 | "## Find unique values"
923 | ]
924 | },
925 | {
926 | "cell_type": "code",
927 | "execution_count": null,
928 | "metadata": {},
929 | "outputs": [],
930 | "source": [
931 | "# Makes only sense for categorical values...\n",
932 | "df_bub[\"cam1\"][\"Bounding \\nleft\"].unique()"
933 | ]
934 | },
935 | {
936 | "cell_type": "code",
937 | "execution_count": null,
938 | "metadata": {},
939 | "outputs": [],
940 | "source": [
941 | "df_rki[\"Bundesland\"].unique()"
942 | ]
943 | },
944 | {
945 | "cell_type": "markdown",
946 | "metadata": {},
947 | "source": [
948 | "## Get specific rows, columns, elements\n",
949 | "By names (loc), indices (iloc)"
950 | ]
951 | },
952 | {
953 | "cell_type": "code",
954 | "execution_count": null,
955 | "metadata": {},
956 | "outputs": [],
957 | "source": [
958 | "# loc - gets you data by column and row name\n",
959 | "# get one specific element by column_name and row_index\n",
960 | "df_bub.loc[6,(\"cam1\",\"Bounding \\nleft\")]\n",
961 | "\n",
962 | "# get numerical index of column:\n",
963 | "idx=df_bub.columns.get_loc((\"cam1\",\"Bounding \\nleft\"))\n",
964 | "idx\n",
965 | "\n",
966 | "# iloc - gets you data by index\n",
967 | "# get one specific element by column index and row index\n",
968 | "df_bub.iloc[6,idx]"
969 | ]
970 | },
971 | {
972 | "cell_type": "markdown",
973 | "metadata": {},
974 | "source": [
975 | "## And what do we now do with that? More ideas... Try it!\n",
976 | "### Add other tests and combine to one big dataframe\n",
977 | "### Add columns with postprocessed values\n",
978 | "### Plotting\n",
979 | "### ..."
980 | ]
981 | },
982 | {
983 | "cell_type": "code",
984 | "execution_count": null,
985 | "metadata": {},
986 | "outputs": [],
987 | "source": []
988 | },
989 | {
990 | "cell_type": "markdown",
991 | "metadata": {},
992 | "source": [
993 | "## Visualize results\n",
994 | "\n",
995 | "For the bubblecolumn test we plot v_pins over time\n",
996 | "\n",
997 | "More about data visualization in the next session! \n",
998 | "\n",
999 | "### Hint: You can also plot only a portion of the original data and apply the filtering functions upfront."
1000 | ]
1001 | },
1002 | {
1003 | "cell_type": "code",
1004 | "execution_count": null,
1005 | "metadata": {},
1006 | "outputs": [],
1007 | "source": [
1008 | "df_bub.columns.values"
1009 | ]
1010 | },
1011 | {
1012 | "cell_type": "code",
1013 | "execution_count": null,
1014 | "metadata": {},
1015 | "outputs": [],
1016 | "source": [
1017 | "import matplotlib.pyplot as plt\n",
1018 | "%matplotlib inline \n",
1019 | "import datetime\n",
1020 | "\n",
1021 | "x=df_bub[\"erg\",\"Zeit [ms]\"]\n",
1022 | "\n",
1023 | "y1=(df_bub[\"erg\",\"z_bild \"].shift(1)-df_bub[\"erg\",\"z_bild \"])/(df_bub[\"erg\",\"t_Bilder LabV\"].shift(1)-df_bub[\"erg\",\"t_Bilder LabV\"])\n",
1024 | "y2=df_bub[\"erg\",\"z_bild \"]\n",
1025 | "\n",
1026 | "plt.figure(figsize=(15, 10))\n",
1027 | "plt.scatter(x,y1)\n",
1028 | "plt.ylim(0,0.5)\n",
1029 | "plt.xlabel(\"Zeit [ms]\")\n",
1030 | "plt.ylabel(\"v_pins\")"
1031 | ]
1032 | },
1033 | {
1034 | "cell_type": "markdown",
1035 | "metadata": {},
1036 | "source": [
1037 | "## Export\n",
1038 | "### to Excel"
1039 | ]
1040 | },
1041 | {
1042 | "cell_type": "code",
1043 | "execution_count": null,
1044 | "metadata": {},
1045 | "outputs": [],
1046 | "source": [
1047 | "df_all_cols"
1048 | ]
1049 | },
1050 | {
1051 | "cell_type": "code",
1052 | "execution_count": null,
1053 | "metadata": {},
1054 | "outputs": [],
1055 | "source": [
1056 | "# just a tiny example. Of course you can do all kinds of formatting etc...\n",
1057 | "writer = pd.ExcelWriter(\"../Data/df_all_columns.xlsx\",engine='xlsxwriter',options={'remove_timezone': True}) \n",
1058 | "df_all_cols.to_excel(writer,sheet_name=\"all cols\",startrow=1 , startcol=1, index=False)\n",
1059 | "writer.save()"
1060 | ]
1061 | },
1062 | {
1063 | "cell_type": "markdown",
1064 | "metadata": {},
1065 | "source": [
1066 | "## Export notebooks also as\n",
1067 | "- pdf\n",
1068 | "- latex\n",
1069 | "- py\n",
1070 | "\n",
1071 | "--> check out in the menu: File --> Export"
1072 | ]
1073 | },
1074 | {
1075 | "cell_type": "markdown",
1076 | "metadata": {},
1077 | "source": [
1078 | "# More python:\n",
1079 | "\n",
1080 | "Some useful functionalities:\n",
1081 | "- zip() - combines multiple iterables into one data structure by grouping them together according to their index (0 pairs with 0, 1 with 1, etc) \n",
1082 | "- map() - map iterates over an array and executes a function on each element. It's an elegant and concise way to loop through data \n",
1083 | "- filter() - filters through your array and returns all elements who pass your condition \n",
1084 | "- reduce() - you can perform cumulative tasks on the elements of your list, for example the sum of all elements or calculating the product of all entries (has to be imported from functools for python 3) \n",
1085 | "- lambdas - Lambdas are locally defined functions you can use without having to define them globally\n"
1086 | ]
1087 | },
1088 | {
1089 | "cell_type": "code",
1090 | "execution_count": null,
1091 | "metadata": {},
1092 | "outputs": [],
1093 | "source": [
1094 | "%%time\n",
1095 | "from functools import reduce\n",
1096 | "\n",
1097 | "arr = [1, 2, 3, 4]\n",
1098 | "\n",
1099 | "letters = ['A', 'B', 'C', 'D']\n",
1100 | "\n",
1101 | "def someFunction(arg1, arg2):\n",
1102 | " result = arg1 ** arg2\n",
1103 | " return(result)\n",
1104 | "\n",
1105 | "print(someFunction(2, 3))\n",
1106 | "\n",
1107 | "outputZip = list(zip(arr, letters))\n",
1108 | "print(outputZip)\n",
1109 | "\n",
1110 | "outputMap = list(map(lambda x: x*2, arr))\n",
1111 | "print(outputMap)\n",
1112 | "\n",
1113 | "outputFilter = list(filter(lambda x: x % 2 == 0, arr))\n",
1114 | "print(outputFilter)\n",
1115 | "\n",
1116 | "outputReduce = reduce(lambda x, y: x + y, arr)\n",
1117 | "print(outputReduce)"
1118 | ]
1119 | },
1120 | {
1121 | "cell_type": "markdown",
1122 | "metadata": {},
1123 | "source": [
1124 | "# More hacks and best practices:\n",
1125 | "## Command mode\n",
1126 | "```esc``` and then navigate around with arrows\n",
1127 | "\n",
1128 | "\n",
1129 | "## Shell commands\n",
1130 | "```python\n",
1131 | "!ls\n",
1132 | "```\n",
1133 | "## Use virtual environments for more complex projects\n",
1134 | "One big disadavantage of python is it's volatility and dynamic. So lots of functions keep changing and packages are not compatible with each other, depending on the versions.\n",
1135 | "```python\n",
1136 | "python3 -m venv --system-site-packages NAME_ENV\n",
1137 | "```\n",
1138 | "## Use the virtual env with jupyter notebook:\n",
1139 | "```python\n",
1140 | "pip install --user ipykernel\n",
1141 | "python -m ipykernel install --user --name=myenv\n",
1142 | "source env/bin/activate\n",
1143 | "```\n",
1144 | "\n",
1145 | "## Get a working environment\n",
1146 | "requirements.txt\n",
1147 | "pip freeze\n",
1148 | "Besides \n",
1149 | "## Reuse the same structure for your projects --> Cookiecutter templates\n",
1150 | "The way from raw to processed data is well documented, comprehensible and repeatable. \n",
1151 | "\n",
1152 | "### Hacks:\n",
1153 | "- https://www.dataquest.io/blog/jupyter-notebook-tips-tricks-shortcuts/\n",
1154 | "- https://github.com/astridwalle/python_jupyter_basics/blob/main/JupyterHacks/Jupyter-Hacks.ipynb\n",
1155 | "\n",
1156 | "### Combination of tools:\n",
1157 | "https://sehyoun.com/blog/20180904_using-matlab-with-jupyter-notebook.html"
1158 | ]
1159 | },
1160 | {
1161 | "cell_type": "markdown",
1162 | "metadata": {},
1163 | "source": [
1164 | "## Check out markdown possibilities\n",
1165 | "e.g. include pics and gif's easily\n",
1166 | "\n",
1167 | ""
1168 | ]
1169 | },
1170 | {
1171 | "cell_type": "code",
1172 | "execution_count": null,
1173 | "metadata": {},
1174 | "outputs": [],
1175 | "source": [
1176 | "# Check out your variables of a specific type, here now we check all DataFrames in our notebook\n",
1177 | "%who DataFrame"
1178 | ]
1179 | },
1180 | {
1181 | "cell_type": "code",
1182 | "execution_count": null,
1183 | "metadata": {},
1184 | "outputs": [],
1185 | "source": []
1186 | }
1187 | ],
1188 | "metadata": {
1189 | "kernelspec": {
1190 | "display_name": "Python 3",
1191 | "language": "python",
1192 | "name": "python3"
1193 | },
1194 | "language_info": {
1195 | "codemirror_mode": {
1196 | "name": "ipython",
1197 | "version": 3
1198 | },
1199 | "file_extension": ".py",
1200 | "mimetype": "text/x-python",
1201 | "name": "python",
1202 | "nbconvert_exporter": "python",
1203 | "pygments_lexer": "ipython3",
1204 | "version": "3.6.9"
1205 | }
1206 | },
1207 | "nbformat": 4,
1208 | "nbformat_minor": 1
1209 | }
1210 |
--------------------------------------------------------------------------------
/DataVisualization/2_Visualization.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {
6 | "collapsed": true
7 | },
8 | "source": [
9 | "# Visualisation of BubbleColumn testing data and Covid19 data\n",
10 | "\n",
11 | "Now we will look a bit deeper into different visualisation options.\n",
12 | "\n",
13 | "Again we will start by reading in the files with read_excel and read_csv.\n",
14 | "\n",
15 | "## Our data for plotting:\n",
16 | "1. Bubble column test data (combined data from 3 testruns)\n",
17 | "2. RKI Covid19 data"
18 | ]
19 | },
20 | {
21 | "cell_type": "code",
22 | "execution_count": null,
23 | "metadata": {
24 | "ExecuteTime": {
25 | "end_time": "2019-10-16T10:03:28.577240Z",
26 | "start_time": "2019-10-16T10:03:27.996431Z"
27 | }
28 | },
29 | "outputs": [],
30 | "source": [
31 | "import pandas as pd"
32 | ]
33 | },
34 | {
35 | "cell_type": "markdown",
36 | "metadata": {},
37 | "source": [
38 | "### Read bubblecolumn excel files and combine into one pandas dataframe"
39 | ]
40 | },
41 | {
42 | "cell_type": "code",
43 | "execution_count": null,
44 | "metadata": {
45 | "ExecuteTime": {
46 | "end_time": "2019-10-16T10:03:42.527542Z",
47 | "start_time": "2019-10-16T10:03:40.683455Z"
48 | }
49 | },
50 | "outputs": [],
51 | "source": [
52 | "df_bub1 = pd.read_excel(\"../Data/BubbleColumn/Test_01.xlsx\",header=[0,1])\n",
53 | "df_bub2 = pd.read_excel(\"../Data/BubbleColumn/Test_02.xlsx\",header=[0,1])\n",
54 | "df_bub3 = pd.read_excel(\"../Data/BubbleColumn/Test_03.xlsx\",header=[0,1])"
55 | ]
56 | },
57 | {
58 | "cell_type": "code",
59 | "execution_count": null,
60 | "metadata": {
61 | "scrolled": true
62 | },
63 | "outputs": [],
64 | "source": [
65 | "df_bub=pd.concat([df_bub1,df_bub2,df_bub2],keys=[\"Test1\",\"Test2\",\"Test3\"],axis=0,names=[\"Param\",\"Row_Index\"],ignore_index=False)\n",
66 | "df_bub"
67 | ]
68 | },
69 | {
70 | "cell_type": "code",
71 | "execution_count": null,
72 | "metadata": {},
73 | "outputs": [],
74 | "source": [
75 | "df_bub.index.get_level_values(level=0)"
76 | ]
77 | },
78 | {
79 | "cell_type": "code",
80 | "execution_count": null,
81 | "metadata": {},
82 | "outputs": [],
83 | "source": [
84 | "# We convert the Param multiindex into an additional column (easier for filtering)\n",
85 | "df_bub.reset_index(level=0,inplace=True)\n",
86 | "df_bub"
87 | ]
88 | },
89 | {
90 | "cell_type": "markdown",
91 | "metadata": {},
92 | "source": [
93 | "Now we do some data exploration to check for non-numerical or missing values and to check whether the data is as we expect it."
94 | ]
95 | },
96 | {
97 | "cell_type": "code",
98 | "execution_count": null,
99 | "metadata": {},
100 | "outputs": [],
101 | "source": [
102 | "df_bub.describe()"
103 | ]
104 | },
105 | {
106 | "cell_type": "code",
107 | "execution_count": null,
108 | "metadata": {},
109 | "outputs": [],
110 | "source": [
111 | "# Let's check the multiindex column names\n",
112 | "df_bub.columns.values"
113 | ]
114 | },
115 | {
116 | "cell_type": "code",
117 | "execution_count": null,
118 | "metadata": {},
119 | "outputs": [],
120 | "source": []
121 | },
122 | {
123 | "cell_type": "markdown",
124 | "metadata": {},
125 | "source": [
126 | "### We also read in RKI Covid19 data as an example for timeseries and categorical data"
127 | ]
128 | },
129 | {
130 | "cell_type": "code",
131 | "execution_count": null,
132 | "metadata": {},
133 | "outputs": [],
134 | "source": [
135 | "df_rki=pd.read_csv(\"https://www.arcgis.com/sharing/rest/content/items/f10774f1c63e40168479a1feb6c7ca74/data\")\n",
136 | "df_rki"
137 | ]
138 | },
139 | {
140 | "cell_type": "code",
141 | "execution_count": null,
142 | "metadata": {},
143 | "outputs": [],
144 | "source": [
145 | "# Some preprocessing of RKI data to get official results:\n",
146 | "# Is the data up to date?\n",
147 | "print(df_rki[\"Datenstand\"].unique())\n",
148 | "df_rki_temp = df_rki[((df_rki[\"NeuerFall\"]==0) | (df_rki[\"NeuerFall\"]==1))]"
149 | ]
150 | },
151 | {
152 | "cell_type": "code",
153 | "execution_count": null,
154 | "metadata": {},
155 | "outputs": [],
156 | "source": []
157 | },
158 | {
159 | "cell_type": "markdown",
160 | "metadata": {},
161 | "source": [
162 | "## Some general tips:\n",
163 | "Always very useful:\n",
164 | "- Data Dictionary: Metadata for your column names. Explanations, units, etc\n",
165 | "- Data Catalogue: Catalogue for with metadata and storage paths for your testing data."
166 | ]
167 | },
168 | {
169 | "cell_type": "markdown",
170 | "metadata": {},
171 | "source": [
172 | "## Overview of python visualisation packages \n",
173 | "\n",
174 | "- matplotlib\n",
175 | "\n",
176 | " Widely used visualisation library. Easy to use and has a good online community presence.\n",
177 | "\n",
178 | "\n",
179 | "- pandas built-in plotting library\n",
180 | " \n",
181 | " Single line command to plot the dataframe. Easier to plot scatterplotmatrix using this library compared to matplotlib and bokeh. \n",
182 | "\n",
183 | "\n",
184 | "- bokeh\n",
185 | "\n",
186 | " Visualisations are more appealing and has built in plot configuration tools (zoom in, pan, etc). But takes time to load the visualisation and it is more suited for creating dashboard. Moreover, the documentation is not clear\n",
187 | " \n",
188 | "- seaborn\n",
189 | "\n",
190 | " Plotting based on matplotlib, but with lots of nice preformatting. Optimized for statistical, dataframe plotting\n",
191 | "\n",
192 | "- plotly\n",
193 | "\n",
194 | " Can do contourplots and 3D plots\n",
195 | "\n",
196 | "- altair / Vega-Lite\n",
197 | "\n",
198 | " Statistical visualization library, preformatted. Minimum amount of code required for nice plotting results\n",
199 | " \n",
200 | "### Keep in mind:\n",
201 | "\n",
202 | "- Check the documentation of the module by using help() function or the ? in front of the function call!\n",
203 | "- Questions, Problems? --> Google! --> One of the best sources is stackoverflow\n",
204 | "- Module features are dependent on the module version! Check your version:\n",
205 | "```\n",
206 | "import matplotlib\n",
207 | "matplotlib.__version__\n",
208 | "```\n"
209 | ]
210 | },
211 | {
212 | "cell_type": "code",
213 | "execution_count": null,
214 | "metadata": {},
215 | "outputs": [],
216 | "source": [
217 | "# import neessary libraries:\n",
218 | "import matplotlib.pyplot as plt\n",
219 | "import bokeh.plotting as bp # another plotting option \n",
220 | "from bokeh.plotting import figure,output_notebook,show # for plotting\n",
221 | "import seaborn as sb # yeat another plotting option\n",
222 | "import plotly.express as px\n",
223 | "import altair as alt\n",
224 | "\n",
225 | "# interactive widgets\n",
226 | "import ipywidgets as widgets # interactive notebooks - make selection etc\n",
227 | "from IPython.display import display # to display the widgets in notebook\n",
228 | "\n",
229 | "# some more useful stuff:\n",
230 | "import os\n",
231 | "import datetime "
232 | ]
233 | },
234 | {
235 | "cell_type": "markdown",
236 | "metadata": {},
237 | "source": [
238 | "## Plot histograms\n",
239 | "In order to understand the typical distributions of values, you can always start with a histogram\n",
240 | "\n",
241 | "We start with our BubbleColumn testing data\n",
242 | "\n",
243 | "We wil compare the histogram plots from matplotlib library and pandas built-in"
244 | ]
245 | },
246 | {
247 | "cell_type": "code",
248 | "execution_count": null,
249 | "metadata": {},
250 | "outputs": [],
251 | "source": [
252 | "# 1. df_test - matplotlib\n",
253 | "\n",
254 | "plt.figure(figsize=(15,5))\n",
255 | "plt.hist(df_bub['cam0', 'Max Feret Diameter'].dropna(),bins=25, color='green',alpha=0.7) # Remember to dropna!\n",
256 | "plt.xlabel('Max Feret Diameter')\n",
257 | "plt.title('Max Feret Diameter')\n",
258 | "plt.show()\n",
259 | "\n",
260 | "# 2. df_test - matplotlib: Also applying some filtering to zoom into a smaller range \n",
261 | "\n",
262 | "plt.figure(figsize=(15,5))\n",
263 | "plt.hist(df_bub[((df_bub['cam0', 'Max Feret Diameter']>1.2) & (df_bub['cam0', 'Max Feret Diameter']<6.0))]['cam0', 'Max Feret Diameter'].dropna(),bins=25, color='grey',alpha=0.7) # Remember to dropna!\n",
264 | "plt.xlabel('Max Feret Diameter')\n",
265 | "plt.title('Max Feret Diameter')\n",
266 | "plt.show()"
267 | ]
268 | },
269 | {
270 | "cell_type": "code",
271 | "execution_count": null,
272 | "metadata": {},
273 | "outputs": [],
274 | "source": [
275 | "# 2. df_test - pandas built-in \n",
276 | "df_bub.plot(y=(\"cam0\",'Max Feret Diameter'),kind=\"hist\",bins=25,color=\"green\",alpha=0.7,figsize=(15,5),title='Max Feret Diameter')"
277 | ]
278 | },
279 | {
280 | "cell_type": "markdown",
281 | "metadata": {},
282 | "source": [
283 | "## How can histograms be extremely valuable? \n",
284 | "With the help of histograms you can already get an idea about outliers:\n",
285 | "\n",
286 | "If you have the data from multiple tests and you want to know how one specific test compares to the overall amount of tests."
287 | ]
288 | },
289 | {
290 | "cell_type": "code",
291 | "execution_count": null,
292 | "metadata": {},
293 | "outputs": [],
294 | "source": [
295 | "help(plt.hist)"
296 | ]
297 | },
298 | {
299 | "cell_type": "code",
300 | "execution_count": null,
301 | "metadata": {
302 | "scrolled": true
303 | },
304 | "outputs": [],
305 | "source": [
306 | "# Plot 2 overlaying histograms for comparison. \n",
307 | "# To be able to do so, we also need to add the density keyword! Otherwise the bins of the one test will be much much smaller.\n",
308 | "plt.figure(figsize=(15,5))\n",
309 | "plt.hist(df_bub['cam0', 'Max Feret Diameter'].dropna(),bins=20,density=True, color='blue',label=\"All tests\")\n",
310 | "plt.hist(df_bub[df_bub[\"Param\"]==\"Test1\"]['cam0', 'Max Feret Diameter'].dropna(),bins=20, density=True,color='orange',alpha= 0.35, label=\"Test 1\")\n",
311 | "plt.xlabel('Max Feret Diameter')\n",
312 | "plt.title('Comparison of one test with the overall amount of test')\n",
313 | "plt.legend()\n",
314 | "plt.show()"
315 | ]
316 | },
317 | {
318 | "cell_type": "markdown",
319 | "metadata": {},
320 | "source": [
321 | "## Some simple scatter / line plots\n",
322 | "### Created in a loop with filtering of a large dataframe\n",
323 | "\n",
324 | "1. Example: Bubble testing data\n",
325 | "2. Example: RKI Covid10 cases for different Landkreise\n",
326 | "\n",
327 | "Advantages of a scatter plot over a line plot:\n",
328 | "Whenever you look at a distribution or a change over time, you are not able to see the intensity / density of the datapoints, if you just do a line plot. To get a feeling for the data, it is always better to start with 'point' as marker instead of 'line'"
329 | ]
330 | },
331 | {
332 | "cell_type": "code",
333 | "execution_count": null,
334 | "metadata": {},
335 | "outputs": [],
336 | "source": [
337 | "df_bub[df_bub[\"Param\"]==\"Test3\"][\"erg\",\"z_bild \"]"
338 | ]
339 | },
340 | {
341 | "cell_type": "code",
342 | "execution_count": null,
343 | "metadata": {},
344 | "outputs": [],
345 | "source": [
346 | "plt.figure(figsize=(15,5))\n",
347 | "\n",
348 | "for i in df_bub[\"Param\"].unique():\n",
349 | " print(i)\n",
350 | " df_temp=df_bub[df_bub[\"Param\"]==i]\n",
351 | " x=df_temp[\"erg\",\"Zeit [ms]\"]\n",
352 | " y=(df_temp[\"erg\",\"z_bild \"].shift(1)-df_temp[\"erg\",\"z_bild \"])/(df_temp[\"erg\",\"t_Bilder LabV\"].shift(1)-df_temp[\"erg\",\"t_Bilder LabV\"])\n",
353 | " plt.scatter(x,y,label=i)\n",
354 | " \n",
355 | "plt.legend()\n",
356 | "plt.ylim(0,0.5)\n"
357 | ]
358 | },
359 | {
360 | "cell_type": "markdown",
361 | "metadata": {},
362 | "source": [
363 | "## Interactive selection widgets:\n",
364 | "Another option to get the plots for different tests interactively:\n",
365 | "\n",
366 | "In this minimal example you have to run the plot command every time yoiu have changed the Dropdwon values. But of course you can also add a so-called callback to renew the plot automatically, when a dropdown value changes.\n",
367 | "check it out: widget.observe"
368 | ]
369 | },
370 | {
371 | "cell_type": "code",
372 | "execution_count": null,
373 | "metadata": {},
374 | "outputs": [],
375 | "source": [
376 | "# At first we create the selection widget for the Testrun\n",
377 | "Test_selection=widgets.Dropdown(options=df_bub[\"Param\"].unique(), value=\"Test2\", description=\"Select one test\")\n",
378 | "display(Test_selection)"
379 | ]
380 | },
381 | {
382 | "cell_type": "code",
383 | "execution_count": null,
384 | "metadata": {},
385 | "outputs": [],
386 | "source": [
387 | "# Then we create the selection widget for the 0th level of the multiindex columns:\n",
388 | "Parameter1_selection=widgets.Dropdown(options=df_bub.columns.get_level_values(level=0).unique(), value=\"cam0\", description=\"Select one parameter\")\n",
389 | "display(Parameter1_selection)"
390 | ]
391 | },
392 | {
393 | "cell_type": "code",
394 | "execution_count": null,
395 | "metadata": {},
396 | "outputs": [],
397 | "source": [
398 | "# Then we create the selection widget for the 1st level of the multiindex columns:\n",
399 | "Parameter2_selection=widgets.Dropdown(options=df_bub.loc[:,pd.IndexSlice[[\"cam0\"], :]].columns.get_level_values(1).unique(),\n",
400 | " description=\"Select one parameter\")\n",
401 | "display(Parameter2_selection)"
402 | ]
403 | },
404 | {
405 | "cell_type": "code",
406 | "execution_count": null,
407 | "metadata": {},
408 | "outputs": [],
409 | "source": [
410 | "plt.figure(figsize=(15,5))\n",
411 | "\n",
412 | "df_temp=df_bub[df_bub[\"Param\"]==Test_selection.value]\n",
413 | "x=df_temp[\"erg\",\"Zeit [ms]\"]\n",
414 | "y=df_temp[Parameter1_selection.value,Parameter2_selection.value]\n",
415 | "plt.scatter(x,y,label=str(Parameter1_selection.value)+\", \"+str(Parameter2_selection.value))\n",
416 | " \n",
417 | "plt.legend()"
418 | ]
419 | },
420 | {
421 | "cell_type": "markdown",
422 | "metadata": {},
423 | "source": [
424 | "## Now lets have a look at the same plot with different packages\n",
425 | "### Bokeh --> Interactive plots\n",
426 | "Try the different menu options you can see at the right side of the plot"
427 | ]
428 | },
429 | {
430 | "cell_type": "code",
431 | "execution_count": null,
432 | "metadata": {},
433 | "outputs": [],
434 | "source": [
435 | "# This commands let you visualise bokeh below the execution cell\n",
436 | "output_notebook()"
437 | ]
438 | },
439 | {
440 | "cell_type": "code",
441 | "execution_count": null,
442 | "metadata": {},
443 | "outputs": [],
444 | "source": [
445 | "df_temp=df_bub[df_bub[\"Param\"]==Test_selection.value]\n",
446 | "x=df_temp[\"erg\",\"Zeit [ms]\"]\n",
447 | "y=df_temp[Parameter1_selection.value,Parameter2_selection.value]\n",
448 | "\n",
449 | "# 1. Bokeh\n",
450 | "p = figure(title=\"Parameter Selection {}, {} for {}\".\n",
451 | " format(Parameter1_selection.value,Parameter2_selection.value,Test_selection.value),x_axis_type='datetime',\n",
452 | " width=800,height=250)\n",
453 | "p.circle(x=x,\n",
454 | " y=y)\n",
455 | "show(p)"
456 | ]
457 | },
458 | {
459 | "cell_type": "markdown",
460 | "metadata": {},
461 | "source": [
462 | "### Seaborn\n",
463 | "Not interactive, but preformatted for a nice appearance"
464 | ]
465 | },
466 | {
467 | "cell_type": "code",
468 | "execution_count": null,
469 | "metadata": {},
470 | "outputs": [],
471 | "source": [
472 | "df_temp=df_bub[df_bub[\"Param\"]==Test_selection.value]\n",
473 | "x=df_temp[\"erg\",\"Zeit [ms]\"]\n",
474 | "y=df_temp[Parameter1_selection.value,Parameter2_selection.value]\n",
475 | "\n",
476 | "plt.figure(figsize=(15,5))\n",
477 | "sb.scatterplot(x,y)"
478 | ]
479 | },
480 | {
481 | "cell_type": "markdown",
482 | "metadata": {},
483 | "source": [
484 | "### Plotly\n",
485 | "Interactive plots. Here you can see the single values when hovering over the points.\n",
486 | "\n",
487 | "With plotly you can also do 3D plots!"
488 | ]
489 | },
490 | {
491 | "cell_type": "code",
492 | "execution_count": null,
493 | "metadata": {},
494 | "outputs": [],
495 | "source": [
496 | "df_temp=df_bub[df_bub[\"Param\"]==Test_selection.value]\n",
497 | "x=df_temp[\"erg\",\"Zeit [ms]\"]\n",
498 | "y=df_temp[Parameter1_selection.value,Parameter2_selection.value]\n",
499 | "\n",
500 | "fig = px.scatter(x=x, y=y)\n",
501 | "fig.show()"
502 | ]
503 | },
504 | {
505 | "cell_type": "markdown",
506 | "metadata": {},
507 | "source": [
508 | "### And now as well Altair"
509 | ]
510 | },
511 | {
512 | "cell_type": "code",
513 | "execution_count": null,
514 | "metadata": {},
515 | "outputs": [],
516 | "source": [
517 | "# Here we need some additional extensions, so maybe you need to install some additional packages to be able to display the plot.\n",
518 | "df_temp=df_bub[df_bub[\"Param\"]==Test_selection.value]\n",
519 | "x=df_temp[\"erg\",\"Zeit [ms]\"]\n",
520 | "y=df_temp[Parameter1_selection.value,Parameter2_selection.value]\n",
521 | "\n",
522 | "chart=alt.Chart(x=x, y=y).interactive()\n",
523 | "\n",
524 | "chart.show()"
525 | ]
526 | },
527 | {
528 | "cell_type": "markdown",
529 | "metadata": {},
530 | "source": [
531 | "## Doing some line plots in a loop\n",
532 | "Lets look at the Covid19 case numbers for each Landkreis"
533 | ]
534 | },
535 | {
536 | "cell_type": "code",
537 | "execution_count": null,
538 | "metadata": {},
539 | "outputs": [],
540 | "source": [
541 | "# Plotting the casenumbers\n",
542 | "fig=plt.figure(figsize=(12,10))\n",
543 | "ax1=fig.add_subplot(111)\n",
544 | "\n",
545 | "df_rki_lk=df_rki_temp.groupby([\"Landkreis\",\"Meldedatum\"],as_index=False)[[\"AnzahlFall\"]].sum()\n",
546 | "for i in df_rki_lk[\"Landkreis\"].unique():\n",
547 | " df=df_rki_lk[df_rki_lk[\"Landkreis\"]==i]\n",
548 | " df.set_index(\"Meldedatum\", inplace=True, drop=True)\n",
549 | " df.index=pd.to_datetime(df.index,format=\"%Y-%m-%d\")\n",
550 | " df.sort_index(inplace=True)\n",
551 | " ax1.plot(df[\"AnzahlFall\"],color=\"grey\",alpha=0.3)\n",
552 | "\n",
553 | " if \"Berlin\" in i:\n",
554 | " df_b=df\n",
555 | "\n",
556 | "ax1.plot(df_b[\"AnzahlFall\"],color=\"red\",label=\"Berlin\")\n",
557 | "plt.yscale(\"log\")\n",
558 | "plt.title(\"Casenumbers - Reporting Date - for each Landkreis in Germany\")"
559 | ]
560 | },
561 | {
562 | "cell_type": "code",
563 | "execution_count": null,
564 | "metadata": {},
565 | "outputs": [],
566 | "source": [
567 | "df_rki_lk[\"Landkreis\"].unique()"
568 | ]
569 | },
570 | {
571 | "cell_type": "markdown",
572 | "metadata": {},
573 | "source": [
574 | "### Create mulitple subplots\n",
575 | "\n",
576 | "Plot different Landkreise in subplots"
577 | ]
578 | },
579 | {
580 | "cell_type": "code",
581 | "execution_count": null,
582 | "metadata": {},
583 | "outputs": [],
584 | "source": [
585 | "plt.figure(figsize=(15,7))\n",
586 | "\n",
587 | "plt.subplot(2,2,1)\n",
588 | "plt.plot(df_b[\"AnzahlFall\"],'.')\n",
589 | "plt.plot(df_b[\"AnzahlFall\"],'-', color=\"grey\", alpha=0.5)\n",
590 | "plt.title(\"Subplot1 - Berlin\")\n",
591 | "\n",
592 | "plt.subplot(2,2,2)\n",
593 | "plt.plot(df_rki_lk[df_rki_lk[\"Landkreis\"]==\"LK Darmstadt-Dieburg\"][\"AnzahlFall\"],'.')\n",
594 | "plt.plot(df_rki_lk[df_rki_lk[\"Landkreis\"]==\"LK Darmstadt-Dieburg\"][\"AnzahlFall\"],'-', color=\"grey\", alpha=0.5)\n",
595 | "plt.title(\"Subplot 2 - LK Darmstadt-Dieburg\")\n",
596 | "\n",
597 | "plt.subplot(2,2,3)\n",
598 | "plt.plot(df_rki_lk[df_rki_lk[\"Landkreis\"]==\"LK Friesland\"][\"AnzahlFall\"],'.')\n",
599 | "plt.plot(df_rki_lk[df_rki_lk[\"Landkreis\"]==\"LK Friesland\"][\"AnzahlFall\"],'-', color=\"grey\", alpha=0.5)\n",
600 | "plt.title(\"Subplot 3 - LK Friesland\")\n",
601 | "\n",
602 | "plt.subplot(2,2,4)\n",
603 | "plt.plot(df_rki_lk[df_rki_lk[\"Landkreis\"]==\"LK Heinsberg\"][\"AnzahlFall\"],'.')\n",
604 | "plt.plot(df_rki_lk[df_rki_lk[\"Landkreis\"]==\"LK Heinsberg\"][\"AnzahlFall\"],'-', color=\"grey\", alpha=0.5)\n",
605 | "plt.title(\"Subplot 4 - LK Heinsberg\")\n",
606 | "\n",
607 | "plt.show()"
608 | ]
609 | },
610 | {
611 | "cell_type": "markdown",
612 | "metadata": {},
613 | "source": [
614 | "## Create a Correlation / Scatterplot matrix \n",
615 | "\n",
616 | "\"A scatter plot matrix is a grid (or matrix) of scatter plots used to visualize bivariate relationships between combinations of variables. Each scatter plot in the matrix visualizes the relationship between a pair of variables, allowing many relationships to be explored in one chart.\"\n",
617 | "(https://pro.arcgis.com/en/pro-app/latest/help/analysis/geoprocessing/charts/scatter-plot-matrix.htm)"
618 | ]
619 | },
620 | {
621 | "cell_type": "code",
622 | "execution_count": null,
623 | "metadata": {},
624 | "outputs": [],
625 | "source": [
626 | "# for simplicity we just look at a smaller df with just one level of column indices\n",
627 | "df_temp=df_bub[df_bub[\"Param\"]==\"Test1\"]\n",
628 | "df_temp=df_temp.loc[:,pd.IndexSlice[[\"cam0\"], :]]"
629 | ]
630 | },
631 | {
632 | "cell_type": "code",
633 | "execution_count": null,
634 | "metadata": {},
635 | "outputs": [],
636 | "source": [
637 | "df_temp.columns.values"
638 | ]
639 | },
640 | {
641 | "cell_type": "code",
642 | "execution_count": null,
643 | "metadata": {},
644 | "outputs": [],
645 | "source": [
646 | "df_temp=df_temp[[('cam0', 'Waddel Disk Diameter'),('cam0', 'equi Ellipse Minor'),('cam0', 'Max Feret Diameter'),\n",
647 | " ('cam0', 'equi Ellipse Minor Axis (Feret)'),('cam0', 'equi Rect Short Side (Feret)')]]"
648 | ]
649 | },
650 | {
651 | "cell_type": "code",
652 | "execution_count": null,
653 | "metadata": {},
654 | "outputs": [],
655 | "source": [
656 | "pd.plotting.scatter_matrix(df_temp, figsize=(15, 15), marker='o',\n",
657 | " hist_kwds={'bins': 20}, s=1, alpha=.25)\n",
658 | "plt.show()"
659 | ]
660 | },
661 | {
662 | "cell_type": "markdown",
663 | "metadata": {
664 | "collapsed": true
665 | },
666 | "source": [
667 | "### Links to visualisation examples and more:\n",
668 | "\n",
669 | "* Finding the right diagram\n",
670 | "https://www.visual-literacy.org/periodic_table/periodic_table.html\n",
671 | "* Finding the right colormap for our visualisation\n",
672 | "http://colorbrewer2.org/#type=sequential&scheme=BuGn&n=3\n",
673 | "* More visualisation examples: \n",
674 | "https://d3js.org/\n",
675 | "https://docs.bokeh.org/en/latest/docs/gallery.html\n",
676 | "* Broad overview of vsrious tools available in python \n",
677 | "https://github.com/EthicalML/awesome-production-machine-learning\n",
678 | "* Need multiple y-axis?\n",
679 | "https://matplotlib.org/3.1.1/gallery/ticks_and_spines/multiple_yaxis_with_spines.html"
680 | ]
681 | },
682 | {
683 | "cell_type": "code",
684 | "execution_count": null,
685 | "metadata": {},
686 | "outputs": [],
687 | "source": []
688 | }
689 | ],
690 | "metadata": {
691 | "kernelspec": {
692 | "display_name": "Python 3",
693 | "language": "python",
694 | "name": "python3"
695 | },
696 | "language_info": {
697 | "codemirror_mode": {
698 | "name": "ipython",
699 | "version": 3
700 | },
701 | "file_extension": ".py",
702 | "mimetype": "text/x-python",
703 | "name": "python",
704 | "nbconvert_exporter": "python",
705 | "pygments_lexer": "ipython3",
706 | "version": "3.6.9"
707 | }
708 | },
709 | "nbformat": 4,
710 | "nbformat_minor": 2
711 | }
712 |
--------------------------------------------------------------------------------
/JupyterHacks/.ipynb_checkpoints/Jupyter-Hacks-checkpoint.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Intro\n",
8 | "In this document I want to give an overview of magic commands, short cuts and code snippets I find extremely helpful in my day-2-day work with jupyter-notebooks. "
9 | ]
10 | },
11 | {
12 | "cell_type": "markdown",
13 | "metadata": {},
14 | "source": [
15 | "# Command Mode\n",
16 | "I don't know if it is just me or if there are some other people as well, who came across the command mode quite late. So this would have saved me really a lot of time!\n",
17 | "\n",
18 | "To enter the command mode, just press ```esc```.\n",
19 | "\n",
20 | "- Now you can move across the cells with the arrow keys.\n",
21 | "- Enable editing of a specific cell, just by hitting ```enter```.\n",
22 | "- Insert new cells before or after the current cell with ```a```, ```b```.\n",
23 | "- Change between ```code```and ```markdown```with ```y```and ```m```.\n",
24 | "- Deleting a cell is as easy as pressing ```x```.\n",
25 | "- ...\n",
26 | "\n",
27 | "Check out https://www.dataquest.io/blog/jupyter-notebook-tips-tricks-shortcuts/ for more functionalities."
28 | ]
29 | },
30 | {
31 | "cell_type": "markdown",
32 | "metadata": {},
33 | "source": [
34 | "# Toggling code\n",
35 | "Sometimes you like to use the notebook as a nice and clean documentation for sharing your results and maybe you don't want to disturb the viewer with long and maybe messy codecells. Then you can insert one of the following code snippets at the beginning of your notebook.\n",
36 | "\n",
37 | "The first one is for toggling all code in the notebook # https://stackoverflow.com/questions/27934885/how-to-hide-code-from-cells-in-ipython-notebook-visualized-with-nbviewer\n",
38 | "\n",
39 | "The second one is for toggling one specific cell by adding the command ```hide_toggle()``` at the end https://stackoverflow.com/questions/31517194/how-to-hide-one-specific-cell-input-or-output-in-ipython-notebook/48084050"
40 | ]
41 | },
42 | {
43 | "cell_type": "code",
44 | "execution_count": 1,
45 | "metadata": {},
46 | "outputs": [
47 | {
48 | "data": {
49 | "text/html": [
50 | "\n",
62 | ""
63 | ],
64 | "text/plain": [
65 | ""
66 | ]
67 | },
68 | "execution_count": 1,
69 | "metadata": {},
70 | "output_type": "execute_result"
71 | }
72 | ],
73 | "source": [
74 | "from IPython.display import HTML\n",
75 | "\n",
76 | "HTML('''\n",
88 | "''')"
89 | ]
90 | },
91 | {
92 | "cell_type": "code",
93 | "execution_count": 36,
94 | "metadata": {},
95 | "outputs": [
96 | {
97 | "data": {
98 | "text/html": [
99 | "\n",
100 | " \n",
107 | "\n",
108 | " Toggle show/hide\n",
109 | " "
110 | ],
111 | "text/plain": [
112 | ""
113 | ]
114 | },
115 | "execution_count": 36,
116 | "metadata": {},
117 | "output_type": "execute_result"
118 | }
119 | ],
120 | "source": [
121 | "from IPython.display import HTML\n",
122 | "import random\n",
123 | "\n",
124 | "def hide_toggle(for_next=False):\n",
125 | " this_cell = \"\"\"$('div.cell.code_cell.rendered.selected')\"\"\"\n",
126 | " next_cell = this_cell + '.next()'\n",
127 | "\n",
128 | " toggle_text = 'Toggle show/hide' # text shown on toggle link\n",
129 | " target_cell = this_cell # target cell to control with toggle\n",
130 | " js_hide_current = '' # bit of JS to permanently hide code in current cell (only when toggling next cell)\n",
131 | "\n",
132 | " if for_next:\n",
133 | " target_cell = next_cell\n",
134 | " toggle_text += ' next cell'\n",
135 | " js_hide_current = this_cell + '.find(\"div.input\").hide();'\n",
136 | "\n",
137 | " js_f_name = 'code_toggle_{}'.format(str(random.randint(1,2**64)))\n",
138 | "\n",
139 | " html = \"\"\"\n",
140 | " \n",
147 | "\n",
148 | " {toggle_text}\n",
149 | " \"\"\".format(\n",
150 | " f_name=js_f_name,\n",
151 | " cell_selector=target_cell,\n",
152 | " js_hide_current=js_hide_current, \n",
153 | " toggle_text=toggle_text\n",
154 | " )\n",
155 | "\n",
156 | " return HTML(html)\n",
157 | "\n",
158 | "hide_toggle()"
159 | ]
160 | },
161 | {
162 | "cell_type": "markdown",
163 | "metadata": {},
164 | "source": [
165 | "# Toggle output\n",
166 | "Sometimes you also want to toggle the output, especially if you have created some long print statements in a loop and you don't want to scroll down so much.\n",
167 | "\n",
168 | "Therefore you have to enter the command mode again, press ```esc``` and use the shortcut ```esc + o```. The same shortcut is used to untoggle the output."
169 | ]
170 | },
171 | {
172 | "cell_type": "markdown",
173 | "metadata": {},
174 | "source": [
175 | "# Variable output\n",
176 | "One great thing about jupyter-notebook is the output of all variables, including arrays and dataframes. But by default only the last statement will produce a printed output. So, in the end you will end up with lots of cells or un/commenting lines for printing several variables. But there is an easy solution - check out the following code snippet. Found on https://www.dataquest.io/blog/jupyter-notebook-tips-tricks-shortcuts/ - there is also the description how to enable this permanently for all your notebooks."
177 | ]
178 | },
179 | {
180 | "cell_type": "code",
181 | "execution_count": 38,
182 | "metadata": {},
183 | "outputs": [
184 | {
185 | "data": {
186 | "text/html": [
187 | "\n",
188 | " \n",
195 | "\n",
196 | " Toggle show/hide\n",
197 | " "
198 | ],
199 | "text/plain": [
200 | ""
201 | ]
202 | },
203 | "execution_count": 38,
204 | "metadata": {},
205 | "output_type": "execute_result"
206 | }
207 | ],
208 | "source": [
209 | "from IPython.core.interactiveshell import InteractiveShell\n",
210 | "InteractiveShell.ast_node_interactivity = \"all\"\n",
211 | "\n",
212 | "hide_toggle()"
213 | ]
214 | },
215 | {
216 | "cell_type": "code",
217 | "execution_count": 6,
218 | "metadata": {},
219 | "outputs": [
220 | {
221 | "data": {
222 | "text/plain": [
223 | "10"
224 | ]
225 | },
226 | "execution_count": 6,
227 | "metadata": {},
228 | "output_type": "execute_result"
229 | },
230 | {
231 | "data": {
232 | "text/plain": [
233 | "20"
234 | ]
235 | },
236 | "execution_count": 6,
237 | "metadata": {},
238 | "output_type": "execute_result"
239 | },
240 | {
241 | "data": {
242 | "text/plain": [
243 | "30"
244 | ]
245 | },
246 | "execution_count": 6,
247 | "metadata": {},
248 | "output_type": "execute_result"
249 | }
250 | ],
251 | "source": [
252 | "a=10\n",
253 | "b=20\n",
254 | "c=a+b\n",
255 | "\n",
256 | "a\n",
257 | "b\n",
258 | "c"
259 | ]
260 | },
261 | {
262 | "cell_type": "markdown",
263 | "metadata": {},
264 | "source": [
265 | "# Help - man pages\n",
266 | "By adding a ```?``` in front of a function, you will get the documentation for this function, so no need to google it while coding.\n",
267 | "\n",
268 | "--> And if you want to close the documentation, just hit ```esc```."
269 | ]
270 | },
271 | {
272 | "cell_type": "code",
273 | "execution_count": 7,
274 | "metadata": {},
275 | "outputs": [],
276 | "source": [
277 | "?print"
278 | ]
279 | },
280 | {
281 | "cell_type": "markdown",
282 | "metadata": {},
283 | "source": [
284 | "# Execute shell commands\n",
285 | "For executing shell commands, e.g. creating folders, getting filenames, you just have to preset a ```!```"
286 | ]
287 | },
288 | {
289 | "cell_type": "code",
290 | "execution_count": 40,
291 | "metadata": {
292 | "collapsed": true
293 | },
294 | "outputs": [
295 | {
296 | "name": "stdout",
297 | "output_type": "stream",
298 | "text": [
299 | "total 1016\r\n",
300 | "drwxr-xr-x 7 astrid staff 224 Feb 3 08:51 \u001b[34m.\u001b[m\u001b[m\r\n",
301 | "drwxr-xr-x 7 astrid staff 224 Jan 26 20:10 \u001b[34m..\u001b[m\u001b[m\r\n",
302 | "drwxr-xr-x 5 astrid staff 160 Feb 3 08:22 \u001b[34m.ipynb_checkpoints\u001b[m\u001b[m\r\n",
303 | "-rw-r--r--@ 1 astrid staff 485805 Feb 3 08:45 1_DataManagement.ipynb\r\n",
304 | "-rw-r--r-- 1 astrid staff 20686 Feb 3 08:51 Jupyter-Hacks.ipynb\r\n",
305 | "-rw-r--r-- 1 astrid staff 881 Feb 3 08:25 hello_world.ipynb\r\n",
306 | "-rw-r--r--@ 1 astrid staff 241 Feb 3 08:43 hello_world.py\r\n"
307 | ]
308 | }
309 | ],
310 | "source": [
311 | "!ls -al"
312 | ]
313 | },
314 | {
315 | "cell_type": "markdown",
316 | "metadata": {},
317 | "source": [
318 | "# Magic commands\n",
319 | "Ipython has so called magic commands built-in, from which I find some extremely helpful!\n",
320 | "\n",
321 | "Some more examples can also be found here:\n",
322 | "https://towardsdatascience.com/top-8-magic-commands-in-jupyter-notebook-c1582e813560\n",
323 | "\n",
324 | "To get an overview of all available commands, type ```%lsmagic```."
325 | ]
326 | },
327 | {
328 | "cell_type": "code",
329 | "execution_count": 16,
330 | "metadata": {},
331 | "outputs": [
332 | {
333 | "data": {
334 | "application/json": {
335 | "cell": {
336 | "!": "OSMagics",
337 | "HTML": "Other",
338 | "SVG": "Other",
339 | "bash": "Other",
340 | "capture": "ExecutionMagics",
341 | "debug": "ExecutionMagics",
342 | "file": "Other",
343 | "html": "DisplayMagics",
344 | "javascript": "DisplayMagics",
345 | "js": "DisplayMagics",
346 | "latex": "DisplayMagics",
347 | "markdown": "DisplayMagics",
348 | "perl": "Other",
349 | "prun": "ExecutionMagics",
350 | "pypy": "Other",
351 | "python": "Other",
352 | "python2": "Other",
353 | "python3": "Other",
354 | "ruby": "Other",
355 | "script": "ScriptMagics",
356 | "sh": "Other",
357 | "svg": "DisplayMagics",
358 | "sx": "OSMagics",
359 | "system": "OSMagics",
360 | "time": "ExecutionMagics",
361 | "timeit": "ExecutionMagics",
362 | "writefile": "OSMagics"
363 | },
364 | "line": {
365 | "alias": "OSMagics",
366 | "alias_magic": "BasicMagics",
367 | "autoawait": "AsyncMagics",
368 | "autocall": "AutoMagics",
369 | "automagic": "AutoMagics",
370 | "autosave": "KernelMagics",
371 | "bookmark": "OSMagics",
372 | "cat": "Other",
373 | "cd": "OSMagics",
374 | "clear": "KernelMagics",
375 | "colors": "BasicMagics",
376 | "conda": "PackagingMagics",
377 | "config": "ConfigMagics",
378 | "connect_info": "KernelMagics",
379 | "cp": "Other",
380 | "debug": "ExecutionMagics",
381 | "dhist": "OSMagics",
382 | "dirs": "OSMagics",
383 | "doctest_mode": "BasicMagics",
384 | "ed": "Other",
385 | "edit": "KernelMagics",
386 | "env": "OSMagics",
387 | "gui": "BasicMagics",
388 | "hist": "Other",
389 | "history": "HistoryMagics",
390 | "killbgscripts": "ScriptMagics",
391 | "ldir": "Other",
392 | "less": "KernelMagics",
393 | "lf": "Other",
394 | "lk": "Other",
395 | "ll": "Other",
396 | "load": "CodeMagics",
397 | "load_ext": "ExtensionMagics",
398 | "loadpy": "CodeMagics",
399 | "logoff": "LoggingMagics",
400 | "logon": "LoggingMagics",
401 | "logstart": "LoggingMagics",
402 | "logstate": "LoggingMagics",
403 | "logstop": "LoggingMagics",
404 | "ls": "Other",
405 | "lsmagic": "BasicMagics",
406 | "lx": "Other",
407 | "macro": "ExecutionMagics",
408 | "magic": "BasicMagics",
409 | "man": "KernelMagics",
410 | "matplotlib": "PylabMagics",
411 | "mkdir": "Other",
412 | "more": "KernelMagics",
413 | "mv": "Other",
414 | "notebook": "BasicMagics",
415 | "page": "BasicMagics",
416 | "pastebin": "CodeMagics",
417 | "pdb": "ExecutionMagics",
418 | "pdef": "NamespaceMagics",
419 | "pdoc": "NamespaceMagics",
420 | "pfile": "NamespaceMagics",
421 | "pinfo": "NamespaceMagics",
422 | "pinfo2": "NamespaceMagics",
423 | "pip": "PackagingMagics",
424 | "popd": "OSMagics",
425 | "pprint": "BasicMagics",
426 | "precision": "BasicMagics",
427 | "prun": "ExecutionMagics",
428 | "psearch": "NamespaceMagics",
429 | "psource": "NamespaceMagics",
430 | "pushd": "OSMagics",
431 | "pwd": "OSMagics",
432 | "pycat": "OSMagics",
433 | "pylab": "PylabMagics",
434 | "qtconsole": "KernelMagics",
435 | "quickref": "BasicMagics",
436 | "recall": "HistoryMagics",
437 | "rehashx": "OSMagics",
438 | "reload_ext": "ExtensionMagics",
439 | "rep": "Other",
440 | "rerun": "HistoryMagics",
441 | "reset": "NamespaceMagics",
442 | "reset_selective": "NamespaceMagics",
443 | "rm": "Other",
444 | "rmdir": "Other",
445 | "run": "ExecutionMagics",
446 | "save": "CodeMagics",
447 | "sc": "OSMagics",
448 | "set_env": "OSMagics",
449 | "store": "StoreMagics",
450 | "sx": "OSMagics",
451 | "system": "OSMagics",
452 | "tb": "ExecutionMagics",
453 | "time": "ExecutionMagics",
454 | "timeit": "ExecutionMagics",
455 | "unalias": "OSMagics",
456 | "unload_ext": "ExtensionMagics",
457 | "who": "NamespaceMagics",
458 | "who_ls": "NamespaceMagics",
459 | "whos": "NamespaceMagics",
460 | "xdel": "NamespaceMagics",
461 | "xmode": "BasicMagics"
462 | }
463 | },
464 | "text/plain": [
465 | "Available line magics:\n",
466 | "%alias %alias_magic %autoawait %autocall %automagic %autosave %bookmark %cat %cd %clear %colors %conda %config %connect_info %cp %debug %dhist %dirs %doctest_mode %ed %edit %env %gui %hist %history %killbgscripts %ldir %less %lf %lk %ll %load %load_ext %loadpy %logoff %logon %logstart %logstate %logstop %ls %lsmagic %lx %macro %magic %man %matplotlib %mkdir %more %mv %notebook %page %pastebin %pdb %pdef %pdoc %pfile %pinfo %pinfo2 %pip %popd %pprint %precision %prun %psearch %psource %pushd %pwd %pycat %pylab %qtconsole %quickref %recall %rehashx %reload_ext %rep %rerun %reset %reset_selective %rm %rmdir %run %save %sc %set_env %store %sx %system %tb %time %timeit %unalias %unload_ext %who %who_ls %whos %xdel %xmode\n",
467 | "\n",
468 | "Available cell magics:\n",
469 | "%%! %%HTML %%SVG %%bash %%capture %%debug %%file %%html %%javascript %%js %%latex %%markdown %%perl %%prun %%pypy %%python %%python2 %%python3 %%ruby %%script %%sh %%svg %%sx %%system %%time %%timeit %%writefile\n",
470 | "\n",
471 | "Automagic is ON, % prefix IS NOT needed for line magics."
472 | ]
473 | },
474 | "execution_count": 16,
475 | "metadata": {},
476 | "output_type": "execute_result"
477 | }
478 | ],
479 | "source": [
480 | "%lsmagic"
481 | ]
482 | },
483 | {
484 | "cell_type": "markdown",
485 | "metadata": {},
486 | "source": [
487 | "In the following list I describe my favourite commands\n",
488 | "- %%time: Measure cell execution time\n",
489 | "- %env: set and get system environment variables\n",
490 | "- %who: show all variables of a specific type, e.g. str / int\n",
491 | "- %load: load code into your notebook (does not work with .ipynb, but with .py)\n",
492 | "- %run: execute external code (does work with .ipynb)"
493 | ]
494 | },
495 | {
496 | "cell_type": "code",
497 | "execution_count": 29,
498 | "metadata": {},
499 | "outputs": [
500 | {
501 | "name": "stdout",
502 | "output_type": "stream",
503 | "text": [
504 | "CPU times: user 1.31 ms, sys: 33 µs, total: 1.34 ms\n",
505 | "Wall time: 1.38 ms\n"
506 | ]
507 | }
508 | ],
509 | "source": [
510 | "%%time\n",
511 | "for i in range(1,10000):\n",
512 | " i+=1"
513 | ]
514 | },
515 | {
516 | "cell_type": "code",
517 | "execution_count": 31,
518 | "metadata": {
519 | "collapsed": true
520 | },
521 | "outputs": [
522 | {
523 | "name": "stdout",
524 | "output_type": "stream",
525 | "text": [
526 | "env: NEW_ENV=MAGIC\n"
527 | ]
528 | },
529 | {
530 | "data": {
531 | "text/plain": [
532 | "{'TERM_PROGRAM': 'Apple_Terminal',\n",
533 | " 'TERM': 'xterm-color',\n",
534 | " 'SHELL': '/bin/bash',\n",
535 | " 'TMPDIR': '/var/folders/b0/006828j97knc9z1y2jbxf57c0000gn/T/',\n",
536 | " 'CONDA_SHLVL': '1',\n",
537 | " 'CONDA_PROMPT_MODIFIER': '(base) ',\n",
538 | " 'TERM_PROGRAM_VERSION': '440',\n",
539 | " 'TERM_SESSION_ID': '692799A3-08CC-4B2B-AA6E-F76D9DCA9312',\n",
540 | " 'USER': 'astrid',\n",
541 | " 'CONDA_EXE': '/Users/astrid/opt/anaconda3/bin/conda',\n",
542 | " 'SSH_AUTH_SOCK': '/private/tmp/com.apple.launchd.tRpoCDVVO2/Listeners',\n",
543 | " 'AWS_PROFILE': 'admin',\n",
544 | " '_CE_CONDA': '',\n",
545 | " 'PATH': '/Users/astrid/opt/anaconda3/bin:/Users/astrid/opt/anaconda3/condabin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/TeX/texbin:/Applications/Postgres.app/Contents/Versions/latest/bin:/Users/astrid/local/bin',\n",
546 | " 'LaunchInstanceID': '0F4231B1-0052-4D2E-A19D-0A0A8F42DE3C',\n",
547 | " 'CONDA_PREFIX': '/Users/astrid/opt/anaconda3',\n",
548 | " '__CFBundleIdentifier': 'com.apple.Terminal',\n",
549 | " 'PWD': '/Users/astrid/work/CFDsolutions/Verfahrenstechnik_TUB/Mini_AI_Gym/1_DataManagement',\n",
550 | " 'XPC_FLAGS': '0x0',\n",
551 | " '_CE_M': '',\n",
552 | " 'XPC_SERVICE_NAME': '0',\n",
553 | " 'HOME': '/Users/astrid',\n",
554 | " 'SHLVL': '1',\n",
555 | " 'CONDA_PYTHON_EXE': '/Users/astrid/opt/anaconda3/bin/python',\n",
556 | " 'LOGNAME': 'astrid',\n",
557 | " 'LC_CTYPE': 'UTF-8',\n",
558 | " 'CONDA_DEFAULT_ENV': 'base',\n",
559 | " 'SECURITYSESSIONID': '186ab',\n",
560 | " '_': '/Users/astrid/opt/anaconda3/bin/jupyter-notebook',\n",
561 | " 'KERNEL_LAUNCH_TIMEOUT': '40',\n",
562 | " 'JPY_PARENT_PID': '4177',\n",
563 | " 'CLICOLOR': '1',\n",
564 | " 'PAGER': 'cat',\n",
565 | " 'GIT_PAGER': 'cat',\n",
566 | " 'MPLBACKEND': 'module://ipykernel.pylab.backend_inline',\n",
567 | " 'NEW_ENV': 'MAGIC'}"
568 | ]
569 | },
570 | "execution_count": 31,
571 | "metadata": {},
572 | "output_type": "execute_result"
573 | }
574 | ],
575 | "source": [
576 | "%env NEW_ENV=MAGIC\n",
577 | "%env"
578 | ]
579 | },
580 | {
581 | "cell_type": "code",
582 | "execution_count": 32,
583 | "metadata": {},
584 | "outputs": [
585 | {
586 | "name": "stdout",
587 | "output_type": "stream",
588 | "text": [
589 | "b\t c\t i\t \n"
590 | ]
591 | }
592 | ],
593 | "source": [
594 | "%who int"
595 | ]
596 | },
597 | {
598 | "cell_type": "code",
599 | "execution_count": 34,
600 | "metadata": {},
601 | "outputs": [
602 | {
603 | "name": "stdout",
604 | "output_type": "stream",
605 | "text": [
606 | "hello world\n"
607 | ]
608 | }
609 | ],
610 | "source": [
611 | "%load ./hello_world.py "
612 | ]
613 | },
614 | {
615 | "cell_type": "code",
616 | "execution_count": 23,
617 | "metadata": {},
618 | "outputs": [
619 | {
620 | "name": "stdout",
621 | "output_type": "stream",
622 | "text": [
623 | "hello world\n"
624 | ]
625 | }
626 | ],
627 | "source": [
628 | "%run ./hello_world.ipynb"
629 | ]
630 | },
631 | {
632 | "cell_type": "markdown",
633 | "metadata": {},
634 | "source": [
635 | "# Virtual Environments w/ Jupyter\n",
636 | "Python and PYthon packages are quite volatile and dynamic. So depending on versions packages might be or migt not be compatible. Virtual environemnts are very common to ensure a stable python env for your current project with the needed packages without breaking your systemwide python installation.\n",
637 | "\n",
638 | "Setting up the venv is well known\n",
639 | "\n",
640 | "```python\n",
641 | "python3 -m venv --system-site-packages NAME_ENV\n",
642 | "```\n",
643 | "\n",
644 | "But if you use jupyter-notebooks a lot for development, you want to use the same environemnts also in your notebooks:\n",
645 | "\n",
646 | "```python\n",
647 | "pip install --user ipykernel\n",
648 | "python -m ipykernel install --user --name=myenv\n",
649 | "source env/bin/activate\n",
650 | "```"
651 | ]
652 | },
653 | {
654 | "cell_type": "code",
655 | "execution_count": null,
656 | "metadata": {},
657 | "outputs": [],
658 | "source": []
659 | }
660 | ],
661 | "metadata": {
662 | "kernelspec": {
663 | "display_name": "Python 3",
664 | "language": "python",
665 | "name": "python3"
666 | },
667 | "language_info": {
668 | "codemirror_mode": {
669 | "name": "ipython",
670 | "version": 3
671 | },
672 | "file_extension": ".py",
673 | "mimetype": "text/x-python",
674 | "name": "python",
675 | "nbconvert_exporter": "python",
676 | "pygments_lexer": "ipython3",
677 | "version": "3.6.9"
678 | }
679 | },
680 | "nbformat": 4,
681 | "nbformat_minor": 2
682 | }
683 |
--------------------------------------------------------------------------------
/JupyterHacks/Jupyter-Hacks.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Intro\n",
8 | "In this document I want to give an overview of magic commands, short cuts and code snippets I find extremely helpful in my day-2-day work with jupyter-notebooks. "
9 | ]
10 | },
11 | {
12 | "cell_type": "markdown",
13 | "metadata": {},
14 | "source": [
15 | "# Command Mode\n",
16 | "I don't know if it is just me or if there are some other people as well, who came across the command mode quite late. So this would have saved me really a lot of time!\n",
17 | "\n",
18 | "To enter the command mode, just press ```esc```.\n",
19 | "\n",
20 | "- Now you can move across the cells with the arrow keys.\n",
21 | "- Enable editing of a specific cell, just by hitting ```enter```.\n",
22 | "- Insert new cells before or after the current cell with ```a```, ```b```.\n",
23 | "- Change between ```code```and ```markdown```with ```y```and ```m```.\n",
24 | "- Deleting a cell is as easy as pressing ```x```.\n",
25 | "- ...\n",
26 | "\n",
27 | "Check out https://www.dataquest.io/blog/jupyter-notebook-tips-tricks-shortcuts/ for more functionalities."
28 | ]
29 | },
30 | {
31 | "cell_type": "markdown",
32 | "metadata": {},
33 | "source": [
34 | "# Toggling code\n",
35 | "Sometimes you like to use the notebook as a nice and clean documentation for sharing your results and maybe you don't want to disturb the viewer with long and maybe messy codecells. Then you can insert one of the following code snippets at the beginning of your notebook.\n",
36 | "\n",
37 | "The first one is for toggling all code in the notebook # https://stackoverflow.com/questions/27934885/how-to-hide-code-from-cells-in-ipython-notebook-visualized-with-nbviewer\n",
38 | "\n",
39 | "The second one is for toggling one specific cell by adding the command ```hide_toggle()``` at the end https://stackoverflow.com/questions/31517194/how-to-hide-one-specific-cell-input-or-output-in-ipython-notebook/48084050"
40 | ]
41 | },
42 | {
43 | "cell_type": "code",
44 | "execution_count": 1,
45 | "metadata": {},
46 | "outputs": [
47 | {
48 | "data": {
49 | "text/html": [
50 | "\n",
62 | ""
63 | ],
64 | "text/plain": [
65 | ""
66 | ]
67 | },
68 | "execution_count": 1,
69 | "metadata": {},
70 | "output_type": "execute_result"
71 | }
72 | ],
73 | "source": [
74 | "from IPython.display import HTML\n",
75 | "\n",
76 | "HTML('''\n",
88 | "''')"
89 | ]
90 | },
91 | {
92 | "cell_type": "code",
93 | "execution_count": 36,
94 | "metadata": {},
95 | "outputs": [
96 | {
97 | "data": {
98 | "text/html": [
99 | "\n",
100 | " \n",
107 | "\n",
108 | " Toggle show/hide\n",
109 | " "
110 | ],
111 | "text/plain": [
112 | ""
113 | ]
114 | },
115 | "execution_count": 36,
116 | "metadata": {},
117 | "output_type": "execute_result"
118 | }
119 | ],
120 | "source": [
121 | "from IPython.display import HTML\n",
122 | "import random\n",
123 | "\n",
124 | "def hide_toggle(for_next=False):\n",
125 | " this_cell = \"\"\"$('div.cell.code_cell.rendered.selected')\"\"\"\n",
126 | " next_cell = this_cell + '.next()'\n",
127 | "\n",
128 | " toggle_text = 'Toggle show/hide' # text shown on toggle link\n",
129 | " target_cell = this_cell # target cell to control with toggle\n",
130 | " js_hide_current = '' # bit of JS to permanently hide code in current cell (only when toggling next cell)\n",
131 | "\n",
132 | " if for_next:\n",
133 | " target_cell = next_cell\n",
134 | " toggle_text += ' next cell'\n",
135 | " js_hide_current = this_cell + '.find(\"div.input\").hide();'\n",
136 | "\n",
137 | " js_f_name = 'code_toggle_{}'.format(str(random.randint(1,2**64)))\n",
138 | "\n",
139 | " html = \"\"\"\n",
140 | " \n",
147 | "\n",
148 | " {toggle_text}\n",
149 | " \"\"\".format(\n",
150 | " f_name=js_f_name,\n",
151 | " cell_selector=target_cell,\n",
152 | " js_hide_current=js_hide_current, \n",
153 | " toggle_text=toggle_text\n",
154 | " )\n",
155 | "\n",
156 | " return HTML(html)\n",
157 | "\n",
158 | "hide_toggle()"
159 | ]
160 | },
161 | {
162 | "cell_type": "markdown",
163 | "metadata": {},
164 | "source": [
165 | "# Toggle output\n",
166 | "Sometimes you also want to toggle the output, especially if you have created some long print statements in a loop and you don't want to scroll down so much.\n",
167 | "\n",
168 | "Therefore you have to enter the command mode again, press ```esc``` and use the shortcut ```esc + o```. The same shortcut is used to untoggle the output."
169 | ]
170 | },
171 | {
172 | "cell_type": "markdown",
173 | "metadata": {},
174 | "source": [
175 | "# Variable output\n",
176 | "One great thing about jupyter-notebook is the output of all variables, including arrays and dataframes. But by default only the last statement will produce a printed output. So, in the end you will end up with lots of cells or un/commenting lines for printing several variables. But there is an easy solution - check out the following code snippet. Found on https://www.dataquest.io/blog/jupyter-notebook-tips-tricks-shortcuts/ - there is also the description how to enable this permanently for all your notebooks."
177 | ]
178 | },
179 | {
180 | "cell_type": "code",
181 | "execution_count": 38,
182 | "metadata": {},
183 | "outputs": [
184 | {
185 | "data": {
186 | "text/html": [
187 | "\n",
188 | " \n",
195 | "\n",
196 | " Toggle show/hide\n",
197 | " "
198 | ],
199 | "text/plain": [
200 | ""
201 | ]
202 | },
203 | "execution_count": 38,
204 | "metadata": {},
205 | "output_type": "execute_result"
206 | }
207 | ],
208 | "source": [
209 | "from IPython.core.interactiveshell import InteractiveShell\n",
210 | "InteractiveShell.ast_node_interactivity = \"all\"\n",
211 | "\n",
212 | "hide_toggle()"
213 | ]
214 | },
215 | {
216 | "cell_type": "code",
217 | "execution_count": 6,
218 | "metadata": {},
219 | "outputs": [
220 | {
221 | "data": {
222 | "text/plain": [
223 | "10"
224 | ]
225 | },
226 | "execution_count": 6,
227 | "metadata": {},
228 | "output_type": "execute_result"
229 | },
230 | {
231 | "data": {
232 | "text/plain": [
233 | "20"
234 | ]
235 | },
236 | "execution_count": 6,
237 | "metadata": {},
238 | "output_type": "execute_result"
239 | },
240 | {
241 | "data": {
242 | "text/plain": [
243 | "30"
244 | ]
245 | },
246 | "execution_count": 6,
247 | "metadata": {},
248 | "output_type": "execute_result"
249 | }
250 | ],
251 | "source": [
252 | "a=10\n",
253 | "b=20\n",
254 | "c=a+b\n",
255 | "\n",
256 | "a\n",
257 | "b\n",
258 | "c"
259 | ]
260 | },
261 | {
262 | "cell_type": "markdown",
263 | "metadata": {},
264 | "source": [
265 | "# Help - man pages\n",
266 | "By adding a ```?``` in front of a function, you will get the documentation for this function, so no need to google it while coding.\n",
267 | "\n",
268 | "--> And if you want to close the documentation, just hit ```esc```."
269 | ]
270 | },
271 | {
272 | "cell_type": "code",
273 | "execution_count": 7,
274 | "metadata": {},
275 | "outputs": [],
276 | "source": [
277 | "?print"
278 | ]
279 | },
280 | {
281 | "cell_type": "markdown",
282 | "metadata": {},
283 | "source": [
284 | "# Execute shell commands\n",
285 | "For executing shell commands, e.g. creating folders, getting filenames, you just have to preset a ```!```"
286 | ]
287 | },
288 | {
289 | "cell_type": "code",
290 | "execution_count": 40,
291 | "metadata": {
292 | "collapsed": true
293 | },
294 | "outputs": [
295 | {
296 | "name": "stdout",
297 | "output_type": "stream",
298 | "text": [
299 | "total 1016\r\n",
300 | "drwxr-xr-x 7 astrid staff 224 Feb 3 08:51 \u001b[34m.\u001b[m\u001b[m\r\n",
301 | "drwxr-xr-x 7 astrid staff 224 Jan 26 20:10 \u001b[34m..\u001b[m\u001b[m\r\n",
302 | "drwxr-xr-x 5 astrid staff 160 Feb 3 08:22 \u001b[34m.ipynb_checkpoints\u001b[m\u001b[m\r\n",
303 | "-rw-r--r--@ 1 astrid staff 485805 Feb 3 08:45 1_DataManagement.ipynb\r\n",
304 | "-rw-r--r-- 1 astrid staff 20686 Feb 3 08:51 Jupyter-Hacks.ipynb\r\n",
305 | "-rw-r--r-- 1 astrid staff 881 Feb 3 08:25 hello_world.ipynb\r\n",
306 | "-rw-r--r--@ 1 astrid staff 241 Feb 3 08:43 hello_world.py\r\n"
307 | ]
308 | }
309 | ],
310 | "source": [
311 | "!ls -al"
312 | ]
313 | },
314 | {
315 | "cell_type": "markdown",
316 | "metadata": {},
317 | "source": [
318 | "# Magic commands\n",
319 | "Ipython has so called magic commands built-in, from which I find some extremely helpful!\n",
320 | "\n",
321 | "Some more examples can also be found here:\n",
322 | "https://towardsdatascience.com/top-8-magic-commands-in-jupyter-notebook-c1582e813560\n",
323 | "\n",
324 | "To get an overview of all available commands, type ```%lsmagic```."
325 | ]
326 | },
327 | {
328 | "cell_type": "code",
329 | "execution_count": 16,
330 | "metadata": {},
331 | "outputs": [
332 | {
333 | "data": {
334 | "application/json": {
335 | "cell": {
336 | "!": "OSMagics",
337 | "HTML": "Other",
338 | "SVG": "Other",
339 | "bash": "Other",
340 | "capture": "ExecutionMagics",
341 | "debug": "ExecutionMagics",
342 | "file": "Other",
343 | "html": "DisplayMagics",
344 | "javascript": "DisplayMagics",
345 | "js": "DisplayMagics",
346 | "latex": "DisplayMagics",
347 | "markdown": "DisplayMagics",
348 | "perl": "Other",
349 | "prun": "ExecutionMagics",
350 | "pypy": "Other",
351 | "python": "Other",
352 | "python2": "Other",
353 | "python3": "Other",
354 | "ruby": "Other",
355 | "script": "ScriptMagics",
356 | "sh": "Other",
357 | "svg": "DisplayMagics",
358 | "sx": "OSMagics",
359 | "system": "OSMagics",
360 | "time": "ExecutionMagics",
361 | "timeit": "ExecutionMagics",
362 | "writefile": "OSMagics"
363 | },
364 | "line": {
365 | "alias": "OSMagics",
366 | "alias_magic": "BasicMagics",
367 | "autoawait": "AsyncMagics",
368 | "autocall": "AutoMagics",
369 | "automagic": "AutoMagics",
370 | "autosave": "KernelMagics",
371 | "bookmark": "OSMagics",
372 | "cat": "Other",
373 | "cd": "OSMagics",
374 | "clear": "KernelMagics",
375 | "colors": "BasicMagics",
376 | "conda": "PackagingMagics",
377 | "config": "ConfigMagics",
378 | "connect_info": "KernelMagics",
379 | "cp": "Other",
380 | "debug": "ExecutionMagics",
381 | "dhist": "OSMagics",
382 | "dirs": "OSMagics",
383 | "doctest_mode": "BasicMagics",
384 | "ed": "Other",
385 | "edit": "KernelMagics",
386 | "env": "OSMagics",
387 | "gui": "BasicMagics",
388 | "hist": "Other",
389 | "history": "HistoryMagics",
390 | "killbgscripts": "ScriptMagics",
391 | "ldir": "Other",
392 | "less": "KernelMagics",
393 | "lf": "Other",
394 | "lk": "Other",
395 | "ll": "Other",
396 | "load": "CodeMagics",
397 | "load_ext": "ExtensionMagics",
398 | "loadpy": "CodeMagics",
399 | "logoff": "LoggingMagics",
400 | "logon": "LoggingMagics",
401 | "logstart": "LoggingMagics",
402 | "logstate": "LoggingMagics",
403 | "logstop": "LoggingMagics",
404 | "ls": "Other",
405 | "lsmagic": "BasicMagics",
406 | "lx": "Other",
407 | "macro": "ExecutionMagics",
408 | "magic": "BasicMagics",
409 | "man": "KernelMagics",
410 | "matplotlib": "PylabMagics",
411 | "mkdir": "Other",
412 | "more": "KernelMagics",
413 | "mv": "Other",
414 | "notebook": "BasicMagics",
415 | "page": "BasicMagics",
416 | "pastebin": "CodeMagics",
417 | "pdb": "ExecutionMagics",
418 | "pdef": "NamespaceMagics",
419 | "pdoc": "NamespaceMagics",
420 | "pfile": "NamespaceMagics",
421 | "pinfo": "NamespaceMagics",
422 | "pinfo2": "NamespaceMagics",
423 | "pip": "PackagingMagics",
424 | "popd": "OSMagics",
425 | "pprint": "BasicMagics",
426 | "precision": "BasicMagics",
427 | "prun": "ExecutionMagics",
428 | "psearch": "NamespaceMagics",
429 | "psource": "NamespaceMagics",
430 | "pushd": "OSMagics",
431 | "pwd": "OSMagics",
432 | "pycat": "OSMagics",
433 | "pylab": "PylabMagics",
434 | "qtconsole": "KernelMagics",
435 | "quickref": "BasicMagics",
436 | "recall": "HistoryMagics",
437 | "rehashx": "OSMagics",
438 | "reload_ext": "ExtensionMagics",
439 | "rep": "Other",
440 | "rerun": "HistoryMagics",
441 | "reset": "NamespaceMagics",
442 | "reset_selective": "NamespaceMagics",
443 | "rm": "Other",
444 | "rmdir": "Other",
445 | "run": "ExecutionMagics",
446 | "save": "CodeMagics",
447 | "sc": "OSMagics",
448 | "set_env": "OSMagics",
449 | "store": "StoreMagics",
450 | "sx": "OSMagics",
451 | "system": "OSMagics",
452 | "tb": "ExecutionMagics",
453 | "time": "ExecutionMagics",
454 | "timeit": "ExecutionMagics",
455 | "unalias": "OSMagics",
456 | "unload_ext": "ExtensionMagics",
457 | "who": "NamespaceMagics",
458 | "who_ls": "NamespaceMagics",
459 | "whos": "NamespaceMagics",
460 | "xdel": "NamespaceMagics",
461 | "xmode": "BasicMagics"
462 | }
463 | },
464 | "text/plain": [
465 | "Available line magics:\n",
466 | "%alias %alias_magic %autoawait %autocall %automagic %autosave %bookmark %cat %cd %clear %colors %conda %config %connect_info %cp %debug %dhist %dirs %doctest_mode %ed %edit %env %gui %hist %history %killbgscripts %ldir %less %lf %lk %ll %load %load_ext %loadpy %logoff %logon %logstart %logstate %logstop %ls %lsmagic %lx %macro %magic %man %matplotlib %mkdir %more %mv %notebook %page %pastebin %pdb %pdef %pdoc %pfile %pinfo %pinfo2 %pip %popd %pprint %precision %prun %psearch %psource %pushd %pwd %pycat %pylab %qtconsole %quickref %recall %rehashx %reload_ext %rep %rerun %reset %reset_selective %rm %rmdir %run %save %sc %set_env %store %sx %system %tb %time %timeit %unalias %unload_ext %who %who_ls %whos %xdel %xmode\n",
467 | "\n",
468 | "Available cell magics:\n",
469 | "%%! %%HTML %%SVG %%bash %%capture %%debug %%file %%html %%javascript %%js %%latex %%markdown %%perl %%prun %%pypy %%python %%python2 %%python3 %%ruby %%script %%sh %%svg %%sx %%system %%time %%timeit %%writefile\n",
470 | "\n",
471 | "Automagic is ON, % prefix IS NOT needed for line magics."
472 | ]
473 | },
474 | "execution_count": 16,
475 | "metadata": {},
476 | "output_type": "execute_result"
477 | }
478 | ],
479 | "source": [
480 | "%lsmagic"
481 | ]
482 | },
483 | {
484 | "cell_type": "markdown",
485 | "metadata": {},
486 | "source": [
487 | "In the following list I describe my favourite commands\n",
488 | "- %%time: Measure cell execution time\n",
489 | "- %env: set and get system environment variables\n",
490 | "- %who: show all variables of a specific type, e.g. str / int\n",
491 | "- %load: load code into your notebook (does not work with .ipynb, but with .py)\n",
492 | "- %run: execute external code (does work with .ipynb)"
493 | ]
494 | },
495 | {
496 | "cell_type": "code",
497 | "execution_count": 29,
498 | "metadata": {},
499 | "outputs": [
500 | {
501 | "name": "stdout",
502 | "output_type": "stream",
503 | "text": [
504 | "CPU times: user 1.31 ms, sys: 33 µs, total: 1.34 ms\n",
505 | "Wall time: 1.38 ms\n"
506 | ]
507 | }
508 | ],
509 | "source": [
510 | "%%time\n",
511 | "for i in range(1,10000):\n",
512 | " i+=1"
513 | ]
514 | },
515 | {
516 | "cell_type": "code",
517 | "execution_count": 31,
518 | "metadata": {
519 | "collapsed": true
520 | },
521 | "outputs": [
522 | {
523 | "name": "stdout",
524 | "output_type": "stream",
525 | "text": [
526 | "env: NEW_ENV=MAGIC\n"
527 | ]
528 | },
529 | {
530 | "data": {
531 | "text/plain": [
532 | "{'TERM_PROGRAM': 'Apple_Terminal',\n",
533 | " 'TERM': 'xterm-color',\n",
534 | " 'SHELL': '/bin/bash',\n",
535 | " 'TMPDIR': '/var/folders/b0/006828j97knc9z1y2jbxf57c0000gn/T/',\n",
536 | " 'CONDA_SHLVL': '1',\n",
537 | " 'CONDA_PROMPT_MODIFIER': '(base) ',\n",
538 | " 'TERM_PROGRAM_VERSION': '440',\n",
539 | " 'TERM_SESSION_ID': '692799A3-08CC-4B2B-AA6E-F76D9DCA9312',\n",
540 | " 'USER': 'astrid',\n",
541 | " 'CONDA_EXE': '/Users/astrid/opt/anaconda3/bin/conda',\n",
542 | " 'SSH_AUTH_SOCK': '/private/tmp/com.apple.launchd.tRpoCDVVO2/Listeners',\n",
543 | " 'AWS_PROFILE': 'admin',\n",
544 | " '_CE_CONDA': '',\n",
545 | " 'PATH': '/Users/astrid/opt/anaconda3/bin:/Users/astrid/opt/anaconda3/condabin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/TeX/texbin:/Applications/Postgres.app/Contents/Versions/latest/bin:/Users/astrid/local/bin',\n",
546 | " 'LaunchInstanceID': '0F4231B1-0052-4D2E-A19D-0A0A8F42DE3C',\n",
547 | " 'CONDA_PREFIX': '/Users/astrid/opt/anaconda3',\n",
548 | " '__CFBundleIdentifier': 'com.apple.Terminal',\n",
549 | " 'PWD': '/Users/astrid/work/CFDsolutions/Verfahrenstechnik_TUB/Mini_AI_Gym/1_DataManagement',\n",
550 | " 'XPC_FLAGS': '0x0',\n",
551 | " '_CE_M': '',\n",
552 | " 'XPC_SERVICE_NAME': '0',\n",
553 | " 'HOME': '/Users/astrid',\n",
554 | " 'SHLVL': '1',\n",
555 | " 'CONDA_PYTHON_EXE': '/Users/astrid/opt/anaconda3/bin/python',\n",
556 | " 'LOGNAME': 'astrid',\n",
557 | " 'LC_CTYPE': 'UTF-8',\n",
558 | " 'CONDA_DEFAULT_ENV': 'base',\n",
559 | " 'SECURITYSESSIONID': '186ab',\n",
560 | " '_': '/Users/astrid/opt/anaconda3/bin/jupyter-notebook',\n",
561 | " 'KERNEL_LAUNCH_TIMEOUT': '40',\n",
562 | " 'JPY_PARENT_PID': '4177',\n",
563 | " 'CLICOLOR': '1',\n",
564 | " 'PAGER': 'cat',\n",
565 | " 'GIT_PAGER': 'cat',\n",
566 | " 'MPLBACKEND': 'module://ipykernel.pylab.backend_inline',\n",
567 | " 'NEW_ENV': 'MAGIC'}"
568 | ]
569 | },
570 | "execution_count": 31,
571 | "metadata": {},
572 | "output_type": "execute_result"
573 | }
574 | ],
575 | "source": [
576 | "%env NEW_ENV=MAGIC\n",
577 | "%env"
578 | ]
579 | },
580 | {
581 | "cell_type": "code",
582 | "execution_count": 32,
583 | "metadata": {},
584 | "outputs": [
585 | {
586 | "name": "stdout",
587 | "output_type": "stream",
588 | "text": [
589 | "b\t c\t i\t \n"
590 | ]
591 | }
592 | ],
593 | "source": [
594 | "%who int"
595 | ]
596 | },
597 | {
598 | "cell_type": "code",
599 | "execution_count": 34,
600 | "metadata": {},
601 | "outputs": [
602 | {
603 | "name": "stdout",
604 | "output_type": "stream",
605 | "text": [
606 | "hello world\n"
607 | ]
608 | }
609 | ],
610 | "source": [
611 | "%load ./hello_world.py "
612 | ]
613 | },
614 | {
615 | "cell_type": "code",
616 | "execution_count": 23,
617 | "metadata": {},
618 | "outputs": [
619 | {
620 | "name": "stdout",
621 | "output_type": "stream",
622 | "text": [
623 | "hello world\n"
624 | ]
625 | }
626 | ],
627 | "source": [
628 | "%run ./hello_world.ipynb"
629 | ]
630 | },
631 | {
632 | "cell_type": "markdown",
633 | "metadata": {},
634 | "source": [
635 | "# Virtual Environments w/ Jupyter\n",
636 | "Python and PYthon packages are quite volatile and dynamic. So depending on versions packages might be or migt not be compatible. Virtual environemnts are very common to ensure a stable python env for your current project with the needed packages without breaking your systemwide python installation.\n",
637 | "\n",
638 | "Setting up the venv is well known\n",
639 | "\n",
640 | "```python\n",
641 | "python3 -m venv --system-site-packages NAME_ENV\n",
642 | "```\n",
643 | "\n",
644 | "But if you use jupyter-notebooks a lot for development, you want to use the same environemnts also in your notebooks:\n",
645 | "\n",
646 | "```python\n",
647 | "pip install --user ipykernel\n",
648 | "python -m ipykernel install --user --name=myenv\n",
649 | "source env/bin/activate\n",
650 | "```"
651 | ]
652 | },
653 | {
654 | "cell_type": "code",
655 | "execution_count": null,
656 | "metadata": {},
657 | "outputs": [],
658 | "source": []
659 | }
660 | ],
661 | "metadata": {
662 | "kernelspec": {
663 | "display_name": "Python 3",
664 | "language": "python",
665 | "name": "python3"
666 | },
667 | "language_info": {
668 | "codemirror_mode": {
669 | "name": "ipython",
670 | "version": 3
671 | },
672 | "file_extension": ".py",
673 | "mimetype": "text/x-python",
674 | "name": "python",
675 | "nbconvert_exporter": "python",
676 | "pygments_lexer": "ipython3",
677 | "version": "3.6.9"
678 | }
679 | },
680 | "nbformat": 4,
681 | "nbformat_minor": 2
682 | }
683 |
--------------------------------------------------------------------------------
/JupyterHacks/README.md:
--------------------------------------------------------------------------------
1 | Try the notebook python_hacks.ipynb to get an overview of some tips, tricks and magic commands.
2 |
--------------------------------------------------------------------------------
/JupyterHacks/hello_world.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "metadata": {},
7 | "outputs": [],
8 | "source": [
9 | "from IPython.core.interactiveshell import InteractiveShell\n",
10 | "InteractiveShell.ast_node_interactivity = \"all\""
11 | ]
12 | },
13 | {
14 | "cell_type": "code",
15 | "execution_count": null,
16 | "metadata": {},
17 | "outputs": [],
18 | "source": [
19 | "a=\"hello world\"\n",
20 | "\n",
21 | "%pinfo a\n",
22 | "\n",
23 | "print(a)"
24 | ]
25 | }
26 | ],
27 | "metadata": {
28 | "kernelspec": {
29 | "display_name": "Python 3",
30 | "language": "python",
31 | "name": "python3"
32 | },
33 | "language_info": {
34 | "codemirror_mode": {
35 | "name": "ipython",
36 | "version": 3
37 | },
38 | "file_extension": ".py",
39 | "mimetype": "text/x-python",
40 | "name": "python",
41 | "nbconvert_exporter": "python",
42 | "pygments_lexer": "ipython3",
43 | "version": "3.6.9"
44 | }
45 | },
46 | "nbformat": 4,
47 | "nbformat_minor": 2
48 | }
49 |
--------------------------------------------------------------------------------
/JupyterHacks/hello_world.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # coding: utf-8
3 |
4 | # In[ ]:
5 |
6 |
7 | from IPython.core.interactiveshell import InteractiveShell
8 | InteractiveShell.ast_node_interactivity = "all"
9 |
10 |
11 | # In[ ]:
12 |
13 |
14 | a="hello world"
15 |
16 | get_ipython().run_line_magic('pinfo', 'a')
17 |
18 | print(a)
19 |
20 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2021 Astrid Walle
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/ML/3_ML.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Introduction to Machine Learning"
8 | ]
9 | },
10 | {
11 | "cell_type": "markdown",
12 | "metadata": {},
13 | "source": [
14 | "(The examples in this notebook were inspired by my work for EmergentAlliance, the Scikit-Learn documentation and Jason Brownlee's \"Machine Learning Mastery with Python\")"
15 | ]
16 | },
17 | {
18 | "cell_type": "markdown",
19 | "metadata": {},
20 | "source": [
21 | "In this short intro course we will focus on predictive modeling. That means that we want to use the models to make predictions, e.g. a system's future behaviour or a system's response to specific inputs, aka classification and regression."
22 | ]
23 | },
24 | {
25 | "cell_type": "markdown",
26 | "metadata": {},
27 | "source": [
28 | "So from all the various types of machine learning categories we will look at **supervised learning**. So we will train a model based on labelled training data. For example when training an image recognition model for recognizing cats vs dogs you need to label a lot of pictures for training purpose upfront.\n",
29 | ""
30 | ]
31 | },
32 | {
33 | "cell_type": "markdown",
34 | "metadata": {},
35 | "source": [
36 | "The other categories cover **unsupervised learning**, e.g. clustering and **Reinforcement learning**, e.g. Deepmind's AlphaGo."
37 | ]
38 | },
39 | {
40 | "cell_type": "markdown",
41 | "metadata": {},
42 | "source": [
43 | ""
44 | ]
45 | },
46 | {
47 | "cell_type": "markdown",
48 | "metadata": {},
49 | "source": [
50 | "## Datasets:\n",
51 | "We will look at two different datasets:\n",
52 | "1. Iris Flower Dataset\n",
53 | "2. Boston Housing Prices\n",
54 | "\n",
55 | "These datasets are so called toy datasets, well known machine learning examples, and already included in the Python machine learning library scikitlearn https://scikit-learn.org/stable/datasets/toy_dataset.html. The Iris Flower dataset is an example for a classification problem, whereas the Boston Housing Price dataset is a regression example."
56 | ]
57 | },
58 | {
59 | "cell_type": "markdown",
60 | "metadata": {},
61 | "source": [
62 | "## What does a ML project always look like?\n",
63 | "* Idea --> Problem Definition / Hypothesis formulation\n",
64 | "* Analyze and Visualize your data\n",
65 | " - Understand your data (dimensions, data types, class distributions (bias!), data summary, correllations, skewness)\n",
66 | " - Visualize your data (box and whisker / violine / distribution / scatter matrix)\n",
67 | "* Data Preprocessing including data cleansing, data wrangling, data compilation, normalization, standardization\n",
68 | "* Apply algorithms and make predictions\n",
69 | "* Improve, validate and present results"
70 | ]
71 | },
72 | {
73 | "cell_type": "markdown",
74 | "metadata": {},
75 | "source": [
76 | "## Let's get started\n",
77 | "Load some libraries"
78 | ]
79 | },
80 | {
81 | "cell_type": "code",
82 | "execution_count": null,
83 | "metadata": {},
84 | "outputs": [],
85 | "source": [
86 | "import pandas as pd # data analysis\n",
87 | "import numpy as np # math operations on arrays and vectors\n",
88 | "import matplotlib.pyplot as plt # plotting\n",
89 | "# display plots directly in the notebook\n",
90 | "%matplotlib inline \n",
91 | "import sklearn # the library we use for all ML related functions, algorithms"
92 | ]
93 | },
94 | {
95 | "cell_type": "markdown",
96 | "metadata": {},
97 | "source": [
98 | "## Example 1: Iris flower dataset\n",
99 | "https://scikit-learn.org/stable/datasets/toy_dataset.html#iris-dataset\n",
100 | "4 numeric, predictive attributes (sepal length in cm, sepal width in cm, petal length in cm, petal width in cm) and the class (Iris-Setosa, Iris-Versicolour, Iris-Virginica)\n",
101 | "\n",
102 | "**Hypothesis:** One can predict the class of Iris Flower based on their attributes.\n",
103 | "\n",
104 | "Here this is just one sentence, but formulating this hypothesis is a non-trivial, iterative task, which is the basis for data and feature selection and extremely important for the overall success!\n",
105 | "\n",
106 | "### 1. Load the data"
107 | ]
108 | },
109 | {
110 | "cell_type": "code",
111 | "execution_count": null,
112 | "metadata": {},
113 | "outputs": [],
114 | "source": [
115 | "# check here again with autocompletion --> then you can see all availbale datasets\n",
116 | "# https://scikit-learn.org/stable/datasets/toy_dataset.html\n",
117 | "from sklearn.datasets import load_iris"
118 | ]
119 | },
120 | {
121 | "cell_type": "code",
122 | "execution_count": null,
123 | "metadata": {},
124 | "outputs": [],
125 | "source": [
126 | "(data, target) =load_iris(return_X_y=True, as_frame=True)"
127 | ]
128 | },
129 | {
130 | "cell_type": "code",
131 | "execution_count": null,
132 | "metadata": {},
133 | "outputs": [],
134 | "source": [
135 | "data"
136 | ]
137 | },
138 | {
139 | "cell_type": "code",
140 | "execution_count": null,
141 | "metadata": {},
142 | "outputs": [],
143 | "source": [
144 | "target"
145 | ]
146 | },
147 | {
148 | "cell_type": "markdown",
149 | "metadata": {},
150 | "source": [
151 | "We will combine this now into one dataframe and check the classes"
152 | ]
153 | },
154 | {
155 | "cell_type": "code",
156 | "execution_count": null,
157 | "metadata": {},
158 | "outputs": [],
159 | "source": [
160 | "data[\"class\"]=target"
161 | ]
162 | },
163 | {
164 | "cell_type": "code",
165 | "execution_count": null,
166 | "metadata": {},
167 | "outputs": [],
168 | "source": [
169 | "data"
170 | ]
171 | },
172 | {
173 | "cell_type": "markdown",
174 | "metadata": {},
175 | "source": [
176 | "### 2. Understand your data"
177 | ]
178 | },
179 | {
180 | "cell_type": "code",
181 | "execution_count": null,
182 | "metadata": {},
183 | "outputs": [],
184 | "source": [
185 | "data.describe()"
186 | ]
187 | },
188 | {
189 | "cell_type": "markdown",
190 | "metadata": {},
191 | "source": [
192 | "This is a classification problem, so we will check the class distribution. This is important to avoid bias due to over- oder underrepresentation of classes. Well known example of this problem are predictive maintenance (very less errors compared to normal runs, Amazon's hiring AI https://www.reuters.com/article/us-amazon-com-jobs-automation-insight-idUSKCN1MK08G)"
193 | ]
194 | },
195 | {
196 | "cell_type": "code",
197 | "execution_count": null,
198 | "metadata": {},
199 | "outputs": [],
200 | "source": [
201 | "class_counts = data.groupby('class').size()\n",
202 | "class_counts"
203 | ]
204 | },
205 | {
206 | "cell_type": "markdown",
207 | "metadata": {},
208 | "source": [
209 | "Now let's check for correlations\n",
210 | "Correlation means the relationship between two variables and how they may or may not change together.\n",
211 | "There are different methods available (--> check with ?data.corr)"
212 | ]
213 | },
214 | {
215 | "cell_type": "code",
216 | "execution_count": null,
217 | "metadata": {},
218 | "outputs": [],
219 | "source": [
220 | "correlations = data.corr(method='pearson')"
221 | ]
222 | },
223 | {
224 | "cell_type": "code",
225 | "execution_count": null,
226 | "metadata": {},
227 | "outputs": [],
228 | "source": [
229 | "correlations"
230 | ]
231 | },
232 | {
233 | "cell_type": "markdown",
234 | "metadata": {},
235 | "source": [
236 | "Let's do a heatmap plot for the correlation matrix (pandas built-in)"
237 | ]
238 | },
239 | {
240 | "cell_type": "code",
241 | "execution_count": null,
242 | "metadata": {},
243 | "outputs": [],
244 | "source": [
245 | "correlations.style.background_gradient(cmap='coolwarm').set_precision(2)"
246 | ]
247 | },
248 | {
249 | "cell_type": "markdown",
250 | "metadata": {},
251 | "source": [
252 | "Now we will also check the skewness of the distributions, assuming a normal Gaussian distribution. \n",
253 | "The skew results show a positive (right) or negative (left) skew. Values closer to zero show less skew."
254 | ]
255 | },
256 | {
257 | "cell_type": "code",
258 | "execution_count": null,
259 | "metadata": {},
260 | "outputs": [],
261 | "source": [
262 | "skew=data.skew()\n",
263 | "skew"
264 | ]
265 | },
266 | {
267 | "cell_type": "markdown",
268 | "metadata": {},
269 | "source": [
270 | "## 2. Visualize your data\n",
271 | "- Histogram\n",
272 | "- Paiplot\n",
273 | "- Density"
274 | ]
275 | },
276 | {
277 | "cell_type": "code",
278 | "execution_count": null,
279 | "metadata": {},
280 | "outputs": [],
281 | "source": [
282 | "data.hist()"
283 | ]
284 | },
285 | {
286 | "cell_type": "code",
287 | "execution_count": null,
288 | "metadata": {},
289 | "outputs": [],
290 | "source": [
291 | "data.plot(kind=\"density\", subplots=True, layout=(3,2),sharex=False)"
292 | ]
293 | },
294 | {
295 | "cell_type": "markdown",
296 | "metadata": {},
297 | "source": [
298 | "Another nice plot is the box and whisker plot, visualizing the quartiles of a distribution"
299 | ]
300 | },
301 | {
302 | "cell_type": "code",
303 | "execution_count": null,
304 | "metadata": {},
305 | "outputs": [],
306 | "source": [
307 | "data.plot(kind=\"box\", subplots=True, layout=(3,2),sharex=False)"
308 | ]
309 | },
310 | {
311 | "cell_type": "markdown",
312 | "metadata": {},
313 | "source": [
314 | "Another option are the seaborn violine plots, which give a more intuitive feeling about the distribution of values"
315 | ]
316 | },
317 | {
318 | "cell_type": "code",
319 | "execution_count": null,
320 | "metadata": {},
321 | "outputs": [],
322 | "source": [
323 | "import seaborn as sns\n",
324 | "sns.violinplot(data=data,x=\"class\", y=\"sepal length (cm)\")"
325 | ]
326 | },
327 | {
328 | "cell_type": "markdown",
329 | "metadata": {},
330 | "source": [
331 | "And last but not least a scatterplot matrix, similar to the pairplot we did already in the last session. This should also give insights about correllations."
332 | ]
333 | },
334 | {
335 | "cell_type": "code",
336 | "execution_count": null,
337 | "metadata": {},
338 | "outputs": [],
339 | "source": [
340 | "sns.pairplot(data)"
341 | ]
342 | },
343 | {
344 | "cell_type": "markdown",
345 | "metadata": {},
346 | "source": [
347 | "## 3. Data Preprocessing\n",
348 | "For this dataset, there are already some steps we don't need to take, like:\n",
349 | "Conglomeration of multiple datasources to one table, including the adaption of formats and granularities. Also we don't need to take care for missing values or NaN's. But among preprocessing there are as well\n",
350 | "- Rescaling\n",
351 | "- Normalization\n",
352 | "\n",
353 | "The goal of these transformtions is bringing the data into a format, which is most beneficial for the later applied algorithms. So for example optimization algorithms for multivariate optimizations perform better, when all attributes / parameters have the same scale. And other methods assume that input variables have a Gaussian distribution, so it is better to transform the input parameters to meet these requirements."
354 | ]
355 | },
356 | {
357 | "cell_type": "markdown",
358 | "metadata": {},
359 | "source": [
360 | "At first we look at **rescaling**. This is done to rescale all attributes (parameters) into the same range, most of the times this is the range [0,1].\n",
361 | "\n",
362 | "For applying these preprocessing steps at first we need to transform the dataframe into an array and split the arry in input and output values, here the descriptive parameters and the class."
363 | ]
364 | },
365 | {
366 | "cell_type": "code",
367 | "execution_count": null,
368 | "metadata": {},
369 | "outputs": [],
370 | "source": [
371 | "# transform into array\n",
372 | "array = data.values\n",
373 | "array"
374 | ]
375 | },
376 | {
377 | "cell_type": "code",
378 | "execution_count": null,
379 | "metadata": {},
380 | "outputs": [],
381 | "source": [
382 | "# separate array into input and output components\n",
383 | "X = array[:,0:4]\n",
384 | "Y = array[:,4]"
385 | ]
386 | },
387 | {
388 | "cell_type": "code",
389 | "execution_count": null,
390 | "metadata": {},
391 | "outputs": [],
392 | "source": [
393 | "# Now we apply the MinMaxScaler with a range of [0,1], so that afterwards all columns have a min of 0 and a max of 1.\n",
394 | "from sklearn.preprocessing import MinMaxScaler\n",
395 | "scaler = MinMaxScaler(feature_range=(0, 1))\n",
396 | "rescaledX = scaler.fit_transform(X)\n",
397 | "rescaledX"
398 | ]
399 | },
400 | {
401 | "cell_type": "markdown",
402 | "metadata": {},
403 | "source": [
404 | "Now we will apply Normalization by using the Standard Scaler, which means that each column (each attribute / parameter) will be transformed, such that afterwards each attribute has a standard distribution with mean = 0 and std. dev. = 1.\n",
405 | "Given the distribution of the data, each value in the dataset will have the mean value subtracted, and then divided by the standard deviation of the whole dataset (or feature in the multivariate case)"
406 | ]
407 | },
408 | {
409 | "cell_type": "code",
410 | "execution_count": null,
411 | "metadata": {},
412 | "outputs": [],
413 | "source": [
414 | "from sklearn.preprocessing import StandardScaler\n",
415 | "scaler = StandardScaler().fit(X)\n",
416 | "rescaledX = scaler.transform(X)\n",
417 | "rescaledX"
418 | ]
419 | },
420 | {
421 | "cell_type": "markdown",
422 | "metadata": {},
423 | "source": [
424 | "## 4. Feature Selection (Parameter Sensitivity)\n",
425 | "Now we come to an extremely interesting part, which is about finding out which parameters do really have an impact onto my outputs. This is the first time we can validate our assumptions. So we will get a qualitative and a quantitative answer to the question which parameters are important. This is also important as having irrelevant features in your data can decrease the accuracy of many models and increases the training time."
426 | ]
427 | },
428 | {
429 | "cell_type": "code",
430 | "execution_count": null,
431 | "metadata": {},
432 | "outputs": [],
433 | "source": [
434 | "# Feature Extraction with Univariate Statistical Tests (Chi-squared for classification)\n",
435 | "from sklearn.feature_selection import SelectKBest\n",
436 | "from sklearn.feature_selection import chi2\n",
437 | "# feature extraction\n",
438 | "test = SelectKBest(score_func=chi2, k=3)\n",
439 | "fit = test.fit(X, Y)\n",
440 | "# summarize scores\n",
441 | "print(fit.scores_)\n",
442 | "features = fit.transform(X)\n",
443 | "# summarize selected features\n",
444 | "print(features[0:5,:])"
445 | ]
446 | },
447 | {
448 | "cell_type": "markdown",
449 | "metadata": {},
450 | "source": [
451 | "Here we can see the scores of the features. The higher the score, the more impact they have. As we have selected to take 3 attributes into account, we can see the values of the three selected features (sepal length (cm), sepal width (cm), petal length (cm), petal width (cm)). This result also makes sense, when remembering the correlation heatmap..."
452 | ]
453 | },
454 | {
455 | "cell_type": "markdown",
456 | "metadata": {},
457 | "source": [
458 | "Another very interesting transformation, which fulfills the same job as feature extraction in terms of data reduction is the PCA. Here the complete dataset is transformed into a reduced dataset (you set the number of resulting principal components). A Singular Value Decomposition of the data is performed to project it to a lower dimensional space. "
459 | ]
460 | },
461 | {
462 | "cell_type": "code",
463 | "execution_count": null,
464 | "metadata": {},
465 | "outputs": [],
466 | "source": [
467 | "from sklearn.decomposition import PCA\n",
468 | "pca = PCA(n_components=3)\n",
469 | "fit = pca.fit(X)\n",
470 | "# summarize components\n",
471 | "print(\"Explained Variance: %s\" % fit.explained_variance_ratio_)\n",
472 | "print(fit.components_)"
473 | ]
474 | },
475 | {
476 | "cell_type": "markdown",
477 | "metadata": {},
478 | "source": [
479 | "Of course there are even more possibilities, especially when you consider that the application of ML algorithms itself will give the feature importance. So there are multiple built-in methods available in sklearn."
480 | ]
481 | },
482 | {
483 | "cell_type": "markdown",
484 | "metadata": {},
485 | "source": [
486 | "## 5. Apply ML algorithms\n",
487 | "- The first step is to split our data into **training and testing data**. We need to have a separate testing dataset, which was not used for training purpose to validate the performance and accuracy of our trained model.\n",
488 | "- **Which algorithm to take?** There is no simple answer to that. Based on your problem (classification vs regression), there are different clases of algorithms, but you cannot know beforehand whoch algorithm will perform best on your data. So it is alwyas a good idea to try different algorithms and check the performance.\n",
489 | "- How to evaluate the performance? There are different metrics available to check the **performance of a ML model**"
490 | ]
491 | },
492 | {
493 | "cell_type": "code",
494 | "execution_count": null,
495 | "metadata": {},
496 | "outputs": [],
497 | "source": [
498 | "# specifying the size of the testing data set\n",
499 | "# seed: reproducable random split --> especially important when comparing different algorithms with each other.\n",
500 | "from sklearn.model_selection import train_test_split\n",
501 | "from sklearn.linear_model import LogisticRegression\n",
502 | "test_size = 0.33\n",
503 | "seed = 7 # we set a seed to get a reproducable split - especially important when you want to compare diff. algorithms with each other\n",
504 | "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=test_size,\n",
505 | "random_state=seed)\n",
506 | "model = LogisticRegression(solver='liblinear') \n",
507 | "model.fit(X_train, Y_train)\n",
508 | "result = model.score(X_test, Y_test) \n",
509 | "print(\"Accuracy: %.3f%%\" % (result*100.0))"
510 | ]
511 | },
512 | {
513 | "cell_type": "code",
514 | "execution_count": null,
515 | "metadata": {},
516 | "outputs": [],
517 | "source": [
518 | "# Let's compare the accuracy, when we use the same data for training and testing\n",
519 | "model = LogisticRegression(solver='liblinear') \n",
520 | "model.fit(X, Y)\n",
521 | "result = model.score(X, Y) \n",
522 | "print(\"Accuracy: %.3f%%\" % (result*100.0))"
523 | ]
524 | },
525 | {
526 | "cell_type": "code",
527 | "execution_count": null,
528 | "metadata": {},
529 | "outputs": [],
530 | "source": [
531 | "# get importance\n",
532 | "model = LogisticRegression(solver='liblinear') \n",
533 | "model.fit(X_train, Y_train)\n",
534 | "importance = model.coef_[0]\n",
535 | "# summarize feature importance\n",
536 | "for i,v in enumerate(importance):\n",
537 | " print('Feature: %0d, Score: %.5f' % (i,v))\n",
538 | "# print(\"Feature: \"+str(i)+\", Score: \"+str(v))\n",
539 | "# plot feature importance\n",
540 | "plt.bar([x for x in range(len(importance))], importance)"
541 | ]
542 | },
543 | {
544 | "cell_type": "code",
545 | "execution_count": null,
546 | "metadata": {},
547 | "outputs": [],
548 | "source": [
549 | "# decision tree for feature importance on a regression problem\n",
550 | "from sklearn.datasets import make_regression\n",
551 | "from sklearn.tree import DecisionTreeRegressor\n",
552 | "model = DecisionTreeRegressor()\n",
553 | "# fit the model\n",
554 | "model.fit(X_train, Y_train)\n",
555 | "# get importance\n",
556 | "importance = model.feature_importances_\n",
557 | "# summarize feature importance\n",
558 | "for i,v in enumerate(importance):\n",
559 | "\tprint('Feature: %0d, Score: %.5f' % (i,v))\n",
560 | "# plot feature importance\n",
561 | "plt.bar([x for x in range(len(importance))], importance)"
562 | ]
563 | },
564 | {
565 | "cell_type": "markdown",
566 | "metadata": {},
567 | "source": [
568 | "### Test-Train-Splits\n",
569 | "Performing just one test-train-split and checking the performance or feature importance might be not good enough, as the result could be very good or very bad by coincidence due to this specific split. So the easiest solution is to repeat this process several times and check the averaged accuracy or use some of the ready-to-use built-in tools in scikit-learn, like KFold, cross-val-score, LeaveOneOut, ShuffleSplit."
570 | ]
571 | },
572 | {
573 | "cell_type": "markdown",
574 | "metadata": {},
575 | "source": [
576 | "### Which ML model to use?\n",
577 | "Here is just a tiny overview of some mosdels one can use for classification and regression problems. For more models, which are just built-in in sciki-learn, please refer to https://scikit-learn.org/stable/index.html and https://machinelearningmastery.com\n",
578 | "\n",
579 | "- Logistic / Linear Regression\n",
580 | "- k-nearest neighbour\n",
581 | "- Classification and Regression Trees\n",
582 | "- Support Vector Machines\n",
583 | "- Neural Networks\n",
584 | "\n",
585 | "In the following we will just use logistic regression (https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression) for our classification example and linear regression (https://scikit-learn.org/stable/modules/linear_model.html#generalized-linear-regression) for our regression example.\n"
586 | ]
587 | },
588 | {
589 | "cell_type": "markdown",
590 | "metadata": {},
591 | "source": [
592 | "### ML model evaluation\n",
593 | "For evaluating the model performance, there are different metrics available, depending on your type of problem (classification vs regression)\n",
594 | "\n",
595 | "For classification, there are for example:\n",
596 | "- Classification accuracy\n",
597 | "- Logistic Loss\n",
598 | "- Confusion Matrix\n",
599 | "- ...\n",
600 | "\n",
601 | "For regression, there are for example:\n",
602 | "- Mean Absolute Error\n",
603 | "- Mean Squared Error (R)MSE\n",
604 | "- R^2 \n",
605 | "\n",
606 | "\n",
607 | "So the accuracy alone does by far not tell you the whole story, you need to check other metrics as well!\n",
608 | "\n",
609 | "The confusion matrix is a handy presentation of the accuracy of a model with two or more classes. The table presents predictions on the x-axis and true outcomes on the y-axis. --> false negative, false positive\n",
610 | "https://www.dataschool.io/simple-guide-to-confusion-matrix-terminology/"
611 | ]
612 | },
613 | {
614 | "cell_type": "code",
615 | "execution_count": null,
616 | "metadata": {},
617 | "outputs": [],
618 | "source": [
619 | "from sklearn.model_selection import KFold\n",
620 | "from sklearn.model_selection import cross_val_score\n",
621 | "from sklearn.metrics import confusion_matrix\n",
622 | "\n",
623 | "#Lets have a look at our classification problem:\n",
624 | "kfold = KFold(n_splits=10, random_state=7, shuffle=True)\n",
625 | "model = LogisticRegression(solver='liblinear')\n",
626 | "\n",
627 | "# Classification accuracy:\n",
628 | "scoring = 'accuracy'\n",
629 | "results = cross_val_score(model, X, Y, cv=kfold, scoring=scoring) \n",
630 | "print(\"Accuracy: %.3f (%.3f)\" % (results.mean(), results.std()))\n",
631 | "\n",
632 | "# Logistic Loss\n",
633 | "scoring = 'neg_log_loss'\n",
634 | "results = cross_val_score(model, X, Y, cv=kfold, scoring=scoring) \n",
635 | "print(\"Logloss: %.3f (%.3f)\" % (results.mean(), results.std()))\n",
636 | "\n",
637 | "# Confusion Matrix\n",
638 | "model.fit(X_train, Y_train)\n",
639 | "predicted = model.predict(X_test)\n",
640 | "matrix = confusion_matrix(Y_test, predicted)\n",
641 | "print(matrix)"
642 | ]
643 | },
644 | {
645 | "cell_type": "markdown",
646 | "metadata": {},
647 | "source": [
648 | "## Regression Example: Boston Housing Example"
649 | ]
650 | },
651 | {
652 | "cell_type": "code",
653 | "execution_count": null,
654 | "metadata": {},
655 | "outputs": [],
656 | "source": [
657 | "import sklearn\n",
658 | "from sklearn.datasets import load_boston "
659 | ]
660 | },
661 | {
662 | "cell_type": "code",
663 | "execution_count": null,
664 | "metadata": {},
665 | "outputs": [],
666 | "source": [
667 | "data =load_boston(return_X_y=False)"
668 | ]
669 | },
670 | {
671 | "cell_type": "code",
672 | "execution_count": null,
673 | "metadata": {},
674 | "outputs": [],
675 | "source": [
676 | "print(data.DESCR)"
677 | ]
678 | },
679 | {
680 | "cell_type": "code",
681 | "execution_count": null,
682 | "metadata": {},
683 | "outputs": [],
684 | "source": [
685 | "df=pd.DataFrame(data.data)"
686 | ]
687 | },
688 | {
689 | "cell_type": "code",
690 | "execution_count": null,
691 | "metadata": {},
692 | "outputs": [],
693 | "source": [
694 | "df.columns=data.feature_names"
695 | ]
696 | },
697 | {
698 | "cell_type": "code",
699 | "execution_count": null,
700 | "metadata": {},
701 | "outputs": [],
702 | "source": [
703 | "df"
704 | ]
705 | },
706 | {
707 | "cell_type": "code",
708 | "execution_count": null,
709 | "metadata": {},
710 | "outputs": [],
711 | "source": [
712 | "df[\"MEDV\"]=data.target"
713 | ]
714 | },
715 | {
716 | "cell_type": "code",
717 | "execution_count": null,
718 | "metadata": {},
719 | "outputs": [],
720 | "source": [
721 | "df"
722 | ]
723 | },
724 | {
725 | "cell_type": "markdown",
726 | "metadata": {},
727 | "source": [
728 | "Now we start again with our procedure:\n",
729 | "* Hypothesis\n",
730 | "* Understand and visualize the data \n",
731 | "* Preprocessing\n",
732 | "* Feature Selection\n",
733 | "* Apply Model\n",
734 | "* Evaluate Results\n",
735 | "\n",
736 | "Our **Hypothesis** here is, that we can actually predict the price of a house based on attributes of the geographic area, population and the property."
737 | ]
738 | },
739 | {
740 | "cell_type": "code",
741 | "execution_count": null,
742 | "metadata": {},
743 | "outputs": [],
744 | "source": [
745 | "df.describe()"
746 | ]
747 | },
748 | {
749 | "cell_type": "code",
750 | "execution_count": null,
751 | "metadata": {},
752 | "outputs": [],
753 | "source": [
754 | "sns.pairplot(df[[\"DIS\",\"RM\",\"CRIM\",\"LSTAT\",\"MEDV\"]])"
755 | ]
756 | },
757 | {
758 | "cell_type": "code",
759 | "execution_count": null,
760 | "metadata": {},
761 | "outputs": [],
762 | "source": [
763 | "from sklearn.linear_model import LinearRegression\n",
764 | "\n",
765 | "# Now we do the \n",
766 | "# preprocessing\n",
767 | "# feature selection\n",
768 | "# training-test-split\n",
769 | "# ML model application\n",
770 | "# evaluation\n",
771 | "array = df.values\n",
772 | "X = array[:,0:13]\n",
773 | "Y = array[:,13]\n",
774 | "\n",
775 | "# preprocessing\n",
776 | "scaler = StandardScaler().fit(X)\n",
777 | "rescaledX = scaler.transform(X)\n",
778 | "\n",
779 | "# feature selection\n",
780 | "test = SelectKBest(k=6)\n",
781 | "fit = test.fit(rescaledX, Y)\n",
782 | "features = fit.transform(X)\n",
783 | "\n",
784 | "# train-test-split\n",
785 | "X_train, X_test, Y_train, Y_test = train_test_split(features, Y, test_size=0.3,\n",
786 | "random_state=5)\n",
787 | "\n",
788 | "# build model\n",
789 | "kfold = KFold(n_splits=10, random_state=7, shuffle=True)\n",
790 | "model = LinearRegression()\n",
791 | "model.fit(X_train,Y_train)\n",
792 | "acc = model.score(X_test, Y_test) \n",
793 | "\n",
794 | "# evaluate model\n",
795 | "model = LinearRegression()\n",
796 | "scoring = 'neg_mean_squared_error'\n",
797 | "results = cross_val_score(model, X, Y, cv=kfold, scoring=scoring) \n",
798 | "\n",
799 | "print(\"Accuracy: %.3f%%\" % (acc*100.0))\n",
800 | "print(\"MSE: %.3f (%.3f)\" % (results.mean(), results.std()))\n",
801 | "\n",
802 | "\n",
803 | "# And now:\n",
804 | "# Make predictions\n",
805 | "# make predictions\n",
806 | "# model.predict(new_data)"
807 | ]
808 | },
809 | {
810 | "cell_type": "markdown",
811 | "metadata": {},
812 | "source": [
813 | "### What comes next?\n",
814 | "---> Hyperparameter optimization.\n",
815 | "For advanced ML algorithms you have to provide options and settings by yourself. These of course also have an impact onto your model performance and accuracy. Here you can perform so-called grid searches to find the optimal settings for your dataset.\n",
816 | "\n",
817 | "**GridSearchCV**"
818 | ]
819 | },
820 | {
821 | "cell_type": "markdown",
822 | "metadata": {},
823 | "source": [
824 | "## What does a typical project look like:\n",
825 | "* Data engineering - **A LOT**\n",
826 | "* Applying actual ML algorithms - 5% of the time. \n",
827 | "(If you have your dataset ready to apply algorithms you have already done like 100% of the work. Of course afterwards you still need to validate and present your results)"
828 | ]
829 | },
830 | {
831 | "cell_type": "markdown",
832 | "metadata": {},
833 | "source": [
834 | ""
835 | ]
836 | },
837 | {
838 | "cell_type": "markdown",
839 | "metadata": {},
840 | "source": [
841 | "### Example: Emergent Alliance - Health Risk Index for Europe\n",
842 | "https://emergentalliance.org\n",
843 | "What we wanted to do: Predict the risk of getting infected, when travelling to a specific region.\n",
844 | "\n",
845 | "We actually spent weeks formulating and reformulatin our hypothesis to (re-)consider influencing attributes, trying to distinguish between causes and effects.\n",
846 | "\n",
847 | "In the end we spent most of the time with data engineering for:\n",
848 | "Population density, intensive care units, mobility, case numbers, sentiment, acceptance of governemnt orders.\n",
849 | "The biggest amount of time was spent on checking data sources, getting the data, reading data dictionaries and understanding the data, creating automatic downloads and data pipelines, data preprocessig, bringing the preprocessed data into a database. We had to fight lots of issues with data quality and data granularity (time and geographic) for different countries.\n",
850 | "\n",
851 | "Also afterwards the visual and textual processing and presentation took quite some time (writing blogs, building dashboards, cleaning up databases, ...)"
852 | ]
853 | },
854 | {
855 | "cell_type": "markdown",
856 | "metadata": {},
857 | "source": [
858 | "## Image Recognition\n",
859 | "It is actually quite easy to build a simple classification model (cats vs dogs), so when you are interested in applying something like this maybe to your experimental data (bubble column pictures or postprocessing contour plots), here are some links to get started:\n",
860 | "https://medium.com/@nina95dan/simple-image-classification-with-resnet-50-334366e7311a\n",
861 | "https://medium.com/abraia/getting-started-with-image-recognition-and-convolutional-neural-networks-in-5-minutes-28c1dfdd401"
862 | ]
863 | },
864 | {
865 | "cell_type": "code",
866 | "execution_count": null,
867 | "metadata": {},
868 | "outputs": [],
869 | "source": []
870 | },
871 | {
872 | "cell_type": "code",
873 | "execution_count": null,
874 | "metadata": {},
875 | "outputs": [],
876 | "source": []
877 | },
878 | {
879 | "cell_type": "code",
880 | "execution_count": null,
881 | "metadata": {},
882 | "outputs": [],
883 | "source": []
884 | },
885 | {
886 | "cell_type": "code",
887 | "execution_count": null,
888 | "metadata": {},
889 | "outputs": [],
890 | "source": []
891 | }
892 | ],
893 | "metadata": {
894 | "kernelspec": {
895 | "display_name": "sklearn_024",
896 | "language": "python",
897 | "name": "sklearn_024"
898 | },
899 | "language_info": {
900 | "codemirror_mode": {
901 | "name": "ipython",
902 | "version": 3
903 | },
904 | "file_extension": ".py",
905 | "mimetype": "text/x-python",
906 | "name": "python",
907 | "nbconvert_exporter": "python",
908 | "pygments_lexer": "ipython3",
909 | "version": "3.6.9"
910 | }
911 | },
912 | "nbformat": 4,
913 | "nbformat_minor": 2
914 | }
915 |
--------------------------------------------------------------------------------
/ML/Bereiche-des-Machine-Learnings.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/astridwalle/python_jupyter_basics/27bbd1591fed3ad9b1def6b9eb31efe8fbef1f3a/ML/Bereiche-des-Machine-Learnings.png
--------------------------------------------------------------------------------
/ML/HealthRiskIndex.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/astridwalle/python_jupyter_basics/27bbd1591fed3ad9b1def6b9eb31efe8fbef1f3a/ML/HealthRiskIndex.png
--------------------------------------------------------------------------------
/ML/deepmind_parkour.0.gif.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/astridwalle/python_jupyter_basics/27bbd1591fed3ad9b1def6b9eb31efe8fbef1f3a/ML/deepmind_parkour.0.gif.mp4
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # python_jupyter_basics
2 | Notebooks for showcasing basic functionalities of python and jupyter
3 |
4 | - Jupyter Hacks
5 | [](https://mybinder.org/v2/gh/astridwalle/python_jupyter_basics.git/HEAD?filepath=JupyterHacks%2FJupyter-Hacks.ipynb)
6 | - Data Management
7 | [](https://mybinder.org/v2/gh/astridwalle/python_jupyter_basics/HEAD?filepath=DataManagement%2F1_DataManagement.ipynb)
8 | - Visualization
9 | [](https://mybinder.org/v2/gh/astridwalle/python_jupyter_basics/HEAD?filepath=DataVisualization%2F2_Visualization.ipynb)
10 | - ML
11 | [](https://mybinder.org/v2/gh/astridwalle/python_jupyter_basics/master)
12 |
--------------------------------------------------------------------------------