\n",
240 | "\n",
241 | " \n",
242 | "
\n",
271 | "
"
272 | ],
273 | "output_type": "pyout",
274 | "prompt_number": 11,
275 | "text": [
276 | " firstNames heights\n",
277 | "0 jeff 188.2\n",
278 | "1 roger 181.3\n",
279 | "2 andrew 193.4\n",
280 | "3 brian 192.3"
281 | ]
282 | }
283 | ],
284 | "prompt_number": 11
285 | },
286 | {
287 | "cell_type": "code",
288 | "collapsed": false,
289 | "input": [
290 | "# factors is equivalent to pandas Categorical\n",
291 | "smoker = np.array(['yes', 'no', 'yes', 'yes'])\n",
292 | "smokerFactor = pd.Categorical.from_array(smoker)\n",
293 | "smokerFactor"
294 | ],
295 | "language": "python",
296 | "metadata": {},
297 | "outputs": [
298 | {
299 | "output_type": "pyout",
300 | "prompt_number": 12,
301 | "text": [
302 | "Categorical: \n",
303 | "array(['yes', 'no', 'yes', 'yes'], dtype=object)\n",
304 | "Levels (2): Index(['no', 'yes'], dtype=object)"
305 | ]
306 | }
307 | ],
308 | "prompt_number": 12
309 | },
310 | {
311 | "cell_type": "code",
312 | "collapsed": false,
313 | "input": [
314 | "# R's NA missing values is equivalent to NaN\n",
315 | "vector1 = np.array([188.2, 181.3, 193.4, NaN])\n",
316 | "print vector1\n",
317 | "print isnan(vector1)"
318 | ],
319 | "language": "python",
320 | "metadata": {},
321 | "outputs": [
322 | {
323 | "output_type": "stream",
324 | "stream": "stdout",
325 | "text": [
326 | "[ 188.2 181.3 193.4 nan]\n",
327 | "[False False False True]\n"
328 | ]
329 | }
330 | ],
331 | "prompt_number": 13
332 | },
333 | {
334 | "cell_type": "code",
335 | "collapsed": false,
336 | "input": [
337 | "# subsetting\n",
338 | "vector1 = np.array([188.2, 181.3, 193.4, 192.3])\n",
339 | "vector2 = np.array(['jeff', 'roger', 'andrew', 'brian'])\n",
340 | "\n",
341 | "myDataFrame = pd.DataFrame(dict(heights = vector1, firstNames = vector2))\n",
342 | "\n",
343 | "print '------------------'\n",
344 | "print vector1[0]\n",
345 | "print '------------------'\n",
346 | "print vector1[[0, 1, 3]]\n",
347 | "print '------------------'\n",
348 | "print myDataFrame.ix[0, 0:2] # appears transposed as compared to R\n",
349 | "print '------------------'\n",
350 | "print myDataFrame['firstNames'] # there's no 'Levels' as in R\n",
351 | "print '------------------'\n",
352 | "print myDataFrame[myDataFrame['firstNames'] == 'jeff']\n",
353 | "print '------------------'\n",
354 | "print myDataFrame[myDataFrame['heights'] < 190]"
355 | ],
356 | "language": "python",
357 | "metadata": {},
358 | "outputs": [
359 | {
360 | "output_type": "stream",
361 | "stream": "stdout",
362 | "text": [
363 | "------------------\n",
364 | "188.2\n",
365 | "------------------\n",
366 | "[ 188.2 181.3 192.3]\n",
367 | "------------------\n",
368 | "firstNames jeff\n",
369 | "heights 188.2\n",
370 | "Name: 0\n",
371 | "------------------\n",
372 | "0 jeff\n",
373 | "1 roger\n",
374 | "2 andrew\n",
375 | "3 brian\n",
376 | "Name: firstNames\n",
377 | "------------------\n",
378 | " firstNames heights\n",
379 | "0 jeff 188.2\n",
380 | "------------------\n",
381 | " firstNames heights\n",
382 | "0 jeff 188.2\n",
383 | "1 roger 181.3\n"
384 | ]
385 | }
386 | ],
387 | "prompt_number": 14
388 | },
389 | {
390 | "cell_type": "code",
391 | "collapsed": true,
392 | "input": [],
393 | "language": "python",
394 | "metadata": {},
395 | "outputs": []
396 | }
397 | ],
398 | "metadata": {}
399 | }
400 | ]
401 | }
--------------------------------------------------------------------------------
/week1/simulation.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "metadata": {
3 | "name": "simulation"
4 | },
5 | "nbformat": 3,
6 | "nbformat_minor": 0,
7 | "worksheets": [
8 | {
9 | "cells": [
10 | {
11 | "cell_type": "heading",
12 | "level": 2,
13 | "metadata": {},
14 | "source": [
15 | "Simulation Basics -- equivalent in Python"
16 | ]
17 | },
18 | {
19 | "cell_type": "code",
20 | "collapsed": false,
21 | "input": [
22 | "import numpy as np\n",
23 | "import scipy.stats as s"
24 | ],
25 | "language": "python",
26 | "metadata": {},
27 | "outputs": [],
28 | "prompt_number": 1
29 | },
30 | {
31 | "cell_type": "code",
32 | "collapsed": false,
33 | "input": [
34 | "# normal distribution\n",
35 | "# normal(mean, stdev, size)\n",
36 | "heights = np.random.normal(188, 3, 10)\n",
37 | "print heights"
38 | ],
39 | "language": "python",
40 | "metadata": {},
41 | "outputs": [
42 | {
43 | "output_type": "stream",
44 | "stream": "stdout",
45 | "text": [
46 | "[ 182.67139438 192.17804988 190.09008635 186.57278872 194.03622068\n",
47 | " 191.88272193 183.61975607 186.97670622 187.82621993 189.9658577 ]\n"
48 | ]
49 | }
50 | ],
51 | "prompt_number": 2
52 | },
53 | {
54 | "cell_type": "code",
55 | "collapsed": false,
56 | "input": [
57 | "# binomial distribution\n",
58 | "# binomial(n, p, size)\n",
59 | "coinFlips = np.random.binomial(10, 0.5, 10)\n",
60 | "print coinFlips"
61 | ],
62 | "language": "python",
63 | "metadata": {},
64 | "outputs": [
65 | {
66 | "output_type": "stream",
67 | "stream": "stdout",
68 | "text": [
69 | "[4 6 5 6 7 3 4 6 5 6]\n"
70 | ]
71 | }
72 | ],
73 | "prompt_number": 3
74 | },
75 | {
76 | "cell_type": "code",
77 | "collapsed": false,
78 | "input": [
79 | "# normal density\n",
80 | "x = np.linspace(-5, 5, num=10)\n",
81 | "normalDensity = s.norm.pdf(x, 0, 1)\n",
82 | "print np.around(normalDensity, decimals=2)"
83 | ],
84 | "language": "python",
85 | "metadata": {},
86 | "outputs": [
87 | {
88 | "output_type": "stream",
89 | "stream": "stdout",
90 | "text": [
91 | "[ 0. 0. 0.01 0.1 0.34 0.34 0.1 0.01 0. 0. ]\n"
92 | ]
93 | }
94 | ],
95 | "prompt_number": 4
96 | },
97 | {
98 | "cell_type": "code",
99 | "collapsed": false,
100 | "input": [
101 | "# binomial density\n",
102 | "x = np.arange(0, 11, 1) # note that it's (0, 11, 1) instead of (0, 10, 1)\n",
103 | "binomialDensity = s.binom.pmf(x, 10, 0.5)\n",
104 | "print np.around(binomialDensity, decimals=2)"
105 | ],
106 | "language": "python",
107 | "metadata": {},
108 | "outputs": [
109 | {
110 | "output_type": "stream",
111 | "stream": "stdout",
112 | "text": [
113 | "[ 0. 0.01 0.04 0.12 0.21 0.25 0.21 0.12 0.04 0.01 0. ]\n"
114 | ]
115 | }
116 | ],
117 | "prompt_number": 5
118 | },
119 | {
120 | "cell_type": "code",
121 | "collapsed": false,
122 | "input": [
123 | "# 'sample' draws a random sample with and without replacement --> numpy.random.choice in numpy 1.7\n",
124 | "heights = np.random.normal(188, 3, 10)\n",
125 | "\n",
126 | "# with replacement\n",
127 | "print 'random sampling with replacement:'\n",
128 | "print np.random.choice(heights, size=10, replace=True, p=None)\n",
129 | "\n",
130 | "# without replacement\n",
131 | "print '\\nrandom sampling without replacement'\n",
132 | "print np.random.choice(heights, size=10, replace=False, p=None)\n",
133 | "\n",
134 | "# sample according to a set of probability\n",
135 | "probs = [0.4, 0.3, 0.2, 0.1, 0, 0, 0, 0, 0, 0]\n",
136 | "print '\\nrandom sampling with replacement according to a set of probability:'\n",
137 | "print probs\n",
138 | "print '\\nsum of probabilities:'\n",
139 | "print sum(probs)\n",
140 | "print '\\nsampled:'\n",
141 | "print np.random.choice(heights, size=10, replace=True, p=probs)"
142 | ],
143 | "language": "python",
144 | "metadata": {},
145 | "outputs": [
146 | {
147 | "output_type": "stream",
148 | "stream": "stdout",
149 | "text": [
150 | "random sampling with replacement:\n",
151 | "[ 188.51730245 187.57465384 187.7094007 187.34057076 189.44858637\n",
152 | " 187.57465384 188.51730245 184.62068782 187.57465384 189.4947812 ]\n",
153 | "\n",
154 | "random sampling without replacement\n",
155 | "[ 187.57465384 187.7094007 189.92494462 186.40851384 188.8397573\n",
156 | " 189.44858637 188.51730245 187.34057076 189.4947812 184.62068782]\n",
157 | "\n",
158 | "random sampling with replacement according to a set of probability:\n",
159 | "[0.4, 0.3, 0.2, 0.1, 0, 0, 0, 0, 0, 0]\n",
160 | "\n",
161 | "sum of probabilities:\n",
162 | "1.0\n",
163 | "\n",
164 | "sampled:\n",
165 | "[ 189.44858637 189.44858637 184.62068782 189.44858637 189.44858637\n",
166 | " 184.62068782 189.44858637 186.40851384 189.44858637 189.44858637]\n"
167 | ]
168 | }
169 | ],
170 | "prompt_number": 8
171 | },
172 | {
173 | "cell_type": "code",
174 | "collapsed": false,
175 | "input": [
176 | "# setting a seed\n",
177 | "np.random.seed(12345)\n",
178 | "print np.random.normal(0, 1, 5)\n",
179 | "print '-------------------------------------------------------------'\n",
180 | "\n",
181 | "np.random.seed(12345)\n",
182 | "print np.random.normal(0, 1, 5)"
183 | ],
184 | "language": "python",
185 | "metadata": {},
186 | "outputs": [
187 | {
188 | "output_type": "stream",
189 | "stream": "stdout",
190 | "text": [
191 | "[-0.20470766 0.47894334 -0.51943872 -0.5557303 1.96578057]\n",
192 | "-------------------------------------------------------------\n",
193 | "[-0.20470766 0.47894334 -0.51943872 -0.5557303 1.96578057]\n"
194 | ]
195 | }
196 | ],
197 | "prompt_number": 9
198 | },
199 | {
200 | "cell_type": "code",
201 | "collapsed": false,
202 | "input": [],
203 | "language": "python",
204 | "metadata": {},
205 | "outputs": []
206 | }
207 | ],
208 | "metadata": {}
209 | }
210 | ]
211 | }
--------------------------------------------------------------------------------
/week1/wk1_quiz.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "metadata": {
3 | "name": "wk1_quiz"
4 | },
5 | "nbformat": 3,
6 | "nbformat_minor": 0,
7 | "worksheets": [
8 | {
9 | "cells": [
10 | {
11 | "cell_type": "heading",
12 | "level": 2,
13 | "metadata": {},
14 | "source": [
15 | "Week 1 quiz"
16 | ]
17 | },
18 | {
19 | "cell_type": "code",
20 | "collapsed": false,
21 | "input": [
22 | "import pandas as pd"
23 | ],
24 | "language": "python",
25 | "metadata": {},
26 | "outputs": [],
27 | "prompt_number": 2
28 | },
29 | {
30 | "cell_type": "code",
31 | "collapsed": false,
32 | "input": [
33 | "%load_ext rmagic"
34 | ],
35 | "language": "python",
36 | "metadata": {},
37 | "outputs": [],
38 | "prompt_number": 3
39 | },
40 | {
41 | "cell_type": "markdown",
42 | "metadata": {},
43 | "source": [
44 | "Question 3. Here we'll use R just to generate the same data as required in the course quiz."
45 | ]
46 | },
47 | {
48 | "cell_type": "code",
49 | "collapsed": false,
50 | "input": [
51 | "%%R -o dataFrame\n",
52 | "set.seed(31);\n",
53 | "heightsCM = rnorm(30,mean=188, sd=5);\n",
54 | "weightsK = rnorm(30,mean=84,sd=3); \n",
55 | "hasDaughter = sample(c(TRUE,FALSE),size=30,replace=T); \n",
56 | "dataFrame = data.frame(heightsCM,weightsK,hasDaughter);"
57 | ],
58 | "language": "python",
59 | "metadata": {},
60 | "outputs": [],
61 | "prompt_number": 4
62 | },
63 | {
64 | "cell_type": "code",
65 | "collapsed": false,
66 | "input": [
67 | "p = pd.DataFrame(dataFrame.T, columns=['heightsCM', 'weightsK', 'hasDaughter'])"
68 | ],
69 | "language": "python",
70 | "metadata": {},
71 | "outputs": [],
72 | "prompt_number": 5
73 | },
74 | {
75 | "cell_type": "markdown",
76 | "metadata": {},
77 | "source": [
78 | "Question 4. Here we'll use R just to generate the same data as required in the course quiz."
79 | ]
80 | },
81 | {
82 | "cell_type": "code",
83 | "collapsed": false,
84 | "input": [
85 | "%%R -o s\n",
86 | "set.seed(41);\n",
87 | "cauchyValues = rcauchy(100)\n",
88 | "\n",
89 | "set.seed(415);\n",
90 | "s = sample(cauchyValues,size=10,replace=T)"
91 | ],
92 | "language": "python",
93 | "metadata": {},
94 | "outputs": [],
95 | "prompt_number": 7
96 | },
97 | {
98 | "cell_type": "code",
99 | "collapsed": false,
100 | "input": [
101 | "s"
102 | ],
103 | "language": "python",
104 | "metadata": {},
105 | "outputs": [
106 | {
107 | "output_type": "pyout",
108 | "prompt_number": 8,
109 | "text": [
110 | "array([ 0.80847186, -1.11228626, 0.37166707, -1.16507204, -2.97017695,\n",
111 | " -1.02310311, 0.31152624, 0.28154888, 0.2414012 , -1.11228626])"
112 | ]
113 | }
114 | ],
115 | "prompt_number": 8
116 | },
117 | {
118 | "cell_type": "code",
119 | "collapsed": false,
120 | "input": [],
121 | "language": "python",
122 | "metadata": {},
123 | "outputs": []
124 | }
125 | ],
126 | "metadata": {}
127 | }
128 | ]
129 | }
--------------------------------------------------------------------------------
/week2/getting_data.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "metadata": {
3 | "name": "getting_data"
4 | },
5 | "nbformat": 3,
6 | "nbformat_minor": 0,
7 | "worksheets": [
8 | {
9 | "cells": [
10 | {
11 | "cell_type": "heading",
12 | "level": 2,
13 | "metadata": {},
14 | "source": [
15 | "Getting data"
16 | ]
17 | },
18 | {
19 | "cell_type": "markdown",
20 | "metadata": {},
21 | "source": [
22 | "Getting and setting directory, equivalent to R getwd() and setwd() commands\n",
23 | "\n",
24 | "This is the more general Python way. In IPython we can use system commands directly prefixed by an exclamation mark (!)"
25 | ]
26 | },
27 | {
28 | "cell_type": "code",
29 | "collapsed": false,
30 | "input": [
31 | "import os\n",
32 | "\n",
33 | "os.getcwd()"
34 | ],
35 | "language": "python",
36 | "metadata": {},
37 | "outputs": [
38 | {
39 | "output_type": "pyout",
40 | "prompt_number": 2,
41 | "text": [
42 | "'/Users/erriza/dataanalysis/week2'"
43 | ]
44 | }
45 | ],
46 | "prompt_number": 2
47 | },
48 | {
49 | "cell_type": "code",
50 | "collapsed": false,
51 | "input": [
52 | "os.chdir('..')\n",
53 | "os.getcwd()"
54 | ],
55 | "language": "python",
56 | "metadata": {},
57 | "outputs": [
58 | {
59 | "output_type": "pyout",
60 | "prompt_number": 3,
61 | "text": [
62 | "'/Users/erriza/dataanalysis'"
63 | ]
64 | }
65 | ],
66 | "prompt_number": 3
67 | },
68 | {
69 | "cell_type": "code",
70 | "collapsed": false,
71 | "input": [
72 | "os.chdir('./week2/')\n",
73 | "os.getcwd()"
74 | ],
75 | "language": "python",
76 | "metadata": {},
77 | "outputs": [
78 | {
79 | "output_type": "pyout",
80 | "prompt_number": 4,
81 | "text": [
82 | "'/Users/erriza/dataanalysis/week2'"
83 | ]
84 | }
85 | ],
86 | "prompt_number": 4
87 | },
88 | {
89 | "cell_type": "markdown",
90 | "metadata": {},
91 | "source": [
92 | "Load CSV data. We'll use mostly `pandas`."
93 | ]
94 | },
95 | {
96 | "cell_type": "code",
97 | "collapsed": false,
98 | "input": [
99 | "import pandas as pd\n",
100 | "\n",
101 | "fileUrl = 'https://data.baltimorecity.gov/api/views/dz54-2aru/rows.csv?accessType=DOWNLOAD'\n",
102 | "\n",
103 | "# we can directly use read_csv to download the file\n",
104 | "# this is equivalent to R's combined download.file() and read.table() or read.csv() commands\n",
105 | "cameraData = pd.read_csv(fileUrl)\n",
106 | "\n",
107 | "# save data locally\n",
108 | "cameraData.to_csv('../data/cameras.csv', index=False)\n",
109 | "\n",
110 | "# for simplicity I'll use IPython tricks to list folder contents\n",
111 | "!ls ../data\n",
112 | "\n",
113 | "\n",
114 | "# get current date and time\n",
115 | "# this is equivalent to R date() command\n",
116 | "# note that I use IPython ! prefix to run my system's command\n",
117 | "dateDownloaded = !date\n",
118 | "print '\\nDate downloaded: ' + str(dateDownloaded)\n",
119 | "\n",
120 | "cameraData.head()"
121 | ],
122 | "language": "python",
123 | "metadata": {},
124 | "outputs": [
125 | {
126 | "output_type": "stream",
127 | "stream": "stdout",
128 | "text": [
129 | "camera.xls face.rda loansData.csv samsungData.csv\r\n",
130 | "camera.xlsx gaData.csv movies.txt samsungData.rda\r\n",
131 | "cameras.csv gaData.rda ravensData.csv ss06pid.csv\r\n",
132 | "camerasModified.csv galton.csv ravensData.rda warpbreaks.csv\r\n"
133 | ]
134 | },
135 | {
136 | "output_type": "stream",
137 | "stream": "stdout",
138 | "text": [
139 | "\n",
140 | "Date downloaded: ['Mon Mar 18 21:29:11 CET 2013']\n"
141 | ]
142 | },
143 | {
144 | "html": [
145 | "| \n", 244 | " | firstNames | \n", 245 | "heights | \n", 246 | "
|---|---|---|
| 0 | \n", 251 | "jeff | \n", 252 | "188.2 | \n", 253 | "
| 1 | \n", 256 | "roger | \n", 257 | "181.3 | \n", 258 | "
| 2 | \n", 261 | "andrew | \n", 262 | "193.4 | \n", 263 | "
| 3 | \n", 266 | "brian | \n", 267 | "192.3 | \n", 268 | "
\n",
146 | "\n",
147 | " \n",
148 | "
\n",
206 | "
"
207 | ],
208 | "output_type": "pyout",
209 | "prompt_number": 7,
210 | "text": [
211 | " address direction street crossStreet \\\n",
212 | "0 S CATON AVE & BENSON AVE N/B Caton Ave Benson Ave \n",
213 | "1 S CATON AVE & BENSON AVE S/B Caton Ave Benson Ave \n",
214 | "2 WILKENS AVE & PINE HEIGHTS AVE E/B Wilkens Ave Pine Heights \n",
215 | "3 THE ALAMEDA & E 33RD ST S/B The Alameda 33rd St \n",
216 | "4 E 33RD ST & THE ALAMEDA E/B E 33rd The Alameda \n",
217 | "\n",
218 | " intersection Location 1 \n",
219 | "0 Caton Ave & Benson Ave (39.2693779962, -76.6688185297) \n",
220 | "1 Caton Ave & Benson Ave (39.2693157898, -76.6689698176) \n",
221 | "2 Wilkens Ave & Pine Heights (39.2720252302, -76.676960806) \n",
222 | "3 The Alameda & 33rd St (39.3285013141, -76.5953545714) \n",
223 | "4 E 33rd & The Alameda (39.3283410623, -76.5953594625) "
224 | ]
225 | }
226 | ],
227 | "prompt_number": 7
228 | },
229 | {
230 | "cell_type": "markdown",
231 | "metadata": {},
232 | "source": [
233 | "Read Excel file\n",
234 | "\n",
235 | "This is equivalent to R `read.xlsx()` and `read.xlsx2()` commands.\n",
236 | "\n",
237 | "We need openpyxl 1.5.8 (don't use the latest version due to a bug) and xlrd packages. Install with: \n",
238 | "\n",
239 | "| \n", 150 | " | address | \n", 151 | "direction | \n", 152 | "street | \n", 153 | "crossStreet | \n", 154 | "intersection | \n", 155 | "Location 1 | \n", 156 | "
|---|---|---|---|---|---|---|
| 0 | \n", 161 | "S CATON AVE & BENSON AVE | \n", 162 | "N/B | \n", 163 | "Caton Ave | \n", 164 | "Benson Ave | \n", 165 | "Caton Ave & Benson Ave | \n", 166 | "(39.2693779962, -76.6688185297) | \n", 167 | "
| 1 | \n", 170 | "S CATON AVE & BENSON AVE | \n", 171 | "S/B | \n", 172 | "Caton Ave | \n", 173 | "Benson Ave | \n", 174 | "Caton Ave & Benson Ave | \n", 175 | "(39.2693157898, -76.6689698176) | \n", 176 | "
| 2 | \n", 179 | "WILKENS AVE & PINE HEIGHTS AVE | \n", 180 | "E/B | \n", 181 | "Wilkens Ave | \n", 182 | "Pine Heights | \n", 183 | "Wilkens Ave & Pine Heights | \n", 184 | "(39.2720252302, -76.676960806) | \n", 185 | "
| 3 | \n", 188 | "THE ALAMEDA & E 33RD ST | \n", 189 | "S/B | \n", 190 | "The Alameda | \n", 191 | "33rd St | \n", 192 | "The Alameda & 33rd St | \n", 193 | "(39.3285013141, -76.5953545714) | \n", 194 | "
| 4 | \n", 197 | "E 33RD ST & THE ALAMEDA | \n", 198 | "E/B | \n", 199 | "E 33rd | \n", 200 | "The Alameda | \n", 201 | "E 33rd & The Alameda | \n", 202 | "(39.3283410623, -76.5953594625) | \n", 203 | "
\n",
240 | "sudo pip install openpyxl==1.5.8\n",
241 | "sudo pip install xlrd\n",
242 | "\n",
272 | "\n",
273 | " \n",
274 | "
\n",
332 | "
"
333 | ],
334 | "output_type": "pyout",
335 | "prompt_number": 8,
336 | "text": [
337 | " address direction street crossStreet \\\n",
338 | "0 S CATON AVE & BENSON AVE N/B Caton Ave Benson Ave \n",
339 | "1 S CATON AVE & BENSON AVE S/B Caton Ave Benson Ave \n",
340 | "2 WILKENS AVE & PINE HEIGHTS AVE E/B Wilkens Ave Pine Heights \n",
341 | "3 THE ALAMEDA & E 33RD ST S/B The Alameda 33rd St \n",
342 | "4 E 33RD ST & THE ALAMEDA E/B E 33rd The Alameda \n",
343 | "\n",
344 | " intersection Location 1 \n",
345 | "0 Caton Ave & Benson Ave (39.2693779962, -76.6688185297) \n",
346 | "1 Caton Ave & Benson Ave (39.2693157898, -76.6689698176) \n",
347 | "2 Wilkens Ave & Pine Heights (39.2720252302, -76.676960806) \n",
348 | "3 The Alameda & 33rd St (39.3285013141, -76.5953545714) \n",
349 | "4 E 33rd & The Alameda (39.3283410623, -76.5953594625) "
350 | ]
351 | }
352 | ],
353 | "prompt_number": 8
354 | },
355 | {
356 | "cell_type": "markdown",
357 | "metadata": {},
358 | "source": [
359 | "The course video describes R's `readLines()` for reading a text file, which is similar to standard Python file access, so I'm not going to detail it here.\n",
360 | "\n",
361 | "Similarly, R's `readLines()` to read data from a website is similar to Python with urllib2 package as in xls example above`"
362 | ]
363 | },
364 | {
365 | "cell_type": "markdown",
366 | "metadata": {},
367 | "source": [
368 | "| \n", 276 | " | address | \n", 277 | "direction | \n", 278 | "street | \n", 279 | "crossStreet | \n", 280 | "intersection | \n", 281 | "Location 1 | \n", 282 | "
|---|---|---|---|---|---|---|
| 0 | \n", 287 | "S CATON AVE & BENSON AVE | \n", 288 | "N/B | \n", 289 | "Caton Ave | \n", 290 | "Benson Ave | \n", 291 | "Caton Ave & Benson Ave | \n", 292 | "(39.2693779962, -76.6688185297) | \n", 293 | "
| 1 | \n", 296 | "S CATON AVE & BENSON AVE | \n", 297 | "S/B | \n", 298 | "Caton Ave | \n", 299 | "Benson Ave | \n", 300 | "Caton Ave & Benson Ave | \n", 301 | "(39.2693157898, -76.6689698176) | \n", 302 | "
| 2 | \n", 305 | "WILKENS AVE & PINE HEIGHTS AVE | \n", 306 | "E/B | \n", 307 | "Wilkens Ave | \n", 308 | "Pine Heights | \n", 309 | "Wilkens Ave & Pine Heights | \n", 310 | "(39.2720252302, -76.676960806) | \n", 311 | "
| 3 | \n", 314 | "THE ALAMEDA & E 33RD ST | \n", 315 | "S/B | \n", 316 | "The Alameda | \n", 317 | "33rd St | \n", 318 | "The Alameda & 33rd St | \n", 319 | "(39.3285013141, -76.5953545714) | \n", 320 | "
| 4 | \n", 323 | "E 33RD ST & THE ALAMEDA | \n", 324 | "E/B | \n", 325 | "E 33rd | \n", 326 | "The Alameda | \n", 327 | "E 33rd & The Alameda | \n", 328 | "(39.3283410623, -76.5953594625) | \n", 329 | "
Read JSON file\n", 369 | "\n", 370 | "This is equivalent to R's `fromJSON()` command." 371 | ] 372 | }, 373 | { 374 | "cell_type": "code", 375 | "collapsed": false, 376 | "input": [ 377 | "import json\n", 378 | "\n", 379 | "# first we get the json file from the website\n", 380 | "fileUrl = 'https://data.baltimorecity.gov/api/views/dz54-2aru/rows.json?accessType=DOWNLOAD'\n", 381 | "req = urllib2.Request(fileUrl)\n", 382 | "opener = urllib2.build_opener()\n", 383 | "f = opener.open(req)\n", 384 | "\n", 385 | "# then we read it into a data structure\n", 386 | "jsonCamera = json.loads(f.read())\n", 387 | "\n", 388 | "# json is loadad as dictionary\n", 389 | "print jsonCamera['meta']['view']['id']\n", 390 | "print jsonCamera['meta']['view']['name']\n", 391 | "print jsonCamera['meta']['view']['attribution']" 392 | ], 393 | "language": "python", 394 | "metadata": {}, 395 | "outputs": [ 396 | { 397 | "output_type": "stream", 398 | "stream": "stdout", 399 | "text": [ 400 | "dz54-2aru\n", 401 | "Baltimore Fixed Speed Cameras\n", 402 | "Department of Transportation\n" 403 | ] 404 | } 405 | ], 406 | "prompt_number": 9 407 | }, 408 | { 409 | "cell_type": "markdown", 410 | "metadata": {}, 411 | "source": [ 412 | "
Writing data" 413 | ] 414 | }, 415 | { 416 | "cell_type": "code", 417 | "collapsed": false, 418 | "input": [ 419 | "# first read the csv file\n", 420 | "cameraData = pd.read_csv('../data/cameras.csv')\n", 421 | "\n", 422 | "# take a subset of the columns\n", 423 | "tmpData = cameraData.ix[:,2:]\n", 424 | "\n", 425 | "# then save it to a different csv file\n", 426 | "# this is equivalent to R's write.table() command\n", 427 | "tmpData.to_csv('../data/camerasModified.csv', sep=',', index=False)\n", 428 | "\n", 429 | "cameraData2 = pd.read_csv('../data/camerasModified.csv')\n", 430 | "cameraData2.head()" 431 | ], 432 | "language": "python", 433 | "metadata": {}, 434 | "outputs": [ 435 | { 436 | "html": [ 437 | "
\n",
438 | "\n",
439 | " \n",
440 | "
\n",
486 | "
"
487 | ],
488 | "output_type": "pyout",
489 | "prompt_number": 10,
490 | "text": [
491 | " street crossStreet intersection \\\n",
492 | "0 Caton Ave Benson Ave Caton Ave & Benson Ave \n",
493 | "1 Caton Ave Benson Ave Caton Ave & Benson Ave \n",
494 | "2 Wilkens Ave Pine Heights Wilkens Ave & Pine Heights \n",
495 | "3 The Alameda 33rd St The Alameda & 33rd St \n",
496 | "4 E 33rd The Alameda E 33rd & The Alameda \n",
497 | "\n",
498 | " Location 1 \n",
499 | "0 (39.2693779962, -76.6688185297) \n",
500 | "1 (39.2693157898, -76.6689698176) \n",
501 | "2 (39.2720252302, -76.676960806) \n",
502 | "3 (39.3285013141, -76.5953545714) \n",
503 | "4 (39.3283410623, -76.5953594625) "
504 | ]
505 | }
506 | ],
507 | "prompt_number": 10
508 | },
509 | {
510 | "cell_type": "markdown",
511 | "metadata": {},
512 | "source": [
513 | "The course video explains R commands to save and load the workspace. I don't think we have the equivalent for that."
514 | ]
515 | },
516 | {
517 | "cell_type": "markdown",
518 | "metadata": {},
519 | "source": [
520 | "| \n", 442 | " | street | \n", 443 | "crossStreet | \n", 444 | "intersection | \n", 445 | "Location 1 | \n", 446 | "
|---|---|---|---|---|
| 0 | \n", 451 | "Caton Ave | \n", 452 | "Benson Ave | \n", 453 | "Caton Ave & Benson Ave | \n", 454 | "(39.2693779962, -76.6688185297) | \n", 455 | "
| 1 | \n", 458 | "Caton Ave | \n", 459 | "Benson Ave | \n", 460 | "Caton Ave & Benson Ave | \n", 461 | "(39.2693157898, -76.6689698176) | \n", 462 | "
| 2 | \n", 465 | "Wilkens Ave | \n", 466 | "Pine Heights | \n", 467 | "Wilkens Ave & Pine Heights | \n", 468 | "(39.2720252302, -76.676960806) | \n", 469 | "
| 3 | \n", 472 | "The Alameda | \n", 473 | "33rd St | \n", 474 | "The Alameda & 33rd St | \n", 475 | "(39.3285013141, -76.5953545714) | \n", 476 | "
| 4 | \n", 479 | "E 33rd | \n", 480 | "The Alameda | \n", 481 | "E 33rd & The Alameda | \n", 482 | "(39.3283410623, -76.5953594625) | \n", 483 | "
The course video explains R's `paste()` and `paste0()` commands, which look like standard Python's string manipulations:" 521 | ] 522 | }, 523 | { 524 | "cell_type": "code", 525 | "collapsed": false, 526 | "input": [ 527 | "print ['../data' + str(i) + '.csv' for i in range(1, 6)]" 528 | ], 529 | "language": "python", 530 | "metadata": {}, 531 | "outputs": [ 532 | { 533 | "output_type": "stream", 534 | "stream": "stdout", 535 | "text": [ 536 | "['../data1.csv', '../data2.csv', '../data3.csv', '../data4.csv', '../data5.csv']\n" 537 | ] 538 | } 539 | ], 540 | "prompt_number": 11 541 | }, 542 | { 543 | "cell_type": "markdown", 544 | "metadata": {}, 545 | "source": [ 546 | "
Getting data off webpages" 547 | ] 548 | }, 549 | { 550 | "cell_type": "code", 551 | "collapsed": false, 552 | "input": [ 553 | "from lxml.html import parse\n", 554 | "\n", 555 | "url = 'http://scholar.google.com/citations?user=HI-I6C0AAAAJ&hl=en'\n", 556 | "\n", 557 | "# this is equivalent to the combined R's opening/reading/closing connection and htmlTreeParse() commands\n", 558 | "html3 = parse(url).getroot()\n", 559 | "\n", 560 | "# get the title text using xpath expression\n", 561 | "# this is equivalent to R xpathSApply() command\n", 562 | "title = html3.xpath('//title')\n", 563 | "print [x.text_content() for x in title]\n", 564 | "\n", 565 | "# get the texts of col-citedby elements using xpath expression\n", 566 | "citedby = html3.xpath(\"//td[@id='col-citedby']\")\n", 567 | "print [x.text_content() for x in citedby]" 568 | ], 569 | "language": "python", 570 | "metadata": {}, 571 | "outputs": [ 572 | { 573 | "output_type": "stream", 574 | "stream": "stdout", 575 | "text": [ 576 | "['Jeff Leek - Google Scholar Citations']\n", 577 | "['Cited by', '344', '183', '147', '143', '111', '96', '87', '80', '59', '18', '11', '10', '10', '8', '8', '8', '7', '6', '5', '3']\n" 578 | ] 579 | } 580 | ], 581 | "prompt_number": 12 582 | }, 583 | { 584 | "cell_type": "code", 585 | "collapsed": false, 586 | "input": [], 587 | "language": "python", 588 | "metadata": {}, 589 | "outputs": [] 590 | } 591 | ], 592 | "metadata": {} 593 | } 594 | ] 595 | } -------------------------------------------------------------------------------- /week2/wk2_quiz.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "wk2_quiz" 4 | }, 5 | "nbformat": 3, 6 | "nbformat_minor": 0, 7 | "worksheets": [ 8 | { 9 | "cells": [ 10 | { 11 | "cell_type": "code", 12 | "collapsed": false, 13 | "input": [ 14 | "import pandas as pd\n", 15 | "import numpy as np" 16 | ], 17 | "language": "python", 18 | "metadata": {}, 19 | "outputs": [], 20 | "prompt_number": 1 21 | }, 22 | { 23 | "cell_type": "code", 24 | "collapsed": false, 25 | "input": [ 26 | "import urllib2\n", 27 | "\n", 28 | "url = 'http://simplystatistics.tumblr.com/'\n", 29 | "f = urllib2.urlopen(url)\n", 30 | "simplyStats = f.read()\n", 31 | "with open('../data/simplystats', 'wb') as w:\n", 32 | " w.write(simplyStats)" 33 | ], 34 | "language": "python", 35 | "metadata": {}, 36 | "outputs": [], 37 | "prompt_number": 16 38 | }, 39 | { 40 | "cell_type": "code", 41 | "collapsed": false, 42 | "input": [ 43 | "with open('simplystats', 'r') as w:\n", 44 | " l = w.readlines()" 45 | ], 46 | "language": "python", 47 | "metadata": {}, 48 | "outputs": [], 49 | "prompt_number": 17 50 | }, 51 | { 52 | "cell_type": "code", 53 | "collapsed": false, 54 | "input": [ 55 | "print ','.join([str(len(l[1])), str(len(l[44])), str(len(l[121]))])" 56 | ], 57 | "language": "python", 58 | "metadata": {}, 59 | "outputs": [ 60 | { 61 | "output_type": "stream", 62 | "stream": "stdout", 63 | "text": [ 64 | "920,7,26\n" 65 | ] 66 | } 67 | ], 68 | "prompt_number": 24 69 | }, 70 | { 71 | "cell_type": "code", 72 | "collapsed": false, 73 | "input": [ 74 | "idaho_housing = pd.read_csv('https://dl.dropbox.com/u/7710864/data/csv_hid/ss06hid.csv')\n", 75 | "idaho_housing" 76 | ], 77 | "language": "python", 78 | "metadata": {}, 79 | "outputs": [ 80 | { 81 | "output_type": "pyout", 82 | "prompt_number": 26, 83 | "text": [ 84 | "
\n",
365 | "\n",
366 | " \n",
367 | "
\n",
762 | "
"
763 | ],
764 | "output_type": "pyout",
765 | "prompt_number": 68,
766 | "text": [
767 | " RT_x SERIALNO DIVISION PUMA_x REGION ST_x ADJUST_x WGTP NP TYPE ACR AGS BDS \\\n",
768 | "0 H 186 8 700 4 16 1015675 89 4 1 1 NaN 4 \n",
769 | "1 H 186 8 700 4 16 1015675 89 4 1 1 NaN 4 \n",
770 | "2 H 186 8 700 4 16 1015675 89 4 1 1 NaN 4 \n",
771 | "3 H 186 8 700 4 16 1015675 89 4 1 1 NaN 4 \n",
772 | "4 H 306 8 700 4 16 1015675 310 1 1 NaN NaN 1 \n",
773 | "5 H 395 8 100 4 16 1015675 106 2 1 1 NaN 3 \n",
774 | "6 H 395 8 100 4 16 1015675 106 2 1 1 NaN 3 \n",
775 | "7 H 506 8 700 4 16 1015675 240 4 1 1 NaN 4 \n",
776 | "8 H 506 8 700 4 16 1015675 240 4 1 1 NaN 4 \n",
777 | "9 H 506 8 700 4 16 1015675 240 4 1 1 NaN 4 \n",
778 | "10 H 506 8 700 4 16 1015675 240 4 1 1 NaN 4 \n",
779 | "11 H 835 8 800 4 16 1015675 118 4 1 2 1 5 \n",
780 | "12 H 835 8 800 4 16 1015675 118 4 1 2 1 5 \n",
781 | "13 H 835 8 800 4 16 1015675 118 4 1 2 1 5 \n",
782 | "14 H 835 8 800 4 16 1015675 118 4 1 2 1 5 \n",
783 | "15 H 989 8 700 4 16 1015675 115 4 1 1 NaN 3 \n",
784 | "\n",
785 | " BLD BUS CONP ELEP FS FULP GASP \n",
786 | "0 2 2 NaN 180 0 2 3 \n",
787 | "1 2 2 NaN 180 0 2 3 \n",
788 | "2 2 2 NaN 180 0 2 3 \n",
789 | "3 2 2 NaN 180 0 2 3 \n",
790 | "4 7 NaN NaN 60 0 2 3 \n",
791 | "5 2 2 NaN 70 0 2 30 \n",
792 | "6 2 2 NaN 70 0 2 30 \n",
793 | "7 2 2 NaN 40 0 2 80 \n",
794 | "8 2 2 NaN 40 0 2 80 \n",
795 | "9 2 2 NaN 40 0 2 80 \n",
796 | "10 2 2 NaN 40 0 2 80 \n",
797 | "11 2 2 NaN 250 0 2 3 \n",
798 | "12 2 2 NaN 250 0 2 3 \n",
799 | "13 2 2 NaN 250 0 2 3 \n",
800 | "14 2 2 NaN 250 0 2 3 \n",
801 | "15 2 2 NaN 130 0 2 3 "
802 | ]
803 | }
804 | ],
805 | "prompt_number": 68
806 | },
807 | {
808 | "cell_type": "code",
809 | "collapsed": false,
810 | "input": [
811 | "merged.shape"
812 | ],
813 | "language": "python",
814 | "metadata": {},
815 | "outputs": [
816 | {
817 | "output_type": "pyout",
818 | "prompt_number": 69,
819 | "text": [
820 | "(15451, 426)"
821 | ]
822 | }
823 | ],
824 | "prompt_number": 69
825 | },
826 | {
827 | "cell_type": "code",
828 | "collapsed": false,
829 | "input": [],
830 | "language": "python",
831 | "metadata": {},
832 | "outputs": []
833 | }
834 | ],
835 | "metadata": {}
836 | }
837 | ]
838 | }
--------------------------------------------------------------------------------
/week3/twoPanel.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/herrfz/dataanalysis/fa2d42045f364f1a5b0683b7fb908399d69ff1cb/week3/twoPanel.pdf
--------------------------------------------------------------------------------
/week3/twoPanel.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/herrfz/dataanalysis/fa2d42045f364f1a5b0683b7fb908399d69ff1cb/week3/twoPanel.png
--------------------------------------------------------------------------------
/week5/anova_with_multiple_factors.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "metadata": {
3 | "name": "anova_with_multiple_factors"
4 | },
5 | "nbformat": 3,
6 | "nbformat_minor": 0,
7 | "worksheets": [
8 | {
9 | "cells": [
10 | {
11 | "cell_type": "code",
12 | "collapsed": false,
13 | "input": [
14 | "import pandas as pd\n",
15 | "import numpy as np"
16 | ],
17 | "language": "python",
18 | "metadata": {},
19 | "outputs": [],
20 | "prompt_number": 1
21 | },
22 | {
23 | "cell_type": "code",
24 | "collapsed": false,
25 | "input": [
26 | "movies = pd.read_csv('http://www.rossmanchance.com/iscam2/data/movies03RT.txt', sep='\\t')\n",
27 | "movies.columns = ['X', 'score', 'rating', 'genre', 'box_office', 'running_time']\n",
28 | "movies.head()"
29 | ],
30 | "language": "python",
31 | "metadata": {},
32 | "outputs": [
33 | {
34 | "html": [
35 | "| \n", 369 | " | RT_x | \n", 370 | "SERIALNO | \n", 371 | "DIVISION | \n", 372 | "PUMA_x | \n", 373 | "REGION | \n", 374 | "ST_x | \n", 375 | "ADJUST_x | \n", 376 | "WGTP | \n", 377 | "NP | \n", 378 | "TYPE | \n", 379 | "ACR | \n", 380 | "AGS | \n", 381 | "BDS | \n", 382 | "BLD | \n", 383 | "BUS | \n", 384 | "CONP | \n", 385 | "ELEP | \n", 386 | "FS | \n", 387 | "FULP | \n", 388 | "GASP | \n", 389 | "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", 394 | "H | \n", 395 | "186 | \n", 396 | "8 | \n", 397 | "700 | \n", 398 | "4 | \n", 399 | "16 | \n", 400 | "1015675 | \n", 401 | "89 | \n", 402 | "4 | \n", 403 | "1 | \n", 404 | "1 | \n", 405 | "NaN | \n", 406 | "4 | \n", 407 | "2 | \n", 408 | "2 | \n", 409 | "NaN | \n", 410 | "180 | \n", 411 | "0 | \n", 412 | "2 | \n", 413 | "3 | \n", 414 | "
| 1 | \n", 417 | "H | \n", 418 | "186 | \n", 419 | "8 | \n", 420 | "700 | \n", 421 | "4 | \n", 422 | "16 | \n", 423 | "1015675 | \n", 424 | "89 | \n", 425 | "4 | \n", 426 | "1 | \n", 427 | "1 | \n", 428 | "NaN | \n", 429 | "4 | \n", 430 | "2 | \n", 431 | "2 | \n", 432 | "NaN | \n", 433 | "180 | \n", 434 | "0 | \n", 435 | "2 | \n", 436 | "3 | \n", 437 | "
| 2 | \n", 440 | "H | \n", 441 | "186 | \n", 442 | "8 | \n", 443 | "700 | \n", 444 | "4 | \n", 445 | "16 | \n", 446 | "1015675 | \n", 447 | "89 | \n", 448 | "4 | \n", 449 | "1 | \n", 450 | "1 | \n", 451 | "NaN | \n", 452 | "4 | \n", 453 | "2 | \n", 454 | "2 | \n", 455 | "NaN | \n", 456 | "180 | \n", 457 | "0 | \n", 458 | "2 | \n", 459 | "3 | \n", 460 | "
| 3 | \n", 463 | "H | \n", 464 | "186 | \n", 465 | "8 | \n", 466 | "700 | \n", 467 | "4 | \n", 468 | "16 | \n", 469 | "1015675 | \n", 470 | "89 | \n", 471 | "4 | \n", 472 | "1 | \n", 473 | "1 | \n", 474 | "NaN | \n", 475 | "4 | \n", 476 | "2 | \n", 477 | "2 | \n", 478 | "NaN | \n", 479 | "180 | \n", 480 | "0 | \n", 481 | "2 | \n", 482 | "3 | \n", 483 | "
| 4 | \n", 486 | "H | \n", 487 | "306 | \n", 488 | "8 | \n", 489 | "700 | \n", 490 | "4 | \n", 491 | "16 | \n", 492 | "1015675 | \n", 493 | "310 | \n", 494 | "1 | \n", 495 | "1 | \n", 496 | "NaN | \n", 497 | "NaN | \n", 498 | "1 | \n", 499 | "7 | \n", 500 | "NaN | \n", 501 | "NaN | \n", 502 | "60 | \n", 503 | "0 | \n", 504 | "2 | \n", 505 | "3 | \n", 506 | "
| 5 | \n", 509 | "H | \n", 510 | "395 | \n", 511 | "8 | \n", 512 | "100 | \n", 513 | "4 | \n", 514 | "16 | \n", 515 | "1015675 | \n", 516 | "106 | \n", 517 | "2 | \n", 518 | "1 | \n", 519 | "1 | \n", 520 | "NaN | \n", 521 | "3 | \n", 522 | "2 | \n", 523 | "2 | \n", 524 | "NaN | \n", 525 | "70 | \n", 526 | "0 | \n", 527 | "2 | \n", 528 | "30 | \n", 529 | "
| 6 | \n", 532 | "H | \n", 533 | "395 | \n", 534 | "8 | \n", 535 | "100 | \n", 536 | "4 | \n", 537 | "16 | \n", 538 | "1015675 | \n", 539 | "106 | \n", 540 | "2 | \n", 541 | "1 | \n", 542 | "1 | \n", 543 | "NaN | \n", 544 | "3 | \n", 545 | "2 | \n", 546 | "2 | \n", 547 | "NaN | \n", 548 | "70 | \n", 549 | "0 | \n", 550 | "2 | \n", 551 | "30 | \n", 552 | "
| 7 | \n", 555 | "H | \n", 556 | "506 | \n", 557 | "8 | \n", 558 | "700 | \n", 559 | "4 | \n", 560 | "16 | \n", 561 | "1015675 | \n", 562 | "240 | \n", 563 | "4 | \n", 564 | "1 | \n", 565 | "1 | \n", 566 | "NaN | \n", 567 | "4 | \n", 568 | "2 | \n", 569 | "2 | \n", 570 | "NaN | \n", 571 | "40 | \n", 572 | "0 | \n", 573 | "2 | \n", 574 | "80 | \n", 575 | "
| 8 | \n", 578 | "H | \n", 579 | "506 | \n", 580 | "8 | \n", 581 | "700 | \n", 582 | "4 | \n", 583 | "16 | \n", 584 | "1015675 | \n", 585 | "240 | \n", 586 | "4 | \n", 587 | "1 | \n", 588 | "1 | \n", 589 | "NaN | \n", 590 | "4 | \n", 591 | "2 | \n", 592 | "2 | \n", 593 | "NaN | \n", 594 | "40 | \n", 595 | "0 | \n", 596 | "2 | \n", 597 | "80 | \n", 598 | "
| 9 | \n", 601 | "H | \n", 602 | "506 | \n", 603 | "8 | \n", 604 | "700 | \n", 605 | "4 | \n", 606 | "16 | \n", 607 | "1015675 | \n", 608 | "240 | \n", 609 | "4 | \n", 610 | "1 | \n", 611 | "1 | \n", 612 | "NaN | \n", 613 | "4 | \n", 614 | "2 | \n", 615 | "2 | \n", 616 | "NaN | \n", 617 | "40 | \n", 618 | "0 | \n", 619 | "2 | \n", 620 | "80 | \n", 621 | "
| 10 | \n", 624 | "H | \n", 625 | "506 | \n", 626 | "8 | \n", 627 | "700 | \n", 628 | "4 | \n", 629 | "16 | \n", 630 | "1015675 | \n", 631 | "240 | \n", 632 | "4 | \n", 633 | "1 | \n", 634 | "1 | \n", 635 | "NaN | \n", 636 | "4 | \n", 637 | "2 | \n", 638 | "2 | \n", 639 | "NaN | \n", 640 | "40 | \n", 641 | "0 | \n", 642 | "2 | \n", 643 | "80 | \n", 644 | "
| 11 | \n", 647 | "H | \n", 648 | "835 | \n", 649 | "8 | \n", 650 | "800 | \n", 651 | "4 | \n", 652 | "16 | \n", 653 | "1015675 | \n", 654 | "118 | \n", 655 | "4 | \n", 656 | "1 | \n", 657 | "2 | \n", 658 | "1 | \n", 659 | "5 | \n", 660 | "2 | \n", 661 | "2 | \n", 662 | "NaN | \n", 663 | "250 | \n", 664 | "0 | \n", 665 | "2 | \n", 666 | "3 | \n", 667 | "
| 12 | \n", 670 | "H | \n", 671 | "835 | \n", 672 | "8 | \n", 673 | "800 | \n", 674 | "4 | \n", 675 | "16 | \n", 676 | "1015675 | \n", 677 | "118 | \n", 678 | "4 | \n", 679 | "1 | \n", 680 | "2 | \n", 681 | "1 | \n", 682 | "5 | \n", 683 | "2 | \n", 684 | "2 | \n", 685 | "NaN | \n", 686 | "250 | \n", 687 | "0 | \n", 688 | "2 | \n", 689 | "3 | \n", 690 | "
| 13 | \n", 693 | "H | \n", 694 | "835 | \n", 695 | "8 | \n", 696 | "800 | \n", 697 | "4 | \n", 698 | "16 | \n", 699 | "1015675 | \n", 700 | "118 | \n", 701 | "4 | \n", 702 | "1 | \n", 703 | "2 | \n", 704 | "1 | \n", 705 | "5 | \n", 706 | "2 | \n", 707 | "2 | \n", 708 | "NaN | \n", 709 | "250 | \n", 710 | "0 | \n", 711 | "2 | \n", 712 | "3 | \n", 713 | "
| 14 | \n", 716 | "H | \n", 717 | "835 | \n", 718 | "8 | \n", 719 | "800 | \n", 720 | "4 | \n", 721 | "16 | \n", 722 | "1015675 | \n", 723 | "118 | \n", 724 | "4 | \n", 725 | "1 | \n", 726 | "2 | \n", 727 | "1 | \n", 728 | "5 | \n", 729 | "2 | \n", 730 | "2 | \n", 731 | "NaN | \n", 732 | "250 | \n", 733 | "0 | \n", 734 | "2 | \n", 735 | "3 | \n", 736 | "
| 15 | \n", 739 | "H | \n", 740 | "989 | \n", 741 | "8 | \n", 742 | "700 | \n", 743 | "4 | \n", 744 | "16 | \n", 745 | "1015675 | \n", 746 | "115 | \n", 747 | "4 | \n", 748 | "1 | \n", 749 | "1 | \n", 750 | "NaN | \n", 751 | "3 | \n", 752 | "2 | \n", 753 | "2 | \n", 754 | "NaN | \n", 755 | "130 | \n", 756 | "0 | \n", 757 | "2 | \n", 758 | "3 | \n", 759 | "
\n",
36 | "\n",
37 | " \n",
38 | "
\n",
96 | "
"
97 | ],
98 | "output_type": "pyout",
99 | "prompt_number": 17,
100 | "text": [
101 | " X score rating genre box_office running_time\n",
102 | "0 2 Fast 2 Furious 48.9 PG-13 action/adventure 127.146 107\n",
103 | "1 28 Days Later 78.2 R horror 45.065 113\n",
104 | "2 A Guy Thing 39.5 PG-13 rom comedy 15.545 101\n",
105 | "3 A Man Apart 42.9 R action/adventure 26.248 110\n",
106 | "4 A Mighty Wind 79.9 PG-13 comedy 17.781 91"
107 | ]
108 | }
109 | ],
110 | "prompt_number": 17
111 | },
112 | {
113 | "cell_type": "code",
114 | "collapsed": false,
115 | "input": [
116 | "from statsmodels.stats.anova import anova_lm\n",
117 | "from statsmodels.formula.api import ols\n",
118 | "\n",
119 | "lm = ols('score ~ rating', movies).fit()\n",
120 | "\n",
121 | "aovObject = anova_lm(lm)\n",
122 | "aovObject"
123 | ],
124 | "language": "python",
125 | "metadata": {},
126 | "outputs": [
127 | {
128 | "html": [
129 | "| \n", 40 | " | X | \n", 41 | "score | \n", 42 | "rating | \n", 43 | "genre | \n", 44 | "box_office | \n", 45 | "running_time | \n", 46 | "
|---|---|---|---|---|---|---|
| 0 | \n", 51 | "2 Fast 2 Furious | \n", 52 | "48.9 | \n", 53 | "PG-13 | \n", 54 | "action/adventure | \n", 55 | "127.146 | \n", 56 | "107 | \n", 57 | "
| 1 | \n", 60 | "28 Days Later | \n", 61 | "78.2 | \n", 62 | "R | \n", 63 | "horror | \n", 64 | "45.065 | \n", 65 | "113 | \n", 66 | "
| 2 | \n", 69 | "A Guy Thing | \n", 70 | "39.5 | \n", 71 | "PG-13 | \n", 72 | "rom comedy | \n", 73 | "15.545 | \n", 74 | "101 | \n", 75 | "
| 3 | \n", 78 | "A Man Apart | \n", 79 | "42.9 | \n", 80 | "R | \n", 81 | "action/adventure | \n", 82 | "26.248 | \n", 83 | "110 | \n", 84 | "
| 4 | \n", 87 | "A Mighty Wind | \n", 88 | "79.9 | \n", 89 | "PG-13 | \n", 90 | "comedy | \n", 91 | "17.781 | \n", 92 | "91 | \n", 93 | "
\n",
130 | "\n",
131 | " \n",
132 | "
\n",
160 | "
"
161 | ],
162 | "output_type": "pyout",
163 | "prompt_number": 24,
164 | "text": [
165 | " df sum_sq mean_sq F PR(>F)\n",
166 | "rating 3 570.123813 190.041271 0.918184 0.433975\n",
167 | "Residual 136 28148.635044 206.975258 NaN NaN"
168 | ]
169 | }
170 | ],
171 | "prompt_number": 24
172 | },
173 | {
174 | "cell_type": "code",
175 | "collapsed": false,
176 | "input": [
177 | "lm.params"
178 | ],
179 | "language": "python",
180 | "metadata": {},
181 | "outputs": [
182 | {
183 | "output_type": "pyout",
184 | "prompt_number": 25,
185 | "text": [
186 | "Intercept 67.650000\n",
187 | "rating[T.PG] -12.592857\n",
188 | "rating[T.PG-13] -11.814615\n",
189 | "rating[T.R] -12.020000"
190 | ]
191 | }
192 | ],
193 | "prompt_number": 25
194 | },
195 | {
196 | "cell_type": "code",
197 | "collapsed": false,
198 | "input": [
199 | "lm2 = ols('score ~ rating + genre', movies).fit()\n",
200 | "\n",
201 | "aovObject2 = anova_lm(lm2)\n",
202 | "aovObject2"
203 | ],
204 | "language": "python",
205 | "metadata": {},
206 | "outputs": [
207 | {
208 | "html": [
209 | "| \n", 134 | " | df | \n", 135 | "sum_sq | \n", 136 | "mean_sq | \n", 137 | "F | \n", 138 | "PR(>F) | \n", 139 | "
|---|---|---|---|---|---|
| rating | \n", 144 | "3 | \n", 145 | "570.123813 | \n", 146 | "190.041271 | \n", 147 | "0.918184 | \n", 148 | "0.433975 | \n", 149 | "
| Residual | \n", 152 | "136 | \n", 153 | "28148.635044 | \n", 154 | "206.975258 | \n", 155 | "NaN | \n", 156 | "NaN | \n", 157 | "
\n",
210 | "\n",
211 | " \n",
212 | "
\n",
248 | "
"
249 | ],
250 | "output_type": "pyout",
251 | "prompt_number": 28,
252 | "text": [
253 | " df sum_sq mean_sq F PR(>F)\n",
254 | "rating 3 570.123813 190.041271 0.973214 0.407720\n",
255 | "genre 12 3934.928021 327.910668 1.679252 0.079134\n",
256 | "Residual 124 24213.707023 195.271831 NaN NaN"
257 | ]
258 | }
259 | ],
260 | "prompt_number": 28
261 | },
262 | {
263 | "cell_type": "code",
264 | "collapsed": false,
265 | "input": [
266 | "lm3 = ols('score ~ genre + rating', movies).fit()\n",
267 | "\n",
268 | "aovObject3 = anova_lm(lm3)\n",
269 | "aovObject3"
270 | ],
271 | "language": "python",
272 | "metadata": {},
273 | "outputs": [
274 | {
275 | "html": [
276 | "| \n", 214 | " | df | \n", 215 | "sum_sq | \n", 216 | "mean_sq | \n", 217 | "F | \n", 218 | "PR(>F) | \n", 219 | "
|---|---|---|---|---|---|
| rating | \n", 224 | "3 | \n", 225 | "570.123813 | \n", 226 | "190.041271 | \n", 227 | "0.973214 | \n", 228 | "0.407720 | \n", 229 | "
| genre | \n", 232 | "12 | \n", 233 | "3934.928021 | \n", 234 | "327.910668 | \n", 235 | "1.679252 | \n", 236 | "0.079134 | \n", 237 | "
| Residual | \n", 240 | "124 | \n", 241 | "24213.707023 | \n", 242 | "195.271831 | \n", 243 | "NaN | \n", 244 | "NaN | \n", 245 | "
\n",
277 | "\n",
278 | " \n",
279 | "
\n",
315 | "
"
316 | ],
317 | "output_type": "pyout",
318 | "prompt_number": 29,
319 | "text": [
320 | " df sum_sq mean_sq F PR(>F)\n",
321 | "genre 12 4221.505277 351.792106 1.801551 0.054737\n",
322 | "rating 3 283.546557 94.515519 0.484020 0.693992\n",
323 | "Residual 124 24213.707023 195.271831 NaN NaN"
324 | ]
325 | }
326 | ],
327 | "prompt_number": 29
328 | },
329 | {
330 | "cell_type": "code",
331 | "collapsed": false,
332 | "input": [
333 | "lm4 = ols('score ~ genre + rating + box_office', movies).fit()\n",
334 | "\n",
335 | "aovObject4 = anova_lm(lm4)\n",
336 | "aovObject4"
337 | ],
338 | "language": "python",
339 | "metadata": {},
340 | "outputs": [
341 | {
342 | "html": [
343 | "| \n", 281 | " | df | \n", 282 | "sum_sq | \n", 283 | "mean_sq | \n", 284 | "F | \n", 285 | "PR(>F) | \n", 286 | "
|---|---|---|---|---|---|
| genre | \n", 291 | "12 | \n", 292 | "4221.505277 | \n", 293 | "351.792106 | \n", 294 | "1.801551 | \n", 295 | "0.054737 | \n", 296 | "
| rating | \n", 299 | "3 | \n", 300 | "283.546557 | \n", 301 | "94.515519 | \n", 302 | "0.484020 | \n", 303 | "0.693992 | \n", 304 | "
| Residual | \n", 307 | "124 | \n", 308 | "24213.707023 | \n", 309 | "195.271831 | \n", 310 | "NaN | \n", 311 | "NaN | \n", 312 | "
\n",
344 | "\n",
345 | " \n",
346 | "
\n",
390 | "
"
391 | ],
392 | "output_type": "pyout",
393 | "prompt_number": 30,
394 | "text": [
395 | " df sum_sq mean_sq F PR(>F)\n",
396 | "genre 12 4221.505277 351.792106 2.186135 0.016198\n",
397 | "rating 3 283.546557 94.515519 0.587346 0.624421\n",
398 | "box_office 1 4420.588612 4420.588612 27.470780 0.000001\n",
399 | "Residual 123 19793.118411 160.919662 NaN NaN"
400 | ]
401 | }
402 | ],
403 | "prompt_number": 30
404 | },
405 | {
406 | "cell_type": "code",
407 | "collapsed": false,
408 | "input": [],
409 | "language": "python",
410 | "metadata": {},
411 | "outputs": []
412 | }
413 | ],
414 | "metadata": {}
415 | }
416 | ]
417 | }
--------------------------------------------------------------------------------
/week6/quiz.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "metadata": {
3 | "name": "quiz"
4 | },
5 | "nbformat": 3,
6 | "nbformat_minor": 0,
7 | "worksheets": [
8 | {
9 | "cells": [
10 | {
11 | "cell_type": "code",
12 | "collapsed": false,
13 | "input": [
14 | "import pandas.rpy.common as com"
15 | ],
16 | "language": "python",
17 | "metadata": {},
18 | "outputs": [],
19 | "prompt_number": 1
20 | },
21 | {
22 | "cell_type": "code",
23 | "collapsed": false,
24 | "input": [
25 | "SAheart = com.load_data('SAheart', package='ElemStatLearn')\n",
26 | "SAheart.head()"
27 | ],
28 | "language": "python",
29 | "metadata": {},
30 | "outputs": [
31 | {
32 | "html": [
33 | "| \n", 348 | " | df | \n", 349 | "sum_sq | \n", 350 | "mean_sq | \n", 351 | "F | \n", 352 | "PR(>F) | \n", 353 | "
|---|---|---|---|---|---|
| genre | \n", 358 | "12 | \n", 359 | "4221.505277 | \n", 360 | "351.792106 | \n", 361 | "2.186135 | \n", 362 | "0.016198 | \n", 363 | "
| rating | \n", 366 | "3 | \n", 367 | "283.546557 | \n", 368 | "94.515519 | \n", 369 | "0.587346 | \n", 370 | "0.624421 | \n", 371 | "
| box_office | \n", 374 | "1 | \n", 375 | "4420.588612 | \n", 376 | "4420.588612 | \n", 377 | "27.470780 | \n", 378 | "0.000001 | \n", 379 | "
| Residual | \n", 382 | "123 | \n", 383 | "19793.118411 | \n", 384 | "160.919662 | \n", 385 | "NaN | \n", 386 | "NaN | \n", 387 | "
\n",
34 | "\n",
35 | " \n",
36 | "
\n",
118 | "
"
119 | ],
120 | "output_type": "pyout",
121 | "prompt_number": 11,
122 | "text": [
123 | " sbp tobacco ldl adiposity famhist typea obesity alcohol age chd\n",
124 | "1 160 12.00 5.73 23.11 Present 49 25.30 97.20 52 1\n",
125 | "2 144 0.01 4.41 28.61 Absent 55 28.87 2.06 63 1\n",
126 | "3 118 0.08 3.48 32.28 Present 52 29.14 3.81 46 0\n",
127 | "4 170 7.50 6.41 38.03 Present 51 31.99 24.26 58 1\n",
128 | "5 134 13.60 3.50 27.78 Present 60 25.99 57.34 49 1"
129 | ]
130 | }
131 | ],
132 | "prompt_number": 11
133 | },
134 | {
135 | "cell_type": "code",
136 | "collapsed": false,
137 | "input": [
138 | "%load_ext rmagic"
139 | ],
140 | "language": "python",
141 | "metadata": {},
142 | "outputs": [],
143 | "prompt_number": 12
144 | },
145 | {
146 | "cell_type": "code",
147 | "collapsed": false,
148 | "input": [
149 | "%%R -o train\n",
150 | "set.seed(8484)\n",
151 | "train = sample(1:dim(SAheart)[1],size=dim(SAheart)[1]/2,replace=F)"
152 | ],
153 | "language": "python",
154 | "metadata": {},
155 | "outputs": [],
156 | "prompt_number": 13
157 | },
158 | {
159 | "cell_type": "code",
160 | "collapsed": false,
161 | "input": [
162 | "trainSA = SAheart.ix[train,:]\n",
163 | "test = filter(lambda x: x not in train, SAheart.index)\n",
164 | "testSA = SAheart.ix[test, :]"
165 | ],
166 | "language": "python",
167 | "metadata": {},
168 | "outputs": [],
169 | "prompt_number": 37
170 | },
171 | {
172 | "cell_type": "code",
173 | "collapsed": false,
174 | "input": [
175 | "from statsmodels.formula.api import glm\n",
176 | "from statsmodels.api import families as f"
177 | ],
178 | "language": "python",
179 | "metadata": {},
180 | "outputs": [],
181 | "prompt_number": 38
182 | },
183 | {
184 | "cell_type": "code",
185 | "collapsed": false,
186 | "input": [
187 | "lm = glm('chd ~ age + alcohol + obesity + tobacco + typea + ldl', trainSA, family=f.Binomial()).fit()"
188 | ],
189 | "language": "python",
190 | "metadata": {},
191 | "outputs": [],
192 | "prompt_number": 39
193 | },
194 | {
195 | "cell_type": "code",
196 | "collapsed": false,
197 | "input": [
198 | "def missClass(values, prediction):\n",
199 | " return float( sum( ((prediction > 0.5) * 1) != values ) ) / float(len(values))"
200 | ],
201 | "language": "python",
202 | "metadata": {},
203 | "outputs": [],
204 | "prompt_number": 40
205 | },
206 | {
207 | "cell_type": "code",
208 | "collapsed": false,
209 | "input": [
210 | "print 'training set misclassification %.4f' % missClass(trainSA['chd'], lm.fittedvalues)\n",
211 | "print 'test set misclassification %.4f' % missClass(testSA['chd'], lm.predict(testSA.ix[:,:-1]))"
212 | ],
213 | "language": "python",
214 | "metadata": {},
215 | "outputs": [
216 | {
217 | "output_type": "stream",
218 | "stream": "stdout",
219 | "text": [
220 | "training set misclassification 0.2597\n",
221 | "test set misclassification 0.3203\n"
222 | ]
223 | }
224 | ],
225 | "prompt_number": 52
226 | },
227 | {
228 | "cell_type": "code",
229 | "collapsed": false,
230 | "input": [
231 | "olive = com.load_data('olive', package='pgmm')\n",
232 | "olive = olive.ix[:,:-1]\n",
233 | "olive.head()"
234 | ],
235 | "language": "python",
236 | "metadata": {},
237 | "outputs": [
238 | {
239 | "html": [
240 | "| \n", 38 | " | sbp | \n", 39 | "tobacco | \n", 40 | "ldl | \n", 41 | "adiposity | \n", 42 | "famhist | \n", 43 | "typea | \n", 44 | "obesity | \n", 45 | "alcohol | \n", 46 | "age | \n", 47 | "chd | \n", 48 | "
|---|---|---|---|---|---|---|---|---|---|---|
| 1 | \n", 53 | "160 | \n", 54 | "12.00 | \n", 55 | "5.73 | \n", 56 | "23.11 | \n", 57 | "Present | \n", 58 | "49 | \n", 59 | "25.30 | \n", 60 | "97.20 | \n", 61 | "52 | \n", 62 | "1 | \n", 63 | "
| 2 | \n", 66 | "144 | \n", 67 | "0.01 | \n", 68 | "4.41 | \n", 69 | "28.61 | \n", 70 | "Absent | \n", 71 | "55 | \n", 72 | "28.87 | \n", 73 | "2.06 | \n", 74 | "63 | \n", 75 | "1 | \n", 76 | "
| 3 | \n", 79 | "118 | \n", 80 | "0.08 | \n", 81 | "3.48 | \n", 82 | "32.28 | \n", 83 | "Present | \n", 84 | "52 | \n", 85 | "29.14 | \n", 86 | "3.81 | \n", 87 | "46 | \n", 88 | "0 | \n", 89 | "
| 4 | \n", 92 | "170 | \n", 93 | "7.50 | \n", 94 | "6.41 | \n", 95 | "38.03 | \n", 96 | "Present | \n", 97 | "51 | \n", 98 | "31.99 | \n", 99 | "24.26 | \n", 100 | "58 | \n", 101 | "1 | \n", 102 | "
| 5 | \n", 105 | "134 | \n", 106 | "13.60 | \n", 107 | "3.50 | \n", 108 | "27.78 | \n", 109 | "Present | \n", 110 | "60 | \n", 111 | "25.99 | \n", 112 | "57.34 | \n", 113 | "49 | \n", 114 | "1 | \n", 115 | "
\n",
241 | "\n",
242 | " \n",
243 | "
\n",
319 | "
"
320 | ],
321 | "output_type": "pyout",
322 | "prompt_number": 56,
323 | "text": [
324 | " Region Area Palmitic Palmitoleic Stearic Oleic Linoleic Linolenic Arachidic\n",
325 | "1 1 1 1075 75 226 7823 672 36 60\n",
326 | "2 1 1 1088 73 224 7709 781 31 61\n",
327 | "3 1 1 911 54 246 8113 549 31 63\n",
328 | "4 1 1 966 57 240 7952 619 50 78\n",
329 | "5 1 1 1051 67 259 7771 672 50 80"
330 | ]
331 | }
332 | ],
333 | "prompt_number": 56
334 | },
335 | {
336 | "cell_type": "code",
337 | "collapsed": false,
338 | "input": [
339 | "from sklearn import tree\n",
340 | "import patsy as pt\n",
341 | "import pandas as pd"
342 | ],
343 | "language": "python",
344 | "metadata": {},
345 | "outputs": [],
346 | "prompt_number": 60
347 | },
348 | {
349 | "cell_type": "code",
350 | "collapsed": false,
351 | "input": [
352 | "y, X = pt.dmatrices('Area ~ Region + Palmitic + Palmitoleic + Stearic + \\\n",
353 | " Oleic + Linoleic + Linolenic + Arachidic - 1', olive)\n",
354 | "\n",
355 | "clf = tree.DecisionTreeClassifier().fit(X, y)"
356 | ],
357 | "language": "python",
358 | "metadata": {},
359 | "outputs": [],
360 | "prompt_number": 96
361 | },
362 | {
363 | "cell_type": "code",
364 | "collapsed": false,
365 | "input": [
366 | "import StringIO, pydot\n",
367 | "from IPython.core.display import HTML\n",
368 | "\n",
369 | "dot_data = StringIO.StringIO()\n",
370 | "tree.export_graphviz(clf, out_file=dot_data)\n",
371 | "graph = pydot.graph_from_dot_data(dot_data.getvalue())\n",
372 | "graph.write_png('tree3.png')\n",
373 | "#HTML('| \n", 245 | " | Region | \n", 246 | "Area | \n", 247 | "Palmitic | \n", 248 | "Palmitoleic | \n", 249 | "Stearic | \n", 250 | "Oleic | \n", 251 | "Linoleic | \n", 252 | "Linolenic | \n", 253 | "Arachidic | \n", 254 | "
|---|---|---|---|---|---|---|---|---|---|
| 1 | \n", 259 | "1 | \n", 260 | "1 | \n", 261 | "1075 | \n", 262 | "75 | \n", 263 | "226 | \n", 264 | "7823 | \n", 265 | "672 | \n", 266 | "36 | \n", 267 | "60 | \n", 268 | "
| 2 | \n", 271 | "1 | \n", 272 | "1 | \n", 273 | "1088 | \n", 274 | "73 | \n", 275 | "224 | \n", 276 | "7709 | \n", 277 | "781 | \n", 278 | "31 | \n", 279 | "61 | \n", 280 | "
| 3 | \n", 283 | "1 | \n", 284 | "1 | \n", 285 | "911 | \n", 286 | "54 | \n", 287 | "246 | \n", 288 | "8113 | \n", 289 | "549 | \n", 290 | "31 | \n", 291 | "63 | \n", 292 | "
| 4 | \n", 295 | "1 | \n", 296 | "1 | \n", 297 | "966 | \n", 298 | "57 | \n", 299 | "240 | \n", 300 | "7952 | \n", 301 | "619 | \n", 302 | "50 | \n", 303 | "78 | \n", 304 | "
| 5 | \n", 307 | "1 | \n", 308 | "1 | \n", 309 | "1051 | \n", 310 | "67 | \n", 311 | "259 | \n", 312 | "7771 | \n", 313 | "672 | \n", 314 | "50 | \n", 315 | "80 | \n", 316 | "
![](\"files/tree3.png\")
')"
374 | ],
375 | "language": "python",
376 | "metadata": {},
377 | "outputs": [
378 | {
379 | "output_type": "pyout",
380 | "prompt_number": 97,
381 | "text": [
382 | "True"
383 | ]
384 | }
385 | ],
386 | "prompt_number": 97
387 | },
388 | {
389 | "cell_type": "code",
390 | "collapsed": false,
391 | "input": [
392 | "olive.mean()"
393 | ],
394 | "language": "python",
395 | "metadata": {},
396 | "outputs": [
397 | {
398 | "output_type": "pyout",
399 | "prompt_number": 104,
400 | "text": [
401 | "Region 1.699301\n",
402 | "Area 4.599650\n",
403 | "Palmitic 1231.741259\n",
404 | "Palmitoleic 126.094406\n",
405 | "Stearic 228.865385\n",
406 | "Oleic 7311.748252\n",
407 | "Linoleic 980.527972\n",
408 | "Linolenic 31.888112\n",
409 | "Arachidic 58.097902"
410 | ]
411 | }
412 | ],
413 | "prompt_number": 104
414 | },
415 | {
416 | "cell_type": "code",
417 | "collapsed": false,
418 | "input": [
419 | "newdata = olive[['Region', 'Palmitic', 'Palmitoleic', 'Stearic', 'Oleic', 'Linoleic', 'Linolenic', 'Arachidic']].mean()"
420 | ],
421 | "language": "python",
422 | "metadata": {},
423 | "outputs": [],
424 | "prompt_number": 111
425 | },
426 | {
427 | "cell_type": "code",
428 | "collapsed": false,
429 | "input": [
430 | "clf.predict_proba(newdata)"
431 | ],
432 | "language": "python",
433 | "metadata": {},
434 | "outputs": [
435 | {
436 | "output_type": "pyout",
437 | "prompt_number": 112,
438 | "text": [
439 | "array([[ 0., 1., 0., 0., 0., 0., 0., 0., 0.]])"
440 | ]
441 | }
442 | ],
443 | "prompt_number": 112
444 | },
445 | {
446 | "cell_type": "code",
447 | "collapsed": false,
448 | "input": [],
449 | "language": "python",
450 | "metadata": {},
451 | "outputs": []
452 | }
453 | ],
454 | "metadata": {}
455 | }
456 | ]
457 | }
--------------------------------------------------------------------------------
/week6/tree.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/herrfz/dataanalysis/fa2d42045f364f1a5b0683b7fb908399d69ff1cb/week6/tree.png
--------------------------------------------------------------------------------
/week6/tree2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/herrfz/dataanalysis/fa2d42045f364f1a5b0683b7fb908399d69ff1cb/week6/tree2.png
--------------------------------------------------------------------------------
/week6/tree3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/herrfz/dataanalysis/fa2d42045f364f1a5b0683b7fb908399d69ff1cb/week6/tree3.png
--------------------------------------------------------------------------------
/week8/multiple_testing.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "metadata": {
3 | "name": "multiple_testing"
4 | },
5 | "nbformat": 3,
6 | "nbformat_minor": 0,
7 | "worksheets": [
8 | {
9 | "cells": [
10 | {
11 | "cell_type": "code",
12 | "collapsed": false,
13 | "input": [
14 | "import pandas as pd\n",
15 | "import numpy as np\n",
16 | "import patsy as pt\n",
17 | "import statsmodels.api as sm\n",
18 | "from statsmodels.sandbox.stats.multicomp import multipletests"
19 | ],
20 | "language": "python",
21 | "metadata": {},
22 | "outputs": [],
23 | "prompt_number": 1
24 | },
25 | {
26 | "cell_type": "markdown",
27 | "metadata": {},
28 | "source": [
29 | "\n",
52 | "\n",
53 | "Test"
54 | ]
55 | },
56 | {
57 | "cell_type": "code",
58 | "collapsed": false,
59 | "input": [
60 | "np.random.seed(1010093)\n",
61 | "\n",
62 | "pValues = []\n",
63 | "\n",
64 | "for i in xrange(1000):\n",
65 | " x = np.random.normal(size=20)\n",
66 | " y = np.random.normal(size=20)\n",
67 | " y, x = pt.dmatrices('y ~ x')\n",
68 | "\n",
69 | " pValues.append(sm.OLS(y, x).fit().pvalues[1])\n",
70 | " \n",
71 | "pValues = np.array(pValues)"
72 | ],
73 | "language": "python",
74 | "metadata": {},
75 | "outputs": [],
76 | "prompt_number": 17
77 | },
78 | {
79 | "cell_type": "code",
80 | "collapsed": false,
81 | "input": [
82 | "sum(pValues < .05)"
83 | ],
84 | "language": "python",
85 | "metadata": {},
86 | "outputs": [
87 | {
88 | "output_type": "pyout",
89 | "prompt_number": 18,
90 | "text": [
91 | "54"
92 | ]
93 | }
94 | ],
95 | "prompt_number": 18
96 | },
97 | {
98 | "cell_type": "code",
99 | "collapsed": false,
100 | "input": [
101 | "_, p_adjust, _, _ = multipletests(pValues, method='bonferroni')\n",
102 | "sum(p_adjust < .05)"
103 | ],
104 | "language": "python",
105 | "metadata": {},
106 | "outputs": [
107 | {
108 | "output_type": "pyout",
109 | "prompt_number": 19,
110 | "text": [
111 | "0"
112 | ]
113 | }
114 | ],
115 | "prompt_number": 19
116 | },
117 | {
118 | "cell_type": "code",
119 | "collapsed": false,
120 | "input": [
121 | "_, p_adjust, _, _ = multipletests(pValues, method='fdr_bh')\n",
122 | "sum(p_adjust < .05)"
123 | ],
124 | "language": "python",
125 | "metadata": {},
126 | "outputs": [
127 | {
128 | "output_type": "pyout",
129 | "prompt_number": 20,
130 | "text": [
131 | "0"
132 | ]
133 | }
134 | ],
135 | "prompt_number": 20
136 | },
137 | {
138 | "cell_type": "code",
139 | "collapsed": false,
140 | "input": [
141 | "np.random.seed(1010093)\n",
142 | "\n",
143 | "pValues = []\n",
144 | "\n",
145 | "for i in xrange(1000):\n",
146 | " x = np.random.normal(size=20)\n",
147 | " \n",
148 | " # first 500 beta = 0, last 500 beta = 2\n",
149 | " if i < 500:\n",
150 | " y = np.random.normal(size=20)\n",
151 | " else:\n",
152 | " y = np.random.normal(loc=2*x, size=20)\n",
153 | "\n",
154 | " y, x = pt.dmatrices('y ~ x')\n",
155 | " \n",
156 | " pValues.append(sm.OLS(y, x).fit().pvalues[1])\n",
157 | " \n",
158 | "pValues = np.array(pValues)"
159 | ],
160 | "language": "python",
161 | "metadata": {},
162 | "outputs": [],
163 | "prompt_number": 21
164 | },
165 | {
166 | "cell_type": "code",
167 | "collapsed": false,
168 | "input": [
169 | "trueStatus = np.concatenate([np.repeat('zero', 500), np.repeat('not zero', 500)])\n",
170 | "pd.crosstab(pValues < .05, trueStatus)"
171 | ],
172 | "language": "python",
173 | "metadata": {},
174 | "outputs": [
175 | {
176 | "html": [
177 | "\n",
178 | "\n",
179 | " \n",
180 | "
\n",
204 | "
"
205 | ],
206 | "output_type": "pyout",
207 | "prompt_number": 22,
208 | "text": [
209 | "col_0 not zero zero\n",
210 | "row_0 \n",
211 | "False 0 470\n",
212 | "True 500 30"
213 | ]
214 | }
215 | ],
216 | "prompt_number": 22
217 | },
218 | {
219 | "cell_type": "code",
220 | "collapsed": false,
221 | "input": [
222 | "_, p_adjust, _, _ = multipletests(pValues, method='bonferroni')\n",
223 | "pd.crosstab(p_adjust < .05, trueStatus)"
224 | ],
225 | "language": "python",
226 | "metadata": {},
227 | "outputs": [
228 | {
229 | "html": [
230 | "| col_0 | \n", 182 | "not zero | \n", 183 | "zero | \n", 184 | "
|---|---|---|
| row_0 | \n", 187 | "\n", 188 | " | \n", 189 | " |
| False | \n", 194 | "0 | \n", 195 | "470 | \n", 196 | "
| True | \n", 199 | "500 | \n", 200 | "30 | \n", 201 | "
\n",
231 | "\n",
232 | " \n",
233 | "
\n",
257 | "
"
258 | ],
259 | "output_type": "pyout",
260 | "prompt_number": 23,
261 | "text": [
262 | "col_0 not zero zero\n",
263 | "row_0 \n",
264 | "False 29 500\n",
265 | "True 471 0"
266 | ]
267 | }
268 | ],
269 | "prompt_number": 23
270 | },
271 | {
272 | "cell_type": "code",
273 | "collapsed": false,
274 | "input": [
275 | "plot(pValues, p_adjust, 'ok')\n",
276 | "xlim(-.05, 1.05)\n",
277 | "ylim(-.05, 1.05)\n",
278 | "xlabel('pValues')\n",
279 | "ylabel('p_adjust');"
280 | ],
281 | "language": "python",
282 | "metadata": {},
283 | "outputs": [
284 | {
285 | "output_type": "display_data",
286 | "png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHPxJREFUeJzt3X9M03f+B/BnoZUyZeAPJtpyUQvhx/glgkg8Tb1Fy2Ry\nibocLvOmU8aZE9gut+TiXU7QbDeX7A+BeWHzx86bMqZngme1t9Ot81ABFRVPjMeInJWbRlSmmxah\nfL5/OPul0pZC+6Hl83k+kib03Xc/fb1xe/bN58f7oxAEQQAREclKkL8LICKikcfwJyKSIYY/EZEM\nMfyJiGSI4U9EJENKfxfgCYVC4e8SiIhGJVcndI6amb8gCMN6bNy4cdjvHa0PjlkeD45Z+g9vx+vO\nqAl/IiLyHYY/EZEMST789Xq9v0sYcRyzPHDM0ifmeBXCYDuGAoBCoRh0/xURETlyl52Sn/kTEZET\ngohWr14tPPfcc0JSUpLLPkVFRUJMTIyQkpIiNDU1Oe0znDI3btwoKBQKAYDbR3BwsDBmzJhB+4nx\nUKlUwrhx44TIyEghKChoWNtwN8bg4OAhbev5558Xxo0bN6T3BQUFCSqVyuX4/PF7HYlH/9+7QqEQ\nQkNDhcjISL/X5e1jzJgxglKp9Oj/naH+vsaMGSOEhIQM6T2hoaFCeHi4Q7tKpRLUarXT96jVamHs\n2LED/ht21ubN4+ltqVQqr7YfHBwshIaGOt3uzJkzhUOHDg05AwfLTlHD//jx40JTU5PL8DcajcKL\nL74oCIIg1NfXC1lZWU77uRuAMxs3bvT7/0R88MEHH756REVFDesLAHCdnaLu9pk3bx7Gjx/v8vWD\nBw/itddeAwBkZWWhq6sLN2/e9PpzKysrvd4GEVGguHHjBioqKny6Tb9e4dvR0YHo6Gj7c61Wi+vX\nr2Py5MkD+paWltp/1uv1bo+C9/b2+rJMIiK/s1qtg/Yxm80wm80ebc/vyzsITx2JdrWUQ//wH4xS\n6fdhERH5lFqtHrTP0xPjsrIyl339eraPRqOBxWKxP79+/To0Go3X212/fr3X2yAiChRRUVEoKiry\n6Tb9Gv55eXnYvXs3AKC+vh4RERFOd/kMVWZmJiIiIjzqGxwcjDFjxnj9mcOhUqkQFhaGyMhIBAUN\n75/C3aJ3wcHBQ9rO888/j7CwsCG9LygoCCqVyulrrtqloP/vXaFQIDQ0FJGRkX6syDfGjBkDlUrl\n88UUFQoFxowZg5CQkCG9JzQ0FOHh4Q7tKpXK5SxYrVZj7NixDv8NKxSKAW3eCg4OdvgdqVQqr7Yf\nHByM0NDQAdtQqVRIT0/H9u3bkZubO+ztOyPq/pEVK1bg66+/RmdnJ6Kjo1FWVoaenh4AQGFhIRYv\nXozDhw8jJiYGY8eOxa5du3zyueXl5ejq6hrQbjAYYDKZfPIZRESjmajhX11dPWgfMc7M6e7udtru\nyQETIiI5kOQVvq7+tPTkgAkRkRxIMvyLi4uh0+kc2nQ6nc8PmBARjVaSPCfyyYGRiooKWK1WqNVq\nFBUV+fyACRHRaMVVPYmIJIqrehIRkQOGPxGRDDH8iYhkiOFPRCRDDH8iIhli+BMRyRDDn4hIhhj+\nREQyxPAnIpIhhj8RkQwx/ImIZIjhT0QkQwx/IiIZYvgTEckQw5+ISIYY/kREMsTwJyKSIYY/EZEM\nMfyJiGSI4U9EJEMMfyIiGWL4ExHJEMOfiEiGGP5ERDLE8CcikiGGPxGRDDH8iYhkSPTwN5lMiI+P\nR2xsLLZs2TLg9c7OTuTk5CAtLQ1JSUn45JNPxC6JiEj2FIIgCGJt3GazIS4uDkePHoVGo0FmZiaq\nq6uRkJBg71NaWoru7m786U9/QmdnJ+Li4nDz5k0olcr/L1KhgIhlEhFJkrvsFHXm39jYiJiYGEyb\nNg0qlQr5+fmora116DNlyhTcu3cPAHDv3j1MnDjRIfiJiMj3RE3Zjo4OREdH259rtVo0NDQ49Cko\nKMDPfvYzTJ06Fffv38fnn3/udFulpaX2n/V6PfR6vRglExGNWmazGWaz2aO+ooa/QqEYtM+7776L\ntLQ0mM1mtLW1YeHChbhw4QLCwsIc+vUPfyIiGujpiXFZWZnLvqLu9tFoNLBYLPbnFosFWq3Woc/J\nkyfx8ssvAwB0Oh2mT5+OK1euiFkWEZHsiRr+GRkZaG1tRXt7Ox49eoSamhrk5eU59ImPj8fRo0cB\nADdv3sSVK1cwY8YMrz/baDTCYDBAr9fDYDDAaDR6vU0iIqkQdbePUqlEZWUlDAYDbDYb1qxZg4SE\nBFRVVQEACgsLsWHDBqxevRqpqano6+vD+++/jwkTJnj1uUajESUlJWhra7O3Pfk5NzfXq20TEUmB\nqKd6+spQT/U0GAz44osvnLabTCZflkZEFLD8dqqnv3R3dzttt1qtI1wJEVFgkmT4h4SEOG1Xq9Uj\nXAkRUWCSZPgXFxdDp9M5tOl0OhQVFfmpIiKiwCLJS2mfHNStqKiA1WqFWq1GUVERD/YSEf1Ikgd8\niYhIhgd8iYjIPYY/EZEMMfyJiGSI4U9EJEMMfyIiGWL4ExHJkGTDn6t6EhG5JsmLvLiqJxGRe5Kc\n+ZeXlzsEP/A4/CsqKvxUERFRYJFk+HNVTyIi9yQZ/lzVk4jIPUmGP1f1JCJyT5IHfLmqJxGRe1zV\nk4hIoriqJxEROWD4ExHJEMOfiEiGGP5ERDLE8CcikiGGPxGRDDH8iYhkiOFPRCRDDH8iIhli+BMR\nyRDDn4hIhhj+REQyxPAnIpIh0cPfZDIhPj4esbGx2LJli9M+ZrMZM2fORFJSEvR6vdglERHJnqhL\nOttsNsTFxeHo0aPQaDTIzMxEdXU1EhIS7H26urowd+5c/OMf/4BWq0VnZycmTZrkWCSXdCYiGjK/\nLenc2NiImJgYTJs2DSqVCvn5+aitrXXos3fvXixbtgxarRYABgQ/ERH5nqh38uro6EB0dLT9uVar\nRUNDg0Of1tZW9PT0YMGCBbh//z5KSkqwcuXKAdsqLS21/6zX67l7iIjoKWazGWaz2aO+ooa/QqEY\ntE9PTw+amppw7NgxPHjwANnZ2ZgzZw5iY2Md+vUPfyIiGujpiXFZWZnLvh7t9nnhhRc8anuaRqOB\nxWKxP7dYLPbdO09ER0dj0aJFCA0NxcSJEzF//nxcuHDBk7KIiGiY3Ib/w4cPcfv2bdy6dQt37tyx\nP9rb29HR0THoxjMyMtDa2or29nY8evQINTU1yMvLc+jz85//HHV1dbDZbHjw4AEaGhqQmJjo3aiI\niMgtt7t9qqqqsHXrVvzvf//DrFmz7O1hYWFYv3794BtXKlFZWQmDwQCbzYY1a9YgISEBVVVVAIDC\nwkLEx8cjJycHKSkpCAoKQkFBAcOfiEhkHp3qWVFRgaKiopGoxyme6klENHRen+o5efJk3L9/HwCw\nefNmLF26FE1NTb6rkIiIRpRH4b9582aEhYWhrq4Ox44dw+uvv45f/epXYtdGREQi8Sj8g4ODAQCH\nDh1CQUEBXnrpJfT09IhaGBERicej8NdoNHjjjTdQU1OD3NxcWK1W9PX1iV0bERGJxKMDvj/88ANM\nJhNSUlIQGxuLb7/9FhcvXsSiRYtGokYe8CUiGgZ32elR+F+7dg2CIAy4YvcnP/mJbyocBMOfiGjo\nvA7/pKQke/BbrVZcvXoVcXFxuHTpkm8rdYHhT0Q0dO6y06O1ff797387PG9qasKHH37ofWVEROQX\nw17PPykpacCXglg48yciGjqvZ/4ffPCB/ee+vj40NTVBo9H4pjoiIhpxHoX//fv37fv8lUolXnrp\nJSxbtkzUwoiISDyi3sbRV7jbh4ho6Ia926ekpARbt27FkiVLnG504sSJKCwsxJw5c3xTKRERjQi3\nM/8zZ84gIyPD5W3Bbt++jT/84Q+4fPmyWPUB4MyfiGg4hj3zz8jIAAC398tVqVTDr4yIiPzC7cw/\nOTnZ9RsVCjQ3N4tSlLPP4syfiGhohj3z//vf/w4A2LZtGwBg5cqVEAQBe/bs8XGJvmc0GlFeXo7u\n7m6EhISguLgYubm5/i6LiCggeHS2T1paGs6fP+/QNnPmTJw7d060wvob6szfaDSipKQEbW1t9jad\nToetW7fyC4CIZMPrO3kJgoC6ujr78xMnTgT0bpjy8nKH4AeAtrY2VFRU+KkiIqLA4tFFXjt37sTq\n1avx3XffAQAiIiKwa9cuUQvzRnd3t9N2q9U6wpUQEQUmj8J/1qxZaG5uRldXFxQKBcLDw8Wuyysh\nISFO29Vq9QhXQkQUmDwKf+DxLRxbWlocZs9//OMfRSnKW8XFxWhraxuwz7+oqMiPVRERBQ6Pwr+w\nsBAPHz7El19+iYKCAuzbtw9ZWVli1zZsTw7qVlRUwGq1Qq1Wo6ioiAd7iYh+5NHZPsnJybh48SJS\nUlLQ3NyM77//Hjk5OQ4HgcXE8/yJiIbO67N9QkNDAQDPPPMMOjo6oFQqcePGDd9VSEREI8qj3T5L\nlizB3bt38fbbb2PWrFkAgIKCAlELIyIi8Qx5SWer1Qqr1YqIiAh72z//+U8sXLjQ58U9wd0+RERD\n5/UN3Acj9tW+DH8ioqHzep8/ERFJC8OfiEiGGP5ERDLk0dk+Dx8+xLZt21BXVweFQoF58+Zh3bp1\n9uUSpk+fLmqRRETkWx7N/H/5y1+ipaUFxcXFWL9+PS5duoSVK1faXz9w4IDL95pMJsTHxyM2NhZb\ntmxx2e/06dNQKpVut0VERL7h0dk+iYmJaGlpGbTtaTabDXFxcTh69Cg0Gg0yMzNRXV2NhISEAf0W\nLlyIZ555BqtXr8ayZcsci+TZPkREQzbsO3k9kZ6ejlOnTiE7OxsAUF9fb7/Yy53GxkbExMRg2rRp\nAID8/HzU1tYOCP+KigosX74cp0+fdrmt0tJS+896vd7tfYWJiOTIbDbDbDZ71Nej8D9z5gzmzp2L\n6OhoKBQKXLt2DXFxcUhOTnZ7L9+Ojg5ER0fbn2u1WjQ0NAzoU1tbiy+//BKnT5+GQqFwuq3+4U9E\nRAM9PTEuKytz2dej8DeZTMMqxFWQ9/fmm2/ivffes/95wt07RETi8yj8n+y2GSqNRgOLxWJ/brFY\noNVqHfqcPXsW+fn5AIDOzk4cOXIEKpUKeXl5w/pMIiIanE+Wd3Clt7cXcXFxOHbsGKZOnYrZs2c7\nPeD7xOrVq7FkyRIsXbrUsUge8CUiGjKvD/gOl1KpRGVlJQwGA2w2G9asWYOEhARUVVUBeHyTGCIi\nGnmizvx9ZTgzf6PRiPLycnR3dyMkJATFxcW8kxcRyYroq3qKbajhbzQasXbtWocbzkRFRWH79u38\nAiAi2ZBd+KenpztdYjo9PR1nz571ZWlERAFLdks6t7e3O22/evXqyBZCRBSgJBn+RETkniTD39V1\nCVx9lIjoMUmG/+bNmxEVFeXQFhUVhU2bNvmpIiKiwCLJA77A4zN+KioqYLVaoVarUVRUxDN9iEhW\nZHe2DxERyfBsHyIico/hT0QkQ5INf6PRCIPBAL1eD4PBAKPR6O+SiIgChqgLu/mL0WhESUkJ2tra\n7G1PfuZBXyIiic78y8vLHYIfeBz+FRUVfqqIiCiwSDL8u7u7nbZbrdYRroSIKDBJMvxDQkKctqvV\n6hGuhIgoMEky/IuLi6HT6RzadDodioqK/FQREVFgkeQB3ycHdXmFLxGRc7zCl4hIoniFLxEROWD4\nExHJEMOfiEiGGP5ERDLE8CcikiGGPxGRDDH8iYhkiOFPRCRDDH8iIhli+BMRyZBkw7+0tBSTJk1C\nREQEJk2ahNLSUn+XREQUMCS5sFtpaSneeecd9Pb22tveeecd+2tERHInyYXdJk2ahNu3bzttv3Xr\nli9LIyIKWH5d2M1kMiE+Ph6xsbHYsmXLgNf37NmD1NRUpKSkYO7cuWhubvb6M/vP+Pvr6enxettE\nRFIg6m4fm82G9evX4+jRo9BoNMjMzEReXh4SEhLsfWbMmIHjx48jPDwcJpMJb7zxBurr6736XKXS\n+bBUKpVX2yUikgpRZ/6NjY2IiYnBtGnToFKpkJ+fj9raWoc+2dnZCA8PBwBkZWXh+vXrXn/uokWL\nnLYvXLjQ620TEUmBqDP/jo4OREdH259rtVo0NDS47L9jxw4sXrzY6Wv9D9Tq9Xro9XqX23G2vx8A\n7ty5475gIqJRzGw2w2w2e9RX1PBXKBQe9/3qq6+wc+dOnDhxwunrQzlLp6Ojw2m7L/6qICIKVE9P\njMvKylz2FTX8NRoNLBaL/bnFYoFWqx3Qr7m5GQUFBTCZTBg/frzXn3vjxg2n7Tdv3vR620REUiDq\nPv+MjAy0traivb0djx49Qk1NDfLy8hz6XLt2DUuXLsWnn36KmJgYn3zu1KlTnbZPmTLFJ9snIhrt\nRJ35K5VKVFZWwmAwwGazYc2aNUhISEBVVRUAoLCwEJs2bcLdu3exbt06AI/PyGlsbPTqc6dMmYKL\nFy8OaHf1pUBEJDeSvMjLaDRi7dq1Drt/oqKisH37duTm5opRIhFRwPHrRV5ERBR4JBn+5eXlAw76\n3rhxAxUVFX6qiIgosEgy/Lu7u522W63WEa6EiCgwSTL8Q0JCnLar1eoRroSIKDBJMvyzs7MHrO+j\nVCoxZ84cP1VERBRYJBn+p06dGrCyZ29vr9cLxhERSYUkw5/7/ImI3JNk+HOfPxGRe5IM/+LiYuh0\nOoc2nU6HoqIiP1VERBRYJBn+ubm5mD17NpRKJYKDg6FUKjF79mxe3UtE9CNJhn9paSn27duH3t5e\n2Gw29Pb2Yt++fbx5OxHRjyS5tg9v4E5EJMO1fXgDdyIi9yQZ/jabzWl7X1/fCFdCRBSYJBn+zz33\n3JDaiYjkRpLh72rm72p3EBGR3EjygK9KpXIa9CqVCo8ePfJlaUREAUt2B3wVCoW/SyAiCmiSnPm7\nC/9RMFwiIp+Q3cyfiIjcY/gTEckQw5+ISIYY/kREMiTJ8A8Kcj6s4ODgEa6EiCgwSTL8n332Waft\nYWFhI1wJEVFgkmT4u7pdI2/jSET0GM/zJyKSKJ7nT0REDhj+REQyxPAnIpIhhj8RkQwx/ImIZEj0\n8DeZTIiPj0dsbCy2bNnitE9xcTFiY2ORmpqKc+fOiV0SEZHsiRr+NpsN69evh8lkQktLC6qrq3H5\n8mWHPocPH8Y333yD1tZWfPTRR1i3bp2YJREREUQO/8bGRsTExGDatGlQqVTIz89HbW2tQ5+DBw/i\ntddeAwBkZWWhq6sLN2/eFLMsIiLZU4q58Y6ODkRHR9ufa7VaNDQ0DNrn+vXrmDx5skO/0tJS+896\nvR56vV6UmomIRiuz2Qyz2exRX1HD39PbKT59BZqz9/UPfyIiGujpiXFZWZnLvqLu9tFoNLBYLPbn\nFosFWq3WbZ/r169Do9GIWRYRkeyJGv4ZGRlobW1Fe3s7Hj16hJqaGuTl5Tn0ycvLw+7duwEA9fX1\niIiIGLDLZ6hcrWXBdX2IiB4TdbePUqlEZWUlDAYDbDYb1qxZg4SEBFRVVQEACgsLsXjxYhw+fBgx\nMTEYO3Ysdu3a5ZPPZtATEbkmyVU9iYhI5qt6enrkW0o4ZnngmKVPzPEy/CWIY5YHjln6GP5ERORT\nDH8iIhkaNQd8iYho6FxFvKinevrKKPh+IiIaVbjbh4hIhhj+REQyxPAnIpIhyYS/HO8YNtiY9+zZ\ng9TUVKSkpGDu3Llobm72Q5W+5cm/MwCcPn0aSqUSBw4cGMHqxOHJmM1mM2bOnImkpCRJLHc+2Jg7\nOzuRk5ODtLQ0JCUl4ZNPPhn5In3o9ddfx+TJk5GcnOyyj8/zS5CA3t5eQafTCVevXhUePXokpKam\nCi0tLQ59jEaj8OKLLwqCIAj19fVCVlaWP0r1GU/GfPLkSaGrq0sQBEE4cuSILMb8pN+CBQuE3Nxc\nYf/+/X6o1Hc8GfPdu3eFxMREwWKxCIIgCLdu3fJHqT7jyZg3btwo/O53vxME4fF4J0yYIPT09Pij\nXJ84fvy40NTUJCQlJTl9XYz8ksTMX453DPNkzNnZ2QgPDwfweMzXr1/3R6k+48mYAaCiogLLly9H\nZGSkH6r0LU/GvHfvXixbtsy+XPqkSZP8UarPeDLmKVOm4N69ewCAe/fuYeLEiVAqR8XJi07NmzcP\n48ePd/m6GPklifB3djewjo6OQfuM5jD0ZMz97dixA4sXLx6J0kTj6b9zbW2t/V7Qo/0aEU/G3Nra\nijt37mDBggXIyMjAX//615Eu06c8GXNBQQEuXbqEqVOnIjU1FVu3bh3pMkeUGPk1er8q+/HlHcNG\ni6HU/tVXX2Hnzp04ceKEiBWJz5Mxv/nmm3jvvffsqxk+/W8+2ngy5p6eHjQ1NeHYsWN48OABsrOz\nMWfOHMTGxo5Ahb7nyZjfffddpKWlwWw2o62tDQsXLsSFCxcQFhY2AhX6h6/zSxLhL8c7hnkyZgBo\nbm5GQUEBTCaT2z8rRwNPxnz27Fnk5+cDeHxQ8MiRI1CpVANuIjRaeDLm6OhoTJo0CaGhoQgNDcX8\n+fNx4cKFURv+noz55MmT+P3vfw8A0Ol0mD59Oq5cuYKMjIwRrXWkiJJfXh81CAA9PT3CjBkzhKtX\nrwrd3d2DHvA9derUqD/46cmY//vf/wo6nU44deqUn6r0LU/G3N+qVauEv/3tbyNYoe95MubLly8L\nL7zwgtDb2yv88MMPQlJSknDp0iU/Vew9T8b81ltvCaWlpYIgCMKNGzcEjUYj3L592x/l+szVq1c9\nOuDrq/ySxMzfn3cM8xdPxrxp0ybcvXvXvv9bpVKhsbHRn2V7xZMxS40nY46Pj0dOTg5SUlIQFBSE\ngoICJCYm+rny4fNkzBs2bMDq1auRmpqKvr4+vP/++5gwYYKfKx++FStW4Ouvv0ZnZyeio6NRVlaG\nnp4eAOLl16hY2I2IiHxLEmf7EBHR0DD8iYhkiOFPRCRDDH8iIhli+BP9qKysDBs2bHBoO3/+vNsz\nZ0pLS/HBBx+IXRqRzzH8iX70yiuvoKamxqHts88+wyuvvOLyPaP5KnGSN4Y/yVJ7ezvi4+Px6quv\nIjExES+//DK0Wi3Gjx/vcC3Evn37sGLFCnz88ceYPXs20tLSsHz5cjx8+NDe58kXgF6vx9mzZwE8\nvrp4+vTpAACbzYa3334bs2fPRmpqKj766CMAwLfffov58+dj5syZSE5ORl1d3UgNn4jhT/L1n//8\nB7/+9a/R0tKCZ599Ftu2bcOKFSvw2WefAQDq6+sxYcIE6HQ6LFu2DI2NjTh//jwSEhKwY8eOAdtT\nKBRO/xLYsWMHIiIi0NjYiMbGRnz88cdob29HdXU1cnJycO7cOTQ3NyMtLU30MRM9wfAn2YqOjkZ2\ndjYA4NVXX0VdXR1+8YtfYP/+/RAEwWGXz8WLFzFv3jykpKRgz549aGlp8fhzvvjiC+zevRszZ87E\nnDlzcOfOHXzzzTfIzMzErl27UFZWhubmZowbN06UcRI5I4nlHYiGo/8sXRAEKBQKaLVaTJ8+HWaz\nGQcOHEB9fT0AYNWqVTh48CCSk5Pxl7/8BWazecD2lEol+vr6AABWq9XhtcrKSixcuHDAe/71r3/h\n0KFDWLVqFX7zm99g5cqVPhwhkWuc+ZNsXbt2zR7ue/fuxbx58wA8Xmflrbfegk6nw9SpUwEA33//\nPaKiotDT04NPP/3U/sUh9Fs2etq0aThz5gwAYP/+/fbPMRgM2LZtG3p7ewE83t304MEDXLt2DZGR\nkVi7di3Wrl0riVuL0ujB8CfZiouLw4cffojExER899139gXwli9fjpaWFqxYscLed/PmzcjKysJP\nf/pTJCQk2Nv77+f/7W9/iz//+c9IT0/H7du37e1r165FYmIi0tPTkZycjHXr1qG3txdmsxlpaWlI\nT0/H559/jpKSkhEcPckdF3YjWWpvb8eSJUtw8eJFf5dC5Bec+ZNs8Rx9kjPO/ImIZIgzfyIiGWL4\nExHJEMOfiEiGGP5ERDLE8CcikiGGPxGRDP0fqiVcfC2OPvEAAAAASUVORK5CYII=\n"
287 | }
288 | ],
289 | "prompt_number": 24
290 | },
291 | {
292 | "cell_type": "code",
293 | "collapsed": false,
294 | "input": [
295 | "_, p_adjust, _, _ = multipletests(pValues, method='fdr_bh')\n",
296 | "pd.crosstab(p_adjust < .05, trueStatus)"
297 | ],
298 | "language": "python",
299 | "metadata": {},
300 | "outputs": [
301 | {
302 | "html": [
303 | "| col_0 | \n", 235 | "not zero | \n", 236 | "zero | \n", 237 | "
|---|---|---|
| row_0 | \n", 240 | "\n", 241 | " | \n", 242 | " |
| False | \n", 247 | "29 | \n", 248 | "500 | \n", 249 | "
| True | \n", 252 | "471 | \n", 253 | "0 | \n", 254 | "
\n",
304 | "\n",
305 | " \n",
306 | "
\n",
330 | "
"
331 | ],
332 | "output_type": "pyout",
333 | "prompt_number": 25,
334 | "text": [
335 | "col_0 not zero zero\n",
336 | "row_0 \n",
337 | "False 0 483\n",
338 | "True 500 17"
339 | ]
340 | }
341 | ],
342 | "prompt_number": 25
343 | },
344 | {
345 | "cell_type": "code",
346 | "collapsed": false,
347 | "input": [
348 | "plot(pValues, p_adjust, 'ok')\n",
349 | "xlim(-.05, 1.05)\n",
350 | "ylim(-.05, 1.05)\n",
351 | "xlabel('pValues')\n",
352 | "ylabel('p_adjust');"
353 | ],
354 | "language": "python",
355 | "metadata": {},
356 | "outputs": [
357 | {
358 | "output_type": "display_data",
359 | "png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtUVWX+P/D3PhxMML7gpREFvIAsLnKTvC5HpRrDMaSW\nVw5pZUqOKy/lmpzMybPBMdNZzVoqOcvSbErF2zSDso0praNjhnhLUlwNXkEmnVTwEiK3/fvDOD8R\nOGefyz7X92st1+rAcw6fPTVvHj/72c8jyLIsg4iIvIrG2QUQEZHjMfyJiLwQw5+IyAsx/ImIvBDD\nn4jIC2mdXYASgiA4uwQiIrfU3oJOt5n5y7Js1R+9Xm/1e931D6/ZO/7wmj3/j63Xa4rbhD8REdkP\nw5+IyAt5fPinpKQ4uwSH4zV7B16z51PzegXZXGPIBQiCYLZ/RURELZnKTo+f+RMRUWuqhv/LL7+M\n7t27Iz4+vt0x8+bNQ2RkJBITE3HixAk1yyEicguSJCE1NRUpKSlITU2FJEl2/xmqrvOfPn065s6d\nixdeeKHN7+/Zswdnz55FWVkZDh8+jNmzZ6OoqEjNkoiIXI4kSXj77bdx6tQp1NfXt/r+uXPnAADP\nPPOM3X6mqjP/ESNGoHPnzu1+f9euXXjxxRcBAEOGDEF1dTWuXr2qZklERC5FkiTMnDkTJ06caDP4\ngfvhv2bNGrv+XKc+4VtZWYmwsDDj69DQUFy+fBndu3dvNVYUReM/p6SkeN1dfyLyHM0z/bKyMty5\nc0fRe2pra82OMRgMMBgMij7P6ds7PHwnur2tHB4MfyIidyOKInJzc3Hz5k00NDRY/P6OHTuaHfPw\nxDg7O7vdsU4N/5CQEFRUVBhfX758GSEhIU6siIjIviRJwvPPP4+bN29a/Rl+fn6YO3euHaty8lLP\n9PR0fPLJJwCAoqIiBAUFtdnyISJyF5IkITg4GIIgQBAEpKWl2RT8Pj4+WLhwoV1v9gIqz/x1Oh32\n79+Pa9euISwsDNnZ2cYbGrNmzcLYsWOxZ88e9OvXD506dcLGjRvVLIeISFWSJGHy5Mmoqamxy+cF\nBwdj/fr1dg9+gE/4EhHZTJIkzJgxw+bVivYOez7hS0SkAkmSkJycjHHjxtkc/N27d1dtlt8Whj8R\nkYVEUYS/vz/S0tJw4sQJqzsTgiDAz88Per0eV65ccVjwAy6w1JOIyB00r80/c+aMojX37fHz88PC\nhQudvnyd4U9EZIIoili+fDnq6uqs/gytVouEhATk5OQ4dHZvCsOfiOgXoihi5cqVuHv3rl0+T6fT\nYcuWLXb5LHtj+BORV8vMzMT27dvR2Nhot8/08fHB5MmTXTb4AYY/EXmxUaNG4cCBA3b5LK1Wi8WL\nFzu9l68UV/sQkVdpXqkjCIJdgl+j0SA8PBz//Oc/3Sb4Ac78icgLiKKIv/zlL7hz547dHhjt3r07\nNmzY4DI3cC3F8Ccij5aZmYm8vDy7fqZer3erWX5buL0DEXksSZKQnp6OpqYmu3xe//79cerUKbt8\nliNwewci8hqSJKFHjx7QaDRIS0uzOPgDAwNRUFAAWZZb/XGn4DeH4U9EHqF5n5309HRcuXLF4m5B\nREQECgoKUF1d7bZ9fEsw/InI7YmiiEmTJuHEiRMWz/Q7dOgAvV6Ps2fPekXoN+MNXyJyO7ZuueDr\n64v4+HiX2m7B0Rj+ROQ2JEnCvHnzcP78eYvfKwgCnn76acydO9drA/9BDH8icgu27ruTkZHh0tst\nOBrDn4hcmq1777jDPjvOwBu+ROSSmpds5uXlWRz8zQekyLKMhoYGBn8bOPMnIpdi66EpERERWLVq\nFfv6ZjD8icgl2HoIeocOHbBo0SK333bBUdj2ISKnevA8XEuC/8HWjizLuHfvHoPfApz5E5HDiaKI\n3Nxc3Lhxw6p9u9x9R01XwPAnIodo7uWfPHnS6o3WuHLHftj2ISLViaKI8ePHW7X9AgAkJyejoKCA\nK3fsiDN/IlKNPWb7rnwIujvjzJ+I7E6SJAQHByMtLc3q2X7zDV0Gvzo48yciu7HHgej+/v7Yvn07\nb+aqjDN/IrLag4eh23oguq+vL3Q6HX7++WcGvwNw5k9EFhNFEX/605+s3m+nGVfvOA9n/kSkmCiK\neOSRR5CdnW118Dev3OG+O86levgXFhYiOjoakZGRWLFiRavvX7t2DWPGjEFSUhLi4uLw8ccfq10S\nEVlo1KhREAQB2dnZVh+gAtxfuXPs2DG2dVyAIFvzeJ1CjY2NiIqKwt69exESEoJBgwYhLy8PMTEx\nxjGiKOLevXtYvnw5rl27hqioKFy9ehVa7f/vSJk6gZ6I1NW7d2+Ul5fb9Bncd8c5TGWnqjP/4uJi\n9OvXD3369IGvry8yMjKQn5/fYkyPHj1w69YtAMCtW7fQtWvXFsFPRI7XPNMXBMHm4Nfr9dx3xwWp\nmrKVlZUICwszvg4NDcXhw4dbjMnKysKTTz6Jnj174vbt29i+fXubn/XgfzgpKSlISUlRo2Qir2av\nG7l+fn5YuHAhA9/BDAYDDAaDorGqhr8gCGbHvPPOO0hKSoLBYMC5c+cwevRonDx5EgEBAS3G8T8i\nIvXYcjYucP//6+Hh4dxH38kenhhnZ2e3O1bVtk9ISAgqKiqMrysqKhAaGtpizKFDhzBp0iQA9w9h\n6Nu3L3744Qc1yyIi3J9QabVaCIKAtLQ0q4I/PDwcBQUFaGpqwtmzZxn8bkTVmf/AgQNRVlaGixcv\nomfPnti2bRvy8vJajImOjsbevXsxfPhwXL16FT/88APCw8PVLIvIa9l6YEqzXr164dKlS3aqipxB\n1Zm/VqtFbm4uUlNTERsbiylTpiAmJgbr1q3DunXrAABvvfUWjh49isTERPzmN7/BypUr0aVLFzXL\nIvIqzfvsNM/wrQ1+QRDQrVs36PV6Br8HUHWpp71wqSeR5Wzt4z9o5MiR2L9/vx2qIkdy2lJPInIO\nURTx3HPPWR38Go0GAQEBxmMSGfyehwvqiTxMZmZmq3trlggMDMTmzZt589bDMfyJPEBmZia2bdtm\n9YEpzdje8R5s+xC5ud69eyMvL8+mc3HZ3vE+DH8iN/TgCh5rt18IDg42novLhyi9D8OfyM2Ioohx\n48ZZtWQzIiLCuJ3yjz/+yL6+F2PPn8hN2NLXDw4Oxvr16xn2ZMTwJ3ID1p6NGxAQgAULFrCtQ60w\n/IlclCiKWLlyJe7evWvxezt27IidO3dypk/tYvgTuZC4uDicPn3aps/Q6XQ8GpHM4g1fIhcgSRI6\nduxodfBrNBrodDrIsszgJ0UY/kROJEkSIiIiMG7cONy7d8/i9zcv12xsbGTok0XY9iFyEltPzeLT\nuGQLzvyJHEySJHTq1AnZ2dlWBX/zASoMfrIFw5/IAZrbOz4+PkhLS0NNTY3Fn6HRaKDX63Hu3Dmu\n4iGbse1DpDJRFLFs2TI0NDRY9X6NRoMpU6awp092xZk/kQoePB83Ozvb4uBvbu3IssybuaQKzvyJ\n7EwURWRnZ1v9fq7TJ0fgzJ/ITkRRNM70rSEIAvR6PYOfHIIzfyIb2eOsXC7bJEfjzJ/ISpIkITk5\nGRMnTrQ4+AVBaLG9MoOfHI0zfyIrWLu9siAIWLJkCXfZJKdj+BNZyNoD0oOCgrBp0yau0SeXwPAn\nUsja3n6vXr1w6dIllaoisg57/kRmiKIIf39/pKWlMfjJYzD8iUxo3nxN6YEqycnJxpu4siwz+Mll\nse1D1AZJkjBjxgyLDknnw1nkThj+RA+QJAnPP/88bt68adH7GPzkbtj2IULLvr4lwd+hQwc+lUtu\niTN/8mqSJGH8+PGoq6uz+L0RERFYtWoVl26SW+LMn7xS8/76aWlpFge/n58fCgoKcPbsWQY/uS2G\nP3kdURStWrYJAI8++ih27NjB0Ce3p3r4FxYWIjo6GpGRkVixYkWbYwwGAwYMGIC4uDikpKSoXRJ5\nqebZvrW7bgYFBWHr1q0MfvIIgizLslof3tjYiKioKOzduxchISEYNGgQ8vLyEBMTYxxTXV2N4cOH\n41//+hdCQ0Nx7do1dOvWrWWRggAVyyQvIEkSpkyZgp9//tni9wYHB2P9+vUMfXI7prJT1Zl/cXEx\n+vXrhz59+sDX1xcZGRnIz89vMWbLli2YMGECQkNDAaBV8BPZonnnzfT0dIuDX6/XQ5Zl/Pjjjwx+\n8jiqrvaprKxEWFiY8XVoaCgOHz7cYkxZWRnq6+vxxBNP4Pbt25g/fz6mTZvW6rMe3AUxJSWF7SEy\ny9oN2Pz9/bF9+3YGPrkdg8EAg8GgaKyq4S8Igtkx9fX1OH78OPbt24eamhoMGzYMQ4cORWRkZItx\n3AKXlGrekqGxsdGi93Xu3BmffvopQ5/c1sMTY1P3txS1fZ566ilFX3tYSEgIKioqjK8rKiqM7Z1m\nYWFhePrpp+Hn54euXbti5MiROHnypJKyiFoRRRE5OTkWBb8gCNDpdLhx4waDn7yGyfC/e/curl+/\njp9++gk3btww/rl48SIqKyvNfvjAgQNRVlaGixcvoq6uDtu2bUN6enqLMc8++ywOHjyIxsZG1NTU\n4PDhw4iNjbXtqsgrSZKEZcuWWbQ4QKfToampiU/oktcx2fZZt24dVq1ahf/+9794/PHHjV8PCAjA\nnDlzzH+4Vovc3FykpqaisbERM2bMQExMDNatWwcAmDVrFqKjozFmzBgkJCRAo9EgKyuL4U8WsWYT\ntoCAACxYsIDtRPJaipZ6rlmzBnPnznVEPW3iUk9qy6hRo3DgwAGL3sNjFMmb2LzUs3v37rh9+zYA\nYOnSpRg/fjyOHz9uvwqJLCBJErRarcXBHxgYiN27dzP4iaAw/JcuXYqAgAAcPHgQ+/btw8svv4zf\n/e53atdG1IIkSejRowfS0tIsuqHr5+cHvV6P6upq3tAl+oWi8Pfx8QEAFBQUICsrC2lpaaivr1e1\nMKJmoihCq9UiLS0NV65cUfw+jUaDgoIC1NTUcLZP9BBF6/xDQkLwyiuv4Msvv8Sbb76J2tpaNDU1\nqV0beTlJkjB16lRUV1db9f4pU6Zwpk/UDkUz/+3btyM1NRVffPEFgoKCUFVVhT//+c9q10ZeTBRF\nPPvss1YFv0aj4QErRGYoWu1TXl4OWZZbPbHbq1cv1Qp7EFf7eA9bZ/sjR47E/v377VwVkXsylZ2K\nwj8uLs4Y/LW1tbhw4QKioqJw+vRp+1baDoa/d5AkCRkZGbhz545F7xMEARkZGZzpEz3EVHYq6vmf\nOnWqxevjx4/j/ffft70yol+IooilS5dadC/Jx8cHf/zjH3kzl8gKVu/nHxcX1+qXglo48/dc1jyd\nC7C9Q6SEzTP/9957z/jPTU1NOH78OEJCQuxTHXktSZIwc+ZMi4I/PDwcq1ev5ioeIhspCv/bt28b\ne/7N660nTJigamHk2SRJgk6nMz45roRer2eLh8hOVD3G0V7Y9vEslu7J06FDByxatIjBT2Qhq9s+\n8+fPx6pVqzBu3Lg2P7Rr166YNWsWhg4dap9KyaNJkoTJkyejpqZG0XgerkKkHpMz/6NHj2LgwIHt\nHgt2/fp1/PGPf8SZM2fUqg8AZ/6eQJIkTJs2DVVVVYrG89B0ItvZvM7flF27drU6oMXeGP7uLzk5\nGSdOnDA7LiAgAHl5eQx9IjuwOvzj4+NNfmhJSYnt1SnA8Hdfoihi+fLlqKurMzuWs30i+7K65797\n924AwNq1awEA06ZNgyzL2Lx5s51LJE8UFxen+CnwwMBABj+RAylq+yQlJeG7775r8bUBAwYo+mu8\nPXDm716a1+8r3X65f//+DntgkMib2HySlyzLOHjwoPH1N998wzCmNkmShPnz5ysKfh8fH+j1egY/\nkRMoesjro48+wvTp03Hz5k0AQFBQEDZu3KhqYeReJEnC888/b/xvxByNRoP8/Hy2eYicxKLVPtXV\n1RAEAYGBgWrW1ArbPq4tMzMTeXl5Fr1Hp9NxF04ildm8tw9w/wjH0tJS1NbWGr+2ZMkS26sjtyaK\nokXBr9Fo8Pbbb/NpXSInUxT+s2bNwt27d/HVV18hKysLO3bswJAhQ9SujVycKIrIzs5WNNbHxweT\nJ0/mbJ/IRShq+8THx+P7779HQkICSkpKcOfOHYwZM6bFTWA1se3jWiRJwrx583D+/HlF4wsKCtjb\nJ3ICm9s+fn5+AAB/f39UVlaia9euipfxkWfJzMzEtm3bFB+6MnLkSAY/kQtStNRz3LhxqKqqwhtv\nvIHHH38cffr0gU6nU7s2ciGiKEIQBOTl5SkO/l69evHAFSIXZfHePrW1taitrUVQUJDxa19++SVG\njx5t9+Kase3jXJZuwezn54eFCxfypi6Rk6m6sRug/tO+DH/nsWQZp1arxeLFixn6RC7C5id8yfuI\nogiNRqM4+AMCAhj8RG5E8Tp/8h6WLOEE+MAWkTti24dakCQJ6enpFt3UvXTpkspVEZE1bF7qeffu\nXaxduxYHDx6EIAgYMWIEZs+ejY4dOwIA+vbta79qyWmaN2WzZBknV/MQuSdFPf8XXngBpaWlmDdv\nHubMmYPTp09j2rRpxu9/9tln7b63sLAQ0dHRiIyMxIoVK9odd+TIEWi1WpOfReqaN28ezp07Z3ac\nTqeDLMsMfiI3pmjmf/r0aZSWlhpfP/nkk4iNjTX7vsbGRsyZMwd79+5FSEgIBg0ahPT0dMTExLQa\n94c//AFjxoxhe8dJRFHEhQsXzI7r378/+/tEHkBR+CcnJ+Pbb7/FsGHDAABFRUV4/PHHzb6vuLgY\n/fr1Q58+fQAAGRkZyM/PbxX+a9aswcSJE3HkyJF2P+vBVSQpKSlISUlRUjoplJuba/YXLw9dIXJt\nBoMBBoNB0VhF4X/06FEMHz4cYWFhEAQB5eXliIqKQnx8vMmzfCsrKxEWFmZ8HRoaisOHD7cak5+f\nj6+++gpHjhyBIAhtfhaXENqfJElYvXo17t27Z3If/uTkZOTk5HCbBiIX9/DE2NSqPUXhX1hYaFUh\n7QX5g1577TW8++67xrvSbPs4htKHtwICAnDs2DEHVEREjqQo/JvbNpYKCQlBRUWF8XVFRQVCQ0Nb\njDl27BgyMjIAANeuXcPnn38OX19fpKenW/UzyTylB6trtVosWLDAARURkaPZZZ1/exoaGhAVFYV9\n+/ahZ8+eGDx4MPLy8lr1/JtNnz4d48aNw/jx41sWyXX+dqH0qMXAwED4+vri1VdfZbuNyI3Z5SQv\na2i1WuTm5iI1NRWNjY2YMWMGYmJisG7dOgD3D4khxxBFEcuWLUNDQ4PJcT4+PqiurnZQVUTkLKrO\n/O2FM3/biKKInJwcRf8bBgQE4NatWw6oiojUxo3dvJgoili6dKniX57s8RN5B27s5sEs2Y5ZEAQs\nWbKEPX4iL8Hw91CiKCoOfu7RQ+R92PbxQJIkYdmyZWbH+fj4QK/XM/iJvBDD38OIoohJkyaZXdUT\nHh6O/Px8tnmIvBTbPh7Ckv5+eHi4ot07ichzMfw9gCUHrD/66KNYvXq1yhURkatj28fNiaKoKPgF\nQUBERAS2bt3KDdqIiA95uTNJkvDcc8+Z7e/7+flhx44dDH0iL8OHvDyQJEl48cUXzQY/ACxcuJDB\nT0QtcObvhkRRxPLly1FXV2d2rE6n48lbRF7KaRu7kf1Zsk8Pg5+I2sO2jxuxZJ8eBj8RmcKZv5tQ\nupwzICAACxYs4MNbRGQSw98N9O7dG+Xl5WbHJScn88hFIlKEbR8XN2rUKEXBHxQUhJycHAdURESe\ngOHvwpQ+wBUeHo5NmzZxOScRKca2jwuSJAlTp05VdJyiXq9nf5+ILMZ1/i5GkiRkZGTgzp07ZscG\nBgbyvF0iahef8HUjq1evVhT8giBg8+bNDqiIiDwRw9+FSJKEQ4cOmR0XGBiI3bt3s8dPRFZj+LuI\nzMxMpKWlmZ316/V6VFdXM/iJyCbs+bsAURSRnZ1tdhzP2iUiS5jKToa/C/Dz80Ntba3JMdyugYgs\nxRu+LkqSJERERJgN/tTUVAY/EdkVw99JJEnC/Pnzcf78eZPjgoODMXfuXAdVRUTegg95OUHzQSzX\nr183OS44OBjr16/nzV0isjv2/B1MFEWsXLkSd+/eNTnOz88PNTU1DqqKiDwRD3NxEc378Tc1NZkc\np9VqsXDhQgdVRUTeiDN/B5EkCePHjzd59KJGo0GnTp24Hz8R2QWXerqAiIgIkzd3u3Xrho8//pj9\nfSKyG6cu9SwsLER0dDQiIyOxYsWKVt/fvHkzEhMTkZCQgOHDh6OkpETtkhxOFEWTwe/n58fgJyKH\nUrXn39jYiDlz5mDv3r0ICQnBoEGDkJ6ejpiYGOOY8PBwHDhwAIGBgSgsLMQrr7yCoqIiNctyKCXH\nLy5cuJDBT0QOperMv7i4GP369UOfPn3g6+uLjIwM5OfntxgzbNgwBAYGAgCGDBmCy5cvq1mSw0iS\nhM6dO5sN/vDwcPb3icjhVJ35V1ZWIiwszPg6NDQUhw8fbnf8hg0bMHbs2Da/92BApqSkICUlxV5l\n2p0kSZg2bZrZvfY7dOiA1atXO6gqIvJ0BoMBBoNB0VhVw18QBMVjv/76a3z00Uf45ptv2vy+O82O\n582bh6qqKpNjNBoNFi1axHYPEdnNwxNjUxtGqhr+ISEhqKioML6uqKhAaGhoq3ElJSXIyspCYWEh\nOnfurGZJqjN3c7fZ22+/7Va/0IjIs6i61LOhoQFRUVHYt28fevbsicGDByMvL6/FDd/y8nI8+eST\n2LRpE4YOHdp2kW6y1DMzMxN5eXlmx3FrZiJyBKc94avVapGbm4vU1FQ0NjZixowZiImJwbp16wAA\ns2bNQk5ODqqqqjB79mwAgK+vL4qLi9UsSxUMfiJyJ3zIyw4kSUJ6errJbRsEQcCSJUvY6iEih+He\nPiqSJAk6nc5k8Gs0Gvb4icilcD9/GzSfu3v79m2T4xj8RORqGP5WEkVRUY9fp9Mx+InI5bDnbyV/\nf3+ze/Lz3F0iciae4WtnkiSZDf7k5GQGPxG5LIa/hTIzM5Genm52XE5OjgOqISKyDts+FujduzfK\ny8vNjuNafiJyBTzMxQ6UBn///v1x6tQpB1RERGQae/42iouLMxv8vr6+0Ov1DH4icgt8yMuMzMxM\nnD592uQYjUaDf/zjH9yhk4jcBmf+Jihdyz9lyhQGPxG5Ffb82yGKosm9sJv16tULly5dckBFRESW\n4Q1fKzzyyCOoq6szOYbBT0SujDd8LTRq1Cizwd+/f38GPxG5LYb/Q0RRNHvoeocOHbiqh4jcGlf7\nPEDpgSyLFi1yQDVEROphz/8XcXFxZpd0AtysjYjcB3v+ZihZyw8w+InIc3h9+Ctdyz9y5EgGPxF5\nDK9u+0iShPHjx5td2RMeHo5z587Z/ecTEamJbZ92rF692mzw+/v7Y/Xq1Q6qiIjIMbw2/CVJwpEj\nR0yOEQQB27dv59YNRORxvDL8JUnC/PnzUVVV1e4YjUaD3bt3M/iJyCN53Tp/SZKQkZGBO3futDtG\nq9Vi8eLFDH4i8lheFf6SJGHy5Mmoqalp8/tarRYJCQnIyclh8BORR/Oq1T4RERE4f/58u99PTU1F\nYWGhzT+HiMgVcLXPL0wFf8eOHTF37lwHVkNE5DxeE/6ZmZkmvx8bG8tWDxF5Da/o+Y8aNcrkTp2C\nICAnJ8eBFREROZfHh/+vfvUr/PTTTybHZGRkcNZPRF7Fo2/49u7dG+Xl5SbHaDQaNDY2WlsaEZHL\nMpWdHjvz9/f3x927d82OmzJligOqISJyLarf8C0sLER0dDQiIyOxYsWKNsfMmzcPkZGRSExMxIkT\nJ2z+mb1791YU/NyimYi8larh39jYiDlz5qCwsBClpaXIy8vDmTNnWozZs2cPzp49i7KyMnzwwQeY\nPXu2zT/XXKsH4BbNROTdVA3/4uJi9OvXD3369IGvry8yMjKQn5/fYsyuXbvw4osvAgCGDBmC6upq\nXL16Vc2y8Nhjj2H//v2q/gwiIlemas+/srISYWFhxtehoaE4fPiw2TGXL19G9+7dW4wTRdH4zykp\nKUhJSbG6rv/9739Wv5eIyFUZDAYYDAZFY1UNf0EQFI17+G50W+97MPxtodfr7fI5RESu5uGJcXZ2\ndrtjVW37hISEoKKiwvi6oqICoaGhJsdcvnwZISEhqtVkr18iRETuTNXwHzhwIMrKynDx4kXU1dVh\n27ZtSE9PbzEmPT0dn3zyCQCgqKgIQUFBrVo+lmpvXasbPNJAROQQqrZ9tFotcnNzkZqaisbGRsyY\nMQMxMTFYt24dAGDWrFkYO3Ys9uzZg379+qFTp07YuHGjXX42g56IqH0e/YQvEZE38+otnZXe+fYk\nvGbvwGv2fGpeL8PfA/GavQOv2fMx/ImIyK4Y/kREXshtbvgSEZHl3HpLZzf4/URE5FbY9iEi8kIM\nfyIiL8TwJyLyQh4T/s44MczZzF3z5s2bkZiYiISEBAwfPhwlJSVOqNK+lPx7BoAjR45Aq9Xis88+\nc2B16lByzQaDAQMGDEBcXJxN2527CnPXfO3aNYwZMwZJSUmIi4vDxx9/7Pgi7ejll19G9+7dER8f\n3+4Yu+eX7AEaGhrkiIgI+cKFC3JdXZ2cmJgol5aWthgjSZL829/+VpZlWS4qKpKHDBnijFLtRsk1\nHzp0SK6urpZlWZY///xzr7jm5nFPPPGE/Mwzz8g7d+50QqX2o+Saq6qq5NjYWLmiokKWZVn+6aef\nnFGq3Si5Zr1eL7/55puyLN+/3i5dusj19fXOKNcuDhw4IB8/flyOi4tr8/tq5JdHzPxd9cQwNSm5\n5mHDhiEwMBDA/Wu+fPmyM0q1GyXXDABr1qzBxIkT8dhjjzmhSvtScs1btmzBhAkTjNuld+vWzRml\n2o2Sa+7Rowdu3boFALh16xa6du0KrdYtFi+2acSIEejcuXO731cjvzwi/Ns6DayystLsGHcOQyXX\n/KANGzZg7NixjihNNUr/Pefn5xvPgnb3Z0SUXHNZWRlu3LiBJ554AgMHDsSnn37q6DLtSsk1Z2Vl\n4fTp0+h9yni1AAAFJklEQVTZsycSExOxatUqR5fpUGrkl/v+qnyAPU8McxeW1P7111/jo48+wjff\nfKNiRepTcs2vvfYa3n33XeNuhg//O3c3Sq65vr4ex48fx759+1BTU4Nhw4Zh6NChiIyMdECF9qfk\nmt955x0kJSXBYDDg3LlzGD16NE6ePImAgAAHVOgc9s4vjwh/VzwxTG1KrhkASkpKkJWVhcLCQpN/\nrXQHSq752LFjyMjIAHD/puDnn38OX1/fVocIuQsl1xwWFoZu3brBz88Pfn5+GDlyJE6ePOm24a/k\nmg8dOoTFixcDACIiItC3b1/88MMPGDhwoENrdRRV8svmuwYuoL6+Xg4PD5cvXLgg37t3z+wN32+/\n/dbtb34queZLly7JERER8rfffuukKu1LyTU/6KWXXpL//ve/O7BC+1NyzWfOnJGfeuopuaGhQf75\n55/luLg4+fTp006q2HZKrvn111+XRVGUZVmWr1y5IoeEhMjXr193Rrl2c+HCBUU3fO2VXx4x83fm\niWHOouSac3JyUFVVZex/+/r6ori42Jll20TJNXsaJdccHR2NMWPGICEhARqNBllZWYiNjXVy5dZT\ncs1vvfUWpk+fjsTERDQ1NWHlypXo0qWLkyu3nk6nw/79+3Ht2jWEhYUhOzsb9fX1ANTLL7fY2I2I\niOzLI1b7EBGRZRj+REReiOFPROSFGP5ERF6I4U/0i+zsbLz11lstvvbdd9+ZXDkjiiLee+89tUsj\nsjuGP9EvMjMzsW3bthZf27p1KzIzM9t9jzs/JU7ejeFPXunixYuIjo7G1KlTERsbi0mTJiE0NBSd\nO3du8SzEjh07oNPp8OGHH2Lw4MFISkrCxIkTcffuXeOY5l8AKSkpOHbsGID7Txf37dsXANDY2Ig3\n3ngDgwcPRmJiIj744AMAwI8//oiRI0diwIABiI+Px8GDBx11+UQMf/Je//nPf/Dqq6+itLQU//d/\n/4e1a9dCp9Nh69atAICioiJ06dIFERERmDBhAoqLi/Hdd98hJiYGGzZsaPV5giC0+TeBDRs2ICgo\nCMXFxSguLsaHH36IixcvIi8vD2PGjMGJEydQUlKCpKQk1a+ZqBnDn7xWWFgYhg0bBgCYOnUqDh48\niClTpmDnzp2QZblFy+f777/HiBEjkJCQgM2bN6O0tFTxz/niiy/wySefYMCAARg6dChu3LiBs2fP\nYtCgQdi4cSOys7NRUlKCRx99VJXrJGqLR2zvQGSNB2fpsixDEASEhoaib9++MBgM+Oyzz1BUVAQA\neOmll7Br1y7Ex8fjb3/7GwwGQ6vP02q1aGpqAgDU1ta2+F5ubi5Gjx7d6j3//ve/UVBQgJdeegkL\nFizAtGnT7HiFRO3jzJ+8Vnl5uTHct2zZghEjRgC4v8/K66+/joiICPTs2RMAcOfOHQQHB6O+vh6b\nNm0y/uKQH9g2uk+fPjh69CgAYOfOncafk5qairVr16KhoQHA/XZTTU0NysvL8dhjj2HmzJmYOXOm\nRxwtSu6D4U9eKyoqCu+//z5iY2Nx8+ZN4wZ4EydORGlpKXQ6nXHs0qVLMWTIEPz6179GTEyM8esP\n9vl///vf469//SuSk5Nx/fp149dnzpyJ2NhYJCcnIz4+HrNnz0ZDQwMMBgOSkpKQnJyM7du3Y/78\n+Q68evJ23NiNvNLFixcxbtw4fP/9984uhcgpOPMnr8U1+uTNOPMnIvJCnPkTEXkhhj8RkRdi+BMR\neSGGPxGRF2L4ExF5IYY/EZEX+n+XKHfV6au9XQAAAABJRU5ErkJggg==\n"
360 | }
361 | ],
362 | "prompt_number": 26
363 | },
364 | {
365 | "cell_type": "code",
366 | "collapsed": false,
367 | "input": [],
368 | "language": "python",
369 | "metadata": {},
370 | "outputs": []
371 | }
372 | ],
373 | "metadata": {}
374 | }
375 | ]
376 | }
--------------------------------------------------------------------------------
| col_0 | \n", 308 | "not zero | \n", 309 | "zero | \n", 310 | "
|---|---|---|
| row_0 | \n", 313 | "\n", 314 | " | \n", 315 | " |
| False | \n", 320 | "0 | \n", 321 | "483 | \n", 322 | "
| True | \n", 325 | "500 | \n", 326 | "17 | \n", 327 | "