├── .github
├── FUNDING.yml
├── ISSUE_TEMPLATE
│ └── chapter-xxx.md
└── PULL_REQUEST_TEMPLATE
│ └── pull_request_template.md
├── .gitignore
├── LICENSE
├── README.md
├── Simulated-Data.ipynb
└── causal-inference-for-the-brave-and-true
├── 01-Introduction-To-Causality.ipynb
├── 02-Randomised-Experiments.ipynb
├── 03-Stats-Review-The-Most-Dangerous-Equation.ipynb
├── 04-Graphical-Causal-Models.ipynb
├── 05-The-Unreasonable-Effectiveness-of-Linear-Regression.ipynb
├── 06-Grouped-and-Dummy-Regression.ipynb
├── 07-Beyond-Confounders.ipynb
├── 08-Instrumental-Variables.ipynb
├── 09-Non-Compliance-and-LATE.ipynb
├── 10-Matching.ipynb
├── 11-Propensity-Score.ipynb
├── 12-Doubly-Robust-Estimation.ipynb
├── 13-Difference-in-Differences.ipynb
├── 14-Panel-Data-and-Fixed-Effects.ipynb
├── 15-Synthetic-Control.ipynb
├── 16-Regression-Discontinuity-Design.ipynb
├── 17-Predictive-Models-101.ipynb
├── 18-Heterogeneous-Treatment-Effects-and-Personalization.ipynb
├── 19-Evaluating-Causal-Models.ipynb
├── 20-Plug-and-Play-Estimators.ipynb
├── 21-Meta-Learners.ipynb
├── 22-Debiased-Orthogonal-Machine-Learning.ipynb
├── 23-Challenges-with-Effect-Heterogeneity-and-Nonlinearity.ipynb
├── 24-The-Diff-in-Diff-Saga.ipynb
├── 25-Synthetic-Diff-in-Diff.ipynb
├── Conformal-Inference-for-Synthetic-Control.ipynb
├── Debiasing-with-Orthogonalization.ipynb
├── Debiasing-with-Propensity-Score.ipynb
├── Prediction-Metrics-For-Causal-Models.ipynb
├── When-Prediction-Fails.ipynb
├── _config.yml
├── _toc.yml
├── data
├── ak91.csv
├── app_engagement_push.csv
├── billboard_impact.csv
├── collections_email.csv
├── customer_features.csv
├── customer_transactions.csv
├── drinking.csv
├── enem_scores.csv
├── hospital_treatment.csv
├── ice_cream_sales.csv
├── ice_cream_sales_rnd.csv
├── img
│ ├── beyond-conf
│ │ ├── cop-ex1.png
│ │ ├── cop-ex2.png
│ │ ├── cop.png
│ │ ├── selection.png
│ │ ├── shocked.png
│ │ └── y-pred.png
│ ├── brave-and-true.png
│ ├── causal-graph
│ │ ├── both_crap.png
│ │ ├── selection_bias.png
│ │ └── three_bias.png
│ ├── causal-model
│ │ ├── customers.png
│ │ ├── economists.png
│ │ ├── elast-partition.png
│ │ ├── elast-split.png
│ │ ├── elasticity.png
│ │ ├── linear-fix.png
│ │ ├── y-partition.png
│ │ └── y-split.png
│ ├── curse-of-dimensionality.jpg
│ ├── debias-ps
│ │ ├── again.png
│ │ ├── fear-no-man.png
│ │ └── ps-experiment.png
│ ├── debiased-ml
│ │ ├── avengers.png
│ │ ├── cf-pred.png
│ │ ├── cross-prediction.png
│ │ ├── diagram.png
│ │ ├── drown-ols.png
│ │ ├── linear-aprox.png
│ │ ├── ml-problem.png
│ │ ├── more-metrics.png
│ │ ├── non-linear.png
│ │ └── non-sci.png
│ ├── did-saga
│ │ ├── awful.jpeg
│ │ ├── death.png
│ │ ├── promise.png
│ │ └── twfeworking.png
│ ├── diff-in-diff
│ │ ├── cant-read.png
│ │ ├── non-parallel.png
│ │ └── secrets.png
│ ├── doubly-robust
│ │ └── double.png
│ ├── dummy
│ │ ├── heterosk.png
│ │ └── you_little_shit.png
│ ├── evaluate-causal-models
│ │ ├── ate_bins.png
│ │ ├── cumm_elast.png
│ │ ├── cumm_gain.png
│ │ ├── sneak.png
│ │ └── uncivilised.png
│ ├── fixed-effects
│ │ ├── control-unit.png
│ │ ├── demeaned.png
│ │ ├── fe-graph.png
│ │ └── time-travel.png
│ ├── graph-flow.png
│ ├── hte-binary-outcome
│ │ ├── logistic.png
│ │ └── slide-susan-athey.png
│ ├── industry-ml
│ │ ├── cascade-1.png
│ │ ├── cascade-2.png
│ │ ├── cashflow-1.png
│ │ ├── cross-validation.png
│ │ ├── ml-flow.png
│ │ ├── ml-meme.png
│ │ ├── model-fit.png
│ │ ├── overfitting.jpg
│ │ ├── profit.png
│ │ ├── test.png
│ │ └── translation.png
│ ├── intro
│ │ ├── anatomy1.png
│ │ ├── anatomy2.png
│ │ ├── anatomy3.png
│ │ ├── anatomy4.png
│ │ ├── potential_outcomes.png
│ │ └── tougher-up-cupcake1.jpg
│ ├── iv
│ │ ├── good-iv.png
│ │ ├── incomplete-files.png
│ │ ├── qob.png
│ │ └── weak-iv.png
│ ├── late
│ │ ├── always_takers.png
│ │ ├── defiers.png
│ │ └── double_index.png
│ ├── linear-regression
│ │ ├── confused_cat.png
│ │ └── you_vs.png
│ ├── matching
│ │ ├── explain.png
│ │ ├── its-a-match.png
│ │ └── ubiquitous-ols.png
│ ├── meta-learners
│ │ ├── learned-new-move.png
│ │ ├── not-great-not-terrible.jpeg
│ │ ├── s-learner.png
│ │ ├── second-stage-x.png
│ │ ├── t-learner-problem.png
│ │ ├── t-learner.png
│ │ ├── t-vs-x-learner.png
│ │ ├── x-learner.png
│ │ └── zero-bias-s-learner.png
│ ├── orthogonal
│ │ ├── athlas.png
│ │ ├── kfold-cv.png
│ │ └── nazare-confusa.jpg
│ ├── plug-and-play-estimators
│ │ ├── genious.jpeg
│ │ ├── infeasible.png
│ │ ├── linear-case.png
│ │ ├── non-linear-case.png
│ │ └── second-estimator.png
│ ├── poetry.png
│ ├── ps
│ │ ├── balance.png
│ │ ├── bootstrap.png
│ │ ├── iptw.png
│ │ └── ml-trap.png
│ ├── rct
│ │ ├── indep.png
│ │ ├── lurking_bias.png
│ │ └── science_dream.png
│ ├── rdd
│ │ ├── ols.png
│ │ ├── rdd.png
│ │ ├── rdd_fuzzy.png
│ │ └── smooth.png
│ ├── sc-conformal-inf
│ │ ├── aug-data.png
│ │ └── block-perm.png
│ ├── sdid
│ │ └── both-pills.png
│ ├── stats-review
│ │ ├── ci_xkcd.png
│ │ ├── normal_dist.jpeg
│ │ └── p_value.png
│ ├── synth-control
│ │ ├── allways.png
│ │ ├── extrapolation.png
│ │ ├── out-of-sample.png
│ │ ├── regr_space.png
│ │ ├── regr_space_x.png
│ │ ├── regr_time.png
│ │ └── synth_img.png
│ └── when-prediction-fails
│ │ ├── flat-curves.png
│ │ ├── graph-1.png
│ │ ├── graph-2.png
│ │ ├── graph-3.png
│ │ ├── horse-meme.png
│ │ ├── ml-in-5.png
│ │ ├── model-opt.png
│ │ ├── opt-deriv.png
│ │ ├── pandas-magic.png
│ │ ├── partitions.png
│ │ ├── personalise.png
│ │ ├── slice-1.png
│ │ ├── slice-2.png
│ │ └── waiting-time.png
├── invest_email.csv
├── invest_email_biased.csv
├── invest_email_rnd.csv
├── learning_mindset.csv
├── medicine_impact_recovery.csv
├── online_classroom.csv
├── sheepskin.csv
├── smoking.csv
├── trainees.csv
├── wage.csv
└── when-prediction-fails
│ ├── graph-1.png
│ ├── graph-2.png
│ ├── graph-3.png
│ ├── horse-meme.png
│ ├── ml-in-5.png
│ ├── model-opt.png
│ ├── opt-deriv.png
│ ├── pandas-magic.png
│ ├── partitions.png
│ ├── personalise.png
│ ├── slice-1.png
│ ├── slice-2.png
│ └── waiting-time.png
├── landing-page.md
├── nb18.py
├── nb21.py
└── sc.py
/.github/FUNDING.yml:
--------------------------------------------------------------------------------
1 | # These are supported funding model platforms
2 |
3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
4 | patreon: causal_inference_for_the_brave_and_true
5 | open_collective: # Replace with a single Open Collective username
6 | ko_fi: # Replace with a single Ko-fi username
7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
9 | liberapay: # Replace with a single Liberapay username
10 | issuehunt: # Replace with a single IssueHunt username
11 | otechie: # Replace with a single Otechie username
12 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
13 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/chapter-xxx.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Chapter XXX
3 | about: Propose a correction
4 | title: ''
5 | labels: ''
6 | assignees: matheusfacure
7 |
8 | ---
9 | There is an issue on chapter XXX, in the following paragraph
10 | [paragraph here]
11 |
12 | It should be
13 | [suggestion here]
14 |
--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md:
--------------------------------------------------------------------------------
1 | ## Status
2 | WIP/Review Required/Ready for Merge
3 |
4 |
5 | ## Description
6 |
7 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # =============================================================================
2 | # Python
3 | # https://github.com/github/gitignore/blob/master/Python.gitignore
4 | # =============================================================================
5 |
6 | # Byte-compiled / optimized / DLL files
7 | __pycache__/
8 | *.py[cod]
9 | *$py.class
10 |
11 | # C extensions
12 | *.so
13 |
14 | # Distribution / packaging
15 | .Python
16 | build/
17 | develop-eggs/
18 | dist/
19 | downloads/
20 | eggs/
21 | .eggs/
22 | lib/
23 | lib64/
24 | parts/
25 | sdist/
26 | var/
27 | wheels/
28 | pip-wheel-metadata/
29 | share/python-wheels/
30 | *.egg-info/
31 | .installed.cfg
32 | *.egg
33 | MANIFEST
34 |
35 | # PyInstaller
36 | # Usually these files are written by a python script from a template
37 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
38 | *.manifest
39 | *.spec
40 |
41 | # Installer logs
42 | pip-log.txt
43 | pip-delete-this-directory.txt
44 |
45 | # Unit test / coverage reports
46 | htmlcov/
47 | .tox/
48 | .nox/
49 | .coverage
50 | .coverage.*
51 | .cache
52 | nosetests.xml
53 | coverage.xml
54 | *.cover
55 | *.py,cover
56 | .hypothesis/
57 | .pytest_cache/
58 |
59 | # Translations
60 | *.mo
61 | *.pot
62 |
63 | # Django stuff:
64 | *.log
65 | local_settings.py
66 | db.sqlite3
67 | db.sqlite3-journal
68 |
69 | # Flask stuff:
70 | instance/
71 | .webassets-cache
72 |
73 | # Scrapy stuff:
74 | .scrapy
75 |
76 | # Sphinx documentation
77 | docs/_build/
78 |
79 | # PyBuilder
80 | target/
81 |
82 | # Jupyter Notebook
83 | .ipynb_checkpoints
84 |
85 | # IPython
86 | profile_default/
87 | ipython_config.py
88 |
89 | # pyenv
90 | .python-version
91 |
92 | # pipenv
93 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
94 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
95 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
96 | # install all needed dependencies.
97 | #Pipfile.lock
98 |
99 | # celery beat schedule file
100 | celerybeat-schedule
101 |
102 | # SageMath parsed files
103 | *.sage.py
104 |
105 | # Environments
106 | .env
107 | .venv
108 | env/
109 | venv/
110 | ENV/
111 | env.bak/
112 | venv.bak/
113 |
114 | # Spyder project settings
115 | .spyderproject
116 | .spyproject
117 |
118 | # Rope project settings
119 | .ropeproject
120 |
121 | # mkdocs documentation
122 | /site
123 |
124 | # mypy
125 | .mypy_cache/
126 | .dmypy.json
127 | dmypy.json
128 |
129 | # Pyre type checker
130 | .pyre/
131 |
132 |
133 | # =============================================================================
134 | # JetBrains IDEs
135 | # https://github.com/github/gitignore/blob/master/Global/JetBrains.gitignore
136 | # =============================================================================
137 |
138 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
139 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
140 |
141 | # User-specific stuff
142 | .idea/**/workspace.xml
143 | .idea/**/tasks.xml
144 | .idea/**/usage.statistics.xml
145 | .idea/**/dictionaries
146 | .idea/**/shelf
147 |
148 | # Generated files
149 | .idea/**/contentModel.xml
150 |
151 | # Sensitive or high-churn files
152 | .idea/**/dataSources/
153 | .idea/**/dataSources.ids
154 | .idea/**/dataSources.local.xml
155 | .idea/**/sqlDataSources.xml
156 | .idea/**/dynamic.xml
157 | .idea/**/uiDesigner.xml
158 | .idea/**/dbnavigator.xml
159 |
160 | # Gradle
161 | .idea/**/gradle.xml
162 | .idea/**/libraries
163 |
164 | # Gradle and Maven with auto-import
165 | # When using Gradle or Maven with auto-import, you should exclude module files,
166 | # since they will be recreated, and may cause churn. Uncomment if using
167 | # auto-import.
168 | # .idea/modules.xml
169 | # .idea/*.iml
170 | # .idea/modules
171 | # *.iml
172 | # *.ipr
173 |
174 | # CMake
175 | cmake-build-*/
176 |
177 | # Mongo Explorer plugin
178 | .idea/**/mongoSettings.xml
179 |
180 | # File-based project format
181 | *.iws
182 |
183 | # IntelliJ
184 | out/
185 |
186 | # VS Code
187 | .vscode/
188 |
189 | # mpeltonen/sbt-idea plugin
190 | .idea_modules/
191 |
192 | # JIRA plugin
193 | atlassian-ide-plugin.xml
194 |
195 | # Cursive Clojure plugin
196 | .idea/replstate.xml
197 |
198 | # Crashlytics plugin (for Android Studio and IntelliJ)
199 | com_crashlytics_export_strings.xml
200 | crashlytics.properties
201 | crashlytics-build.properties
202 | fabric.properties
203 |
204 | # Editor-based Rest Client
205 | .idea/httpRequests
206 |
207 | # Android studio 3.1+ serialized cache file
208 | .idea/caches/build_file_checksums.ser
209 |
210 |
211 | # =============================================================================
212 | # macOS
213 | # https://github.com/github/gitignore/blob/master/Global/macOS.gitignore
214 | # =============================================================================
215 |
216 | # General
217 | .DS_Store
218 | .AppleDouble
219 | .LSOverride
220 |
221 | # Icon must end with two \r
222 | Icon
223 |
224 |
225 | # Thumbnails
226 | ._*
227 |
228 | # Files that might appear in the root of a volume
229 | .DocumentRevisions-V100
230 | .fseventsd
231 | .Spotlight-V100
232 | .TemporaryItems
233 | .Trashes
234 | .VolumeIcon.icns
235 | .com.apple.timemachine.donotpresent
236 |
237 | # Directories potentially created on remote AFP share
238 | .AppleDB
239 | .AppleDesktop
240 | Network Trash Folder
241 | Temporary Items
242 | .apdisk
243 |
244 |
245 | # =============================================================================
246 | # Custom
247 | # =============================================================================
248 |
249 | .idea/*
250 | causal-inference-for-the-brave-and-true/_build/*
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 Matheus Facure
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Causal Inference for The Brave and True
2 |
3 | 
4 |
5 | [](https://zenodo.org/badge/latestdoi/255903310)
6 |
7 | A light-hearted yet rigorous approach to learning impact estimation and sensitivity analysis. All in Python and with as many memes as I could find.
8 |
9 | [Check out the book here!](https://matheusfacure.github.io/python-causality-handbook/landing-page.html)
10 |
11 | If you want to read the book in Brazilian Portuguese, @rdemarqui made this awesome translation:
12 | [Inferência Causal para os Corajosos e Verdadeiros](https://github.com/rdemarqui/python-causality-handbook-ptbr)
13 |
14 | If you want to read the book in French, Arthur Mello put a lot of effort into this beautiful translation:
15 | [L'Inférence Causale pour les Courageux et les Vrais](https://github.com/arthurmello/python-causality-handbook)
16 |
17 | If you want to read the book in Chinese, @xieliaing was very kind to make a translation:
18 | [因果推断：从概念到实践](https://github.com/xieliaing/CausalInferenceIntro)
19 |
20 | If you want to read the book in Spanish, @donelianc was very kind to make a translation:
21 | [Inferencia Causal para los Valientes y Verdaderos](https://github.com/donelianc/introduccion-inferencia-causal)
22 |
23 | If you want to read it in Korean, @jsshin2019 has put up a team to make the that translation possible:
24 | [Python으로 하는 인과추론 : 개념부터 실습까지](https://github.com/TeamCausality/Causal-Inference-with-Python)
25 |
26 | Also, some really kind folks (@vietecon, @dinhtrang24 and @anhpham52) also translated this content into Vietnamese:
27 | [Nhân quả Python](https://github.com/vietecon/NhanQuaPython)
28 |
29 |
30 |
31 |
32 |
33 |
34 | I like to think of this entire series as a tribute to Joshua Angrist, Alberto Abadie and Christopher Walters for their amazing Econometrics class. Most of the ideas here are taken from their classes at the American Economic Association. Watching them is what is keeping me sane during this tough year of 2020.
35 | * [Cross-Section Econometrics](https://www.aeaweb.org/conference/cont-ed/2017-webcasts)
36 | * [Mastering Mostly Harmless Econometrics](https://www.aeaweb.org/conference/cont-ed/2020-webcasts)
37 |
38 | I'd also like to reference the amazing books from Angrist. They have shown me that Econometrics, or 'Metrics as they call it, is not only extremely useful but also profoundly fun.
39 |
40 | * [Mostly Harmless Econometrics](https://www.mostlyharmlesseconometrics.com/)
41 | * [Mastering 'Metrics](https://www.masteringmetrics.com/)
42 |
43 | Finally, I'd like to reference Miguel Hernan and Jamie Robins' book. It has been my trustworthy companion in the most thorny causal inference questions I've had to answer.
44 |
45 | * [Causal Inference Book](https://www.hsph.harvard.edu/miguel-hernan/causal-inference-book/)
46 |
47 | ## How to Support This Work
48 |
49 | Causal Inference for the Brave and True is an open-source resource primarily focused on econometrics and the statistics of science. It exclusively utilizes free software, grounded in Python. The primary objective is to ensure accessibility, not only from a financial standpoint but also from an intellectual perspective. I've tried my best to keep the content entertaining while maintaining the necessary scientific rigor.
50 |
51 |
52 | If you want to show your appreciation for this work, consider going to https://www.patreon.com/causal_inference_for_the_brave_and_true. Alternatively, you can purchase my book, [Causal Inference in Python](https://www.amazon.com/Causal-Inference-Python-Applying-Industry/dp/1098140257), which provides more insights into applying causal inference in the industry.
53 |
54 |
55 |
--------------------------------------------------------------------------------
/Simulated-Data.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "## collections_email"
8 | ]
9 | },
10 | {
11 | "cell_type": "code",
12 | "execution_count": 1,
13 | "metadata": {},
14 | "outputs": [],
15 | "source": [
16 | "import pandas as pd\n",
17 | "import numpy as np\n",
18 | "\n",
19 | "np.random.seed(24)\n",
20 | "n = 5000\n",
21 | "email = np.random.binomial(1, 0.5, n)\n",
22 | "\n",
23 | "credit_limit = np.random.gamma(6, 200, n)\n",
24 | "risk_score = np.random.beta(credit_limit, credit_limit.mean(), n)\n",
25 | "\n",
26 | "opened = np.random.normal(5 + 0.001*credit_limit - 4*risk_score, 2)\n",
27 | "opened = (opened > 4).astype(float) * email\n",
28 | "\n",
29 | "\n",
30 | "agreement = np.random.normal(30 +(-0.003*credit_limit - 10*risk_score), 7) * 2 * opened\n",
31 | "agreement = (agreement > 40).astype(float)\n",
32 | "\n",
33 | "payments = (np.random.normal(500 + 0.16*credit_limit - 40*risk_score + 11*agreement + email, 75).astype(int) // 10) * 10\n",
34 | "\n",
35 | "data = pd.DataFrame(dict(payments=payments,\n",
36 | " email=email,\n",
37 | " opened=opened,\n",
38 | " agreement=agreement,\n",
39 | " credit_limit=credit_limit,\n",
40 | " risk_score=risk_score))\n",
41 | "\n",
42 | "data.to_csv(\"collections_email.csv\", index=False)"
43 | ]
44 | },
45 | {
46 | "cell_type": "markdown",
47 | "metadata": {},
48 | "source": [
49 | "## hospital_treatment"
50 | ]
51 | },
52 | {
53 | "cell_type": "code",
54 | "execution_count": 2,
55 | "metadata": {},
56 | "outputs": [],
57 | "source": [
58 | "import pandas as pd\n",
59 | "import numpy as np\n",
60 | "\n",
61 | "np.random.seed(24)\n",
62 | "n = 80\n",
63 | "\n",
64 | "hospital = np.random.binomial(1, 0.5, n)\n",
65 | "\n",
66 | "treatment = np.where(hospital.astype(bool),\n",
67 | " np.random.binomial(1, 0.9, n),\n",
68 | " np.random.binomial(1, 0.1, n))\n",
69 | "\n",
70 | "severity = np.where(hospital.astype(bool), \n",
71 | " np.random.normal(20, 5, n),\n",
72 | " np.random.normal(10, 5, n))\n",
73 | "\n",
74 | "days = np.random.normal(15 + -5*treatment + 2*severity, 7).astype(int)\n",
75 | "\n",
76 | "hospital = pd.DataFrame(dict(hospital=hospital,\n",
77 | " treatment=treatment,\n",
78 | " severity=severity,\n",
79 | " days=days))\n",
80 | "\n",
81 | "hospital.to_csv(\"hospital_treatment.csv\", index=False)"
82 | ]
83 | },
84 | {
85 | "cell_type": "markdown",
86 | "metadata": {},
87 | "source": [
88 | "## app engagement push"
89 | ]
90 | },
91 | {
92 | "cell_type": "code",
93 | "execution_count": 3,
94 | "metadata": {},
95 | "outputs": [],
96 | "source": [
97 | "import pandas as pd\n",
98 | "import numpy as np\n",
99 | "\n",
100 | "np.random.seed(24)\n",
101 | "n = 10000\n",
102 | "\n",
103 | "push_assigned = np.random.binomial(1, 0.5, n)\n",
104 | "\n",
105 | "income = np.random.gamma(6, 200, n)\n",
106 | "\n",
107 | "push_delivered = np.random.normal(5 + 0.3+income, 500)\n",
108 | "push_delivered = ((push_delivered > 800) & (push_assigned == 1)).astype(int)\n",
109 | "\n",
110 | "in_app_purchase = (np.random.normal(100 + 20*push_delivered + 0.5*income, 75).astype(int) // 10)\n",
111 | "\n",
112 | "data = pd.DataFrame(dict(in_app_purchase=in_app_purchase,\n",
113 | " push_assigned=push_assigned,\n",
114 | " push_delivered=push_delivered))\n",
115 | "\n",
116 | "data.to_csv(\"app_engagement_push.csv\", index=False)"
117 | ]
118 | },
119 | {
120 | "cell_type": "markdown",
121 | "metadata": {},
122 | "source": [
123 | "## Drug Impact"
124 | ]
125 | },
126 | {
127 | "cell_type": "code",
128 | "execution_count": 17,
129 | "metadata": {},
130 | "outputs": [],
131 | "source": [
132 | "import numpy as np\n",
133 | "import pandas as pd\n",
134 | "\n",
135 | "def make_confounded_data(N):\n",
136 | "\n",
137 | " def get_severity(df):\n",
138 | " return ((np.random.beta(1, 3, size=df.shape[0]) * (df[\"age\"] < 30)) +\n",
139 | " (np.random.beta(3, 1.5, size=df.shape[0]) * (df[\"age\"] >= 30)))\n",
140 | "\n",
141 | " def get_treatment(df):\n",
142 | " return ((.33 * df[\"sex\"] +\n",
143 | " 1.5 * df[\"severity\"] + df[\"severity\"] ** 2 +\n",
144 | " 0.15 * np.random.normal(size=df.shape[0])) > 1.5).astype(int)\n",
145 | "\n",
146 | " def get_recovery(df):\n",
147 | " return ((2 +\n",
148 | " 0.5 * df[\"sex\"] +\n",
149 | " 0.03 * df[\"age\"] + 0.03 * ((df[\"age\"] * 0.1) ** 2) +\n",
150 | " df[\"severity\"] + np.log(df[\"severity\"]) +\n",
151 | " df[\"sex\"] * df[\"severity\"] -\n",
152 | " df[\"medication\"]) * 10).astype(int)\n",
153 | "\n",
154 | " np.random.seed(1111)\n",
155 | " sexes = np.random.randint(0, 2, size=N)\n",
156 | " ages = np.random.gamma(8, scale=4, size=N)\n",
157 | " meds = np.random.beta(1, 1, size=N)\n",
158 | "\n",
159 | " # dados com designação aleatória\n",
160 | " df_rnd = pd.DataFrame(dict(sex=sexes, age=ages, medication=meds))\n",
161 | " df_rnd['severity'] = get_severity(df_rnd)\n",
162 | " df_rnd['recovery'] = get_recovery(df_rnd)\n",
163 | "\n",
164 | " features = ['sex', 'age', 'severity', 'medication', 'recovery']\n",
165 | " df_rnd = df_rnd[features] # to enforce column order\n",
166 | "\n",
167 | " # dados observacionais\n",
168 | " df_obs = df_rnd.copy()\n",
169 | " df_obs['medication'] = get_treatment(df_obs)\n",
170 | " df_obs['recovery'] = get_recovery(df_obs)\n",
171 | "\n",
172 | " # dados contrafactuais data\n",
173 | " df_ctf = df_obs.copy()\n",
174 | " df_ctf['medication'] = ((df_ctf['medication'] == 1) ^ 1).astype(float)\n",
175 | " df_ctf['recovery'] = get_recovery(df_ctf)\n",
176 | "\n",
177 | " return df_rnd, df_obs, df_ctf\n",
178 | "\n",
179 | "np.random.seed(1234)\n",
180 | "df_rnd, df_obs, df_ctf = make_confounded_data(20000)\n",
181 | "\n",
182 | "df_obs.to_csv(\"medicine_impact_recovery.csv\", index=False)"
183 | ]
184 | },
185 | {
186 | "cell_type": "markdown",
187 | "metadata": {},
188 | "source": [
189 | "## Bilboard Mkt"
190 | ]
191 | },
192 | {
193 | "cell_type": "code",
194 | "execution_count": 5,
195 | "metadata": {},
196 | "outputs": [],
197 | "source": [
198 | "import pandas as pd\n",
199 | "import numpy as np\n",
200 | "np.random.seed(123)\n",
201 | "POAMay = np.random.gamma(7,10, 500) * np.random.binomial(1, .7, 500)\n",
202 | "POAJul = np.random.gamma(7,15, 800) * np.random.binomial(1, .8, 800)\n",
203 | "FLMay = np.random.gamma(10,20, 1300) * np.random.binomial(1, .85, 1300)\n",
204 | "FLJul = np.random.gamma(11,21, 2000) * np.random.binomial(1, .9, 2000)\n",
205 | "\n",
206 | "data = pd.concat([\n",
207 | " pd.DataFrame(dict(deposits = POAMay.astype(int), poa=1, jul=0)),\n",
208 | " pd.DataFrame(dict(deposits = POAJul.astype(int), poa=1, jul=1)),\n",
209 | " pd.DataFrame(dict(deposits = FLMay.astype(int), poa=0, jul=0)),\n",
210 | " pd.DataFrame(dict(deposits = FLJul.astype(int), poa=0, jul=1))\n",
211 | "])\n",
212 | "data.to_csv(\"billboard_impact.csv\", index=False)"
213 | ]
214 | },
215 | {
216 | "cell_type": "markdown",
217 | "metadata": {},
218 | "source": [
219 | "## Customer Lifecycle"
220 | ]
221 | },
222 | {
223 | "cell_type": "code",
224 | "execution_count": 13,
225 | "metadata": {},
226 | "outputs": [
227 | {
228 | "name": "stdout",
229 | "output_type": "stream",
230 | "text": [
231 | "0.3721\n",
232 | "[1. 1. 1. 1. 1. 1. 1. 0.9999 0.9994 0.9984\n",
233 | " 0.9966 0.994 0.9886 0.9791 0.9663 0.944 0.9128 0.8726 0.8205 0.7603\n",
234 | " 0.6932 0.6138 0.5295 0.4424 0.3618 0.2919 0.2308 0.1769 0.1286 0.0942]\n"
235 | ]
236 | }
237 | ],
238 | "source": [
239 | "import pandas as pd\n",
240 | "import numpy as np\n",
241 | "from matplotlib import pyplot as plt\n",
242 | "from toolz import merge\n",
243 | "from sklearn.preprocessing import LabelEncoder\n",
244 | "\n",
245 | "np.random.seed(12)\n",
246 | "\n",
247 | "n = 10000\n",
248 | "t = 30\n",
249 | "\n",
250 | "age = 18 + np.random.poisson(10, n)\n",
251 | "income = 500+np.random.exponential(2000, size=n).astype(int)\n",
252 | "region = np.random.choice(np.random.lognormal(4, size=50), size=n)\n",
253 | "\n",
254 | "freq = np.random.lognormal((1 + age/(18+10)).astype(int))\n",
255 | "churn = np.random.poisson((income-500)/2000 + 22, n)\n",
256 | "\n",
257 | "ones = np.ones((n, t))\n",
258 | "alive = (np.cumsum(ones, axis=1) <= churn.reshape(n, 1)).astype(int)\n",
259 | "buy = np.random.binomial(1, ((1/(freq+1)).reshape(n, 1) * ones))\n",
260 | "\n",
261 | "cacq = -1*abs(np.random.normal(region, 2, size=n).astype(int))\n",
262 | "transactions = np.random.lognormal(((income.mean() - 500) / 1000), size=(n, t)).astype(int) * buy * alive\n",
263 | "\n",
264 | "data = pd.DataFrame(merge({\"customer_id\": range(n), \"cacq\":cacq},\n",
265 | " {f\"day_{day}\": trans \n",
266 | " for day, trans in enumerate(transactions.T)}))\n",
267 | "\n",
268 | "encoced = {value:index for index, value in\n",
269 | " enumerate(np.random.permutation(np.unique(region)))}\n",
270 | "\n",
271 | "customer_features = pd.DataFrame(dict(customer_id=range(n), \n",
272 | " region=region,\n",
273 | " income=income,\n",
274 | " age=age)).replace({\"region\":encoced}).astype(int)\n",
275 | "\n",
276 | "print((data.drop(columns=[\"customer_id\"]).sum(axis=1) > 0).mean()) # proportion of profitable customers\n",
277 | "print((alive).mean(axis=0)) # alive customer per days\n",
278 | "\n",
279 | "data.to_csv(\"./causal-inference-for-the-brave-and-true/data/customer_transactions.csv\", index=False)\n",
280 | "customer_features.to_csv(\"./causal-inference-for-the-brave-and-true/data/customer_features.csv\", index=False)"
281 | ]
282 | },
283 | {
284 | "cell_type": "markdown",
285 | "metadata": {},
286 | "source": [
287 | "## Price and Sales"
288 | ]
289 | },
290 | {
291 | "cell_type": "code",
292 | "execution_count": 1,
293 | "metadata": {},
294 | "outputs": [],
295 | "source": [
296 | "import numpy as np\n",
297 | "import pandas as pd\n",
298 | "import seaborn as sns\n",
299 | "from matplotlib import pyplot as plt\n",
300 | "\n",
301 | "np.random.seed(5)\n",
302 | "\n",
303 | "def price_elast(price, temp, weekday, cost):\n",
304 | " return -4 + 0.2*price + 0.05*temp + 2*np.isin(weekday, [1,7]) + 0.3 * cost\n",
305 | "\n",
306 | "def sales(price, temp, weekday, cost):\n",
307 | " elast = -abs(price_elast(price, temp, weekday, cost))\n",
308 | " output = np.random.normal(200 + 20*np.isin(weekday, [1,7]) + 1.3 * temp +\n",
309 | " 5*elast * price, 5).astype(int)\n",
310 | " \n",
311 | " return output\n",
312 | "\n",
313 | "\n",
314 | "n_rnd = 5000\n",
315 | "\n",
316 | "temp = np.random.normal(24, 4, n_rnd).round(1)\n",
317 | "weekday = np.random.choice(list(range(1, 8)), n_rnd)\n",
318 | "cost = np.random.choice([0.3, 0.5, 1.0, 1.5], n_rnd)\n",
319 | "price_rnd = np.random.choice(list(range(3, 11)), n_rnd)\n",
320 | "\n",
321 | "price_df_rnd = pd.DataFrame(dict(temp=temp, weekday=weekday, cost=cost,\n",
322 | " price=price_rnd, sales=sales(price_rnd, temp, weekday, cost)))\n",
323 | "\n",
324 | "n = 10000\n",
325 | "temp = np.random.normal(24, 4, n).round(1)\n",
326 | "weekday = np.random.choice(list(range(1, 8)), n)\n",
327 | "cost = np.random.choice([0.3, 0.5, 1.0, 1.5], n)\n",
328 | "price = np.random.normal(5 + cost + np.isin(weekday, [1,7])).round(1)\n",
329 | "\n",
330 | "price_df = pd.DataFrame(dict(temp=temp, weekday=weekday, cost=cost,\n",
331 | " price=price, sales=sales(price, temp, weekday, cost)))\n",
332 | "\n",
333 | "price_df_rnd.to_csv(\"./causal-inference-for-the-brave-and-true/data/ice_cream_sales_rnd.csv\", index=False)\n",
334 | "price_df.to_csv(\"./causal-inference-for-the-brave-and-true/data/ice_cream_sales.csv\", index=False)"
335 | ]
336 | },
337 | {
338 | "cell_type": "markdown",
339 | "metadata": {},
340 | "source": [
341 | "## Marketing Email"
342 | ]
343 | },
344 | {
345 | "cell_type": "code",
346 | "execution_count": 216,
347 | "metadata": {},
348 | "outputs": [],
349 | "source": [
350 | "import numpy as np\n",
351 | "import pandas as pd\n",
352 | "import seaborn as sns\n",
353 | "from matplotlib import pyplot as plt\n",
354 | "from sklearn.preprocessing import MinMaxScaler\n",
355 | "\n",
356 | "scaler = MinMaxScaler((0, 1))\n",
357 | "\n",
358 | "np.random.seed(12321)\n",
359 | "\n",
360 | "n_rnd=5000\n",
361 | "\n",
362 | "age = 18 + np.random.normal(24, 4, n_rnd).round(1)\n",
363 | "income = 500 + np.random.gamma(1, age * 100, n_rnd).round(2)\n",
364 | "insurance = np.random.gamma(30/age, age*1000, n_rnd).round(2)\n",
365 | "invested = np.random.gamma(age/10, income/2, n_rnd).round(2)\n",
366 | "\n",
367 | "em1_ps = income.min()/(income + 10)\n",
368 | "em2_ps = invested/(invested.max())\n",
369 | "em3_ps = np.where(age > 40, scaler.fit_transform(-income.reshape(-1,1)).ravel(), 0)\n",
370 | "\n",
371 | "em1 = np.random.binomial(1, em1_ps)\n",
372 | "em2 = np.random.binomial(1, em2_ps)\n",
373 | "em3 = np.random.binomial(1, em3_ps)\n",
374 | "\n",
375 | "elast_em1 = scaler.fit_transform((-3*age + 0.005*invested).reshape(-1,1)).ravel()\n",
376 | "elast_em2 = scaler.fit_transform((age + income*0.005).reshape(-1,1)).ravel()\n",
377 | "elast_em3 = scaler.fit_transform((-insurance).reshape(-1,1)).ravel()\n",
378 | "\n",
379 | "buy = scaler.fit_transform((1 + 0.4*age - invested/10000).reshape(-1,1)).ravel()\n",
380 | "buy += elast_em1*em1 + elast_em2*em2 + elast_em3*em3\n",
381 | "buy = scaler.fit_transform(buy.reshape(-1,1)).ravel()\n",
382 | "buy = np.random.binomial(1, buy).round(2)\n",
383 | "\n",
384 | "df = pd.DataFrame(dict(age=age, income=income, insurance=insurance, invested=invested,\n",
385 | " em1_ps=em1_ps, em2_ps=em2_ps, em3_ps=em3_ps,\n",
386 | " em1=em1, em2=em2, em3=em3,\n",
387 | " converted=buy))\n",
388 | "\n",
389 | "df.to_csv(\"./causal-inference-for-the-brave-and-true/data/invest_email.csv\", index=False)"
390 | ]
391 | },
392 | {
393 | "cell_type": "code",
394 | "execution_count": 9,
395 | "metadata": {},
396 | "outputs": [],
397 | "source": [
398 | "import numpy as np\n",
399 | "import pandas as pd\n",
400 | "import seaborn as sns\n",
401 | "from matplotlib import pyplot as plt\n",
402 | "from sklearn.preprocessing import MinMaxScaler\n",
403 | "\n",
404 | "scaler = MinMaxScaler((0.001, 0.999))\n",
405 | "\n",
406 | "np.random.seed(12321)\n",
407 | "\n",
408 | "n_rnd=15000\n",
409 | "\n",
410 | "age = 18 + np.random.normal(24, 4, n_rnd).round(1)\n",
411 | "income = 500 + np.random.gamma(1, age * 100, n_rnd).round(2)\n",
412 | "insurance = np.random.gamma(30/age, age*1000, n_rnd).round(2)\n",
413 | "invested = np.random.gamma(age/10, income/2, n_rnd).round(2)\n",
414 | "\n",
415 | "em1 = np.random.binomial(1, 0.5, n_rnd)\n",
416 | "em2 = np.random.binomial(1, 0.2, n_rnd)\n",
417 | "em3 = np.random.binomial(1, 0.9, n_rnd)\n",
418 | "\n",
419 | "elast_em1 = scaler.fit_transform((-3*age + 0.005*invested).reshape(-1,1)).ravel()\n",
420 | "elast_em2 = scaler.fit_transform((age + income*0.005).reshape(-1,1)).ravel()\n",
421 | "elast_em3 = scaler.fit_transform((-insurance).reshape(-1,1)).ravel()\n",
422 | "\n",
423 | "buy = (200*elast_em1*em1 + 100*elast_em2*em2 + 10*elast_em3*em3 \n",
424 | " + 1.5*age + 0.0005*invested - 0.0001*income)\n",
425 | "\n",
426 | "buy = scaler.fit_transform(buy.reshape(-1,1)).ravel()\n",
427 | "\n",
428 | "buy = np.random.binomial(1, buy)\n",
429 | "\n",
430 | "df = pd.DataFrame(dict(age=age, income=income, insurance=insurance, invested=invested,\n",
431 | " em1=em1, em2=em2, em3=em3,\n",
432 | " converted=buy))\n",
433 | "\n",
434 | "df.to_csv(\"./causal-inference-for-the-brave-and-true/data/invest_email_rnd.csv\", index=False)"
435 | ]
436 | },
437 | {
438 | "cell_type": "code",
439 | "execution_count": 12,
440 | "metadata": {},
441 | "outputs": [],
442 | "source": [
443 | "import numpy as np\n",
444 | "import pandas as pd\n",
445 | "import seaborn as sns\n",
446 | "from matplotlib import pyplot as plt\n",
447 | "from sklearn.preprocessing import MinMaxScaler\n",
448 | "\n",
449 | "scaler = MinMaxScaler((0.001, 0.999))\n",
450 | "\n",
451 | "np.random.seed(12321)\n",
452 | "\n",
453 | "n_rnd=15000\n",
454 | "\n",
455 | "age = 18 + np.random.normal(24, 4, n_rnd).round(1)\n",
456 | "income = 500 + np.random.gamma(1, age * 100, n_rnd).round(2)\n",
457 | "insurance = np.random.gamma(30/age, age*1000, n_rnd).round(2)\n",
458 | "invested = np.random.gamma(age/10, income/2, n_rnd).round(2)\n",
459 | "\n",
460 | "em1_ps = income.min()/(income + 10)\n",
461 | "em2_ps = invested/(invested.max())\n",
462 | "em3_ps = np.where(age > 40, scaler.fit_transform(-income.reshape(-1,1)).ravel(), 0)\n",
463 | "\n",
464 | "\n",
465 | "em1 = np.random.binomial(1, em1_ps)\n",
466 | "em2 = np.random.binomial(1, em2_ps)\n",
467 | "em3 = np.random.binomial(1, em3_ps)\n",
468 | "\n",
469 | "elast_em1 = scaler.fit_transform((-3*age + 0.005*invested).reshape(-1,1)).ravel()\n",
470 | "elast_em2 = scaler.fit_transform((age + income*0.005).reshape(-1,1)).ravel()\n",
471 | "elast_em3 = scaler.fit_transform((-insurance).reshape(-1,1)).ravel()\n",
472 | "\n",
473 | "buy = (200*elast_em1*em1 + 100*elast_em2*em2 + 10*elast_em3*em3 \n",
474 | " + 1.5*age + 0.0005*invested - 0.0001*income)\n",
475 | "\n",
476 | "buy = scaler.fit_transform(buy.reshape(-1,1)).ravel()\n",
477 | "\n",
478 | "buy = np.random.binomial(1, buy)\n",
479 | "\n",
480 | "df = pd.DataFrame(dict(age=age, income=income, insurance=insurance, invested=invested,\n",
481 | " em1=em1, em2=em2, em3=em3,\n",
482 | " converted=buy))\n",
483 | "\n",
484 | "df.to_csv(\"./causal-inference-for-the-brave-and-true/data/invest_email_biased.csv\", index=False)"
485 | ]
486 | },
487 | {
488 | "cell_type": "code",
489 | "execution_count": null,
490 | "metadata": {},
491 | "outputs": [],
492 | "source": []
493 | }
494 | ],
495 | "metadata": {
496 | "kernelspec": {
497 | "display_name": "causal-glory",
498 | "language": "python",
499 | "name": "causal-glory"
500 | },
501 | "language_info": {
502 | "codemirror_mode": {
503 | "name": "ipython",
504 | "version": 3
505 | },
506 | "file_extension": ".py",
507 | "mimetype": "text/x-python",
508 | "name": "python",
509 | "nbconvert_exporter": "python",
510 | "pygments_lexer": "ipython3",
511 | "version": "3.6.9"
512 | }
513 | },
514 | "nbformat": 4,
515 | "nbformat_minor": 2
516 | }
517 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/02-Randomised-Experiments.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# 02 - Randomised Experiments\n",
8 | "\n",
9 | "\n",
10 | "## The Gold Standard\n",
11 | "\n",
12 | "In the previous session, we saw why and how association is different from causation. We also saw what is required to make association be causation.\n",
13 | "\n",
14 | "$\n",
15 | "E[Y|T=1] - E[Y|T=0] = \\underbrace{E[Y_1 - Y_0|T=1]}_{ATT} + \\underbrace{\\{ E[Y_0|T=1] - E[Y_0|T=0] \\}}_{BIAS}\n",
16 | "$\n",
17 | "\n",
18 | "\n",
19 | "To recap, association becomes causation if there is no bias. There will be no bias if $E[Y_0|T=0]=E[Y_0|T=1]$. In words, association will be causation if the treated and control are equal or comparable, except for their treatment. Or, in more technical words, when the outcome of the untreated is equal to the counterfactual outcome of the treated. Remember that this counterfactual outcome is the outcome of the treated group if they had not received the treatment.\n",
20 | "\n",
21 | "I think we did an OK job explaining how to make association equal to causation in math terms. But that was only in theory. Now, we look at the first tool we have to make the bias vanish: **Randomised Experiments**. Randomised experiments randomly assign individuals in a population to a treatment or to a control group. The proportion that receives the treatment doesn't have to be 50%. You could have an experiment where only 10% of your samples get the treatment.\n",
22 | "\n",
23 | "Randomisation annihilates bias by making the potential outcomes independent of the treatment.\n",
24 | "\n",
25 | "$\n",
26 | "(Y_0, Y_1) \\perp\\!\\!\\!\\perp T\n",
27 | "$\n",
28 | "\n",
29 | "This can be confusing at first (it was for me). But don't worry, my brave and true fellow, I'll explain it further. If the outcome is independent of the treatment, doesn't this also imply that the treatment has no effect? Well, yes! But notice I'm not talking about the outcomes. Instead, I'm talking about the **potential** outcomes. The potential outcome is how the outcome **would have been** under treatment ($Y_1$) or under control ($Y_0$). In randomized trials, we **don't** want the outcome to be **independent** of the treatment since we think **the treatment causes** the outcome. But we want the **potential** outcomes to be independent of the treatment. \n",
30 | "\n",
31 | "\n",
32 | "\n",
33 | "Saying that the potential outcomes are independent of the treatment is saying that they would be, in expectation, the same in the treatment or the control group. In simpler terms, it means that treatment and control groups are comparable. Or that knowing the treatment assignment doesn't give me any information on how the outcome was previous to the treatment. Consequently, $(Y_0, Y_1)\\perp T$ means that the treatment is the only thing generating a difference between the outcome in the treated and in the control group. To see this, notice that independence implies precisely that\n",
34 | "\n",
35 | "$\n",
36 | "E[Y_0|T=0]=E[Y_0|T=1]=E[Y_0]\n",
37 | "$\n",
38 | "\n",
39 | "Which, as we've seen, makes it so that\n",
40 | "\n",
41 | "$\n",
42 | "E[Y|T=1] - E[Y|T=0] = E[Y_1 - Y_0]=ATE\n",
43 | "$\n",
44 | "\n",
45 | "So, randomization gives us a way to use a simple difference in means between treatment and control and call that the treatment effect.\n",
46 | "\n",
47 | "\n",
48 | "## In a School Far, Far Away\n",
49 | "\n",
50 | "In 2020, the Coronavirus Pandemic forced businesses to adapt to social distancing. Delivery services became widespread, and big corporations shifted to a remote work strategy. With schools, it wasn't different. Many started their own online repository of classes. \n",
51 | "\n",
52 | "Four months into the crisis, many wonder if the introduced changes could be maintained. There is no question that online learning has its benefits. It is cheaper since it can save on real estate and transportation. It can also be more digital, leveraging world-class content from around the globe, not just from a fixed set of teachers. Despite all of that, we still need to answer if online learning has a negative or positive impact on the student's academic performance.\n",
53 | "\n",
54 | "One way to answer this is to take students from schools that give mostly online classes and compare them with students from schools that provide lectures in traditional classrooms. As we know by now, this is not the best approach. It could be that online schools attract only the well-disciplined students that do better than average even if the class were presential. In this case, we would have a positive bias, where the treated are academically better than the untreated: $E[Y_0|T=1] > E[Y_0|T=0]$.\n",
55 | "\n",
56 | "Or on the flip side, it could be that online classes are cheaper and are composed chiefly of less wealthy students, who might have to work besides studying. In this case, these students would do worse than those from the presential schools even if they took presential classes. If this was the case, we would have a bias in the other direction, where the treated are academically worse than the untreated: $E[Y_0|T=1] < E[Y_0|T=0]$. \n",
57 | "\n",
58 | "So, although we could make simple comparisons, it wouldn't be compelling. One way or another, we could never be sure if there wasn't any bias lurking around and masking our causal effect.\n",
59 | "\n",
60 | "\n",
61 | "\n",
62 | "To solve that, we need to make the treated and untreated comparable $E[Y_0|T=1] = E[Y_0|T=0]$. One way to force this is by randomly assigning the online and presential classes to students. If we managed to do that, the treatment and untreated would be, on average, the same, except for the treatment they receive. \n",
63 | "\n",
64 | "Fortunately, some economists have done that for us. They've randomized classes so that some students were assigned to have face-to-face lectures, others to have only online lessons, and a third group to have a blended format of both online and face-to-face classes. They collected data on a standard exam at the end of the semester.\n",
65 | "\n",
66 | "Here is what the data looks like:"
67 | ]
68 | },
69 | {
70 | "cell_type": "code",
71 | "execution_count": 1,
72 | "metadata": {
73 | "tags": [
74 | "hide-input"
75 | ]
76 | },
77 | "outputs": [
78 | {
79 | "name": "stdout",
80 | "output_type": "stream",
81 | "text": [
82 | "(323, 10)\n"
83 | ]
84 | },
85 | {
86 | "data": {
87 | "text/html": [
88 | "<div>\n",
89 | "<style scoped>\n",
90 | " .dataframe tbody tr th:only-of-type {\n",
91 | " vertical-align: middle;\n",
92 | " }\n",
93 | "\n",
94 | " .dataframe tbody tr th {\n",
95 | " vertical-align: top;\n",
96 | " }\n",
97 | "\n",
98 | " .dataframe thead th {\n",
99 | " text-align: right;\n",
100 | " }\n",
101 | "</style>\n",
102 | "<table border=\"1\" class=\"dataframe\">\n",
103 | " <thead>\n",
104 | " <tr style=\"text-align: right;\">\n",
105 | " <th></th>\n",
106 | " <th>gender</th>\n",
107 | " <th>asian</th>\n",
108 | " <th>black</th>\n",
109 | " <th>hawaiian</th>\n",
110 | " <th>hispanic</th>\n",
111 | " <th>unknown</th>\n",
112 | " <th>white</th>\n",
113 | " <th>format_ol</th>\n",
114 | " <th>format_blended</th>\n",
115 | " <th>falsexam</th>\n",
116 | " </tr>\n",
117 | " </thead>\n",
118 | " <tbody>\n",
119 | " <tr>\n",
120 | " <th>0</th>\n",
121 | " <td>0</td>\n",
122 | " <td>0.0</td>\n",
123 | " <td>0.0</td>\n",
124 | " <td>0.0</td>\n",
125 | " <td>0.0</td>\n",
126 | " <td>0.0</td>\n",
127 | " <td>1.0</td>\n",
128 | " <td>0</td>\n",
129 | " <td>0.0</td>\n",
130 | " <td>63.29997</td>\n",
131 | " </tr>\n",
132 | " <tr>\n",
133 | " <th>1</th>\n",
134 | " <td>1</td>\n",
135 | " <td>0.0</td>\n",
136 | " <td>0.0</td>\n",
137 | " <td>0.0</td>\n",
138 | " <td>0.0</td>\n",
139 | " <td>0.0</td>\n",
140 | " <td>1.0</td>\n",
141 | " <td>0</td>\n",
142 | " <td>0.0</td>\n",
143 | " <td>79.96000</td>\n",
144 | " </tr>\n",
145 | " <tr>\n",
146 | " <th>2</th>\n",
147 | " <td>1</td>\n",
148 | " <td>0.0</td>\n",
149 | " <td>0.0</td>\n",
150 | " <td>0.0</td>\n",
151 | " <td>0.0</td>\n",
152 | " <td>0.0</td>\n",
153 | " <td>1.0</td>\n",
154 | " <td>0</td>\n",
155 | " <td>1.0</td>\n",
156 | " <td>83.37000</td>\n",
157 | " </tr>\n",
158 | " <tr>\n",
159 | " <th>3</th>\n",
160 | " <td>1</td>\n",
161 | " <td>0.0</td>\n",
162 | " <td>0.0</td>\n",
163 | " <td>0.0</td>\n",
164 | " <td>0.0</td>\n",
165 | " <td>0.0</td>\n",
166 | " <td>1.0</td>\n",
167 | " <td>0</td>\n",
168 | " <td>1.0</td>\n",
169 | " <td>90.01994</td>\n",
170 | " </tr>\n",
171 | " <tr>\n",
172 | " <th>4</th>\n",
173 | " <td>1</td>\n",
174 | " <td>0.0</td>\n",
175 | " <td>0.0</td>\n",
176 | " <td>0.0</td>\n",
177 | " <td>0.0</td>\n",
178 | " <td>0.0</td>\n",
179 | " <td>1.0</td>\n",
180 | " <td>1</td>\n",
181 | " <td>0.0</td>\n",
182 | " <td>83.30000</td>\n",
183 | " </tr>\n",
184 | " </tbody>\n",
185 | "</table>\n",
186 | "</div>"
187 | ],
188 | "text/plain": [
189 | " gender asian black hawaiian hispanic unknown white format_ol \\\n",
190 | "0 0 0.0 0.0 0.0 0.0 0.0 1.0 0 \n",
191 | "1 1 0.0 0.0 0.0 0.0 0.0 1.0 0 \n",
192 | "2 1 0.0 0.0 0.0 0.0 0.0 1.0 0 \n",
193 | "3 1 0.0 0.0 0.0 0.0 0.0 1.0 0 \n",
194 | "4 1 0.0 0.0 0.0 0.0 0.0 1.0 1 \n",
195 | "\n",
196 | " format_blended falsexam \n",
197 | "0 0.0 63.29997 \n",
198 | "1 0.0 79.96000 \n",
199 | "2 1.0 83.37000 \n",
200 | "3 1.0 90.01994 \n",
201 | "4 0.0 83.30000 "
202 | ]
203 | },
204 | "execution_count": 1,
205 | "metadata": {},
206 | "output_type": "execute_result"
207 | }
208 | ],
209 | "source": [
210 | "import pandas as pd\n",
211 | "import numpy as np\n",
212 | "\n",
213 | "data = pd.read_csv(\"./data/online_classroom.csv\")\n",
214 | "print(data.shape)\n",
215 | "data.head()"
216 | ]
217 | },
218 | {
219 | "cell_type": "markdown",
220 | "metadata": {},
221 | "source": [
222 | "We can see that we have 323 samples. It's not exactly big data, but something we can work with. To estimate the causal effect, we can simply compute the mean score for each of the treatment groups."
223 | ]
224 | },
225 | {
226 | "cell_type": "code",
227 | "execution_count": 4,
228 | "metadata": {},
229 | "outputs": [
230 | {
231 | "data": {
232 | "text/html": [
233 | "<div>\n",
234 | "<style scoped>\n",
235 | " .dataframe tbody tr th:only-of-type {\n",
236 | " vertical-align: middle;\n",
237 | " }\n",
238 | "\n",
239 | " .dataframe tbody tr th {\n",
240 | " vertical-align: top;\n",
241 | " }\n",
242 | "\n",
243 | " .dataframe thead th {\n",
244 | " text-align: right;\n",
245 | " }\n",
246 | "</style>\n",
247 | "<table border=\"1\" class=\"dataframe\">\n",
248 | " <thead>\n",
249 | " <tr style=\"text-align: right;\">\n",
250 | " <th></th>\n",
251 | " <th>gender</th>\n",
252 | " <th>asian</th>\n",
253 | " <th>black</th>\n",
254 | " <th>hawaiian</th>\n",
255 | " <th>hispanic</th>\n",
256 | " <th>unknown</th>\n",
257 | " <th>white</th>\n",
258 | " <th>format_ol</th>\n",
259 | " <th>format_blended</th>\n",
260 | " <th>falsexam</th>\n",
261 | " </tr>\n",
262 | " <tr>\n",
263 | " <th>class_format</th>\n",
264 | " <th></th>\n",
265 | " <th></th>\n",
266 | " <th></th>\n",
267 | " <th></th>\n",
268 | " <th></th>\n",
269 | " <th></th>\n",
270 | " <th></th>\n",
271 | " <th></th>\n",
272 | " <th></th>\n",
273 | " <th></th>\n",
274 | " </tr>\n",
275 | " </thead>\n",
276 | " <tbody>\n",
277 | " <tr>\n",
278 | " <th>blended</th>\n",
279 | " <td>0.550459</td>\n",
280 | " <td>0.217949</td>\n",
281 | " <td>0.102564</td>\n",
282 | " <td>0.025641</td>\n",
283 | " <td>0.012821</td>\n",
284 | " <td>0.012821</td>\n",
285 | " <td>0.628205</td>\n",
286 | " <td>0.0</td>\n",
287 | " <td>1.0</td>\n",
288 | " <td>77.093731</td>\n",
289 | " </tr>\n",
290 | " <tr>\n",
291 | " <th>face_to_face</th>\n",
292 | " <td>0.633333</td>\n",
293 | " <td>0.202020</td>\n",
294 | " <td>0.070707</td>\n",
295 | " <td>0.000000</td>\n",
296 | " <td>0.010101</td>\n",
297 | " <td>0.000000</td>\n",
298 | " <td>0.717172</td>\n",
299 | " <td>0.0</td>\n",
300 | " <td>0.0</td>\n",
301 | " <td>78.547485</td>\n",
302 | " </tr>\n",
303 | " <tr>\n",
304 | " <th>online</th>\n",
305 | " <td>0.542553</td>\n",
306 | " <td>0.228571</td>\n",
307 | " <td>0.028571</td>\n",
308 | " <td>0.014286</td>\n",
309 | " <td>0.028571</td>\n",
310 | " <td>0.000000</td>\n",
311 | " <td>0.700000</td>\n",
312 | " <td>1.0</td>\n",
313 | " <td>0.0</td>\n",
314 | " <td>73.635263</td>\n",
315 | " </tr>\n",
316 | " </tbody>\n",
317 | "</table>\n",
318 | "</div>"
319 | ],
320 | "text/plain": [
321 | " gender asian black hawaiian hispanic unknown \\\n",
322 | "class_format \n",
323 | "blended 0.550459 0.217949 0.102564 0.025641 0.012821 0.012821 \n",
324 | "face_to_face 0.633333 0.202020 0.070707 0.000000 0.010101 0.000000 \n",
325 | "online 0.542553 0.228571 0.028571 0.014286 0.028571 0.000000 \n",
326 | "\n",
327 | " white format_ol format_blended falsexam \n",
328 | "class_format \n",
329 | "blended 0.628205 0.0 1.0 77.093731 \n",
330 | "face_to_face 0.717172 0.0 0.0 78.547485 \n",
331 | "online 0.700000 1.0 0.0 73.635263 "
332 | ]
333 | },
334 | "execution_count": 4,
335 | "metadata": {},
336 | "output_type": "execute_result"
337 | }
338 | ],
339 | "source": [
340 | "(data\n",
341 | " .assign(class_format = np.select(\n",
342 | " [data[\"format_ol\"].astype(bool), data[\"format_blended\"].astype(bool)],\n",
343 | " [\"online\", \"blended\"],\n",
344 | " default=\"face_to_face\"\n",
345 | " ))\n",
346 | " .groupby([\"class_format\"])\n",
347 | " .mean())"
348 | ]
349 | },
350 | {
351 | "cell_type": "markdown",
352 | "metadata": {},
353 | "source": [
354 | "Yup. It's that simple. We can see that face-to-face classes yield a 78.54 average score, while online courses yield a 73.63 average score. Not so good news for the proponents of online learning. The $ATE$ for an online class is thus -4.91. This means that **online classes cause students to perform about 5 points lower**, on average. That's it. You don't need to worry that online courses might have poorer students that can't afford face-to-face classes or, for that matter, you don't have to worry that the students from the different treatments are different in any way other than the treatment they received. By design, the random experiment is made to wipe out those differences. \n",
355 | "\n",
356 | "For this reason, a good sanity check to see if the randomisation was done right (or if you are looking at the correct data) is to check if the treated are equal to the untreated in pre-treatment variables. Our data has information on gender and ethnicity to see if they are similar across groups. We can say that they look pretty similar for the `gender`, `asian`, `hispanic`, and `white` variables. The `black` variable, however, seems a little bit different. This draws attention to what happens with a small dataset. Even under randomisation, it could be that, by chance, one group is different from another. In large samples, this difference tends to disappear.\n",
357 | "\n",
358 | "## The Ideal Experiment\n",
359 | "\n",
360 | "Randomised experiments or Randomised Controlled Trials (RCT) are the most reliable way to get causal effects. It's a straightforward technique and absurdly convincing. It is so powerful that most countries have it as a requirement for showing the effectiveness of new medicine. To make a terrible analogy, you can think of RCT as Aang, from Avatar: The Last Airbender, while other techniques are more like Sokka. Sokka is cool and can pull some neat tricks here and there, but Aang can bend the four elements and connect with the spiritual world. Think of it this way, if we could, RCT would be all we would ever do to uncover causality. A well designed RCT is the dream of any scientist.\n",
361 | "\n",
362 | "\n",
363 | "\n",
364 | "Unfortunately, they tend to be either very expensive or just plain unethical. Sometimes, we simply can't control the assignment mechanism. Imagine yourself as a physician trying to estimate the effect of smoking during pregnancy on baby weight at birth. You can't simply force a random portion of moms to smoke during pregnancy. Or say you work for a big bank, and you need to estimate the impact of the credit line on customer churn. It would be too expensive to give random credit lines to your customers. Or that you want to understand the impact of increasing the minimum wage on unemployment. You can't simply assign countries to have one or another minimum wage. You get the point.\n",
365 | "\n",
366 | "We will later see how to lower the randomisation cost by using conditional randomisation, but there is nothing we can do about unethical or unfeasible experiments. Still, whenever we deal with causal questions, it is worth thinking about the **ideal experiment**. Always ask yourself, if you could, **what would be the perfect experiment you would run to uncover this causal effect?** This tends to shed some light on the way how we can discover the causal effect even without the ideal experiment.\n",
367 | "\n",
368 | "\n",
369 | "## The Assignment Mechanism\n",
370 | "\n",
371 | "In a randomised experiment, the mechanism that assigns units to one treatment or the other is, well, random. As we will see later, all causal inference techniques will somehow try to identify the assignment mechanisms of the treatments. When we know for sure how this mechanism behaves, causal inference will be much more confident, even if the assignment mechanism isn't random.\n",
372 | "\n",
373 | "Unfortunately, the assignment mechanism can't be discovered by simply looking at the data. For example, if you have a dataset where higher education correlates with wealth, you can't know for sure which one caused which by just looking at the data. You will have to use your knowledge about how the world works to argue in favor of a plausible assignment mechanism: is it the case that schools educate people, making them more productive and leading them to higher-paying jobs. Or, if you are pessimistic about education, you can say that schools do nothing to increase productivity, and this is just a spurious correlation because only wealthy families can afford to have a kid get a higher degree.\n",
374 | "\n",
375 | "In causal questions, we usually can argue in both ways: that X causes Y, or that it is a third variable Z that causes both X and Y, and hence the X and Y correlation is just spurious. For this reason, knowing the assignment mechanism leads to a much more convincing causal answer. This is also what makes causal inference so exciting. While applied ML is usually just pressing some buttons in the proper order, applied causal inference requires you to seriously think about the mechanism generating that data. \n",
376 | "\n",
377 | "\n",
378 | "## Key Ideas\n",
379 | "\n",
380 | "We looked at how randomised experiments are the simplest and most effective way to uncover causal impact. It does this by making the treatment and control groups comparable. Unfortunately, we can't do randomised experiments all the time, but it is still helpful to think about what is the ideal experiment we would do if we could.\n",
381 | "\n",
382 | "Someone familiar with statistics might be protesting right now that I didn't look at the variance of my causal effect estimate. How can I know that a 4.91 points decrease is not due to chance? In other words, how can I know if the difference is statistically significant? And they would be right. Don't worry. I intend to review some statistical concepts next. \n",
383 | "\n",
384 | "\n",
385 | "## References\n",
386 | "\n",
387 | "I like to think of this entire book as a tribute to Joshua Angrist, Alberto Abadie and Christopher Walters for their amazing Econometrics class. Most of the ideas here are taken from their classes at the American Economic Association. Watching them is what is keeping me sane during this tough year of 2020.\n",
388 | "* [Cross-Section Econometrics](https://www.aeaweb.org/conference/cont-ed/2017-webcasts)\n",
389 | "* [Mastering Mostly Harmless Econometrics](https://www.aeaweb.org/conference/cont-ed/2020-webcasts)\n",
390 | "\n",
391 | "I'll also like to reference the amazing books from Angrist. They have shown me that Econometrics, or 'Metrics as they call it, is not only extremely useful but also profoundly fun.\n",
392 | "\n",
393 | "* [Mostly Harmless Econometrics](https://www.mostlyharmlesseconometrics.com/)\n",
394 | "* [Mastering 'Metrics](https://www.masteringmetrics.com/)\n",
395 | "\n",
396 | "My final reference is Miguel Hernan and Jamie Robins' book. It has been my trustworthy companion in the most thorny causal questions I had to answer.\n",
397 | "\n",
398 | "* [Causal Inference Book](https://www.hsph.harvard.edu/miguel-hernan/causal-inference-book/)\n",
399 | "\n",
400 | "The data used here is from a study of Alpert, William T., Kenneth A. Couch, and Oskar R. Harmon. 2016. [\"A Randomized Assessment of Online Learning\"](https://www.aeaweb.org/articles?id=10.1257/aer.p20161057). American Economic Review, 106 (5): 378-82.\n",
401 | "\n",
402 | "\n",
403 | "\n",
404 | "## Contribute\n",
405 | "\n",
406 | "Causal Inference for the Brave and True is an open-source material on causal inference, the statistics of science. Its goal is to be accessible monetarily and intellectually. It uses only free software based on Python.\n",
407 | "If you found this book valuable and want to support it, please go to [Patreon](https://www.patreon.com/causal_inference_for_the_brave_and_true). If you are not ready to contribute financially, you can also help by fixing typos, suggesting edits, or giving feedback on passages you didn't understand. Go to the book's repository and [open an issue](https://github.com/matheusfacure/python-causality-handbook/issues). Finally, if you liked this content, please share it with others who might find it helpful and give it a [star on GitHub](https://github.com/matheusfacure/python-causality-handbook/stargazers)."
408 | ]
409 | },
410 | {
411 | "cell_type": "code",
412 | "execution_count": null,
413 | "metadata": {},
414 | "outputs": [],
415 | "source": []
416 | }
417 | ],
418 | "metadata": {
419 | "celltoolbar": "Tags",
420 | "kernelspec": {
421 | "display_name": "Python 3 (ipykernel)",
422 | "language": "python",
423 | "name": "python3"
424 | },
425 | "language_info": {
426 | "codemirror_mode": {
427 | "name": "ipython",
428 | "version": 3
429 | },
430 | "file_extension": ".py",
431 | "mimetype": "text/x-python",
432 | "name": "python",
433 | "nbconvert_exporter": "python",
434 | "pygments_lexer": "ipython3",
435 | "version": "3.7.12"
436 | },
437 | "toc": {
438 | "base_numbering": 1,
439 | "nav_menu": {},
440 | "number_sections": true,
441 | "sideBar": true,
442 | "skip_h1_title": false,
443 | "title_cell": "Table of Contents",
444 | "title_sidebar": "Contents",
445 | "toc_cell": false,
446 | "toc_position": {},
447 | "toc_section_display": true,
448 | "toc_window_display": false
449 | },
450 | "varInspector": {
451 | "cols": {
452 | "lenName": 16,
453 | "lenType": 16,
454 | "lenVar": 40
455 | },
456 | "kernels_config": {
457 | "python": {
458 | "delete_cmd_postfix": "",
459 | "delete_cmd_prefix": "del ",
460 | "library": "var_list.py",
461 | "varRefreshCmd": "print(var_dic_list())"
462 | },
463 | "r": {
464 | "delete_cmd_postfix": ") ",
465 | "delete_cmd_prefix": "rm(",
466 | "library": "var_list.r",
467 | "varRefreshCmd": "cat(var_dic_list()) "
468 | }
469 | },
470 | "types_to_exclude": [
471 | "module",
472 | "function",
473 | "builtin_function_or_method",
474 | "instance",
475 | "_Feature"
476 | ],
477 | "window_display": false
478 | }
479 | },
480 | "nbformat": 4,
481 | "nbformat_minor": 4
482 | }
483 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/Prediction-Metrics-For-Causal-Models.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 1,
6 | "id": "36722953",
7 | "metadata": {},
8 | "outputs": [],
9 | "source": [
10 | "import pandas as pd\n",
11 | "import numpy as np\n",
12 | "from matplotlib import pyplot as plt\n",
13 | "import statsmodels.formula.api as smf\n",
14 | "from toolz import *"
15 | ]
16 | },
17 | {
18 | "cell_type": "markdown",
19 | "id": "22f4484b",
20 | "metadata": {},
21 | "source": [
22 | "# Why Prediction Metrics are Dangerous For Causal Models\n",
23 | " \n",
24 | "A common misconception I often hear is that, if we have random data, to evaluate a causal model we could just evaluate the predictive performance of such model on the random dataset, using a metric like $R^2$. Unfortunately things are not that simple and I'll try to explain why. \n",
25 | " \n",
26 | "Generally speaking, we can say any outcome is a function of the treatment and covariates\n",
27 | " \n",
28 | "$\n",
29 | "Y = F(x, t)\n",
30 | "$\n",
31 | " \n",
32 | "Let's say we can decompose this function into two additive pieces. One piece that doesn't depend on the treatment and another that depends only on the treatment and possible interactions.\n",
33 | " \n",
34 | "$\n",
35 | "Y = g(x) + f(t,x)\n",
36 | "$\n",
37 | " \n",
38 | "This additive structure places some restriction on the functional form but not much, so we can argue it is a pretty general way of describing a Data Generating Process (DGP).\n",
39 | " \n",
40 | "The point is that if the treatment effect is weaker than the covariates effect, then, **even if we have random data**, we can have a model which has higher predictive power but is bad for causal inference. All that model has to do is approximate $g(x)$ while disregarding $f(t,x)$.\n",
41 | " \n",
42 | "In other words, **predictive performance on a random dataset does not translate our preference for how good a model is for causal inference**.\n",
43 | " \n",
44 | "To show that, let's use some simulated data.\n",
45 | " \n",
46 | "## Simulating Data\n",
47 | " \n",
48 | "In the following DGP, we have covariates $X$ that have a high predictive power but don't interact with the treatment. In other words, they don't dictate the treatment effect heterogeneity. We also have features $W$ that only impact the outcome through the treatment (are not confounders). Since the treatment has low predictive power, $W$ also doesn't have much predictive power.\n",
49 | " \n",
50 | "$\n",
51 | "Y_i = g(X_i) + f(T_i,W_i) + e_i\n",
52 | "$"
53 | ]
54 | },
55 | {
56 | "cell_type": "code",
57 | "execution_count": 2,
58 | "id": "fe6e5192",
59 | "metadata": {},
60 | "outputs": [],
61 | "source": [
62 | "n = 100000\n",
63 | "n_features = 20\n",
64 | "n_heter = 10\n",
65 | "\n",
66 | "np.random.seed(12321)\n",
67 | "\n",
68 | "X = np.random.normal(1, 10, (n, n_features))\n",
69 | "nuisance = np.random.uniform(-1,1, (n_features, 1))\n",
70 | "\n",
71 | "W = np.random.normal(1, 10, (n, n_heter))\n",
72 | "heter_y = np.random.uniform(-1,1, (n_heter, 1))\n",
73 | "\n",
74 | "T = np.random.normal(10, 2, (n, 1)) # T is random!\n",
75 | "Y = np.random.normal(T + T*W.dot(heter_y) + 20*X.dot(nuisance), 0.1)\n",
76 | "\n",
77 | "df = pd.concat([\n",
78 | " pd.DataFrame(X, columns=[f\"f{f}\" for f in range(n_features)]),\n",
79 | " pd.DataFrame(W, columns=[f\"w{f}\" for f in range(n_heter)])\n",
80 | "], axis=1).assign(T=T, Y=Y)"
81 | ]
82 | },
83 | {
84 | "cell_type": "markdown",
85 | "id": "e943c9d8",
86 | "metadata": {},
87 | "source": [
88 | "Now, let's break that dataset into a training and a test set"
89 | ]
90 | },
91 | {
92 | "cell_type": "code",
93 | "execution_count": 3,
94 | "id": "48674d02",
95 | "metadata": {},
96 | "outputs": [
97 | {
98 | "data": {
99 | "text/plain": [
100 | "((50000, 32), (50000, 32))"
101 | ]
102 | },
103 | "execution_count": 3,
104 | "metadata": {},
105 | "output_type": "execute_result"
106 | }
107 | ],
108 | "source": [
109 | "from sklearn.model_selection import train_test_split\n",
110 | "\n",
111 | "train, test = train_test_split(df, test_size=0.5)\n",
112 | "train.shape, test.shape"
113 | ]
114 | },
115 | {
116 | "cell_type": "markdown",
117 | "id": "c85db5a8",
118 | "metadata": {},
119 | "source": [
120 | "and train two models for treatment effect heterogeneity, $M1$ and $M2$. $M1$ will include the highly predictive features that don't affect the treatment heterogeneity and $M2$ will include the low predictive features that do affect treatment heterogeneity."
121 | ]
122 | },
123 | {
124 | "cell_type": "code",
125 | "execution_count": 4,
126 | "id": "12490bcf",
127 | "metadata": {},
128 | "outputs": [],
129 | "source": [
130 | "m1 = smf.ols(\"Y~T*(\" + \"+\".join([f\"f{f}\" for f in range(n_features)])+\")\", data=df).fit()\n",
131 | "m2 = smf.ols(\"Y~T*(\" + \"+\".join([f\"w{f}\" for f in range(n_heter)])+\")\", data=df).fit()"
132 | ]
133 | },
134 | {
135 | "cell_type": "markdown",
136 | "id": "642418c0",
137 | "metadata": {},
138 | "source": [
139 | "If we look at the predictive power of both models using the $R^2$, indeed, $M1$ is much better than $M2$. "
140 | ]
141 | },
142 | {
143 | "cell_type": "code",
144 | "execution_count": 5,
145 | "id": "5b4facc6",
146 | "metadata": {},
147 | "outputs": [
148 | {
149 | "name": "stdout",
150 | "output_type": "stream",
151 | "text": [
152 | "M1: 0.9160516511287358\n",
153 | "M2: 0.08378351037639298\n"
154 | ]
155 | }
156 | ],
157 | "source": [
158 | "from sklearn.metrics import r2_score\n",
159 | "\n",
160 | "print(\"M1:\", r2_score(test[\"Y\"], m1.predict(test)))\n",
161 | "print(\"M2:\", r2_score(test[\"Y\"], m2.predict(test)))"
162 | ]
163 | },
164 | {
165 | "cell_type": "markdown",
166 | "id": "0e2c5510",
167 | "metadata": {},
168 | "source": [
169 | "Now, let's calculate the cumulative elasticity curve for both models. For that, we will need Conditional Average Treatment Effect predictions. Since all the models here are linear, we can use the following formula to get CATE predictions\n",
170 | " \n",
171 | "$\n",
172 | "\\hat{CATE_i} = M(X, W, t) - M(X, W, t-1)\n",
173 | "$"
174 | ]
175 | },
176 | {
177 | "cell_type": "code",
178 | "execution_count": 6,
179 | "id": "1afeeee1",
180 | "metadata": {},
181 | "outputs": [],
182 | "source": [
183 | "@curry\n",
184 | "def elast(data, y, t):\n",
185 | " return (np.sum((data[t] - data[t].mean())*(data[y] - data[y].mean())) /\n",
186 | " np.sum((data[t] - data[t].mean())**2))\n",
187 | " \n",
188 | "\n",
189 | "def cumulative_gain(dataset, prediction, y, t, min_periods=30, steps=100):\n",
190 | " size = dataset.shape[0]\n",
191 | " ordered_df = dataset.sort_values(prediction, ascending=False).reset_index(drop=True)\n",
192 | " n_rows = list(range(min_periods, size, size // steps)) + [size]\n",
193 | " return np.array([elast(ordered_df.head(rows), y, t) * (rows/size) for rows in n_rows])\n"
194 | ]
195 | },
196 | {
197 | "cell_type": "code",
198 | "execution_count": 7,
199 | "id": "4261421b",
200 | "metadata": {},
201 | "outputs": [],
202 | "source": [
203 | "test_pred = test.assign(\n",
204 | " cate_1 = m1.predict(test) - m1.predict(test.assign(T=test[\"T\"]-1)),\n",
205 | " cate_2 = m2.predict(test) - m2.predict(test.assign(T=test[\"T\"]-1))\n",
206 | ")"
207 | ]
208 | },
209 | {
210 | "cell_type": "markdown",
211 | "id": "945b269f",
212 | "metadata": {},
213 | "source": [
214 | "Once we have those CATE predictions, we can evaluate them using the cumulative elasticity curve"
215 | ]
216 | },
217 | {
218 | "cell_type": "code",
219 | "execution_count": 8,
220 | "id": "f2605203",
221 | "metadata": {},
222 | "outputs": [],
223 | "source": [
224 | "cumelast_1 = cumulative_gain(test_pred, \"cate_1\", \"Y\", \"T\", steps=100)\n",
225 | "cumelast_2 = cumulative_gain(test_pred, \"cate_2\", \"Y\", \"T\", steps=100)"
226 | ]
227 | },
228 | {
229 | "cell_type": "code",
230 | "execution_count": 9,
231 | "id": "2d2407dc",
232 | "metadata": {},
233 | "outputs": [
234 | {
235 | "data": {
236 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD4CAYAAADmWv3KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABQc0lEQVR4nO3dd3hURdvA4d+kN5JAeiX0EiCU0IuA0gRREOlIEfC1gKiIfnZf9bX3giICoghIURBBpfcaCL0lkJAAqUAgCSmbzPfHSUICSVhCtmQz93XtlezZ3XOew4ZnZ+fMPCOklCiKoiiWx8rUASiKoiiGoRK8oiiKhVIJXlEUxUKpBK8oimKhVIJXFEWxUDamDqA4T09PGRISYuowFEVRqoyIiIgUKaVXaY+ZVYIPCQlh3759pg5DURSlyhBCxJb1mOqiURRFsVAqwSuKolgoleAVRVEslErwiqIoFkoleEVRFAulEryiKIqFUgleURTFQqkEr1QvUsLBRRCv5lsols9gCV4I0UgIEVnsdlUIMc1Qx1OU28rLhRVPw++Pw+x74Y+nID3Z1FEpisEYbCarlPIk0BJACGENnAd+N9TxFAWA7HSwcwYhSm7PugpLxkL0Bug6HfJzYee3cPxPeOBzaDa47H1Kqd2s1BdepWoxVqmCe4FoKWWZU2oV5a6lRsN3XcDFB1qOhOZD4FoCRK2Ho8vhciwM/ApaP6o9v+VoWPk0LHsMrKyh6YO37jP3Osx/CFy8YNgvRj0dRblbxkrww4GFpT0ghJgMTAYIDg42UjiKRVr7OiDALRA2vqvdAIQ1BLaF/p9AvZ43nu/VEEYvh18Gw9LHYLgjNOx943EpYdWzELdLux+1Hurfa7TTUZS7JQy9JqsQwg64AIRKKRPLe254eLhUxcaUConZDvPuh56vQrcX4Mo5OPEXuAZAnW7g6F72a7PS4KeBkHQcHvwaQgeDtQ3s+QFWT4cuz8GRZWDvCo9v1lr7imImhBARUsrw0h4zRqdiP2D/7ZK7olRYfj7887KWzDs8pW1zD4YOT0DTgeUndwAHNxjzu9aiXz4JvgiDf1+Fv1+Chn2h52tw7+uQeBgO/XbjddevQEbq3cefeQkOLdG6gxSlEhkjwY+gjO4ZRakUh5fAxUi49w2wc6rYPpxqweTNMGwB1KoDO77SPiQGfa9dXA0dDP6tYMM7Wot/+xfweXP4rvPdJfn8fFgyDpZPhC9bw7652mgfRakEBu2iEUI4A+eAulLKtNs9X3XRKHrLvAQX9sOFA1pXiqs/TNxQeSNdUqK0lr1LsXUUzm6Bnx4AuxqQcw3qdofYHVC3B4xcfOvIHdC6dk7+Da3HQEjXW5+z/UtY+xp0mgLndkP8HvBqDI/9qx1fUW6jvC4ag15klVJmAB6GPIZSzSQcge2fw5HlIPO0bZ4NYcBnlTuM0bP+rdvqdIPmj0DyCbjvLe2C6+7vYc0M2DUTOj5Z8vnHVsKyiYCAw7+Bd1NoNwmaDwV7F7gQCev/C40HQK+3tdccWaaN6on8VetiUpS7YPCLrHdCteCVMl1LhJVT4PQ/YOcCbcZBwz7g2+L2feyGJCUsGgmn18LEtVo3Dmgjbn4dBgGtYcQiOLla+zBIOKRdrA0bDtEbIScDntiudREVmt0LMlPh6X1q7L1yW+W14FWCV8zflTiY/yBcu6iNaGk3ERxrmjqqGzIvwczOkJ6ofZvwaQonVmvfAsauuvEBJCXE74W9s+Ho71pf+6MroO49Jfd3aInWJz96GdS/z+ino1QtKsErVVdqtJbcs67CqCUQ3N7UEZUuJQoO/qp1ISUegRq+MGJxyT784jJS4OoF8Gtx62O6HPgsVGv9j1xs2LiVKs9kffCKUiZdttaX7RdW9nPS4mFuP8jXwbg/y3+uqXnW14ZS6svZU7uVxsZO64La8hFcOquN6lGUClAdfIrx5efD0gnwfTc49W/Zz9v0Ply/DOP+Mu/kbgjh47UJVXtna99ejiyDdW9B4jFTR6ZUIaoFr1S+yzHa0MCA1uDXUmuRFrfpf3BilXaxcc0MbXSKrUPJ56RGayNJ2k0C7ybGitx8uPpDkwe0IaC7v9eKowFs+1SrmXPPi+ATatoYFbOnWvBKxV06o40Nv9nfL8PfL8KPveD9IG3s+P752gShw0u1rodWY2DofLh8FnZ8ees+Nr0P1nbaRdXqqstz4Nsc2j8O4/+GF6K1MgxRG7SLuv+8ArlZ2nPz87XW/qdNteGZFXHxkPZtQbEY6iKrUjFSwg89IPEoPHNQa3GCNuLlixYQ/pjWMj+3SxvamBoFNg4g87XCX2P+0Fr2v42FU3/DU7uhZoi2j6QT8G0H6DwVev3XVGdovjIvwYa3Yd8c8A6F+97UPiRjtmqTsJAwaQN4NdJ/nzHbtA/i0EEwZI6hIlcMwNS1aBRLFLVOm0Wal6PNxiwUMVf72XmqVgem7/+08dwT12ut9pAuMPTnG902ff6nVXtcXVAgLD9P68Kxc4HO04x+WlWCUy1tYtfIJZCRDL8+ok2aeuBLeGqX9kG6eIxWG/9mGSlw4BetC6xQerJWTVPma/XxK6O+jmIWVAteuXNSwuz7ID0JgjtoSWHaIW1q/WehWgt9xB2UH9r+RUGpX8DaHvKytT7mHi8bJn5LkpECB36GZkPAPUjbdmYz/PyQ1lff623IvqrNITi4CI6t0D6UbZ2g73vah+4vD8O5nVqt/OWTtA/djk+Z9LQU/alx8Erlilqv1VAf8LlWX+WbttBpqtZfvOyxO5+gI6XWlZNyUuvXz0rTEpODq8FOweJt+wzWvVlyW+EM2qYPwuYP4exmbWJWyil44AttaOYP90JOOjy5q/TaOorZUePglcojJWz+AFwDoeUorasldLB2ga9WXe1Wt+ft91OcEFC7o3ZTKkfnaVCzjtZ6t3fVPiwD22k1cACCO8Gub7RaOC2GQ+ux2vY2Y7WSEHF7zHdSmaI3leAV/Ump1VSJ262tjlTYj971eTiyVKuz0vtdVT/FHAgBoQ+V/biVlVbBsuUocHC/0VoPHQx//5826kkl+CpP/U9Ubi/ltHYR9MuWWmEtt2Ct77aQT1NtzLatk7YWqlJ1ONUq+YFs76ItQH50uRoyaQFUglfKl58HC0fA/p/Bqwn0/xQmrQcb+5LPG/g1TN5UsiqiUjW1Hge5mbDza218vVJlqS4apXzH/4TU0/DIPG2MdFkc3U1btlepPAGtod692rWWqHXaqJrgDqaOSqkA1YJXyialNjXeoz40GWjqaBRjEQJGLYWHZmoVL+f0gWWTtPHyhXIyYOsnBWUU8kwXq1Iu1YJXyha9Hi4e1LpfrKxNHY1iTFZW2vWUpg9qQy63fQ5Ra7WL6DJfW5s2PUF77tE/YNDMGzORFbOhWvBK2bZ+Bq4B0GKYqSNRTMXOGXq+Cv/Zpo2ZX/EkrHxam1Q14V946Dut/v3MztoyiopZMWgLXgjhDswGmgESmCCl3GnIYyqV5NwuiN0Gfd+/tRqkUv14N9YKnh1drpVCaNxf68oJbg8hnWHJeFjxtFaKwsX7xus2fwS7vi0Yhlkw32HYLyY7jerG0C34L4C/pZSNgTDguIGPp1QGXbZWOsCxFrR+1NTRKObCygqaD4EmA0rOcnUPhsGztBITm96/sf38ftj4rlbWOHSwdi3n+J9q+KURGSzBCyHcgG7AjwBSyhwp5RVDHU+pJFIWzGTcDfd/pH1FV5Tb8agHbcZDxDxt3kR+Hqx6VmvND18A/T+GrgWlnxOPmDTU6sSQLfg6QDIwVwhxQAgxWwhxS7YQQkwWQuwTQuxLTk6+dS+KcW16Dw4t1vpdmw8xdTRKVXLPi2DrqNXA2fsjXIzUCpo5uGmPF67KdfGQqSKsdgyZ4G2A1sBMKWUrIAN46eYnSSlnSSnDpZThXl5lLFCsGMe+udrY51ajoet0U0ejVDUuXtD5GW21rrWvQ90eWtdMoRq+4OytjcxSjMKQCT4eiJdS7i64vxQt4SvmJjsdVjwFq6ZpE1wGfK4qCSoV0/EpcPHRhlL2/+TWvyO/MK1mkWIUBhtFI6VMEELECSEaSSlPAvcCasVgc3PxECwdry0A0XU6dH8JrG1NHZVSVdk5w6glcP2K1i9/M78WEL1BW2rw5nV4lUpn6IlOU4AFQgg74Aww3sDHU+5EehLMf1Ab9jZ2pbbEnqLcrcK+9rIek3mQdEwriaAYlEETvJQyEii1EL1iYlJqoxxyMmDC33e2fqeiVJRvC+1nwiGV4I1AzWStro4s0y6G9XhZJXfFeGqGgL2butBqJCrBV0fXEmH1dAgI1xZ9UBRjEULrh1dDJY1CJfjq6O8XISdTqxaoiogpxubbQpvslKczdSQWTyX46ibrqjZdvN0k8Gpo6miU6sgvDHRZ2joDikGpBF/dxGyDfB007GvqSJTqyq/gQqvqhzc4leCrm+j1YOsMQWpBZcVEPBqAjaPqhzcCleCrm+gNUKerKgGsmI61jVZhUrXgDU4l+Ork0lm4dAbq9TR1JEp1598KzkdAoprcbkgqwVcnZzZqP1WCV0yty7Nalclfh2rDdhWDUAm+OoneAG7B2sILimJKbgEwchFkpsLCYdqwXaXSqQRfXeTp4MwWqNdDVYpUzIN/K3j4R7gQCYtGwKl/VaKvZLdN8EIID2MEohjY+QjITlPdM4p5aXy/VlY4bg/8+gh8WAeWTVSToCqJPi34XUKIJUKI+4VQTb8qK3oDCCuoe4+pI1GUkto+BjPOwujl0GIoHF4CBxeaOiqLoE+CbwjMAsYAp4UQ/xNCqCmQVUl6MhxfCQFtwLGmqaNRlFvZOkD9e+GBL8G/tbaymC7b1FFVebdN8FKzVko5ApgEjAX2CCE2CyE6GjxCpeJyMmDzR/BlS0g+CW0nmjoiRSmfENp6wGlxsH++qaOp8m5bD76gD340Wgs+EW0Rj5VAS2AJ2uLairk5HwGLH4Wr8dB4ANz3Jng2MHVUinJ79XpC7c6w5SNoOQrsnEwdUZWlTxfNTsAVeEhK2V9KuVxKqZNS7gO+M2x4SoUc+AXm9NP63Mf/DcMXqOSuVB2Frfj0RNg729TRVGn6JPhXpZRvSynjCzcIIR4BkFJ+YLDIlDsnJayeoS2gHdwBJm+C2qoXTamCanfSFoDf9hlkpZk6mipLnwT/Uinb/q+yA1Eqwcb/wZ7vocOT2ogEZzXCVanC7n0drl+CbZ+bOpIqq8w+eCFEP+B+IEAI8WWxh1wBvQapCiFigGtAHqCTUqr1WQ0lciFs+RBajYY+/1OTmZSqz78ltBgGu76F8AngHqRtP7Ea1v8Xxq0CZ0+ThmjuymvBXwD2AVlARLHbSqDPHRyjh5SypUruBhSzHVZOgZCu0P8zldwVy9HzNa3rccM72v3z+2HpBEg+Dmc2mTS0qqDMFryU8iBwUAixQEqpppWZq6sXYfFobTHjYT+rMsCKZXEPgo5Pan3xTR6Av54DZy+4fhlit0PzIaaO0KyV2YIXQvxW8OsBIcShm2967l8C/wohIoQQk8s4zmQhxD4hxL7k5OQ7DL+ak1K7oJp7HUYsVJOYFMvU5Vlw8oDFo7S/9VFLIKgdxO40dWRmr7xx8M8U/BxwF/vvIqU8L4TwBtYKIU5IKbcUf4KUchbaTFnCw8PlXRyr+tk7W1uh6f6P1TBIxXI5uMG9b8CaF2HoT+DdWBsdtuEdyLwETrVMHaHZKrMFL6W8WOw5iVLKWCllLJAE6NXJK6U8X/AzCfgdaHd34SpFUqLg39e0oWRqhqpi6dqMhRdjbhTLq91Z+3lul8lCqgr0GSa5BMgvdj+vYFu5hBDOQogahb8DvYEjFQlSuYkuG36fDDb28OA36qKqUj3YOtz43b81WNtp/fBKmW5bqgCwkVLmFN6RUuYIIfS5kucD/F5QgNIG+FVK+XfFwlSK5Om0UQTnI+CRn8DVz9QRKYrx2TpoxfPOqX748uiT4JOFEAOllCsBhBAPAim3e5GU8gwQdpfxKcXl58OKJ+HEKuj3IYQ+ZOqIFMV0gjvCji+1onp2zqaOxizp00XzH+BlIcQ5IUQc8CLwuGHDUkrIz9eqQf45FQ4t1up0tFdvgVLN1e4M+TqI32vqSMzWbVvwUspooIMQwqXgfrrBo1I0GanaMMjY7ZB9VdvWeRp0nW7SsBTFLAS10wrqxe6Aut1NHY1ZKq9UwWgp5S9CiOdu2g6AlPJTA8dWvUkJK5/WhkG2HAWB4RDYFrwamToyRTEPDq7g00xL8EqpymvBF3Zq1SjlMTVe3dD2zYGTq7W6Mh2fMnU0imKeaneCiJ9Al6NmcZeivFIF3xf8uk5KWWIskhCis0Gjqu6ST8I/r2hjfts/YepoFMV81e4Mu7+Dc6qbpjT6XGT9Ss9tSmXITodlj2mr2Dw0E6z0eYsUpZpq0Eub6Rrxk6kjMUvl9cF3BDoBXjf1w7sC1oYOrFpKOgG/PQqpp2H4Qqjha+qIFMW82TpC2AjY+6O2uLyLl6kjMivlNQ/tABe0D4EaxW5XAVXCrbIdWgI/9NAWOBjzBzTqa+qIFKVqaDMe8nMhcoGpIzE75fXBbwY2CyHmFdSgQQhhBbhIKa8aK8Bq4dxuWD5Rm7gxZK6anaood8K7MQR3goh50Gmq6tYsRp9/ifeEEK4F9WSOAMeEEC8YOK7qJWqdNp535GKV3BWlIsInwOWzcHaTqSMxK/ok+KYFLfaHgDVAHWCMIYOqdmK3g28L7WKRoih3rulAcKwF++aaOhKzok+CtxVC2KIl+JVSylzUOPjKk5sF8fsgpIupI1GUqsvGHlqN0uaOXL1g6mjMhj4J/nsgBm3i0xYhRG20C61KZTi/D/Kyb9S3VhSlYsIf07o6//4/U0diNm6b4KWUX0opA6SU90tNLNDDCLFVDzHbAaGtUKMoSsXVqgP3vAjH/oDjf5o6GrOgT7lghBD9gVCgWMV9/muQiKqb2G3g20ytp6oolaHzM1qC/+t5rduzmv+/um0LXgjxHTAMmIK2VN8jQG0Dx1U96HIgbi/UVv3vilIprG1h4NeQkQL/vmrqaExOnz74TlLKR4HLUsq3gI5AQ8OGVU1c2A+66xCi+t8VpdL4t4TOU+HALxC3x9TRmJQ+Cf56wc9MIYQ/kAuowdqVIWab9jO4k2njUBRL0+0FsHXWknw1pk+CXyWEcAc+AvajjahZaMCYqo/Y7eDdFJw9TB2JolgWO2do3F/rj9dlmzoak9FnFM3bUsorUsplaH3vjaWUr+l7ACGEtRDigBBi1d0EanHycrUSBWp4pKIYRouhkJUGp9eaOhKTKa+a5OByHkNKuVzPYzwDHEerQqkUit0BuRmq/11RDKVuD3DyhMO/QZMBpo7GJMobJvlAOY9J4LYJXggRCPQH3gWeu83Tq4+MVFjxNLgGaIt6KIpS+axtoNlgrVZ8Vlq1LAVSXjXJ8ZWw/8+BGZS+7B8AQojJwGSA4ODgSjikmcvTwbIJkJ4IE9ZUyz86RTGa5kNhzyxt4lOr0aaOxujK7IMXQnxe7Pdnbnps3u12LIQYACRJKSPKe56UcpaUMlxKGe7lVQ2K9W/4L5zZBP0/gYA2po5GUSxbYDjUrAOHl5g6EpMo7yJrt2K/j73psRZ67LszMFAIEQMsAnoKIar3mKWYbbD9C620aWtVkFNRDE4IaP4InN1SLYuQlZfgRRm/60VK+X9SykApZQgwHNggpax+35GKO7YSbByhz/9MHYmiVB9hw7UiZL8OhWsJJR9LPqU1vKLWwal/ISfTNDEaSHkXWa2EEDXRPgQKfy9M9GpN1oqI3qCNmrF1NHUkilJ9eNSDEYu19Y5n94LRSyHzEmz+AM5sLPnckK4wejnY2Jkm1kpWXoJ3AyK4kdT3F3vsjurBSyk3AZvu5DUW50qctph2eGVcu1YU5Y40uA/G/wULhsJ3XSAvB5y94L63IKA12DjAxYOwejqsmgYPfqN171Rx5Y2iCTFiHJavsKVQV1VaVhST8G8FE9dq9eJDumiLdds53Xg8qB1kpsKm97TSw92q/sqkepULVipB9EZw8QXvJqaORFGqr5ohMKKcSiv3vAiXzsCGd8CnGTTqZ7TQDEEtP24M+XlaC75eT4v42qcoFksIGPgV1KoH2780dTR3TSV4Y7h4EK5fhnqqe0ZRzJ6NvTYp6twOSI02dTR3Ra8EL4ToIoQYX/C7lxCijmHDsjBF/e/dTRqGoih6KhxaebBqF87VZ0WnN4AXgcKVbG2B6j1h6U5FbwSf5uDibepIFEXRh6u/1qUauRDy800dTYXp04IfBAwEMgCklBcop7aMcpOcDDi3S3XPKEpV03IkXI2HmC2mjqTC9EnwOVJKScHYdyGEs2FDsjDHVkB+rkrwilLVNOoP9m4Q+aupI6kwfRL8b0KI7wF3IcQkYB3wg2HDshD752tlgf3C1MIeilLV2DpA84e1EiNZaaaOpkL0WdHpY2ApsAxoBLwupfzK0IFVaVLCxv/ByinahdVxf2lX5hVFqVpajgbddTii7/pG5uW2E52EEM8Bi6WU1Xfdqzu1+QPt1mo0DPgcrG1NHZGiKBUR0Fr7Br71E21kTRWrI6VPF00N4F8hxFYhxNNCCB9DB1WlndkMm96HsBEw8GuV3BWlKhMCer8DaXGw8xtTR3PH9OmieUtKGQo8BfgBm4UQ6wweWVWUngTLJ4FnQ21BDzVrVVGqvjrdoPEA2PYZXEs0dTR35E5msiYBCUAqoAZ03yw/T0vuWVfhkXlgpwYbKYrF6PVf0GXDhrdNHckd0Wei05NCiE3AesADmCSl1GdFp+pDSvj3VW0pvvs/BJ+mpo5IUZTK5FEP2j8OB36Bi4dMHY3e9GnBBwHTpJShUso3pZTHDB1UlSKlVn5017fQ/glopZbiUxSL1O0FcKwJ/76i/b+vAspbdNu14NePgHNCiFrFb8YJz8xJCatfgN0zocNT0Pc91e+uKJbK0R26/5+2vuvpf00djV7Ka8EXTt+KAPYV/Iwodl/Z+TXs/QE6TYE+76rkrgCwfH880xYdIFuXZ+pQlMoWPl4rJfzva5CnM3U0t1Xeik4DCn5WqHKkEMIB2ALYFxxnqZTyjYrsyyzl5cLOb6HOPdDrbZXcFQB0efl88PcJEq9mY2NtxUdDWiDU34blsLbVLrguHgX7f4K2j5k6onLpc5F1vT7bSpEN9JRShgEtgb5CiA53HKG5OrEKrl2ADk+o5K4U2XgymcSr2XSs68HSiHjmbI8xdUhKZWvcXys9svF/2qg5M1ZeH7xDQV+7pxCiZrH+9xAg4HY7lpr0gru2BbeqcWVCH3t+APfa0KC3qSNRzMivu2PxrmHPTxPa0SfUh3f/OsaWU8mmDkupTIWTnzJTYNunpo6mXOW14B9H629vTMn+9xXA1/rsXAhhLYSIRBtDv1ZKufuuojUXCUcgdju0nQhW1qaOxuD2n7vM9qgUoxxry6lkHvpmO0cvVL3iTuevXGfTqWSGtQ3CzsaKT4e2pKFPDaYtjiQrV/XHW5SA1tps9R1fQ8ppU0dTpvL64L8AvhBCTKlocTEpZR7QUgjhDvwuhGgmpTxS/DlCiMnAZIDg4OCKHMb49swCG0et1kwVkJ8vGTNnN1FJ6dRwsMXVwQY/d0caeLtQ39uFy5m57Iu5RETsZdrUrsnHj4Rha6199kclXWP07N1YC8GeV+7D0c5wH2g7o1OZNH8f2bp8Jszby+9Pdsbf/Ubtj9y8/KK4zNHiPecAGNY2CABnexteH9CUkbN3s/rwRQa3DjRleEpl6/VfOLEa/noeHl1hll21+pQq+EoI0UwIMVQI8Wjh7U4OIqW8AmwE+pby2CwpZbiUMtzLy+tOdmsamZfg0G/QYig4VY3RoquPXGR7VCrNA9xp6OOCo501h+PT+GL9aZ7+9QCv/XGEHdGp1PF0ZkXkBV5efhgpJRnZOv7zy36khGvZOtYcuVipcaVn68jP13rt9sVc4rGf9hJcy4lfJ7UnMzuPcXP3kHY9lwtXrvPcb5E0ee1vNp5MqtQYKosuL5/F++K4p6EXgTWdirZ3qOtBiIcTCwuSv2JBXLzh3tfg7GY4sszU0ZRKn2qSbwDdgabAaqAfsA2Yf5vXeQG5UsorQghHoBfwwd0GbHIHF2rlQ9tNNnUkesnPl3y5/jT1vV34fkwbrK1utDKu5+QRnZxODQcbgms5IYTgs7Wn+GL9aXzdHIi7lEl0cjo/T2jPq38cZtHeuEprhc7aEs17a05ga2WFn7sDydey8XV1YMGk9njXcOD7MW0YO3cPD8/cQdylTCTg6mjL+6tP0K2BV4nzMAcbTiSReDWbtx8s+S3Uykowol0w7605wenEazTwUYuhWZTwCdrs1n9e0a7HObje/jVGpM/33SHAvUCClHI8EAa46fE6P2CjEOIQsBetD35VhSM1FydWg29z8G1m6kj08vfRBE4lpjOlZ/1bkqKjnTXNAtyo7eFcNJRv2n0NGN42iK82RPFH5AWeu68hXRp4MrRtEHvOXuJMcnrR6/PzJYlXs9Dl3dmalbvPpPL+mhN0beDF+C4hNA9wo0cj76LkDtCpvicfPNyCmJQM7m/ux4bn7+G/D4ZyMvEaKyLP3+W/SuU6m5LBW38ew9fVgZ6Nby3TNKRNILbWgl9VK97yWFnDgE8hPRG2fGTqaG5x2xY8cF1KmS+E0BXMbk1CK19QLinlIaDV3QZoVrKvQdxu6PiUqSPRS2Hrva6XMwNa+Ov1GiEE7zzUjGxdPvlS8lSP+gAMaR3IJ/+e4rd98bzUrzHZujzGz93LjuhUrAR41bCnbUgtPn4kDAfbG/30P++MYcHuczzbqyG9m/qQmpHDlIUHqO3hzDcjW1HDoexyyoNbBzKghT92Nlo7xN/NkWYB0Xy69lSJ7cYUm5pBeraOJr6uWFkJjpxPY+ycPUhg3vi22JRyjcDDxZ4+ob4s33+eF/s2LvHvo1iAgDYQOkhbwa3nq2a1uI8+CX5fwUXSH9BG0aQDOw0ZlNmK2V6wvmpPU0eil3+PJXAi4RqfDQu7oy4NG2srPhvWssQ274LW6dKIeJ7t1YDnfzvIjuhUnuxeDxsrQdzl6/wReZ4cXT4zR2tdQX8fSeD1lUdxsrXm8Z8j6NnYm+s5eaRdz2Xe+HblJvdCxZO4lZXghT6NGTtnDwv3nGNspxC9z+luSCnZcjqFudvPsumkNuTRw9mOjvU82HQyGVcHG36e2J56Xi5l7mNk+2BWHbpY6sXWv49c5P+WH6ZDXQ/ua+JDz8be1HS2M+g5KZWs1Wg4uhxOroHQh0wdTZHbJngp5ZMFv34nhPgbcC1onVc/ZzZqo2eCzX++Vn6+5Iv1UdTxdOYBPVvvtzMsPIi1xxIZ9cNu9sVe5qV+jfnPPfWKHg8LdOPNP4/x2oojPNImkGmLDxAW6M7Pj7Vj8d44Plt7ioycPN4f3Jym/hXrq+zWwJMOdWvx1YbTDGkTiLO9Pm2UitsZncrbq45x7OJVPF3smXZfAwJrOrHtdDLbolII8XTih0fD8XMrf6WfjnU9qOPpzK+7z92S4OcWTIbaf+4ya44k4GBrxbzx7ehQ18NQp6VUtrrdoYa/do2uKiR4IUTr8h6TUu43TEhmLHoDhHQ2q69gZVm2P57jF6/yxfCWpXYbVET3Rl5417BnX+xlxnaszePd6pZ4fFznOiRey2bmpmiWRcTj7WrP7LHh1HCwZWLXugxo4c/h82nc16TiywkIobXiH565gyX74hjXuUKVNG7rbEoG7685zj9HEwlwd+TjR8J4IMwPexute2VIm0CklHqXIRBCMLJdMO+uPs7RC2mE+muXsc5fuc7us5d4rldDpvSsz+HzaTz320Em/bSPxY93rPAHoWJkVtYQNgy2f6kt/ONiHktmlNf8+aScxyRQNfopKktaPKScgjbjTB3JbWVk6/jon5OEBblXWusdtK6bVwc05eiFNGb0aVxqcpvRpxGX0nNYfyKRuePa4ely48PQ180BXzeHu46jTe2ahAW6sXBPHGM7hVRarZfMHB1rDiewJCKOXWcu4WhrzfO9GjKpW91S+83v9LhDw4P4bN0p5m6P4eNHwgD444B2wXhQqwCEELQIdGf+hHY8PHMHY+fuYfkTnQiq5VTebhVzETZSW/Xp0G/Q6WlTRwOUP9GphzEDMXvRG7Wfdc3vn+VaVi7OdjZYFfSzf785mqRr2cwc3aZoW2UZGObPwLCyPzSEEHwwpIXBJyWNbB/Mi8sOs//cZdrUvvv5CJk5Oh74ahvRyRnU9nDi+V4NGdY2CG/Xu/9AKuTmZMuQNoEs2hPHi30b4+lix+8HztM2pGaJJO7v7sj8Ce0Y8t1Oxvy4m18ndSgx4as8qenZeLiY/zdMi+TVEALCIXKBNhDDDCY+6VNs7NHSbsYIzqxEbwAXX/BuYupISjideI12765n4Dfb2H0mlfNXrvP9ljM8EOZPm9o1TRaXoWecDmjhj4u9Db/ujquU/X38zymikzOYOao1m6Z3Z8q9DSo1uRca37kOOXn5/LIrlqMXrhKVlM6gVrfOLWjgU4M549qSmp7D4G93cCKh/KJWWbl5vL7iCG3eWVf0rUAxgZYjIOkYXDxo6kgA/cbBty126wq8CQw0YEzmJz9fW46vXg+z+FQulKPLZ9riSBxsrbiUnsOwWbsY/O12AF7s28jE0RmWs70ND7b0Z9WhC6Rl5t7VviJiLzF3x1nGdKhNv+Z+Bi3vW8fTmXsbe7NgdyyL9p7DztqK/s39Sn1um9o1+e0/HZFIHpm5kx3RpdcDikq6xkPfbGf+zlhcHWz4bnM0soqsOGRxmj0M1nba5CczoM8ominF7xcMmVxkqIDMUsJBuH7J7IZHfrn+NEcvXOX7MW24p6EXs7eeYeamaJ7qUb/EdHlLNaJdMAt2n+P3A/EVvtialZvHC0sP4e/myIv9GldyhKWb0KUOo2bv5pdd5+gb6oubU9nDRZv4ubL8yc6Mm7OH0bN3U8/LhVB/V0I8nYm7dJ2TiVc5mXANVwdb5o5rS9K1LF5cdpid0al0qu9plPNRinGsqZUxiZgLrR8FP9MuX12RMWYZgGGGLpirqILy93W7mzSM4iJiL/PtpigeaRNIn1BfAJ7u2YD/3FPP7KbxG0qzALcKXWyNu5TJ8YtXuZqlY8upZM4kZzB/QjtcDDzkslCneh408qnBycRrDGp928rbBLg7svQ/nZi74yyH49PYdeYSf0RewKuGPY19azC+cx0e61IHH1cHsnLz+ODvk8zZflYleFPp9Tac+hf+eBImb9QWCTERfWrR/MmNOu5WaDVpfjNkUGYlT6fNUAtsZzZDn3adSeWFpQfxd3fk9QealnissoZEVhUj2gXz0vLDfLH+NCPbBxeVOihL/OVMen+2hevFyveO6xRCt4bGK3QnhOD53g35cdtZejTS72/KzcmWafc1LLqflZtX6sgeB1trRrcP5quNUZxNyaCOp/MdxZZ2PZctp5LZfCoZKwH1CyqOdqrnqWbg6supFjzwOSwaCVs/he4vmiwUcbu+OiHEPcXu6oBYKWW8IYIJDw+X+/aZ2XKvh36D5ZNg+K/aSi4mFBF7mU/XnmR7VCreNeyZObqNSS+kmoPMHB2T5u9je1Qq1laCHo28ePuhZmVOPHpyQQQbTiQxb3w7/N0ccXW0wd3JsmaNJl3LovP7GxjZLpi3HtSvZlJmjo7nFh9k7fFE8vIl7k622FgJUtJzALi3sTc/jmtryLAtz7KJcPR3mLxJq19lIEKICClleGmP6dMHv7lgJ66FzxdC1JJSXqrUKM1Rfr72CezVBBr2M1kYUkq+3RTNR/+cxNPFjlf7N2F0h9qqRQU42dmwYGIHopLSWRoRz9ztZ/nk31NF48yL2xGVwurDCTzXq6FFzxL1ruHAA2H+LImI57nejXBzLL+LIFuXx+M/R7A9KoWJXevSJ9SHlkE1sbYSXMnM4YetZ/hmYzQRsZcqZUhqtdHvQ21wxpoXYfxqk4SgzzDJyUKIBOAQsA+tHo2ZNbMN5NQaSD4OXZ8DK9N0feTo8pm+5BAf/XOSgWH+bJnRg4ldS594U53V93bhpX6NebhNIH8evMDljJwSj+vy8nnrz2ME1nRk8k0zcC3RY13qkJmTx9O/7ic9W1fiseRr2WQUbNPl5fPMwki2nk7h/cEtePn+JrSpXavoOo67kx1P9aiPh7Mdn62985WLcnT5XM26u1FOVZZTLej4tLb6m4lWfdIna70ANJNShkgp60op60gpLf9/iJSw9RNt3dXQwSYJIfFqFqN/3M2y/fFMu68BXwxviZOdcS4EVlWPdqxNti6fJRElx8cv2H2Ok4nXeLV/02rx4Rjq78aHQ1qwIzqVEbN2kXwtm/jLmTz/20Ha/W8dzd78h/s+3cyQ73by99EEXhvQlKFtSy8S62RnwxPd67EtKoXdZ1L1jiE9W8fQ73cS/s46Ziw9eNux/BYpbDgIa23ykwnok+CjgUxDB2J2zm6G8xHQZRpYGzepSilZEXme3p9t4VD8FT4f1pJp9zU06PhsS9HY15V2IbX4Zde5otWiYlMz+Pjfk3Sp70mfUB8TR2g8Q8OD+OHRNkQlpdP/y630/Hgzfx66wITOdXjm3gaEeDhxOTOHl/o15rEu5Q+MG9W+Nl417Pl07albxthLKXlm0QEGf7udU4nXAO0i8KSf9nH4fBp9Qn1ZefACfT/fytg5e6pXoq/hC/Xvg4OLIN/46/Lqk7n+D9ghhNgNZBdulFJONVhUppZ7XVuhxcVXqy9hRNdz8pi+9CB/HbpIq2B3PnkkjLrllKFVbjWmY22mLDzA5lPJtKtTi8nzI7ASgv8Nal7tPiR7NvZh4eQOPLPoAPc09OLZXg31LntQnKOdNU91r8ebfx67ZYz9isgLrIi8gL2NFQO+2sbzvRqyN+Yyu86m8tnQljzUKoArmTks2H2OWVvOcP8XWxnWNpjnezcsUavIYrUaBb/9o82Gb9DLqIfWZxTNHrQl+g4DRUv3SCl/quxgzGYUzZ/TtIkKI5dAw95GO6yUkulLDrH8QDzTezfi8W51q92wx8qQo8un8wcbCPV3xcHGmn+PJTB/Qnu6NFDjwu9GVm4ePT7ehLWV4NeJHQj2cCIlPZten26mjqcz341uw6t/HOHfY4kAvP1gKGM6hpTYx5XMHD5fd5pfdsViYy0Y0MKfke2DaRHgxrGLV9l1JpXUjByGhgeVW1+/StHlwKeNIaQrDK30tFnuKBp9EvwBKaVRVmYyiwR/eCkseww6T4Nebxn10Iv3nuPFZYeZ2rM+z/W27FIDhvbp2lN8uV67sPVq/yZM7Gr5l42M4XB8GmPm7MbBxppfJrbns3WnWHs0kdXPdKG+dw2klPx1+CI5uvxy1++NSkrnx21nWRl5noycPOxsrMjRae1HaytBXr7kviY+PNG9rmWM3FnzEuz7EZ4/qV18rUR3m+D/B8QAf1Kyi6bSh0maPMGnRMGse8CnGYxbZdQZaMcuXGXQt9sJD6nJ/Antq81sVENJSMuix8eb6NvMl0+HhlW7rhlDOpFwldGz95Cdm8e1bB3Tezfk6Z4NKrSv9GwdKyLPczoxnda1a9KhTi2srATzd8by884YLmfmMqFzHV7q19gkSzRWmoTD8F0Xbehk+8crddd3m+DPlrJZ3m4kjRAiCJgP+KDNhJ0lpfyivNeYNMFLCT/2htQo+M9WcCu79VHZrmXl8sBX27iem8dfU7tWj35JI7iUkUNNJ1uV3A3gTHI6o2bvxtPFnuVPdjJI9dDrOXl88PcJ5u2IoUWgG1+NaEVtjzubmWtWvuuqLRY0cV2l7vZuJzpVtO6MDnheSrlfCFEDiBBCrJVSHqvg/gzr2B8QvwcGfm3U5A7w7aZoYlIz+e3xjiq5V6Jaal1Tg6nr5cLG6d0Bw5WGdrSz5s2BoXSo68GMpQfp/dkWujbwpHsjb3o29i71YvGdrLJldHXvgd3fQ16u0XoH9KlFU2rtdynl/PJeJ6W8CFws+P2aEOI4EACYX4LXZcPaN7SumZbGHTVz4cp15mw7y0Mt/WlXxwL6GpVqw1jzCfo286VZgCuztpxhw4kk1h1PwsZK8M2o1kWF9gBWHbrA6yuO8vXIVnSqZ4YX1H3DIC9HWxnOJ9QohzRKPXghRAjQCthdymOThRD7hBD7kpOT72S3lWfPLLgSC73f1tZWNKJP/j2FlDC9j7qoqihlCazpxH8fbMbWGT1Y91w3QgPcmLrwAPtitEuBW04l8+ziSC5l5PDiskNFM3XNSmE9mouHjHbI2yZ4KeWUYrdJQGtA7/FLQggXYBkwTUp5ywwHKeUsKWW4lDLcy8t4Ff2KZF6CLR9B/V5Gr/d+7MJVlh+IZ1znkGpRv11R7pYQgvreNZg7ri0B7o5MmLeXpRHx/OeXCOp5ufDj2HDiLl3no39OmjrUW3k2ABtH7YKrkVSk80zvevBCCFu05L5ASrm8AscyvK2fQPY1rfVuZO+tOY6rgy1Pda9v9GMrSlVWy9mOnya0w97WmulLDuLpYs/8Ce24t4kPj3aszU87Y4pa92bDylrrmkkwoxa8EOJPIcTKgtsq4CTwux6vE8CPwHEp5ad3H6oB5Odr5YCbDDT6WqsRsZfYejqFKT3rl7uij6IopQuq5cT8Ce24v7kv8ye0K1pDd0bfxvi7OTJj2SGyco1fHqBcvs21BG+kJRX1acF/DHxScHsP6CalfEmP13UGxgA9hRCRBbf7Kx6qASQchIwkaGT8UsA7orSiTY+El17gSVGU22vi58q3o9oQUmxhExd7G94d1IwzyRks2Vc5i7JXGr8WkJUGV84Z5XBljqIRQtQHfArrwRfb3lkIYS+ljC5vx1LKbYCZjlcqcLpgPGq9e41+6Mi4K9T3drltrW5FUe7cPQ29aBHoxk87Yxndobb5DJ30LVinIOEQ1Kxt8MOV14L/HCit7NvVgseqvqi14N8KXIx7cVdKSWTcFVoGuRv1uIpSXQghGNsxhKikdLZH6V/i2OC8m4CwMtpImvISvI+U8pbLvQXbQgwWkbFkXoL4vdroGSOLv3yd1IwcleAVxYAGhPnh4WzHvB0xpg7lBjsn8GxotJE05SV493Ieu/N6o+bmzEaQ+UYv3wmw/9xlAFoFuxv92IpSXdjbWDOiXTDrTyQSd8mMlrTwbWG0kTTlJfh9QohJN28UQkxEW7avaju9DhxrQkAbox86Mu4KjrbWNPKpYfRjK0p1MqpDMFZC8POuWIMfK1uXx8qDFxj5wy5e/ePwLQujFPFtDlfPQ4bhu47KK1UwDfhdCDGKGwk9HLADBhk4LsPKz4eoddrFVSPPXAUtwTcPcFO13hXFwPzcHOkb6suiPeeYdl8Dgy15uXDPOT7+5ySpGTl4ONuxIzqV4FpOTO5Wr5SgWmg/Ew5BvR4GiadQmRlGSpkopewEvIVWLjgGeEtK2VFKmWDQqAwt4ZA2PNIE3TPZujyOnr9KS9U9oyhGMb5zCFezdLy+4mjRMo6V6UpmDm+sPEqwhxM/TWjHnlfu4/7mvry/5gRbT5dSfsW3WII3MH1KFWyUUn5VcNtg8IiM4fRa7acJhkcev3iNnLx8WqkLrIpiFOEhtZh2XwOWRsTz8u+HKz3J/37gPDm6fN55qBn3NPTC2krw0ZAwGnjXYMrCA7f2/zvVAtdAo4ykMe5q0uYier1JhkcCRBZcYLXEFnxubi7x8fFkZWWZOhRFTw4ODgQGBmJra9nzMZ65twG6PMnXG6MQQtC5vgeH4tM4k5zBhM4hJdaYvRNSSn7dfY6wIHdC/d2Ktjvb2zDr0TYM/Ho7zy6OZMl/OpYci+/XAs7v02a0GnCMfvVL8LnXIX4fdHzSJIc/EHcFH1d7/Nyq/kCkm8XHx1OjRg1CQkLMZ2KJUiYpJampqcTHx1OnTkWXfagahBA837shufn5fL/5DAv3nMPOxgoXext2RKfw66QOFRq2HBF7mdNJ6Xz4cItbHqvt4cz0Po147Y8j7DpziY71PG482Oh+OLka4vZAcPu7OLPyVb+rfOcjID8Xanc2yeEj467QKqimSY5taFlZWXh4eKjkXkUIIfDw8Kg237iEELzUtzELJ3Xgz6e7cOTNPvw9rSseLnaMn7uH6OT0O97nr7vPUcPehgFhfqU+/kibQDxd7Jm5+aaJ/6GDwM4FDpS7rMZdq34JPnYHICDIcJ+aZbmUkUNsaqZFds8UUsm9aqlu75cQgo71PGge6IadjRXeNRz4eUJ7rITg0R/3sDfmEnl69tFfycxh1eGLPNQqoMzROQ621jzWpQ5bTiVz5HzajQfsXbQkf+R3rZqtgVTDBL9dW7nJ0d3ohy68oq5msCqK+QjxdOanCe24mpXLI9/tpM07a5m68ABRSeUn3uX7tYurI9oFl/u8UR2CqWFvw8xNWis+KzePN1ce5bNLHSA3A47etjhvhVWvBJ+Xq/V51e5k9EMnX8vm7VXHaOjjQutgy+yiMQdCCEaPHl10X6fT4eXlxYABAwA4ceIEHTt2xN7eno8//thUYSpmplmAG9te7Mk3I1tzXxMfNp1MYvy8vaRl5pb6/PRsHXN3nKVlkDtN/V3L3bergy1jOtZm9ZGL/HM0gYFfb2Pejhi+OOlOkn0Icv/PhjgloLol+IuHIDdTrwS/MzqV11ccYeycPXT/aCPj5+6pcG3p/HzJC0sPcjVLx1cjWmNnU73+2Y3J2dmZI0eOcP36dQDWrl1LQEBA0eO1atXiyy+/ZPr06aYKUTFTbo629G/hx8ePhDFvQjsS0rJ4fknkLcMqpZS8tOwQ5y9f56V+jfXa9/jOdbCztuLxnyO4lJHD/Ant+M899ZmV3hkRvweSDbMCVfUaRRO7XftZToLPys3j/TUnmLcjBhd7G0I8najvXYN1xxN568+jvDf41qvltzNvRwybTibz3wdDaeRbPcoTvPXnUY5dKK0YacU19XfljQduv1jx/fffz19//cWQIUNYuHAhI0aMYOvWrQB4e3vj7e3NX3/9VamxKZaldXBNXrm/CW/+eYzvt5zhie43ZqT+siuWVYcu8kKfRnSo61HOXm7wqmHPs70acvh8Gm880BTvGg50ru/Js3EDiD72K9ZrZxIy8vNKP49qluB3gEd9cPEu9eFTidd4asF+TielM65TCC/1a1y0cvwHf59g5qZo2tWpxaBWgXof8tiFq7y/5gT3NfFmTAfD139WYPjw4fz3v/9lwIABHDp0iAkTJhQleEXR19hOIeyNvcxH/5wgR5dPi0A3hIC3Vx2nRyMvnrinlDIExeTl5ZGQkEBcXBxxcXG0CAjgPyM7kZGRQYcOHYiLiyMhIYGv8vPZ9dQivB76H85Olbs2c/VJ8Pn5cG4nNB1Y5lOe/+0glzNz+GlCO+5pWHIS1PO9GhIRc5mXlx+hgXcNrmblsufsJaKTM8iXEiklAe6OPN+7UdGHwvWcPKYuOoC7ky0fDgmrViMW9GlpG0qLFi2IiYlh4cKF3H+/eS0iplQdQgg+eLgFF69c57N1p4q2B7g78skjYSQnJxUl77i4OHx9fRk6dCgATZo0ISoqCp1OV/S6sWPH0qlTJ5ycnKhVqxZNmjQhKCgIV1sdWW3CcHas/Lkx1SfBJx+HrCsQXHr3zMmEaxw+n8brA5rektwBbKyt+HJEK/p/uZUBX20DwEpo60LaWAmEEKw+nEBsaibfjmqNjbUV7/x1jKikdH55rD21nO0MeXbKTQYOHMj06dPZtGkTqalmtOCDUiVcvnyZ2NhY4uLiuI84QnQxZFvZ0/nhSbSrU4tuHdpw9OjREq/p06dPUYLv378/dnZ2BAUFFd1q19a+wQshWL16tVHOo/ok+Ngd2s8y+t+X7Y/HxkrwYEv/Mnfh6+bAD2PDWX88kTa1axIeUgtXhxtTvH/aEcMbK48yY+kheof6sGD3OR7vVpcuDSo2DVqpuAkTJuDu7k7z5s3ZtGmTqcNRzMzZs2c5efJkiRa4lJJ58+YBMGTIEDZsuFF6y8bGhq5du/LNh28DMHXqVLKzs0skcC+vGw1DcxmhZbAEL4SYAwwAkqSUzQx1HL3F7tAK/LjfOmZVl5fP7wfO06OxNx4u9uXupnVwzTKHOY7tFMK1rFw+/vcUf0Sep1mAK8/3blQp4St3JjAwkKlTp96yPSEhgfDwcK5evYqVlRWff/45x44dw9W1/KFuStVy5MgR9uzZQ3x8fFECT05OZt++fQgheOutt/jpp58AsLKyws/Pj4YNGxa9fsaMGTz55JMEBgYSFBSEj48P1tY3SotPnjzZ6OdUEYZswc8DvgYMOxdXH1Jq/e8hXUot7LP1dArJ17J5uLX+F0/L8lSP+mTm5LF4bxxfDG+lhkQaWXr6rdPNu3fvTvfu3QHw9fUlPj7eyFEplSE7OxsbGxusra2JjIxk9erVJVrgcXFxnD59Gg8PDxYtWsS7774LgI+PD4GBgQQHB5OVlYWjoyPPP/88kyZNIigoCD8/v1uKrfXp08cUp1jpDJbgpZRbhBAhhtr/HbkcA9cuQnDHUh9euj+emk629Gxc+uiaOyGEYEbfxkzv3Qgrq+pzUVVR7kZubi4XLlzA09MTZ2dnDh48yOzZs4mLiytqhSclJXHgwAFatmzJ7t27eeWVV/Dw8CAoKIjg4GA6d+5MXp42V+Wpp55iwoQJBAQEYG9/67fy5s2bG/sUTcLkffBCiMnAZIDg4PKn/FbYuV3az1ISfFpmLmuPJTKyXXCltrZVclcUTfHhgsHBwfj7+3P8+HFee+21opZ3QkICUkr+/PNPBgwYwPnz5/n555+L+rfbtGlDYGAgHh7auPMxY8YwZswYnMoYVujnV3rxr+rG5AleSjkLmAUQHh5e+cutgNY94+AGXrfOOlt1+AI5uvxK6Z5RlOomPz+f5OTkokRdv359mjdvTlxcHMOHDycuLo4LFy4Utay//fZbnnjiCfLz8zly5AhBQUH07du3KJG3aKFNJOzXrx9Xrlwp87hlJXalJJMneKM4twuCOoDVrS303/efp6GPC80C1EU2RSlOSsmlS5dK9HHXr1+f3r17k5GRQYsWLYiPjycnJ6foNS+//DLNmzenRo0a2NnZ0b179xIjTVq1agVAaGgoJ06cKPPY1WnOiCFZfoLPSIWUk9ByxC0PXUy7zr7Yyzzfq6H6g1KqHZ1OR25uLtevX2fWrFnExcURFBRUNELE19eXpKSkEq8ZM2YMvXv3xsnJia5du+Lj41MigdetWxcAd3d3Nm7caPRzUkoy5DDJhUB3wFMIEQ+8IaX80VDHK1Nc2f3vaw5ra4ff30L11ymWJzs7m+zsbHJycopuNjY2BAZq3ZEnTpwgKyuLlJQUHn/8caysrBg8eHBRgn/22WdxcHAokcB9fHwArYVdOGZcMV+GHEVza5PZFM7tBGt7bQ3Wm6w+fJHGvjWo5+VigsAUQxBCMGrUKH755RdAa6X6+fnRvn17Vq1axYIFC/jggw+QUlKjRg1mzpxJWFiYiaOumMzMTDIzM8nNzS1K4AANGjQAIDY2lqtXbxR8s7W1xdnZuei+v782qc/GxoZz587h5+eHjc2NlPDSSy8Z4zQUA7L8LppzuyCgNdiUHCqVkJZV1D2jWI7i5YIdHR1vKRdcp04dNm/eTM2aNVmzZg2TJ09m9+7dJoy4pPz8fITQSl+kp6eTnp5eogWu0+lo3rw5QgiSkpJISUkBtCRtZ2dXYkigv78/fn5+2NnZYWtri9VN16Bq1aoFQGJiIkFBQcY7ScVoLDvB52TChQPQacotD605chFQ3TMGs+YlSDhcufv0bQ793r/t08orF9yp041SFR06dDDqpCcpJbm5udjY2GBlZUV6ejqXLl0q0QLPzc0lLCwMW1tb0tLSuHjxItbW1tjZ2WFnZ4ezszNSSoQQ+Pn54evri52d3S3JG8DFRX0zre4sO8Gfj4B8XakFxv46pLpnLJW+5YJ//PFH+vXrVynHlFKi0+nIycnB3t4eGxsb0tPTSUxMLErchV0oTZo0wdnZmevXr5OSklKUvN3c3LCzsyu64O/j44Ovr2+JKfLFlTaBR1GKs+wEf24X2gLbbUtsVt0zRqBHS9tQ9CkXvHHjRn788Ue2bdt22/0VJu/CJO3o6Ii9vT2ZmZmcO3euaLuU2jSOevXqUbNmTfLz88nMzMTOzq5o2KCtrS12dlplUU9PTzw9PcscwVW8P1xRKsKy/4LO7QTvpuBYsjiY6p6xfOWVCz506BATJ05kzZo11KpVq6jlXTyBu7i4kJ2dzalTp8jNzSU/P7/o9cHBwXh7exd1izg7O1OzZs0S3SgArq6u5U6JV0NzFUOz3ASfna5VkGz9aInNWbl5LNxzTnXPWLjCcsFNmzZl3bp16HQ6kpOTiYmJYdiwYfzyyy/Uq1ePAwcOlEjeoI3/dnFxwcbGBicnp6LEXXgr7BpxcHCgcWP91uRUFFOw3AR/cg3orkPooBKb3151jFOJ6fw4NtxEgSmVqfhY79zcXKSUJCQkFJULPnDgADExMWRkZBAbG8u7775LamoqTz75JKCNWvn3339LJPDCyoLW1tbUq1f+smyKYs4sN8EfWQquARDUvmjTisjz2iIc99Tl3iY+JgxO0VdGRgZZWVklulBsbGwICQkB4PTp02RlZRU9f/v27SXuF5aJHTlyJHZ2dixfvrzUESeKYoksM8FnXoKo9dD+8aL6M1FJ6fzf8sOE167JdLUIh0kVDvMDuHbtGhkZGSWGCYI20gTg/PnzRZN1CocLFr/4WFiBtLD1fXPyLr7KjqJUN5aZ4E+sgvxcaD4EgPXHE3ntjyM42Frz1chW2FqrFpyhFI71trW1RQjB1atXSUtLK5HAdTodrVq1QghBamoqKSkpWFlZ3dLHDRRNwLGzsyt1uKBaiUlRymaZCf7IMqhVl/OOjXhz/j7WHkukgbcL344Ow8+t8lcury6klKSmphIXF0fjxo1xdHRk/fr1zJkzh7i4OF5//XUyMzORUtKiRQvs7OxIT08nKSmpKHkXDhcsbMUHBAQQGBiItbV1qaNKHA2w0ryiVBeWl+CvJcLZLeR0fJaHZ+4k7XouL/VrzITOddTyeeWQUpKWllZUFrZNmzb4+PiwdetW3njjjaKVdQr7t/fs2UPbtm25ePEiO3fuJCgoCHt7e3x8fEp0lfj6+uLn51fmkMCbl0pTFKXyWF6CP7YCZD7zr4WTcDWLZU90ok3t0hfJrk6uXbtWYvmzjh070rRpU/bv38+oUaOIi4sjIyOj6PnLli1j8ODBCCHIysqidevWPPjgg0VVBQtHl4wePZrRo0cDcPz48aJKhYXUBU1FMR3LS/BHlpLj0YQP9wuGtAmoFsn9+vXrJVaPj4uLo1OnTvTs2ZOzZ8/SqlUr0tLSSrzm888/p2nTptSqVYumTZvSp0+fEmVhmzZtCkCXLl3YsWOHKU5LUZS7ZFkJPvkUxO1mZc2J2NlYMaOvZYyWOXv2bInkHR8fT3h4OOPGjSMjI6PUolKvvfYaPXv2xMfHh9GjR5dI3oGBgUUVFkNCQli2bJmxT8lgrK2tad68OTqdjjp16vDzzz/j7u5+1/udN28e+/bt4+uvv777IIvp3r07Z86cITY2tqgb66GHHmLdunWkp6frvZ9x48YxYMAAhgwZclfPUSyLZSX4iHnkCxvev9iGKf3q413DwdQR6WXv3r2cOXOmRAJv0qQJ77zzDgBt2rTh8uXLRc93d3cvuvjo7OzMhx9+WGJlncDAQBwctHN3cnKq9KRkzhwdHYmMjARg7NixfPPNN7zyyiumDeo23N3d2b59O126dOHKlStcvHjR1CEpFsJyEnxuFjLyVzZbtcPV05/xneuYNJziY73Xr1/P4cOHS7TC/f39i1rOEydO5NChQ4BW4jUoKIjatWsX7Wv27NlF24OCgm5psb/wwgtGOqs7071791u2DR06lCeffJLMzMxSC4GNGzeOcePGkZKScktLc9OmTXd0/I4dOxb9u+7Zs4dnnnmGrKwsHB0dmTt3Lo0aNWLevHmsXLmSzMxMoqOjGTRoEB9++CEAc+fO5b333sPd3Z2wsLCi4ZsxMTFMmDCBlJQUvLy8mDt3LsHBwYwbNw5HR0cOHDhAUlISc+bMYf78+ezcuZP27duXuQLS8OHDWbRoEV26dGH58uUMHjyYo0ePAtrf0YwZM1izZg1CCF599VWGDRuGlJIpU6awdu1agoKCigqYAURERPDcc8+Rnp6Op6cn8+bNw89P1V2qjiwmwctjfyCyLjMntwfvjWpu0BEzUkouX75ctGDCX3/9xZYtW0okcBsbG6KiogD49NNPWb16dYnlzwrXrgStbK29vT1BQUG4ubndMuJk8ODBBjsXS5WXl8f69et57LHHAGjcuDFbt27FxsaGdevW8fLLLxd9wEZGRnLgwAHs7e1p1KgRU6ZMwcbGhjfeeIOIiAjc3Nzo0aNH0YLRU6ZMYezYsYwdO5Y5c+YwdepU/vjjDwAuX77Mzp07WblyJQMHDmT79u3Mnj2btm3bEhkZScuWLW+J9d5772XSpEnk5eWxaNEiZs2axdtvvw3A8uXLiYyM5ODBg6SkpNC2bVu6devGzp07OXnyJMeOHSMxMZGmTZsyYcIEcnNzmTJlCitWrMDLy4vFixfzyiuvMGfOHMP/oytmx2ISfMKGmWTn+3BPn8G0r+tR4f1IKbly5QpxcXGEhoZibW3NypUrWbZsWYkulNzcXLKzs7G2tubPP/9kzpw5BAYGEhQURJcuXahT58Y3iO+//x5HR0dq1apV6nDB8HDLrItTXovbycmp3Mc9PT3vuMUO2gXnli1bcv78eZo0aUKvXr0ASEtLY+zYsZw+fRohRNGMWdASrJubGwBNmzYlNjaWlJQUunfvXjQTdtiwYZw6dQqAnTt3snz5ckBbhHrGjBlF+3rggQcQQtC8eXN8fHyKqkmGhoYSExNTaoK3tramS5cuLFq0iOvXrxeVYQDYtm0bI0aMwNraGh8fH+655x727t3Lli1birb7+/vTs2dPAE6ePMmRI0eKzjsvL0+13qsxgyZ4IURf4AvAGpgtpTRIkfD9+3bSOi2S370n81jX8otDFQ4XLBxpUqNGDVatWsUXX3xRNBKlcLhgfHw8AQEBHD9+nI0bNxIUFESbNm146KGHCAoKQqfTYW1tzWeffca3335b5pDAm4cOKoZT2AefmZlJnz59+Oabb5g6dSqvvfYaPXr04PfffycmJqZE91HxmbPW1tbodLoKH79wX1ZWViX2a2VlVe5+hw8fzqBBg3jzzTcrfGzQGiihoaHs3LnzrvajWAaD9WMIIayBb4B+QFNghBCiaWUf50pmDif++opcbOg66AlOnz7N+vXrmTdvHnFxcQCsW7eO5s2b4+7ujqurK6GhofTt27eonzM7O5urV68SGhrKpEmT+Pjjj1m8eHHRNPgXX3yRc+fOsX37dhYtWsRHH33E1KlTi/4DOzo6qvHeZsbJyYkvv/ySTz75BJ1OR1paWtHIobL6wotr3749mzdvJjU1ldzcXJYsWVL0WKdOnVi0aBEACxYsoGvXrncdb9euXfm///s/RowYccv2xYsXk5eXR3JyMlu2bKFdu3Z069ataPvFixfZuHEjAI0aNSI5Obkowefm5hb9nSvVjyFb8O2AKCnlGQAhxCLgQeBYZR7E3TaP2hfX4bsgg0tvlWy9L1myhKCgINzd3alXrx7du3cvGmUSFBREs2bNAHj44Yd5+OGHKzMsxQy0atWKFi1asHDhQmbMmMHYsWN555136N+//21f6+fnx5tvvknHjh1xd3cv0bXy1VdfMX78eD766KOii6x3SwjB9OnTb9k+aNAgdu7cSVhYGEIIPvzwQ3x9fRk0aBAbNmygadOmBAcH07FjR0Cr2bN06VKmTp1KWloaOp2OadOmERoaetcxKlWPKFxmrNJ3LMQQoK+UcmLB/TFAeynl0zc9bzIwGSA4OLhNbGzsnR0oT8f5LfN5Z+4aghq3KjFUsHD6vGIcx48fL6oCqVQd6n2r2oQQEVLKUi/kmfwiq5RyFjALIDw8/M4/baxtCOgxgZk9JlR2aIqiKFWaITuOzwNBxe4HFmxTFEVRjMCQCX4v0EAIUUcIYQcMB1Ya8HiKGTBUl59iGOr9smwGS/BSSh3wNPAPcBz4TUqpLudbMAcHB1JTU1XSqCIK6/sXlrVQLI9B++CllKuB1YY8hmI+AgMDiY+PJzk52dShKHpycHBQ8zQsmMkvsiqWw9bWtsQMXkVRTEvNzlEURbFQKsEriqJYKJXgFUVRLJTBZrJWhBAiGbjDqaxFPIGUSgynKlDnbPmq2/mCOuc7VVtK6VXaA2aV4O+GEGJfWdN1LZU6Z8tX3c4X1DlXJtVFoyiKYqFUglcURbFQlpTgZ5k6ABNQ52z5qtv5gjrnSmMxffCKoihKSZbUglcURVGKUQleURTFQlX5BC+E6CuEOCmEiBJCvGTqeAxBCBEkhNgohDgmhDgqhHimYHstIcRaIcTpgp81TR1rZRNCWAshDgghVhXcryOE2F3wfi8uKEVtMYQQ7kKIpUKIE0KI40KIjpb+Pgshni34uz4ihFgohHCwtPdZCDFHCJEkhDhSbFup76vQfFlw7oeEEK0retwqneCNtbC3GdABz0spmwIdgKcKzvMlYL2UsgGwvuC+pXkGrdx0oQ+Az6SU9YHLwGMmicpwvgD+llI2BsLQzt1i32chRAAwFQiXUjYDrNHWjrC093ke0PembWW9r/2ABgW3ycDMih60Sid4ii3sLaXMAQoX9rYoUsqLUsr9Bb9fQ/tPH4B2rj8VPO0n4CGTBGggQohAoD8wu+C+AHoCSwueYlHnLIRwA7oBPwJIKXOklFew8PcZraqtoxDCBnACLmJh77OUcgtw6abNZb2vDwLzpWYX4C6E8KvIcat6gg8A4ordjy/YZrGEECFAK2A34COlvFjwUALgY6q4DORzYAaQX3DfA7hSsJgMWN77XQdIBuYWdEvNFkI4Y8Hvs5TyPPAxcA4tsacBEVj2+1yorPe10vJaVU/w1YoQwgVYBkyTUl4t/pjUxrtazJhXIcQAIElKGWHqWIzIBmgNzJRStgIyuKk7xgLf55poLdY6gD/gzK1dGRbPUO9rVU/w1WZhbyGELVpyXyClXF6wObHwq1vBzyRTxWcAnYGBQogYtK63nmj90+4FX+XB8t7veCBeSrm74P5StIRvye/zfcBZKWWylDIXWI723lvy+1yorPe10vJaVU/w1WJh74K+5x+B41LKT4s9tBIYW/D7WGCFsWMzFCnl/0kpA6WUIWjv6wYp5ShgIzCk4GmWds4JQJwQolHBpnuBY1jw+4zWNdNBCOFU8HdeeM4W+z4XU9b7uhJ4tGA0TQcgrVhXzp2RUlbpG3A/cAqIBl4xdTwGOscuaF/fDgGRBbf70fqk1wOngXVALVPHaqDz7w6sKvi9LrAHiAKWAPamjq+Sz7UlsK/gvf4DqGnp7zPwFnACOAL8DNhb2vsMLES7xpCL9k3tsbLeV0CgjQ6MBg6jjTCq0HFVqQJFURQLVdW7aBRFUZQyqASvKIpioVSCVxRFsVAqwSuKolgoleAVRVEslErwiqIoFkoleEVRFAv1/8+kMZhjFJ+HAAAAAElFTkSuQmCC\n",
237 | "text/plain": [
238 | "<Figure size 432x288 with 1 Axes>"
239 | ]
240 | },
241 | "metadata": {
242 | "needs_background": "light"
243 | },
244 | "output_type": "display_data"
245 | }
246 | ],
247 | "source": [
248 | "plt.plot(range(len(cumelast_1)), cumelast_1, label=\"M1\")\n",
249 | "plt.plot(range(len(cumelast_2)), cumelast_2, label=\"M2\")\n",
250 | "plt.plot([0, 100], [0, elast(test_pred, \"Y\", \"T\")], linestyle=\"--\", label=\"Random Model\", color=\"black\")\n",
251 | "plt.legend()\n",
252 | "plt.ylabel(\"Cumulative Elasticity\");"
253 | ]
254 | },
255 | {
256 | "cell_type": "markdown",
257 | "id": "10ac5842",
258 | "metadata": {},
259 | "source": [
260 | "As we can see, now $M1$ is much worse than $M2$, even though it has a higher $R^2$ on the test set. This shows that predictive power doesn't translate directly to a good causal model, even if we use random data."
261 | ]
262 | },
263 | {
264 | "cell_type": "markdown",
265 | "id": "4e6aab85",
266 | "metadata": {},
267 | "source": [
268 | "## Predictive Metric For Causal Inference\n",
269 | " \n",
270 | "This doesn't mean we can't come up with a way to correctly evaluate a causal model using predictive metrics. In order to do so, let's go back to our additive assumption about de DGP.\n",
271 | " \n",
272 | "$\n",
273 | "Y = g(x) + f(t,x)\n",
274 | "$\n",
275 | " \n",
276 | "To use a predictive metric, we need to somehow transform the outcome in order to remove the $g(x)$ component from it. That way, all the remaining predictive power will necessarily be used to learn the causal relationship.\n",
277 | " \n",
278 | "$\n",
279 | "\\tilde{Y} = Y - g(x) = f(t,x)\n",
280 | "$\n",
281 | " \n",
282 | "One way of doing that is by orthogonalizing $Y$. We can use any ML model to estimate $g$ and get out of fold residuals\n",
283 | " \n",
284 | "$\n",
285 | "\\tilde{Y} = Y - \\hat{g}(x)\n",
286 | "$"
287 | ]
288 | },
289 | {
290 | "cell_type": "code",
291 | "execution_count": 10,
292 | "id": "aa28a557",
293 | "metadata": {},
294 | "outputs": [],
295 | "source": [
296 | "denoise_m = smf.ols(\"Y~\"+\n",
297 | " \"+\".join([f\"w{f}\" for f in range(n_heter)])+\n",
298 | " \"+\"+\n",
299 | " \"+\".join([f\"f{f}\" for f in range(n_features)]), data=train).fit()\n",
300 | "\n",
301 | "test_res = test.assign(Y_res = test[\"Y\"] - denoise_m.predict(test) + test[\"Y\"].mean())"
302 | ]
303 | },
304 | {
305 | "cell_type": "markdown",
306 | "id": "f6302a6a",
307 | "metadata": {},
308 | "source": [
309 | "Once we do that, we can evaluate the power of each model in predicting the residualized outcome $\\tilde{Y}$. The predictive performance on this new outcome will directly translate to a better causal model."
310 | ]
311 | },
312 | {
313 | "cell_type": "code",
314 | "execution_count": 11,
315 | "id": "0c6ed700",
316 | "metadata": {},
317 | "outputs": [
318 | {
319 | "name": "stdout",
320 | "output_type": "stream",
321 | "text": [
322 | "M1: -281.7950185092476\n",
323 | "M2: -24.04912481654577\n"
324 | ]
325 | }
326 | ],
327 | "source": [
328 | "print(\"M1:\", r2_score(test_res[\"Y_res\"], m1.predict(test_res)))\n",
329 | "print(\"M2:\", r2_score(test_res[\"Y_res\"], m2.predict(test_res)))"
330 | ]
331 | },
332 | {
333 | "cell_type": "markdown",
334 | "id": "86f12479",
335 | "metadata": {},
336 | "source": [
337 | "The downside of this approach is that it depends on how well you can estimate $g(x)$."
338 | ]
339 | }
340 | ],
341 | "metadata": {
342 | "kernelspec": {
343 | "display_name": "causal-glory",
344 | "language": "python",
345 | "name": "causal-glory"
346 | },
347 | "language_info": {
348 | "codemirror_mode": {
349 | "name": "ipython",
350 | "version": 3
351 | },
352 | "file_extension": ".py",
353 | "mimetype": "text/x-python",
354 | "name": "python",
355 | "nbconvert_exporter": "python",
356 | "pygments_lexer": "ipython3",
357 | "version": "3.9.7"
358 | }
359 | },
360 | "nbformat": 4,
361 | "nbformat_minor": 5
362 | }
363 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/_config.yml:
--------------------------------------------------------------------------------
1 | # in _config.yml
2 | title: "Causal Inference for the Brave and True"
3 | author: Matheus Facure Alves
4 |
5 | execute:
6 | execute_notebooks: "off"
7 |
8 | repository:
9 | url: https://github.com/matheusfacure/python-causality-handbook
10 | html:
11 | use_issues_button: true
12 | google_analytics_id: UA-97848161-1
13 |
14 | exclude_patterns: [_build, .DS_Store, "**.ipynb_checkpoints"]
15 |
16 | sphinx:
17 | config:
18 | language: en
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/_toc.yml:
--------------------------------------------------------------------------------
1 | format: jb-book
2 | root: landing-page
3 | parts:
4 | - caption: Part I - The Yang
5 | chapters:
6 | - file: 01-Introduction-To-Causality.ipynb
7 | - file: 02-Randomised-Experiments.ipynb
8 | - file: 03-Stats-Review-The-Most-Dangerous-Equation.ipynb
9 | - file: 04-Graphical-Causal-Models.ipynb
10 | - file: 05-The-Unreasonable-Effectiveness-of-Linear-Regression.ipynb
11 | - file: 06-Grouped-and-Dummy-Regression.ipynb
12 | - file: 07-Beyond-Confounders.ipynb
13 | - file: 08-Instrumental-Variables.ipynb
14 | - file: 09-Non-Compliance-and-LATE.ipynb
15 | - file: 10-Matching.ipynb
16 | - file: 11-Propensity-Score.ipynb
17 | - file: 12-Doubly-Robust-Estimation.ipynb
18 | - file: 13-Difference-in-Differences.ipynb
19 | - file: 14-Panel-Data-and-Fixed-Effects.ipynb
20 | - file: 15-Synthetic-Control.ipynb
21 | - file: 16-Regression-Discontinuity-Design.ipynb
22 | - caption: Part II - The Yin
23 | chapters:
24 | - file: 17-Predictive-Models-101.ipynb
25 | - file: 18-Heterogeneous-Treatment-Effects-and-Personalization.ipynb
26 | - file: 19-Evaluating-Causal-Models.ipynb
27 | - file: 20-Plug-and-Play-Estimators.ipynb
28 | - file: 21-Meta-Learners.ipynb
29 | - file: 22-Debiased-Orthogonal-Machine-Learning.ipynb
30 | - file: 23-Challenges-with-Effect-Heterogeneity-and-Nonlinearity.ipynb
31 | - file: 24-The-Diff-in-Diff-Saga.ipynb
32 | - file: 25-Synthetic-Diff-in-Diff.ipynb
33 | - caption: Appendix
34 | chapters:
35 | - file: Debiasing-with-Orthogonalization.ipynb
36 | - file: Debiasing-with-Propensity-Score.ipynb
37 | - file: When-Prediction-Fails.ipynb
38 | - file: Prediction-Metrics-For-Causal-Models.ipynb
39 | - file: Conformal-Inference-for-Synthetic-Control.ipynb
40 | - caption: Contribute
41 | chapters:
42 | - url: https://www.patreon.com/causal_inference_for_the_brave_and_true
43 | title: Patreon
44 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/drinking.csv:
--------------------------------------------------------------------------------
1 | agecell,all,allfitted,internal,internalfitted,external,externalfitted,alcohol,alcoholfitted,homicide,homicidefitted,suicide,suicidefitted,mva,mvafitted,drugs,drugsfitted,externalother,externalotherfitted
2 | 19.068493,92.8254,91.70615,16.61759,16.73813,76.20782,74.96801,0.63913804,0.7943445,16.316818,16.284573,11.203714,11.5921,35.829327,34.81778,3.8724246,3.448835,8.534373,8.388236
3 | 19.150684,95.10074,91.88372,18.327684,16.920654,76.773056,74.963066,0.6774093,0.8375749,16.859964,16.270697,12.193368,11.593611,35.639256,34.63389,3.2365112,3.4700222,8.655786,8.530174
4 | 19.232876,92.144295,92.049065,18.911053,17.098843,73.23324,74.950226,0.8664426,0.8778347,15.219254,16.262882,11.715812,11.595129,34.20565,34.446735,3.2020707,3.4920695,8.513741,8.662681
5 | 19.31507,88.42776,92.20214,16.10177,17.27268,72.32598,74.92947,0.86730844,0.9151149,16.742825,16.261148,11.27501,11.596655,32.278957,34.2563,3.2806885,3.5149798,8.258285,8.7857275
6 | 19.39726,88.70494,92.34292,17.36352,17.442156,71.341415,74.90076,1.0191631,0.94940656,14.947726,16.26551,10.984314,11.598189,32.650967,34.062588,3.5481975,3.5387552,8.417533,8.899288
7 | 19.479452,90.19179,92.471344,17.872105,17.607254,72.31968,74.86409,1.1713219,0.98070073,15.642815,16.27599,12.166634,11.599731,32.721443,33.86558,3.211689,3.5633986,7.9725456,9.003332
8 | 19.561644,96.22031,92.58739,16.414942,17.767965,79.80537,74.81942,0.8699163,1.0089884,16.263653,16.292604,12.405763,11.601281,36.385197,33.66527,3.8578897,3.5889127,10.287705,9.097831
9 | 19.643835,89.615555,92.69102,15.977087,17.924273,73.638466,74.76675,1.0979514,1.0342605,15.825645,16.31537,10.979514,11.602839,34.187935,33.461647,3.4831562,3.6153,8.670031,9.182756
10 | 19.726027,93.3817,92.782196,17.433271,18.076166,75.948425,74.706024,1.174851,1.0565081,16.789,16.344309,11.900103,11.604405,31.910467,33.254696,4.0551305,3.6425629,10.76315,9.2580805
11 | 19.80822,90.857956,92.86087,18.2854,18.22363,72.572556,74.637245,0.9484129,1.0757217,16.616194,16.379436,11.570638,11.60598,30.576832,33.044415,3.5660326,3.6707044,9.863494,9.323772
12 | 19.890411,95.81015,92.927025,18.6076,18.366652,77.20255,74.56037,1.3291142,1.0918926,17.278484,16.420774,11.468357,11.607562,33.531654,32.830788,4.101267,3.6997268,9.835445,9.379805
13 | 19.972603,94.158066,92.980606,18.43628,18.505219,75.72179,74.47539,1.2164142,1.1050112,16.953773,16.468338,11.631961,11.609152,33.603443,32.613808,3.7632816,3.729633,9.123107,9.426147
14 | 20.054794,92.4646,93.02158,19.634459,18.639317,72.83014,74.38227,1.0273844,1.1150686,17.199177,16.52215,11.263178,11.61075,31.772816,32.393467,3.729025,3.7604258,8.295178,9.46277
15 | 20.136986,98.2456,93.09767,20.88386,18.778563,77.36174,74.319115,0.9527308,1.1226311,17.034826,16.590736,11.928189,11.618317,34.29831,32.186256,3.9252508,3.7940538,9.679745,9.494516
16 | 20.219177,94.9604,93.16124,18.473003,18.913473,76.4874,74.24777,1.3740251,1.1271195,16.297464,16.6657,11.87005,11.625911,33.32011,31.975483,4.2365775,3.828613,9.809012,9.516518
17 | 20.30137,93.38532,93.212234,18.883484,19.044024,74.50184,74.168205,0.9938675,1.1285197,15.940106,16.747074,12.46157,11.633533,32.759403,31.761126,3.4403107,3.8641078,9.250613,9.528728
18 | 20.383562,90.12092,93.25059,18.56782,19.170197,71.5531,74.08039,1.2250932,1.1268175,16.883316,16.834883,11.102407,11.641183,30.589046,31.543165,3.254154,3.9005423,8.9967785,9.531103
19 | 20.465754,92.40623,93.276245,19.36313,19.291971,73.04309,73.984276,0.9969136,1.1219985,17.33096,16.92916,10.889364,11.648862,31.28775,31.32159,4.1793685,3.9379208,8.972222,9.523599
20 | 20.547945,93.354485,93.289154,19.968874,19.409328,73.38561,73.87982,0.960042,1.1140486,16.743132,17.029932,11.482102,11.6565695,30.298925,31.096382,4.3777914,3.976248,9.83083,9.506168
21 | 20.630136,92.22878,93.289246,19.422657,19.522247,72.80612,73.767,0.9615177,1.1029531,15.884273,17.137232,12.345887,11.664306,30.230118,30.867521,3.6537673,4.015528,10.192088,9.478767
22 | 20.712328,92.332245,93.276474,20.377037,19.630705,71.95521,73.64577,1.0015179,1.0886977,17.411003,17.25109,10.978177,11.6720705,30.122578,30.634996,3.6593924,4.055765,9.39886,9.441348
23 | 20.794521,92.66751,93.25076,19.521132,19.734684,73.14638,73.51608,1.1959587,1.0712676,17.9008,17.37153,11.303739,11.679864,29.74465,30.398787,4.3594623,4.096964,9.220456,9.393866
24 | 20.876713,93.3902,93.212074,18.623945,19.834162,74.76625,73.37791,0.96597224,1.0506482,18.160278,17.498592,12.209889,11.687687,30.717916,30.158875,3.9411666,4.139129,9.157417,9.336272
25 | 20.958904,94.26991,93.160324,20.200695,19.92912,74.06922,73.23121,1.2383568,1.0268247,17.143503,17.6323,11.880486,11.6955385,30.41714,29.915245,4.4890437,4.182265,9.48117,9.268521
26 | 20.99999,,93.13445,,19.976597,,73.15786,,1.014913,,17.699154,,11.699464,,29.79343,,4.203832,,9.234646
27 | 21.0,,102.89176,,21.10804,,81.78372,,1.8173605,,17.76202,,13.547069,,34.785976,,4.4421463,,10.353402
28 | 21.041096,105.26835,102.58908,21.937366,21.104536,83.330986,81.48454,2.5193088,1.7885877,18.332817,17.72146,12.402751,13.520511,36.316807,34.590244,4.224687,4.4496655,10.697372,10.321973
29 | 21.123287,101.06651,101.98372,21.121853,21.097532,79.944664,80.88619,1.5919044,1.7310421,17.316814,17.64034,14.83189,13.467395,32.575798,34.198776,4.426271,4.464704,9.978522,10.259112
30 | 21.205479,96.96652,101.412476,19.25328,21.102318,77.71324,80.31016,1.3613431,1.6776136,16.880655,17.563643,12.213193,13.416207,33.022293,33.815987,4.745253,4.4810143,10.268416,10.200633
31 | 21.287672,102.8269,100.87552,21.508316,21.118961,81.31858,79.75656,1.4416807,1.6283239,17.378096,17.49139,14.4557705,13.36696,35.106873,33.441917,4.1691847,4.498604,9.624192,10.146556
32 | 21.369864,100.97943,100.37305,22.366144,21.14752,78.61329,79.225525,1.3271359,1.5831952,17.018566,17.423609,13.583627,13.319661,32.358696,33.07662,4.410775,4.5174794,10.578053,10.096907
33 | 21.452055,101.236946,99.905235,21.232779,21.18806,80.00417,78.71717,1.7205198,1.5422496,17.478916,17.360321,13.803261,13.2743225,32.45526,32.72013,4.8487377,4.5376472,10.284016,10.051706
34 | 21.534246,99.4976,99.47227,21.27055,21.240648,78.22704,78.23162,1.1359962,1.5055093,18.410973,17.30155,12.848509,13.230954,31.533688,32.372505,4.622329,4.5591145,10.341482,10.010981
35 | 21.616438,95.20131,99.07433,22.093296,21.305344,73.10801,77.76899,1.0202943,1.472997,16.599403,17.24732,11.811869,13.189567,30.098682,32.03379,4.8660192,4.581888,9.496586,9.974753
36 | 21.69863,97.25825,98.71162,21.267872,21.382215,75.99038,77.32941,1.6117979,1.444735,18.044275,17.197657,13.3268175,13.15017,30.388288,31.704031,4.4029603,4.6059756,8.963169,9.943048
37 | 21.780823,98.92873,98.384315,19.454931,21.471325,79.47379,76.91299,1.7328279,1.4207466,17.17075,17.152584,14.76842,13.112777,31.742258,31.383278,5.0803366,4.6313834,9.806231,9.915888
38 | 21.863014,102.26221,98.092606,20.515566,21.572742,81.74664,76.51987,1.3019494,1.4010543,17.16206,17.112127,13.926913,13.077396,34.481934,31.071575,4.8132677,4.6581187,10.573407,9.893299
39 | 21.945206,96.833374,97.83669,22.093819,21.686531,74.739555,76.15016,1.1461909,1.3856815,16.718578,17.07631,12.370957,13.04404,31.540012,30.768976,4.2685733,4.6861887,9.446195,9.875306
40 | 22.027397,99.50197,97.61676,21.024887,21.812757,78.47707,75.80401,1.3462263,1.3746514,17.223778,17.045156,12.274417,13.012717,31.834293,30.475529,4.9097667,4.7156005,11.482519,9.861933
41 | 22.109589,96.07229,97.43374,23.40324,21.951653,72.669044,75.48208,1.6263269,1.3679978,17.016933,17.01882,11.622288,12.983537,29.55155,30.191505,4.4029827,4.746397,9.440629,9.853277
42 | 22.19178,95.96821,97.28708,20.624224,22.103125,75.34398,75.18395,1.8279659,1.3657339,16.570908,16.9972,12.597069,12.956413,30.836988,29.916727,4.609653,4.7785506,9.497475,9.849293
43 | 22.273973,98.61067,97.177,23.129915,22.267242,75.48076,74.90976,1.6322316,1.3678838,17.198147,16.980322,12.420884,12.931358,29.857895,29.651249,4.9365053,4.8120685,10.0720625,9.850004
44 | 22.356165,97.95247,97.10369,23.092215,22.444073,74.86025,74.659615,1.5554341,1.3744715,17.109776,16.96821,13.879258,12.908381,28.75559,29.395117,5.1050143,4.846958,9.372488,9.855436
45 | 22.438356,97.41173,97.06736,24.37291,22.633688,73.03882,74.43367,1.6381792,1.3855213,16.701437,16.960892,13.86459,12.887496,27.64927,29.148384,4.4350705,4.883228,9.469475,9.865617
46 | 22.520548,97.509476,97.0682,23.49674,22.836157,74.01273,74.23204,1.5611123,1.4010576,17.532492,16.958393,12.809127,12.868712,28.260136,28.911102,4.523223,4.920884,9.887045,9.88057
47 | 22.60274,95.56239,97.10642,22.817877,23.051548,72.744514,74.05487,1.2431532,1.4211051,16.040686,16.960741,12.15082,12.852041,29.234152,28.683325,5.0127144,4.9599347,10.025429,9.900323
48 | 22.68493,97.02306,97.18223,21.815123,23.279934,75.20794,73.90229,0.883854,1.4456885,16.793226,16.967962,12.856058,12.837497,29.528757,28.4651,5.4236493,5.0003877,10.204496,9.924901
49 | 22.767124,99.37457,97.29582,24.350594,23.521387,75.02398,73.77444,1.2074674,1.4748328,17.226536,16.980082,14.610356,12.8250885,27.44976,28.256483,4.789621,5.042251,10.504967,9.954332
50 | 22.849316,94.31531,97.447426,22.94374,23.77598,71.37157,73.67144,1.3709793,1.5085632,16.69369,16.997128,12.338814,12.814829,26.855064,28.057526,5.564563,5.0855317,9.435563,9.988643
51 | 22.931507,97.397644,97.63723,24.23832,24.043783,73.159325,73.59345,1.7774768,1.5469048,17.774767,17.019129,11.917173,12.806729,27.389301,27.868282,4.9688554,5.130238,10.099299,10.027859
52 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/hospital_treatment.csv:
--------------------------------------------------------------------------------
1 | hospital,treatment,severity,days
2 | 1,1,29.68661809190674,82
3 | 1,1,20.05034002189231,57
4 | 1,1,20.30239881248276,49
5 | 0,0,10.60311785194925,44
6 | 0,0,8.332793071329407,15
7 | 1,1,21.758550579956434,47
8 | 1,1,21.327448392585648,48
9 | 0,0,4.303175222038317,7
10 | 0,1,7.207283266459029,24
11 | 0,1,6.070862178525765,14
12 | 0,0,4.789457939363819,23
13 | 0,0,4.212921227504667,29
14 | 1,1,23.464904341428472,49
15 | 1,1,20.92600219571415,48
16 | 1,1,20.53715556398321,55
17 | 0,0,5.4312073991796685,16
18 | 1,1,14.342576565794099,47
19 | 1,1,21.555786241020208,52
20 | 1,1,15.988342151856092,30
21 | 1,1,11.124161856729355,29
22 | 1,1,19.30868453439712,56
23 | 0,0,-4.030355814517009,0
24 | 1,1,18.43053213846864,52
25 | 1,0,18.459255366409295,47
26 | 1,1,13.496527762740396,29
27 | 1,1,17.05937147227385,44
28 | 1,1,16.716497106468633,51
29 | 1,1,11.631451456633389,26
30 | 1,1,21.59907406097571,41
31 | 0,0,10.990871585259548,40
32 | 0,0,10.662402581085297,41
33 | 1,1,18.7122356433277,49
34 | 1,1,19.34971607037092,53
35 | 1,1,20.851634146711035,50
36 | 1,1,13.974703759301523,33
37 | 0,0,16.738970681321835,45
38 | 1,1,17.35338037885459,55
39 | 1,1,14.435092546669193,37
40 | 1,1,18.702688315989445,48
41 | 0,0,4.277110911803915,27
42 | 0,0,14.507148103884719,51
43 | 1,1,23.653800862521376,51
44 | 0,0,8.567537012901722,29
45 | 0,0,8.78531987243314,26
46 | 0,0,13.569290501083774,53
47 | 1,1,25.62129332283528,60
48 | 1,1,14.530000549011794,35
49 | 0,0,6.883272739837913,36
50 | 1,1,20.097209378373694,48
51 | 1,1,20.762651131416032,54
52 | 1,0,18.33307712322896,63
53 | 0,0,9.850827762530315,43
54 | 1,1,23.32180296971828,68
55 | 1,1,18.6917819575034,42
56 | 1,1,14.932152734747788,36
57 | 0,0,2.956746677935584,22
58 | 1,1,16.01711908876523,45
59 | 1,0,31.067416491153757,67
60 | 1,1,22.568000830990364,54
61 | 0,0,16.565257818798667,58
62 | 1,1,19.0837814111937,49
63 | 1,1,16.519551452099265,38
64 | 0,0,9.393171029539495,31
65 | 0,0,3.882968917283679,13
66 | 1,1,18.27549742570222,54
67 | 0,0,14.264939751887525,49
68 | 0,0,8.621892715657257,19
69 | 0,0,7.891128046822556,35
70 | 1,1,21.57854623631077,59
71 | 1,1,24.912477749048726,68
72 | 0,0,2.3519666762624283,7
73 | 1,1,26.650685824747153,72
74 | 1,1,27.242014029255106,73
75 | 0,0,9.820683378340444,35
76 | 1,1,20.456665254823022,48
77 | 1,1,18.879461912449205,41
78 | 1,1,22.497424230704993,41
79 | 0,0,2.9026514590083883,27
80 | 1,1,24.634592485036222,39
81 | 1,1,16.183497861445662,41
82 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/beyond-conf/cop-ex1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/beyond-conf/cop-ex1.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/beyond-conf/cop-ex2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/beyond-conf/cop-ex2.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/beyond-conf/cop.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/beyond-conf/cop.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/beyond-conf/selection.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/beyond-conf/selection.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/beyond-conf/shocked.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/beyond-conf/shocked.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/beyond-conf/y-pred.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/beyond-conf/y-pred.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/brave-and-true.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/brave-and-true.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/causal-graph/both_crap.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/causal-graph/both_crap.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/causal-graph/selection_bias.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/causal-graph/selection_bias.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/causal-graph/three_bias.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/causal-graph/three_bias.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/causal-model/customers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/causal-model/customers.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/causal-model/economists.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/causal-model/economists.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/causal-model/elast-partition.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/causal-model/elast-partition.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/causal-model/elast-split.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/causal-model/elast-split.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/causal-model/elasticity.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/causal-model/elasticity.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/causal-model/linear-fix.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/causal-model/linear-fix.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/causal-model/y-partition.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/causal-model/y-partition.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/causal-model/y-split.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/causal-model/y-split.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/curse-of-dimensionality.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/curse-of-dimensionality.jpg
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debias-ps/again.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debias-ps/again.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debias-ps/fear-no-man.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debias-ps/fear-no-man.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debias-ps/ps-experiment.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debias-ps/ps-experiment.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debiased-ml/avengers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debiased-ml/avengers.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debiased-ml/cf-pred.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debiased-ml/cf-pred.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debiased-ml/cross-prediction.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debiased-ml/cross-prediction.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debiased-ml/diagram.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debiased-ml/diagram.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debiased-ml/drown-ols.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debiased-ml/drown-ols.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debiased-ml/linear-aprox.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debiased-ml/linear-aprox.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debiased-ml/ml-problem.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debiased-ml/ml-problem.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debiased-ml/more-metrics.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debiased-ml/more-metrics.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debiased-ml/non-linear.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debiased-ml/non-linear.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/debiased-ml/non-sci.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/debiased-ml/non-sci.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/did-saga/awful.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/did-saga/awful.jpeg
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/did-saga/death.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/did-saga/death.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/did-saga/promise.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/did-saga/promise.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/did-saga/twfeworking.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/did-saga/twfeworking.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/diff-in-diff/cant-read.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/diff-in-diff/cant-read.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/diff-in-diff/non-parallel.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/diff-in-diff/non-parallel.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/diff-in-diff/secrets.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/diff-in-diff/secrets.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/doubly-robust/double.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/doubly-robust/double.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/dummy/heterosk.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/dummy/heterosk.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/dummy/you_little_shit.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/dummy/you_little_shit.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/evaluate-causal-models/ate_bins.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/evaluate-causal-models/ate_bins.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/evaluate-causal-models/cumm_elast.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/evaluate-causal-models/cumm_elast.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/evaluate-causal-models/cumm_gain.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/evaluate-causal-models/cumm_gain.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/evaluate-causal-models/sneak.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/evaluate-causal-models/sneak.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/evaluate-causal-models/uncivilised.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/evaluate-causal-models/uncivilised.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/fixed-effects/control-unit.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/fixed-effects/control-unit.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/fixed-effects/demeaned.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/fixed-effects/demeaned.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/fixed-effects/fe-graph.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/fixed-effects/fe-graph.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/fixed-effects/time-travel.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/fixed-effects/time-travel.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/graph-flow.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/graph-flow.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/hte-binary-outcome/logistic.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/hte-binary-outcome/logistic.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/hte-binary-outcome/slide-susan-athey.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/hte-binary-outcome/slide-susan-athey.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/industry-ml/cascade-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/industry-ml/cascade-1.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/industry-ml/cascade-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/industry-ml/cascade-2.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/industry-ml/cashflow-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/industry-ml/cashflow-1.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/industry-ml/cross-validation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/industry-ml/cross-validation.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/industry-ml/ml-flow.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/industry-ml/ml-flow.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/industry-ml/ml-meme.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/industry-ml/ml-meme.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/industry-ml/model-fit.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/industry-ml/model-fit.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/industry-ml/overfitting.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/industry-ml/overfitting.jpg
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/industry-ml/profit.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/industry-ml/profit.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/industry-ml/test.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/industry-ml/test.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/industry-ml/translation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/industry-ml/translation.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/intro/anatomy1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/intro/anatomy1.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/intro/anatomy2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/intro/anatomy2.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/intro/anatomy3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/intro/anatomy3.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/intro/anatomy4.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/intro/anatomy4.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/intro/potential_outcomes.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/intro/potential_outcomes.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/intro/tougher-up-cupcake1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/intro/tougher-up-cupcake1.jpg
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/iv/good-iv.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/iv/good-iv.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/iv/incomplete-files.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/iv/incomplete-files.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/iv/qob.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/iv/qob.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/iv/weak-iv.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/iv/weak-iv.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/late/always_takers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/late/always_takers.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/late/defiers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/late/defiers.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/late/double_index.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/late/double_index.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/linear-regression/confused_cat.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/linear-regression/confused_cat.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/linear-regression/you_vs.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/linear-regression/you_vs.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/matching/explain.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/matching/explain.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/matching/its-a-match.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/matching/its-a-match.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/matching/ubiquitous-ols.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/matching/ubiquitous-ols.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/meta-learners/learned-new-move.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/meta-learners/learned-new-move.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/meta-learners/not-great-not-terrible.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/meta-learners/not-great-not-terrible.jpeg
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/meta-learners/s-learner.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/meta-learners/s-learner.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/meta-learners/second-stage-x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/meta-learners/second-stage-x.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/meta-learners/t-learner-problem.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/meta-learners/t-learner-problem.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/meta-learners/t-learner.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/meta-learners/t-learner.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/meta-learners/t-vs-x-learner.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/meta-learners/t-vs-x-learner.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/meta-learners/x-learner.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/meta-learners/x-learner.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/meta-learners/zero-bias-s-learner.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/meta-learners/zero-bias-s-learner.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/orthogonal/athlas.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/orthogonal/athlas.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/orthogonal/kfold-cv.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/orthogonal/kfold-cv.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/orthogonal/nazare-confusa.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/orthogonal/nazare-confusa.jpg
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/plug-and-play-estimators/genious.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/plug-and-play-estimators/genious.jpeg
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/plug-and-play-estimators/infeasible.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/plug-and-play-estimators/infeasible.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/plug-and-play-estimators/linear-case.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/plug-and-play-estimators/linear-case.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/plug-and-play-estimators/non-linear-case.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/plug-and-play-estimators/non-linear-case.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/plug-and-play-estimators/second-estimator.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/plug-and-play-estimators/second-estimator.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/poetry.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/poetry.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/ps/balance.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/ps/balance.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/ps/bootstrap.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/ps/bootstrap.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/ps/iptw.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/ps/iptw.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/ps/ml-trap.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/ps/ml-trap.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/rct/indep.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/rct/indep.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/rct/lurking_bias.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/rct/lurking_bias.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/rct/science_dream.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/rct/science_dream.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/rdd/ols.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/rdd/ols.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/rdd/rdd.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/rdd/rdd.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/rdd/rdd_fuzzy.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/rdd/rdd_fuzzy.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/rdd/smooth.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/rdd/smooth.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/sc-conformal-inf/aug-data.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/sc-conformal-inf/aug-data.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/sc-conformal-inf/block-perm.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/sc-conformal-inf/block-perm.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/sdid/both-pills.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/sdid/both-pills.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/stats-review/ci_xkcd.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/stats-review/ci_xkcd.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/stats-review/normal_dist.jpeg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/stats-review/normal_dist.jpeg
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/stats-review/p_value.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/stats-review/p_value.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/synth-control/allways.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/synth-control/allways.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/synth-control/extrapolation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/synth-control/extrapolation.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/synth-control/out-of-sample.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/synth-control/out-of-sample.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/synth-control/regr_space.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/synth-control/regr_space.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/synth-control/regr_space_x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/synth-control/regr_space_x.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/synth-control/regr_time.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/synth-control/regr_time.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/synth-control/synth_img.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/synth-control/synth_img.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/flat-curves.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/flat-curves.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/graph-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/graph-1.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/graph-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/graph-2.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/graph-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/graph-3.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/horse-meme.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/horse-meme.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/ml-in-5.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/ml-in-5.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/model-opt.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/model-opt.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/opt-deriv.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/opt-deriv.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/pandas-magic.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/pandas-magic.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/partitions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/partitions.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/personalise.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/personalise.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/slice-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/slice-1.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/slice-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/slice-2.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/waiting-time.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/img/when-prediction-fails/waiting-time.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/online_classroom.csv:
--------------------------------------------------------------------------------
1 | gender,asian,black,hawaiian,hispanic,unknown,white,format_ol,format_blended,falsexam
2 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,63.299969999999995
3 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,79.96
4 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,83.37
5 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,90.01994
6 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,83.3
7 | 0,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,88.34996
8 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,78.4
9 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,90.0
10 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,88.31
11 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,73.37
12 | 1,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,83.3
13 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,86.64
14 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,83.37
15 | 1,,,,,,,0,1.0,76.64
16 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,89.98
17 | 0,,,,,,,0,1.0,66.64
18 | 1,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,80.05
19 | 1,,,,,,,0,0.0,78.38
20 | 0,,,,,,,0,1.0,86.64
21 | 1,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,74.97
22 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,63.35
23 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,71.73899999999999
24 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,81.65
25 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,88.33
26 | 0,0.0,1.0,0.0,0.0,0.0,0.0,1,0.0,59.94
27 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,80.04997
28 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,84.99
29 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,71.68
30 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,71.63
31 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,73.29899999999999
32 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,68.27
33 | 0,,,,,,,0,1.0,83.3
34 | 0,0.0,1.0,0.0,0.0,0.0,0.0,0,1.0,56.7
35 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,73.3
36 | 1,,,,,,,0,0.0,73.3
37 | 1,0.0,1.0,0.0,0.0,0.0,0.0,0,0.0,68.27
38 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,74.97
39 | 1,,,,,,,1,0.0,66.62
40 | 0,,,,,,,0,0.0,66.69
41 | 0,,,,,,,0,1.0,81.63
42 | 1,,,,,,,0,1.0,56.6
43 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,76.64
44 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,81.74
45 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,65.04
46 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,84.99
47 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,96.66
48 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,88.33
49 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,88.31
50 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,76.64
51 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,68.31
52 | 1,0.0,1.0,0.0,0.0,0.0,0.0,0,1.0,71.61
53 | 0,,,,,,,0,0.0,60.01
54 | 0,,,,,,,0,1.0,61.72
55 | 1,0.0,1.0,0.0,0.0,0.0,0.0,0,0.0,79.96
56 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,88.31
57 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,63.26
58 | 1,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,56.63998
59 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,83.32
60 | 0,,,,,,,1,0.0,78.31
61 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,69.95998
62 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,83.32
63 | 0,,,,,,,0,0.0,96.68
64 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,76.63996
65 | 0,,,,,,,1,0.0,78.36
66 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,65.04
67 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,84.99
68 | 0,0.0,1.0,0.0,0.0,0.0,0.0,0,1.0,68.34
69 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,88.38
70 | 1,0.0,0.0,0.0,1.0,0.0,0.0,0,0.0,79.959
71 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,63.35
72 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,70.08
73 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,68.34
74 | 1,,,,,,,0,1.0,90.0
75 | 0,,,,,,,1,0.0,86.66
76 | 0,,,,,,,0,1.0,59.99
77 | 0,0.0,1.0,0.0,0.0,0.0,0.0,0,1.0,55.07
78 | 1,,,,,,,0,1.0,88.33
79 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,81.76
80 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,66.69
81 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,85.04
82 | 1,,,,,,,1,0.0,90.0
83 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,73.3
84 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,73.28
85 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,73.3
86 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,91.69
87 | 1,0.0,0.0,1.0,0.0,0.0,0.0,0,1.0,69.98
88 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,83.32
89 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,54.949980000000004
90 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,70.0
91 | 1,,,,,,,0,1.0,51.66
92 | 1,,,,,,,0,0.0,81.63
93 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,81.72
94 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,90.05
95 | 0,,,,,,,1,0.0,51.67998
96 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,73.3
97 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,76.64
98 | 0,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,58.34
99 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,78.31
100 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,83.3
101 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,51.678999999999995
102 | 0,,,,,,,0,0.0,61.67997
103 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,76.66
104 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,56.70997
105 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,79.99996999999999
106 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,79.98
107 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,73.3
108 | 0,,,,,,,0,0.0,73.39
109 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,71.64996
110 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,79.97995
111 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,71.62997
112 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,79.98
113 | 0,,,,,,,0,0.0,86.64
114 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,83.39
115 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,64.979
116 | 1,,,,,,,1,0.0,93.39
117 | 0,,,,,,,0,0.0,70.03
118 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,84.97
119 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,89.98
120 | 1,,,,,,,1,0.0,56.65
121 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,78.34995
122 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,74.90996
123 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,78.31
124 | 0,0.0,0.0,1.0,0.0,0.0,0.0,1,0.0,81.7
125 | 0,0.0,1.0,0.0,0.0,0.0,0.0,0,0.0,66.64
126 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,86.64
127 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,79.98
128 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,81.65
129 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,84.97
130 | 0,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,76.71
131 | 0,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,60.08
132 | 0,,,,,,,0,1.0,91.65
133 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,75.06
134 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,80.05
135 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,81.67
136 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,86.65994
137 | 1,,,,,,,0,1.0,83.3
138 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,74.95997
139 | 0,,,,,,,0,0.0,69.95996
140 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,88.32994000000001
141 | 0,,,,,,,1,0.0,64.95
142 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,66.64
143 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,76.64
144 | 0,,,,,,,1,0.0,74.99
145 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,59.94
146 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,75.02
147 | 0,,,,,,,1,0.0,58.31
148 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,76.76
149 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,71.68
150 | 1,,,,,,,0,0.0,89.98
151 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,84.99
152 | 1,,,,,,,0,0.0,81.63
153 | 1,0.0,1.0,0.0,0.0,0.0,0.0,0,1.0,83.32
154 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,73.31997
155 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,73.27996999999999
156 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,88.38
157 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,68.36
158 | 0,,,,,,,0,1.0,68.29
159 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,71.63
160 | 1,,,,,,,0,1.0,71.63
161 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,91.65
162 | 1,,,,,,,0,0.0,68.31
163 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,75.02
164 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,73.44
165 | 0,0.0,1.0,0.0,0.0,0.0,0.0,0,1.0,50.1
166 | 1,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,63.31998000000001
167 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,70.03
168 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,73.32
169 | 0,,,,,,,1,0.0,83.3
170 | 1,,,,,,,1,0.0,75.06
171 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,86.66
172 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,68.32996
173 | 0,,,,,,,0,1.0,78.329
174 | 0,,,,,,,1,0.0,68.31
175 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,75.02
176 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,78.33
177 | 0,,,,,,,0,1.0,73.28
178 | 0,,,,,,,0,0.0,74.97
179 | 0,0.0,1.0,0.0,0.0,0.0,0.0,0,1.0,70.05
180 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,76.63996999999999
181 | 1,,,,,,,0,1.0,84.99
182 | 0,,,,,,,0,1.0,71.74
183 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,88.33
184 | 0,,,,,,,1,0.0,63.28
185 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,91.74
186 | 1,,,,,,,1,0.0,0.0
187 | 1,0.0,1.0,0.0,0.0,0.0,0.0,0,0.0,79.98
188 | 1,,,,,,,0,0.0,69.94
189 | 1,0.0,0.0,0.0,1.0,0.0,0.0,0,1.0,63.27997
190 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,84.99
191 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,73.3
192 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,88.33
193 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,83.29997
194 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,84.97
195 | 1,,,,,,,1,0.0,81.65
196 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,83.3
197 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,89.99994000000001
198 | 0,0.0,0.0,0.0,1.0,0.0,0.0,1,0.0,79.93996
199 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,78.29
200 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,86.64
201 | 1,,,,,,,0,0.0,91.67
202 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,76.75
203 | 0,0.0,1.0,0.0,0.0,0.0,0.0,0,1.0,73.28
204 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,70.03
205 | 1,,,,,,,0,0.0,83.32
206 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,90.0
207 | 1,,,,,,,0,1.0,83.32
208 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,83.3
209 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,78.31
210 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,88.33
211 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,83.37
212 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,86.66
213 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,86.66
214 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,84.99
215 | 1,0.0,1.0,0.0,0.0,0.0,0.0,0,0.0,88.33
216 | 1,0.0,0.0,1.0,0.0,0.0,0.0,0,1.0,88.33
217 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,81.65
218 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,78.229
219 | 1,,,,,,,0,1.0,76.73
220 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,89.98
221 | 1,,,,,,,0,1.0,74.95
222 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,75.02
223 | 0,,,,,,,0,1.0,56.759980000000006
224 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,81.669
225 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,58.27
226 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,81.63
227 | 0,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,58.299969999999995
228 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,78.38
229 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,81.72
230 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,84.99
231 | 0,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,81.67
232 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,86.66
233 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,88.31
234 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,65.02
235 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,80.0
236 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,78.37995
237 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,86.66
238 | 1,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,83.34
239 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,84.99
240 | 0,,,,,,,0,1.0,89.98
241 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,66.69
242 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,83.37
243 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,88.33
244 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,86.64
245 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,90.0
246 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,84.97
247 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,73.32
248 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,85.01
249 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,75.06
250 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,81.65
251 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,68.27
252 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,83.34
253 | 1,,,,,,,1,0.0,80.05
254 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,86.64
255 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,81.65
256 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,68.36
257 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,80.0
258 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,73.35
259 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,68.36
260 | 0,,,,,,,0,1.0,61.63
261 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,56.71
262 | 1,,,,,,,0,1.0,78.31
263 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,90.0
264 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,70.03
265 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,68.28996
266 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,76.73
267 | 1,,,,,,,1,0.0,86.65996
268 | 0,0.0,0.0,0.0,0.0,1.0,0.0,0,1.0,48.33998
269 | 1,0.0,1.0,0.0,0.0,0.0,0.0,1,0.0,81.7
270 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,81.72
271 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,56.59996999999999
272 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,81.65
273 | 1,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,94.99
274 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,59.92
275 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,61.68
276 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,86.64
277 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,86.66
278 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,88.31
279 | 1,,,,,,,1,0.0,81.7
280 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,68.34
281 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,78.4
282 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,74.97
283 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,83.32
284 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,84.99
285 | 1,0.0,0.0,0.0,1.0,0.0,0.0,1,0.0,76.62
286 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,68.36
287 | 1,,,,,,,0,1.0,76.62
288 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,66.62
289 | 0,,,,,,,1,0.0,74.90997
290 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,79.96
291 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,88.33
292 | 0,,,,,,,0,1.0,76.71
293 | 1,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,65.02
294 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,83.32
295 | 1,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,86.75
296 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,83.44
297 | 1,0.0,1.0,0.0,0.0,0.0,0.0,0,0.0,95.01
298 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,69.99996999999999
299 | 0,1.0,0.0,0.0,0.0,0.0,0.0,1,0.0,86.68
300 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,66.65899999999999
301 | 1,,,,,,,0,1.0,76.62
302 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,71.61
303 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,90.0
304 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,60.01
305 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,1.0,94.99
306 | 0,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,83.3
307 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,83.32
308 | 1,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,70.0
309 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,73.37
310 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,83.39
311 | 0,0.0,0.0,0.0,0.0,0.0,1.0,1,0.0,0.0
312 | 1,,,,,,,0,1.0,69.96
313 | 1,1.0,0.0,0.0,0.0,0.0,0.0,0,0.0,86.71
314 | 1,,,,,,,0,0.0,90.0
315 | 0,0.0,1.0,0.0,0.0,0.0,0.0,0,0.0,61.68
316 | 0,,,,,,,0,0.0,73.37
317 | 1,,,,,,,0,1.0,88.31
318 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,0.0,91.65
319 | 1,,,,,,,0,0.0,84.99
320 | 0,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,68.36
321 | 1,,,,,,,1,0.0,70.05
322 | 0,,,,,,,1,0.0,66.69
323 | 1,,,,,,,1,0.0,83.29997
324 | 1,0.0,0.0,0.0,0.0,0.0,1.0,0,1.0,96.68
325 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/sheepskin.csv:
--------------------------------------------------------------------------------
1 | minscore,person_years,avgearnings,receivehsd,n
2 | -30.0,24.0,11845.086,0.41666666,12
3 | -29.0,104.0,9205.679,0.38709676,31
4 | -28.0,146.0,8407.745,0.3181818,44
5 | -27.0,137.0,11114.087,0.37777779,45
6 | -26.0,227.0,10814.624,0.30666667,75
7 | -25.0,341.0,11666.925,0.3018868,106
8 | -24.0,419.0,11336.057,0.38059703,134
9 | -23.0,479.0,10719.352,0.275,160
10 | -22.0,550.0,10684.487,0.26666668,180
11 | -21.0,756.0,10342.762,0.23673469,245
12 | -20.0,839.0,11012.137,0.26335877,262
13 | -19.0,1029.0,11721.943,0.28783384,337
14 | -18.0,1175.0,11712.557,0.28350514,388
15 | -17.0,1424.0,10564.079,0.26444444,450
16 | -16.0,1549.0,10737.608,0.29554656,494
17 | -15.0,1854.0,11423.25,0.30993152,584
18 | -14.0,2092.0,11717.597,0.30225563,665
19 | -13.0,2443.0,11931.299,0.28188777,784
20 | -12.0,2924.0,12174.198,0.28349945,903
21 | -11.0,3058.0,11953.884,0.29771784,964
22 | -10.0,3505.0,12705.653,0.30483273,1076
23 | -9.0,3986.0,12060.475,0.34561834,1221
24 | -8.0,4394.0,12214.287,0.34626648,1366
25 | -7.0,4931.0,13038.381,0.35818908,1502
26 | -6.0,5574.0,12683.077,0.37507418,1685
27 | -5.0,5706.0,13179.544,0.39025787,1745
28 | -4.0,6530.0,13083.809,0.39937598,1923
29 | -3.0,6438.0,13143.415,0.4405812,1927
30 | -2.0,6869.0,13983.55,0.45413533,1995
31 | -1.0,6667.0,13732.518,0.46518824,1939
32 | 0.0,6458.0,14063.868,0.916222,1874
33 | 1.0,6548.0,14257.279,0.9062334,1877
34 | 2.0,5850.0,13829.307,0.9275449,1670
35 | 3.0,5484.0,14631.656,0.9151671,1556
36 | 4.0,5219.0,14847.078,0.9280677,1418
37 | 5.0,4654.0,14983.756,0.9230159,1260
38 | 6.0,3874.0,14936.146,0.9325736,1053
39 | 7.0,3392.0,15824.566,0.93134,903
40 | 8.0,2653.0,15175.247,0.9410072,695
41 | 9.0,2220.0,15253.268,0.9256637,565
42 | 10.0,1869.0,16395.117,0.93432206,472
43 | 11.0,1466.0,15541.161,0.9222222,360
44 | 12.0,1052.0,15487.525,0.96564883,262
45 | 13.0,761.0,17235.902,0.9673913,184
46 | 14.0,474.0,16985.027,0.9357798,109
47 | 15.0,308.0,17854.611,0.87323946,71
48 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/trainees.csv:
--------------------------------------------------------------------------------
1 | unit,trainees,age,earnings
2 | 1,1,28,17700
3 | 2,1,34,10200
4 | 3,1,29,14400
5 | 4,1,25,20800
6 | 5,1,29,6100
7 | 6,1,23,28600
8 | 7,1,33,21900
9 | 8,1,27,28800
10 | 9,1,31,20300
11 | 10,1,26,28100
12 | 11,1,25,9400
13 | 12,1,27,14300
14 | 13,1,29,12500
15 | 14,1,24,19700
16 | 15,1,25,10100
17 | 16,1,43,10700
18 | 17,1,28,11500
19 | 18,1,27,10700
20 | 19,1,28,16300
21 | 20,0,43,20900
22 | 21,0,50,31000
23 | 22,0,30,21000
24 | 23,0,27,9300
25 | 24,0,54,41100
26 | 25,0,48,29800
27 | 26,0,39,42000
28 | 27,0,28,8800
29 | 28,0,24,25500
30 | 29,0,33,15500
31 | 31,0,26,400
32 | 32,0,31,26600
33 | 33,0,26,16500
34 | 34,0,34,24200
35 | 35,0,25,23300
36 | 36,0,24,9700
37 | 37,0,29,6200
38 | 38,0,35,30200
39 | 39,0,32,17800
40 | 40,0,23,9500
41 | 41,0,32,25900
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/graph-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/graph-1.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/graph-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/graph-2.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/graph-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/graph-3.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/horse-meme.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/horse-meme.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/ml-in-5.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/ml-in-5.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/model-opt.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/model-opt.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/opt-deriv.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/opt-deriv.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/pandas-magic.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/pandas-magic.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/partitions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/partitions.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/personalise.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/personalise.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/slice-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/slice-1.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/slice-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/slice-2.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/data/when-prediction-fails/waiting-time.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/matheusfacure/python-causality-handbook/ebc1cec295d4b71324bc377eebf72fcce0ef1b30/causal-inference-for-the-brave-and-true/data/when-prediction-fails/waiting-time.png
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/landing-page.md:
--------------------------------------------------------------------------------
1 | # Causal Inference for The Brave and True
2 |
3 | 
4 |
5 | A light-hearted yet rigorous approach to learning impact estimation and sensitivity analysis. Everything in Python and with as many memes as I could find.
6 |
7 | Part I of the book contains core concepts and models for causal inference. You will learn how to represent causal questions with potential outcome notation, learn about causal graphs, what is bias and how to deal with it. Most of the content here is well established. It's an amalgamation of materials I've found on books, university curriculums and online courses. You can think of Part I as the solid and safe foundation to your causal inquiries.
8 |
9 | Part II (WIP) contains modern development and applications of causal inference to the (mostly tech) industry. While Part I focuses mostly on identifying average treatment effects, Part II takes a shift to personalization and heterogeneous effect estimating with CATE models. Most of what is written there was taken from my personal experience and is by no means well established science. It's much more experimental and subject to change, afterall, I, too, am learning.
10 |
11 |
12 | ## Acknowledgments
13 |
14 | I like to think of this entire series as a tribute to Joshua Angrist, Alberto Abadie and Christopher Walters for their amazing Econometrics class. Most of the ideas from Part I are taken from their classes at the American Economic Association. Watching them was what kept me sane during the tough year of 2020.
15 | * [Cross-Section Econometrics](https://www.aeaweb.org/conference/cont-ed/2017-webcasts)
16 | * [Mastering Mostly Harmless Econometrics](https://www.aeaweb.org/conference/cont-ed/2020-webcasts)
17 |
18 | I'll also like to reference the amazing books from Angrist. They have shown me that Econometrics, or 'Metrics as they call it, is not only extremely useful but also profoundly fun.
19 |
20 | * [Mostly Harmless Econometrics](https://www.mostlyharmlesseconometrics.com/)
21 | * [Mastering 'Metrics](https://www.masteringmetrics.com/)
22 |
23 | My final reference is Miguel Hernan and Jamie Robins' book. It has been my trustworthy companion in the most thorny causal questions I had to answer.
24 |
25 | * [Causal Inference Book](https://www.hsph.harvard.edu/miguel-hernan/causal-inference-book/)
26 |
27 |
28 | ## Contribute
29 |
30 | Causal Inference for the Brave and True is an open-source material on causal inference, the statistics of science. It uses only free software, based in Python. Its goal is to be accessible monetarily and intellectually.
31 | If you found this book valuable and you want to support it, please go to [Patreon](https://www.patreon.com/causal_inference_for_the_brave_and_true). If you are not ready to contribute financially, you can also help by fixing typos, suggesting edits or giving feedback on passages you didn't understand. Just go to the book's repository and [open an issue](https://github.com/matheusfacure/python-causality-handbook/issues). Finally, if you liked this content, please share it with others who might find it useful and give it a [star on GitHub](https://github.com/matheusfacure/python-causality-handbook/stargazers).
32 |
33 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/nb18.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | from matplotlib import pyplot as plt
4 | from toolz import merge
5 | from sklearn.preprocessing import LabelEncoder
6 |
7 | def ltv_with_coupons(coupons=None):
8 |
9 | n = 10000
10 | t = 30
11 |
12 | np.random.seed(12)
13 |
14 | age = 18 + np.random.poisson(10, n)
15 | income = 500+np.random.exponential(2000, size=n).astype(int)
16 | region = np.random.choice(np.random.lognormal(4, size=50), size=n)
17 |
18 | if coupons is None:
19 | coupons = np.clip(np.random.normal((age-18), 0.01, size=n) // 5 * 5, 0, 15)
20 |
21 | assert len(coupons) == n
22 |
23 | np.random.seed(12)
24 |
25 | # treatment effect on freq
26 | freq_mu = 0.5*coupons * age + age
27 | freq_mu = (freq_mu - 150) / 30
28 | freq_mu += 2
29 |
30 |
31 | freq = np.random.lognormal(freq_mu.astype(int))
32 |
33 | churn = np.random.poisson((income-500)/2000 + 22, n)
34 |
35 | ones = np.ones((n, t))
36 | alive = (np.cumsum(ones, axis=1) <= churn.reshape(n, 1)).astype(int)
37 | buy = np.random.binomial(1, ((1/(freq+1)).reshape(n, 1) * ones))
38 |
39 | cacq = -1*abs(np.random.normal(region, 2, size=n).astype(int))
40 |
41 | # treatment effect on transactions
42 | np.random.seed(12)
43 | transactions = np.random.lognormal(2, size=(n, t)).astype(int) * buy * alive
44 |
45 | transaction_mu = 0.1 + (((income - 500) / 900) * (coupons/8)) + coupons/9
46 | transaction_mu = np.clip(transaction_mu, 0, 5)
47 | transaction_mu = np.tile(transaction_mu.reshape(-1,1), t)
48 |
49 | np.random.seed(12)
50 | transactions = np.random.lognormal(transaction_mu, size=(n, t)).astype(int) * buy * alive
51 |
52 | data = pd.DataFrame(merge({"customer_id": range(n), "cacq":cacq},
53 | {f"day_{day}": trans
54 | for day, trans in enumerate(transactions.T)}))
55 |
56 | encoded = {value:index for index, value in
57 | enumerate(np.random.permutation(np.unique(region)))}
58 |
59 | customer_features = pd.DataFrame(dict(customer_id=range(n),
60 | region=region,
61 | income=income,
62 | coupons=coupons,
63 | age=age)).replace({"region":encoded}).astype(int)
64 |
65 | return data, customer_features
66 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/nb21.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | from matplotlib import pyplot as plt
4 | from toolz import merge, curry
5 | from sklearn.preprocessing import LabelEncoder
6 |
7 | @curry
8 | def elast(data, y, t):
9 | return (np.sum((data[t] - data[t].mean())*(data[y] - data[y].mean())) /
10 | np.sum((data[t] - data[t].mean())**2))
11 |
12 |
13 | def elast_ci(df, y, t, z=1.96):
14 | n = df.shape[0]
15 | t_bar = df[t].mean()
16 | beta1 = elast(df, y, t)
17 | beta0 = df[y].mean() - beta1 * t_bar
18 | e = df[y] - (beta0 + beta1*df[t])
19 | se = np.sqrt(((1/(n-2))*np.sum(e**2))/np.sum((df[t]-t_bar)**2))
20 | return np.array([beta1 - z*se, beta1 + z*se])
21 |
22 |
23 | def cumulative_elast_curve_ci(dataset, prediction, y, t, min_periods=30, steps=100):
24 | size = dataset.shape[0]
25 | ordered_df = dataset.sort_values(prediction, ascending=False).reset_index(drop=True)
26 | n_rows = list(range(min_periods, size, size // steps)) + [size]
27 |
28 | return np.array([elast_ci(ordered_df.head(rows), y, t) for rows in n_rows])
29 |
30 |
31 | def cumulative_gain_ci(dataset, prediction, y, t, min_periods=30, steps=100):
32 | size = dataset.shape[0]
33 | ordered_df = dataset.sort_values(prediction, ascending=False).reset_index(drop=True)
34 | n_rows = list(range(min_periods, size, size // steps)) + [size]
35 | return np.array([elast_ci(ordered_df.head(rows), y, t) * (rows/size) for rows in n_rows])
36 |
37 |
38 | def cumulative_gain(dataset, prediction, y, t, min_periods=30, steps=100):
39 | size = dataset.shape[0]
40 | ordered_df = dataset.sort_values(prediction, ascending=False).reset_index(drop=True)
41 | n_rows = list(range(min_periods, size, size // steps)) + [size]
42 | return np.array([elast(ordered_df.head(rows), y, t) * (rows/size) for rows in n_rows])
43 |
--------------------------------------------------------------------------------
/causal-inference-for-the-brave-and-true/sc.py:
--------------------------------------------------------------------------------
1 | from sklearn.base import BaseEstimator, RegressorMixin
2 | from sklearn.utils.validation import check_X_y, check_array, check_is_fitted
3 | import cvxpy as cp
4 |
5 | class SyntheticControl(BaseEstimator, RegressorMixin):
6 |
7 | def __init__(self,):
8 | pass
9 |
10 | def fit(self, X, y):
11 |
12 | X, y = check_X_y(X, y)
13 |
14 | w = cp.Variable(X.shape[1])
15 | objective = cp.Minimize(cp.sum_squares(X@w - y))
16 |
17 | constraints = [cp.sum(w) == 1, w >= 0]
18 |
19 | problem = cp.Problem(objective, constraints)
20 | problem.solve(verbose=False)
21 |
22 | self.X_ = X
23 | self.y_ = y
24 | self.w_ = w.value
25 |
26 | self.is_fitted_ = True
27 | return self
28 |
29 |
30 | def predict(self, X):
31 |
32 | check_is_fitted(self)
33 | X = check_array(X)
34 |
35 | return X @ self.w_
--------------------------------------------------------------------------------