├── exps ├── .gitignore ├── cmnist_label_noise_sweep.sh └── cmnist_with_specified_label_noise.sh ├── opt_env ├── .gitignore ├── cmnist_results │ ├── .gitignore │ └── acc_table.py ├── utils │ ├── .gitignore │ ├── model_utils.py │ ├── env_utils.py │ └── opt_utils.py └── irm_cmnist.py ├── InvariantRiskMinimization ├── code │ ├── colored_mnist │ │ ├── .gitignore │ │ ├── optimize_envs.sh │ │ └── main_optenv.py │ ├── experiment_synthetic │ │ ├── .gitignore │ │ ├── synthetic_results.pt │ │ ├── run_sems.sh │ │ ├── sem.py │ │ ├── plot.py │ │ ├── main.py │ │ └── models.py │ └── figure_1 │ │ └── penalties.py ├── CODE_OF_CONDUCT.md ├── README.md ├── CONTRIBUTING.md └── LICENSE ├── .gitignore ├── requirements.txt ├── README.md ├── LICENSE └── notebooks └── sem_results.ipynb /exps/.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints/* 2 | -------------------------------------------------------------------------------- /opt_env/.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | __pycache__/* 3 | -------------------------------------------------------------------------------- /opt_env/cmnist_results/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/* 2 | -------------------------------------------------------------------------------- /opt_env/utils/.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | __pycache__/* 3 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/code/colored_mnist/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/* 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | __pycache__/* 3 | plots/* 4 | results/* 5 | slurm_output/* 6 | old_scripts/* 7 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/code/experiment_synthetic/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/* 2 | slurm_output/* 3 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/code/experiment_synthetic/synthetic_results.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ecreager/eiil/HEAD/InvariantRiskMinimization/code/experiment_synthetic/synthetic_results.pt -------------------------------------------------------------------------------- /InvariantRiskMinimization/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please read the [full text](https://code.fb.com/codeofconduct/)) so that you can understand what actions will and will not be tolerated. 4 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/code/colored_mnist/optimize_envs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "EIIL + IRM" 4 | python -u main_optenv.py \ 5 | --hidden_dim=390 \ 6 | --l2_regularizer_weight=0.00110794568 \ 7 | --lr=0.0004898536566546834 \ 8 | --penalty_anneal_iters=190 \ 9 | --penalty_weight=191257.18613115903 \ 10 | --steps=501 \ 11 | --n_restarts=10 \ 12 | --eiil \ 13 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/README.md: -------------------------------------------------------------------------------- 1 | # Code repository for Invariant Risk Minimization 2 | 3 | Source code for [the paper](https://arxiv.org/abs/1907.02893v1): 4 | 5 | ``` 6 | @article{InvariantRiskMinimization, 7 | title={Invariant Risk Minimization}, 8 | author={Arjovsky, Martin and Bottou, L{\'e}on and Gulrajani, Ishaan and Lopez-Paz, David}, 9 | journal={arXiv}, 10 | year={2019} 11 | } 12 | ``` 13 | 14 | Repository licensed under [LICENSE](LICENSE). 15 | -------------------------------------------------------------------------------- /exps/cmnist_label_noise_sweep.sh: -------------------------------------------------------------------------------- 1 | ./exps/cmnist_with_specified_label_noise.sh 0.00 2 | ./exps/cmnist_with_specified_label_noise.sh 0.05 3 | ./exps/cmnist_with_specified_label_noise.sh 0.10 4 | ./exps/cmnist_with_specified_label_noise.sh 0.15 5 | ./exps/cmnist_with_specified_label_noise.sh 0.20 6 | ./exps/cmnist_with_specified_label_noise.sh 0.25 7 | ./exps/cmnist_with_specified_label_noise.sh 0.30 8 | ./exps/cmnist_with_specified_label_noise.sh 0.35 9 | ./exps/cmnist_with_specified_label_noise.sh 0.40 10 | ./exps/cmnist_with_specified_label_noise.sh 0.45 11 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/code/experiment_synthetic/run_sems.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo RUNNING ALPHA SWEEP EXPER 3 | RESULTS_DIR=/scratch/gobi1/creager/opt_env/run_sems_alpha_sweep/$RANDOM$RANDOM 4 | echo $RESULTS_DIR 5 | SETUP_HETERO=2 6 | N_REPS=5 7 | mkdir -p $RESULTS_DIR 8 | echo results found here 9 | echo $RESULTS_DIR 10 | echo RUNNING ALL METHODS FOR $N_REPS RESTARTS IN HETEROSKEDASTIC SETTING 11 | python -u main.py --verbose 1 --methods "EIIL,ERM,ICP,IRM" --setup_hetero $SETUP_HETERO --results_dir $RESULTS_DIR --n_reps $N_REPS 12 | for alpha in 0.0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0 13 | do 14 | echo RUNNING EIIL WITH $alpha SPURIOUS REFERENCE CLASSIFIER FOR $N_REPS RESTARTS IN HETEROSKEDASTIC SETTING 15 | python -u main.py --verbose 1 --methods "EIIL" --setup_hetero $SETUP_HETERO --results_dir $RESULTS_DIR --eiil_ref_alpha $alpha --n_reps $N_REPS 16 | done 17 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to InvariantRiskMinimization 2 | 3 | We want to make contributing to this project as easy and transparent as 4 | possible. 5 | 6 | ## Pull Requests 7 | We actively welcome your pull requests. 8 | 9 | 1. Fork the repo and create your branch from `master`. 10 | 2. If you've added code that should be tested, add tests. 11 | 3. If you've changed APIs, update the documentation. 12 | 4. Ensure the test suite passes. 13 | 5. Make sure your code lints. 14 | 6. If you haven't already, complete the Contributor License Agreement ("CLA"). 15 | 16 | ## Contributor License Agreement ("CLA") 17 | In order to accept your pull request, we need you to submit a CLA. You only need 18 | to do this once to work on any of Facebook's open source projects. 19 | 20 | Complete your CLA here: 21 | 22 | ## Issues 23 | We use GitHub issues to track public bugs. Please ensure your description is 24 | clear and has sufficient instructions to be able to reproduce the issue. 25 | 26 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 27 | disclosure of security bugs. In those cases, please go through the process 28 | outlined on that page and do not file a public issue. 29 | 30 | ## License 31 | By contributing to GradientEpisodicMemory, you agree that your contributions 32 | will be licensed under the LICENSE file in the root directory of this source 33 | tree. 34 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | argon2-cffi==20.1.0 2 | attrs==20.1.0 3 | backcall==0.2.0 4 | bleach==3.1.5 5 | certifi==2020.6.20 6 | cffi==1.14.2 7 | cycler==0.10.0 8 | decorator==4.4.2 9 | defusedxml==0.6.0 10 | entrypoints==0.3 11 | future==0.18.2 12 | importlib-metadata==1.7.0 13 | ipykernel==5.3.4 14 | ipython==7.16.1 15 | ipython-genutils==0.2.0 16 | ipywidgets==7.5.1 17 | jedi==0.17.2 18 | Jinja2==2.11.2 19 | joblib==0.16.0 20 | jsonschema==3.2.0 21 | jupyter==1.0.0 22 | jupyter-client==6.1.7 23 | jupyter-console==6.1.0 24 | jupyter-core==4.6.3 25 | jupyter-http-over-ws==0.0.8 26 | kiwisolver==1.2.0 27 | MarkupSafe==1.1.1 28 | matplotlib==3.3.1 29 | mistune==0.8.4 30 | nbconvert==5.6.1 31 | nbformat==5.0.7 32 | notebook==6.1.3 33 | numpy==1.19.1 34 | packaging==20.4 35 | pandas==1.1.1 36 | pandocfilters==1.4.2 37 | parso==0.7.1 38 | pexpect==4.8.0 39 | pickleshare==0.7.5 40 | Pillow==7.2.0 41 | prometheus-client==0.8.0 42 | prompt-toolkit==3.0.6 43 | ptyprocess==0.6.0 44 | pycparser==2.20 45 | Pygments==2.6.1 46 | pyparsing==2.4.7 47 | pyrsistent==0.16.0 48 | python-dateutil==2.8.1 49 | pytz==2020.1 50 | pyzmq==19.0.2 51 | qtconsole==4.7.6 52 | QtPy==1.9.0 53 | scikit-learn==0.23.2 54 | scipy==1.5.2 55 | seaborn==0.10.1 56 | Send2Trash==1.5.0 57 | six==1.15.0 58 | terminado==0.8.3 59 | testpath==0.4.4 60 | threadpoolctl==2.1.0 61 | torch==1.6.0 62 | torchvision==0.7.0 63 | tornado==6.0.4 64 | tqdm==4.48.2 65 | traitlets==4.3.3 66 | wcwidth==0.2.5 67 | webencodings==0.5.1 68 | widgetsnbextension==3.5.1 69 | zipp==3.1.0 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Environment Inference for Invariant Learning 2 | This code accompanies the paper [Environment Inference for Invariant Learning](https://arxiv.org/abs/2010.07249), which appears at ICML 2021. 3 | 4 | Thanks to my wonderful co-authors [Jörn-Henrik Jacobsen](https://github.com/jhjacobsen/) and [Richard Zemel](https://www.cs.toronto.edu/~zemel/inquiry/home.php). 5 | 6 | The InvariantRiskMinimization subdirectory is modified from https://github.com/facebookresearch/InvariantRiskMinimization, and has its own license. 7 | 8 | ## Reproducing paper results 9 | 10 | ### Sythetic data 11 | To produce results 12 | ``` 13 | cd InvariantRiskMinimization/code/experiment_synthetic/ 14 | ./run_sems.sh 15 | ``` 16 | To analyze results 17 | ``` 18 | noteooks/sem_results.ipynb 19 | ``` 20 | 21 | ### Color MNIST 22 | To produce results 23 | ``` 24 | ./exps/cmnist_label_noise_sweep.sh 25 | ``` 26 | To analyze results 27 | ``` 28 | notebooks/plot_cmnist_label_noise_sweep.ipynb 29 | ``` 30 | As an alternative, `InvariantRiskMinimization/code/colored_mnist/optimize_envs.sh` also runs EIIL+IRM on CMNIST with 25% label noise (the default from the IRM paper). 31 | 32 | ## Citing this work 33 | If you find this code to your research useful please consider citing our workshop paper using the following bibtex entry 34 | ``` 35 | @inproceedings{creager21environment, 36 | title={Environment Inference for Invariant Learning}, 37 | author={Creager, Elliot and Jacobsen, J{\"o}rn-Henrik and Zemel, Richard}, 38 | booktitle={International Conference on Machine Learning}, 39 | year={2021}, 40 | } 41 | 42 | ``` 43 | -------------------------------------------------------------------------------- /opt_env/utils/model_utils.py: -------------------------------------------------------------------------------- 1 | """Model utils.""" 2 | import os 3 | import pickle 4 | 5 | import torch 6 | from torch import nn 7 | 8 | 9 | class ColorBasedClassifier(nn.Module): 10 | LOG_CONFIDENCE = .5 11 | def forward(self, input): 12 | # estimate color based on arg max over the two channels 13 | input = input.sum((-1, -2)) # add pixels to get per-channel sums 14 | if len(input) == 0: # hack to handle size zero batches 15 | hard_prediction = torch.zeros(0, 1).to(input.device) 16 | else: 17 | hard_prediction = torch.argmax(input, -1, keepdim=True) # in {0, 1} 18 | hard_prediction = hard_prediction.float() 19 | hard_prediction = hard_prediction * 2 - 1. # in {-1, 1} 20 | return hard_prediction * self.LOG_CONFIDENCE 21 | 22 | 23 | def load_mlp(results_dir=None, flags=None, basename='params.p'): 24 | if flags is None: 25 | assert results_dir is not None, "flags and results_dir cannot both be None." 26 | flags = pickle.load(open(os.path.join(results_dir, 'flags.p'), 'rb')) 27 | 28 | class MLP(nn.Module): 29 | def __init__(self): 30 | super(MLP, self).__init__() 31 | if flags.grayscale_model: 32 | lin1 = nn.Linear(14 * 14, flags.hidden_dim) 33 | else: 34 | lin1 = nn.Linear(2 * 14 * 14, flags.hidden_dim) 35 | lin2 = nn.Linear(flags.hidden_dim, flags.hidden_dim) 36 | lin3 = nn.Linear(flags.hidden_dim, 1) 37 | for lin in [lin1, lin2, lin3]: 38 | nn.init.xavier_uniform_(lin.weight) 39 | nn.init.zeros_(lin.bias) 40 | self._main = nn.Sequential(lin1, nn.ReLU(True), lin2, nn.ReLU(True), lin3) 41 | def forward(self, input): 42 | if flags.grayscale_model: 43 | out = input.view(input.shape[0], 2, 14 * 14).sum(dim=1) 44 | else: 45 | out = input.view(input.shape[0], 2 * 14 * 14) 46 | out = self._main(out) 47 | return out 48 | 49 | mlp = MLP() 50 | if torch.cuda.is_available(): 51 | mlp = mlp.cuda() 52 | 53 | if results_dir is not None: 54 | mlp.load_state_dict(torch.load(os.path.join(results_dir, basename))) 55 | print('Model params loaded from %s.' % results_dir) 56 | else: 57 | print('Model built with randomly initialized parameters.') 58 | mlp.eval() 59 | 60 | return mlp 61 | 62 | 63 | -------------------------------------------------------------------------------- /opt_env/cmnist_results/acc_table.py: -------------------------------------------------------------------------------- 1 | """Build results table for CMNIST experiment.""" 2 | import argparse 3 | import os 4 | import pickle 5 | 6 | import numpy as np 7 | import pandas as pd 8 | 9 | 10 | def load_results(dirname): 11 | return pickle.load(open(os.path.join(dirname, 'metrics.p'), 'rb')) 12 | 13 | def main(flags): 14 | # load results from disk 15 | results = dict( 16 | erm=load_results(flags.erm_results_dir), 17 | irm=load_results(flags.irm_results_dir), 18 | eiil=load_results(flags.eiil_results_dir), 19 | eiil_cb=load_results(flags.eiil_cb_results_dir), 20 | cb=load_results(flags.cb_results_dir), 21 | gray=load_results(flags.gray_results_dir), 22 | ) 23 | num_methods = len(results) 24 | def mean_plus_minus_std(x): 25 | """Format list as its mean plus minus one std dev.""" 26 | return r"""%.1f $\pm$ %.1f""" % (100. * np.mean(x), 100. * np.std(x)) 27 | results = pd.DataFrame.from_dict(results) 28 | results = results.T 29 | results_tex = results.to_latex( 30 | formatters=[mean_plus_minus_std, ] * 2, 31 | escape=False 32 | ) 33 | print(results_tex) 34 | if not os.path.exists(flags.results_dir): 35 | os.makedirs(flags.results_dir) 36 | print(results_tex, file=open(os.path.join(flags.results_dir, 'results.tex'), 'w')) 37 | 38 | if __name__ == '__main__': 39 | parser = argparse.ArgumentParser(description='Build results table for CMNIST experiment.') 40 | parser.add_argument('--erm_results_dir', type=str, default='/scratch/gobi1/creager/opt_env/cmnist/erm') 41 | parser.add_argument('--irm_results_dir', type=str, default='/scratch/gobi1/creager/opt_env/cmnist/irm') 42 | parser.add_argument('--eiil_results_dir', type=str, default='/scratch/gobi1/creager/opt_env/cmnist/eiil') 43 | parser.add_argument('--eiil_cb_results_dir', type=str, default='/scratch/gobi1/creager/opt_env/cmnist/eiil_cb') 44 | parser.add_argument('--cb_results_dir', type=str, default='/scratch/gobi1/creager/opt_env/cmnist/cb') 45 | parser.add_argument('--gray_results_dir', type=str, default='/scratch/gobi1/creager/opt_env/cmnist/gray') 46 | parser.add_argument('--results_dir', type=str, default='/scratch/gobi1/creager/opt_env/cmnist_results/acc_table', 47 | help='where tex tables should be saved') 48 | flags = parser.parse_args() 49 | main(flags) -------------------------------------------------------------------------------- /InvariantRiskMinimization/code/figure_1/penalties.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | 8 | from sklearn.linear_model import Ridge 9 | import matplotlib.pyplot as plt 10 | import numpy as np 11 | 12 | 13 | def ls(x, y, reg=.1): 14 | return Ridge(alpha=reg, fit_intercept=False).fit(x, y).coef_ 15 | 16 | 17 | def sample(n=100000, e=1): 18 | x = np.random.randn(n, 1) * e 19 | y = x + np.random.randn(n, 1) * e 20 | z = y + np.random.randn(n, 1) 21 | return np.hstack((x, z)), y 22 | 23 | 24 | def penalty_ls(x1, y1, x2, y2, t=1, reg=.1): 25 | phi = np.diag([1, t]) 26 | w = np.array([1, 0]).reshape(1, 2) 27 | p1 = np.linalg.norm(ls(x1 @ phi, y1, reg) - w) 28 | p2 = np.linalg.norm(ls(x2 @ phi, y2, reg) - w) 29 | return (p1 + p2) / 2 30 | 31 | 32 | def penalty_g(x1, y1, x2, y2, t=1): 33 | phi = np.diag([1, t]) 34 | w = np.array([1, 0]).reshape(2, 1) 35 | p1 = (phi.T @ x1.T @ x1 @ phi @ w - phi.T @ x1.T @ y1) / x1.shape[0] 36 | p2 = (phi.T @ x2.T @ x2 @ phi @ w - phi.T @ x2.T @ y2) / x2.shape[0] 37 | return np.linalg.norm(p1) ** 2 + np.linalg.norm(p2) ** 2 38 | 39 | 40 | if __name__ == "__main__": 41 | x1, y1 = sample(e=1) 42 | x2, y2 = sample(e=2) 43 | 44 | plot_x = np.linspace(-1, 1, 100 + 1) 45 | plot_y_ls = [] 46 | plot_y_ls_reg = [] 47 | plot_y_1 = [] 48 | 49 | for t in plot_x: 50 | plot_y_ls.append(penalty_ls(x1, y1, x2, y2, t)) 51 | plot_y_ls_reg.append(penalty_ls(x1, y1, x2, y2, t, reg=1000)) 52 | plot_y_1.append(penalty_g(x1, y1, x2, y2, t)) 53 | 54 | plt.rcParams.update({'text.latex.preamble' : [r'\usepackage{amsmath, amsfonts}']}) 55 | plt.rcParams["font.family"] = "Times New Roman" 56 | plt.rc('text', usetex=True) 57 | plt.rc('font', size=12) 58 | 59 | plt.figure(figsize=(8, 4)) 60 | plt.plot(plot_x, plot_y_ls, lw=2, label=r'$\mathbb{D}_{\text{dist}}((1, 0), \Phi, e)$') 61 | plt.plot(plot_x, plot_y_ls_reg, ls="--", lw=2, label=r'$\mathbb{D}_{\text{dist}}$ (heavy regularization)') 62 | plt.plot(plot_x, plot_y_1, '.', lw=2, label=r'$\mathbb{D}_{\text{lin}}((1, 0), \Phi, e)$') 63 | plt.ylim(-1, 12) 64 | plt.xlabel( 65 | r'$c$, the weight of $\Phi$ on the input with varying correlation', labelpad=10) 66 | plt.ylabel(r'invariance penalty') 67 | plt.tight_layout(0, 0, 0) 68 | plt.legend(prop={'size': 11}, loc="upper right") 69 | plt.savefig("different_penalties.pdf") 70 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/code/experiment_synthetic/sem.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | 8 | import torch 9 | import numpy as np 10 | 11 | 12 | class ChainEquationModel(object): 13 | def __init__(self, dim, scramble=False, hetero=True, hidden=False): 14 | self.hetero = hetero 15 | self.hidden = hidden 16 | self.dim = dim // 2 17 | ones = True 18 | 19 | if ones: 20 | self.wxy = torch.eye(self.dim) 21 | self.wyz = torch.eye(self.dim) 22 | else: 23 | self.wxy = torch.randn(self.dim, self.dim) / dim 24 | self.wyz = torch.randn(self.dim, self.dim) / dim 25 | 26 | if scramble: 27 | self.scramble, _ = torch.qr(torch.randn(dim, dim)) 28 | else: 29 | self.scramble = torch.eye(dim) 30 | 31 | if hidden: 32 | self.whx = torch.randn(self.dim, self.dim) / dim 33 | self.why = torch.randn(self.dim, self.dim) / dim 34 | self.whz = torch.randn(self.dim, self.dim) / dim 35 | else: 36 | self.whx = torch.eye(self.dim, self.dim) 37 | self.why = torch.zeros(self.dim, self.dim) 38 | self.whz = torch.zeros(self.dim, self.dim) 39 | 40 | def solution(self): 41 | w = torch.cat((self.wxy.sum(1), torch.zeros(self.dim))).view(-1, 1) 42 | return self.scramble.t() @ w 43 | 44 | def __call__(self, n, env): 45 | h = torch.randn(n, self.dim) * env 46 | 47 | if self.hetero == 2: 48 | x = torch.randn(n, self.dim) * 5. 49 | y = x @ self.wxy + torch.randn(n, self.dim) * env 50 | z = y @ self.wyz + torch.randn(n, self.dim) 51 | elif self.hetero == 1: 52 | x = h @ self.whx + torch.randn(n, self.dim) * env 53 | y = x @ self.wxy + h @ self.why + torch.randn(n, self.dim) * env 54 | z = y @ self.wyz + h @ self.whz + torch.randn(n, self.dim) 55 | else: 56 | x = h @ self.whx + torch.randn(n, self.dim) * env 57 | y = x @ self.wxy + h @ self.why + torch.randn(n, self.dim) 58 | z = y @ self.wyz + h @ self.whz + torch.randn(n, self.dim) * env 59 | 60 | variances = dict( 61 | h=h.var().item(), 62 | x=x.var().item(), 63 | y=y.var().item(), 64 | z=z.var().item(), 65 | e=(torch.randn(n, self.dim) * env).var().item() # any env dependent noise we might add 66 | ) 67 | from pprint import pprint 68 | print('in setting %d data in env %d have following variances' % (self.hetero, env)) 69 | pprint(variances) 70 | return torch.cat((x, z), 1) @ self.scramble, y.sum(1, keepdim=True) 71 | -------------------------------------------------------------------------------- /exps/cmnist_with_specified_label_noise.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CMNIST Experiment. 3 | 4 | # Hyperparameters 5 | N_RESTARTS=10 6 | HIDDEN_DIM=390 7 | L2_REGULARIZER_WEIGHT=0.00110794568 8 | LR=0.0004898536566546834 9 | LABEL_NOISE=${1-0.05} 10 | PENALTY_ANNEAL_ITERS=190 11 | PENALTY_WEIGHT=191257.18613115903 12 | STEPS=501 13 | ROOT=${2-/scratch/gobi1/creager/opt_env/cmnist} 14 | # RNUM=$(printf "%05d" $(($RANDOM$RANDOM$RANDOM % 100000))) 15 | TAG=$(date +'%Y-%m-%d')--$LABEL_NOISE 16 | ROOT=$ROOT/label_noise_sweep/$TAG 17 | 18 | # ERM 19 | python -u -m opt_env.irm_cmnist \ 20 | --results_dir $ROOT/erm \ 21 | --n_restarts $N_RESTARTS \ 22 | --hidden_dim $HIDDEN_DIM \ 23 | --l2_regularizer_weight $L2_REGULARIZER_WEIGHT \ 24 | --lr $LR \ 25 | --label_noise $LABEL_NOISE \ 26 | --penalty_anneal_iters 0 \ 27 | --penalty_weight 0.0 \ 28 | --steps $STEPS 29 | 30 | # IRM 31 | python -u -m opt_env.irm_cmnist \ 32 | --results_dir $ROOT/irm \ 33 | --n_restarts $N_RESTARTS \ 34 | --hidden_dim $HIDDEN_DIM \ 35 | --l2_regularizer_weight $L2_REGULARIZER_WEIGHT \ 36 | --lr $LR \ 37 | --label_noise $LABEL_NOISE \ 38 | --penalty_anneal_iters $PENALTY_ANNEAL_ITERS \ 39 | --penalty_weight $PENALTY_WEIGHT \ 40 | --steps $STEPS 41 | 42 | # EIIL 43 | python -u -m opt_env.irm_cmnist \ 44 | --results_dir $ROOT/eiil \ 45 | --n_restarts $N_RESTARTS \ 46 | --hidden_dim $HIDDEN_DIM \ 47 | --l2_regularizer_weight $L2_REGULARIZER_WEIGHT \ 48 | --lr $LR \ 49 | --label_noise $LABEL_NOISE \ 50 | --penalty_anneal_iters $PENALTY_ANNEAL_ITERS \ 51 | --penalty_weight $PENALTY_WEIGHT \ 52 | --steps $STEPS \ 53 | --eiil 54 | 55 | # EIIL with color-based reference classifier 56 | python -u -m opt_env.irm_cmnist \ 57 | --results_dir $ROOT/eiil_cb \ 58 | --n_restarts $N_RESTARTS \ 59 | --hidden_dim $HIDDEN_DIM \ 60 | --l2_regularizer_weight $L2_REGULARIZER_WEIGHT \ 61 | --lr $LR \ 62 | --label_noise $LABEL_NOISE \ 63 | --penalty_anneal_iters $PENALTY_ANNEAL_ITERS \ 64 | --penalty_weight $PENALTY_WEIGHT \ 65 | --steps $STEPS \ 66 | --eiil \ 67 | --color_based 68 | 69 | # Evaluate color-based classifier on its own 70 | python -u -m opt_env.irm_cmnist \ 71 | --results_dir $ROOT/cb \ 72 | --n_restarts $N_RESTARTS \ 73 | --hidden_dim $HIDDEN_DIM \ 74 | --l2_regularizer_weight $L2_REGULARIZER_WEIGHT \ 75 | --lr $LR \ 76 | --label_noise $LABEL_NOISE \ 77 | --penalty_anneal_iters $PENALTY_ANNEAL_ITERS \ 78 | --penalty_weight $PENALTY_WEIGHT \ 79 | --steps $STEPS \ 80 | --color_based_eval \ 81 | 82 | # Grayscale baseline 83 | python -u -m opt_env.irm_cmnist \ 84 | --results_dir $ROOT/gray \ 85 | --n_restarts $N_RESTARTS \ 86 | --hidden_dim $HIDDEN_DIM \ 87 | --l2_regularizer_weight $L2_REGULARIZER_WEIGHT \ 88 | --lr $LR \ 89 | --label_noise $LABEL_NOISE \ 90 | --penalty_anneal_iters $PENALTY_ANNEAL_ITERS \ 91 | --penalty_weight 0.0 \ 92 | --steps $STEPS \ 93 | --grayscale_model 94 | 95 | # Build latex tables 96 | # accuracy 97 | python -u -m opt_env.cmnist_results.acc_table \ 98 | --erm_results_dir $ROOT/erm \ 99 | --irm_results_dir $ROOT/irm \ 100 | --eiil_results_dir $ROOT/eiil \ 101 | --eiil_cb_results_dir $ROOT/eiil_cb \ 102 | --cb_results_dir $ROOT/cb \ 103 | --gray_results_dir $ROOT/gray \ 104 | --results_dir $ROOT/acc_table 105 | -------------------------------------------------------------------------------- /opt_env/utils/env_utils.py: -------------------------------------------------------------------------------- 1 | """Build environments.""" 2 | import attr 3 | import numpy as np 4 | import torch 5 | from torchvision import datasets 6 | 7 | 8 | def get_envs(cuda=True, flags=None): 9 | 10 | if flags is None: # configure data generation like in original IRM paper 11 | @attr.s 12 | class DefaultFlags(object): 13 | """Specify spurrious correlations as original IRM paper.""" 14 | train_env_1__color_noise = attr.ib(default=0.2) 15 | train_env_2__color_noise = attr.ib(default=0.1) 16 | test_env__color_noise = attr.ib(default=0.9) 17 | label_noise = attr.ib(default=0.25) 18 | flags = DefaultFlags() 19 | 20 | def _make_environment(images, labels, e): 21 | 22 | # NOTE: low e indicates a spurious correlation from color to (noisy) label 23 | 24 | def torch_bernoulli(p, size): 25 | return (torch.rand(size) < p).float() 26 | 27 | def torch_xor(a, b): 28 | return (a-b).abs() # Assumes both inputs are either 0 or 1 29 | 30 | samples = dict() 31 | # 2x subsample for computational convenience 32 | images = images.reshape((-1, 28, 28))[:, ::2, ::2] 33 | # Assign a binary label based on the digit; flip label with probability 0.25 34 | labels = (labels < 5).float() 35 | samples.update(preliminary_labels=labels) 36 | label_noise = torch_bernoulli(flags.label_noise, len(labels)) 37 | labels = torch_xor(labels, label_noise) 38 | samples.update(final_labels=labels) 39 | samples.update(label_noise=label_noise) 40 | # Assign a color based on the label; flip the color with probability e 41 | color_noise = torch_bernoulli(e, len(labels)) 42 | colors = torch_xor(labels, color_noise) 43 | samples.update(colors=colors) 44 | samples.update(color_noise=color_noise) 45 | # Apply the color to the image by zeroing out the other color channel 46 | images = torch.stack([images, images], dim=1) 47 | images[torch.tensor(range(len(images))), (1-colors).long(), :, :] *= 0 48 | images = (images.float() / 255.) 49 | labels = labels[:, None] 50 | if cuda and torch.cuda.is_available(): 51 | images = images.cuda() 52 | labels = labels.cuda() 53 | samples.update(images=images, labels=labels) 54 | return samples 55 | 56 | mnist = datasets.MNIST('~/datasets/mnist', train=True, download=True) 57 | mnist_train = (mnist.data[:50000], mnist.targets[:50000]) 58 | mnist_val = (mnist.data[50000:], mnist.targets[50000:]) 59 | 60 | rng_state = np.random.get_state() 61 | np.random.shuffle(mnist_train[0].numpy()) 62 | np.random.set_state(rng_state) 63 | np.random.shuffle(mnist_train[1].numpy()) 64 | 65 | envs = [ 66 | _make_environment(mnist_train[0][::2], mnist_train[1][::2], flags.train_env_1__color_noise), 67 | _make_environment(mnist_train[0][1::2], mnist_train[1][1::2], flags.train_env_2__color_noise), 68 | _make_environment(mnist_val[0], mnist_val[1], flags.test_env__color_noise) 69 | ] 70 | return envs 71 | 72 | 73 | def get_envs_with_indices(): 74 | """Return IRM envs but with indices and environment indicators.""" 75 | envs = get_envs() 76 | examples_so_far = 0 77 | for i, env in enumerate(envs): 78 | num_examples = len(env['images']) 79 | env['idx'] = idx = torch.tensor( 80 | np.arange(examples_so_far, examples_so_far + num_examples), 81 | dtype=torch.int32 82 | ) 83 | examples_so_far += num_examples 84 | # here "env" is a label indicating which env each example belongs to 85 | env['env'] = torch.tensor(i * np.ones_like(env['idx']), dtype=torch.uint8) 86 | return envs 87 | 88 | 89 | def split_by_noise(env, noise_var='label'): 90 | assert noise_var in ('label', 'color'), 'Unexpected noise variable.' 91 | noise_name = '%s_noise' % noise_var 92 | clean_idx = (env[noise_name] == 0.) 93 | noisy_idx = (env[noise_name] == 1.) 94 | from copy import deepcopy 95 | clean_env, noisy_env = deepcopy(env), deepcopy(env) 96 | for k, v in clean_env.items(): 97 | if v.numel() > 1: 98 | clean_env[k] = v[clean_idx] 99 | for k, v in noisy_env.items(): 100 | if v.numel() > 1: 101 | noisy_env[k] = v[noisy_idx] 102 | return clean_env, noisy_env 103 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/code/experiment_synthetic/plot.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | 8 | import matplotlib.pyplot as plt 9 | import matplotlib.ticker as mticker 10 | import numpy as np 11 | import torch 12 | import math 13 | import sys 14 | 15 | from matplotlib.patches import Patch 16 | 17 | 18 | def parse_title(title): 19 | result = "" 20 | fields = title.split("_") 21 | 22 | if fields[1].split("=")[1] == "1": 23 | result += "P" 24 | else: 25 | result += "F" 26 | 27 | if fields[2].split("=")[1] == "1": 28 | result += "E" 29 | else: 30 | result += "O" 31 | 32 | if fields[3].split("=")[1] == "1": 33 | result += "S" 34 | else: 35 | result += "U" 36 | 37 | return result 38 | 39 | 40 | def plot_bars(results, category, which, sep=1.1): 41 | models = list(results[list(results.keys())[0]].keys()) 42 | 43 | if "SEM" in models: 44 | models.remove("SEM") 45 | models.sort() 46 | 47 | setups = list(results.keys()) 48 | setups.sort() 49 | 50 | if which == "causal": 51 | hatch = None 52 | offset = 0 53 | idx = 0 54 | else: 55 | hatch = "//" 56 | offset = 4 57 | idx = 1 58 | 59 | counter = 1 60 | for s, setup in enumerate(setups): 61 | title = parse_title(setup) 62 | 63 | if category not in title: 64 | continue 65 | 66 | boxes = [] 67 | boxes_means = [] 68 | boxes_colors = [] 69 | boxes_vars = [] 70 | ax = plt.subplot(2, 4, counter + offset) 71 | counter += 1 72 | 73 | for m, model in enumerate(models): 74 | boxes.append(np.array(results[setup][model])[:, idx]) 75 | boxes_means.append( 76 | np.mean(np.array(results[setup][model])[:, idx])) 77 | boxes_vars.append(np.std(np.array(results[setup][model])[:, idx])) 78 | boxes_colors.append("C" + str(m)) 79 | 80 | plt.bar([0, 1, 2], 81 | boxes_means, 82 | yerr=np.array(boxes_vars), 83 | color=boxes_colors, 84 | hatch=hatch, 85 | alpha=0.7, 86 | log=True) 87 | 88 | if which == "causal": 89 | plt.xticks([1], [title]) 90 | else: 91 | ax.xaxis.set_ticks_position('top') 92 | plt.xticks([1], [""]) 93 | 94 | if (counter + offset) == 2 or (counter + offset) == 6: 95 | if which == "causal": 96 | plt.ylabel("causal error") 97 | else: 98 | plt.ylabel("non-causal error") 99 | 100 | if title == "PES" and which != "causal": 101 | legends = [] 102 | for m, model in enumerate(models): 103 | legends.append( 104 | Patch(facecolor="C" + str(m), alpha=0.7, label=model)) 105 | plt.legend(handles=legends, loc="lower center") 106 | 107 | if title == "POU" and which != "causal": 108 | plt.minorticks_off() 109 | ax.set_yticks([0.1, 0.01]) 110 | 111 | def get_results(all_solutions): 112 | results = {} 113 | 114 | for line in all_solutions: 115 | words = line.split(" ") 116 | setup = str(words[0]) 117 | model = str(words[1]) 118 | err_causal = float(words[-2]) 119 | err_noncausal = float(words[-1]) 120 | 121 | if setup not in results: 122 | results[setup] = {} 123 | 124 | if model not in results[setup]: 125 | results[setup][model] = [] 126 | 127 | results[setup][model].append([err_causal, err_noncausal]) 128 | return results 129 | 130 | 131 | 132 | def plot_experiment(all_solutions, fname): 133 | plt.rcParams["font.family"] = "serif" 134 | plt.rc('text', usetex=True) 135 | plt.rc('font', size=10) 136 | 137 | results = get_results(all_solutions) 138 | 139 | plt.figure(figsize=(7, 2)) 140 | plot_bars(results, category, "causal") 141 | plot_bars(results, category, "noncausal") 142 | plt.tight_layout(0, 0, 0.5) 143 | 144 | if fname is None: 145 | plt.show() 146 | else: 147 | plt.savefig(fname) 148 | 149 | 150 | if __name__ == "__main__": 151 | if len(sys.argv) == 1: 152 | fname = "synthetic_results.pt" 153 | else: 154 | fname = sys.argv[1] 155 | lines = torch.load(fname) 156 | plot_experiment(lines, "F", "results_f.pdf") 157 | plot_experiment(lines, "P", "results_p.pdf") 158 | -------------------------------------------------------------------------------- /opt_env/utils/opt_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from torch import autograd 4 | from torch import nn 5 | from torch import optim 6 | from tqdm import tqdm 7 | 8 | def nll(logits, y, reduction='mean'): 9 | return nn.functional.binary_cross_entropy_with_logits(logits, y, reduction=reduction) 10 | 11 | def mean_accuracy(logits, y): 12 | preds = (logits > 0.).float() 13 | return ((preds - y).abs() < 1e-2).float().mean() 14 | 15 | def penalty(logits, y): 16 | scale = torch.tensor(1.).cuda().requires_grad_() 17 | loss = nll(logits * scale, y) 18 | grad = autograd.grad(loss, [scale], create_graph=True)[0] 19 | return torch.sum(grad**2) 20 | 21 | def split_data_opt(envs, model, n_steps=10000, n_samples=-1, lr=0.001, 22 | batch_size=None, join=True, no_tqdm=False): 23 | """Learn soft environment assignment.""" 24 | 25 | if join: # assumes first two entries in envs list are the train sets to joined 26 | print('pooling envs') 27 | # pool all training envs (defined as each env in envs[:-1]) 28 | joined_train_envs = dict() 29 | for k in envs[0].keys(): 30 | if envs[0][k].numel() > 1: # omit scalars previously stored during training 31 | joined_values = torch.cat((envs[0][k][:n_samples], 32 | envs[1][k][:n_samples]), 33 | 0) 34 | joined_train_envs[k] = joined_values 35 | print('size of pooled envs: %d' % len(joined_train_envs['images'])) 36 | else: 37 | if not isinstance(envs, dict): 38 | raise ValueError(('When join=False, first argument should be a dict' 39 | ' corresponding to the only environment.' 40 | )) 41 | print('splitting data from single env of size %d' % len(envs['images'])) 42 | joined_train_envs = envs 43 | 44 | scale = torch.tensor(1.).cuda().requires_grad_() 45 | if batch_size: 46 | logits = [] 47 | i = 0 48 | num_examples = len(joined_train_envs['images']) 49 | while i < num_examples: 50 | images = joined_train_envs['images'][i:i+64] 51 | images = images.cuda() 52 | logits.append(model(images).detach()) 53 | i += 64 54 | logits = torch.cat(logits) 55 | else: 56 | logits = model(joined_train_envs['images']) 57 | logits = logits.detach() 58 | 59 | loss = nll(logits * scale, joined_train_envs['labels'].cuda(), reduction='none') 60 | 61 | env_w = torch.randn(len(logits)).cuda().requires_grad_() 62 | optimizer = optim.Adam([env_w], lr=lr) 63 | 64 | with tqdm(total=n_steps, position=1, bar_format='{desc}', desc='AED Loss: ', disable=no_tqdm) as desc: 65 | for i in tqdm(range(n_steps), disable=no_tqdm): 66 | # penalty for env a 67 | lossa = (loss.squeeze() * env_w.sigmoid()).mean() 68 | grada = autograd.grad(lossa, [scale], create_graph=True)[0] 69 | penaltya = torch.sum(grada**2) 70 | # penalty for env b 71 | lossb = (loss.squeeze() * (1-env_w.sigmoid())).mean() 72 | gradb = autograd.grad(lossb, [scale], create_graph=True)[0] 73 | penaltyb = torch.sum(gradb**2) 74 | # negate 75 | npenalty = - torch.stack([penaltya, penaltyb]).mean() 76 | # step 77 | optimizer.zero_grad() 78 | npenalty.backward(retain_graph=True) 79 | optimizer.step() 80 | desc.set_description('AED Loss: %.8f' % npenalty.cpu().item()) 81 | 82 | print('Final AED Loss: %.8f' % npenalty.cpu().item()) 83 | 84 | # split envs based on env_w threshold 85 | new_envs = [] 86 | idx0 = (env_w.sigmoid()>.5) 87 | idx1 = (env_w.sigmoid()<=.5) 88 | # train envs 89 | # NOTE: envs include original data indices for qualitative investigation 90 | for _idx in (idx0, idx1): 91 | new_env = dict() 92 | for k, v in joined_train_envs.items(): 93 | if k == 'paths': # paths is formatted as a list of str, not ndarray or tensor 94 | v_ = np.array(v) 95 | v_ = v_[_idx.cpu().numpy()] 96 | v_ = list(v_) 97 | new_env[k] = v_ 98 | else: 99 | new_env[k] = v[_idx] 100 | new_envs.append(new_env) 101 | print('size of env0: %d' % len(new_envs[0]['images'])) 102 | print('size of env1: %d' % len(new_envs[1]['images'])) 103 | 104 | if join: #NOTE: assume the user includes test set as part of arguments only if join=True 105 | new_envs.append(envs[-1]) 106 | print('size of env2: %d' % len(new_envs[2]['images'])) 107 | return new_envs 108 | 109 | 110 | def train_irm_batch(model, envs, flags): 111 | """Batch version of the IRM algo for CMNIST expers.""" 112 | def _pretty_print(*values): 113 | col_width = 13 114 | def format_val(v): 115 | if not isinstance(v, str): 116 | v = np.array2string(v, precision=5, floatmode='fixed') 117 | return v.ljust(col_width) 118 | str_values = [format_val(v) for v in values] 119 | print(" ".join(str_values)) 120 | 121 | if flags.color_based_eval: # skip IRM and evaluate color-based model 122 | from opt_env.utils.model_utils import ColorBasedClassifier 123 | model = ColorBasedClassifier() 124 | if not flags.color_based_eval: 125 | optimizer = optim.Adam(model.parameters(), lr=flags.lr) 126 | for step in range(flags.steps): 127 | for env in envs: 128 | logits = model(env['images']) 129 | env['nll'] = nll(logits, env['labels']) 130 | env['acc'] = mean_accuracy(logits, env['labels']) 131 | env['penalty'] = penalty(logits, env['labels']) 132 | 133 | train_nll = torch.stack([envs[0]['nll'], envs[1]['nll']]).mean() 134 | train_acc = torch.stack([envs[0]['acc'], envs[1]['acc']]).mean() 135 | train_penalty = torch.stack([envs[0]['penalty'], envs[1]['penalty']]).mean() 136 | 137 | weight_norm = torch.tensor(0.).cuda() 138 | for w in model.parameters(): 139 | weight_norm += w.norm().pow(2) 140 | loss = train_nll.clone() 141 | loss += flags.l2_regularizer_weight * weight_norm 142 | penalty_weight = (flags.penalty_weight 143 | if step >= flags.penalty_anneal_iters else 1.0) 144 | loss += penalty_weight * train_penalty 145 | if penalty_weight > 1.0: 146 | # Rescale the entire loss to keep gradients in a reasonable range 147 | loss /= penalty_weight 148 | 149 | if not flags.color_based_eval: 150 | optimizer.zero_grad() 151 | loss.backward() 152 | optimizer.step() 153 | 154 | test_acc = envs[2]['acc'] 155 | if step % 100 == 0: 156 | _pretty_print( 157 | np.int32(step), 158 | train_nll.detach().cpu().numpy(), 159 | train_acc.detach().cpu().numpy(), 160 | train_penalty.detach().cpu().numpy(), 161 | test_acc.detach().cpu().numpy() 162 | ) 163 | 164 | final_train_acc = train_acc.detach().cpu().numpy() 165 | final_test_acc = test_acc.detach().cpu().numpy() 166 | return model, final_train_acc, final_test_acc 167 | 168 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/code/experiment_synthetic/main.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | 8 | import os 9 | import pickle 10 | import sys 11 | 12 | from sem import ChainEquationModel 13 | from models import * 14 | 15 | import argparse 16 | import torch 17 | import numpy 18 | 19 | 20 | def pretty(vector): 21 | vlist = vector.view(-1).tolist() 22 | return "[" + ", ".join("{:+.3f}".format(vi) for vi in vlist) + "]" 23 | 24 | 25 | def errors(w, w_hat): 26 | w = w.view(-1) 27 | w_hat = w_hat.view(-1) 28 | 29 | i_causal = (w != 0).nonzero().view(-1) 30 | i_noncausal = (w == 0).nonzero().view(-1) 31 | 32 | if len(i_causal): 33 | error_causal = (w[i_causal] - w_hat[i_causal]).pow(2).mean() 34 | error_causal = error_causal.item() 35 | else: 36 | error_causal = 0 37 | 38 | if len(i_noncausal): 39 | error_noncausal = (w[i_noncausal] - w_hat[i_noncausal]).pow(2).mean() 40 | error_noncausal = error_noncausal.item() 41 | else: 42 | error_noncausal = 0 43 | 44 | return error_causal, error_noncausal 45 | 46 | 47 | def run_experiment(args): 48 | if args["seed"] >= 0: 49 | torch.manual_seed(args["seed"]) 50 | numpy.random.seed(args["seed"]) 51 | torch.set_num_threads(1) 52 | 53 | if args["setup_sem"] == "chain": 54 | setup_str = "chain_hidden={}_hetero={}_scramble={}".format( 55 | args["setup_hidden"], 56 | args["setup_hetero"], 57 | args["setup_scramble"]) 58 | elif args["setup_sem"] == "icp": 59 | setup_str = "sem_icp" 60 | else: 61 | raise NotImplementedError 62 | 63 | args['results_dir'] = os.path.join(args['results_dir'], setup_str) 64 | if args['eiil_ref_alpha'] >= 0 and args['eiil_ref_alpha'] <= 1: 65 | args['results_dir'] = '{results_dir}_alpha_{eiil_ref_alpha:.1f}'.format(**args) 66 | 67 | if not os.path.exists(args['results_dir']): 68 | os.makedirs(args['results_dir']) 69 | pickle.dump(args, open(os.path.join(args['results_dir'], 'flags.p'), 'wb')) 70 | for f in sys.stdout, open(os.path.join(args['results_dir'], 'flags.txt'), 'w'): 71 | print('Flags:', file=f) 72 | for k,v in sorted(args.items()): 73 | print("\t{}: {}".format(k, v), file=f) 74 | print('results will be found here:') 75 | print(args['results_dir']) 76 | 77 | all_methods = { 78 | "ERM": EmpiricalRiskMinimizer, 79 | "ICP": InvariantCausalPrediction, 80 | "IRM": InvariantRiskMinimization, 81 | "EIIL": LearnedEnvInvariantRiskMinimization 82 | } 83 | 84 | if args["methods"] == "all": 85 | methods = all_methods 86 | else: 87 | methods = {m: all_methods[m] for m in args["methods"].split(',')} 88 | 89 | all_sems = [] 90 | all_solutions = [] 91 | all_environments = [] 92 | from collections import defaultdict 93 | all_err_causal = defaultdict(list) 94 | all_err_noncausal = defaultdict(list) 95 | 96 | for rep_i in range(args["n_reps"]): 97 | if args["setup_sem"] == "chain": 98 | sem = ChainEquationModel(args["dim"], 99 | hidden=args["setup_hidden"], 100 | scramble=args["setup_scramble"], 101 | hetero=args["setup_hetero"]) 102 | environments = [sem(args["n_samples"], .2), 103 | sem(args["n_samples"], 2.), 104 | sem(args["n_samples"], 5.)] 105 | else: 106 | raise NotImplementedError 107 | 108 | all_sems.append(sem) 109 | all_environments.append(environments) 110 | 111 | for sem, environments in zip(all_sems, all_environments): 112 | soln = sem.solution() 113 | solutions = [ 114 | "{} {:<5} {} {:.5f} {:.5f}".format(setup_str, 115 | "SEM", 116 | pretty(sem.solution()), 0, 0) 117 | ] 118 | 119 | 120 | for method_name, method_constructor in methods.items(): 121 | method = method_constructor(environments, args) 122 | msolution = method.solution() 123 | err_causal, err_noncausal = errors(sem.solution(), msolution) 124 | all_err_causal[method_name].append(err_causal) 125 | all_err_noncausal[method_name].append(err_noncausal) 126 | 127 | solutions.append("{} {:<5} {} {:.5f} {:.5f}".format(setup_str, 128 | method_name, 129 | pretty(msolution), 130 | err_causal, 131 | err_noncausal)) 132 | 133 | all_solutions += solutions 134 | 135 | # save results 136 | results = dict() 137 | results.update(setup_str=setup_str) 138 | results.update(all_sems=all_sems) 139 | results.update(all_solutions=all_solutions) 140 | results.update(all_environments=all_environments) 141 | results.update(all_environments=all_environments) 142 | results.update(all_err_causal=all_err_causal) 143 | results.update(all_err_noncausal=all_err_noncausal) 144 | with open(os.path.join(args['results_dir'], 'results.p'), 'wb') as f: 145 | pickle.dump(results, f) 146 | 147 | return all_solutions 148 | 149 | 150 | if __name__ == '__main__': 151 | parser = argparse.ArgumentParser(description='Invariant regression') 152 | parser.add_argument('--dim', type=int, default=10) 153 | parser.add_argument('--n_samples', type=int, default=1000) 154 | parser.add_argument('--n_reps', type=int, default=1) 155 | parser.add_argument('--skip_reps', type=int, default=0) 156 | parser.add_argument('--seed', type=int, default=0) # Negative is random 157 | parser.add_argument('--print_vectors', type=int, default=1) 158 | parser.add_argument('--n_iterations', type=int, default=10000) 159 | parser.add_argument('--lr', type=float, default=1e-3) 160 | parser.add_argument('--verbose', type=int, default=0) 161 | parser.add_argument('--methods', type=str, default="EIIL,IRM,ERM") 162 | parser.add_argument('--alpha', type=float, default=0.05) 163 | parser.add_argument('--setup_sem', type=str, default="chain") 164 | parser.add_argument('--setup_hidden', type=int, default=0) 165 | parser.add_argument('--setup_hetero', type=int, default=1) 166 | parser.add_argument('--setup_scramble', type=int, default=0) 167 | parser.add_argument('--results_dir', type=str, default="/tmp/experiment_synthetic") 168 | parser.add_argument('--eiil_ref_alpha', type=float, default=-1, 169 | help=('Value between zero and one to hard code the reference ' 170 | 'classifier propensity to use the spurious feature. Set ' 171 | 'to value outside zero one interval to disable.')) 172 | args = dict(vars(parser.parse_args())) 173 | 174 | all_solutions = run_experiment(args) 175 | print("\n".join(all_solutions)) 176 | print("\n".join(all_solutions), file=open( 177 | os.path.join(args['results_dir'], 'all_solutions.txt'), 'w') 178 | ) 179 | -------------------------------------------------------------------------------- /opt_env/irm_cmnist.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import pdb 4 | import pickle 5 | import sys 6 | 7 | import numpy as np 8 | import torch 9 | from torchvision import datasets 10 | from torch import nn, optim, autograd 11 | from tqdm import tqdm 12 | 13 | from opt_env.utils.env_utils import get_envs 14 | from opt_env.utils.opt_utils import split_data_opt 15 | from opt_env.utils.opt_utils import train_irm_batch 16 | from opt_env.utils.model_utils import load_mlp 17 | from opt_env.utils.model_utils import ColorBasedClassifier 18 | 19 | def main(flags): 20 | 21 | if not os.path.exists(flags.results_dir): 22 | os.makedirs(flags.results_dir) 23 | 24 | # save this file and command for reproducibility 25 | if flags.results_dir != '.': 26 | with open(__file__, 'r') as f: 27 | this_file = f.readlines() 28 | with open(os.path.join(flags.results_dir, 'irm_cmnist.py'), 'w') as f: 29 | f.write(''.join(this_file)) 30 | cmd = 'python ' + ' '.join(sys.argv) 31 | with open(os.path.join(flags.results_dir, 'command.sh'), 'w') as f: 32 | f.write(cmd) 33 | # save params for later 34 | if not os.path.exists(flags.results_dir): 35 | os.makedirs(flags.results_dir) 36 | pickle.dump(flags, open(os.path.join(flags.results_dir, 'flags.p'), 'wb')) 37 | for f in sys.stdout, open(os.path.join(flags.results_dir, 'flags.txt'), 'w'): 38 | print('Flags:', file=f) 39 | for k,v in sorted(vars(flags).items()): 40 | print("\t{}: {}".format(k, v), file=f) 41 | 42 | print('results will be found here:') 43 | print(flags.results_dir) 44 | 45 | final_train_accs = [] 46 | final_test_accs = [] 47 | for restart in range(flags.n_restarts): 48 | print("Restart", restart) 49 | 50 | rng_state = np.random.get_state() 51 | np.random.set_state(rng_state) 52 | 53 | # Build environments 54 | envs = get_envs(flags=flags) 55 | 56 | # Define and instantiate the model 57 | if flags.color_based: # use color-based reference classifier without trainable params 58 | mlp_pre = ColorBasedClassifier() 59 | else: 60 | mlp_pre = load_mlp(results_dir=None, flags=flags).cuda() # reference classifier 61 | mlp = load_mlp(results_dir=None, flags=flags).cuda() # invariant representation learner 62 | mlp_pre.train() 63 | mlp.train() 64 | 65 | # Define loss function helpers 66 | 67 | def nll(logits, y, reduction='mean'): 68 | return nn.functional.binary_cross_entropy_with_logits(logits, y, reduction=reduction) 69 | 70 | def mean_accuracy(logits, y): 71 | preds = (logits > 0.).float() 72 | return ((preds - y).abs() < 1e-2).float().mean() 73 | 74 | def penalty(logits, y): 75 | scale = torch.tensor(1.).cuda().requires_grad_() 76 | loss = nll(logits * scale, y) 77 | grad = autograd.grad(loss, [scale], create_graph=True)[0] 78 | return torch.sum(grad**2) 79 | 80 | # Train loop 81 | 82 | def pretty_print(*values): 83 | col_width = 13 84 | def format_val(v): 85 | if not isinstance(v, str): 86 | v = np.array2string(v, precision=5, floatmode='fixed') 87 | return v.ljust(col_width) 88 | str_values = [format_val(v) for v in values] 89 | print(" ".join(str_values)) 90 | 91 | 92 | pretty_print('step', 'train nll', 'train acc', 'train penalty', 'test acc') 93 | 94 | if flags.eiil: 95 | if flags.color_based: 96 | print('Color-based refernece classifier was specified, to skipping pre-training.') 97 | else: 98 | optimizer_pre = optim.Adam(mlp_pre.parameters(), lr=flags.lr) 99 | for step in range(flags.steps): 100 | for env in envs: 101 | logits = mlp_pre(env['images']) 102 | env['nll'] = nll(logits, env['labels']) 103 | env['acc'] = mean_accuracy(logits, env['labels']) 104 | env['penalty'] = penalty(logits, env['labels']) 105 | 106 | train_nll = torch.stack([envs[0]['nll'], envs[1]['nll']]).mean() 107 | train_acc = torch.stack([envs[0]['acc'], envs[1]['acc']]).mean() 108 | train_penalty = torch.stack([envs[0]['penalty'], envs[1]['penalty']]).mean() 109 | 110 | weight_norm = torch.tensor(0.).cuda() 111 | for w in mlp_pre.parameters(): 112 | weight_norm += w.norm().pow(2) 113 | 114 | loss = train_nll.clone() 115 | loss += flags.l2_regularizer_weight * weight_norm 116 | 117 | optimizer_pre.zero_grad() 118 | loss.backward() 119 | optimizer_pre.step() 120 | 121 | test_acc = envs[2]['acc'] 122 | if step % 100 == 0: 123 | pretty_print( 124 | np.int32(step), 125 | train_nll.detach().cpu().numpy(), 126 | train_acc.detach().cpu().numpy(), 127 | train_penalty.detach().cpu().numpy(), 128 | test_acc.detach().cpu().numpy() 129 | ) 130 | torch.save(mlp_pre.state_dict(), 131 | os.path.join(flags.results_dir, 'mlp_pre.%d.p' % restart)) 132 | envs = split_data_opt(envs, mlp_pre) 133 | mlp, final_train_acc, final_test_acc = train_irm_batch(mlp, envs, flags) 134 | final_train_accs.append(final_train_acc) 135 | final_test_accs.append(final_test_acc) 136 | print('Final train acc (mean/std across restarts so far):') 137 | print(np.mean(final_train_accs), np.std(final_train_accs)) 138 | print('Final test acc (mean/std across restarts so far):') 139 | print(np.mean(final_test_accs), np.std(final_test_accs)) 140 | print('done with restart %d' % restart) 141 | torch.save(mlp.state_dict(), 142 | os.path.join(flags.results_dir, 'mlp.%s.p' % restart)) 143 | 144 | print('done with all restarts') 145 | final_train_accs = [t.item() for t in final_train_accs] 146 | final_test_accs = [t.item() for t in final_test_accs] 147 | metrics = {'Train accs': final_train_accs, 148 | 'Test accs': final_test_accs} 149 | with open(os.path.join(flags.results_dir, 'metrics.p'), 'wb') as f: 150 | pickle.dump(metrics, f) 151 | 152 | print('results are here:') 153 | print(flags.results_dir) 154 | 155 | 156 | if __name__ == '__main__': 157 | parser = argparse.ArgumentParser(description='IRM Colored MNIST') 158 | parser.add_argument('--hidden_dim', type=int, default=256) 159 | parser.add_argument('--l2_regularizer_weight', type=float,default=0.001) 160 | parser.add_argument('--lr', type=float, default=0.001) 161 | parser.add_argument('--n_restarts', type=int, default=1) 162 | parser.add_argument('--penalty_anneal_iters', type=int, default=100) 163 | parser.add_argument('--penalty_weight', type=float, default=10000.0) 164 | parser.add_argument('--steps', type=int, default=5001) 165 | parser.add_argument('--grayscale_model', action='store_true') 166 | parser.add_argument('--eiil', action='store_true') 167 | parser.add_argument('--results_dir', type=str, default='/tmp/opt_env/irm_cmnist', 168 | help='Directory where results should be saved.') 169 | parser.add_argument('--label_noise', type=float, default=0.25) 170 | parser.add_argument('--train_env_1__color_noise', type=float, default=0.2) 171 | parser.add_argument('--train_env_2__color_noise', type=float, default=0.1) 172 | parser.add_argument('--test_env__color_noise', type=float, default=0.9) 173 | parser.add_argument('--color_based', action='store_true') # use color-based reference classifier without trainable params 174 | parser.add_argument('--color_based_eval', action='store_true') # skip IRM phase and evaluate color-based classifier 175 | flags = parser.parse_args() 176 | torch.cuda.set_device(0) 177 | main(flags) 178 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/code/colored_mnist/main_optenv.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import torch 4 | from torchvision import datasets 5 | from torch import nn, optim, autograd 6 | import pdb 7 | from tqdm import tqdm 8 | 9 | parser = argparse.ArgumentParser(description='Colored MNIST') 10 | parser.add_argument('--hidden_dim', type=int, default=256) 11 | parser.add_argument('--l2_regularizer_weight', type=float,default=0.001) 12 | parser.add_argument('--lr', type=float, default=0.001) 13 | parser.add_argument('--n_restarts', type=int, default=1) 14 | parser.add_argument('--penalty_anneal_iters', type=int, default=100) 15 | parser.add_argument('--penalty_weight', type=float, default=10000.0) 16 | parser.add_argument('--steps', type=int, default=5001) 17 | parser.add_argument('--grayscale_model', action='store_true') 18 | parser.add_argument('--eiil', action='store_true') 19 | flags = parser.parse_args() 20 | torch.cuda.set_device(0) 21 | 22 | print('Flags:') 23 | for k,v in sorted(vars(flags).items()): 24 | print("\t{}: {}".format(k, v)) 25 | 26 | def split_data_opt(envs, model, n_steps=10000, n_samples=-1): 27 | """Learn soft environment assignment.""" 28 | images = torch.cat((envs[0]['images'][:n_samples],envs[1]['images'][:n_samples]),0) 29 | labels = torch.cat((envs[0]['labels'][:n_samples],envs[1]['labels'][:n_samples]),0) 30 | print('size of pooled envs: '+str(len(images))) 31 | 32 | scale = torch.tensor(1.).cuda().requires_grad_() 33 | logits = model(images) 34 | loss = nll(logits * scale, labels, reduction='none') 35 | 36 | env_w = torch.randn(len(logits)).cuda().requires_grad_() 37 | optimizer = optim.Adam([env_w], lr=0.001) 38 | 39 | print('learning soft environment assignments') 40 | for i in tqdm(range(n_steps)): 41 | # penalty for env a 42 | lossa = (loss.squeeze() * env_w.sigmoid()).mean() 43 | grada = autograd.grad(lossa, [scale], create_graph=True)[0] 44 | penaltya = torch.sum(grada**2) 45 | # penalty for env b 46 | lossb = (loss.squeeze() * (1-env_w.sigmoid())).mean() 47 | gradb = autograd.grad(lossb, [scale], create_graph=True)[0] 48 | penaltyb = torch.sum(gradb**2) 49 | # negate 50 | npenalty = - torch.stack([penaltya, penaltyb]).mean() 51 | 52 | optimizer.zero_grad() 53 | npenalty.backward(retain_graph=True) 54 | optimizer.step() 55 | 56 | # split envs based on env_w threshold 57 | new_envs = [] 58 | idx0 = (env_w.sigmoid()>.5) 59 | idx1 = (env_w.sigmoid()<=.5) 60 | # train envs 61 | for idx in (idx0, idx1): 62 | new_envs.append(dict(images=images[idx], labels=labels[idx])) 63 | # test env 64 | new_envs.append(dict(images=envs[-1]['images'], 65 | labels=envs[-1]['labels'])) 66 | print('size of env0: '+str(len(new_envs[0]['images']))) 67 | print('size of env1: '+str(len(new_envs[1]['images']))) 68 | print('size of env2: '+str(len(new_envs[2]['images']))) 69 | return new_envs 70 | 71 | final_train_accs = [] 72 | final_test_accs = [] 73 | for restart in range(flags.n_restarts): 74 | print("Restart", restart) 75 | 76 | # Load MNIST, make train/val splits, and shuffle train set examples 77 | 78 | mnist = datasets.MNIST('~/datasets/mnist', train=True, download=True) 79 | mnist_train = (mnist.data[:50000], mnist.targets[:50000]) 80 | mnist_val = (mnist.data[50000:], mnist.targets[50000:]) 81 | 82 | rng_state = np.random.get_state() 83 | np.random.shuffle(mnist_train[0].numpy()) 84 | np.random.set_state(rng_state) 85 | np.random.shuffle(mnist_train[1].numpy()) 86 | 87 | # Build environments 88 | 89 | def make_environment(images, labels, e): 90 | def torch_bernoulli(p, size): 91 | return (torch.rand(size) < p).float() 92 | def torch_xor(a, b): 93 | return (a-b).abs() # Assumes both inputs are either 0 or 1 94 | # 2x subsample for computational convenience 95 | images = images.reshape((-1, 28, 28))[:, ::2, ::2] 96 | # Assign a binary label based on the digit; flip label with probability 0.25 97 | labels = (labels < 5).float() 98 | labels = torch_xor(labels, torch_bernoulli(0.25, len(labels))) 99 | # Assign a color based on the label; flip the color with probability e 100 | colors = torch_xor(labels, torch_bernoulli(e, len(labels))) 101 | # Apply the color to the image by zeroing out the other color channel 102 | images = torch.stack([images, images], dim=1) 103 | images[torch.tensor(range(len(images))), (1-colors).long(), :, :] *= 0 104 | return { 105 | 'images': (images.float() / 255.).cuda(), 106 | 'labels': labels[:, None].cuda() 107 | } 108 | envs = [ 109 | make_environment(mnist_train[0][::2], mnist_train[1][::2], 0.2), 110 | make_environment(mnist_train[0][1::2], mnist_train[1][1::2], 0.1), 111 | make_environment(mnist_val[0], mnist_val[1], 0.9) 112 | ] 113 | 114 | # Define and instantiate the model 115 | 116 | class MLP(nn.Module): 117 | def __init__(self): 118 | super(MLP, self).__init__() 119 | if flags.grayscale_model: 120 | lin1 = nn.Linear(14 * 14, flags.hidden_dim) 121 | else: 122 | lin1 = nn.Linear(2 * 14 * 14, flags.hidden_dim) 123 | lin2 = nn.Linear(flags.hidden_dim, flags.hidden_dim) 124 | lin3 = nn.Linear(flags.hidden_dim, 1) 125 | for lin in [lin1, lin2, lin3]: 126 | nn.init.xavier_uniform_(lin.weight) 127 | nn.init.zeros_(lin.bias) 128 | self._main = nn.Sequential(lin1, nn.ReLU(True), lin2, nn.ReLU(True), lin3) 129 | def forward(self, input): 130 | if flags.grayscale_model: 131 | out = input.view(input.shape[0], 2, 14 * 14).sum(dim=1) 132 | else: 133 | out = input.view(input.shape[0], 2 * 14 * 14) 134 | out = self._main(out) 135 | return out 136 | 137 | mlp_pre = MLP().cuda() 138 | mlp = MLP().cuda() 139 | 140 | # Define loss function helpers 141 | 142 | def nll(logits, y, reduction='mean'): 143 | return nn.functional.binary_cross_entropy_with_logits(logits, y, reduction=reduction) 144 | 145 | def mean_accuracy(logits, y): 146 | preds = (logits > 0.).float() 147 | return ((preds - y).abs() < 1e-2).float().mean() 148 | 149 | def penalty(logits, y): 150 | scale = torch.tensor(1.).cuda().requires_grad_() 151 | loss = nll(logits * scale, y) 152 | grad = autograd.grad(loss, [scale], create_graph=True)[0] 153 | return torch.sum(grad**2) 154 | 155 | # Train loop 156 | 157 | def pretty_print(*values): 158 | col_width = 13 159 | def format_val(v): 160 | if not isinstance(v, str): 161 | v = np.array2string(v, precision=5, floatmode='fixed') 162 | return v.ljust(col_width) 163 | str_values = [format_val(v) for v in values] 164 | print(" ".join(str_values)) 165 | 166 | optimizer_pre = optim.Adam(mlp_pre.parameters(), lr=flags.lr) 167 | optimizer = optim.Adam(mlp.parameters(), lr=flags.lr) 168 | 169 | pretty_print('step', 'train nll', 'train acc', 'train penalty', 'test acc') 170 | 171 | if flags.eiil: 172 | # Pre-train reference model 173 | for step in range(flags.steps): 174 | for env in envs: 175 | logits = mlp_pre(env['images']) 176 | env['nll'] = nll(logits, env['labels']) 177 | env['acc'] = mean_accuracy(logits, env['labels']) 178 | env['penalty'] = penalty(logits, env['labels']) 179 | 180 | train_nll = torch.stack([envs[0]['nll'], envs[1]['nll']]).mean() 181 | train_acc = torch.stack([envs[0]['acc'], envs[1]['acc']]).mean() 182 | train_penalty = torch.stack([envs[0]['penalty'], envs[1]['penalty']]).mean() 183 | 184 | weight_norm = torch.tensor(0.).cuda() 185 | for w in mlp_pre.parameters(): 186 | weight_norm += w.norm().pow(2) 187 | 188 | loss = train_nll.clone() 189 | loss += flags.l2_regularizer_weight * weight_norm 190 | # NOTE: IRM penalties not used in pre-training 191 | #penalty_weight = (flags.penalty_weight 192 | # if step >= flags.penalty_anneal_iters else 1.0) 193 | #loss += penalty_weight * train_penalty 194 | #if penalty_weight > 1.0: 195 | # # Rescale the entire loss to keep gradients in a reasonable range 196 | # loss /= penalty_weight 197 | 198 | optimizer_pre.zero_grad() 199 | loss.backward() 200 | optimizer_pre.step() 201 | 202 | test_acc = envs[2]['acc'] 203 | if step % 100 == 0: 204 | pretty_print( 205 | np.int32(step), 206 | train_nll.detach().cpu().numpy(), 207 | train_acc.detach().cpu().numpy(), 208 | train_penalty.detach().cpu().numpy(), 209 | test_acc.detach().cpu().numpy() 210 | ) 211 | 212 | envs = split_data_opt(envs, mlp_pre) 213 | 214 | for step in range(flags.steps): 215 | for env in envs: 216 | logits = mlp(env['images']) 217 | env['nll'] = nll(logits, env['labels']) 218 | env['acc'] = mean_accuracy(logits, env['labels']) 219 | env['penalty'] = penalty(logits, env['labels']) 220 | 221 | train_nll = torch.stack([envs[0]['nll'], envs[1]['nll']]).mean() 222 | train_acc = torch.stack([envs[0]['acc'], envs[1]['acc']]).mean() 223 | train_penalty = torch.stack([envs[0]['penalty'], envs[1]['penalty']]).mean() 224 | 225 | weight_norm = torch.tensor(0.).cuda() 226 | for w in mlp.parameters(): 227 | weight_norm += w.norm().pow(2) 228 | 229 | loss = train_nll.clone() 230 | loss += flags.l2_regularizer_weight * weight_norm 231 | penalty_weight = (flags.penalty_weight 232 | if step >= flags.penalty_anneal_iters else 1.0) 233 | loss += penalty_weight * train_penalty 234 | if penalty_weight > 1.0: 235 | # Rescale the entire loss to keep gradients in a reasonable range 236 | loss /= penalty_weight 237 | 238 | optimizer.zero_grad() 239 | loss.backward() 240 | optimizer.step() 241 | 242 | test_acc = envs[2]['acc'] 243 | if step % 100 == 0: 244 | pretty_print( 245 | np.int32(step), 246 | train_nll.detach().cpu().numpy(), 247 | train_acc.detach().cpu().numpy(), 248 | train_penalty.detach().cpu().numpy(), 249 | test_acc.detach().cpu().numpy() 250 | ) 251 | 252 | final_train_accs.append(train_acc.detach().cpu().numpy()) 253 | final_test_accs.append(test_acc.detach().cpu().numpy()) 254 | print('Final train acc (mean/std across restarts so far):') 255 | print(np.mean(final_train_accs), np.std(final_train_accs)) 256 | print('Final test acc (mean/std across restarts so far):') 257 | print(np.mean(final_test_accs), np.std(final_test_accs)) 258 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/code/experiment_synthetic/models.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Facebook, Inc. and its affiliates. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | 8 | import numpy as np 9 | import torch 10 | import math 11 | 12 | from sklearn.linear_model import LinearRegression 13 | from itertools import chain, combinations 14 | from scipy.stats import f as fdist 15 | from scipy.stats import ttest_ind 16 | 17 | from torch.autograd import grad 18 | 19 | import scipy.optimize 20 | 21 | import matplotlib 22 | import matplotlib.pyplot as plt 23 | from tqdm import tqdm 24 | import pdb 25 | 26 | 27 | def pretty(vector): 28 | vlist = vector.view(-1).tolist() 29 | return "[" + ", ".join("{:+.4f}".format(vi) for vi in vlist) + "]" 30 | 31 | 32 | 33 | class InvariantRiskMinimization(object): 34 | def __init__(self, environments, args): 35 | best_reg = 0 36 | best_err = 1e6 37 | 38 | x_val = environments[-1][0] 39 | y_val = environments[-1][1] 40 | 41 | for reg in [0, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1]: 42 | reg = 1. - reg # change of variables for consistency with old codebase 43 | self.train(environments[:-1], args, reg=reg) 44 | err = (x_val @ self.solution() - y_val).pow(2).mean().item() 45 | 46 | if args["verbose"]: 47 | print("IRM (reg={:.6f}) has {:.3f} validation error.".format( 48 | reg, err)) 49 | 50 | if err < best_err: 51 | best_err = err 52 | best_reg = reg 53 | best_phi = self.phi.clone() 54 | self.phi = best_phi 55 | 56 | def train(self, environments, args, reg=0): 57 | print('learning representation with', self, 'and reg', reg) 58 | dim_x = environments[0][0].size(1) 59 | 60 | self.phi = torch.nn.Parameter(torch.eye(dim_x, dim_x)) 61 | self.w = torch.ones(dim_x, 1) 62 | self.w.requires_grad = True 63 | 64 | opt = torch.optim.Adam([self.phi], lr=args["lr"]) 65 | loss = torch.nn.MSELoss() 66 | 67 | for iteration in range(args["n_iterations"]): 68 | penalty = 0 69 | error = 0 70 | for x_e, y_e in environments: 71 | error_e = loss(x_e @ self.phi @ self.w, y_e) 72 | penalty += grad(error_e, self.w, 73 | create_graph=True)[0].pow(2).mean() 74 | error += error_e 75 | 76 | opt.zero_grad() 77 | # (reg * error + (1 - reg) * penalty).backward() # dumb; zero reg means regularize 100% 78 | ((1 - reg) * error + reg * penalty).backward() # good 79 | opt.step() 80 | 81 | if args["verbose"] and iteration % 1000 == 0: 82 | w_str = pretty(self.solution()) 83 | print("{:05d} | {:.5f} | {:.5f} | {:.5f} | {}".format(iteration, 84 | reg, 85 | error, 86 | penalty, 87 | w_str)) 88 | 89 | def solution(self): 90 | return self.phi @ self.w 91 | 92 | 93 | class LearnedEnvInvariantRiskMinimization(InvariantRiskMinimization): 94 | def __init__(self, environments, args, pretrain=False): 95 | best_reg = 0 96 | best_err = 1e6 97 | 98 | x_val = environments[-1][0] 99 | y_val = environments[-1][1] 100 | 101 | if args['eiil_ref_alpha'] >= 0 and args['eiil_ref_alpha'] <= 1: 102 | print('Using hard-coded reference classifier with alpha={:.2f}'.format( 103 | args['eiil_ref_alpha'] 104 | )) 105 | alpha = args['eiil_ref_alpha'] 106 | w_causal = (1. - alpha) * np.ones((1, 5)) 107 | w_noncausal = alpha * np.ones((1, 5)) # spurious contribution to prediction 108 | w_ref = np.hstack((w_causal, w_noncausal)) 109 | w_ref = torch.tensor(w_ref, dtype=torch.float32) 110 | else: 111 | print('Using ERM soln as reference classifier.') 112 | w_ref = EmpiricalRiskMinimizer(environments, args).solution() 113 | 114 | self.phi = torch.nn.Parameter(torch.diag(w_ref.squeeze())) 115 | dim_x = environments[0][0].size(1) 116 | self.w = torch.ones(dim_x, 1) 117 | self.w.requires_grad = True 118 | err = (x_val @ self.solution() - y_val).pow(2).mean().item() 119 | 120 | if args["verbose"]: 121 | print("EIIL's reference classifier has {:.3f} validation error.".format( 122 | err)) 123 | print("EIIL's reference classifier has the following solution:\n.", 124 | pretty(self.solution())) 125 | 126 | self.phi = self.phi.clone() 127 | 128 | environments = self.split(environments, args) 129 | if args["verbose"]: 130 | print("EIIL+ERM ref clf still has the following solution after AED (sanity check):\n.", pretty(self.solution())) 131 | best_reg = 0 132 | best_err = 1e6 133 | 134 | for reg in [0, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1]: 135 | reg = 1. - reg # change of variables for consistency with old codebase 136 | self.train(environments[:-1], args, reg=reg) 137 | err = (x_val @ self.solution() - y_val).pow(2).mean().item() 138 | 139 | if args["verbose"]: 140 | print("EIIL+IRM (reg={:.6f}) has {:.3f} validation error.".format( 141 | reg, err)) 142 | 143 | if err < best_err: 144 | best_err = err 145 | best_reg = reg 146 | best_phi = self.phi.clone() 147 | self.phi = best_phi 148 | 149 | 150 | def split(self, environments, args, n_samples=-1): 151 | """Learn soft environment assignment.""" 152 | envs = environments 153 | test_env = envs[-1] 154 | x = torch.cat((envs[0][0][:n_samples],envs[1][0][:n_samples]),0) 155 | y = torch.cat((envs[0][1][:n_samples],envs[1][1][:n_samples]),0) 156 | print('size of pooled envs: '+str(len(x))) 157 | 158 | loss = torch.nn.MSELoss(reduction='none') 159 | error = loss(x @ self.phi @ self.w, y) 160 | 161 | env_w = torch.randn(len(error)).requires_grad_() 162 | optimizer = torch.optim.Adam([env_w], lr=0.001) 163 | 164 | print('learning soft environment assignments') 165 | with tqdm(total=args['n_iterations'], 166 | position=1, 167 | bar_format='{desc}', 168 | desc='negative penalty: ', 169 | ) as desc: 170 | for i in tqdm(range(args["n_iterations"])): 171 | # penalty for env a 172 | error_a = (error.squeeze() * env_w.sigmoid()).mean() 173 | penalty_a = grad(error_a, self.w, create_graph=True)[0].pow(2).mean() 174 | # penalty for env b 175 | error_b = (error.squeeze() * (1-env_w.sigmoid())).mean() 176 | penalty_b = grad(error_b, self.w, create_graph=True)[0].pow(2).mean() 177 | # negate 178 | npenalty = - torch.stack([penalty_a, penalty_b]).mean() 179 | desc.set_description('negative penalty: '+ str(npenalty)) 180 | 181 | optimizer.zero_grad() 182 | npenalty.backward(retain_graph=True) 183 | optimizer.step() 184 | 185 | envs = [] 186 | idx0 = (env_w.sigmoid()>.5) 187 | idx1 = (env_w.sigmoid()<=.5) 188 | envs.append((x[idx0],y[idx0])) 189 | print('size of env 0: '+str(len(x[idx0]))) 190 | envs.append((x[idx1],y[idx1])) 191 | print('size of env 1: '+str(len(x[idx1]))) 192 | print('weights: '+str(env_w.sigmoid())) 193 | envs.append(test_env) 194 | return envs 195 | 196 | 197 | class InvariantCausalPrediction(object): 198 | def __init__(self, environments, args): 199 | self.coefficients = None 200 | self.alpha = args["alpha"] 201 | 202 | x_all = [] 203 | y_all = [] 204 | e_all = [] 205 | 206 | for e, (x, y) in enumerate(environments): 207 | x_all.append(x.numpy()) 208 | y_all.append(y.numpy()) 209 | e_all.append(np.full(x.shape[0], e)) 210 | 211 | x_all = np.vstack(x_all) 212 | y_all = np.vstack(y_all) 213 | e_all = np.hstack(e_all) 214 | 215 | dim = x_all.shape[1] 216 | 217 | accepted_subsets = [] 218 | for subset in self.powerset(range(dim)): 219 | if len(subset) == 0: 220 | continue 221 | 222 | x_s = x_all[:, subset] 223 | reg = LinearRegression(fit_intercept=False).fit(x_s, y_all) 224 | 225 | p_values = [] 226 | for e in range(len(environments)): 227 | e_in = np.where(e_all == e)[0] 228 | e_out = np.where(e_all != e)[0] 229 | 230 | res_in = (y_all[e_in] - reg.predict(x_s[e_in, :])).ravel() 231 | res_out = (y_all[e_out] - reg.predict(x_s[e_out, :])).ravel() 232 | 233 | p_values.append(self.mean_var_test(res_in, res_out)) 234 | 235 | # TODO: Jonas uses "min(p_values) * len(environments) - 1" 236 | p_value = min(p_values) * len(environments) 237 | 238 | if p_value > self.alpha: 239 | accepted_subsets.append(set(subset)) 240 | if args["verbose"]: 241 | print("Accepted subset:", subset) 242 | 243 | if len(accepted_subsets): 244 | accepted_features = list(set.intersection(*accepted_subsets)) 245 | if args["verbose"]: 246 | print("Intersection:", accepted_features) 247 | self.coefficients = np.zeros(dim) 248 | 249 | if len(accepted_features): 250 | x_s = x_all[:, list(accepted_features)] 251 | reg = LinearRegression(fit_intercept=False).fit(x_s, y_all) 252 | self.coefficients[list(accepted_features)] = reg.coef_ 253 | 254 | self.coefficients = torch.Tensor(self.coefficients) 255 | else: 256 | self.coefficients = torch.zeros(dim) 257 | 258 | def mean_var_test(self, x, y): 259 | pvalue_mean = ttest_ind(x, y, equal_var=False).pvalue 260 | pvalue_var1 = 1 - fdist.cdf(np.var(x, ddof=1) / np.var(y, ddof=1), 261 | x.shape[0] - 1, 262 | y.shape[0] - 1) 263 | 264 | pvalue_var2 = 2 * min(pvalue_var1, 1 - pvalue_var1) 265 | 266 | return 2 * min(pvalue_mean, pvalue_var2) 267 | 268 | def powerset(self, s): 269 | return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1)) 270 | 271 | def solution(self): 272 | return self.coefficients 273 | 274 | 275 | class EmpiricalRiskMinimizer(object): 276 | def __init__(self, environments, args): 277 | x_all = torch.cat([x for (x, y) in environments]).numpy() 278 | y_all = torch.cat([y for (x, y) in environments]).numpy() 279 | 280 | w = LinearRegression(fit_intercept=False).fit(x_all, y_all).coef_ 281 | self.w = torch.Tensor(w) 282 | if args['verbose']: 283 | print('Done training ERM.') 284 | err = np.mean((x_all.dot(self.solution().T) - y_all) ** 2.).item() 285 | print("ERM has {:.3f} train error.".format(err)) 286 | print("ERM has the following solution:\n ", pretty(self.solution())) 287 | 288 | def solution(self): 289 | return self.w 290 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Attribution-NonCommercial 4.0 International 2 | 3 | ======================================================================= 4 | 5 | Creative Commons Corporation ("Creative Commons") is not a law firm and 6 | does not provide legal services or legal advice. Distribution of 7 | Creative Commons public licenses does not create a lawyer-client or 8 | other relationship. Creative Commons makes its licenses and related 9 | information available on an "as-is" basis. Creative Commons gives no 10 | warranties regarding its licenses, any material licensed under their 11 | terms and conditions, or any related information. Creative Commons 12 | disclaims all liability for damages resulting from their use to the 13 | fullest extent possible. 14 | 15 | Using Creative Commons Public Licenses 16 | 17 | Creative Commons public licenses provide a standard set of terms and 18 | conditions that creators and other rights holders may use to share 19 | original works of authorship and other material subject to copyright 20 | and certain other rights specified in the public license below. The 21 | following considerations are for informational purposes only, are not 22 | exhaustive, and do not form part of our licenses. 23 | 24 | Considerations for licensors: Our public licenses are 25 | intended for use by those authorized to give the public 26 | permission to use material in ways otherwise restricted by 27 | copyright and certain other rights. Our licenses are 28 | irrevocable. Licensors should read and understand the terms 29 | and conditions of the license they choose before applying it. 30 | Licensors should also secure all rights necessary before 31 | applying our licenses so that the public can reuse the 32 | material as expected. Licensors should clearly mark any 33 | material not subject to the license. This includes other CC- 34 | licensed material, or material used under an exception or 35 | limitation to copyright. More considerations for licensors: 36 | wiki.creativecommons.org/Considerations_for_licensors 37 | 38 | Considerations for the public: By using one of our public 39 | licenses, a licensor grants the public permission to use the 40 | licensed material under specified terms and conditions. If 41 | the licensor's permission is not necessary for any reason--for 42 | example, because of any applicable exception or limitation to 43 | copyright--then that use is not regulated by the license. Our 44 | licenses grant only permissions under copyright and certain 45 | other rights that a licensor has authority to grant. Use of 46 | the licensed material may still be restricted for other 47 | reasons, including because others have copyright or other 48 | rights in the material. A licensor may make special requests, 49 | such as asking that all changes be marked or described. 50 | Although not required by our licenses, you are encouraged to 51 | respect those requests where reasonable. More_considerations 52 | for the public: 53 | wiki.creativecommons.org/Considerations_for_licensees 54 | 55 | ======================================================================= 56 | 57 | Creative Commons Attribution-NonCommercial 4.0 International Public 58 | License 59 | 60 | By exercising the Licensed Rights (defined below), You accept and agree 61 | to be bound by the terms and conditions of this Creative Commons 62 | Attribution-NonCommercial 4.0 International Public License ("Public 63 | License"). To the extent this Public License may be interpreted as a 64 | contract, You are granted the Licensed Rights in consideration of Your 65 | acceptance of these terms and conditions, and the Licensor grants You 66 | such rights in consideration of benefits the Licensor receives from 67 | making the Licensed Material available under these terms and 68 | conditions. 69 | 70 | Section 1 -- Definitions. 71 | 72 | a. Adapted Material means material subject to Copyright and Similar 73 | Rights that is derived from or based upon the Licensed Material 74 | and in which the Licensed Material is translated, altered, 75 | arranged, transformed, or otherwise modified in a manner requiring 76 | permission under the Copyright and Similar Rights held by the 77 | Licensor. For purposes of this Public License, where the Licensed 78 | Material is a musical work, performance, or sound recording, 79 | Adapted Material is always produced where the Licensed Material is 80 | synched in timed relation with a moving image. 81 | 82 | b. Adapter's License means the license You apply to Your Copyright 83 | and Similar Rights in Your contributions to Adapted Material in 84 | accordance with the terms and conditions of this Public License. 85 | 86 | c. Copyright and Similar Rights means copyright and/or similar rights 87 | closely related to copyright including, without limitation, 88 | performance, broadcast, sound recording, and Sui Generis Database 89 | Rights, without regard to how the rights are labeled or 90 | categorized. For purposes of this Public License, the rights 91 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 92 | Rights. 93 | d. Effective Technological Measures means those measures that, in the 94 | absence of proper authority, may not be circumvented under laws 95 | fulfilling obligations under Article 11 of the WIPO Copyright 96 | Treaty adopted on December 20, 1996, and/or similar international 97 | agreements. 98 | 99 | e. Exceptions and Limitations means fair use, fair dealing, and/or 100 | any other exception or limitation to Copyright and Similar Rights 101 | that applies to Your use of the Licensed Material. 102 | 103 | f. Licensed Material means the artistic or literary work, database, 104 | or other material to which the Licensor applied this Public 105 | License. 106 | 107 | g. Licensed Rights means the rights granted to You subject to the 108 | terms and conditions of this Public License, which are limited to 109 | all Copyright and Similar Rights that apply to Your use of the 110 | Licensed Material and that the Licensor has authority to license. 111 | 112 | h. Licensor means the individual(s) or entity(ies) granting rights 113 | under this Public License. 114 | 115 | i. NonCommercial means not primarily intended for or directed towards 116 | commercial advantage or monetary compensation. For purposes of 117 | this Public License, the exchange of the Licensed Material for 118 | other material subject to Copyright and Similar Rights by digital 119 | file-sharing or similar means is NonCommercial provided there is 120 | no payment of monetary compensation in connection with the 121 | exchange. 122 | 123 | j. Share means to provide material to the public by any means or 124 | process that requires permission under the Licensed Rights, such 125 | as reproduction, public display, public performance, distribution, 126 | dissemination, communication, or importation, and to make material 127 | available to the public including in ways that members of the 128 | public may access the material from a place and at a time 129 | individually chosen by them. 130 | 131 | k. Sui Generis Database Rights means rights other than copyright 132 | resulting from Directive 96/9/EC of the European Parliament and of 133 | the Council of 11 March 1996 on the legal protection of databases, 134 | as amended and/or succeeded, as well as other essentially 135 | equivalent rights anywhere in the world. 136 | 137 | l. You means the individual or entity exercising the Licensed Rights 138 | under this Public License. Your has a corresponding meaning. 139 | 140 | Section 2 -- Scope. 141 | 142 | a. License grant. 143 | 144 | 1. Subject to the terms and conditions of this Public License, 145 | the Licensor hereby grants You a worldwide, royalty-free, 146 | non-sublicensable, non-exclusive, irrevocable license to 147 | exercise the Licensed Rights in the Licensed Material to: 148 | 149 | a. reproduce and Share the Licensed Material, in whole or 150 | in part, for NonCommercial purposes only; and 151 | 152 | b. produce, reproduce, and Share Adapted Material for 153 | NonCommercial purposes only. 154 | 155 | 2. Exceptions and Limitations. For the avoidance of doubt, where 156 | Exceptions and Limitations apply to Your use, this Public 157 | License does not apply, and You do not need to comply with 158 | its terms and conditions. 159 | 160 | 3. Term. The term of this Public License is specified in Section 161 | 6(a). 162 | 163 | 4. Media and formats; technical modifications allowed. The 164 | Licensor authorizes You to exercise the Licensed Rights in 165 | all media and formats whether now known or hereafter created, 166 | and to make technical modifications necessary to do so. The 167 | Licensor waives and/or agrees not to assert any right or 168 | authority to forbid You from making technical modifications 169 | necessary to exercise the Licensed Rights, including 170 | technical modifications necessary to circumvent Effective 171 | Technological Measures. For purposes of this Public License, 172 | simply making modifications authorized by this Section 2(a) 173 | (4) never produces Adapted Material. 174 | 175 | 5. Downstream recipients. 176 | 177 | a. Offer from the Licensor -- Licensed Material. Every 178 | recipient of the Licensed Material automatically 179 | receives an offer from the Licensor to exercise the 180 | Licensed Rights under the terms and conditions of this 181 | Public License. 182 | 183 | b. No downstream restrictions. You may not offer or impose 184 | any additional or different terms or conditions on, or 185 | apply any Effective Technological Measures to, the 186 | Licensed Material if doing so restricts exercise of the 187 | Licensed Rights by any recipient of the Licensed 188 | Material. 189 | 190 | 6. No endorsement. Nothing in this Public License constitutes or 191 | may be construed as permission to assert or imply that You 192 | are, or that Your use of the Licensed Material is, connected 193 | with, or sponsored, endorsed, or granted official status by, 194 | the Licensor or others designated to receive attribution as 195 | provided in Section 3(a)(1)(A)(i). 196 | 197 | b. Other rights. 198 | 199 | 1. Moral rights, such as the right of integrity, are not 200 | licensed under this Public License, nor are publicity, 201 | privacy, and/or other similar personality rights; however, to 202 | the extent possible, the Licensor waives and/or agrees not to 203 | assert any such rights held by the Licensor to the limited 204 | extent necessary to allow You to exercise the Licensed 205 | Rights, but not otherwise. 206 | 207 | 2. Patent and trademark rights are not licensed under this 208 | Public License. 209 | 210 | 3. To the extent possible, the Licensor waives any right to 211 | collect royalties from You for the exercise of the Licensed 212 | Rights, whether directly or through a collecting society 213 | under any voluntary or waivable statutory or compulsory 214 | licensing scheme. In all other cases the Licensor expressly 215 | reserves any right to collect such royalties, including when 216 | the Licensed Material is used other than for NonCommercial 217 | purposes. 218 | 219 | Section 3 -- License Conditions. 220 | 221 | Your exercise of the Licensed Rights is expressly made subject to the 222 | following conditions. 223 | 224 | a. Attribution. 225 | 226 | 1. If You Share the Licensed Material (including in modified 227 | form), You must: 228 | 229 | a. retain the following if it is supplied by the Licensor 230 | with the Licensed Material: 231 | 232 | i. identification of the creator(s) of the Licensed 233 | Material and any others designated to receive 234 | attribution, in any reasonable manner requested by 235 | the Licensor (including by pseudonym if 236 | designated); 237 | 238 | ii. a copyright notice; 239 | 240 | iii. a notice that refers to this Public License; 241 | 242 | iv. a notice that refers to the disclaimer of 243 | warranties; 244 | 245 | v. a URI or hyperlink to the Licensed Material to the 246 | extent reasonably practicable; 247 | 248 | b. indicate if You modified the Licensed Material and 249 | retain an indication of any previous modifications; and 250 | 251 | c. indicate the Licensed Material is licensed under this 252 | Public License, and include the text of, or the URI or 253 | hyperlink to, this Public License. 254 | 255 | 2. You may satisfy the conditions in Section 3(a)(1) in any 256 | reasonable manner based on the medium, means, and context in 257 | which You Share the Licensed Material. For example, it may be 258 | reasonable to satisfy the conditions by providing a URI or 259 | hyperlink to a resource that includes the required 260 | information. 261 | 262 | 3. If requested by the Licensor, You must remove any of the 263 | information required by Section 3(a)(1)(A) to the extent 264 | reasonably practicable. 265 | 266 | 4. If You Share Adapted Material You produce, the Adapter's 267 | License You apply must not prevent recipients of the Adapted 268 | Material from complying with this Public License. 269 | 270 | Section 4 -- Sui Generis Database Rights. 271 | 272 | Where the Licensed Rights include Sui Generis Database Rights that 273 | apply to Your use of the Licensed Material: 274 | 275 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 276 | to extract, reuse, reproduce, and Share all or a substantial 277 | portion of the contents of the database for NonCommercial purposes 278 | only; 279 | 280 | b. if You include all or a substantial portion of the database 281 | contents in a database in which You have Sui Generis Database 282 | Rights, then the database in which You have Sui Generis Database 283 | Rights (but not its individual contents) is Adapted Material; and 284 | 285 | c. You must comply with the conditions in Section 3(a) if You Share 286 | all or a substantial portion of the contents of the database. 287 | 288 | For the avoidance of doubt, this Section 4 supplements and does not 289 | replace Your obligations under this Public License where the Licensed 290 | Rights include other Copyright and Similar Rights. 291 | 292 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 293 | 294 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 295 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 296 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 297 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 298 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 299 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 300 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 301 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 302 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 303 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 304 | 305 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 306 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 307 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 308 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 309 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 310 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 311 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 312 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 313 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 314 | 315 | c. The disclaimer of warranties and limitation of liability provided 316 | above shall be interpreted in a manner that, to the extent 317 | possible, most closely approximates an absolute disclaimer and 318 | waiver of all liability. 319 | 320 | Section 6 -- Term and Termination. 321 | 322 | a. This Public License applies for the term of the Copyright and 323 | Similar Rights licensed here. However, if You fail to comply with 324 | this Public License, then Your rights under this Public License 325 | terminate automatically. 326 | 327 | b. Where Your right to use the Licensed Material has terminated under 328 | Section 6(a), it reinstates: 329 | 330 | 1. automatically as of the date the violation is cured, provided 331 | it is cured within 30 days of Your discovery of the 332 | violation; or 333 | 334 | 2. upon express reinstatement by the Licensor. 335 | 336 | For the avoidance of doubt, this Section 6(b) does not affect any 337 | right the Licensor may have to seek remedies for Your violations 338 | of this Public License. 339 | 340 | c. For the avoidance of doubt, the Licensor may also offer the 341 | Licensed Material under separate terms or conditions or stop 342 | distributing the Licensed Material at any time; however, doing so 343 | will not terminate this Public License. 344 | 345 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 346 | License. 347 | 348 | Section 7 -- Other Terms and Conditions. 349 | 350 | a. The Licensor shall not be bound by any additional or different 351 | terms or conditions communicated by You unless expressly agreed. 352 | 353 | b. Any arrangements, understandings, or agreements regarding the 354 | Licensed Material not stated herein are separate from and 355 | independent of the terms and conditions of this Public License. 356 | 357 | Section 8 -- Interpretation. 358 | 359 | a. For the avoidance of doubt, this Public License does not, and 360 | shall not be interpreted to, reduce, limit, restrict, or impose 361 | conditions on any use of the Licensed Material that could lawfully 362 | be made without permission under this Public License. 363 | 364 | b. To the extent possible, if any provision of this Public License is 365 | deemed unenforceable, it shall be automatically reformed to the 366 | minimum extent necessary to make it enforceable. If the provision 367 | cannot be reformed, it shall be severed from this Public License 368 | without affecting the enforceability of the remaining terms and 369 | conditions. 370 | 371 | c. No term or condition of this Public License will be waived and no 372 | failure to comply consented to unless expressly agreed to by the 373 | Licensor. 374 | 375 | d. Nothing in this Public License constitutes or may be interpreted 376 | as a limitation upon, or waiver of, any privileges and immunities 377 | that apply to the Licensor or You, including from the legal 378 | processes of any jurisdiction or authority. 379 | 380 | ======================================================================= 381 | 382 | Creative Commons is not a party to its public 383 | licenses. Notwithstanding, Creative Commons may elect to apply one of 384 | its public licenses to material it publishes and in those instances 385 | will be considered the “Licensor.” The text of the Creative Commons 386 | public licenses is dedicated to the public domain under the CC0 Public 387 | Domain Dedication. Except for the limited purpose of indicating that 388 | material is shared under a Creative Commons public license or as 389 | otherwise permitted by the Creative Commons policies published at 390 | creativecommons.org/policies, Creative Commons does not authorize the 391 | use of the trademark "Creative Commons" or any other trademark or logo 392 | of Creative Commons without its prior written consent including, 393 | without limitation, in connection with any unauthorized modifications 394 | to any of its public licenses or any other arrangements, 395 | understandings, or agreements concerning use of licensed material. For 396 | the avoidance of doubt, this paragraph does not form part of the 397 | public licenses. 398 | 399 | Creative Commons may be contacted at creativecommons.org. 400 | 401 | -------------------------------------------------------------------------------- /InvariantRiskMinimization/LICENSE: -------------------------------------------------------------------------------- 1 | Attribution-NonCommercial 4.0 International 2 | 3 | ======================================================================= 4 | 5 | Creative Commons Corporation ("Creative Commons") is not a law firm and 6 | does not provide legal services or legal advice. Distribution of 7 | Creative Commons public licenses does not create a lawyer-client or 8 | other relationship. Creative Commons makes its licenses and related 9 | information available on an "as-is" basis. Creative Commons gives no 10 | warranties regarding its licenses, any material licensed under their 11 | terms and conditions, or any related information. Creative Commons 12 | disclaims all liability for damages resulting from their use to the 13 | fullest extent possible. 14 | 15 | Using Creative Commons Public Licenses 16 | 17 | Creative Commons public licenses provide a standard set of terms and 18 | conditions that creators and other rights holders may use to share 19 | original works of authorship and other material subject to copyright 20 | and certain other rights specified in the public license below. The 21 | following considerations are for informational purposes only, are not 22 | exhaustive, and do not form part of our licenses. 23 | 24 | Considerations for licensors: Our public licenses are 25 | intended for use by those authorized to give the public 26 | permission to use material in ways otherwise restricted by 27 | copyright and certain other rights. Our licenses are 28 | irrevocable. Licensors should read and understand the terms 29 | and conditions of the license they choose before applying it. 30 | Licensors should also secure all rights necessary before 31 | applying our licenses so that the public can reuse the 32 | material as expected. Licensors should clearly mark any 33 | material not subject to the license. This includes other CC- 34 | licensed material, or material used under an exception or 35 | limitation to copyright. More considerations for licensors: 36 | wiki.creativecommons.org/Considerations_for_licensors 37 | 38 | Considerations for the public: By using one of our public 39 | licenses, a licensor grants the public permission to use the 40 | licensed material under specified terms and conditions. If 41 | the licensor's permission is not necessary for any reason--for 42 | example, because of any applicable exception or limitation to 43 | copyright--then that use is not regulated by the license. Our 44 | licenses grant only permissions under copyright and certain 45 | other rights that a licensor has authority to grant. Use of 46 | the licensed material may still be restricted for other 47 | reasons, including because others have copyright or other 48 | rights in the material. A licensor may make special requests, 49 | such as asking that all changes be marked or described. 50 | Although not required by our licenses, you are encouraged to 51 | respect those requests where reasonable. More_considerations 52 | for the public: 53 | wiki.creativecommons.org/Considerations_for_licensees 54 | 55 | ======================================================================= 56 | 57 | Creative Commons Attribution-NonCommercial 4.0 International Public 58 | License 59 | 60 | By exercising the Licensed Rights (defined below), You accept and agree 61 | to be bound by the terms and conditions of this Creative Commons 62 | Attribution-NonCommercial 4.0 International Public License ("Public 63 | License"). To the extent this Public License may be interpreted as a 64 | contract, You are granted the Licensed Rights in consideration of Your 65 | acceptance of these terms and conditions, and the Licensor grants You 66 | such rights in consideration of benefits the Licensor receives from 67 | making the Licensed Material available under these terms and 68 | conditions. 69 | 70 | Section 1 -- Definitions. 71 | 72 | a. Adapted Material means material subject to Copyright and Similar 73 | Rights that is derived from or based upon the Licensed Material 74 | and in which the Licensed Material is translated, altered, 75 | arranged, transformed, or otherwise modified in a manner requiring 76 | permission under the Copyright and Similar Rights held by the 77 | Licensor. For purposes of this Public License, where the Licensed 78 | Material is a musical work, performance, or sound recording, 79 | Adapted Material is always produced where the Licensed Material is 80 | synched in timed relation with a moving image. 81 | 82 | b. Adapter's License means the license You apply to Your Copyright 83 | and Similar Rights in Your contributions to Adapted Material in 84 | accordance with the terms and conditions of this Public License. 85 | 86 | c. Copyright and Similar Rights means copyright and/or similar rights 87 | closely related to copyright including, without limitation, 88 | performance, broadcast, sound recording, and Sui Generis Database 89 | Rights, without regard to how the rights are labeled or 90 | categorized. For purposes of this Public License, the rights 91 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 92 | Rights. 93 | d. Effective Technological Measures means those measures that, in the 94 | absence of proper authority, may not be circumvented under laws 95 | fulfilling obligations under Article 11 of the WIPO Copyright 96 | Treaty adopted on December 20, 1996, and/or similar international 97 | agreements. 98 | 99 | e. Exceptions and Limitations means fair use, fair dealing, and/or 100 | any other exception or limitation to Copyright and Similar Rights 101 | that applies to Your use of the Licensed Material. 102 | 103 | f. Licensed Material means the artistic or literary work, database, 104 | or other material to which the Licensor applied this Public 105 | License. 106 | 107 | g. Licensed Rights means the rights granted to You subject to the 108 | terms and conditions of this Public License, which are limited to 109 | all Copyright and Similar Rights that apply to Your use of the 110 | Licensed Material and that the Licensor has authority to license. 111 | 112 | h. Licensor means the individual(s) or entity(ies) granting rights 113 | under this Public License. 114 | 115 | i. NonCommercial means not primarily intended for or directed towards 116 | commercial advantage or monetary compensation. For purposes of 117 | this Public License, the exchange of the Licensed Material for 118 | other material subject to Copyright and Similar Rights by digital 119 | file-sharing or similar means is NonCommercial provided there is 120 | no payment of monetary compensation in connection with the 121 | exchange. 122 | 123 | j. Share means to provide material to the public by any means or 124 | process that requires permission under the Licensed Rights, such 125 | as reproduction, public display, public performance, distribution, 126 | dissemination, communication, or importation, and to make material 127 | available to the public including in ways that members of the 128 | public may access the material from a place and at a time 129 | individually chosen by them. 130 | 131 | k. Sui Generis Database Rights means rights other than copyright 132 | resulting from Directive 96/9/EC of the European Parliament and of 133 | the Council of 11 March 1996 on the legal protection of databases, 134 | as amended and/or succeeded, as well as other essentially 135 | equivalent rights anywhere in the world. 136 | 137 | l. You means the individual or entity exercising the Licensed Rights 138 | under this Public License. Your has a corresponding meaning. 139 | 140 | Section 2 -- Scope. 141 | 142 | a. License grant. 143 | 144 | 1. Subject to the terms and conditions of this Public License, 145 | the Licensor hereby grants You a worldwide, royalty-free, 146 | non-sublicensable, non-exclusive, irrevocable license to 147 | exercise the Licensed Rights in the Licensed Material to: 148 | 149 | a. reproduce and Share the Licensed Material, in whole or 150 | in part, for NonCommercial purposes only; and 151 | 152 | b. produce, reproduce, and Share Adapted Material for 153 | NonCommercial purposes only. 154 | 155 | 2. Exceptions and Limitations. For the avoidance of doubt, where 156 | Exceptions and Limitations apply to Your use, this Public 157 | License does not apply, and You do not need to comply with 158 | its terms and conditions. 159 | 160 | 3. Term. The term of this Public License is specified in Section 161 | 6(a). 162 | 163 | 4. Media and formats; technical modifications allowed. The 164 | Licensor authorizes You to exercise the Licensed Rights in 165 | all media and formats whether now known or hereafter created, 166 | and to make technical modifications necessary to do so. The 167 | Licensor waives and/or agrees not to assert any right or 168 | authority to forbid You from making technical modifications 169 | necessary to exercise the Licensed Rights, including 170 | technical modifications necessary to circumvent Effective 171 | Technological Measures. For purposes of this Public License, 172 | simply making modifications authorized by this Section 2(a) 173 | (4) never produces Adapted Material. 174 | 175 | 5. Downstream recipients. 176 | 177 | a. Offer from the Licensor -- Licensed Material. Every 178 | recipient of the Licensed Material automatically 179 | receives an offer from the Licensor to exercise the 180 | Licensed Rights under the terms and conditions of this 181 | Public License. 182 | 183 | b. No downstream restrictions. You may not offer or impose 184 | any additional or different terms or conditions on, or 185 | apply any Effective Technological Measures to, the 186 | Licensed Material if doing so restricts exercise of the 187 | Licensed Rights by any recipient of the Licensed 188 | Material. 189 | 190 | 6. No endorsement. Nothing in this Public License constitutes or 191 | may be construed as permission to assert or imply that You 192 | are, or that Your use of the Licensed Material is, connected 193 | with, or sponsored, endorsed, or granted official status by, 194 | the Licensor or others designated to receive attribution as 195 | provided in Section 3(a)(1)(A)(i). 196 | 197 | b. Other rights. 198 | 199 | 1. Moral rights, such as the right of integrity, are not 200 | licensed under this Public License, nor are publicity, 201 | privacy, and/or other similar personality rights; however, to 202 | the extent possible, the Licensor waives and/or agrees not to 203 | assert any such rights held by the Licensor to the limited 204 | extent necessary to allow You to exercise the Licensed 205 | Rights, but not otherwise. 206 | 207 | 2. Patent and trademark rights are not licensed under this 208 | Public License. 209 | 210 | 3. To the extent possible, the Licensor waives any right to 211 | collect royalties from You for the exercise of the Licensed 212 | Rights, whether directly or through a collecting society 213 | under any voluntary or waivable statutory or compulsory 214 | licensing scheme. In all other cases the Licensor expressly 215 | reserves any right to collect such royalties, including when 216 | the Licensed Material is used other than for NonCommercial 217 | purposes. 218 | 219 | Section 3 -- License Conditions. 220 | 221 | Your exercise of the Licensed Rights is expressly made subject to the 222 | following conditions. 223 | 224 | a. Attribution. 225 | 226 | 1. If You Share the Licensed Material (including in modified 227 | form), You must: 228 | 229 | a. retain the following if it is supplied by the Licensor 230 | with the Licensed Material: 231 | 232 | i. identification of the creator(s) of the Licensed 233 | Material and any others designated to receive 234 | attribution, in any reasonable manner requested by 235 | the Licensor (including by pseudonym if 236 | designated); 237 | 238 | ii. a copyright notice; 239 | 240 | iii. a notice that refers to this Public License; 241 | 242 | iv. a notice that refers to the disclaimer of 243 | warranties; 244 | 245 | v. a URI or hyperlink to the Licensed Material to the 246 | extent reasonably practicable; 247 | 248 | b. indicate if You modified the Licensed Material and 249 | retain an indication of any previous modifications; and 250 | 251 | c. indicate the Licensed Material is licensed under this 252 | Public License, and include the text of, or the URI or 253 | hyperlink to, this Public License. 254 | 255 | 2. You may satisfy the conditions in Section 3(a)(1) in any 256 | reasonable manner based on the medium, means, and context in 257 | which You Share the Licensed Material. For example, it may be 258 | reasonable to satisfy the conditions by providing a URI or 259 | hyperlink to a resource that includes the required 260 | information. 261 | 262 | 3. If requested by the Licensor, You must remove any of the 263 | information required by Section 3(a)(1)(A) to the extent 264 | reasonably practicable. 265 | 266 | 4. If You Share Adapted Material You produce, the Adapter's 267 | License You apply must not prevent recipients of the Adapted 268 | Material from complying with this Public License. 269 | 270 | Section 4 -- Sui Generis Database Rights. 271 | 272 | Where the Licensed Rights include Sui Generis Database Rights that 273 | apply to Your use of the Licensed Material: 274 | 275 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 276 | to extract, reuse, reproduce, and Share all or a substantial 277 | portion of the contents of the database for NonCommercial purposes 278 | only; 279 | 280 | b. if You include all or a substantial portion of the database 281 | contents in a database in which You have Sui Generis Database 282 | Rights, then the database in which You have Sui Generis Database 283 | Rights (but not its individual contents) is Adapted Material; and 284 | 285 | c. You must comply with the conditions in Section 3(a) if You Share 286 | all or a substantial portion of the contents of the database. 287 | 288 | For the avoidance of doubt, this Section 4 supplements and does not 289 | replace Your obligations under this Public License where the Licensed 290 | Rights include other Copyright and Similar Rights. 291 | 292 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 293 | 294 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 295 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 296 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 297 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 298 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 299 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 300 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 301 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 302 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 303 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 304 | 305 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 306 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 307 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 308 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 309 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 310 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 311 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 312 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 313 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 314 | 315 | c. The disclaimer of warranties and limitation of liability provided 316 | above shall be interpreted in a manner that, to the extent 317 | possible, most closely approximates an absolute disclaimer and 318 | waiver of all liability. 319 | 320 | Section 6 -- Term and Termination. 321 | 322 | a. This Public License applies for the term of the Copyright and 323 | Similar Rights licensed here. However, if You fail to comply with 324 | this Public License, then Your rights under this Public License 325 | terminate automatically. 326 | 327 | b. Where Your right to use the Licensed Material has terminated under 328 | Section 6(a), it reinstates: 329 | 330 | 1. automatically as of the date the violation is cured, provided 331 | it is cured within 30 days of Your discovery of the 332 | violation; or 333 | 334 | 2. upon express reinstatement by the Licensor. 335 | 336 | For the avoidance of doubt, this Section 6(b) does not affect any 337 | right the Licensor may have to seek remedies for Your violations 338 | of this Public License. 339 | 340 | c. For the avoidance of doubt, the Licensor may also offer the 341 | Licensed Material under separate terms or conditions or stop 342 | distributing the Licensed Material at any time; however, doing so 343 | will not terminate this Public License. 344 | 345 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 346 | License. 347 | 348 | Section 7 -- Other Terms and Conditions. 349 | 350 | a. The Licensor shall not be bound by any additional or different 351 | terms or conditions communicated by You unless expressly agreed. 352 | 353 | b. Any arrangements, understandings, or agreements regarding the 354 | Licensed Material not stated herein are separate from and 355 | independent of the terms and conditions of this Public License. 356 | 357 | Section 8 -- Interpretation. 358 | 359 | a. For the avoidance of doubt, this Public License does not, and 360 | shall not be interpreted to, reduce, limit, restrict, or impose 361 | conditions on any use of the Licensed Material that could lawfully 362 | be made without permission under this Public License. 363 | 364 | b. To the extent possible, if any provision of this Public License is 365 | deemed unenforceable, it shall be automatically reformed to the 366 | minimum extent necessary to make it enforceable. If the provision 367 | cannot be reformed, it shall be severed from this Public License 368 | without affecting the enforceability of the remaining terms and 369 | conditions. 370 | 371 | c. No term or condition of this Public License will be waived and no 372 | failure to comply consented to unless expressly agreed to by the 373 | Licensor. 374 | 375 | d. Nothing in this Public License constitutes or may be interpreted 376 | as a limitation upon, or waiver of, any privileges and immunities 377 | that apply to the Licensor or You, including from the legal 378 | processes of any jurisdiction or authority. 379 | 380 | ======================================================================= 381 | 382 | Creative Commons is not a party to its public 383 | licenses. Notwithstanding, Creative Commons may elect to apply one of 384 | its public licenses to material it publishes and in those instances 385 | will be considered the “Licensor.” The text of the Creative Commons 386 | public licenses is dedicated to the public domain under the CC0 Public 387 | Domain Dedication. Except for the limited purpose of indicating that 388 | material is shared under a Creative Commons public license or as 389 | otherwise permitted by the Creative Commons policies published at 390 | creativecommons.org/policies, Creative Commons does not authorize the 391 | use of the trademark "Creative Commons" or any other trademark or logo 392 | of Creative Commons without its prior written consent including, 393 | without limitation, in connection with any unauthorized modifications 394 | to any of its public licenses or any other arrangements, 395 | understandings, or agreements concerning use of licensed material. For 396 | the avoidance of doubt, this paragraph does not form part of the 397 | public licenses. 398 | 399 | Creative Commons may be contacted at creativecommons.org. 400 | -------------------------------------------------------------------------------- /notebooks/sem_results.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "sem_results.ipynb", 7 | "provenance": [], 8 | "collapsed_sections": [], 9 | "include_colab_link": true 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "markdown", 19 | "metadata": { 20 | "id": "view-in-github", 21 | "colab_type": "text" 22 | }, 23 | "source": [ 24 | "\"Open" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "metadata": { 30 | "id": "Uhz1a5KgB5hg", 31 | "colab_type": "code", 32 | "colab": { 33 | "base_uri": "https://localhost:8080/", 34 | "height": 1000 35 | }, 36 | "outputId": "19285df6-b2bc-42b9-d38e-2213aa76552a" 37 | }, 38 | "source": [ 39 | "\"\"\"Load results from disk. See the run_sems.sh experiment producing results.\"\"\"\n", 40 | "from glob import glob\n", 41 | "import os\n", 42 | "import pickle\n", 43 | "from pprint import pprint\n", 44 | "\n", 45 | "import matplotlib.pyplot as plt\n", 46 | "\n", 47 | "def load_results(dirname):\n", 48 | " return pickle.load(open(os.path.join(dirname, 'metrics.p'), 'rb'))\n", 49 | " \n", 50 | "def load_flags(dirname):\n", 51 | " return pickle.load(open(os.path.join(dirname, 'flags.p'), 'rb'))\n", 52 | "\n", 53 | "# ROOT = '/scratch/gobi1/creager/opt_env/run_sems_alpha_sweep/43878915'\n", 54 | "ROOT = '/PATH/TO/SWEEP/DIR'\n", 55 | "\n", 56 | "# baselines include HandCrafted->ERM, ICP, IRM, and ERM->EIIL->IRM\n", 57 | "BASELINE_GLOB_PATTERN = '{}/chain_hidden=0_hetero=2_scramble=0/all_solutions.txt'.format(ROOT) # TODO(): change\n", 58 | "\n", 59 | "ALPHAS = (\n", 60 | " 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0\n", 61 | ") # NOTE: UDL workshop paper missed two x-tick marks on this plot\n", 62 | "from collections import OrderedDict\n", 63 | "ALPHA_SWEEP_GLOB_PATTERNS = OrderedDict()\n", 64 | "for alpha in ALPHAS:\n", 65 | " ALPHA_SWEEP_GLOB_PATTERNS[alpha] = \\\n", 66 | " '{}/chain_hidden=0_hetero=2_scramble=0_alpha_{}/all_solutions.txt'.format(ROOT, str(alpha)) # TODO(): change\n", 67 | "\n", 68 | "def read_lines(results_glob):\n", 69 | " all_lines = []\n", 70 | " for fn in glob(results_glob):\n", 71 | " lines = open(fn, 'r').read().splitlines()\n", 72 | " all_lines.extend(lines)\n", 73 | " return all_lines\n", 74 | "\n", 75 | "def get_results(all_solutions):\n", 76 | " results = {}\n", 77 | "\n", 78 | " for line in all_solutions:\n", 79 | " words = line.split(\" \")\n", 80 | " setup = str(words[0])\n", 81 | " model = str(words[1])\n", 82 | " err_causal = float(words[-2])\n", 83 | " err_noncausal = float(words[-1])\n", 84 | "\n", 85 | " if setup not in results:\n", 86 | " results[setup] = {}\n", 87 | "\n", 88 | " if model not in results[setup]:\n", 89 | " results[setup][model] = []\n", 90 | "\n", 91 | " results[setup][model].append([err_causal, err_noncausal])\n", 92 | " return results\n", 93 | "\n", 94 | "results_lines = dict()\n", 95 | "results_lines['baselines'] = read_lines(BASELINE_GLOB_PATTERN)\n", 96 | "results_lines['alpha_sweep'] = OrderedDict()\n", 97 | "for a, agp in ALPHA_SWEEP_GLOB_PATTERNS.items():\n", 98 | " results_lines['alpha_sweep'][a] = read_lines(agp)\n", 99 | "\n", 100 | "SETTING = 'chain_hidden=0_hetero=2_scramble=0' # apply filter to yield only this data setting\n", 101 | "# results = {k: get_results(rl)[SETTING] for k, rl in results_lines.items()}\n", 102 | "results = dict()\n", 103 | "results['baselines'] = get_results(results_lines['baselines'])[SETTING]\n", 104 | "results['alpha_sweep'] = OrderedDict()\n", 105 | "for a in ALPHAS:\n", 106 | " results['alpha_sweep'][a] = get_results(results_lines['alpha_sweep'][a])[SETTING]\n", 107 | "\n", 108 | "from pprint import pprint\n", 109 | "pprint(results)\n", 110 | "for a in ALPHAS:\n", 111 | " print(a, results['alpha_sweep'][a])" 112 | ], 113 | "execution_count": null, 114 | "outputs": [ 115 | { 116 | "output_type": "stream", 117 | "text": [ 118 | "{'alpha_sweep': OrderedDict([(0.1,\n", 119 | " {'EIIL': [[1.20987, 1.08879],\n", 120 | " [1.03908, 0.92657],\n", 121 | " [1.27709, 1.12844],\n", 122 | " [1.56064, 1.35345],\n", 123 | " [0.58332, 0.56332]],\n", 124 | " 'SEM': [[0.0, 0.0],\n", 125 | " [0.0, 0.0],\n", 126 | " [0.0, 0.0],\n", 127 | " [0.0, 0.0],\n", 128 | " [0.0, 0.0]]}),\n", 129 | " (0.2,\n", 130 | " {'EIIL': [[0.58675, 0.56536],\n", 131 | " [0.42208, 0.41769],\n", 132 | " [0.47998, 0.46702],\n", 133 | " [0.49813, 0.49398],\n", 134 | " [0.47181, 0.47596]],\n", 135 | " 'SEM': [[0.0, 0.0],\n", 136 | " [0.0, 0.0],\n", 137 | " [0.0, 0.0],\n", 138 | " [0.0, 0.0],\n", 139 | " [0.0, 0.0]]}),\n", 140 | " (0.3,\n", 141 | " {'EIIL': [[1.13695, 1.00236],\n", 142 | " [0.57304, 0.54475],\n", 143 | " [0.46865, 0.45732],\n", 144 | " [0.4946, 0.48862],\n", 145 | " [0.4725, 0.47559]],\n", 146 | " 'SEM': [[0.0, 0.0],\n", 147 | " [0.0, 0.0],\n", 148 | " [0.0, 0.0],\n", 149 | " [0.0, 0.0],\n", 150 | " [0.0, 0.0]]}),\n", 151 | " (0.4,\n", 152 | " {'EIIL': [[1.07029, 0.94224],\n", 153 | " [0.61972, 0.56385],\n", 154 | " [0.46864, 0.45667],\n", 155 | " [0.50413, 0.49556],\n", 156 | " [0.72768, 0.6986]],\n", 157 | " 'SEM': [[0.0, 0.0],\n", 158 | " [0.0, 0.0],\n", 159 | " [0.0, 0.0],\n", 160 | " [0.0, 0.0],\n", 161 | " [0.0, 0.0]]}),\n", 162 | " (0.5,\n", 163 | " {'EIIL': [[1.20125, 1.06908],\n", 164 | " [0.66164, 0.58418],\n", 165 | " [0.47249, 0.46022],\n", 166 | " [0.49825, 0.48423],\n", 167 | " [1.28127, 1.16425]],\n", 168 | " 'SEM': [[0.0, 0.0],\n", 169 | " [0.0, 0.0],\n", 170 | " [0.0, 0.0],\n", 171 | " [0.0, 0.0],\n", 172 | " [0.0, 0.0]]}),\n", 173 | " (0.6,\n", 174 | " {'EIIL': [[1.2228, 1.05965],\n", 175 | " [1.44376, 1.32061],\n", 176 | " [0.46628, 0.45464],\n", 177 | " [0.47986, 0.47615],\n", 178 | " [0.83696, 0.75114]],\n", 179 | " 'SEM': [[0.0, 0.0],\n", 180 | " [0.0, 0.0],\n", 181 | " [0.0, 0.0],\n", 182 | " [0.0, 0.0],\n", 183 | " [0.0, 0.0]]}),\n", 184 | " (0.7,\n", 185 | " {'EIIL': [[0.56202, 0.57157],\n", 186 | " [0.12392, 0.14817],\n", 187 | " [0.45657, 0.44559],\n", 188 | " [0.46264, 0.45731],\n", 189 | " [0.46125, 0.46519]],\n", 190 | " 'SEM': [[0.0, 0.0],\n", 191 | " [0.0, 0.0],\n", 192 | " [0.0, 0.0],\n", 193 | " [0.0, 0.0],\n", 194 | " [0.0, 0.0]]}),\n", 195 | " (0.8,\n", 196 | " {'EIIL': [[0.61086, 0.49274],\n", 197 | " [0.06665, 0.06252],\n", 198 | " [0.50138, 0.483],\n", 199 | " [0.70804, 0.63456],\n", 200 | " [0.56308, 0.55354]],\n", 201 | " 'SEM': [[0.0, 0.0],\n", 202 | " [0.0, 0.0],\n", 203 | " [0.0, 0.0],\n", 204 | " [0.0, 0.0],\n", 205 | " [0.0, 0.0]]}),\n", 206 | " (0.9,\n", 207 | " {'EIIL': [[0.18227, 0.1929],\n", 208 | " [0.03049, 0.01854],\n", 209 | " [0.51175, 0.49186],\n", 210 | " [0.0655, 0.06096],\n", 211 | " [0.08848, 0.10323]],\n", 212 | " 'SEM': [[0.0, 0.0],\n", 213 | " [0.0, 0.0],\n", 214 | " [0.0, 0.0],\n", 215 | " [0.0, 0.0],\n", 216 | " [0.0, 0.0]]}),\n", 217 | " (1.0,\n", 218 | " {'EIIL': [[0.08395, 0.0869],\n", 219 | " [0.03107, 0.03012],\n", 220 | " [0.4985, 0.48285],\n", 221 | " [0.03681, 0.03995],\n", 222 | " [0.07054, 0.08129]],\n", 223 | " 'SEM': [[0.0, 0.0],\n", 224 | " [0.0, 0.0],\n", 225 | " [0.0, 0.0],\n", 226 | " [0.0, 0.0],\n", 227 | " [0.0, 0.0]]})]),\n", 228 | " 'baselines': {'EIIL': [[0.09301, 0.09893],\n", 229 | " [0.02745, 0.0234],\n", 230 | " [0.51578, 0.49595],\n", 231 | " [0.06457, 0.05908],\n", 232 | " [0.03721, 0.04549]],\n", 233 | " 'ERM': [[0.85369, 0.84856],\n", 234 | " [0.82197, 0.81206],\n", 235 | " [0.82215, 0.81623],\n", 236 | " [0.82626, 0.82619],\n", 237 | " [0.81081, 0.81841]],\n", 238 | " 'ICP': [[1.0, 0.95135],\n", 239 | " [1.0, 0.0],\n", 240 | " [1.0, 0.93675],\n", 241 | " [1.0, 0.943],\n", 242 | " [1.0, 0.95139]],\n", 243 | " 'IRM': [[0.79471, 0.74555],\n", 244 | " [0.56653, 0.55343],\n", 245 | " [0.65777, 0.62865],\n", 246 | " [0.66595, 0.65153],\n", 247 | " [0.64409, 0.64043]],\n", 248 | " 'SEM': [[0.0, 0.0],\n", 249 | " [0.0, 0.0],\n", 250 | " [0.0, 0.0],\n", 251 | " [0.0, 0.0],\n", 252 | " [0.0, 0.0]]}}\n", 253 | "0.1 {'SEM': [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], 'EIIL': [[1.20987, 1.08879], [1.03908, 0.92657], [1.27709, 1.12844], [1.56064, 1.35345], [0.58332, 0.56332]]}\n", 254 | "0.2 {'SEM': [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], 'EIIL': [[0.58675, 0.56536], [0.42208, 0.41769], [0.47998, 0.46702], [0.49813, 0.49398], [0.47181, 0.47596]]}\n", 255 | "0.3 {'SEM': [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], 'EIIL': [[1.13695, 1.00236], [0.57304, 0.54475], [0.46865, 0.45732], [0.4946, 0.48862], [0.4725, 0.47559]]}\n", 256 | "0.4 {'SEM': [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], 'EIIL': [[1.07029, 0.94224], [0.61972, 0.56385], [0.46864, 0.45667], [0.50413, 0.49556], [0.72768, 0.6986]]}\n", 257 | "0.5 {'SEM': [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], 'EIIL': [[1.20125, 1.06908], [0.66164, 0.58418], [0.47249, 0.46022], [0.49825, 0.48423], [1.28127, 1.16425]]}\n", 258 | "0.6 {'SEM': [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], 'EIIL': [[1.2228, 1.05965], [1.44376, 1.32061], [0.46628, 0.45464], [0.47986, 0.47615], [0.83696, 0.75114]]}\n", 259 | "0.7 {'SEM': [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], 'EIIL': [[0.56202, 0.57157], [0.12392, 0.14817], [0.45657, 0.44559], [0.46264, 0.45731], [0.46125, 0.46519]]}\n", 260 | "0.8 {'SEM': [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], 'EIIL': [[0.61086, 0.49274], [0.06665, 0.06252], [0.50138, 0.483], [0.70804, 0.63456], [0.56308, 0.55354]]}\n", 261 | "0.9 {'SEM': [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], 'EIIL': [[0.18227, 0.1929], [0.03049, 0.01854], [0.51175, 0.49186], [0.0655, 0.06096], [0.08848, 0.10323]]}\n", 262 | "1.0 {'SEM': [[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], 'EIIL': [[0.08395, 0.0869], [0.03107, 0.03012], [0.4985, 0.48285], [0.03681, 0.03995], [0.07054, 0.08129]]}\n" 263 | ], 264 | "name": "stdout" 265 | } 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "metadata": { 271 | "id": "ZgDWX6D63qqd", 272 | "colab_type": "code", 273 | "colab": { 274 | "base_uri": "https://localhost:8080/", 275 | "height": 301 276 | }, 277 | "outputId": "e4915737-eb2f-4895-bb77-58bfa4d2eb75" 278 | }, 279 | "source": [ 280 | "\"\"\"Produce table from baseline errors.\"\"\"\n", 281 | "from collections import defaultdict\n", 282 | "import numpy as np\n", 283 | "import pandas as pd\n", 284 | "\n", 285 | "LATEX_SYMBOLS = dict(\n", 286 | " ERM=r'\\textsc{ERM}',\n", 287 | " ICP=r'\\textsc{ICP}',\n", 288 | " IRM=r'\\textsc{IRM}($e_{\\textsc{HC}}$)',\n", 289 | " EIIL=r'\\textsc{IRM}($e_{\\textsc{EIIL}}| \\tilde \\Phi = \\Phi_{ERM}$)',\n", 290 | " alpha=r'\\textsc{IRM}($e_{\\textsc{EIIL}}| \\tilde \\Phi = \\Phi_{\\alpha-Spurious}$)',\n", 291 | ")\n", 292 | "\n", 293 | "baseline_errors = defaultdict(dict)\n", 294 | "for method, v in results['baselines'].items():\n", 295 | " if method == 'SEM':\n", 296 | " continue\n", 297 | " results_ = np.stack(v)\n", 298 | " mean = np.mean(results_, 0)\n", 299 | " std = np.std(results_, 0)\n", 300 | " baseline_errors['Causal err.'][LATEX_SYMBOLS[method]] = (mean[0], std[0])\n", 301 | " baseline_errors['Noncausal err.'][LATEX_SYMBOLS[method]] = (mean[1], std[1])\n", 302 | "\n", 303 | "def f(xy):\n", 304 | " \"\"\"Format mean plus/minus std dev.\"\"\"\n", 305 | " x, y = xy\n", 306 | " return r\"\"\"%.3f $\\pm$ %.3f\"\"\" % (x, y)\n", 307 | "\n", 308 | "re = pd.DataFrame.from_dict(baseline_errors)\n", 309 | "results_tex = re.to_latex(\n", 310 | " formatters=[f, ] * len(re.T),\n", 311 | " escape=False,\n", 312 | " caption=(\n", 313 | " 'Results on synthetic data. MSE on causal and non-causal portions of the '\n", 314 | " 'ground truth regression coefficient are reported, plus/minus a standard '\n", 315 | " 'deviation across ten runs. '\n", 316 | " )\n", 317 | ")\n", 318 | "print(results_tex)" 319 | ], 320 | "execution_count": null, 321 | "outputs": [ 322 | { 323 | "output_type": "stream", 324 | "text": [ 325 | "\\begin{table}\n", 326 | "\\centering\n", 327 | "\\caption{Results on synthetic data. MSE on causal and non-causal portions of the ground truth regression coefficient are reported, plus/minus a standard deviation across ten runs. }\n", 328 | "\\begin{tabular}{lll}\n", 329 | "\\toprule\n", 330 | "{} & Causal err. & Noncausal err. \\\\\n", 331 | "\\midrule\n", 332 | "\\textsc{IRM}($e_{\\textsc{EIIL}}| \\tilde \\Phi = ... & 0.148 $\\pm$ 0.186 & 0.145 $\\pm$ 0.177 \\\\\n", 333 | "\\textsc{ERM} & 0.827 $\\pm$ 0.014 & 0.824 $\\pm$ 0.013 \\\\\n", 334 | "\\textsc{ICP} & 1.000 $\\pm$ 0.000 & 0.756 $\\pm$ 0.378 \\\\\n", 335 | "\\textsc{IRM}($e_{\\textsc{HC}}$) & 0.666 $\\pm$ 0.073 & 0.644 $\\pm$ 0.061 \\\\\n", 336 | "\\bottomrule\n", 337 | "\\end{tabular}\n", 338 | "\\end{table}\n", 339 | "\n" 340 | ], 341 | "name": "stdout" 342 | } 343 | ] 344 | }, 345 | { 346 | "cell_type": "code", 347 | "metadata": { 348 | "id": "3VfKtY2HJ5xw", 349 | "colab_type": "code", 350 | "colab": { 351 | "base_uri": "https://localhost:8080/", 352 | "height": 1000 353 | }, 354 | "outputId": "ac275104-516c-4b90-98ed-07186d795b4f" 355 | }, 356 | "source": [ 357 | "\"\"\"Aggregate error measurements.\"\"\"\n", 358 | "\n", 359 | "alpha_sweep_errors = dict()\n", 360 | "for k in ('Causal err.', 'Noncausal err.'):\n", 361 | " alpha_sweep_errors[k] = defaultdict(list)\n", 362 | "\n", 363 | "for a, v in results['alpha_sweep'].items():\n", 364 | " v = v['EIIL']\n", 365 | " results_ = np.stack(v)\n", 366 | " mean = np.mean(results_, 0)\n", 367 | " std = np.std(results_, 0)\n", 368 | " alpha_sweep_errors['Causal err.']['mean'].append(mean[0])\n", 369 | " alpha_sweep_errors['Causal err.']['upper'].append(mean[0] + std[0])\n", 370 | " alpha_sweep_errors['Causal err.']['lower'].append(mean[0] - std[0])\n", 371 | " alpha_sweep_errors['Noncausal err.']['mean'].append(mean[1])\n", 372 | " alpha_sweep_errors['Noncausal err.']['upper'].append(mean[1] + std[1])\n", 373 | " alpha_sweep_errors['Noncausal err.']['lower'].append(mean[1] - std[1])\n", 374 | "\n", 375 | "pprint(alpha_sweep_errors)\n" 376 | ], 377 | "execution_count": null, 378 | "outputs": [ 379 | { 380 | "output_type": "stream", 381 | "text": [ 382 | "{'Causal err.': defaultdict(,\n", 383 | " {'lower': [0.8113512859470844,\n", 384 | " 0.4379805852737823,\n", 385 | " 0.3724721939410729,\n", 386 | " 0.4618428616988267,\n", 387 | " 0.4744283322088388,\n", 388 | " 0.4980206186699855,\n", 389 | " 0.263303767349623,\n", 390 | " 0.26781109357941757,\n", 391 | " 0.0003069626691261096,\n", 392 | " -0.03410407140531893],\n", 393 | " 'mean': [1.134,\n", 394 | " 0.49175,\n", 395 | " 0.629148,\n", 396 | " 0.678092,\n", 397 | " 0.8229799999999999,\n", 398 | " 0.889932,\n", 399 | " 0.41328,\n", 400 | " 0.49000199999999994,\n", 401 | " 0.175698,\n", 402 | " 0.144174],\n", 403 | " 'upper': [1.4566487140529154,\n", 404 | " 0.5455194147262178,\n", 405 | " 0.8858238060589272,\n", 406 | " 0.8943411383011733,\n", 407 | " 1.1715316677911611,\n", 408 | " 1.2818433813300145,\n", 409 | " 0.5632562326503769,\n", 410 | " 0.7121929064205823,\n", 411 | " 0.3510890373308739,\n", 412 | " 0.32245207140531895]}),\n", 413 | " 'Noncausal err.': defaultdict(,\n", 414 | " {'lower': [0.7495668451801467,\n", 415 | " 0.4361083092255358,\n", 416 | " 0.38733498796713106,\n", 417 | " 0.45549572562106355,\n", 418 | " 0.4505679074295096,\n", 419 | " 0.47651567949122525,\n", 420 | " 0.27549248922476793,\n", 421 | " 0.24641143916402125,\n", 422 | " 0.004178671109291238,\n", 423 | " -0.026545143842133745],\n", 424 | " 'mean': [1.012114,\n", 425 | " 0.484002,\n", 426 | " 0.593728,\n", 427 | " 0.631384,\n", 428 | " 0.7523920000000001,\n", 429 | " 0.812438,\n", 430 | " 0.41756600000000005,\n", 431 | " 0.445272,\n", 432 | " 0.173498,\n", 433 | " 0.14422200000000002],\n", 434 | " 'upper': [1.2746611548198532,\n", 435 | " 0.5318956907744642,\n", 436 | " 0.800121012032869,\n", 437 | " 0.8072722743789363,\n", 438 | " 1.0542160925704906,\n", 439 | " 1.1483603205087747,\n", 440 | " 0.5596395107752321,\n", 441 | " 0.6441325608359787,\n", 442 | " 0.3428173288907088,\n", 443 | " 0.3149891438421338]})}\n" 444 | ], 445 | "name": "stdout" 446 | } 447 | ] 448 | }, 449 | { 450 | "cell_type": "code", 451 | "metadata": { 452 | "id": "-28z06pUENzh", 453 | "colab_type": "code", 454 | "colab": { 455 | "base_uri": "https://localhost:8080/", 456 | "height": 755 457 | }, 458 | "outputId": "60708d07-2422-4870-e8c4-973b6904f4af" 459 | }, 460 | "source": [ 461 | "from matplotlib import rc\n", 462 | "rc('font',**{'family':'sans-serif','sans-serif':['Helvetica']})\n", 463 | "rc('text', usetex=True)\n", 464 | "\n", 465 | "import matplotlib.pyplot as plt\n", 466 | "import matplotlib.patches as mpatches\n", 467 | "from matplotlib.colors import colorConverter as cc\n", 468 | "import numpy as np\n", 469 | "import seaborn as sns\n", 470 | "\n", 471 | "TITLE_FONTSIZE = 16\n", 472 | "AXIS_FONTSIZE = 16\n", 473 | "LEGEND_FONTSIZE = 10\n", 474 | "FIGSIZE = (6, 4)\n", 475 | "SHADING_OPACITY = .5 # how opaque should the uncertainty fills be\n", 476 | "\n", 477 | "\n", 478 | "class LegendObject(object):\n", 479 | " def __init__(self, facecolor='red', edgecolor='white', dashed=False):\n", 480 | " self.facecolor = facecolor\n", 481 | " self.edgecolor = edgecolor\n", 482 | " self.dashed = dashed\n", 483 | " \n", 484 | " def legend_artist(self, legend, orig_handle, fontsize, handlebox):\n", 485 | " x0, y0 = handlebox.xdescent, handlebox.ydescent\n", 486 | " width, height = handlebox.width, handlebox.height\n", 487 | " patch = mpatches.Rectangle(\n", 488 | " # create a rectangle that is filled with color\n", 489 | " [x0, y0], width, height, facecolor=self.facecolor,\n", 490 | " # and whose edges are the faded color\n", 491 | " edgecolor=self.edgecolor, lw=3)\n", 492 | " handlebox.add_artist(patch)\n", 493 | " \n", 494 | " # if we're creating the legend for a dashed line,\n", 495 | " # manually add the dash in to our rectangle\n", 496 | " if self.dashed:\n", 497 | " patch1 = mpatches.Rectangle(\n", 498 | " [x0 + 2*width/5, y0], width/5, height, facecolor=self.edgecolor,\n", 499 | " transform=handlebox.get_transform())\n", 500 | " handlebox.add_artist(patch1)\n", 501 | " \n", 502 | " return patch\n", 503 | " \n", 504 | " \n", 505 | "def plot_mean_and_CI(x_axis_vec, mean, lb, ub, color_mean=None, color_shading=None):\n", 506 | " # plot the shaded range of the confidence intervals\n", 507 | " plt.fill_between(x_axis_vec, ub, lb,\n", 508 | " color=color_shading, alpha=.5)\n", 509 | " # plt.fill_between(range(mean.shape[0]), ub, lb,\n", 510 | " # color=color_shading, alpha=.5)\n", 511 | " # plot the mean on top\n", 512 | " plt.plot(x_axis_vec, mean, color=color_mean)\n", 513 | " \n", 514 | "# plot the data\n", 515 | "a = lambda x: np.array(x)\n", 516 | "bg = np.array([1, 1, 1]) # background of the legend is white\n", 517 | "for metric in ('Causal err.', 'Noncausal err.'):\n", 518 | " from itertools import cycle\n", 519 | " handler_map = dict()\n", 520 | " colors = iter('mgkb')\n", 521 | " legend_names = []\n", 522 | " plt.figure(figsize=FIGSIZE)\n", 523 | " color = next(colors)\n", 524 | " color_faded = (np.array(cc.to_rgb(color)) + bg) * SHADING_OPACITY\n", 525 | " handler_map[len(legend_names)] = LegendObject(color, color_faded)\n", 526 | " print(handler_map)\n", 527 | " legend_names.append(LATEX_SYMBOLS['alpha'])\n", 528 | " a = lambda x: np.array(x)\n", 529 | " plot_mean_and_CI(ALPHAS,\n", 530 | " a(alpha_sweep_errors[metric]['mean']),\n", 531 | " a(alpha_sweep_errors[metric]['lower']),\n", 532 | " a(alpha_sweep_errors[metric]['upper']),\n", 533 | " color_mean=color, color_shading=color_faded)\n", 534 | " \n", 535 | " for m, v in baseline_errors[metric].items():\n", 536 | " if m == LATEX_SYMBOLS['ICP']: # don't plot this baseline b/c it doesn't work anyways\n", 537 | " continue\n", 538 | " color = next(colors)\n", 539 | " color_faded = (np.array(cc.to_rgb(color)) + bg) * SHADING_OPACITY\n", 540 | " handler_map[len(legend_names)] = LegendObject(color, color_faded)\n", 541 | " print(handler_map)\n", 542 | " legend_names.append(m)\n", 543 | " mean = v[0] * np.ones(len(ALPHAS))\n", 544 | " upper = (v[0] + v[1]) * np.ones(len(ALPHAS))\n", 545 | " lower = (v[0] - v[1]) * np.ones(len(ALPHAS))\n", 546 | " plot_mean_and_CI(ALPHAS,\n", 547 | " mean,\n", 548 | " lower,\n", 549 | " upper,\n", 550 | " color_mean=color, color_shading=color_faded)\n", 551 | " plt.legend(range(len(legend_names)), legend_names, handler_map=handler_map,\n", 552 | " loc='upper right', fontsize=LEGEND_FONTSIZE)\n", 553 | " plt.xlabel(r'$\\alpha$', fontsize=AXIS_FONTSIZE)\n", 554 | " if metric == 'Causal err.':\n", 555 | " plt.ylabel('MSE', fontsize=AXIS_FONTSIZE)\n", 556 | " # plt.title(metric, fontsize=TITLE_FONTSIZE)\n", 557 | " plt.tight_layout()\n", 558 | " plt.grid()\n", 559 | " plt.show()" 560 | ], 561 | "execution_count": null, 562 | "outputs": [ 563 | { 564 | "output_type": "stream", 565 | "text": [ 566 | "{0: <__main__.LegendObject object at 0x7f59bab72f98>}\n", 567 | "{0: <__main__.LegendObject object at 0x7f59bab72f98>, 1: <__main__.LegendObject object at 0x7f59bab51898>}\n", 568 | "{0: <__main__.LegendObject object at 0x7f59bab72f98>, 1: <__main__.LegendObject object at 0x7f59bab51898>, 2: <__main__.LegendObject object at 0x7f59bab51ef0>}\n", 569 | "{0: <__main__.LegendObject object at 0x7f59bab72f98>, 1: <__main__.LegendObject object at 0x7f59bab51898>, 2: <__main__.LegendObject object at 0x7f59bab51ef0>, 3: <__main__.LegendObject object at 0x7f59b8aeb550>}\n" 570 | ], 571 | "name": "stdout" 572 | }, 573 | { 574 | "output_type": "stream", 575 | "text": [ 576 | "findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans.\n" 577 | ], 578 | "name": "stderr" 579 | }, 580 | { 581 | "output_type": "display_data", 582 | "data": { 583 | "text/plain": [ 584 | "
" 585 | ], 586 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAABQGUlEQVR4nO29e3Rb13Xn/zn34kmCJEhKot4PyvIjaRKZouNm0sSKQyXuI3Eyla1m+kt/6Uwqx5nfdHVlUmvctHXSTOJKv7TTZK1mIqaPyS9tp7I0jZ0006aiY+Xth0TLyaRybIuSZb0lkiABkgDu4/z+uADENwEST2J/1uICcO/FwcEh7v3evc8+eyutNYIgCIJQbRiV7oAgCIIgzIYIlCAIglCViEAJgiAIVYkIlCAIglCViEAJgiAIVYmv0h1YLCtWrNCbN2+udDeKztjYGI2NjZXuRk0gY5U/MlaFIeOVP8UYqxMnTlzXWq+cvr1mBWrz5s0cP3680t0oOseOHWPnzp2V7kZNIGOVPzJWhSHjlT/FGCul1KuzbRcXnyAIglCViEAJgiAIVYkIlCAIglCV1OwclCAI+aOU4syZMySTyUp3pSZoaWnh1KlTle5GTVDIWIVCIdavX4/f78/reBEoQagDGhsbaWpqYvPmzSilKt2dqicej9PU1FTpbtQE+Y6V1prBwUHOnz/Pli1b8mpbXHyCUAeYpkl7e7uIk1AxlFK0t7cXZMWLBSUIdYKIU/E4cuQI0WiUzs5OOjs7K92dmqHQ36BYUIIgCAXS09MDQH9/f4V7srypWwsqWwdL7ioFQSiUaDSaEymhdNStQFlXLKwhi8bXSToTob44+6mzJWt78yObS9Z2JTly5Ai7d++udDfqjrp18WlHkzyfRCoKC0LpaW1tBSAWi6GUYteuXezatYutW7eya9eu3HH9/f0opWa4zrZu3coDDzwwZdu+fftK3/EMQ0NDU14fOXKE1tZWjhw5QiwWW3L7xW6vUvT29hbV7Vm3AgXgjDo4cafS3RCEuqKzs5OjR49y9OhRTp8+TTQapbe3d8r+Q4cO5V7PdsE7cOAAe/bsKUt/p5MVkLa2NqLR6AzxqnR7lWTv3r08+uijRWuv7C4+pVSX1npeiVVKPaS1PlDqvrhJl/TVNL7muvV0CnXO2U+eXXIbmz+5eUnvj8ViUyLhurq66Ovry70+ePAgu3fvnmJZHD16lIceegiAvr4+BgYGaGtrK7kbLhaLcd9993H06FH27dtHT08P9913Hw8//DBdXV0Vb68aaGtrY2BgoCjRjWW9MiuleoD9wI4FjrmjLP3xKdIX0jTc1FCOjxMEARgYGMi59Y4fP05PT8+UgIO2tjY6Ozvp6+ujp6eH48ePs3//fg4fPgx4F/VoNJpr6+DBgxw+fJgjR46UvO+PPfbYjAvvHXfcwaFDh6YIysDAwBSRnczevXsLbi8fBgYG6O/vJxqNcvToUfbv31/Q+6e3lbXoCmXHjh309/fXnkBprfuUUlVjvxohAytm4aZcjGBdezsFoWxkXXxZduzYMeOOe8+ePRw8eBBgRrRc9uIJ3txNVsyyF/RYLEZra+uM92U/s7W1leHh4RnP+/v72bdv35S+Tef+++/PCWWW5557jocffnjGd5wsREttLx8OHjzIww8/TDQazY3PYlmKuHR2dhZtHqqqfFsZ91+fUuqBhY8uEhrSg2lCa0Nl+0hBEG7Q3d094467q6uL48ePMzQ0xP79+xkYGMjt6+zszM3TZMO9p1sb00WwWESjUQ4fPpyzjvr6+ti/f/+MC/rAwMCcFl3WNZlPe729vXR3dzMwMEBXV9e8wvHwww/nAkey1lN/fz+HDh1iz549uZuAQ4cOsX//fg4cOMBDDz1Ef38/Bw8e5IEHHphxzPQ+zPb+7DFtbW10dXUVzb0HVSZQwNJkfxEon8K6ZIlACXXJUuePisHWrVs5evTojPmjrBXV2dk5RaCi0WhuPur+++/nscceA7xIu3KsTcp+/tDQELFYbFZrpbOzc4oQLaa9AwcOsHv3bjo7O3n00UdzIjxd+CZ/54MHD+bm5Lq6uujq6uLgwYN0dXXx6KOPcvjw4VwAyuDgIODdDESj0dzxQO6YAwcO0NPTQ2dnJwcPHuTgwYMz3p8l+z88ceJE0SIsq0agstbTAsfsBfYCdHR0cOzYsUV/nrY0Dg5KK/R5jS9eHUORSCSW9L3qCRmr/GlubiYej5f8c+b6jHPnzhGPxzFNk5MnT0457sEHH8y9d9u2bXzuc58jHo/z4IMP8uCDDxKPx7nzzju58847c+97+9vfzve+9z22b9/OBz7wgSmfH4/HGRgY4O67785t37x5M5///Odn7Wf2+djYGI7jEI/Hc49ZksnklNfvfve7OXfu3ILfO19ma+/WW2/lwoULXLhwgfXr15NIJIjH47z73e+e8f6nnnqKkZERWlpaOHv2LL/5m79JPB7n5MmTWJbF17/+dX77t3+beDxOJBLh61//OpFIhB//+Me0traybt263HeIxWK5fbfeeivnz5/n+9//Pr//+78/6/sHBga47bbbCIfDxONxrl69ysqVK+cck2Qymfd5q8q9DkgpdVRrvWvS66jWOqaUmnz79DDwW/NF+3V3d+ullHxPXUgx+twovqgPe8Qm+rYo/rb8UsCXEik1nT8yVvnz/PPPc/vttwPLZ6Huvn37Zg0EiMVi7Nixg9OnT8/6vnzmoKZn6K7VhboPPPAADzzwQEkjAiePVdYdON/nnTp1ittuu23KNqXUCa119/Rjyx3FtxvoVkrt1lpn7dQngR3Z1xkrKVrWfqFIX0tXhUAJQqlZLtkelhKlVii1KE5ALtCkXOQTGFII5Y7iOwIcmbZtx7TXvUAvZUSFFKnXUjTeImmPBGE5MDAwwI4dU1ezPPnkkwuGTR8/fpwdO3bgui6GYfDlL3+5ZtcjLQeqY+KlwqiAwhlxcMYdzAaz0t0RBGEJRKPReVOYZV160593dXXlXkvBwupAFv9wI6O5dd2qcE8EQRCELCJQGVRAkbqYqnQ3BEEQhAzi4stghAysqxba1iif1IgSBGFupKJueRALKoMyFLhgDYubTxCE+ZGKuuVBLKhJaEOTvpImsDJQ6a4IglDFSEXd8iACNQkjbJC6kKLx9Y1SCl5Ytnzq2KdK1vYjOx8pWduVpFYX6tY64uKbhOE30BMaJyFFDAWhmEhF3fkpV0XdYle8LTUiUNPQaAk3F4QSIhV1S9vefBS74m2pERffNIygQep8ivCWcKW7Iggl55Pf+eTS27hraW1IRd3yVtQtZsXbUiMW1DRUSGENWbhpt9JdEYRlSbai7q5du2htbZ0RcNDW1kZPT09OpI4fPz7FDThbRd1i54Cbi/kq4E5mYGCA3t7eWf8KaS+b+Lavr4++vr4pbs5YLMauXbty+yaL+q5du6aUKHnggQdy+7MVb2sBsaCmoZQCDfaQTWC1RPMJQrGRirr5t5dda9XT08PAwMCUebfp+44cOZL7zrt27coVgcwKVXZfMSvelhoRqFlQpiJ1KSUCJSwLtKtzj8qovuhUqag7d3vZcent7c0VDszS399PNBrlyJEjDAwM5Nrt7+9n9+7duYCL6QUVa8W9ByJQs2I0GKQupoi8KVKVJ7Qg5IObdEmeTzLxygSs9F5PT4a81PmjYiAVdeduL+vWywZQ9Pf358Q4u6+zs5P77rsv956hoSG6uroYHBzMCfnkzO7FrHhbakSgZkGZCmywR238UakRJdQOWmvsYZvkmSTJ80nQ3g0XgJt2McKVmXbOutGi0eiMQoKTL+TZEuXZ7dl9PT09U8Qn68Lq6uqa1ZWWnefKki1ZXix2795d1ICM2dqLxWIcOnSIrq4uBgYGOHz4cK4GViwWmyLqbW1tOWvr6NGjU9x5+/bt44477si1OzQ0JBZUraO1xrpqiUAJNYFruaQvpRl/eRwn7qBMhdlk3vAAKECDtspbQbtUPPTQQ+zbt2/OSLdiu/hms5JKTTQa5cSJE7nXkwU6Go1O+X6TxTcrYtMfwVsHNX2+rJoRgZoDI+yFmzfc3FDprgjCnNijNslXkyTPJtGOxggZmC3m7JlQlGdFLZdsD1JRt3DKFe1YLESg5kAFFPaIjZN0MENSxFCoHrTj5YycOD2BNWihDIXRaHiu6XlQhkJbumqDJYqJVNRdHohAzcHkIobmehEoofI44w7J15IkTydx0y4qoOa2lmYjc5i2NCq4fAVKKuouH8ouUEqpLq31rEH4Sqms/blVa13xMBPlV6QvpgmtD1W6K0KdorXGGrSYGJggfTENgNlo4mtY3KmrDIWb8sRNEiIL1U5ZBUop1QPsB3bMsa9Paz2glDqslOrRWvfNaKSMGGGD9JU02tELuk8EoZi4KZfUxRQTL0/gjDsoX4HW0lwoz0WIg/hPhKqnrD9RrXWfUmquTIidmb9eYCDzvKIoQ6FdjR2z8bdLNJ9QWrTW2CNeiHjqtRTa1RgNBr5oEU9T5f25lovpE9e1UN1UzT2U1npykqou4NBcx5YThSJ9OS0CJZQM13ZJX04z8fIE9ojtBT1EjJIFMnz2858tSbsAjzyyPCIEheqgagQqi1KqCzg61zxVuVFhRepCiobXNYjPXigqdsIm9WqKiTMTaHuBEHFBqEOqTqCAHq31gdl2ZIIo9gJ0dHRw7NixRX+ItjQODiqxwMVAeznMfE/5ypL7PZFILOl71RO1Olba1rgp11s0qzxXMj7AzvyVgGhTlHFnvDSNTyIej8+6PRaLsXHjRrZv357b1trayhNPPDFjXywWY8uWLXzlK18hGo3y1FNPce+99/Ld7353yvs3btzI+9//fj7/+c8X/Xs4jjPndxGmUuhYJZPJvM/biguUUiqqtY5lnu/NitNsQRIZN2AvQHd3t965c+eiPzd1IcXoc6P4IgsPgR2ziWyNEN5U+hpRx44dYynfq56opbFykg6p11JMvDLhRdH5FUaLUTZr6aw6S4M5c9H5Jz/5ySW3PbmNuUKzHcehs7OT559/Pq999913H//0T//E3r17aWhooLOzk7/5m7/hbW97G+AlRG1ra8Pv95ckHFzCzPOn0LEKhULcfvvteR1b1sRcSqndQHfmMcuTmX09wH6l1Gml1PCsDVQIFVSkz6cr3Q2hxsiGiI8eH2XoW0OM/XQMfOCL+jAbxZU3H7MVMTx+/HjudbaIobC8KatAaa2PaK1btdZHJm3bkXnsy+zbmnmsaIj5ZIyQgXXdwrWkiKGwMK7lMvHqBMNPDhP7Xoz0pTRmk4kv6sPw12+N0Gx2h+zf5OJ7k/fNVsQQmLeIobA8qbiLrxZQSqHRXhHDDqkRJcxN+lqa0WdGvaCHcJFDxGuczs7OKclP59tXaBFDYXkiZ0++GJC6nBKBEuYkPZhm9EejqKDCjNTGGqNizEGVgkKLGArLk/r1NxSIGTZJX0jPm+NLqF+sYYvRH46iAgojKKfVbEx38e3YsSNXeHA62SKG09mzZ88M4RKWL2JB5YnyKZyEgzPq4GuRYRNuYI/Ynjj5FUZIxGk2FkrgOl8Rw8nFCucrYigsP+rySmuP2Fz6i0sENwcLniNIX0uLQAk57LjNyA9G0IaumbIsn/jYJ6a81o7G8BuYjbXRf6F+qMvbvdTFFGc/eZbhJwuLZjdCBukLEm4ueDgJh5Hvj6DRmOHavbgrQ+FaLtoV97VQXdSlQDXe1kjzv2lm+FvDXmbnPFFBhTVs4aYk3LzeccYdYj+IoV2N2VC74gTcKAdvi0AJ1UVdChTA6t9YjXXVIv5s/ik6sgsr09fFiqpnnAnHc+tZetm4xbJ1ogShmqhbgWp9dyu+Nh/Xn7he0PuUT5G+JAJVr7hJl5EfjOAm3ZoJJc8L5VlQhXgUBKHU1O1sv+E3aH1XK9cOXSN1IUVwXTC/94UN0pfTaFeXrByCUJ24KZeRH47gjruYzbUrTp/509Kd9lJtQygmdWtBAbTuagUDBr8xmPd7lKnQjlfEUKgf3LTLyI9GcBJOTYuTINQSdS1Q/jY/Lb/QwtA/DRXkf1co0lfFzVcvuJbL6DOj2CO2iNMiyc7fxmIxlFK5hbpbt25l165duQW7fX19KKXo759aDq61tXVK7j6Affv2laSvvb29nDx5siRtC4VRty6+LCvet4KR74wQeypG2z1teb1HhRWp8ykabpEihssdbWtGnx3FGrQwW5afOBUj01GhbUzPu3fffffx2GOPsXfv3tz+gwcP5vLuZUtrTObAgQPs2bNnSf2ei7179/L+978/V9pDqBx1bUEBNL6pkeCmYEHBEsrvZZVwJyTqaTmjHc3oc6PY12ypdFtCFlNa4+jRo3R1dQGe1dXb28uRI0coFq2trZLrrwqoe4FSSrHi3hVMvDjB+Iv5VRyVcPPlj3Y08eNx0pfTZS0sWA8stbRGLBYjGo3m2jp48GDO+ioW27dvn+FmFMpP3QsUQOu7WjHCRmFWVEBJVollinY18efjpC6mMKNiORWbrIvvxIkTDA8PMzAwMMNa2bNnD4cPH6avr2+GeA0MDORcfkeOHKGzs5O+vr6cRZUVwMns27eP3t7eKW3s2rUrNw82fT5r8+bNPPfcc0X7zsLiqPs5KACz0aR1VytD/zzE2o+szSvXnhEysK5ZuLaL4ROdXy5orUm8kCD1Wn2IUzVU2yi0tEZnZydDQ0MAOesrK075smvXLo4ePZr7zF27dnHkyJGcK/Hs2bPccccdS/1qwhKRK2uG9nvb0WnN0LeG8jpeGQrtauxhCTdfLmitGfvxGMmzyboQp2qh0NIa0Wg0F/V3//33c/z4cfr7+3MuwYU4cuQIPT09U9o9fPjwFEvt5MmTBYueUHzEgsoQ7gzT+MZGBp8YZOXulfktwjUgfTlNYKUUMax1tNaM/XSMidMTIk4lIFtqIxqNFqW0xq5du+jv76erq2vW+af+/v4pbr6BgQH279+fe75169Ypx2fntLIMDw9LzakqQARqEu33tnPu0+eIH4/T/ObmBY83wyapCykaf65RLmg1jNaa8RfHmXgpI07LPEPIJz6Wn9WvbY3ZZFalC/uhhx5i3759c1o5XV1dU0LZJ88xdXZ2zrDY+vv7OX78OHv37qW3t5ePfexjpem4UBBl/+Uppea0m5VSu5VSPUqph+Y6ppS0vK0FX6uPwcfzyyyh/Ao36eLEnRL3TCgVWmvGXx5n/NR4XYhTQSjQ6erNzZe1iApl9+7d9PX1TYnS27dvXy7wYu/evWzfvr0YXRSWSFkFSinVA3x5jn1dAFrrPiA2n5CVCsNv0PYrbYw+PUr6cv4RetZ1q4S9EkpJciDJ+P8Z99Y5iThNQRkKN+3OWwm3Vjl8+DD79u3LRfF1dXXNWGslVJ6yuvi01n1KqbmiEPYAWbt7AOgByr4Qof097Vz926sMfmOQNb+1ZsHjjaBB6nyKcGe4DL0TisnEmQkSLyQ8cTJFnGaQrRNlaVSgdsZneqYKmGltdXV1zRqYIVQX1eRcjgKTxau9Ep0IrAzQ8tYWBr85iJteOFOECimsISuvY4XqIXkuSeJkArO5PsRJa704S0ghv22haBT6G6ypIAml1F5gL0BHRwfHjh1bdFva0jg4qMQsF6d3gfk9kxe+9QL6HQsPqEZjfsdE+Zd+oUskEkv6XvXEYsdKWxpnzEH5FCSL369qpDHYyKXhS7Q0txQe0GOBGlWeRVUnOI5DPJ5/MdN6Jt+x0lozMjLC2NhY3udtNQlUDMhmhIwCMyIVtNa9QC9Ad3e33rlz56I/LHUhxehzo/giM4dAv1Xzsw0/w/yWybb3bFuwLSfhEGwP0tTVtOj+ZDl27BhL+V71xGLGKnUpxegzoxiNBoa/mhwIpeX54edJ+pPEhwq/6GpXYwQMT9DrhGQySSgUqnQ3aoJCxioUCvGmN70Jv9+f1/EVFyilVFRrHQMOAd2ZzZ1AfqvuStMn2t/bzsU/v8j4y+M0bGuY93gjbJC6mCKyPSIT7VVM6kpGnBrqS5zAE5mViZWLeq+bclE+RevdrXWznOLYsWPcfvvtle5GTVDKsSp3FN9uoDvzmOVJAK11f+aYHiCWfV0p2u5pwwgZDD6xcMi5MhXYYI9IVolqJX0tTfzpuCdOgfoSp6WiAgon7uCMynIKobyU9UzVWh/RWrdqrY9M2rZj0vNerXVfxpVXUcyISfSdUYb7hrHjCwuPRmNdlXDzaiQ9mGb0R6OokBJxWgRKKZRSJM/XyYSdUDXI2ToPK963Ap3SDH9reMFjjbBB6kKqDL0SCsGKWYz+cBQVUBhB+bkvFqPRIHk2iXaW35oooXqRM3YewjeFaXh9A9efuI525z8xVUDhjDo4E+IGqRbsEZuR74+gfAojJD/1paB8Cm1pWZQulBU5axdgxb0rSJ9Pk+hPzHucUspz88kJXBXYcZuRH4yA6Vm3wtJRPsXEqxOV7oZQR8iZuwAtd7Xgi/ryKmao/IrURXHzVRon4TDy/RFvfVrYrHR3lg1Gg4F1ycJNycJdoTyIQC2AETBo+6U2Rn84Svrq/Pn5jLCBdcUSP30FccYdYj+IoV2N2SDiVEyUodBak7osN2FCeRCByoP293hZlwa/MX/IebaIoTUsbr5K4Ew4jPxgBG1pzEYRp1JghAySA8llmUBWqD5EoPIgsDpA8883M/TNIVxrAfeGgvSV/DOhC8XBTbqM/HAEN+liRkScSoUKKuwRGychwUBC6RGBypP2e9uxh21Gvjsy73FGgxduLneY5cNNuYz8aAR3zMVsEnEqJdlMEqnz4uYTSo8IVJ40dTcRWBdYMLOE8inccRdnTO4wy4Gb9sTJiTuYzSJO5cBsNEmeSS649EIQlooIVJ4oQ7HivSsY+8kYE6fnDrXN3mFa12QequRoGH1mFHvEFnFaiCLeLym/V8jQGpTfuFBaKp4stpZovaeVS395icEnBln/sfVzHqeCitSFFOEt9VPEUGvtFbdzNbgFPDo3HnN/tveIM3X79Nd20sbSFmaLiNN0rEGLxMkEiZMJxl4Yw7hiMPHFCcJbi/ObVKYidS5FYGWgKO0JwmyIQBWAr9lH9O4ow0eHWbN3zZyT8UbIwB60cS13WWTNtkdtxv51DG1lRMKdJhpu5pGMy0dlH7zFy7nHzGJmpdWNY3XmT039U0rN/9pUKEPha5KfMIA15AnS2MkxEicTpF7z5oiMRoPIGyMkY0kufvEinZ/rLEpGcqPRm2ttfEOj5DcUSoac3QWy4n0rGP7nYYaPDrPi/StmPUYpb72IPWQT6KjtO0w37TL69ChOyvHEdrJI+BT4b4hH2Usx1Eflh1nJCdILGUE6lxGkBoPGNzbS9sttRLZHCN8URpmK5//n8yR6E8SfjtP8luYlf352SUX6SprQBqmbJJQGEagCabilgYZbvfx87e9rn/OirExF6lKqpgVKu5p4fxxnwsHXIj+VSmINWTkxSryQIPXqJEF6QyNtvzRVkKajf1ET/KcgFw9epOnNTUUpc6+CiuSZpAiUUDLkqrMI2u9t57X9rzF2cozI7ZFZjzHCBumLafSbdM0WeRt/eZz0pTRmVOZ4yo01PEmQTk4SpHDGQronI0jbZhekGfhgzd41nP2Dswz+4yAr7p3d+i8EI2RgDVk4CUfWngklQQRqEUTvjnLxv1/k+uPX5xQo5VM4Yw7OiIMvWnvDnLqSYvxfxzGbzZoV2FrCjtkkXkjk5pGSZ73aSzlBendGkG7OU5BmofmtzTRub+TyX1+m9Z2tSxaV3Jqoiykabp6/6rQgLIbau3JWAUbAoO0X27h2+BrWNQv/Sv+cx6avpWtOoJwxh/hzXvXZYriChJnYI3bOXTd2cozkmYwghTxBan1XK41vaqTh5gZvrq8IKKVY++BaXv7Iy1z5uyus3bt2yW0ajQYTAxOeJSc3MkKRqa0rZxXR/t52rj12jcFvDrL6Q6tnPcYIGqTOp2jYVjt3l67tMvrsKAop8FdM7BE7J0aJFxIkByYJ0hsaifZEiWyPFFWQZqPh5gZad7Vy/ch1Vrx3BYHVS5sjNfwG9piNPWTjb5/7Rk0QFoOq1ZQ83d3d+vjx44t+f+pCim898S0mjMXXt4n+eRTfaz6uf+Y6zOYt0aAtTXBdsCYskWzkoTPmoALV399Ae4D0YHXmPVQJReCVAIGXAvhf8uO/4F28dUCT3pomfXMa62YLa5M1+2+nyEweK2PYYMUjK0i9KcXIf5g/dVc+6LSXnFcEqj4xTZP3vOc9S2pDKXVCa909fXvdWlAff+TjPPmdJ/GZSxiCBBgjBu5nXWia/RDtaoxg/q4yx3EwzcpMOGtb46ZdlFH94gSgTY1yqqivDqghhRpTkE1Vp0CHNawA3aAhBLjAi5m/MjF9rFSzQh1XuFddKMbaXTdTGLKK/h1LoZLnYa2xYcOGJQvUXJRVoJRSu4EY0KW1PjDP/k6tdW85+7YoInjrgIYVuml2S1QphbZ11VtQ2q0tcaoqtPcbUIPKy4rRME2QqnBIdZtGxRTqqkJvWroXRaPRbvX/zoXaYl6BUkp9HDiitT47adt2YEBrPTpp2zuBh7TW756nrS4ArXWfUqpTKdWlte6ftn9Aa92vlOqZvr/YfO5Tn+Nr/+trtDS3LKkd3//24T/sJ/nBJHrdLCe667n5wtvCeV38E4kEkcjskYGlwrXdXNRYKec/io3VZOGPV9Ct5IL5QxPf13wYQwbOGxys+yz0hupzm882VuZ3TQJ/HSD1zhTuHUurkqstz1MQ2rg81kRV4jysVWKxWMnaXsiC2g8MAGcBlFItwAlgF/DtScd1Aj0LtLUHOJp5PpA5froA7c+03am17lugvSVhhAwafA2MTowu6Q5XvVmx8msrcY+6xH8tPusxOq1JXk1ihPILOijlP3w6Wmusq14ZbxVQYJfto5dMoDHARHLxc4iLRkPgpwEij0fwX/BjbbKI/UYM65ZM8tRk+bu0ELOOVTe0/Usb5iGT2K0xWIrWa9ATmkBDAMO3PIJrynke1jKldIUuJFCzXboXezmPAkOTXrdP3pmxnAaUUqeBfbN2Rqm9wF6Ajo4Ojh07tsiueES2RGhILT1qSr9dE/5RmOBHgzBLwJ52vLvLfASq3HdubtLF7XA910ztGE8AjDvjNK9fetqegngZjP9hoH6s0Ks17kMuxlsNWoylWeKlZs6xegDMPzRZeXIl+v1Ls/y0rTHCxrKI/hQLKn8SicSSr8VzUTVBEkqpKN7800Hgy0qpfq31wORjMvNSveBF8e3cuXNJn+naLrHvxHBTLmbD4u8Cxn51jFe+/Qobf7Rx1hX6btpFKUXrXa0LrhU5duwYS/1e+ZK8kCT+TBwzatbk3FN/op+uSFdZPit1IcXlv7xM7KkYZotJx3/qoP097TWTDHjOsXobDNw5wNihMW57721LSmnlpl0U+f3Oq51ynoe1TinHqpxnVwxoyzyPAtMr/+0FHs0ET9wH7C51hwyfQXN3MzrtlXhYLA23NRDeFmbwicFZK+kqv5dVwh1fmp+/mNijNokTCYwmoybFqVzYMZsLX7jAzz70M0Z/NMqqD67itr+9jZX/dmXNiNNCrP3IWtwJlytfubKkdoyAgTPuYMdqyE8sVDXlPMMO4c1VkXnsg5zlNIXM/FOsHJ3ytfiIvCGCE3cWXaZdKUX7+9pJnkky9uOxWfcrFOnr1bFmx027jD4zCj6WzUW22DgTDle+eoVTv36K609cp/WeVm79m1tZ8+/XYDYur/Dj0OYQ7b/SzvWvXyd5bmkTaEqpXKkPQVgq+VydZrtqF3wlz0bkKaV6gNikCL0nM/sPAHuVUruVUnvLGWYe6gwR6Ajgxhdv4bTe3YrZZM5dEj4A6QuVFyitNYnnEzjjzpLcmssV7WgGvzHIix98kct/dZlIV4Rb/voWNvznDct6IWrHhzowggaXDl5aUjtGxCB5Lrkkj4QgZMnH4XxEKRWbZdvk19F8Pmw20dFa75j0fMbaqHKglCJye4Thbw/jptxFTfIaIYO2e9q49g/XWDu0Fn+bf8Z+65qFa7sVjXKaeHmC1IUUZquI02S01oz+YJRLX75E6lyKhtc3sPmRzTS+obHSXSsL/lY/q359FZe/fJnEyQSR7YsLEFCmt+4vfS1NcE2wyL0U6o2FBKr6F8sWCTNs0rSjidEfjaL8alHzMu3vbefa4WsM/eMQHb/RMWWfMiYVMVxVmRpRqaspxn46htkiGconM/aTMS4evMj4T8cJbgyy+dObaX5rc92N0crdKxn8+iAXv3iRbV/atui5SeVXJM8mRaCEJTOvQGmtP1KujlQDwdVBwlvDTAxMLCoDeXB9kEh3hMFvDLLq11fNXFVvQPpyuiIC5Yw7JJ5LSIbySSTPJbn05UuMfn8UX5uP9R9bT9svtdXt+BgBgzW/tYZz//Ucw/8yTNs9bQu/abZ2GgzSV9I4SQczJJa6sHhkhnwaDa9rwGwyccadRb1/xftWYF23GPnBzCScZoNJ6kJq0cEYi0XbmtFnR3N5Aesda9DitT95jZ/95s9I9CdY/e9Xc+vf3Er7e9rrVpyyRO+O0nBrA5f/8jJucnFzsrmgoEuVn3MVapu8zASl1N3AkNb6ZOZ1M/BloAsvs8R/mZwOqVbo64NEYvpWAzcdJX057S3gLfR63hpmRdtFXvq7GLFVM+vt6LQmENcY82QLf/zxAj9zHm5kKA/XRIbyQgi0G/zL+fxLmagJm4ajF2l88hI4mom7VpP4xfVcavLDqyXsaBVQyFj5f2ULbZ/7KT/6YoyxX1q/qM/TjkZdVATW1G5F6WKeh8uZUubUXVCglFLfwktL1As8mNncjxcqfgR4F7BbKbVVa11Tp3kiAdHobHsMLJ9J+kraszgKPL/cd6wi+L/O0zI8hl4zNVW0i0vA7+CPzq58iQQUcwG7FbNJp9MYTQao5RVZZRnQHMrjLt92MY9dxf/EBVTCxr6zHfvfrketCmWS0FfP+rRSkfdYAbw+gtPdRuO/XMC8ewVEF+GS1uCmHMJhP0ao9gSq2OfhcqaUGaHmtQ+UUr+LlxvvXVrrBzPb3kkm957W+n6tdRterr79petm+fG1+jAjJjpd+EXdfvsqtKnwfXvmwkflUziji3MfFooz4WBdthYlsssCV2M+M0jwEz8m8Lev4m5oIPmHr8f6yE3oVcsjqWmpsO7bALbG/w/nF9eAAoXCHpVFu8LiWciB9QCwX2v95KRt9wGntdaTk8UeAXawjFBKEVgT8Or5FLqmo9mPc0cb5g+uQ2qqGClT4Uw4JV8nom1N6kLKK4ZXh9NOxqkRgp/+KYEvvQJBk9THbiH9u7eit8htcT7oVSHsng7M719DnZu5+DwfVEBhx2y0W1uWu3Y0NVrHddmx0KWrE3hu2rYePEGazGluZIlYNhg+g8DaANrSBS9Ndu7uQE04mE9PW7ibubN0JkpnRWmtSV1KeQX0/PVlOqnXxgn86YsED7wIoxbpD3eS+uTP4b4hCjU6F1Ip7PesgwYf/kPnWNQV28jUGZuoDReqdjXWsMXE6Qkv/VmNCetyZKE5qBg38udly210cqNsRpateCU0lh2+iA+3zcUatvIulwHg3hTB3dCA78krOG9fOfXiaIATd/A1lSZXr3Xdwkk4BfW31lGDKXxfO4/5w+sQNrHu34jd0wGSymnxNPqw7l1H4O9exfjxCO6bogU3oQyFFbOqOj2U1hon4XhlZ9IuRsDwxOqaRaCjMmsWBY+Fzt4TeG6+LA8Depp7D7zEriUrLlhp/Cv93o+2kPkopbDv7sB4bRzjlamhgsqncBKLz/03H3bcxrqeEdN6MBjGbHyPnSP4X17AfGYQ+541JA9sx/7FNSJORcB5xyrcjhD+x14Fp/Dfq/Ir3LhbtamPnAmH1LkUqfPe8g8jbICZEdYhCzsuc2iVZKEzeB+wQyn1nFLqEPAQk7JLKKW2Z7Z34pXJWJYoQxFcG/RM/gK8Fc5b2tFhE3N6sETW9bHIdSZz4aZc0hfTXjj5chYnrWE4TeAJCO07ie+fL+Hc2U7qj9+Eff9GaKyaKjK1j8/Aun8DxsUk5neuFv5+lVnqkKiuC71ruaQupkieTXrpzULGjLpwKuCt5XKt2nBRLkcWyiTRr5R6F54w7QB6s9F8Gfrx3IAPzGJVLSuMkEGgI0D6cjp/6yRo4rx1BeZTV7E+YEHzjfx8Cq8EhxkujutDO5mgCMXyWmxquahLExjnxjFeG0e9lnlM2IDCeUME676N6A35r4cSCsO9vRXnlib8j5/HeUs7hAu7AVB+hT1s42vxVXxNlHY01pCFNWihlJr3XM7lFbyYJrghKGVpKoBaiptJKbVFa32miP3Jm+7ubn38+PFFv/93fsdbqOsr6FzTuCntuTry9R6lXYwzY+gVQXT7JH+2BhQY01LBOE4M04wW0qlcv7SjUbXs1XI0JF1UyoGUg0q6kJ5096qAoIEOmhA0cBuTqECZK+rWKNocRTlLGKuki/HqGLotgF5ZeI497YIZMqBiF3mv5pu2vAi9hc6TyeOlXTD8CiUu41nZsOE83/jG4hZ0Z1FKndBad0/fPu/lOZMxYj4GJx+jtR5dZP9qBIURADepcwKzIAED3WCiYumpAqW8H753tiyx5LxdY+KkgXRWiFxIOqiUO3WOw6c8IYr4IGiiQwYEpn1BMwnlWVImhAx0sx81lEZHA7CI6FDvN1pugdJoB3TazZ1qhZ4nygDX0hiGXl7eiRpgIfthuMD2qjdUZxp/9mdeKpPZM0nMh8JOaFKvpfJ29RnHRwn++cuk3n0z7u2tue1u0iW4JjilzHYicZJIZGfevXHGHJLnkt5i3GoUqISN8dpYzjVnnBtHXZxAZSbNtU+h14ZxNzTgbmhAZx5pmlyuxGW2yT+r6ST++M+X53vUOEUZqyGL0MM/xlnZivWRmwp7r+PNu4ZvCpfNzecmXdJX0zhjjlehwJf/504fL217N6WhzSEp8jmNWOwVYGkW1FwsJFDZ/+hpvCCIirjzqo1CQ8/d21vRrX58375CepJAKUNhx+0pAlUI2Yle5V9EzsBi42rUleTUeaJz4xjDNxKG6mYf7oYGnJ7VOSHSa0JQwfpYQgG0BbHfvRr/Ny5i71qN3lrAomcTtKVxx92Sh5y7lot13cKO2Shz/nmmfFE+hU5p0pcy81Gypq4sLHRlbMULM78fOIAnVEeAQ9nEsfWKf6UfZ8xBp/XCSVhNhb2zA//XzqMuJ9GrvTQ7yq9wx1y0W7jrQ7uZoAgXVLDMJ8u4jXE+I0BZMbowgcrMF2kD9Jow7i1NOBkhcjc2QIusKal17F9ai++71/D//aukf+91BbmnlVLYI3bJBEo73kJbe9AGTdGXWqiAF9hkDVkE2uW3XA4WiuIbwROmA5lFunvw1jztU0oNA4eAg1rrF0re0ypDGYrguiATZyZQ7sIWjP32lfi+fgHz2BXsX9uUaQQvqeZEYXeVWmtvUeGEW/rFuK7G+OkIxiuJG9bR9dSNvjR6VpG7c9UNN93asKxBWq6ETKz3ryfwP85gHB/CvaM977eqgMKJO95cVBHncrTWOKMO6atpL3tKoEQeBQVG0KuMbYZNzIaamdGoWfL2LWXEqhfoVUpF8ayq3cDzWbHSWn+0JL2sUoxgAaHn0QBOVyu+713Dfv96CHo/bq20F25egEA5o84N92KpjCetMX4cw/8P5zHOjaMV6I4Q7pZGnLtW5sSI1oCkEKoznLetxO27jP/wa6S2t+Z/M6JuZG1YrFt7MlprnHEH64qFm3JRAVX61F6G5+5LXUwR3hwuaF5LKJxF/Uq01jFuiNVv4WUyfwCYV6CUUrvx1k11aa0PzLK/i0xOP6319Hx/VYkv6sNJOLjj7oKuNuedHfieG8J8ZhDn7asA78duj9r4V/rz8mu7SZfUpRRGoETipDXGv47i/9p5jNMJ3JVB0h/uxOluy4mqUOcYCuvXNhH83Iv4nryCfc+avN+qTC+B7FIFyk1lAiASDsqnvAwQZUL5FDqpverY6wIyH1VCCv6vKqWalVL/Vin1LaWUA/wx8BheXaj53tcFoLXuA2LZ19N4ICNMnXPsrzpyWc+NhbOeuzc34a4Le2U4MuvPlKnAJq80StrWpM6nvPeUQCuMl+IE9p8i+LkXYThN+kNbSH32jThvXSniJEzBfX0Lzhta8H3jAiSsvN+nfF6iZDe9uOwMru2Svpxm4swE7ngmA0QFEiKroBfgZA9XV4aM5UZeApURpQ9nihfGgL/Ai+h7l9a6XWv9kWklOWZjT+a94CWW7Zn2GbvxgjDQWh/QWtdMbj/DZxBYk0fW82x+vlfHUQM3Shho9IIl5rMZyrWti35CqoEEgT95keCj/4pxOUn61zeR+uM34dy1SiLshDmx9myECQf/Exfyf1Pmp+vEC1vApl2NNWiRPJ3EjtkYQcPzWFTKeMnOR121cJKyGK9ULLRQ98N49Z968MTlMWCH1vr5RXxWFBia9Hr67Oodmc/swiuGOMMFWM3kG3ruvGUF+vA5fN++gpUJ01WmV8TQ3+qf833WYPEzlKtzY/i/dh7zZAwd8XkZwO9eJdaSkBd6XQPOzlWYT13FfmcHenV44TfhRa9awxa+toVTH2mtceIO6StptKM913a13DMZgAnpC2lCm0OyiLcELOQI7sWzCQaAvsy2vXP8qHQRgiQGM/n/epRSu6fPQyml9gJ7ATo6Ojh27NgSP84r7VwsdANon8aZLztEE5h3gf/b15j48DV0Ng+Hq7HiBq6bIJE4NrVdB1ztolapoiROMM5D8DHw/1ChGzTJX4P0L1sQfhV4tQifUB60mcBqerrS3agJSjVW9v8FkafB/IcXmNhXQH9cjRM35szqoDXgZrKkuBrVCqji/P7z6l8B46VdjTXiBWjU53RUoijX4tlYSKDOcCOpz64Fjl1oEiXGjdpSUWBaJT9Oc8PCGsCzqKYIlNa6l0w29e7ubr1z584FPnJ++vqKK1AAruP5yJVv7lDX9JvHWfGtF+AbG5l41zrAm4Pyr/Rjhr+Hbe+80Z7ltYex9CSw5rUkjd88T+jZa+iAQeIX1zD+zrXoRh8k8P5qiED7D0kP/ptKd6MmKOlYvesCTU+cI/6D12Hd3JLXW3RaY0ZM/G0zvQau5WLHbNxx1yt9YZbflVfQeGnv+/jaffgi9ZdJ3zSPsdRr8VwstA5qaxE/6xCQTQbYScYiU0pFM1GBfXhh69n90yv5Fp2enoWPKRyDiQFN4mQcs9Wc3YVxK7zy9UYCT1/mrf8x6rn44g7B9UFOjED2f+3aLiPfHcEJO5iRxbvd0lfSXPnqFYb+eQhlKlbcv5JVv7YKX9QHpDN/tUd/wqXr1vFKd6MmKOVYuVtaePFpP+v+8QzbvrQtr0Xn2vGySrTd05ZLHeQmXcZfGic5kITVYESMikXIFTpe2vaWi7S+oxVfc32JVImMJ6CM3txs0INSqgeITQqCeDKzfwAvum935nVNhJnPRmhLiMDaAG587kilFfeuIH0pTfzZOABG2CB1cdICWK1JvJDAHrUXLU7WoMWFL1zgxQ++yPC/DLPivSu47e9uY+1H1mbESRCWjhE0WPPhNUy8PMHw0fzSdypToR1N+moabWvGXxln6OgQEwMTGM0GZtMcN3dVivJ5uf5GnxvFtaV+VLEo61Uq46Kbvm3HfPtrEaUUTdubGP72sFcMLTjzPqDlbS342nxcf+I6zW9p9tZWJLys5ADJM0lSr6YwWwsXJ3vE5ur/vMr1x6+jbU3bPW10fLBDylcLJSN6d5Rr/+sal//iMtG7onkF86iAYuKlCcZ+MoabdDEjZk0vfDUbTeyYzdhPxohsj9SUwFYr1RIPs+wwQgaRHRHccS/X3nSUT9H+K+3En41PtZxsTfp6msSPE5jNhd1FOgmHS391iVMfOMW1w9eI3hXl1q/cyoaPbxBxEkqKMhRrP7oW67rFtceu5fUeI2xgx2xQ3oL3WhanLGaL6d1cXkgtfLCwICJQJSTYESR8UxhndPbYo/ZfaQcFg1/34kWMsIFOa+LPxWctQT0XzrjDla9e4dQHTnH1q1dpurOJW/7qFjY+vJHgusKLywnCYoi8IULL21u4+j+vYg0uvHhXKYWv1Terh6FWUUphNBkknk9UXZn7WmT5/DKqlMbbGvE1+3DGZoqUf6Wfll9oYeifhnK5xLTtVf7Mq4xH0uXqoauc+sApLv/VZRrf2MjNX76ZzY9sJrQpVIqvIwjzsmbvGrStufzXlyvdlYph+L00ZPHn4jmXvbA4RKBKjPIpmrqbcsIznfZ723FGHWLHYijlTbQakfn/LW7a5frXrnPq109x6UuXaLilgW1f3MaWz2whfFN+iyUFoRQE1wVZ8f4VDP3vISZOT1S6OxXDjGTmo06NLXywMCciUGXA1+wj8oYIzqiD1lNFKnJ7hODGIIOPZ5aFKeacd9K2ZvCbg7z4wRe58IULBNcH2fr5rXQe6KThtoZSfw1ByItVH1yF2WRy8UsXZ/ze6wmzxWTipQlSl2Q+arGIQJWJXOj56NQQVKUU7fe2M/7iOOM/m33dhXY0Q/8yxIv/94uc/9x5/O1+Oj/XydY/20rkjQVUNRWEMuBr8tHxwQ4SxxO5ZRT1iDI8b0j8RHzBXJvC7IhAlYls6LnyK9zUVJFqe1cbRsjg+uPXp2zXrib2VIyf/fuf8dqjr2E2mmz57BZu+vObaNrRJGGsQtXSfm87gfUBLv73i3U9D2MEDNAQPx6fNZpXmB8RqDIyV+i5GTFp3dVK7NsxiHuLdEd+MMJLv/USr/7Rq6Bg06c2se1L27w1UyJMQpVj+A3WPrCW1KspBr85PatZfWFEDKxBa04PiTA3kk6gzAQ7goS3hZl4ZWJKNof297Yz+I1B1F8qXr7wMhMvThBYF2DjJzYSfUdUMiULi0ZrrwyM1rqsNzfNb22m8Y2NXPnrK7S+s7WgqtHLCaUUZovJ+Ivj+Np9BFfJ0o98EQuqAjTeOjP0PHxTmIafa8B40sAetln/u+u59Su30trTKuIkzEC7Gm1p3JSLM+7gxB3smI0ds3FGHOyRzGPM9uY9NTjDTlndbUp5i3ftmM3Vv7tats9dDNrVjL80zpWvXuHMJ87A2eK2rwyF0WCQOJ7AmZD5qHwRC6oCZEPPh58a9goQZhbkbvjdDZx64RS33nNrLoGmUB9orb3yEq4GxwuM0Y73PHsbqVBoNEp7mfJVSGGGTYyQgRE2MBoMjKCB4TdQAYUR8KrNKr/C/I5Jw9oGxn867qUUKlMV2oZbGmjd1cq1w9dof087gdXVk9HEjtskjicYfXaU+DPxXHVcFVQYLxlYB61Zs60vFiNo4Iw6JJ5P0PzzzXkl1a13RKAqhK/ZR+SNERInE5hRL6VRaGMI3aZFnJYJWmfExr0hNtq5UXVZqYzgoDz3m195AhMxPOEJZ4Qn6AlNVnCMgOGVoSjQXdd4cyNm2CR+Ip5rtxys/vBqYt+JcekvLrHp9zeV5TNnQ2tN8nSS0Wc8QRr76Ri4YDaZNHU30XRnE013NGFdt3jpt1/izCfOcNN/u6moRUKNJoP05TTjr4zTeHNj0dpdrohAVZDQ5hDWFYv01TRmc33656uNrCWDzlgz2cJ5s23nhkhkhWZKO8qL4jJCBkaT92g2mKiQJzJGIGPpZCyectxRhzaEMEIGo0+P4jgOZkPpf3eBVQFW7lnJ1a9eZeWvrizrmj0n4RA/ESf+bJzRZ0exr3tWUnhbmFX/bhXNb26m4XUNU9zo/jY/7n92mXh0gnN/fI5Nf7ipaP+b3HzUv47jb/MTWFE9FmU1IgJVQZRSRLZH5s16LsyDvmGRzCYmk0UF8BZBZ9xk3ssb4pJtTykFPs8Na/gy+RD9eM8z7rLsn+EzvAtbpqjelL9AtsJq9blxAisDRO+KMvLDEZzE0mqN5cuqX1vF0D8OceGLF7jpCzeVbFy01iTPJok/7QnS2E/GwAGj0aCpu4nmO5tpenMT/vYFXHdvgTUPrOHSly5x+a8us+bDa4rWR2UqjJBB/Hic1ne0ynk/DyJQFSYbej76w9GyzQvUMlpr3AkXndZoNDqpweeFNWdr8kwREX9GWHw3xGO6oEx+PV8mj+WEr9lH9K4ooz8axR6xC86cXyhmg8nq/7Ca8587z8h3R4jeFS1a286EQ+JEwnPdPRvHuuolqg11hli1ZxVNb26i8fWNBWdLX3n/SlKvpbj6t1cJrg/Sdk/bwm/KEyNk4Ix481FNd8qaxrkQgaoCcqHnL09A8eZklw2TRQnlWQDBjUF8L/tof0d7pbtXs5hhk5ZfaCF+PO65mVtKK1Jt97Rx/R+uc6n3Es1vafbm0haB1prUudSNuaQfj3kJlhsMmnY00fQbTTS9uYnAyqW5z5RSrP+d9aQvpjn/J+cJrA4Q2V68zC1Gs1ek1D/gJ7xVcmjOhghUldB4ayPWVQsdl9XmMLcoBVYGbrhEXqlsH5cDRsCg+c5mEi8kSL6a9ESqRHNhylSs/chaBh4aYPDxQVbevzLv9zoTDmMnxxh92rOS0pfTgDePu+JXV9B8Z7O3TKPIAUbKp9j0qU288h9f4ewjZ9n259sIri/OOialFGazydhPxvC1+/BH5e50OiJQVUI29JynwI7ZXj2oYHXOYZSKWUVpQ5DAqoD46UuIMhWR2yMYYYPxU+Oeu69ExQOb7vCi5a589Qqt727F1zL7JUhrTfp8mtFnRhl9ZpSxF8bQlleGJtIVYdUHVtF0Z1NZCnH6mnxseXQLL3/0ZQYeHmDbn2/D11ycS6fyKQhC/Nk40Z3RRVuVy5WyCpRSajcQA7q01gfmOe6h+fYvV3zNPnwRH40bGklfTmPH7FwWACOYEaxltnZihiityFhKIkplRSlFw60N3mLS/gRGo1Gyi+XaB9bysw//jCv/3xXW/ad1ue1uyiVxMpFz3aUvelZScGOQ9nvbaf75Zhrf0FiRi3hwXZDNn97MwMcHOPvIWToPdBbNWjPDJvaITeLHCcmxOY2yCZRSqgtAa92nlOpUSnVprftnOa4HuKNc/ao6TGi4uYGGmxvQtsYetbGGLNKX01hDFkpn1swEvEigWhQsrTVu0kWnMqLUHiC4SUSp0iilCG8KexFmz8RxHRcjXPz/R2hLiPZfbuf6E9dp/jfNufmkxPMJdFqjgp5Ft/K+lTTd2URwTXWkBoq8McKG393Auc+e4/yfnmfDQxuKJiZms0nqXAr/Sj/hTTIflaWcFtQe4Gjm+QDQA8wQKOEGyqfwt/nxt/lpuKkB7XiCZQ/bnmANWrlwahXwFnlWa1qkKaIE+Ff4Cb0+5IlSERdCCksn2BHEfLvJyA9HcBPuggU0F0PHhzoYfnKYgY8PABBYH6D9V9ppurOJyPZI1bq6Wne1knwtydWvXiW0McSqD6wqSrvZ+ajEyQT+Vn/RXIi1TjlHIQoMTXo9I/wqY1X1KaUeKFuvaghlKvytfvytfsKdYbSrcUYdrGGL9JU01jUrtx5I+TMWVgUFa4YotYso1Qq+qBeGPvKjEZxRB6PJKKrryd/mZ/MnN5N8LUnzm5uLFnhQDlZ/aDXp82ku9V4isC5A9O3RorSbXSYx+two0buiGD45R6pNpou30KAOUIbCF/Xhi/oIb8kIVtzBimVcgtcstJPJ3eanLIKltbc2KVvzyt/uJ/h6z31nhiRbRi1hNppE3xZl9NlRrEGr6GHoTXd4qYVqDWUoNuzbQPpKmnOfPUegI0DDLcXJjmE2mjgxh7GfjBHZHqn7+ShVrpLMSqn9wNGMhbQb6JwcCDF5TkopdVhrfd8sbewF9gJ0dHTs+Pu///uy9L2cJBIJIpEiVsl1vVLx2vayX2syGReUd6JRjN+/5kY2BzJZGDJ544rS/hwUfayWMUsdK3fcxU27ucXMy51xZ5wGcwHRGQbj4wZY4P6JC/lHzc9PJkOK2Vi+pL5LoRjn4Tve8Y4TWuvu6dvLaUEdArId6AT6AJRSUa11DOhUSnVm988WRKG17gV6Abq7u/XOnTvL0e+ycuzYMUr1vbTWOGMO9rCNddXLAeimvVIMyufNYeHLL5PCrJZSJvrODJfHUirlWC03ljpW2tWMnRpj4mcTniVVpXOdxaI/0U9XpGv+gyIw8ccTvPL/vEL4M2Fu+sJNRctt6FouOqlpfUtrWVJRLYVSnodlEyitdb9SqjsTpRebJD5PAju01kcgZyVFy9WvekIphS/ihbKHNoS8OaJxF2vYwrrmzWO5Y57gKFOhQpkUQdmEqBlR0inPEvO3+Wm4rYFAR/lESagMylA0vs7Lhp54IYERMSTrPhDeEmbTI5s48/AZXv30q2z5r1uKIt6G38BJOd581Nvrt2BpWeegMhbQ9G07ZjlmxnFC8VFKYTaamI0mofWh3JokO2aTvpb2BGvEEyyNV43V1+ojeFtQRKkOUUoR7gxjhA3iz8Zxg64EuwDNb25m3X9ax4XPX+Dily6y7j+uW/hNeWBGTOxhm7FTY0R+rj5d2dUWJCFUEKUUZoOJ2WASXBvMReHZMRs37RJYGShLeQahugmuCaJ+QRF/Oo4z5tRtKffJrHjfClKvpbh+5DrB9UFW3LuiKO2aLSYTL094LvQqWQ9WTuT2R5gTpbyKrcE1QcKbwiJOQo5Ae4CWu1o8V9SolDAHWPvRtTTd2cSFL1xg9NnRorSpDIXRaBA/EccZr79xFoESBGFR+CI+om+PYrZ4qXrKFRFcrShTsekPNxHaHOLVP3qV5JlkUdo1AgZovKzzV9JYwxbOuOPVQlvmiItPEIRFY4QMWt7aQuJEgtTFVEmzodcCZoPJls9u4eUHX+bM753hpi/ehL916VnKjYiBHbMZfXp0SoFNw29gNHh/2flkI2jk/lRwaqBTrSECJQjCkjB8Bk13NGH8H4OJV+ojDH0+Ah0BtnxmC6/8ziuc/YOzbP3TrUtO3ZRNhTSZbMVo13JxBh2vXI+tpxTd1Fp7FXzDGRFr8ETMDJuooLohZIHqFLGyLdQtNt3d3fr48eNLaqPvdB+JdKJIPSoSrwKbKt2JGkHGKn/KMFYaL/WWHbO9BaY1PIEQSARIR9JLaiP4nSDRT0aZuHuC0d8frdwCZ31DzLKL6mfFBMPMZJvxz1J12phdxMxzJu9593uW1EWlVMUX6lYdiXSCaCha6W5MIWEkiITqM6S0UGSs8qdsYxUCO2STvpT2Fn2XqK5UqbGwaKZ5aY3cBdaHLcJ/Eca3wYf9Ibs4nSsUlfmb74Yhmw3G1pAGPZ7J6Ym64VIkI1J+heH3MsWogGLMHStZ1+taoARBKD6+Zh/Kp0hdSKEtXRPpekqF/e9s1HmF/yt+9DqNs6tKI/EyIpadP1STzL3c80kpzZwJBz2mwbmR4qwU1LARLghCtWI2mIQ2hkCRy2ZflyiwPmbhvMnB///6MX5Sw5fcjBWmzIwVFTRKfvNRw6MlCEI1YwQNQptCqIDCTbpzz30sd/yQ/lQa3aEJ/EEAdbF+LcpCEYESBKFkGD6D0MYQZoNZ3yLVAunPpsGFwMMBqLLYrGpFBEoQhJKiDEVwfRBf1FfXIqU3aNJ/lEZdVAQ+GYAKxUzUEiJQgiCUHKUUgdUB/Cv8nki5le5RZXC3u1gfszBPmPi/4K9bsc4XieITBKEsKBSBFQEMv0HqcsoLU67DBb3OLzpY5y38f+dHr9fY94spNRciUIIglBVfiw98kL6QRrtepgOv0PMkc2K6ZTHf61zmHz37sdOYvLYnG0Kde51diFpiC8/+DzbGeQPfl3y461zct9apSbkAIlCCIJQdX6MPY5NB6lIK7Xi1xpS6UU4+JxRq6p9Czdw27dhcO7Ptn/aYa2/yZwH2gI1OalSoRBaeAemH0wSvBAn81wCpL6TQ28TfNx0RKEEQKoIRNAhvDle6G7Oi/AoVVKUVqRCkPpMi+NEgwd8LkvxiElaW5qNqFQmSEARBmIZSitD6ECrkiVTJaIf0Z9IwBsFPBGGidB9Vi4hACYIgzIIyyyNS+iZN+g/SqNOKwGcDdRvhOBsiUIIgCHMwRaRKmLLJfYuL9aCF+X0TX6/MvGQp60gopXYDMaBLa31glv17M0+3aq33lbNvgiAIs5EVqeRrSXRKo4KlmZNyftXBfs3Gf8iP3qBxfrlKE8uWkbJZUEqpLgCtdR8Qy76etL8H6NNa9wKdmdeCIAgVR5mK0AYvr2DJLCkF1m9bON0O/v/mx+gXB1c5R2APnvUEMABMF6DOSdsGMq8FQRCqgrKIlAnpR9Lo9ZrAIwHUufpbyDyZcgpUFBia9Lp98k6tdW/GegLoApZWLlcQBKHIlEWkIpB+1Cv4GHg4ACOl+ZhaoOpm4zKuv6Na6/5Z9u0F9gJ0dHRw7NixpX1Y0qs0Wk04SYfEy9XVp2pFxip/ZKwKY6Hx0lqj0xpSlOY2vwmcTzg0/F4D/k/4Gf/0OPhL8DlLRYNy1dKvxXNQToGKAW2Z51FgcI7jemYLoADPygJ6Abq7u/XOnTuX1KHHTz1edSXDEy8niGyrrj5VKzJW+SNjVRj5jJd2tBc4kS5R4EQ3WP/FIvDpAA3/vQHrYetG1osqQVuaVCrFUq/Fc1FOgToEdGeedwJ9AEqpqNY6lnm+NytOSqmeTECFIAhC1ZF195Uyus+5O5NY9q/9EAD3TS56tUav0ugVGsyif2RVUTaB0lr3K6W6M9F5sUkuvCeBHZnt+5VS+/AsrfvK1TdBEITFUA6Rsj9oo64pfP/og2/e2K4NT6R0R0awso+rdE7EaCx6d8pKWeegJgVBTN62I/PYB7SWsz+CIAhLpeQipcD6zxbWRy3UNYW6olBXM4+Z58YpA/UdhbKnfrZunCZgk0TMXe16pkAVW2FVFyQhCIJQa5TDkiIMeqNGb5wjetABhvEE64oxVciuKoyfGqj4NAEzNHrlTPGaLGo0FP+r5EvdCtTv/PPv0DfQh8+oriFwJhzMgSq+pakiZKzyR8aqMBY9XhrcVKZicCXX2bZm/m6dtt0F0oAFKq1yz0lnXlvAcObvxcx7TMAPOqAh4D0nANqvwQ8bIxt5P+8vydeorquzIAhCLaO8MiJVIVKzYQAh709Pq+yovaqRYOMJlpURsMmCliCXzDZb7NHXWjoZqVuB+rN7/ozHTz1ONBStdFemIOHA+SNjlT8yVoWx1PEqeQh6JUngzYVdVqhLiuTmZMk+qm4FShAEoVSUZU6qUkRARzR6i0ZbGidVuqS21WaACoIgLAvKkhZpmSMCJQiCUCJy9aREpBaFCJQgCEIJUT4RqcUiAiUIglBiRKQWhwiUIAhCGRCRKhwRKEEQhDKREym/iFQ+iEAJgiCUEeXLRPeJSC2ICJQgCEKZUT5FcENQRGoBRKAEQRAqgOEzRKQWQARKEAShQohIzY8IlCAIQgURkZobEShBEIQKIyI1OyJQgiAIVYCI1ExEoARBEKoEEamplFWglFK7lVI9SqmHFrNfEARhuSMidYOyCZRSqgtAa90HxLKv890vCIJQL4hIeZTTgtoDxDLPB4CeAvcLgiDUDSJS5RWoKDA06XV7gfsFQRDqihkiVWc6VVMl35VSe4G9AB0dHRw7dmxpDSYhYSSW3rEi4iQdEi9XV5+qFRmr/JGxKoxqGy+tvfLqOqVR3Cgfr8m8zm4qd2V5DcpVS78Wz0E5BSoGtGWeR4HBAvejte4FegG6u7v1zp07l9Shx089TiQUWVIbxSbxcoLIturqU7UiY5U/MlaFUa3jpV2Ntm/8uZaLTmnctOsJmDvNxFKgDAVG5rHIAqYtTSqVYqnX4rkop0AdArozzzuBPgClVFRrHZtrvyAIguChDIUKKAjM3KfR4HqioW2Na7votMZNZcQrPdVFqNEoNUm8DMpvgS1A2QRKa92vlOpWSvUAMa11f2bXk8COefYLgiAIC6DIiE1QQRBMzCn7NRqcqQLmpjIilnZnDcSYYX2VWcDKOgeVcdFN37Zjvv2CIAjC0lEoMEGZnsrMEDCt0Y7OCZi2bgiXa7loZ+b8V6mDNmoqSKLYRAIRYslYpbsxFZfq61O1ImOVPzJWhVHP46UAf+av4cbmrIBh4wlZZg4sFA+VrCt1LVA9W6tvqdWxK8fYedvOSnejJpCxyh8Zq8KQ8cqfUkXwgeTiEwRBEKoUEShBEAShKhGBEgRBEKoSEShBEAShKhGBEgRBEKoSEShBEAShKhGBEgRBEKoSEShBEAShKhGBEgRBEKoSpXVtVsBSSl0DXq10P0rACuB6pTtRI8hY5Y+MVWHIeOVPMcZqk9Z65fSNNStQyxWl1HGtdffCRwoyVvkjY1UYMl75U8qxEhefIAiCUJWIQAmCIAhViQhU9SE1sfJHxip/ZKwKQ8Yrf0o2VjIHJQiCIFQlYkEJNYFSardSqkcp9dACx827XxCExaOU6ppnX17naCGIQFWQhf6hSqm9mb/95e5bNZE9KbTWfUBsrpNEKdUD3FHOvlUjefyuujLH7C5336qNPMYqu39vuftWbWTOry/PsS+vc7RQRKAqxEL/0MyPoU9r3Qt0Zl7XK3uAWOb5AFDPYzEveV4oHtBaH8H7XRXlQlKL5HEOdgEDmf0D9TxWkBunoTl2l+QcFYGqHAv9QzsnbRvIvK5Xokw9MdqnH6CU6sqcQPXOvL+rjNV0GkBrfUBr3V/W3lUX+VxUs96Lzjofq4WIssA5uhhEoCpHlHn+oVrr3oz1BNAFHC9Tv2qVtkp3oEqIMv+F4g6gPePmq/f5uijzn4P9eJbTaea2HIQSIgJV5WTcCkfr/O4txg0BigKDk3eK9VQwg9nfk8xDzY1SKor32zsIfFkpVc9ejIWIMc85ulhEoCpHjPz+oT1a6wPl6FAVc4gbLs5OoA9yFxDw5lKyk/51Pa/Cwr+r03juLDKP9RxUEmP+sdoLPJo5/+4DRMynMekcnPUcXSoiUJVjoYsuSqm9WXGq5yCJSXf7PUBskjX5ZGb/kcykfxvehaaeWeh31Tdt/3Pl7FyVseA5mCUbSFGujlUjmRvA7mlWd/YcnOscXdpnykLdypEJXR3Am4DtzWw7obXekflHH8bzfbcB94kbS8iH+X5Xk/YPZfbXtXWex1g9lNnfNmlOWCgTIlCCIAhCVSIuPkEQBKEqEYESBEEQqhIRKEEQBKEqEYESBEEQqhIRKEEQBKEqEYESBEEQqhIRKEEQBKEqEYESBEEQqhIRKEEQBKEq8VW6A4IgTCWTCy6XggfolzRXQj0iqY4EoYrIlHQ4DLxTax3LbDsN7CCTDLfOS68IdYS4+AShujiMV+IhNmlbttprj4iTUE+IQAlClZCxnroypUMmE8Or2yRVlYW6QgRKEKqHLmAuC0lce0LdIQIlCNVDP9MKLmasqihwogL9EYSKIkESglBFZArogTfvFM08xoB9wAkpmifUEyJQgiAIQlUiLj5BEAShKhGBEgRBEKoSEShBEAShKhGBEgRBEKoSEShBEAShKhGBEgRBEKoSEShBEAShKhGBEgRBEKoSEShBEAShKvn/AdjHsMbHPHY0AAAAAElFTkSuQmCC\n" 587 | }, 588 | "metadata": { 589 | "tags": [], 590 | "needs_background": "light" 591 | } 592 | }, 593 | { 594 | "output_type": "stream", 595 | "text": [ 596 | "{0: <__main__.LegendObject object at 0x7f5a0dfa3fd0>}\n", 597 | "{0: <__main__.LegendObject object at 0x7f5a0dfa3fd0>, 1: <__main__.LegendObject object at 0x7f5a0cebfa58>}\n", 598 | "{0: <__main__.LegendObject object at 0x7f5a0dfa3fd0>, 1: <__main__.LegendObject object at 0x7f5a0cebfa58>, 2: <__main__.LegendObject object at 0x7f5a0cecd080>}\n", 599 | "{0: <__main__.LegendObject object at 0x7f5a0dfa3fd0>, 1: <__main__.LegendObject object at 0x7f5a0cebfa58>, 2: <__main__.LegendObject object at 0x7f5a0cecd080>, 3: <__main__.LegendObject object at 0x7f5a0cecd668>}\n" 600 | ], 601 | "name": "stdout" 602 | }, 603 | { 604 | "output_type": "display_data", 605 | "data": { 606 | "text/plain": [ 607 | "
" 608 | ], 609 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAABL/ElEQVR4nO29e5wc11nn/T1VfZuenpmeq+62NJLjW+zEozGJg0MUMloIJIFkZSuB9yXvwr7ywoZsFhYJAyHJBvBKCyQky4ImwC6wIZYlEu9muSwaB+Uld0tDnITYxFZLlnWdm3pmemb6UlXn/aO6Wz33vt/m+X4+I3V3VZ06c6arfvU85znPo7TWCIIgCEK9YdS6A4IgCIKwEiJQgiAIQl0iAiUIgiDUJSJQgiAIQl0iAiUIgiDUJZ5qnqynp0fv3LmzmqesOnNzc7S2tta6Gw2JjF3xyNgVj4xd8ZRr7M6dOzehte5d+nlVBWrnzp2cPXu2mqesOmfOnGHfvn217kZDImNXPDJ2xSNjVzzlGjul1MsrfS4uPkEQBKEuEYESBEEQ6hIRKEEQBKEuqeoclCAItSGVSnH58mXi8Xitu1J3dHR08Pzzz9e6Gw1JoWMXCATYvn07Xq83r/1FoARhA3D58mXa2trYuXMnSqlad6eumJ2dpa2trdbdaEgKGTutNZOTk1y+fJldu3bldYy4+ARhAxCPx+nu7hZxEmqGUoru7u6CrHixoARhgyDiVD5OnTpFOBymv7+f/v7+WnenYSj0OygWlCAIQoEMDQ0BMDo6WuOeNDcNZ0Fl6lfJ06AgCLUiHA5nRUqoHA0nUNZNi/jLcdoekElNQSiGix+5WLG2d35oZ8XariWnTp3iwIEDte7GhqPhXHza0SSvJnGSTq27IghCnnR2dgIQjUZRSrF//37279/P7t272b9/f3a/0dFRlFLLXGe7d+/mscceW/TZkSNHKt/xNFNTU4venzp1is7OTk6dOkU0Gi25/XK3VyuGh4fL6vZsOIECsOYskuPJWndDEIQi6O/v5/Tp05w+fZrz588TDocZHh5etP3EiRPZ9yvd8I4dO8bBgwer0t+lZASkq6uLcDi8TLxq3V4tOXToEE888UTZ2ms4Fx8ANiQuJwhsC9S6J4LQ0Fz88MWS29j54Z0lHR+NRhdFwg0MDDAyMpJ9f/z4cQ4cOLDIsjh9+jSHDx8GYGRkhEgkQldXV8XdcNFolEceeYTTp09z5MgRhoaGeOSRR3j88ccZGBioeXv1QFdXF5FIpCzRjQ0pUEaLQep6CiflYHgb0ggUhA1LJBLJuvXOnj3L0NDQooCDrq4u+vv7GRkZYWhoiLNnz3L06FFOnjwJuDf1cDicbev48eOcPHmSU6dOVbzvTz311LIb74MPPsiJEycWCUokElkksrkcOnSo4PbyIRKJMDo6Sjgc5vTp0xw9erSg45e2lbHoCmXv3r2Mjo5uXIHCABxIjicJbBUrShAaiYyLL8PevXuXPXEfPHiQ48ePAyyLlsvcPMGdu8mIWeaGHo1G6ezsXHZc5pydnZ3cvHkz+/rSpUuA60o8cuTIor4t5dFHH80KZYZnn32Wxx9/fNnvmCtEpbaXD8ePH+fxxx8nHA5nx6dYShGX/v7+ss1DNaZAAXgh8UpCBEoQGpzBwcFlT9wDAwOcPXuWqakpjh49SiQSyW7r7+/PztNkwr2XWhtLRbBchMNhTp48mbWORkZGOHr06LIbeiQSWdWiy7gm82lveHiYwcFBIpEIAwMDawrH448/ng0cyVhPo6OjnDhxgoMHD2YfAk6cOMHRo0c5duwYhw8fZnR0lOPHj/PYY48t22dpH5Ye/7M/+7PZfbq6uhgYGCibew/yFCil1IDWekVJVEplHhN2a62rFlYjbj5BKJ1S54/Kwe7duzl9+vSy+aOMFdXf379IoMLhcHY+6tFHH+Wpp54C3Ei7aqxNypx/amqKaDS6orXS39+/SIiKae/YsWMcOHCA/v5+nnjiiawILxW+3N/5+PHj2Tm5gYEBBgYGOH78OAMDAzzxxBOcPHkyG4AyOTkJuA8D4XA4uz+Q3efYsWMMDQ3R39/P8ePHOX78+LLjM2T+hufOnStbhOW6AqWUGgKOAntX2TaitY4opU4qpYa01is7XsuMMhRaa1LjKfxb/dU4pSAIRZJxqYXDYc6fP79oW+6NPHNDzXye2bZ0nmr//v2Mjo4yMDCwoistd54LyN5gy8WBAwfKGpCxUnsDAwNEo9Fl1uVK5x0ZGcnOzUUikWWZLkZGRrJuw+7ubkZGRuju7s66S3fv3p1tKxqNZrcNDAwwNTWVnQdcevyFCxcYGxtjcHAwK6xTU1PVs6C01iNKqdXiHvvTP8NAJP26aiivIn45LgIlCBuMw4cPc+TIkVUDCcrt4it1TqcYcgV5vYCJ1SzHjOsu9/hc0c+QK/LhcDi7z0pCk3v87Ows999/f7aPw8PDRc2frUZJc1Ba6+GctwPAidX2rQTi5hOEwmmWbA+lRKkVSqNmkSin1ZgP+QSGFEJZgiSUUgPA6dXmqSqFMhTa0aQmUvi3iBUlCIJLJBJh797FsxLPPPPMumHTZ8+eXXTcpz71qYZdj9QMqEzy1TV3Uuq01nr/GtsPa62PrbLtEHAIYNOmTXuffPLJYvsKgLY09pyNMt1ksdrWGF4DI1gfFlQsFiMUCtW6Gw2JjF3xrDd2HR0d7Nmzp4o9ahxs28Y0zVp3oyEpZuxeeuklpqenF3325je/+ZzWenDpvkVZUEqpsNY6mn59KCNOKwVJpN2AwwCDg4N63759xZwyS3IiyfRXpvGE3K5rR+PEHLoe7sLw1F6kzpw5Q6m/40ZFxq541hu7559/XqrGroJU1C2eYsYuEAjwwAMP5LXvund0pdQBYDD9f4Zn0tuGgKNKqfNKqZsF9bJMZNx81oRVi9MLgiAIFSKfKL5TwKkln+1N/z8CdFama/mjPG40n2+zr9ZdEQRhAyAVdatD7X1iZcAIGiSvJdHW+vNpgiAIpSIVdatD46Y6ykEZCm1rUpMpfJvEihIEobJIRd3q0BQCBa6bL3E5IQIlCOvwkTMfqVjbH9r3oYq1XUukom5taAoXH7iLdhNXE+LmE4Q6RCrqrk21KuqWu+JtpWkagVJm2s03lap1VwShajiWQ/xyHGfBQTuN8XAmFXUr295alLvibaVpGhcfuCKVuJzA1yduPqF50VpjRS3il+IkLiXQtsbBITmWxL+5sIwqH/7ih0vuz4ffVFobUlG3uhV1y1nxttI0jQUFbjRf4op7wQpCs+HEHRYuLHBz5CbRL0ZJvJzACBp4wh5QED8fr3UX8yKTaXz//v10dnYuCzjo6upiaGgoK1Jnz55d5AZcqaJuuXPArcZaFXBziUQiDA8Pr/hTSHvRaJS9e/cyMjLCyMjIIjdnNBpl//792W25or5///5FJUoee+yx7PZMxdtGoOksqGw0n1hRQhOQyTW5cGGB5PUkSitUi3JFKQdlKJLjSayYlc2yUq9IRd3828ustRoaGiISiSyad1u67dSpU9nfOVOOJLeWVmZbOSveVpr6/iYXgTIViSvi5hMaG2vWInE5QfxCHCfpoLwKs91EKbXyAQqUUsRfjhO6t7HyGUpF3dXby4zL8PBwtnBghtHRUcLhMKdOnSISiWTbHR0d5cCBA9mAi6WlQhrFvQdNKFAZN1/o/lA2oawgNAJOyiF5I0k8Es8G+5ghE08wv8vUCBnEI3GCdwbzzktZ6vxROZCKuqu3l3HrZQIoMkUac7f19/fzyCOPZI+ZmppiYGCAycnJrJDnZmgvZ8XbStN0AqVMBTakplL4esWKEuobrTXWzZyAB0dj+A3MjjWspVXIuLiT15IEdgQq1OPikIq6hbcXjUY5ceIEAwMDRCIRTp48ma2BFY1GF4l6V1dX1to6ffr0InfekSNHePDBB7PtlrPibaVpOoECwMB184lACXWKHbdJXkmycH4Be94GA8xWs2Sr3wgYzL84j3+7v2CBayQ2QkXdcDjMuXPnsu9zBTocDi/6/XLFNyNiS/8Hyl7xttI0pUAZQYPE5QSh+8TNJ9QP2tYkx5PEX46TvJYEuBWFVyaUX2FP21g3Lbxd3hX3aZZsD1JRt3CqFe1YLppSoJSp0JYWN59Qc7TW2DGbxOUEC5EFdEq7AQ9FuPDyQSmFMhXxC/FVBWojIBV1m4OmFCjAdfNdFTefUBuclEPyepKFyALWTQulFEargWqtvEVvtBrEL8dpvbcVI9BUSx3zIhwOs1al8Mx8WOb17Ows4M5/5W4Tak/TCpTZapJ4Je3mM8TNJ1SebMDDy3ESr5QW8FAKylCgIX45TnBPsGrnFYRy07QClZubz9cjVpRQOewFm8SVBPFI3A14MN3w8Fo+GBmtBgsvLdDS3yIPaELD0rQCBYCC5JWkCJRQdrIBDxfiJMeSoMsf8FAKhtfAmrOKys8nCPVCfVxNFcIMmq4v/r5WeYoUSkZrjT1rk3glwcLFnICHtTI81BDlVcQj8WUC9ZGPVLAe1IeaI0JQqA+aWqCUR6Fj7ryAt3vjRjQJpZOcSDL3nTmsaHUDHkrBCBokx9z8fILQiDR/iE960a4gFEtyPMn0l6exF2zMDtMNevDUtzhBOuQ8nZ+v1mQKFe7duzf7k8n0sHRbpohhJp3RyMjIikUMOzs7lxUxFJqLpragwHXzJV5J0PpqcfMJhZOcSDL9lWmMgIHhb7znuUx+Pr175bDrD3/4wyWfI982+vv7F2VGWGvbI488wlNPPZVdWJpJVZTJmDA6OlqT7A5CdWm8K65AlEfhpBysm+LmEAqj0cUJbkWzYte6J4WxUhHDs2fPZt9nihgKzU1eV51SatWl1EqpA0qpIaVUfql7a4AyFImr4uYT8ic5kWTmKzMY/sYVpwxGwEBbes3Fq9Ugk90h85PrnsvdtlIRQ2DNIoZCc7Kui08pNQQcBfausG0AQGs9opTqV0oNaK3rrhKWETRcN9+94uYT1ic56YqT8quGFydw8/NpR6NtXdO5s0JcfIUWMRSak3UFKi0+U6tsPghkUupGgCGgYgKVnEgSORIheE+Qtr1teR+nPAp7zsaKrp5AUxAgLU5fnkH5mkOcgGwIvE7oZVd8OeagKkGhRQyF5qTUKzAM5IpXd4ntrYnhMxg/Nc7EZycKPlahSFwTN5+wOovEqcly2CnDnYvVTu3cfEtdfHv37s1G6i0lU8RwKQcPHlwmXELz0lBRfJ52D5v/1WaufOIKicsJ/NvzXyFvtBokXk7Qere4+YTlJKfSbr0mFKcsGpykU5NTr5fAda0ihrnFCtcqYig0HyqfiVOl1Gmt9bIZSaXUUeB02g14AOjXWh9bss8h4BDApk2b9j755JMldViPa/hJ0D+o0e8r7GlQWxqzrfSicGsRi8UIhUIVa7+ZqdXYadstiaGUati41nl7nqC5emLYrq1d7O7fDSA10pZg2zamada6Gw1JMWP30ksvMT09veizN7/5zee01oNL9y3KglJKhbXWUeAEkGm0HxhZuq/WehgYBhgcHNT79u0r5pRZkhNJnv/B54l+Icrdh+4uaE7JnrYJbA4QurdyN8EzZ85Q6u+4UanF2CWn0m49f2NbTqOxUQZCq9ctuqquEjSD7kNawMTwNu7vWm5mZ2dpa8t/Tlu4RTFjFwgEeOCBB/Lad91vadoyGkz/n+EZgEzEXjrSL1qtCL7ud3ajbc3EXxY2F2UEDRKXEjUPtxXqg1Q05YqTt7HFqSAUOInauPkEoVDyieI7BZxa8tnenNfDFejXmvi3+Ol4YwcT/3OCvp/ow2zNz8RUXoUz77jRfJ0SzbeRSUVTTH9pemOJE26whE6lQ87F1SfUOQ17Zfa9pw9nzmHy85MFH5u8lqxAj4RGwYpazHxpBmVuLHECQLk/tQqWEIRCaKgovlyCdwYJPRBi/NQ4Pe/qwfDld6NRQUX8Upzg3cG6LJEgVBYrahH9UtQVp5YNJk5pfuv3Kuc9kGobQjlp6Cu07z19WJMWN0du5n2M4TVw4g7WtOTm22hY0xbTX57e0OIkCI1EQ1+locEQLXe0MP7keEELEJVS4ubbYFjTFtNfmkYbWsSpBmS8FeUsrXHkyJGK9HV4eHjZ+YXa0LAuPnC/9L3v7uXSRy8x85UZOh7uyO+4oFsjJ3iXuPk2ArniZLbIepdcypHpqNA2ylFa49ixYxw8eLCkfq/GoUOHeOSRRzh58mRF2hfyp+EfJcNvCuPb4mPsM2N5h49n3Hz2dIPVIBAKxpoRcap3iimtcfr0aQYG3HVfIyMjDA8Pc+rUomDjkujq6pJcf3VAwwuUMhW9j/Yy/9155r41l/9xSpG4Lrn5mpmsOCkRp3qi1NIa0WiUcDicbev48eNZ66tc7N27V9x8dUDDCxRA11u78IQ9jH1mLO9jMm4+WbTbnGTFCY0ZFHGqJzIuvnPnznHz5k0ikcgya+XgwYOcPHmSkZGRZeIViUSyLr9Tp07R39/PyMhI1qLKCGAuR44cYXh4eFEb+/fvZ+/evdx///3L5rP6+/t59tlny/Y7C8XR0HNQGQy/Qc87e7j+366zEFmgpb9l/WO8BlbUwp6x8XQ0xTAIaUSc8qceqm0UWlqjv7+fqSm3iELG+sqIU77s37+f06dP09/fz+zsLO9617s4depU1pUYiUR48MEHy/DbCaXQFBYUQPePd2MEDMafHC/oOHHzNRfWrIhTo1FoaY1wOJyN+nv00Uc5e/Yso6OjWZfgepw6dYqhoaFF7Z48eXKRpXbu3LmCRU8oP01jOnjaPXS9rYuJz06w+ac349vsW/cYI+iW4Ai+SqL5mgFr1mL6H0Sc6pGMKz0cDpeltMb+/fsZHR1lYGBgxfmn0dHRRW6+SCTC0aNHs6937969aP/MnFaGqakpqTlVBzSNQAH0PtLLxOcmGD81zrb3bVt3f8Mnbr5mQSyn/PnVX1iySF27JUc87Z6Gyc93+PBhjhw5sqqVMzAwsCiUPXeOqb+/f5nFNjo6ytmzZzl06BDDw8M8/vjjlem4UBBN4+ID8PX56HxLJ1N/NZV/pggFyeuyaLeRsWJpcXJEnIqiQfPzZSyiQjlw4AAjIyOLovSOHDmSDbw4dOiQuPfqhKYSKIC+d/fhxB0mns6vFIfRYkg0XwOzSJzyzGovLEcZCifhbJjr4OTJkxw5ciQbxTcwMLBsrZVQe5rOrxXYFaD9oXYmPjdB38G+dbNVK6/CnraxZ2087U03HE2NHbNdcbJFnEpG4br6Uhrlaww332oszVQBy62tgYGBrJtPChbWL01nQQH0vqcXe9pm6m+m1t1XKSVuvgbEjtlEvxQVcSqA9awjZSic+MaxooTqU+h3qykFqvXVrQTvDTL+1DjaXn9AxM3XWNhzIk6F4rW9TE1Prf0dV26wRD7XjCAUitaayclJAoFA3sc0pU9LKUXfu/u4+MGLRP8+SudQ59r7exXOtIMds/G0NeWQNA0ZccICMyTilC9dc11M3ZhiYmLtuVntaJRH5V1frRmIx+MF3TSFWxQ6doFAgO3bt+e9f9Pejdvf0I7/dj9jT44Rfkt4zXVOSik0muT1pAhUHZMVpxQYoY1zAy0HpjbpjfWuu592NM6sQ9cPdW2YasNnzpzhgQceqHU3GpJKj13TfgOVoeg72Ef8fJzZZ2fX3V/cfPWNiFN1UIZCa038crzWXRGE5hUogPBQGG+PN6/0R8qnsGM2dkxKcNQb9rwbrSfiVB2MVoOFlxYKKgIqCJWgqa92w2vQ80gPsX+MMf/C/Jr7ZlyAyRsSzVdPZMTJSTkiTlUiUy8tOSbXglBbVDVdWoODgzq3EFkxJCeS/O3Jv2VBLeS1v4oren6lh+RdSaYPTa+5r7Y1Sil8W3ySm68OcCyH1I2UO3Hvlb/HWvi6fSQnyycoOqUx/Aa+vvVzWgobF9M0efvb315yO0qpc1rrwaWfN1xEwC/+6i/yzBeewWPm33XVolD/qHCOObDO9aaddHG7Iu+Htm1jmhJdVgyLxk6DE3dA0+R2fnnQpkbZ5RXxUq+FRkGu2eLZsWNHWQRqNTbEpa87XStRTeV3pck6kBoj4lQXKBTakmtBqB3rmiFKqQNAFBjQWh9bY3u/1np46fZy8zu/+Tt89i8+S0d7R0HHef/Ui/klk/i/iUN49f20pVGmIrAzUJSbLxaLEQqFCj5OcMcu6A+SuJRw3a0NnnKnGqhJhfE9g2R/EnNTma0Ax3X1tdzRgjKa928h12zxZOpyVYo1BUopNQCgtR5RSvUrpQa01qNLtke01qNKqaGl2ytF0BtkJj5T0DHmm026v9iN8zcOsXfGVt9Rg57XLEwsYHiLe3yv9B+tadEwfsnN/qG8CiTSeTEazAkT74tefC/68L3ow5x0RckT8jDx6xPotvJaPDqhWbi20PQZO+SaLY5Ku0bXs6AOApnCKRFgCFgqQEeB/bgWVH4lLUvkoW0PFZ7Y9S64+AMXMb9s8rqff92aWQisqEXo/lBepeOXcubMGfbt21fwcfWE1toVakeDg/sknX6ttV78Pvf/nGO0rd3X1q30OSu+t9MuVQfOxc7x8I6HMdua+2aYL1prEq8kmPvmHLFvxZh7bo7URAoAs8MkdH+I1vtb8W32ceHDF7hz5E5ue/y2svbBiTsYPoPwm9de7N7INMM1WyvOnDlT0fbXu8uHgdyMq925G9OWU0QpdR44wgoopQ4BhwA2bdpU8i+kLY2tbVSsiIvlx8H8osm3Pvst9LvWeNI0gH8C81LhN8pYLFbxP1qxaEvjJNJzO7Ds/2xEZ+7QFDLMmmxW7DWPV5n/1KLPFljgOfUcrGHgNjUOcAnUdxR8B9Q/KVTUHSPdqdGv1vBq0Pdq7B02U8YUU+nL0/pxi5t/eZOJN03A/WXskwYd15hfMBummGGh1PM1W+9UeuxKiuJTSoVx55+OA59SSo1qrSO5+6TnpYbBDTMv9UklOZFk+ivTeEJFdP21cH7gPPH/Fefud9+9ar4xrTX2tE3Xg10Fuzbq9WksOZVk+kvTKFOhPCpbpA51K6N79gdq8rQ8GhtlILRxCsVpW7NwfoG55+aIPRdj7ttz2DPuQnFvn5fQgyFaX9NK6DUhfNvWXvow+u5RfF/xof5Q8ao/elVZc+lZsxaBzgBtA81ZkqJer9lGoNJjt95dPgp0pV+Hgckl2w8BT2ito0qpUeAAsCyQop7oe08fkV+KcPP0Tbp/tHvFfbKLdseStOwq3M1Xb1izFrNfncXwGRsmv1o9oi3N/PfmbwnSd+Zw5twqtr6tPtrf0E7oNSFCrw3h21zg+qMAbPvANi4cucDYZ8bY/N7NZeu32WqSeCVB6z2t8v0Rqsp6AnUCyCye6gdGwLWctNbR3B0zgRRl72GZCe0N0XJHC+Mnxul6a9eq0UlGwM3N1+gCZc/bTH9lGq00ZkDmdqqJk3SYf2GeuW+5gjT/nXk3fB7w3+an8wc7ab2/ldbXtOLrLX1BbPv3tdOxr4OxT4/R+ZZO/Nv9JbcJi/PzBfcEy9KmIOTDmgKVnmMaVEoNAdGcCL1ngL1a62NKqcNKqQjQVY0w81JRStH77l4uffQSM1+eoeONK4erK7/CilrY8zZmsDFv7E7CYfqr0+iklsCDKuDEHea+O+daSN+KMf/deXTSnZAL9AfofGtnNrDB2+WtSB+2vW8bs8/Ocvljl+n/7f6yuWqNoJufr6W/uUPOhfpi3YmclURHa70353Vdu/RWIvymMNf/6Dpjnxmj/eH2FS/i3Nx8jWhFOSmH6a9N48w5mO0iTpXAnreZ+85c1mW38M8L7sJWA1p2t9D9jm5CrwnRel8rno7qJG3xdnvZ8jNbuPKJK0RHonTuX7sWWr4YPgMrapEcS+LfXB7LTBDWo+FSHZUDZSp6H+3lyu9dYe65OUKvXXmRnhEwSFxKNJxAaVsze3YW66aF2SHiVC7smM3ct10xij0XY+F7C27knQHBO4P0HOhxBenVrTUtptj9jm6m/m6Kq39wlbbXt5WtxpnyKuKRuAiUUDU2pEABdL21ixt/eoOxJ8dWFSjlV1g3G8vNpx3N7DdnSd5IYnaYTbt2pVpoW3Pz9E0mPjfBwosLoN0bdfDuIH0/0UfoNSGC9wbdnHV1gjIV239hOy/+mxe5NnyNHb+4oyztGkGD5FgSO2ZLNWOhKjSmQDmlN2H4DXre1cP1P7nOwvkFWnYvt5IWRfPtrH8rSmvN3D/NkXg5gdkp4lQK2tJM/d0UY58eI3k1SaA/wKb3bnIF6e4ghr++o9mCdwTpeVcPE6cm6PqhLlpf3Vpym0oplFLEX47Tem/p7QnCetT3VbYCnnYPZtDEni+9sGD3j3VjBAzGT6xe0FAFFIlLiZLPVQ3mX5xn4cUFzLCIU7E4KYfJz0/ywv/9Apf/82XMkMnO39jJq/7oVWx+72ZCrw3VvThl2PzTm/H2ern8u5fLlvTVCBksRBZwrDI8JQrCOjTGlZaD4TNof107OqVxUqVdJJ52D11v6+LmMzdJXl+5lk7WzbdQ35V2F15eYP47865bT6KsCsZJOkz8zwle+L9e4PLvXsYT9rDriV3c8Yd30PH9HQ0p+GaLybb3byN+Ic74qfWrSueDMt0M58lrUsxQqDwNJ1AAng4PbQNtOLNOyWWpex/pBQXjJ1e+gJVy14DUc6XdxNUEsdEYZnvzpqOpFE7SYfyz47zwky9w5eNX8PZ62XV0F3v+6x7aX79yhGcj0fFwB+1vaOfGn95Y9SGsUIwWg4UXF6hmsVNhY9KQAgXg3+6nZU8L9oxd0oXi6/PROdTJ1F9PYU1bK+5j+I26dfMlx5PMPjuLETLcFEZCXjhxh/FT4zz/E89z9ZNX8W3x0f/b/ez55B7av6/xhSmXbe/fBsCVT1wpi6gov8KasbCiK18vglAuGlaglFK03tuKp8uDEyvN1dd3sA8n7jDx9MTK5wooUlOpunPzWVGLma/NoAKq6NIgGw17wWbsqTFXmH7/Kv4dfnZ/bDe7f283bXvbmkqYMvg2+dj0rzYx89UZpv9huuT2lFJgQvyC1EMRKktD39WUqWh/sB1lqmwKmWII7ArQ/lA7E5+dWFGEcqP56gU75qYwUh7VMJP2tcResBn7zBjPv+d5rv3BNQL9AXZ/fDd7PraH0GtDTSlMufT+y14CuwNc/eTVsgQYZfLzOQkJlhAqR8Pf2cwWk7YH23AWnJJKtfe+pxd7xmbqb6ZW3K789RPNZy+k8+tpjdHS8H/CimLP2dz4Hzd4/t3Pc234GsFXBdnzyT3s/u3dhF6zcaqoZtZGpSZTXP+T66W3l8nP94pYUULlaMh1UCMjEFtUM8iHNduJdclyy4QX8zDsDdLZP8Yrn57kH+/cAeaSG792y1/7xhyMpduW8PTTRZw/T7StSY5ZaCvoVp1tInzdBn93uTzJSNW8RfDMdYLPXMWYt0m8OkzsR7ZzY1cbLwG8UJbT1A15jZ0RpO2Ns4x/9gb//KotWLeVJtDa1nAF/N/WDW+BVvKabWYqXFC3MQUqFoNwePFnusMkadpYs1bRJQH027ZgfuJ7dH5rAvuhnmXbHeXg9zh4OlZvPxaDUIUezLXjVlj1KRujzWBxZcDGJ2VAe6BEl9GchefvruM5fR21YGO/Nkz8HdvQu0K4t+/mdEnlPXYHt8NzU3R+JkLig/dCiUsSnLiD32sXV5+tTqjkNdvsRKOVbb9xv1VLUErh2+zDSTjopHYtqQJxXhPG2dqC52+uYb++G5Y8FSpTYU1bVUv8mYvWmuTVJPaCLXNOKxFL4fk/1/GMXEfFHeyBTlLv2Ia+XTIeLCLoIfWe2/H94UuYX7iBPVRa3ShluOsEG1mghPqlqb5VylT4t/mJX4yjbV34miBDYb11C74/jmB8ZxrnvvDi9j0Ke95GW7qqId1aa5LXk7esw8b2ppSXmRSe/3MNzxduQMLBHuzCevs29A6pW7Qa9vd1Yf9DB96/fAV7bxd0Fl+LSnkV9pyNk3TKWsVXEKAJgiSWYvgNfFt9bh2eIjxg9uu70Z1ePH99dfnGtDDYc9ULN9dakxpPYUVFnBYxncJz4hKBX/qma/G+ppPER+8j9XN3iDith1Kkfmon2BrfX7xcYlugULImSqgITWVBZfC0eXC6HVKTqcJv6h4D619swXviEioSQ/cvdk5X281n3bSK+z2alWgS799cwzwzBikH+/XdrsW0pf6T+dYTui+A9fZteD97GeO5KM5rwkW3pXyuQHl7vJJmSygrTWdBZfD2ejFbzWxF00Kw3tSHDpp4V7Cict18lcaatkjeSLpzThv9ur+ZxPvpiwR+6ZuYI9exH+wi8Vv3kzq0R8SpSKy3bsHZGsD7Py5CogSvgOEG8Niz9bWQXWh8mlaglFL4tvrAdMPDC6LFxPrBTRijN1HXFpY07P5XaTefFbNIXE244tS0f6X1UZMJvH9+gcDhb2L+/Rj2Qz0knngNqX+9G71ZhKkkPAapn9qFMZHA87+ulNSU8rjZViQ/n1BOVDW/UIODg/rs2bMltfGBD7jroDx5eti0o3HiDqrQm7ylMc7H0B1e9ObAkkYBY+UMDrYdxTTDBZ5sSfOORscdVww3kOWkzRmU3e6+SWnUZAI1nXK3dXjR3T6QlE4rsmjsCkRdi6NmUjg7W6GECFHtuFWoG83NV45rdqOyY8dlPv/57SW3o5Q6p7UeXPp5U85B5aIMheFTOEldmEh5FDrsRUVT6B4/5EbtqfQiRa2XhaKXjKPRiY0nTllSDmoyeUuYwl50lx+abFFyPaH7/KiYhboeR99eWoCJtopb4lEbNNrGvY7RbMwLrr5pOIH6+MfdVd9LF+quhdaK5PUU9rSNCuT/JVRjSfy//BzWq7ZgPXrbom1O3MG/1Y+nffEQxmLfJBTal3/ncttMOsRfdlPHNFuWiFXRGmYsjGsLqG98F+8XFRgK+y19pH5kC3QZQKrWvax7Um3fxDv7+qKPN780je+PIyQf2oX9pr7iGtGgE5qWPS11m1lfa42z4GDNWG4lBEdjdf8jLephfD3Fh9tvVKLRl4DSLajVaDiBKgalFL5NPuLxeEGLeHVfAHuwC8+ZMay3bYXgreHKRvO1l2cIteVmiUDTQE+gBZByUGNx1LU4xvWFRf+rdIJe7QN7aDOpH95S0tocoXDs7+/B/tI43pOXsB/ohHZv4Y0o0GisGQtvVxHHVwitXTe/PWNjTVtuDTkFhtd1RypDkRpLYfgNPG0b4pbYMGyYv4YyilvEa/3IVjzPTrki9SNbb7XnUThzTnELgpegbU38ctx1j/gbWJy0hpkUxrU46toCxvU46rr7Wk0kUDnTnbrTi7O5BeehbpzNLejNAZKvfgGPur12/d/IKEXqvbvwf/DbeJ98mdShPcU143WDJTydnprm59NaoxMaazZdtyod06R8asU5MuVTJK8mMW43ik6VJpSfdQVKKXUAiAIDWutjK2wfAPoBtNanyt3BcmL43EW8iVcSrhDk8T3UO1ux72nHc/o61v7Ntybp00+L9pxdkhWlHU3iSgIn7jTOhZFyXOG5fkuEjGsL7mc55Uq0z0BvCuDsbEU/1IPeHMDZ0oLeFICW5VkmdRswW8XfQ1iE3tKC9SNb8H7+KvbDvTj3dBTcRqb0jT1X/fx8Wmt0UmPHbKyohZNyUCjwru8yV6Zyr8XLCQI7A3XrotxorPkNSosPWusRpVS/UmpAaz26ZLfHtNaPKaUOr7K9rvCEPDi9Dqnx/Be/Wj+yFf9vv4D5lYlF/nlllObm01qTvJbEnq/D/HpaQzTluuGux10hyojQEmvI6fShtwRwHurB2RJAbw6gt7SgO30lJyMVqov1tm2YX5/E+2cXSXz0vqKiJqudn89JOrdEKekmzFVeVfADn/IqdEKTuJrAv93fcNGIzch636CDwOn06wgwBGQFKG1dnQdYybqqV7zdXpwF9ykvny+xc087zm1BPH97DfuNvdmbrvIW7+bTWpO6kcKaqXEKo6SDupG2hha55hZQOUUgtc9wLaD+EPoNS6yhQIVz7gvVw+eujfL/9gt4/uoq1o8XPgFejfx8TiotStNWtlhptnhnCdeS8rl9T42n8PZ5G76MSKOznkCFgdwKft1Ltj8IWUtraBUX4CHgEMCmTZs4c+ZMsX1dxOJ6UIWjw6BbNDb5hYrrfwnBjyn081/HygmW0o7Gjhmo9D3atmPEYmfWbku7QRHa1Kg+RVXW3ztgfg+Ml8G4AuZVMK6CGgelb/3+To/G2QrO3WBvxX29DXSXDcYcMFexLmozRqrtaxVrv5kp69i9HoyHwfNXl0n84GWcbUW04WhicwojWb4bvNaAk752HO3qUAhoK3EOeOnYtYPlaBLTSlx96xIr2z19Jcphg09qrUeVUkNKqQNL56G01sPAMLgLdfft21fyCZ9+ujz1Wxyvw8LFBQyvAesZAfdqnN7nCHzWQ+Kee7OippMaI2gQ2O4u5o3FzqwbZp66mSJ5PVkdyynlYH5lAs/fXcO46oawa396bmhXC/oh1x3nbAm41pB/8UAY6Z8K6tKtrrZ9raRQ6Y1MucfOOpDEc+5btPxhK8lfuqvw9X5pIWnZ01KSq0zb7jyvNW1ls7coMy0cZbp2Vhw7x3UdBm4LYAbFQ7Aa0egZynFPX431BCoKdKVfh4HJJdvPc8vCiuBaVHUdKJGLETDwb/G7KYXWEwtTYf3wFnx/fhHjn2dx7nJX7Rfq5rNmLFecKp1fbyaF5ws38HzhBmrWwrktSPJf92Pf3e6GcIvrQliLDh+pR3bg+7OLmF+dwH5Db2HH5+TnKzSxsnZuiZITc9C411ZVc1IarsswccUNmjAkg0lNWO+bcwLIpJ/oB0YAlFJhrXU0/f5AzvZnK9DHZYRC5avkqLWJ5fdhz9jrl4sf7KPn6Svw+WvM7AzfaiOpWbjhZJ+0VuubvWCTGrfAY6LK6PrIxbw2T/AL12j52jjK0iTu62TuLVtIvar9liglKnLqkvG1wkJcbgTFUJGxe91mOv9hAs9nLhG9swvdWtjaJm0rYldtfI657lyOdjROwp0XdubTc58KlMd0r0kbKuULX2vsdEozez6Fr88nQRMrUNOS72nX3aBSagiI5kToPQPs1VpHlFLRdLBE1cLMh4bK2ZpC2x6iX4phz9iYbWuP+I1Hu7n+x9d5o3eSlt1uslJ7zsbb46XjdR2cOQMrWbzJqSTTX5rG6DbKPnGstSY2GmP85DizX59F+RRdb+2i50APgdsyeQQX1myjHhiNOQzcNV/rbjQklRq7hV/dyvcOfY+7/j7Cjv+wo6BjtdbY0zbhN4Xxdi4XN+1oUlMpEpcTJF5JoA2NalMYwerm81tv7OyojW+Hj7aBNgmaWEIFp5+APOag0nNISz/bu9b2RkOZivYH24meieIknDVDvrvf0c3Yp8cYe3KM23/VXVRqtBikrqdwUs6Kx1izFrNfncXwlVecnJRD9O+jjJ8cJ/5SHE+nh03/zyZ6fqwHT3jDrMEWKkjL7hZ6H+ll/MQ4nT/USei+/Cd/lXLXGsYvxLMCpR2NFbVIXEkQvxR3Kw2YYAbNkhe8VwqjwyDxcgJPu4fgHVIMs5rIXSyNGTRpe7CN6S9Nozxq1YvF0+6h621dTHx2gi0/swXfZtf0z1S+XYo9bzP95Wm00phlCse2Ziwm//ckE5+bwJqw8N/uZ/t/2E7n/k4puy2UnU3v3UT0TJQrv3uFO4bvKGg+xgyZJF5J4N/hJzWWIn4pjpN0UCptKbXWpyjlopTC7DCZ+84cZruJf5O/1l3aMMjdLAdfr4/WV7diT9tr1rXpfaQXFIw/NZ79THndCdVcnITD9Fen0SldlkigxJUEl3/vMs8ffJ7rn7pOYGeAXUd3ced/u5PuH+0WcRIqgtlisu3924hfjDN+cnz9A3JQhkIrzfRXpll4aQHlUXg6PJjtZkOFcCtTYbQazH5jFmtWyttXC7GgltCypwXrpkXiWmLV6CNfn4/OoU4m/3qSTe/dhKfDg9FikLyWhDZ3HyflMP21aZw5B7O9eHHSWjP3nTkmTk641p2pCA+F6T3Qm50DE4RK0/GGDjre2MGNP7tB+M1h/FvytyLKlVC51hg+AztlM/P1GcI/EJYHwiogI7wEpRShB0KYreaaVXP73t2HTmgmPjfhHme4uby0pdG2ZvbsLNZNC6OtuCHWtubmF27y0s+9xPn3nyf2XIy+n+zj7ifv5rYjt4k4CVVn689vRRmKK793ZcNWzs3cF2ZHZ92s6EJFEYFaAcNr0P66drBZNfAhsDNA+xvamfjcBHY6QaryKDcs9ZuzJK8nMTvWD69dij1nM/7UOM//5PNc+uglrJjFtg9s4+4n72bLz2zB210/ZQyEjYWv18fmn97M7Ndnmf7idK27UzPMdpPktSTzL0jEaaURgVoFT5uH0GAIZ9ZZ9Ump79192DM2U3/trlU2ggZO0iHxcgIzXJg4JW8kufpfr/LdR7/L1T+4im+Tj50f3cldf3oXPT/Wg7lC9m9BqDY97+yh5Y4WrvyXK9ixqiTpqjsyQRPzL8wTvxyvdXeamuZwDleIwNYA1p0WC99bWFFwWu9rJfjqIOMnx+n5sR43+s9Q7gRwnuI0/8I840+NE/1iFIDwvjC9j/YSvFPCWYX6Q5mK7b+wnRd/7kWu/ck1tr+/ctVU6xllKIw2g9lzs5ghE29YPBuVQARqHVrvbsW6aWFNWSsGO/S9u4+Lv3aR6N9H6dzf6aZIWWeRobY1M1+dYfypcea+PYfRatD7SC897+rB1yeVZIX6JnhXkO4f72by6Um6/kUXwbs25sOU4TXQPs3M12YI7wuXbRmJcAtx8a2DMhRtg21uzr348vmo9ofa8d/uZ+zJsXUnju0Fm4nPTfDCe1/g4gcvkhxLsvXfbuWeE/ew9d9sFXESGoYtP70FT5eHy797GW1v3GABs8VEJzWzz85u6HGoFCJQeWAGTNq+rw0n7qCtxV9CZSj63t1HPBJn9hsrl4NNTaS49qlrPH/wea584gqedg+3f+h27v703fQe6MVslScvobEwQybb3reNhRcXspGsGxWjzSA1mWLu23MbNrqxUoiLL0983T5C94eY/eYsnk7Pojmm8FvCXP+T64x9Zgx+49YxCy8tMH5ynOgXomhb0/Fwhzu/dG9QcnoJDU/Hmzpoe10b1//kOh1v6sDXuzE9AJmgiYXIAmaHScsuWQJSLkSgCiCwK0DqZorEK4lFue4Mr0HvgV6u/sFVeAFmbHd+KTYawwgYdL+9m55/2YN/m6RIEZoHpRTb/t02/vlf/TNXP3mVnf9xZ627VDOUcoOjYs/FMEPmhhXrciMuvgJQShF6TQhPuwcntng+quttXZghE+NXDC788gXil+JsObSFu5+6m23v3ybiJDQl/i1+Nv3UJqb/YZrpr2zctVGQLjnf4qZDWmuRv5A/IlAFYngM2r+vHY3GSd4SKTNosvmnN8MeuO1XbuPuv7ibvvf04WkTI1Vobvoe7SOwM8CVT1zJLlrfqBh+A+1oZr4xs+oifyF/RKCKwAyZtA22udU+cyJ3et7Zg3PMcbOKSwVOoQJordEpjRN3slVnraiVTbFVC5THXRuVupHixp/eqEkf6gmzzcSasYh9MyZBEyUij/dF4t/sJ3hPkPnvzmN2Fp7SSBBy0VqD466R07YGy32NAoVCo0G7UaMqoDBbTcygiREyMAIGxgsG9oyNETTWrGdWKVrva6XrR7sYPzlO51AnLXs2dqCA2W6SuJzA7DBpfVVrrbvTsIhAlUDwVUGsmxapsRRmh4SKCyujtQabrJWjbVeMlEoLD7ji41WYLSZGu+GKT6vhvvcZKL/C8Bsor1rxYci4YND+QLs7/2HZNVm6sOXQFma+PMPlj11mzyf3bOgS6Zmgifl/msfT5iko+7twCxGoElCGom2gjeiZKPa8XZaaTxuVjAXhLDigcH8g+1q5/yz+PP1/raxXrV2rJutey1g9Of1T6RdZq6clbfW0GJgBE+VzhcfwGyVXlPX3+TH3mcx8bQZr2ioo5VY58LR72PKzW3jliVeY/N+T9Lyjp2rnrkeyNaTOzmK+yWyasiPVREasRAy/m/n85hdvuuWrhbzInUtRTlp8TFxL1Ma98TtpAbBz/ndwXWE653XaCsm6wnLcYhmBWPRaaZS+9fmiY90Pczqa/l+RdbFl2tJaozxpq6fNwAgamCHTFR6/wvClhce3stVTCTwhD+EfCDM7OkvyWjqjfhUtmc79ndz825tcG75Gx8MdeLvqL0edtjXWlEVyPOlWwb4NyL+SfUEYPgPbyqkhVQP3ayMjAlUGPGEPbQ+0MXt2FurveqwbtKNxFpyskJtBk5bdLfj6fHg6PZhfMgm/IVx4u2lLJitcmkUCt+x1RtxWer20nfQ2x3HcBKEBI2vxGD6jLqvCGj430nTu+Tk30XHIRHmr00+lFNs+sI3v/evvcfX3r3L7B2+vynkzaEdjRS1S4ylSYymSY8nlrydS7t81jRE2SPx+Av/WyrjhzKCJPW0ze26W9te3b2jXZ6GIQJUJ/w4/1k0LfUG7T9YSNOFaSUmNjqfHxFR4+7z4t/rxdnsxWoyyjJNSaQvMuOVS2+goQ9F6Tyuedg+x0ZjrSmypztN74LYAfT/Rx40/vUHXD3fR9mBbWdrVWmPP2KTG02Izllr2OjWRWubJUF73e+fr8xF6bch93evD2+dFeRXnP3KeyOEIez65B29nZZ4wjXaD5I0kc9+do/XeVrk/5IkIVJlQStH66lbUZYU9Y99yAZnKdfFU0c1TS7TlWknYrvvM0+HBv8uPt8eLJ+yRp8cqopQisCOAGXLnpexZGyNUnoeC9ej7iT5uPnOTyx+/zJ1/cmderi07Zrtut7G0xZPzOjXhCpFOLBEfj8Lb48Xb5yV4T/CW+PR6s6K0XuFQ59cdUr+W4sIvX2D3x3ZXZC45mw7pewt4OjwEdgTKfo5mZF2BUkodAKLAgNb62Br7HV5r+0ZAme5EeM8be7BjNtaM5Ub5TaawZix3Jw2YrhumGURLa3ceKXPjMHwG/u1+fJt9eLu84nOvA7ydXsL7wsx8YwbrplVUpedCMXwG2//9diK/GOHGp2/Q956+5a62Je+d+SULWw3wdrtCE+gP0P76drx9Xry9rvB4+7xuXsxSH3rugp0f2smFX7vAxQ9dZNdv7arIOkZlKMw2k9iomw6pUtZaM7GmQCmlBgC01iNKqX6l1IDWenSF/YaAByvUx4ZDmQpPhwdPhwd2uJ9pW7uiNevWlkpNpbCmc0TLoOoT6sWgtb5lJaVDpT3dHvx3uW47s03WhNUjZotJ+PvDzD4361Z87jBLjhpcj7aBNjr3dzL252OM/fnYsu2eTo8rPrcFaNvbtkx8vN3eivcxQ/tD7ez4pR28cvQVXjn6Crf9ym0VsfaV1/WmZGtISaXsNVnPgjoInE6/jgBDwDKBEtZnkWili5BqW7vZANKWljVpkZpOpTdyS7S8qqausTWDG8IeyZrRICiPuyzC0+HJFso0fJX92239t1vxdHsw28zsvI+314u3x1vxcxdK1w93kZpKcf1T1/F0etj6c1sr8rBltBjYszYz35gh/P3hugy0qRfWE6gwMJXzvnvpDmmrakQp9Vg5O7YRUKbC0+5x10dkRMtxLS171iZ1M4U1YZGaSWUj0DBy3IMVEq01gxu6vBjB6sxjCOVHKUVwTxCzzXQX9aYqu6jX0+Fh62NbK9Z+uel7Tx/WpMXEqQm83V763t1XkfMYIQPrpkXs2zFCrw3J9bQK5QiS6CpDG0IaZdwSrUwGdO24lpY9a7uuwUkLa9pyQ6fTi1dLFa0Vgxt2+vH2evF0eKrmahGqg39TbRf11itKKbb+261YNy2uHb+Gp8tD178o/y0uEzQRvxDH0+6hZffGTg21GusJVJRbAhQGJnM3ZqyntRpQSh0CDgFs2rSJM2fOFNPPhiEWi1Xnd2xnUe42belbEU6aWxkWcrMv5JK7Poh0jjevm+dNedLHXUv/VImqjV0TUvTYecD22Ohp11LeiFH68/Y8o7ElMxc/D8aUwaVjl7jovwh7K3RyL+hva8wLZkO6+ip9za4nUCeAwfTrfmAEQCkV1lpHgX6lVH9m+0pBFFrrYWAYYHBwUO/bt69MXa9Pzpw5Q61+x1xLKxs9GLVuLTzNuPy1u17I0+1x3XY99RHcUMuxa3RKGTvtaOa+m17U296YN8pSGI2NMhAaWPa5/Zs25//9eRL/KcHu391N8O5gRc7vJBx0XNO5rxMz1FhBE5W+ZtcUKK31qFJqMB2lF80Rn2eAvVrrU5C1ksIV66WQF8pQeNo8bnLKrTnuwflboqW1luAGYRHKULTem17U+48xN01TQL4bZqvJrv+0i5fe9xKRxyPc8ck78O8of7YJw2/gpBxmvj5Dxw90yHWZw7ojobUe1lqPpC2hzGd7V9hn90oh6EJtUYbCE3KzKbfe00ro3hC+Xp9cBMIilFIEbgvQ/nA7OGDPbuzCgxm8XV76/3M/SikihyOkJlMVOY8RMrBiFrHRWNbtLkjBQkEQcvB1+9z1OW0mdtSWgnuAf5ufXf9pF1bUInIkgh2rjHib7SaJqwnmvzdfkfYbEREoQRAWYQZNOh7uwLfdh33Trlml3noieGeQnf9xJ/GLcS588AJOsvzl3DORffPfnSd+NV729hsREShBEJZheAza9rbRel8r9oxdkRtyo9H2YBu3HbmNuW/Ocem3LlVEuJWhMNoMYmdjtzLNbGBEoARBWBGlFME7grQ/1I5OuME2G53O/Z1s/dmtTH9xmiv/5UpFXKCG1wAPzHxtBiexsR8MRKAEQVgT/2Y/4TeFMTwG9ozMS/U+2kvvwV4mn55k7NPLcwyWAzNo4iQcZp6dwUk6G3bMpdyGIAjr4mn3EH5TmJmzM6TGUlWv1FtvbDm0BWvK4vofu3n7un90WRa4kjHaDFKTKSb/etItHx80MIMmZquJGTLd4pk5BTSbcf2aCJQgCHlh+A06Hupg7jtzLLy0MRf1ZlCGYsfhHVhRi8u/exlPp4eON3SU9xzKTTAN7npGbWlS0XRRRksvKtCptUZ5FWbQdIUsZLqvW3IqQPuNhktZJgIlCELeKEPRel8rZodJ7B9j2af4jYjyKG7/yO2c/4XzvPyRl9n9O7tpfXVrZc5luGU6VkNrDQ44KQd70iZ5I5kth6O5lQLN8LlWmNHqipgRNDADpiteAeXm9Kwjy1hV07c5ODioz549W3I7I+dHiCVjZehRBXgZuL3WnWhQZOyKpwZjZ8dtd+GqduscNSq+mI9kKFn08Sqq6Pr5Loxpg6lPTGHvrNNgkkz+zbSYsdqt3wTDdF2GmR9Mt/qCMtKv05abecnk7T/09pK7ppQ6p7UeXPp5Q1pQsWSMcCBc626sSMyIEQqEat2NhkTGrnhqMnYBcFodElcSOHHHrZ7cgDqVIkU77cU3EAbrmIX/fX66DneR+P0E9Jate+UjN3H0ain/MiJmaUiyKKuFyvnjKq8rXAv2QqV6C0gUnyAIJWB4DQK3BfC0eXDibpXljYjeokkcTaDmFP7DfpitdY+KROHObXlcl2I2ECPgugBVQKH8rlA5Cbc8TyURgRIEoSSUofBt9eHt9bqZuTdo5gm9R5P8jSTqisL3qz5I1LpHFULhWmBVSLwuAiUIQskoFL5uH/7tfrBApzamSDkPOCR/JYnxHQPfR30VtzCaHREoQRDKhifkIXB7wI0eS+jVJ+KbGGefQ+rnU5hfNvF+zLshx6BciEAJglBWDL9BYGcAI2C481Ib8AZtv9Mm9ZMpPH/lwfPfGzIWrS6QkRMEoewoU+Hf4Sc1liJ1M+VOuhvKfSRuwEi/YrB+xkJNKbx/5kV3a+x3iL+vUESgBEGoCEopvJu8GAEDa9rCSTnohF4UrqzRoMiKlzJU8wiYgtQvplBRhffjXnRY4/zABg1zLBIRKEEQKobCTdeTTdmDdoMobDd1j5Ny0CmNTua8Tq+9yabxQS8XsEYRMROSv57E9x98+H7DR/JYEue1IlL5IgIlCELVUCjwuOts8IO5JFZZ42Y50JYrYNrW6JR2M3pnRMxavHhUo1Eq7T6sRyssAMnfTOL/d358v+Yj8YkEun8DTswVgQiUIAh1gyJtKfkU+FbeR2udtcCyP8m0iKWtsZVSuNXUCuuA5NEk/vf58R/xk/hkAr1ZRGo9RKAEQWgolErnh1vl7qXRYLNYxJZaYbmLiTN56SosWHqTJnEsgf/9fnyHfSQ+mYDyJkBvOkSgBEFoKhS3kpuuaYWlxWvuwpybSzBQ+VyCepcm+ZvunJT/cT+J30lAS2XP2cjIOihBEDYcSikMr4HZYqK8Cm+nt2prtpz7HZK/nkT9s8L3ER9YlT9noyICJQjChkbhhsN7u6ooUg87pD6Qwvy6ife3JdvEaqzr4lNKHQCiwIDW+tgK2w+lX+7WWh8pb/cEQRAqj0Lh7fMCkJpKVcXdZ7/dJjWVwvvf3YW81v8rptRS1rSglFIDAFrrESCaeZ+zfQgY0VoPA/3p94IgCA1HRqS83dWzpKyfsrDebuH9Cy/mX1YhPXiDsZ6L7yCu9QQQAZYKUH/OZ5H0e0EQhIZEofD2VlGkFKT+XQr7jTbe3/difkFEKpf1BCoMTOW8787dqLUeTltPAANA6fXcBUEQakhWpHqqJFImJH8tiXOfg/cJL8Y5CQ3IUJYw87Tr77TWenSFbYeAQwCbNm3izJkzpZ8w7pa4rkfsuE3sxfrsW70jY1c8MnbFs9rYadwFv9as5WaqqPCcVOrxFK2/3Irvgz7mnpjD2V3nKZE0KEeV556+CusJVBToSr8OA5Or7De0UgAFuFYWMAwwODio9+3bV3Anl/L0808TCoRKbqcSxF6MEbqjPvtW78jYFY+MXfGsNXYaTWoiRWoiheE3Khv3HILkf07i/3k/rR9pJfFfEuit9Rvep21NYj5BOe7pq7HecJ/g1rxSPzACoJQKZ3ZQSh3KiJMESQiC0EwoFN4eb7acPZU2anoheSwJNvgO++Bmhc9X56wpUBmXXVp4ojkuvGdyPj+qlDqvlNrgQykIQjOiUHi7vfh6fVURKX2bJvFEAjWh8P+CH88feTA/b2J8w0C9rGChsuevJ9adg8oJgsj9bG/6/xGgswL9EgRBqBsUCk+3BxQkx5IVd/fpezTJjyTxfsKL5zMelLN4Aky3a/SmnJ8+jbPJQW92XxOmvjK6F4nk4hMEQcgDhcLT5d4yqyFSzuscEp9OgA1qUqFuLPkZU6hXFMZZAxVfImC+5QK26H2PBm/l+l4uRKAEQRDyRKHwdnldS+pG5UUKABNXYPo03LfCdg3M4grW9VviZdwwUDcUxnkDdXOJgCkN3bhW1yoiRmuFf688EIESBEEoEG+na35UTaTWQgHtabffnlWi/pJpAVtqgd1QGC8YqP9PoawlIhZKi9bmxeKVETXaK/+rNZxAfeBvP8BIZASPUZ9dtxdszIisBi8GGbvikbErnlLGTltunSllNMiEjwlsTf8sJeX+qKSCZPp9Mv0+BSSAS+kfAAU7W3byzre/s2Ldrc+7vCAIQgOgPAoDo7FEajW87o8OLrbCdCaVhs1i0Uq6c12VpOEE6uM//HGefv5pwoFwrbuyIrJgsnhk7IpHxq54yjF2qekUyWt14O6rIpmFupVkgwylIAhC5fB2ePFv8VdnMe8GQgRKEAShDHg6PPi3ikiVk4Zz8QmCINQrnnb3lpq4msDwGW5QglA0IlCCIAhlJFeklE+hzAYPnqghIlCCIAhlxtPupkVKXEmADxGpIpE5KEEQhArgafPg3+ZHpzTart+yGfWMCJQgCEKFEJEqDREoQRCECuIJiUgViwiUIAhChRGRKg4RKEEQhCrgCXnwb/dDChGpPBGBEgRBqBKeVg++7T4RqTwRgRIEQagintYcS8oSkVoLEShBEIQqY7aa+Hf40ZYWkVoDEShBEIQaYAZNAjsCaFtEajVEoARBEGqEGTQJbA+ALe6+lRCBEgRBqCFm0HX3iUgtRwRKEAShxpgtIlIrsa5AKaUOKKWGlFKHi9kuCIIgrI+I1HLWFCil1ACA1noEiGbe57tdEARByB8RqcWsZ0EdBKLp1xFgqMDtgiAIQgGYLSb+2/zggE5tbJFaT6DCwFTO++4CtwuCIAgFYgbcEHQ06LjGiTvoZDoc3QE2iG5VvGChUuoQcAhg06ZNnDlzpvRG4xAzYqW3UwHsuE3sxfrsW70jY1c8MnbFU89jp7V2RUprsHGFyVlhPzQKBbl1EStdI1GDclR57umrsJ5ARYGu9OswMFngdrTWw8AwwODgoN63b18x/VzE088/TSgQKrmdShB7MUbojvrsW70jY1c8MnbF04hjpx2dzUKhLY2TctAJjU5pnKSDdpabWMpQYKT/V5QsYNrWJOYTlOOevhrrCdQJYDD9uh8YAVBKhbXW0dW2C4IgCJVDGQrlU+BbeftqAuak0q7CKghYOVhToLTWo0qpQaXUEBDVWo+mNz0D7F1juyAIglAjKi5gVVpBu+4cVNpFt/SzvWttFwRBEOqXtQRM4wZiLBKwpCtcywSswlZWxYMkKkHIFyIaj9a6GyvjUL99q3dk7IpHxq54ZOxWx8AVsRwhywqYrfFcrayENKRADe2u3+VWZ26cYd/d+2rdjYZExq54ZOyKR8aueM5Mnalo+5KLTxAEQahLRKAEQRCEukQEShAEQahLRKAEQRCEukQEShAEQahLRKAEQRCEukQEShAEQahLRKAEQRCEukQEShAEQahLlNbVq3yllBoHXq7aCWtDDzBR6040KDJ2xSNjVzwydsVTrrG7XWvdu/TDqgrURkApdVZrPbj+nsJSZOyKR8aueGTsiqfSYycuPkEQBKEuEYESBEEQ6hIRqPIj9bGKR8aueGTsikfGrngqOnYyByUIgiDUJWJBCTVBKXVAKTWklDq8zn5rbhcEoToopQbW2JbX9VwoIlAlsN4fRSl1KP1ztNp9q2cyX3St9QgQXe2Lr5QaAh6sZt8agTy+dwPpfQ5Uu2/1Th5jl9l+qNp9q2fS1+KnVtmW1/VcDCJQRbLeHyX9Bx3RWg8D/en3gstBIJp+HQFkbPIkz5vBY1rrU7jfu7LdLBqdPK7ZASCS3h6RsbtFekymVtlcsetZBKp41vuj9Od8Fkm/F1zCLP6ydy/dQSk1kL4ohMWs+b1LW03nAbTWx7TWo1XtXX2Tz4004+3ol7HLmzDrXM/FIgJVPGHW+KNorYfT1hPAAHC2Sv1qFrpq3YE6JczaN4MHge60m0/m7xYTZu1rdhTXcjrP6taCUEVEoCpM2k1wWp7GFhHllgCFgcncjWI9lcxk5vsm81D5o5QK4343jwOfUkqJ1yM/oqxxPZeCCFTxRMnvjzKktT5WjQ41ECe45fLsB0Yge4MAd+4kM8kv8yiLibL29+48rvuK9P8SZHKLKGuP3SHgifT1+ggg4r4GOdfritdzORCBKp71brIopQ5lxEmCJG6R83Q/BERzrMtn0ttPpSf5u3BvJMIt1vvejSzZ/mw1O1fnrHvNZsgEUlSrY/VO+mFxcIlFnrleV7ueSz+vLNQtnnQoagR3QnU4/dk5rfXe9B/rJK4vuwt4RNxWQjlY63uXs30qvV2s9xzyGLvD6e1dOXPIQo0QgRIEQRDqEnHxCYIgCHWJCJQgCIJQl4hACYIgCHWJCJQgCIJQl4hACYIgCHWJCJQgCIJQl4hACYIgCHWJCJQgCIJQl4hACYIgCHWJp9YdEISNQjrnWzbVDjAq6a8EYXUk1ZEgVIF06YaTwFu01tH0Z+eBvaST4kpJFkFYjLj4BKE6nMQt5RDN+SxT1XVIxEkQliMCJQgVJm09DaRLiOQSxa3XJNWWBWEFRKAEofIMAKtZSOLaE4RVEIEShMozypLCi2mrKgycq0F/BKEhkCAJQagC6UJ54M47hdP/R4EjwDkpjicIyxGBEgRBEOoScfEJgiAIdYkIlCAIglCXiEAJgiAIdYkIlCAIglCXiEAJgiAIdYkIlCAIglCXiEAJgiAIdYkIlCAIglCXiEAJgiAIdcn/D0+wgQHLztTTAAAAAElFTkSuQmCC\n" 610 | }, 611 | "metadata": { 612 | "tags": [], 613 | "needs_background": "light" 614 | } 615 | } 616 | ] 617 | } 618 | ] 619 | } --------------------------------------------------------------------------------