├── models ├── __init__.py ├── README.md └── bayesian_IRT.py ├── src ├── __init__.py ├── kaggle_submision_to_csv.py ├── utils.py ├── generate_IRT_model_results.py └── IRT_model_plots_analysis.py ├── analysis ├── __init__.py ├── 5-learning.ipynb ├── 3-misc.ipynb ├── 8-action-traces.ipynb ├── 6-incomplete-data.ipynb └── 0-arc-dataset.ipynb ├── .cursorignore ├── figures ├── arc-preprint-figure.png ├── arc-preprint-figure2.png ├── arc-preprint-figure2-background.png ├── bayes_IRT_model_burn2000_N10000_imputed_4_trace_plot.png └── bayes_IRT_model_burn2000_N10000_imputed_4_irt_parameters.png ├── .gitignore ├── .vscode └── launch.json ├── results └── bayes_IRT_model_burn2000_N10000_imputed_4_stats.md ├── polars_cfg.json ├── requirements.txt ├── survey └── readme.md ├── data └── readme.md ├── arc_data ├── ARC_training_tasks_ordered.json └── ARC_evaluation_tasks_ordered.json └── README.md /models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /analysis/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.cursorignore: -------------------------------------------------------------------------------- 1 | # Add directories or file patterns to ignore during indexing (e.g. foo/ or *.csv) 2 | .venv/ -------------------------------------------------------------------------------- /figures/arc-preprint-figure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Le-Gris/h-arc/HEAD/figures/arc-preprint-figure.png -------------------------------------------------------------------------------- /figures/arc-preprint-figure2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Le-Gris/h-arc/HEAD/figures/arc-preprint-figure2.png -------------------------------------------------------------------------------- /figures/arc-preprint-figure2-background.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Le-Gris/h-arc/HEAD/figures/arc-preprint-figure2-background.png -------------------------------------------------------------------------------- /figures/bayes_IRT_model_burn2000_N10000_imputed_4_trace_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Le-Gris/h-arc/HEAD/figures/bayes_IRT_model_burn2000_N10000_imputed_4_trace_plot.png -------------------------------------------------------------------------------- /figures/bayes_IRT_model_burn2000_N10000_imputed_4_irt_parameters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Le-Gris/h-arc/HEAD/figures/bayes_IRT_model_burn2000_N10000_imputed_4_irt_parameters.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # virtual environment 2 | .venv/ 3 | penv 4 | 5 | # cache 6 | __pycache__ 7 | .DS_Store 8 | 9 | # data 10 | data/ 11 | data_/ 12 | survey/ 13 | *.pkl 14 | *tar.gz 15 | 16 | # excluded analyses 17 | analysis/excluded/ 18 | 19 | # some source code 20 | src/make_dataset_csv.py 21 | 22 | figures/ 23 | 24 | models/*.pkl 25 | # ignore 26 | steps.txt 27 | *.out 28 | *.sbatch 29 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python Debugger: Current File with Arguments", 9 | "type": "debugpy", 10 | "request": "launch", 11 | "program": "${file}", 12 | "args": "${command:pickArgs}" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /results/bayes_IRT_model_burn2000_N10000_imputed_4_stats.md: -------------------------------------------------------------------------------- 1 | === IRT Model Statistics: bayes_IRT_model_burn2000_N10000_imputed_4 === 2 | 3 | Feedback Effects 4 | -------------------------------------------------- 5 | First additional attempt ($\epsilon_1 =1.25$):\n 6 | - Probability increase: 27.7% 7 | (94% HDI: [26.1%, 29.3%]) 8 | 9 | 10 | Second additional attempt ($\epsilon_2 =1.69$):\n 11 | - Probability increase: 34.4% 12 | (94% HDI: [33.1%, 35.6%]) 13 | 14 | 15 | Task Type Differences 16 | -------------------------------------------------- 17 | Training Tasks: 18 | 19 | - Mean difficulty ($\beta = -0.29$)\n 20 | - Success probability increase: 7.1% 21 | (94% HDI: [3.4%, 10.6%]) 22 | 23 | 24 | Evaluation Tasks: 25 | 26 | - Mean difficulty ($\beta = 0.24$)\n 27 | - Success probability increase: -6.1% 28 | (94% HDI: [-9.5%, -2.6%]) 29 | 30 | 31 | Mean Task Accuracy by Shot 32 | 33 | -------------------------------------------------- 34 | 35 | 1-shot: 36 | 37 | - Training Tasks: 54.6% 38 | (94% HDI: [53.3%, 55.8%]) 39 | 40 | - Evaluation Tasks: 49.2% 41 | (94% HDI: [47.9%, 50.4%]) 42 | 43 | 44 | 2-shots: 45 | 46 | - Training Tasks: 66.6% 47 | (94% HDI: [65.4%, 67.8%]) 48 | 49 | - Evaluation Tasks: 61.6% 50 | (94% HDI: [60.5%, 62.8%]) 51 | 52 | 53 | 3-shots: 54 | 55 | - Training Tasks: 70.5% 56 | (94% HDI: [69.3%, 71.6%]) 57 | 58 | - Evaluation Tasks: 65.7% 59 | (94% HDI: [64.6%, 66.8%]) 60 | -------------------------------------------------------------------------------- /polars_cfg.json: -------------------------------------------------------------------------------- 1 | { 2 | "environment": { 3 | "POLARS_AUTO_STRUCTIFY": null, 4 | "POLARS_FMT_MAX_COLS": null, 5 | "POLARS_FMT_MAX_ROWS": "15", 6 | "POLARS_FMT_NUM_DECIMAL": null, 7 | "POLARS_FMT_NUM_GROUP_SEPARATOR": null, 8 | "POLARS_FMT_NUM_LEN": null, 9 | "POLARS_FMT_STR_LEN": "45", 10 | "POLARS_FMT_TABLE_CELL_ALIGNMENT": null, 11 | "POLARS_FMT_TABLE_CELL_LIST_LEN": null, 12 | "POLARS_FMT_TABLE_CELL_NUMERIC_ALIGNMENT": null, 13 | "POLARS_FMT_TABLE_DATAFRAME_SHAPE_BELOW": null, 14 | "POLARS_FMT_TABLE_FORMATTING": null, 15 | "POLARS_FMT_TABLE_HIDE_COLUMN_DATA_TYPES": null, 16 | "POLARS_FMT_TABLE_HIDE_COLUMN_NAMES": null, 17 | "POLARS_FMT_TABLE_HIDE_COLUMN_SEPARATOR": null, 18 | "POLARS_FMT_TABLE_HIDE_DATAFRAME_SHAPE_INFORMATION": null, 19 | "POLARS_FMT_TABLE_INLINE_COLUMN_DATA_TYPE": null, 20 | "POLARS_FMT_TABLE_ROUNDED_CORNERS": null, 21 | "POLARS_MAX_EXPR_DEPTH": null, 22 | "POLARS_STREAMING_CHUNK_SIZE": null, 23 | "POLARS_TABLE_WIDTH": null, 24 | "POLARS_VERBOSE": null, 25 | "POLARS_WARN_UNSTABLE": null 26 | }, 27 | "direct": { 28 | "set_fmt_float": "mixed", 29 | "set_float_precision": null, 30 | "set_thousands_separator": "", 31 | "set_decimal_separator": ".", 32 | "set_trim_decimal_zeros": false 33 | } 34 | } -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appnope==0.1.4 2 | arviz==0.19.0 3 | asttokens==2.4.1 4 | cachetools==5.5.0 5 | cloudpickle==3.0.0 6 | comm==0.2.2 7 | cons==0.4.6 8 | contourpy==1.2.1 9 | cycler==0.12.1 10 | debugpy==1.8.5 11 | decorator==5.1.1 12 | dm-tree==0.1.8 13 | etuples==0.3.9 14 | executing==2.0.1 15 | filelock==3.16.1 16 | fonttools==4.53.1 17 | h5netcdf==1.3.0 18 | h5py==3.11.0 19 | ipykernel==6.29.5 20 | ipython==8.26.0 21 | ipywidgets==8.1.5 22 | jedi==0.19.1 23 | jupyter-client==8.6.2 24 | jupyter-core==5.7.2 25 | jupyterlab-widgets==3.0.13 26 | kiwisolver==1.4.5 27 | levenshtein==0.25.1 28 | llvmlite==0.43.0 29 | logical-unification==0.4.6 30 | markdown-it-py==3.0.0 31 | matplotlib==3.9.2 32 | matplotlib-inline==0.1.7 33 | mdurl==0.1.2 34 | minikanren==1.0.3 35 | multipledispatch==1.0.0 36 | nest-asyncio==1.6.0 37 | numba==0.60.0 38 | numpy==1.26.4 39 | packaging==24.1 40 | pandas==2.2.2 41 | parso==0.8.4 42 | pexpect==4.9.0 43 | pillow==10.4.0 44 | pip==24.2 45 | platformdirs==4.2.2 46 | polars==1.5.0 47 | preliz==0.9.1 48 | prompt-toolkit==3.0.47 49 | psutil==6.0.0 50 | ptyprocess==0.7.0 51 | pure-eval==0.2.3 52 | pyarrow==17.0.0 53 | pygments==2.18.0 54 | pymc==5.16.2 55 | pyparsing==3.1.2 56 | pytensor==2.25.4 57 | python-dateutil==2.9.0.post0 58 | pytz==2024.1 59 | pyzmq==26.1.0 60 | rapidfuzz==3.9.6 61 | rich==13.8.1 62 | scipy==1.12.0 63 | seaborn==0.13.2 64 | setuptools==75.1.0 65 | six==1.16.0 66 | stack-data==0.6.3 67 | threadpoolctl==3.5.0 68 | toolz==0.12.1 69 | tornado==6.4.1 70 | traitlets==5.14.3 71 | typing-extensions==4.12.2 72 | tzdata==2024.1 73 | wcwidth==0.2.13 74 | widgetsnbextension==4.0.13 75 | xarray==2024.9.0 76 | xarray-einstats==0.8.0 77 | -------------------------------------------------------------------------------- /src/kaggle_submision_to_csv.py: -------------------------------------------------------------------------------- 1 | import json 2 | import polars as pl 3 | from argparse import ArgumentParser 4 | from utils import grid2str 5 | import os 6 | from pathlib import Path 7 | 8 | basepath = Path(__file__).parent.parent 9 | 10 | 11 | def get_args(): 12 | parser = ArgumentParser() 13 | parser.add_argument("--submission_id", type=str, required=True) 14 | return parser.parse_args() 15 | 16 | 17 | def kaggle_submision_to_csv(input_json, submission_id): 18 | dataframe = { 19 | "task_name": [], 20 | "test_number": [], 21 | "submission_id": [], 22 | "test_output_grid": [], 23 | "attempt_number": [], 24 | } 25 | for k, test_list in input_json.items(): 26 | for i, test_submissions in enumerate(test_list): 27 | for attempt, grid in test_submissions.items(): 28 | dataframe["task_name"].append(k + ".json") 29 | dataframe["test_number"].append(i + 1) 30 | dataframe["submission_id"].append(submission_id) 31 | dataframe["test_output_grid"].append(grid2str(grid)) 32 | dataframe["attempt_number"].append(int(attempt.split("_")[-1])) 33 | 34 | df = pl.DataFrame(dataframe) 35 | output_path = os.path.join( 36 | basepath, "data", "kaggle_solutions", submission_id, "submission.csv" 37 | ) 38 | df.write_csv(output_path) 39 | 40 | 41 | if __name__ == "__main__": 42 | args = get_args() 43 | input_json_path = os.path.join( 44 | basepath, "data", "kaggle_solutions", args.submission_id, "submission.json" 45 | ) 46 | with open(input_json_path, "r") as f: 47 | input_json = json.load(f) 48 | kaggle_submision_to_csv(input_json, args.submission_id) 49 | -------------------------------------------------------------------------------- /survey/readme.md: -------------------------------------------------------------------------------- 1 | # Survey directory description 2 | 3 | This document describes the CSV files in the `/survey` directory 4 | 5 | ## feedback_data.csv 6 | 7 | This file contains participant feedback. 8 | 9 | This file contains participant feedback. 10 | 11 | | Column name | Description | 12 | | ----------- | ----------------------------------------- | 13 | | exp_name | Experiment name | 14 | | task_type | Type of task (training or evaluation set) | 15 | | hashed_id | Anonymized participant identifier | 16 | | feedback | Feedback provided by the participant | 17 | 18 | ## demographics_data.csv 19 | 20 | This file contains demographic information about the participants. 21 | 22 | | Column name | Description | 23 | | --------------- | ---------------------------------------------------------- | 24 | | exp_name | Experiment name (internal identifier) | 25 | | task_type | Type of task (training or evaluation set) | 26 | | hashed_id | Anonymized participant identifier | 27 | | age | Age of the participant | 28 | | gender | Gender of the participant | 29 | | race | Race of the participant | 30 | | education_level | Education level of the participant | 31 | | normal_vision | Boolean indicating if the participant has normal vision | 32 | | color_blind | Boolean indicating if the participant is color blind | 33 | | fluent_english | Boolean indicating if the participant is fluent in English | 34 | 35 | ## withdraw_data.csv 36 | 37 | This file contains information about participants who withdrew from the experiment. 38 | 39 | | Column name | Description | 40 | | ---------------- | ---------------------------------------------- | 41 | | exp_name | Experiment name | 42 | | task_type | Type of task (training or evaluation set) | 43 | | hashed_id | Anonymized participant identifier | 44 | | withdraw | Boolean indicating if the participant withdrew | 45 | | withdraw_reason | Reason for withdrawal | 46 | | withdraw_comment | Comment on withdrawal | 47 | -------------------------------------------------------------------------------- /models/README.md: -------------------------------------------------------------------------------- 1 | # Bayesian Item Response Theory (IRT) Model 2 | 3 | This directory contains the implementation of a Bayesian Item Response Theory (IRT) model used to analyze participant performance on the Abstraction and Reasoning Corpus (ARC) tasks. The model is implemented in Python using the `pymc` library. 4 | 5 | ## Model Overview 6 | 7 | Intuitively, IRT models disambiguate between different latent variables that are hypothesized to drive probability of success on tasks within a test. Due to the limitations of random task-participant pairings in a finite experimental setup, certain tasks/participants can potentially be under- or overestimated with respect to ground-truth difficulty/ability when simply considering empirical success rate. Fitting an IRT model using a Bayesian framework allowed us to extract credible intervals for each parameter, which is useful for dealing with the inherent uncertainty of empirical data in a principled way. Through Bayesian data imputation, missing values were treated as additional parameters to be estimated and were sampled, conditioned on values of the model parameters during inference. Additionally, the inferred item difficulties allow us to examine difficulty distributions across ARC tasks and datasets, as well as lay out a task difficulty ordering to get a better sense of which kinds of tasks are easier or harder for people. Participant and item difficulties were given $\mathcal{N}(0, \sigma_{\alpha})$ and $\mathcal{N}(0, \sigma_{\beta})$ priors with $\sigma_{\alpha}, \sigma_{\beta} \sim \mathcal{N}^+(1)$ hyperpriors. The feedback effect was modeled as follows: $\gamma_0 = 0$, $\gamma_1 \sim \mathcal{N}^+(1)$ and $\gamma_2 = \gamma_1 + \delta$, where $\delta \sim \mathcal{N}^+(1)$. 8 | 9 | ## Mean Probability of Success 10 | 11 | The model calculates the mean probability of success for both the training and evaluation sets for each attempt. This is computed by averaging the predicted probabilities over all participants and tasks within a given set. 12 | 13 | The formula for the mean probability of success is: 14 | 15 | $$P_{\text{set}}(k) = \frac{1}{N_p N_t} \sum_{i=1}^{N_p} \sum_{j \in \mathcal{T}_{\text{set}}} \text{logit}^{-1}(\hat{\alpha}_i - \hat{\beta}_j + \hat{\gamma}_k)$$ 16 | 17 | where $`\mathcal{T}_{\text{set}}`$ represents either training or evaluation tasks, $`k \in \{0,1,2\}`$ is the attempt number and $`P_{\text{set}}(k)`$ is the mean probability of success for the training or evaluation set at attempt $k$. 18 | 19 | In this equation: 20 | 21 | - $\hat{\alpha}_i$ is the estimated ability for participant $i$. 22 | - $\hat{\beta}_j$ is the estimated difficulty for task $j$. 23 | - $\hat{\gamma}_k$ is the estimated effect for attempt $k$. 24 | 25 | ## Implementation 26 | 27 | The model is defined in `bayesian_IRT.py`. The script takes a DataFrame of participant responses and fits the IRT model using PyMC, returning the model object and the inference trace. 28 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime as dt 2 | import polars as pl 3 | import hashlib 4 | 5 | 6 | def parse_mixed_datetime(datetime_str): 7 | formats = ["%m/%d/%Y, %I:%M:%S %p", "%m/%d/%Y, %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"] 8 | for fmt in formats: 9 | try: 10 | return dt.strptime(datetime_str, fmt) 11 | except ValueError: 12 | continue 13 | return None 14 | 15 | 16 | def get_summary(df, verbose=False): 17 | """Get ARC summary data frame by filtering out traces where last action is invalid""" 18 | # list of final actions 19 | final_actions = [ 20 | "no_last_description", 21 | "write_description", 22 | "write_last_description", 23 | ] 24 | df = df.with_columns( 25 | attempt_number=pl.when(pl.col("attempt_number") == 4) 26 | .then(3) 27 | .otherwise(pl.col("attempt_number")) 28 | ) 29 | complete_task_joint_ids = ( 30 | df.select( 31 | pl.all() 32 | .top_k_by(["attempt_number", "num_actions"], k=1) 33 | .over(["joint_id_task"], mapping_strategy="explode") 34 | ) 35 | .filter(pl.col("action").is_in(final_actions)) 36 | .select("joint_id_task") 37 | ) 38 | df_summary = df.join(complete_task_joint_ids, on="joint_id_task").select( 39 | pl.all() 40 | .top_k_by("num_actions", k=1) 41 | .over(["joint_id_task", "attempt_number"], mapping_strategy="explode") 42 | ) 43 | if verbose: 44 | print( 45 | f"Filtered out {df.n_unique('joint_id_task') - df_summary.n_unique('joint_id_task')}/{df.n_unique('joint_id_task')} participant task attempts" 46 | ) 47 | df_summary = df_summary[ 48 | [ 49 | "hashed_id", 50 | "task_name", 51 | "joint_id_task", 52 | "task_number", 53 | "attempt_number", 54 | "action", 55 | "solved", 56 | "test_output_grid", 57 | "first_written_solution", 58 | "last_written_solution", 59 | "num_actions", 60 | "exp_name", 61 | "task_type", 62 | "complete", 63 | ] 64 | ] 65 | return df_summary 66 | 67 | 68 | def get_errors(df): 69 | """Take ARC summary data frame and filter for all incorrect attempts""" 70 | df_errors = df.filter(pl.col("solved") == False) 71 | 72 | # get frequency of errors 73 | df_errors = df_errors.group_by(["task_name", "test_output_grid"]).agg( 74 | pl.count().alias("count"), 75 | pl.first("task_type").alias("task_type"), 76 | ) 77 | 78 | # get hashed output grid 79 | df_errors = df_errors.with_columns( 80 | pl.col("test_output_grid") 81 | .map_elements( 82 | lambda x: hashlib.md5(x.encode()).hexdigest(), return_dtype=pl.Utf8 83 | ) 84 | .alias("hashed_output_grid") 85 | ) 86 | 87 | return df_errors 88 | 89 | 90 | def grid2str(grid): 91 | """Converts an ARC grid in numpy form to a string representation""" 92 | grid_str = "|" 93 | for row in grid: 94 | for num in row: 95 | grid_str += str(num) 96 | grid_str += "|" 97 | return grid_str 98 | 99 | 100 | def include_incomplete(df_summary, df_incomplete, verbose=False): 101 | df_summary = df_summary.drop("condition") 102 | df_incomplete_summary = get_summary(df_incomplete, verbose=verbose) 103 | df_incomplete_summary = df_incomplete_summary.select(df_summary.columns) 104 | df_summary = df_summary.with_columns(pl.lit(False).alias("incomplete")) 105 | df_incomplete_summary = df_incomplete_summary.with_columns( 106 | pl.lit(True).alias("incomplete") 107 | ) 108 | df_summary = df_summary.vstack(df_incomplete_summary) 109 | if verbose: 110 | print( 111 | f"Included {df_incomplete_summary.n_unique('joint_id_task')}/{df_incomplete.n_unique('joint_id_task')} incomplete participant task attempts" 112 | ) 113 | return df_summary, df_incomplete_summary 114 | 115 | 116 | def md5(grid): 117 | """Converts a string representation of a grid to an md5 hash for indexing""" 118 | return hashlib.md5(bytes(grid, encoding="utf-8")).hexdigest() 119 | -------------------------------------------------------------------------------- /models/bayesian_IRT.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import pymc as pm 3 | import numpy as np 4 | 5 | 6 | def bayes_irt(df, n_samples=5000, tune=1000, seed=0): 7 | """ 8 | Bayesian Item Response Theory model 9 | :param df: pandas DataFrame 10 | :param n_samples: int, number of samples 11 | :param tune: int, number of burn in samples 12 | :param seed: int, seed for random number generator 13 | :return: model, trace 14 | """ 15 | 16 | np.random.seed(seed) 17 | # create indices for participants 18 | participants_idx, participants = pd.factorize(df["hashed_id"], sort=True) 19 | 20 | # create task index 21 | task_idx, tasks = pd.factorize(df["task_name"], sort=True) 22 | 23 | # task type 24 | task_type_idx, _ = pd.factorize( 25 | df["task_type"], sort=True 26 | ) # sorted means evaluation=0, training=1 27 | 28 | # training and eval tasks 29 | training_task_idx = task_idx[task_type_idx == 1] 30 | training_task_idx = np.unique(training_task_idx) 31 | training_tasks = tasks[training_task_idx] 32 | eval_task_idx = task_idx[task_type_idx == 0] 33 | eval_task_idx = np.unique(eval_task_idx) 34 | eval_tasks = tasks[eval_task_idx] 35 | 36 | # training and eval participants 37 | training_participants_idx = participants_idx[task_type_idx == 1] 38 | training_participants_idx = np.unique(training_participants_idx) 39 | eval_participants_idx = participants_idx[task_type_idx == 0] 40 | eval_participants_idx = np.unique(eval_participants_idx) 41 | 42 | # coords 43 | coords = { 44 | "participants": participants, 45 | "tasks": tasks, 46 | "shots": ["1-shot", "2-shots", "3-shots"], 47 | "obs": np.arange(len(df)), 48 | "training_tasks": training_tasks, 49 | "eval_tasks": eval_tasks, 50 | } 51 | 52 | with pm.Model(coords=coords) as model: 53 | 54 | # hyperpriors 55 | mu_alpha = 0 56 | mu_beta = 0 57 | 58 | sigma_alpha = pm.HalfNormal("sigma_alpha", sigma=1) 59 | sigma_beta = pm.HalfNormal("sigma_beta", sigma=1) 60 | 61 | # Ability (alpha) for each participant 62 | alpha = pm.Normal("alpha", mu=mu_alpha, sigma=sigma_alpha, dims="participants") 63 | 64 | # Difficulty (beta) for each task 65 | beta = pm.Normal("beta", mu=mu_beta, sigma=sigma_beta, dims="tasks") 66 | 67 | # Learning rate (epsilon) for each shot 68 | epsilon_zero = 0 69 | epsilon_one = pm.HalfNormal("epsilon_one", sigma=1) 70 | delta = pm.HalfNormal("epsilon_delta", sigma=1) 71 | epsilon_two = epsilon_one + delta 72 | pm.Deterministic("epsilon_two", epsilon_two) 73 | 74 | # Stack epsilons as a vector 75 | epsilon = pm.math.stack([epsilon_zero, epsilon_one, epsilon_two]) 76 | 77 | # Likelihood 78 | p = pm.math.invlogit( 79 | alpha[participants_idx, None] - beta[task_idx, None] + epsilon[None, :] 80 | ) 81 | observed = df[["1-shot", "2-shots", "3-shots"]].values 82 | pm.Bernoulli("outcomes", p=p, observed=observed, dims=("obs", "shots")) 83 | 84 | # Calculate logits for every participant on every train task 85 | logits_all_training = ( 86 | alpha[:, None, None] # Participant abilities (N_participants, 1, 1) 87 | - beta[ 88 | None, training_task_idx, None 89 | ] # Task difficulties (1, N__training_tasks, 1) 90 | + epsilon[None, None, :] # Learning rates (1, 1, N_shots) 91 | ) 92 | 93 | # Apply invlogit to get probabilities 94 | p_all_training = pm.math.invlogit(logits_all_training) 95 | pm.Deterministic( 96 | "mean_task_acc_training", 97 | p_all_training.mean( 98 | axis=(0, 1) 99 | ), # Average across participants and tasks for each attempt 100 | dims="shots", 101 | ) 102 | 103 | # Calculate logits for every participant on every eval task 104 | logits_all_eval = ( 105 | alpha[:, None, None] # Participant abilities (N_participants, 1, 1) 106 | - beta[None, eval_task_idx, None] # Task difficulties (1, N_eval_tasks, 1) 107 | + epsilon[None, None, :] # Learning rates (1, 1, N_shots) 108 | ) 109 | 110 | # Apply invlogit to get probabilities 111 | p_all_eval = pm.math.invlogit(logits_all_eval) 112 | pm.Deterministic( 113 | "mean_task_acc_eval", 114 | p_all_eval.mean( 115 | axis=(0, 1) 116 | ), # Average across participants and tasks for each attempt 117 | dims="shots", 118 | ) 119 | 120 | # Sampling 121 | trace = pm.sample( 122 | n_samples, 123 | tune=tune, 124 | return_inferencedata=True, 125 | random_seed=seed, 126 | progressbar=True, 127 | ) 128 | 129 | return model, trace 130 | -------------------------------------------------------------------------------- /analysis/5-learning.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": { 7 | "vscode": { 8 | "languageId": "r" 9 | } 10 | }, 11 | "outputs": [ 12 | { 13 | "name": "stderr", 14 | "output_type": "stream", 15 | "text": [ 16 | "Le chargement a n'ecessit'e le package : Matrix\n", 17 | "\n", 18 | "************\n", 19 | "Welcome to afex. For support visit: http://afex.singmann.science/\n", 20 | "\n", 21 | "- Functions for ANOVAs: aov_car(), aov_ez(), and aov_4()\n", 22 | "- Methods for calculating p-values with mixed(): 'S', 'KR', 'LRT', and 'PB'\n", 23 | "- 'afex_aov' and 'mixed' objects can be passed to emmeans() for follow-up tests\n", 24 | "- Get and set global package options with: afex_options()\n", 25 | "- Set sum-to-zero contrasts globally: set_sum_contrasts()\n", 26 | "- For example analyses see: browseVignettes(\"afex\")\n", 27 | "************\n", 28 | "\n", 29 | "\n", 30 | "Attachement du package : 'afex'\n", 31 | "\n", 32 | "\n", 33 | "L'objet suivant est masqu'e depuis 'package:lme4':\n", 34 | "\n", 35 | " lmer\n", 36 | "\n", 37 | "\n" 38 | ] 39 | } 40 | ], 41 | "source": [ 42 | "library(lme4)\n", 43 | "library(ggplot2)\n", 44 | "library(afex)\n" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 2, 50 | "metadata": { 51 | "vscode": { 52 | "languageId": "r" 53 | } 54 | }, 55 | "outputs": [], 56 | "source": [ 57 | "# load data\n", 58 | "df_task_num_success <- read.csv(\"../data/task_number_outcomes.csv\")\n" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": 3, 64 | "metadata": { 65 | "vscode": { 66 | "languageId": "r" 67 | } 68 | }, 69 | "outputs": [ 70 | { 71 | "data": { 72 | "text/plain": [ 73 | "Generalized linear mixed model fit by maximum likelihood (Laplace\n", 74 | " Approximation) [glmerMod]\n", 75 | " Family: binomial ( logit )\n", 76 | "Formula: success ~ task_number + (1 | hashed_id)\n", 77 | " Data: df_task_num_success\n", 78 | "Control: glmerControl(optimizer = \"bobyqa\")\n", 79 | "\n", 80 | " AIC BIC logLik deviance df.resid \n", 81 | " 7368.7 7389.3 -3681.4 7362.7 7143 \n", 82 | "\n", 83 | "Scaled residuals: \n", 84 | " Min 1Q Median 3Q Max \n", 85 | "-2.7843 -0.4753 0.2765 0.4093 2.0394 \n", 86 | "\n", 87 | "Random effects:\n", 88 | " Groups Name Variance Std.Dev.\n", 89 | " hashed_id (Intercept) 3.913 1.978 \n", 90 | "Number of obs: 7146, groups: hashed_id, 1632\n", 91 | "\n", 92 | "Fixed effects:\n", 93 | " Estimate Std. Error z value Pr(>|z|) \n", 94 | "(Intercept) 1.37745 0.09073 15.181 <2e-16 ***\n", 95 | "task_number 0.02333 0.02044 1.141 0.254 \n", 96 | "---\n", 97 | "Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1\n", 98 | "\n", 99 | "Correlation of Fixed Effects:\n", 100 | " (Intr)\n", 101 | "task_number -0.649" 102 | ] 103 | }, 104 | "metadata": {}, 105 | "output_type": "display_data" 106 | } 107 | ], 108 | "source": [ 109 | "# fit mixed model\n", 110 | "mixed_model <- glmer(\n", 111 | " success ~ task_number +\n", 112 | " (1 | hashed_id),\n", 113 | " data = df_task_num_success,\n", 114 | " family = \"binomial\",\n", 115 | " control = glmerControl(optimizer = \"bobyqa\")\n", 116 | ")\n", 117 | "summary(mixed_model)\n" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 4, 123 | "metadata": { 124 | "vscode": { 125 | "languageId": "r" 126 | } 127 | }, 128 | "outputs": [ 129 | { 130 | "name": "stderr", 131 | "output_type": "stream", 132 | "text": [ 133 | "Contrasts set to contr.sum for the following variables: hashed_id\n", 134 | "\n", 135 | "Numerical variables NOT centered on 0: task_number\n", 136 | "If in interactions, interpretation of lower order (e.g., main) effects difficult.\n", 137 | "\n" 138 | ] 139 | } 140 | ], 141 | "source": [ 142 | "mixed_model_lrt <- mixed(\n", 143 | " success ~ task_number +\n", 144 | " (1 | hashed_id),\n", 145 | " data = df_task_num_success,\n", 146 | " family = \"binomial\",\n", 147 | " method = \"LRT\",\n", 148 | " control = glmerControl(optimizer = \"bobyqa\")\n", 149 | ")\n" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 5, 155 | "metadata": { 156 | "vscode": { 157 | "languageId": "r" 158 | } 159 | }, 160 | "outputs": [ 161 | { 162 | "data": { 163 | "text/plain": [ 164 | "Mixed Model Anova Table (Type 3 tests, LRT-method)\n", 165 | "\n", 166 | "Model: success ~ task_number + (1 | hashed_id)\n", 167 | "Data: df_task_num_success\n", 168 | "Df full model: 3\n", 169 | " Effect df Chisq p.value\n", 170 | "1 task_number 1 1.30 .254\n", 171 | "---\n", 172 | "Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '+' 0.1 ' ' 1" 173 | ] 174 | }, 175 | "metadata": {}, 176 | "output_type": "display_data" 177 | } 178 | ], 179 | "source": [ 180 | "mixed_model_lrt\n" 181 | ] 182 | } 183 | ], 184 | "metadata": { 185 | "kernelspec": { 186 | "display_name": "R", 187 | "language": "R", 188 | "name": "ir" 189 | }, 190 | "language_info": { 191 | "codemirror_mode": "r", 192 | "file_extension": ".r", 193 | "mimetype": "text/x-r-source", 194 | "name": "R", 195 | "pygments_lexer": "r", 196 | "version": "4.3.2" 197 | } 198 | }, 199 | "nbformat": 4, 200 | "nbformat_minor": 2 201 | } 202 | -------------------------------------------------------------------------------- /data/readme.md: -------------------------------------------------------------------------------- 1 | # Data directory description 2 | 3 | This document describes the CSV files in the `/data` directory. 4 | 5 | ## data.csv 6 | 7 | This file contains all collected task data for complete and incomplete participant data, respectively. Participant data was deemed incomplete if the data file had an insufficient number of attempted tasks (5 or 10 depending on the experiment). 8 | 9 | Each row represents a single, numbered action taken by a unique participant on a given task and attempt with all relevant experiment, participant, task and action information. 10 | 11 | | Column name | Description | 12 | | ------------------ | ----------------------------------------------------------------------------------------- | 13 | | exp_name | Experiment name (internal identifier) | 14 | | task_type | Type of task (training or evaluation set) | 15 | | hashed_id | Anonymized participant identifier | 16 | | joint_id_task | Combined identifier for participant and task | 17 | | task_name | Name of the task | 18 | | task_number | Number of the task (i.e., 3 is the third task completed) | 19 | | time | Timestamp of the action | 20 | | attempt_number | Number of the attempt | 21 | | action_id | Number of the action taken | 22 | | solved | Boolean indicating if the task was solved at this action | 23 | | done | Boolean indicating if the attempt is complete (last action) | 24 | | test_input_grid | Input grid for the task | 25 | | test_input_size_x | X-dimension of the input grid | 26 | | test_input_size_y | Y-dimension of the input grid | 27 | | test_output_grid | Output grid for the task in string format | 28 | | test_output_size_x | X-dimension of the output grid | 29 | | test_output_size_y | Y-dimension of the output grid | 30 | | action | Action taken by the participant | 31 | | action_x | X-coordinate of the action | 32 | | action_y | Y-coordinate of the action | 33 | | select_loc | Selected location | 34 | | selected_data | Data selected by the participant | 35 | | selected_symbol | Symbol selected by the participant (i.e., color in the experiment interface) | 36 | | selected_tool | Tool selected by the participant | 37 | | copy_paste_data | Data used in copy-paste actions | 38 | | complete | Boolean indicating if the data is from a participant that completed the experiment or not | 39 | 40 | ## summary_data.csv 41 | 42 | This file contains summary data for complete and incomplete participant data, respectively. Each row represents a summary of an attempt by a unique participant at a given task. 43 | 44 | | Column name | Description | 45 | | ---------------------- | ----------------------------------------------------------------------------------------- | 46 | | exp_name | Experiment name (internal identifier) | 47 | | task_type | Type of task (training or evaluation set) | 48 | | hashed_id | Anonymized participant identifier | 49 | | joint_id_task | Combined identifier for participant and task | 50 | | task_name | Name of the task | 51 | | task_number | Number of the task (i.e., 3 is the third completed task) | 52 | | attempt_number | Number of the attempt | 53 | | num_actions | Number of actions taken until submission | 54 | | solved | Boolean indicating if the task was solved | 55 | | test_output_grid | Output grid for the task | 56 | | first_written_solution | First solution written by the participant | 57 | | last_written_solution | Last solution written by the participant | 58 | | complete | Boolean indicating if the data is from a participant that completed the experiment or not | 59 | 60 | ## incorrect_submissions.csv 61 | 62 | These files contain error information for complete and incomplete participant data, respectively. 63 | 64 | | Column name | Description | 65 | | ---------------- | ----------------------------------------- | 66 | | task_name | Name of the task | 67 | | task_type | Type of task (training or evaluation set) | 68 | | test_output_grid | Output grid for the task | 69 | | count | Number of occurrences of this error | 70 | -------------------------------------------------------------------------------- /src/generate_IRT_model_results.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import argparse as ap 4 | import os 5 | import json 6 | from pathlib import Path 7 | import cloudpickle as cpkl 8 | import sys 9 | import polars as pl 10 | 11 | basepath = Path(__file__).parent.parent 12 | sys.path.append(str(basepath)) 13 | 14 | from models.bayesian_IRT import bayes_irt 15 | 16 | 17 | def get_args(): 18 | parser = ap.ArgumentParser(description="Generate IRT model results") 19 | parser.add_argument("--n_samples", type=int, default=5000, help="number of samples") 20 | parser.add_argument( 21 | "--n_burn", type=int, default=1000, help="number of burn in samples" 22 | ) 23 | parser.add_argument( 24 | "--seed", type=int, default=0, help="seed for random number generator" 25 | ) 26 | parser.add_argument("--impute", action="store_true", help="impute missing data") 27 | return parser.parse_args() 28 | 29 | 30 | def load_data(): 31 | # load data 32 | df_summary = pl.read_csv(os.path.join(basepath, "data", "clean_summary_data.csv")) 33 | df_summary_incomplete = pl.read_csv( 34 | os.path.join(basepath, "data", "clean_summary_data_incomplete.csv") 35 | ) 36 | 37 | # combine data 38 | df_summary = pl.concat([df_summary, df_summary_incomplete]) 39 | 40 | # filter data 41 | columns = [ 42 | "hashed_id", 43 | "task_name", 44 | "attempt_number", 45 | "solved", 46 | "task_type", 47 | "complete", 48 | ] 49 | df = ( 50 | df_summary.select(columns) 51 | .with_columns( 52 | pl.col("attempt_number").cast(pl.Int32), 53 | pl.col("solved").cast(pl.Int32), 54 | ) 55 | .to_pandas() 56 | ) 57 | 58 | # reshape for bernouilli 59 | df = df.pivot_table( 60 | values="solved", 61 | index=["hashed_id", "task_name", "task_type", "complete"], 62 | columns="attempt_number", 63 | aggfunc="first", 64 | ).reset_index() 65 | df.columns.name = None 66 | df = df.rename(columns={1: "1-shot", 2: "2-shots", 3: "3-shots"}) 67 | df = df.fillna(1) 68 | return df 69 | 70 | 71 | def get_incomplete_tasks(df, ordered_tasks_list): 72 | incomplete = df.groupby("hashed_id").size() 73 | incomplete = incomplete.reset_index() 74 | incomplete = incomplete.rename(columns={0: "n_complete"}) 75 | incomplete = incomplete[incomplete["n_complete"] < 5] 76 | repeats = 5 - incomplete["n_complete"].to_numpy().flatten() 77 | 78 | tasks = [] 79 | for h, m, r in zip(incomplete["hashed_id"], incomplete["n_complete"], repeats): 80 | # Get completed tasks for this participant 81 | completed_tasks = df[df["hashed_id"] == h]["task_name"].to_numpy() 82 | # Find indices of completed tasks in ordered list 83 | completed_indices = np.where(np.isin(ordered_tasks_list, completed_tasks))[0] 84 | 85 | if len(completed_indices) == 0: 86 | # If no tasks completed, start from beginning 87 | next_tasks = [] 88 | idx = 0 89 | while len(next_tasks) < r: 90 | if ordered_tasks_list[idx] not in completed_tasks: 91 | next_tasks.append(ordered_tasks_list[idx]) 92 | idx = (idx + 1) % len(ordered_tasks_list) 93 | else: 94 | # Start from the highest completed index 95 | next_tasks = [] 96 | idx = (completed_indices.max() + 1) % len(ordered_tasks_list) 97 | while len(next_tasks) < r: 98 | if ordered_tasks_list[idx] not in completed_tasks: 99 | next_tasks.append(ordered_tasks_list[idx]) 100 | idx = (idx + 1) % len(ordered_tasks_list) 101 | 102 | tasks.extend(next_tasks) 103 | 104 | hashed_ids = incomplete["hashed_id"].repeat(repeats) 105 | return hashed_ids, np.array(tasks) 106 | 107 | 108 | def load_ordered_tasks(): 109 | df_training_tasks_ordered = json.load( 110 | open(os.path.join(basepath, "data", "ARC_training_tasks_ordered.json")) 111 | ) 112 | df_eval_tasks_ordered = json.load( 113 | open(os.path.join(basepath, "data", "ARC_evaluation_tasks_ordered.json")) 114 | ) 115 | return df_training_tasks_ordered, df_eval_tasks_ordered 116 | 117 | 118 | def fill_na(df, hashed_ids, tasks, task_type="training"): 119 | nan_rows = pd.DataFrame( 120 | { 121 | "hashed_id": hashed_ids, 122 | "task_name": tasks, 123 | "task_type": task_type, 124 | "complete": False, 125 | "1-shot": np.nan, 126 | "2-shots": np.nan, 127 | "3-shots": np.nan, 128 | } 129 | ) 130 | df = pd.concat([df, nan_rows]) 131 | return df 132 | 133 | 134 | def include_missing_data(df): 135 | # separate training and evaluation data 136 | df_training = df[df["task_type"] == "training"] 137 | df_eval = df[df["task_type"] == "evaluation"] 138 | # load ordered tasks 139 | df_training_tasks_ordered, df_eval_tasks_ordered = load_ordered_tasks() 140 | # get incomplete tasks 141 | hashed_ids_training, tasks_training = get_incomplete_tasks( 142 | df_training, np.array(df_training_tasks_ordered) 143 | ) 144 | hashed_ids_eval, tasks_eval = get_incomplete_tasks( 145 | df_eval, np.array(df_eval_tasks_ordered) 146 | ) 147 | # fill missing data 148 | df_training = fill_na( 149 | df_training, hashed_ids_training, tasks_training, task_type="training" 150 | ) 151 | df_eval = fill_na(df_eval, hashed_ids_eval, tasks_eval, task_type="evaluation") 152 | # combine data 153 | df = pd.concat([df_training, df_eval]) 154 | return df 155 | 156 | 157 | if __name__ == "__main__": 158 | args = get_args() 159 | # load data 160 | df = load_data() 161 | if args.impute: 162 | df = include_missing_data(df) 163 | # run IRT model 164 | model, trace = bayes_irt( 165 | df, n_samples=args.n_samples, tune=args.n_burn, seed=args.seed 166 | ) 167 | # save model 168 | imputed = "_imputed" if args.impute else "" 169 | with open( 170 | os.path.join( 171 | basepath, 172 | "models", 173 | f"bayes_IRT_model_burn{args.n_burn}_N{args.n_samples}{imputed}_{args.seed}.pkl", 174 | ), 175 | "wb", 176 | ) as f: 177 | cpkl.dump((model, trace), f) 178 | -------------------------------------------------------------------------------- /arc_data/ARC_training_tasks_ordered.json: -------------------------------------------------------------------------------- 1 | [ 2 | "007bbfb7.json", 3 | "00d62c1b.json", 4 | "017c7c7b.json", 5 | "045e512c.json", 6 | "0520fde7.json", 7 | "05269061.json", 8 | "05f2a901.json", 9 | "06df4c85.json", 10 | "08ed6ac7.json", 11 | "09629e4f.json", 12 | "0962bcdd.json", 13 | "0a938d79.json", 14 | "0b148d64.json", 15 | "0ca9ddb6.json", 16 | "0d3d703e.json", 17 | "0e206a2e.json", 18 | "10fcaaa3.json", 19 | "11852cab.json", 20 | "1190e5a7.json", 21 | "137eaa0f.json", 22 | "178fcbfb.json", 23 | "1a07d186.json", 24 | "1b2d62fb.json", 25 | "1b60fb0c.json", 26 | "1bfc4729.json", 27 | "1c786137.json", 28 | "1cf80156.json", 29 | "1e32b0e9.json", 30 | "1f0c79e5.json", 31 | "1f642eb9.json", 32 | "1f85a75f.json", 33 | "2013d3e2.json", 34 | "2204b7a8.json", 35 | "22168020.json", 36 | "22233c11.json", 37 | "22eb0ac0.json", 38 | "234bbc79.json", 39 | "23581191.json", 40 | "239be575.json", 41 | "23b5c85d.json", 42 | "253bf280.json", 43 | "25d487eb.json", 44 | "25d8a9c8.json", 45 | "25ff71a9.json", 46 | "264363fd.json", 47 | "272f95fa.json", 48 | "27a28665.json", 49 | "28bf18c6.json", 50 | "29623171.json", 51 | "29c11459.json", 52 | "2bcee788.json", 53 | "2bee17df.json", 54 | "2c608aff.json", 55 | "2dc579da.json", 56 | "2dd70a9a.json", 57 | "2dee498d.json", 58 | "321b1fc6.json", 59 | "32597951.json", 60 | "3345333e.json", 61 | "3618c87e.json", 62 | "363442ee.json", 63 | "36d67576.json", 64 | "36fdfd69.json", 65 | "3906de3d.json", 66 | "3ac3eb23.json", 67 | "3bd67248.json", 68 | "3bdb4ada.json", 69 | "3befdf3e.json", 70 | "3c9b0459.json", 71 | "3de23699.json", 72 | "3e980e27.json", 73 | "3eda0437.json", 74 | "3f7978a0.json", 75 | "40853293.json", 76 | "4093f84a.json", 77 | "41e4d17e.json", 78 | "4258a5f9.json", 79 | "42a50994.json", 80 | "4347f46a.json", 81 | "445eab21.json", 82 | "447fd412.json", 83 | "44d8ac46.json", 84 | "44f52bb0.json", 85 | "4522001f.json", 86 | "4612dd53.json", 87 | "46442a0e.json", 88 | "46f33fce.json", 89 | "48d8fb45.json", 90 | "4938f0c2.json", 91 | "496994bd.json", 92 | "49d1d64f.json", 93 | "4be741c5.json", 94 | "4c4377d9.json", 95 | "4c5c2cf0.json", 96 | "50846271.json", 97 | "508bd3b6.json", 98 | "50cb2852.json", 99 | "5117e062.json", 100 | "5168d44c.json", 101 | "539a4f51.json", 102 | "53b68214.json", 103 | "543a7ed5.json", 104 | "54d82841.json", 105 | "54d9e175.json", 106 | "5521c0d9.json", 107 | "5582e5ca.json", 108 | "5614dbcf.json", 109 | "56dc2b01.json", 110 | "56ff96f3.json", 111 | "57aa92db.json", 112 | "5ad4f10b.json", 113 | "5bd6f4ac.json", 114 | "5daaa586.json", 115 | "60b61512.json", 116 | "623ea044.json", 117 | "63613498.json", 118 | "6430c8c4.json", 119 | "6455b5f5.json", 120 | "662c240a.json", 121 | "67385a82.json", 122 | "673ef223.json", 123 | "67a3c6ac.json", 124 | "67a423a3.json", 125 | "67e8384a.json", 126 | "681b3aeb.json", 127 | "6855a6e4.json", 128 | "694f12f3.json", 129 | "6a1e5592.json", 130 | "6aa20dc0.json", 131 | "6b9890af.json", 132 | "6cdd2623.json", 133 | "6cf79266.json", 134 | "6d0160f0.json", 135 | "6d0aefbc.json", 136 | "6d58a25d.json", 137 | "6d75e8bb.json", 138 | "6e02f1e3.json", 139 | "6e19193c.json", 140 | "6ecd11f4.json", 141 | "6f8cd79b.json", 142 | "6fa7a44f.json", 143 | "72322fa7.json", 144 | "72ca375d.json", 145 | "7447852a.json", 146 | "7468f01a.json", 147 | "746b3537.json", 148 | "74dd1130.json", 149 | "75b8110e.json", 150 | "760b3cac.json", 151 | "776ffc46.json", 152 | "77fdfe62.json", 153 | "780d0b14.json", 154 | "7837ac64.json", 155 | "794b24be.json", 156 | "7b6016b9.json", 157 | "7b7f7511.json", 158 | "7df24a62.json", 159 | "7e0986d6.json", 160 | "7f4411dc.json", 161 | "7fe24cdd.json", 162 | "80af3007.json", 163 | "810b9b61.json", 164 | "82819916.json", 165 | "834ec97d.json", 166 | "8403a5d5.json", 167 | "846bdb03.json", 168 | "855e0971.json", 169 | "85c4e7cd.json", 170 | "868de0fa.json", 171 | "8731374e.json", 172 | "88a10436.json", 173 | "88a62173.json", 174 | "890034e9.json", 175 | "8a004b2b.json", 176 | "8be77c9e.json", 177 | "8d5021e8.json", 178 | "8d510a79.json", 179 | "8e1813be.json", 180 | "8e5a5113.json", 181 | "8eb1be9a.json", 182 | "8efcae92.json", 183 | "8f2ea7aa.json", 184 | "90c28cc7.json", 185 | "90f3ed37.json", 186 | "913fb3ed.json", 187 | "91413438.json", 188 | "91714a58.json", 189 | "9172f3a0.json", 190 | "928ad970.json", 191 | "93b581b8.json", 192 | "941d9a10.json", 193 | "94f9d214.json", 194 | "952a094c.json", 195 | "9565186b.json", 196 | "95990924.json", 197 | "963e52fc.json", 198 | "97a05b5b.json", 199 | "98cf29f8.json", 200 | "995c5fa3.json", 201 | "99fa7670.json", 202 | "9aec4887.json", 203 | "9af7a82c.json", 204 | "9d9215db.json", 205 | "9dfd6313.json", 206 | "9ecd008a.json", 207 | "9f236235.json", 208 | "a1570a43.json", 209 | "a2fd1cf0.json", 210 | "a3df8b1e.json", 211 | "a416b8f3.json", 212 | "a48eeaf7.json", 213 | "a5f85a15.json", 214 | "a61ba2ce.json", 215 | "a61f2674.json", 216 | "a65b410d.json", 217 | "a68b268e.json", 218 | "a699fb00.json", 219 | "a740d043.json", 220 | "a79310a0.json", 221 | "a85d4709.json", 222 | "a8c38be5.json", 223 | "a8d7556c.json", 224 | "a9f96cdd.json", 225 | "aabf363d.json", 226 | "aba27056.json", 227 | "ae3edfdc.json", 228 | "ae4f1146.json", 229 | "aedd82e4.json", 230 | "af902bf9.json", 231 | "b0c4d837.json", 232 | "b190f7f5.json", 233 | "b1948b0a.json", 234 | "b230c067.json", 235 | "b27ca6d3.json", 236 | "b2862040.json", 237 | "b527c5c6.json", 238 | "b548a754.json", 239 | "b60334d2.json", 240 | "b6afb2da.json", 241 | "b7249182.json", 242 | "b782dc8a.json", 243 | "b91ae062.json", 244 | "b9b7f026.json", 245 | "ba26e723.json", 246 | "ba97ae07.json", 247 | "bb43febb.json", 248 | "bbc9ae5d.json", 249 | "bc1d5164.json", 250 | "bd4472b8.json", 251 | "bda2d7a6.json", 252 | "bdad9b1f.json", 253 | "be94b721.json", 254 | "beb8660c.json", 255 | "c0f76784.json", 256 | "c1d99e64.json", 257 | "c3e719e8.json", 258 | "c444b776.json", 259 | "c59eb873.json", 260 | "c8cbb738.json", 261 | "c909285e.json", 262 | "c9e6f938.json", 263 | "c9f8e694.json", 264 | "cbded52d.json", 265 | "cce03e0d.json", 266 | "cdecee7f.json", 267 | "ce22a75a.json", 268 | "ce4f8723.json", 269 | "ce602527.json", 270 | "ce9e57f2.json", 271 | "cf98881b.json", 272 | "d037b0a7.json", 273 | "d06dbe63.json", 274 | "d0f5fe59.json", 275 | "d10ecb37.json", 276 | "d13f3404.json", 277 | "d23f8c26.json", 278 | "d2abd087.json", 279 | "d364b489.json", 280 | "d406998b.json", 281 | "d43fd935.json", 282 | "d4469b4b.json", 283 | "d4a91cb9.json", 284 | "d4f3cd78.json", 285 | "d511f180.json", 286 | "d5d6de2d.json", 287 | "d687bc17.json", 288 | "d6ad076f.json", 289 | "d89b689b.json", 290 | "d8c310e9.json", 291 | "d90796e8.json", 292 | "d9f24cd1.json", 293 | "dae9d2b5.json", 294 | "db3e9e38.json", 295 | "dbc1a6ce.json", 296 | "dc1df850.json", 297 | "dc433765.json", 298 | "ddf7fa4f.json", 299 | "de1cd16c.json", 300 | "e21d9049.json", 301 | "e26a3af2.json", 302 | "e3497940.json", 303 | "e40b9e2f.json", 304 | "e48d4e1a.json", 305 | "e509e548.json", 306 | "e6721834.json", 307 | "e73095fd.json", 308 | "e76a88a6.json", 309 | "e8593010.json", 310 | "e8dc4411.json", 311 | "e9614598.json", 312 | "e98196ab.json", 313 | "e9afcf9a.json", 314 | "ea32f347.json", 315 | "ea786f4a.json", 316 | "eb5a1d5d.json", 317 | "ec883f72.json", 318 | "ecdecbb3.json", 319 | "ed36ccf7.json", 320 | "ef135b50.json", 321 | "f15e1fac.json", 322 | "f1cefba8.json", 323 | "f25fbde4.json", 324 | "f25ffba3.json", 325 | "f2829549.json", 326 | "f35d900a.json", 327 | "f5b8619d.json", 328 | "f8a8fe49.json", 329 | "f8b3ba0a.json", 330 | "f8c80d96.json", 331 | "f8ff0b80.json", 332 | "fafffa47.json", 333 | "fcb5c309.json", 334 | "fcc82909.json", 335 | "feca6190.json", 336 | "ff28f65a.json", 337 | "ff805c23.json" 338 | ] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Human Abstraction and Reasoning Corpus (H-ARC) 2 | 3 | This repository contains scripts and additional information about H-ARC to accompany our paper [published in](https://www.nature.com/articles/s41597-025-05687-1) the Scientific Data journal. 4 | 5 | ![Figure 2: Example of a human actions traces from H-ARC and corresponding natural language descriptions](figures/arc-preprint-figure2-background.png) 6 | 7 | The data can be downloaded from our [OSF repository](https://osf.io/bh8yq). 8 | 9 | Participant responses, natural language descriptions, errors and state space graphs can all be explored visually on our [project webpage](https://arc-visualizations.github.io/index.html). 10 | 11 | H-ARC consists of action by action traces of humans solving ARC tasks from the both the training and evaluation sets using an interface and setup similar to François Chollet's initial proposal. The experiment platform can be viewed at [this link](https://exps.gureckislab.org/e/assumption-fast-natural/#/) and the underlying code is available in an [accompanying Github repo](https://github.com/Le-Gris/harc-viewer). 12 | 13 | The original ARC dataset can be found [here](https://github.com/fchollet/ARC-AGI). 14 | 15 | ## Citing our work 16 | 17 | ``` 18 | @article{legrisComprehensiveBehavioralDataset2025, 19 | title = {A {{Comprehensive Behavioral Dataset}} for the {{Abstraction}} and {{Reasoning Corpus}}}, 20 | author = {LeGris, Solim and Vong, Wai Keen and Lake, Brenden M. and Gureckis, Todd M.}, 21 | year = {2025}, 22 | month = aug, 23 | journal = {Scientific Data}, 24 | volume = {12}, 25 | number = {1}, 26 | pages = {1380}, 27 | issn = {2052-4463}, 28 | doi = {10.1038/s41597-025-05687-1}, 29 | abstract = {The Abstraction and Reasoning Corpus (ARC) is a visual program synthesis benchmark designed to test out-of-distribution generalization in machines. Comparing AI algorithms to human performance is essential to measure progress on these problems. In this paper, we present H-ARC (Human-ARC): a novel large-scale dataset containing solution attempts from over 1700 humans on ARC problems. The dataset spans the full set of 400 training and 400 evaluation tasks from the original ARC benchmark, and it is the largest human evaluation to date. By publishing the dataset, we contribute human responses to each problem, step-by-step behavioral action traces from the ARC user-interface, and natural-language solution descriptions of the inferred program/rule. We believe this dataset will be of value to researchers, both in cognitive science and AI, since it offers the potential to facilitate the discovery of underlying mechanisms supporting abstraction and reasoning in people. The insights to be gained from these data not only have value for cognitive science, but could in turn inform the design of more efficient, human-like AI algorithms.} 30 | } 31 | ``` 32 | 33 | ## Getting started 34 | 35 | ### Setting up the Python Environment 36 | 37 | 1. Ensure you have Python 3.10 or later installed on your system. 38 | 39 | 2. Clone this repository to your local machine: 40 | 41 | ```bash 42 | gh repo clone le-gris/h-arc 43 | cd h-arc 44 | ``` 45 | 46 | 3. Create a virtual environment: 47 | 48 | ```bash 49 | python -m venv .venv 50 | ``` 51 | 52 | 4. Activate the virtual environment: 53 | 54 | - On Windows: 55 | ```bash 56 | venv\Scripts\activate 57 | ``` 58 | - On macOS and Linux: 59 | ```bash 60 | source .venv/bin/activate 61 | ``` 62 | 63 | 5. Install the required packages using pip and the requirements.txt file: 64 | ```bash 65 | pip install -r requirements.txt 66 | ``` 67 | 68 | ### Extracting the dataset 69 | 70 | The H-ARC dataset can be downloaded as a zip archive from our OSF repository. To extract it: 71 | 72 | 1. Navigate to the project root directory if you're not already there and move the zip archive there. Make sure it is named `osfstorage-archive.zip`. 73 | 74 | 2. Use the following command to extract the dataset: 75 | - On Windows: 76 | ```bash 77 | tar -xf data/osfstorage-archive.zip 78 | ``` 79 | - On macOS and Linux: 80 | ```bash 81 | unzip data/osfstorage-archive.zip 82 | ``` 83 | 84 | After extraction, you should see several CSV files in the `data` and `survey` folders. 85 | 86 | ## Dataset 87 | 88 | The H-ARC dataset consists of several CSV files containing different aspects of human performance on ARC tasks. 89 | 90 | All files are in CSV format. In the `data` folder, there are the following files: 91 | 92 | - `data.csv`: All collected data from complete / incomplete participant data 93 | - `incorrect_submissions.csv`: All unique errors on each task and their counts from complete/incomplete participant data 94 | - `summary_data.csv`: Attempt by attempt summary data for complete/incomplete participant data 95 | - [`readme.md`](data/readme.md): data directory description 96 | 97 | In the `survey` folder, there are the following files: 98 | 99 | - `feedback.csv`: Participant feedback 100 | - `demographics.csv`: Demographic information 101 | - `withdraw.csv`: Withdrawal information 102 | - [`readme.md`](survey/readme.md): survey directory description 103 | 104 | For more detailed information about the dataset, see each of the readme files. 105 | 106 | ## Bayesian IRT Model 107 | 108 | To analyze performance using a Bayesian Item Response Theory model: 109 | 110 | 1. Generate the model: 111 | 112 | ```bash 113 | python src/generate_IRT_model_results.py --n_samples 10000 --n_burn 2000 --seed 4 --impute 114 | ``` 115 | 116 | Remove the `--impute` flag to exclude missing data from the analysis. 117 | 118 | 2. Generate plots and statistics: 119 | ```bash 120 | python src/IRT_model_plots_analysis.py --model_path models/bayes_IRT_model_burn2000_N10000_imputed_4.pkl --verbose 121 | ``` 122 | 123 | This will create trace plots, parameter visualizations, and detailed statistics in the `figures/` and `results/` directories. 124 | 125 | For more details, see the model [`readme.md`](models/README.md) file. 126 | 127 | ## Analyses 128 | 129 | We include in this repository the main Jupyter notebooks used to compute reported results from our paper. 130 | 131 | ### Notebooks 132 | 133 | #### [0-arc-dataset.ipynb](analysis/0-arc-dataset.ipynb) 134 | 135 | This notebook looks at some aspects of the ARC dataset structure. 136 | 137 | #### [1-basic-results](analysis/1-basic-results.ipynb) 138 | 139 | This notebook computes basic performance metrics on the H-ARC dataset, including overall solve rates, action counts, and time-related statistics for both training and evaluation tasks. 140 | 141 | #### [2-demogrpahics](analysis/2-demographics.ipynb) 142 | 143 | This notebook looks at some basic demographics data from our pool of participants. 144 | 145 | #### [3-misc](analysis/3-misc.ipynb) 146 | 147 | This notebook contains miscellaneous analyses, including participant counts for different experimental conditions and various data processing steps. 148 | 149 | #### [4-errors](analysis/4-errors.ipynb) 150 | 151 | This notebook analyzes error patterns in participant responses, including copy errors and other common mistake types across both training and evaluation tasks. 152 | 153 | #### [5-learning](analysis/5-learning.ipynb) 154 | 155 | This notebook examines learning effects across tasks using mixed-effects logistic regression models. It analyzes how task success rates change as participants progress through the experiment. 156 | 157 | #### [6-incomplete-data-analysis](analysis/6-incomplete-data.ipynb) 158 | 159 | This notebook focuses on analyzing incomplete task attempts, comparing performance metrics between participants who completed all tasks and those who didn't, and examining factors that might contribute to task incompletion. 160 | 161 | #### [7-human-machine](analysis/7-human-machine.ipynb) 162 | 163 | This notebook compares the performance of human participants with that of algorithmic solutions to evaluation set ARC tasks. It analyzes success rates, error patterns, and solution strategies between humans and AI systems. 164 | 165 | #### [8-action-traces](analysis/8-action-traces.ipynb) 166 | 167 | This notebook shows how to use the data to extract action traces for further analysis. 168 | 169 | ## Processing Kaggle Submission 170 | 171 | Follow these steps to process a Kaggle submission file. This will faciliate downstream human-machine comparisons. Here we use the "Claude-3.5 (Baseline)" approach from the [ARC Prize leaderboard](https://arcprize.org/leaderboard) as an example. 172 | 173 | 1. Create the necessary directories: 174 | 175 | ```bash 176 | mkdir -p data/kaggle_solutions/claude3_5-langchain 177 | ``` 178 | 179 | 2. Visit the following webpage: 180 | [Claude 3.5 Langchain ARC Submission](https://www.kaggle.com/code/gregkamradt/using-frontier-models-on-arc-agi-via-langchain/output) 181 | 182 | 3. Download the `submission.json` file from the webpage into the `data/kaggle_solutions/claude3_5-langchain` directory. 183 | 184 | 4. Run the `kaggle_submision_to_csv.py` script with the appropriate submission ID: 185 | ```bash 186 | python src/kaggle_submision_to_csv.py --submission_id claude3_5-langchain 187 | ``` 188 | 189 | This will process the JSON file and create a CSV file in the same directory with a similar format to our human data. 190 | 191 | ## License 192 | 193 | This dataset is licensed under the [CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/) and can be used for any purposes. 194 | -------------------------------------------------------------------------------- /analysis/3-misc.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# imports\n", 10 | "import sys\n", 11 | "\n", 12 | "sys.path.append(\"..\")\n", 13 | "from src.utils import *\n", 14 | "import polars as pl\n", 15 | "from datetime import datetime as dt\n", 16 | "import seaborn as sns\n", 17 | "import numpy as np\n", 18 | "from matplotlib import pyplot as plt\n", 19 | "from scipy.stats import ttest_ind, permutation_test" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 2, 25 | "metadata": {}, 26 | "outputs": [ 27 | { 28 | "data": { 29 | "text/plain": [ 30 | "" 31 | ] 32 | }, 33 | "execution_count": 2, 34 | "metadata": {}, 35 | "output_type": "execute_result" 36 | } 37 | ], 38 | "source": [ 39 | "# polars config\n", 40 | "pl.Config.load_from_file(\"../polars_cfg.json\")" 41 | ] 42 | }, 43 | { 44 | "cell_type": "code", 45 | "execution_count": 3, 46 | "metadata": {}, 47 | "outputs": [], 48 | "source": [ 49 | "# data file paths\n", 50 | "data_path = \"/Users/solimlegris/Projets/h-arc-osf/data/data.csv\"\n", 51 | "summary_path = \"/Users/solimlegris/Projets/h-arc-osf/data/summary_data.csv\"" 52 | ] 53 | }, 54 | { 55 | "cell_type": "code", 56 | "execution_count": 4, 57 | "metadata": {}, 58 | "outputs": [], 59 | "source": [ 60 | "df_summary = pl.read_csv(summary_path)\n", 61 | "df_all = pl.read_csv(data_path)\n", 62 | "\n", 63 | "# parse time\n", 64 | "df_all = df_all.with_columns(pl.col(\"time\").cast(pl.Datetime))\n", 65 | "\n", 66 | "columns = [\n", 67 | " \"exp_name\",\n", 68 | " \"hashed_id\",\n", 69 | " \"joint_id_task\",\n", 70 | " \"task_name\",\n", 71 | " \"task_number\",\n", 72 | " \"task_type\",\n", 73 | " \"attempt_number\",\n", 74 | " \"action\",\n", 75 | " \"action_id\",\n", 76 | " \"solved\",\n", 77 | " \"time\",\n", 78 | " \"test_input_grid\",\n", 79 | " \"test_output_grid\",\n", 80 | "]\n", 81 | "df_all = df_all.select(columns)" 82 | ] 83 | }, 84 | { 85 | "cell_type": "code", 86 | "execution_count": 5, 87 | "metadata": {}, 88 | "outputs": [ 89 | { 90 | "name": "stdout", 91 | "output_type": "stream", 92 | "text": [ 93 | "Number of participants given 5 tasks on training set: 542\n", 94 | "Number of participants given 10 tasks on training set: 241\n" 95 | ] 96 | } 97 | ], 98 | "source": [ 99 | "df_by_participant = df_summary.group_by(\"hashed_id\").agg(\n", 100 | " pl.max(\"task_number\").alias(\"tasks_completed\"),\n", 101 | " pl.min(\"task_number\").alias(\"first_task\"),\n", 102 | " pl.first(\"task_type\"),\n", 103 | " pl.first(\"exp_name\"),\n", 104 | ")\n", 105 | "\n", 106 | "# number of participants given 5 tasks on training set\n", 107 | "five = df_by_participant.filter(\n", 108 | " (pl.col(\"task_type\") == \"training\")\n", 109 | " & ~(pl.col(\"exp_name\").is_in([\"expv0\", \"expv1\"]))\n", 110 | ")\n", 111 | "ten = df_by_participant.filter(\n", 112 | " (pl.col(\"task_type\") == \"training\") & (pl.col(\"exp_name\").is_in([\"expv0\", \"expv1\"]))\n", 113 | ")\n", 114 | "print(\"Number of participants given 5 tasks on training set: \", len(five))\n", 115 | "print(\"Number of participants given 10 tasks on training set: \", len(ten))" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 6, 121 | "metadata": {}, 122 | "outputs": [ 123 | { 124 | "name": "stdout", 125 | "output_type": "stream", 126 | "text": [ 127 | "Number of participants not prevented from submitting the same grid after an incorrect attempt: 405/1729\n", 128 | "\n" 129 | ] 130 | } 131 | ], 132 | "source": [ 133 | "# get number of participants who were prevented from submitting the same task\n", 134 | "num_copy_allowed = len(\n", 135 | " df_all.group_by(\"hashed_id\")\n", 136 | " .agg(pl.max(\"time\").alias(\"last_time\"), pl.first(\"task_type\"))\n", 137 | " .filter((pl.col(\"last_time\") < dt(2023, 12, 1, 15, 43, 10)))\n", 138 | ")\n", 139 | "total = len(df_summary[\"hashed_id\"].unique())\n", 140 | "print(\n", 141 | " f\"Number of participants not prevented from submitting the same grid after an incorrect attempt: {num_copy_allowed}/{total}\\n\"\n", 142 | ")" 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": 7, 148 | "metadata": {}, 149 | "outputs": [ 150 | { 151 | "name": "stdout", 152 | "output_type": "stream", 153 | "text": [ 154 | "Proportion of incorrect submissions that were copies of previous submissions outputs: 0.06\n", 155 | "\n" 156 | ] 157 | } 158 | ], 159 | "source": [ 160 | "# rate of copied outputs\n", 161 | "test_outputs = (\n", 162 | " df_summary.filter(~pl.col(\"solved\"))\n", 163 | " .group_by(\"joint_id_task\")\n", 164 | " .agg(pl.col(\"test_output_grid\"), pl.len())\n", 165 | ")\n", 166 | "total = test_outputs.select(pl.sum(\"len\")).item()\n", 167 | "# apply set to remove duplicates\n", 168 | "test_outputs = test_outputs.with_columns(\n", 169 | " pl.col(\"test_output_grid\").list.n_unique().alias(\"unique_count\")\n", 170 | ")\n", 171 | "num_copied = (\n", 172 | " test_outputs.select((pl.col(\"len\") - pl.col(\"unique_count\")).alias(\"num_copies\"))\n", 173 | " .select(pl.sum(\"num_copies\"))\n", 174 | " .item()\n", 175 | ")\n", 176 | "print(\n", 177 | " f\"Proportion of incorrect submissions that were copies of previous submissions outputs: {num_copied/total:.2f}\\n\"\n", 178 | ")" 179 | ] 180 | }, 181 | { 182 | "cell_type": "code", 183 | "execution_count": 8, 184 | "metadata": {}, 185 | "outputs": [ 186 | { 187 | "data": { 188 | "text/html": [ 189 | "
\n", 196 | "shape: (10, 3)
task_namesolvedcount
strf64u32
"b4a43f3b.json"0.010
"31d5ba1a.json"0.09
"a8610ef7.json"0.08
"8719f442.json"0.07
"79fb03f4.json"0.06
"fea12743.json"0.09090911
"e6de6e8f.json"0.110
"34b99a2b.json"0.1111119
"1c56ad9f.json"0.1111119
"0c9aba6e.json"0.1111119
" 197 | ], 198 | "text/plain": [ 199 | "shape: (10, 3)\n", 200 | "┌───────────────┬──────────┬───────┐\n", 201 | "│ task_name ┆ solved ┆ count │\n", 202 | "│ --- ┆ --- ┆ --- │\n", 203 | "│ str ┆ f64 ┆ u32 │\n", 204 | "╞═══════════════╪══════════╪═══════╡\n", 205 | "│ b4a43f3b.json ┆ 0.0 ┆ 10 │\n", 206 | "│ 31d5ba1a.json ┆ 0.0 ┆ 9 │\n", 207 | "│ a8610ef7.json ┆ 0.0 ┆ 8 │\n", 208 | "│ 8719f442.json ┆ 0.0 ┆ 7 │\n", 209 | "│ 79fb03f4.json ┆ 0.0 ┆ 6 │\n", 210 | "│ fea12743.json ┆ 0.090909 ┆ 11 │\n", 211 | "│ e6de6e8f.json ┆ 0.1 ┆ 10 │\n", 212 | "│ 34b99a2b.json ┆ 0.111111 ┆ 9 │\n", 213 | "│ 1c56ad9f.json ┆ 0.111111 ┆ 9 │\n", 214 | "│ 0c9aba6e.json ┆ 0.111111 ┆ 9 │\n", 215 | "└───────────────┴──────────┴───────┘" 216 | ] 217 | }, 218 | "execution_count": 8, 219 | "metadata": {}, 220 | "output_type": "execute_result" 221 | } 222 | ], 223 | "source": [ 224 | "# top ten hardest tasks ordered by number of participants\n", 225 | "df_summary.filter(pl.col(\"task_type\") == \"evaluation\").select(\n", 226 | " pl.col([\"joint_id_task\", \"task_name\", \"attempt_number\", \"solved\"])\n", 227 | " .top_k_by(\"attempt_number\", k=1)\n", 228 | " .over(\"joint_id_task\", mapping_strategy=\"explode\")\n", 229 | ").group_by(\"task_name\").agg(pl.sum(\"solved\") / pl.len(), pl.len().alias(\"count\")).sort(\n", 230 | " [\"solved\", \"count\"], descending=[False, True]\n", 231 | ").head(\n", 232 | " 10\n", 233 | ")" 234 | ] 235 | } 236 | ], 237 | "metadata": { 238 | "kernelspec": { 239 | "display_name": ".venv", 240 | "language": "python", 241 | "name": "python3" 242 | }, 243 | "language_info": { 244 | "codemirror_mode": { 245 | "name": "ipython", 246 | "version": 3 247 | }, 248 | "file_extension": ".py", 249 | "mimetype": "text/x-python", 250 | "name": "python", 251 | "nbconvert_exporter": "python", 252 | "pygments_lexer": "ipython3", 253 | "version": "3.12.5" 254 | } 255 | }, 256 | "nbformat": 4, 257 | "nbformat_minor": 2 258 | } 259 | -------------------------------------------------------------------------------- /arc_data/ARC_evaluation_tasks_ordered.json: -------------------------------------------------------------------------------- 1 | [ 2 | "f0afb749.json", 3 | "94414823.json", 4 | "dc2e9a9d.json", 5 | "f83cb3f6.json", 6 | "baf41dbf.json", 7 | "93b4f4b3.json", 8 | "ff72ca3e.json", 9 | "50f325b5.json", 10 | "da515329.json", 11 | "60a26a3e.json", 12 | "14754a24.json", 13 | "4ff4c9da.json", 14 | "f9d67f8b.json", 15 | "5ffb2104.json", 16 | "2037f2c7.json", 17 | "00dbd492.json", 18 | "9c1e755f.json", 19 | "6a11f6da.json", 20 | "e760a62e.json", 21 | "7bb29440.json", 22 | "19bb5feb.json", 23 | "6ad5bdfd.json", 24 | "891232d6.json", 25 | "292dd178.json", 26 | "67b4a34d.json", 27 | "94be5b80.json", 28 | "df8cc377.json", 29 | "ce8d95cc.json", 30 | "72a961c9.json", 31 | "6f473927.json", 32 | "18419cfa.json", 33 | "45bbe264.json", 34 | "7c8af763.json", 35 | "f8be4b64.json", 36 | "e7dd8335.json", 37 | "103eff5b.json", 38 | "a57f2f04.json", 39 | "52fd389e.json", 40 | "7d1f7ee8.json", 41 | "95a58926.json", 42 | "8dae5dfc.json", 43 | "2753e76c.json", 44 | "c6e1b8da.json", 45 | "516b51b7.json", 46 | "351d6448.json", 47 | "c48954c1.json", 48 | "dc2aa30b.json", 49 | "712bf12e.json", 50 | "cb227835.json", 51 | "cd3c21df.json", 52 | "20981f0e.json", 53 | "03560426.json", 54 | "ca8de6ea.json", 55 | "e2092e0c.json", 56 | "195ba7dc.json", 57 | "fc754716.json", 58 | "09c534e7.json", 59 | "ac0c5833.json", 60 | "27a77e38.json", 61 | "7e02026e.json", 62 | "a680ac02.json", 63 | "ac605cbb.json", 64 | "5b6cbef5.json", 65 | "17b80ad2.json", 66 | "4acc7107.json", 67 | "67c52801.json", 68 | "ce039d91.json", 69 | "506d28a5.json", 70 | "5a5a2103.json", 71 | "0c9aba6e.json", 72 | "55783887.json", 73 | "ecaa0ec1.json", 74 | "929ab4e9.json", 75 | "ae58858e.json", 76 | "c658a4bd.json", 77 | "477d2879.json", 78 | "281123b4.json", 79 | "12422b43.json", 80 | "47996f11.json", 81 | "73c3b0d8.json", 82 | "137f0df0.json", 83 | "94133066.json", 84 | "ed98d772.json", 85 | "fea12743.json", 86 | "e69241bd.json", 87 | "64a7c07e.json", 88 | "7d419a02.json", 89 | "9772c176.json", 90 | "b457fec5.json", 91 | "310f3251.json", 92 | "c92b942c.json", 93 | "140c817e.json", 94 | "b7999b51.json", 95 | "ac3e2b04.json", 96 | "3d31c5b3.json", 97 | "2546ccf6.json", 98 | "626c0bcc.json", 99 | "de493100.json", 100 | "90347967.json", 101 | "88207623.json", 102 | "45737921.json", 103 | "fb791726.json", 104 | "c3202e5a.json", 105 | "642d658d.json", 106 | "456873bc.json", 107 | "782b5218.json", 108 | "9b365c51.json", 109 | "b9630600.json", 110 | "c7d4e6ad.json", 111 | "c35c1b4c.json", 112 | "60c09cac.json", 113 | "d19f7514.json", 114 | "8ba14f53.json", 115 | "0c786b71.json", 116 | "a04b2602.json", 117 | "e6de6e8f.json", 118 | "7039b2d7.json", 119 | "7d18a6fb.json", 120 | "4c177718.json", 121 | "c97c0139.json", 122 | "1e81d6f9.json", 123 | "4364c1c4.json", 124 | "72207abc.json", 125 | "e4075551.json", 126 | "31d5ba1a.json", 127 | "896d5239.json", 128 | "4e45f183.json", 129 | "009d5c81.json", 130 | "a406ac07.json", 131 | "5af49b42.json", 132 | "b942fd60.json", 133 | "11e1fe23.json", 134 | "b7cb93ac.json", 135 | "cfb2ce5a.json", 136 | "62b74c02.json", 137 | "7953d61e.json", 138 | "c663677b.json", 139 | "96a8c0cd.json", 140 | "a8610ef7.json", 141 | "0a1d4ef5.json", 142 | "69889d6e.json", 143 | "a934301b.json", 144 | "97239e3d.json", 145 | "4f537728.json", 146 | "a096bf4d.json", 147 | "575b1a71.json", 148 | "13713586.json", 149 | "8719f442.json", 150 | "40f6cd08.json", 151 | "12eac192.json", 152 | "770cc55f.json", 153 | "bc4146bd.json", 154 | "0b17323b.json", 155 | "ca8f78db.json", 156 | "e9bb6954.json", 157 | "639f5a19.json", 158 | "85b81ff1.json", 159 | "551d5bf1.json", 160 | "55059096.json", 161 | "5783df64.json", 162 | "3a301edc.json", 163 | "22a4bbc2.json", 164 | "4aab4007.json", 165 | "f9a67cb5.json", 166 | "f823c43c.json", 167 | "642248e4.json", 168 | "705a3229.json", 169 | "ad7e01d0.json", 170 | "73182012.json", 171 | "e99362f0.json", 172 | "c64f1187.json", 173 | "4e469f39.json", 174 | "e5c44e8f.json", 175 | "ccd554ac.json", 176 | "7ee1c6ea.json", 177 | "e5790162.json", 178 | "29700607.json", 179 | "9ddd00f0.json", 180 | "3194b014.json", 181 | "aa18de87.json", 182 | "af24b4cc.json", 183 | "e1baa8a4.json", 184 | "414297c0.json", 185 | "e133d23d.json", 186 | "1d398264.json", 187 | "e88171ec.json", 188 | "0e671a1a.json", 189 | "8e2edd66.json", 190 | "15696249.json", 191 | "e7b06bea.json", 192 | "48f8583b.json", 193 | "7c9b52a0.json", 194 | "3391f8c0.json", 195 | "f5c89df1.json", 196 | "42918530.json", 197 | "c074846d.json", 198 | "5207a7b5.json", 199 | "bf32578f.json", 200 | "8b28cd80.json", 201 | "fe9372f3.json", 202 | "a59b95c0.json", 203 | "93c31fbe.json", 204 | "1c56ad9f.json", 205 | "bf89d739.json", 206 | "e78887d1.json", 207 | "bd14c3bf.json", 208 | "c87289bb.json", 209 | "2a5f8217.json", 210 | "f21745ec.json", 211 | "59341089.json", 212 | "833dafe3.json", 213 | "505fff84.json", 214 | "79369cc6.json", 215 | "af22c60d.json", 216 | "aab50785.json", 217 | "b4a43f3b.json", 218 | "b0722778.json", 219 | "85fa5666.json", 220 | "fd4b2b02.json", 221 | "b1fc8b8e.json", 222 | "d56f2372.json", 223 | "1a2e2828.json", 224 | "358ba94e.json", 225 | "b20f7c8b.json", 226 | "8ee62060.json", 227 | "bbb1b8b6.json", 228 | "9b2a60aa.json", 229 | "25094a63.json", 230 | "d5c634a2.json", 231 | "0692e18c.json", 232 | "d304284e.json", 233 | "0f63c0b9.json", 234 | "9def23fe.json", 235 | "9b4c17c4.json", 236 | "27f8ce4f.json", 237 | "05a7bcf2.json", 238 | "42a15761.json", 239 | "c62e2108.json", 240 | "817e6c09.json", 241 | "ba9d41b8.json", 242 | "ea9794b1.json", 243 | "8cb8642d.json", 244 | "845d6e51.json", 245 | "e345f17b.json", 246 | "e95e3d8e.json", 247 | "9110e3c5.json", 248 | "e9b4f6fc.json", 249 | "d2acf2cb.json", 250 | "0934a4d8.json", 251 | "e9c9d9a1.json", 252 | "070dd51e.json", 253 | "762cd429.json", 254 | "da2b0fe3.json", 255 | "5289ad53.json", 256 | "e21a174a.json", 257 | "79fb03f4.json", 258 | "c1990cce.json", 259 | "20818e16.json", 260 | "bcb3040b.json", 261 | "2685904e.json", 262 | "3490cc26.json", 263 | "58743b76.json", 264 | "15113be4.json", 265 | "d017b73f.json", 266 | "cad67732.json", 267 | "12997ef3.json", 268 | "fd096ab6.json", 269 | "5b692c0f.json", 270 | "3f23242b.json", 271 | "992798f6.json", 272 | "1d0a4b61.json", 273 | "aa300dc3.json", 274 | "e74e1818.json", 275 | "4b6b68e5.json", 276 | "b15fca0b.json", 277 | "f5aa3634.json", 278 | "3b4c2228.json", 279 | "aa4ec2a5.json", 280 | "2b01abd0.json", 281 | "21f83797.json", 282 | "1acc24af.json", 283 | "15663ba9.json", 284 | "f3b10344.json", 285 | "6ea4a07e.json", 286 | "0bb8deee.json", 287 | "54db823b.json", 288 | "ef26cbf6.json", 289 | "f3cdc58f.json", 290 | "423a55dc.json", 291 | "2697da3f.json", 292 | "08573cc6.json", 293 | "0a2355a6.json", 294 | "256b0a75.json", 295 | "50aad11f.json", 296 | "f45f5ca7.json", 297 | "e66aafb8.json", 298 | "1da012fc.json", 299 | "1e97544e.json", 300 | "d931c21c.json", 301 | "68b67ca3.json", 302 | "58e15b12.json", 303 | "e7a25a18.json", 304 | "b0f4d537.json", 305 | "332efdb3.json", 306 | "16b78196.json", 307 | "9c56f360.json", 308 | "4cd1b7b2.json", 309 | "0607ce86.json", 310 | "5b526a93.json", 311 | "136b0064.json", 312 | "92e50de0.json", 313 | "81c0276b.json", 314 | "3979b1a8.json", 315 | "d37a1ef5.json", 316 | "bb52a14b.json", 317 | "9bebae7a.json", 318 | "66e6c45b.json", 319 | "604001fa.json", 320 | "981571dc.json", 321 | "0becf7df.json", 322 | "9356391f.json", 323 | "695367ec.json", 324 | "50a16a69.json", 325 | "ac2e8ecf.json", 326 | "a3f84088.json", 327 | "212895b5.json", 328 | "ea959feb.json", 329 | "62ab2642.json", 330 | "319f2597.json", 331 | "0d87d2a6.json", 332 | "dd2401ed.json", 333 | "c8b7cc0f.json", 334 | "5d2a5c43.json", 335 | "4852f2fa.json", 336 | "17cae0c1.json", 337 | "696d4842.json", 338 | "3ed85e70.json", 339 | "692cd3b6.json", 340 | "d47aa2ff.json", 341 | "e619ca6e.json", 342 | "1c02dbbe.json", 343 | "37d3e8b2.json", 344 | "b7fb29bc.json", 345 | "48131b3c.json", 346 | "2c737e39.json", 347 | "f4081712.json", 348 | "67636eac.json", 349 | "e1d2900e.json", 350 | "2c0b0aff.json", 351 | "f0df5ff0.json", 352 | "d492a647.json", 353 | "d94c3b52.json", 354 | "e9ac8c9e.json", 355 | "e0fb7511.json", 356 | "2072aba6.json", 357 | "99306f82.json", 358 | "6df30ad6.json", 359 | "ed74f2f2.json", 360 | "1a6449f1.json", 361 | "e872b94a.json", 362 | "e41c6fd3.json", 363 | "31adaf00.json", 364 | "73ccf9c2.json", 365 | "903d1b4a.json", 366 | "1990f7a8.json", 367 | "8597cfd7.json", 368 | "3ee1011a.json", 369 | "917bccba.json", 370 | "9f27f097.json", 371 | "8a371977.json", 372 | "32e9702f.json", 373 | "9caba7c3.json", 374 | "e633a9e5.json", 375 | "e681b708.json", 376 | "184a9768.json", 377 | "1c0d0a4b.json", 378 | "84f2aca1.json", 379 | "00576224.json", 380 | "84db8fc4.json", 381 | "2f0c5170.json", 382 | "d4c90558.json", 383 | "33b52de3.json", 384 | "be03b35f.json", 385 | "b7f8a4d8.json", 386 | "8fbca751.json", 387 | "cf133acc.json", 388 | "aee291af.json", 389 | "fafd9572.json", 390 | "963f59bc.json", 391 | "bf699163.json", 392 | "759f3fd3.json", 393 | "d282b262.json", 394 | "5833af48.json", 395 | "34b99a2b.json", 396 | "f3e62deb.json", 397 | "9a4bb226.json", 398 | "e7639916.json", 399 | "66f2d22f.json", 400 | "d4b1c2b1.json", 401 | "e57337a4.json" 402 | ] -------------------------------------------------------------------------------- /analysis/8-action-traces.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# imports\n", 10 | "import polars as pl\n", 11 | "import seaborn as sns\n", 12 | "import matplotlib.pyplot as plt\n", 13 | "from matplotlib.lines import Line2D\n", 14 | "from datetime import timezone\n", 15 | "from datetime import datetime as dt\n", 16 | "import numpy as np\n", 17 | "import sys\n", 18 | "from scipy.stats import pearsonr\n", 19 | "from scipy import stats\n", 20 | "\n", 21 | "sys.path.append(\"../\")\n", 22 | "from src.utils import *" 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": 2, 28 | "metadata": {}, 29 | "outputs": [ 30 | { 31 | "data": { 32 | "text/plain": [ 33 | "" 34 | ] 35 | }, 36 | "execution_count": 2, 37 | "metadata": {}, 38 | "output_type": "execute_result" 39 | } 40 | ], 41 | "source": [ 42 | "# polars config\n", 43 | "pl.Config.load_from_file(\"../polars_cfg.json\")" 44 | ] 45 | }, 46 | { 47 | "cell_type": "markdown", 48 | "metadata": {}, 49 | "source": [ 50 | "# 1. Data loading\n", 51 | "\n", 52 | "Choose whether to include participants that didn't complete all five tasks in the analyses with the True / False toggle.\n" 53 | ] 54 | }, 55 | { 56 | "cell_type": "code", 57 | "execution_count": 3, 58 | "metadata": {}, 59 | "outputs": [], 60 | "source": [ 61 | "# data file paths\n", 62 | "clean_data_path = \"/Users/solimlegris/Projets/h-arc-osf/data/data.csv\"" 63 | ] 64 | }, 65 | { 66 | "cell_type": "code", 67 | "execution_count": 4, 68 | "metadata": {}, 69 | "outputs": [], 70 | "source": [ 71 | "df_all = pl.read_csv(clean_data_path)" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 5, 77 | "metadata": {}, 78 | "outputs": [ 79 | { 80 | "data": { 81 | "text/html": [ 82 | "
\n", 89 | "shape: (5, 40)
exp_nametask_typehashed_idjoint_id_tasktask_nametask_numberis_tutorialtimeattempt_numberaction_idsolveddonetest_input_gridtest_input_size_xtest_input_size_ytest_output_gridtest_output_size_xtest_output_size_yactionaction_xaction_yselect_locselected_dataselected_symbolselected_toolcopy_paste_datafirst_written_solutionlast_written_solutionwithdrawwithdraw_reasonwithdraw_commentagegenderraceeducation_levelhousehold_incomenormal_visioncolor_blindfluent_englishcomplete
strstrstrstrstri64boolstri64i64boolboolstri64i64stri64i64strf64f64strstri64strstrstrstrboolstrstrstrstrstrstrstrstrstrstrbool
"expv2""training""59e36641c3391529505bdf7af902cec0""59e36641c3391529505bdf7af902cec0_32597951.jso…"32597951.json"1false"2023-08-03T04:22:11.000000"11falsetrue"|000|000|000|"33"|000|000|000|"33"reset_grid"nullnullnull"[]"0"edit""[]""YES DONE EASILY""INTERESTING TO COMPLETE"falsenullnullnullnullnullnullnullnullnullnulltrue
"expv2""training""59e36641c3391529505bdf7af902cec0""59e36641c3391529505bdf7af902cec0_32597951.jso…"32597951.json"1false"2023-08-03T04:22:34.000000"12falsetrue"|10010010010010010|01001001001001001|00100100…1717"|000|000|000|"33"change_color"nullnullnull"[]"1"edit""[]""YES DONE EASILY""INTERESTING TO COMPLETE"falsenullnullnullnullnullnullnullnullnullnulltrue
"expv2""training""59e36641c3391529505bdf7af902cec0""59e36641c3391529505bdf7af902cec0_32597951.jso…"32597951.json"1false"2023-08-03T04:22:46.000000"13falsetrue"|10010010010010010|01001001001001001|00100100…1717"|000|000|000|000|000|000|000|000|000|000|000|…173"change_height"nullnullnull"[]"1"edit""[]""YES DONE EASILY""INTERESTING TO COMPLETE"falsenullnullnullnullnullnullnullnullnullnulltrue
"expv2""training""59e36641c3391529505bdf7af902cec0""59e36641c3391529505bdf7af902cec0_32597951.jso…"32597951.json"1false"2023-08-03T04:23:01.000000"14falsetrue"|10010010010010010|01001001001001001|00100100…1717"|00000000000000000|00000000000000000|00000000…1717"change_width"nullnullnull"[]"1"edit""[]""YES DONE EASILY""INTERESTING TO COMPLETE"falsenullnullnullnullnullnullnullnullnullnulltrue
"expv2""training""59e36641c3391529505bdf7af902cec0""59e36641c3391529505bdf7af902cec0_32597951.jso…"32597951.json"1false"2023-08-03T04:23:41.000000"15falsetrue"|10010010010010010|01001001001001001|00100100…1717"|00000000000000000|00000000000000000|00000000…1717"change_color"nullnullnull"[]"1"edit""[]""YES DONE EASILY""INTERESTING TO COMPLETE"falsenullnullnullnullnullnullnullnullnullnulltrue
" 90 | ], 91 | "text/plain": [ 92 | "shape: (5, 40)\n", 93 | "┌──────────┬───────────┬────────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n", 94 | "│ exp_name ┆ task_type ┆ hashed_id ┆ joint_id_ ┆ … ┆ normal_vi ┆ color_bli ┆ fluent_en ┆ complete │\n", 95 | "│ --- ┆ --- ┆ --- ┆ task ┆ ┆ sion ┆ nd ┆ glish ┆ --- │\n", 96 | "│ str ┆ str ┆ str ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ bool │\n", 97 | "│ ┆ ┆ ┆ str ┆ ┆ str ┆ str ┆ str ┆ │\n", 98 | "╞══════════╪═══════════╪════════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n", 99 | "│ expv2 ┆ training ┆ 59e36641c3 ┆ 59e36641c ┆ … ┆ null ┆ null ┆ null ┆ true │\n", 100 | "│ ┆ ┆ 391529505b ┆ 339152950 ┆ ┆ ┆ ┆ ┆ │\n", 101 | "│ ┆ ┆ df7af902ce ┆ 5bdf7af90 ┆ ┆ ┆ ┆ ┆ │\n", 102 | "│ ┆ ┆ c0 ┆ 2cec0_325 ┆ ┆ ┆ ┆ ┆ │\n", 103 | "│ ┆ ┆ ┆ 97951.jso ┆ ┆ ┆ ┆ ┆ │\n", 104 | "│ ┆ ┆ ┆ … ┆ ┆ ┆ ┆ ┆ │\n", 105 | "│ expv2 ┆ training ┆ 59e36641c3 ┆ 59e36641c ┆ … ┆ null ┆ null ┆ null ┆ true │\n", 106 | "│ ┆ ┆ 391529505b ┆ 339152950 ┆ ┆ ┆ ┆ ┆ │\n", 107 | "│ ┆ ┆ df7af902ce ┆ 5bdf7af90 ┆ ┆ ┆ ┆ ┆ │\n", 108 | "│ ┆ ┆ c0 ┆ 2cec0_325 ┆ ┆ ┆ ┆ ┆ │\n", 109 | "│ ┆ ┆ ┆ 97951.jso ┆ ┆ ┆ ┆ ┆ │\n", 110 | "│ ┆ ┆ ┆ … ┆ ┆ ┆ ┆ ┆ │\n", 111 | "│ expv2 ┆ training ┆ 59e36641c3 ┆ 59e36641c ┆ … ┆ null ┆ null ┆ null ┆ true │\n", 112 | "│ ┆ ┆ 391529505b ┆ 339152950 ┆ ┆ ┆ ┆ ┆ │\n", 113 | "│ ┆ ┆ df7af902ce ┆ 5bdf7af90 ┆ ┆ ┆ ┆ ┆ │\n", 114 | "│ ┆ ┆ c0 ┆ 2cec0_325 ┆ ┆ ┆ ┆ ┆ │\n", 115 | "│ ┆ ┆ ┆ 97951.jso ┆ ┆ ┆ ┆ ┆ │\n", 116 | "│ ┆ ┆ ┆ … ┆ ┆ ┆ ┆ ┆ │\n", 117 | "│ expv2 ┆ training ┆ 59e36641c3 ┆ 59e36641c ┆ … ┆ null ┆ null ┆ null ┆ true │\n", 118 | "│ ┆ ┆ 391529505b ┆ 339152950 ┆ ┆ ┆ ┆ ┆ │\n", 119 | "│ ┆ ┆ df7af902ce ┆ 5bdf7af90 ┆ ┆ ┆ ┆ ┆ │\n", 120 | "│ ┆ ┆ c0 ┆ 2cec0_325 ┆ ┆ ┆ ┆ ┆ │\n", 121 | "│ ┆ ┆ ┆ 97951.jso ┆ ┆ ┆ ┆ ┆ │\n", 122 | "│ ┆ ┆ ┆ … ┆ ┆ ┆ ┆ ┆ │\n", 123 | "│ expv2 ┆ training ┆ 59e36641c3 ┆ 59e36641c ┆ … ┆ null ┆ null ┆ null ┆ true │\n", 124 | "│ ┆ ┆ 391529505b ┆ 339152950 ┆ ┆ ┆ ┆ ┆ │\n", 125 | "│ ┆ ┆ df7af902ce ┆ 5bdf7af90 ┆ ┆ ┆ ┆ ┆ │\n", 126 | "│ ┆ ┆ c0 ┆ 2cec0_325 ┆ ┆ ┆ ┆ ┆ │\n", 127 | "│ ┆ ┆ ┆ 97951.jso ┆ ┆ ┆ ┆ ┆ │\n", 128 | "│ ┆ ┆ ┆ … ┆ ┆ ┆ ┆ ┆ │\n", 129 | "└──────────┴───────────┴────────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘" 130 | ] 131 | }, 132 | "execution_count": 5, 133 | "metadata": {}, 134 | "output_type": "execute_result" 135 | } 136 | ], 137 | "source": [ 138 | "df_all.head()" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 9, 144 | "metadata": {}, 145 | "outputs": [], 146 | "source": [ 147 | "sorted_df = df_all.sort(\n", 148 | " by=[\"joint_id_task\", \"task_number\", \"attempt_number\", \"action_id\"]\n", 149 | ")\n", 150 | "aggregated_df = sorted_df.group_by(\n", 151 | " \"joint_id_task\", \"attempt_number\", maintain_order=True\n", 152 | ").agg(pl.col(\"test_input_grid\").alias(\"grid_states\"))" 153 | ] 154 | }, 155 | { 156 | "cell_type": "code", 157 | "execution_count": 10, 158 | "metadata": {}, 159 | "outputs": [ 160 | { 161 | "data": { 162 | "text/html": [ 163 | "
\n", 170 | "shape: (5, 3)
joint_id_taskattempt_numbergrid_states
stri64list[str]
"00136a0f6142ddb48fd80ea8ff22f12d_27a77e38.jso…1["|000|000|000|", "|912849821|443127679|216978436|986342917|555555555|000000000|000000000|000000000|000000000|", … "|912849821|443127679|216978436|986342917|555555555|000000000|000000000|000000000|000000000|"]
"00136a0f6142ddb48fd80ea8ff22f12d_27a77e38.jso…2["|912849821|443127679|216978436|986342917|555555555|000000000|000000000|000000000|000000000|", "|912849821|443127679|216978436|986342917|555555555|000000000|000000000|000000000|000000000|", … "|912849821|443127679|216978436|986342917|555555555|000000000|000000000|000000000|000000000|"]
"00136a0f6142ddb48fd80ea8ff22f12d_27a77e38.jso…3["|912849821|443127679|216978436|986342917|555555555|000000000|000000000|000000000|000000000|", "|912849821|443127679|216978436|986342917|555555555|000000000|000000000|000000000|000000000|", … "|912849821|443127679|216978436|986342917|555555555|000000000|000000000|000000000|000000000|"]
"00136a0f6142ddb48fd80ea8ff22f12d_7e02026e.jso…1["|000|000|000|", "|808888808080|088800808000|888800088888|800080880080|088808088088|008880000000|808808808000|080800888888|000880080800|000080880880|000880880888|888080000888|", … "|808888808080|088800808000|888800088888|800080880080|088808088088|008880000000|808808808000|080800888888|000880080800|000080880880|000880880888|888080000888|"]
"00136a0f6142ddb48fd80ea8ff22f12d_7e02026e.jso…2["|808888808080|088800808000|888800088888|800080880080|088808088088|008880000000|808808808000|080800888888|000880080800|000080880880|000880880888|888080000888|", "|808888808080|088800808000|888800088888|800080880080|088808088088|008880000000|808808808000|080800888888|000880080800|000080880880|000880880888|888080000888|", "|808888808080|088800808000|888800088888|800080880080|088808088088|008880000000|808808808000|080800888888|000880080800|000080880880|000880880888|888080000888|"]
" 171 | ], 172 | "text/plain": [ 173 | "shape: (5, 3)\n", 174 | "┌────────────────────────────────────────┬────────────────┬────────────────────────────────────────┐\n", 175 | "│ joint_id_task ┆ attempt_number ┆ grid_states │\n", 176 | "│ --- ┆ --- ┆ --- │\n", 177 | "│ str ┆ i64 ┆ list[str] │\n", 178 | "╞════════════════════════════════════════╪════════════════╪════════════════════════════════════════╡\n", 179 | "│ 00136a0f6142ddb48fd80ea8ff22f12d_27a77 ┆ 1 ┆ [\"|000|000|000|\", │\n", 180 | "│ e38.jso… ┆ ┆ \"|912849821|443127679|21697… │\n", 181 | "│ 00136a0f6142ddb48fd80ea8ff22f12d_27a77 ┆ 2 ┆ [\"|912849821|443127679|216978436|98634 │\n", 182 | "│ e38.jso… ┆ ┆ 2917|55… │\n", 183 | "│ 00136a0f6142ddb48fd80ea8ff22f12d_27a77 ┆ 3 ┆ [\"|912849821|443127679|216978436|98634 │\n", 184 | "│ e38.jso… ┆ ┆ 2917|55… │\n", 185 | "│ 00136a0f6142ddb48fd80ea8ff22f12d_7e020 ┆ 1 ┆ [\"|000|000|000|\", │\n", 186 | "│ 26e.jso… ┆ ┆ \"|808888808080|088800808000… │\n", 187 | "│ 00136a0f6142ddb48fd80ea8ff22f12d_7e020 ┆ 2 ┆ [\"|808888808080|088800808000|888800088 │\n", 188 | "│ 26e.jso… ┆ ┆ 888|800… │\n", 189 | "└────────────────────────────────────────┴────────────────┴────────────────────────────────────────┘" 190 | ] 191 | }, 192 | "execution_count": 10, 193 | "metadata": {}, 194 | "output_type": "execute_result" 195 | } 196 | ], 197 | "source": [ 198 | "aggregated_df.head()" 199 | ] 200 | }, 201 | { 202 | "cell_type": "code", 203 | "execution_count": 11, 204 | "metadata": {}, 205 | "outputs": [], 206 | "source": [ 207 | "output_json_path = \"../data/grid_states.json\"\n", 208 | "\n", 209 | "# Write the aggregated DataFrame to a JSON file\n", 210 | "aggregated_df.write_json(output_json_path)" 211 | ] 212 | } 213 | ], 214 | "metadata": { 215 | "kernelspec": { 216 | "display_name": ".venv", 217 | "language": "python", 218 | "name": "python3" 219 | }, 220 | "language_info": { 221 | "codemirror_mode": { 222 | "name": "ipython", 223 | "version": 3 224 | }, 225 | "file_extension": ".py", 226 | "mimetype": "text/x-python", 227 | "name": "python", 228 | "nbconvert_exporter": "python", 229 | "pygments_lexer": "ipython3", 230 | "version": "3.12.9" 231 | } 232 | }, 233 | "nbformat": 4, 234 | "nbformat_minor": 2 235 | } 236 | -------------------------------------------------------------------------------- /src/IRT_model_plots_analysis.py: -------------------------------------------------------------------------------- 1 | import argparse as ap 2 | import os 3 | import cloudpickle as cpkl 4 | import arviz as az 5 | import polars as pl 6 | import matplotlib.pyplot as plt 7 | from matplotlib.patches import Patch 8 | from pathlib import Path 9 | import numpy as np 10 | import seaborn as sns 11 | 12 | basepath = Path(__file__).parent.parent 13 | 14 | LABEL_SIZE = 20 15 | TITLE_SIZE = 24 16 | TICK_SIZE = 18 17 | LEGEND_SIZE = 18 18 | 19 | plt.rcParams["text.usetex"] = True 20 | 21 | 22 | def get_args(): 23 | parser = ap.ArgumentParser(description="Generate IRT model plots and analysis") 24 | parser.add_argument("--model_path", type=str, help="path to model") 25 | parser.add_argument("--verbose", action="store_true", help="print verbose output") 26 | return parser.parse_args() 27 | 28 | 29 | def load_model(model_path): 30 | with open(os.path.join(basepath, model_path), "rb") as f: 31 | model, trace = cpkl.load(f) 32 | return model, trace 33 | 34 | 35 | def plot_trace(trace, model_name): 36 | ax = az.plot_trace( 37 | trace, 38 | var_names=[ 39 | "sigma_alpha", 40 | "sigma_beta", 41 | "epsilon_one", 42 | "epsilon_delta", 43 | "epsilon_two", 44 | ], 45 | ) 46 | fig = ax.ravel()[0].figure 47 | plt.tight_layout() 48 | fig.savefig( 49 | os.path.join(basepath, "figures", f"{model_name}_trace_plot.png"), 50 | bbox_inches="tight", 51 | dpi=300, 52 | ) 53 | plt.close() 54 | 55 | 56 | def get_participant_success_rate(): 57 | # load data 58 | df_summary = pl.read_csv(os.path.join(basepath, "data", "clean_summary_data.csv")) 59 | df_summary_incomplete = pl.read_csv( 60 | os.path.join(basepath, "data", "clean_summary_data_incomplete.csv") 61 | ) 62 | df_summary = pl.concat([df_summary, df_summary_incomplete]) 63 | 64 | # filter data 65 | participants_max_tasks = df_summary.group_by("hashed_id").agg( 66 | pl.max("task_number"), pl.first("task_type") 67 | ) 68 | participants_max_tasks = participants_max_tasks.rename( 69 | {"task_number": "max_task_number"} 70 | ) 71 | df_summary_ = df_summary.join(participants_max_tasks, on="hashed_id") 72 | participant_success = df_summary_.group_by("hashed_id").agg( 73 | pl.sum("solved") / pl.max("max_task_number"), pl.first("complete") 74 | ) 75 | return participant_success 76 | 77 | 78 | def generate_irt_dataframes(trace): 79 | # Load task success rate 80 | mean_task_acc_three_shot = pl.read_csv( 81 | os.path.join(basepath, "data", "mean_task_acc_three_attempts.csv") 82 | ) 83 | # Extract necessary data from the trace 84 | alpha_mean = trace.posterior["alpha"].mean(dim=["chain", "draw"]) 85 | alpha_hdi = az.hdi(trace.posterior["alpha"], hdi_prob=0.94) 86 | participants = trace.posterior["alpha"].coords["participants"].values 87 | participant_success = get_participant_success_rate() 88 | participant_success_values = ( 89 | participant_success.select("hashed_id", "solved") 90 | .to_pandas() 91 | .set_index("hashed_id") 92 | .loc[participants] 93 | .values.flatten() 94 | ) 95 | complete = ( 96 | participant_success.select("hashed_id", "complete") 97 | .to_pandas() 98 | .set_index("hashed_id") 99 | .loc[participants] 100 | .values.flatten() 101 | ) 102 | 103 | beta_mean = trace.posterior["beta"].mean(dim=["chain", "draw"]) 104 | beta_hdi = az.hdi(trace.posterior["beta"], hdi_prob=0.94) 105 | tasks = trace.posterior["beta"].coords["tasks"].values 106 | 107 | # order success rate by tasks 108 | task_success_values = ( 109 | mean_task_acc_three_shot.select("task_name", "mean_solved") 110 | .to_pandas() 111 | .set_index("task_name") 112 | .loc[tasks] 113 | .values.flatten() 114 | ) 115 | 116 | task_type_values = ( 117 | mean_task_acc_three_shot.select("task_name", "task_type") 118 | .to_pandas() 119 | .set_index("task_name") 120 | .loc[tasks] 121 | .values.flatten() 122 | ) 123 | epsilon_one_mean = trace.posterior["epsilon_one"].mean(dim=["chain", "draw"]) 124 | epsilon_two_mean = trace.posterior["epsilon_two"].mean(dim=["chain", "draw"]) 125 | epsilon_one_hdi = az.hdi(trace.posterior["epsilon_one"], hdi_prob=0.94) 126 | epsilon_two_hdi = az.hdi(trace.posterior["epsilon_two"], hdi_prob=0.94) 127 | 128 | epsilon_one = trace.posterior["epsilon_one"].values.flatten() 129 | epsilon_two = trace.posterior["epsilon_two"].values.flatten() 130 | 131 | # Create DataFrame for participant abilities 132 | plot_df_ability = pl.DataFrame( 133 | { 134 | "hashed_id": participants, 135 | "ability_mean": alpha_mean.values, 136 | "success_rate": participant_success_values, 137 | "ability_hdi_lower": alpha_hdi.sel(hdi="lower").alpha.values, 138 | "ability_hdi_upper": alpha_hdi.sel(hdi="higher").alpha.values, 139 | "complete": complete, 140 | } 141 | ) 142 | 143 | # Create DataFrame for task difficulties 144 | plot_df_difficulty = pl.DataFrame( 145 | { 146 | "tasks": tasks, 147 | "task_type": task_type_values, 148 | "success_rate": task_success_values, 149 | "diff_mean": beta_mean.values, 150 | "diff_hdi_lower": beta_hdi.sel(hdi="lower").beta.values, 151 | "diff_hdi_upper": beta_hdi.sel(hdi="higher").beta.values, 152 | } 153 | ) 154 | 155 | # Create DataFrame for epsilon values 156 | plot_df_epsilon = pl.DataFrame( 157 | { 158 | "epsilon": list(epsilon_one) + list(epsilon_two), 159 | "attempt": [r"$\epsilon_1$"] * len(epsilon_one) 160 | + [r"$\epsilon_2$"] * len(epsilon_two), 161 | } 162 | ) 163 | 164 | # create alternative dataframe for epsilon 165 | plot_df_epsilon_alt = pl.DataFrame( 166 | { 167 | "attempt": [1, 2], 168 | "epsilon_mean": [ 169 | epsilon_one_mean.values.item(), 170 | epsilon_two_mean.values.item(), 171 | ], 172 | "epsilon_hdi_lower": [ 173 | epsilon_one_hdi.sel(hdi="lower").epsilon_one.values.item(), 174 | epsilon_two_hdi.sel(hdi="lower").epsilon_two.values.item(), 175 | ], 176 | "epsilon_hdi_upper": [ 177 | epsilon_one_hdi.sel(hdi="higher").epsilon_one.values.item(), 178 | epsilon_two_hdi.sel(hdi="higher").epsilon_two.values.item(), 179 | ], 180 | } 181 | ) 182 | 183 | return plot_df_ability, plot_df_difficulty, plot_df_epsilon, plot_df_epsilon_alt 184 | 185 | 186 | def plot_irt_parameters(df_epsilon, plot_df_ability, plot_df_difficulty, model_name): 187 | # Create figure and gridspec for the main layout 188 | fig = plt.figure(figsize=(20, 12)) 189 | gs_main = plt.GridSpec( 190 | 2, 2, width_ratios=[3, 5], height_ratios=[1, 1], hspace=0.25, wspace=0.175 191 | ) 192 | 193 | eps_palette = sns.color_palette("Paired", 2) 194 | others_palette = sns.color_palette("tab10") 195 | 196 | # Create subgridspecs for the right plots 197 | gs_ability = gs_main[0, 1].subgridspec(1, 2, width_ratios=[4, 1], wspace=0.01) 198 | gs_diff = gs_main[1, 1].subgridspec(1, 2, width_ratios=[4, 1], wspace=0.01) 199 | 200 | # Left plot (epsilon distributions) 201 | ax_eps = fig.add_subplot(gs_main[0, 0]) 202 | 203 | sns.histplot( 204 | df_epsilon, 205 | x="epsilon", 206 | hue="attempt", 207 | bins=50, 208 | alpha=0.8, 209 | palette=eps_palette, 210 | stat="density", 211 | kde=True, 212 | ax=ax_eps, 213 | legend=True, 214 | ) 215 | ax_eps.set_title("(a) Feedback effect", fontsize=TITLE_SIZE) 216 | ax_eps.set_xlabel("Parameter estimates", fontsize=LABEL_SIZE) 217 | ax_eps.set_ylabel("Count", fontsize=LABEL_SIZE) 218 | ax_eps.set_xticklabels(ax_eps.get_xticks(), fontsize=TICK_SIZE) 219 | ax_eps.set_yticklabels(ax_eps.get_yticks(), fontsize=TICK_SIZE) 220 | handles = ax_eps.get_legend().legend_handles 221 | labels = [r"$\gamma_1$", r"$\gamma_2$"] 222 | ax_eps.legend(handles, labels, fontsize=LEGEND_SIZE) 223 | ax_eps.grid(True, alpha=0.1) 224 | 225 | # Top right plots (ability) 226 | ax_ability_main = fig.add_subplot(gs_ability[0, 0]) 227 | ax_ability_kde = fig.add_subplot(gs_ability[0, 1]) 228 | 229 | # Jittered success rates for scatter plots 230 | jittered_success_rate_ability = plot_df_ability.select( 231 | "success_rate" 232 | ).to_numpy().flatten() + np.random.normal(0, 0.01, len(plot_df_ability)) 233 | 234 | # Create ability scatter plot 235 | ax_ability_main.scatter( 236 | jittered_success_rate_ability, 237 | plot_df_ability.select("ability_mean").to_numpy().flatten(), 238 | s=20, 239 | alpha=0.075, 240 | color="black", 241 | zorder=2, 242 | ) 243 | 244 | ax_ability_main.set_title("(b) Participant ability", fontsize=TITLE_SIZE) 245 | ax_ability_main.set_xlabel("Mean participant accuracy", fontsize=LABEL_SIZE) 246 | ax_ability_main.set_xticklabels([]) 247 | ax_ability_main.set_ylabel(r"$\alpha$", fontsize=LABEL_SIZE) 248 | ax_ability_main.set_yticklabels(ax_ability_main.get_yticks(), fontsize=TICK_SIZE) 249 | ax_ability_main.grid(True, alpha=0.1) 250 | ax_ability_main.axhline(0, color="black", linestyle="--", alpha=0.2) 251 | 252 | # Create ability KDE 253 | sns.kdeplot( 254 | plot_df_ability, 255 | y="ability_mean", 256 | ax=ax_ability_kde, 257 | fill=True, 258 | color=others_palette[4], 259 | alpha=0.1, 260 | ) 261 | ax_ability_kde.set_yticklabels([]) 262 | ax_ability_kde.set_xticklabels([]) 263 | ax_ability_kde.set_xlabel("") 264 | ax_ability_kde.set_ylabel("") 265 | ax_ability_kde.spines["left"].set_visible(False) 266 | ax_ability_kde.spines["top"].set_visible(False) 267 | ax_ability_kde.spines["right"].set_visible(False) 268 | ax_ability_kde.spines["bottom"].set_visible(False) 269 | ax_ability_kde.tick_params(axis="y", left=False) 270 | ax_ability_kde.tick_params(axis="x", bottom=False) 271 | ax_ability_kde.set_ylim(ax_ability_main.get_ylim()) 272 | 273 | # Bottom right plots (difficulty) 274 | ax_diff_main = fig.add_subplot(gs_diff[0, 0]) 275 | ax_diff_kde = fig.add_subplot(gs_diff[0, 1]) 276 | 277 | # Jittered success rates for scatter plots 278 | jittered_success_rate_difficulty = plot_df_difficulty.select( 279 | "success_rate" 280 | ).to_numpy().flatten() + np.random.normal(0, 0.01, len(plot_df_difficulty)) 281 | 282 | # Create difficulty scatter plot 283 | ax_diff_main.scatter( 284 | jittered_success_rate_difficulty, 285 | plot_df_difficulty.select("diff_mean").to_numpy().flatten(), 286 | s=20, 287 | alpha=0.2, 288 | color="black", 289 | zorder=2, 290 | ) 291 | ax_diff_main.errorbar( 292 | jittered_success_rate_difficulty, 293 | plot_df_difficulty["diff_mean"], 294 | yerr=[ 295 | plot_df_difficulty["diff_mean"] - plot_df_difficulty["diff_hdi_lower"], 296 | plot_df_difficulty["diff_hdi_upper"] - plot_df_difficulty["diff_mean"], 297 | ], 298 | fmt="none", 299 | ecolor="gray", 300 | alpha=0.1, 301 | zorder=1, 302 | ) 303 | 304 | # add mean difficulty for training and eval tasks 305 | mean_diff_training = ( 306 | plot_df_difficulty.filter(pl.col("task_type") == "training") 307 | .select(pl.mean("diff_mean")) 308 | .item() 309 | ) 310 | mean_diff_eval = ( 311 | plot_df_difficulty.filter(pl.col("task_type") == "evaluation") 312 | .select(pl.mean("diff_mean")) 313 | .item() 314 | ) 315 | 316 | ax_diff_main.axhline( 317 | mean_diff_training, color=others_palette[0], linestyle="--", alpha=0.5 318 | ) 319 | ax_diff_main.axhline( 320 | mean_diff_eval, color=others_palette[1], linestyle="--", alpha=0.5 321 | ) 322 | 323 | ax_diff_main.set_title("(c) Task difficulty", fontsize=TITLE_SIZE) 324 | ax_diff_main.set_xlabel("Mean task accuracy", fontsize=LABEL_SIZE) 325 | ax_diff_main.set_ylabel(r"$\beta$", fontsize=LABEL_SIZE) 326 | ax_diff_main.set_xticklabels(ax_diff_main.get_xticklabels(), fontsize=TICK_SIZE) 327 | ax_diff_main.set_yticklabels(ax_diff_main.get_yticklabels(), fontsize=TICK_SIZE) 328 | ax_diff_main.grid(True, alpha=0.1) 329 | ax_diff_main.axhline(0, color="black", linestyle="--", alpha=0.2) 330 | 331 | # create legend for kde 332 | legend_elements = [ 333 | Patch(edgecolor=others_palette[0], label="Training set", fill=False), 334 | Patch(edgecolor=others_palette[1], label="Evaluation set", fill=False), 335 | ] 336 | 337 | # Add legend with custom handles 338 | ax_diff_main.legend(handles=legend_elements, loc="upper right") 339 | 340 | # Create difficulty KDE 341 | sns.kdeplot( 342 | plot_df_difficulty.filter(pl.col("task_type") == "training"), 343 | y="diff_mean", 344 | ax=ax_diff_kde, 345 | fill=True, 346 | color=others_palette[0], 347 | alpha=0.1, 348 | ) 349 | sns.kdeplot( 350 | plot_df_difficulty.filter(pl.col("task_type") == "evaluation"), 351 | y="diff_mean", 352 | ax=ax_diff_kde, 353 | fill=True, 354 | color=others_palette[1], 355 | alpha=0.1, 356 | ) 357 | ax_diff_kde.set_yticklabels([]) 358 | ax_diff_kde.set_xticklabels([]) 359 | ax_diff_kde.set_xlabel("") 360 | ax_diff_kde.set_ylabel("") 361 | ax_diff_kde.spines["left"].set_visible(False) 362 | ax_diff_kde.spines["top"].set_visible(False) 363 | ax_diff_kde.spines["right"].set_visible(False) 364 | ax_diff_kde.spines["bottom"].set_visible(False) 365 | ax_diff_kde.tick_params(axis="y", left=False) 366 | ax_diff_kde.tick_params(axis="x", bottom=False) 367 | ax_diff_kde.set_ylim(ax_diff_main.get_ylim()) 368 | 369 | # make sure ability and diff share the same xlim 370 | ax_ability_main.set_xlim(ax_diff_main.get_xlim()) 371 | 372 | # plt.tight_layout() 373 | plt.savefig( 374 | os.path.join(basepath, "figures", f"{model_name}_irt_parameters.png"), 375 | bbox_inches="tight", 376 | dpi=300, 377 | ) 378 | plt.close() 379 | 380 | 381 | def forest_plots(plot_df_ability, plot_df_difficulty, model_name): 382 | # Create figure with two subplots 383 | fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 12)) 384 | 385 | # Ability Forest Plot 386 | ability_sorted = plot_df_ability.sort("ability_mean").with_row_index("idx") 387 | ability_sorted_complete = ability_sorted.filter(pl.col("complete")) 388 | 389 | # plot abiltiies for complete participants 390 | ax1.errorbar( 391 | x=ability_sorted_complete.select("ability_mean").to_numpy().flatten(), 392 | y=ability_sorted_complete.select("idx").to_numpy().flatten(), 393 | xerr=[ 394 | ability_sorted_complete.select("ability_mean").to_numpy().flatten() 395 | - ability_sorted_complete.select("ability_hdi_lower").to_numpy().flatten(), 396 | ability_sorted_complete.select("ability_hdi_upper").to_numpy().flatten() 397 | - ability_sorted_complete.select("ability_mean").to_numpy().flatten(), 398 | ], 399 | fmt="o", 400 | color="black", 401 | markersize=1, 402 | elinewidth=0.5, 403 | capsize=0, 404 | alpha=0.1, 405 | label="Complete", 406 | ) 407 | # add ability for incomplete participants with color 408 | ability_sorted_incomplete = ability_sorted.filter(~pl.col("complete")).sort( 409 | "ability_mean" 410 | ) 411 | ax1.errorbar( 412 | x=ability_sorted_incomplete.select("ability_mean").to_numpy().flatten(), 413 | y=ability_sorted_incomplete.select("idx").to_numpy().flatten(), 414 | xerr=[ 415 | ability_sorted_incomplete.select("ability_mean").to_numpy().flatten() 416 | - ability_sorted_incomplete.select("ability_hdi_lower") 417 | .to_numpy() 418 | .flatten(), 419 | ability_sorted_incomplete.select("ability_hdi_upper").to_numpy().flatten() 420 | - ability_sorted_incomplete.select("ability_mean").to_numpy().flatten(), 421 | ], 422 | fmt="o", 423 | color="red", 424 | markersize=1, 425 | elinewidth=0.5, 426 | capsize=0, 427 | alpha=0.5, 428 | label="Incomplete", 429 | ) 430 | 431 | ax1.axvline(x=0, color="black", linestyle="--", alpha=0.2) 432 | ax1.set_xlabel(r"Ability Parameter ($\alpha$)", fontsize=LABEL_SIZE) 433 | ax1.set_yticks([]) 434 | ax1.grid(True, alpha=0.1) 435 | ax1.set_title("(a) Participant Abilities with 94% HDI", fontsize=TITLE_SIZE) 436 | ax1.tick_params(axis="x", labelsize=TICK_SIZE) 437 | ax1.legend(fontsize=LEGEND_SIZE) 438 | 439 | # Difficulty Forest Plot 440 | difficulty_sorted = plot_df_difficulty.sort("diff_mean") 441 | ax2.errorbar( 442 | x=difficulty_sorted.select("diff_mean").to_numpy().flatten(), 443 | y=range(len(difficulty_sorted)), 444 | xerr=[ 445 | difficulty_sorted.select("diff_mean").to_numpy().flatten() 446 | - difficulty_sorted.select("diff_hdi_lower").to_numpy().flatten(), 447 | difficulty_sorted.select("diff_hdi_upper").to_numpy().flatten() 448 | - difficulty_sorted.select("diff_mean").to_numpy().flatten(), 449 | ], 450 | fmt="o", 451 | markersize=2, 452 | elinewidth=0.5, 453 | capsize=0, 454 | alpha=0.3, 455 | ) 456 | ax2.axvline(x=0, color="black", linestyle="--", alpha=0.2) 457 | ax2.set_xlabel(r"Difficulty Parameter ($\beta$)", fontsize=LABEL_SIZE) 458 | ax2.set_yticks([]) 459 | ax2.grid(True, alpha=0.1) 460 | ax2.set_title("(b) Task Difficulties with 94% HDI", fontsize=TITLE_SIZE) 461 | ax2.tick_params(axis="x", labelsize=TICK_SIZE) 462 | 463 | plt.tight_layout() 464 | plt.savefig( 465 | os.path.join(basepath, "figures", f"{model_name}_forest_plots.png"), 466 | bbox_inches="tight", 467 | dpi=300, 468 | ) 469 | plt.close() 470 | 471 | 472 | def logit_to_prob(logit): 473 | """Convert logit to probability.""" 474 | # Convert from logit scale to probability 475 | prob = 1 / (1 + np.exp(-(logit))) 476 | return prob 477 | 478 | 479 | def get_stats(trace, model_name, df, verbose=False): 480 | """Calculate and output key statistics from the IRT model trace. 481 | 482 | Args: 483 | trace: ArviZ trace object containing posterior samples 484 | model_name: Name of the model (used for output file naming) 485 | verbose: Whether to print results to console 486 | """ 487 | # Initialize output string to store all results 488 | output = [] 489 | output.append(f"=== IRT Model Statistics: {model_name} ===\n") 490 | 491 | # 1. Feedback Effects (Epsilon) 492 | output.append("Feedback Effects") 493 | output.append("-" * 50) 494 | 495 | # Calculate epsilon statistics 496 | epsilon1 = trace.posterior["epsilon_one"].mean(dim=("chain", "draw")).values.item() 497 | epsilon2 = trace.posterior["epsilon_two"].mean(dim=("chain", "draw")).values.item() 498 | 499 | # Calculate probability changes for average participant/task 500 | change1 = logit_to_prob(epsilon1) - logit_to_prob(0) 501 | change2 = logit_to_prob(epsilon2) - logit_to_prob(0) 502 | 503 | # Calculate HDI 504 | epsilon1_hdi = az.hdi(trace.posterior["epsilon_one"], hdi_prob=0.94) 505 | epsilon2_hdi = az.hdi(trace.posterior["epsilon_two"], hdi_prob=0.94) 506 | 507 | # Convert to probability differences 508 | epsilon1_hdi_prob = ( 509 | logit_to_prob(epsilon1_hdi.sel(hdi="lower").epsilon_one.values) 510 | - logit_to_prob(0), 511 | logit_to_prob(epsilon1_hdi.sel(hdi="higher").epsilon_one.values) 512 | - logit_to_prob(0), 513 | ) 514 | epsilon2_hdi_prob = ( 515 | logit_to_prob(epsilon2_hdi.sel(hdi="lower").epsilon_two.values) 516 | - logit_to_prob(0), 517 | logit_to_prob(epsilon2_hdi.sel(hdi="higher").epsilon_two.values) 518 | - logit_to_prob(0), 519 | ) 520 | 521 | output.append(rf"First additional attempt ($\epsilon_1 ={epsilon1:.2f}$):\n") 522 | output.append(f"- Probability increase: {change1:.1%}") 523 | output.append( 524 | f" (94% HDI: [{epsilon1_hdi_prob[0]:.1%}, {epsilon1_hdi_prob[1]:.1%}])\n\n" 525 | ) 526 | output.append(rf"Second additional attempt ($\epsilon_2 ={epsilon2:.2f}$):\n") 527 | output.append(f"- Probability increase: {change2:.1%}") 528 | output.append( 529 | f" (94% HDI: [{epsilon2_hdi_prob[0]:.1%}, {epsilon2_hdi_prob[1]:.1%}])\n\n" 530 | ) 531 | 532 | # 2. Task Type Differences 533 | output.append("Task Type Differences") 534 | output.append("-" * 50) 535 | 536 | # Calculate mean difficulties 537 | beta_training = trace.posterior["beta"].sel( 538 | tasks=df.filter(pl.col("task_type") == "training") 539 | .select("tasks") 540 | .to_numpy() 541 | .flatten() 542 | ) 543 | beta_eval = trace.posterior["beta"].sel( 544 | tasks=df.filter(pl.col("task_type") == "evaluation") 545 | .select("tasks") 546 | .to_numpy() 547 | .flatten() 548 | ) 549 | 550 | beta_training_mean = beta_training.mean().values 551 | beta_eval_mean = beta_eval.mean().values 552 | 553 | # Calculate probability differences for average participant 554 | p_training = logit_to_prob(0 - beta_training_mean) - logit_to_prob(0) 555 | p_eval = logit_to_prob(0 - beta_eval_mean) - logit_to_prob(0) 556 | 557 | # Calculate HDI 558 | training_hdi = az.hdi( 559 | trace.posterior["beta"] 560 | .sel( 561 | tasks=df.filter(pl.col("task_type") == "training") 562 | .select("tasks") 563 | .to_numpy() 564 | .flatten() 565 | ) 566 | .mean(dim=["tasks"]) 567 | .values.flatten(), 568 | hdi_prob=0.94, 569 | ) 570 | eval_hdi = az.hdi( 571 | trace.posterior["beta"] 572 | .sel( 573 | tasks=df.filter(pl.col("task_type") == "evaluation") 574 | .select("tasks") 575 | .to_numpy() 576 | .flatten() 577 | ) 578 | .mean(dim=["tasks"]) 579 | .values.flatten(), 580 | hdi_prob=0.94, 581 | ) 582 | 583 | p_training_hdi = ( 584 | logit_to_prob(0 - training_hdi[1]) - logit_to_prob(0), 585 | logit_to_prob(0 - training_hdi[0]) - logit_to_prob(0), 586 | ) 587 | p_eval_hdi = ( 588 | logit_to_prob(0 - eval_hdi[1]) - logit_to_prob(0), 589 | logit_to_prob(0 - eval_hdi[0]) - logit_to_prob(0), 590 | ) 591 | 592 | output.append("Training Tasks:\n") 593 | output.append(rf"- Mean difficulty ($\beta = {beta_training_mean:.2f}$)\n") 594 | output.append(f"- Success probability increase: {p_training:.1%}") 595 | output.append(f" (94% HDI: [{p_training_hdi[0]:.1%}, {p_training_hdi[1]:.1%}])\n\n") 596 | 597 | output.append("Evaluation Tasks:\n") 598 | output.append(rf"- Mean difficulty ($\beta = {beta_eval_mean:.2f}$)\n") 599 | output.append(f"- Success probability increase: {p_eval:.1%}") 600 | output.append(f" (94% HDI: [{p_eval_hdi[0]:.1%}, {p_eval_hdi[1]:.1%}])\n\n") 601 | 602 | # 3. Mean Task Accuracy by Shot 603 | output.append("Mean Task Accuracy by Shot\n") 604 | output.append("-" * 50) 605 | 606 | # Calculate mean task accuracies 607 | mean_task_acc_training = trace.posterior["mean_task_acc_training"].mean( 608 | dim=["chain", "draw"] 609 | ) 610 | mean_task_acc_training_hdi = az.hdi( 611 | trace.posterior["mean_task_acc_training"], hdi_prob=0.94 612 | ) 613 | mean_task_acc_eval = trace.posterior["mean_task_acc_eval"].mean( 614 | dim=["chain", "draw"] 615 | ) 616 | mean_task_acc_eval_hdi = az.hdi( 617 | trace.posterior["mean_task_acc_eval"], hdi_prob=0.94 618 | ) 619 | 620 | # Create formatted output for each shot 621 | for shot_idx, shot in enumerate(mean_task_acc_training.shots.values): 622 | output.append(f"\n{shot}:\n") 623 | 624 | # Training tasks 625 | train_acc = mean_task_acc_training.values[shot_idx] * 100 626 | train_hdi_lower = ( 627 | mean_task_acc_training_hdi.sel(hdi="lower").mean_task_acc_training.values[ 628 | shot_idx 629 | ] 630 | * 100 631 | ) 632 | train_hdi_upper = ( 633 | mean_task_acc_training_hdi.sel(hdi="higher").mean_task_acc_training.values[ 634 | shot_idx 635 | ] 636 | * 100 637 | ) 638 | 639 | output.append(f"- Training Tasks: {train_acc:.1f}%") 640 | output.append(f" (94% HDI: [{train_hdi_lower:.1f}%, {train_hdi_upper:.1f}%])\n") 641 | 642 | # Evaluation tasks 643 | eval_acc = mean_task_acc_eval.values[shot_idx] * 100 644 | eval_hdi_lower = ( 645 | mean_task_acc_eval_hdi.sel(hdi="lower").mean_task_acc_eval.values[shot_idx] 646 | * 100 647 | ) 648 | eval_hdi_upper = ( 649 | mean_task_acc_eval_hdi.sel(hdi="higher").mean_task_acc_eval.values[shot_idx] 650 | * 100 651 | ) 652 | 653 | output.append(f"- Evaluation Tasks: {eval_acc:.1f}%") 654 | output.append(f" (94% HDI: [{eval_hdi_lower:.1f}%, {eval_hdi_upper:.1f}%])\n") 655 | 656 | # Write results to file 657 | output_path = os.path.join(basepath, "results", f"{model_name}_stats.md") 658 | os.makedirs(os.path.dirname(output_path), exist_ok=True) 659 | with open(output_path, "w") as f: 660 | f.write("\n".join(output)) 661 | 662 | # Print to console if verbose 663 | if verbose: 664 | print("\n".join(output)) 665 | 666 | return output_path 667 | 668 | 669 | if __name__ == "__main__": 670 | args = get_args() 671 | model_name = Path(args.model_path).stem 672 | model, trace = load_model(args.model_path) 673 | # total_params = ( 674 | # len(trace.posterior.participants) # ability parameters 675 | # + len(trace.posterior.tasks) # difficulty parameters 676 | # + 2 # epsilon parameters 677 | # ) 678 | # print(f"Total number of parameters: {total_params}") 679 | plot_trace(trace, model_name) 680 | plot_df_ability, plot_df_difficulty, plot_df_epsilon, plot_df_epsilon_alt = ( 681 | generate_irt_dataframes(trace) 682 | ) 683 | # save IRT model parameters 684 | plot_df_ability.write_csv( 685 | os.path.join(basepath, "data", f"{model_name}_ability_parameters.csv") 686 | ) 687 | plot_df_difficulty.write_csv( 688 | os.path.join(basepath, "data", f"{model_name}_difficulty_parameters.csv") 689 | ) 690 | plot_df_epsilon_alt.write_csv( 691 | os.path.join(basepath, "data", f"{model_name}_epsilon_parameters.csv") 692 | ) 693 | plot_irt_parameters( 694 | plot_df_epsilon, plot_df_ability, plot_df_difficulty, model_name 695 | ) 696 | forest_plots(plot_df_ability, plot_df_difficulty, model_name) 697 | get_stats(trace, model_name, plot_df_difficulty, args.verbose) 698 | -------------------------------------------------------------------------------- /analysis/6-incomplete-data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# imports\n", 10 | "import seaborn as sns\n", 11 | "import matplotlib.pyplot as plt\n", 12 | "import sys\n", 13 | "import numpy as np\n", 14 | "\n", 15 | "sys.path.append(\"..\")\n", 16 | "from src.utils import *\n", 17 | "from scipy.stats import ttest_ind, permutation_test\n", 18 | "from scipy import stats" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 3, 24 | "metadata": {}, 25 | "outputs": [ 26 | { 27 | "data": { 28 | "text/plain": [ 29 | "" 30 | ] 31 | }, 32 | "execution_count": 3, 33 | "metadata": {}, 34 | "output_type": "execute_result" 35 | } 36 | ], 37 | "source": [ 38 | "# polars config\n", 39 | "pl.Config.load_from_file(\"../polars_cfg.json\")" 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": 4, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "# data file paths\n", 49 | "data_path = \"/Users/solimlegris/Projets/h-arc-osf/data/data.csv\"\n", 50 | "summary_path = \"/Users/solimlegris/Projets/h-arc-osf/data/summary_data.csv\"" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 5, 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "df_summary = pl.read_csv(summary_path)\n", 60 | "df_all = pl.read_csv(data_path)\n", 61 | "\n", 62 | "# parse time\n", 63 | "df_all = df_all.with_columns(pl.col(\"time\").cast(pl.Datetime))\n", 64 | "\n", 65 | "columns = [\n", 66 | " \"exp_name\",\n", 67 | " \"hashed_id\",\n", 68 | " \"joint_id_task\",\n", 69 | " \"task_name\",\n", 70 | " \"task_number\",\n", 71 | " \"task_type\",\n", 72 | " \"attempt_number\",\n", 73 | " \"action\",\n", 74 | " \"action_id\",\n", 75 | " \"solved\",\n", 76 | " \"time\",\n", 77 | " \"test_input_grid\",\n", 78 | " \"test_output_grid\",\n", 79 | "]\n", 80 | "df_all = df_all.select(columns)\n", 81 | "\n", 82 | "# load task accuracy data\n", 83 | "mean_task_acc = pl.read_csv(\"../data/mean_task_acc_three_attempts.csv\")" 84 | ] 85 | }, 86 | { 87 | "cell_type": "code", 88 | "execution_count": 7, 89 | "metadata": {}, 90 | "outputs": [ 91 | { 92 | "data": { 93 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABlRUlEQVR4nO3dd1gUV9sG8HvpTUCkq0hVsaKgBHsUs/YSe1SKNSpRgpqIsTeiUYMaI9G8do0mxmjeGLEQ0ajETkwUiRoQoxQxAlIEYef7Ix/zZmVRyuICc/+uay+dM2fOPLMzu/swc86MTBAEAUREREQSoqXpAIiIiIheNyZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAarZo0SLIZLLXsq5u3bqhW7du4nR0dDRkMhkOHDjwWtYfEBAAR0fH17KuisrOzsaECRNga2sLmUyG4OBgjcYTEBAAExMTjcagTtu3b4dMJsPly5c1HUq5OTo6ol+/fpoOo0aRyWRYtGhRuZYp/l6Kjo6ukpiqq8TERMhkMmzfvl3ToVS5F3+LagomQC9R/OVe/DIwMIC9vT3kcjnWr1+Pp0+fqmU9Dx8+xKJFixAbG6uW9tSpOsdWFitWrMD27dsxZcoU7Nq1C2PHjn1p3UOHDr2+4IiqoR9//LHcSU5NUtu3T0pu3ryJRYsWITExsULLMwEqgyVLlmDXrl3YtGkT3nvvPQBAcHAwWrZsievXryvVnTdvHvLy8srV/sOHD7F48eJyJxnHjx/H8ePHy7VMeb0sti1btiA+Pr5K119ZP/30E9544w0sXLgQY8aMgaenZ6l1mQAR/ZMgLF68WOW8vLw8zJs3r1ztdenSBXl5eejSpYs6wqu0l20f1Sw3b97E4sWLK5wA6ag3nNqpd+/e8PLyEqdDQ0Px008/oV+/fhgwYADi4uJgaGgIANDR0YGOTtW+rbm5uTAyMoKenl6VrudVdHV1Nbr+skhLS0OzZs00HQZVUk5ODoyNjTUdRrkVFhZCoVBo/LNaFmV5jw0MDMrdrpaWVoWWk4qaemzXBjwDVEHdu3fH/Pnzce/ePezevVssV9UH6MSJE+jUqRPMzc1hYmKCJk2aYO7cuQD+uT7erl07AEBgYKB4ua34unG3bt3QokULXLlyBV26dIGRkZG4bGnXXYuKijB37lzY2trC2NgYAwYMwP3795XqODo6IiAgoMSy/27zVbGp6gOUk5ODmTNnomHDhtDX10eTJk2wevVqCIKgVE8mkyEoKAiHDh1CixYtoK+vj+bNmyMyMlL1G/6CtLQ0jB8/HjY2NjAwMEDr1q2xY8cOcX5xv4OEhAQcOXJEjL20vxRkMhlycnKwY8cOsW7x+3Pv3j1MnToVTZo0gaGhIerVq4dhw4aVaOv58+dYvHgx3NzcYGBggHr16qFTp044ceLES7clNjYWVlZW6NatG7Kzs0utV9x/6MGDBxg0aBBMTExgZWWFWbNmoaioqMS2v9jnQlWfhOI2k5KS0K9fP5iYmKB+/frYuHEjAOC3335D9+7dYWxsjEaNGmHv3r0qY8vNzcXkyZNRr149mJqaws/PD0+ePClR7+jRo+jcuTOMjY1Rp04d9O3bFzdu3FC5nXfv3kWfPn1Qp04djB49WuV6r1+/DplMhu+//14su3LlCmQyGdq2batUt3fv3vD29i7RxtmzZ9G+fXsYGBjA2dkZO3fuLFEnIyMDwcHB4nHt6uqKlStXQqFQiHWK39/Vq1cjPDwcLi4u0NfXx82bNwEAt27dwtChQ2FhYQEDAwN4eXkpxV2af7f76aefolGjRjA0NETXrl3x+++/l3g/AgIC4OzsDAMDA9ja2mLcuHF4/PixUr3i76mbN2/inXfeQd26ddGpUycEBASI+/7fl/+LqeoD9ODBA4wfPx729vbQ19eHk5MTpkyZgoKCAgCqj8d/f6916NABhoaGcHJyQkREhFLbBQUFWLBgATw9PWFmZgZjY2N07twZp06dKvU92rx5s/jet2vXDpcuXRLrvWr79u3bB09PT9SpUwempqZo2bIl1q1b98p9lJGRgYCAAJiZmcHc3Bz+/v7IyMgoUe9lx3Z5vzv37NmDJk2awMDAAJ6enjhz5oxSveJ9fOvWLQwfPhympqaoV68eZsyYgWfPnpWIbffu3fD09IShoSEsLCwwcuTIEr8bAMT319DQEO3bt8fPP//8yvenWGFhIZYuXSruH0dHR8ydOxf5+fkltlHVZcp//25t374dw4YNAwC8+eab4r4sT18zngGqhLFjx2Lu3Lk4fvw4Jk6cqLLOjRs30K9fP7Rq1QpLliyBvr4+7ty5g3PnzgEA3N3dsWTJEixYsACTJk1C586dAQAdOnQQ23j8+DF69+6NkSNHYsyYMbCxsXlpXMuXL4dMJsOHH36ItLQ0hIeHw9fXF7GxseKZqrIoS2z/JggCBgwYgFOnTmH8+PHw8PDAsWPHMHv2bDx48ACffvqpUv2zZ8/i4MGDmDp1KurUqYP169djyJAhSEpKQr169UqNKy8vD926dcOdO3cQFBQEJycnfPPNNwgICEBGRgZmzJgBd3d37Nq1C++//z4aNGiAmTNnAgCsrKxUtrlr1y5MmDAB7du3x6RJkwAALi4uAIBLly7h/PnzGDlyJBo0aIDExERs2rQJ3bp1w82bN2FkZATgny+csLAwsZ2srCxcvnwZV69eRc+ePVWu99KlS5DL5fDy8sLhw4dfuX+Kioogl8vh7e2N1atX4+TJk1izZg1cXFwwZcqUly77sjZ79+6NLl26YNWqVdizZw+CgoJgbGyMjz76CKNHj8bbb7+NiIgI+Pn5wcfHB05OTkptBAUFwdzcHIsWLUJ8fDw2bdqEe/fuiT9+xe+xv78/5HI5Vq5cidzcXGzatAmdOnXCtWvXlJLpwsJCyOVydOrUCatXrxbf4xe1aNEC5ubmOHPmDAYMGAAA+Pnnn6GlpYVff/0VWVlZMDU1hUKhwPnz58V9W+zOnTsYOnQoxo8fD39/f2zduhUBAQHw9PRE8+bNAfyT3HXt2hUPHjzA5MmT4eDggPPnzyM0NBTJyckIDw9XanPbtm149uwZJk2aBH19fVhYWODGjRvo2LEj6tevjzlz5sDY2Bhff/01Bg0ahG+//RaDBw9+5X7auXMnnj59imnTpuHZs2dYt24dunfvjt9++038Tjhx4gT+/PNPBAYGwtbWFjdu3MDmzZtx48YN/PLLLyX+OBs2bBjc3NywYsUKCIKANm3a4OHDhzhx4gR27dr1ypgePnyI9u3bIyMjA5MmTULTpk3x4MEDHDhwALm5uS898/XkyRP06dMHw4cPx6hRo/D1119jypQp0NPTw7hx4wAAWVlZ+PLLLzFq1ChMnDgRT58+xX/+8x/I5XJcvHgRHh4eSm3u3bsXT58+xeTJkyGTybBq1Sq8/fbb+PPPP6Grq4vJkyeXun0nTpzAqFGj0KNHD6xcuRIAEBcXh3PnzmHGjBmlbocgCBg4cCDOnj2Ld999F+7u7vjuu+/g7++vsr6qY7u8352nT5/G/v37MX36dOjr6+Pzzz9Hr169cPHiRbRo0UKp7vDhw+Ho6IiwsDD88ssvWL9+PZ48eaKU6C9fvhzz58/H8OHDMWHCBDx69AgbNmxAly5dcO3aNZibmwMA/vOf/2Dy5Mno0KEDgoOD8eeff2LAgAGwsLBAw4YNS32Pik2YMAE7duzA0KFDMXPmTFy4cAFhYWGIi4vDd99998rl/61Lly6YPn061q9fj7lz58Ld3R0AxH/LRKBSbdu2TQAgXLp0qdQ6ZmZmQps2bcTphQsXCv9+Wz/99FMBgPDo0aNS27h06ZIAQNi2bVuJeV27dhUACBERESrnde3aVZw+deqUAECoX7++kJWVJZZ//fXXAgBh3bp1YlmjRo0Ef3//V7b5stj8/f2FRo0aidOHDh0SAAjLli1Tqjd06FBBJpMJd+7cEcsACHp6ekplv/76qwBA2LBhQ4l1/Vt4eLgAQNi9e7dYVlBQIPj4+AgmJiZK296oUSOhb9++L22vmLGxscr3JDc3t0RZTEyMAEDYuXOnWNa6detXrsvf318wNjYWBEEQzp49K5iamgp9+/YVnj179sr4/P39BQDCkiVLlMrbtGkjeHp6itPFx8GpU6eU6iUkJJTYl8VtrlixQix78uSJYGhoKMhkMmHfvn1i+a1btwQAwsKFC8Wy4s+Ip6enUFBQIJavWrVKACAcPnxYEARBePr0qWBubi5MnDhRKaaUlBTBzMxMqbw4pjlz5rzyPREEQejbt6/Qvn17cfrtt98W3n77bUFbW1s4evSoIAiCcPXqVaV4BOGfYwOAcObMGbEsLS1N0NfXF2bOnCmWLV26VDA2Nhb++OMPpfXOmTNH0NbWFpKSkgRB+N/7a2pqKqSlpSnV7dGjh9CyZUul/axQKIQOHToIbm5uL92+4nYNDQ2Fv/76Syy/cOGCAEB4//33xTJVx+pXX31VYjuLv6dGjRpVov60adOE0n4aXtz/fn5+gpaWlsrvSIVCIQiC6uOx+HttzZo1Yll+fr7g4eEhWFtbi8dSYWGhkJ+fr9TukydPBBsbG2HcuHFiWfF7VK9ePeHvv/8Wyw8fPiwAEP773/++cvtmzJghmJqaCoWFhSq3vTTF33urVq0SywoLC4XOnTuX+nl78dgu73cnAOHy5cti2b179wQDAwNh8ODBYlnxPh4wYIBSm1OnThUACL/++qsgCIKQmJgoaGtrC8uXL1eq99tvvwk6OjpieUFBgWBtbS14eHgo7ZPNmzcLAJR+N1SJjY0VAAgTJkxQKp81a5YAQPjpp5+UtvHfx1mxF3+3vvnmG5XfdWXFS2CVZGJi8tLRYMWZ8+HDh5VOl5eHvr4+AgMDy1zfz88PderUEaeHDh0KOzs7/PjjjxVaf1n9+OOP0NbWxvTp05XKZ86cCUEQcPToUaVyX19f8SwLALRq1Qqmpqb4888/X7keW1tbjBo1SizT1dXF9OnTkZ2djdOnT6tha/7n32dlnj9/jsePH8PV1RXm5ua4evWqOM/c3Bw3btzA7du3X9nmqVOnIJfL0aNHDxw8eBD6+vpljufdd99Vmu7cufMr37NXmTBhgvh/c3NzNGnSBMbGxhg+fLhY3qRJE5ibm6tc16RJk5T6hE2ZMgU6OjriMXfixAlkZGRg1KhRSE9PF1/a2trw9vYucUmjuI2y6Ny5M65evYqcnBwA/5xZ7NOnDzw8PMTT8z///DNkMhk6deqktGyzZs3EM5vAP2cImzRporSN33zzDTp37oy6desqxe7r64uioqISlx6GDBmidKbx77//xk8//YThw4fj6dOn4vKPHz+GXC7H7du38eDBg1du56BBg1C/fn1xun379vD29lb6XP/7WH327BnS09PxxhtvAIDSsVrsxWOpPBQKBQ4dOoT+/fsr9ZEs9qrbgejo6GDy5MnitJ6eHiZPnoy0tDRcuXIFAKCtrS2eRVIoFPj7779RWFgILy8vldszYsQI1K1bV5wu3rdl+XyYm5sjJyfnlZesX/Tjjz9CR0dH6XjV1tYWB8yo8uKxXd7vTh8fH6UBHQ4ODhg4cCCOHTumdDkcAKZNm6Y0XRxX8XFz8OBBKBQKDB8+XOn4trW1hZubm/jZvHz5MtLS0vDuu+8qndkrvvT3KsXrCwkJKbGNAHDkyJFXtqFuTIAqKTs7WynZeNGIESPQsWNHTJgwATY2Nhg5ciS+/vrrciVD9evXL1cnSjc3N6VpmUwGV1fXCveUL6t79+7B3t6+xPtRfEry3r17SuUODg4l2qhbt67KviMvrsfNzQ1aWsqHb2nrqay8vDwsWLBAvDZvaWkJKysrZGRkIDMzU6y3ZMkSZGRkoHHjxmjZsiVmz55dYpQg8M8PU9++fdGmTRt8/fXX5dq3BgYGJS7jleU9K2+bZmZmaNCgQYkfMTMzM5XrevGYMzExgZ2dnXjMFSeF3bt3h5WVldLr+PHjSEtLU1peR0cHDRo0KFP8nTt3RmFhIWJiYhAfH4+0tDR07twZXbp0UUqAmjVrBgsLC6Vly3IM3r59G5GRkSXi9vX1BYASsb94efDOnTsQBAHz588v0cbChQtVtqHKi+8xADRu3Fjpc/33339jxowZsLGxgaGhIaysrMR4/n2slhZreTx69AhZWVklLrmUlb29fYnOv40bNwYApW3asWMHWrVqJfars7KywpEjR1Ruz4v7szgZKsvnY+rUqWjcuDF69+6NBg0aYNy4cWXqk3jv3j3Y2dmVuL9XkyZNVNZXdWyX97uztGMhNzcXjx49emldFxcXaGlpKX02BUGAm5tbieMzLi5OPDaLY3ixPV1dXTg7O6vc1he3UUtLC66urkrltra2MDc3V/v3dlmwD1Al/PXXX8jMzCyxQ//N0NAQZ86cwalTp3DkyBFERkZi//796N69O44fPw5tbe1Xrqc8/XbKqrS/zoqKisoUkzqUth7hhU5/mvbee+9h27ZtCA4Oho+PD8zMzCCTyTBy5EilRLZLly64e/cuDh8+jOPHj+PLL7/Ep59+ioiICKUzLPr6+ujTpw8OHz6MyMjIct2Mryz75mX7tjxtqnP/FL9Pu3btgq2tbYn5L46c1NfXL5HglsbLywsGBgY4c+YMHBwcYG1tjcaNG6Nz5874/PPPkZ+fj59//lllP5uybKNCoUDPnj3xwQcfqKxb/KNd7MXPa/G2z5o1C3K5XGUbL/sOKY/hw4fj/PnzmD17Njw8PGBiYgKFQoFevXqp/KOrKr5b1Gn37t0ICAjAoEGDMHv2bFhbW0NbWxthYWG4e/duifqVOWatra0RGxuLY8eO4ejRozh69Ci2bdsGPz8/pQEWlVWeY7sqvPj9oFAoIJPJcPToUZXvn7pv3FqZGwWX9h1WUUyAKqG4E11pX2rFtLS00KNHD/To0QNr167FihUr8NFHH+HUqVPw9fVV+52jX7wEIwgC7ty5g1atWolldevWVTlK4d69e0rZfHlia9SoEU6ePImnT58q/SVz69Ytcb46NGrUCNevX4dCoVD6Iqnsekrb1gMHDsDf3x9r1qwRy549e6by/bOwsEBgYCACAwORnZ2NLl26YNGiRUoJkEwmw549ezBw4EAMGzYMR48eVetdVIv/6n0xvqr8C+v27dt48803xens7GwkJyejT58+AP7Xodza2lo8c6Iuenp64mgUBwcH8bJH586dkZ+fjz179iA1NbXC96FxcXFBdnZ2heMu/jzp6upWattVXVr9448/xM7jT548QVRUFBYvXowFCxa8dLmXKetn3srKCqampiVGopXVw4cPSwwB/+OPPwBA3KYDBw7A2dkZBw8eVIqr+MxZRbxs+/T09NC/f3/0798fCoUCU6dOxRdffIH58+eXmqQ2atQIUVFRyM7OVkoWynOPtPJ+d5Z2LBgZGZU4m3v79m2lM3137tyBQqEQ32MXFxcIggAnJ6cSyfyLMRa31717d7H8+fPnSEhIQOvWrV+5jQqFArdv31bqqJyamoqMjAylbVT1+1RQUIDk5GSlssr+dvISWAX99NNPWLp0KZycnEodogv8c0r6RcUjF4qH/hV/Aaj6Qa2I4tEixQ4cOIDk5GT07t1bLHNxccEvv/wiDlUFgB9++KHEsMfyxNanTx8UFRXhs88+Uyr/9NNPIZPJlNZfGX369EFKSgr2798vlhUWFmLDhg0wMTFB165dK9SusbGxyu3U1tYu8Rfkhg0bSvw18uJQYxMTE7i6upYY4gn880V78OBBtGvXDv3798fFixcrFLMqjRo1gra2dom+KZ9//rna1vGizZs34/nz5+L0pk2bUFhYKO5zuVwOU1NTrFixQqlesRdP25dX586dceHCBZw6dUpMgCwtLeHu7i6O6Pl3X5/yGD58OGJiYnDs2LES8zIyMlBYWPjS5a2trdGtWzd88cUXJb7AgbJv+6FDh5T6Cl28eBEXLlwQ3+Piv95fPFZfHKX2KmX9zGtpaWHQoEH473//q/JRKK8661JYWIgvvvhCnC4oKMAXX3wBKysrsX+Lqm26cOECYmJiyrQtqpS2fS9+frW0tMQ/GlV9hov16dMHhYWF2LRpk1hWVFSEDRs2lDmm8n53xsTEKPWBun//Pg4fPoy33nqrxFmc4mH/xYrjKm7z7bffhra2NhYvXlxinwmCIL4vXl5esLKyQkREhNLvxvbt28v8+wCUPB7Xrl0LAOjbt69Y5uLiUuL7a/PmzSW+cyv728kzQGVw9OhR3Lp1C4WFhUhNTcVPP/2EEydOoFGjRvj+++9fepOvJUuW4MyZM+jbty8aNWqEtLQ0fP7552jQoIHYIdPFxQXm5uaIiIhAnTp1YGxsDG9v7wpfn7ewsECnTp0QGBiI1NRUhIeHw9XVVWmo/oQJE3DgwAH06tULw4cPx927d7F7926lTsnlja1///5488038dFHHyExMRGtW7fG8ePHcfjwYQQHB5dou6ImTZqEL774AgEBAbhy5QocHR1x4MABnDt3DuHh4S/tk/Uynp6eOHnyJNauXQt7e3s4OTnB29sb/fr1w65du2BmZoZmzZohJiYGJ0+eLDFUv1mzZujWrRs8PT1hYWGBy5cv48CBAwgKClK5PkNDQ/zwww/o3r07evfujdOnT1e4P8W/mZmZYdiwYdiwYQNkMhlcXFzwww8/lKmfSUUVFBSgR48eGD58OOLj4/H555+jU6dO4tB0U1NTbNq0CWPHjkXbtm0xcuRIWFlZISkpCUeOHEHHjh1LfPmXR+fOnbF8+XLcv39fKdHp0qULvvjiCzg6Opa5T9GLZs+eje+//x79+vUTh8jn5OTgt99+w4EDB5CYmAhLS8uXtrFx40Z06tQJLVu2xMSJE+Hs7IzU1FTExMTgr7/+wq+//vrKOFxdXdGpUydMmTIF+fn5CA8PR7169cRLc6ampuKtDJ4/f4769evj+PHjSEhIKNf2Ficf06dPh1wuh7a2NkaOHKmy7ooVK3D8+HF07doVkyZNgru7O5KTk/HNN9/g7Nmz4iAQVezt7bFy5UokJiaicePG2L9/P2JjY7F582axQ32/fv1w8OBBDB48GH379kVCQgIiIiLQrFmzl94zqyLbN2HCBPz999/o3r07GjRogHv37mHDhg3w8PB46dDq/v37o2PHjpgzZw4SExPRrFkzHDx4UGUfpZe1UZ7vzhYtWkAulysNgweg8g7XCQkJGDBgAHr16oWYmBjs3r0b77zzjnjGxsXFBcuWLUNoaCgSExMxaNAg1KlTBwkJCfjuu+8wadIkzJo1C7q6uli2bBkmT56M7t27Y8SIEUhISMC2bdvK1AeodevW8Pf3x+bNm5GRkYGuXbvi4sWL2LFjBwYNGqR0BnnChAl49913MWTIEPTs2RO//vorjh07VuJz5uHhAW1tbaxcuRKZmZnQ19dH9+7dYW1tXbY3vkJjxySieIhv8UtPT0+wtbUVevbsKaxbt05puHWxF4fBR0VFCQMHDhTs7e0FPT09wd7eXhg1alSJIbWHDx8WmjVrJujo6CgNnezatavQvHlzlfGVNgz+q6++EkJDQwVra2vB0NBQ6Nu3r3Dv3r0Sy69Zs0aoX7++oK+vL3Ts2FG4fPlyiTZfFtuLw+AF4Z/hzu+//75gb28v6OrqCm5ubsInn3wiDoktBkCYNm1aiZhKG57/otTUVCEwMFCwtLQU9PT0hJYtW6ocql+eYfC3bt0SunTpIhgaGgoAxDiePHkirsvExESQy+XCrVu3SsS6bNkyoX379oK5ublgaGgoNG3aVFi+fLnS8PB/D4Mvlp6eLjRr1kywtbUVbt++XWp8qpYVhJLHnCAIwqNHj4QhQ4YIRkZGQt26dYXJkycLv//+u8phuaraLO24e/H9LP6MnD59Wpg0aZJQt25dwcTERBg9erTw+PHjEsufOnVKkMvlgpmZmWBgYCC4uLgIAQEBSkN6S4vpZbKysgRtbW2hTp06SsOYd+/eLQAQxo4d+8pt+fe2v/gZePr0qRAaGiq4uroKenp6gqWlpdChQwdh9erV4v4tHor9ySefqIzx7t27gp+fn2Brayvo6uoK9evXF/r16yccOHDgpdv273bXrFkjNGzYUNDX1xc6d+4sDmUu9tdffwmDBw8WzM3NBTMzM2HYsGHCw4cPSwwrLj5mVN2eo7CwUHjvvfcEKysrQSaTKR1bL7YjCP8Mwfbz8xOsrKwEfX19wdnZWZg2bZo4VLq0YfDNmzcXLl++LPj4+AgGBgZCo0aNhM8++0ypbYVCIaxYsUJo1KiRoK+vL7Rp00b44YcfSnz3vOy9fzHm0rbvwIEDwltvvSVYW1sLenp6goODgzB58mQhOTlZ5X75t8ePHwtjx44VTE1NBTMzM2Hs2LHCtWvXyvx5E4Tyf3fu3r1bcHNzE9+XF4eCF+/jmzdvCkOHDhXq1Kkj1K1bVwgKChLy8vJKrP/bb78VOnXqJBgbGwvGxsZC06ZNhWnTpgnx8fFK9T7//HPByclJ0NfXF7y8vIQzZ86o/Myo8vz5c2Hx4sWCk5OToKurKzRs2FAIDQ0tcRuQoqIi4cMPPxQsLS0FIyMjQS6XC3fu3FH5+7BlyxbB2dlZ0NbWLveQeJkgVLMep0REJEpMTISTkxM++eQTzJo1S9PhqEW3bt2Qnp5e4f5DUiaTyTBt2rRXnjFdtGgRFi9ejEePHr3yDKVUsQ8QERERSQ4TICIiIpIcJkBEREQkOewDRERERJJTLc4Abdy4EY6OjjAwMIC3t/dL74ly8OBBeHl5wdzcHMbGxvDw8CjxVN+AgADIZDKlV69evap6M4iIiKiG0Ph9gPbv34+QkBBERETA29sb4eHhkMvliI+PVzmW38LCAh999BGaNm0KPT09/PDDDwgMDIS1tbXSHZl79eqFbdu2idPledgkERER1W4avwTm7e2Ndu3aiUP6FAoFGjZsiPfeew9z5swpUxtt27ZF3759sXTpUgD/nAHKyMjAoUOHKhSTQqHAw4cPUadOHbU/poKIiIiqhiAIePr0Kezt7V/5zDWNngEqKCjAlStXEBoaKpZpaWnB19e3TLc6FwQBP/30E+Lj48Xb3ReLjo6GtbU16tati+7du2PZsmUl7txbmocPH6Jhw4bl2xgiIiKqFu7fv//Ku79rNAFKT09HUVERbGxslMptbGzEh8CpkpmZifr16yM/Px/a2tr4/PPP0bNnT3F+r1698Pbbb8PJyQl3797F3Llz0bt3b8TExKh82m1+fr7Ss16KT4rdv38fpqamld1MIiIieg2ysrLQsGHDMj0SSeN9gCqiTp06iI2NRXZ2NqKiohASEgJnZ2fxidr/fm5Ny5Yt0apVK7i4uCA6Oho9evQo0V5YWJjKZ6iYmpoyASIiIqphytJ9RaOjwCwtLaGtrY3U1FSl8tTUVNja2pa6nJaWFlxdXeHh4YGZM2di6NChCAsLK7W+s7MzLC0tcefOHZXzQ0NDkZmZKb5efCI6ERER1S4aTYD09PTg6emJqKgosUyhUCAqKgo+Pj5lbkehUChdwnrRX3/9hcePH8POzk7lfH19ffFsD8/6EBER1X4avwQWEhICf39/eHl5oX379ggPD0dOTg4CAwMBAH5+fqhfv754hicsLAxeXl5wcXFBfn4+fvzxR+zatQubNm0CAGRnZ2Px4sUYMmQIbG1tcffuXXzwwQdwdXVVGiZPRERE0qXxBGjEiBF49OgRFixYgJSUFHh4eCAyMlLsGJ2UlKQ0lC0nJwdTp07FX3/9BUNDQzRt2hS7d+/GiBEjAADa2tq4fv06duzYgYyMDNjb2+Ott97C0qVLeS8gIiIJKCoqwvPnzzUdBlUBXV1dlYOZKkLj9wGqjrKysmBmZobMzExeDiMiqiEEQUBKSgoyMjI0HQpVIXNzc9ja2qrs6Fye32+NnwEiIiJSh+Lkx9raGkZGRryRbS0jCAJyc3ORlpYGAKX26y0rJkBERFTjFRUViclPWW96SzWPoaEhACAtLQ3W1taVuhxWLR6GSkREVBnFfX6MjIw0HAlVteJ9XNl+XkyAiIio1uBlr9pPXfuYCRARERFJDhMgIiKiamLRokXw8PDQdBiSwASIiIiogrp164bg4GBNhwEA2L59O8zNzTUdRo3BBIiIiIgkhwkQERFRBQQEBOD06dNYt24dZDIZZDIZ7t69i/Hjx8PJyQmGhoZo0qQJ1q1bp7RcdHQ02rdvD2NjY5ibm6Njx464d++eynXcvXsXzs7OCAoKwsvuWxwdHY3AwEBkZmaKsSxatAhLlixBixYtStT38PDA/Pnzxe0YNGgQFi9eDCsrK5iamuLdd99FQUGBWF+hUCAsLEzcrtatW+PAgQMVeduqDd4HqBZISkpCenq6Wtu0tLSEg4ODWtskIqpN1q1bhz/++AMtWrTAkiVLAAB169ZFgwYN8M0336BevXo4f/48Jk2aBDs7OwwfPhyFhYUYNGgQJk6ciK+++goFBQW4ePGiypFN169fh1wux/jx47Fs2bKXxtKhQweEh4djwYIFiI+PBwCYmJggIyMDixcvxqVLl9CuXTsAwLVr13D9+nUcPHhQXD4qKgoGBgaIjo5GYmIiAgMDUa9ePSxfvhzAP8/h3L17NyIiIuDm5oYzZ85gzJgxsLKyQteuXdXyfr5uTIBquKSkJLg3bYLcvGdqbdfI0ABxt+KZBBERlcLMzAx6enowMjKCra2tWL548WLx/05OToiJicHXX3+N4cOHIysrC5mZmejXrx9cXFwAAO7u7iXaPn/+PPr164ePPvoIM2fOfGUsenp6MDMzg0wmU4rFxMQEcrkc27ZtExOgbdu2oWvXrnB2dlZafuvWrTAyMkLz5s2xZMkSzJ49G0uXLsXz58+xYsUKnDx5Ej4+PgAAZ2dnnD17Fl988QUTINKM9PR05OY9w+5JHnC3M1FLm3HJ2RizORbp6elMgIiIymnjxo3YunUrkpKSkJeXh4KCAnFkl4WFBQICAiCXy9GzZ0/4+vpi+PDhSo91SEpKQs+ePbF8+XK1dLCeOHEixo0bh7Vr10JLSwt79+7Fp59+qlSndevWSjeR9PHxQXZ2Nu7fv4/s7Gzk5uaiZ8+eSssUFBSgTZs2lY5PU5gA1RLudiZo62im6TCIiCRt3759mDVrFtasWQMfHx/UqVMHn3zyCS5cuCDW2bZtG6ZPn47IyEjs378f8+bNw4kTJ/DGG28AAKysrGBvb4+vvvoK48aNq/RDufv37w99fX1899130NPTw/PnzzF06NAyL5+dnQ0AOHLkCOrXr680T19fv1KxaRITICIiogrS09NDUVGROH3u3Dl06NABU6dOFcvu3r1bYrk2bdqgTZs2CA0NhY+PD/bu3SsmQIaGhvjhhx/Qp08fyOVyHD9+HHXq1Cl3LMV0dHTg7++Pbdu2QU9PDyNHjhSfqVXs119/RV5enlj+yy+/wMTEBA0bNoSFhQX09fWRlJRUYy93qcIEiIiIqIIcHR1x4cIFJCYmwsTEBG5ubti5cyeOHTsGJycn7Nq1C5cuXYKTkxMAICEhAZs3b8aAAQNgb2+P+Ph43L59G35+fkrtGhsb48iRI+jduzd69+6NyMhImJi8vJuDo6MjsrOzERUVJV7SKr6sNWHCBLGv0blz50osW1BQgPHjx2PevHlITEzEwoULERQUBC0tLdSpUwezZs3C+++/D4VCgU6dOiEzMxPnzp2Dqakp/P391fFWvnYcBk9ERFRBs2bNgra2Npo1awYrKyvI5XK8/fbbGDFiBLy9vfH48WOls0FGRka4desWhgwZgsaNG2PSpEmYNm0aJk+eXKJtExMTHD16FIIgoG/fvsjJyXlpLB06dMC7776LESNGwMrKCqtWrRLnubm5oUOHDmjatCm8vb1LLNujRw+4ubmhS5cuGDFiBAYMGIBFixaJ85cuXYr58+cjLCwM7u7u6NWrF44cOSImdjWRTHjZjQUkKisrC2ZmZsjMzKz0tdeqdvXqVXh6euLKwk5q6wN0NTETnovP4sqVK2jbtq1a2iQiqkrPnj1DQkICnJycYGBgoOlwqh1BEODm5oapU6ciJCREaV5AQAAyMjJw6NAhzQRXTi/b1+X5/eYlMCIiolrs0aNH2LdvH1JSUhAYGKjpcKoNXgIjIiKqAXr37g0TExOVrxUrVpS6nLW1NZYsWYLNmzejbt26rzHi6o1ngIiIiGqAL7/8Enl5eSrnWVhYlLrcq3q6bN++vTJh1VhMgIiIiGqAF+/BQ5XDS2BEREQkOUyAiIiISHKYABEREZHkMAEiIiIiyWECRERERJLDUWBERFRrJSUlIT09/bWtz9LSEg4ODq9tfS9ydHREcHAwgoODy1Q/Ojoab775Jp48eQJzc/Mqja26YQJERES1UlJSEpo2dUdeXu5rW6ehoRFu3YorVxLUrVs3eHh4IDw8vNLrv3TpEoyNjctcv0OHDkhOToaZmXoepVSTMAEiIqJaKT09HXl5ufAetxCmdo5Vvr6s5ERc2LoY6enpaj0LJAgCioqKoKPz6p9sKyurcrWtp6cHW1vbioZWo7EPEBER1Wqmdo6wcGhS5a+KJFkBAQE4ffo01q1bB5lMBplMhu3bt0Mmk+Ho0aPw9PSEvr4+zp49i7t372LgwIGwsbGBiYkJ2rVrh5MnTyq15+joqHQmSSaT4csvv8TgwYNhZGQENzc3fP/99+L86OhoyGQyZGRkAPjnrtDm5uY4duwY3N3dYWJigl69eiE5OVlcprCwENOnT4e5uTnq1auHDz/8EP7+/hg0aFC5t1+TmAARERFpyLp16+Dj44OJEyciOTkZycnJaNiwIQBgzpw5+PjjjxEXF4dWrVohOzsbffr0QVRUFK5du4ZevXqhf//+SEpKeuk6Fi9ejOHDh+P69evo06cPRo8ejb///rvU+rm5uVi9ejV27dqFM2fOICkpCbNmzRLnr1y5Env27MG2bdtw7tw5ZGVl1Zgnyf8bEyAiIiINMTMzg56eHoyMjGBrawtbW1toa2sDAJYsWYKePXvCxcUFFhYWaN26NSZPnowWLVrAzc0NS5cuhYuLi9IZHVUCAgIwatQouLq6YsWKFcjOzsbFixdLrf/8+XNERETAy8sLbdu2RVBQEKKiosT5GzZsQGhoKAYPHoymTZvis88+q5EdqJkAERERVUNeXl5K09nZ2Zg1axbc3d1hbm4OExMTxMXFvfIMUKtWrcT/Gxsbw9TUFGlpaaXWNzIygouLizhtZ2cn1s/MzERqairat28vztfW1oanp2e5tq06YCdoIiKiaujF0VyzZs3CiRMnsHr1ari6usLQ0BBDhw5FQUHBS9vR1dVVmpbJZFAoFOWq/6onytdEPANERESkQXp6eigqKnplvXPnziEgIACDBw9Gy5YtYWtri8TExKoP8F/MzMxgY2ODS5cuiWVFRUW4evXqa41DHXgGiIiISIMcHR1x4cIFJCYmwsTEpNSzM25ubjh48CD69+8PmUyG+fPnv/RMTlV57733EBYWBldXVzRt2hQbNmzAkydPIJPJXnsslcEEiIiIarWs5MRqvZ5Zs2bB398fzZo1Q15eHrZt26ay3tq1azFu3Dh06NABlpaW+PDDD5GVlVWJiCvmww8/REpKCvz8/KCtrY1JkyZBLpeLnbdrCplQGy/sVVJWVhbMzMyQmZkJU1NTTYfzUlevXoWnpyeuLOyEto7quZPn1cRMeC4+iytXrqBt27ZqaZOIqCo9e/YMCQkJcHJygoGBAYCacyfomk6hUMDd3R3Dhw/H0qVLq3x9qvZ1sfL8fvMMEBER1UoODg64dStOUs8Cex3u3buH48ePo2vXrsjPz8dnn32GhIQEvPPOO5oOrVyYABERUa3l4OBQ6xOS101LSwvbt2/HrFmzIAgCWrRogZMnT8Ld3V3ToZULEyAiIiIqs4YNG+LcuXOaDqPSOAyeiIiIJKdaJEAbN26Eo6MjDAwM4O3t/dJbdB88eBBeXl4wNzeHsbExPDw8sGvXLqU6giBgwYIFsLOzg6GhIXx9fXH79u2q3gwiIiKqITSeAO3fvx8hISFYuHAhrl69itatW0Mul5d6m24LCwt89NFHiImJwfXr1xEYGIjAwEAcO3ZMrLNq1SqsX78eERERuHDhAoyNjSGXy/Hs2bPXtVlERERUjWk8AVq7di0mTpyIwMBANGvWDBERETAyMsLWrVtV1u/WrRsGDx4Md3d3uLi4YMaMGWjVqhXOnj0L4J+zP+Hh4Zg3bx4GDhyIVq1aYefOnXj48GGNfFotERERqZ9GE6CCggJcuXIFvr6+YpmWlhZ8fX0RExPzyuUFQUBUVBTi4+PRpUsXAEBCQgJSUlKU2jQzM4O3t3epbebn5yMrK0vpRURERLWXRhOg9PR0FBUVwcbGRqncxsYGKSkppS6XmZkJExMT6OnpoW/fvtiwYQN69uwJAOJy5WkzLCwMZmZm4qthw4aV2SwiIiKq5mrkMPg6deogNjYW2dnZiIqKQkhICJydndGtW7cKtRcaGoqQkBBxOisri0kQEVEtkJSUJPkbIW7fvh3BwcHIyMio8nUFBAQgIyOjRnQ50WgCZGlpCW1tbaSmpiqVp6amwtbWttTltLS04OrqCgDw8PBAXFwcwsLC0K1bN3G51NRU2NnZKbXp4eGhsj19fX3o6+tXcmuIiKg6SUpKgnvTJsjNe30DYIwMDRB3K77aJUHqlpiYCCcnJ1y7dk3pt3XdunWoKU/Y0mgCpKenB09PT0RFRWHQoEEA/nmmSFRUFIKCgsrcjkKhQH5+PgDAyckJtra2iIqKEndKVlYWLly4gClTpqh7E4iIqJpKT09Hbt4z7J7kAXc7kypfX1xyNsZsjkV6enqtT4BKY2amnmdSvg4aHwUWEhKCLVu2YMeOHYiLi8OUKVOQk5ODwMBAAICfnx9CQ0PF+mFhYThx4gT+/PNPxMXFYc2aNdi1axfGjBkDAJDJZAgODsayZcvw/fff47fffoOfnx/s7e3FJIuIiKTD3c4EbR3NqvxV0SRLoVAgLCwMTk5OMDQ0ROvWrXHgwAEoFAo0aNAAmzZtUqp/7do1aGlp4d69ewD+GU3dsmVLGBsbo2HDhpg6dSqys7NLXV9AQECJ38Pg4GClbiSRkZHo1KkTzM3NUa9ePfTr1w93794V5zs5OQEA2rRpA5lMJi77Ytv5+fmYPn06rK2tYWBggE6dOuHSpUvi/OjoaMhkMkRFRcHLywtGRkbo0KED4uPjy/MWVojGE6ARI0Zg9erVWLBgATw8PBAbG4vIyEixE3NSUhKSk5PF+jk5OZg6dSqaN2+Ojh074ttvv8Xu3bsxYcIEsc4HH3yA9957D5MmTUK7du2QnZ2NyMjIEk+NJSIi0rSwsDDs3LkTERERuHHjBt5//32MGTMGP//8M0aNGoW9e/cq1d+zZw86duyIRo0aAfinW8j69etx48YN7NixAz/99BM++OCDSsWUk5ODkJAQXL58GVFRUdDS0sLgwYOhUCgAQLxh8cmTJ5GcnIyDBw+qbOeDDz7At99+ix07duDq1atwdXWFXC7H33//rVTvo48+wpo1a3D58mXo6Ohg3LhxlYq/LKpFJ+igoKBSL3lFR0crTS9btgzLli17aXsymQxLlizBkiVL1BUiERGR2uXn52PFihU4efIkfHx8AADOzs44e/YsvvjiC3zwwQdYs2YNkpKS4ODgAIVCgX379mHevHliG8HBweL/HR0dsWzZMrz77rv4/PPPKxzXkCFDlKa3bt0KKysr3Lx5Ey1atICVlRUAoF69eqX22c3JycGmTZuwfft29O7dGwCwZcsWnDhxAv/5z38we/Zsse7y5cvRtWtXAMCcOXPQt29fPHv2rEpPXGj8DBAREZFU3blzB7m5uejZsydMTEzE186dO3H37l14eHjA3d1dPAt0+vRppKWlYdiwYWIbJ0+eRI8ePVC/fn3UqVMHY8eOxePHj5Gbm1vhuG7fvo1Ro0bB2dkZpqamcHR0BPDPVZmyunv3Lp4/f46OHTuKZbq6umjfvj3i4uKU6rZq1Ur8f/EAptKeCKEu1eIMEBERkRQV99U5cuQI6tevrzSveHTy6NGjsXfvXsyZMwd79+5Fr169UK9ePQD/jMbq168fpkyZguXLl8PCwgJnz57F+PHjUVBQACMjoxLr1NLSKjFS6/nz50rT/fv3R6NGjbBlyxbY29tDoVCgRYsWKCgoUNu2/5uurq74f5lMBgDi5baqwjNAREREGtKsWTPo6+sjKSkJrq6uSq/i+9G98847+P3333HlyhUcOHAAo0ePFpe/cuUKFAoF1qxZgzfeeAONGzfGw4cPX7pOKysrpb61ABAbGyv+//Hjx4iPj8e8efPQo0cPuLu748mTJ0r19fT0AABFRUWlrsfFxQV6eno4d+6cWPb8+XNcunQJzZo1e/kb8xrwDBAREZGG1KlTB7NmzcL7778PhUKBTp06ITMzE+fOnYOpqSn8/f3h6OiIDh06YPz48SgqKsKAAQPE5V1dXfH8+XNs2LAB/fv3x7lz5xAREfHSdXbv3h2ffPIJdu7cCR8fH+zevRu///472rRpAwCoW7cu6tWrh82bN8POzg5JSUmYM2eOUhvW1tYwNDREZGQkGjRoAAMDgxJD4I2NjTFlyhTMnj0bFhYWcHBwwKpVq5Cbm4vx48er6R2sOCZARERUq8Ullz4kvDqsZ+nSpbCyskJYWBj+/PNPmJubo23btpg7d65YZ/To0Zg6dSr8/PxgaGgolrdu3Rpr167FypUrERoaii5duiAsLAx+fn6lrk8ul2P+/Pn44IMP8OzZM4wbNw5+fn747bffAPxziWzfvn2YPn06WrRogSZNmmD9+vVKw+R1dHSwfv16LFmyBAsWLEDnzp1LDFoCgI8//hgKhQJjx47F06dP4eXlhWPHjqFu3boVeq/USSbUlFs2vkZZWVkwMzNDZmYmTE1NNR3OS129ehWenp64srAT2jqq5wZUVxMz4bn4LK5cuYK2bduqpU0ioqr07NkzJCQkwMnJSRw5xDtB106q9nWx8vx+8wyQBqjz2TQv9qQnIqJ/ODg4IO5WvOSfBUaqMQF6zZKSktC0qTvy8io+PLEkGZIznwGoObcgJyJ6HRwcHJiQkEpMgF6z9PR05OXlwnvcQpjaOVa6vUcJtxC7dyUycgsrHxwREZFEMAHSEFM7R1g4NKl0O3k56jyTREREJA28DxAREdUaHNdT+6lrHzMBIiKiGq/4TsKVefwD1QzF+/jfd4+uCF4CIyKiGk9bWxvm5ubi86OMjIzERypQ7SAIAnJzc5GWlgZzc3Noa2tXqj0mQEREVCsUP5W8qh+iSZplbm5e6hPoy4MJEBER1QoymQx2dnawtrYu8XBPqh10dXUrfeanGBMgIiKqVbS1tdX2I0m1FxMgohpEnXcRL8Y71xKRFDEBIqohquq5Rnx2ERFJERMgohoiPT0duXnPsHuSB9ztTNTSZlxyNsZsjkV6ejoTICKSFCZARDWMu50J2jryuW9ERJXBGyESERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSnGqRAG3cuBGOjo4wMDCAt7c3Ll68WGrdLVu2oHPnzqhbty7q1q0LX1/fEvUDAgIgk8mUXr169arqzSAiIqIaQuMJ0P79+xESEoKFCxfi6tWraN26NeRyOdLS0lTWj46OxqhRo3Dq1CnExMSgYcOGeOutt/DgwQOler169UJycrL4+uqrr17H5hAREVENoPEEaO3atZg4cSICAwPRrFkzREREwMjICFu3blVZf8+ePZg6dSo8PDzQtGlTfPnll1AoFIiKilKqp6+vD1tbW/FVt27d17E5REREVANoNAEqKCjAlStX4OvrK5ZpaWnB19cXMTExZWojNzcXz58/h4WFhVJ5dHQ0rK2t0aRJE0yZMgWPHz8utY38/HxkZWUpvYiIiKj20mgClJ6ejqKiItjY2CiV29jYICUlpUxtfPjhh7C3t1dKonr16oWdO3ciKioKK1euxOnTp9G7d28UFRWpbCMsLAxmZmbiq2HDhhXfKCIiIqr2dDQdQGV8/PHH2LdvH6Kjo2FgYCCWjxw5Uvx/y5Yt0apVK7i4uCA6Oho9evQo0U5oaChCQkLE6aysLCZBREREtZhGzwBZWlpCW1sbqampSuWpqamwtbV96bKrV6/Gxx9/jOPHj6NVq1Yvrevs7AxLS0vcuXNH5Xx9fX2YmpoqvYiIiKj20mgCpKenB09PT6UOzMUdmn18fEpdbtWqVVi6dCkiIyPh5eX1yvX89ddfePz4Mezs7NQSNxEREdVsGh8FFhISgi1btmDHjh2Ii4vDlClTkJOTg8DAQACAn58fQkNDxforV67E/PnzsXXrVjg6OiIlJQUpKSnIzs4GAGRnZ2P27Nn45ZdfkJiYiKioKAwcOBCurq6Qy+Ua2UYiIiKqXjTeB2jEiBF49OgRFixYgJSUFHh4eCAyMlLsGJ2UlAQtrf/laZs2bUJBQQGGDh2q1M7ChQuxaNEiaGtr4/r169ixYwcyMjJgb2+Pt956C0uXLoW+vv5r3TYiIiKqnjSeAAFAUFAQgoKCVM6Ljo5Wmk5MTHxpW4aGhjh27JiaIiMiIqLaSOOXwIiIiIheNyZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIsnR0XQAJA1JSUlIT09Xa5uWlpZwcHBQa5tERCQNTICoyiUlJcG9aRPk5j1Ta7tGhgaIuxXPJIiIiMqtQglQUVERtm/fjqioKKSlpUGhUCjN/+mnn9QSHNUO6enpyM17ht2TPOBuZ6KWNuOSszFmcyzS09OZABERUblVKAGaMWMGtm/fjr59+6JFixaQyWTqjotqIXc7E7R1NNN0GERERBVLgPbt24evv/4affr0UUsQGzduxCeffIKUlBS0bt0aGzZsQPv27VXW3bJlC3bu3Inff/8dAODp6YkVK1Yo1RcEAQsXLsSWLVuQkZGBjh07YtOmTXBzc1NLvERERFSzVWgUmJ6eHlxdXdUSwP79+xESEoKFCxfi6tWraN26NeRyOdLS0lTWj46OxqhRo3Dq1CnExMSgYcOGeOutt/DgwQOxzqpVq7B+/XpERETgwoULMDY2hlwux7Nn6u2DQkRERDVThRKgmTNnYt26dRAEodIBrF27FhMnTkRgYCCaNWuGiIgIGBkZYevWrSrr79mzB1OnToWHhweaNm2KL7/8EgqFAlFRUQD+OfsTHh6OefPmYeDAgWjVqhV27tyJhw8f4tChQ5WOl4iIiGq+Cl0CO3v2LE6dOoWjR4+iefPm0NXVVZp/8ODBMrVTUFCAK1euIDQ0VCzT0tKCr68vYmJiytRGbm4unj9/DgsLCwBAQkICUlJS4OvrK9YxMzODt7c3YmJiMHLkyBJt5OfnIz8/X5zOysoq07qJiIioZqpQAmRubo7BgwdXeuXp6ekoKiqCjY2NUrmNjQ1u3bpVpjY+/PBD2NvbiwlPSkqK2MaLbRbPe1FYWBgWL15c3vCJiIiohqpQArRt2zZ1x1EhH3/8Mfbt24fo6GgYGBhUuJ3Q0FCEhISI01lZWWjYsKE6QiQiIqJqqFI3Qnz06BHi4+MBAE2aNIGVlVW5lre0tIS2tjZSU1OVylNTU2Fra/vSZVevXo2PP/4YJ0+eRKtWrcTy4uVSU1NhZ2en1KaHh4fKtvT19aGvr1+u2ImIiKjmqlAn6JycHIwbNw52dnbo0qULunTpAnt7e4wfPx65ubllbkdPTw+enp5iB2YAYodmHx+fUpdbtWoVli5disjISHh5eSnNc3Jygq2trVKbWVlZuHDhwkvbJCIiIumoUAIUEhKC06dP47///S8yMjKQkZGBw4cP4/Tp05g5c2a529qyZQt27NiBuLg4TJkyBTk5OQgMDAQA+Pn5KXWSXrlyJebPn4+tW7fC0dERKSkpSElJQXZ2NgBAJpMhODgYy5Ytw/fff4/ffvsNfn5+sLe3x6BBgyqyuURERFTLVOgS2LfffosDBw6gW7duYlmfPn1gaGiI4cOHY9OmTWVua8SIEXj06BEWLFiAlJQUeHh4IDIyUuzEnJSUBC2t/+VpmzZtQkFBAYYOHarUzsKFC7Fo0SIAwAcffICcnBxMmjQJGRkZ6NSpEyIjIyvVT4iIiIhqjwolQLm5uSVGWQGAtbV1uS6BFQsKCkJQUJDKedHR0UrTiYmJr2xPJpNhyZIlWLJkSbljISIiotqvQpfAfHx8sHDhQqU7K+fl5WHx4sXsZ0NERETVXoXOAK1btw5yuRwNGjRA69atAQC//vorDAwMcOzYMbUGSERERKRuFUqAWrRogdu3b2PPnj3iDQtHjRqF0aNHw9DQUK0BEhEREalbhe8DZGRkhIkTJ6ozFiIiIqLXoswJ0Pfff4/evXtDV1cX33///UvrDhgwoNKBEREREVWVMidAgwYNQkpKCqytrV96Px2ZTIaioiJ1xEZERERUJcqcACkUCpX/JyIiIqppKjQMfufOncjPzy9RXlBQgJ07d1Y6KCIiIqKqVKEEKDAwEJmZmSXKnz59Kj7CgoiIiKi6qlACJAgCZDJZifK//voLZmZmlQ6KiIiIqCqVaxh8mzZtIJPJIJPJ0KNHD+jo/G/xoqIiJCQkoFevXmoPkoiIiEidypUAFY/+io2NhVwuh4mJiThPT08Pjo6OGDJkiFoDJHpdkpKSkJ6ertY2LS0t4eDgoNY2iYio8sqVAC1cuBBFRUVwdHTEW2+9BTs7u6qKi+i1SkpKgnvTJsjNe/bqyuVgZGiAuFvxTIKIagn+oVR7lPtO0Nra2pg8eTLi4uKqIh4ijUhPT0du3jPsnuQBdzuTVy9QBnHJ2RizORbp6en8ciOqBZKSktC0qTvy8nLV2q6hoRFu3Yrj98RrVuFngf35559wcnJSdzxEGuVuZ4K2juzIT0QlpaenIy8vF97jFsLUzlEtbWYlJ+LC1sX8Q0kDKpQALVu2DLNmzcLSpUvh6ekJY2NjpfmmpqZqCY6IiKi6MbVzhIVDE02HQZVUoQSoT58+AP555te/h8MXD4/nozCIiIioOqtQAnTq1Cl1x0FERET02lQoAeratau64yAiIiJ6bSqUAAFARkYG/vOf/4ijwZo3b45x48bxTtBERERU7VXoURiXL1+Gi4sLPv30U/z999/4+++/sXbtWri4uODq1avqjpGIiIhIrSp0Buj999/HgAEDsGXLFvFxGIWFhZgwYQKCg4Nx5swZtQZJREREpE4VSoAuX76slPwAgI6ODj744AN4eXmpLTgiIiKiqlChS2CmpqZISkoqUX7//n3UqVOn0kERERERVaUKJUAjRozA+PHjsX//fty/fx/379/Hvn37MGHCBIwaNUrdMRIRERGpVYUuga1evRoymQx+fn4oLCwEAOjq6mLKlCn4+OOP1RogERERkbpVKAHS09PDunXrEBYWhrt37wIAXFxcYGRkpNbgiIiIiKpChe8DBABGRkYwNzcX/09ERERUE1SoD1BhYSHmz58PMzMzODo6wtHREWZmZpg3bx6eP3+u7hiJiIiI1KpCZ4Dee+89HDx4EKtWrYKPjw8AICYmBosWLcLjx4+xadMmtQZJREREpE4VSoD27t2Lffv2oXfv3mJZq1at0LBhQ4waNYoJEBEREVVrFboEpq+vD0dHxxLlTk5O0NPTq2xMRERERFWqQglQUFAQli5divz8fLEsPz8fy5cvR1BQkNqCIyIiIqoKFboEdu3aNURFRaFBgwZo3bo1AODXX39FQUEBevTogbfffluse/DgQfVESkRERKQmFUqAzM3NMWTIEKWyhg0bqiUgIiIioqpWoQRo27Zt6o6DqELi4uKqVTtERFQzVOpGiI8ePUJ8fDwAoEmTJrCyslJLUESvkpz5DIAMY8aMUWOrsv9v10yNbRIRUXVUoQQoJycH7733Hnbu3AmFQgEA0NbWhp+fHzZs2MC7QlOVy8gtBCDA450PYeXUtNLtPUq4hdi9K/+/XSIiqu0qlACFhITg9OnT+O9//4uOHTsCAM6ePYvp06dj5syZvA8QvTYm1g6wcGhS6XbycnLVEA0REdUUFUqAvv32Wxw4cADdunUTy/r06QNDQ0MMHz6cCRARERFVaxW6D1Bubi5sbGxKlFtbWyM3l39JExERUfVWoQTIx8cHCxcuxLNnz8SyvLw8LF68WHw2GBEREVF1VaEEKDw8HOfOnUODBg3Qo0cP9OjRAw0bNsT58+exbt26crW1ceNGODo6wsDAAN7e3rh48WKpdW/cuIEhQ4bA0dERMpkM4eHhJeosWrQIMplM6dW0aeU7yRIREVHtUaE+QC1btsTt27exZ88e3Lp1CwAwatQojB49GoaGhmVuZ//+/QgJCUFERAS8vb0RHh4OuVyO+Ph4WFtbl6ifm5sLZ2dnDBs2DO+//36p7TZv3hwnT54Up3V0KjXan4iIiGqZcmcGz58/R9OmTfHDDz9g4sSJlVr52rVrMXHiRAQGBgIAIiIicOTIEWzduhVz5swpUb9du3Zo164dAKicX0xHRwe2traVio2IiIhqr3JfAtPV1VXq+1NRBQUFuHLlCnx9ff8XjJYWfH19ERMTU6m2b9++DXt7ezg7O2P06NFISkp6af38/HxkZWUpvYiIiKj2qlAfoGnTpmHlypUoLKz4TePS09NRVFRUYjSZjY0NUlJSKtyut7c3tm/fjsjISGzatAkJCQno3Lkznj59WuoyYWFhMDMzE198rhkREVHtVqHOMZcuXUJUVBSOHz+Oli1bwtjYWGm+Jp8A37t3b/H/rVq1gre3Nxo1aoSvv/4a48ePV7lMaGgoQkJCxOmsrCwmQURERLWY2p4GX16WlpbQ1tZGamqqUnlqaqpa+++Ym5ujcePGuHPnTql19PX1oa+vr7Z1EhERUfVWrgRIoVDgk08+wR9//IGCggJ0794dixYtKtfIr2J6enrw9PREVFQUBg0aJLYfFRWFoKCgcrdXmuzsbNy9exdjx45VW5tERERUs5WrD9Dy5csxd+5cmJiYoH79+li/fj2mTZtW4ZWHhIRgy5Yt2LFjB+Li4jBlyhTk5OSIo8L8/PwQGhoq1i8oKEBsbCxiY2NRUFCABw8eIDY2VunszqxZs3D69GkkJibi/PnzGDx4MLS1tTFq1KgKx0lERES1S7nOAO3cuROff/45Jk+eDAA4efIk+vbtiy+//BJaWuXvTz1ixAg8evQICxYsQEpKCjw8PBAZGSl2jE5KSlJq9+HDh2jTpo04vXr1aqxevRpdu3ZFdHQ0AOCvv/7CqFGj8PjxY1hZWaFTp0745ZdfYGVlVe74iIiIqHYqVwKUlJSEPn36iNO+vr6QyWR4+PAhGjRoUKEAgoKCSr3kVZzUFHN0dIQgCC9tb9++fRWKg4iIiKSjXKdtCgsLYWBgoFSmq6uL58+fqzUoIiIioqpUrjNAgiAgICBAacTUs2fP8O677yoNhdfkMHgiIiKiVylXAuTv71+ibMyYMWoLhoiIiOh1KFcCtG3btqqKg4iIiOi1qdCjMIiIiIhqMiZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhyyvUsMJKWuLi4atUOERGRujABohKSM58BkGHMmDFqbFX2/+2aqbFNIiKiimECRCVk5BYCEODxzoewcmpa6fYeJdxC7N6V/98uERGR5jEBolKZWDvAwqFJpdvJy8lVQzRERETqw07QREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJEfjCdDGjRvh6OgIAwMDeHt74+LFi6XWvXHjBoYMGQJHR0fIZDKEh4dXuk0iIiKSHo0mQPv370dISAgWLlyIq1evonXr1pDL5UhLS1NZPzc3F87Ozvj4449ha2urljaJiIhIejSaAK1duxYTJ05EYGAgmjVrhoiICBgZGWHr1q0q67dr1w6ffPIJRo4cCX19fbW0SURERNKjsQSooKAAV65cga+v7/+C0dKCr68vYmJiXmub+fn5yMrKUnoRERFR7aWxBCg9PR1FRUWwsbFRKrexsUFKSsprbTMsLAxmZmbiq2HDhhVaPxEREdUMGu8EXR2EhoYiMzNTfN2/f1/TIREREVEV0tHUii0tLaGtrY3U1FSl8tTU1FI7OFdVm/r6+qX2KSIiIqLaR2NngPT09ODp6YmoqCixTKFQICoqCj4+PtWmTSIiIqp9NHYGCABCQkLg7+8PLy8vtG/fHuHh4cjJyUFgYCAAwM/PD/Xr10dYWBiAfzo537x5U/z/gwcPEBsbCxMTE7i6upapTSIiIiKNJkAjRozAo0ePsGDBAqSkpMDDwwORkZFiJ+akpCRoaf3vJNXDhw/Rpk0bcXr16tVYvXo1unbtiujo6DK1SURERKTRBAgAgoKCEBQUpHJecVJTzNHREYIgVKpNIiIiIo4CIyIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5Gj8RohEtV1cXFy1aoeIiJgAEVWZ5MxnAGQYM2aMGluV/X+7Zmpsk4hIepgAEVWRjNxCAAI83vkQVk5NK93eo4RbiN278v/bJSKiymACRFTFTKwdYOHQpNLt5OXkqiEaIiIC2AmaiIiIJIgJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5OhoOgAA2LhxIz755BOkpKSgdevW2LBhA9q3b19q/W+++Qbz589HYmIi3NzcsHLlSvTp00ecHxAQgB07digtI5fLERkZWWXbQERU3SUlJSE9PV1t7VlaWsLBwUFt7ZF6qHs/A7VzX2s8Adq/fz9CQkIQEREBb29vhIeHQy6XIz4+HtbW1iXqnz9/HqNGjUJYWBj69euHvXv3YtCgQbh69SpatGgh1uvVqxe2bdsmTuvr67+W7SEiqo6SkpLQtKk78vJy1damoaERbt2Kq3U/jDVZVexnoHbua40nQGvXrsXEiRMRGBgIAIiIiMCRI0ewdetWzJkzp0T9devWoVevXpg9ezYAYOnSpThx4gQ+++wzREREiPX09fVha2v7ejaCiKiaS09PR15eLrzHLYSpnWOl28tKTsSFrYuRnp5eq34Uazp172eg9u5rjSZABQUFuHLlCkJDQ8UyLS0t+Pr6IiYmRuUyMTExCAkJUSqTy+U4dOiQUll0dDSsra1Rt25ddO/eHcuWLUO9evVUtpmfn4/8/HxxOisrq4JbRERUvZnaOcLCoYmmw6Aqxv38ahrtBJ2eno6ioiLY2NgoldvY2CAlJUXlMikpKa+s36tXL+zcuRNRUVFYuXIlTp8+jd69e6OoqEhlm2FhYTAzMxNfDRs2rOSWERERUXWm8UtgVWHkyJHi/1u2bIlWrVrBxcUF0dHR6NGjR4n6oaGhSmeVsrKymAQRERHVYho9A2RpaQltbW2kpqYqlaemppbaf8fW1rZc9QHA2dkZlpaWuHPnjsr5+vr6MDU1VXoRERFR7aXRBEhPTw+enp6IiooSyxQKBaKiouDj46NyGR8fH6X6AHDixIlS6wPAX3/9hcePH8POzk49gRMREVGNpvEbIYaEhGDLli3YsWMH4uLiMGXKFOTk5Iijwvz8/JQ6Sc+YMQORkZFYs2YNbt26hUWLFuHy5csICgoCAGRnZ2P27Nn45ZdfkJiYiKioKAwcOBCurq6Qy+Ua2UYiIiKqXjTeB2jEiBF49OgRFixYgJSUFHh4eCAyMlLs6JyUlAQtrf/laR06dMDevXsxb948zJ07F25ubjh06JB4DyBtbW1cv34dO3bsQEZGBuzt7fHWW29h6dKlvBcQERERAagGCRAABAUFiWdwXhQdHV2ibNiwYRg2bJjK+oaGhjh27Jg6wyMiIqJaRuOXwIiIiIheNyZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhymAARERGR5DABIiIiIslhAkRERESSo6PpAIhI8+Li4tTWlqWlJRwcHNTWHklLUlIS0tPT1dYej0cqDRMgIglLznwGQIYxY8aorU1DQyPcuhXHHx0qt6SkJDRt6o68vFy1tcnjkUrDBIhIwjJyCwEI8HjnQ1g5Na10e1nJibiwdTHS09P5g0Pllp6ejry8XHiPWwhTO8dKt8fjkV6GCRARwcTaARYOTTQdBhEAwNTOkccjVTl2giYiIiLJYQJEREREksMEiIiIiCSHCRARERFJDhMgIiIikhwmQERERCQ5TICIiIhIcngfICIiNeAjHIhqFiZARESVxEc4ENU8TICIiCqJj3AgqnmYABERqQkf4UBUc7ATNBEREUkOEyAiIiKSHCZAREREJDlMgIiIiEhyqkUCtHHjRjg6OsLAwADe3t64ePHiS+t/8803aNq0KQwMDNCyZUv8+OOPSvMFQcCCBQtgZ2cHQ0ND+Pr64vbt21W5CURERFSDaDwB2r9/P0JCQrBw4UJcvXoVrVu3hlwuR1pamsr658+fx6hRozB+/Hhcu3YNgwYNwqBBg/D777+LdVatWoX169cjIiICFy5cgLGxMeRyOZ49e/a6NouIiIiqMY0nQGvXrsXEiRMRGBiIZs2aISIiAkZGRti6davK+uvWrUOvXr0we/ZsuLu7Y+nSpWjbti0+++wzAP+c/QkPD8e8efMwcOBAtGrVCjt37sTDhw9x6NCh17hlREREVF1pNAEqKCjAlStX4OvrK5ZpaWnB19cXMTExKpeJiYlRqg8AcrlcrJ+QkICUlBSlOmZmZvD29i61TSIiIpIWjd4IMT09HUVFRbCxsVEqt7Gxwa1bt1Quk5KSorJ+SkqKOL+4rLQ6L8rPz0d+fr44nZmZCQDIysoqx9aUTXZ2NgDg73vxKMzPq3R7mQ/+6dsUl/wUZ+IfV7q94raK29bVkVW6PcYonRizUpIAAFeuXBGPdXXQ0tKCQqGotu3Fx8cDUN/nuireR8ZY+RjVHR8gzRgBwNbWFra2tmpp69+Kf7cFQXh1ZUGDHjx4IAAQzp8/r1Q+e/ZsoX379iqX0dXVFfbu3atUtnHjRsHa2loQBEE4d+6cAEB4+PChUp1hw4YJw4cPV9nmwoULBQB88cUXX3zxxVcteN2/f/+VOYhGzwBZWlpCW1sbqampSuWpqamlZoa2trYvrV/8b2pqKuzs7JTqeHh4qGwzNDQUISEh4rRCocDff/+NevXqQSar/F/FtVFWVhYaNmyI+/fvw9TUVNPhSB73R/XC/VG9cH9UL1W5PwRBwNOnT2Fvb//KuhpNgPT09ODp6YmoqCgMGjQIwD/JR1RUFIKCglQu4+Pjg6ioKAQHB4tlJ06cgI+PDwDAyckJtra2iIqKEhOerKwsXLhwAVOmTFHZpr6+PvT19ZXKzM3NK7VtUmFqasovlGqE+6N64f6oXrg/qpeq2h9mZmZlqqfxh6GGhITA398fXl5eaN++PcLDw5GTk4PAwEAAgJ+fH+rXr4+wsDAAwIwZM9C1a1esWbMGffv2xb59+3D58mVs3rwZACCTyRAcHIxly5bBzc0NTk5OmD9/Puzt7cUki4iIiKRN4wnQiBEj8OjRIyxYsAApKSnw8PBAZGSk2Ik5KSkJWlr/G6zWoUMH7N27F/PmzcPcuXPh5uaGQ4cOoUWLFmKdDz74ADk5OZg0aRIyMjLQqVMnREZGwsDA4LVvHxEREVU/MkEoS1dpImX5+fkICwtDaGhoicuH9Ppxf1Qv3B/VC/dH9VJd9gcTICIiIpIcjd8JmoiIiOh1YwJEREREksMEiIiIiCSHCRARERFJDhMgKrOwsDC0a9cOderUgbW1NQYNGiQ+d4Y07+OPPxbvg0Wa8+DBA4wZMwb16tWDoaEhWrZsicuXL2s6LEkqKirC/Pnz4eTkBENDQ7i4uGDp0qVle04UVdqZM2fQv39/2NvbQyaT4dChQ0rzBUHAggULYGdnB0NDQ/j6+uL27duvLT4mQFRmp0+fxrRp0/DLL7/gxIkTeP78Od566y3k5ORoOjTJu3TpEr744gu0atVK06FI2pMnT9CxY0fo6uri6NGjuHnzJtasWYO6detqOjRJWrlyJTZt2oTPPvsMcXFxWLlyJVatWoUNGzZoOjRJyMnJQevWrbFx40aV81etWoX169cjIiICFy5cgLGxMeRyOZ49e/Za4uMweKqwR48ewdraGqdPn0aXLl00HY5kZWdno23btvj888+xbNkyeHh4IDw8XNNhSdKcOXNw7tw5/Pzzz5oOhQD069cPNjY2+M9//iOWDRkyBIaGhti9e7cGI5MemUyG7777TnwigyAIsLe3x8yZMzFr1iwAQGZmJmxsbLB9+3aMHDmyymPiGSCqsMzMTACAhYWFhiORtmnTpqFv377w9fXVdCiS9/3338PLywvDhg2DtbU12rRpgy1btmg6LMnq0KEDoqKi8McffwAAfv31V5w9exa9e/fWcGSUkJCAlJQUpe8tMzMzeHt7IyYm5rXEoPFHYVDNpFAoEBwcjI4dOyo9hoRer3379uHq1au4dOmSpkMhAH/++Sc2bdqEkJAQzJ07F5cuXcL06dOhp6cHf39/TYcnOXPmzEFWVhaaNm0KbW1tFBUVYfny5Rg9erSmQ5O8lJQUABAfe1XMxsZGnFfVmABRhUybNg2///47zp49q+lQJOv+/fuYMWMGTpw4wefcVRMKhQJeXl5YsWIFAKBNmzb4/fffERERwQRIA77++mvs2bMHe/fuRfPmzREbG4vg4GDY29tzfxAvgVH5BQUF4YcffsCpU6fQoEEDTYcjWVeuXEFaWhratm0LHR0d6Ojo4PTp01i/fj10dHRQVFSk6RAlx87ODs2aNVMqc3d3R1JSkoYikrbZs2djzpw5GDlyJFq2bImxY8fi/fffR1hYmKZDkzxbW1sAQGpqqlJ5amqqOK+qMQGiMhMEAUFBQfjuu+/w008/wcnJSdMhSVqPHj3w22+/ITY2Vnx5eXlh9OjRiI2Nhba2tqZDlJyOHTuWuDXEH3/8gUaNGmkoImnLzc2Flpbyz5y2tjYUCoWGIqJiTk5OsLW1RVRUlFiWlZWFCxcuwMfH57XEwEtgVGbTpk3D3r17cfjwYdSpU0e8TmtmZgZDQ0MNRyc9derUKdH/ytjYGPXq1WO/LA15//330aFDB6xYsQLDhw/HxYsXsXnzZmzevFnToUlS//79sXz5cjg4OKB58+a4du0a1q5di3Hjxmk6NEnIzs7GnTt3xOmEhATExsbCwsICDg4OCA4OxrJly+Dm5gYnJyfMnz8f9vb24kixKicQlREAla9t27ZpOjT6f127dhVmzJih6TAk7b///a/QokULQV9fX2jatKmwefNmTYckWVlZWcKMGTMEBwcHwcDAQHB2dhY++ugjIT8/X9OhScKpU6dU/mb4+/sLgiAICoVCmD9/vmBjYyPo6+sLPXr0EOLj419bfLwPEBEREUkO+wARERGR5DABIiIiIslhAkRERESSwwSIiIiIJIcJEBEREUkOEyAiIiKSHCZAREREJDlMgIio2lm0aBE8PDw0HUa5bd++Hebm5poOg4jKgAkQEVVat27dEBwcrOkwiIjKjAkQEVE19/z5c02HQFTrMAEiokoJCAjA6dOnsW7dOshkMshkMty9exfjx4+Hk5MTDA0N0aRJE6xbt05puejoaLRv3x7GxsYwNzdHx44dce/ePZXruHv3LpydnREUFIRXPb2n+DLUsWPH4O7uDhMTE/Tq1QvJycliHVVnrAYNGoSAgABx2tHREcuWLYOfnx9MTEzQqFEjfP/993j06BEGDhwIExMTtGrVCpcvXy4Rw6FDh+Dm5gYDAwPI5XLcv39faf7hw4fRtm1bGBgYwNnZGYsXL0ZhYaE4XyaTYdOmTRgwYACMjY2xfPnyl24zEZUfEyAiqpR169bBx8cHEydORHJyMpKTk9GgQQM0aNAA33zzDW7evIkFCxZg7ty5+PrrrwEAhYWFGDRoELp27Yrr168jJiYGkyZNgkwmK9H+9evX0alTJ7zzzjv47LPPVNZ5UW5uLlavXo1du3bhzJkzSEpKwqxZs8q9bZ9++ik6duyIa9euoW/fvhg7diz8/PwwZswYXL16FS4uLvDz81NKynJzc7F8+XLs3LkT586dQ0ZGBkaOHCnO//nnn+Hn54cZM2bg5s2b+OKLL7B9+/YSSc6iRYswePBg/Pbbb3x6OVFVeG2PXSWiWqssT6GfNm2aMGTIEEEQBOHx48cCACE6Olpl3YULFwqtW7cWzp07J9StW1dYvXp1mWPZtm2bAEC4c+eOWLZx40bBxsbmpfEOHDhQfEq1IAhCo0aNhDFjxojTycnJAgBh/vz5YllMTIwAQEhOTlZa9y+//CLWiYuLEwAIFy5cEARBEHr06CGsWLFCad27du0S7OzsxGkAQnBwcJm3mYjKT0eDuRcR1WIbN27E1q1bkZSUhLy8PBQUFIgjuywsLBAQEAC5XI6ePXvC19cXw4cPh52dnbh8UlISevbsieXLl5e7g7WRkRFcXFzEaTs7O6SlpZV7G1q1aiX+38bGBgDQsmXLEmVpaWmwtbUFAOjo6KBdu3ZinaZNm8Lc3BxxcXFo3749fv31V5w7d07pjE9RURGePXuG3NxcGBkZAQC8vLzKHS8RlR0vgRGR2u3btw+zZs3C+PHjcfz4ccTGxiIwMBAFBQVinW3btiEmJgYdOnTA/v370bhxY/zyyy/ifCsrK7Rv3x5fffUVsrKyyrV+XV1dpWmZTKZ0mUpLS6tEXyJVHY3/3U7xpTdVZQqFosyxZWdnY/HixYiNjRVfv/32G27fvg0DAwOxnrGxcZnbJKLyYwJERJWmp6eHoqIicfrcuXPo0KEDpk6dijZt2sDV1RV3794tsVybNm0QGhqK8+fPo0WLFti7d684z9DQED/88IPYkfjp06dqi9fKykqpU3RRURF+//13tbRdWFio1DE6Pj4eGRkZcHd3BwC0bdsW8fHxcHV1LfHS0uJXMtHrwk8bEVWao6MjLly4gMTERKSnp8PNzQ2XL1/GsWPH8Mcff2D+/Pm4dOmSWD8hIQGhoaGIiYnBvXv3cPz4cdy+fVtMEooZGxvjyJEj0NHRQe/evZGdna2WeLt3744jR47gyJEjuHXrFqZMmYKMjAy1tK2rq4v33nsPFy5cwJUrVxAQEIA33ngD7du3BwAsWLAAO3fuxOLFi3Hjxg3ExcVh3759mDdvnlrWT0RlwwSIiCpt1qxZ0NbWRrNmzWBlZQW5XI63334bI0aMgLe3Nx4/foypU6eK9Y2MjHDr1i0MGTIEjRs3xqRJkzBt2jRMnjy5RNsmJiY4evQoBEFA3759kZOTU+l4x40bB39/f/j5+aFr165wdnbGm2++Wel2gX+27cMPP8Q777yDjh07wsTEBPv37xfny+Vy/PDDDzh+/DjatWuHN954A59++ikaNWqklvUTUdnIhBcvhBMRERHVcjwDRERERJLDBIiIapTevXvDxMRE5WvFihWaDo+IagheAiOiGuXBgwfIy8tTOc/CwgIWFhavOSIiqomYABEREZHk8BIYERERSQ4TICIiIpIcJkBEREQkOUyAiIiISHKYABEREZHkMAEiIiIiyWECRERERJLDBIiIiIgk5/8AD4ZSN5Lek/8AAAAASUVORK5CYII=", 94 | "text/plain": [ 95 | "
" 96 | ] 97 | }, 98 | "metadata": {}, 99 | "output_type": "display_data" 100 | } 101 | ], 102 | "source": [ 103 | "# plot distribution of task number where participants dropped out\n", 104 | "sns.histplot(\n", 105 | " df_summary.filter(~pl.col(\"complete\")).select(\n", 106 | " pl.all()\n", 107 | " .top_k_by(\"task_number\", k=1)\n", 108 | " .over(\"hashed_id\", mapping_strategy=\"explode\")\n", 109 | " ),\n", 110 | " x=\"task_number\",\n", 111 | " hue=\"task_type\",\n", 112 | " multiple=\"dodge\",\n", 113 | " stat=\"proportion\",\n", 114 | " common_norm=False,\n", 115 | ")\n", 116 | "plt.title(\"Distribution of task number where participants dropped out\")\n", 117 | "plt.show()" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": 8, 123 | "metadata": {}, 124 | "outputs": [ 125 | { 126 | "data": { 127 | "text/html": [ 128 | "
\n", 135 | "shape: (5, 3)
task_typetask_numberlen
stri64u32
"evaluation"181
"evaluation"250
"evaluation"344
"evaluation"467
"training"127
" 136 | ], 137 | "text/plain": [ 138 | "shape: (5, 3)\n", 139 | "┌────────────┬─────────────┬─────┐\n", 140 | "│ task_type ┆ task_number ┆ len │\n", 141 | "│ --- ┆ --- ┆ --- │\n", 142 | "│ str ┆ i64 ┆ u32 │\n", 143 | "╞════════════╪═════════════╪═════╡\n", 144 | "│ evaluation ┆ 1 ┆ 81 │\n", 145 | "│ evaluation ┆ 2 ┆ 50 │\n", 146 | "│ evaluation ┆ 3 ┆ 44 │\n", 147 | "│ evaluation ┆ 4 ┆ 67 │\n", 148 | "│ training ┆ 1 ┆ 27 │\n", 149 | "└────────────┴─────────────┴─────┘" 150 | ] 151 | }, 152 | "execution_count": 8, 153 | "metadata": {}, 154 | "output_type": "execute_result" 155 | } 156 | ], 157 | "source": [ 158 | "# for each task_type, compute how many drop after 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 attempts\n", 159 | "# total_training =\n", 160 | "dropout_after = (\n", 161 | " df_summary.filter(~pl.col(\"complete\"))\n", 162 | " .group_by(\"hashed_id\")\n", 163 | " .agg(pl.max(\"task_number\"), pl.first(\"task_type\"))\n", 164 | " .group_by([\"task_type\", \"task_number\"])\n", 165 | " .agg(pl.len())\n", 166 | ")\n", 167 | "dropout_after.sort([\"task_type\", \"task_number\"]).head()" 168 | ] 169 | }, 170 | { 171 | "cell_type": "code", 172 | "execution_count": 9, 173 | "metadata": {}, 174 | "outputs": [ 175 | { 176 | "name": "stdout", 177 | "output_type": "stream", 178 | "text": [ 179 | "Training dropout rate: 94/783 (0.120, 12.0%)\n", 180 | "Evaluation dropout rate: 242/946 (0.256, 25.6%)\n" 181 | ] 182 | } 183 | ], 184 | "source": [ 185 | "# dropout rates\n", 186 | "training_incomplete = (\n", 187 | " df_summary.filter(~pl.col(\"complete\"))\n", 188 | " .filter(pl.col(\"task_type\") == \"training\")\n", 189 | " .n_unique(\"hashed_id\")\n", 190 | ")\n", 191 | "training_total = df_summary.filter(pl.col(\"task_type\") == \"training\").n_unique(\n", 192 | " \"hashed_id\"\n", 193 | ")\n", 194 | "training_dropout_rate = training_incomplete / training_total\n", 195 | "\n", 196 | "evaluation_incomplete = (\n", 197 | " df_summary.filter(~pl.col(\"complete\"))\n", 198 | " .filter(pl.col(\"task_type\") == \"evaluation\")\n", 199 | " .n_unique(\"hashed_id\")\n", 200 | ")\n", 201 | "evaluation_total = df_summary.filter(pl.col(\"task_type\") == \"evaluation\").n_unique(\n", 202 | " \"hashed_id\"\n", 203 | ")\n", 204 | "evaluation_dropout_rate = evaluation_incomplete / evaluation_total\n", 205 | "\n", 206 | "print(\n", 207 | " f\"Training dropout rate: {training_incomplete}/{training_total} ({training_dropout_rate:.3f}, {round(training_dropout_rate * 100, 1)}%)\"\n", 208 | ")\n", 209 | "print(\n", 210 | " f\"Evaluation dropout rate: {evaluation_incomplete}/{evaluation_total} ({evaluation_dropout_rate:.3f}, {round(evaluation_dropout_rate * 100, 1)}%)\"\n", 211 | ")" 212 | ] 213 | }, 214 | { 215 | "cell_type": "code", 216 | "execution_count": 10, 217 | "metadata": {}, 218 | "outputs": [ 219 | { 220 | "name": "stdout", 221 | "output_type": "stream", 222 | "text": [ 223 | "Proportion of task data missing: 10.3%\n", 224 | "Proportion of task data expected on training set: 7.5%\n", 225 | "Proportion of task data expected on evaluation set: 13.3%\n" 226 | ] 227 | } 228 | ], 229 | "source": [ 230 | "# amount of missing data\n", 231 | "df_summary_ten = df_summary.filter(pl.col(\"exp_name\").is_in([\"expv0\", \"expv1\"]))\n", 232 | "df_summary_five = df_summary.filter(~pl.col(\"exp_name\").is_in([\"expv0\", \"expv1\"]))\n", 233 | "total_expected_task_data = (\n", 234 | " df_summary_ten.n_unique(\"hashed_id\") * 10\n", 235 | " + df_summary_five.n_unique(\"hashed_id\") * 5\n", 236 | ")\n", 237 | "total_expected_task_data_training = (\n", 238 | " df_summary_ten.n_unique(\"hashed_id\") * 10\n", 239 | " + df_summary_five.filter(pl.col(\"task_type\") == \"training\").n_unique(\"hashed_id\")\n", 240 | " * 5\n", 241 | ")\n", 242 | "total_expected_task_data_evaluation = (\n", 243 | " df_summary_five.filter(pl.col(\"task_type\") == \"evaluation\").n_unique(\"hashed_id\")\n", 244 | " * 5\n", 245 | ")\n", 246 | "observed_task_data = df_summary.n_unique(\"joint_id_task\")\n", 247 | "observed_task_data_training = df_summary.filter(\n", 248 | " pl.col(\"task_type\") == \"training\"\n", 249 | ").n_unique(\"joint_id_task\")\n", 250 | "observed_task_data_evaluation = df_summary.filter(\n", 251 | " pl.col(\"task_type\") == \"evaluation\"\n", 252 | ").n_unique(\"joint_id_task\")\n", 253 | "print(\n", 254 | " f\"Proportion of task data missing: {round((1 - observed_task_data / total_expected_task_data) * 100, 1)}%\"\n", 255 | ")\n", 256 | "print(\n", 257 | " f\"Proportion of task data expected on training set: {round((1 - observed_task_data_training / total_expected_task_data_training) * 100, 1)}%\"\n", 258 | ")\n", 259 | "print(\n", 260 | " f\"Proportion of task data expected on evaluation set: {round((1 - observed_task_data_evaluation / total_expected_task_data_evaluation) * 100, 1)}%\"\n", 261 | ")" 262 | ] 263 | } 264 | ], 265 | "metadata": { 266 | "kernelspec": { 267 | "display_name": ".venv", 268 | "language": "python", 269 | "name": "python3" 270 | }, 271 | "language_info": { 272 | "codemirror_mode": { 273 | "name": "ipython", 274 | "version": 3 275 | }, 276 | "file_extension": ".py", 277 | "mimetype": "text/x-python", 278 | "name": "python", 279 | "nbconvert_exporter": "python", 280 | "pygments_lexer": "ipython3", 281 | "version": "3.12.5" 282 | } 283 | }, 284 | "nbformat": 4, 285 | "nbformat_minor": 2 286 | } 287 | -------------------------------------------------------------------------------- /analysis/0-arc-dataset.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "# imports\n", 10 | "import polars as pl\n", 11 | "import seaborn as sns\n", 12 | "from matplotlib import pyplot as plt\n", 13 | "from scipy.stats import ttest_ind, permutation_test" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 2, 19 | "metadata": {}, 20 | "outputs": [], 21 | "source": [ 22 | "# load data\n", 23 | "training_set = pl.read_csv(\"../arc_data/ARC_training_tasks.csv\")\n", 24 | "evaluation_set = pl.read_csv(\"../arc_data/ARC_evaluation_tasks.csv\")" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 3, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "def get_min_max(df, example_type):\n", 34 | " return (\n", 35 | " df.filter(pl.col(\"example_type\") == example_type)\n", 36 | " .group_by(\"task_name\")\n", 37 | " .agg(pl.max(\"example_number\"))\n", 38 | " .select(\n", 39 | " pl.min(\"example_number\").alias(\"min_examples\"),\n", 40 | " pl.max(\"example_number\").alias(\"max_examples\"),\n", 41 | " )\n", 42 | " .row(0)\n", 43 | " )" 44 | ] 45 | }, 46 | { 47 | "cell_type": "code", 48 | "execution_count": 4, 49 | "metadata": {}, 50 | "outputs": [ 51 | { 52 | "name": "stdout", 53 | "output_type": "stream", 54 | "text": [ 55 | "Training set:\n", 56 | "\tmin examples: 2\n", 57 | "\tmax examples: 10\n", 58 | "Evaluation set:\n", 59 | "\tmin examples: 2\n", 60 | "\tmax examples: 7\n" 61 | ] 62 | } 63 | ], 64 | "source": [ 65 | "# Get min and max number of training examples for both sets\n", 66 | "train_min, train_max = get_min_max(training_set, \"train\")\n", 67 | "eval_min, eval_max = get_min_max(evaluation_set, \"train\")\n", 68 | "\n", 69 | "print(f\"Training set:\\n\\tmin examples: {train_min}\\n\\tmax examples: {train_max}\")\n", 70 | "print(f\"Evaluation set:\\n\\tmin examples: {eval_min}\\n\\tmax examples: {eval_max}\")" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "There are between 2 and 10 inference examples for training set tasks and 2 and 7 for evaluation set tasks.\n" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 5, 83 | "metadata": {}, 84 | "outputs": [ 85 | { 86 | "name": "stdout", 87 | "output_type": "stream", 88 | "text": [ 89 | "Training set:\n", 90 | "\tmin examples: 1\n", 91 | "\tmax examples: 3\n", 92 | "Evaluation set:\n", 93 | "\tmin examples: 1\n", 94 | "\tmax examples: 2\n" 95 | ] 96 | } 97 | ], 98 | "source": [ 99 | "# get min and max number of training examples on training/eval set for test examples\n", 100 | "train_min, train_max = get_min_max(training_set, \"test\")\n", 101 | "eval_min, eval_max = get_min_max(evaluation_set, \"test\")\n", 102 | "\n", 103 | "print(f\"Training set:\\n\\tmin examples: {train_min}\\n\\tmax examples: {train_max}\")\n", 104 | "print(f\"Evaluation set:\\n\\tmin examples: {eval_min}\\n\\tmax examples: {eval_max}\")" 105 | ] 106 | }, 107 | { 108 | "cell_type": "markdown", 109 | "metadata": {}, 110 | "source": [ 111 | "There are between 1 and 3 test examples on the training set and 1 and 2 on the test set.\n" 112 | ] 113 | }, 114 | { 115 | "cell_type": "code", 116 | "execution_count": 6, 117 | "metadata": {}, 118 | "outputs": [ 119 | { 120 | "name": "stdout", 121 | "output_type": "stream", 122 | "text": [ 123 | "We did not collect data for 16 tests over 14 tasks in the training set\n", 124 | "We did not collect data for 19 tests over 19 tasks in the evaluation set\n" 125 | ] 126 | } 127 | ], 128 | "source": [ 129 | "training_set_no_data_tests = (\n", 130 | " training_set.filter(pl.col(\"example_type\") == \"test\")\n", 131 | " .group_by(\"task_name\")\n", 132 | " .agg((pl.max(\"example_number\") - 1).alias(\"missed_examples\"))\n", 133 | " .filter(pl.col(\"missed_examples\") > 0)\n", 134 | ")\n", 135 | "num_missed_training = training_set_no_data_tests.select(\n", 136 | " pl.sum(\"missed_examples\")\n", 137 | ").item()\n", 138 | "evaluation_set_no_data_tests = (\n", 139 | " evaluation_set.filter(pl.col(\"example_type\") == \"test\")\n", 140 | " .group_by(\"task_name\")\n", 141 | " .agg((pl.max(\"example_number\") - 1).alias(\"missed_examples\"))\n", 142 | " .filter(pl.col(\"missed_examples\") > 0)\n", 143 | ")\n", 144 | "num_missed_eval = evaluation_set_no_data_tests.select(pl.sum(\"missed_examples\")).item()\n", 145 | "\n", 146 | "print(\n", 147 | " f\"We did not collect data for {num_missed_training} tests over {training_set_no_data_tests.height} tasks in the training set\"\n", 148 | ")\n", 149 | "print(\n", 150 | " f\"We did not collect data for {num_missed_eval} tests over {evaluation_set_no_data_tests.height} tasks in the evaluation set\"\n", 151 | ")" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 7, 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "def get_set_tests(dataset):\n", 161 | " # grid size\n", 162 | " filtered_tests = dataset.filter((pl.col(\"example_type\") == \"test\")).with_columns(\n", 163 | " (pl.col(\"output_height\") * pl.col(\"output_width\")).alias(\"grid_size\")\n", 164 | " )\n", 165 | " # number of examples per task\n", 166 | " num_examples_per_task = (\n", 167 | " dataset.filter(pl.col(\"example_type\") == \"train\")\n", 168 | " .group_by(\"task_name\")\n", 169 | " .agg(pl.max(\"example_number\").alias(\"num_examples\"))\n", 170 | " )\n", 171 | " filtered_tests = filtered_tests.join(num_examples_per_task, on=\"task_name\")\n", 172 | " return filtered_tests" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 8, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [ 181 | "training_set_tests = get_set_tests(training_set)\n", 182 | "training_set_tests = training_set_tests.with_columns(pl.lit(\"training\").alias(\"set\"))\n", 183 | "evaluation_set_tests = get_set_tests(evaluation_set)\n", 184 | "evaluation_set_tests = evaluation_set_tests.with_columns(\n", 185 | " pl.lit(\"evaluation\").alias(\"set\")\n", 186 | ")\n", 187 | "dataset = pl.concat([training_set_tests, evaluation_set_tests])" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 9, 193 | "metadata": {}, 194 | "outputs": [ 195 | { 196 | "data": { 197 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+0AAAIjCAYAAAB20vpjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB5GUlEQVR4nO3de3zP9f//8ft7ZueTMduwg7M5n+VQyMopSUo0x5SUQ4hQjVKZc3LISkWF+tQnhOQ0Z8kxhdZIY32wsZiZsc32/v3h5/3tzcbwnvfL3K6Xy+ty8X69nq/X6/F6v1+z3d+v1/P5MpnNZrMAAAAAAIDhONi7AAAAAAAAkDtCOwAAAAAABkVoBwAAAADAoAjtAAAAAAAYFKEdAAAAAACDIrQDAAAAAGBQhHYAAAAAAAyK0A4AAAAAgEER2gEAAAAAMChCOwAAd6B3794KDQ29abujR4/KZDJp/vz5d7zPt956SyaT6Y63YwstWrRQixYtLK9teZw3M3/+fJlMJh09etQyLzQ0VI899liB71uSNm7cKJPJpI0bN96V/QEA7k+EdgDAfSk+Pl4DBw5UpUqV5ObmJjc3N1WtWlUDBgzQb7/9Zu/y7jsffvjhXQn6t8PItQEACj+T2Ww227sIAADuphUrVuiZZ56Ro6OjIiIiVKtWLTk4OOiPP/7Q4sWLdezYMcXHxyskJOSm28rKylJOTo6cnZ1v2O7o0aMqW7as5s2bp969e99R/ZcvX9bly5fl4uJyR9uxhatX2a9ebTabzcrIyFDRokVVpEiRfG+nevXqKlGixC1dtc7OzlZWVpacnZ0tdx6EhoaqevXqWrFiRb63c7u15eTkKDMzU05OTnJw4DoIAKBgONq7AAAA7qYjR46oa9euCgkJUUxMjAIDA62WT5w4UR9++OFNQ9iFCxfk7u6uokWLFmS5uXJ0dJSjozF/hZtMpgL/MuHqe1+kSJFb+mLA1hwcHAzxxQkAoHDja2EAwH1l0qRJunDhgubNm3ddYJeuBOLBgwcrKCjIMq93797y8PDQkSNH1K5dO3l6eioiIsKy7No+7SkpKerdu7e8vb3l4+OjXr16KSUlJV/1ZWVl6e2331bFihXl4uKi4sWLq1mzZlq7dq2lzbV92nv37i2TyZTr9NZbb1naZWRkaOzYsapQoYKcnZ0VFBSk1157TRkZGfmq7eOPP1b58uXl6uqqhg0basuWLde1ya1Pe2Jiovr06aMyZcrI2dlZgYGB6tixo6UvemhoqA4ePKhNmzZZ6r56Bf9qv/VNmzbp5ZdfVsmSJVWmTBmrZf/u037VmjVrVLt2bbm4uKhq1apavHix1fK8xgW4dps3qi2vPu3ffvut6tWrJ1dXV5UoUULdu3fX8ePHrdpcPaeOHz+uJ554Qh4eHvLz89Pw4cOVnZ2dxycAALgfGfNregAACsiKFStUoUIFNWrU6JbWu3z5slq3bq1mzZppypQpcnNzy7Wd2WxWx44dtXXrVvXv319hYWFasmSJevXqla/9vPXWW4qKitLzzz+vhg0bKjU1Vbt379bevXv1yCOP5LrOiy++qPDwcKt5q1at0sKFC1WyZElJV27lfvzxx7V161b169dPYWFh2r9/v95//30dOnRIS5cuvWFdn376qV588UU1adJEQ4YM0V9//aXHH39cvr6+Vl9w5KZz5846ePCgBg0apNDQUJ06dUpr165VQkKCQkNDNX36dA0aNEgeHh564403JEn+/v5W23j55Zfl5+enMWPG6MKFCzfc3+HDh/XMM8+of//+6tWrl+bNm6enn35aq1atyvM9zEt+avu3+fPnq0+fPmrQoIGioqKUlJSkDz74QNu2bdMvv/wiHx8fS9vs7Gy1bt1ajRo10pQpU7Ru3TpNnTpV5cuX10svvXRLdQIACjEzAAD3iXPnzpklmZ944onrlp09e9Z8+vRpy5Senm5Z1qtXL7Mk86hRo65br1evXuaQkBDL66VLl5olmSdNmmSZd/nyZfODDz5olmSeN2/eDWusVauWuX379jdsM3bsWPONfoUfPnzY7O3tbX7kkUfMly9fNpvNZvOXX35pdnBwMG/ZssWqbXR0tFmSedu2bXluLzMz01yyZElz7dq1zRkZGZb5H3/8sVmSuXnz5pZ58fHxVsd59uxZsyTz5MmTb3hM1apVs9rOVfPmzTNLMjdr1sxyLNcui4+Pt8wLCQkxSzJ/9913lnnnzp0zBwYGmuvUqWOZl9d7mNs286ptw4YNZknmDRs2mM3m/3ufqlevbr548aKl3YoVK8ySzGPGjLHMu3pOjRs3zmqbderUMderV++6fQEA7l/cHg8AuG+kpqZKkjw8PK5b1qJFC/n5+Vmm2bNnX9cmP1c/V65cKUdHR6u2RYoU0aBBg/JVo4+Pjw4ePKjDhw/nq/21Lly4oE6dOqlYsWL66quvLH2+v/32W4WFhalKlSpKTk62TA8//LAkacOGDXluc/fu3Tp16pT69+8vJycny/yrXQBuxNXVVU5OTtq4caPOnj17W8ckSS+88EK++6+XKlVKnTp1srz28vJSz5499csvvygxMfG2a7iZq+/Tyy+/bNXXvX379qpSpYp++OGH69bp37+/1esHH3xQf/31V4HVCAC49xDaAQD3DU9PT0lSWlradcs++ugjrV27VgsWLMh1XUdHR0tf6hs5duyYAgMDr/tioHLlyvmqcdy4cUpJSVGlSpVUo0YNjRgx4pYeQffCCy/oyJEjWrJkiYoXL26Zf/jwYR08eNDqiwk/Pz9VqlRJknTq1KkbHpMkVaxY0Wp+0aJFVa5cuRvW4+zsrIkTJ+rHH3+Uv7+/HnroIU2aNOmWw3PZsmXz3bZChQrX9Ve/epy59X+3lavvU26fdZUqVSzLr3JxcZGfn5/VvGLFit3RlxsAgMKHPu0AgPuGt7e3AgMDdeDAgeuWXe3jnleoc3Z2viuP9XrooYd05MgRff/991qzZo0++eQTvf/++4qOjtbzzz9/w3U/+OADffXVV1qwYIFq165ttSwnJ0c1atTQtGnTcl33Zv3S78SQIUPUoUMHLV26VKtXr1ZkZKSioqK0fv161alTJ1/bcHV1tWlNuQ1CJ+muDgJnz5HvAQD3Dq60AwDuK+3bt9eff/6pnTt3Fsj2Q0JCdPLkyeuu5sfFxeV7G76+vurTp4+++uor/f3336pZs6bVKPC52bJli4YPH64hQ4ZYRrb/t/Lly+vMmTNq1aqVwsPDr5tudCfA1efVX3vLflZWluLj4/N1TOXLl9err76qNWvW6MCBA8rMzNTUqVMty/MK0bfjzz//lNlstpp36NAhSbKM9F+sWDFJum5U/2uvht9KbVffp9w+67i4OMtyAABuBaEdAHBfee211+Tm5qbnnntOSUlJ1y2/Nuzdqnbt2uny5cuaM2eOZV52drZmzpyZr/X/+ecfq9ceHh6qUKHCDR/LdvLkSXXp0kXNmjXT5MmTc23TpUsXHT9+XHPnzr1u2cWLF284Inv9+vXl5+en6OhoZWZmWubPnz//po+yS09P16VLl6zmlS9fXp6enlbH5O7unu/H4t3MiRMntGTJEsvr1NRUffHFF6pdu7YCAgIsNUjS5s2bLe0uXLigzz///Lrt5be2+vXrq2TJkoqOjrY6th9//FGxsbFq37797R4SAOA+xu3xAID7SsWKFbVo0SJ169ZNlStXVkREhGrVqiWz2az4+HgtWrRIDg4O+eq/npsOHTqoadOmGjVqlI4ePWp5Rvi5c+fytX7VqlXVokUL1atXT76+vtq9e7f++9//auDAgXmuM3jwYJ0+fVqvvfaavv76a6tlNWvWVM2aNdWjRw9988036t+/vzZs2KCmTZsqOztbf/zxh7755hutXr1a9evXz3X7RYsW1bvvvqsXX3xRDz/8sJ555hnFx8dr3rx5N+3TfujQIbVq1UpdunRR1apV5ejoqCVLligpKUldu3a1tKtXr57mzJmjd999VxUqVFDJkiUtg+TdqkqVKqlv377atWuX/P399dlnnykpKUnz5s2ztHn00UcVHBysvn37asSIESpSpIg+++wz+fn5KSEhwWp7+a2taNGimjhxovr06aPmzZurW7dulke+hYaGaujQobd1PACA+xuhHQBw3+nYsaP279+vqVOnas2aNfrss89kMpkUEhKi9u3bq3///qpVq9ZtbdvBwUHLli3TkCFDtGDBAplMJj3++OOaOnVqvvpvDx48WMuWLdOaNWuUkZGhkJAQvfvuuxoxYkSe65w+fVrZ2dkaNmzYdcvGjh2rmjVrysHBQUuXLtX777+vL774QkuWLJGbm5vKlSunV155xTJQW1769eun7OxsTZ48WSNGjFCNGjW0bNkyRUZG3nC9oKAgdevWTTExMfryyy/l6OioKlWq6JtvvlHnzp0t7caMGaNjx45p0qRJOn/+vJo3b37bob1ixYqaOXOmRowYobi4OJUtW1b/+c9/1Lp1a0ubokWLasmSJXr55ZcVGRmpgIAADRkyRMWKFVOfPn2stncrtfXu3Vtubm6aMGGCRo4cKXd3d3Xq1EkTJ060ekY7AAD5ZTLf6X2AAAAAAACgQNCnHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAbFc9ol5eTk6MSJE/L09JTJZLJ3OQAAAACAQs5sNuv8+fMqVaqUHBzyvp5OaJd04sQJBQUF2bsMAAAAAMB95u+//1aZMmXyXE5ol+Tp6Snpypvl5eVl52oAAAAAAIVdamqqgoKCLHk0L4R2yXJLvJeXF6EdAAAAAHDX3KyLNgPRAQAAAABgUIR2AAAAAAAMitAOAAAAAIBB0acdAAAAAOwgOztbWVlZ9i4DBaRIkSJydHS848eKE9oBAAAA4C5LS0vT//73P5nNZnuXggLk5uamwMBAOTk53fY2CO0AAAAAcBdlZ2frf//7n9zc3OTn53fHV2JhPGazWZmZmTp9+rTi4+NVsWJFOTjcXu90QjsAAAAA3EVZWVkym83y8/OTq6urvctBAXF1dVXRokV17NgxZWZmysXF5ba2w0B0AAAAAGAHXGEv/G736rrVNmxQBwAAAAAAKAB2De2bN29Whw4dVKpUKZlMJi1duvS6NrGxsXr88cfl7e0td3d3NWjQQAkJCZblly5d0oABA1S8eHF5eHioc+fOSkpKuotHAQAAAABAwbBraL9w4YJq1aql2bNn57r8yJEjatasmapUqaKNGzfqt99+U2RkpFVfgKFDh2r58uX69ttvtWnTJp04cUJPPvnk3ToEAAAAAAAKjF0Homvbtq3atm2b5/I33nhD7dq106RJkyzzypcvb/n3uXPn9Omnn2rRokV6+OGHJUnz5s1TWFiYfv75Zz3wwAMFVzwAAAAA4I4cPXpUZcuW1S+//KLatWvbuxxDMmyf9pycHP3www+qVKmSWrdurZIlS6pRo0ZWt9Dv2bNHWVlZCg8Pt8yrUqWKgoODtX379jy3nZGRodTUVKsJAAAAAACjMWxoP3XqlNLS0jRhwgS1adNGa9asUadOnfTkk09q06ZNkqTExEQ5OTnJx8fHal1/f38lJibmue2oqCh5e3tbpqCgoII8FAAAAAAo1P773/+qRo0acnV1VfHixRUeHq4LFy5Ikj755BOFhYXJxcVFVapU0YcffmhZr2zZspKkOnXqyGQyqUWLFvYo39AM+5z2nJwcSVLHjh01dOhQSVLt2rX1008/KTo6Ws2bN7/tbY8ePVrDhg2zvE5NTSW4AwAAAMBtOHnypLp166ZJkyapU6dOOn/+vLZs2SKz2ayFCxdqzJgxmjVrlurUqaNffvlFL7zwgtzd3dWrVy/t3LlTDRs21Lp161StWjU5OTnZ+3AMx7ChvUSJEnJ0dFTVqlWt5oeFhWnr1q2SpICAAGVmZiolJcXqantSUpICAgLy3Lazs7OcnZ0LpG4AAAAAuJ+cPHlSly9f1pNPPqmQkBBJUo0aNSRJY8eO1dSpUy2DhZctW1a///67PvroI/Xq1Ut+fn6SpOLFi98ww93PDHt7vJOTkxo0aKC4uDir+YcOHbKcCPXq1VPRokUVExNjWR4XF6eEhAQ1btz4rtYLAAAAAPejWrVqqVWrVqpRo4aefvppzZ07V2fPntWFCxd05MgR9e3bVx4eHpbp3Xff1ZEjR+xd9j3Drlfa09LS9Oeff1pex8fHa9++ffL19VVwcLBGjBihZ555Rg899JBatmypVatWafny5dq4caMkydvbW3379tWwYcPk6+srLy8vDRo0SI0bN2bkeAAAAAC4C4oUKaK1a9fqp59+0po1azRz5ky98cYbWr58uSRp7ty5atSo0XXrIH/sGtp3796tli1bWl5f7Wfeq1cvzZ8/X506dVJ0dLSioqI0ePBgVa5cWd99952aNWtmWef999+Xg4ODOnfurIyMDLVu3dpqYIPCJiEhQcnJyfYuw6ZKlCih4OBge5cBAAAA4DaZTCY1bdpUTZs21ZgxYxQSEqJt27apVKlS+uuvvxQREZHrelf7sGdnZ9/Ncu8pdg3tLVq0kNlsvmGb5557Ts8991yey11cXDR79mzNnj3b1uUZTkJCgqqEhelierq9S7EpVzc3/REbS3AHAAAA7kE7duxQTEyMHn30UZUsWVI7duzQ6dOnFRYWprfffluDBw+Wt7e32rRpo4yMDO3evVtnz57VsGHDVLJkSbm6umrVqlUqU6aMXFxc5O3tbe9DMhTDDkSH6yUnJ+tieroiRk6Wf3B5e5djE0kJR7Rw4gglJycT2gEAAIB7kJeXlzZv3qzp06crNTVVISEhmjp1qtq2bStJcnNz0+TJkzVixAi5u7urRo0aGjJkiCTJ0dFRM2bM0Lhx4zRmzBg9+OCDlu7QuILQfg/yDy6vMhWr2bsMAAAAAFBYWJhWrVqV5/Jnn31Wzz77bJ7Ln3/+eT3//PMFUVqhYNjR4wEAAAAAuN8R2gEAAAAAMChCOwAAAAAABkVoBwAAAADAoAjtAAAAAAAYFKEdAAAAAACDIrQDAAAAAGBQhHYAAAAAAAzK0d4FAAAAAACkhIQEJScn37X9lShRQsHBwXdtf7g9hHYAAAAAsLOEhARVCQvTxfT0u7ZPVzc3/REba5fgHhoaqiFDhmjIkCH5ar9x40a1bNlSZ8+elY+PT4HWZjSEdgAAAACws+TkZF1MT1fEyMnyDy5f4PtLSjiihRNHKDk5Od+hvUWLFqpdu7amT59+x/vftWuX3N3d892+SZMmOnnypLy9ve943/caQjsAAAAAGIR/cHmVqVjN3mXcFrPZrOzsbDk63jxm+vn53dK2nZycFBAQcLul3dMYiA4AAAAAcEO9e/fWpk2b9MEHH8hkMslkMmn+/PkymUz68ccfVa9ePTk7O2vr1q06cuSIOnbsKH9/f3l4eKhBgwZat26d1fZCQ0OtrtibTCZ98skn6tSpk9zc3FSxYkUtW7bMsnzjxo0ymUxKSUmRJM2fP18+Pj5avXq1wsLC5OHhoTZt2ujkyZOWdS5fvqzBgwfLx8dHxYsX18iRI9WrVy898cQTBflW2RyhHQAAAABwQx988IEaN26sF154QSdPntTJkycVFBQkSRo1apQmTJig2NhY1axZU2lpaWrXrp1iYmL0yy+/qE2bNurQoYMSEhJuuI+3335bXbp00W+//aZ27dopIiJCZ86cybN9enq6pkyZoi+//FKbN29WQkKChg8fblk+ceJELVy4UPPmzdO2bduUmpqqpUuX2uT9uJsI7QAAAACAG/L29paTk5Pc3NwUEBCggIAAFSlSRJI0btw4PfLIIypfvrx8fX1Vq1Ytvfjii6pevboqVqyod955R+XLl7e6cp6b3r17q1u3bqpQoYLGjx+vtLQ07dy5M8/2WVlZio6OVv369VW3bl0NHDhQMTExluUzZ87U6NGj1alTJ1WpUkWzZs26JwexI7QDAAAAAG5b/fr1rV6npaVp+PDhCgsLk4+Pjzw8PBQbG3vTK+01a9a0/Nvd3V1eXl46depUnu3d3NxUvvz/DdoXGBhoaX/u3DklJSWpYcOGluVFihRRvXr1bunYjICB6AAAAAAAt+3aUeCHDx+utWvXasqUKapQoYJcXV311FNPKTMz84bbKVq0qNVrk8mknJycW2pvNptvsXrj40o7AAAAAOCmnJyclJ2dfdN227ZtU+/evdWpUyfVqFFDAQEBOnr0aMEX+C/e3t7y9/fXrl27LPOys7O1d+/eu1qHLXClHQAAAAAMIinhiGH3Exoaqh07dujo0aPy8PDI8yp4xYoVtXjxYnXo0EEmk0mRkZE3vGJeUAYNGqSoqChVqFBBVapU0cyZM3X27FmZTKa7XsudILQDAAAAgJ2VKFFCrm5uWjhxxF3bp6ubm0qUKJHv9sOHD1evXr1UtWpVXbx4UfPmzcu13bRp0/Tcc8+pSZMmKlGihEaOHKnU1FRblZ1vI0eOVGJionr27KkiRYqoX79+at26tWUAvXuFyVwYb/q/RampqfL29ta5c+fk5eVl73LytHfvXtWrV0/DZi9WmYrV7F2OTfzv8EFNG/Ck9uzZo7p169q7HAAAAKDAXbp0SfHx8SpbtqxcXFws8xMSEpScnHzX6ihRooSCg4Pv2v7sLScnR2FhYerSpYveeeedu7LPvD5rKf85lCvtAAAAAGAAwcHB91WILmjHjh3TmjVr1Lx5c2VkZGjWrFmKj4/Xs88+a+/SbgkD0QEAAAAACh0HBwfNnz9fDRo0UNOmTbV//36tW7dOYWFh9i7tlnClHQAAAABQ6AQFBWnbtm32LuOOcaUdAAAAAACDIrQDAAAAAGBQhHYAAAAAAAyK0A4AAAAAgEER2gEAAAAAMChGjwcAAAAAA0hISFBycvJd21+JEiV4Lvw9gNAOAAAAAHaWkJCgsLAqSk+/eNf26ebmqtjYPwwV3OfPn68hQ4YoJSWlwPfVu3dvpaSkaOnSpQW+rztBaAcAAAAAO0tOTlZ6+kUteL2LwoL9Cnx/sQmn1X38N0pOTjZUaC8IR48eVdmyZfXLL7+odu3alvkffPCBzGaz/QrLJ0I7AAAAABhEWLCf6lYqbe8y7gve3t72LiFfGIgOAAAAAJAvOTk5ioqKUtmyZeXq6qpatWrpv//9r3JyclSmTBnNmTPHqv0vv/wiBwcHHTt2TJI0bdo01ahRQ+7u7goKCtLLL7+stLS0PPfXu3dvPfHEE1bzhgwZohYtWlher1q1Ss2aNZOPj4+KFy+uxx57TEeOHLEsL1u2rCSpTp06MplMlnWv3XZGRoYGDx6skiVLysXFRc2aNdOuXbssyzdu3CiTyaSYmBjVr19fbm5uatKkieLi4m7lLbxlhHYAAAAAQL5ERUXpiy++UHR0tA4ePKihQ4eqe/fu2rJli7p166ZFixZZtV+4cKGaNm2qkJAQSZKDg4NmzJihgwcP6vPPP9f69ev12muv3VFNFy5c0LBhw7R7927FxMTIwcFBnTp1Uk5OjiRp586dkqR169bp5MmTWrx4ca7bee211/Tdd9/p888/1969e1WhQgW1bt1aZ86csWr3xhtvaOrUqdq9e7ccHR313HPP3VH9N8Pt8QAAAACAm8rIyND48eO1bt06NW7cWJJUrlw5bd26VR999JFee+01TZ06VQkJCQoODlZOTo6+/vprvfnmm5ZtDBkyxPLv0NBQvfvuu+rfv78+/PDD266rc+fOVq8/++wz+fn56ffff1f16tXl53dljIDixYsrICAg121cuHBBc+bM0fz589W2bVtJ0ty5c7V27Vp9+umnGjFihKXte++9p+bNm0uSRo0apfbt2+vSpUtycXG57WO4Ea60AwAAAABu6s8//1R6eroeeeQReXh4WKYvvvhCR44cUe3atRUWFma52r5p0yadOnVKTz/9tGUb69atU6tWrVS6dGl5enqqR48e+ueff5Senn7bdR0+fFjdunVTuXLl5OXlpdDQUElXRuTPryNHjigrK0tNmza1zCtatKgaNmyo2NhYq7Y1a9a0/DswMFCSdOrUqduu/2a40g4AAAAAuKmrfc9/+OEHlS5tPVies7OzJCkiIkKLFi3SqFGjtGjRIrVp00bFixeXdGUU98cee0wvvfSS3nvvPfn6+mrr1q3q27evMjMz5ebmdt0+HRwcrhvhPSsry+p1hw4dFBISorlz56pUqVLKyclR9erVlZmZabNj/7eiRYta/m0ymSTJcit+QeBKOwAAAADgpqpWrSpnZ2clJCSoQoUKVlNQUJAk6dlnn9WBAwe0Z88e/fe//1VERIRl/T179ignJ0dTp07VAw88oEqVKunEiRM33Kefn59OnjxpNW/fvn2Wf//zzz+Ki4vTm2++qVatWiksLExnz561au/k5CRJys7OznM/5cuXl5OTk7Zt22aZl5WVpV27dqlq1ao3fmMKGFfaAQAAAMAgYhNOG3Y/np6eGj58uIYOHaqcnBw1a9ZM586d07Zt2+Tl5aVevXopNDRUTZo0Ud++fZWdna3HH3/csn6FChWUlZWlmTNnqkOHDtq2bZuio6NvuM+HH35YkydP1hdffKHGjRtrwYIFOnDggOrUqSNJKlasmIoXL66PP/5YgYGBSkhI0KhRo6y2UbJkSbm6umrVqlUqU6aMXFxcrnvcm7u7u1566SWNGDFCvr6+Cg4O1qRJk5Senq6+ffve8ntlS4R2AAAAALCzEiVKyM3NVd3Hf3PX9unm5qoSJUrc0jrvvPOO/Pz8FBUVpb/++ks+Pj6qW7euXn/9dUubiIgIvfzyy+rZs6dcXV0t82vVqqVp06Zp4sSJGj16tB566CFFRUWpZ8+eee6vdevWioyM1GuvvaZLly7pueeeU8+ePbV//35JV26f//rrrzV48GBVr15dlStX1owZM6weCefo6KgZM2Zo3LhxGjNmjB588EFt3Ljxun1NmDBBOTk56tGjh86fP6/69etr9erVKlas2C29R7ZmMl/bQeA+lJqaKm9vb507d05eXl72LidPe/fuVb169TRs9mKVqVjN3uXYxP8OH9S0AU9qz549qlu3rr3LAQAAAArcpUuXFB8fr7Jly1qNOJ6QkKDk5OS7VkeJEiUUHBx81/Z3P8rrs5byn0O50g4AAAAABhAcHEyIxnXsOhDd5s2b1aFDB5UqVUomk0lLly7Ns23//v1lMpk0ffp0q/lnzpxRRESEvLy85OPjo759+1pGNQQAAAAA4F5m19B+4cIF1apVS7Nnz75huyVLlujnn39WqVKlrlsWERGhgwcPau3atVqxYoU2b96sfv36FVTJAAAAAADcNXa9Pb5t27Zq27btDdscP35cgwYN0urVq9W+fXurZbGxsVq1apV27dql+vXrS5Jmzpypdu3aacqUKbmGfAAAAAAA7hWGfk771ZH7RowYoWrVrh94bfv27fLx8bEEdkkKDw+Xg4ODduzYked2MzIylJqaajUBAAAAwN3EmOCFny0+Y0OH9okTJ8rR0VGDBw/OdXliYqJKlixpNc/R0VG+vr5KTEzMc7tRUVHy9va2TEFBQTatGwAAAADyUqRIEUlSZmamnStBQUtPT5ckFS1a9La3YdjR4/fs2aMPPvhAe/fulclksum2R48erWHDhllep6amEtwBAAAA3BWOjo5yc3PT6dOnVbRoUTk4GPpaKm6D2WxWenq6Tp06JR8fH8sXNbfDsKF9y5YtOnXqlNUjD7Kzs/Xqq69q+vTpOnr0qAICAnTq1Cmr9S5fvqwzZ84oICAgz207OzvL2dm5wGoHAAAAgLyYTCYFBgYqPj5ex44ds3c5KEA+Pj43zKb5YdjQ3qNHD4WHh1vNa926tXr06KE+ffpIkho3bqyUlBTt2bNH9erVkyStX79eOTk5atSo0V2vGQAAAADyw8nJSRUrVuQW+UKsaNGid3SF/Sq7hva0tDT9+eefltfx8fHat2+ffH19FRwcrOLFi1u1L1q0qAICAlS5cmVJUlhYmNq0aaMXXnhB0dHRysrK0sCBA9W1a1dGjgcAAABgaA4ODnJxcbF3GTA4u3ae2L17t+rUqaM6depIkoYNG6Y6depozJgx+d7GwoULVaVKFbVq1Urt2rVTs2bN9PHHHxdUyQAAAAAA3DV2vdLeokWLWxoC/+jRo9fN8/X11aJFi2xYFQAAAAAAxsAwhQAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBB2TW0b968WR06dFCpUqVkMpm0dOlSy7KsrCyNHDlSNWrUkLu7u0qVKqWePXvqxIkTVts4c+aMIiIi5OXlJR8fH/Xt21dpaWl3+UgAAAAAALA9u4b2CxcuqFatWpo9e/Z1y9LT07V3715FRkZq7969Wrx4seLi4vT4449btYuIiNDBgwe1du1arVixQps3b1a/fv3u1iEAAAAAAFBgHO2587Zt26pt27a5LvP29tbatWut5s2aNUsNGzZUQkKCgoODFRsbq1WrVmnXrl2qX7++JGnmzJlq166dpkyZolKlSuW67YyMDGVkZFhep6am2uiIAAAAAACwnXuqT/u5c+dkMpnk4+MjSdq+fbt8fHwsgV2SwsPD5eDgoB07duS5naioKHl7e1umoKCggi4dAAAAAIBbds+E9kuXLmnkyJHq1q2bvLy8JEmJiYkqWbKkVTtHR0f5+voqMTExz22NHj1a586ds0x///13gdYOAAAAAMDtsOvt8fmVlZWlLl26yGw2a86cOXe8PWdnZzk7O9ugMgAAAAAACo7hQ/vVwH7s2DGtX7/ecpVdkgICAnTq1Cmr9pcvX9aZM2cUEBBwt0sFAAAAAMCmDH17/NXAfvjwYa1bt07Fixe3Wt64cWOlpKRoz549lnnr169XTk6OGjVqdLfLBQAAAADApux6pT0tLU1//vmn5XV8fLz27dsnX19fBQYG6qmnntLevXu1YsUKZWdnW/qp+/r6ysnJSWFhYWrTpo1eeOEFRUdHKysrSwMHDlTXrl3zHDkexhQbG2vvEmyqRIkSCg4OtncZAAAAAO5xdg3tu3fvVsuWLS2vhw0bJknq1auX3nrrLS1btkySVLt2bav1NmzYoBYtWkiSFi5cqIEDB6pVq1ZycHBQ586dNWPGjLtSP+5c6pnTkqTu3bvbuRLbcnVz0x+xsQR3AAAAAHfErqG9RYsWMpvNeS6/0bKrfH19tWjRIluWhbvoYlqqJKn9i2+ocs16dq7GNpISjmjhxBFKTk4mtAMAAAC4I4YfiA73h+KlQlSmYjV7lwEAAAAAhmLogegAAAAAALifEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGJSjvQvArUtOTlYRj5P2LsMmUlJS7F0CAAAAABgWof0ecvLklaC+ePFiFfHwtXM1tpF5Kl6SlJ6ebudKAAAAAMB4CO33kKtXpVvWKacqlSvatxgb+fmnLG05LGVkZtq7FAAAAAAwHEL7PaiYh4sCi3vZuwyb8HJ1sncJAAAAAGBYDEQHAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAou4b2zZs3q0OHDipVqpRMJpOWLl1qtdxsNmvMmDEKDAyUq6urwsPDdfjwYas2Z86cUUREhLy8vOTj46O+ffsqLS3tLh4FAAAAAAAFw66h/cKFC6pVq5Zmz56d6/JJkyZpxowZio6O1o4dO+Tu7q7WrVvr0qVLljYRERE6ePCg1q5dqxUrVmjz5s3q16/f3ToEAAAAAAAKjKM9d962bVu1bds212Vms1nTp0/Xm2++qY4dO0qSvvjiC/n7+2vp0qXq2rWrYmNjtWrVKu3atUv169eXJM2cOVPt2rXTlClTVKpUqbt2LAAAAAAA2Jph+7THx8crMTFR4eHhlnne3t5q1KiRtm/fLknavn27fHx8LIFdksLDw+Xg4KAdO3bkue2MjAylpqZaTQAAAAAAGI1hQ3tiYqIkyd/f32q+v7+/ZVliYqJKlixptdzR0VG+vr6WNrmJioqSt7e3ZQoKCrJx9QAAAAAA3DnDhvaCNHr0aJ07d84y/f333/YuCQAAAACA6xg2tAcEBEiSkpKSrOYnJSVZlgUEBOjUqVNWyy9fvqwzZ85Y2uTG2dlZXl5eVhMAAAAAAEZj2NBetmxZBQQEKCYmxjIvNTVVO3bsUOPGjSVJjRs3VkpKivbs2WNps379euXk5KhRo0Z3vWYAAAAAAGzJrqPHp6Wl6c8//7S8jo+P1759++Tr66vg4GANGTJE7777ripWrKiyZcsqMjJSpUqV0hNPPCFJCgsLU5s2bfTCCy8oOjpaWVlZGjhwoLp27crI8QAAAACAe55dQ/vu3bvVsmVLy+thw4ZJknr16qX58+frtdde04ULF9SvXz+lpKSoWbNmWrVqlVxcXCzrLFy4UAMHDlSrVq3k4OCgzp07a8aMGXf9WAAAAAAAsDW7hvYWLVrIbDbnudxkMmncuHEaN25cnm18fX21aNGigigPAAAAAAC7MmyfdgAAAAAA7neEdgAAAAAADIrQDgAAAACAQRHaAQAAAAAwKEI7AAAAAAAGRWgHAAAAAMCgCO0AAAAAABgUoR0AAAAAAIMitAMAAAAAYFCEdgAAAAAADIrQDgAAAACAQRHaAQAAAAAwKEI7AAAAAAAGRWgHAAAAAMCgCO0AAAAAABgUoR0AAAAAAIMitAMAAAAAYFCEdgAAAAAADIrQDgAAAACAQRHaAQAAAAAwKEI7AAAAAAAGRWgHAAAAAMCgCO0AAAAAABjUbYX2cuXK6Z9//rlufkpKisqVK3fHRQEAAAAAgNsM7UePHlV2dvZ18zMyMnT8+PE7LgoAAAAAAEiOt9J42bJlln+vXr1a3t7eltfZ2dmKiYlRaGiozYoDAAAAAOB+dkuh/YknnpAkmUwm9erVy2pZ0aJFFRoaqqlTp9qsOAAAJCk0NFRDhgzRkCFD8tV+48aNatmypc6ePSsfH58CrQ0AAKAg3dLt8Tk5OcrJyVFwcLBOnTpleZ2Tk6OMjAzFxcXpscceK6haAQAGZzKZbji99dZbt7XdXbt2qV+/fvlu36RJE508edLqjjAjatGiRb6/iLiXnTx5Us8++6wqVaokBweHXI958eLFql+/vnx8fOTu7q7atWvryy+/vK5dbGysHn/8cXl7e8vd3V0NGjRQQkJCnvueO3euHnzwQRUrVkzFihVTeHi4du7cecvHsGnTJj388MPy9fWVm5ubKlasqF69eikzM1PSlS+Krp7nDg4O8vb2Vp06dfTaa6/p5MmTt7y/f/v444/VokULeXl5yWQyKSUlJdd2P/zwgxo1aiRXV1cVK1bMcrHlWv/884/KlClzw21dtXfvXj3yyCPy8fFR8eLF1a9fP6Wlpd3R8QAAbs1t9WmPj49XiRIlbF0LAOAed/LkScs0ffp0eXl5Wc0bPny4pa3ZbNbly5fztV0/Pz+5ubnluw4nJycFBATIZDLd8jHA9jIyMuTn56c333xTtWrVyrWNr6+v3njjDW3fvl2//fab+vTpoz59+mj16tWWNkeOHFGzZs1UpUoVbdy4Ub/99psiIyPl4uKS5743btyobt26acOGDdq+fbuCgoL06KOP3tIYPL///rvatGmj+vXra/Pmzdq/f79mzpwpJyen68b4iYuL04kTJ7Rr1y6NHDlS69atU/Xq1bV///587+9a6enpatOmjV5//fU823z33Xfq0aOH+vTpo19//VXbtm3Ts88+m2vbvn37qmbNmjfd74kTJxQeHq4KFSpox44dWrVqlQ4ePKjevXvf7qEAAG7DbT/yLSYmRq+//rqef/55Pffcc1YTAOD+FBAQYJm8vb1lMpksr//44w95enrqxx9/VL169eTs7KytW7fqyJEj6tixo/z9/eXh4aEGDRpo3bp1VtsNDQ3V9OnTLa9NJpM++eQTderUyXLV89/jrly96nn1KuL8+fPl4+Oj1atXKywsTB4eHmrTpo3VFdDLly9r8ODBliuKI0eOVK9evfK8WilJx44dU4cOHVSsWDG5u7urWrVqWrlypWX5gQMH1LZtW3l4eMjf3189evRQcnKyJKl3797atGmTPvjgA8sV2qNHj+a6n9DQUL377rvq2bOnPDw8FBISomXLlun06dPq2LGjPDw8VLNmTe3evdtqva1bt+rBBx+Uq6urgoKCNHjwYF24cMGy/Msvv1T9+vXl6empgIAAPfvsszp16tR172NMTIzq168vNzc3NWnSRHFxcXm+J3nV/8EHH6hnz5553v3QokULderUSWFhYSpfvrxeeeUV1axZU1u3brW0eeONN9SuXTtNmjRJderUUfny5fX444+rZMmSee574cKFevnll1W7dm1VqVJFn3zyiXJychQTE5Pv+tesWaOAgABNmjRJ1atXV/ny5dWmTRvNnTtXrq6uVm1LliypgIAAVapUSV27dtW2bdvk5+enl156Kd/7u9aQIUM0atQoPfDAA7kuv3z5sl555RVNnjxZ/fv3V6VKlVS1alV16dLlurZz5sxRSkqK1RdoeVmxYoWKFi2q2bNnq3LlymrQoIGio6P13Xff6c8//7zt4wEA3JrbCu1vv/22Hn30UcXExCg5OVlnz561mgAAyMuoUaM0YcIExcbGqmbNmkpLS1O7du0UExOjX375RW3atFGHDh1ueMuzdOV3UZcuXfTbb7+pXbt2ioiI0JkzZ/Jsn56erilTpujLL7/U5s2blZCQYBVcJk6cqIULF2revHnatm2bUlNTtXTp0hvWMGDAAGVkZFiuvk6cOFEeHh6SrjwG9eGHH1adOnW0e/durVq1SklJSZYg9cEHH6hx48Z64YUXLHciBAUF5bmv999/X02bNtUvv/yi9u3bq0ePHurZs6e6d++uvXv3qnz58urZs6fMZrOkK1el27Rpo86dO+u3337Tf/7zH23dulUDBw60bDMrK0vvvPOOfv31Vy1dulRHjx7N9SrqG2+8oalTp2r37t1ydHS0+oL+6NGjMplM2rhx4w3fq1thNpsVExOjuLg4PfTQQ5KudNH74YcfVKlSJbVu3VolS5ZUo0aNbvoZXSs9PV1ZWVny9fXN9zoBAQE6efKkNm/efEv7kiRXV1f1799f27Zts3whsnDhQnl4eNxw2rJlS773sXfvXh0/flwODg6qU6eOAgMD1bZtWx04cMCq3e+//65x48bpiy++kIPDzf8EzMjIkJOTk1Xbq19S/PvLFABAwbqlgeiuio6O1vz589WjRw9b1wMAKOTGjRunRx55xPLa19fX6pbpd955R0uWLNGyZcusAua1evfurW7dukmSxo8frxkzZmjnzp1q06ZNru2zsrIUHR2t8uXLS5IGDhyocePGWZbPnDlTo0ePVqdOnSRJs2bNsrpqnpuEhAR17txZNWrUkCSVK1fOsmzWrFmqU6eOxo8fb5n32WefKSgoSIcOHVKlSpXk5OQkNzc3BQQE3HA/ktSuXTu9+OKLkqQxY8Zozpw5atCggZ5++mlJ0siRI9W4cWMlJSUpICBAUVFRioiIsPQfr1ixombMmKHmzZtrzpw5cnFxsQrf5cqV04wZM9SgQQOlpaVZvnyQpPfee0/NmzeXdOVLl/bt2+vSpUtycXFR0aJFVbly5VvqvpCXc+fOqXTp0srIyFCRIkX04YcfWs6VU6dOKS0tTRMmTNC7776riRMnatWqVXryySe1YcMGS303M3LkSJUqVUrh4eH5ruvpp5/W6tWr1bx5cwUEBOiBBx5Qq1at1LNnT3l5ed10/SpVqki68gVHyZIl9fjjj6tRo0Y3XKd06dL5ru+vv/6SJL311luaNm2aZWDgFi1a6NChQ/L19VVGRoa6deumyZMnKzg42LLOjTz88MMaNmyYJk+erFdeeUUXLlzQqFGjJOmO++kDAPLvtq60Z2ZmqkmTJrauBQBwH6hfv77V67S0NA0fPlxhYWHy8fGRh4eHYmNjb3ql/d99ct3d3eXl5WV1a/e13NzcLIFdkgIDAy3tz507p6SkJDVs2NCyvEiRIqpXr94Naxg8eLDeffddNW3aVGPHjtVvv/1mWfbrr79qw4YNVldPr4a3I0eO3HC7ufn38fr7+0uS5cuCf8+7eky//vqr5s+fb7X/1q1bKycnR/Hx8ZKkPXv2qEOHDgoODpanp6cl+F773v9734GBgVb7KV26tP744w+r9+52eXp6at++fdq1a5fee+89DRs2zHIFPycnR5LUsWNHDR06VLVr19aoUaP02GOPKTo6Ol/bnzBhgr7++mstWbLkhv3gr1WkSBHNmzdP//vf/zRp0iSVLl1a48ePV7Vq1fIVXq/e/XB1jAVPT09VqFDhhtO1t93fyNX35o033lDnzp1Vr149zZs3TyaTSd9++60kafTo0QoLC1P37t3zvd1q1arp888/19SpUy1fLpUtW1b+/v75ulIPALCN2/of9/nnn9eiRYtsXQsA4D7g7u5u9Xr48OFasmSJxo8fry1btmjfvn2qUaOGZVTuvBQtWtTqtclksoSX/La/GqZu1/PPP6+//vpLPXr00P79+1W/fn3NnDlT0pUvIzp06KB9+/ZZTYcPH7bc8n0r/l3/1fCX27yr70FaWppefPFFq33/+uuvOnz4sMqXL68LFy6odevW8vLy0sKFC7Vr1y4tWbJEkq5772+0H1tycHBQhQoVVLt2bb366qt66qmnFBUVJUkqUaKEHB0dVbVqVat1wsLCbvoFjyRNmTJFEyZM0Jo1a/I1CFtuSpcurR49emjWrFk6ePCgLl26lK8vDGJjYyVd6dsv2f72+KtfpPz7vXF2dla5cuUs78369ev17bffytHRUY6OjmrVqpWkK+/r2LFj89z2s88+q8TERB0/flz//POP3nrrLZ0+fdrqrhIAQMG6rdvjL126pI8//ljr1q1TzZo1r/tDaNq0aTYpDveP86mpheZWu6uDTAHIn23btql3796W29LT0tLyHJCtoHh7e8vf31+7du2yBOrs7Gzt3btXtWvXvuG6QUFB6t+/v/r376/Ro0dr7ty5GjRokOrWravvvvtOoaGhcnTM/ddtbqOP20rdunX1+++/q0KFCrku379/v/755x9NmDDB0pf+2oHs7O3qI2WlK+9VgwYNrhsE79ChQwoJCbnhdiZNmqT33ntPq1evvu5Oj9tVrFgxBQYGWg3sl5uLFy/q448/1kMPPSQ/Pz9Jsvnt8VcHdoyLi1OzZs0kXekOcvToUct789133+nixYuWdXbt2qXnnntOW7ZssboDJS9X7+T47LPP5OLiYtXFBQBQsG4rtP/222+WP2KuHeSEx+vgVlzKuvLH6u7du/XLHzfvX3cvyE67MhBWYfkSAihoFStW1OLFi9WhQweZTCZFRkYWyFXcmxk0aJCioqJUoUIFValSRTNnztTZs2dv+HttyJAhatu2rSpVqqSzZ89qw4YNCgsLk3RlkLq5c+eqW7dueu211+Tr66s///xTX3/9tT755BMVKVJEoaGh2rFjh44ePSoPDw/5+vra7LbjkSNH6oEHHtDAgQP1/PPPy93dXb///rvWrl2rWbNmKTg4WE5OTpo5c6b69++vAwcO6J133rnl/Rw/flytWrXSF198ccNb5Pft2yfpypcyp0+f1r59++Tk5GS5OhwVFaX69eurfPnyysjI0MqVK/Xll19qzpw5lm2MGDFCzzzzjB566CG1bNlSq1at0vLly60GwevZs6dKly5tuUI/ceJEjRkzRosWLVJoaKgSExMlyXJFOz8++ugj7du3T506dVL58uV16dIlffHFFzp48KDlzoqrTp06pUuXLun8+fPas2ePJk2apOTkZC1evNjSxtPTU56envnatyQlJiYqMTHRMmL7/v375enpqeDgYPn6+srLy0v9+/fX2LFjFRQUpJCQEE2ePFmSLGMeXBvMr37BfLVbiiTt3LlTPXv2VExMjOVLg1mzZqlJkyby8PDQ2rVrNWLECE2YMMGyDgCg4N1WaN+wYYOt68B9KvPyldBeq3xJNWlQx87V2MYfcYe1/FdZHjUF4MamTZum5557Tk2aNFGJEiU0cuRIpaam3vU6Ro4cqcTERPXs2VNFihRRv3791Lp1axUpUiTPdbKzszVgwAD973//k5eXl9q0aaP3339fklSqVClt27ZNI0eO1KOPPqqMjAyFhISoTZs2lmA+fPhw9erVS1WrVtXFixcVHx9vuYX6TtWsWVObNm3SG2+8oQcffFBms1nly5fXM888I0ny8/PT/Pnz9frrr2vGjBmqW7eupkyZoscff/yW9pOVlaW4uDilp6ffsF2dOv/3f/yePXu0aNEihYSEWO6quHDhgl5++WX973//k6urq6pUqaIFCxZY6pWkTp06KTo6WlFRURo8eLAqV66s7777znJ1WbrSH//fX3zMmTNHmZmZeuqpp6zqGTt2rN566y1JVwZwmz9/fp53eDRs2FBbt25V//79deLECXl4eKhatWpaunTpdQPgVa5cWSaTSR4eHipXrpweffRRDRs2LF+DDeYlOjpab7/9tuX11btB5s2bZxntf/LkyXJ0dFSPHj108eJFNWrUSOvXr1exYsXyvZ/09HTFxcUpKyvLMm/nzp0aO3as0tLSVKVKFX300UcMRAwAd5nJfKcd+gqB1NRUeXt769y5c/kaBdZeFi5cqO7du6vnwGGqXbOavcuxiSXLV2nL8m/16LN91aZF4RjccN9vB/XFrGlasGCBIiIi7F0OgNuUk5OjsLAwdenS5bauQOPe0atXL5lMJs2fP9/epQAA7iP5zaG3daW9ZcuWN7xdcP369bezWQAA7ObYsWNas2aNmjdvroyMDM2aNUvx8fF69tln7V0aCpDZbNbGjRt57jgAwLBuK7RfOyhPVlaW9u3bpwMHDqhXr162qAsAgLvKwcFB8+fP1/Dhw2U2m1W9enWtW7fO0kcdhZPJZNKxY8fsXQYAAHm6rdB+tb/etd566y2lpaXdUUEAANhDUFCQtm3bZu8yAAAArNhmiNr/r3v37vrss89suUkAAAAAAO5bNg3t27dvl4uLiy03CQC4jxw9elQmk8nyeLCCNH/+fB5bBQAADO+2QvuTTz5pNXXq1EkPPPCA+vTpoxdffNHWNQIADKB3794ymUzXTW3atLF3aTcVGhqq6dOnW8175plndOjQIfsUlE8bN26UyWQq9I+QPHPmjAYNGqTKlSvL1dVVwcHBGjx4sM6dO2dp888//6hNmzYqVaqUnJ2dFRQUpIEDB1o9HvDq+3XtdPXZ7Lm5dOmSevfurRo1asjR0VFPPPHEbR3D6dOn9dJLLyk4OFjOzs4KCAhQ69atrbpchIaGWmpydXVVaGiounTpcscD+M6dO1cPPvigihUrpmLFiik8PFw7d+7Ms33//v1lMpmu+5mQpB9++EGNGjWSq6urihUrdtP3I7f322QyWZ4TDwC4c7fVp93b29vqtYODgypXrqxx48bp0UcftUlhAADjadOmjebNm2c1z9nZ2U7V3BlXV1e5urrauwxIOnHihE6cOKEpU6aoatWqOnbsmOWZ6P/9738lXflbo2PHjnr33Xfl5+enP//8UwMGDNCZM2e0aNEiq+3FxcVZPTqnZMmSee47Oztbrq6uGjx4sL777rvbPobOnTsrMzNTn3/+ucqVK6ekpCTFxMTon3/+sWo3btw4vfDCC8rMzNTRo0e1YMEChYeH65133tEbb7xxW/veuHGjunXrpiZNmsjFxUUTJ07Uo48+qoMHD6p06dJWbZcsWaKff/5ZpUqVum473333nV544QWNHz9eDz/8sC5fvqwDBw7ccN8nT560ev3jjz+qb9++6ty5820dCwDgerd1pX3evHlW06effqoJEybYPLBnZ2crMjJSZcuWlaurq8qXL6933nlH/360vNls1pgxYxQYGChXV1eFh4fr8OHDNq0DAHDF1SuI/56KFSsmSXr22Wf1zDPPWLXPyspSiRIl9MUXX0iSVq1apWbNmsnHx0fFixfXY489piNHjuS5v9xuYV+6dKnVY0ePHDmijh07yt/fXx4eHmrQoIHWrVtnWd6iRQsdO3ZMQ4cOtVwFzGvbc+bMUfny5eXk5KTKlSvryy+/tFpuMpn0ySefqFOnTnJzc1PFihW1bNmyG75nH374oSpWrCgXFxf5+/vrqaeesizLyclRVFSU5fdcrVq1LCH16NGjatmypSSpWLFiMplM6t279w3fpxUrVqhy5cpyc3PTU089pfT0dH3++ecKDQ1VsWLFNHjwYGVnZ1vWy8jI0PDhw1W6dGm5u7urUaNG2rhxo2X5P//8o27duql06dJyc3NTjRo19NVXX1ntu0WLFho8eLBee+01+fr6KiAgQG+99dYN35NrVa9eXd999506dOig8uXL6+GHH9Z7772n5cuX6/Lly5b34KWXXlL9+vUVEhKiVq1a6eWXX9aWLVuu217JkiWtzlEHh7z/3HF3d9ecOXP0wgsvKCAg4JbqviolJUVbtmzRxIkT1bJlS4WEhKhhw4YaPXq0Hn/8cau2np6eCggIUHBwsB566CF9/PHHioyM1JgxYxQXF3db+1+4cKFefvll1a5dW1WqVNEnn3yinJwcxcTEWLU7fvy4Bg0apIULF6po0aJWyy5fvqxXXnlFkydPVv/+/VWpUiVVrVpVXbp0ueG+r/3/4Pvvv1fLli1Vrly52zoWAMD17qhP+549e7RgwQItWLBAv/zyi61qspg4caLmzJmjWbNmKTY2VhMnTtSkSZM0c+ZMS5tJkyZpxowZio6O1o4dO+Tu7q7WrVvr0qVLNq8HAJC3iIgILV++3OopIqtXr1Z6ero6deokSbpw4YKGDRum3bt3KyYmRg4ODurUqZNycnJue79paWlq166dYmJi9Msvv6hNmzbq0KGDEhISJEmLFy9WmTJlNG7cOJ08efK6K4NXLVmyRK+88opeffVVHThwQC+++KL69OmjDRs2WLV7++231aVLF/32229q166dIiIidObMmVy3uXv3bg0ePFjjxo1TXFycVq1apYceesiyPCoqSl988YWio6N18OBBDR06VN27d9emTZsUFBRkufIbFxenkydP6oMPPsjzfUhPT9eMGTP09ddfa9WqVdq4caM6deqklStXauXKlfryyy/10UcfWb4UkKSBAwdq+/bt+vrrr/Xbb7/p6aefVps2bSxffl+6dEn16tXTDz/8oAMHDqhfv37q0aPHdbdef/7553J3d9eOHTs0adIkjRs3TmvXrrUs7927t1q0aJFn7bk5d+6cvLy85OiY+02BJ06c0OLFi9W8efPrltWuXVuBgYF65JFH7soTATw8POTh4aGlS5cqIyPjltd/5ZVXZDab9f3331+3zbym/v3757m99PR0ZWVlydfX1zIvJydHPXr00IgRI1StWrXr1tm7d6+OHz8uBwcH1alTR4GBgWrbtu1Nr7T/W1JSkn744Qf17ds33+sAAG7utm6PP3XqlLp27aqNGzdarlKkpKSoZcuW+vrrr+Xn52eT4n766Sd17NhR7du3l3SlL9hXX31l+WPBbDZr+vTpevPNN9WxY0dJ0hdffCF/f38tXbpUXbt2tUkdAIArVqxYIQ8PD6t5r7/+ul5//XW1bt1a7u7uWrJkiXr06CFJWrRokR5//HF5enpK0nW3zH722Wfy8/PT77//rurVq99WTbVq1VKtWrUsr9955x0tWbJEy5Yt08CBA+Xr66siRYpYrnDmZcqUKerdu7defvllSdKwYcP0888/a8qUKZYr3tKVANqtWzdJ0vjx4zVjxgzt3Lkz1779CQkJcnd312OPPSZPT0+FhISoTp06kq5c5R4/frzWrVunxo0bS5LKlSunrVu36qOPPlLz5s0toatkyZI3HTQvKyvLcqeAJD311FP68ssvlZSUJA8PD1WtWlUtW7bUhg0b9MwzzyghIUHz5s1TQkKC5Vbp4cOHa9WqVZo3b57Gjx+v0qVLa/jw4ZZ9DBo0SKtXr9Y333yjhg0bWubXrFlTY8eOlSRVrFhRs2bNUkxMjB555BFJUmBg4C19MZOcnKx33nlH/fr1u25Zt27d9P333+vixYvq0KGDPvnkE8uywMBARUdHq379+srIyNAnn3yiFi1aaMeOHapbt26+93+rHB0dNX/+fL3wwguKjo5W3bp11bx5c3Xt2lU1a9a86fq+vr4qWbKkjh49apl3s8EY/337/7VGjhypUqVKKTw83DJv4sSJcnR01ODBg3Nd56+//pJ05fG906ZNU2hoqKZOnaoWLVro0KFDVl8A5OXzzz+Xp6ennnzyyZu2BQDk321daR80aJDOnz+vgwcP6syZMzpz5owOHDig1NTUPH8Z3I4mTZooJibGMlDQr7/+qq1bt6pt27aSpPj4eCUmJlr9UvL29lajRo20ffv2PLebkZGh1NRUqwkAcHMtW7bUvn37rKarV/wcHR3VpUsXLVy4UNKVq+rff/+9IiIiLOsfPnxY3bp1U7ly5eTl5aXQ0FBJslwVvx1paWkaPny4wsLC5OPjIw8PD8XGxt7yNmNjY9W0aVOreU2bNlVsbKzVvH+HMHd3d3l5eenUqVO5bvORRx5RSEiIypUrpx49emjhwoVKT0+XJP35559KT0/XI488YnUF9Ysvvrhhl4G8uLm5WQK7JPn7+ys0NNTqSxZ/f39Lrfv371d2drYqVapktf9NmzZZ9p+dna133nlHNWrUkK+vrzw8PLR69err3ttrg2lgYKDVe3L1joL8SE1NVfv27VW1atVcb7N///33tXfvXn3//fc6cuSIhg0bZllWuXJlvfjii6pXr56aNGmizz77TE2aNNH777+fr33fic6dO+vEiRNatmyZ2rRpo40bN6pu3bqaP39+vtY3m81W3T4qVKhwwymvfvoTJkzQ119/rSVLllie6LNnzx598MEHmj9/vtU+/u3qlypvvPGGOnfurHr16mnevHkymUz69ttv83UMn332mSIiIniSEADY2G1daV+1apXWrVunsLAwy7yqVatq9uzZNu3XPmrUKKWmpqpKlSoqUqSIsrOz9d5771n+ALw6Gqy/v7/Vev7+/jccKTYqKkpvv/22zeoEgPuFu7u7KlSokOfyiIgINW/eXKdOndLatWvl6upqdQW6Q4cOCgkJ0dy5c1WqVCnl5OSoevXqyszMzHV7Dg4OVuOYSFeuKP/b8OHDtXbtWk2ZMkUVKlSQq6urnnrqqTy3eaeu7QtsMpnyvIrs6empvXv3auPGjVqzZo3GjBmjt956S7t27bJ0I/jhhx+uGyzsdgb3y62uG9WalpamIkWKaM+ePSpSpIhVu6tBf/Lkyfrggw80ffp01ahRQ+7u7hoyZMh17+2tvCc3cv78ebVp00aenp5asmTJdduV/q8PdZUqVeTr66sHH3xQkZGRCgwMzHWbDRs21NatW2+5ltvh4uKiRx55RI888ogiIyP1/PPPa+zYsXmORXDVP//8o9OnT6ts2bKWedfe0XKt7t27Kzo62mrelClTNGHCBK1bt87qi5QtW7bo1KlTCg4OtszLzs7Wq6++qunTp+vo0aOW969q1aqWNs7OzipXrly+vgDbsmWL4uLi9J///OembQEAt+a2QntOTk6uv0iLFi16R/0Sr/XNN99o4cKFWrRokapVq6Z9+/ZpyJAhKlWqlHr16nXb2x09erTVN/OpqakKCgqyRckAcF9r0qSJgoKC9J///Ec//vijnn76acvvi3/++UdxcXGWx1NJummY8vPz0/nz53XhwgW5u7tLuv624W3btql3796WfvNpaWlWtxlLkpOTk9UAbLkJCwvTtm3brH6/bNu2zSrE3A5HR0eFh4crPDxcY8eOlY+Pj9avX69HHnlEzs7OSkhIyLVf9tW6Jd209ttRp04dZWdn69SpU5bP41rbtm1Tx44d1b17d0lXfv8fOnTojt+T3KSmpqp169ZydnbWsmXL8nW19urfHDfqR75v3748A31Bq1q1qpYuXXrTdh988IEcHBysHq92q7fHT5o0Se+9955Wr16t+vXrWy3r0aOH1V2JktS6dWv16NFDffr0kSTVq1dPzs7OiouLU7NmzSRd+YLs6NGjCgkJuekxfPrpp6pXr55VVxUAgG3cVmh/+OGH9corr+irr76y9IM7fvy4hg4dqlatWtmsuBEjRmjUqFGWvuk1atTQsWPHFBUVpV69eln6JiYlJVn9Qk5KSlLt2rXz3K6zs/M9+4giALCnjIyM6+5kcnR0VIkSJSyvn332WUVHR+vQoUNWg7gVK1ZMxYsX18cff6zAwEAlJCRo1KhRN9xfo0aN5Obmptdff12DBw/Wjh07rrvduGLFilq8eLE6dOggk8mkyMjI675ADg0N1ebNm9W1a1c5Oztb1XvViBEj1KVLF9WpU0fh4eFavny5Fi9ebDUS/a1asWKF/vrrLz300EMqVqyYVq5cqZycHFWuXFmenp4aPny4hg4dqpycHDVr1kznzp3Ttm3b5OXlpV69eikkJEQmk0krVqxQu3bt5OrqetMrsPlVqVIlRUREqGfPnpo6darq1Kmj06dPKyYmRjVr1lT79u1VsWJF/fe//9VPP/2kYsWKadq0aUpKSrrl0D569GgdP348z1vkU1NT9eijjyo9PV0LFiyw6rrm5+enIkWKaOXKlUpKSlKDBg3k4eGhgwcPasSIEWratKmlm8X06dNVtmxZVatWTZcuXdInn3yi9evXa82aNZZ9zZo1S0uWLLEaWf33339XZmamzpw5o/Pnz1sC843+lvi3f/75R08//bSee+451axZU56entq9e7cmTZpkGXPnqvPnzysxMVFZWVmKj4/XggUL9MknnygqKsrqLpYb3dFyrYkTJ2rMmDFatGiRQkNDLT+jV7s8FC9eXMWLF7dap2jRogoICFDlypUlXfkSoH///ho7dqyCgoIUEhJiedb6008/bVmvSpUqioqKsnxJJl35/L799ltNnTo13zUDAPLvtvq0z5o1S6mpqQoNDVX58uVVvnx5lS1bVqmpqVYju9+p9PT06x7TUqRIEcsfY2XLllVAQIDVL97U1FTt2LHDMqgPAMB2Vq1apcDAQKvp6lW5qyIiIvT777+rdOnSVn3EHRwc9PXXX2vPnj2qXr26hg4dagkFefH19dWCBQu0cuVKy+PGru3nPG3aNBUrVkxNmjRRhw4d1Lp16+sGHRs3bpyOHj2q8uXL5zlY6hNPPKEPPvhAU6ZMUbVq1fTRRx9p3rx5tzzq+b/5+Pho8eLFevjhhxUWFqbo6Gh99dVXltG733nnHUVGRioqKkphYWFq06aNfvjhB8tt0qVLl9bbb7+tUaNGyd/fXwMHDrztWnIzb9489ezZU6+++qoqV66sJ554Qrt27bLcRv3mm2+qbt26at26tVq0aKGAgACrq8H5dfLkyRveYr13717t2LFD+/fvV4UKFazOr7///luS5Orqqrlz56pZs2YKCwvT0KFD9fjjj2vFihWW7WRmZurVV19VjRo11Lx5c/36669at26d1QWF5OTk68YMaNeunerUqaPly5dr48aNqlOnjmXAQOnK4/dMJpPV4/D+zcPDQ40aNdL777+vhx56SNWrV1dkZKReeOEFzZo1y6rt1cfUVqhQQT169NC5c+cUExOjkSNH5vv9vNacOXOUmZmpp556yuq9mzJlyi1tZ/Lkyeratat69OihBg0a6NixY1q/fr3lsY7SlScZnDt3zmq9r7/+Wmaz2TJAIwDAtkzmazsL5pPZbNa6dev0xx9/SLpyW+G1t17dqd69e2vdunX66KOPVK1aNf3yyy/q16+fnnvuOU2cOFHSlW+XJ0yYoM8//1xly5ZVZGSkfvvtN/3+++/5HgglNTVV3t7elsfLGNXChQvVvXt39Rw4TLVrXv+4lnvRkuWrtGX5t3r02b5q06KJvcuxiX2/HdQXs6ZpwYIFVgNwAQDuTRs2bNCTTz6pv/76yyrAAgBwJ/KbQ2/p9vj169dr4MCB+vnnn+Xl5WUZbEW68jzVatWqKTo6Os++cbdq5syZioyM1Msvv6xTp06pVKlSevHFFzVmzBhLm9dee00XLlxQv379lJKSombNmmnVqlWMXAoAAGxi5cqVev311wnsAAC7uKXQPn36dL3wwgu5fgvg7e2tF198UdOmTbNZaPf09NT06dM1ffr0PNuYTCaNGzdO48aNs8k+AQAA/u1m3TgAAChIt9Sn/ddff7V6dM+1Hn30Ue3Zs+eOiwIAAAAAALcY2pOSknJ91NtVjo6OOn369B0XBQAAAAAAbjG0ly5dWgcOHMhz+W+//Wa3Z6ECAAAAAFDY3FJob9eunSIjI3Xp0qXrll28eFFjx47VY489ZrPiAAAAAAC4n93SQHRvvvmmFi9erEqVKmngwIGqXLmyJOmPP/7Q7NmzlZ2drTfeeKNACgUAAAAA4H5zS6Hd399fP/30k1566SWNHj1aVx/xbjKZ1Lp1a82ePVv+/v4FUigAAAAAAPebWwrtkhQSEqKVK1fq7Nmz+vPPP2U2m1WxYkWeXQoAAAAAgI3dcmi/qlixYmrQoIEtawEAAAAAAP9ySwPRAQAAAACAu4fQDgAAAACAQRHaAQAAAAAwKEI7AAAAAAAGRWgHAAAAAMCgCO0AAAAAABgUoR0AAAAAAIMitAMAAAAAYFCEdgAAAAAADIrQDgAAAACAQRHaAQAAAAAwKEI7AAAAAAAGRWgHAAAAAMCgCO0AAAAAABgUoR0AAAAAAIMitAMAAAAAYFCEdgAAAAAADIrQDgAAAACAQRHaAQAAAAAwKEI7AAAAAAAGRWgHAAAAAMCgCO0AAAAAABgUoR0AAAAAAIMitAMAAAAAYFCEdgAAAAAADIrQDgAAAACAQRHaAQAAAAAwKEI7AAAAAAAGRWgHAAAAAMCgCO0AAAAAABgUoR0AAAAAAIMitAMAAAAAYFCEdgAAAAAADIrQDgAAAACAQRHaAQAAAAAwKEI7AAAAAAAGRWgHAAAAAMCgDB/ajx8/ru7du6t48eJydXVVjRo1tHv3bstys9msMWPGKDAwUK6urgoPD9fhw4ftWDEAAAAAALZh6NB+9uxZNW3aVEWLFtWPP/6o33//XVOnTlWxYsUsbSZNmqQZM2YoOjpaO3bskLu7u1q3bq1Lly7ZsXIAAAAAAO6co70LuJGJEycqKChI8+bNs8wrW7as5d9ms1nTp0/Xm2++qY4dO0qSvvjiC/n7+2vp0qXq2rVrrtvNyMhQRkaG5XVqamoBHQEAAAAAALfP0Ffaly1bpvr16+vpp59WyZIlVadOHc2dO9eyPD4+XomJiQoPD7fM8/b2VqNGjbR9+/Y8txsVFSVvb2/LFBQUVKDHAQAAAADA7TB0aP/rr780Z84cVaxYUatXr9ZLL72kwYMH6/PPP5ckJSYmSpL8/f2t1vP397csy83o0aN17tw5y/T3338X3EEAAAAAAHCbDH17fE5OjurXr6/x48dLkurUqaMDBw4oOjpavXr1uu3tOjs7y9nZ2VZlAgAAAABQIAx9pT0wMFBVq1a1mhcWFqaEhARJUkBAgCQpKSnJqk1SUpJlGQAAAAAA9ypDh/amTZsqLi7Oat6hQ4cUEhIi6cqgdAEBAYqJibEsT01N1Y4dO9S4ceO7WisAAAAAALZm6Nvjhw4dqiZNmmj8+PHq0qWLdu7cqY8//lgff/yxJMlkMmnIkCF69913VbFiRZUtW1aRkZEqVaqUnnjiCfsWDwAAAADAHTJ0aG/QoIGWLFmi0aNHa9y4cSpbtqymT5+uiIgIS5vXXntNFy5cUL9+/ZSSkqJmzZpp1apVcnFxsWPlAAAAAADcOUOHdkl67LHH9Nhjj+W53GQyady4cRo3btxdrAoAAAAAgIJn6D7tAAAAAADczwjtAAAAAAAYFKEdAAAAAACDIrQDAAAAAGBQhHYAAAAAAAyK0A4AAAAAgEER2gEAAAAAMChCOwAAAAAABkVoBwAAAADAoAjtAAAAAAAYFKEdAAAAAACDIrQDAAAAAGBQhHYAAAAAAAyK0A4AAAAAgEER2gEAAAAAMChCOwAAAAAABkVoBwAAAADAoAjtAAAAAAAYFKEdAAAAAACDIrQDAAAAAGBQhHYAAAAAAAyK0A4AAAAAgEER2gEAAAAAMChCOwAAAAAABkVoBwAAAADAoAjtAAAAAAAYFKEdAAAAAACDIrQDAAAAAGBQjvYuACis4uPjtXfvXnuXYTMlSpRQcHCwvcsAAAAA7iuEdsDG0jMuS5IiIyMVGRlp52psx83NVbGxfxDcAQAAgLuI0A7YWEbWldA+8ukH1KVVPTtXYxuxCafVffw3Sk5OJrQDAAAAdxGhHSggZfw8VbdSaXuXAQAAAOAexkB0AAAAAAAYFKEdAAAAAACDIrQDAAAAAGBQhHYAAAAAAAyK0A4AAAAAgEER2gEAAAAAMChCOwAAAAAABkVoBwAAAADAoAjtAAAAAAAYFKEdAAAAAACDIrQDAAAAAGBQhHYAAAAAAAzqngrtEyZMkMlk0pAhQyzzLl26pAEDBqh48eLy8PBQ586dlZSUZL8iAQAAAACwkXsmtO/atUsfffSRatasaTV/6NChWr58ub799ltt2rRJJ06c0JNPPmmnKgEAAAAAsJ17IrSnpaUpIiJCc+fOVbFixSzzz507p08//VTTpk3Tww8/rHr16mnevHn66aef9PPPP9uxYgAAAAAA7tw9EdoHDBig9u3bKzw83Gr+nj17lJWVZTW/SpUqCg4O1vbt2/PcXkZGhlJTU60mAAAAAACMxtHeBdzM119/rb1792rXrl3XLUtMTJSTk5N8fHys5vv7+ysxMTHPbUZFRentt9+2dakAAAAAANiUoa+0//3333rllVe0cOFCubi42Gy7o0eP1rlz5yzT33//bbNtAwAAAABgK4YO7Xv27NGpU6dUt25dOTo6ytHRUZs2bdKMGTPk6Ogof39/ZWZmKiUlxWq9pKQkBQQE5LldZ2dneXl5WU0AAAAAABiNoW+Pb9Wqlfbv3281r0+fPqpSpYpGjhypoKAgFS1aVDExMercubMkKS4uTgkJCWrcuLE9SgYAAAAAwGYMHdo9PT1VvXp1q3nu7u4qXry4ZX7fvn01bNgw+fr6ysvLS4MGDVLjxo31wAMP2KNkAAAAAABsxtChPT/ef/99OTg4qHPnzsrIyFDr1q314Ycf2rssQH+fTtXeQ8ftXYZNxCactncJAAAAwH3pngvtGzdutHrt4uKi2bNna/bs2fYpCLhGetp5SdKkb3do0rc77FyNbZ08edLeJQAAAAD3lXsutANGl5lxSZL0QJuOalKvlp2rsY24w3/ph28WXDfoIwAAAICCRWgHCoiXbwmVCQmxdxk2kXwuTZIUHx+vvXv32rka2yhRooSCg4PtXQYAAABwQ4R2ADeVnnFZkhQZGanIyEg7V2Mbbm6uio39g+AOAAAAQyO0A7ipjKwroX3k0w+oS6t6dq7mzsUmnFb38d8oOTmZ0A4AAABDI7QDyLcyfp6qW6m0vcsAAAAA7hsO9i4AAAAAAADkjtAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBOdq7AAD3jr9Pp2rvoeP2LuOOxSactncJAAAAQL4Q2gHcVHraeUnSpG93aNK3O+xcje2cPHnS3iUAAAAAN0RoB3BTmRmXJEkPtOmoJvVq2bmaOxd3+C/98M0CpaSk2LsUAAAA4IYI7QDyzcu3hMqEhNi7jDuWfC7N3iUAAAAA+cJAdAAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAAAAAAGBShHQAAAAAAgzJ8aI+KilKDBg3k6empkiVL6oknnlBcXJxVm0uXLmnAgAEqXry4PDw81LlzZyUlJdmpYgAAAAAAbMPwoX3Tpk0aMGCAfv75Z61du1ZZWVl69NFHdeHCBUuboUOHavny5fr222+1adMmnThxQk8++aQdqwYAAAAA4M452ruAm1m1apXV6/nz56tkyZLas2ePHnroIZ07d06ffvqpFi1apIcffliSNG/ePIWFhennn3/WAw88YI+yAQAAAAC4Y4YP7dc6d+6cJMnX11eStGfPHmVlZSk8PNzSpkqVKgoODtb27dtzDe0ZGRnKyMiwvE5NTS3gqgEYUXx8vPbu3WvvMmymRIkSCg4OtncZAAAAsKF7KrTn5ORoyJAhatq0qapXry5JSkxMlJOTk3x8fKza+vv7KzExMdftREVF6e233y7ocgEYVHrGZUlSZGSkIiMj7VyN7bi5uSo29g+COwAAQCFyT4X2AQMG6MCBA9q6desdbWf06NEaNmyY5XVqaqqCgoLutDwA94iMrCuhfeTTD6hLq3p2rsY2YhNOq/v4b5ScnExoBwAAKETumdA+cOBArVixQps3b1aZMmUs8wMCApSZmamUlBSrq+1JSUkKCAjIdVvOzs5ydnYu6JIBGFwZP0/VrVTa3mUAAADYVUJCgpKTk+1dhk0Vpm6Dhg/tZrNZgwYN0pIlS7Rx40aVLVvWanm9evVUtGhRxcTEqHPnzpKkuLg4JSQkqHHjxvYoGQAAAADuCQkJCQoLq6L09Iv2LsWmClO3QcOH9gEDBmjRokX6/vvv5enpaemn7u3tLVdXV3l7e6tv374aNmyYfH195eXlpUGDBqlx48aMHA/gvhMbG2vvEmyqMH1LfhVXMwAARpKcnKz09Ita8HoXhQX72bscmyhs3QYNH9rnzJkjSWrRooXV/Hnz5ql3796SpPfff18ODg7q3LmzMjIy1Lp1a3344Yd3uVIAsJ+TZ85Lkrp3727nSmzL1c1Nf8TGFopfuNKVwF4lLEwX09PtXYpNFbbPCQDuR2HBfnQbNCjDh3az2XzTNi4uLpo9e7Zmz559FyoCAONJSbskSWr/4huqXLNwDK6XlHBECyeOKDTfkktXrmZcTE9XxMjJ8g8ub+9ybKIwfk4AABiJ4UM7ACD/ipcKUZmK1exdBm7CP7g8nxMAAMgXB3sXAAAAAAAAckdoBwAAAADAoLg9HsB96+/Tqdp76Li9y7CJ+MSzkqSzKSk6efKknauxjcI2wvq/JScnq4gHnxMAALg5QjuA+0562pWR1id9u0OTvt1h52psa8P69dq8c5+9y7CJ7LQzklRovoSQ/u9YFi9erCIevnauxjYK4+cEAICRENoB3HcyM66MtP5Am45qUq+WnauxjQ2bf9IvW9apVvmSatKgjr3LsYk/4g5r+a9SSkqKvUuxmavH0rJOOVWpXNG+xdhIYfycAAAwEkI7gPuWl28JlQkJsXcZNuHhEytJcncpqsDiXnauxjaSPFzsXUKBKebhwucEAADyhYHoAAAAAAAwKEI7AAAAAAAGRWgHAAAAAMCgCO0AAAAAABgUoR0AAAAAAIMitAMAAAAAYFCEdgAAAAAADIrntAMADC0+Pl579+61dxk2ER8fb+8SgEIjISFBycnJ9i7DpkqUKKHg4GB7lwHAYAjtAABDSs+4LEmKjIxUZGSknauxraysy/YuAbinJSQkKCysitLTL9q7FJtyc3NVbOwfBHcAVgjtAABDyvj/wXbk0w+oS6t6dq7GNj5ZuVtzlu1QdjahHbgTycnJSk+/qAWvd1FYsJ+9y7GJ2ITT6j7+GyUnJxPaAVghtAMADK2Mn6fqVipt7zJsotTOQ/YuocAUpm4MErcp3yvCgv0Kzf8PV8XGxtq7BJviZwm4c4R2AABw2wprNwZuU8bddvLMeZkkde/e3d6l2BQ/S8CdI7QDAIDbVhi7MXCbMuwhJe2SzJJmvfyoGtesaO9ybIKfJcA2CO0AAOCOFaZuDIA9VShVjJ8lAFZ4TjsAAAAAAAZFaAcAAAAAwKC4PR4AAKCQS0hIUHJysr3LsJnCNsI6ANwIoR0AAKAQS0hIUFhYFaWnX7R3KTZ3Pi3N3iUAQIEjtAMAAOSisFzNjY2NVXr6RS14vYvCgv3sXY5NrNx5SJGfrdWlS5fsXQoAFDhCOwAAwL8U1udll/F1LTSjkscmnLZ3CQBw1xDaAQAA/qWwPS+bq9IAcG8jtAMAAOSisDwvm6vSAHBv45FvAAAAAAAYFKEdAAAAAACDIrQDAAAAAGBQhHYAAAAAAAyK0A4AAAAAgEExejwAAAAA5FNCQoKSk5PtXYbNxMbG2rsE3AShHQAAAADyISEhQWFhVZSeftHepdjc+bQ0e5eAPBDaAQCG9vfpVO09dNzeZdjEieRUe5cAALgDycnJSk+/qAWvd1FYsJ+9y7GJlTsPKfKztdr/1wl5enjYuxybiE04be8SbIrQDgAwpPS085KkSd/u0KRvd9i5Gtu6mHnZ3iUAAO5AWLCf6lYqbe8ybGLL/qOSpEEfrpW01q612NrJkyftXYJNENoBAIaUmXFJkvRAm45qUq+WnauxjQ2bf9IvW9Yp43K2vUsBAECSdP5ipiQp/ImnVbNamJ2rsY24w3/ph28WKCUlxd6l2AShHQBgaF6+JVQmJMTeZdiEh0/hHeynMHVjiE88K0k6fOKM/ArBMV09nsKosHxGUuH+nHBv8C3hV2h+3yafK1z98wntAADgthXmbgyF7VbRUykX7F2CzSSfu3Ishe0zkgrX5wTANgjtAADgthXmbgyF5Zh2/xarzcu/1bn0DHuXYjOF8Xbewvg5AbANQjsAALhjhbEbQ2E5psPHrzxPujB1Ybj6JIbCdDtvYfycro7gXZieA371WArT6OQ82cT4CO0AAACFWGHuwlCYnsRQmD+n7t2727sEm+s+/ht7l2BzhennqbAhtAMAABRihbkLQ2F6EkNh/Jyu3vI/8ukH1KVVPXuXYxOfrNytOct2FKquGYXx56mwKTShffbs2Zo8ebISExNVq1YtzZw5Uw0bNrR3WQAAAIZQWG73lwr3kxgK0+d09ZZ/s8x2rsT2ClPXjML881RYFIrQ/p///EfDhg1TdHS0GjVqpOnTp6t169aKi4tTyZIl7V0eAAAAcN8pzLf8cys57qZCEdqnTZumF154QX369JEkRUdH64cfftBnn32mUaNG2bk6AAAA4P5TGG/551Zy2MM9H9ozMzO1Z88ejR492jLPwcFB4eHh2r59e67rZGRkKCPj/x6nce7cOUlSaqqxR05MT0+XJP3551/KzCwcjwNJOnFCknTi77+1c/deO1djGxyT8RW245E4pnsFx3RvKGzHVNiOR+KY7hVXjyntwkWdOP2PnauxjQvpFyUVzs+pMB1TQsKVJzCkp6cbOuNdrc1svnEXEpP5Zi0M7sSJEypdurR++uknNW7c2DL/tdde06ZNm7Rjx/W34rz11lt6++2372aZAAAAAABc5++//1aZMmXyXH7PX2m/HaNHj9awYcMsr3NycnTmzBkVL15cJpPJjpXlLTU1VUFBQfr777/l5eVl73KAAsX5jvsJ5zvuJ5zvuJ9wvuNmzGazzp8/r1KlSt2w3T0f2kuUKKEiRYooKSnJan5SUpICAgJyXcfZ2VnOzs5W83x8fAqqRJvy8vLihx73Dc533E8433E/4XzH/YTzHTfi7e190zYOd6GOAuXk5KR69eopJibGMi8nJ0cxMTFWt8sDAAAAAHCvueevtEvSsGHD1KtXL9WvX18NGzbU9OnTdeHCBcto8gAAAAAA3IsKRWh/5plndPr0aY0ZM0aJiYmqXbu2Vq1aJX9/f3uXZjPOzs4aO3bsdbf1A4UR5zvuJ5zvuJ9wvuN+wvkOW7nnR48HAAAAAKCwuuf7tAMAAAAAUFgR2gEAAAAAMChCOwAAAAAABkVoBwAAAADAoAjt94jZs2crNDRULi4uatSokXbu3GnvkoBbEhUVpQYNGsjT01MlS5bUE088obi4OKs2ly5d0oABA1S8eHF5eHioc+fOSkpKsmqTkJCg9u3by83NTSVLltSIESN0+fLlu3kowC2bMGGCTCaThgwZYpnH+Y7C5Pjx4+revbuKFy8uV1dX1ahRQ7t377YsN5vNGjNmjAIDA+Xq6qrw8HAdPnzYahtnzpxRRESEvLy85OPjo759+yotLe1uHwpwQ9nZ2YqMjFTZsmXl6uqq8uXL65133tG/x/bmfIetEdrvAf/5z380bNgwjR07Vnv37lWtWrXUunVrnTp1yt6lAfm2adMmDRgwQD///LPWrl2rrKwsPfroo7pw4YKlzdChQ7V8+XJ9++232rRpk06cOKEnn3zSsjw7O1vt27dXZmamfvrpJ33++eeaP3++xowZY49DAvJl165d+uijj1SzZk2r+ZzvKCzOnj2rpk2bqmjRovrxxx/1+++/a+rUqSpWrJilzaRJkzRjxgxFR0drx44dcnd3V+vWrXXp0iVLm4iICB08eFBr167VihUrtHnzZvXr188ehwTkaeLEiZozZ45mzZql2NhYTZw4UZMmTdLMmTMtbTjfYXNmGF7Dhg3NAwYMsLzOzs42lypVyhwVFWXHqoA7c+rUKbMk86ZNm8xms9mckpJiLlq0qPnbb7+1tImNjTVLMm/fvt1sNpvNK1euNDs4OJgTExMtbebMmWP28vIyZ2Rk3N0DAPLh/Pnz5ooVK5rXrl1rbt68ufmVV14xm82c7yhcRo4caW7WrFmey3NycswBAQHmyZMnW+alpKSYnZ2dzV999ZXZbDabf//9d7Mk865duyxtfvzxR7PJZDIfP3684IoHblH79u3Nzz33nNW8J5980hwREWE2mznfUTC40m5wmZmZ2rNnj8LDwy3zHBwcFB4eru3bt9uxMuDOnDt3TpLk6+srSdqzZ4+ysrKszvUqVaooODjYcq5v375dNWrUkL+/v6VN69atlZqaqoMHD97F6oH8GTBggNq3b291Xkuc7yhcli1bpvr16+vpp59WyZIlVadOHc2dO9eyPD4+XomJiVbnu7e3txo1amR1vvv4+Kh+/fqWNuHh4XJwcNCOHTvu3sEAN9GkSRPFxMTo0KFDkqRff/1VW7duVdu2bSVxvqNgONq7ANxYcnKysrOzrf5okyR/f3/98ccfdqoKuDM5OTkaMmSImjZtqurVq0uSEhMT5eTkJB8fH6u2/v7+SkxMtLTJ7Wfh6jLASL7++mvt3btXu3btum4Z5zsKk7/++ktz5szRsGHD9Prrr2vXrl0aPHiwnJyc1KtXL8v5mtv5/O/zvWTJklbLHR0d5evry/kOQxk1apRSU1NVpUoVFSlSRNnZ2XrvvfcUEREhSZzvKBCEdgB33YABA3TgwAFt3brV3qUABeLvv//WK6+8orVr18rFxcXe5QAFKicnR/Xr19f48eMlSXXq1NGBAwcUHR2tXr162bk6wLa++eYbLVy4UIsWLVK1atW0b98+DRkyRKVKleJ8R4Hh9niDK1GihIoUKXLdiMJJSUkKCAiwU1XA7Rs4cKBWrFihDRs2qEyZMpb5AQEByszMVEpKilX7f5/rAQEBuf4sXF0GGMWePXt06tQp1a1bV46OjnJ0dNSmTZs0Y8YMOTo6yt/fn/MdhUZgYKCqVq1qNS8sLEwJCQmS/u98vdHfMgEBAdcNsHv58mWdOXOG8x2GMmLECI0aNUpdu3ZVjRo11KNHDw0dOlRRUVGSON9RMAjtBufk5KR69eopJibGMi8nJ0cxMTFq3LixHSsDbo3ZbNbAgQO1ZMkSrV+/XmXLlrVaXq9ePRUtWtTqXI+Li1NCQoLlXG/cuLH2799v9Ytu7dq18vLyuu4PRsCeWrVqpf3792vfvn2WqX79+oqIiLD8m/MdhUXTpk2ve4TnoUOHFBISIkkqW7asAgICrM731NRU7dixw+p8T0lJ0Z49eyxt1q9fr5ycHDVq1OguHAWQP+np6XJwsI5QRYoUUU5OjiTOdxQQe4+Eh5v7+uuvzc7Ozub58+ebf//9d3O/fv3MPj4+ViMKA0b30ksvmb29vc0bN240nzx50jKlp6db2vTv398cHBxsXr9+vXn37t3mxo0bmxs3bmxZfvnyZXP16tXNjz76qHnfvn3mVatWmf38/MyjR4+2xyEBt+Tfo8ebzZzvKDx27txpdnR0NL/33nvmw4cPmxcuXGh2c3MzL1iwwNJmwoQJZh8fH/P3339v/u2338wdO3Y0ly1b1nzx4kVLmzZt2pjr1Klj3rFjh3nr1q3mihUrmrt162aPQwLy1KtXL3Pp0qXNK1asMMfHx5sXL15sLlGihPm1116ztOF8h60R2u8RM2fONAcHB5udnJzMDRs2NP/888/2Lgm4JZJynebNm2dpc/HiRfPLL79sLlasmNnNzc3cqVMn88mTJ622c/ToUXPbtm3Nrq6u5hIlSphfffVVc1ZW1l0+GuDWXRvaOd9RmCxfvtxcvXp1s7Ozs7lKlSrmjz/+2Gp5Tk6OOTIy0uzv7292dnY2t2rVyhwXF2fV5p9//jF369bN7OHhYfby8jL36dPHfP78+bt5GMBNpaamml955RVzcHCw2cXFxVyuXDnzG2+8YfUoTs532JrJbDab7XmlHwAAAAAA5I4+7QAAAAAAGBShHQAAAAAAgyK0AwAAAABgUIR2AAAAAAAMitAOAAAAAIBBEdoBAAAAADAoQjsAAAAAAAZFaAcAAAAAwKAI7QAA3OeOHj0qk8mkffv25dlm48aNMplMSklJuaN92Wo7AADcLxztXQAAALCvoKAgnTx5UiVKlCjwfTVp0kQnT56Ut7d3ge8LAIDCgNAOAMB9LDMzU05OTgoICLgr+7ub+wIAoDDg9ngAAAqR8+fPKyIiQu7u7goMDNT777+vFi1aaMiQIZKk0NBQvfPOO+rZs6e8vLzUr1+/XG+PX7lypSpVqiRXV1e1bNlSR48ezXcNx44dU4cOHVSsWDG5u7urWrVqWrlypaTrb49v0aKFTCbTddPV/aWkpOj555+Xn5+fvLy89PDDD+vXX3+1wTsFAMC9gdAOAEAhMmzYMG3btk3Lli3T2rVrtWXLFu3du9eqzZQpU1SrVi398ssvioyMvG4bf//9t5588kl16NBB+/bt0/PPP69Ro0blu4YBAwYoIyNDmzdv1v79+zVx4kR5eHjk2nbx4sU6efKkZXryySdVuXJl+fv7S5KefvppnTp1Sj/++KP27NmjunXrqlWrVjpz5swtvCsAANy7uD0eAIBC4vz58/r888+1aNEitWrVSpI0b948lSpVyqrdww8/rFdffdXy+tqr6HPmzFH58uU1depUSVLlypUt4Ts/EhIS1LlzZ9WoUUOSVK5cuTzb+vr6Wv79/vvva/369dqxY4dcXV21detW7dy5U6dOnZKzs7OkK184LF26VP/973/Vr1+/fNUDAMC9jNAOAEAh8ddffykrK0sNGza0zPP29lblypWt2tWvX/+G24mNjVWjRo2s5jVu3DjfdQwePFgvvfSS1qxZo/DwcHXu3Fk1a9a84To//vijRo0apeXLl6tSpUqSpF9//VVpaWkqXry4VduLFy/qyJEj+a4HAIB7GaEdAID7jLu7e4Fu//nnn1fr1q31ww8/aM2aNYqKitLUqVM1aNCgXNv//vvv6tq1qyZMmKBHH33UMj8tLU2BgYHauHHjdev4+PgUUPUAABgLfdoBACgkypUrp6JFi2rXrl2WeefOndOhQ4duaTthYWHauXOn1byff/75lrYRFBSk/v37a/HixXr11Vc1d+7cXNslJyerQ4cO6ty5s4YOHWq1rG7dukpMTJSjo6MqVKhgNd2Nx9MBAGAEhHYAAAoJT09P9erVSyNGjNCGDRt08OBB9e3bVw4ODjKZTPneTv/+/XX48GGNGDFCcXFxWrRokebPn5/v9YcMGaLVq1crPj5ee/fu1YYNGxQWFpZr286dO8vNzU1vvfWWEhMTLVN2drbCw8PVuHFjPfHEE1qzZo2OHj2qn376SW+88YZ2796d73oAALiXEdoBAChEpk2bpsaNG+uxxx5TeHi4mjZtqrCwMLm4uOR7G8HBwfruu++0dOlS1apVS9HR0Ro/fny+18/OztaAAQMUFhamNm3aqFKlSvrwww9zbbt582YdOHBAISEhCgwMtEx///23TCaTVq5cqYceekh9+vRRpUqV1LVrVx07dswyujwAAIWdyWw2m+1dBAAAKBgXLlxQ6dKlNXXqVPXt29fe5QAAgFvEQHQAABQiv/zyi/744w81bNhQ586d07hx4yRJHTt2tHNlAADgdnB7PAAAhcyUKVNUq1YthYeH68KFC9qyZYtNB25r27atPDw8cp1u5TZ6AABwc9weDwAAbsnx48d18eLFXJf5+vrK19f3LlcEAEDhRWgHAAAAAMCguD0eAAAAAACDIrQDAAAAAGBQhHYAAAAAAAyK0A4AAAAAgEER2gEAAAAAMChCOwAAAAAABkVoBwAAAADAoP4fOsz80eeGA8UAAAAASUVORK5CYII=", 198 | "text/plain": [ 199 | "
" 200 | ] 201 | }, 202 | "metadata": {}, 203 | "output_type": "display_data" 204 | } 205 | ], 206 | "source": [ 207 | "# plot grid size distribution\n", 208 | "plt.figure(figsize=(12, 6))\n", 209 | "sns.histplot(dataset, x=\"grid_size\", bins=16, hue=\"set\")\n", 210 | "plt.title(\"Grid size distribution\")\n", 211 | "training_set_mean = training_set_tests.select(pl.mean(\"grid_size\")).item()\n", 212 | "evaluation_set_mean = evaluation_set_tests.select(pl.mean(\"grid_size\")).item()\n", 213 | "training_set_sd = training_set_tests.select(pl.std(\"grid_size\")).item()\n", 214 | "evaluation_set_sd = evaluation_set_tests.select(pl.std(\"grid_size\")).item()\n", 215 | "plt.text(\n", 216 | " 0.5,\n", 217 | " 0.5,\n", 218 | " f\"Training set mean: {round(training_set_mean, 1)}, SD={round(training_set_sd, 1)}\\nEvaluation set mean: {round(evaluation_set_mean, 1)}, SD={round(evaluation_set_sd, 1)}\",\n", 219 | " horizontalalignment=\"center\",\n", 220 | " verticalalignment=\"center\",\n", 221 | " transform=plt.gca().transAxes,\n", 222 | ")\n", 223 | "plt.plot()\n", 224 | "plt.show()" 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "execution_count": 10, 230 | "metadata": {}, 231 | "outputs": [ 232 | { 233 | "name": "stdout", 234 | "output_type": "stream", 235 | "text": [ 236 | "Test statistic: -6.81421447779164\n", 237 | "P-value: 1.9843266953504276e-11\n", 238 | "Degrees of freedom: 730.1473878533382\n" 239 | ] 240 | } 241 | ], 242 | "source": [ 243 | "# run permutation test\n", 244 | "training_set_grid_sizes = training_set_tests.select(\"grid_size\").to_numpy().flatten()\n", 245 | "evaluation_set_grid_sizes = (\n", 246 | " evaluation_set_tests.select(\"grid_size\").to_numpy().flatten()\n", 247 | ")\n", 248 | "\n", 249 | "stats = ttest_ind(\n", 250 | " training_set_grid_sizes,\n", 251 | " evaluation_set_grid_sizes,\n", 252 | " equal_var=False,\n", 253 | ")\n", 254 | "print(f\"Test statistic: {stats.statistic}\")\n", 255 | "print(f\"P-value: {stats.pvalue}\")\n", 256 | "print(f\"Degrees of freedom: {stats.df}\")" 257 | ] 258 | }, 259 | { 260 | "cell_type": "markdown", 261 | "metadata": {}, 262 | "source": [ 263 | "Permutation tests show that output grid sizes are significantly smaller in the training set than in the evaluation set (p < .001).\n" 264 | ] 265 | }, 266 | { 267 | "cell_type": "code", 268 | "execution_count": 11, 269 | "metadata": {}, 270 | "outputs": [], 271 | "source": [ 272 | "dataset.write_csv(\"../data/ARC_all_tasks_tests_stats.csv\")" 273 | ] 274 | } 275 | ], 276 | "metadata": { 277 | "kernelspec": { 278 | "display_name": ".venv", 279 | "language": "python", 280 | "name": "python3" 281 | }, 282 | "language_info": { 283 | "codemirror_mode": { 284 | "name": "ipython", 285 | "version": 3 286 | }, 287 | "file_extension": ".py", 288 | "mimetype": "text/x-python", 289 | "name": "python", 290 | "nbconvert_exporter": "python", 291 | "pygments_lexer": "ipython3", 292 | "version": "3.12.5" 293 | } 294 | }, 295 | "nbformat": 4, 296 | "nbformat_minor": 2 297 | } 298 | --------------------------------------------------------------------------------