├── .gitignore ├── LICENSE ├── README.md ├── analysis ├── analysis_blimp.py ├── analysis_sg.py └── results │ ├── blimp │ ├── gpt-2.txt │ ├── sclm-next_bllip-lg_rand-init_1101_5.txt │ ├── sclm-next_bllip-lg_rand-init_1102_5.txt │ ├── sclm-next_bllip-lg_rand-init_1103_5.txt │ ├── sclm-next_bllip-lg_rand-init_1104_5.txt │ ├── sclm-next_bllip-md_rand-init_1101_5.txt │ ├── sclm-next_bllip-md_rand-init_1102_5.txt │ ├── sclm-next_bllip-md_rand-init_1103_5.txt │ ├── sclm-next_bllip-md_rand-init_1104_5.txt │ ├── sclm-past_bllip-lg_rand-init_1101_5.txt │ ├── sclm-past_bllip-lg_rand-init_1102_5.txt │ ├── sclm-past_bllip-lg_rand-init_1103_5.txt │ ├── sclm-past_bllip-lg_rand-init_1104_5.txt │ ├── sclm-past_bllip-md_rand-init_1101_5.txt │ ├── sclm-past_bllip-md_rand-init_1102_5.txt │ ├── sclm-past_bllip-md_rand-init_1103_5.txt │ ├── sclm-past_bllip-md_rand-init_1104_5.txt │ ├── xlm_bllip-lg_rand-init_1101_5.txt │ ├── xlm_bllip-lg_rand-init_1102_5.txt │ ├── xlm_bllip-lg_rand-init_1103_5.txt │ ├── xlm_bllip-md_rand-init_1101_5.txt │ ├── xlm_bllip-md_rand-init_1102_5.txt │ ├── xlm_bllip-md_rand-init_1103_5.txt │ ├── xplm-mask_bllip-lg_rand-init_1101_5.txt │ ├── xplm-mask_bllip-lg_rand-init_1102_5.txt │ ├── xplm-mask_bllip-lg_rand-init_1103_5.txt │ ├── xplm-mask_bllip-md_rand-init_1101_5.txt │ ├── xplm-mask_bllip-md_rand-init_1102_5.txt │ ├── xplm-mask_bllip-md_rand-init_1103_5.txt │ ├── xplm_-mask_bllip-lg_rand-init_1101_5.txt │ ├── xplm_-mask_bllip-lg_rand-init_1102_5.txt │ ├── xplm_-mask_bllip-lg_rand-init_1103_5.txt │ ├── xplm_-mask_bllip-md_rand-init_1101_5.txt │ ├── xplm_-mask_bllip-md_rand-init_1102_5.txt │ ├── xplm_-mask_bllip-md_rand-init_1103_5.txt │ ├── xplm_bllip-lg_rand-init_1101_5.txt │ ├── xplm_bllip-lg_rand-init_1102_5.txt │ ├── xplm_bllip-lg_rand-init_1103_5.txt │ ├── xplm_bllip-md_rand-init_1101_5.txt │ ├── xplm_bllip-md_rand-init_1102_5.txt │ └── xplm_bllip-md_rand-init_1103_5.txt │ ├── model_ppl_list.txt │ └── sg │ ├── gpt-2.txt │ ├── sclm-next_bllip-lg_rand-init_1101_5.txt │ ├── sclm-next_bllip-lg_rand-init_1102_5.txt │ ├── sclm-next_bllip-lg_rand-init_1103_5.txt │ ├── sclm-next_bllip-lg_rand-init_1104_5.txt │ ├── sclm-next_bllip-md_rand-init_1101_5.txt │ ├── sclm-next_bllip-md_rand-init_1102_5.txt │ ├── sclm-next_bllip-md_rand-init_1103_5.txt │ ├── sclm-next_bllip-md_rand-init_1104_5.txt │ ├── sclm-past_bllip-lg_rand-init_1101_5.txt │ ├── sclm-past_bllip-lg_rand-init_1102_5.txt │ ├── sclm-past_bllip-lg_rand-init_1103_5.txt │ ├── sclm-past_bllip-lg_rand-init_1104_5.txt │ ├── sclm-past_bllip-md_rand-init_1101_5.txt │ ├── sclm-past_bllip-md_rand-init_1102_5.txt │ ├── sclm-past_bllip-md_rand-init_1103_5.txt │ ├── sclm-past_bllip-md_rand-init_1104_5.txt │ ├── syntaxgym_results_rnng_bllip-lg.csv │ ├── syntaxgym_results_rnng_bllip-md.csv │ ├── xlm_bllip-lg_rand-init_1101_5.txt │ ├── xlm_bllip-lg_rand-init_1102_5.txt │ ├── xlm_bllip-lg_rand-init_1103_5.txt │ ├── xlm_bllip-md_rand-init_1101_5.txt │ ├── xlm_bllip-md_rand-init_1102_5.txt │ ├── xlm_bllip-md_rand-init_1103_5.txt │ ├── xplm-mask_bllip-lg_rand-init_1101_5.txt │ ├── xplm-mask_bllip-lg_rand-init_1102_5.txt │ ├── xplm-mask_bllip-lg_rand-init_1103_5.txt │ ├── xplm-mask_bllip-md_rand-init_1101_5.txt │ ├── xplm-mask_bllip-md_rand-init_1102_5.txt │ ├── xplm-mask_bllip-md_rand-init_1103_5.txt │ ├── xplm_bllip-lg_rand-init_1101_5.txt │ ├── xplm_bllip-lg_rand-init_1102_5.txt │ ├── xplm_bllip-lg_rand-init_1103_5.txt │ ├── xplm_bllip-md_rand-init_1101_5.txt │ ├── xplm_bllip-md_rand-init_1102_5.txt │ └── xplm_bllip-md_rand-init_1103_5.txt ├── requirements.txt ├── scripts ├── download_and_patch_transformers.sh └── generate_patch.sh └── src ├── get_action_ngram_list.py ├── get_oracle.py ├── lm-sc.py ├── lm.py ├── plm-gen.py ├── transformers-v3.3.1-x.patch └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.params 2 | *.zip 3 | logs/ 4 | model_params/ 5 | data/ 6 | output/ 7 | oracle_gen/ 8 | pretrained/ 9 | src/transformers/ 10 | train_logs/ 11 | eval_logs 12 | *.sh 13 | surprisals/ 14 | eval_scripts/ 15 | util_scripts/ 16 | backup/ 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 International Business Machines 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Structural Guidance for Transformer Language Models 2 | 3 | This repository accompanies the paper, [Structural Guidance for Transformer Language Models](https://arxiv.org/pdf/2108.00104.pdf), published in ACL 2021. It includes inplementation of parsing-as-language-modelling and structural scaffolding for Transformer language models. 4 | 5 | ## Environment 6 | 7 | The code is based on Python3. You can install the different modules with 8 | ``` 9 | bash scripts/download_and_patch_transformers.sh 10 | pip install -r requirements.txt 11 | python -c "import nltk;nltk.download('punkt')" 12 | ``` 13 | 14 | The Huggingface transformers is updated indirectly through a patch. If you 15 | modifiy the code, to commit changes run 16 | 17 | ``` 18 | bash scripts/generate_patch.sh 19 | ``` 20 | 21 | and then just commit this patch 22 | 23 | 24 | ## Data preparation 25 | 26 | ### Prepare parsing oracle files 27 | 28 | PLM and ScLM require syntactic parses to derive the action sequence oracle. The following command demonstrates how to prepare oracle files for these models. 29 | 30 | ``` 31 | python src/get_oracle.py --gen --fpath train.txt > train_gen.oracle 32 | python src/get_oracle.py --gen --fpath dev.txt > dev_gen.oracle 33 | python src/get_oracle.py --gen --fpath test.txt > test_gen.oracle 34 | ``` 35 | 36 | ### Prepare action ngram list 37 | 38 | The following command generates the action ngram list for ScLM models. The training code of ScLM assumes that the action ngram list is stored in the root folder. 39 | 40 | ``` 41 | python src/get_action_ngram_list.py -f path/to/bllip-lg_train_gen.oracle path/to/bllip-lg_dev_gen.oracle -o bllip-lg_action_ngram_list.txt 42 | ``` 43 | 44 | ## Vanilla Language Models (LM) 45 | 46 | The script `src/lm.py` implements a vanilla Transformer language model. Below are the commands for model training and evaluation, as well as commands to compute word-level surprisals from a trained model. 47 | 48 | ``` 49 | # Model training 50 | python src/lm.py --train_data train.txt --dev_data dev.txt --lr 1e-5 --epochs ${EPOCHS} --seed ${SEED} --do_train --random_init --batch_size ${BATCH_SIZE} --report ${REPORT} --sample_every ${SAMPLE_EVERY} --model_path ${MODEL_PATH} 51 | 52 | # Compute word-level perplexity 53 | python src/lm.py --restore_from ${MODEL_PATH} --test_data test.txt --do_test 54 | 55 | # Estimate word surprisals 56 | python src/lm.py --restore_from ${MODEL_PATH} --do_eval --fpath ${TEST_SUITE_PATH} --pretokenized > ${OUTPUT_PATH} 57 | ``` 58 | 59 | ## Scaffolded Language Models (ScLM) 60 | 61 | The script `src/lm-sc.py` implements Transformer language model with structural prediction as an auxilliary task, referred as ScLM. The commanline variable, ${SCAFFOLD_TYPE}, can be set as `past` or `next`, which corresponds to `ScLM-past` or `ScLM-next` respectively in the paper. 62 | 63 | ``` 64 | # Model training 65 | python src/lm-sc.py --train_data train_gen.oracle --dev_data dev_gen.oracle --lr 1e-5 --epochs ${EPOCHS} --seed ${SEED} --do_train --random_init --batch_size ${BATCH_SIZE} --report ${REPORT} --sample_every ${SAMPLE_EVERY} --alpha 0.5 --scaffold_type ${SCAFFOLD_TYPE} --model_path ${MODEL_PATH} 66 | 67 | # Compute word-level perplexity 68 | python src/lm-sc.py --restore_from ${MODEL_PATH} --test_data test_gen.oracle --do_test 69 | 70 | # Estimate word surprisals 71 | python src/lm-sc.py --restore_from ${MODEL_PATH} --do_eval --fpath ${TEST_SUITE_PATH} --pretokenized > ${OUTPUT_PATH} 72 | ``` 73 | 74 | ## Parsing as Language Modelling (PLM/PLM-mask) 75 | 76 | The script `src/plm-gen.py` implements the idea of generative parsing as language modelling, a probabilistic model of top-down parsing action sequence. There are two variants, PLM and PLM-mask. 77 | 78 | For PLM: 79 | ``` 80 | # Model training for PLM 81 | python src/plm-gen.py --train_data train_gen.oracle --dev_data dev_gen.oracle --lr 1e-5 --epochs ${EPOCHS} --seed ${SEED} --do_train --batch_size ${BATCH_SIZE} --random_init --report ${REPORT} --sample_every ${SAMPLE_EVERY} --model_path ${MODEL_PATH} 82 | 83 | # Estimate word-level perplexity with PLM 84 | python src/plm-gen.py --restore_from ${MODEL_PATH} --test_data test_gen.oracle --do_test 85 | 86 | # Estimate word surprisals with PLM 87 | python src/plm-gen.py --restore_from ${MODEL_PATH} --do_eval --beam_size 100 --word_beam_size 10 --fast_track_size 5 --pretokenized --fpath ${TEST_SUITE_PATH} > ${OUTPUT_PATH} 2>${EVAL_LOG_PATH} 88 | ``` 89 | 90 | For PLM-mask: 91 | ``` 92 | # Model training for PLM-mask 93 | python src/plm-gen.py --train_data train_gen.oracle --dev_data dev_gen.oracle --lr 1e-5 --epochs ${EPOCHS} --seed ${SEED} --do_train --batch_size ${BATCH_SIZE} --random_init --add_structured_mask --buffer_head 0 --stack_head 1 --report ${REPORT} --sample_every ${SAMPLE_EVERY} --model_path ${MODEL_PATH} 94 | 95 | # Estimate word-level perplexity with PLM-mask 96 | python src/plm-gen.py --restore_from ${MODEL_PATH} --add_structured_mask --buffer_head 0 --stack_head 1 --test_data test_gen.oracle --do_test 97 | 98 | # Estimate word surprisals with PLM-mask 99 | python src/plm-gen.py --restore_from ${MODEL_PATH} --add_structured_mask --buffer_head 0 --stack_head 1 --do_eval --beam_size 100 --word_beam_size 10 --fast_track_size 5 --pretokenized --fpath ${TEST_SUITE_PATH} > ${OUTPUT_PATH} 2>>${EVAL_LOG_PATH} 100 | ``` 101 | 102 | ## Trained models 103 | 104 | Parameters of trained models are available [here](https://www.dropbox.com/sh/akprbzhr0touxz0/AACUHwzA0UYopEPR6QtJYdu8a?dl=0). 105 | 106 | ## Plot figures 107 | 108 | The `analysis` folder contains the code and model evaluation results for generating the figures in the paper. The following commands run the plotting scripts and generate figures in the `figs` folder. Python packages `matplotlib` and `pandas` are required to run the plotting scripts. RNNG results are taken from [Hu et al., (2020)](https://github.com/cpllab/syntactic-generalization/tree/master/data/raw/sg_results). 109 | 110 | ``` 111 | cd analysis 112 | mkdir -p figs 113 | 114 | # Plot results on SG Test Suites and BLiMP-10%. 115 | python analysis_sg.py 116 | python analysis_blimp.py 117 | ``` 118 | 119 | ## Acknowledgements 120 | 121 | We thank Ramon Astudillo and Tahira Naseem for their contributions to the repository. 122 | -------------------------------------------------------------------------------- /analysis/analysis_blimp.py: -------------------------------------------------------------------------------- 1 | # This script generates figures of model performance comparisons on BLiMP-10%. 2 | 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | from matplotlib.lines import Line2D 6 | import matplotlib.patches as mpatches 7 | from matplotlib import rcParams 8 | from matplotlib.patches import Patch 9 | rcParams['font.family'] = 'Arial' 10 | 11 | 12 | def load_results(path): 13 | with open(path) as f: 14 | lines = f.readlines() 15 | lines = [line.strip().split() for line in lines] 16 | lines = [(line[0], float(line[1])) for line in lines] 17 | return lines 18 | 19 | 20 | phenomena = { 21 | "Anaphor Agreement": ["anaphor_gender_agreement", "anaphor_number_agreement"], 22 | "Argument Structure": ["animate_subject_passive", "animate_subject_trans", "causative", "drop_argument", "inchoative", "intransitive", 23 | "passive_1", "passive_2", "transitive"], 24 | "Binding": ["principle_A_c_command", "principle_A_case_1", "principle_A_case_2", 25 | "principle_A_domain_1", "principle_A_domain_2", "principle_A_domain_3", "principle_A_reconstruction"], 26 | "Control/Raising": ["existential_there_object_raising", "existential_there_subject_raising", "expletive_it_object_raising", 27 | "tough_vs_raising_1", "tough_vs_raising_2"], 28 | "Determiner-Noun Agreement": ["determiner_noun_agreement_1", "determiner_noun_agreement_2", 29 | "determiner_noun_agreement_irregular_1", "determiner_noun_agreement_irregular_2", 30 | "determiner_noun_agreement_with_adjective_1", "determiner_noun_agreement_with_adj_2", 31 | "determiner_noun_agreement_with_adj_irregular_1", "determiner_noun_agreement_with_adj_irregular_2"], 32 | "Ellipsis": ["ellipsis_n_bar_1", "ellipsis_n_bar_2"], 33 | "Filler Gap": ["wh_questions_object_gap", "wh_questions_subject_gap", "wh_questions_subject_gap_long_distance", 34 | "wh_vs_that_no_gap", "wh_vs_that_no_gap_long_distance", "wh_vs_that_with_gap", "wh_vs_that_with_gap_long_distance"], 35 | "Irregular Forms": ["irregular_past_participle_adjectives", "irregular_past_participle_verbs"], 36 | "Island Effects": ["adjunct_island", "complex_NP_island", "coordinate_structure_constraint_complex_left_branch", 37 | "coordinate_structure_constraint_object_extraction", "left_branch_island_echo_question", 38 | "left_branch_island_simple_question", "sentential_subject_island", "wh_island"], 39 | "NPI Licensing": ["matrix_question_npi_licensor_present", "npi_present_1", "npi_present_2", "only_npi_licensor_present", 40 | "only_npi_scope", "sentential_negation_npi_licensor_present", "sentential_negation_npi_scope"], 41 | "Quantifiers": ["existential_there_quantifiers_1", "existential_there_quantifiers_2", "superlative_quantifiers_1", "superlative_quantifiers_2"], 42 | "Subject-Verb Agreement": ["distractor_agreement_relational_noun", "distractor_agreement_relative_clause", 43 | "irregular_plural_subject_verb_agreement_1", "irregular_plural_subject_verb_agreement_2", 44 | "regular_plural_subject_verb_agreement_1", "regular_plural_subject_verb_agreement_2"], 45 | } 46 | 47 | corpora = ['bllip-md', 'bllip-lg'] 48 | models = ['xlm', 'sclm-past', 'sclm-next', 'xplm', 'xplm-mask'] 49 | model2seed = {'bllip-md':{'xlm':['1101', '1102', '1103'], 'sclm-past':['1101','1102', '1103', '1104'], 'sclm-next':['1101','1102', '1103','1104'], 'xplm':['1101', '1102', '1103'], 'xplm-mask':['1101', '1102', '1103']}, 50 | 'bllip-lg':{'xlm':['1101', '1102', '1103'], 'sclm-past':['1101', '1102', '1103', '1104'], 'sclm-next':['1101','1102','1103', '1104'], 'xplm':['1101', '1102', '1103'], 'xplm-mask':['1101', '1102', '1103']},} 51 | model2name = {'xlm':'LM', 'sclm-past':'ScLM-past', 'sclm-next':'ScLM-next', 'xplm':'PLM', 'xplm-mask':'PLM-mask'} 52 | 53 | rs = {} 54 | 55 | for corpus in corpora: 56 | rs[corpus] = {} 57 | for model in models: 58 | rs[corpus][model] = {} 59 | 60 | for corpus in corpora: 61 | for model in models: 62 | for seed in model2seed[corpus][model]: 63 | rs[corpus][model][seed] = {} 64 | path = 'results/blimp/{}_{}_rand-init_{}_5.txt'.format(model, corpus, seed) 65 | results = load_results(path) 66 | acc_all = [line[1] for line in results] 67 | acc_mean = np.mean(acc_all) 68 | rs[corpus][model][seed]['score'] = acc_mean 69 | rs[corpus][model][seed]['results'] = results 70 | 71 | 72 | results = load_results('results/blimp/gpt-2.txt') 73 | acc_all = [line[1] for line in results] 74 | acc_mean = np.mean(acc_all) 75 | rs['gpt-2'] = {} 76 | rs['gpt-2']['score'] = acc_mean 77 | rs['gpt-2']['results'] = results 78 | 79 | 80 | # Print the mean and standard deviation of model performances 81 | for corpus in corpora: 82 | for model in models: 83 | rs_list = [rs[corpus][model][seed]['score'] for seed in model2seed[corpus][model]] 84 | print(corpus, model, 'mean:', np.mean(rs_list), 'std:', np.std(rs_list)) 85 | 86 | 87 | # Bar plot of model performances 88 | model2style = {'xlm':{'color':plt.cm.Set2(7)}, 'sclm-past':{'color':plt.cm.Set2(0)}, 'sclm-next':{'color':plt.cm.Set2(1)}, 89 | 'xplm':{'color':plt.cm.Set2(2)}, 'xplm-mask':{'color':plt.cm.Set2(3)}} 90 | fig = plt.figure(figsize=(4, 2)) 91 | ax = plt.gca() 92 | ax.grid(linestyle='--') 93 | ax.set_axisbelow(True) 94 | bar_width = 0.10 95 | 96 | corpus2color = {'bllip-lg':plt.cm.tab20(0), 'bllip-md':plt.cm.tab20(1)} 97 | for i, corpus in enumerate(corpora): 98 | for j, model in enumerate(models): 99 | acc_mean_list = [] 100 | for seed in sorted(list(rs[corpus][model].keys())): 101 | acc_mean = rs[corpus][model][seed]['score'] 102 | acc_mean_list.append(acc_mean) 103 | 104 | bar_position = i*1 + (bar_width+0.02)*(j-2) 105 | ax.bar(bar_position, np.mean(acc_mean_list), width=bar_width, **model2style[model], label=model) 106 | plt.plot([bar_position for _ in range(len(acc_mean_list))], acc_mean_list, 'o', linestyle='None', mec='k',color='k', mfc='None', markersize=5) 107 | 108 | 109 | ax.bar(bar_position + bar_width*4, rs['gpt-2']['score'], width=bar_width, color='white', edgecolor='dimgrey',hatch='///', label='gpt-2') 110 | ax.set_xlim(-0.5,len(corpora)-0.2) 111 | ax.spines['right'].set_visible(False) 112 | ax.spines['top'].set_visible(False) 113 | 114 | ax.set_ylim((0.55, 0.83)) 115 | ax.set_yticks(np.arange(0.55, 0.82, 0.05)) 116 | 117 | ax.set_xticks(np.arange(0, len(corpora))) 118 | ax.set_xticklabels([corpus.upper() for corpus in corpora], fontsize=10) 119 | ax.set_ylabel('Accuracy', fontsize=10) 120 | 121 | item_legend_elements = [] 122 | for model in models: 123 | patch = mpatches.Patch(**model2style[model], label=model2name[model]) 124 | # item_legend_elements.append(Line2D([0], [0], **tag_cat2style[tag_cat],label=' '.join(tag_cat.split('_')), marker='o', linestyle = 'None', markersize=5)) 125 | item_legend_elements.append(patch) 126 | 127 | patch = Patch(facecolor='white', edgecolor='dimgrey',hatch='///', label='GPT-2') 128 | item_legend_elements.append(patch) 129 | 130 | ax.legend(handles=item_legend_elements, ncol=3, loc = 'center', bbox_to_anchor=(0.5, -0.36)) 131 | 132 | plt.title('Model Performance on BLiMP-10% Test Suites') 133 | plt.savefig('figs/blimp_comparison_by_corpus.pdf'.format(corpus), bbox_inches="tight") 134 | # plt.show() 135 | plt.close() 136 | 137 | 138 | # Bar plot of model performances on each cluster of phenomena 139 | total_test_suite_count = 0 140 | n_suite_dict = {} 141 | 142 | for k, phenomenon in enumerate(list(phenomena.keys())): 143 | n_suite_dict[phenomenon] = 0 144 | tag_set = set(phenomena[phenomenon]) 145 | for test_suite_name, _ in rs['bllip-md']['xlm']['1101']['results']: 146 | test_suite_cat = test_suite_name 147 | if test_suite_cat in tag_set: 148 | total_test_suite_count += 1 149 | n_suite_dict[phenomenon] += 1 150 | assert total_test_suite_count == 67 151 | 152 | fig, axes = plt.subplots(3, 4, sharex=True, figsize=(15,6)) 153 | plt.subplots_adjust(hspace = 0.3, wspace=0.2) 154 | for k, phenomenon in enumerate(list(phenomena.keys())): 155 | tag_set = set(phenomena[phenomenon]) 156 | 157 | ax = axes[k // 4, k % 4] 158 | 159 | ax.grid(linestyle='--') 160 | ax.set_axisbelow(True) 161 | bar_width = 0.10 162 | 163 | corpus2color = {'bllip-lg':plt.cm.tab20(0), 'bllip-md':plt.cm.tab20(1)} 164 | 165 | phenomenon_score_all = [] 166 | 167 | for i, corpus in enumerate(corpora): 168 | for j, model in enumerate(models): 169 | acc_mean_list = [] 170 | for seed in sorted(list(rs[corpus][model].keys())): 171 | score_list = [] 172 | for test_suite_name, score in rs[corpus][model][seed]['results']: 173 | test_suite_cat = test_suite_name 174 | if test_suite_cat in tag_set: 175 | score_list.append(score) 176 | acc_mean = np.mean(score_list) 177 | acc_mean_list.append(acc_mean) 178 | phenomenon_score_all.append(acc_mean) 179 | 180 | bar_position = i*1 + (bar_width+0.02)*(j-2) 181 | ax.bar(bar_position, np.mean(acc_mean_list), width=bar_width, **model2style[model], label=model) 182 | ax.plot([bar_position for _ in range(len(acc_mean_list))], acc_mean_list, 'o', linestyle='None', mec='k',color='k', mfc='None', markersize=5) 183 | 184 | ax.set_xlim(-0.5,len(corpora)-0.5) 185 | ax.spines['right'].set_visible(False) 186 | ax.spines['top'].set_visible(False) 187 | ax.set_xticks(np.arange(0, len(corpora))) 188 | ymin = np.max([(np.min(phenomenon_score_all) // 0.05 - 1)*0.05, 0]) 189 | ymax = np.min([(np.max(phenomenon_score_all) // 0.05 + 1)*0.05, 1]) 190 | ax.set_ylim((ymin, ymax)) 191 | ax.set_yticks(np.arange(ymin, ymax, 0.05)) 192 | ax.set_xticklabels([corpus.upper() for corpus in corpora], fontsize=10) 193 | if k % 4 == 0: 194 | ax.set_ylabel('Accuracy', fontsize=10) 195 | n_suite = n_suite_dict[phenomenon] 196 | if n_suite == 1: 197 | ax.set_title(phenomenon+' ({} suite)'.format(n_suite), fontsize=11.5) 198 | else: 199 | ax.set_title(phenomenon+' ({} suites)'.format(n_suite), fontsize=11.5) 200 | 201 | if k == 10: 202 | item_legend_elements = [] 203 | for model in models: 204 | patch = mpatches.Patch(**model2style[model], label=model2name[model]) 205 | item_legend_elements.append(patch) 206 | fig.legend(handles=item_legend_elements, ncol=5, loc = 'center', bbox_to_anchor=(0.5,0.03)) 207 | 208 | fig.suptitle('Model Performance on Specific Clusters of BLiMP-10% Test Suites', x=0.5, y=0.97, fontsize=14) 209 | plt.savefig('figs/blimp_comparison_by_test_suite_cluster_3_by_4.pdf', bbox_inches="tight") 210 | # plt.show() 211 | plt.close() 212 | 213 | 214 | # Plot the model performance against word-level perplexity 215 | with open('results/model_ppl_list.txt') as f: 216 | lines = f.readlines() 217 | lines = [line.strip().split() for line in lines if line.strip() != ''] 218 | ppl_results = [(line[0], line[1], line[2], float(line[3])) for line in lines] 219 | 220 | model2style = {'xlm':{'marker':'o', 'ms':5}, 'xplm':{'marker':'x', 'ms':5}, 'xplm-mask':{'marker':'^', 'ms':5}, 221 | 'sclm-past':{'marker':'s', 'ms':5}, 'sclm-next':{'marker':'d', 'ms':5},} 222 | corpus2style = {'bllip-lg':{'color':plt.cm.tab20(0)}, 'bllip-md':{'color':plt.cm.tab20(1)}} 223 | 224 | fig = plt.figure(figsize=(2.5,2.5)) 225 | 226 | for model, corpus, seed, ppl in ppl_results: 227 | if seed not in rs[corpus][model]: 228 | print(model, corpus, seed) 229 | plt.plot(ppl, rs[corpus][model][seed]['score'], **model2style[model], **corpus2style[corpus]) 230 | ax = plt.gca() 231 | ax.spines['right'].set_visible(False) 232 | ax.spines['top'].set_visible(False) 233 | plt.xlabel('Word-level Perplexity') 234 | plt.ylabel('BLiMP-10% Accuracy') 235 | 236 | item_legend_elements = [] 237 | for model in models: 238 | item_legend_elements.append(Line2D([0], [0], **model2style[model],label=model2name[model], color='gray', linestyle = 'None')) 239 | legend = plt.legend(handles=item_legend_elements, loc = 'center left', bbox_to_anchor=(1.1, 0.25), title='Model') 240 | ax.add_artist(legend) 241 | 242 | item_legend_elements = [] 243 | for corpus in corpora: 244 | patch = mpatches.Patch(**corpus2style[corpus], label=corpus.upper()) 245 | item_legend_elements.append(patch) 246 | legend = plt.legend(handles=item_legend_elements, loc = 'center left', bbox_to_anchor=(1.1, 0.8), title='Corpus') 247 | ax.add_artist(legend) 248 | 249 | plt.savefig('figs/ppl_blimp.pdf', bbox_inches="tight") 250 | # plt.show() 251 | plt.close() 252 | -------------------------------------------------------------------------------- /analysis/analysis_sg.py: -------------------------------------------------------------------------------- 1 | # This script generates figures of model performance comparisons on SG Test Suites. 2 | 3 | import numpy as np 4 | import pandas as pd 5 | import re 6 | import matplotlib.pyplot as plt 7 | from matplotlib.lines import Line2D 8 | import matplotlib.patches as mpatches 9 | from matplotlib.patches import Patch 10 | from matplotlib import rcParams 11 | rcParams['font.family'] = 'Arial' 12 | 13 | 14 | def load_results(path): 15 | with open(path) as f: 16 | lines = f.readlines() 17 | lines = [line.strip().split() for line in lines] 18 | lines = [(line[0], float(line[1])) for line in lines] 19 | return lines 20 | 21 | 22 | circuits = { 23 | "Licensing": ["npi", "reflexive"], 24 | "Long-Distance Dependencies": ["fgd", "cleft"], 25 | "Agreement": ["number"], 26 | "Garden-Path Effects": ["npz", "mvrr"], 27 | "Gross Syntactic State": ["subordination"], 28 | "Center Embedding": ["center"], 29 | } 30 | 31 | corpora = ['bllip-md', 'bllip-lg'] 32 | models = ['xlm', 'sclm-past', 'sclm-next', 'xplm', 'xplm-mask'] 33 | model2seed = {'bllip-md':{'xlm':['1101', '1102', '1103'], 'sclm-past':['1101', '1102', '1103', '1104'], 'sclm-next':['1101', '1102', '1103', '1104'], 'xplm':['1101', '1102', '1103'], 'xplm-mask':['1101', '1102', '1103']}, 34 | 'bllip-lg':{'xlm':['1101', '1102', '1103'], 'sclm-past':['1101', '1102', '1103', '1104'], 'sclm-next':['1101', '1102', '1103', '1104'], 'xplm':['1101', '1102', '1103'], 'xplm-mask':['1101', '1102', '1103']},} 35 | 36 | model2name = {'xlm':'LM', 'sclm-past':'ScLM-past', 'sclm-next':'ScLM-next', 'xplm':'PLM', 'xplm-mask':'PLM-mask', 'rnng':'RNNG'} 37 | 38 | rs = {} 39 | 40 | for corpus in corpora: 41 | rs[corpus] = {} 42 | for model in models: 43 | rs[corpus][model] = {} 44 | 45 | for corpus in corpora: 46 | for model in models: 47 | for seed in model2seed[corpus][model]: 48 | rs[corpus][model][seed] = {} 49 | path = 'results/sg/{}_{}_rand-init_{}_5.txt'.format(model, corpus, seed) 50 | results = load_results(path) 51 | acc_all = [line[1] for line in results if line[0] != 'nn-nv-rpl'] # filter one test suite 52 | acc_mean = np.mean(acc_all) 53 | rs[corpus][model][seed]['score'] = acc_mean 54 | rs[corpus][model][seed]['results'] = [line for line in results if line[0] != 'nn-nv-rpl'] # filter one test suite 55 | 56 | 57 | results = load_results('results/sg/gpt-2.txt') 58 | acc_all = [line[1] for line in results if line[0] != 'nn-nv-rpl'] # filter one test suite 59 | acc_mean = np.mean(acc_all) 60 | rs['gpt-2'] = {} 61 | rs['gpt-2']['score'] = acc_mean 62 | rs['gpt-2']['results'] = [line for line in results if line[0] != 'nn-nv-rpl'] # filter one test suite 63 | 64 | 65 | # Print the mean and standard deviation of model performances 66 | for corpus in corpora: 67 | for model in models: 68 | rs_list = [rs[corpus][model][seed]['score'] for seed in model2seed[corpus][model]] 69 | print(corpus, model, 'mean:', np.mean(rs_list), 'std:', np.std(rs_list)) 70 | 71 | 72 | # Plot the SG score against word-level perplexity 73 | with open('results/model_ppl_list.txt') as f: 74 | lines = f.readlines() 75 | lines = [line.strip().split() for line in lines if line.strip() != ''] 76 | ppl_results = [(line[0], line[1], line[2], float(line[3])) for line in lines] 77 | 78 | model2style = {'xlm':{'marker':'o', 'ms':5}, 'xplm':{'marker':'x', 'ms':5}, 'xplm-mask':{'marker':'^', 'ms':5}, 79 | 'sclm-past':{'marker':'s', 'ms':5}, 'sclm-next':{'marker':'d', 'ms':5},} 80 | corpus2style = {'bllip-lg':{'color':plt.cm.tab20(0)}, 'bllip-md':{'color':plt.cm.tab20(1)}} 81 | 82 | fig = plt.figure(figsize=(2.5,2.5)) 83 | 84 | for model, corpus, seed, ppl in ppl_results: 85 | if seed not in rs[corpus][model]: 86 | print(model, corpus, seed) 87 | plt.plot(ppl, rs[corpus][model][seed]['score'], **model2style[model], **corpus2style[corpus]) 88 | ax = plt.gca() 89 | ax.spines['right'].set_visible(False) 90 | ax.spines['top'].set_visible(False) 91 | plt.xlabel('Word-level Perplexity') 92 | plt.ylabel('SG Accuracy') 93 | 94 | item_legend_elements = [] 95 | for model in models: 96 | item_legend_elements.append(Line2D([0], [0], **model2style[model],label=model2name[model], color='gray', linestyle = 'None')) 97 | legend = plt.legend(handles=item_legend_elements, loc = 'center left', bbox_to_anchor=(1.1, 0.25), title='Model') 98 | ax.add_artist(legend) 99 | 100 | item_legend_elements = [] 101 | for corpus in corpora: 102 | patch = mpatches.Patch(**corpus2style[corpus], label=corpus.upper()) 103 | item_legend_elements.append(patch) 104 | legend = plt.legend(handles=item_legend_elements, loc = 'center left', bbox_to_anchor=(1.1, 0.8), title='Corpus') 105 | ax.add_artist(legend) 106 | 107 | plt.savefig('figs/ppl_sg.pdf', bbox_inches="tight") 108 | # plt.show() 109 | plt.close() 110 | 111 | 112 | # Load RNNG results 113 | rnng_rs = {} 114 | 115 | rnng2seed = {'bllip-md':['44862', '3602'], 'bllip-lg':['38435', '62488', '7245']} 116 | 117 | for corpus in corpora: 118 | rnng_rs[corpus] = {} 119 | path = 'results/sg/syntaxgym_results_rnng_{}.csv'.format(corpus) 120 | df = pd.read_csv(path) 121 | 122 | test_suite_name_list = list(set(df['suite'].tolist())) 123 | 124 | for seed in rnng2seed[corpus]: 125 | rnng_rs[corpus][seed] = {} 126 | acc_list = [] 127 | for test_suite_name in test_suite_name_list: 128 | data = df[df['model'].isin(['rnng_{}_{}'.format(corpus, seed)]) & df['suite'].isin([test_suite_name])]['correct'].tolist() 129 | acc_list.append([test_suite_name, np.mean(data)]) 130 | rnng_rs[corpus][seed]['results'] = acc_list 131 | rnng_rs[corpus][seed]['score'] = np.mean([acc for _, acc in acc_list]) 132 | 133 | 134 | # Bar plot of model performances 135 | model2style = {'xlm':{'color':plt.cm.Set2(7)}, 'sclm-past':{'color':plt.cm.Set2(0)}, 'sclm-next':{'color':plt.cm.Set2(1)}, 136 | 'xplm':{'color':plt.cm.Set2(2)}, 'xplm-mask':{'color':plt.cm.Set2(3)}, 'rnng':{'color':plt.cm.Set2(4)},} 137 | fig = plt.figure(figsize=(4, 2)) 138 | ax = plt.gca() 139 | ax.grid(linestyle='--') 140 | ax.set_axisbelow(True) 141 | bar_width = 0.10 142 | 143 | corpus2color = {'bllip-lg':plt.cm.tab20(0), 'bllip-md':plt.cm.tab20(1)} 144 | for i, corpus in enumerate(corpora): 145 | for j, model in enumerate(['rnng']+models): 146 | acc_mean_list = [] 147 | 148 | if model == 'rnng': 149 | for seed in rnng2seed[corpus]: 150 | acc_mean = rnng_rs[corpus][seed]['score'] 151 | acc_mean_list.append(acc_mean) 152 | else: 153 | for seed in sorted(list(rs[corpus][model].keys())): 154 | acc_mean = rs[corpus][model][seed]['score'] 155 | acc_mean_list.append(acc_mean) 156 | 157 | bar_position = i*1 + (bar_width+0.02)*(j-2.5) 158 | 159 | if model == 'rnng': 160 | ax.bar(bar_position, np.mean(acc_mean_list), width=bar_width, color=plt.cm.Set2(4), label=model) 161 | else: 162 | ax.bar(bar_position, np.mean(acc_mean_list), width=bar_width, **model2style[model], label=model) 163 | plt.plot([bar_position for _ in range(len(acc_mean_list))], acc_mean_list, 'o', linestyle='None', mec='k',color='k', mfc='None', markersize=5) 164 | 165 | ax.bar(bar_position + bar_width*4, rs['gpt-2']['score'], width=bar_width, color='white', edgecolor='dimgrey',hatch='///', label='gpt-2') 166 | 167 | ax.set_xlim(-0.5,len(corpora)-0.2) 168 | ax.spines['right'].set_visible(False) 169 | ax.spines['top'].set_visible(False) 170 | 171 | ax.set_ylim((0.55, 0.82)) 172 | ax.set_yticks(np.arange(0.55, 0.82, 0.05)) 173 | ax.set_xticks(np.arange(0, len(corpora))) 174 | ax.set_xticklabels([corpus.upper() for corpus in corpora], fontsize=10) 175 | ax.set_ylabel('Accuracy', fontsize=10) 176 | 177 | item_legend_elements = [] 178 | for model in ['rnng'] + models: 179 | patch = mpatches.Patch(**model2style[model], label=model2name[model]) 180 | item_legend_elements.append(patch) 181 | 182 | patch = Patch(facecolor='white', edgecolor='dimgrey',hatch='///', label='GPT-2') 183 | 184 | item_legend_elements.append(patch) 185 | 186 | ax.legend(handles=item_legend_elements, ncol=3, loc = 'center', bbox_to_anchor=(0.5, -0.36)) 187 | 188 | plt.title('Model Performance on SG Test Suites') 189 | plt.savefig('figs/sg_comparison_by_corpus_with_rnng.pdf'.format(corpus), bbox_inches="tight") 190 | # plt.show() 191 | plt.close() 192 | 193 | 194 | # Bar plot of model performances on test suite clusters 195 | total_test_suite_count = 0 196 | 197 | n_suite_dict = {} 198 | 199 | for k, circuit in enumerate(list(circuits.keys())): 200 | n_suite_dict[circuit] = 0 201 | tag_set = set(circuits[circuit]) 202 | for test_suite_name, _ in rs['bllip-md']['xlm']['1101']['results']: 203 | test_suite_cat = re.split('_|-',test_suite_name) 204 | if test_suite_cat[0] in tag_set: 205 | total_test_suite_count += 1 206 | n_suite_dict[circuit] += 1 207 | assert total_test_suite_count == 33 208 | 209 | fig, axes = plt.subplots(2, 4, sharex=True, figsize=(15, 4)) 210 | plt.subplots_adjust(hspace = 0.3, wspace=0.2) 211 | for k, circuit in enumerate(list(circuits.keys())): 212 | tag_set = set(circuits[circuit]) 213 | 214 | ax = axes[k // 3, k % 3] 215 | 216 | ax.grid(linestyle='--') 217 | ax.set_axisbelow(True) 218 | bar_width = 0.10 219 | 220 | corpus2color = {'bllip-lg':plt.cm.tab20(0), 'bllip-md':plt.cm.tab20(1)} 221 | 222 | circuit_score_all = [] 223 | 224 | for i, corpus in enumerate(corpora): 225 | for j, model in enumerate(['rnng']+models): 226 | acc_mean_list = [] 227 | if model == 'rnng': 228 | seeds = rnng2seed[corpus] 229 | else: 230 | seeds = sorted(list(rs[corpus][model].keys())) 231 | for seed in seeds: 232 | score_list = [] 233 | if model == 'rnng': 234 | results_by_suites = rnng_rs[corpus][seed]['results'] 235 | else: 236 | results_by_suites = rs[corpus][model][seed]['results'] 237 | for test_suite_name, score in results_by_suites: 238 | test_suite_cat = re.split('_|-',test_suite_name) 239 | if test_suite_cat[0] in tag_set: 240 | score_list.append(score) 241 | acc_mean = np.mean(score_list) 242 | acc_mean_list.append(acc_mean) 243 | circuit_score_all.append(acc_mean) 244 | 245 | bar_position = i*1 + (bar_width+0.02)*(j-2) 246 | ax.bar(bar_position, np.mean(acc_mean_list), width=bar_width, **model2style[model], label=model) 247 | ax.plot([bar_position for _ in range(len(acc_mean_list))], acc_mean_list, 'o', linestyle='None', mec='k',color='k', mfc='None', markersize=5) 248 | 249 | ax.set_xlim(-0.5,len(corpora)-0.5) 250 | ax.spines['right'].set_visible(False) 251 | ax.spines['top'].set_visible(False) 252 | ax.set_xticks(np.arange(0, len(corpora))) 253 | ymin = (np.min(circuit_score_all) // 0.05 - 1)*0.05 254 | ymax = np.min([(np.max(circuit_score_all) // 0.05 + 1)*0.05, 1]) 255 | ax.set_ylim((ymin, ymax)) 256 | 257 | ax.set_yticks(np.arange(ymin, ymax, 0.05)) 258 | 259 | n_yticks = (ymax - ymin)/0.05 260 | if n_yticks > 8: 261 | yticklabels = [] 262 | for j, y_value in enumerate(np.arange(ymin, ymax, 0.05)): 263 | if j % 2 == 0: 264 | yticklabels.append(np.round(y_value,2)) 265 | else: 266 | yticklabels.append('') 267 | ax.set_yticklabels(yticklabels) 268 | ax.set_xticklabels([corpus.upper() for corpus in corpora], fontsize=10) 269 | if k % 3 == 0: 270 | ax.set_ylabel('Accuracy', fontsize=10) 271 | 272 | n_suite = n_suite_dict[circuit] 273 | if n_suite == 1: 274 | ax.set_title(circuit+' ({} suite)'.format(n_suite), fontsize=11.5) 275 | else: 276 | ax.set_title(circuit+' ({} suites)'.format(n_suite), fontsize=11.5) 277 | 278 | if k == 4: 279 | item_legend_elements = [] 280 | for model in ['rnng']+models: 281 | patch = mpatches.Patch(**model2style[model], label=model2name[model]) 282 | item_legend_elements.append(patch) 283 | fig.legend(handles=item_legend_elements, ncol=1, loc = 'center left', bbox_to_anchor=(0.72, 0.75)) 284 | 285 | axes[0, -1].axis('off') 286 | axes[1, -1].axis('off') 287 | 288 | fig.suptitle('Model Performance on Specific Clusters of SG Test Suites', x=0.5, y=1, fontsize=14) 289 | 290 | plt.savefig('figs/sg_comparison_by_test_suite_cluster_2_by_4_with_rnng.pdf', bbox_inches="tight") 291 | # plt.show() 292 | plt.close() -------------------------------------------------------------------------------- /analysis/results/blimp/gpt-2.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.88 2 | anaphor_gender_agreement 1.0 3 | anaphor_number_agreement 1.0 4 | animate_subject_passive 0.84 5 | animate_subject_trans 0.86 6 | causative 0.7 7 | complex_NP_island 0.67 8 | coordinate_structure_constraint_complex_left_branch 0.73 9 | coordinate_structure_constraint_object_extraction 0.88 10 | determiner_noun_agreement_1 1.0 11 | determiner_noun_agreement_2 1.0 12 | determiner_noun_agreement_irregular_1 0.92 13 | determiner_noun_agreement_irregular_2 0.97 14 | determiner_noun_agreement_with_adj_2 0.95 15 | determiner_noun_agreement_with_adj_irregular_1 0.9 16 | determiner_noun_agreement_with_adj_irregular_2 0.96 17 | determiner_noun_agreement_with_adjective_1 0.95 18 | distractor_agreement_relational_noun 0.83 19 | distractor_agreement_relative_clause 0.73 20 | drop_argument 0.86 21 | ellipsis_n_bar_1 0.81 22 | ellipsis_n_bar_2 0.87 23 | existential_there_object_raising 0.79 24 | existential_there_quantifiers_1 0.99 25 | existential_there_quantifiers_2 0.22 26 | existential_there_subject_raising 0.83 27 | expletive_it_object_raising 0.79 28 | inchoative 0.68 29 | intransitive 0.84 30 | irregular_past_participle_adjectives 0.99 31 | irregular_past_participle_verbs 0.93 32 | irregular_plural_subject_verb_agreement_1 0.97 33 | irregular_plural_subject_verb_agreement_2 0.94 34 | left_branch_island_echo_question 0.55 35 | left_branch_island_simple_question 0.84 36 | matrix_question_npi_licensor_present 0.64 37 | npi_present_1 0.59 38 | npi_present_2 0.67 39 | only_npi_licensor_present 0.97 40 | only_npi_scope 0.81 41 | passive_1 0.9 42 | passive_2 0.91 43 | principle_A_c_command 0.71 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.97 46 | principle_A_domain_1 0.98 47 | principle_A_domain_2 0.79 48 | principle_A_domain_3 0.74 49 | principle_A_reconstruction 0.42 50 | regular_plural_subject_verb_agreement_1 0.95 51 | regular_plural_subject_verb_agreement_2 0.93 52 | sentential_negation_npi_licensor_present 0.97 53 | sentential_negation_npi_scope 0.78 54 | sentential_subject_island 0.29 55 | superlative_quantifiers_1 0.91 56 | superlative_quantifiers_2 0.81 57 | tough_vs_raising_1 0.67 58 | tough_vs_raising_2 0.92 59 | transitive 0.87 60 | wh_island 0.82 61 | wh_questions_object_gap 0.87 62 | wh_questions_subject_gap 0.91 63 | wh_questions_subject_gap_long_distance 0.87 64 | wh_vs_that_no_gap 0.98 65 | wh_vs_that_no_gap_long_distance 1.0 66 | wh_vs_that_with_gap 0.55 67 | wh_vs_that_with_gap_long_distance 0.53 -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-next_bllip-lg_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.78 2 | anaphor_gender_agreement 0.61 3 | anaphor_number_agreement 0.88 4 | animate_subject_passive 0.73 5 | animate_subject_trans 0.55 6 | causative 0.6 7 | complex_NP_island 0.45 8 | coordinate_structure_constraint_complex_left_branch 0.47 9 | coordinate_structure_constraint_object_extraction 0.66 10 | determiner_noun_agreement_1 0.93 11 | determiner_noun_agreement_2 0.9 12 | determiner_noun_agreement_irregular_1 0.75 13 | determiner_noun_agreement_irregular_2 0.85 14 | determiner_noun_agreement_with_adj_2 0.76 15 | determiner_noun_agreement_with_adj_irregular_1 0.76 16 | determiner_noun_agreement_with_adj_irregular_2 0.82 17 | determiner_noun_agreement_with_adjective_1 0.86 18 | distractor_agreement_relational_noun 0.87 19 | distractor_agreement_relative_clause 0.63 20 | drop_argument 0.77 21 | ellipsis_n_bar_1 0.88 22 | ellipsis_n_bar_2 0.64 23 | existential_there_object_raising 0.73 24 | existential_there_quantifiers_1 0.98 25 | existential_there_quantifiers_2 0.21 26 | existential_there_subject_raising 0.87 27 | expletive_it_object_raising 0.72 28 | inchoative 0.66 29 | intransitive 0.75 30 | irregular_past_participle_adjectives 0.81 31 | irregular_past_participle_verbs 0.76 32 | irregular_plural_subject_verb_agreement_1 0.84 33 | irregular_plural_subject_verb_agreement_2 0.88 34 | left_branch_island_echo_question 0.31 35 | left_branch_island_simple_question 0.62 36 | matrix_question_npi_licensor_present 0.41 37 | npi_present_1 0.71 38 | npi_present_2 0.7 39 | only_npi_licensor_present 0.84 40 | only_npi_scope 0.69 41 | passive_1 0.83 42 | passive_2 0.8 43 | principle_A_c_command 0.72 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.8 46 | principle_A_domain_1 0.92 47 | principle_A_domain_2 0.74 48 | principle_A_domain_3 0.55 49 | principle_A_reconstruction 0.7 50 | regular_plural_subject_verb_agreement_1 0.83 51 | regular_plural_subject_verb_agreement_2 0.94 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.69 54 | sentential_subject_island 0.53 55 | superlative_quantifiers_1 0.85 56 | superlative_quantifiers_2 0.7 57 | tough_vs_raising_1 0.55 58 | tough_vs_raising_2 0.89 59 | transitive 0.73 60 | wh_island 0.47 61 | wh_questions_object_gap 0.8 62 | wh_questions_subject_gap 0.86 63 | wh_questions_subject_gap_long_distance 0.88 64 | wh_vs_that_no_gap 0.95 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.52 67 | wh_vs_that_with_gap_long_distance 0.21 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-next_bllip-lg_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.71 2 | anaphor_gender_agreement 0.46 3 | anaphor_number_agreement 0.85 4 | animate_subject_passive 0.67 5 | animate_subject_trans 0.57 6 | causative 0.56 7 | complex_NP_island 0.52 8 | coordinate_structure_constraint_complex_left_branch 0.53 9 | coordinate_structure_constraint_object_extraction 0.77 10 | determiner_noun_agreement_1 0.93 11 | determiner_noun_agreement_2 0.91 12 | determiner_noun_agreement_irregular_1 0.75 13 | determiner_noun_agreement_irregular_2 0.85 14 | determiner_noun_agreement_with_adj_2 0.76 15 | determiner_noun_agreement_with_adj_irregular_1 0.84 16 | determiner_noun_agreement_with_adj_irregular_2 0.84 17 | determiner_noun_agreement_with_adjective_1 0.86 18 | distractor_agreement_relational_noun 0.85 19 | distractor_agreement_relative_clause 0.72 20 | drop_argument 0.74 21 | ellipsis_n_bar_1 0.88 22 | ellipsis_n_bar_2 0.59 23 | existential_there_object_raising 0.74 24 | existential_there_quantifiers_1 1.0 25 | existential_there_quantifiers_2 0.17 26 | existential_there_subject_raising 0.8 27 | expletive_it_object_raising 0.71 28 | inchoative 0.55 29 | intransitive 0.77 30 | irregular_past_participle_adjectives 0.6 31 | irregular_past_participle_verbs 0.71 32 | irregular_plural_subject_verb_agreement_1 0.79 33 | irregular_plural_subject_verb_agreement_2 0.9 34 | left_branch_island_echo_question 0.27 35 | left_branch_island_simple_question 0.69 36 | matrix_question_npi_licensor_present 0.35 37 | npi_present_1 0.71 38 | npi_present_2 0.73 39 | only_npi_licensor_present 0.78 40 | only_npi_scope 0.65 41 | passive_1 0.72 42 | passive_2 0.81 43 | principle_A_c_command 0.81 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.83 46 | principle_A_domain_1 0.94 47 | principle_A_domain_2 0.7 48 | principle_A_domain_3 0.5 49 | principle_A_reconstruction 0.54 50 | regular_plural_subject_verb_agreement_1 0.88 51 | regular_plural_subject_verb_agreement_2 0.93 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.77 54 | sentential_subject_island 0.59 55 | superlative_quantifiers_1 0.93 56 | superlative_quantifiers_2 0.76 57 | tough_vs_raising_1 0.5 58 | tough_vs_raising_2 0.91 59 | transitive 0.74 60 | wh_island 0.47 61 | wh_questions_object_gap 0.77 62 | wh_questions_subject_gap 0.85 63 | wh_questions_subject_gap_long_distance 0.87 64 | wh_vs_that_no_gap 0.95 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.48 67 | wh_vs_that_with_gap_long_distance 0.2 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-next_bllip-lg_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.73 2 | anaphor_gender_agreement 0.64 3 | anaphor_number_agreement 0.89 4 | animate_subject_passive 0.72 5 | animate_subject_trans 0.55 6 | causative 0.62 7 | complex_NP_island 0.59 8 | coordinate_structure_constraint_complex_left_branch 0.51 9 | coordinate_structure_constraint_object_extraction 0.8 10 | determiner_noun_agreement_1 0.92 11 | determiner_noun_agreement_2 0.87 12 | determiner_noun_agreement_irregular_1 0.71 13 | determiner_noun_agreement_irregular_2 0.9 14 | determiner_noun_agreement_with_adj_2 0.83 15 | determiner_noun_agreement_with_adj_irregular_1 0.81 16 | determiner_noun_agreement_with_adj_irregular_2 0.79 17 | determiner_noun_agreement_with_adjective_1 0.88 18 | distractor_agreement_relational_noun 0.86 19 | distractor_agreement_relative_clause 0.62 20 | drop_argument 0.81 21 | ellipsis_n_bar_1 0.87 22 | ellipsis_n_bar_2 0.54 23 | existential_there_object_raising 0.65 24 | existential_there_quantifiers_1 0.99 25 | existential_there_quantifiers_2 0.28 26 | existential_there_subject_raising 0.78 27 | expletive_it_object_raising 0.72 28 | inchoative 0.59 29 | intransitive 0.77 30 | irregular_past_participle_adjectives 0.77 31 | irregular_past_participle_verbs 0.75 32 | irregular_plural_subject_verb_agreement_1 0.83 33 | irregular_plural_subject_verb_agreement_2 0.9 34 | left_branch_island_echo_question 0.24 35 | left_branch_island_simple_question 0.71 36 | matrix_question_npi_licensor_present 0.46 37 | npi_present_1 0.7 38 | npi_present_2 0.78 39 | only_npi_licensor_present 0.75 40 | only_npi_scope 0.67 41 | passive_1 0.8 42 | passive_2 0.83 43 | principle_A_c_command 0.72 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.82 46 | principle_A_domain_1 0.95 47 | principle_A_domain_2 0.73 48 | principle_A_domain_3 0.5 49 | principle_A_reconstruction 0.68 50 | regular_plural_subject_verb_agreement_1 0.84 51 | regular_plural_subject_verb_agreement_2 0.92 52 | sentential_negation_npi_licensor_present 0.99 53 | sentential_negation_npi_scope 0.71 54 | sentential_subject_island 0.58 55 | superlative_quantifiers_1 0.93 56 | superlative_quantifiers_2 0.77 57 | tough_vs_raising_1 0.53 58 | tough_vs_raising_2 0.93 59 | transitive 0.74 60 | wh_island 0.64 61 | wh_questions_object_gap 0.81 62 | wh_questions_subject_gap 0.87 63 | wh_questions_subject_gap_long_distance 0.92 64 | wh_vs_that_no_gap 0.96 65 | wh_vs_that_no_gap_long_distance 1.0 66 | wh_vs_that_with_gap 0.36 67 | wh_vs_that_with_gap_long_distance 0.09 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-next_bllip-lg_rand-init_1104_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.67 2 | anaphor_gender_agreement 0.53 3 | anaphor_number_agreement 0.91 4 | animate_subject_passive 0.71 5 | animate_subject_trans 0.55 6 | causative 0.55 7 | complex_NP_island 0.52 8 | coordinate_structure_constraint_complex_left_branch 0.39 9 | coordinate_structure_constraint_object_extraction 0.73 10 | determiner_noun_agreement_1 0.96 11 | determiner_noun_agreement_2 0.87 12 | determiner_noun_agreement_irregular_1 0.74 13 | determiner_noun_agreement_irregular_2 0.83 14 | determiner_noun_agreement_with_adj_2 0.83 15 | determiner_noun_agreement_with_adj_irregular_1 0.73 16 | determiner_noun_agreement_with_adj_irregular_2 0.77 17 | determiner_noun_agreement_with_adjective_1 0.83 18 | distractor_agreement_relational_noun 0.85 19 | distractor_agreement_relative_clause 0.62 20 | drop_argument 0.77 21 | ellipsis_n_bar_1 0.83 22 | ellipsis_n_bar_2 0.61 23 | existential_there_object_raising 0.62 24 | existential_there_quantifiers_1 0.98 25 | existential_there_quantifiers_2 0.27 26 | existential_there_subject_raising 0.89 27 | expletive_it_object_raising 0.75 28 | inchoative 0.61 29 | intransitive 0.75 30 | irregular_past_participle_adjectives 0.74 31 | irregular_past_participle_verbs 0.81 32 | irregular_plural_subject_verb_agreement_1 0.78 33 | irregular_plural_subject_verb_agreement_2 0.85 34 | left_branch_island_echo_question 0.22 35 | left_branch_island_simple_question 0.52 36 | matrix_question_npi_licensor_present 0.29 37 | npi_present_1 0.69 38 | npi_present_2 0.72 39 | only_npi_licensor_present 0.85 40 | only_npi_scope 0.71 41 | passive_1 0.83 42 | passive_2 0.83 43 | principle_A_c_command 0.78 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.9 46 | principle_A_domain_1 0.98 47 | principle_A_domain_2 0.74 48 | principle_A_domain_3 0.52 49 | principle_A_reconstruction 0.62 50 | regular_plural_subject_verb_agreement_1 0.77 51 | regular_plural_subject_verb_agreement_2 0.91 52 | sentential_negation_npi_licensor_present 0.99 53 | sentential_negation_npi_scope 0.71 54 | sentential_subject_island 0.54 55 | superlative_quantifiers_1 0.95 56 | superlative_quantifiers_2 0.74 57 | tough_vs_raising_1 0.57 58 | tough_vs_raising_2 0.89 59 | transitive 0.67 60 | wh_island 0.44 61 | wh_questions_object_gap 0.72 62 | wh_questions_subject_gap 0.83 63 | wh_questions_subject_gap_long_distance 0.85 64 | wh_vs_that_no_gap 0.95 65 | wh_vs_that_no_gap_long_distance 0.99 66 | wh_vs_that_with_gap 0.48 67 | wh_vs_that_with_gap_long_distance 0.21 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-next_bllip-md_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.64 2 | anaphor_gender_agreement 0.45 3 | anaphor_number_agreement 0.8 4 | animate_subject_passive 0.69 5 | animate_subject_trans 0.47 6 | causative 0.58 7 | complex_NP_island 0.49 8 | coordinate_structure_constraint_complex_left_branch 0.27 9 | coordinate_structure_constraint_object_extraction 0.71 10 | determiner_noun_agreement_1 0.92 11 | determiner_noun_agreement_2 0.92 12 | determiner_noun_agreement_irregular_1 0.69 13 | determiner_noun_agreement_irregular_2 0.76 14 | determiner_noun_agreement_with_adj_2 0.8 15 | determiner_noun_agreement_with_adj_irregular_1 0.77 16 | determiner_noun_agreement_with_adj_irregular_2 0.85 17 | determiner_noun_agreement_with_adjective_1 0.82 18 | distractor_agreement_relational_noun 0.87 19 | distractor_agreement_relative_clause 0.65 20 | drop_argument 0.78 21 | ellipsis_n_bar_1 0.83 22 | ellipsis_n_bar_2 0.5 23 | existential_there_object_raising 0.61 24 | existential_there_quantifiers_1 0.98 25 | existential_there_quantifiers_2 0.29 26 | existential_there_subject_raising 0.84 27 | expletive_it_object_raising 0.62 28 | inchoative 0.53 29 | intransitive 0.66 30 | irregular_past_participle_adjectives 0.68 31 | irregular_past_participle_verbs 0.75 32 | irregular_plural_subject_verb_agreement_1 0.72 33 | irregular_plural_subject_verb_agreement_2 0.85 34 | left_branch_island_echo_question 0.35 35 | left_branch_island_simple_question 0.53 36 | matrix_question_npi_licensor_present 0.36 37 | npi_present_1 0.54 38 | npi_present_2 0.66 39 | only_npi_licensor_present 0.7 40 | only_npi_scope 0.69 41 | passive_1 0.74 42 | passive_2 0.82 43 | principle_A_c_command 0.75 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.77 46 | principle_A_domain_1 0.93 47 | principle_A_domain_2 0.73 48 | principle_A_domain_3 0.46 49 | principle_A_reconstruction 0.71 50 | regular_plural_subject_verb_agreement_1 0.81 51 | regular_plural_subject_verb_agreement_2 0.9 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.84 54 | sentential_subject_island 0.6 55 | superlative_quantifiers_1 0.78 56 | superlative_quantifiers_2 0.69 57 | tough_vs_raising_1 0.47 58 | tough_vs_raising_2 0.92 59 | transitive 0.69 60 | wh_island 0.45 61 | wh_questions_object_gap 0.75 62 | wh_questions_subject_gap 0.88 63 | wh_questions_subject_gap_long_distance 0.89 64 | wh_vs_that_no_gap 0.92 65 | wh_vs_that_no_gap_long_distance 0.96 66 | wh_vs_that_with_gap 0.35 67 | wh_vs_that_with_gap_long_distance 0.25 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-next_bllip-md_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.57 2 | anaphor_gender_agreement 0.47 3 | anaphor_number_agreement 0.77 4 | animate_subject_passive 0.67 5 | animate_subject_trans 0.49 6 | causative 0.59 7 | complex_NP_island 0.44 8 | coordinate_structure_constraint_complex_left_branch 0.28 9 | coordinate_structure_constraint_object_extraction 0.68 10 | determiner_noun_agreement_1 0.85 11 | determiner_noun_agreement_2 0.89 12 | determiner_noun_agreement_irregular_1 0.69 13 | determiner_noun_agreement_irregular_2 0.8 14 | determiner_noun_agreement_with_adj_2 0.77 15 | determiner_noun_agreement_with_adj_irregular_1 0.68 16 | determiner_noun_agreement_with_adj_irregular_2 0.84 17 | determiner_noun_agreement_with_adjective_1 0.85 18 | distractor_agreement_relational_noun 0.77 19 | distractor_agreement_relative_clause 0.62 20 | drop_argument 0.77 21 | ellipsis_n_bar_1 0.82 22 | ellipsis_n_bar_2 0.5 23 | existential_there_object_raising 0.58 24 | existential_there_quantifiers_1 0.94 25 | existential_there_quantifiers_2 0.27 26 | existential_there_subject_raising 0.82 27 | expletive_it_object_raising 0.63 28 | inchoative 0.58 29 | intransitive 0.65 30 | irregular_past_participle_adjectives 0.93 31 | irregular_past_participle_verbs 0.75 32 | irregular_plural_subject_verb_agreement_1 0.8 33 | irregular_plural_subject_verb_agreement_2 0.9 34 | left_branch_island_echo_question 0.29 35 | left_branch_island_simple_question 0.45 36 | matrix_question_npi_licensor_present 0.3 37 | npi_present_1 0.57 38 | npi_present_2 0.58 39 | only_npi_licensor_present 0.74 40 | only_npi_scope 0.75 41 | passive_1 0.75 42 | passive_2 0.81 43 | principle_A_c_command 0.79 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.78 46 | principle_A_domain_1 0.89 47 | principle_A_domain_2 0.7 48 | principle_A_domain_3 0.44 49 | principle_A_reconstruction 0.71 50 | regular_plural_subject_verb_agreement_1 0.85 51 | regular_plural_subject_verb_agreement_2 0.91 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.53 54 | sentential_subject_island 0.64 55 | superlative_quantifiers_1 0.91 56 | superlative_quantifiers_2 0.84 57 | tough_vs_raising_1 0.46 58 | tough_vs_raising_2 0.86 59 | transitive 0.73 60 | wh_island 0.44 61 | wh_questions_object_gap 0.69 62 | wh_questions_subject_gap 0.83 63 | wh_questions_subject_gap_long_distance 0.82 64 | wh_vs_that_no_gap 0.94 65 | wh_vs_that_no_gap_long_distance 1.0 66 | wh_vs_that_with_gap 0.29 67 | wh_vs_that_with_gap_long_distance 0.19 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-next_bllip-md_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.6 2 | anaphor_gender_agreement 0.48 3 | anaphor_number_agreement 0.77 4 | animate_subject_passive 0.66 5 | animate_subject_trans 0.55 6 | causative 0.65 7 | complex_NP_island 0.51 8 | coordinate_structure_constraint_complex_left_branch 0.31 9 | coordinate_structure_constraint_object_extraction 0.63 10 | determiner_noun_agreement_1 0.89 11 | determiner_noun_agreement_2 0.91 12 | determiner_noun_agreement_irregular_1 0.7 13 | determiner_noun_agreement_irregular_2 0.85 14 | determiner_noun_agreement_with_adj_2 0.8 15 | determiner_noun_agreement_with_adj_irregular_1 0.77 16 | determiner_noun_agreement_with_adj_irregular_2 0.77 17 | determiner_noun_agreement_with_adjective_1 0.87 18 | distractor_agreement_relational_noun 0.8 19 | distractor_agreement_relative_clause 0.63 20 | drop_argument 0.72 21 | ellipsis_n_bar_1 0.73 22 | ellipsis_n_bar_2 0.51 23 | existential_there_object_raising 0.63 24 | existential_there_quantifiers_1 0.95 25 | existential_there_quantifiers_2 0.16 26 | existential_there_subject_raising 0.81 27 | expletive_it_object_raising 0.62 28 | inchoative 0.58 29 | intransitive 0.69 30 | irregular_past_participle_adjectives 0.68 31 | irregular_past_participle_verbs 0.77 32 | irregular_plural_subject_verb_agreement_1 0.72 33 | irregular_plural_subject_verb_agreement_2 0.87 34 | left_branch_island_echo_question 0.3 35 | left_branch_island_simple_question 0.55 36 | matrix_question_npi_licensor_present 0.3 37 | npi_present_1 0.56 38 | npi_present_2 0.6 39 | only_npi_licensor_present 0.67 40 | only_npi_scope 0.68 41 | passive_1 0.78 42 | passive_2 0.75 43 | principle_A_c_command 0.82 44 | principle_A_case_1 0.99 45 | principle_A_case_2 0.77 46 | principle_A_domain_1 0.92 47 | principle_A_domain_2 0.67 48 | principle_A_domain_3 0.51 49 | principle_A_reconstruction 0.63 50 | regular_plural_subject_verb_agreement_1 0.79 51 | regular_plural_subject_verb_agreement_2 0.88 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.66 54 | sentential_subject_island 0.59 55 | superlative_quantifiers_1 0.93 56 | superlative_quantifiers_2 0.76 57 | tough_vs_raising_1 0.51 58 | tough_vs_raising_2 0.91 59 | transitive 0.65 60 | wh_island 0.61 61 | wh_questions_object_gap 0.7 62 | wh_questions_subject_gap 0.81 63 | wh_questions_subject_gap_long_distance 0.89 64 | wh_vs_that_no_gap 0.92 65 | wh_vs_that_no_gap_long_distance 0.99 66 | wh_vs_that_with_gap 0.34 67 | wh_vs_that_with_gap_long_distance 0.13 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-next_bllip-md_rand-init_1104_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.69 2 | anaphor_gender_agreement 0.42 3 | anaphor_number_agreement 0.82 4 | animate_subject_passive 0.65 5 | animate_subject_trans 0.53 6 | causative 0.63 7 | complex_NP_island 0.52 8 | coordinate_structure_constraint_complex_left_branch 0.3 9 | coordinate_structure_constraint_object_extraction 0.61 10 | determiner_noun_agreement_1 0.9 11 | determiner_noun_agreement_2 0.92 12 | determiner_noun_agreement_irregular_1 0.76 13 | determiner_noun_agreement_irregular_2 0.83 14 | determiner_noun_agreement_with_adj_2 0.78 15 | determiner_noun_agreement_with_adj_irregular_1 0.8 16 | determiner_noun_agreement_with_adj_irregular_2 0.83 17 | determiner_noun_agreement_with_adjective_1 0.82 18 | distractor_agreement_relational_noun 0.89 19 | distractor_agreement_relative_clause 0.62 20 | drop_argument 0.77 21 | ellipsis_n_bar_1 0.76 22 | ellipsis_n_bar_2 0.55 23 | existential_there_object_raising 0.66 24 | existential_there_quantifiers_1 0.93 25 | existential_there_quantifiers_2 0.28 26 | existential_there_subject_raising 0.77 27 | expletive_it_object_raising 0.67 28 | inchoative 0.58 29 | intransitive 0.75 30 | irregular_past_participle_adjectives 0.73 31 | irregular_past_participle_verbs 0.72 32 | irregular_plural_subject_verb_agreement_1 0.72 33 | irregular_plural_subject_verb_agreement_2 0.83 34 | left_branch_island_echo_question 0.32 35 | left_branch_island_simple_question 0.42 36 | matrix_question_npi_licensor_present 0.35 37 | npi_present_1 0.58 38 | npi_present_2 0.67 39 | only_npi_licensor_present 0.83 40 | only_npi_scope 0.67 41 | passive_1 0.72 42 | passive_2 0.79 43 | principle_A_c_command 0.82 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.74 46 | principle_A_domain_1 0.96 47 | principle_A_domain_2 0.68 48 | principle_A_domain_3 0.44 49 | principle_A_reconstruction 0.77 50 | regular_plural_subject_verb_agreement_1 0.78 51 | regular_plural_subject_verb_agreement_2 0.86 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.64 54 | sentential_subject_island 0.49 55 | superlative_quantifiers_1 0.84 56 | superlative_quantifiers_2 0.8 57 | tough_vs_raising_1 0.47 58 | tough_vs_raising_2 0.88 59 | transitive 0.71 60 | wh_island 0.49 61 | wh_questions_object_gap 0.74 62 | wh_questions_subject_gap 0.85 63 | wh_questions_subject_gap_long_distance 0.9 64 | wh_vs_that_no_gap 0.93 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.36 67 | wh_vs_that_with_gap_long_distance 0.18 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-past_bllip-lg_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.77 2 | anaphor_gender_agreement 0.58 3 | anaphor_number_agreement 0.89 4 | animate_subject_passive 0.66 5 | animate_subject_trans 0.56 6 | causative 0.58 7 | complex_NP_island 0.61 8 | coordinate_structure_constraint_complex_left_branch 0.46 9 | coordinate_structure_constraint_object_extraction 0.64 10 | determiner_noun_agreement_1 0.95 11 | determiner_noun_agreement_2 0.91 12 | determiner_noun_agreement_irregular_1 0.69 13 | determiner_noun_agreement_irregular_2 0.91 14 | determiner_noun_agreement_with_adj_2 0.86 15 | determiner_noun_agreement_with_adj_irregular_1 0.67 16 | determiner_noun_agreement_with_adj_irregular_2 0.8 17 | determiner_noun_agreement_with_adjective_1 0.85 18 | distractor_agreement_relational_noun 0.82 19 | distractor_agreement_relative_clause 0.63 20 | drop_argument 0.76 21 | ellipsis_n_bar_1 0.86 22 | ellipsis_n_bar_2 0.57 23 | existential_there_object_raising 0.6 24 | existential_there_quantifiers_1 0.99 25 | existential_there_quantifiers_2 0.19 26 | existential_there_subject_raising 0.85 27 | expletive_it_object_raising 0.7 28 | inchoative 0.59 29 | intransitive 0.73 30 | irregular_past_participle_adjectives 0.74 31 | irregular_past_participle_verbs 0.76 32 | irregular_plural_subject_verb_agreement_1 0.79 33 | irregular_plural_subject_verb_agreement_2 0.89 34 | left_branch_island_echo_question 0.36 35 | left_branch_island_simple_question 0.6 36 | matrix_question_npi_licensor_present 0.41 37 | npi_present_1 0.54 38 | npi_present_2 0.77 39 | only_npi_licensor_present 0.6 40 | only_npi_scope 0.68 41 | passive_1 0.75 42 | passive_2 0.83 43 | principle_A_c_command 0.75 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.83 46 | principle_A_domain_1 0.97 47 | principle_A_domain_2 0.73 48 | principle_A_domain_3 0.52 49 | principle_A_reconstruction 0.62 50 | regular_plural_subject_verb_agreement_1 0.82 51 | regular_plural_subject_verb_agreement_2 0.9 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.79 54 | sentential_subject_island 0.33 55 | superlative_quantifiers_1 0.87 56 | superlative_quantifiers_2 0.74 57 | tough_vs_raising_1 0.52 58 | tough_vs_raising_2 0.87 59 | transitive 0.7 60 | wh_island 0.43 61 | wh_questions_object_gap 0.72 62 | wh_questions_subject_gap 0.85 63 | wh_questions_subject_gap_long_distance 0.75 64 | wh_vs_that_no_gap 0.95 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.38 67 | wh_vs_that_with_gap_long_distance 0.22 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-past_bllip-lg_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.79 2 | anaphor_gender_agreement 0.55 3 | anaphor_number_agreement 0.88 4 | animate_subject_passive 0.71 5 | animate_subject_trans 0.48 6 | causative 0.57 7 | complex_NP_island 0.59 8 | coordinate_structure_constraint_complex_left_branch 0.39 9 | coordinate_structure_constraint_object_extraction 0.73 10 | determiner_noun_agreement_1 0.93 11 | determiner_noun_agreement_2 0.96 12 | determiner_noun_agreement_irregular_1 0.7 13 | determiner_noun_agreement_irregular_2 0.85 14 | determiner_noun_agreement_with_adj_2 0.84 15 | determiner_noun_agreement_with_adj_irregular_1 0.74 16 | determiner_noun_agreement_with_adj_irregular_2 0.84 17 | determiner_noun_agreement_with_adjective_1 0.85 18 | distractor_agreement_relational_noun 0.81 19 | distractor_agreement_relative_clause 0.64 20 | drop_argument 0.78 21 | ellipsis_n_bar_1 0.87 22 | ellipsis_n_bar_2 0.66 23 | existential_there_object_raising 0.59 24 | existential_there_quantifiers_1 0.96 25 | existential_there_quantifiers_2 0.21 26 | existential_there_subject_raising 0.84 27 | expletive_it_object_raising 0.77 28 | inchoative 0.62 29 | intransitive 0.77 30 | irregular_past_participle_adjectives 0.6 31 | irregular_past_participle_verbs 0.77 32 | irregular_plural_subject_verb_agreement_1 0.8 33 | irregular_plural_subject_verb_agreement_2 0.87 34 | left_branch_island_echo_question 0.23 35 | left_branch_island_simple_question 0.63 36 | matrix_question_npi_licensor_present 0.6 37 | npi_present_1 0.7 38 | npi_present_2 0.66 39 | only_npi_licensor_present 0.82 40 | only_npi_scope 0.77 41 | passive_1 0.85 42 | passive_2 0.82 43 | principle_A_c_command 0.76 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.83 46 | principle_A_domain_1 0.95 47 | principle_A_domain_2 0.76 48 | principle_A_domain_3 0.53 49 | principle_A_reconstruction 0.57 50 | regular_plural_subject_verb_agreement_1 0.81 51 | regular_plural_subject_verb_agreement_2 0.91 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.74 54 | sentential_subject_island 0.37 55 | superlative_quantifiers_1 0.9 56 | superlative_quantifiers_2 0.62 57 | tough_vs_raising_1 0.53 58 | tough_vs_raising_2 0.89 59 | transitive 0.69 60 | wh_island 0.49 61 | wh_questions_object_gap 0.75 62 | wh_questions_subject_gap 0.79 63 | wh_questions_subject_gap_long_distance 0.81 64 | wh_vs_that_no_gap 0.95 65 | wh_vs_that_no_gap_long_distance 0.99 66 | wh_vs_that_with_gap 0.46 67 | wh_vs_that_with_gap_long_distance 0.24 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-past_bllip-lg_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.79 2 | anaphor_gender_agreement 0.58 3 | anaphor_number_agreement 0.87 4 | animate_subject_passive 0.65 5 | animate_subject_trans 0.56 6 | causative 0.6 7 | complex_NP_island 0.45 8 | coordinate_structure_constraint_complex_left_branch 0.43 9 | coordinate_structure_constraint_object_extraction 0.81 10 | determiner_noun_agreement_1 0.94 11 | determiner_noun_agreement_2 0.87 12 | determiner_noun_agreement_irregular_1 0.67 13 | determiner_noun_agreement_irregular_2 0.85 14 | determiner_noun_agreement_with_adj_2 0.75 15 | determiner_noun_agreement_with_adj_irregular_1 0.78 16 | determiner_noun_agreement_with_adj_irregular_2 0.82 17 | determiner_noun_agreement_with_adjective_1 0.89 18 | distractor_agreement_relational_noun 0.82 19 | distractor_agreement_relative_clause 0.64 20 | drop_argument 0.77 21 | ellipsis_n_bar_1 0.82 22 | ellipsis_n_bar_2 0.47 23 | existential_there_object_raising 0.64 24 | existential_there_quantifiers_1 0.98 25 | existential_there_quantifiers_2 0.13 26 | existential_there_subject_raising 0.82 27 | expletive_it_object_raising 0.69 28 | inchoative 0.63 29 | intransitive 0.74 30 | irregular_past_participle_adjectives 0.96 31 | irregular_past_participle_verbs 0.75 32 | irregular_plural_subject_verb_agreement_1 0.8 33 | irregular_plural_subject_verb_agreement_2 0.88 34 | left_branch_island_echo_question 0.23 35 | left_branch_island_simple_question 0.63 36 | matrix_question_npi_licensor_present 0.41 37 | npi_present_1 0.46 38 | npi_present_2 0.61 39 | only_npi_licensor_present 0.76 40 | only_npi_scope 0.67 41 | passive_1 0.83 42 | passive_2 0.8 43 | principle_A_c_command 0.79 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.81 46 | principle_A_domain_1 0.96 47 | principle_A_domain_2 0.71 48 | principle_A_domain_3 0.58 49 | principle_A_reconstruction 0.54 50 | regular_plural_subject_verb_agreement_1 0.84 51 | regular_plural_subject_verb_agreement_2 0.91 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.48 54 | sentential_subject_island 0.51 55 | superlative_quantifiers_1 0.88 56 | superlative_quantifiers_2 0.66 57 | tough_vs_raising_1 0.56 58 | tough_vs_raising_2 0.84 59 | transitive 0.73 60 | wh_island 0.54 61 | wh_questions_object_gap 0.71 62 | wh_questions_subject_gap 0.79 63 | wh_questions_subject_gap_long_distance 0.83 64 | wh_vs_that_no_gap 0.96 65 | wh_vs_that_no_gap_long_distance 0.99 66 | wh_vs_that_with_gap 0.52 67 | wh_vs_that_with_gap_long_distance 0.23 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-past_bllip-lg_rand-init_1104_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.68 2 | anaphor_gender_agreement 0.54 3 | anaphor_number_agreement 0.89 4 | animate_subject_passive 0.68 5 | animate_subject_trans 0.53 6 | causative 0.6 7 | complex_NP_island 0.6 8 | coordinate_structure_constraint_complex_left_branch 0.35 9 | coordinate_structure_constraint_object_extraction 0.81 10 | determiner_noun_agreement_1 0.95 11 | determiner_noun_agreement_2 0.92 12 | determiner_noun_agreement_irregular_1 0.74 13 | determiner_noun_agreement_irregular_2 0.88 14 | determiner_noun_agreement_with_adj_2 0.83 15 | determiner_noun_agreement_with_adj_irregular_1 0.82 16 | determiner_noun_agreement_with_adj_irregular_2 0.83 17 | determiner_noun_agreement_with_adjective_1 0.85 18 | distractor_agreement_relational_noun 0.84 19 | distractor_agreement_relative_clause 0.67 20 | drop_argument 0.81 21 | ellipsis_n_bar_1 0.85 22 | ellipsis_n_bar_2 0.58 23 | existential_there_object_raising 0.58 24 | existential_there_quantifiers_1 0.99 25 | existential_there_quantifiers_2 0.19 26 | existential_there_subject_raising 0.83 27 | expletive_it_object_raising 0.74 28 | inchoative 0.66 29 | intransitive 0.75 30 | irregular_past_participle_adjectives 0.8 31 | irregular_past_participle_verbs 0.78 32 | irregular_plural_subject_verb_agreement_1 0.81 33 | irregular_plural_subject_verb_agreement_2 0.91 34 | left_branch_island_echo_question 0.28 35 | left_branch_island_simple_question 0.65 36 | matrix_question_npi_licensor_present 0.48 37 | npi_present_1 0.57 38 | npi_present_2 0.62 39 | only_npi_licensor_present 0.78 40 | only_npi_scope 0.78 41 | passive_1 0.74 42 | passive_2 0.8 43 | principle_A_c_command 0.73 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.86 46 | principle_A_domain_1 0.94 47 | principle_A_domain_2 0.72 48 | principle_A_domain_3 0.52 49 | principle_A_reconstruction 0.54 50 | regular_plural_subject_verb_agreement_1 0.87 51 | regular_plural_subject_verb_agreement_2 0.88 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.59 54 | sentential_subject_island 0.34 55 | superlative_quantifiers_1 0.99 56 | superlative_quantifiers_2 0.76 57 | tough_vs_raising_1 0.43 58 | tough_vs_raising_2 0.92 59 | transitive 0.69 60 | wh_island 0.45 61 | wh_questions_object_gap 0.69 62 | wh_questions_subject_gap 0.85 63 | wh_questions_subject_gap_long_distance 0.77 64 | wh_vs_that_no_gap 0.94 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.5 67 | wh_vs_that_with_gap_long_distance 0.18 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-past_bllip-md_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.71 2 | anaphor_gender_agreement 0.49 3 | anaphor_number_agreement 0.82 4 | animate_subject_passive 0.64 5 | animate_subject_trans 0.49 6 | causative 0.58 7 | complex_NP_island 0.54 8 | coordinate_structure_constraint_complex_left_branch 0.28 9 | coordinate_structure_constraint_object_extraction 0.68 10 | determiner_noun_agreement_1 0.88 11 | determiner_noun_agreement_2 0.87 12 | determiner_noun_agreement_irregular_1 0.68 13 | determiner_noun_agreement_irregular_2 0.77 14 | determiner_noun_agreement_with_adj_2 0.76 15 | determiner_noun_agreement_with_adj_irregular_1 0.75 16 | determiner_noun_agreement_with_adj_irregular_2 0.81 17 | determiner_noun_agreement_with_adjective_1 0.8 18 | distractor_agreement_relational_noun 0.89 19 | distractor_agreement_relative_clause 0.63 20 | drop_argument 0.78 21 | ellipsis_n_bar_1 0.73 22 | ellipsis_n_bar_2 0.51 23 | existential_there_object_raising 0.59 24 | existential_there_quantifiers_1 0.95 25 | existential_there_quantifiers_2 0.13 26 | existential_there_subject_raising 0.79 27 | expletive_it_object_raising 0.7 28 | inchoative 0.58 29 | intransitive 0.73 30 | irregular_past_participle_adjectives 0.8 31 | irregular_past_participle_verbs 0.75 32 | irregular_plural_subject_verb_agreement_1 0.77 33 | irregular_plural_subject_verb_agreement_2 0.86 34 | left_branch_island_echo_question 0.37 35 | left_branch_island_simple_question 0.49 36 | matrix_question_npi_licensor_present 0.39 37 | npi_present_1 0.53 38 | npi_present_2 0.59 39 | only_npi_licensor_present 0.54 40 | only_npi_scope 0.78 41 | passive_1 0.75 42 | passive_2 0.8 43 | principle_A_c_command 0.79 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.76 46 | principle_A_domain_1 0.91 47 | principle_A_domain_2 0.69 48 | principle_A_domain_3 0.49 49 | principle_A_reconstruction 0.77 50 | regular_plural_subject_verb_agreement_1 0.79 51 | regular_plural_subject_verb_agreement_2 0.87 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.56 54 | sentential_subject_island 0.44 55 | superlative_quantifiers_1 0.9 56 | superlative_quantifiers_2 0.78 57 | tough_vs_raising_1 0.49 58 | tough_vs_raising_2 0.9 59 | transitive 0.71 60 | wh_island 0.63 61 | wh_questions_object_gap 0.72 62 | wh_questions_subject_gap 0.81 63 | wh_questions_subject_gap_long_distance 0.83 64 | wh_vs_that_no_gap 0.92 65 | wh_vs_that_no_gap_long_distance 0.97 66 | wh_vs_that_with_gap 0.49 67 | wh_vs_that_with_gap_long_distance 0.19 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-past_bllip-md_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.66 2 | anaphor_gender_agreement 0.45 3 | anaphor_number_agreement 0.78 4 | animate_subject_passive 0.61 5 | animate_subject_trans 0.45 6 | causative 0.57 7 | complex_NP_island 0.48 8 | coordinate_structure_constraint_complex_left_branch 0.21 9 | coordinate_structure_constraint_object_extraction 0.68 10 | determiner_noun_agreement_1 0.84 11 | determiner_noun_agreement_2 0.87 12 | determiner_noun_agreement_irregular_1 0.69 13 | determiner_noun_agreement_irregular_2 0.9 14 | determiner_noun_agreement_with_adj_2 0.83 15 | determiner_noun_agreement_with_adj_irregular_1 0.81 16 | determiner_noun_agreement_with_adj_irregular_2 0.82 17 | determiner_noun_agreement_with_adjective_1 0.79 18 | distractor_agreement_relational_noun 0.84 19 | distractor_agreement_relative_clause 0.55 20 | drop_argument 0.77 21 | ellipsis_n_bar_1 0.78 22 | ellipsis_n_bar_2 0.6 23 | existential_there_object_raising 0.55 24 | existential_there_quantifiers_1 0.99 25 | existential_there_quantifiers_2 0.18 26 | existential_there_subject_raising 0.8 27 | expletive_it_object_raising 0.69 28 | inchoative 0.51 29 | intransitive 0.65 30 | irregular_past_participle_adjectives 0.71 31 | irregular_past_participle_verbs 0.73 32 | irregular_plural_subject_verb_agreement_1 0.83 33 | irregular_plural_subject_verb_agreement_2 0.88 34 | left_branch_island_echo_question 0.14 35 | left_branch_island_simple_question 0.5 36 | matrix_question_npi_licensor_present 0.33 37 | npi_present_1 0.7 38 | npi_present_2 0.55 39 | only_npi_licensor_present 0.63 40 | only_npi_scope 0.58 41 | passive_1 0.75 42 | passive_2 0.8 43 | principle_A_c_command 0.8 44 | principle_A_case_1 0.99 45 | principle_A_case_2 0.74 46 | principle_A_domain_1 0.93 47 | principle_A_domain_2 0.73 48 | principle_A_domain_3 0.56 49 | principle_A_reconstruction 0.58 50 | regular_plural_subject_verb_agreement_1 0.84 51 | regular_plural_subject_verb_agreement_2 0.85 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.73 54 | sentential_subject_island 0.53 55 | superlative_quantifiers_1 0.94 56 | superlative_quantifiers_2 0.73 57 | tough_vs_raising_1 0.47 58 | tough_vs_raising_2 0.87 59 | transitive 0.67 60 | wh_island 0.62 61 | wh_questions_object_gap 0.69 62 | wh_questions_subject_gap 0.87 63 | wh_questions_subject_gap_long_distance 0.81 64 | wh_vs_that_no_gap 0.92 65 | wh_vs_that_no_gap_long_distance 0.97 66 | wh_vs_that_with_gap 0.4 67 | wh_vs_that_with_gap_long_distance 0.17 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-past_bllip-md_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.65 2 | anaphor_gender_agreement 0.45 3 | anaphor_number_agreement 0.75 4 | animate_subject_passive 0.68 5 | animate_subject_trans 0.54 6 | causative 0.59 7 | complex_NP_island 0.56 8 | coordinate_structure_constraint_complex_left_branch 0.36 9 | coordinate_structure_constraint_object_extraction 0.56 10 | determiner_noun_agreement_1 0.91 11 | determiner_noun_agreement_2 0.88 12 | determiner_noun_agreement_irregular_1 0.68 13 | determiner_noun_agreement_irregular_2 0.83 14 | determiner_noun_agreement_with_adj_2 0.81 15 | determiner_noun_agreement_with_adj_irregular_1 0.73 16 | determiner_noun_agreement_with_adj_irregular_2 0.84 17 | determiner_noun_agreement_with_adjective_1 0.85 18 | distractor_agreement_relational_noun 0.79 19 | distractor_agreement_relative_clause 0.66 20 | drop_argument 0.78 21 | ellipsis_n_bar_1 0.82 22 | ellipsis_n_bar_2 0.55 23 | existential_there_object_raising 0.64 24 | existential_there_quantifiers_1 0.94 25 | existential_there_quantifiers_2 0.23 26 | existential_there_subject_raising 0.8 27 | expletive_it_object_raising 0.71 28 | inchoative 0.47 29 | intransitive 0.6 30 | irregular_past_participle_adjectives 0.63 31 | irregular_past_participle_verbs 0.72 32 | irregular_plural_subject_verb_agreement_1 0.77 33 | irregular_plural_subject_verb_agreement_2 0.83 34 | left_branch_island_echo_question 0.35 35 | left_branch_island_simple_question 0.59 36 | matrix_question_npi_licensor_present 0.26 37 | npi_present_1 0.57 38 | npi_present_2 0.56 39 | only_npi_licensor_present 0.55 40 | only_npi_scope 0.69 41 | passive_1 0.74 42 | passive_2 0.83 43 | principle_A_c_command 0.76 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.8 46 | principle_A_domain_1 0.92 47 | principle_A_domain_2 0.66 48 | principle_A_domain_3 0.43 49 | principle_A_reconstruction 0.69 50 | regular_plural_subject_verb_agreement_1 0.78 51 | regular_plural_subject_verb_agreement_2 0.86 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.73 54 | sentential_subject_island 0.58 55 | superlative_quantifiers_1 0.82 56 | superlative_quantifiers_2 0.79 57 | tough_vs_raising_1 0.36 58 | tough_vs_raising_2 0.9 59 | transitive 0.69 60 | wh_island 0.64 61 | wh_questions_object_gap 0.78 62 | wh_questions_subject_gap 0.84 63 | wh_questions_subject_gap_long_distance 0.84 64 | wh_vs_that_no_gap 0.92 65 | wh_vs_that_no_gap_long_distance 0.94 66 | wh_vs_that_with_gap 0.42 67 | wh_vs_that_with_gap_long_distance 0.21 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/sclm-past_bllip-md_rand-init_1104_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.63 2 | anaphor_gender_agreement 0.39 3 | anaphor_number_agreement 0.81 4 | animate_subject_passive 0.68 5 | animate_subject_trans 0.53 6 | causative 0.57 7 | complex_NP_island 0.59 8 | coordinate_structure_constraint_complex_left_branch 0.22 9 | coordinate_structure_constraint_object_extraction 0.76 10 | determiner_noun_agreement_1 0.83 11 | determiner_noun_agreement_2 0.88 12 | determiner_noun_agreement_irregular_1 0.7 13 | determiner_noun_agreement_irregular_2 0.82 14 | determiner_noun_agreement_with_adj_2 0.76 15 | determiner_noun_agreement_with_adj_irregular_1 0.85 16 | determiner_noun_agreement_with_adj_irregular_2 0.78 17 | determiner_noun_agreement_with_adjective_1 0.8 18 | distractor_agreement_relational_noun 0.76 19 | distractor_agreement_relative_clause 0.61 20 | drop_argument 0.81 21 | ellipsis_n_bar_1 0.68 22 | ellipsis_n_bar_2 0.49 23 | existential_there_object_raising 0.57 24 | existential_there_quantifiers_1 0.97 25 | existential_there_quantifiers_2 0.17 26 | existential_there_subject_raising 0.8 27 | expletive_it_object_raising 0.64 28 | inchoative 0.57 29 | intransitive 0.72 30 | irregular_past_participle_adjectives 0.82 31 | irregular_past_participle_verbs 0.82 32 | irregular_plural_subject_verb_agreement_1 0.74 33 | irregular_plural_subject_verb_agreement_2 0.84 34 | left_branch_island_echo_question 0.32 35 | left_branch_island_simple_question 0.53 36 | matrix_question_npi_licensor_present 0.36 37 | npi_present_1 0.33 38 | npi_present_2 0.52 39 | only_npi_licensor_present 0.59 40 | only_npi_scope 0.58 41 | passive_1 0.77 42 | passive_2 0.81 43 | principle_A_c_command 0.74 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.76 46 | principle_A_domain_1 0.96 47 | principle_A_domain_2 0.68 48 | principle_A_domain_3 0.55 49 | principle_A_reconstruction 0.7 50 | regular_plural_subject_verb_agreement_1 0.82 51 | regular_plural_subject_verb_agreement_2 0.86 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.66 54 | sentential_subject_island 0.52 55 | superlative_quantifiers_1 0.97 56 | superlative_quantifiers_2 0.84 57 | tough_vs_raising_1 0.52 58 | tough_vs_raising_2 0.92 59 | transitive 0.72 60 | wh_island 0.55 61 | wh_questions_object_gap 0.65 62 | wh_questions_subject_gap 0.87 63 | wh_questions_subject_gap_long_distance 0.83 64 | wh_vs_that_no_gap 0.94 65 | wh_vs_that_no_gap_long_distance 0.95 66 | wh_vs_that_with_gap 0.33 67 | wh_vs_that_with_gap_long_distance 0.2 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xlm_bllip-lg_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.63 2 | anaphor_gender_agreement 0.57 3 | anaphor_number_agreement 0.87 4 | animate_subject_passive 0.72 5 | animate_subject_trans 0.54 6 | causative 0.6 7 | complex_NP_island 0.54 8 | coordinate_structure_constraint_complex_left_branch 0.32 9 | coordinate_structure_constraint_object_extraction 0.75 10 | determiner_noun_agreement_1 0.93 11 | determiner_noun_agreement_2 0.91 12 | determiner_noun_agreement_irregular_1 0.78 13 | determiner_noun_agreement_irregular_2 0.88 14 | determiner_noun_agreement_with_adj_2 0.77 15 | determiner_noun_agreement_with_adj_irregular_1 0.81 16 | determiner_noun_agreement_with_adj_irregular_2 0.79 17 | determiner_noun_agreement_with_adjective_1 0.86 18 | distractor_agreement_relational_noun 0.85 19 | distractor_agreement_relative_clause 0.64 20 | drop_argument 0.65 21 | ellipsis_n_bar_1 0.76 22 | ellipsis_n_bar_2 0.55 23 | existential_there_object_raising 0.74 24 | existential_there_quantifiers_1 0.98 25 | existential_there_quantifiers_2 0.32 26 | existential_there_subject_raising 0.82 27 | expletive_it_object_raising 0.73 28 | inchoative 0.56 29 | intransitive 0.71 30 | irregular_past_participle_adjectives 0.65 31 | irregular_past_participle_verbs 0.7 32 | irregular_plural_subject_verb_agreement_1 0.79 33 | irregular_plural_subject_verb_agreement_2 0.9 34 | left_branch_island_echo_question 0.34 35 | left_branch_island_simple_question 0.53 36 | matrix_question_npi_licensor_present 0.26 37 | npi_present_1 0.76 38 | npi_present_2 0.65 39 | only_npi_licensor_present 0.72 40 | only_npi_scope 0.59 41 | passive_1 0.84 42 | passive_2 0.82 43 | principle_A_c_command 0.74 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.82 46 | principle_A_domain_1 0.98 47 | principle_A_domain_2 0.7 48 | principle_A_domain_3 0.52 49 | principle_A_reconstruction 0.6 50 | regular_plural_subject_verb_agreement_1 0.76 51 | regular_plural_subject_verb_agreement_2 0.86 52 | sentential_negation_npi_licensor_present 0.99 53 | sentential_negation_npi_scope 0.69 54 | sentential_subject_island 0.51 55 | superlative_quantifiers_1 0.94 56 | superlative_quantifiers_2 0.8 57 | tough_vs_raising_1 0.46 58 | tough_vs_raising_2 0.9 59 | transitive 0.72 60 | wh_island 0.42 61 | wh_questions_object_gap 0.7 62 | wh_questions_subject_gap 0.81 63 | wh_questions_subject_gap_long_distance 0.88 64 | wh_vs_that_no_gap 0.96 65 | wh_vs_that_no_gap_long_distance 1.0 66 | wh_vs_that_with_gap 0.45 67 | wh_vs_that_with_gap_long_distance 0.19 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xlm_bllip-lg_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.59 2 | anaphor_gender_agreement 0.48 3 | anaphor_number_agreement 0.86 4 | animate_subject_passive 0.69 5 | animate_subject_trans 0.55 6 | causative 0.57 7 | complex_NP_island 0.52 8 | coordinate_structure_constraint_complex_left_branch 0.31 9 | coordinate_structure_constraint_object_extraction 0.71 10 | determiner_noun_agreement_1 0.95 11 | determiner_noun_agreement_2 0.92 12 | determiner_noun_agreement_irregular_1 0.7 13 | determiner_noun_agreement_irregular_2 0.83 14 | determiner_noun_agreement_with_adj_2 0.84 15 | determiner_noun_agreement_with_adj_irregular_1 0.8 16 | determiner_noun_agreement_with_adj_irregular_2 0.8 17 | determiner_noun_agreement_with_adjective_1 0.85 18 | distractor_agreement_relational_noun 0.84 19 | distractor_agreement_relative_clause 0.64 20 | drop_argument 0.73 21 | ellipsis_n_bar_1 0.66 22 | ellipsis_n_bar_2 0.54 23 | existential_there_object_raising 0.65 24 | existential_there_quantifiers_1 0.98 25 | existential_there_quantifiers_2 0.22 26 | existential_there_subject_raising 0.85 27 | expletive_it_object_raising 0.74 28 | inchoative 0.59 29 | intransitive 0.69 30 | irregular_past_participle_adjectives 0.75 31 | irregular_past_participle_verbs 0.7 32 | irregular_plural_subject_verb_agreement_1 0.81 33 | irregular_plural_subject_verb_agreement_2 0.9 34 | left_branch_island_echo_question 0.29 35 | left_branch_island_simple_question 0.49 36 | matrix_question_npi_licensor_present 0.36 37 | npi_present_1 0.78 38 | npi_present_2 0.69 39 | only_npi_licensor_present 0.69 40 | only_npi_scope 0.65 41 | passive_1 0.77 42 | passive_2 0.82 43 | principle_A_c_command 0.77 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.87 46 | principle_A_domain_1 0.93 47 | principle_A_domain_2 0.65 48 | principle_A_domain_3 0.51 49 | principle_A_reconstruction 0.64 50 | regular_plural_subject_verb_agreement_1 0.86 51 | regular_plural_subject_verb_agreement_2 0.91 52 | sentential_negation_npi_licensor_present 0.99 53 | sentential_negation_npi_scope 0.6 54 | sentential_subject_island 0.51 55 | superlative_quantifiers_1 0.9 56 | superlative_quantifiers_2 0.76 57 | tough_vs_raising_1 0.47 58 | tough_vs_raising_2 0.89 59 | transitive 0.76 60 | wh_island 0.48 61 | wh_questions_object_gap 0.77 62 | wh_questions_subject_gap 0.89 63 | wh_questions_subject_gap_long_distance 0.9 64 | wh_vs_that_no_gap 0.96 65 | wh_vs_that_no_gap_long_distance 0.99 66 | wh_vs_that_with_gap 0.28 67 | wh_vs_that_with_gap_long_distance 0.13 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xlm_bllip-lg_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.59 2 | anaphor_gender_agreement 0.53 3 | anaphor_number_agreement 0.83 4 | animate_subject_passive 0.65 5 | animate_subject_trans 0.56 6 | causative 0.62 7 | complex_NP_island 0.46 8 | coordinate_structure_constraint_complex_left_branch 0.38 9 | coordinate_structure_constraint_object_extraction 0.71 10 | determiner_noun_agreement_1 0.92 11 | determiner_noun_agreement_2 0.89 12 | determiner_noun_agreement_irregular_1 0.7 13 | determiner_noun_agreement_irregular_2 0.85 14 | determiner_noun_agreement_with_adj_2 0.78 15 | determiner_noun_agreement_with_adj_irregular_1 0.83 16 | determiner_noun_agreement_with_adj_irregular_2 0.76 17 | determiner_noun_agreement_with_adjective_1 0.81 18 | distractor_agreement_relational_noun 0.8 19 | distractor_agreement_relative_clause 0.62 20 | drop_argument 0.78 21 | ellipsis_n_bar_1 0.77 22 | ellipsis_n_bar_2 0.52 23 | existential_there_object_raising 0.69 24 | existential_there_quantifiers_1 0.94 25 | existential_there_quantifiers_2 0.24 26 | existential_there_subject_raising 0.86 27 | expletive_it_object_raising 0.74 28 | inchoative 0.6 29 | intransitive 0.72 30 | irregular_past_participle_adjectives 0.88 31 | irregular_past_participle_verbs 0.75 32 | irregular_plural_subject_verb_agreement_1 0.75 33 | irregular_plural_subject_verb_agreement_2 0.89 34 | left_branch_island_echo_question 0.19 35 | left_branch_island_simple_question 0.57 36 | matrix_question_npi_licensor_present 0.34 37 | npi_present_1 0.76 38 | npi_present_2 0.72 39 | only_npi_licensor_present 0.9 40 | only_npi_scope 0.73 41 | passive_1 0.81 42 | passive_2 0.88 43 | principle_A_c_command 0.79 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.77 46 | principle_A_domain_1 0.92 47 | principle_A_domain_2 0.67 48 | principle_A_domain_3 0.57 49 | principle_A_reconstruction 0.47 50 | regular_plural_subject_verb_agreement_1 0.78 51 | regular_plural_subject_verb_agreement_2 0.89 52 | sentential_negation_npi_licensor_present 0.98 53 | sentential_negation_npi_scope 0.59 54 | sentential_subject_island 0.4 55 | superlative_quantifiers_1 0.91 56 | superlative_quantifiers_2 0.71 57 | tough_vs_raising_1 0.51 58 | tough_vs_raising_2 0.92 59 | transitive 0.72 60 | wh_island 0.65 61 | wh_questions_object_gap 0.68 62 | wh_questions_subject_gap 0.88 63 | wh_questions_subject_gap_long_distance 0.91 64 | wh_vs_that_no_gap 0.99 65 | wh_vs_that_no_gap_long_distance 1.0 66 | wh_vs_that_with_gap 0.36 67 | wh_vs_that_with_gap_long_distance 0.21 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xlm_bllip-md_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.56 2 | anaphor_gender_agreement 0.39 3 | anaphor_number_agreement 0.82 4 | animate_subject_passive 0.68 5 | animate_subject_trans 0.5 6 | causative 0.56 7 | complex_NP_island 0.53 8 | coordinate_structure_constraint_complex_left_branch 0.19 9 | coordinate_structure_constraint_object_extraction 0.67 10 | determiner_noun_agreement_1 0.9 11 | determiner_noun_agreement_2 0.84 12 | determiner_noun_agreement_irregular_1 0.69 13 | determiner_noun_agreement_irregular_2 0.77 14 | determiner_noun_agreement_with_adj_2 0.71 15 | determiner_noun_agreement_with_adj_irregular_1 0.8 16 | determiner_noun_agreement_with_adj_irregular_2 0.76 17 | determiner_noun_agreement_with_adjective_1 0.86 18 | distractor_agreement_relational_noun 0.81 19 | distractor_agreement_relative_clause 0.57 20 | drop_argument 0.73 21 | ellipsis_n_bar_1 0.75 22 | ellipsis_n_bar_2 0.55 23 | existential_there_object_raising 0.7 24 | existential_there_quantifiers_1 0.96 25 | existential_there_quantifiers_2 0.23 26 | existential_there_subject_raising 0.81 27 | expletive_it_object_raising 0.7 28 | inchoative 0.52 29 | intransitive 0.65 30 | irregular_past_participle_adjectives 0.91 31 | irregular_past_participle_verbs 0.76 32 | irregular_plural_subject_verb_agreement_1 0.74 33 | irregular_plural_subject_verb_agreement_2 0.89 34 | left_branch_island_echo_question 0.32 35 | left_branch_island_simple_question 0.36 36 | matrix_question_npi_licensor_present 0.28 37 | npi_present_1 0.66 38 | npi_present_2 0.59 39 | only_npi_licensor_present 0.74 40 | only_npi_scope 0.82 41 | passive_1 0.69 42 | passive_2 0.78 43 | principle_A_c_command 0.76 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.72 46 | principle_A_domain_1 0.92 47 | principle_A_domain_2 0.76 48 | principle_A_domain_3 0.55 49 | principle_A_reconstruction 0.67 50 | regular_plural_subject_verb_agreement_1 0.82 51 | regular_plural_subject_verb_agreement_2 0.86 52 | sentential_negation_npi_licensor_present 0.99 53 | sentential_negation_npi_scope 0.61 54 | sentential_subject_island 0.61 55 | superlative_quantifiers_1 0.8 56 | superlative_quantifiers_2 0.65 57 | tough_vs_raising_1 0.38 58 | tough_vs_raising_2 0.87 59 | transitive 0.75 60 | wh_island 0.59 61 | wh_questions_object_gap 0.69 62 | wh_questions_subject_gap 0.85 63 | wh_questions_subject_gap_long_distance 0.91 64 | wh_vs_that_no_gap 0.91 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.21 67 | wh_vs_that_with_gap_long_distance 0.22 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xlm_bllip-md_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.58 2 | anaphor_gender_agreement 0.48 3 | anaphor_number_agreement 0.83 4 | animate_subject_passive 0.6 5 | animate_subject_trans 0.53 6 | causative 0.57 7 | complex_NP_island 0.43 8 | coordinate_structure_constraint_complex_left_branch 0.29 9 | coordinate_structure_constraint_object_extraction 0.66 10 | determiner_noun_agreement_1 0.89 11 | determiner_noun_agreement_2 0.93 12 | determiner_noun_agreement_irregular_1 0.68 13 | determiner_noun_agreement_irregular_2 0.84 14 | determiner_noun_agreement_with_adj_2 0.83 15 | determiner_noun_agreement_with_adj_irregular_1 0.75 16 | determiner_noun_agreement_with_adj_irregular_2 0.79 17 | determiner_noun_agreement_with_adjective_1 0.79 18 | distractor_agreement_relational_noun 0.82 19 | distractor_agreement_relative_clause 0.53 20 | drop_argument 0.76 21 | ellipsis_n_bar_1 0.73 22 | ellipsis_n_bar_2 0.56 23 | existential_there_object_raising 0.78 24 | existential_there_quantifiers_1 1.0 25 | existential_there_quantifiers_2 0.18 26 | existential_there_subject_raising 0.8 27 | expletive_it_object_raising 0.71 28 | inchoative 0.58 29 | intransitive 0.68 30 | irregular_past_participle_adjectives 0.6 31 | irregular_past_participle_verbs 0.7 32 | irregular_plural_subject_verb_agreement_1 0.8 33 | irregular_plural_subject_verb_agreement_2 0.89 34 | left_branch_island_echo_question 0.35 35 | left_branch_island_simple_question 0.43 36 | matrix_question_npi_licensor_present 0.29 37 | npi_present_1 0.69 38 | npi_present_2 0.57 39 | only_npi_licensor_present 0.69 40 | only_npi_scope 0.71 41 | passive_1 0.69 42 | passive_2 0.82 43 | principle_A_c_command 0.79 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.81 46 | principle_A_domain_1 0.9 47 | principle_A_domain_2 0.7 48 | principle_A_domain_3 0.54 49 | principle_A_reconstruction 0.5 50 | regular_plural_subject_verb_agreement_1 0.76 51 | regular_plural_subject_verb_agreement_2 0.88 52 | sentential_negation_npi_licensor_present 0.99 53 | sentential_negation_npi_scope 0.7 54 | sentential_subject_island 0.48 55 | superlative_quantifiers_1 0.74 56 | superlative_quantifiers_2 0.77 57 | tough_vs_raising_1 0.43 58 | tough_vs_raising_2 0.84 59 | transitive 0.69 60 | wh_island 0.54 61 | wh_questions_object_gap 0.62 62 | wh_questions_subject_gap 0.78 63 | wh_questions_subject_gap_long_distance 0.92 64 | wh_vs_that_no_gap 0.87 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.29 67 | wh_vs_that_with_gap_long_distance 0.2 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xlm_bllip-md_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.5 2 | anaphor_gender_agreement 0.38 3 | anaphor_number_agreement 0.85 4 | animate_subject_passive 0.67 5 | animate_subject_trans 0.56 6 | causative 0.6 7 | complex_NP_island 0.4 8 | coordinate_structure_constraint_complex_left_branch 0.2 9 | coordinate_structure_constraint_object_extraction 0.58 10 | determiner_noun_agreement_1 0.88 11 | determiner_noun_agreement_2 0.87 12 | determiner_noun_agreement_irregular_1 0.69 13 | determiner_noun_agreement_irregular_2 0.87 14 | determiner_noun_agreement_with_adj_2 0.76 15 | determiner_noun_agreement_with_adj_irregular_1 0.76 16 | determiner_noun_agreement_with_adj_irregular_2 0.8 17 | determiner_noun_agreement_with_adjective_1 0.85 18 | distractor_agreement_relational_noun 0.81 19 | distractor_agreement_relative_clause 0.64 20 | drop_argument 0.75 21 | ellipsis_n_bar_1 0.73 22 | ellipsis_n_bar_2 0.47 23 | existential_there_object_raising 0.66 24 | existential_there_quantifiers_1 0.99 25 | existential_there_quantifiers_2 0.23 26 | existential_there_subject_raising 0.85 27 | expletive_it_object_raising 0.74 28 | inchoative 0.53 29 | intransitive 0.65 30 | irregular_past_participle_adjectives 0.71 31 | irregular_past_participle_verbs 0.81 32 | irregular_plural_subject_verb_agreement_1 0.75 33 | irregular_plural_subject_verb_agreement_2 0.88 34 | left_branch_island_echo_question 0.36 35 | left_branch_island_simple_question 0.48 36 | matrix_question_npi_licensor_present 0.32 37 | npi_present_1 0.41 38 | npi_present_2 0.5 39 | only_npi_licensor_present 0.29 40 | only_npi_scope 0.74 41 | passive_1 0.79 42 | passive_2 0.81 43 | principle_A_c_command 0.78 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.78 46 | principle_A_domain_1 0.87 47 | principle_A_domain_2 0.67 48 | principle_A_domain_3 0.56 49 | principle_A_reconstruction 0.62 50 | regular_plural_subject_verb_agreement_1 0.8 51 | regular_plural_subject_verb_agreement_2 0.84 52 | sentential_negation_npi_licensor_present 0.97 53 | sentential_negation_npi_scope 0.63 54 | sentential_subject_island 0.59 55 | superlative_quantifiers_1 0.91 56 | superlative_quantifiers_2 0.86 57 | tough_vs_raising_1 0.4 58 | tough_vs_raising_2 0.87 59 | transitive 0.75 60 | wh_island 0.65 61 | wh_questions_object_gap 0.74 62 | wh_questions_subject_gap 0.81 63 | wh_questions_subject_gap_long_distance 0.91 64 | wh_vs_that_no_gap 0.93 65 | wh_vs_that_no_gap_long_distance 0.99 66 | wh_vs_that_with_gap 0.24 67 | wh_vs_that_with_gap_long_distance 0.17 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm-mask_bllip-lg_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.76 2 | anaphor_gender_agreement 0.69 3 | anaphor_number_agreement 0.96 4 | animate_subject_passive 0.75 5 | animate_subject_trans 0.64 6 | causative 0.66 7 | complex_NP_island 0.58 8 | coordinate_structure_constraint_complex_left_branch 0.46 9 | coordinate_structure_constraint_object_extraction 0.69 10 | determiner_noun_agreement_1 0.92 11 | determiner_noun_agreement_2 0.9 12 | determiner_noun_agreement_irregular_1 0.72 13 | determiner_noun_agreement_irregular_2 0.92 14 | determiner_noun_agreement_with_adj_2 0.83 15 | determiner_noun_agreement_with_adj_irregular_1 0.76 16 | determiner_noun_agreement_with_adj_irregular_2 0.86 17 | determiner_noun_agreement_with_adjective_1 0.81 18 | distractor_agreement_relational_noun 0.94 19 | distractor_agreement_relative_clause 0.85 20 | drop_argument 0.81 21 | ellipsis_n_bar_1 0.76 22 | ellipsis_n_bar_2 0.6 23 | existential_there_object_raising 0.47 24 | existential_there_quantifiers_1 0.98 25 | existential_there_quantifiers_2 0.25 26 | existential_there_subject_raising 0.77 27 | expletive_it_object_raising 0.62 28 | inchoative 0.65 29 | intransitive 0.78 30 | irregular_past_participle_adjectives 0.82 31 | irregular_past_participle_verbs 0.78 32 | irregular_plural_subject_verb_agreement_1 0.8 33 | irregular_plural_subject_verb_agreement_2 0.89 34 | left_branch_island_echo_question 0.26 35 | left_branch_island_simple_question 0.64 36 | matrix_question_npi_licensor_present 0.82 37 | npi_present_1 0.75 38 | npi_present_2 0.9 39 | only_npi_licensor_present 0.98 40 | only_npi_scope 0.85 41 | passive_1 0.81 42 | passive_2 0.8 43 | principle_A_c_command 0.78 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.81 46 | principle_A_domain_1 0.95 47 | principle_A_domain_2 0.59 48 | principle_A_domain_3 0.48 49 | principle_A_reconstruction 0.38 50 | regular_plural_subject_verb_agreement_1 0.8 51 | regular_plural_subject_verb_agreement_2 0.95 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.89 54 | sentential_subject_island 0.46 55 | superlative_quantifiers_1 0.9 56 | superlative_quantifiers_2 0.74 57 | tough_vs_raising_1 0.6 58 | tough_vs_raising_2 0.93 59 | transitive 0.74 60 | wh_island 0.65 61 | wh_questions_object_gap 0.77 62 | wh_questions_subject_gap 0.87 63 | wh_questions_subject_gap_long_distance 0.85 64 | wh_vs_that_no_gap 0.97 65 | wh_vs_that_no_gap_long_distance 0.99 66 | wh_vs_that_with_gap 0.43 67 | wh_vs_that_with_gap_long_distance 0.26 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm-mask_bllip-lg_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.81 2 | anaphor_gender_agreement 0.58 3 | anaphor_number_agreement 0.9 4 | animate_subject_passive 0.68 5 | animate_subject_trans 0.66 6 | causative 0.65 7 | complex_NP_island 0.68 8 | coordinate_structure_constraint_complex_left_branch 0.55 9 | coordinate_structure_constraint_object_extraction 0.71 10 | determiner_noun_agreement_1 0.92 11 | determiner_noun_agreement_2 0.84 12 | determiner_noun_agreement_irregular_1 0.73 13 | determiner_noun_agreement_irregular_2 0.9 14 | determiner_noun_agreement_with_adj_2 0.85 15 | determiner_noun_agreement_with_adj_irregular_1 0.85 16 | determiner_noun_agreement_with_adj_irregular_2 0.9 17 | determiner_noun_agreement_with_adjective_1 0.89 18 | distractor_agreement_relational_noun 0.95 19 | distractor_agreement_relative_clause 0.81 20 | drop_argument 0.79 21 | ellipsis_n_bar_1 0.78 22 | ellipsis_n_bar_2 0.6 23 | existential_there_object_raising 0.59 24 | existential_there_quantifiers_1 0.91 25 | existential_there_quantifiers_2 0.12 26 | existential_there_subject_raising 0.77 27 | expletive_it_object_raising 0.67 28 | inchoative 0.71 29 | intransitive 0.8 30 | irregular_past_participle_adjectives 0.8 31 | irregular_past_participle_verbs 0.85 32 | irregular_plural_subject_verb_agreement_1 0.82 33 | irregular_plural_subject_verb_agreement_2 0.91 34 | left_branch_island_echo_question 0.22 35 | left_branch_island_simple_question 0.76 36 | matrix_question_npi_licensor_present 0.82 37 | npi_present_1 0.79 38 | npi_present_2 0.93 39 | only_npi_licensor_present 0.98 40 | only_npi_scope 0.83 41 | passive_1 0.83 42 | passive_2 0.8 43 | principle_A_c_command 0.76 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.86 46 | principle_A_domain_1 1.0 47 | principle_A_domain_2 0.63 48 | principle_A_domain_3 0.59 49 | principle_A_reconstruction 0.4 50 | regular_plural_subject_verb_agreement_1 0.85 51 | regular_plural_subject_verb_agreement_2 0.91 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.83 54 | sentential_subject_island 0.57 55 | superlative_quantifiers_1 0.97 56 | superlative_quantifiers_2 0.74 57 | tough_vs_raising_1 0.52 58 | tough_vs_raising_2 0.87 59 | transitive 0.76 60 | wh_island 0.68 61 | wh_questions_object_gap 0.85 62 | wh_questions_subject_gap 0.86 63 | wh_questions_subject_gap_long_distance 0.81 64 | wh_vs_that_no_gap 0.95 65 | wh_vs_that_no_gap_long_distance 0.96 66 | wh_vs_that_with_gap 0.58 67 | wh_vs_that_with_gap_long_distance 0.34 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm-mask_bllip-lg_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.83 2 | anaphor_gender_agreement 0.64 3 | anaphor_number_agreement 0.92 4 | animate_subject_passive 0.71 5 | animate_subject_trans 0.65 6 | causative 0.62 7 | complex_NP_island 0.67 8 | coordinate_structure_constraint_complex_left_branch 0.63 9 | coordinate_structure_constraint_object_extraction 0.71 10 | determiner_noun_agreement_1 0.91 11 | determiner_noun_agreement_2 0.89 12 | determiner_noun_agreement_irregular_1 0.75 13 | determiner_noun_agreement_irregular_2 0.86 14 | determiner_noun_agreement_with_adj_2 0.8 15 | determiner_noun_agreement_with_adj_irregular_1 0.73 16 | determiner_noun_agreement_with_adj_irregular_2 0.84 17 | determiner_noun_agreement_with_adjective_1 0.83 18 | distractor_agreement_relational_noun 0.94 19 | distractor_agreement_relative_clause 0.83 20 | drop_argument 0.79 21 | ellipsis_n_bar_1 0.75 22 | ellipsis_n_bar_2 0.61 23 | existential_there_object_raising 0.59 24 | existential_there_quantifiers_1 0.96 25 | existential_there_quantifiers_2 0.14 26 | existential_there_subject_raising 0.78 27 | expletive_it_object_raising 0.61 28 | inchoative 0.62 29 | intransitive 0.78 30 | irregular_past_participle_adjectives 0.7 31 | irregular_past_participle_verbs 0.87 32 | irregular_plural_subject_verb_agreement_1 0.84 33 | irregular_plural_subject_verb_agreement_2 0.88 34 | left_branch_island_echo_question 0.29 35 | left_branch_island_simple_question 0.71 36 | matrix_question_npi_licensor_present 0.81 37 | npi_present_1 0.74 38 | npi_present_2 0.84 39 | only_npi_licensor_present 0.99 40 | only_npi_scope 0.81 41 | passive_1 0.81 42 | passive_2 0.82 43 | principle_A_c_command 0.7 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.82 46 | principle_A_domain_1 0.96 47 | principle_A_domain_2 0.53 48 | principle_A_domain_3 0.49 49 | principle_A_reconstruction 0.31 50 | regular_plural_subject_verb_agreement_1 0.87 51 | regular_plural_subject_verb_agreement_2 0.93 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.8 54 | sentential_subject_island 0.46 55 | superlative_quantifiers_1 0.96 56 | superlative_quantifiers_2 0.67 57 | tough_vs_raising_1 0.48 58 | tough_vs_raising_2 0.85 59 | transitive 0.71 60 | wh_island 0.72 61 | wh_questions_object_gap 0.78 62 | wh_questions_subject_gap 0.82 63 | wh_questions_subject_gap_long_distance 0.8 64 | wh_vs_that_no_gap 0.99 65 | wh_vs_that_no_gap_long_distance 1.0 66 | wh_vs_that_with_gap 0.39 67 | wh_vs_that_with_gap_long_distance 0.38 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm-mask_bllip-md_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.78 2 | anaphor_gender_agreement 0.61 3 | anaphor_number_agreement 0.9 4 | animate_subject_passive 0.67 5 | animate_subject_trans 0.7 6 | causative 0.56 7 | complex_NP_island 0.61 8 | coordinate_structure_constraint_complex_left_branch 0.5 9 | coordinate_structure_constraint_object_extraction 0.64 10 | determiner_noun_agreement_1 0.89 11 | determiner_noun_agreement_2 0.86 12 | determiner_noun_agreement_irregular_1 0.73 13 | determiner_noun_agreement_irregular_2 0.81 14 | determiner_noun_agreement_with_adj_2 0.8 15 | determiner_noun_agreement_with_adj_irregular_1 0.82 16 | determiner_noun_agreement_with_adj_irregular_2 0.8 17 | determiner_noun_agreement_with_adjective_1 0.88 18 | distractor_agreement_relational_noun 0.96 19 | distractor_agreement_relative_clause 0.85 20 | drop_argument 0.77 21 | ellipsis_n_bar_1 0.74 22 | ellipsis_n_bar_2 0.58 23 | existential_there_object_raising 0.59 24 | existential_there_quantifiers_1 0.99 25 | existential_there_quantifiers_2 0.28 26 | existential_there_subject_raising 0.84 27 | expletive_it_object_raising 0.62 28 | inchoative 0.58 29 | intransitive 0.73 30 | irregular_past_participle_adjectives 0.84 31 | irregular_past_participle_verbs 0.78 32 | irregular_plural_subject_verb_agreement_1 0.82 33 | irregular_plural_subject_verb_agreement_2 0.94 34 | left_branch_island_echo_question 0.36 35 | left_branch_island_simple_question 0.59 36 | matrix_question_npi_licensor_present 0.61 37 | npi_present_1 0.73 38 | npi_present_2 0.79 39 | only_npi_licensor_present 0.84 40 | only_npi_scope 0.8 41 | passive_1 0.83 42 | passive_2 0.83 43 | principle_A_c_command 0.8 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.81 46 | principle_A_domain_1 0.88 47 | principle_A_domain_2 0.66 48 | principle_A_domain_3 0.5 49 | principle_A_reconstruction 0.33 50 | regular_plural_subject_verb_agreement_1 0.84 51 | regular_plural_subject_verb_agreement_2 0.89 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.82 54 | sentential_subject_island 0.41 55 | superlative_quantifiers_1 0.93 56 | superlative_quantifiers_2 0.83 57 | tough_vs_raising_1 0.44 58 | tough_vs_raising_2 0.88 59 | transitive 0.72 60 | wh_island 0.58 61 | wh_questions_object_gap 0.77 62 | wh_questions_subject_gap 0.88 63 | wh_questions_subject_gap_long_distance 0.88 64 | wh_vs_that_no_gap 0.97 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.43 67 | wh_vs_that_with_gap_long_distance 0.19 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm-mask_bllip-md_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.71 2 | anaphor_gender_agreement 0.52 3 | anaphor_number_agreement 0.83 4 | animate_subject_passive 0.71 5 | animate_subject_trans 0.65 6 | causative 0.64 7 | complex_NP_island 0.51 8 | coordinate_structure_constraint_complex_left_branch 0.41 9 | coordinate_structure_constraint_object_extraction 0.67 10 | determiner_noun_agreement_1 0.91 11 | determiner_noun_agreement_2 0.83 12 | determiner_noun_agreement_irregular_1 0.69 13 | determiner_noun_agreement_irregular_2 0.85 14 | determiner_noun_agreement_with_adj_2 0.81 15 | determiner_noun_agreement_with_adj_irregular_1 0.81 16 | determiner_noun_agreement_with_adj_irregular_2 0.8 17 | determiner_noun_agreement_with_adjective_1 0.82 18 | distractor_agreement_relational_noun 0.96 19 | distractor_agreement_relative_clause 0.82 20 | drop_argument 0.8 21 | ellipsis_n_bar_1 0.7 22 | ellipsis_n_bar_2 0.57 23 | existential_there_object_raising 0.6 24 | existential_there_quantifiers_1 0.94 25 | existential_there_quantifiers_2 0.1 26 | existential_there_subject_raising 0.86 27 | expletive_it_object_raising 0.61 28 | inchoative 0.59 29 | intransitive 0.71 30 | irregular_past_participle_adjectives 0.66 31 | irregular_past_participle_verbs 0.87 32 | irregular_plural_subject_verb_agreement_1 0.75 33 | irregular_plural_subject_verb_agreement_2 0.87 34 | left_branch_island_echo_question 0.32 35 | left_branch_island_simple_question 0.65 36 | matrix_question_npi_licensor_present 0.55 37 | npi_present_1 0.8 38 | npi_present_2 0.78 39 | only_npi_licensor_present 0.88 40 | only_npi_scope 0.76 41 | passive_1 0.76 42 | passive_2 0.77 43 | principle_A_c_command 0.81 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.79 46 | principle_A_domain_1 0.94 47 | principle_A_domain_2 0.67 48 | principle_A_domain_3 0.49 49 | principle_A_reconstruction 0.45 50 | regular_plural_subject_verb_agreement_1 0.84 51 | regular_plural_subject_verb_agreement_2 0.88 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.79 54 | sentential_subject_island 0.52 55 | superlative_quantifiers_1 0.78 56 | superlative_quantifiers_2 0.81 57 | tough_vs_raising_1 0.45 58 | tough_vs_raising_2 0.88 59 | transitive 0.67 60 | wh_island 0.67 61 | wh_questions_object_gap 0.75 62 | wh_questions_subject_gap 0.85 63 | wh_questions_subject_gap_long_distance 0.74 64 | wh_vs_that_no_gap 0.91 65 | wh_vs_that_no_gap_long_distance 0.96 66 | wh_vs_that_with_gap 0.45 67 | wh_vs_that_with_gap_long_distance 0.29 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm-mask_bllip-md_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.69 2 | anaphor_gender_agreement 0.63 3 | anaphor_number_agreement 0.94 4 | animate_subject_passive 0.64 5 | animate_subject_trans 0.63 6 | causative 0.62 7 | complex_NP_island 0.51 8 | coordinate_structure_constraint_complex_left_branch 0.33 9 | coordinate_structure_constraint_object_extraction 0.56 10 | determiner_noun_agreement_1 0.9 11 | determiner_noun_agreement_2 0.88 12 | determiner_noun_agreement_irregular_1 0.7 13 | determiner_noun_agreement_irregular_2 0.88 14 | determiner_noun_agreement_with_adj_2 0.79 15 | determiner_noun_agreement_with_adj_irregular_1 0.8 16 | determiner_noun_agreement_with_adj_irregular_2 0.84 17 | determiner_noun_agreement_with_adjective_1 0.84 18 | distractor_agreement_relational_noun 0.95 19 | distractor_agreement_relative_clause 0.84 20 | drop_argument 0.76 21 | ellipsis_n_bar_1 0.73 22 | ellipsis_n_bar_2 0.53 23 | existential_there_object_raising 0.57 24 | existential_there_quantifiers_1 0.98 25 | existential_there_quantifiers_2 0.29 26 | existential_there_subject_raising 0.8 27 | expletive_it_object_raising 0.59 28 | inchoative 0.63 29 | intransitive 0.79 30 | irregular_past_participle_adjectives 0.85 31 | irregular_past_participle_verbs 0.78 32 | irregular_plural_subject_verb_agreement_1 0.78 33 | irregular_plural_subject_verb_agreement_2 0.89 34 | left_branch_island_echo_question 0.27 35 | left_branch_island_simple_question 0.41 36 | matrix_question_npi_licensor_present 0.7 37 | npi_present_1 0.64 38 | npi_present_2 0.8 39 | only_npi_licensor_present 0.66 40 | only_npi_scope 0.6 41 | passive_1 0.77 42 | passive_2 0.79 43 | principle_A_c_command 0.77 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.84 46 | principle_A_domain_1 0.93 47 | principle_A_domain_2 0.63 48 | principle_A_domain_3 0.53 49 | principle_A_reconstruction 0.29 50 | regular_plural_subject_verb_agreement_1 0.86 51 | regular_plural_subject_verb_agreement_2 0.91 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.71 54 | sentential_subject_island 0.46 55 | superlative_quantifiers_1 0.91 56 | superlative_quantifiers_2 0.83 57 | tough_vs_raising_1 0.46 58 | tough_vs_raising_2 0.95 59 | transitive 0.66 60 | wh_island 0.62 61 | wh_questions_object_gap 0.74 62 | wh_questions_subject_gap 0.91 63 | wh_questions_subject_gap_long_distance 0.79 64 | wh_vs_that_no_gap 0.96 65 | wh_vs_that_no_gap_long_distance 0.97 66 | wh_vs_that_with_gap 0.32 67 | wh_vs_that_with_gap_long_distance 0.21 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_-mask_bllip-lg_rand-init_1101_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/transformers-struct-guidance/17716afec2f7b060163cf5e45a4ebef6b6709555/analysis/results/blimp/xplm_-mask_bllip-lg_rand-init_1101_5.txt -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_-mask_bllip-lg_rand-init_1102_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/transformers-struct-guidance/17716afec2f7b060163cf5e45a4ebef6b6709555/analysis/results/blimp/xplm_-mask_bllip-lg_rand-init_1102_5.txt -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_-mask_bllip-lg_rand-init_1103_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/transformers-struct-guidance/17716afec2f7b060163cf5e45a4ebef6b6709555/analysis/results/blimp/xplm_-mask_bllip-lg_rand-init_1103_5.txt -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_-mask_bllip-md_rand-init_1101_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/transformers-struct-guidance/17716afec2f7b060163cf5e45a4ebef6b6709555/analysis/results/blimp/xplm_-mask_bllip-md_rand-init_1101_5.txt -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_-mask_bllip-md_rand-init_1102_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/transformers-struct-guidance/17716afec2f7b060163cf5e45a4ebef6b6709555/analysis/results/blimp/xplm_-mask_bllip-md_rand-init_1102_5.txt -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_-mask_bllip-md_rand-init_1103_5.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/transformers-struct-guidance/17716afec2f7b060163cf5e45a4ebef6b6709555/analysis/results/blimp/xplm_-mask_bllip-md_rand-init_1103_5.txt -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_bllip-lg_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.81 2 | anaphor_gender_agreement 0.7 3 | anaphor_number_agreement 0.94 4 | animate_subject_passive 0.74 5 | animate_subject_trans 0.65 6 | causative 0.63 7 | complex_NP_island 0.65 8 | coordinate_structure_constraint_complex_left_branch 0.61 9 | coordinate_structure_constraint_object_extraction 0.71 10 | determiner_noun_agreement_1 0.93 11 | determiner_noun_agreement_2 0.8 12 | determiner_noun_agreement_irregular_1 0.72 13 | determiner_noun_agreement_irregular_2 0.9 14 | determiner_noun_agreement_with_adj_2 0.84 15 | determiner_noun_agreement_with_adj_irregular_1 0.8 16 | determiner_noun_agreement_with_adj_irregular_2 0.87 17 | determiner_noun_agreement_with_adjective_1 0.86 18 | distractor_agreement_relational_noun 0.96 19 | distractor_agreement_relative_clause 0.83 20 | drop_argument 0.76 21 | ellipsis_n_bar_1 0.71 22 | ellipsis_n_bar_2 0.58 23 | existential_there_object_raising 0.66 24 | existential_there_quantifiers_1 0.95 25 | existential_there_quantifiers_2 0.26 26 | existential_there_subject_raising 0.89 27 | expletive_it_object_raising 0.67 28 | inchoative 0.6 29 | intransitive 0.74 30 | irregular_past_participle_adjectives 0.79 31 | irregular_past_participle_verbs 0.82 32 | irregular_plural_subject_verb_agreement_1 0.84 33 | irregular_plural_subject_verb_agreement_2 0.93 34 | left_branch_island_echo_question 0.11 35 | left_branch_island_simple_question 0.72 36 | matrix_question_npi_licensor_present 0.76 37 | npi_present_1 0.92 38 | npi_present_2 0.83 39 | only_npi_licensor_present 0.98 40 | only_npi_scope 0.82 41 | passive_1 0.82 42 | passive_2 0.84 43 | principle_A_c_command 0.82 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.82 46 | principle_A_domain_1 0.98 47 | principle_A_domain_2 0.62 48 | principle_A_domain_3 0.47 49 | principle_A_reconstruction 0.44 50 | regular_plural_subject_verb_agreement_1 0.89 51 | regular_plural_subject_verb_agreement_2 0.9 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.77 54 | sentential_subject_island 0.35 55 | superlative_quantifiers_1 0.95 56 | superlative_quantifiers_2 0.7 57 | tough_vs_raising_1 0.48 58 | tough_vs_raising_2 0.9 59 | transitive 0.73 60 | wh_island 0.71 61 | wh_questions_object_gap 0.8 62 | wh_questions_subject_gap 0.8 63 | wh_questions_subject_gap_long_distance 0.87 64 | wh_vs_that_no_gap 0.97 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.41 67 | wh_vs_that_with_gap_long_distance 0.24 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_bllip-lg_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.82 2 | anaphor_gender_agreement 0.59 3 | anaphor_number_agreement 0.86 4 | animate_subject_passive 0.74 5 | animate_subject_trans 0.57 6 | causative 0.65 7 | complex_NP_island 0.58 8 | coordinate_structure_constraint_complex_left_branch 0.51 9 | coordinate_structure_constraint_object_extraction 0.61 10 | determiner_noun_agreement_1 0.93 11 | determiner_noun_agreement_2 0.9 12 | determiner_noun_agreement_irregular_1 0.72 13 | determiner_noun_agreement_irregular_2 0.88 14 | determiner_noun_agreement_with_adj_2 0.77 15 | determiner_noun_agreement_with_adj_irregular_1 0.82 16 | determiner_noun_agreement_with_adj_irregular_2 0.91 17 | determiner_noun_agreement_with_adjective_1 0.88 18 | distractor_agreement_relational_noun 0.94 19 | distractor_agreement_relative_clause 0.84 20 | drop_argument 0.74 21 | ellipsis_n_bar_1 0.77 22 | ellipsis_n_bar_2 0.57 23 | existential_there_object_raising 0.51 24 | existential_there_quantifiers_1 0.97 25 | existential_there_quantifiers_2 0.24 26 | existential_there_subject_raising 0.83 27 | expletive_it_object_raising 0.62 28 | inchoative 0.59 29 | intransitive 0.75 30 | irregular_past_participle_adjectives 0.8 31 | irregular_past_participle_verbs 0.82 32 | irregular_plural_subject_verb_agreement_1 0.83 33 | irregular_plural_subject_verb_agreement_2 0.94 34 | left_branch_island_echo_question 0.21 35 | left_branch_island_simple_question 0.66 36 | matrix_question_npi_licensor_present 0.72 37 | npi_present_1 0.8 38 | npi_present_2 0.9 39 | only_npi_licensor_present 0.99 40 | only_npi_scope 0.85 41 | passive_1 0.84 42 | passive_2 0.82 43 | principle_A_c_command 0.85 44 | principle_A_case_1 0.99 45 | principle_A_case_2 0.83 46 | principle_A_domain_1 0.93 47 | principle_A_domain_2 0.67 48 | principle_A_domain_3 0.55 49 | principle_A_reconstruction 0.47 50 | regular_plural_subject_verb_agreement_1 0.84 51 | regular_plural_subject_verb_agreement_2 0.92 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.8 54 | sentential_subject_island 0.45 55 | superlative_quantifiers_1 0.96 56 | superlative_quantifiers_2 0.72 57 | tough_vs_raising_1 0.55 58 | tough_vs_raising_2 0.87 59 | transitive 0.72 60 | wh_island 0.75 61 | wh_questions_object_gap 0.75 62 | wh_questions_subject_gap 0.81 63 | wh_questions_subject_gap_long_distance 0.85 64 | wh_vs_that_no_gap 0.97 65 | wh_vs_that_no_gap_long_distance 0.99 66 | wh_vs_that_with_gap 0.51 67 | wh_vs_that_with_gap_long_distance 0.29 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_bllip-lg_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.83 2 | anaphor_gender_agreement 0.73 3 | anaphor_number_agreement 0.91 4 | animate_subject_passive 0.74 5 | animate_subject_trans 0.68 6 | causative 0.63 7 | complex_NP_island 0.62 8 | coordinate_structure_constraint_complex_left_branch 0.4 9 | coordinate_structure_constraint_object_extraction 0.79 10 | determiner_noun_agreement_1 0.95 11 | determiner_noun_agreement_2 0.86 12 | determiner_noun_agreement_irregular_1 0.72 13 | determiner_noun_agreement_irregular_2 0.9 14 | determiner_noun_agreement_with_adj_2 0.82 15 | determiner_noun_agreement_with_adj_irregular_1 0.84 16 | determiner_noun_agreement_with_adj_irregular_2 0.88 17 | determiner_noun_agreement_with_adjective_1 0.91 18 | distractor_agreement_relational_noun 0.95 19 | distractor_agreement_relative_clause 0.85 20 | drop_argument 0.78 21 | ellipsis_n_bar_1 0.76 22 | ellipsis_n_bar_2 0.62 23 | existential_there_object_raising 0.66 24 | existential_there_quantifiers_1 0.93 25 | existential_there_quantifiers_2 0.16 26 | existential_there_subject_raising 0.82 27 | expletive_it_object_raising 0.73 28 | inchoative 0.66 29 | intransitive 0.77 30 | irregular_past_participle_adjectives 0.71 31 | irregular_past_participle_verbs 0.83 32 | irregular_plural_subject_verb_agreement_1 0.8 33 | irregular_plural_subject_verb_agreement_2 0.94 34 | left_branch_island_echo_question 0.2 35 | left_branch_island_simple_question 0.52 36 | matrix_question_npi_licensor_present 0.74 37 | npi_present_1 0.61 38 | npi_present_2 0.8 39 | only_npi_licensor_present 0.95 40 | only_npi_scope 0.78 41 | passive_1 0.86 42 | passive_2 0.86 43 | principle_A_c_command 0.81 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.77 46 | principle_A_domain_1 0.99 47 | principle_A_domain_2 0.6 48 | principle_A_domain_3 0.5 49 | principle_A_reconstruction 0.29 50 | regular_plural_subject_verb_agreement_1 0.86 51 | regular_plural_subject_verb_agreement_2 0.95 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.77 54 | sentential_subject_island 0.42 55 | superlative_quantifiers_1 0.95 56 | superlative_quantifiers_2 0.66 57 | tough_vs_raising_1 0.43 58 | tough_vs_raising_2 0.96 59 | transitive 0.72 60 | wh_island 0.66 61 | wh_questions_object_gap 0.74 62 | wh_questions_subject_gap 0.86 63 | wh_questions_subject_gap_long_distance 0.83 64 | wh_vs_that_no_gap 0.97 65 | wh_vs_that_no_gap_long_distance 0.99 66 | wh_vs_that_with_gap 0.49 67 | wh_vs_that_with_gap_long_distance 0.33 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_bllip-md_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.8 2 | anaphor_gender_agreement 0.51 3 | anaphor_number_agreement 0.9 4 | animate_subject_passive 0.72 5 | animate_subject_trans 0.63 6 | causative 0.64 7 | complex_NP_island 0.56 8 | coordinate_structure_constraint_complex_left_branch 0.47 9 | coordinate_structure_constraint_object_extraction 0.68 10 | determiner_noun_agreement_1 0.88 11 | determiner_noun_agreement_2 0.83 12 | determiner_noun_agreement_irregular_1 0.72 13 | determiner_noun_agreement_irregular_2 0.84 14 | determiner_noun_agreement_with_adj_2 0.77 15 | determiner_noun_agreement_with_adj_irregular_1 0.86 16 | determiner_noun_agreement_with_adj_irregular_2 0.79 17 | determiner_noun_agreement_with_adjective_1 0.86 18 | distractor_agreement_relational_noun 0.98 19 | distractor_agreement_relative_clause 0.82 20 | drop_argument 0.72 21 | ellipsis_n_bar_1 0.67 22 | ellipsis_n_bar_2 0.47 23 | existential_there_object_raising 0.62 24 | existential_there_quantifiers_1 0.95 25 | existential_there_quantifiers_2 0.28 26 | existential_there_subject_raising 0.84 27 | expletive_it_object_raising 0.67 28 | inchoative 0.56 29 | intransitive 0.7 30 | irregular_past_participle_adjectives 0.82 31 | irregular_past_participle_verbs 0.91 32 | irregular_plural_subject_verb_agreement_1 0.8 33 | irregular_plural_subject_verb_agreement_2 0.87 34 | left_branch_island_echo_question 0.26 35 | left_branch_island_simple_question 0.58 36 | matrix_question_npi_licensor_present 0.55 37 | npi_present_1 0.52 38 | npi_present_2 0.76 39 | only_npi_licensor_present 0.86 40 | only_npi_scope 0.77 41 | passive_1 0.8 42 | passive_2 0.78 43 | principle_A_c_command 0.78 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.8 46 | principle_A_domain_1 0.94 47 | principle_A_domain_2 0.66 48 | principle_A_domain_3 0.52 49 | principle_A_reconstruction 0.38 50 | regular_plural_subject_verb_agreement_1 0.83 51 | regular_plural_subject_verb_agreement_2 0.91 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.75 54 | sentential_subject_island 0.45 55 | superlative_quantifiers_1 0.92 56 | superlative_quantifiers_2 0.67 57 | tough_vs_raising_1 0.45 58 | tough_vs_raising_2 0.88 59 | transitive 0.63 60 | wh_island 0.47 61 | wh_questions_object_gap 0.76 62 | wh_questions_subject_gap 0.94 63 | wh_questions_subject_gap_long_distance 0.83 64 | wh_vs_that_no_gap 0.97 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.46 67 | wh_vs_that_with_gap_long_distance 0.21 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_bllip-md_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.82 2 | anaphor_gender_agreement 0.62 3 | anaphor_number_agreement 0.87 4 | animate_subject_passive 0.7 5 | animate_subject_trans 0.64 6 | causative 0.59 7 | complex_NP_island 0.55 8 | coordinate_structure_constraint_complex_left_branch 0.32 9 | coordinate_structure_constraint_object_extraction 0.65 10 | determiner_noun_agreement_1 0.92 11 | determiner_noun_agreement_2 0.92 12 | determiner_noun_agreement_irregular_1 0.72 13 | determiner_noun_agreement_irregular_2 0.88 14 | determiner_noun_agreement_with_adj_2 0.82 15 | determiner_noun_agreement_with_adj_irregular_1 0.8 16 | determiner_noun_agreement_with_adj_irregular_2 0.85 17 | determiner_noun_agreement_with_adjective_1 0.86 18 | distractor_agreement_relational_noun 0.93 19 | distractor_agreement_relative_clause 0.85 20 | drop_argument 0.76 21 | ellipsis_n_bar_1 0.76 22 | ellipsis_n_bar_2 0.58 23 | existential_there_object_raising 0.63 24 | existential_there_quantifiers_1 0.97 25 | existential_there_quantifiers_2 0.06 26 | existential_there_subject_raising 0.84 27 | expletive_it_object_raising 0.61 28 | inchoative 0.61 29 | intransitive 0.72 30 | irregular_past_participle_adjectives 0.75 31 | irregular_past_participle_verbs 0.73 32 | irregular_plural_subject_verb_agreement_1 0.84 33 | irregular_plural_subject_verb_agreement_2 0.9 34 | left_branch_island_echo_question 0.25 35 | left_branch_island_simple_question 0.45 36 | matrix_question_npi_licensor_present 0.62 37 | npi_present_1 0.55 38 | npi_present_2 0.73 39 | only_npi_licensor_present 0.9 40 | only_npi_scope 0.7 41 | passive_1 0.79 42 | passive_2 0.78 43 | principle_A_c_command 0.78 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.78 46 | principle_A_domain_1 0.94 47 | principle_A_domain_2 0.73 48 | principle_A_domain_3 0.59 49 | principle_A_reconstruction 0.25 50 | regular_plural_subject_verb_agreement_1 0.82 51 | regular_plural_subject_verb_agreement_2 0.9 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.74 54 | sentential_subject_island 0.49 55 | superlative_quantifiers_1 0.87 56 | superlative_quantifiers_2 0.66 57 | tough_vs_raising_1 0.43 58 | tough_vs_raising_2 0.91 59 | transitive 0.68 60 | wh_island 0.67 61 | wh_questions_object_gap 0.73 62 | wh_questions_subject_gap 0.82 63 | wh_questions_subject_gap_long_distance 0.8 64 | wh_vs_that_no_gap 0.96 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.42 67 | wh_vs_that_with_gap_long_distance 0.21 68 | -------------------------------------------------------------------------------- /analysis/results/blimp/xplm_bllip-md_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | adjunct_island 0.79 2 | anaphor_gender_agreement 0.64 3 | anaphor_number_agreement 0.96 4 | animate_subject_passive 0.66 5 | animate_subject_trans 0.63 6 | causative 0.59 7 | complex_NP_island 0.56 8 | coordinate_structure_constraint_complex_left_branch 0.45 9 | coordinate_structure_constraint_object_extraction 0.71 10 | determiner_noun_agreement_1 0.83 11 | determiner_noun_agreement_2 0.86 12 | determiner_noun_agreement_irregular_1 0.72 13 | determiner_noun_agreement_irregular_2 0.84 14 | determiner_noun_agreement_with_adj_2 0.75 15 | determiner_noun_agreement_with_adj_irregular_1 0.84 16 | determiner_noun_agreement_with_adj_irregular_2 0.81 17 | determiner_noun_agreement_with_adjective_1 0.83 18 | distractor_agreement_relational_noun 0.96 19 | distractor_agreement_relative_clause 0.81 20 | drop_argument 0.75 21 | ellipsis_n_bar_1 0.76 22 | ellipsis_n_bar_2 0.54 23 | existential_there_object_raising 0.51 24 | existential_there_quantifiers_1 0.95 25 | existential_there_quantifiers_2 0.21 26 | existential_there_subject_raising 0.83 27 | expletive_it_object_raising 0.62 28 | inchoative 0.55 29 | intransitive 0.7 30 | irregular_past_participle_adjectives 0.65 31 | irregular_past_participle_verbs 0.84 32 | irregular_plural_subject_verb_agreement_1 0.8 33 | irregular_plural_subject_verb_agreement_2 0.9 34 | left_branch_island_echo_question 0.22 35 | left_branch_island_simple_question 0.64 36 | matrix_question_npi_licensor_present 0.61 37 | npi_present_1 0.84 38 | npi_present_2 0.95 39 | only_npi_licensor_present 0.75 40 | only_npi_scope 0.68 41 | passive_1 0.83 42 | passive_2 0.77 43 | principle_A_c_command 0.75 44 | principle_A_case_1 1.0 45 | principle_A_case_2 0.81 46 | principle_A_domain_1 0.92 47 | principle_A_domain_2 0.65 48 | principle_A_domain_3 0.56 49 | principle_A_reconstruction 0.23 50 | regular_plural_subject_verb_agreement_1 0.89 51 | regular_plural_subject_verb_agreement_2 0.9 52 | sentential_negation_npi_licensor_present 1.0 53 | sentential_negation_npi_scope 0.74 54 | sentential_subject_island 0.43 55 | superlative_quantifiers_1 0.85 56 | superlative_quantifiers_2 0.85 57 | tough_vs_raising_1 0.52 58 | tough_vs_raising_2 0.86 59 | transitive 0.75 60 | wh_island 0.52 61 | wh_questions_object_gap 0.8 62 | wh_questions_subject_gap 0.82 63 | wh_questions_subject_gap_long_distance 0.86 64 | wh_vs_that_no_gap 0.98 65 | wh_vs_that_no_gap_long_distance 0.98 66 | wh_vs_that_with_gap 0.46 67 | wh_vs_that_with_gap_long_distance 0.25 68 | -------------------------------------------------------------------------------- /analysis/results/model_ppl_list.txt: -------------------------------------------------------------------------------- 1 | xlm bllip-md 1101 67.76539311438349 2 | xlm bllip-md 1102 67.33157810365111 3 | xlm bllip-md 1103 67.72760499055684 4 | xlm bllip-lg 1101 49.17506099990381 5 | xlm bllip-lg 1102 48.811130898360545 6 | xlm bllip-lg 1103 49.025735379088914 7 | sclm-past bllip-md 1101 67.27234472574018 8 | sclm-past bllip-md 1102 67.42877757453118 9 | sclm-past bllip-md 1103 67.24684697537238 10 | sclm-past bllip-md 1104 67.75107870483356 11 | sclm-next bllip-md 1101 67.77726881718898 12 | sclm-next bllip-md 1102 67.8607775941199 13 | sclm-next bllip-md 1103 67.64091701541993 14 | sclm-next bllip-md 1104 68.00710695753381 15 | sclm-past bllip-lg 1101 48.92779418455921 16 | sclm-past bllip-lg 1102 49.21657233139239 17 | sclm-past bllip-lg 1103 48.96724384223701 18 | sclm-past bllip-lg 1104 48.342084362895186 19 | sclm-next bllip-lg 1101 48.90353694040945 20 | sclm-next bllip-lg 1102 48.73836796710418 21 | sclm-next bllip-lg 1103 49.3221141778685 22 | sclm-next bllip-lg 1104 49.57105950626835 23 | xplm bllip-md 1101 68.13931734627926 24 | xplm bllip-md 1102 68.31621426240034 25 | xplm bllip-md 1103 67.84109632706752 26 | xplm bllip-lg 1101 48.90395620424249 27 | xplm bllip-lg 1102 49.55243502963192 28 | xplm bllip-lg 1103 49.29769429883979 29 | xplm-mask bllip-md 1101 67.32035945454638 30 | xplm-mask bllip-md 1102 67.61009335107133 31 | xplm-mask bllip-md 1103 67.63392824487995 32 | xplm-mask bllip-lg 1101 48.903336132755754 33 | xplm-mask bllip-lg 1102 48.96186029061371 34 | xplm-mask bllip-lg 1103 49.47523268080935 -------------------------------------------------------------------------------- /analysis/results/sg/gpt-2.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.9285714286 2 | center_embed_mod 0.8571428571 3 | cleft 1 4 | cleft_modifier 0.95 5 | fgd-embed3 0.7142857143 6 | fgd-embed4 0.619047619 7 | fgd_hierarchy 0.7083333333 8 | fgd_object 0.9166666667 9 | fgd_pp 0.8333333333 10 | fgd_subject 0.4583333333 11 | mvrr 0.6428571429 12 | mvrr_mod 0.6785714286 13 | nn-nv-rpl 1 14 | npi_orc_any 1 15 | npi_orc_ever 1 16 | npi_src_any 1 17 | npi_src_ever 1 18 | npz_ambig 0.9166666667 19 | npz_ambig_mod 0.9166666667 20 | npz_obj 1 21 | npz_obj_mod 0.9583333333 22 | number_orc 0.6315789474 23 | number_prep 0.8421052632 24 | number_src 0.7894736842 25 | reflexive_orc_fem 0.5263157895 26 | reflexive_orc_masc 0.8947368421 27 | reflexive_prep_fem 0.3157894737 28 | reflexive_prep_masc 0.7894736842 29 | reflexive_src_fem 0.3157894737 30 | reflexive_src_masc 0.6842105263 31 | subordination 0.9565217391 32 | subordination_orc-orc 0.9130434783 33 | subordination_pp-pp 1 34 | subordination_src-src 0.9130434783 -------------------------------------------------------------------------------- /analysis/results/sg/sclm-next_bllip-lg_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.7857142857142857 2 | center_embed_mod 0.7857142857142857 3 | cleft 1.0 4 | cleft_modifier 0.825 5 | fgd-embed3 0.47619047619047616 6 | fgd-embed4 0.6190476190476191 7 | fgd_hierarchy 0.5833333333333334 8 | fgd_object 0.875 9 | fgd_pp 0.5833333333333334 10 | fgd_subject 0.4166666666666667 11 | mvrr 0.5357142857142857 12 | mvrr_mod 0.7142857142857143 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.9473684210526315 15 | npi_orc_ever 1.0 16 | npi_src_any 0.9736842105263158 17 | npi_src_ever 1.0 18 | npz_ambig 0.7916666666666666 19 | npz_ambig_mod 0.6666666666666666 20 | npz_obj 0.875 21 | npz_obj_mod 0.7083333333333334 22 | number_orc 0.631578947368421 23 | number_prep 0.631578947368421 24 | number_src 0.8421052631578947 25 | reflexive_orc_fem 0.05263157894736842 26 | reflexive_orc_masc 0.05263157894736842 27 | reflexive_prep_fem 0.3157894736842105 28 | reflexive_prep_masc 0.6842105263157895 29 | reflexive_src_fem 0.05263157894736842 30 | reflexive_src_masc 0.21052631578947367 31 | subordination 0.9565217391304348 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 0.9565217391304348 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-next_bllip-lg_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.75 2 | center_embed_mod 0.6071428571428571 3 | cleft 1.0 4 | cleft_modifier 0.95 5 | fgd-embed3 0.5238095238095238 6 | fgd-embed4 0.42857142857142855 7 | fgd_hierarchy 0.5416666666666666 8 | fgd_object 0.8333333333333334 9 | fgd_pp 0.625 10 | fgd_subject 0.5416666666666666 11 | mvrr 0.5 12 | mvrr_mod 0.5714285714285714 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.9736842105263158 15 | npi_orc_ever 1.0 16 | npi_src_any 0.8157894736842105 17 | npi_src_ever 1.0 18 | npz_ambig 0.75 19 | npz_ambig_mod 0.5833333333333334 20 | npz_obj 0.875 21 | npz_obj_mod 0.5833333333333334 22 | number_orc 0.3157894736842105 23 | number_prep 0.5789473684210527 24 | number_src 0.8421052631578947 25 | reflexive_orc_fem 0.05263157894736842 26 | reflexive_orc_masc 0.2631578947368421 27 | reflexive_prep_fem 0.10526315789473684 28 | reflexive_prep_masc 0.631578947368421 29 | reflexive_src_fem 0.0 30 | reflexive_src_masc 0.15789473684210525 31 | subordination 1.0 32 | subordination_orc-orc 0.9565217391304348 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-next_bllip-lg_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.75 2 | center_embed_mod 0.7857142857142857 3 | cleft 1.0 4 | cleft_modifier 0.9 5 | fgd-embed3 0.7619047619047619 6 | fgd-embed4 0.7619047619047619 7 | fgd_hierarchy 0.375 8 | fgd_object 0.875 9 | fgd_pp 0.6666666666666666 10 | fgd_subject 0.4166666666666667 11 | mvrr 0.5357142857142857 12 | mvrr_mod 0.7142857142857143 13 | nn-nv-rpl 1.0 14 | npi_orc_any 1.0 15 | npi_orc_ever 1.0 16 | npi_src_any 0.8157894736842105 17 | npi_src_ever 1.0 18 | npz_ambig 0.875 19 | npz_ambig_mod 0.6666666666666666 20 | npz_obj 0.9166666666666666 21 | npz_obj_mod 0.7083333333333334 22 | number_orc 0.5789473684210527 23 | number_prep 0.47368421052631576 24 | number_src 0.7368421052631579 25 | reflexive_orc_fem 0.15789473684210525 26 | reflexive_orc_masc 0.5789473684210527 27 | reflexive_prep_fem 0.10526315789473684 28 | reflexive_prep_masc 0.7894736842105263 29 | reflexive_src_fem 0.05263157894736842 30 | reflexive_src_masc 0.47368421052631576 31 | subordination 1.0 32 | subordination_orc-orc 0.9565217391304348 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-next_bllip-lg_rand-init_1104_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.75 2 | center_embed_mod 0.6785714285714286 3 | cleft 1.0 4 | cleft_modifier 0.95 5 | fgd-embed3 0.5714285714285714 6 | fgd-embed4 0.47619047619047616 7 | fgd_hierarchy 0.5416666666666666 8 | fgd_object 0.9166666666666666 9 | fgd_pp 0.4583333333333333 10 | fgd_subject 0.4166666666666667 11 | mvrr 0.5357142857142857 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 0.0 14 | npi_orc_any 0.9473684210526315 15 | npi_orc_ever 1.0 16 | npi_src_any 0.5 17 | npi_src_ever 1.0 18 | npz_ambig 0.6666666666666666 19 | npz_ambig_mod 0.5 20 | npz_obj 0.9166666666666666 21 | npz_obj_mod 0.7083333333333334 22 | number_orc 0.5263157894736842 23 | number_prep 0.5263157894736842 24 | number_src 0.5263157894736842 25 | reflexive_orc_fem 0.0 26 | reflexive_orc_masc 0.6842105263157895 27 | reflexive_prep_fem 0.05263157894736842 28 | reflexive_prep_masc 0.6842105263157895 29 | reflexive_src_fem 0.05263157894736842 30 | reflexive_src_masc 0.7894736842105263 31 | subordination 1.0 32 | subordination_orc-orc 0.9565217391304348 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-next_bllip-md_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.7142857142857143 2 | center_embed_mod 0.6428571428571429 3 | cleft 1.0 4 | cleft_modifier 0.85 5 | fgd-embed3 0.6190476190476191 6 | fgd-embed4 0.47619047619047616 7 | fgd_hierarchy 0.5 8 | fgd_object 0.7916666666666666 9 | fgd_pp 0.5416666666666666 10 | fgd_subject 0.6666666666666666 11 | mvrr 0.4642857142857143 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.6052631578947368 15 | npi_orc_ever 0.9736842105263158 16 | npi_src_any 0.5789473684210527 17 | npi_src_ever 0.9473684210526315 18 | npz_ambig 0.7083333333333334 19 | npz_ambig_mod 0.4166666666666667 20 | npz_obj 0.875 21 | npz_obj_mod 0.5833333333333334 22 | number_orc 0.6842105263157895 23 | number_prep 0.7368421052631579 24 | number_src 0.7368421052631579 25 | reflexive_orc_fem 0.10526315789473684 26 | reflexive_orc_masc 0.6842105263157895 27 | reflexive_prep_fem 0.2631578947368421 28 | reflexive_prep_masc 0.631578947368421 29 | reflexive_src_fem 0.05263157894736842 30 | reflexive_src_masc 0.5263157894736842 31 | subordination 1.0 32 | subordination_orc-orc 0.9130434782608695 33 | subordination_pp-pp 1.0 34 | subordination_src-src 0.9565217391304348 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-next_bllip-md_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.6785714285714286 2 | center_embed_mod 0.7142857142857143 3 | cleft 1.0 4 | cleft_modifier 0.7 5 | fgd-embed3 0.6666666666666666 6 | fgd-embed4 0.5238095238095238 7 | fgd_hierarchy 0.4583333333333333 8 | fgd_object 0.7916666666666666 9 | fgd_pp 0.4583333333333333 10 | fgd_subject 0.75 11 | mvrr 0.6071428571428571 12 | mvrr_mod 0.5 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.5526315789473685 15 | npi_orc_ever 0.9210526315789473 16 | npi_src_any 0.42105263157894735 17 | npi_src_ever 0.9473684210526315 18 | npz_ambig 0.7083333333333334 19 | npz_ambig_mod 0.5416666666666666 20 | npz_obj 0.9583333333333334 21 | npz_obj_mod 0.8333333333333334 22 | number_orc 0.47368421052631576 23 | number_prep 0.6842105263157895 24 | number_src 0.631578947368421 25 | reflexive_orc_fem 0.2631578947368421 26 | reflexive_orc_masc 0.47368421052631576 27 | reflexive_prep_fem 0.5263157894736842 28 | reflexive_prep_masc 0.8421052631578947 29 | reflexive_src_fem 0.5263157894736842 30 | reflexive_src_masc 0.8421052631578947 31 | subordination 0.9565217391304348 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-next_bllip-md_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.5714285714285714 2 | center_embed_mod 0.5714285714285714 3 | cleft 1.0 4 | cleft_modifier 0.775 5 | fgd-embed3 0.5714285714285714 6 | fgd-embed4 0.5714285714285714 7 | fgd_hierarchy 0.5416666666666666 8 | fgd_object 0.8333333333333334 9 | fgd_pp 0.5 10 | fgd_subject 0.5833333333333334 11 | mvrr 0.6071428571428571 12 | mvrr_mod 0.6071428571428571 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.6052631578947368 15 | npi_orc_ever 1.0 16 | npi_src_any 0.4473684210526316 17 | npi_src_ever 1.0 18 | npz_ambig 0.7083333333333334 19 | npz_ambig_mod 0.4166666666666667 20 | npz_obj 0.8333333333333334 21 | npz_obj_mod 0.7083333333333334 22 | number_orc 0.42105263157894735 23 | number_prep 0.5263157894736842 24 | number_src 0.6842105263157895 25 | reflexive_orc_fem 0.2631578947368421 26 | reflexive_orc_masc 0.5263157894736842 27 | reflexive_prep_fem 0.10526315789473684 28 | reflexive_prep_masc 0.5789473684210527 29 | reflexive_src_fem 0.2631578947368421 30 | reflexive_src_masc 0.7368421052631579 31 | subordination 1.0 32 | subordination_orc-orc 0.9565217391304348 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-next_bllip-md_rand-init_1104_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.6785714285714286 2 | center_embed_mod 0.75 3 | cleft 1.0 4 | cleft_modifier 0.8 5 | fgd-embed3 0.6190476190476191 6 | fgd-embed4 0.5714285714285714 7 | fgd_hierarchy 0.2916666666666667 8 | fgd_object 0.8333333333333334 9 | fgd_pp 0.4166666666666667 10 | fgd_subject 0.6666666666666666 11 | mvrr 0.5357142857142857 12 | mvrr_mod 0.5714285714285714 13 | nn-nv-rpl 0.0 14 | npi_orc_any 0.8947368421052632 15 | npi_orc_ever 1.0 16 | npi_src_any 0.6842105263157895 17 | npi_src_ever 1.0 18 | npz_ambig 0.6666666666666666 19 | npz_ambig_mod 0.375 20 | npz_obj 0.75 21 | npz_obj_mod 0.7083333333333334 22 | number_orc 0.631578947368421 23 | number_prep 0.631578947368421 24 | number_src 0.631578947368421 25 | reflexive_orc_fem 0.2631578947368421 26 | reflexive_orc_masc 0.5789473684210527 27 | reflexive_prep_fem 0.3157894736842105 28 | reflexive_prep_masc 0.8421052631578947 29 | reflexive_src_fem 0.3157894736842105 30 | reflexive_src_masc 0.5263157894736842 31 | subordination 1.0 32 | subordination_orc-orc 0.9565217391304348 33 | subordination_pp-pp 0.9565217391304348 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-past_bllip-lg_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.8928571428571429 2 | center_embed_mod 0.7142857142857143 3 | cleft 1.0 4 | cleft_modifier 0.975 5 | fgd-embed3 0.6190476190476191 6 | fgd-embed4 0.47619047619047616 7 | fgd_hierarchy 0.375 8 | fgd_object 0.9166666666666666 9 | fgd_pp 0.4583333333333333 10 | fgd_subject 0.5416666666666666 11 | mvrr 0.7142857142857143 12 | mvrr_mod 0.8214285714285714 13 | nn-nv-rpl 0.0 14 | npi_orc_any 0.9473684210526315 15 | npi_orc_ever 1.0 16 | npi_src_any 0.8157894736842105 17 | npi_src_ever 1.0 18 | npz_ambig 0.875 19 | npz_ambig_mod 0.6666666666666666 20 | npz_obj 0.9583333333333334 21 | npz_obj_mod 0.8333333333333334 22 | number_orc 0.5789473684210527 23 | number_prep 0.5263157894736842 24 | number_src 0.7894736842105263 25 | reflexive_orc_fem 0.05263157894736842 26 | reflexive_orc_masc 0.10526315789473684 27 | reflexive_prep_fem 0.10526315789473684 28 | reflexive_prep_masc 0.631578947368421 29 | reflexive_src_fem 0.0 30 | reflexive_src_masc 0.5263157894736842 31 | subordination 1.0 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-past_bllip-lg_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.7142857142857143 2 | center_embed_mod 0.6428571428571429 3 | cleft 1.0 4 | cleft_modifier 0.95 5 | fgd-embed3 0.6190476190476191 6 | fgd-embed4 0.6190476190476191 7 | fgd_hierarchy 0.625 8 | fgd_object 0.8333333333333334 9 | fgd_pp 0.5 10 | fgd_subject 0.5 11 | mvrr 0.5357142857142857 12 | mvrr_mod 0.5714285714285714 13 | nn-nv-rpl 0.0 14 | npi_orc_any 1.0 15 | npi_orc_ever 1.0 16 | npi_src_any 0.7368421052631579 17 | npi_src_ever 1.0 18 | npz_ambig 0.75 19 | npz_ambig_mod 0.5833333333333334 20 | npz_obj 0.9166666666666666 21 | npz_obj_mod 0.8333333333333334 22 | number_orc 0.5263157894736842 23 | number_prep 0.5263157894736842 24 | number_src 0.6842105263157895 25 | reflexive_orc_fem 0.0 26 | reflexive_orc_masc 0.6842105263157895 27 | reflexive_prep_fem 0.2631578947368421 28 | reflexive_prep_masc 0.42105263157894735 29 | reflexive_src_fem 0.05263157894736842 30 | reflexive_src_masc 0.47368421052631576 31 | subordination 1.0 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-past_bllip-lg_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.6785714285714286 2 | center_embed_mod 0.7142857142857143 3 | cleft 1.0 4 | cleft_modifier 0.925 5 | fgd-embed3 0.7619047619047619 6 | fgd-embed4 0.6190476190476191 7 | fgd_hierarchy 0.4166666666666667 8 | fgd_object 0.9166666666666666 9 | fgd_pp 0.7916666666666666 10 | fgd_subject 0.5416666666666666 11 | mvrr 0.6071428571428571 12 | mvrr_mod 0.75 13 | nn-nv-rpl 1.0 14 | npi_orc_any 1.0 15 | npi_orc_ever 1.0 16 | npi_src_any 0.9736842105263158 17 | npi_src_ever 1.0 18 | npz_ambig 0.8333333333333334 19 | npz_ambig_mod 0.625 20 | npz_obj 0.9583333333333334 21 | npz_obj_mod 0.7083333333333334 22 | number_orc 0.6842105263157895 23 | number_prep 0.6842105263157895 24 | number_src 0.7894736842105263 25 | reflexive_orc_fem 0.0 26 | reflexive_orc_masc 0.3684210526315789 27 | reflexive_prep_fem 0.10526315789473684 28 | reflexive_prep_masc 0.631578947368421 29 | reflexive_src_fem 0.0 30 | reflexive_src_masc 0.631578947368421 31 | subordination 1.0 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-past_bllip-lg_rand-init_1104_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.8214285714285714 2 | center_embed_mod 0.75 3 | cleft 1.0 4 | cleft_modifier 0.975 5 | fgd-embed3 0.5714285714285714 6 | fgd-embed4 0.47619047619047616 7 | fgd_hierarchy 0.5416666666666666 8 | fgd_object 0.75 9 | fgd_pp 0.5 10 | fgd_subject 0.75 11 | mvrr 0.6785714285714286 12 | mvrr_mod 0.6428571428571429 13 | nn-nv-rpl 1.0 14 | npi_orc_any 1.0 15 | npi_orc_ever 1.0 16 | npi_src_any 1.0 17 | npi_src_ever 1.0 18 | npz_ambig 0.7916666666666666 19 | npz_ambig_mod 0.625 20 | npz_obj 0.9583333333333334 21 | npz_obj_mod 0.9166666666666666 22 | number_orc 0.5263157894736842 23 | number_prep 0.47368421052631576 24 | number_src 0.6842105263157895 25 | reflexive_orc_fem 0.10526315789473684 26 | reflexive_orc_masc 0.2631578947368421 27 | reflexive_prep_fem 0.0 28 | reflexive_prep_masc 0.42105263157894735 29 | reflexive_src_fem 0.0 30 | reflexive_src_masc 0.3157894736842105 31 | subordination 1.0 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-past_bllip-md_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.6785714285714286 2 | center_embed_mod 0.6428571428571429 3 | cleft 1.0 4 | cleft_modifier 0.7 5 | fgd-embed3 0.42857142857142855 6 | fgd-embed4 0.3333333333333333 7 | fgd_hierarchy 0.4166666666666667 8 | fgd_object 0.7916666666666666 9 | fgd_pp 0.3333333333333333 10 | fgd_subject 0.6666666666666666 11 | mvrr 0.75 12 | mvrr_mod 0.75 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.8947368421052632 15 | npi_orc_ever 0.9473684210526315 16 | npi_src_any 0.9210526315789473 17 | npi_src_ever 1.0 18 | npz_ambig 0.625 19 | npz_ambig_mod 0.5416666666666666 20 | npz_obj 0.875 21 | npz_obj_mod 0.7916666666666666 22 | number_orc 0.3684210526315789 23 | number_prep 0.631578947368421 24 | number_src 0.5263157894736842 25 | reflexive_orc_fem 0.0 26 | reflexive_orc_masc 0.47368421052631576 27 | reflexive_prep_fem 0.10526315789473684 28 | reflexive_prep_masc 0.3684210526315789 29 | reflexive_src_fem 0.10526315789473684 30 | reflexive_src_masc 0.3684210526315789 31 | subordination 1.0 32 | subordination_orc-orc 0.9565217391304348 33 | subordination_pp-pp 1.0 34 | subordination_src-src 0.9565217391304348 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-past_bllip-md_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.6071428571428571 2 | center_embed_mod 0.6071428571428571 3 | cleft 1.0 4 | cleft_modifier 0.975 5 | fgd-embed3 0.6666666666666666 6 | fgd-embed4 0.5238095238095238 7 | fgd_hierarchy 0.20833333333333334 8 | fgd_object 0.875 9 | fgd_pp 0.2916666666666667 10 | fgd_subject 0.7916666666666666 11 | mvrr 0.7142857142857143 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.8947368421052632 15 | npi_orc_ever 0.9736842105263158 16 | npi_src_any 0.4473684210526316 17 | npi_src_ever 1.0 18 | npz_ambig 0.7916666666666666 19 | npz_ambig_mod 0.5 20 | npz_obj 0.9583333333333334 21 | npz_obj_mod 0.875 22 | number_orc 0.47368421052631576 23 | number_prep 0.6842105263157895 24 | number_src 0.631578947368421 25 | reflexive_orc_fem 0.10526315789473684 26 | reflexive_orc_masc 0.47368421052631576 27 | reflexive_prep_fem 0.21052631578947367 28 | reflexive_prep_masc 0.5789473684210527 29 | reflexive_src_fem 0.21052631578947367 30 | reflexive_src_masc 0.47368421052631576 31 | subordination 1.0 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-past_bllip-md_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.6428571428571429 2 | center_embed_mod 0.6785714285714286 3 | cleft 1.0 4 | cleft_modifier 0.725 5 | fgd-embed3 0.6190476190476191 6 | fgd-embed4 0.42857142857142855 7 | fgd_hierarchy 0.375 8 | fgd_object 0.75 9 | fgd_pp 0.5416666666666666 10 | fgd_subject 0.5833333333333334 11 | mvrr 0.6785714285714286 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 0.0 14 | npi_orc_any 0.9473684210526315 15 | npi_orc_ever 0.9473684210526315 16 | npi_src_any 0.9736842105263158 17 | npi_src_ever 0.868421052631579 18 | npz_ambig 0.5833333333333334 19 | npz_ambig_mod 0.4583333333333333 20 | npz_obj 0.8333333333333334 21 | npz_obj_mod 0.7083333333333334 22 | number_orc 0.5789473684210527 23 | number_prep 0.5263157894736842 24 | number_src 0.6842105263157895 25 | reflexive_orc_fem 0.0 26 | reflexive_orc_masc 0.10526315789473684 27 | reflexive_prep_fem 0.05263157894736842 28 | reflexive_prep_masc 0.3157894736842105 29 | reflexive_src_fem 0.0 30 | reflexive_src_masc 0.3684210526315789 31 | subordination 1.0 32 | subordination_orc-orc 0.9565217391304348 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/sclm-past_bllip-md_rand-init_1104_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.6428571428571429 2 | center_embed_mod 0.7142857142857143 3 | cleft 1.0 4 | cleft_modifier 0.65 5 | fgd-embed3 0.5714285714285714 6 | fgd-embed4 0.5238095238095238 7 | fgd_hierarchy 0.375 8 | fgd_object 0.7916666666666666 9 | fgd_pp 0.5416666666666666 10 | fgd_subject 0.7916666666666666 11 | mvrr 0.6428571428571429 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.9473684210526315 15 | npi_orc_ever 0.9736842105263158 16 | npi_src_any 0.8157894736842105 17 | npi_src_ever 0.9473684210526315 18 | npz_ambig 0.625 19 | npz_ambig_mod 0.4166666666666667 20 | npz_obj 0.9166666666666666 21 | npz_obj_mod 0.875 22 | number_orc 0.3157894736842105 23 | number_prep 0.3684210526315789 24 | number_src 0.3684210526315789 25 | reflexive_orc_fem 0.15789473684210525 26 | reflexive_orc_masc 0.631578947368421 27 | reflexive_prep_fem 0.10526315789473684 28 | reflexive_prep_masc 0.631578947368421 29 | reflexive_src_fem 0.21052631578947367 30 | reflexive_src_masc 0.7368421052631579 31 | subordination 1.0 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xlm_bllip-lg_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.7857142857142857 2 | center_embed_mod 0.7142857142857143 3 | cleft 1.0 4 | cleft_modifier 0.925 5 | fgd-embed3 0.6666666666666666 6 | fgd-embed4 0.6666666666666666 7 | fgd_hierarchy 0.375 8 | fgd_object 0.75 9 | fgd_pp 0.5416666666666666 10 | fgd_subject 0.625 11 | mvrr 0.7142857142857143 12 | mvrr_mod 0.6071428571428571 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.9736842105263158 15 | npi_orc_ever 1.0 16 | npi_src_any 0.8947368421052632 17 | npi_src_ever 1.0 18 | npz_ambig 0.8333333333333334 19 | npz_ambig_mod 0.6666666666666666 20 | npz_obj 0.8333333333333334 21 | npz_obj_mod 0.7083333333333334 22 | number_orc 0.42105263157894735 23 | number_prep 0.5263157894736842 24 | number_src 0.6842105263157895 25 | reflexive_orc_fem 0.10526315789473684 26 | reflexive_orc_masc 0.3157894736842105 27 | reflexive_prep_fem 0.21052631578947367 28 | reflexive_prep_masc 0.6842105263157895 29 | reflexive_src_fem 0.0 30 | reflexive_src_masc 0.631578947368421 31 | subordination 1.0 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xlm_bllip-lg_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.7857142857142857 2 | center_embed_mod 0.7142857142857143 3 | cleft 1.0 4 | cleft_modifier 0.925 5 | fgd-embed3 0.38095238095238093 6 | fgd-embed4 0.3333333333333333 7 | fgd_hierarchy 0.3333333333333333 8 | fgd_object 0.875 9 | fgd_pp 0.375 10 | fgd_subject 0.5416666666666666 11 | mvrr 0.6428571428571429 12 | mvrr_mod 0.6428571428571429 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.9473684210526315 15 | npi_orc_ever 1.0 16 | npi_src_any 0.5 17 | npi_src_ever 1.0 18 | npz_ambig 0.7916666666666666 19 | npz_ambig_mod 0.375 20 | npz_obj 0.875 21 | npz_obj_mod 0.6666666666666666 22 | number_orc 0.5263157894736842 23 | number_prep 0.5263157894736842 24 | number_src 0.7368421052631579 25 | reflexive_orc_fem 0.05263157894736842 26 | reflexive_orc_masc 0.2631578947368421 27 | reflexive_prep_fem 0.0 28 | reflexive_prep_masc 0.42105263157894735 29 | reflexive_src_fem 0.05263157894736842 30 | reflexive_src_masc 0.5263157894736842 31 | subordination 1.0 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xlm_bllip-lg_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.7857142857142857 2 | center_embed_mod 0.75 3 | cleft 1.0 4 | cleft_modifier 0.95 5 | fgd-embed3 0.47619047619047616 6 | fgd-embed4 0.42857142857142855 7 | fgd_hierarchy 0.3333333333333333 8 | fgd_object 0.75 9 | fgd_pp 0.4583333333333333 10 | fgd_subject 0.2916666666666667 11 | mvrr 0.6071428571428571 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 1.0 14 | npi_orc_any 1.0 15 | npi_orc_ever 1.0 16 | npi_src_any 0.9736842105263158 17 | npi_src_ever 1.0 18 | npz_ambig 0.625 19 | npz_ambig_mod 0.375 20 | npz_obj 0.9583333333333334 21 | npz_obj_mod 0.6666666666666666 22 | number_orc 0.7368421052631579 23 | number_prep 0.5789473684210527 24 | number_src 0.6842105263157895 25 | reflexive_orc_fem 0.0 26 | reflexive_orc_masc 0.47368421052631576 27 | reflexive_prep_fem 0.15789473684210525 28 | reflexive_prep_masc 0.6842105263157895 29 | reflexive_src_fem 0.10526315789473684 30 | reflexive_src_masc 0.6842105263157895 31 | subordination 1.0 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xlm_bllip-md_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.7857142857142857 2 | center_embed_mod 0.7857142857142857 3 | cleft 1.0 4 | cleft_modifier 0.9 5 | fgd-embed3 0.7142857142857143 6 | fgd-embed4 0.5714285714285714 7 | fgd_hierarchy 0.2916666666666667 8 | fgd_object 0.7916666666666666 9 | fgd_pp 0.5 10 | fgd_subject 0.7083333333333334 11 | mvrr 0.6071428571428571 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 0.0 14 | npi_orc_any 0.9210526315789473 15 | npi_orc_ever 0.9736842105263158 16 | npi_src_any 0.5789473684210527 17 | npi_src_ever 0.7368421052631579 18 | npz_ambig 0.8333333333333334 19 | npz_ambig_mod 0.5833333333333334 20 | npz_obj 0.7916666666666666 21 | npz_obj_mod 0.8333333333333334 22 | number_orc 0.47368421052631576 23 | number_prep 0.5263157894736842 24 | number_src 0.3684210526315789 25 | reflexive_orc_fem 0.15789473684210525 26 | reflexive_orc_masc 0.15789473684210525 27 | reflexive_prep_fem 0.3157894736842105 28 | reflexive_prep_masc 0.8421052631578947 29 | reflexive_src_fem 0.21052631578947367 30 | reflexive_src_masc 0.3684210526315789 31 | subordination 0.9565217391304348 32 | subordination_orc-orc 0.9565217391304348 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xlm_bllip-md_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.75 2 | center_embed_mod 0.5714285714285714 3 | cleft 1.0 4 | cleft_modifier 0.675 5 | fgd-embed3 0.6666666666666666 6 | fgd-embed4 0.5238095238095238 7 | fgd_hierarchy 0.2916666666666667 8 | fgd_object 0.6666666666666666 9 | fgd_pp 0.4166666666666667 10 | fgd_subject 0.75 11 | mvrr 0.5714285714285714 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.868421052631579 15 | npi_orc_ever 1.0 16 | npi_src_any 0.7894736842105263 17 | npi_src_ever 1.0 18 | npz_ambig 0.7916666666666666 19 | npz_ambig_mod 0.5 20 | npz_obj 0.7916666666666666 21 | npz_obj_mod 0.6666666666666666 22 | number_orc 0.3684210526315789 23 | number_prep 0.631578947368421 24 | number_src 0.47368421052631576 25 | reflexive_orc_fem 0.10526315789473684 26 | reflexive_orc_masc 0.6842105263157895 27 | reflexive_prep_fem 0.3157894736842105 28 | reflexive_prep_masc 0.8947368421052632 29 | reflexive_src_fem 0.3157894736842105 30 | reflexive_src_masc 0.8421052631578947 31 | subordination 1.0 32 | subordination_orc-orc 1.0 33 | subordination_pp-pp 1.0 34 | subordination_src-src 1.0 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xlm_bllip-md_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.7142857142857143 2 | center_embed_mod 0.6428571428571429 3 | cleft 1.0 4 | cleft_modifier 0.75 5 | fgd-embed3 0.8095238095238095 6 | fgd-embed4 0.47619047619047616 7 | fgd_hierarchy 0.25 8 | fgd_object 0.875 9 | fgd_pp 0.5 10 | fgd_subject 0.6666666666666666 11 | mvrr 0.6071428571428571 12 | mvrr_mod 0.75 13 | nn-nv-rpl 1.0 14 | npi_orc_any 1.0 15 | npi_orc_ever 0.9210526315789473 16 | npi_src_any 0.5263157894736842 17 | npi_src_ever 0.631578947368421 18 | npz_ambig 0.75 19 | npz_ambig_mod 0.4166666666666667 20 | npz_obj 0.7083333333333334 21 | npz_obj_mod 0.6666666666666666 22 | number_orc 0.5789473684210527 23 | number_prep 0.631578947368421 24 | number_src 0.5263157894736842 25 | reflexive_orc_fem 0.05263157894736842 26 | reflexive_orc_masc 0.3684210526315789 27 | reflexive_prep_fem 0.21052631578947367 28 | reflexive_prep_masc 0.8421052631578947 29 | reflexive_src_fem 0.10526315789473684 30 | reflexive_src_masc 0.6842105263157895 31 | subordination 0.9565217391304348 32 | subordination_orc-orc 0.9130434782608695 33 | subordination_pp-pp 0.9565217391304348 34 | subordination_src-src 0.9565217391304348 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm-mask_bllip-lg_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.8571428571428571 2 | center_embed_mod 0.8214285714285714 3 | cleft 1.0 4 | cleft_modifier 0.925 5 | fgd-embed3 0.6190476190476191 6 | fgd-embed4 0.42857142857142855 7 | fgd_hierarchy 0.6666666666666666 8 | fgd_object 0.875 9 | fgd_pp 0.6666666666666666 10 | fgd_subject 0.6666666666666666 11 | mvrr 0.6785714285714286 12 | mvrr_mod 0.5357142857142857 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.9736842105263158 15 | npi_orc_ever 0.9736842105263158 16 | npi_src_any 0.9736842105263158 17 | npi_src_ever 1.0 18 | npz_ambig 1.0 19 | npz_ambig_mod 0.7083333333333334 20 | npz_obj 1.0 21 | npz_obj_mod 0.9583333333333334 22 | number_orc 0.6842105263157895 23 | number_prep 0.8421052631578947 24 | number_src 0.8421052631578947 25 | reflexive_orc_fem 0.05263157894736842 26 | reflexive_orc_masc 0.6842105263157895 27 | reflexive_prep_fem 0.2631578947368421 28 | reflexive_prep_masc 0.7368421052631579 29 | reflexive_src_fem 0.15789473684210525 30 | reflexive_src_masc 0.7894736842105263 31 | subordination 1.0 32 | subordination_orc-orc 0.6086956521739131 33 | subordination_pp-pp 1.0 34 | subordination_src-src 0.5217391304347826 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm-mask_bllip-lg_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.8928571428571429 2 | center_embed_mod 0.8214285714285714 3 | cleft 1.0 4 | cleft_modifier 0.95 5 | fgd-embed3 0.5714285714285714 6 | fgd-embed4 0.5714285714285714 7 | fgd_hierarchy 0.6666666666666666 8 | fgd_object 0.9583333333333334 9 | fgd_pp 0.7083333333333334 10 | fgd_subject 0.5833333333333334 11 | mvrr 0.7142857142857143 12 | mvrr_mod 0.6428571428571429 13 | nn-nv-rpl 0.0 14 | npi_orc_any 0.9473684210526315 15 | npi_orc_ever 1.0 16 | npi_src_any 1.0 17 | npi_src_ever 1.0 18 | npz_ambig 0.8333333333333334 19 | npz_ambig_mod 0.9583333333333334 20 | npz_obj 0.9583333333333334 21 | npz_obj_mod 1.0 22 | number_orc 0.6842105263157895 23 | number_prep 0.8421052631578947 24 | number_src 0.7894736842105263 25 | reflexive_orc_fem 0.05263157894736842 26 | reflexive_orc_masc 0.631578947368421 27 | reflexive_prep_fem 0.10526315789473684 28 | reflexive_prep_masc 0.8421052631578947 29 | reflexive_src_fem 0.0 30 | reflexive_src_masc 0.7368421052631579 31 | subordination 1.0 32 | subordination_orc-orc 0.9130434782608695 33 | subordination_pp-pp 1.0 34 | subordination_src-src 0.6086956521739131 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm-mask_bllip-lg_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.7857142857142857 2 | center_embed_mod 0.7857142857142857 3 | cleft 1.0 4 | cleft_modifier 0.95 5 | fgd-embed3 0.6190476190476191 6 | fgd-embed4 0.38095238095238093 7 | fgd_hierarchy 0.625 8 | fgd_object 0.9583333333333334 9 | fgd_pp 0.625 10 | fgd_subject 0.6666666666666666 11 | mvrr 0.7142857142857143 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.9210526315789473 15 | npi_orc_ever 0.9736842105263158 16 | npi_src_any 0.9736842105263158 17 | npi_src_ever 1.0 18 | npz_ambig 0.9166666666666666 19 | npz_ambig_mod 0.875 20 | npz_obj 1.0 21 | npz_obj_mod 0.9583333333333334 22 | number_orc 0.5789473684210527 23 | number_prep 0.8421052631578947 24 | number_src 0.6842105263157895 25 | reflexive_orc_fem 0.0 26 | reflexive_orc_masc 0.5789473684210527 27 | reflexive_prep_fem 0.15789473684210525 28 | reflexive_prep_masc 0.7894736842105263 29 | reflexive_src_fem 0.05263157894736842 30 | reflexive_src_masc 0.7894736842105263 31 | subordination 1.0 32 | subordination_orc-orc 0.9130434782608695 33 | subordination_pp-pp 1.0 34 | subordination_src-src 0.782608695652174 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm-mask_bllip-md_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.75 2 | center_embed_mod 0.6428571428571429 3 | cleft 1.0 4 | cleft_modifier 0.975 5 | fgd-embed3 0.6190476190476191 6 | fgd-embed4 0.23809523809523808 7 | fgd_hierarchy 0.4583333333333333 8 | fgd_object 0.7916666666666666 9 | fgd_pp 0.5833333333333334 10 | fgd_subject 0.7083333333333334 11 | mvrr 0.6428571428571429 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 0.0 14 | npi_orc_any 0.8157894736842105 15 | npi_orc_ever 0.8947368421052632 16 | npi_src_any 1.0 17 | npi_src_ever 1.0 18 | npz_ambig 0.9166666666666666 19 | npz_ambig_mod 0.6666666666666666 20 | npz_obj 1.0 21 | npz_obj_mod 0.7916666666666666 22 | number_orc 0.631578947368421 23 | number_prep 0.8947368421052632 24 | number_src 0.7894736842105263 25 | reflexive_orc_fem 0.15789473684210525 26 | reflexive_orc_masc 0.5263157894736842 27 | reflexive_prep_fem 0.42105263157894735 28 | reflexive_prep_masc 0.7368421052631579 29 | reflexive_src_fem 0.3684210526315789 30 | reflexive_src_masc 0.8421052631578947 31 | subordination 0.9565217391304348 32 | subordination_orc-orc 0.8695652173913043 33 | subordination_pp-pp 0.9130434782608695 34 | subordination_src-src 0.5217391304347826 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm-mask_bllip-md_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.8571428571428571 2 | center_embed_mod 0.7142857142857143 3 | cleft 1.0 4 | cleft_modifier 0.825 5 | fgd-embed3 0.5238095238095238 6 | fgd-embed4 0.42857142857142855 7 | fgd_hierarchy 0.75 8 | fgd_object 0.875 9 | fgd_pp 0.6666666666666666 10 | fgd_subject 0.75 11 | mvrr 0.6428571428571429 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.6842105263157895 15 | npi_orc_ever 0.6052631578947368 16 | npi_src_any 0.9736842105263158 17 | npi_src_ever 1.0 18 | npz_ambig 0.875 19 | npz_ambig_mod 0.5833333333333334 20 | npz_obj 0.9166666666666666 21 | npz_obj_mod 0.75 22 | number_orc 0.5789473684210527 23 | number_prep 0.8947368421052632 24 | number_src 0.8947368421052632 25 | reflexive_orc_fem 0.0 26 | reflexive_orc_masc 0.7368421052631579 27 | reflexive_prep_fem 0.05263157894736842 28 | reflexive_prep_masc 0.7894736842105263 29 | reflexive_src_fem 0.15789473684210525 30 | reflexive_src_masc 0.8947368421052632 31 | subordination 0.9130434782608695 32 | subordination_orc-orc 0.8695652173913043 33 | subordination_pp-pp 0.8695652173913043 34 | subordination_src-src 0.6956521739130435 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm-mask_bllip-md_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.7857142857142857 2 | center_embed_mod 0.6428571428571429 3 | cleft 1.0 4 | cleft_modifier 0.9 5 | fgd-embed3 0.5238095238095238 6 | fgd-embed4 0.38095238095238093 7 | fgd_hierarchy 0.5 8 | fgd_object 0.7916666666666666 9 | fgd_pp 0.4583333333333333 10 | fgd_subject 0.5833333333333334 11 | mvrr 0.5714285714285714 12 | mvrr_mod 0.6071428571428571 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.6842105263157895 15 | npi_orc_ever 0.9210526315789473 16 | npi_src_any 0.9736842105263158 17 | npi_src_ever 1.0 18 | npz_ambig 0.875 19 | npz_ambig_mod 0.7083333333333334 20 | npz_obj 1.0 21 | npz_obj_mod 0.9166666666666666 22 | number_orc 0.631578947368421 23 | number_prep 0.9473684210526315 24 | number_src 0.8421052631578947 25 | reflexive_orc_fem 0.10526315789473684 26 | reflexive_orc_masc 0.3157894736842105 27 | reflexive_prep_fem 0.42105263157894735 28 | reflexive_prep_masc 0.8947368421052632 29 | reflexive_src_fem 0.3684210526315789 30 | reflexive_src_masc 0.8421052631578947 31 | subordination 1.0 32 | subordination_orc-orc 0.9565217391304348 33 | subordination_pp-pp 0.9130434782608695 34 | subordination_src-src 0.6521739130434783 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm_bllip-lg_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.8214285714285714 2 | center_embed_mod 0.75 3 | cleft 1.0 4 | cleft_modifier 0.95 5 | fgd-embed3 0.5714285714285714 6 | fgd-embed4 0.38095238095238093 7 | fgd_hierarchy 0.5833333333333334 8 | fgd_object 0.875 9 | fgd_pp 0.7083333333333334 10 | fgd_subject 0.5833333333333334 11 | mvrr 0.5357142857142857 12 | mvrr_mod 0.5714285714285714 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.8947368421052632 15 | npi_orc_ever 0.9473684210526315 16 | npi_src_any 0.9736842105263158 17 | npi_src_ever 1.0 18 | npz_ambig 0.9583333333333334 19 | npz_ambig_mod 0.8333333333333334 20 | npz_obj 1.0 21 | npz_obj_mod 0.9166666666666666 22 | number_orc 0.631578947368421 23 | number_prep 0.8947368421052632 24 | number_src 0.7368421052631579 25 | reflexive_orc_fem 0.0 26 | reflexive_orc_masc 0.631578947368421 27 | reflexive_prep_fem 0.15789473684210525 28 | reflexive_prep_masc 0.7894736842105263 29 | reflexive_src_fem 0.0 30 | reflexive_src_masc 0.631578947368421 31 | subordination 1.0 32 | subordination_orc-orc 0.8695652173913043 33 | subordination_pp-pp 1.0 34 | subordination_src-src 0.7391304347826086 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm_bllip-lg_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.8214285714285714 2 | center_embed_mod 0.8571428571428571 3 | cleft 1.0 4 | cleft_modifier 0.95 5 | fgd-embed3 0.3333333333333333 6 | fgd-embed4 0.19047619047619047 7 | fgd_hierarchy 0.7083333333333334 8 | fgd_object 0.9166666666666666 9 | fgd_pp 0.625 10 | fgd_subject 0.7083333333333334 11 | mvrr 0.6071428571428571 12 | mvrr_mod 0.5714285714285714 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.9210526315789473 15 | npi_orc_ever 1.0 16 | npi_src_any 0.9736842105263158 17 | npi_src_ever 1.0 18 | npz_ambig 0.8333333333333334 19 | npz_ambig_mod 0.5833333333333334 20 | npz_obj 1.0 21 | npz_obj_mod 0.875 22 | number_orc 0.7894736842105263 23 | number_prep 0.8947368421052632 24 | number_src 0.8947368421052632 25 | reflexive_orc_fem 0.0 26 | reflexive_orc_masc 0.7894736842105263 27 | reflexive_prep_fem 0.05263157894736842 28 | reflexive_prep_masc 0.8947368421052632 29 | reflexive_src_fem 0.0 30 | reflexive_src_masc 0.8947368421052632 31 | subordination 1.0 32 | subordination_orc-orc 0.782608695652174 33 | subordination_pp-pp 0.9130434782608695 34 | subordination_src-src 0.6521739130434783 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm_bllip-lg_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.8214285714285714 2 | center_embed_mod 0.8214285714285714 3 | cleft 1.0 4 | cleft_modifier 0.975 5 | fgd-embed3 0.5714285714285714 6 | fgd-embed4 0.5238095238095238 7 | fgd_hierarchy 0.5833333333333334 8 | fgd_object 0.875 9 | fgd_pp 0.625 10 | fgd_subject 0.5416666666666666 11 | mvrr 0.6785714285714286 12 | mvrr_mod 0.7142857142857143 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.9736842105263158 15 | npi_orc_ever 0.9736842105263158 16 | npi_src_any 1.0 17 | npi_src_ever 1.0 18 | npz_ambig 0.9583333333333334 19 | npz_ambig_mod 0.7916666666666666 20 | npz_obj 1.0 21 | npz_obj_mod 0.9583333333333334 22 | number_orc 0.5263157894736842 23 | number_prep 0.8421052631578947 24 | number_src 0.7894736842105263 25 | reflexive_orc_fem 0.0 26 | reflexive_orc_masc 0.3157894736842105 27 | reflexive_prep_fem 0.21052631578947367 28 | reflexive_prep_masc 0.5263157894736842 29 | reflexive_src_fem 0.0 30 | reflexive_src_masc 0.5263157894736842 31 | subordination 1.0 32 | subordination_orc-orc 0.8695652173913043 33 | subordination_pp-pp 1.0 34 | subordination_src-src 0.6521739130434783 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm_bllip-md_rand-init_1101_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.8214285714285714 2 | center_embed_mod 0.75 3 | cleft 1.0 4 | cleft_modifier 0.925 5 | fgd-embed3 0.5238095238095238 6 | fgd-embed4 0.3333333333333333 7 | fgd_hierarchy 0.625 8 | fgd_object 0.9166666666666666 9 | fgd_pp 0.75 10 | fgd_subject 0.7083333333333334 11 | mvrr 0.5714285714285714 12 | mvrr_mod 0.5714285714285714 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.6842105263157895 15 | npi_orc_ever 0.9210526315789473 16 | npi_src_any 0.8947368421052632 17 | npi_src_ever 1.0 18 | npz_ambig 0.875 19 | npz_ambig_mod 0.7916666666666666 20 | npz_obj 0.9166666666666666 21 | npz_obj_mod 0.9583333333333334 22 | number_orc 0.7894736842105263 23 | number_prep 0.8947368421052632 24 | number_src 0.8947368421052632 25 | reflexive_orc_fem 0.10526315789473684 26 | reflexive_orc_masc 0.3157894736842105 27 | reflexive_prep_fem 0.21052631578947367 28 | reflexive_prep_masc 0.631578947368421 29 | reflexive_src_fem 0.2631578947368421 30 | reflexive_src_masc 0.6842105263157895 31 | subordination 1.0 32 | subordination_orc-orc 0.8695652173913043 33 | subordination_pp-pp 0.8695652173913043 34 | subordination_src-src 0.6956521739130435 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm_bllip-md_rand-init_1102_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.7857142857142857 2 | center_embed_mod 0.7857142857142857 3 | cleft 1.0 4 | cleft_modifier 0.8 5 | fgd-embed3 0.5238095238095238 6 | fgd-embed4 0.23809523809523808 7 | fgd_hierarchy 0.5 8 | fgd_object 0.875 9 | fgd_pp 0.625 10 | fgd_subject 0.7916666666666666 11 | mvrr 0.6785714285714286 12 | mvrr_mod 0.6785714285714286 13 | nn-nv-rpl 0.0 14 | npi_orc_any 0.8421052631578947 15 | npi_orc_ever 0.631578947368421 16 | npi_src_any 0.9736842105263158 17 | npi_src_ever 1.0 18 | npz_ambig 0.9583333333333334 19 | npz_ambig_mod 0.7083333333333334 20 | npz_obj 1.0 21 | npz_obj_mod 0.9166666666666666 22 | number_orc 0.631578947368421 23 | number_prep 0.8421052631578947 24 | number_src 0.7894736842105263 25 | reflexive_orc_fem 0.05263157894736842 26 | reflexive_orc_masc 0.42105263157894735 27 | reflexive_prep_fem 0.05263157894736842 28 | reflexive_prep_masc 0.8947368421052632 29 | reflexive_src_fem 0.05263157894736842 30 | reflexive_src_masc 1.0 31 | subordination 1.0 32 | subordination_orc-orc 0.782608695652174 33 | subordination_pp-pp 0.9130434782608695 34 | subordination_src-src 0.6521739130434783 35 | -------------------------------------------------------------------------------- /analysis/results/sg/xplm_bllip-md_rand-init_1103_5.txt: -------------------------------------------------------------------------------- 1 | center_embed 0.8214285714285714 2 | center_embed_mod 0.8214285714285714 3 | cleft 1.0 4 | cleft_modifier 0.875 5 | fgd-embed3 0.47619047619047616 6 | fgd-embed4 0.19047619047619047 7 | fgd_hierarchy 0.5 8 | fgd_object 0.8333333333333334 9 | fgd_pp 0.4166666666666667 10 | fgd_subject 0.6666666666666666 11 | mvrr 0.5714285714285714 12 | mvrr_mod 0.75 13 | nn-nv-rpl 1.0 14 | npi_orc_any 0.7894736842105263 15 | npi_orc_ever 0.7894736842105263 16 | npi_src_any 1.0 17 | npi_src_ever 1.0 18 | npz_ambig 0.875 19 | npz_ambig_mod 0.75 20 | npz_obj 1.0 21 | npz_obj_mod 0.9166666666666666 22 | number_orc 0.5263157894736842 23 | number_prep 0.8947368421052632 24 | number_src 0.8421052631578947 25 | reflexive_orc_fem 0.15789473684210525 26 | reflexive_orc_masc 0.3684210526315789 27 | reflexive_prep_fem 0.3157894736842105 28 | reflexive_prep_masc 0.7368421052631579 29 | reflexive_src_fem 0.3157894736842105 30 | reflexive_src_masc 0.631578947368421 31 | subordination 1.0 32 | subordination_orc-orc 0.782608695652174 33 | subordination_pp-pp 0.9130434782608695 34 | subordination_src-src 0.5652173913043478 35 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch==1.4.0 2 | nltk 3 | # for debug 4 | ipdb 5 | # check README.md for install 6 | -e src/transformers/ 7 | -------------------------------------------------------------------------------- /scripts/download_and_patch_transformers.sh: -------------------------------------------------------------------------------- 1 | set -o errexit 2 | cd src 3 | git clone https://github.com/huggingface/transformers.git 4 | cd transformers 5 | git checkout v3.3.1 6 | # apply patch 7 | git apply ../transformers-v3.3.1-x.patch 8 | -------------------------------------------------------------------------------- /scripts/generate_patch.sh: -------------------------------------------------------------------------------- 1 | set -o errexit 2 | cd src/transformers/ 3 | git format-patch v3.3.1 --stdout > ../transformers-v3.3.1-x.patch 4 | -------------------------------------------------------------------------------- /src/get_action_ngram_list.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | 3 | def is_nt(token): 4 | if token.startswith('NT(') and token.endswith(')'): 5 | flag = True 6 | else: 7 | flag = False 8 | return flag 9 | 10 | 11 | def is_reduce(token): 12 | if token == 'REDUCE()': 13 | flag = True 14 | else: 15 | flag = False 16 | return flag 17 | 18 | 19 | def update_ngram_freq_dict(path, action_ngram_freq_dict): 20 | count = 0 21 | with open(path, 'r') as f: 22 | for line in f: 23 | tokens = line.strip().split() 24 | action_seq = [] 25 | for i, token in enumerate(tokens): 26 | if is_nt(token) or is_reduce(token): 27 | action_seq.append(token) 28 | else: 29 | if action_seq != []: 30 | action_ngram = '_'.join(action_seq) 31 | if action_ngram not in action_ngram_freq_dict: 32 | action_ngram_freq_dict[action_ngram] = 1 33 | else: 34 | action_ngram_freq_dict[action_ngram] += 1 35 | action_seq = [] 36 | 37 | count += 1 38 | return count 39 | 40 | 41 | parser = ArgumentParser() 42 | parser.add_argument('-f', '--fpaths', nargs='+', default=['bllip-lg_train_gen.oracle', 'bllip-lg_dev_gen.oracle'], help="List of paths to parsing oracle files.") 43 | parser.add_argument('-o', '--output', default='bllip-lg_action_ngram_list.txt', help='Path to save the action ngram list.') 44 | args = parser.parse_args() 45 | 46 | paths = args.fpaths 47 | output_path = args.output 48 | 49 | action_ngram_freq_dict = {} 50 | 51 | sent_count = 0 52 | for path in paths: 53 | sent_count += update_ngram_freq_dict(path, action_ngram_freq_dict) 54 | print('{} sentences in total.'.format(sent_count)) 55 | 56 | action_ngram_list = [(k, v) for k, v in action_ngram_freq_dict.items()] 57 | 58 | # Sort the action ngram list based on frequency 59 | action_ngram_list_sorted = sorted(action_ngram_list, key = lambda x: x[1], reverse=True) 60 | 61 | with open(output_path, 'w') as f: 62 | for action_ngram, freq in action_ngram_list_sorted: 63 | f.write(action_ngram+'\t'+str(freq)+'\n') 64 | 65 | print(len(action_ngram_list), 'action ngrams.') -------------------------------------------------------------------------------- /src/get_oracle.py: -------------------------------------------------------------------------------- 1 | # This script reuses and modifies some functions from: https://github.com/clab/rnng/blob/master/get_oracle_gen.py 2 | 3 | import argparse 4 | 5 | 6 | def is_valid_action_sequence(action_sequence): 7 | flag = True 8 | for k, action in enumerate(action_sequence): 9 | if action == "REDUCE": 10 | if k <= 1: 11 | flag = False 12 | break 13 | if action_sequence[k-1].startswith('NT('): 14 | flag = False 15 | break 16 | return flag 17 | 18 | 19 | def is_next_open_bracket(line, start_idx): 20 | for char in line[(start_idx + 1):]: 21 | if char == '(': 22 | return True 23 | elif char == ')': 24 | return False 25 | raise IndexError('Bracket possibly not balanced, open bracket not followed by closed bracket') 26 | 27 | 28 | def get_between_brackets(line, start_idx): 29 | output = [] 30 | for char in line[(start_idx + 1):]: 31 | if char == ')': 32 | break 33 | assert not(char == '(') 34 | output.append(char) 35 | return ''.join(output) 36 | 37 | 38 | def get_tags_tokens_lowercase(line): 39 | output = [] 40 | #print 'curr line', line_strip 41 | line_strip = line.rstrip() 42 | #print 'length of the sentence', len(line_strip) 43 | for i in range(len(line_strip)): 44 | if i == 0: 45 | assert line_strip[i] == '(' 46 | if line_strip[i] == '(' and not(is_next_open_bracket(line_strip, i)): # fulfilling this condition means this is a terminal symbol 47 | output.append(get_between_brackets(line_strip, i)) 48 | #print 'output:',output 49 | output_tags = [] 50 | output_tokens = [] 51 | output_lowercase = [] 52 | for terminal in output: 53 | terminal_split = terminal.split() 54 | assert len(terminal_split) == 2 # each terminal contains a POS tag and word 55 | output_tags.append(terminal_split[0]) 56 | output_tokens.append(terminal_split[1]) 57 | output_lowercase.append(terminal_split[1].lower()) 58 | return [output_tags, output_tokens, output_lowercase] 59 | 60 | 61 | def get_nonterminal(line, start_idx): 62 | assert line[start_idx] == '(' # make sure it's an open bracket 63 | output = [] 64 | for char in line[(start_idx + 1):]: 65 | if char == ' ': 66 | break 67 | assert not(char == '(') and not(char == ')') 68 | output.append(char) 69 | return ''.join(output) 70 | 71 | 72 | def get_actions_and_terms(line, is_generative): 73 | output_actions = [] 74 | output_terms = [] 75 | line_strip = line.rstrip() 76 | i = 0 77 | max_idx = (len(line_strip) - 1) 78 | while i <= max_idx: 79 | assert line_strip[i] == '(' or line_strip[i] == ')' 80 | if line_strip[i] == '(': 81 | if is_next_open_bracket(line_strip, i): # open non-terminal 82 | curr_NT = get_nonterminal(line_strip, i) 83 | output_actions.append('NT(' + curr_NT + ')') 84 | i += 1 85 | while line_strip[i] != '(': # get the next open bracket, which may be a terminal or another non-terminal 86 | i += 1 87 | else: # it's a terminal symbol 88 | terminal = get_between_brackets(line_strip, i) 89 | terminal_split = terminal.split() 90 | assert len(terminal_split) == 2 # each terminal contains a POS tag and word 91 | token = terminal_split[1] 92 | output_terms.append(token) 93 | if is_generative: 94 | # generative parsing 95 | output_actions.append(token) 96 | else: 97 | # discriminative parsing 98 | output_actions += ['SHIFT'] 99 | while line_strip[i] != ')': 100 | i += 1 101 | i += 1 102 | while line_strip[i] != ')' and line_strip[i] != '(': 103 | i += 1 104 | else: 105 | output_actions.append('REDUCE()') 106 | if i == max_idx: 107 | break 108 | i += 1 109 | while line_strip[i] != ')' and line_strip[i] != '(': 110 | i += 1 111 | assert i == max_idx 112 | return output_actions, output_terms 113 | 114 | 115 | def main(): 116 | parser = argparse.ArgumentParser() 117 | parser.add_argument('--fpath', type=str, 118 | help='File path to the bracketed constituency trees.') 119 | parser.add_argument('--gen', action='store_true', 120 | help='Get oracle action sequences in the generative mode.') 121 | parser.add_argument('--root', action='store_true', 122 | help='Output root symbol at the beginning of each action sequence.') 123 | args = parser.parse_args() 124 | 125 | f = open(args.fpath, 'r') 126 | lines = f.readlines() 127 | f.close() 128 | 129 | line_ctr = 0 130 | # get the oracle action sequences for the input file 131 | for line in lines: 132 | line_ctr += 1 133 | # assert that the parenthesis are balanced 134 | if line.count('(') != line.count(')'): 135 | raise NotImplementedError('Unbalanced number of parenthesis in line ' + str(line_ctr)) 136 | output_actions, output_terms = get_actions_and_terms(line, is_generative=args.gen) 137 | 138 | if not is_valid_action_sequence(output_actions): 139 | continue 140 | 141 | if len(output_actions) > 500: 142 | continue 143 | 144 | if args.gen: 145 | if args.root: 146 | print('[START]', end=' ') 147 | print(' '.join(output_actions)) 148 | else: 149 | print(' '.join(output_terms) + '\t' + ' '.join(output_actions)) 150 | 151 | 152 | if __name__ == "__main__": 153 | main() 154 | -------------------------------------------------------------------------------- /src/lm-sc.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import argparse 4 | import random 5 | import numpy as np 6 | import nltk 7 | from transformers import GPT2Tokenizer, GPT2LMHeadModel, GPT2Config, AdamW 8 | import torch 9 | import functools 10 | import utils 11 | print = functools.partial(print, flush=True) 12 | 13 | 14 | class Vocabulary(object): 15 | def __init__(self, symbols, pad_index): 16 | self.symbols = symbols 17 | self.pad_index = pad_index 18 | self.token2index = {} 19 | for i, symbol in enumerate(symbols): 20 | self.token2index[symbol] = i 21 | return 22 | 23 | def pad(self): 24 | return self.pad_index 25 | 26 | def convert_tokens_to_ids(self, token_input): 27 | if isinstance(token_input, str): 28 | return self.token2index[token_input] 29 | elif isinstance(token_input, list): 30 | return [self.token2index[token] for token in token_input] 31 | else: 32 | raise NotImplementedError 33 | 34 | def convert_ids_to_tokens(self, id_input): 35 | if isinstance(id_input, int): 36 | return self.symbols[id_input] 37 | elif isinstance(id_input, list): 38 | return [self.symbols[idx] for idx in id_input] 39 | else: 40 | raise NotImplementedError 41 | 42 | 43 | class ScLM: 44 | def __init__(self, is_random_init, action_ngram_list, device='cuda', model_name='gpt2', cache_dir='pretrained/gpt2'): 45 | # Load pretrained tokenizer 46 | self.tokenizer = GPT2Tokenizer.from_pretrained(model_name, cache_dir=cache_dir) 47 | 48 | if is_random_init: 49 | print('Initialize with random weights', file=sys.stderr) 50 | config = GPT2Config(len(self.tokenizer)) 51 | self.model = GPT2LMHeadModel(config).to(device) 52 | else: 53 | print('Initialize with pretrained weights', file=sys.stderr) 54 | self.model = GPT2LMHeadModel.from_pretrained('gpt2', cache_dir=cache_dir).to(device) 55 | 56 | self.action_vocab = Vocabulary(action_ngram_list, 0) 57 | 58 | self.w_boundary_char = b'\xc4\xa0'.decode() 59 | self.model.action_decoder = torch.nn.Linear(768, len(self.action_vocab.symbols)).to(device) 60 | 61 | def get_batch_loss(self, line_batch, scaffold_type, add_eos_token=False): 62 | """ 63 | Assume each line of the batch input contains tokenized sentence paired with action ngram sequence. 64 | """ 65 | if add_eos_token: 66 | tokens_batch = [line[0] + [self.tokenizer.bos_token] for line in line_batch] 67 | else: 68 | tokens_batch = [line[0] for line in line_batch] 69 | 70 | batch_max_len = np.max([len(tokens) for tokens in tokens_batch]) 71 | tokens_padded_batch = [tokens + [self.tokenizer.bos_token for _ in range(batch_max_len - len(tokens))] for tokens in tokens_batch] 72 | 73 | # Mask padded tokens 74 | attention_mask = torch.ones(len(tokens_batch), 12, batch_max_len, batch_max_len).to(device) 75 | for b_idx, tokens in enumerate(tokens_batch): 76 | attention_mask[b_idx, :, :, len(tokens):] = 0 77 | 78 | input_ids_padded_batch = [self.tokenizer.convert_tokens_to_ids(tokens_padded) for tokens_padded in tokens_padded_batch] 79 | input_ids = torch.tensor(input_ids_padded_batch).to(device) 80 | 81 | label_ids_padded_batch = [self.tokenizer.convert_tokens_to_ids(tokens) + [-100 for _ in range(batch_max_len - len(tokens))] for tokens in tokens_batch] 82 | label_ids = torch.tensor(label_ids_padded_batch).to(device) 83 | 84 | output = self.model(input_ids, labels=label_ids, attention_mask=attention_mask, output_hidden_states=True) 85 | 86 | word_prediction_loss = output[0] 87 | 88 | if scaffold_type == 'next': 89 | action_ids_batch = torch.tensor([[self.action_vocab.token2index[a_ngram] for a_ngram in line[1]] + [-100 for _ in range(batch_max_len - len(line[1]))] for line in line_batch]).to(device) 90 | elif scaffold_type == 'past': 91 | action_ids_batch = torch.tensor([[self.action_vocab.pad_index] + [self.action_vocab.token2index[a_ngram] for a_ngram in line[1][:-1]] + [-100 for _ in range(batch_max_len - len(line[1]))] for line in line_batch]).to(device) 92 | else: 93 | raise NotImplementedError 94 | 95 | action_prediction_logits = self.model.action_decoder(output[-1][-1][:, :, :]) 96 | action_prediction_logits = action_prediction_logits.view(len(tokens_batch)*batch_max_len, -1) 97 | action_prediction_loss = cross_entropy_loss(action_prediction_logits, action_ids_batch.view(len(tokens_batch)*batch_max_len, -1).squeeze()) 98 | 99 | return word_prediction_loss, action_prediction_loss 100 | 101 | def get_validation_loss(self, dev_lines, scaffold_type, word_prediction_loss_only=False): 102 | if word_prediction_loss_only: 103 | # Only evaluate word prediction loss 104 | loss_sum = 0 105 | token_count = 0 106 | 107 | for line in dev_lines: 108 | tokens = line[0] 109 | ids = self.tokenizer.convert_tokens_to_ids(tokens) 110 | input_ids = torch.tensor([ids]).to(device) # batch size = 1 111 | 112 | loss = self.model(input_ids, labels=input_ids)[0].item() 113 | loss_sum += loss*(len(tokens)-1) 114 | token_count += len(tokens) - 1 115 | 116 | loss_avg = loss_sum/token_count 117 | 118 | return loss_avg 119 | 120 | else: 121 | word_loss_sum = 0 122 | action_loss_sum = 0 123 | word_token_count = 0 124 | action_token_count = 0 125 | 126 | for line_batch in get_batches(dev_lines, args.batch_size): 127 | n_word_token = np.sum([len(word_tokens) - 1 for word_tokens, _ in line_batch]) 128 | n_action_token = n_word_token + len(line_batch) 129 | 130 | word_prediction_loss, action_prediction_loss = self.get_batch_loss(line_batch, scaffold_type, add_eos_token=False) 131 | 132 | word_loss_sum += word_prediction_loss.item()*n_word_token 133 | action_loss_sum += action_prediction_loss.item()*n_action_token 134 | 135 | word_token_count += n_word_token 136 | action_token_count += n_action_token 137 | 138 | word_loss_avg = word_loss_sum/word_token_count 139 | action_loss_avg = action_loss_sum/action_token_count 140 | loss_avg = (1 - ALPHA)*word_loss_avg + ALPHA*action_loss_avg 141 | 142 | return loss_avg, word_loss_avg, action_loss_avg 143 | 144 | 145 | def get_surprisals(self, tokens, add_bos_token=True): 146 | surprisals = [] 147 | for i in range(len(tokens)): 148 | token_id = self.tokenizer.convert_tokens_to_ids(tokens[i]) 149 | if add_bos_token: 150 | # add BOS token 151 | prefix_tokens = [self.tokenizer.bos_token] + tokens[:i] 152 | else: 153 | if i == 0: 154 | surprisals.append(0.0) 155 | continue 156 | else: 157 | prefix_tokens = tokens[:i] 158 | 159 | ids = self.tokenizer.convert_tokens_to_ids(prefix_tokens) 160 | input_ids = torch.tensor([ids]).to(device) 161 | output = self.model(input_ids) 162 | logits = output[0] 163 | next_token_logits = logits[:, -1, :].squeeze() 164 | log_probs = log_softmax(next_token_logits) 165 | surprisal = -log_probs[token_id]/np.log(2) 166 | surprisals.append(surprisal) 167 | return surprisals 168 | 169 | 170 | def get_word_ppl(self, sents, add_bos_token=True): 171 | nll_total = 0 172 | word_count = 0 173 | 174 | total_token_count = 0 175 | 176 | for sent in sents: 177 | words = sent.split() 178 | if add_bos_token: 179 | tokens = [self.tokenizer.bos_token] + self.tokenizer.tokenize(sent) 180 | else: 181 | tokens = self.tokenizer.tokenize(sent) 182 | if len(tokens) <= 1: 183 | continue 184 | 185 | ids = self.tokenizer.convert_tokens_to_ids(tokens) 186 | input_ids = torch.tensor([ids]).to(device) # batch size = 1 187 | 188 | loss = self.model(input_ids, labels=input_ids)[0].item() # batch size = 1 189 | nll_total += loss*(len(tokens)-1) 190 | word_count += len(words) 191 | 192 | total_token_count += len(tokens)-1 193 | 194 | #print(nll_total, word_count, total_token_count) 195 | nll_avg = nll_total/word_count 196 | return np.exp(nll_avg) 197 | 198 | def generate(self, prompt, max_len=50, top_k=50, top_p=0.92, temperature=1, n_sample=1, device='cuda'): 199 | """ 200 | Sample from the model. 201 | """ 202 | tokens = self.tokenizer.tokenize(prompt) 203 | ids = self.tokenizer.convert_tokens_to_ids(tokens) 204 | input_ids = torch.tensor([ids]).to(device) 205 | output_ids_batch = self.model.generate(input_ids, do_sample=True, max_length=max_len, pad_token_id=50256, 206 | top_k=top_k, top_p=top_p, temperature=temperature, num_return_sequences=n_sample) 207 | samples = [self.tokenizer.decode(output_ids, skip_special_tokens=True).strip() for output_ids in output_ids_batch] 208 | return samples 209 | 210 | 211 | def is_nonterminal(token): 212 | """ 213 | Check if a token is a nonterminal action or word piece token. 214 | """ 215 | if (token.startswith('NT(') and token.endswith(')')) or token == 'REDUCE()': 216 | return True 217 | else: 218 | return False 219 | 220 | 221 | def load_sents(path): 222 | with open(path) as f: 223 | lines = f.readlines() 224 | lines = [line.strip() for line in lines] 225 | lines = [line for line in lines if line != ''] 226 | return lines 227 | 228 | 229 | def load_data(path, tokenizer, BOS_token=None): 230 | 231 | with open(path) as f: 232 | lines = f.readlines() 233 | lines = [line.strip() for line in lines] 234 | lines = [line for line in lines if line != ''] 235 | 236 | data = [] 237 | 238 | for line in lines: 239 | tokens = line.split() 240 | action_ngrams = [] 241 | words = [] 242 | action_seq = [] 243 | for token in tokens: 244 | if is_nonterminal(token): 245 | action_seq.append(token) 246 | else: 247 | if action_seq != []: 248 | action_ngrams.append('_'.join(action_seq)) 249 | action_seq = [] 250 | else: 251 | action_ngrams.append('_') 252 | words.append(token) 253 | 254 | action_ngrams.append('_'.join(action_seq)) # add the action ngram that comes after the last word 255 | 256 | sent = ' '.join(words) 257 | word_pieces = tokenizer.tokenize(sent) 258 | 259 | combined = '' 260 | n_piece = 0 261 | word_index = 0 262 | action_ngram_seq = [] 263 | 264 | for piece in word_pieces: 265 | if piece.startswith(w_boundary_char): 266 | combined += piece[1:] 267 | else: 268 | combined += piece 269 | n_piece += 1 270 | if combined == words[word_index]: 271 | action_ngram_seq += [action_ngrams[word_index]] + ['_' for _ in range(n_piece-1)] 272 | combined = '' 273 | n_piece = 0 274 | word_index += 1 275 | assert combined == '' 276 | assert word_index == len(words) 277 | 278 | action_ngram_seq.append(action_ngrams[-1]) 279 | 280 | assert len(word_pieces) == (len(action_ngram_seq) - 1) 281 | 282 | if BOS_token is None: 283 | data.append([word_pieces, action_ngram_seq]) 284 | else: 285 | data.append([[BOS_token] + word_pieces, action_ngram_seq]) 286 | 287 | return data 288 | 289 | 290 | def get_batches(lines, batch_size): 291 | if len(lines) % batch_size == 0: 292 | num_batches = len(lines) // batch_size 293 | else: 294 | num_batches = len(lines) // batch_size + 1 295 | batches = [] 296 | for i in range(num_batches): 297 | start_index = i*batch_size 298 | end_index = (i+1)*batch_size 299 | batch = lines[start_index:end_index] 300 | batches.append(batch) 301 | return batches 302 | 303 | 304 | if __name__ == "__main__": 305 | parser = argparse.ArgumentParser() 306 | parser.add_argument('--train_data', type=str, help='Path to training data.') 307 | parser.add_argument('--dev_data', type=str, help='Path to validation data.') 308 | parser.add_argument('--test_data', type=str, help='Path to test data.') 309 | parser.add_argument('--fpath', type=str, help='File path for estimating surprisals.') 310 | parser.add_argument('--lr', type=float, default=1e-5, help='Learning rate.') 311 | parser.add_argument('--alpha', type=float, default=0.5, help='Hyerparameter in (0, 1) for weighting the structure prediction loss against the word prediction loss. Default is 0.5.') 312 | parser.add_argument('--scaffold_type', type=str, help='Type of scaffold. (next, past)') 313 | parser.add_argument('--epochs', type=int, default=10, help='Number of training epochs.') 314 | parser.add_argument('--report', type=int, default=1000, help='Frequency of report training status after number of training batches.') 315 | parser.add_argument('--valid_every', type=int, default=None, help='Frequency of validating and saving model parameters after number of training batches.') 316 | parser.add_argument('--sample_every', type=int, default=10000, help='Frequency of generating samples from the model during training.') 317 | parser.add_argument('--seed', type=int, default=None, help='Random seed.') 318 | parser.add_argument('--do_train', action='store_true', help='Whether to train the model.') 319 | parser.add_argument('--do_test', action='store_true', help='Whether to test the model.') 320 | parser.add_argument('--do_eval', action='store_true', help='Whether to use the model for surprisal estimation.') 321 | parser.add_argument('--model_path', type=str, default=None, help='Path of the model to be trained and saved.') 322 | parser.add_argument('--restore_from', type=str, default=None, help='Path to the trained model checkpoint. Will use the pretrained model if path not specified.') 323 | parser.add_argument('--batch_size', type=int, default=5, help="Size of a training batch.") 324 | parser.add_argument('--early_stopping_threshold', type=int, default=2, help='Threshold for early stopping.') 325 | parser.add_argument('--random_init', action='store_true', help="Randomly initialize model parameters.") 326 | parser.add_argument('--pretokenized', action='store_true', help="Whether input sentences for evaluating surprisals are pertokenized or not.") 327 | 328 | args = parser.parse_args() 329 | 330 | log_softmax = torch.nn.LogSoftmax(-1) 331 | cross_entropy_loss = torch.nn.CrossEntropyLoss() 332 | 333 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 334 | 335 | # Set random seed 336 | RANDOM_SEED = args.seed if args.seed is not None else int(np.random.random()*10000) 337 | torch.manual_seed(RANDOM_SEED) 338 | random.seed(RANDOM_SEED) 339 | np.random.seed(RANDOM_SEED) 340 | print('Random seed: {}'.format(RANDOM_SEED), file=sys.stderr) 341 | 342 | # load action ngram list and initialize embeddings 343 | with open('bllip-lg_action_ngram_list.txt') as f: 344 | lines = f.readlines() 345 | symbols = ['', '_'] + [line.strip().split()[0] for line in lines] 346 | 347 | # Initialize the model 348 | sclm = ScLM(is_random_init=args.random_init, action_ngram_list=symbols, device=device, model_name='gpt2') 349 | w_boundary_char = sclm.w_boundary_char 350 | 351 | # Load model checkpoint 352 | if args.restore_from is not None: 353 | print('Load parameters from {}'.format(args.restore_from), file=sys.stderr) 354 | checkpoint = torch.load(args.restore_from) 355 | sclm.model.load_state_dict(checkpoint['model_state_dict']) 356 | 357 | SCAFFOLD_TYPE = args.scaffold_type 358 | print('Scaffold type: {}'.format(SCAFFOLD_TYPE), file=sys.stderr) 359 | ALPHA = args.alpha 360 | print('Interpolation weight of structure prediction loss {}'.format(ALPHA), file=sys.stderr) 361 | 362 | # Train 363 | if args.do_train: 364 | # Path to save the newly trained model 365 | MODEL_PATH = args.model_path if args.model_path is not None else "sclm-{}_pid{}.params".format(SCAFFOLD_TYPE, os.getpid()) 366 | 367 | # print out training settings 368 | print('Training batch size: {}'.format(args.batch_size), file=sys.stderr) 369 | print('Learning rate: {}'.format(args.lr), file=sys.stderr) 370 | print('Model path: {}'.format(MODEL_PATH), file=sys.stderr) 371 | 372 | optimizer = AdamW(sclm.model.parameters(), lr=args.lr) 373 | 374 | # Load train and dev data 375 | train_data_path = args.train_data 376 | dev_data_path = args.dev_data 377 | print("Loading train data from {}".format(train_data_path), file=sys.stderr) 378 | train_lines = load_data(train_data_path, sclm.tokenizer, BOS_token=sclm.tokenizer.bos_token) 379 | print("Loading dev data from {}".format(dev_data_path), file=sys.stderr) 380 | dev_lines = load_data(dev_data_path, sclm.tokenizer, BOS_token=sclm.tokenizer.bos_token) 381 | 382 | if args.restore_from is not None: 383 | sclm.model.eval() 384 | with torch.no_grad(): 385 | validation_loss, _, _ = sclm.get_validation_loss(dev_lines, scaffold_type=SCAFFOLD_TYPE) 386 | best_validation_loss = validation_loss 387 | sclm.model.train() 388 | print('resume training; validation loss: {}'.format(best_validation_loss)) 389 | else: 390 | best_validation_loss = np.inf 391 | 392 | n_epochs = args.epochs 393 | starting_epoch = checkpoint['epoch'] + 1 if (args.restore_from is not None) else 0 394 | no_improvement_count = checkpoint['no_improvement_count'] if (args.restore_from is not None) else 0 395 | VALID_EVERY = None if ((args.valid_every is None) or (args.valid_every < 1)) else args.valid_every 396 | 397 | early_stopping_counter = utils.EarlyStopping(best_validation_loss=best_validation_loss, no_improvement_count=no_improvement_count, threshold=args.early_stopping_threshold) 398 | 399 | for epoch in range(starting_epoch, n_epochs): 400 | random.shuffle(train_lines) 401 | 402 | count = 0 # cumulative count of training examples 403 | batch_count = 0 # cumulative count of training batches 404 | 405 | for line_batch in get_batches(train_lines, args.batch_size): 406 | optimizer.zero_grad() 407 | 408 | word_prediction_loss, action_prediction_loss = sclm.get_batch_loss(line_batch, SCAFFOLD_TYPE, add_eos_token=True) 409 | 410 | loss = (1 - ALPHA) * word_prediction_loss + ALPHA * action_prediction_loss 411 | 412 | loss.backward() 413 | optimizer.step() 414 | 415 | count += len(line_batch) 416 | batch_count += 1 417 | 418 | if batch_count > 0 and batch_count % args.report == 0: 419 | print('Epoch {:.3f} loss: {}'.format(epoch + count/len(train_lines), loss.item())) 420 | 421 | if batch_count > 0 and batch_count % args.sample_every == 0: 422 | sclm.model.eval() 423 | with torch.no_grad(): 424 | samples = sclm.generate(prompt="<|endoftext|>") 425 | for sample in samples: 426 | print(sample) 427 | sclm.model.train() 428 | 429 | if VALID_EVERY is not None: 430 | if batch_count > 0 and batch_count % VALID_EVERY == 0: 431 | sclm.model.eval() 432 | 433 | with torch.no_grad(): 434 | validation_loss, validation_word_prediction_loss, _ = sclm.get_validation_loss(dev_lines, scaffold_type=SCAFFOLD_TYPE) 435 | print('Epoch {:.3f} validation loss: {} validation next-word prediction loss: {}'.format(epoch + count/len(train_lines), validation_loss, validation_word_prediction_loss)) 436 | 437 | is_early_stop = early_stopping_counter.check_stopping_criterion(validation_loss) 438 | if is_early_stop: 439 | print('Validation loss increases for {} epochs in a row.'.format(early_stopping_counter.counter)) 440 | print('EARLY STOPPING...') 441 | sys.exit() 442 | 443 | if validation_loss < best_validation_loss: 444 | best_validation_loss = validation_loss 445 | print("new best... saving model to {}".format(MODEL_PATH)) 446 | torch.save( 447 | {'epoch': epoch, 448 | 'scaffold_type': SCAFFOLD_TYPE, 449 | 'no_improvement_count': early_stopping_counter.counter, 450 | 'model_state_dict': sclm.model.state_dict(), 451 | 'loss': validation_loss}, MODEL_PATH) 452 | 453 | sclm.model.train() 454 | 455 | sclm.model.eval() 456 | 457 | with torch.no_grad(): 458 | validation_loss, validation_word_prediction_loss, _ = sclm.get_validation_loss(dev_lines, scaffold_type=SCAFFOLD_TYPE) 459 | print('Epoch {} validation loss: {} validation next-word prediction loss: {}'.format(epoch, validation_loss, validation_word_prediction_loss)) 460 | 461 | is_early_stop = early_stopping_counter.check_stopping_criterion(validation_loss) 462 | if is_early_stop: 463 | print('Validation loss increases for {} epochs in a row.'.format(early_stopping_counter.counter)) 464 | print('EARLY STOPPING...') 465 | sys.exit() 466 | 467 | if validation_loss < best_validation_loss: 468 | best_validation_loss = validation_loss 469 | print("new best... saving model to {}".format(MODEL_PATH)) 470 | torch.save( 471 | {'epoch': epoch, 472 | 'scaffold_type': SCAFFOLD_TYPE, 473 | 'no_improvement_count': early_stopping_counter.counter, 474 | 'model_state_dict': sclm.model.state_dict(), 475 | 'loss': validation_loss}, MODEL_PATH) 476 | 477 | sclm.model.train() 478 | 479 | if args.do_test: 480 | sclm.model.eval() 481 | if args.test_data is None: 482 | raise ValueError('Test data not specified') 483 | 484 | test_data_path = args.test_data 485 | 486 | test_sents = [] 487 | lines = load_sents(args.test_data) 488 | for line in lines: 489 | tokens = line.split() 490 | words = [token for token in tokens if not is_nonterminal(token)] 491 | test_sents.append(' '.join(words)) 492 | 493 | with torch.no_grad(): 494 | ppl = sclm.get_word_ppl(test_sents) 495 | print('PPL: {}'.format(ppl)) 496 | 497 | # Estimate token surprisal values for unparsed sentences 498 | if args.do_eval: 499 | sclm.model.eval() 500 | 501 | if args.fpath is not None: 502 | sents = load_sents(args.fpath) 503 | else: 504 | sents = ["The dogs under the tree are barking.", "The dogs under the tree is barking.", 505 | "The keys to the cabinet are on the table.", "The keys to the cabinet is on the table.",] 506 | 507 | print('sentence_id\ttoken_id\ttoken\tsurprisal') 508 | 509 | for i, sent in enumerate(sents): 510 | if args.pretokenized: 511 | words = sent.strip().split() 512 | stimulus = sent.strip() 513 | else: 514 | words = nltk.word_tokenize(sent.strip()) 515 | stimulus = ' '.join(words) 516 | 517 | tokens = sclm.tokenizer.tokenize(stimulus) 518 | with torch.no_grad(): 519 | surprisals = sclm.get_surprisals(tokens, add_bos_token=True) 520 | 521 | index = 0 522 | for j, word in enumerate(words): 523 | w_str = '' 524 | w_surprisal = 0 525 | while index < len(tokens) and w_str != word: 526 | token_str = tokens[index] 527 | if token_str.startswith(w_boundary_char): 528 | w_str += token_str[1:] 529 | else: 530 | w_str += token_str 531 | w_surprisal += surprisals[index] 532 | 533 | index += 1 534 | 535 | print('{}\t{}\t{}\t{}'.format(i+1, j+1, word, w_surprisal)) 536 | -------------------------------------------------------------------------------- /src/lm.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import argparse 4 | import random 5 | import numpy as np 6 | import nltk 7 | from transformers import GPT2Tokenizer, GPT2LMHeadModel, GPT2Config, AdamW 8 | import torch 9 | import utils 10 | import functools 11 | print = functools.partial(print, flush=True) 12 | 13 | 14 | class LM: 15 | def __init__(self, is_random_init, device='cuda', model_name='gpt2', cache_dir='pretrained/gpt2'): 16 | # Load pretrained tokenizer 17 | self.tokenizer = GPT2Tokenizer.from_pretrained(model_name, cache_dir=cache_dir) 18 | 19 | if is_random_init: 20 | print('Initialize with random weights', file=sys.stderr) 21 | config = GPT2Config(len(self.tokenizer)) 22 | self.model = GPT2LMHeadModel(config).to(device) 23 | else: 24 | print('Initialize with pretrained weights', file=sys.stderr) 25 | self.model = GPT2LMHeadModel.from_pretrained(model_name, cache_dir=cache_dir).to(device) 26 | 27 | self.w_boundary_char = b'\xc4\xa0'.decode() 28 | 29 | def get_batch_loss(self, data_batch, device='cuda'): 30 | """ 31 | Assume a data batch as a list of sequences. 32 | """ 33 | tokens_batch = [self.tokenizer.tokenize(line) for line in data_batch] 34 | 35 | token_count_batch = [len(tokens) for tokens in tokens_batch] 36 | batch_max_len = np.max(token_count_batch) 37 | 38 | tokens_padded_batch = [tokens + [self.tokenizer.bos_token for _ in range(batch_max_len - len(tokens))] for tokens in tokens_batch] 39 | 40 | attention_mask = torch.ones(len(tokens_batch), 12, batch_max_len, batch_max_len).to(device) 41 | for b_idx, tokens in enumerate(tokens_batch): 42 | attention_mask[b_idx, :, :, len(tokens):] = 0 43 | 44 | input_ids_padded_batch = [self.tokenizer.convert_tokens_to_ids(tokens_padded) for tokens_padded in tokens_padded_batch] 45 | input_ids = torch.tensor(input_ids_padded_batch).to(device) 46 | label_ids_padded_batch = [self.tokenizer.convert_tokens_to_ids(tokens) + [-100 for _ in range(batch_max_len - len(tokens))] for tokens in tokens_batch] 47 | label_ids = torch.tensor(label_ids_padded_batch).to(device) 48 | 49 | output = self.model(input_ids, labels=label_ids, attention_mask=attention_mask, return_dict=True) 50 | 51 | loss = output.loss 52 | batch_token_count = np.sum(token_count_batch) - len(tokens_batch) # substract the count since len(tokens)-1 words are counted 53 | return loss, batch_token_count 54 | 55 | def get_loss(self, data, batch_size, device='cuda'): 56 | total_loss = 0 57 | total_token_count = 0 58 | 59 | for data_batch in get_batches(data, batch_size): 60 | loss, batch_token_count = self.get_batch_loss(data_batch, device=device) 61 | total_loss += loss.item()*batch_token_count 62 | total_token_count += batch_token_count 63 | 64 | return total_loss/total_token_count 65 | 66 | def generate(self, prompt, max_len=50, top_k=50, top_p=0.92, temperature=1, n_sample=1, device='cuda'): 67 | """ 68 | Sample from the model. 69 | """ 70 | tokens = self.tokenizer.tokenize(prompt) 71 | ids = self.tokenizer.convert_tokens_to_ids(tokens) 72 | input_ids = torch.tensor([ids]).to(device) 73 | output_ids_batch = self.model.generate(input_ids, do_sample=True, max_length=max_len, pad_token_id=50256, 74 | top_k=top_k, top_p=top_p, temperature=temperature, num_return_sequences=n_sample) 75 | samples = [self.tokenizer.decode(output_ids, skip_special_tokens=True).strip() for output_ids in output_ids_batch] 76 | return samples 77 | 78 | def get_word_level_perplexity(self, dev_lines, add_bos_token=True): 79 | loss_sum = 0 80 | total_word_count = 0 81 | 82 | for line in dev_lines: 83 | if add_bos_token: 84 | tokens = [self.tokenizer.bos_token] + self.tokenizer.tokenize(line) 85 | else: 86 | tokens = self.tokenizer.tokenize(line) 87 | if len(tokens) <= 1: 88 | continue 89 | 90 | ids = self.tokenizer.convert_tokens_to_ids(tokens) 91 | input_ids = torch.tensor([ids]).to(device) # batch size = 1 92 | 93 | loss = self.model(input_ids, labels=input_ids)[0].item() 94 | loss_sum += loss*(len(tokens)-1) 95 | total_word_count += len(line.strip().split()) 96 | return np.exp(loss_sum/total_word_count) 97 | 98 | def get_surprisals(self, tokens, add_bos_token=True): 99 | surprisals = [] 100 | for i in range(len(tokens)): 101 | token_id = self.tokenizer.convert_tokens_to_ids(tokens[i]) 102 | if add_bos_token: 103 | # add BOS token 104 | prefix_tokens = [self.tokenizer.bos_token] + tokens[:i] 105 | else: 106 | if i == 0: 107 | surprisals.append(0.0) 108 | continue 109 | else: 110 | prefix_tokens = tokens[:i] 111 | ids = self.tokenizer.convert_tokens_to_ids(prefix_tokens) 112 | input_ids = torch.tensor([ids]).to(device) 113 | output = self.model(input_ids) 114 | logits = output[0] 115 | next_token_logits = logits[:, -1, :].squeeze() 116 | log_probs = log_softmax(next_token_logits) 117 | surprisal = -log_probs[token_id]/np.log(2) 118 | surprisals.append(surprisal) 119 | return surprisals 120 | 121 | 122 | def load_data(path): 123 | with open(path) as f: 124 | lines = f.readlines() 125 | lines = [line.strip() for line in lines] 126 | lines = [line for line in lines if line != ''] 127 | return lines 128 | 129 | 130 | def get_batches(lines, batch_size): 131 | if len(lines) % batch_size == 0: 132 | num_batches = len(lines) // batch_size 133 | else: 134 | num_batches = len(lines) // batch_size + 1 135 | batches = [] 136 | for i in range(num_batches): 137 | start_index = i*batch_size 138 | end_index = (i+1)*batch_size 139 | batch = lines[start_index:end_index] 140 | batches.append(batch) 141 | return batches 142 | 143 | 144 | if __name__ == "__main__": 145 | parser = argparse.ArgumentParser() 146 | parser.add_argument('--train_data', type=str, help='Path to training data.') 147 | parser.add_argument('--dev_data', type=str, help='Path to validation data.') 148 | parser.add_argument('--test_data', type=str, help='Path to test data.') 149 | parser.add_argument('--lr', type=float, default=1e-5, help='Learning rate.') 150 | parser.add_argument('--epochs', type=int, default=10, help='Number of training epochs.') 151 | parser.add_argument('--report', type=int, default=1000, help='Frequency of reporting training status after number of training batches.') 152 | parser.add_argument('--sample_every', type=int, default=10000, help='Frequency of generating samples from the model during training.') 153 | parser.add_argument('--seed', type=int, default=None, help='Random seed.') 154 | parser.add_argument('--do_train', action='store_true', help='Whether to train the model.') 155 | parser.add_argument('--do_test', action='store_true', help='Whether to test the model.') 156 | parser.add_argument('--do_eval', action='store_true', help='Whether to use the model for surprisal estimation.') 157 | parser.add_argument('--model_path', type=str, default=None, help='Path of the model to be trained and saved.') 158 | parser.add_argument('--restore_from', type=str, default=None, help='Path to the trained model checkpoint. Will use the pretrained model if path not specified.') 159 | parser.add_argument('--batch_size', type=int, default=5, help="Size of a training batch.") 160 | parser.add_argument('--early_stopping_threshold', type=int, default=2, help='Threshold for early stopping.') 161 | parser.add_argument('--random_init', action='store_true', help="Randomly initialize model parameters.") 162 | parser.add_argument('--fpath', type=str, help='Path to text file for estimating surprisals.') 163 | parser.add_argument('--pretokenized', action='store_true', help="Whether input sentences for evaluating surprisals are pertokenized or not.") 164 | 165 | args = parser.parse_args() 166 | 167 | log_softmax = torch.nn.LogSoftmax(-1) 168 | 169 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 170 | 171 | # Set random seed 172 | RANDOM_SEED = args.seed if args.seed is not None else int(np.random.random()*10000) 173 | torch.manual_seed(RANDOM_SEED) 174 | random.seed(RANDOM_SEED) 175 | np.random.seed(RANDOM_SEED) 176 | print('Random seed: {}'.format(RANDOM_SEED), file=sys.stderr) 177 | 178 | # Initialize language model 179 | lm = LM(is_random_init=args.random_init, device=device, model_name='gpt2', cache_dir='pretrained/gpt2') 180 | 181 | # Restore from a model checkpoint 182 | if args.restore_from is not None: 183 | print('Load parameters from {}'.format(args.restore_from), file=sys.stderr) 184 | checkpoint = torch.load(args.restore_from) 185 | # lm.model.load_state_dict(torch.load(args.restore_from)) 186 | lm.model.load_state_dict(checkpoint['model_state_dict']) 187 | 188 | # Train 189 | if args.do_train: 190 | # Path to save the newly trained model 191 | MODEL_PATH = args.model_path if args.model_path is not None else "lm_pid{}.params".format(os.getpid()) 192 | 193 | # Set the learning rate of the optimizer 194 | optimizer = AdamW(lm.model.parameters(), lr=args.lr) 195 | 196 | # Print out training settings 197 | print('Training batch size: {}'.format(args.batch_size), file=sys.stderr) 198 | print('Learning rate: {}'.format(args.lr), file=sys.stderr) 199 | print('Model path: {}'.format(MODEL_PATH), file=sys.stderr) 200 | 201 | # Load train and dev data 202 | train_data_path = args.train_data 203 | dev_data_path = args.dev_data 204 | print("Loading train data from {}".format(train_data_path), file=sys.stderr) 205 | print("Loading dev data from {}".format(dev_data_path), file=sys.stderr) 206 | train_lines = load_data(train_data_path) 207 | dev_lines = load_data(dev_data_path) 208 | train_lines = [lm.tokenizer.bos_token + line + lm.tokenizer.bos_token for line in train_lines] 209 | dev_lines = [lm.tokenizer.bos_token + line for line in dev_lines] 210 | 211 | if args.restore_from is not None: 212 | lm.model.eval() 213 | with torch.no_grad(): 214 | best_validation_loss = lm.get_loss(dev_lines, batch_size=args.batch_size) 215 | lm.model.train() 216 | print('resume training; validation loss: {}'.format(best_validation_loss)) 217 | else: 218 | best_validation_loss = np.Inf 219 | 220 | n_epochs = args.epochs 221 | starting_epoch = checkpoint['epoch'] + 1 if (args.restore_from is not None) else 0 222 | no_improvement_count = checkpoint['no_improvement_count'] if (args.restore_from is not None) else 0 223 | 224 | early_stopping_counter = utils.EarlyStopping(best_validation_loss=best_validation_loss, no_improvement_count=no_improvement_count, threshold=args.early_stopping_threshold) 225 | 226 | for epoch in range(starting_epoch, n_epochs): 227 | random.shuffle(train_lines) 228 | 229 | count = 0 # cumulative count of training examples 230 | batch_count = 0 # cumulative count of training batches 231 | 232 | for train_data_batch in get_batches(train_lines, args.batch_size): 233 | optimizer.zero_grad() 234 | 235 | loss, batch_token_count = lm.get_batch_loss(train_data_batch) 236 | loss.backward() 237 | optimizer.step() 238 | 239 | batch_count += 1 240 | count += len(train_data_batch) 241 | 242 | if batch_count > 0 and batch_count % args.report == 0: 243 | print('Epoch {:.3f} loss: {}'.format(epoch + count/len(train_lines), loss.item())) 244 | 245 | if batch_count > 0 and batch_count % args.sample_every == 0: 246 | lm.model.eval() 247 | with torch.no_grad(): 248 | samples = lm.generate(prompt="<|endoftext|>") 249 | for sample in samples: 250 | print(sample) 251 | lm.model.train() 252 | 253 | lm.model.eval() 254 | with torch.no_grad(): 255 | validation_loss = lm.get_loss(dev_lines, batch_size=args.batch_size) 256 | print('Epoch', epoch, 'validation loss:', validation_loss) 257 | 258 | is_early_stop = early_stopping_counter.check_stopping_criterion(validation_loss) 259 | if is_early_stop: 260 | print('Validation loss increases for {} epochs in a row.'.format(early_stopping_counter.counter)) 261 | print('EARLY STOPPING...') 262 | break 263 | 264 | if validation_loss < best_validation_loss: 265 | best_validation_loss = validation_loss 266 | print("new best... saving model to {}".format(MODEL_PATH)) 267 | torch.save( 268 | {'epoch': epoch, 269 | 'no_improvement_count': early_stopping_counter.counter, 270 | 'model_state_dict': lm.model.state_dict(), 271 | 'loss': validation_loss}, MODEL_PATH) 272 | 273 | lm.model.train() 274 | 275 | # Test 276 | if args.do_test: 277 | lm.model.eval() 278 | if args.test_data is None: 279 | raise ValueError('Test data not specified') 280 | 281 | test_lines = load_data(args.test_data) 282 | test_lines = [lm.tokenizer.bos_token + '' + line for line in test_lines] 283 | 284 | with torch.no_grad(): 285 | validation_loss = lm.get_loss(test_lines, batch_size=args.batch_size) 286 | print('Test loss: {}'.format(validation_loss)) 287 | #validation_loss = lm.get_loss(test_lines, batch_size=1) 288 | #print('Test loss: {}'.format(validation_loss)) 289 | print('PPL: {}'.format(lm.get_word_level_perplexity(test_lines, add_bos_token=False))) 290 | 291 | # Estimate word-level surprisal values for sentences 292 | if args.do_eval: 293 | lm.model.eval() 294 | 295 | if args.fpath is not None: 296 | sents = load_data(args.fpath) 297 | else: 298 | sents = ["The dogs under the tree are barking.", "The dogs under the tree is barking.", 299 | "The keys to the cabinet are on the table.", "The keys to the cabinet is on the table.",] 300 | 301 | print('sentence_id\ttoken_id\ttoken\tsurprisal') 302 | 303 | for i, sent in enumerate(sents): 304 | if args.pretokenized: 305 | words = sent.strip().split() 306 | stimulus = sent.strip() 307 | else: 308 | words = nltk.word_tokenize(sent.strip()) 309 | stimulus = ' '.join(words) 310 | 311 | tokens = lm.tokenizer.tokenize(stimulus) 312 | with torch.no_grad(): 313 | surprisals = lm.get_surprisals(tokens, add_bos_token=True) 314 | 315 | index = 0 316 | for j, word in enumerate(words): 317 | w_str = '' 318 | w_surprisal = 0 319 | while index < len(tokens) and w_str != word: 320 | token_str = tokens[index] 321 | if token_str.startswith(lm.w_boundary_char): 322 | w_str += token_str[1:] 323 | else: 324 | w_str += token_str 325 | w_surprisal += surprisals[index] 326 | 327 | index += 1 328 | 329 | print('{}\t{}\t{}\t{}'.format(i+1, j+1, word, w_surprisal)) 330 | -------------------------------------------------------------------------------- /src/transformers-v3.3.1-x.patch: -------------------------------------------------------------------------------- 1 | From 4729b6de7436b4cc614a91ee378be0176150ad5d Mon Sep 17 00:00:00 2001 2 | From: ramon-astudillo 3 | Date: Fri, 23 Oct 2020 05:57:07 -0400 4 | Subject: [PATCH] Updated x-former code 5 | 6 | --- 7 | src/transformers/modeling_gpt2.py | 19 +++++++++++-------- 8 | 1 file changed, 11 insertions(+), 8 deletions(-) 9 | 10 | diff --git a/src/transformers/modeling_gpt2.py b/src/transformers/modeling_gpt2.py 11 | index 3ab1660..283e116 100644 12 | --- a/src/transformers/modeling_gpt2.py 13 | +++ b/src/transformers/modeling_gpt2.py 14 | @@ -570,14 +570,17 @@ class GPT2Model(GPT2PreTrainedModel): 15 | 16 | # Attention mask. 17 | if attention_mask is not None: 18 | - assert batch_size > 0, "batch_size has to be defined and > 0" 19 | - attention_mask = attention_mask.view(batch_size, -1) 20 | - # We create a 3D attention mask from a 2D tensor mask. 21 | - # Sizes are [batch_size, 1, 1, to_seq_length] 22 | - # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] 23 | - # this attention mask is more simple than the triangular masking of causal attention 24 | - # used in OpenAI GPT, we just need to prepare the broadcast dimension here. 25 | - attention_mask = attention_mask[:, None, None, :] 26 | + 27 | + # If tensor already has the right size, no need to expand it 28 | + if len(attention_mask.shape) != 4: 29 | + assert batch_size > 0, "batch_size has to be defined and > 0" 30 | + attention_mask = attention_mask.view(batch_size, -1) 31 | + # We create a 3D attention mask from a 2D tensor mask. 32 | + # Sizes are [batch_size, 1, 1, to_seq_length] 33 | + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] 34 | + # this attention mask is more simple than the triangular masking of causal attention 35 | + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. 36 | + attention_mask = attention_mask[:, None, None, :] 37 | 38 | # Since attention_mask is 1.0 for positions we want to attend and 0.0 for 39 | # masked positions, this operation will create a tensor which is 0.0 for 40 | -- 41 | 1.8.3.1 42 | 43 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | class EarlyStopping: 2 | 3 | def __init__(self, best_validation_loss, no_improvement_count, threshold=2): 4 | self.best_validation_loss = best_validation_loss 5 | self.counter = no_improvement_count 6 | self.threshold = threshold 7 | 8 | def check_stopping_criterion(self, validation_loss): 9 | if validation_loss >= self.best_validation_loss: 10 | self.counter += 1 11 | else: 12 | self.best_validation_loss = validation_loss 13 | self.counter = 0 14 | 15 | if self.counter >= self.threshold: 16 | return True 17 | else: 18 | return False 19 | --------------------------------------------------------------------------------