├── LICENSE ├── QA_bias_benchmark.pdf ├── README.md ├── analysis_scripts ├── BBQ_calculate_bias_score.R ├── BBQ_results_metadata.R └── additional_metadata.csv ├── data ├── Age.jsonl ├── Disability_status.jsonl ├── Gender_identity.jsonl ├── Nationality.jsonl ├── Physical_appearance.jsonl ├── Race_ethnicity.jsonl ├── Race_x_SES.jsonl ├── Race_x_gender.jsonl ├── Religion.jsonl ├── SES.jsonl └── Sexual_orientation.jsonl ├── generate_from_template_all_categories.py ├── generate_from_template_intersectional_cats.py ├── results ├── RoBERTa_and_DeBERTaV3 │ └── df_bbq.csv └── UnifiedQA │ ├── preds_Age.jsonl │ ├── preds_Disability_status.jsonl │ ├── preds_Gender_identity.jsonl │ ├── preds_Nationality.jsonl │ ├── preds_Physical_appearance.jsonl │ ├── preds_Race_ethnicity.jsonl │ ├── preds_Race_x_SES.jsonl │ ├── preds_Race_x_gender.jsonl │ ├── preds_Religion.jsonl │ ├── preds_SES.jsonl │ └── preds_Sexual_orientation.jsonl ├── supplemental ├── .gitignore ├── MTurk_validation │ ├── data │ │ ├── qual_batch.csv │ │ ├── validations_finalval.csv │ │ ├── validations_round1.1.csv │ │ ├── validations_round1.2.csv │ │ ├── validations_round1.3.csv │ │ ├── validations_round1.4.csv │ │ ├── validations_round2.1.csv │ │ ├── validations_round2.2.csv │ │ ├── validations_round2.3.csv │ │ ├── validations_round3.csv │ │ ├── validations_round5.csv │ │ └── validations_round6.csv │ ├── for_upload │ │ ├── finalval │ │ │ ├── donotupload_full_data_for_mturk_full_finalval.csv │ │ │ ├── full_data_for_mturk_1_finalval.csv │ │ │ ├── full_data_for_mturk_2_finalval.csv │ │ │ └── full_data_for_mturk_3_finalval.csv │ │ ├── round1 │ │ │ ├── UPLOADED_full_data_for_mturk_1.csv │ │ │ ├── UPLOADED_full_data_for_mturk_2.csv │ │ │ ├── UPLOADED_full_data_for_mturk_3.csv │ │ │ ├── UPLOADED_full_data_for_mturk_4.csv │ │ │ └── donotupload_full_data_for_mturk_full.csv │ │ ├── round2 │ │ │ ├── UPLOADED_full_data_for_mturk_1_round2.csv │ │ │ ├── UPLOADED_full_data_for_mturk_2_round2.csv │ │ │ ├── UPLOADED_full_data_for_mturk_3_round2.csv │ │ │ └── donotupload_full_data_for_mturk_full_round2.csv │ │ ├── round3 │ │ │ ├── donotupload_full_data_for_mturk_full_round3.csv │ │ │ ├── full_data_for_mturk_1_round3.csv │ │ │ └── full_data_for_mturk_2_round3.csv │ │ ├── round4 │ │ │ ├── donotupload_full_data_for_mturk_full_round4.csv │ │ │ └── full_data_for_mturk_1_round4.csv │ │ ├── round5 │ │ │ ├── donotupload_full_data_for_mturk_full_round5.csv │ │ │ ├── full_data_for_mturk_1_round5.csv │ │ │ └── full_data_for_mturk_2_round5.csv │ │ └── round6 │ │ │ ├── donotupload_full_data_for_mturk_full_round6.csv │ │ │ └── full_data_for_mturk_1_round6.csv │ ├── included_workers.csv │ ├── qualification_HIT_template.html │ ├── results_analysis │ │ ├── results_round3.csv │ │ ├── results_round5.csv │ │ ├── results_round6.csv │ │ ├── round1 │ │ │ ├── results_round1.1.csv │ │ │ ├── results_round1.2.csv │ │ │ ├── results_round1.3.csv │ │ │ └── results_round1.4.csv │ │ └── round2 │ │ │ ├── results_round2.1.csv │ │ │ ├── results_round2.2.csv │ │ │ └── results_round2.3.csv │ ├── scripts │ │ ├── all_examples.csv │ │ ├── all_examples_finalval.csv │ │ ├── all_examples_round2.csv │ │ ├── all_examples_round3.csv │ │ ├── all_examples_round4.csv │ │ ├── all_examples_round5.csv │ │ ├── all_examples_round6.csv │ │ ├── all_examples_uid.csv │ │ ├── all_examples_uid_finalval.csv │ │ ├── all_examples_uid_round2.csv │ │ ├── all_examples_uid_round3.csv │ │ ├── all_examples_uid_round4.csv │ │ ├── all_examples_uid_round5.csv │ │ ├── all_examples_uid_round6.csv │ │ ├── analyze_qual_screener.R │ │ ├── analyze_validations.R │ │ ├── anonymize_and_store.R │ │ └── create_validation_HITs.R │ └── validation_HIT_template.html ├── NYC_babyname_Data.R ├── additional_metadata.csv └── name_job_data │ ├── English First Names for Chinese Americans _ Social Science Statistics Blog.html │ ├── Names_2010Census_Top1000.xlsx │ ├── Popular_Baby_Names.csv │ ├── dist.female.first.txt │ ├── dist.male.first.txt │ ├── firstnames.xlsx │ └── job_prestige_score.xlsx ├── templates ├── new_templates - Age.csv ├── new_templates - Disability_status.csv ├── new_templates - Filler_items.csv ├── new_templates - Gender_identity.csv ├── new_templates - Nationality.csv ├── new_templates - Physical_appearance.csv ├── new_templates - Race_ethnicity.csv ├── new_templates - Race_x_SES.csv ├── new_templates - Race_x_gender.csv ├── new_templates - Religion.csv ├── new_templates - SES.csv ├── new_templates - Sexual_orientation.csv ├── vocabulary.csv └── vocabulary_proper_names.csv └── utils.py /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/LICENSE -------------------------------------------------------------------------------- /QA_bias_benchmark.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/QA_bias_benchmark.pdf -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/README.md -------------------------------------------------------------------------------- /analysis_scripts/BBQ_calculate_bias_score.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/analysis_scripts/BBQ_calculate_bias_score.R -------------------------------------------------------------------------------- /analysis_scripts/BBQ_results_metadata.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/analysis_scripts/BBQ_results_metadata.R -------------------------------------------------------------------------------- /analysis_scripts/additional_metadata.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/analysis_scripts/additional_metadata.csv -------------------------------------------------------------------------------- /data/Age.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/data/Age.jsonl -------------------------------------------------------------------------------- /data/Disability_status.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/data/Disability_status.jsonl -------------------------------------------------------------------------------- /data/Gender_identity.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/data/Gender_identity.jsonl -------------------------------------------------------------------------------- /data/Nationality.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/data/Nationality.jsonl -------------------------------------------------------------------------------- /data/Physical_appearance.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/data/Physical_appearance.jsonl -------------------------------------------------------------------------------- /data/Race_ethnicity.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/data/Race_ethnicity.jsonl -------------------------------------------------------------------------------- /data/Race_x_SES.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/data/Race_x_SES.jsonl -------------------------------------------------------------------------------- /data/Race_x_gender.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/data/Race_x_gender.jsonl -------------------------------------------------------------------------------- /data/Religion.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/data/Religion.jsonl -------------------------------------------------------------------------------- /data/SES.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/data/SES.jsonl -------------------------------------------------------------------------------- /data/Sexual_orientation.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/data/Sexual_orientation.jsonl -------------------------------------------------------------------------------- /generate_from_template_all_categories.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/generate_from_template_all_categories.py -------------------------------------------------------------------------------- /generate_from_template_intersectional_cats.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/generate_from_template_intersectional_cats.py -------------------------------------------------------------------------------- /results/RoBERTa_and_DeBERTaV3/df_bbq.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/RoBERTa_and_DeBERTaV3/df_bbq.csv -------------------------------------------------------------------------------- /results/UnifiedQA/preds_Age.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/UnifiedQA/preds_Age.jsonl -------------------------------------------------------------------------------- /results/UnifiedQA/preds_Disability_status.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/UnifiedQA/preds_Disability_status.jsonl -------------------------------------------------------------------------------- /results/UnifiedQA/preds_Gender_identity.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/UnifiedQA/preds_Gender_identity.jsonl -------------------------------------------------------------------------------- /results/UnifiedQA/preds_Nationality.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/UnifiedQA/preds_Nationality.jsonl -------------------------------------------------------------------------------- /results/UnifiedQA/preds_Physical_appearance.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/UnifiedQA/preds_Physical_appearance.jsonl -------------------------------------------------------------------------------- /results/UnifiedQA/preds_Race_ethnicity.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/UnifiedQA/preds_Race_ethnicity.jsonl -------------------------------------------------------------------------------- /results/UnifiedQA/preds_Race_x_SES.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/UnifiedQA/preds_Race_x_SES.jsonl -------------------------------------------------------------------------------- /results/UnifiedQA/preds_Race_x_gender.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/UnifiedQA/preds_Race_x_gender.jsonl -------------------------------------------------------------------------------- /results/UnifiedQA/preds_Religion.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/UnifiedQA/preds_Religion.jsonl -------------------------------------------------------------------------------- /results/UnifiedQA/preds_SES.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/UnifiedQA/preds_SES.jsonl -------------------------------------------------------------------------------- /results/UnifiedQA/preds_Sexual_orientation.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/results/UnifiedQA/preds_Sexual_orientation.jsonl -------------------------------------------------------------------------------- /supplemental/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/.gitignore -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/qual_batch.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/qual_batch.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/validations_finalval.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/validations_finalval.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/validations_round1.1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/validations_round1.1.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/validations_round1.2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/validations_round1.2.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/validations_round1.3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/validations_round1.3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/validations_round1.4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/validations_round1.4.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/validations_round2.1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/validations_round2.1.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/validations_round2.2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/validations_round2.2.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/validations_round2.3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/validations_round2.3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/validations_round3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/validations_round3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/validations_round5.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/validations_round5.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/data/validations_round6.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/data/validations_round6.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/finalval/donotupload_full_data_for_mturk_full_finalval.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/finalval/donotupload_full_data_for_mturk_full_finalval.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/finalval/full_data_for_mturk_1_finalval.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/finalval/full_data_for_mturk_1_finalval.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/finalval/full_data_for_mturk_2_finalval.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/finalval/full_data_for_mturk_2_finalval.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/finalval/full_data_for_mturk_3_finalval.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/finalval/full_data_for_mturk_3_finalval.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round1/UPLOADED_full_data_for_mturk_1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round1/UPLOADED_full_data_for_mturk_1.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round1/UPLOADED_full_data_for_mturk_2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round1/UPLOADED_full_data_for_mturk_2.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round1/UPLOADED_full_data_for_mturk_3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round1/UPLOADED_full_data_for_mturk_3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round1/UPLOADED_full_data_for_mturk_4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round1/UPLOADED_full_data_for_mturk_4.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round1/donotupload_full_data_for_mturk_full.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round1/donotupload_full_data_for_mturk_full.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round2/UPLOADED_full_data_for_mturk_1_round2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round2/UPLOADED_full_data_for_mturk_1_round2.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round2/UPLOADED_full_data_for_mturk_2_round2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round2/UPLOADED_full_data_for_mturk_2_round2.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round2/UPLOADED_full_data_for_mturk_3_round2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round2/UPLOADED_full_data_for_mturk_3_round2.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round2/donotupload_full_data_for_mturk_full_round2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round2/donotupload_full_data_for_mturk_full_round2.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round3/donotupload_full_data_for_mturk_full_round3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round3/donotupload_full_data_for_mturk_full_round3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round3/full_data_for_mturk_1_round3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round3/full_data_for_mturk_1_round3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round3/full_data_for_mturk_2_round3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round3/full_data_for_mturk_2_round3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round4/donotupload_full_data_for_mturk_full_round4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round4/donotupload_full_data_for_mturk_full_round4.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round4/full_data_for_mturk_1_round4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round4/full_data_for_mturk_1_round4.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round5/donotupload_full_data_for_mturk_full_round5.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round5/donotupload_full_data_for_mturk_full_round5.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round5/full_data_for_mturk_1_round5.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round5/full_data_for_mturk_1_round5.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round5/full_data_for_mturk_2_round5.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round5/full_data_for_mturk_2_round5.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round6/donotupload_full_data_for_mturk_full_round6.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round6/donotupload_full_data_for_mturk_full_round6.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/for_upload/round6/full_data_for_mturk_1_round6.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/for_upload/round6/full_data_for_mturk_1_round6.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/included_workers.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/included_workers.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/qualification_HIT_template.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/qualification_HIT_template.html -------------------------------------------------------------------------------- /supplemental/MTurk_validation/results_analysis/results_round3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/results_analysis/results_round3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/results_analysis/results_round5.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/results_analysis/results_round5.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/results_analysis/results_round6.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/results_analysis/results_round6.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/results_analysis/round1/results_round1.1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/results_analysis/round1/results_round1.1.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/results_analysis/round1/results_round1.2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/results_analysis/round1/results_round1.2.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/results_analysis/round1/results_round1.3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/results_analysis/round1/results_round1.3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/results_analysis/round1/results_round1.4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/results_analysis/round1/results_round1.4.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/results_analysis/round2/results_round2.1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/results_analysis/round2/results_round2.1.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/results_analysis/round2/results_round2.2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/results_analysis/round2/results_round2.2.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/results_analysis/round2/results_round2.3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/results_analysis/round2/results_round2.3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_finalval.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_finalval.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_round2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_round2.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_round3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_round3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_round4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_round4.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_round5.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_round5.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_round6.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_round6.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_uid.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_uid.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_uid_finalval.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_uid_finalval.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_uid_round2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_uid_round2.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_uid_round3.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_uid_round3.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_uid_round4.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_uid_round4.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_uid_round5.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_uid_round5.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/all_examples_uid_round6.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/all_examples_uid_round6.csv -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/analyze_qual_screener.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/analyze_qual_screener.R -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/analyze_validations.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/analyze_validations.R -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/anonymize_and_store.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/anonymize_and_store.R -------------------------------------------------------------------------------- /supplemental/MTurk_validation/scripts/create_validation_HITs.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/scripts/create_validation_HITs.R -------------------------------------------------------------------------------- /supplemental/MTurk_validation/validation_HIT_template.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/MTurk_validation/validation_HIT_template.html -------------------------------------------------------------------------------- /supplemental/NYC_babyname_Data.R: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/NYC_babyname_Data.R -------------------------------------------------------------------------------- /supplemental/additional_metadata.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/additional_metadata.csv -------------------------------------------------------------------------------- /supplemental/name_job_data/English First Names for Chinese Americans _ Social Science Statistics Blog.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/name_job_data/English First Names for Chinese Americans _ Social Science Statistics Blog.html -------------------------------------------------------------------------------- /supplemental/name_job_data/Names_2010Census_Top1000.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/name_job_data/Names_2010Census_Top1000.xlsx -------------------------------------------------------------------------------- /supplemental/name_job_data/Popular_Baby_Names.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/name_job_data/Popular_Baby_Names.csv -------------------------------------------------------------------------------- /supplemental/name_job_data/dist.female.first.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/name_job_data/dist.female.first.txt -------------------------------------------------------------------------------- /supplemental/name_job_data/dist.male.first.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/name_job_data/dist.male.first.txt -------------------------------------------------------------------------------- /supplemental/name_job_data/firstnames.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/name_job_data/firstnames.xlsx -------------------------------------------------------------------------------- /supplemental/name_job_data/job_prestige_score.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/supplemental/name_job_data/job_prestige_score.xlsx -------------------------------------------------------------------------------- /templates/new_templates - Age.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - Age.csv -------------------------------------------------------------------------------- /templates/new_templates - Disability_status.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - Disability_status.csv -------------------------------------------------------------------------------- /templates/new_templates - Filler_items.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - Filler_items.csv -------------------------------------------------------------------------------- /templates/new_templates - Gender_identity.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - Gender_identity.csv -------------------------------------------------------------------------------- /templates/new_templates - Nationality.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - Nationality.csv -------------------------------------------------------------------------------- /templates/new_templates - Physical_appearance.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - Physical_appearance.csv -------------------------------------------------------------------------------- /templates/new_templates - Race_ethnicity.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - Race_ethnicity.csv -------------------------------------------------------------------------------- /templates/new_templates - Race_x_SES.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - Race_x_SES.csv -------------------------------------------------------------------------------- /templates/new_templates - Race_x_gender.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - Race_x_gender.csv -------------------------------------------------------------------------------- /templates/new_templates - Religion.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - Religion.csv -------------------------------------------------------------------------------- /templates/new_templates - SES.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - SES.csv -------------------------------------------------------------------------------- /templates/new_templates - Sexual_orientation.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/new_templates - Sexual_orientation.csv -------------------------------------------------------------------------------- /templates/vocabulary.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/vocabulary.csv -------------------------------------------------------------------------------- /templates/vocabulary_proper_names.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/templates/vocabulary_proper_names.csv -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nyu-mll/BBQ/HEAD/utils.py --------------------------------------------------------------------------------