├── LICENSE ├── README.md ├── analysis.py ├── lm ├── __init__.py ├── demo.py └── predict.py ├── logo.png ├── qa ├── __init__.py ├── bert_encoder.py ├── boundary_loss.py ├── data.py ├── demo.py ├── eval.py ├── linear_classifier.py ├── optimizer.py ├── pipeline.py ├── predict.py ├── preprocess.py └── train.py ├── qa_hf ├── __init__.py └── predict.py ├── requirements.txt ├── scripts ├── aggregate_predictions.sh ├── download_predictions.sh ├── generate_lm_predictions.sh ├── generate_predictions.sh ├── generate_qa_predictions_hf.sh └── generate_questions.sh ├── templates ├── __init__.py ├── generate_underspecified_templates.py ├── lists.py └── underspecified_templates.py ├── utils ├── __init__.py ├── convert_hdf5_to_hf.py ├── extract.py ├── holder.py └── util.py ├── visualization ├── Plot_curves.ipynb ├── Plot_ranks.ipynb └── dump_bipartite_vis.py └── word_lists ├── activities ├── biased_country ├── biased_ethnicity ├── biased_religion ├── occupation_demo └── occupation_rev1 ├── fillers └── newsqa ├── nouns └── subjects │ ├── country │ ├── country_bert │ ├── country_demo │ ├── country_roberta │ ├── ethnicity │ ├── ethnicity_bert │ ├── ethnicity_roberta │ ├── female │ ├── female_bert │ ├── female_demo │ ├── female_roberta │ ├── male │ ├── male_bert │ ├── male_demo │ ├── male_roberta │ ├── religion │ ├── religion_bert │ └── religion_roberta └── slots ├── country_demo ├── country_noact ├── country_noact_lm ├── ethnicity_noact ├── ethnicity_noact_lm ├── gender_demo ├── gender_noact ├── gender_noact_lm ├── religion_noact └── religion_noact_lm /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/README.md -------------------------------------------------------------------------------- /analysis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/analysis.py -------------------------------------------------------------------------------- /lm/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm/demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/lm/demo.py -------------------------------------------------------------------------------- /lm/predict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/lm/predict.py -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/logo.png -------------------------------------------------------------------------------- /qa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /qa/bert_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa/bert_encoder.py -------------------------------------------------------------------------------- /qa/boundary_loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa/boundary_loss.py -------------------------------------------------------------------------------- /qa/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa/data.py -------------------------------------------------------------------------------- /qa/demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa/demo.py -------------------------------------------------------------------------------- /qa/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa/eval.py -------------------------------------------------------------------------------- /qa/linear_classifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa/linear_classifier.py -------------------------------------------------------------------------------- /qa/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa/optimizer.py -------------------------------------------------------------------------------- /qa/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa/pipeline.py -------------------------------------------------------------------------------- /qa/predict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa/predict.py -------------------------------------------------------------------------------- /qa/preprocess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa/preprocess.py -------------------------------------------------------------------------------- /qa/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa/train.py -------------------------------------------------------------------------------- /qa_hf/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /qa_hf/predict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/qa_hf/predict.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/aggregate_predictions.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/scripts/aggregate_predictions.sh -------------------------------------------------------------------------------- /scripts/download_predictions.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/scripts/download_predictions.sh -------------------------------------------------------------------------------- /scripts/generate_lm_predictions.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/scripts/generate_lm_predictions.sh -------------------------------------------------------------------------------- /scripts/generate_predictions.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/scripts/generate_predictions.sh -------------------------------------------------------------------------------- /scripts/generate_qa_predictions_hf.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/scripts/generate_qa_predictions_hf.sh -------------------------------------------------------------------------------- /scripts/generate_questions.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/scripts/generate_questions.sh -------------------------------------------------------------------------------- /templates/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /templates/generate_underspecified_templates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/templates/generate_underspecified_templates.py -------------------------------------------------------------------------------- /templates/lists.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/templates/lists.py -------------------------------------------------------------------------------- /templates/underspecified_templates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/templates/underspecified_templates.py -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/convert_hdf5_to_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/utils/convert_hdf5_to_hf.py -------------------------------------------------------------------------------- /utils/extract.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/utils/extract.py -------------------------------------------------------------------------------- /utils/holder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/utils/holder.py -------------------------------------------------------------------------------- /utils/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/utils/util.py -------------------------------------------------------------------------------- /visualization/Plot_curves.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/visualization/Plot_curves.ipynb -------------------------------------------------------------------------------- /visualization/Plot_ranks.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/visualization/Plot_ranks.ipynb -------------------------------------------------------------------------------- /visualization/dump_bipartite_vis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/visualization/dump_bipartite_vis.py -------------------------------------------------------------------------------- /word_lists/activities/biased_country: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/activities/biased_country -------------------------------------------------------------------------------- /word_lists/activities/biased_ethnicity: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/activities/biased_ethnicity -------------------------------------------------------------------------------- /word_lists/activities/biased_religion: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/activities/biased_religion -------------------------------------------------------------------------------- /word_lists/activities/occupation_demo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/activities/occupation_demo -------------------------------------------------------------------------------- /word_lists/activities/occupation_rev1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/activities/occupation_rev1 -------------------------------------------------------------------------------- /word_lists/fillers/newsqa: -------------------------------------------------------------------------------- 1 | [aug] yes no [text] (CNN) -- -------------------------------------------------------------------------------- /word_lists/nouns/subjects/country: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/country -------------------------------------------------------------------------------- /word_lists/nouns/subjects/country_bert: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/country_bert -------------------------------------------------------------------------------- /word_lists/nouns/subjects/country_demo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/country_demo -------------------------------------------------------------------------------- /word_lists/nouns/subjects/country_roberta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/country_roberta -------------------------------------------------------------------------------- /word_lists/nouns/subjects/ethnicity: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/ethnicity -------------------------------------------------------------------------------- /word_lists/nouns/subjects/ethnicity_bert: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/ethnicity_bert -------------------------------------------------------------------------------- /word_lists/nouns/subjects/ethnicity_roberta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/ethnicity_roberta -------------------------------------------------------------------------------- /word_lists/nouns/subjects/female: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/female -------------------------------------------------------------------------------- /word_lists/nouns/subjects/female_bert: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/female_bert -------------------------------------------------------------------------------- /word_lists/nouns/subjects/female_demo: -------------------------------------------------------------------------------- 1 | [subj] female | Alice -------------------------------------------------------------------------------- /word_lists/nouns/subjects/female_roberta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/female_roberta -------------------------------------------------------------------------------- /word_lists/nouns/subjects/male: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/male -------------------------------------------------------------------------------- /word_lists/nouns/subjects/male_bert: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/male_bert -------------------------------------------------------------------------------- /word_lists/nouns/subjects/male_demo: -------------------------------------------------------------------------------- 1 | [subj] male | Justin -------------------------------------------------------------------------------- /word_lists/nouns/subjects/male_roberta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/male_roberta -------------------------------------------------------------------------------- /word_lists/nouns/subjects/religion: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/religion -------------------------------------------------------------------------------- /word_lists/nouns/subjects/religion_bert: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/religion_bert -------------------------------------------------------------------------------- /word_lists/nouns/subjects/religion_roberta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/nouns/subjects/religion_roberta -------------------------------------------------------------------------------- /word_lists/slots/country_demo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/slots/country_demo -------------------------------------------------------------------------------- /word_lists/slots/country_noact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/slots/country_noact -------------------------------------------------------------------------------- /word_lists/slots/country_noact_lm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/slots/country_noact_lm -------------------------------------------------------------------------------- /word_lists/slots/ethnicity_noact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/slots/ethnicity_noact -------------------------------------------------------------------------------- /word_lists/slots/ethnicity_noact_lm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/slots/ethnicity_noact_lm -------------------------------------------------------------------------------- /word_lists/slots/gender_demo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/slots/gender_demo -------------------------------------------------------------------------------- /word_lists/slots/gender_noact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/slots/gender_noact -------------------------------------------------------------------------------- /word_lists/slots/gender_noact_lm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/slots/gender_noact_lm -------------------------------------------------------------------------------- /word_lists/slots/religion_noact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/slots/religion_noact -------------------------------------------------------------------------------- /word_lists/slots/religion_noact_lm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allenai/unqover/HEAD/word_lists/slots/religion_noact_lm --------------------------------------------------------------------------------