├── .gitignore ├── LICENSE ├── README.md ├── data ├── bigbench_ipa_test.txt ├── bigbench_strategyqa_test.txt ├── bigbench_tense_test.txt ├── cnn_test.txt ├── czech_restaurants_test.txt ├── dm_test.txt ├── e2e_nlg_clean_test.txt ├── hupd_test.txt ├── webnlg_en_test.txt ├── wmt_de-en_test.txt ├── wmt_ro-en_test.txt ├── wmt_tr-en_test.txt └── xsum_test.txt ├── fables ├── crowd-sampling.png ├── main-results.png └── mbrd-equation.png ├── ground_truth ├── bigbench_ipa.json ├── bigbench_strategyqa.json ├── bigbench_tense.json ├── cnn.json ├── czech_restaurants.json ├── dm.json ├── e2e_nlg_clean.json ├── webnlg_en.json ├── wmt_de-en.json ├── wmt_ro-en.json ├── wmt_tr-en.json └── xsum.json ├── outputs ├── .DS_Store ├── bigbench │ ├── bigbench_ipa_codex_bertscore_N16.json │ ├── bigbench_ipa_codex_bleurt_N16.json │ ├── bigbench_ipa_codex_choose_first_N16.json │ ├── bigbench_ipa_codex_lcs_N16.json │ ├── bigbench_ipa_codex_majority_N16.json │ ├── bigbench_ipa_codex_original_N16.json │ ├── bigbench_ipa_codex_random_N16.json │ ├── bigbench_strategyqa_codex_bertscore_N16.json │ ├── bigbench_strategyqa_codex_bleurt_N16.json │ ├── bigbench_strategyqa_codex_choose_first_N16.json │ ├── bigbench_strategyqa_codex_lcs_N16.json │ ├── bigbench_strategyqa_codex_majority_N16.json │ ├── bigbench_strategyqa_codex_original_N16.json │ ├── bigbench_strategyqa_codex_random_N16.json │ ├── bigbench_tense_codex_bertscore_N16.json │ ├── bigbench_tense_codex_bleurt_N16.json │ ├── bigbench_tense_codex_choose_first_N16.json │ ├── bigbench_tense_codex_lcs_N16.json │ ├── bigbench_tense_codex_majority_N16.json │ ├── bigbench_tense_codex_original_N16.json │ └── bigbench_tense_codex_random_N16.json ├── data2text │ ├── czech_restaurants_codex_bertscore_N16.json │ ├── czech_restaurants_codex_bleurt_N16.json │ ├── czech_restaurants_codex_choose_first_N16.json │ ├── czech_restaurants_codex_lcs_N16.json │ ├── czech_restaurants_codex_majority_N16.json │ ├── czech_restaurants_codex_original_N16.json │ ├── czech_restaurants_codex_random_N16.json │ ├── e2e_nlg_clean_codex_bertscore_N16.json │ ├── e2e_nlg_clean_codex_bleurt_N16.json │ ├── e2e_nlg_clean_codex_choose_first_N16.json │ ├── e2e_nlg_clean_codex_lcs_N16.json │ ├── e2e_nlg_clean_codex_majority_N16.json │ ├── e2e_nlg_clean_codex_original_N16.json │ ├── e2e_nlg_clean_codex_random_N16.json │ ├── webnlg_en_codex_bertscore_N16.json │ ├── webnlg_en_codex_bleurt_N16.json │ ├── webnlg_en_codex_choose_first_N16.json │ ├── webnlg_en_codex_lcs_N16.json │ ├── webnlg_en_codex_majority_N16.json │ ├── webnlg_en_codex_original_N16.json │ └── webnlg_en_codex_random_N16.json ├── summarization │ ├── cnn_codex_bertscore_N16.json │ ├── cnn_codex_bleurt_N16.json │ ├── cnn_codex_choose_first_N16.json │ ├── cnn_codex_lcs_N16.json │ ├── cnn_codex_majority_N16.json │ ├── cnn_codex_original_N16.json │ ├── cnn_codex_random_N16.json │ ├── dm_codex_bertscore_N16.json │ ├── dm_codex_bleurt_N16.json │ ├── dm_codex_choose_first_N16.json │ ├── dm_codex_lcs_N16.json │ ├── dm_codex_majority_N16.json │ ├── dm_codex_original_N16.json │ ├── dm_codex_random_N16.json │ ├── xsum_codex_bertscore_N16.json │ ├── xsum_codex_bleurt_N16.json │ ├── xsum_codex_choose_first_N16.json │ ├── xsum_codex_lcs_N16.json │ ├── xsum_codex_majority_N16.json │ ├── xsum_codex_original_N16.json │ └── xsum_codex_random_N16.json └── wmt16 │ ├── wmt_de2en_codex_bertscore_N16.json │ ├── wmt_de2en_codex_bleurt_N16.json │ ├── wmt_de2en_codex_choose_first_N16.json │ ├── wmt_de2en_codex_lcs_N16.json │ ├── wmt_de2en_codex_majority_N16.json │ ├── wmt_de2en_codex_original_N16.json │ ├── wmt_de2en_codex_random_N16.json │ ├── wmt_ro2en_codex_bertscore_N16.json │ ├── wmt_ro2en_codex_bleurt_N16.json │ ├── wmt_ro2en_codex_choose_first_N16.json │ ├── wmt_ro2en_codex_lcs_N16.json │ ├── wmt_ro2en_codex_majority_N16.json │ ├── wmt_ro2en_codex_original_N16.json │ ├── wmt_ro2en_codex_random_N16.json │ ├── wmt_tr2en_codex_bertscore_N16.json │ ├── wmt_tr2en_codex_bleurt_N16.json │ ├── wmt_tr2en_codex_choose_first_N16.json │ ├── wmt_tr2en_codex_lcs_N16.json │ ├── wmt_tr2en_codex_majority_N16.json │ ├── wmt_tr2en_codex_original_N16.json │ └── wmt_tr2en_codex_random_N16.json ├── prompts ├── bigbench_ipa_fs.txt ├── bigbench_strategyqa_fs.txt ├── bigbench_tense_fs.txt ├── cnn_fs.txt ├── czech_restaurants_fs.txt ├── dm_fs.txt ├── e2e_nlg_clean_fs.txt ├── gyafc_fs.txt ├── hupd_fs.txt ├── shakespeare_fs.txt ├── webnlg_en_fs.txt ├── wmt_de-en_fs.txt ├── wmt_ro-en_fs.txt ├── wmt_tr-en_fs.txt ├── xsum_fs.txt └── yelp_fs.txt └── run_mbrd.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/README.md -------------------------------------------------------------------------------- /data/bigbench_ipa_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/bigbench_ipa_test.txt -------------------------------------------------------------------------------- /data/bigbench_strategyqa_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/bigbench_strategyqa_test.txt -------------------------------------------------------------------------------- /data/bigbench_tense_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/bigbench_tense_test.txt -------------------------------------------------------------------------------- /data/cnn_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/cnn_test.txt -------------------------------------------------------------------------------- /data/czech_restaurants_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/czech_restaurants_test.txt -------------------------------------------------------------------------------- /data/dm_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/dm_test.txt -------------------------------------------------------------------------------- /data/e2e_nlg_clean_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/e2e_nlg_clean_test.txt -------------------------------------------------------------------------------- /data/hupd_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/hupd_test.txt -------------------------------------------------------------------------------- /data/webnlg_en_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/webnlg_en_test.txt -------------------------------------------------------------------------------- /data/wmt_de-en_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/wmt_de-en_test.txt -------------------------------------------------------------------------------- /data/wmt_ro-en_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/wmt_ro-en_test.txt -------------------------------------------------------------------------------- /data/wmt_tr-en_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/wmt_tr-en_test.txt -------------------------------------------------------------------------------- /data/xsum_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/data/xsum_test.txt -------------------------------------------------------------------------------- /fables/crowd-sampling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/fables/crowd-sampling.png -------------------------------------------------------------------------------- /fables/main-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/fables/main-results.png -------------------------------------------------------------------------------- /fables/mbrd-equation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/fables/mbrd-equation.png -------------------------------------------------------------------------------- /ground_truth/bigbench_ipa.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/bigbench_ipa.json -------------------------------------------------------------------------------- /ground_truth/bigbench_strategyqa.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/bigbench_strategyqa.json -------------------------------------------------------------------------------- /ground_truth/bigbench_tense.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/bigbench_tense.json -------------------------------------------------------------------------------- /ground_truth/cnn.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/cnn.json -------------------------------------------------------------------------------- /ground_truth/czech_restaurants.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/czech_restaurants.json -------------------------------------------------------------------------------- /ground_truth/dm.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/dm.json -------------------------------------------------------------------------------- /ground_truth/e2e_nlg_clean.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/e2e_nlg_clean.json -------------------------------------------------------------------------------- /ground_truth/webnlg_en.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/webnlg_en.json -------------------------------------------------------------------------------- /ground_truth/wmt_de-en.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/wmt_de-en.json -------------------------------------------------------------------------------- /ground_truth/wmt_ro-en.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/wmt_ro-en.json -------------------------------------------------------------------------------- /ground_truth/wmt_tr-en.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/wmt_tr-en.json -------------------------------------------------------------------------------- /ground_truth/xsum.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/ground_truth/xsum.json -------------------------------------------------------------------------------- /outputs/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/.DS_Store -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_ipa_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_ipa_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_ipa_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_ipa_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_ipa_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_ipa_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_ipa_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_ipa_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_ipa_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_ipa_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_ipa_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_ipa_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_ipa_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_ipa_codex_random_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_strategyqa_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_strategyqa_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_strategyqa_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_strategyqa_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_strategyqa_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_strategyqa_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_strategyqa_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_strategyqa_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_strategyqa_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_strategyqa_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_strategyqa_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_strategyqa_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_strategyqa_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_strategyqa_codex_random_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_tense_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_tense_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_tense_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_tense_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_tense_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_tense_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_tense_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_tense_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_tense_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_tense_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_tense_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_tense_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/bigbench/bigbench_tense_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/bigbench/bigbench_tense_codex_random_N16.json -------------------------------------------------------------------------------- /outputs/data2text/czech_restaurants_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/czech_restaurants_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/data2text/czech_restaurants_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/czech_restaurants_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/data2text/czech_restaurants_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/czech_restaurants_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/data2text/czech_restaurants_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/czech_restaurants_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/data2text/czech_restaurants_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/czech_restaurants_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/data2text/czech_restaurants_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/czech_restaurants_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/data2text/czech_restaurants_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/czech_restaurants_codex_random_N16.json -------------------------------------------------------------------------------- /outputs/data2text/e2e_nlg_clean_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/e2e_nlg_clean_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/data2text/e2e_nlg_clean_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/e2e_nlg_clean_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/data2text/e2e_nlg_clean_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/e2e_nlg_clean_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/data2text/e2e_nlg_clean_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/e2e_nlg_clean_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/data2text/e2e_nlg_clean_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/e2e_nlg_clean_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/data2text/e2e_nlg_clean_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/e2e_nlg_clean_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/data2text/e2e_nlg_clean_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/e2e_nlg_clean_codex_random_N16.json -------------------------------------------------------------------------------- /outputs/data2text/webnlg_en_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/webnlg_en_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/data2text/webnlg_en_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/webnlg_en_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/data2text/webnlg_en_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/webnlg_en_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/data2text/webnlg_en_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/webnlg_en_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/data2text/webnlg_en_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/webnlg_en_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/data2text/webnlg_en_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/webnlg_en_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/data2text/webnlg_en_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/data2text/webnlg_en_codex_random_N16.json -------------------------------------------------------------------------------- /outputs/summarization/cnn_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/cnn_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/summarization/cnn_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/cnn_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/summarization/cnn_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/cnn_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/summarization/cnn_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/cnn_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/summarization/cnn_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/cnn_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/summarization/cnn_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/cnn_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/summarization/cnn_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/cnn_codex_random_N16.json -------------------------------------------------------------------------------- /outputs/summarization/dm_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/dm_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/summarization/dm_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/dm_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/summarization/dm_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/dm_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/summarization/dm_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/dm_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/summarization/dm_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/dm_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/summarization/dm_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/dm_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/summarization/dm_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/dm_codex_random_N16.json -------------------------------------------------------------------------------- /outputs/summarization/xsum_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/xsum_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/summarization/xsum_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/xsum_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/summarization/xsum_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/xsum_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/summarization/xsum_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/xsum_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/summarization/xsum_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/xsum_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/summarization/xsum_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/xsum_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/summarization/xsum_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/summarization/xsum_codex_random_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_de2en_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_de2en_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_de2en_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_de2en_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_de2en_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_de2en_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_de2en_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_de2en_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_de2en_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_de2en_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_de2en_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_de2en_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_de2en_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_de2en_codex_random_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_ro2en_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_ro2en_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_ro2en_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_ro2en_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_ro2en_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_ro2en_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_ro2en_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_ro2en_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_ro2en_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_ro2en_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_ro2en_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_ro2en_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_ro2en_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_ro2en_codex_random_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_tr2en_codex_bertscore_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_tr2en_codex_bertscore_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_tr2en_codex_bleurt_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_tr2en_codex_bleurt_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_tr2en_codex_choose_first_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_tr2en_codex_choose_first_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_tr2en_codex_lcs_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_tr2en_codex_lcs_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_tr2en_codex_majority_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_tr2en_codex_majority_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_tr2en_codex_original_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_tr2en_codex_original_N16.json -------------------------------------------------------------------------------- /outputs/wmt16/wmt_tr2en_codex_random_N16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/outputs/wmt16/wmt_tr2en_codex_random_N16.json -------------------------------------------------------------------------------- /prompts/bigbench_ipa_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/bigbench_ipa_fs.txt -------------------------------------------------------------------------------- /prompts/bigbench_strategyqa_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/bigbench_strategyqa_fs.txt -------------------------------------------------------------------------------- /prompts/bigbench_tense_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/bigbench_tense_fs.txt -------------------------------------------------------------------------------- /prompts/cnn_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/cnn_fs.txt -------------------------------------------------------------------------------- /prompts/czech_restaurants_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/czech_restaurants_fs.txt -------------------------------------------------------------------------------- /prompts/dm_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/dm_fs.txt -------------------------------------------------------------------------------- /prompts/e2e_nlg_clean_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/e2e_nlg_clean_fs.txt -------------------------------------------------------------------------------- /prompts/gyafc_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/gyafc_fs.txt -------------------------------------------------------------------------------- /prompts/hupd_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/hupd_fs.txt -------------------------------------------------------------------------------- /prompts/shakespeare_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/shakespeare_fs.txt -------------------------------------------------------------------------------- /prompts/webnlg_en_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/webnlg_en_fs.txt -------------------------------------------------------------------------------- /prompts/wmt_de-en_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/wmt_de-en_fs.txt -------------------------------------------------------------------------------- /prompts/wmt_ro-en_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/wmt_ro-en_fs.txt -------------------------------------------------------------------------------- /prompts/wmt_tr-en_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/wmt_tr-en_fs.txt -------------------------------------------------------------------------------- /prompts/xsum_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/xsum_fs.txt -------------------------------------------------------------------------------- /prompts/yelp_fs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/prompts/yelp_fs.txt -------------------------------------------------------------------------------- /run_mbrd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/suzgunmirac/crowd-sampling/HEAD/run_mbrd.py --------------------------------------------------------------------------------