├── .gitignore ├── README.md ├── cached └── .gitkeep ├── checkpoints └── .gitkeep ├── common ├── config.py ├── custom_squad_feature.py ├── dataset_utils.py ├── interp_utils.py ├── qa_metrics.py └── utils.py ├── docs ├── example_expl.png └── example_latattr.jpg ├── eval_hotpot_exp ├── bridge_testers.py ├── eval_bridge.py ├── eval_yesno.py ├── utils.py └── yesno_testers.py ├── expl_methods ├── latattr_models.py ├── perturb_models.py └── tokig_models.py ├── exps └── .gitkeep ├── hotpot_counterfactuals ├── bridge │ ├── 5a7166395542994082a3e814.json │ ├── 5a722b8655429971e9dc9329.json │ ├── 5a74106b55429979e288289e.json │ ├── 5a7571135542992d0ec05f98.json │ ├── 5a75e05c55429976ec32bc5f.json │ ├── 5a760ab65542994ccc918697.json │ ├── 5a77152355429966f1a36c2e.json │ ├── 5a77724455429972597f153e.json │ ├── 5a79311755429970f5fffe67.json │ ├── 5a7d54165542995f4f402256.json │ ├── 5a80721b554299485f5985ef.json │ ├── 5a80840f554299485f59863b.json │ ├── 5a828c8355429966c78a6a50.json │ ├── 5a85b2d95542997b5ce40028.json │ ├── 5a85ea095542994775f606a8.json │ ├── 5a85fb085542994775f606de.json │ ├── 5a877e5d5542993e715abf7d.json │ ├── 5a87ab905542996e4f3088c1.json │ ├── 5a87c13f5542996e4f30890c.json │ ├── 5a8a43eb5542996c9b8d5e82.json │ ├── 5a8b20335542996c9b8d5fb3.json │ ├── 5a8c7595554299585d9e36b6.json │ ├── 5a8e1027554299653c1aa15f.json │ ├── 5a8e3ea95542995a26add48d.json │ ├── 5a8ef2a75542995a26add583.json │ ├── 5a8f4c8d554299458435d5a3.json │ ├── 5ab2d3df554299194fa9352c.json │ ├── 5ab3b0bf5542992ade7c6e39.json │ ├── 5ab3e45655429976abd1bcd4.json │ ├── 5ab6d09255429954757d337d.json │ ├── 5ab84bf555429916710eb01f.json │ ├── 5ab859a955429934fafe6d7b.json │ ├── 5ab96ab755429970cfb8eacd.json │ ├── 5aba7cfe554299232ef4a2fd.json │ ├── 5abbf698554299114383a0b5.json │ ├── 5abd94525542992ac4f382d2.json │ ├── 5abf63f15542997ec76fd3ea.json │ ├── 5ac23ff0554299636651994d.json │ ├── 5ac2acff55429921a00ab02b.json │ ├── 5add61d65542995b365fab21.json │ ├── 5adddccd5542997dc7907069.json │ ├── 5adf37a95542995ec70e8f97.json │ ├── 5ae0361155429925eb1afc2c.json │ ├── 5ae0d4c9554299603e418468.json │ ├── 5ae22b8d554299234fd0440f.json │ ├── 5ae32e125542991a06ce9946.json │ ├── 5ae33c4d5542992f92d82262.json │ ├── 5ae5aba0554299546bf82f17.json │ ├── 5ae7a8175542993210983ed8.json │ └── 5ae7ba7a5542993210983f12.json └── yesno │ ├── 5a7e1cad55429965cec5ea75.json │ ├── 5a7ec61a55429934daa2fc5f.json │ ├── 5a81c90a5542990a1d231eb4.json │ ├── 5a85a2845542997175ce1fe1.json │ ├── 5a85ebe75542996432c5712a.json │ ├── 5a864d9e5542994775f6073d.json │ ├── 5a866c04554299211dda2b25.json │ ├── 5a8762205542996e4f3087b8.json │ ├── 5a877dd65542993e715abf79.json │ ├── 5a89e2785542992e4fca843d.json │ ├── 5a8a58f355429930ff3c0da3.json │ ├── 5a8b57f25542995d1e6f1371.json │ ├── 5a8c620c554299240d9c2141.json │ ├── 5a8c87aa554299653c1aa0ac.json │ ├── 5a8cdcb0554299441c6b9f33.json │ ├── 5a8cec13554299441c6b9f7d.json │ ├── 5a8f64605542992414482aaa.json │ ├── 5a8f876e5542997ba9cb32a1.json │ ├── 5a8fb3af5542997ba9cb32ee.json │ ├── 5ab52bd1554299637185c4ee.json │ ├── 5ab681ea55429953192ad2e0.json │ ├── 5ab6f222554299110f219a78.json │ ├── 5ab6f9335542991d322236fc.json │ ├── 5ab8337a55429919ba4e225f.json │ ├── 5ab8ee035542991b5579f04b.json │ ├── 5abb8afe5542993f40c73b1e.json │ ├── 5abbd3ac55429931dba1458b.json │ ├── 5abbdaac5542992ccd8e7fc0.json │ ├── 5abd72475542993062266c9a.json │ ├── 5abd7aba55429924427fcfed.json │ ├── 5abfd3d65542994516f4550b.json │ ├── 5ac046c65542996f0d89cb71.json │ ├── 5ac15f6c5542994d76dccdf3.json │ ├── 5ac3a7ce554299391541388f.json │ ├── 5ac4392e5542997ea680ca32.json │ ├── 5adbf81a554299438c868cfd.json │ ├── 5adc69ef554299438c868daf.json │ ├── 5add47bf5542992200553a7f.json │ ├── 5add8ec55542997545bbbd7e.json │ ├── 5adde3a45542997545bbbdc2.json │ ├── 5ade1eda55429975fa854e71.json │ ├── 5ade64c7554299728e26c710.json │ ├── 5ade82ba5542992fa25da7a3.json │ ├── 5adfa12c554299025d62a301.json │ ├── 5adfa92d55429942ec259ae0.json │ ├── 5ae0616255429924de1b70ca.json │ ├── 5ae198bf5542997b2ef7d20d.json │ ├── 5ae1bf46554299234fd042e6.json │ ├── 5ae3fd195542995dadf2428d.json │ └── 5ae4d7fc5542990ba0bbb16f.json ├── make_hpqa_dataset.py ├── misc └── dev_ranker_preds.bin ├── outputs └── .gitkeep ├── run_arch.py ├── run_atattr.py ├── run_interp.sh ├── run_latattr.py ├── run_perturb.py ├── run_qa.py ├── run_qa.sh ├── run_tokig.py └── vis_tools ├── vis_attention.py ├── vis_token.py ├── vis_utils.py └── vis_vanilla_token.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/README.md -------------------------------------------------------------------------------- /cached/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /checkpoints/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /common/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/common/config.py -------------------------------------------------------------------------------- /common/custom_squad_feature.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/common/custom_squad_feature.py -------------------------------------------------------------------------------- /common/dataset_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/common/dataset_utils.py -------------------------------------------------------------------------------- /common/interp_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/common/interp_utils.py -------------------------------------------------------------------------------- /common/qa_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/common/qa_metrics.py -------------------------------------------------------------------------------- /common/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/common/utils.py -------------------------------------------------------------------------------- /docs/example_expl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/docs/example_expl.png -------------------------------------------------------------------------------- /docs/example_latattr.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/docs/example_latattr.jpg -------------------------------------------------------------------------------- /eval_hotpot_exp/bridge_testers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/eval_hotpot_exp/bridge_testers.py -------------------------------------------------------------------------------- /eval_hotpot_exp/eval_bridge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/eval_hotpot_exp/eval_bridge.py -------------------------------------------------------------------------------- /eval_hotpot_exp/eval_yesno.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/eval_hotpot_exp/eval_yesno.py -------------------------------------------------------------------------------- /eval_hotpot_exp/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/eval_hotpot_exp/utils.py -------------------------------------------------------------------------------- /eval_hotpot_exp/yesno_testers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/eval_hotpot_exp/yesno_testers.py -------------------------------------------------------------------------------- /expl_methods/latattr_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/expl_methods/latattr_models.py -------------------------------------------------------------------------------- /expl_methods/perturb_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/expl_methods/perturb_models.py -------------------------------------------------------------------------------- /expl_methods/tokig_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/expl_methods/tokig_models.py -------------------------------------------------------------------------------- /exps/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a7166395542994082a3e814.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a7166395542994082a3e814.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a722b8655429971e9dc9329.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a722b8655429971e9dc9329.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a74106b55429979e288289e.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a74106b55429979e288289e.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a7571135542992d0ec05f98.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a7571135542992d0ec05f98.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a75e05c55429976ec32bc5f.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a75e05c55429976ec32bc5f.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a760ab65542994ccc918697.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a760ab65542994ccc918697.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a77152355429966f1a36c2e.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a77152355429966f1a36c2e.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a77724455429972597f153e.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a77724455429972597f153e.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a79311755429970f5fffe67.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a79311755429970f5fffe67.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a7d54165542995f4f402256.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a7d54165542995f4f402256.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a80721b554299485f5985ef.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a80721b554299485f5985ef.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a80840f554299485f59863b.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a80840f554299485f59863b.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a828c8355429966c78a6a50.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a828c8355429966c78a6a50.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a85b2d95542997b5ce40028.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a85b2d95542997b5ce40028.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a85ea095542994775f606a8.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a85ea095542994775f606a8.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a85fb085542994775f606de.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a85fb085542994775f606de.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a877e5d5542993e715abf7d.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a877e5d5542993e715abf7d.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a87ab905542996e4f3088c1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a87ab905542996e4f3088c1.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a87c13f5542996e4f30890c.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a87c13f5542996e4f30890c.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a8a43eb5542996c9b8d5e82.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a8a43eb5542996c9b8d5e82.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a8b20335542996c9b8d5fb3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a8b20335542996c9b8d5fb3.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a8c7595554299585d9e36b6.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a8c7595554299585d9e36b6.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a8e1027554299653c1aa15f.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a8e1027554299653c1aa15f.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a8e3ea95542995a26add48d.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a8e3ea95542995a26add48d.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a8ef2a75542995a26add583.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a8ef2a75542995a26add583.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5a8f4c8d554299458435d5a3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5a8f4c8d554299458435d5a3.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ab2d3df554299194fa9352c.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ab2d3df554299194fa9352c.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ab3b0bf5542992ade7c6e39.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ab3b0bf5542992ade7c6e39.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ab3e45655429976abd1bcd4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ab3e45655429976abd1bcd4.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ab6d09255429954757d337d.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ab6d09255429954757d337d.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ab84bf555429916710eb01f.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ab84bf555429916710eb01f.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ab859a955429934fafe6d7b.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ab859a955429934fafe6d7b.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ab96ab755429970cfb8eacd.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ab96ab755429970cfb8eacd.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5aba7cfe554299232ef4a2fd.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5aba7cfe554299232ef4a2fd.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5abbf698554299114383a0b5.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5abbf698554299114383a0b5.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5abd94525542992ac4f382d2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5abd94525542992ac4f382d2.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5abf63f15542997ec76fd3ea.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5abf63f15542997ec76fd3ea.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ac23ff0554299636651994d.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ac23ff0554299636651994d.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ac2acff55429921a00ab02b.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ac2acff55429921a00ab02b.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5add61d65542995b365fab21.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5add61d65542995b365fab21.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5adddccd5542997dc7907069.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5adddccd5542997dc7907069.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5adf37a95542995ec70e8f97.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5adf37a95542995ec70e8f97.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ae0361155429925eb1afc2c.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ae0361155429925eb1afc2c.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ae0d4c9554299603e418468.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ae0d4c9554299603e418468.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ae22b8d554299234fd0440f.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ae22b8d554299234fd0440f.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ae32e125542991a06ce9946.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ae32e125542991a06ce9946.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ae33c4d5542992f92d82262.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ae33c4d5542992f92d82262.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ae5aba0554299546bf82f17.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ae5aba0554299546bf82f17.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ae7a8175542993210983ed8.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ae7a8175542993210983ed8.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/bridge/5ae7ba7a5542993210983f12.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/bridge/5ae7ba7a5542993210983f12.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a7e1cad55429965cec5ea75.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a7e1cad55429965cec5ea75.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a7ec61a55429934daa2fc5f.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a7ec61a55429934daa2fc5f.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a81c90a5542990a1d231eb4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a81c90a5542990a1d231eb4.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a85a2845542997175ce1fe1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a85a2845542997175ce1fe1.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a85ebe75542996432c5712a.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a85ebe75542996432c5712a.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a864d9e5542994775f6073d.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a864d9e5542994775f6073d.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a866c04554299211dda2b25.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a866c04554299211dda2b25.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a8762205542996e4f3087b8.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a8762205542996e4f3087b8.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a877dd65542993e715abf79.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a877dd65542993e715abf79.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a89e2785542992e4fca843d.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a89e2785542992e4fca843d.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a8a58f355429930ff3c0da3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a8a58f355429930ff3c0da3.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a8b57f25542995d1e6f1371.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a8b57f25542995d1e6f1371.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a8c620c554299240d9c2141.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a8c620c554299240d9c2141.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a8c87aa554299653c1aa0ac.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a8c87aa554299653c1aa0ac.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a8cdcb0554299441c6b9f33.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a8cdcb0554299441c6b9f33.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a8cec13554299441c6b9f7d.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a8cec13554299441c6b9f7d.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a8f64605542992414482aaa.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a8f64605542992414482aaa.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a8f876e5542997ba9cb32a1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a8f876e5542997ba9cb32a1.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5a8fb3af5542997ba9cb32ee.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5a8fb3af5542997ba9cb32ee.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ab52bd1554299637185c4ee.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ab52bd1554299637185c4ee.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ab681ea55429953192ad2e0.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ab681ea55429953192ad2e0.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ab6f222554299110f219a78.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ab6f222554299110f219a78.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ab6f9335542991d322236fc.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ab6f9335542991d322236fc.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ab8337a55429919ba4e225f.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ab8337a55429919ba4e225f.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ab8ee035542991b5579f04b.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ab8ee035542991b5579f04b.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5abb8afe5542993f40c73b1e.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5abb8afe5542993f40c73b1e.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5abbd3ac55429931dba1458b.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5abbd3ac55429931dba1458b.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5abbdaac5542992ccd8e7fc0.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5abbdaac5542992ccd8e7fc0.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5abd72475542993062266c9a.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5abd72475542993062266c9a.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5abd7aba55429924427fcfed.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5abd7aba55429924427fcfed.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5abfd3d65542994516f4550b.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5abfd3d65542994516f4550b.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ac046c65542996f0d89cb71.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ac046c65542996f0d89cb71.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ac15f6c5542994d76dccdf3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ac15f6c5542994d76dccdf3.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ac3a7ce554299391541388f.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ac3a7ce554299391541388f.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ac4392e5542997ea680ca32.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ac4392e5542997ea680ca32.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5adbf81a554299438c868cfd.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5adbf81a554299438c868cfd.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5adc69ef554299438c868daf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5adc69ef554299438c868daf.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5add47bf5542992200553a7f.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5add47bf5542992200553a7f.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5add8ec55542997545bbbd7e.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5add8ec55542997545bbbd7e.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5adde3a45542997545bbbdc2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5adde3a45542997545bbbdc2.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ade1eda55429975fa854e71.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ade1eda55429975fa854e71.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ade64c7554299728e26c710.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ade64c7554299728e26c710.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ade82ba5542992fa25da7a3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ade82ba5542992fa25da7a3.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5adfa12c554299025d62a301.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5adfa12c554299025d62a301.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5adfa92d55429942ec259ae0.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5adfa92d55429942ec259ae0.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ae0616255429924de1b70ca.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ae0616255429924de1b70ca.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ae198bf5542997b2ef7d20d.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ae198bf5542997b2ef7d20d.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ae1bf46554299234fd042e6.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ae1bf46554299234fd042e6.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ae3fd195542995dadf2428d.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ae3fd195542995dadf2428d.json -------------------------------------------------------------------------------- /hotpot_counterfactuals/yesno/5ae4d7fc5542990ba0bbb16f.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/hotpot_counterfactuals/yesno/5ae4d7fc5542990ba0bbb16f.json -------------------------------------------------------------------------------- /make_hpqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/make_hpqa_dataset.py -------------------------------------------------------------------------------- /misc/dev_ranker_preds.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/misc/dev_ranker_preds.bin -------------------------------------------------------------------------------- /outputs/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /run_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/run_arch.py -------------------------------------------------------------------------------- /run_atattr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/run_atattr.py -------------------------------------------------------------------------------- /run_interp.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/run_interp.sh -------------------------------------------------------------------------------- /run_latattr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/run_latattr.py -------------------------------------------------------------------------------- /run_perturb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/run_perturb.py -------------------------------------------------------------------------------- /run_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/run_qa.py -------------------------------------------------------------------------------- /run_qa.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/run_qa.sh -------------------------------------------------------------------------------- /run_tokig.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/run_tokig.py -------------------------------------------------------------------------------- /vis_tools/vis_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/vis_tools/vis_attention.py -------------------------------------------------------------------------------- /vis_tools/vis_token.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/vis_tools/vis_token.py -------------------------------------------------------------------------------- /vis_tools/vis_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/vis_tools/vis_utils.py -------------------------------------------------------------------------------- /vis_tools/vis_vanilla_token.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xiye17/EvalQAExpl/HEAD/vis_tools/vis_vanilla_token.py --------------------------------------------------------------------------------