├── .gitignore ├── README.md ├── evaluation ├── evaluate_multiple_files.py └── evaluate_single_file.py ├── figures └── example.jpg ├── finetuning ├── main.py ├── scripts │ ├── inference.sh │ └── train.sh └── yamls │ ├── inference.yaml │ └── train.yaml ├── llm_inference ├── hf_model_main.py ├── openai_main.py └── scripts │ ├── run_hf_models.sh │ └── run_openai_models.sh ├── outputs ├── finetuning_inference_outputs │ ├── bart_test_predictions.json │ ├── flan_test_predictions.json │ ├── omnitab_test_predictions.json │ ├── reastap_test_predictions.json │ ├── scores.json │ ├── t5_test_predictions.json │ └── tapex_test_predictions.json └── llm_inference_outputs │ ├── 0_shot │ ├── gpt-3.5-turbo.json │ ├── gpt-4.json │ ├── lemur-70b-chat-v1-awq.json │ ├── llama-2-13b-chat-hf.json │ ├── llama-2-70b-chat-awq.json │ ├── llama-2-7b-chat-hf.json │ ├── mistral-7b-instruct-v0.1.json │ ├── scores.json │ └── vicuna-33b-v1.3.json │ ├── 1_shot │ ├── gpt-3.5-turbo.json │ ├── gpt-4.json │ ├── lemur-70b-chat-v1-awq.json │ ├── llama-2-13b-chat-hf.json │ ├── llama-2-70b-chat-awq.json │ ├── llama-2-7b-chat-hf.json │ ├── mistral-7b-instruct-v0.1.json │ ├── scores.json │ └── vicuna-33b-v1.3.json │ └── 2_shot │ ├── gpt-3.5-turbo.json │ ├── gpt-4.json │ ├── lemur-70b-chat-v1-awq.json │ ├── llama-2-13b-chat-hf.json │ ├── llama-2-70b-chat-awq.json │ ├── llama-2-7b-chat-hf.json │ ├── mistral-7b-instruct-v0.1.json │ └── scores.json ├── requirements.txt └── utils ├── evaluation_utils.py ├── llm_preprocess_utils.py ├── openai_utils.py ├── preprocess_utils.py └── tapas_acc.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/README.md -------------------------------------------------------------------------------- /evaluation/evaluate_multiple_files.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/evaluation/evaluate_multiple_files.py -------------------------------------------------------------------------------- /evaluation/evaluate_single_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/evaluation/evaluate_single_file.py -------------------------------------------------------------------------------- /figures/example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/figures/example.jpg -------------------------------------------------------------------------------- /finetuning/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/finetuning/main.py -------------------------------------------------------------------------------- /finetuning/scripts/inference.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/finetuning/scripts/inference.sh -------------------------------------------------------------------------------- /finetuning/scripts/train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/finetuning/scripts/train.sh -------------------------------------------------------------------------------- /finetuning/yamls/inference.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/finetuning/yamls/inference.yaml -------------------------------------------------------------------------------- /finetuning/yamls/train.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/finetuning/yamls/train.yaml -------------------------------------------------------------------------------- /llm_inference/hf_model_main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/llm_inference/hf_model_main.py -------------------------------------------------------------------------------- /llm_inference/openai_main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/llm_inference/openai_main.py -------------------------------------------------------------------------------- /llm_inference/scripts/run_hf_models.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/llm_inference/scripts/run_hf_models.sh -------------------------------------------------------------------------------- /llm_inference/scripts/run_openai_models.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/llm_inference/scripts/run_openai_models.sh -------------------------------------------------------------------------------- /outputs/finetuning_inference_outputs/bart_test_predictions.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/finetuning_inference_outputs/bart_test_predictions.json -------------------------------------------------------------------------------- /outputs/finetuning_inference_outputs/flan_test_predictions.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/finetuning_inference_outputs/flan_test_predictions.json -------------------------------------------------------------------------------- /outputs/finetuning_inference_outputs/omnitab_test_predictions.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/finetuning_inference_outputs/omnitab_test_predictions.json -------------------------------------------------------------------------------- /outputs/finetuning_inference_outputs/reastap_test_predictions.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/finetuning_inference_outputs/reastap_test_predictions.json -------------------------------------------------------------------------------- /outputs/finetuning_inference_outputs/scores.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/finetuning_inference_outputs/scores.json -------------------------------------------------------------------------------- /outputs/finetuning_inference_outputs/t5_test_predictions.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/finetuning_inference_outputs/t5_test_predictions.json -------------------------------------------------------------------------------- /outputs/finetuning_inference_outputs/tapex_test_predictions.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/finetuning_inference_outputs/tapex_test_predictions.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/0_shot/gpt-3.5-turbo.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/0_shot/gpt-3.5-turbo.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/0_shot/gpt-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/0_shot/gpt-4.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/0_shot/lemur-70b-chat-v1-awq.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/0_shot/lemur-70b-chat-v1-awq.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/0_shot/llama-2-13b-chat-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/0_shot/llama-2-13b-chat-hf.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/0_shot/llama-2-70b-chat-awq.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/0_shot/llama-2-70b-chat-awq.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/0_shot/llama-2-7b-chat-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/0_shot/llama-2-7b-chat-hf.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/0_shot/mistral-7b-instruct-v0.1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/0_shot/mistral-7b-instruct-v0.1.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/0_shot/scores.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/0_shot/scores.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/0_shot/vicuna-33b-v1.3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/0_shot/vicuna-33b-v1.3.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/1_shot/gpt-3.5-turbo.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/1_shot/gpt-3.5-turbo.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/1_shot/gpt-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/1_shot/gpt-4.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/1_shot/lemur-70b-chat-v1-awq.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/1_shot/lemur-70b-chat-v1-awq.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/1_shot/llama-2-13b-chat-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/1_shot/llama-2-13b-chat-hf.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/1_shot/llama-2-70b-chat-awq.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/1_shot/llama-2-70b-chat-awq.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/1_shot/llama-2-7b-chat-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/1_shot/llama-2-7b-chat-hf.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/1_shot/mistral-7b-instruct-v0.1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/1_shot/mistral-7b-instruct-v0.1.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/1_shot/scores.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/1_shot/scores.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/1_shot/vicuna-33b-v1.3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/1_shot/vicuna-33b-v1.3.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/2_shot/gpt-3.5-turbo.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/2_shot/gpt-3.5-turbo.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/2_shot/gpt-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/2_shot/gpt-4.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/2_shot/lemur-70b-chat-v1-awq.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/2_shot/lemur-70b-chat-v1-awq.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/2_shot/llama-2-13b-chat-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/2_shot/llama-2-13b-chat-hf.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/2_shot/llama-2-70b-chat-awq.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/2_shot/llama-2-70b-chat-awq.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/2_shot/llama-2-7b-chat-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/2_shot/llama-2-7b-chat-hf.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/2_shot/mistral-7b-instruct-v0.1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/2_shot/mistral-7b-instruct-v0.1.json -------------------------------------------------------------------------------- /outputs/llm_inference_outputs/2_shot/scores.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/outputs/llm_inference_outputs/2_shot/scores.json -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/requirements.txt -------------------------------------------------------------------------------- /utils/evaluation_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/utils/evaluation_utils.py -------------------------------------------------------------------------------- /utils/llm_preprocess_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/utils/llm_preprocess_utils.py -------------------------------------------------------------------------------- /utils/openai_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/utils/openai_utils.py -------------------------------------------------------------------------------- /utils/preprocess_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/utils/preprocess_utils.py -------------------------------------------------------------------------------- /utils/tapas_acc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yale-nlp/QTSumm/HEAD/utils/tapas_acc.py --------------------------------------------------------------------------------