├── LICENSE ├── README.md ├── data └── summeval.json ├── gpt4_eval.py ├── meta_eval_summeval.py ├── prompts └── summeval │ ├── coh_detailed.txt │ ├── con_detailed.txt │ ├── flu_detailed.txt │ └── rel_detailed.txt └── results ├── gpt4_coh_detailed.json ├── gpt4_con_detailed.json ├── gpt4_flu_detailed.json └── gpt4_rel_detailed.json /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Yang Liu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Code for paper "G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment" [https://arxiv.org/abs/2303.16634] 2 | 3 | ## Experiments on SummEval dataset 4 | ### Evaluate fluency on SummEval dataset 5 | ```python .\gpt4_eval.py --prompt .\prompts\summeval\flu_detailed.txt --save_fp .\results\gpt4_flu_detailed.json --summeval_fp .\data\summeval.json --key XXXXX``` 6 | 7 | ### Meta Evaluate the G-Eval results 8 | 9 | ```python .\meta_eval_summeval.py --input_fp .\results\gpt4_flu_detailed.json --dimension fluency``` 10 | 11 | ## Prompts and Evaluation Results 12 | 13 | Prompts used to evaluate SummEval are in prompts/summeval 14 | 15 | G-eval results on SummEval are in results 16 | 17 | -------------------------------------------------------------------------------- /gpt4_eval.py: -------------------------------------------------------------------------------- 1 | import openai 2 | import json 3 | import argparse 4 | import tqdm 5 | import time 6 | 7 | if __name__ == '__main__': 8 | 9 | argparser = argparse.ArgumentParser() 10 | argparser.add_argument('--prompt_fp', type=str, default='prompts\summeval\con_detailed.txt') 11 | argparser.add_argument('--save_fp', type=str, default='results\gpt4_con_detailed_openai.json') 12 | argparser.add_argument('--summeval_fp', type=str, default='data\summeval.json') 13 | argparser.add_argument('--key', type=str, required=True) 14 | argparser.add_argument('--model', type=str, default='gpt-4-0613') 15 | args = argparser.parse_args() 16 | openai.api_key = args.key 17 | 18 | summeval = json.load(open(args.summeval_fp)) 19 | prompt = open(args.prompt_fp).read() 20 | 21 | ct, ignore = 0, 0 22 | 23 | new_json = [] 24 | for instance in tqdm.tqdm(summeval): 25 | source = instance['source'] 26 | system_output = instance['system_output'] 27 | cur_prompt = prompt.replace('{{Document}}', source).replace('{{Summary}}', system_output) 28 | instance['prompt'] = cur_prompt 29 | while True: 30 | try: 31 | _response = openai.ChatCompletion.create( 32 | model=args.model, 33 | messages=[{"role": "system", "content": cur_prompt}], 34 | temperature=2, 35 | max_tokens=5, 36 | top_p=1, 37 | frequency_penalty=0, 38 | presence_penalty=0, 39 | stop=None, 40 | # logprobs=40, 41 | n=20 42 | ) 43 | time.sleep(0.5) 44 | 45 | all_responses = [_response['choices'][i]['message']['content'] for i in 46 | range(len(_response['choices']))] 47 | instance['all_responses'] = all_responses 48 | new_json.append(instance) 49 | ct += 1 50 | break 51 | except Exception as e: 52 | print(e) 53 | if ("limit" in str(e)): 54 | time.sleep(2) 55 | else: 56 | ignore += 1 57 | print('ignored', ignore) 58 | 59 | break 60 | 61 | print('ignored total', ignore) 62 | with open(args.save_fp, 'w') as f: 63 | json.dump(new_json, f, indent=4) 64 | -------------------------------------------------------------------------------- /meta_eval_summeval.py: -------------------------------------------------------------------------------- 1 | from prettytable import PrettyTable 2 | from scipy.stats import spearmanr, pearsonr, kendalltau 3 | import json 4 | import re 5 | import argparse 6 | 7 | 8 | def calculate_correlation(pred_score, human_score, result): 9 | assert len(pred_score) == len(human_score) 10 | 11 | if (len(result) == 0): 12 | result = {'pearson': 0, 'spearman': 0, 'kendalltau': 0} 13 | result['pearson'] += pearsonr(pred_score, human_score)[0] 14 | result['spearman'] += spearmanr(pred_score, human_score)[0] 15 | result['kendalltau'] += kendalltau(pred_score, human_score)[0] 16 | 17 | return result 18 | 19 | 20 | def print_correlations(result, n): 21 | table = PrettyTable(['Pearson', 'Spearman', 'Kendall']) 22 | if (n == 0): 23 | n = 1 24 | table.add_row( 25 | [round(result['pearson'] / n, 4), round(result['spearman'] / n, 4), round(result['kendalltau'] / n, 4)]) 26 | print(table) 27 | 28 | 29 | def parse_output(output): 30 | matched = re.search("^ ?([\d\.]+)", output) 31 | if (matched): 32 | try: 33 | score = float(matched.group(1)) 34 | except: 35 | score = 0 36 | else: 37 | score = 0 38 | return score 39 | 40 | 41 | if __name__ == '__main__': 42 | parser = argparse.ArgumentParser() 43 | parser.add_argument('--input_fp', type=str, default='results/gpt4_rel_detailed.json') 44 | parser.add_argument('--dimension', type=str, default='relevance') 45 | args = parser.parse_args() 46 | 47 | jobj = json.load(open(args.input_fp)) 48 | pred_scores, human_scores = {}, {} 49 | 50 | print("Calculating correlation for G-Eval") 51 | for item in jobj: 52 | doc_id = item["doc_id"] 53 | if (doc_id not in pred_scores): 54 | pred_scores[doc_id] = [] 55 | human_scores[doc_id] = [] 56 | 57 | all_responses = item["all_responses"] 58 | all_scores = [parse_output(x) for x in all_responses] 59 | score = sum(all_scores) / len(all_scores) 60 | 61 | pred_scores[doc_id].append(score) 62 | human_scores[doc_id].append(item['scores'][args.dimension]) 63 | 64 | print('len(pred_scores): {}'.format(len(pred_scores))) 65 | print('len(human_scores): {}'.format(len(human_scores))) 66 | 67 | results = {'pearson': 0, 'spearman': 0, 'kendalltau': 0} 68 | d_ctr = 0 69 | for doc_id in pred_scores: 70 | pred_scores_doc = pred_scores[doc_id] 71 | human_scores_doc = human_scores[doc_id] 72 | if (len(set(human_scores_doc)) <= 1) or (len(set(pred_scores_doc)) <= 1): 73 | continue 74 | 75 | results = calculate_correlation(pred_scores_doc, human_scores_doc, results) 76 | d_ctr += 1 77 | print_correlations(results, n=d_ctr) 78 | -------------------------------------------------------------------------------- /prompts/summeval/coh_detailed.txt: -------------------------------------------------------------------------------- 1 | You will be given one summary written for a news article. 2 | 3 | Your task is to rate the summary on one metric. 4 | 5 | Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed. 6 | 7 | Evaluation Criteria: 8 | 9 | Coherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby "the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic." 10 | 11 | Evaluation Steps: 12 | 13 | 1. Read the news article carefully and identify the main topic and key points. 14 | 2. Read the summary and compare it to the news article. Check if the summary covers the main topic and key points of the news article, and if it presents them in a clear and logical order. 15 | 3. Assign a score for coherence on a scale of 1 to 5, where 1 is the lowest and 5 is the highest based on the Evaluation Criteria. 16 | 17 | 18 | Example: 19 | 20 | 21 | Source Text: 22 | 23 | {{Document}} 24 | 25 | Summary: 26 | 27 | {{Summary}} 28 | 29 | 30 | Evaluation Form (scores ONLY): 31 | 32 | - Coherence: -------------------------------------------------------------------------------- /prompts/summeval/con_detailed.txt: -------------------------------------------------------------------------------- 1 | You will be given a news article. You will then be given one summary written for this article. 2 | 3 | Your task is to rate the summary on one metric. 4 | 5 | Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed. 6 | 7 | 8 | Evaluation Criteria: 9 | 10 | Consistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. 11 | 12 | Evaluation Steps: 13 | 14 | 1. Read the news article carefully and identify the main facts and details it presents. 15 | 2. Read the summary and compare it to the article. Check if the summary contains any factual errors that are not supported by the article. 16 | 3. Assign a score for consistency based on the Evaluation Criteria. 17 | 18 | 19 | Example: 20 | 21 | 22 | Source Text: 23 | 24 | {{Document}} 25 | 26 | Summary: 27 | 28 | {{Summary}} 29 | 30 | 31 | Evaluation Form (scores ONLY): 32 | 33 | - Consistency: -------------------------------------------------------------------------------- /prompts/summeval/flu_detailed.txt: -------------------------------------------------------------------------------- 1 | You will be given one summary written for a news article. 2 | 3 | Your task is to rate the summary on one metric. 4 | 5 | Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed. 6 | 7 | 8 | Evaluation Criteria: 9 | 10 | Fluency (1-3): the quality of the summary in terms of grammar, spelling, punctuation, word choice, and sentence structure. 11 | 12 | - 1: Poor. The summary has many errors that make it hard to understand or sound unnatural. 13 | - 2: Fair. The summary has some errors that affect the clarity or smoothness of the text, but the main points are still comprehensible. 14 | - 3: Good. The summary has few or no errors and is easy to read and follow. 15 | 16 | 17 | Example: 18 | 19 | Summary: 20 | 21 | {{Summary}} 22 | 23 | 24 | Evaluation Form (scores ONLY): 25 | 26 | - Fluency (1-3): -------------------------------------------------------------------------------- /prompts/summeval/rel_detailed.txt: -------------------------------------------------------------------------------- 1 | You will be given one summary written for a news article. 2 | 3 | Your task is to rate the summary on one metric. 4 | 5 | Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed. 6 | 7 | Evaluation Criteria: 8 | 9 | Relevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information. 10 | 11 | Evaluation Steps: 12 | 13 | 1. Read the summary and the source document carefully. 14 | 2. Compare the summary to the source document and identify the main points of the article. 15 | 3. Assess how well the summary covers the main points of the article, and how much irrelevant or redundant information it contains. 16 | 4. Assign a relevance score from 1 to 5. 17 | 18 | 19 | Example: 20 | 21 | 22 | Source Text: 23 | 24 | {{Document}} 25 | 26 | Summary: 27 | 28 | {{Summary}} 29 | 30 | 31 | Evaluation Form (scores ONLY): 32 | 33 | - Relevance: --------------------------------------------------------------------------------