├── README.md ├── factkg ├── data │ ├── make_type_dict.py │ └── preprocess.py ├── factkg_test.py ├── generated_prompts │ ├── 1_final_evidence.pickle │ └── 1_sentence_divide.pickle ├── openai_api_key.txt └── prompts │ ├── relation_retrieval_prompt.txt │ ├── sentence_divide_prompt.txt │ ├── verify_claim_only.txt │ └── verify_claim_with_evidence.txt └── metaqa ├── data └── preprocess.py ├── meta_1hop_prompts ├── relation_retrieval_prompt.txt ├── sentence_divide_prompt.txt └── verify_claim_with_evidence.txt ├── meta_2hop_prompts ├── relation_retrieval_prompt.txt ├── sentence_divide_prompt.txt └── verify_claim_with_evidence.txt ├── meta_3hop_prompts ├── relation_retrieval_prompt.txt ├── sentence_divide_prompt.txt └── verify_claim_with_evidence.txt ├── meta_generated_prompts ├── 1hop │ ├── 1_final_evidence.pickle │ ├── 1_sentence_divide.pickle │ └── 1_verification_result.txt ├── 2hop │ ├── 1_final_evidence.pickle │ ├── 1_sentence_divide.pickle │ └── 1_verification_result.txt └── 3hop │ ├── 1_final_evidence.pickle │ ├── 1_sentence_divide.pickle │ └── 1_verification_result.txt ├── metaqa_1hop_test.py ├── metaqa_2hop_test.py ├── metaqa_3hop_test.py └── openai_api_key.txt /README.md: -------------------------------------------------------------------------------- 1 | # KG-GPT: A General Framework for Reasoning on Knowledge Graphs Using Large Language Models 2 | We propose KG-GPT, a multi-purpose framework leveraging LLMs for tasks employing KGs. KG-GPT comprises three steps: Sentence Segmentation, Graph Retrieval, and Inference, each aimed at partitioning sentences, retrieving relevant graph components, and deriving logical conclusions, respectively. We evaluate KG-GPT using KG-based fact verification and KGQA benchmarks, with the model showing competitive and robust performance, even outperforming several fully-supervised models. Our work, therefore, marks a significant step in unifying structured and unstructured data processing within the realm of LLMs. 3 | 4 | The code is released along with our [paper](https://arxiv.org/abs/2310.11220) (EMNLP 2023 Findings). For further details, please refer to our paper. 5 | 6 | You can download FactKG from [this link](https://github.com/jiho283/FactKG) and MetaQA from [this link](https://github.com/yuyuz/MetaQA). 7 | 8 | ## Before starting: Enter OpenAI API key 9 | Write your own OpenAI API key in ```factkg/openai_api_key.txt``` and ```metaqa/openai_api_key.txt``` and save them. 10 | 11 | ## 1. FactKG 12 | ```cd factkg``` 13 | ### 1-1) Preprocess 14 | 1. ```python data/preprocess.py --factkg_test /path/to/factkg_test.pickle``` 15 | 16 | 2. ```python data/make_type_dict.py --kg /path/to/dbpedia_2015_undirected.pickle --relations /path/to/relations_for_final.pickle``` 17 | 18 | ### 1-2) Inference 19 | ```python factkg_test.py --model gpt-3.5-turbo-0613 --kg /path/to/dbpedia_2015_undirected.pickle``` 20 | 21 | 22 | ## 2. MetaQA 23 | ```cd metaqa``` 24 | 25 | ### 2-1) Preprocess 26 | ```python data/preprocess.py --test_1_hop /path/to/1-hop/vanilla/qa_test.txt --test_2_hop /path/to/2-hop/vanilla/qa_test.txt --test_3_hop /path/to/3-hop/vanilla/qa_test.txt --kb /path/to/kb.txt``` 27 | 28 | ### 2-2) Inference 29 | 1-hop: ```python metaqa_1hop_test.py``` 30 | 31 | 2-hop: ```python metaqa_2hop_test.py``` 32 | 33 | 3-hop: ```python metaqa_3hop_test.py``` 34 | -------------------------------------------------------------------------------- /factkg/data/make_type_dict.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | from tqdm import tqdm 3 | import argparse 4 | 5 | if __name__ == "__main__": 6 | 7 | parser = argparse.ArgumentParser(description="Parsing input arguments.") 8 | parser.add_argument('--kg', type=str, required=True, help='Path for KG.') 9 | parser.add_argument('--relations', type=str, required=True, help='Path for the relations list.') 10 | 11 | args = parser.parse_args() 12 | 13 | kg_path = args.kg 14 | relations_path = args.relations 15 | 16 | print('Start!') 17 | with open(kg_path, 'rb') as f: 18 | dbp = pickle.load(f) 19 | 20 | with open(relations_path, 'rb') as f: 21 | relations = pickle.load(f) 22 | 23 | entities = list(dbp) 24 | 25 | relation_set = {} 26 | for rel in relations: 27 | if '~' not in rel[0]: 28 | relation_set[rel] = 0 29 | print('Data loading done!') 30 | 31 | total_result = {} 32 | for ent in tqdm(entities): 33 | try: 34 | bpl = dbp[ent] 35 | rels = list(bpl) 36 | head_types = bpl['22-rdf-syntax-ns#type'] 37 | except: 38 | continue 39 | if len(rels) == 1 and 'rdf-schema#label' in rels: 40 | continue 41 | 42 | for rel in rels: 43 | tails = bpl[rel] 44 | for tail in tails: 45 | if '"' in tail: 46 | continue 47 | try: 48 | tail_types = dbp[tail]['22-rdf-syntax-ns#type'] 49 | a = relation_set[rel] 50 | except: 51 | continue 52 | 53 | for head_type in head_types: 54 | for tail_type in tail_types: 55 | try: 56 | total_result[head_type][tail_type][rel] = 0 57 | except: 58 | try: 59 | total_result[head_type][tail_type] = {} 60 | total_result[head_type][tail_type][rel] = 0 61 | except: 62 | total_result[head_type] = {} 63 | total_result[head_type][tail_type] = {} 64 | total_result[head_type][tail_type][rel] = 0 65 | 66 | with open('./type_dict.pickle', 'wb') as f: 67 | pickle.dump(total_result, f) -------------------------------------------------------------------------------- /factkg/data/preprocess.py: -------------------------------------------------------------------------------- 1 | import json 2 | import jsonlines 3 | import pickle 4 | import argparse 5 | 6 | if __name__ == "__main__": 7 | 8 | parser = argparse.ArgumentParser(description="Parsing input arguments.") 9 | parser.add_argument('--factkg_test', type=str, required=True, help='Path for factkg test set.') 10 | 11 | args = parser.parse_args() 12 | 13 | test_set_path = args.factkg_test 14 | 15 | with open(test_set_path, 'rb') as f: 16 | test_set = pickle.load(f) 17 | 18 | claims = list(test_set) 19 | 20 | with jsonlines.open(f'./extracted_test_set.jsonl', mode='w') as w: 21 | for i, sample in enumerate(claims): 22 | new_sample = {} 23 | new_sample["question_id"] = i+1 24 | new_sample["question"] = sample 25 | new_sample["types"] = test_set[sample]["types"] 26 | new_sample["entity_set"] = test_set[sample]["Entity_set"] 27 | new_sample["Label"] = test_set[sample]["Label"] 28 | w.write(new_sample) -------------------------------------------------------------------------------- /factkg/factkg_test.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | import time 5 | import re 6 | import openai 7 | import tqdm 8 | import shortuuid 9 | import pickle 10 | 11 | 12 | def open_file(filepath): 13 | with open(filepath, 'r', encoding='utf-8') as infile: 14 | return infile.read() 15 | 16 | 17 | def claim_divider_parse_answer(answer, gt_entities): 18 | processed_answer_set = {} 19 | answer = answer.strip() 20 | splitted_answers = answer.split('\n') 21 | all_entities = [] 22 | try: 23 | for nth_answer in splitted_answers: 24 | nth_answer = nth_answer.strip() 25 | for i in range(3): 26 | if str(i+1)+'. ' in nth_answer[:5]: 27 | temp_ans = nth_answer.split(str(i+1)+'. ')[1] 28 | temp_split = temp_ans.split(', Entity set: ') 29 | sentence = temp_split[0] 30 | 31 | entity_set = temp_split[1] 32 | entity_set = entity_set.split('[')[1] 33 | entity_set = entity_set.split(']')[0] 34 | entity_set = entity_set.split(' ## ') 35 | 36 | new_entity_set = [] 37 | for ent in entity_set: 38 | new_entity_set.append(ent[1:-1]) 39 | all_entities.append(ent[1:-1]) 40 | break 41 | processed_answer_set[sentence] = new_entity_set 42 | except: 43 | return None 44 | 45 | return processed_answer_set 46 | 47 | 48 | def relation_candidates(KG, type_dict, entity_set): 49 | final_type_set = [] 50 | final_entity_set = [] 51 | new_entity_set = [] 52 | for ent in entity_set: 53 | if '"' in ent: 54 | final_entity_set.append(ent) 55 | new_entity_set.append(ent) 56 | continue 57 | total = [] 58 | ent = ent.strip() 59 | splitted_ent = ent.split(' ') 60 | if len(splitted_ent) == 1: 61 | splitted_ent = ent.split('_') 62 | for spl_ent in splitted_ent: 63 | total.append([spl_ent.strip(), spl_ent.strip()[0].upper() + spl_ent.strip()[1:]]) 64 | 65 | temp_list = [] 66 | for chunk in total: 67 | if len(temp_list) == 0: 68 | temp_list = [chunk[0], chunk[1]] 69 | continue 70 | new_list = [] 71 | for temp in temp_list: 72 | new_list.append(temp+chunk[0]) 73 | new_list.append(temp+chunk[1]) 74 | 75 | temp_list = new_list.copy() 76 | 77 | is_type = 0 78 | for type_ in temp_list: 79 | try: 80 | a = type_dict[type_] 81 | final_type_set.append([type_]) 82 | new_entity_set.append(type_) 83 | is_type = 1 84 | break 85 | except: 86 | continue 87 | 88 | if is_type == 0: 89 | final_entity_set.append(ent) 90 | new_entity_set.append(ent) 91 | 92 | #### For type relations 93 | all_type_list = [] 94 | if len(final_type_set) == 1: 95 | for temp_type in final_type_set[0]: 96 | for type_temp_ele in list(type_dict[temp_type]): 97 | all_type_list += type_dict[temp_type][type_temp_ele] 98 | all_type_list = list(set(all_type_list)) 99 | else: 100 | for final_set in final_type_set: 101 | temp_all_type_list = [] 102 | for other_final_set in final_type_set: 103 | if final_set != other_final_set: 104 | for final_ele in final_set: 105 | for other_ele in other_final_set: 106 | if len(temp_all_type_list) == 0: 107 | try: 108 | temp_all_type_list = type_dict[final_ele][other_ele] 109 | except: 110 | temp_all_type_list = [] 111 | else: 112 | try: 113 | temp_all_type_list = [id_type for id_type in temp_all_type_list if id_type in type_dict[final_ele][other_ele]] 114 | except: 115 | temp_all_type_list = [] 116 | temp_all_type_list_copy = temp_all_type_list.copy() 117 | all_type_list += temp_all_type_list_copy 118 | all_type_list = list(set(all_type_list)) 119 | 120 | #### For entity relations 121 | all_entity_list = [] 122 | if len(final_entity_set) == 1: 123 | try: 124 | all_entity_list = list(KG[final_entity_set[0]]) 125 | except: 126 | all_entity_list = [] 127 | else: 128 | for final_entity in final_entity_set: 129 | for another_final_entity in final_entity_set: 130 | if final_entity != another_final_entity: 131 | try: 132 | for temp_rel in list(KG[final_entity]): 133 | try: 134 | another_final_list = list(KG[another_final_entity]) 135 | if '~' in temp_rel[0]: 136 | if temp_rel.split('~')[1] in another_final_list: 137 | all_entity_list.append(temp_rel.split('~')[1]) 138 | elif '~' + temp_rel in another_final_list: 139 | all_entity_list.append(temp_rel) 140 | except: 141 | pass 142 | except: 143 | pass 144 | all_entity_list = list(set(all_entity_list)) 145 | 146 | final_relation_list = [] 147 | 148 | if len(all_type_list) == 0: 149 | for rel in all_entity_list: 150 | if '~' in rel[0]: 151 | final_relation_list.append(rel.split('~')[1]) 152 | else: 153 | final_relation_list.append(rel) 154 | 155 | elif len(all_entity_list) == 0: 156 | for rel in all_type_list: 157 | if '~' in rel[0]: 158 | final_relation_list.append(rel.split('~')[1]) 159 | else: 160 | final_relation_list.append(rel) 161 | 162 | else: 163 | for rel in all_entity_list: 164 | if '~' in rel[0]: 165 | if rel.split('~')[1] in all_type_list: 166 | final_relation_list.append(rel.split('~')[1]) 167 | elif rel in all_type_list or '~'+rel in all_type_list or len(all_type_list) == 0: 168 | final_relation_list.append(rel) 169 | 170 | return final_relation_list, new_entity_set 171 | 172 | 173 | def retrieval_relation_parse_answer(answer): 174 | pattern = r'\[[^\]]+\]' 175 | matches = re.findall(pattern, answer) 176 | 177 | if len(matches) != 1: 178 | return None 179 | 180 | # Extract the components from the matches and flatten the list 181 | components = [component.strip() for match in matches for component in match.strip('[]').split(',')] 182 | components = [component.strip("''") if "'" in component else component for component in components ] 183 | return components 184 | 185 | 186 | def graph_extractor(target_list): 187 | if len(target_list) == 0: 188 | return target_list 189 | 190 | return_list = [] 191 | filter_dict = {'head': {}, 'tail':{}} 192 | return_list.append(target_list[0]) 193 | used_heads = [] 194 | used_tails = [] 195 | filter_dict['head'][target_list[0][0]] = [target_list[0][1]] 196 | filter_dict['tail'][target_list[0][2]] = [target_list[0][1]] 197 | used_heads.append(target_list[0][0]) 198 | used_tails.append(target_list[0][2]) 199 | 200 | for tar in target_list: 201 | if tar in return_list: 202 | continue 203 | else: 204 | try: 205 | if tar[1] in filter_dict['head'][tar[0]]: 206 | continue 207 | except: 208 | try: 209 | if tar[1] in filter_dict['tail'][tar[2]]: 210 | continue 211 | except: 212 | pass 213 | try: 214 | if tar[1] not in list(filter_dict['head'][tar[0]]): 215 | filter_dict['head'][tar[0]] = [tar[1]] 216 | try: 217 | filter_dict['tail'][tar[2]].append(tar[1]) 218 | except: 219 | filter_dict['tail'][tar[2]] = [tar[1]] 220 | return_list.append(tar) 221 | continue 222 | except: 223 | pass 224 | 225 | try: 226 | if tar[1] not in list(filter_dict['tail'][tar[2]]): 227 | filter_dict['tail'][tar[2]] = [tar[1]] 228 | try: 229 | filter_dict['head'][tar[0]].append(tar[1]) 230 | except: 231 | filter_dict['head'][tar[0]] = [tar[1]] 232 | return_list.append(tar) 233 | continue 234 | except: 235 | pass 236 | 237 | if tar[2] in used_heads or tar[0] in used_tails: 238 | return_list.append(tar) 239 | try: 240 | filter_dict['head'][tar[0]].append(tar[1]) 241 | except: 242 | filter_dict['head'][tar[0]] = [tar[1]] 243 | try: 244 | filter_dict['tail'][tar[2]].append(tar[1]) 245 | except: 246 | filter_dict['tail'][tar[2]] = [tar[1]] 247 | return return_list 248 | 249 | 250 | def get_answer(model_name: str, qid: int, claim: str, gt_entities: list, KG: dict, type_dict: dict, top_k: int, max_tokens: int): 251 | ans = { 252 | "answer_id": shortuuid.uuid(), 253 | } 254 | 255 | sentence_divide_query = open_file('./prompts/sentence_divide_prompt.txt').replace('<<<>>>', claim).replace('<<<>>>', str(gt_entities)) 256 | 257 | #### 1. sentence divide 258 | for _ in range(3): 259 | try: 260 | response = openai.ChatCompletion.create( 261 | model=model_name, 262 | messages=[ 263 | {"role": "system", "content": "You are a helpful assistant."}, 264 | { 265 | "role": "user", 266 | "content": sentence_divide_query, 267 | }, 268 | ], 269 | max_tokens=max_tokens, 270 | temperature=0.2, 271 | top_p = 0.1 272 | ) 273 | divide_claim_result = response["choices"][0]["message"]["content"] 274 | 275 | divided_claim_list = claim_divider_parse_answer(divide_claim_result, gt_entities) 276 | if divided_claim_list: 277 | break 278 | except Exception as e: 279 | print("[ERROR]", e) 280 | time.sleep(5) 281 | with open(f'./generated_prompts/{qid}_sentence_divide.pickle', 'wb') as f: 282 | pickle.dump(divided_claim_list, f) 283 | 284 | 285 | used_all_relations = [] 286 | #### 2. Relation Retrieval 287 | total_evidence = [] 288 | for divided_claim, corresponding_entity_set in divided_claim_list.items(): 289 | rel_candidates, corresponding_entity_set = relation_candidates(KG, type_dict, corresponding_entity_set) 290 | relation_retrieval_query = open_file('./prompts/relation_retrieval_prompt.txt').replace('<<<>>>', str(top_k)).replace('<<<>>>', divided_claim).replace('<<<>>>', str(rel_candidates)) 291 | 292 | if len(rel_candidates) < top_k: 293 | used_all_relations += rel_candidates 294 | total_triples = [] 295 | for ret_rel in rel_candidates: 296 | if len(corresponding_entity_set) == 1: 297 | total_triples.append([corresponding_entity_set[0], ret_rel]) 298 | total_triples.append([corresponding_entity_set[0], '~'+ret_rel]) 299 | for fir_id in range(len(corresponding_entity_set)): 300 | for sec_id in range(len(corresponding_entity_set)): 301 | if fir_id != sec_id: 302 | total_triples.append([corresponding_entity_set[fir_id], ret_rel, corresponding_entity_set[sec_id]]) 303 | total_triples.append([corresponding_entity_set[fir_id], '~'+ret_rel, corresponding_entity_set[sec_id]]) 304 | if len(total_triples) > 0: 305 | total_evidence.append(total_triples) 306 | continue 307 | 308 | 309 | for _ in range(3): 310 | try: 311 | response = openai.ChatCompletion.create( 312 | model=model_name, 313 | messages=[ 314 | {"role": "system", "content": "You are a helpful assistant."}, 315 | { 316 | "role": "user", 317 | "content": relation_retrieval_query, 318 | }, 319 | ], 320 | max_tokens=max_tokens, 321 | temperature=0.2, 322 | top_p = 0.1 323 | ) 324 | relation_retrieval_result = response["choices"][0]["message"]["content"] 325 | retrieved_relations = retrieval_relation_parse_answer(relation_retrieval_result) 326 | 327 | if retrieved_relations: 328 | used_all_relations += retrieved_relations 329 | break 330 | print(_, 'th try!') 331 | except Exception as e: 332 | print("[ERROR]", e) 333 | time.sleep(5) 334 | 335 | total_triples = [] 336 | for ret_rel in retrieved_relations: 337 | if len(corresponding_entity_set) == 1: 338 | total_triples.append([corresponding_entity_set[0], ret_rel]) 339 | total_triples.append([corresponding_entity_set[0], '~'+ret_rel]) 340 | 341 | for fir_id in range(len(corresponding_entity_set)): 342 | for sec_id in range(len(corresponding_entity_set)): 343 | if fir_id != sec_id: 344 | total_triples.append([corresponding_entity_set[fir_id], ret_rel, corresponding_entity_set[sec_id]]) 345 | total_triples.append([corresponding_entity_set[fir_id], '~'+ret_rel, corresponding_entity_set[sec_id]]) 346 | total_evidence.append(total_triples) 347 | 348 | ### for three hop 349 | additional = [] 350 | for evi_set in total_evidence: 351 | try: 352 | a = type_dict[evi_set[0][0]] 353 | b = type_dict[evi_set[0][2]] 354 | for trip in evi_set: 355 | additional.append(trip[1]) 356 | except: 357 | continue 358 | 359 | before_final = [] 360 | for evi_set in total_evidence: 361 | before_final_evi = [] 362 | for trip in evi_set: 363 | try: 364 | a = type_dict[trip[0]] 365 | continue 366 | except: 367 | try: 368 | b = type_dict[trip[2]] 369 | try: 370 | tails = KG[trip[0]][trip[1]] 371 | for tail in tails: 372 | before_final_evi.append([trip[0], trip[1], tail]) 373 | except: 374 | continue 375 | except: 376 | try: 377 | if len(trip) == 2: 378 | before_final_evi.append(trip) 379 | elif trip[2] in KG[trip[0]][trip[1]]: 380 | before_final_evi.append(trip) 381 | except: 382 | pass 383 | if len(before_final_evi) > 0: 384 | before_final.append(before_final_evi) 385 | 386 | final_evidence = [] 387 | for chunk in before_final: 388 | find_gt = 0 389 | for trip in chunk: 390 | if len(trip) != 2 and trip[0] in gt_entities and trip[2] in gt_entities: 391 | final_evidence.append(trip) 392 | find_gt = 1 393 | 394 | if find_gt == 1: 395 | continue 396 | 397 | if len(before_final) == 1: 398 | for trip in chunk: 399 | if len(trip) == 2: 400 | try: 401 | tails = KG[trip[0]][trip[1]] 402 | for tail in tails: 403 | final_evidence.append([trip[0], trip[1], tail]) 404 | except: 405 | continue 406 | break 407 | 408 | additional = list(set(additional)) 409 | 410 | if len(additional) != 0: 411 | for sec_chunk in before_final: 412 | if chunk == sec_chunk: 413 | continue 414 | for trip in chunk: 415 | if len(trip) == 2: 416 | continue 417 | for sec_trip in sec_chunk: 418 | if len(sec_trip) == 2: 419 | continue 420 | for rel_ in additional: 421 | for trip_id in [0, 2]: 422 | for sec_trip_id in [0, 2]: 423 | for rel_add in ['', '~']: 424 | try: 425 | if rel_add == '' and '~' in rel_: 426 | if trip[trip_id] in KG[sec_trip[sec_trip_id]][rel_.split('~')[1]]: 427 | final_evidence.append(trip) 428 | final_evidence.append(sec_trip) 429 | final_evidence.append([sec_trip[sec_trip_id], rel_.split('~')[1], trip[trip_id]]) 430 | except: 431 | pass 432 | try: 433 | if trip[trip_id] in KG[sec_trip[sec_trip_id]][rel_add + rel_]: 434 | final_evidence.append(trip) 435 | final_evidence.append(sec_trip) 436 | final_evidence.append([sec_trip[sec_trip_id], rel_add + rel_, trip[trip_id]]) 437 | except: 438 | pass 439 | else: 440 | for sec_chunk in before_final: 441 | if chunk == sec_chunk: 442 | continue 443 | for trip in chunk: 444 | for sec_trip in sec_chunk: 445 | if len(trip) == 2 or len(sec_trip) == 2: 446 | continue 447 | if (trip[0] in sec_trip and trip[0] not in gt_entities) or (trip[2] in sec_trip and trip[2] not in gt_entities): 448 | final_evidence.append(trip) 449 | final_evidence.append(sec_trip) 450 | 451 | #### Remove duplicated triples 452 | new_final_evidence = [] 453 | if len(final_evidence) != 0: 454 | for trip in final_evidence: 455 | if '~' in trip[1]: 456 | if [trip[2], trip[1].split('~')[1], trip[0]] not in new_final_evidence and [trip[2], trip[1].split('~')[1], trip[0]] not in final_evidence: 457 | new_final_evidence.append([trip[2], trip[1].split('~')[1], trip[0]]) 458 | continue 459 | else: 460 | if trip not in new_final_evidence: 461 | new_final_evidence.append(trip) 462 | 463 | new_final_evidence = graph_extractor(new_final_evidence) 464 | with open(f'./generated_prompts/{qid}_final_evidence.pickle', 'wb') as f: 465 | pickle.dump(new_final_evidence, f) 466 | 467 | #### Verification phase 468 | verify_prompt = open_file('./prompts/verify_claim_with_evidence.txt').replace('<<<>>>', claim).replace('<<<>>>', str(new_final_evidence)).replace('<<<>>>', str(gt_entities)).replace('<<<>>>', str(used_all_relations)) 469 | for _ in range(3): 470 | try: 471 | response = openai.ChatCompletion.create( 472 | model=model_name, 473 | messages=[ 474 | {"role": "system", "content": "You are a helpful assistant."}, 475 | { 476 | "role": "user", 477 | "content": verify_prompt, 478 | }, 479 | ], 480 | max_tokens=max_tokens, 481 | temperature=0.2, 482 | top_p = 0.1 483 | ) 484 | verification = response["choices"][0]["message"]["content"] 485 | 486 | if ('true' in verification.lower() and 'false' not in verification.lower()) or ('not refute' in verification.lower() and 'not support' not in verification.lower()) or ('support' in verification.lower() and 'refute' not in verification.lower()): 487 | return True, verification 488 | elif ('false' in verification.lower() and 'true' not in verification.lower()) or ('not support' in verification.lower() and 'not refute' not in verification.lower()) or ('refute' in verification.lower() and 'support' not in verification.lower()): 489 | return False, verification 490 | else: 491 | continue 492 | 493 | except Exception as e: 494 | print("[ERROR]", e) 495 | time.sleep(5) 496 | 497 | return 'Another Answer', verification 498 | 499 | 500 | if __name__ == "__main__": 501 | 502 | openai.api_key = open_file('./openai_api_key.txt') 503 | parser = argparse.ArgumentParser(description="Parsing input arguments.") 504 | parser.add_argument('--model', type=str, required=True, help='Model name. gpt-4 or gpt-3.5-turbo-0613') 505 | parser.add_argument('--kg', type=str, required=True, help='Path for KG.') 506 | 507 | args = parser.parse_args() 508 | 509 | model_name = args.model 510 | kg_path = args.kg 511 | 512 | print('Start!!!') 513 | with open(kg_path, 'rb') as f: 514 | dbp = pickle.load(f) 515 | 516 | print('Data loading done!!!') 517 | with open('./type_dict.pickle', 'rb') as f: 518 | type_dict = pickle.load(f) 519 | 520 | futures = [] 521 | start_token = 0 522 | 523 | ####For new experiment, use it. 524 | result = {} 525 | questions_dict = {} 526 | entity_set_dict = {} 527 | label_set_dict = {} 528 | with open(os.path.expanduser(f"./extracted_test_set.jsonl")) as f: 529 | for line in f: 530 | if not line: 531 | continue 532 | q = json.loads(line) 533 | questions_dict[q["question_id"]] = q["question"] 534 | entity_set_dict[q["question_id"]] = q["entity_set"] 535 | label_set_dict[q["question_id"]] = q["Label"] 536 | 537 | ####For new experiment, use it. 538 | # with open(f'./result.pickle', 'rb') as f: 539 | # result = pickle.load(f) 540 | 541 | Correct = [] 542 | Wrong = [] 543 | Error = [] 544 | Another = [] 545 | 546 | for qid, question in questions_dict.items(): 547 | ####For new experiment, use it. 548 | # try: 549 | # test_value = result[qid] 550 | # continue 551 | # except: 552 | # pass 553 | if qid == 3532: ## Timeout 554 | Error.append(qid) 555 | print(qid, ': Error...') 556 | result[qid] = 'Error' 557 | continue 558 | try: 559 | future = get_answer(model_name, qid, question, entity_set_dict[qid], KG=dbp, type_dict=type_dict, top_k = 5, max_tokens=1024) 560 | futures.append(future) 561 | if future[0] == 'Another Answer': 562 | Another.append(qid) 563 | print(qid, ': ', future[1]) 564 | result[qid] = future[1] 565 | elif future[0] == label_set_dict[qid][0]: 566 | Correct.append(qid) 567 | print(qid, ': Correct!') 568 | result[qid] = 'Correct' 569 | else: 570 | Wrong.append(qid) 571 | print(qid, ': Wrong...') 572 | result[qid] = 'Wrong' 573 | except: 574 | Error.append(qid) 575 | print(qid, ': Error...') 576 | result[qid] = 'Error' 577 | with open(f'./result.pickle', 'wb') as f: 578 | pickle.dump(result, f) 579 | 580 | tot_corr = 0 581 | for tot_id in list(result): 582 | if result[tot_id] == 'Correct': 583 | tot_corr += 1 584 | 585 | print('Accuracy: ', tot_corr/len(list(result))) 586 | -------------------------------------------------------------------------------- /factkg/generated_prompts/1_final_evidence.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiho283/KG-GPT/ea443df2e9d81e4cbaaa6552cffc74fc54f62f6a/factkg/generated_prompts/1_final_evidence.pickle -------------------------------------------------------------------------------- /factkg/generated_prompts/1_sentence_divide.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiho283/KG-GPT/ea443df2e9d81e4cbaaa6552cffc74fc54f62f6a/factkg/generated_prompts/1_sentence_divide.pickle -------------------------------------------------------------------------------- /factkg/openai_api_key.txt: -------------------------------------------------------------------------------- 1 | <<>> -------------------------------------------------------------------------------- /factkg/prompts/relation_retrieval_prompt.txt: -------------------------------------------------------------------------------- 1 | I will give you a set of words. 2 | Find the top <<<>>> elements from Words set which are most semantically related to the given sentence. You may select up to <<<>>> words. If there is nothing that looks semantically related, pick out any <<<>>> elements and give them to me. 3 | 4 | Examples) 5 | Sentence A: Ahmad Kadhim Assad's club is Al-Zawra'a SC. 6 | Words set: ['club', 'clubs', 'parent', 'spouse', 'birthPlace', 'deathYear', 'leaderName', 'awards', 'award', 'vicepresident', 'vicePresident'] 7 | Top 2 Answer: ['club', 'clubs'] 8 | 9 | 10 | Sentence B: Yeah! I know that Bananaman, which starred Tim Brooke-Taylor, first aired on 3rd October 1983! 11 | Words set: ['OwningCompany', 'owner', 'dean', 'coach', 'writer', 'firstAired', 'director', 'formerTeam', 'starring'] 12 | Top 2 Answer: ['firstAired', 'starring'] 13 | 14 | 15 | Sentence C: 'A film' was produced by Anatole de Grunwald, directed by Harold French, with cinematography done by Bernard Knowles. 16 | Words set: ['composer', 'team', 'editor', 'starring', 'runtime', 'director', 'discoverer', 'founder', 'crewMembers', 'writer', 'producer', 'cinematography'] 17 | Top 3 Answer: ['producer', 'director', 'cinematography'] 18 | 19 | 20 | Sentence D: Really? Jamie Lawrence is the music composer of the 83 minute 'Death on a Factory Farm' film, directed by Sarah Teale! 21 | Words set: ['composer', 'team', 'editor', 'starring', 'runtime', 'director', 'discoverer', 'founder', 'crewMembers', 'writer'] 22 | Top 4 Answer: ['composer', 'runtime', 'director', 'writer'] 23 | 24 | 25 | Sentence E: No, but the leader of the United States is not Olena Serdiuk. 26 | Words set: ['composer', 'team', 'editor', 'starring', 'runtime', 'director', 'discoverer', 'founder', 'leader', 'crewMembers', 'writer', 'producer', 'cinematography'] 27 | Top 1 Answer: ['leader'] 28 | 29 | 30 | Sentence F: Brandon Carter was born in England and graduated from the University of Cambridge where the current Chancellor is Leszek Borysiewicz. 31 | Words set: ['OwningCompany', 'placeOfBirth', 'owner', 'viceChancellor', 'almaMater', 'dean', 'coach', 'writer', 'firstAired', 'director', 'formerTeam', 'starring', 'birthPlace'] 32 | Top 4 Answer: ['birthPlace', 'placeOfBirth', 'viceChancellor', 'almaMater'] 33 | 34 | 35 | Sentence G: AIDA Cruise line operated the ship which was built by Meyer Werft in Townsend, Poulshot, Wiltshire. 36 | Words set: ['location', 'firstAired', 'clubs', 'parent', 'network', 'shipBuilder', 'birthPlace', 'locationCity', 'shipOperator', 'leaderName', 'awards', 'award', 'vicepresident', 'vicePresident'] 37 | Top 4 Answer: ['shipBuilder', 'location', 'locationCity', 'shipOperator',] 38 | 39 | 40 | Sentence H: Yes, with a floor count of 45, 200 Public Square is located in Cleveland in the United States. 41 | Words set: ['producer', 'floorCount', 'country', 'location', 'primeMinister', 'parent', 'spouse', 'nativeName', 'builder', 'manager', 'designer'] 42 | Top 3 Answer: ['floorCount', 'country', 'location'] 43 | 44 | 45 | Sentence I: Bananaman the TV series starred by a person was shown on the company and the company headquarters is called Broadcasting House. 46 | Words set: ['composer', 'team', 'editor', 'starring', 'locationCity', 'runtime', 'network', 'discoverer', 'founder', 'crewMembers', 'writer'] 47 | Top 3 Answer: ['locationCity', 'starring','network'] 48 | 49 | 50 | Sentence J: Do you know Milan Hodža? he had a religion. 51 | Words set: ['deathYear', 'leaderName', 'awards', 'award', 'religion'] 52 | Top 1 Answer: ['religion'] 53 | 54 | 55 | Sentence K: The place, designed by Huseyin Butuner and Hilmi Guner, is located in a country, where the leader is Artur Rasizade. 56 | Words set: ['producer', 'primeMinister', 'parent', 'leaderName' 'spouse', 'nativeName', 'builder', 'manager', 'designer', 'location'] 57 | Top 3 Answer: ['designer', 'leaderName', 'location'] 58 | 59 | 60 | Sentence L: An academic journal with code IJPHDE is also Acta Math. Hungar. 61 | Words set: ['abbreviation', 'placeOfBirth', 'owner', 'coden', 'almaMater', 'dean', 'coach', 'writer', 'firstAired', 'director', 'formerTeam', 'starring', 'birthPlace'] 62 | Top 2 Answer: ['abbreviation', 'coden'] 63 | 64 | 65 | Now let's find the top <<<>>> elements. 66 | Sentence: <<<>>> 67 | Words set: <<<>>> 68 | Top <<<>>> Answer: -------------------------------------------------------------------------------- /factkg/prompts/sentence_divide_prompt.txt: -------------------------------------------------------------------------------- 1 | Please divide the given sentence into several sentences each of which can be represented by one triplet. The generated sentences should be numbered and formatted as follows: #(number). (sentence), (entity set). The entity set for each sentence should contain no more than two entities, with each entity being used only once in all statements. The '##' symbol should be used to indicate an entity set. In the generated sentences, there cannot be more than two entities in the entity set. (i.e., the number of ## must not be larger than two.) 2 | 3 | Examples) 4 | Sentence A: Ahmad Kadhim Assad's club is Al-Zawra'a SC. 5 | Entity set: ['Ahmad_Kadhim_Assad' ## "Al-Zawra'a_SC"] 6 | --> Divided: 7 | 1. Ahmad Kadhim Assad's club is Al-Zawra'a SC., Entity set: ['Ahmad_Kadhim_Assad' ## "Al-Zawra'a_SC"] 8 | 9 | 10 | Sentence B: Yeah! I know that Bananaman, which starred Tim Brooke-Taylor, first aired on 3rd October 1983! 11 | Entity set: ['"1983-10-03"' ## 'Tim_Brooke-Taylor' ## 'Bananaman'] 12 | --> Divided: 13 | 1. Bananaman starred Tim Brooke-Taylor., Entity set: ['Bananaman' ## 'Tim_Brooke-Taylor'] 14 | 2. Bananaman first aired on 3rd October 1983., Entity set: ['Bananaman' ## '"1983-10-03"'] 15 | 16 | 17 | Sentence C: AIDA Cruise line operated the ship which was built by Meyer Werft in Townsend, Poulshot, Wiltshire. 18 | Entity set: ['Meyer_Werft' ## 'AIDA_Cruises' ## '"Townsend, Poulshot, Wiltshire"'] 19 | --> Divided: 20 | 1. AIDA Cruise line operated the ship., Entity set: ['AIDA_Cruises' ## 'ship'] 21 | 2. The ship was built by Meyer Werft., Entity set: ['ship' ## 'Meyer_Werft'] 22 | 3. Meyer Werft is located in Townsend, Poulshot, Wiltshire., Entity set: ['Meyer_Werft' ## '"Townsend, Poulshot, Wiltshire"'] 23 | 24 | 25 | Sentence D: Really? Jamie Lawrence is the music composer of the 83 minute 'Death on a Factory Farm' film, directed by Sarah Teale! 26 | Entity set: ['Jamie_Lawrence' ## '"83.0"' ## 'Death_on_a_Factory_Farm' ## 'Sarah_Teale'] 27 | --> Divided: 28 | 1. Jamie Lawrence is the music composer of 'Death on a Factory Farm' film., Entity set: ['Jamie_Lawrence' ## 'Death_on_a_Factory_Farm'] 29 | 2. 'Death on a Factory Farm' film is directed by Sarah Teale., Entity set: ['Death_on_a_Factory_Farm' ## 'Sarah_Teale'] 30 | 3. 'Death on a Factory Farm' is 83 minute film., Entity set: ['Death_on_a_Factory_Farm' ## '"83.0"'] 31 | 32 | 33 | Sentence E: Brandon Carter was born in England and graduated from the University of Cambridge where the current Chancellor is Leszek Borysiewicz. 34 | Entity set: ['Brandon_Carter' ## 'University_of_Cambridge' ## 'Leszek_Borysiewicz' ## 'England'] 35 | --> Divided: 36 | 1. Brandon Carter was born in England., Entity set: ['Brandon_Carter' ## 'England'] 37 | 2. Brandon Carter graduated from the University of Cambridge., Entity set: ['Brandon_Carter' ## 'University_of_Cambridge'] 38 | 3. The current Chancellor of University of Cambridge is Leszek Borysiewicz., Entity set: ['University_of_Cambridge' ## 'Leszek_Borysiewicz'] 39 | 40 | 41 | Sentence F: No, but the leader of the United States is not Olena Serdiuk. 42 | Entity set: ['United_States' ## '"Olena Serdiuk"'] 43 | --> Divided: 44 | 1. The leader of the United States is not Olena Serdiuk., Entity set: ['United_States' ## '"Olena Serdiuk"'] 45 | 46 | 47 | Sentence G: Yes, with a floor count of 45, 200 Public Square is located in Cleveland in the United States. 48 | Entity set: ['200_Public_Square' ## 'Cleveland' ## 'United_States' ## '"45"'] 49 | --> Divided: 50 | 1. 200 Public Square is with a floor count of 45., Entity set: ['200_Public_Square' ## '"45"'] 51 | 2. 200 Public Square is located in Cleveland., Entity set: ['200_Public_Square' ## 'Cleveland'] 52 | 3. Cleveland is in the United States., Entity set: ['Cleveland' ## 'United_States'] 53 | 54 | 55 | Sentence H: Bananaman the TV series starred by a person was shown on the company and the company headquarters is called Broadcasting House. 56 | Entity set: ['Broadcasting_House' ## 'Bananaman'] 57 | --> Divided: 58 | 1. Bananaman the TV series starred by a person., Entity set: ['Bananaman' ## 'person'] 59 | 2. Bananaman the TV series was shown on the company., Entity set: ['Bananaman' ## 'company'] 60 | 3. The company headquarters is called Broadcasting House., Entity set: ['company' ## 'Broadcasting_House'] 61 | 62 | 63 | Sentence I: Do you know Milan Hodža? he had a religion. 64 | Entity set: ['Milan_Hodža'] 65 | --> Divided: 66 | 1. Milan Hodža had a religion., Entity set: ['Milan_Hodža'] 67 | 68 | 69 | Sentence J: The place, designed by Huseyin Butuner and Hilmi Guner, is located in a country, where the leader is Artur Rasizade. 70 | Entity set: ['"Hüseyin Bütüner and Hilmi Güner"' ## 'Artur_Rasizade'] 71 | --> Divided: 72 | 1. The place was designed by Huseyin Butuner and Hilmi Guner., Entity set: ['place' ## '"Hüseyin Bütüner and Hilmi Güner"'] 73 | 2. The place is located in a country., Entity set: ['place' ## 'country'] 74 | 3. The leader of a country is Artur Rasizade., Entity set: ['country' ## 'Artur_Rasizade'] 75 | 76 | 77 | Sentence K: An academic journal with code IJPHDE is also Acta Math. Hungar. 78 | Entity set: ['"Acta Math. Hungar."' ## '"IJPHDE"'] 79 | --> Divided: 80 | 1. An academic journal is with code IJPHDE., Entity set: ['academic journal' ## '"IJPHDE"'] 81 | 2. An academic journal is also Acta Math. Hungar., Entity set: ['academic journal' ## '"Acta Math. Hungar."'] 82 | 83 | 84 | Sentence L: 'A film' was produced by Anatole de Grunwald, directed by Harold French, with cinematography done by Bernard Knowles. 85 | Entity set: ['Anatole_de_Grunwald' ## 'Bernard_Knowles' ## 'Harold_French'] 86 | --> Divided: 87 | 1. 'A film' was produced by Anatole de Grunwald., Entity set: ['film' ## 'Anatole_de_Grunwald'] 88 | 2. 'A film' was directed by Harold French., Entity set: ['film' ## ''Harold_French'] 89 | 3. 'A film' was with cinematography done by Bernard Knowles., Entity set: ['film' ## 'Bernard_Knowles'] 90 | 91 | 92 | Your Task) 93 | Sentence: <<<>>> 94 | Entity set: <<<>>> 95 | --> Divided: -------------------------------------------------------------------------------- /factkg/prompts/verify_claim_only.txt: -------------------------------------------------------------------------------- 1 | Verify the claim. Is this claim True? or False? 2 | Claim: <<<>>> 3 | Choose one of {True, False}, and give me the one-sentence evidence. -------------------------------------------------------------------------------- /factkg/prompts/verify_claim_with_evidence.txt: -------------------------------------------------------------------------------- 1 | You should verify the claim based on the evidence set. 2 | Each evidence is in the form of [head, relation, tail] and it means "head's relation is tail.". 3 | 4 | Verify the claim based on the evidence set. (True means that everything contained in the claim is supported by the evidence.) 5 | Please note that the unit is not important. (e.g. "98400" is also same as 98.4kg) 6 | Choose one of {True, False}, and give me the one-sentence evidence. 7 | 8 | Examples) 9 | 10 | Claim A: Ahmad Kadhim Assad's club is Al-Zawra'a SC. 11 | Evidence set: [['Ahamad_Kadhim', 'clubs', "Al-Zawra'a SC"]] 12 | Answer: True, based on the evidence set, Ahmad Kadhim Assad's club is Al-Zawra'a SC. 13 | 14 | 15 | Claim B: Yeah! I know that a TV show, which starred Tim Brooke-Taylor, first aired on 3rd October 1983! 16 | Evidence set: [['Bananaman', 'firstAired', '"1983-10-03"'], ['Bananaman', 'starring', 'Tim_Brooke-Taylor']] 17 | Answer: True, the claim is supported by the evidence since Bananaman refers to the TV show. 18 | 19 | 20 | Claim C: Really? Jamie Lawrence is the music composer of the 83 minute 'Death on a Factory Farm' film, directed by Sarah Teale! 21 | Evidence set: [['Jamie_Lawrence', 'composer', 'Death_on_a_Factory_Farm'], ['Death_on_a_Factory_Farm', 'director', 'Sarah_Teale']] 22 | Answer: False, there is no evidence for the 83 minute length. 23 | 24 | 25 | Claim D: Do you know Milan Hodža? he had a religion. 26 | Evidence set: [[]] 27 | Answer: False, there is no evidence that Milan had a religion. 28 | 29 | 30 | Claim E: No, but the leader of the United States is not Olena Serdiuk. 31 | Evidence set: [[]] 32 | Answer: True, based on the evidence set, there is no information that the leader of the United States is Olena Serdiuk. 33 | 34 | 35 | Claim F: Brandon Carter was born in England and graduated from the University of Cambridge where the current Chancellor is Leszek Borysiewicz. 36 | Evidence set: [['Brandon_Carter', 'almaMater', 'University_of_Cambridge'], ['Brandon_Carter', 'birthPlace', 'England'], ['University_of_Cambridge', 'viceChancellor', 'Leszek_Borysiewicz']] 37 | Answer: True, everything of the claim is supported by the evidence set. 38 | 39 | 40 | Claim G: 'A film' was produced by Anatole de Grunwald, directed by Harold French, with cinematography done by Bernard Knowles. 41 | Evidence set: [['Unpublished_Story', 'director', 'Harold_French'], ['Unpublished_Story', 'cinematography', 'Bernard_Knowles']] 42 | Answer: False, there is no information about the producer of 'Unpublished_Story'. 43 | 44 | 45 | Claim H: Yes, with a floor count of 45, 200 Public Square is located in Cleveland in the United States. 46 | Evidence set: [['200_Public_Square', 'location', 'Cleveland'], ['200_Public_Square', 'floorCount', '"45"'], ['Cleveland', 'country', 'United_States']] 47 | Answer: True, everything of the claim is supported by the evidence set. 48 | 49 | 50 | Claim I: Bananaman the TV series starred by a person was shown on the company and the company headquarters is called Broadcasting House. 51 | Evidence set: [['Bananaman', 'starring', 'Bill_Oddie'], ['Bananaman', 'network', 'Broadcasting_House'], ['Bananaman', 'locationCity', 'Broadcasting_House']] 52 | Answer: True, everything of the claim is supported by the evidence set. 53 | 54 | 55 | Claim J: The place, designed by Huseyin Butuner and Hilmi Guner, is located in a country, where the leader is Artur Rasizade. 56 | Evidence set: [['Azerbaijan', 'leaderName', 'Artur_Rasizade'], ["Baku_Turkish_Martyrs'_Memorial", 'designer', '"Hüseyin Bütüner and Hilmi Güner"'], ["Baku_Turkish_Martyrs'_Memorial", 'location', 'Azerbaijan']] 57 | Answer: True, everything of the claim is supported by the evidence set. 58 | 59 | 60 | Claim K: AIDA Cruise line operated the ship which was built by Meyer Werft in Townsend, Poulshot, Wiltshire. 61 | Evidence set: [['AIDAstella', 'shipBuilder', 'Meyer_Werft'], ['AIDAstella', 'shipOperator', 'AIDA_Cruises']] 62 | Answer: False, there is no evidence for Townsend, Poulshot, Wiltshire. 63 | 64 | 65 | Claim L: An academic journal with code IJPHDE is also Acta Math. Hungar. 66 | Evidence set: [[]] 67 | Answer: False, there is no evidence that the academic journal is also Acta Math. Hungar. 68 | 69 | 70 | Now let's verify the Claim based on the Evidence set. 71 | Claim: <<<>>> 72 | Evidence set: <<<>>> 73 | Answer: -------------------------------------------------------------------------------- /metaqa/data/preprocess.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import json 3 | import jsonlines 4 | import argparse 5 | import re 6 | 7 | if __name__ == "__main__": 8 | 9 | parser = argparse.ArgumentParser(description="Parsing input arguments.") 10 | parser.add_argument('--test_1_hop', type=str, required=True, help='Path for metaqa 1-hop test set.') 11 | parser.add_argument('--test_2_hop', type=str, required=True, help='Path for metaqa 2-hop test set.') 12 | parser.add_argument('--test_3_hop', type=str, required=True, help='Path for metaqa 3-hop test set.') 13 | parser.add_argument('--kb', type=str, required=True, help='Path for metaqa kb.') 14 | 15 | args = parser.parse_args() 16 | 17 | test_1_hop = args.test_1_hop 18 | test_2_hop = args.test_2_hop 19 | test_3_hop = args.test_3_hop 20 | kb = args.kb 21 | 22 | KG_construct = {} 23 | with open(kb, 'r') as f: 24 | for line in f: 25 | head = line.strip().split('|')[0] 26 | relation = line.strip().split('|')[1] 27 | tail = line.strip().split('|')[2] 28 | 29 | try: 30 | KG_construct[head][relation].append(tail) 31 | except: 32 | try: 33 | KG_construct[head][relation] = [tail] 34 | except: 35 | KG_construct[head] = {} 36 | KG_construct[head][relation] = [tail] 37 | 38 | try: 39 | KG_construct[tail]['~'+relation].append(head) 40 | except: 41 | try: 42 | KG_construct[tail]['~'+relation] = [head] 43 | except: 44 | KG_construct[tail] = {} 45 | KG_construct[tail]['~'+relation] = [head] 46 | 47 | with open('data/metaqa_kg.pickle', 'wb') as f: 48 | pickle.dump(KG_construct, f) 49 | 50 | 51 | onehop = {} 52 | with open(test_1_hop, 'r') as f: 53 | for line in f: 54 | seperated = line.strip().split('\t') 55 | entities = re.findall(r'\[(.*?)\]', seperated[0]) 56 | labels = seperated[1] 57 | labels = labels.split('|') 58 | onehop[seperated[0]+'?'] = {'entity_set': [entities[0]], 'Label': labels} 59 | 60 | with jsonlines.open(f'data/onehop_test_set.jsonl', mode='w') as w: 61 | total = 0 62 | for i, sample in enumerate(list(onehop)): 63 | new_sample = {} 64 | new_sample["question_id"] = i+1 65 | new_sample["question"] = sample 66 | new_sample["entity_set"] = onehop[sample]["entity_set"] 67 | new_sample["Label"] = onehop[sample]["Label"] 68 | w.write(new_sample) 69 | 70 | 71 | twohop = {} 72 | with open(test_2_hop, 'r') as f: 73 | for line in f: 74 | seperated = line.strip().split('\t') 75 | entities = re.findall(r'\[(.*?)\]', seperated[0]) 76 | labels = seperated[1] 77 | labels = labels.split('|') 78 | twohop[seperated[0]+'?'] = {'entity_set': [entities[0]], 'Label': labels} 79 | 80 | with jsonlines.open(f'data/twohop_test_set.jsonl', mode='w') as w: 81 | total = 0 82 | for i, sample in enumerate(list(twohop)): 83 | new_sample = {} 84 | new_sample["question_id"] = i+1 85 | new_sample["question"] = sample 86 | new_sample["entity_set"] = twohop[sample]["entity_set"] 87 | new_sample["Label"] = twohop[sample]["Label"] 88 | w.write(new_sample) 89 | 90 | 91 | threehop = {} 92 | with open(test_3_hop, 'r') as f: 93 | for line in f: 94 | seperated = line.strip().split('\t') 95 | entities = re.findall(r'\[(.*?)\]', seperated[0]) 96 | labels = seperated[1] 97 | labels = labels.split('|') 98 | threehop[seperated[0]+'?'] = {'entity_set': [entities[0]], 'Label': labels} 99 | 100 | with jsonlines.open(f'data/threehop_test_set.jsonl', mode='w') as w: 101 | total = 0 102 | for i, sample in enumerate(list(threehop)): 103 | new_sample = {} 104 | new_sample["question_id"] = i+1 105 | new_sample["question"] = sample 106 | new_sample["entity_set"] = threehop[sample]["entity_set"] 107 | new_sample["Label"] = threehop[sample]["Label"] 108 | w.write(new_sample) 109 | -------------------------------------------------------------------------------- /metaqa/meta_1hop_prompts/relation_retrieval_prompt.txt: -------------------------------------------------------------------------------- 1 | I will give you a set of words. 2 | Find the elements from Words set which are most semantically related to the given Sentence. You can choose the one that is most similar, or you can choose less than three elements if it is ambiguous. 3 | 4 | Examples) 5 | Sentence A: what words describe [Coming Home]? 6 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 7 | Answer: ['has_tags'] 8 | 9 | 10 | Sentence B: what films does [Faye Wong] appear in? 11 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 12 | Answer: ['starred_actors'] 13 | 14 | 15 | Sentence C: what films are about [haneke]? 16 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 17 | Answer: ['has_tags'] 18 | 19 | 20 | Sentence D: who acted in the movie [Inescapable]? 21 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by', 'release_year'] 22 | Answer: ['starred_actors'] 23 | 24 | 25 | Sentence E: can you name a film directed by [William Cameron Menzies]? 26 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 27 | Answer: ['directed_by'] 28 | 29 | 30 | Sentence F: what sort of movie is [Witness for the Prosecution]? 31 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by', 'has_tags', 'release_year'] 32 | Answer: ['has_genre'] 33 | 34 | 35 | Sentence G: what type of film is [The Mouse That Roared]? 36 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by'] 37 | Answer: ['has_genre'] 38 | 39 | 40 | Sentence H: what is the primary language in the film [Blackboards]? 41 | Words set: ['has_genre', 'directed_by', 'written_by', 'has_tags', 'in_language', 'release_year'] 42 | Answer: ['in_language'] 43 | 44 | 45 | Sentence I: who is the creator of the film script for [The Truth of Lie]? 46 | Words set: ['has_genre', 'directed_by', 'written_by', 'in_language'] 47 | Answer: ['written_by'] 48 | 49 | 50 | Sentence J: what was the release year of the film [The Return of Doctor X]? 51 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by', 'has_tags', 'release_year'] 52 | Answer: ['release_year'] 53 | 54 | 55 | Sentence K: which topics is movie [Topper] about? 56 | Words set: ['starred_actors', 'directed_by', 'written_by', 'has_tags', 'in_language', 'release_year'] 57 | Answer: ['has_tags'] 58 | 59 | 60 | Sentence L: describe the movie [The Mouse on the Moon] in a few words? 61 | Words set: ['starred_actors', 'directed_by', 'written_by', 'has_tags', 'has_genre', 'in_language', 'release_year'] 62 | Answer: ['has_tags'] 63 | 64 | 65 | Now let's find the elements. 66 | Sentence: <<<>>> 67 | Words set: <<<>>> 68 | Answer: -------------------------------------------------------------------------------- /metaqa/meta_1hop_prompts/sentence_divide_prompt.txt: -------------------------------------------------------------------------------- 1 | Please divide the given sentence into several sentences each of which can be represented by one triplet. The generated sentences should be numbered and formatted as follows: #(number). (sentence), (entity set). 2 | 3 | Examples) 4 | Sentence A: what words describe [Coming Home]? 5 | Entity set: ['Coming Home'] 6 | --> Divided: 7 | 1. can you give a few words describing what [Coming Home]?, Entity set: ['Coming Home'] 8 | 9 | 10 | Sentence B: what films does [Faye Wong] appear in? 11 | Entity set: ['Faye Wong'] 12 | --> Divided: 13 | 1. what films does [Faye Wong] appear in?, Entity set: ['Faye Wong'] 14 | 15 | 16 | Sentence C: what films are about [haneke]? 17 | Entity set: ['haneke'] 18 | --> Divided: 19 | 1. what films are about [haneke]?, Entity set: ['haneke'] 20 | 21 | 22 | Sentence D: who acted in the movie [Inescapable]? 23 | Entity set: ['Inescapable'] 24 | --> Divided: 25 | 1. who acted in the movie [Inescapable]?, Entity set: ['Inescapable'] 26 | 27 | 28 | Sentence E: can you name a film directed by [William Cameron Menzies]? 29 | Entity set: ['William Cameron Menzies'] 30 | --> Divided: 31 | 1. can you name a film directed by [William Cameron Menzies]?, Entity set: ['William Cameron Menzies'] 32 | 33 | 34 | Sentence F: what sort of movie is [Witness for the Prosecution]? 35 | Entity set: ['Witness for the Prosecution'] 36 | --> Divided: 37 | 1. what sort of movie is [Witness for the Prosecution]?, Entity set: ['Witness for the Prosecution'] 38 | 39 | 40 | Sentence G: what type of film is [The Mouse That Roared]? 41 | Entity set: ['The Mouse That Roared'] 42 | --> Divided: 43 | 1. what type of film is [The Mouse That Roared]?, Entity set: ['The Mouse That Roared'] 44 | 45 | 46 | Sentence H: what is the primary language in the film [Blackboards]? 47 | Entity set: ['Blackboards'] 48 | --> Divided: 49 | 1. what is the primary language in the film [Blackboards]?, Entity set: ['Blackboards'] 50 | 51 | 52 | Sentence I: who is the creator of the film script for [The Truth of Lie]? 53 | Entity set: ['The Truth of Lie'] 54 | --> Divided: 55 | 1. who is the creator of the film script for [The Truth of Lie]?, Entity set: ['The Truth of Lie'] 56 | 57 | 58 | Sentence J: what was the release year of the film [The Return of Doctor X]? 59 | Entity set: [The Return of Doctor X'] 60 | --> Divided: 61 | 1. what was the release year of the film [The Return of Doctor X]?, Entity set: ['The Return of Doctor X'] 62 | 63 | 64 | Sentence K: which topics is movie [Topper] about? 65 | Entity set: ['Topper'] 66 | --> Divided: 67 | 1. which topics is movie [Topper] about?, Entity set: ['Topper'] 68 | 69 | 70 | Sentence L: describe the movie [The Mouse on the Moon] in a few words? 71 | Entity set: ['The Mouse on the Moon'] 72 | --> Divided: 73 | 1. describe the movie [The Mouse on the Moon] in a few words?, Entity set: ['The Mouse on the Moon'] 74 | 75 | 76 | Your Task) 77 | Sentence: <<<>>> 78 | Entity set: <<<>>> 79 | --> Divided: -------------------------------------------------------------------------------- /metaqa/meta_1hop_prompts/verify_claim_with_evidence.txt: -------------------------------------------------------------------------------- 1 | Answer the questions based on evidence. 2 | Each evidence is in the form of [head, relation, tail] and it means "head's relation is tail.". 3 | If you think a question can have multiple answers, you must choose one and answer it. 4 | 5 | Examples) 6 | 7 | Claim A: what words describe [Coming Home]? 8 | Evidence set: [['Coming Home', 'has_genre', 'Drama'], ['Coming Home', 'has_genre', 'War'], ['Coming Home', 'has_tags', 'vietnam'], ['Coming Home', 'has_tags', 'hal ashby']] 9 | Answer: 'hal ashby' 10 | 11 | 12 | Claim B: what films does [Faye Wong] appear in? 13 | Evidence set: [['Chungking Express', 'starred_actors', 'Faye Wong'], ['Chinese Odyssey 2002', 'starred_actors', 'Faye Wong']] 14 | Answer: 'Chungking Express' 15 | 16 | 17 | Claim C: what films are about [haneke]? 18 | Evidence set: [['Code Unknown', 'has_tag', 'haneke'], ['The Piano Teacher', 'has_tag', 'haneke'], ['Funny Games', 'has_tag', 'haneke'], ['Time of the Wolf', 'has_tag', 'haneke']] 19 | Answer: 'The Piano Teacher' 20 | 21 | 22 | Claim D: who acted in the movie [Inescapable]? 23 | Evidence set: [['Inescapable', 'directed_by', 'Ruba Nadda'], ['Inescapable', 'written_by', 'Ruba Nadda'], ['Inescapable', 'starred_actors', 'Marisa Tomei'], ['Inescapable', 'starred_actors', 'Joshua Jackson'], ['Inescapable', 'starred_actors', 'Alexander Siddig']] 24 | Answer: 'Marisa Tomei' 25 | 26 | 27 | Claim E: can you name a film directed by [William Cameron Menzies]? 28 | Evidence set: [['Things to Come', 'directed_by', 'William Cameron Menzies']] 29 | Answer: 'Things to Come' 30 | 31 | 32 | Claim F: what sort of movie is [Witness for the Prosecution]? 33 | Evidence set: [['Witness for the Prosecution', 'has_tags', 'bd-r'], ['Witness for the Prosecution', 'has_genre', 'Drama'], ['Witness for the Prosecution', 'has_tags', 'courtroom']] 34 | Answer: 'Drama' 35 | 36 | 37 | Claim G: what type of film is [The Mouse That Roared]? 38 | Evidence set: [['The Mouse That Roared', 'has_genre', 'Comedy'], ['The Mouse That Roared', 'has_tags', 'satirical'], ['The Mouse That Roared', 'has_tags', 'peter sellers']] 39 | Answer: 'Comedy' 40 | 41 | 42 | Claim H: what is the primary language in the film [Blackboards]? 43 | Evidence set: [['Blackboards', 'in_language', 'Kurdish'], ['Blackboards', 'has_genre', 'War'], ['Blackboards', 'has_tags', 'samira makhmalbaf']] 44 | Answer: 'Kurdish' 45 | 46 | 47 | Claim I: who is the creator of the film script for [The Truth of Lie]? 48 | Evidence set: [['The Truth of Lie', 'written_by', 'Roland Reber'], ['The Truth of Lie', 'directed_by', 'Roland Reber'], ['The Truth of Lie, 'has_genre', 'Thriller']] 49 | Answer: 'Roland Reber' 50 | 51 | 52 | Claim J: what was the release year of the film [The Return of Doctor X]? 53 | Evidence set: [['The Return of Doctor X', 'written_by', 'William J. Makin'], ['The Return of Doctor X', 'release_year', '1939'], ['The Return of Doctor X', 'has_tags', 'humphrey bogart']] 54 | Answer: '1939' 55 | 56 | 57 | Claim K: which topics is movie [Topper] about? 58 | Evidence set: [['Topper', 'has_tags', 'ghosts'], ['Topper', 'has_tags', 'norman z. mcleod'], ['Topper', 'has_genre', 'Comedy']] 59 | Answer: 'ghosts' 60 | 61 | 62 | Claim L: describe the movie [The Mouse on the Moon] in a few words? 63 | Evidence set: [['The Mouse on the Moon', 'has_tags', 'bd-r'], ['The Mouse on the Moon', 'has_genre', 'Comedy'], ['The Mouse on the Moon', 'written_by', 'Leonard Wibberley']] 64 | Answer: 'bd-r' 65 | 66 | 67 | Now let's verify the Claim based on the Evidence set. Please do not say there is no evdience, you must say the most related one entity from the evidence set. 68 | Claim: <<<>>> 69 | Evidence set: <<<>>> 70 | Answer: -------------------------------------------------------------------------------- /metaqa/meta_2hop_prompts/relation_retrieval_prompt.txt: -------------------------------------------------------------------------------- 1 | I will give you a set of words. 2 | Find the elements from Words set which are most semantically related to the given Sentence. You can choose the one that is most similar, or you can choose less than three elements if it is ambiguous. 3 | 4 | Examples) 5 | Sentence A: which person wrote the films 6 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 7 | Answer: ['written_by'] 8 | 9 | 10 | Sentence B: which directors co-directed movies with 11 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 12 | Answer: ['directed_by'] 13 | 14 | 15 | Sentence C: the movies directed by [Kresten Vestbjerg Andersen] 16 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 17 | Answer: ['directed_by'] 18 | 19 | 20 | Sentence D: the movies were in which genres 21 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by', 'release_year'] 22 | Answer: ['has_genre'] 23 | 24 | 25 | Sentence E: the films were released in which years 26 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 27 | Answer: ['release_year'] 28 | 29 | 30 | Sentence F: the scriptwriter of [First Monday in October] 31 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by', 'has_tags', 'release_year'] 32 | Answer: ['written_by'] 33 | 34 | 35 | Sentence G: what are the languages spoken in the films 36 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by', 'in_language'] 37 | Answer: ['in_language'] 38 | 39 | 40 | Sentence H: [Oliver Cooper] acted films 41 | Words set: ['has_genre', 'directed_by', 'starred_actors', 'has_tags', 'in_language', 'release_year'] 42 | Answer: ['starred_actors'] 43 | 44 | 45 | Sentence I: who co-starred with 46 | Words set: ['has_genre', 'directed_by', 'written_by', 'in_language', 'starred_actors'] 47 | Answer: ['starred_actors'] 48 | 49 | 50 | Sentence J: what types are the films 51 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by', 'has_tags', 'release_year'] 52 | Answer: ['written_by'] 53 | 54 | 55 | Sentence K: when were the movies released 56 | Words set: ['starred_actors', 'directed_by', 'written_by', 'has_tags', 'in_language', 'release_year'] 57 | Answer: ['release_year'] 58 | 59 | 60 | Sentence L: which movies share the screenwrite 61 | Words set: ['starred_actors', 'directed_by', 'written_by', 'has_tags', 'has_genre', 'in_language', 'release_year'] 62 | Answer: ['written_by'] 63 | 64 | 65 | Now let's find the elements. 66 | Sentence: <<<>>> 67 | Words set: <<<>>> 68 | Answer: -------------------------------------------------------------------------------- /metaqa/meta_2hop_prompts/sentence_divide_prompt.txt: -------------------------------------------------------------------------------- 1 | Please divide the given sentence into several sentences each of which can be represented by one triplet. The generated sentences should be numbered and formatted as follows: #(number). (sentence), (entity set). 2 | 3 | Examples) 4 | Sentence A: which person wrote the films directed by [Yuriy Norshteyn]? 5 | Entity set: ['Yuriy Norshteyn'] 6 | --> Divided: 7 | 1. the films directed by [Yuriy Norshteyn], Entity set: ['Yuriy Norshteyn'] 8 | 2. which person wrote the films, Entity set: [] 9 | 10 | 11 | Sentence B: which directors co-directed movies with [Ridley Scott]? 12 | Entity set: ['Ridley Scott'] 13 | --> Divided: 14 | 1. which directors co-directed movies with [Ridley Scott], Entity set: ['Ridley Scott'] 15 | 2. which directors co-directed movies with, Entity set: [] 16 | 17 | 18 | Sentence C: who are the writers of the movies directed by [Kresten Vestbjerg Andersen]? 19 | Entity set: ['Kresten Vestbjerg Andersen'] 20 | --> Divided: 21 | 1. the movies directed by [Kresten Vestbjerg Andersen], Entity set: ['Kresten Vestbjerg Andersen'] 22 | 2. who are the writers of the movies, Entity set: [] 23 | 24 | 25 | Sentence D: the movies directed by [David Atkins] were in which genres? 26 | Entity set: ['David Atkins'] 27 | --> Divided: 28 | 1. the movies directed by [David Atkins], Entity set: ['David Atkins'] 29 | 2. the movies were in which genres, Entity set: [] 30 | 31 | 32 | Sentence E: the films written by [Scott Lobdell] were released in which years? 33 | Entity set: ['Scott Lobdell'] 34 | --> Divided: 35 | 1. the films written by [Scott Lobdell], Entity set: ['Scott Lobdell'] 36 | 2. the films were released in which years, Entity set: [] 37 | 38 | 39 | Sentence F: the scriptwriter of [First Monday in October] also wrote movies? 40 | Entity set: ['First Monday in October'] 41 | --> Divided: 42 | 1. the scriptwriter of [First Monday in October] also wrote movies, Entity set: ['First Monday in October'] 43 | 2. also wrote movies, Entity set: [] 44 | 45 | 46 | Sentence G: what are the languages spoken in the films starred by [Terence Hill]? 47 | Entity set: ['Terence Hill'] 48 | --> Divided: 49 | 1. the films starred by [Terence Hill], Entity set: ['Terence Hill'] 50 | 2. what are the languages spoken in the films, Entity set: [] 51 | 52 | 53 | Sentence H: who is listed as director of [Oliver Cooper] acted films? 54 | Entity set: ['Oliver Cooper'] 55 | --> Divided: 56 | 1. [Oliver Cooper] acted films, Entity set: ['Oliver Cooper'] 57 | 2. who is listed as director of, Entity set: [] 58 | 59 | 60 | Sentence I: who co-starred with [Stephen Furst]? 61 | Entity set: ['Stephen Furst'] 62 | --> Divided: 63 | 1. co-starred with [Stephen Furst], Entity set: ['Stephen Furst'] 64 | 2. who co-starred, Entity set: [] 65 | 66 | 67 | Sentence J: what types are the films written by [Polaris Banks]? 68 | Entity set: ['Polaris Banks'] 69 | --> Divided: 70 | 1. the films written by [Polaris Banks], Entity set: ['Polaris Banks'] 71 | 2. what types are the films, Entity set: [] 72 | 73 | 74 | Sentence K: when were the movies written by [Phillip Borsos] released? 75 | Entity set: ['Phillip Borsos'] 76 | --> Divided: 77 | 1. the movies written by [Phillip Borsos], Entity set: ['Phillip Borsos'] 78 | 2. when were the movies released, Entity set: [] 79 | 80 | 81 | Sentence L: which movies share the screenwriter with [Jesus Henry Christ]? 82 | Entity set: ['Jesus Henry Christ'] 83 | --> Divided: 84 | 1. screenwriter with [Jesus Henry Christ], Entity set: ['Jesus Henry Christ'] 85 | 2. which movies share the screenwriter, Entity set: [] 86 | 87 | 88 | Your Task) 89 | Sentence: <<<>>> 90 | Entity set: <<<>>> 91 | --> Divided: -------------------------------------------------------------------------------- /metaqa/meta_2hop_prompts/verify_claim_with_evidence.txt: -------------------------------------------------------------------------------- 1 | Answer the questions based on evidence. 2 | Each evidence is in the form of [head, relation, tail] and it means "head's relation is tail.". 3 | If you think a question can have multiple answers, you must choose one and answer it. 4 | 5 | Examples) 6 | 7 | Claim A: which person wrote the films directed by [Yuriy Norshteyn]? 8 | Evidence set: [['Tale of Tales', 'directed_by', 'Yuriy Norshteyn'], ['Tale of Tales', 'written_by', 'Yuriy Norshteyn'], ['Hedgehog in the Fog', 'written_by', 'Sergei Kozlov'], ['Hedgehog in the Fog', 'directed_by', 'Yuriy Norshteyn']] 9 | Answer: 'Sergei Kozlov' 10 | 11 | 12 | Claim B: who are the writers of the movies directed by [Kresten Vestbjerg Andersen]? 13 | Evidence set: [['Ronal the Barbarian', 'written_by', 'Philip Einstein Lipski'], ['Ronal the Barbarian', 'directed_by', 'Kresten Vestbjerg Andersen'], ['Ronal the Barbarian', 'written_by', 'Kresten Vestbjerg Andersen'], ['Ronal the Barbarian', 'written_by', 'Thorbjørn Christoffersen']] 14 | Answer: 'Philip Einstein Lipski' 15 | 16 | 17 | Claim C: the movies directed by [David Atkins] were in which genres? 18 | Evidence set: [['Novocaine', 'has_genre', 'Comedy'], ['Novocaine', 'written_by', 'David Atkins'], ['Novocaine', 'directed_by', 'David Atkins']] 19 | Answer: 'Comedy' 20 | 21 | 22 | Claim D: the films written by [Scott Lobdell] were released in which years? 23 | Evidence set: [['Man of the House', 'release_year', '2005'], ['Man of the House', 'written_by', 'Scott Lobdell'], ['Man of the House', 'release_year', '1995'], ['Man of the House', 'has_genre', 'Comedy']] 24 | Answer: '1995' 25 | 26 | 27 | Claim E: what are the languages spoken in the films starred by [Terence Hill]? 28 | Evidence set: [['Terence Hill', 'starred_actors', 'They Call Me Trinity'], ['Terence Hill', 'starred_actors', 'They Call Me Renegade'], ['Terence Hill', 'starred_actors', 'Go for It'], ['They Call Me Renegade', 'in_language', 'Italian']] 29 | Answer: 'Italian' 30 | 31 | 32 | Claim F: who is listed as director of [Oliver Cooper] acted films? 33 | Evidence set: [['Project X', 'directed_by', 'Jonathan Kaplan'], ['Project X', 'starred_actors', 'Jonathan Daniel Brown'], ['Project X', 'starred_actors', 'Oliver Cooper'], ['Project X', 'directed_by', 'Nima Nourizadeh'], ['Project X', 'written_by', 'Matt Drake']] 34 | Answer: 'Nima Nourizadeh' 35 | 36 | 37 | Claim G: who co-starred with [Stephen Furst]? 38 | Evidence set: [['The Dream Team', 'starred_actors', 'Michael Keaton'], ['The Dream Team', 'starred_actors', 'Peter Boyle'], ['The Dream Team', 'starred_actors', 'Christopher Lloyd'], ['The Dream Team', 'directed_by', 'Howard Zieff'], ['The Dream Team', 'written_by', 'David Loucka'], ['The Dream Team', 'starred_actors', 'Stephen Furst']] 39 | Answer: 'Peter Boyle' 40 | 41 | 42 | Claim H: what types are the films written by [Polaris Banks]? 43 | Evidence set: [['Casey Jones', 'written_by', 'Polaris Banks'], ['Casey Jones', 'directed_by', 'Polaris Banks'], ['Casey Jones', 'has_genre', 'Short'], ['Casey Jones', 'release_year', '2011']] 44 | Answer: 'Short' 45 | 46 | 47 | Claim I: when were the movies written by [Phillip Borsos] released? 48 | Evidence set: [['The Grey Fox', 'directed_by', 'Phillip Borsos'], ['The Grey Fox', 'release_year', '1982'], ['One Magic Christmas', 'written_by', 'Phillip Borsos'], ['One Magic Christmas', 'release_year', '1985']] 49 | Answer: '1985' 50 | 51 | 52 | Claim J: which movies share the screenwriter with [Jesus Henry Christ]? 53 | Evidence set: [['Jesus Henry Christ', 'directed_by', 'Dennis Lee'], ['Jesus Henry Christ', 'has_genre', 'Comedy'], ['Jesus Henry Christ', 'written_by', 'Dennis Lee'], ['Fireflies in the Garden', 'written_by', 'Dennis Lee']] 54 | Answer: 'Fireflies in the Garden' 55 | 56 | 57 | Claim K: which directors co-directed movies with [Ridley Scott]? 58 | Evidence set: [['The Counselor', 'directed_by', 'Ridley Scott'], ['Legend', 'directed_by', 'Ridley Scott'], ['Body of Lies', 'directed_by', 'Ridley Scott'], ['Blade Runner', 'directed_by', 'Ridley Scott'], ['Someone to Watch Over Me', 'directed_by', 'Ridley Scott'], ['Gladiator', 'directed_by', 'Ridley Scott'], ['Black Hawk Down', 'directed_by', 'Ridley Scott'], ['Black Rain', 'directed_by', 'Ridley Scott'], ['Robin Hood', 'directed_by', 'Ridley Scott'], ['Gladiator', 'directed_by', 'Rowdy Herrington']] 59 | Answer: 'Rowdy Herrington' 60 | 61 | 62 | Claim L: the scriptwriter of [First Monday in October] also wrote movies? 63 | Evidence set: [['First Monday in October', 'written_by', 'Robert E. Lee'], ['First Monday in October', 'written_by', 'Jerome Lawrence'], ['Inherit the Wind', 'written_by', 'Jerome Lawrence']] 64 | Answer: 'Inherit the Wind' 65 | 66 | 67 | Now let's verify the Claim based on the Evidence set. Please do not say there is no evdience, you must say the most related one entity from the evidence set. 68 | Claim: <<<>>> 69 | Evidence set: <<<>>> 70 | Answer: -------------------------------------------------------------------------------- /metaqa/meta_3hop_prompts/relation_retrieval_prompt.txt: -------------------------------------------------------------------------------- 1 | I will give you a set of words. 2 | Find the elements from Words set which are most semantically related to the given Sentence. You can choose the one that is most similar, or you can choose less than three elements if it is ambiguous. 3 | 4 | Examples) 5 | Sentence A: their films also directed [Agatha] 6 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 7 | Answer: ['directed_by'] 8 | 9 | 10 | Sentence B: when did the movies release 11 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 12 | Answer: ['release_year'] 13 | 14 | 15 | Sentence C: what types are the movies 16 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 17 | Answer: ['has_genre'] 18 | 19 | 20 | Sentence D: were directed by who 21 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by', 'release_year'] 22 | Answer: ['directed_by'] 23 | 24 | 25 | Sentence E: what are the languages spoken in the movies 26 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'has_imdb_rating', 'written_by', 'has_tags', 'has_imdb_votes', 'in_language', 'release_year'] 27 | Answer: ['in_language'] 28 | 29 | 30 | Sentence F: the movies that share actors with 31 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by', 'has_tags', 'release_year'] 32 | Answer: ['starred_actors'] 33 | 34 | 35 | Sentence G: the writer of [French Kiss] 36 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by'] 37 | Answer: ['written_by'] 38 | 39 | 40 | Sentence H: movies for the writer 41 | Words set: ['has_genre', 'directed_by', 'written_by', 'has_tags', 'in_language', 'release_year'] 42 | Answer: ['written_by'] 43 | 44 | 45 | Sentence I: what are the languages spoken in the film 46 | Words set: ['has_genre', 'directed_by', 'written_by', 'in_language'] 47 | Answer: ['in_language'] 48 | 49 | 50 | Sentence J: the films that share actors 51 | Words set: ['has_genre', 'starred_actors', 'directed_by', 'written_by', 'has_tags', 'release_year'] 52 | Answer: ['starred_actors'] 53 | 54 | 55 | Sentence K: what were the release dates of movies 56 | Words set: ['starred_actors', 'directed_by', 'written_by', 'has_tags', 'in_language', 'release_year'] 57 | Answer: ['release_year'] 58 | 59 | 60 | Sentence L: who is listed as screenwriter of the films 61 | Words set: ['starred_actors', 'directed_by', 'written_by', 'has_tags', 'has_genre', 'in_language', 'release_year'] 62 | Answer: ['written_by'] 63 | 64 | 65 | Now let's find the elements. 66 | Sentence: <<<>>> 67 | Words set: <<<>>> 68 | Answer: -------------------------------------------------------------------------------- /metaqa/meta_3hop_prompts/sentence_divide_prompt.txt: -------------------------------------------------------------------------------- 1 | Please divide the given sentence into several sentences each of which can be represented by one triplet. The generated sentences should be numbered and formatted as follows: #(number). (sentence), (entity set). 2 | 3 | Examples) 4 | 5 | Sentence A: when did the movies starred by [The Night Porter] actors release? 6 | Entity set: ['The Night Porter'] 7 | --> Divided: 8 | 1. when did the movies release, Entity set: [] 9 | 2. movies starred by, Entity set: [] 10 | 3. [The Night Porter] actors, Entity set: ['The Night Porter'] 11 | 12 | 13 | Sentence B: what types are the movies directed by the director of [Oklahoma Crude]? 14 | Entity set: ['Oklahoma Crude'] 15 | --> Divided: 16 | 1. what types are the movies, Entity set: [] 17 | 2. movies directed by, Entity set: [] 18 | 3. the director of [Oklahoma Crude], Entity set: ['Oklahoma Crude'] 19 | 20 | 21 | Sentence C: the films written by the screenwriter of [To Catch a Thief] were directed by who? 22 | Entity set: ['To Catch a Thief'] 23 | --> Divided: 24 | 1. the films written by, Entity set: [] 25 | 2. the screenwriter of [To Catch a Thief], Entity set: ['To Catch a Thief'] 26 | 3. were directed by who, Entity set: [] 27 | 28 | 29 | Sentence D: what are the languages spoken in the movies written by [The Beekeeper] writers? 30 | Entity set: ['The Beekeeper'] 31 | --> Divided: 32 | 1. what are the languages spoken in the movies, Entity set: [] 33 | 2. the movies written by, Entity set: [] 34 | 3. [The Beekeeper] writers, Entity set: ['The Beekeeper'] 35 | 36 | 37 | Sentence E: what genres do the movies that share actors with [Man of Steel] fall under? 38 | Entity set: ['Man of Steel'] 39 | --> Divided: 40 | 1. what genres do the movies, Entity set: [] 41 | 2. the movies that share actors with, Entity set: [] 42 | 3. actors with [Man of Steel], Entity set: ['Man of Steel'] 43 | 44 | 45 | Sentence F: the films written by the writer of [French Kiss] starred who? 46 | Entity set: ['French Kiss'] 47 | --> Divided: 48 | 1. the films written by, Entity set: [] 49 | 2. the writer of [French Kiss], Entity set: ['French Kiss'] 50 | 3. starred who, Entity set: [] 51 | 52 | 53 | Sentence G: who directed movies for the writer of [King Kelly]? 54 | Entity set: ['King Kelly'] 55 | --> Divided: 56 | 1. who directed movies, Entity set: [] 57 | 2. movies for the writer, Entity set: [] 58 | 3. writer of [King Kelly], Entity set: ['King Kelly'] 59 | 60 | 61 | Sentence H: the films that share actors with the film [Santa Who?] were released in which years? 62 | Entity set: ['Santa Who?'] 63 | --> Divided: 64 | 1. the films that share actors, Entity set: [] 65 | 2. actors with the film [Santa Who?], Entity set: ['Santa Who?'] 66 | 3. were released in which years, Entity set: [] 67 | 68 | 69 | Sentence I: what were the release dates of movies starred by actors in [Charlie Chan at Monte Carlo]? 70 | Entity set: ['Charlie Chan at Monte Carlo'] 71 | --> Divided: 72 | 1. what were the release dates of movies, Entity set: [] 73 | 2. movies starred by, Entity set: [] 74 | 3. actors in [Charlie Chan at Monte Carlo], Entity set: ['Charlie Chan at Monte Carlo'] 75 | 76 | 77 | Sentence J: who is listed as screenwriter of the films directed by the [Blue Collar] director? 78 | Entity set: ['Blue Collar'] 79 | --> Divided: 80 | 1. who is listed as screenwriter of the films, Entity set: [] 81 | 2. films directed by, Entity set: [] 82 | 3. the [Blue Collar] director, Entity set: ['Blue Collar'] 83 | 84 | 85 | Sentence K: who are the writers that the directors of their films also directed [Agatha]? 86 | Entity set: ['Agatha'] 87 | --> Divided: 88 | 1. who are the writers that, Entity set: [] 89 | 2. that the directors of their films, Entity set: [] 90 | 3. their films also directed [Agatha]?, Entity set: ['Agatha'] 91 | 92 | 93 | Sentence L: what are the languages spoken in the films whose actors also appear in [Pascali's Island]? 94 | Entity set: ["Pascali's Island"] 95 | --> Divided: 96 | 1. what are the languages spoken in the films, Entity set: [] 97 | 2. films whose actors, Entity set: [] 98 | 3. actors also appear in [Pascali's Island], Entity set: ["Pascali's Island"] 99 | 100 | 101 | Your Task) 102 | Sentence: <<<>>> 103 | Entity set: <<<>>> 104 | --> Divided: -------------------------------------------------------------------------------- /metaqa/meta_3hop_prompts/verify_claim_with_evidence.txt: -------------------------------------------------------------------------------- 1 | Answer the questions based on evidence. 2 | Each evidence is in the form of [head, relation, tail] and it means "head's relation is tail.". 3 | If you think a question can have multiple answers, you must choose one and answer it. 4 | 5 | Examples) 6 | 7 | Claim A: who are the writers that the directors of their films also directed [Agatha]? 8 | Evidence set: [['Agatha', 'written_by', 'Kathleen Tynan'], ['Agatha', 'directed_by', 'Michael Apted'], ['Gorky Park', 'written_by', 'Dennis Potter'], ['Gorky Park', 'directed_by', 'Michael Apted']] 9 | Answer: 'Dennis Potter' 10 | 11 | 12 | Claim B: when did the movies starred by [The Night Porter] actors release? 13 | Evidence set: ['The Night Porter', 'release_year', '1974'], ['The Night Porter', 'starred_actors', 'Charlotte Rampling'], ['The Mill and the Cross', 'starred_actors', 'Charlotte Rampling'], ['The Mill and the Cross', 'release_year', '2011'], ['The Night Porter', 'starred_actors', 'Dirk Bogarde'], ['Despair', 'starred_actors', 'Dirk Bogarde']] 14 | Answer: '2011' 15 | 16 | 17 | Claim C: what types are the movies directed by the director of [Oklahoma Crude]? 18 | Evidence set: [['The Defiant Ones', 'directed_by', 'Stanley Kramer'], ['Not as a Stranger', 'directed_by', 'Stanley Kramer'], ['Not as a Stranger', 'has_genre', 'Drama'], ['Oklahoma Crude', 'directed_by', 'Stanley Kramer']] 19 | Answer: 'Drama' 20 | 21 | 22 | Claim D: the films written by the screenwriter of [To Catch a Thief] were directed by who? 23 | Evidence set: [['To Catch a Thief', 'written_by', 'John Michael Hayes'], ['To Catch a Thief', 'directed_by', 'Alfred Hitchcock'], ['Torch Song', 'written_by', 'John Michael Hayes'], ['Rear Window', 'written_by', 'John Michael Hayes'], ['To Catch a Thief', 'written_by', 'David Dodge'], ['Torch Song', 'directed_by', 'Charles Walters']] 24 | Answer: 'Charles Walters' 25 | 26 | 27 | Claim E: what are the languages spoken in the movies written by [The Beekeeper] writers? 28 | Evidence set: [['The Beekeeper', 'written_by', 'Theodoros Angelopoulos'], ['The Dust of Time', 'in_language', 'Greek'], ['The Dust of Time', 'written_by', 'Theodoros Angelopoulos'], ['The Dust of Time', 'release_year', '2008']] 29 | Answer: 'Greek' 30 | 31 | 32 | Claim F: what genres do the movies that share actors with [Man of Steel] fall under? 33 | Evidence set: [['Man of Steel', 'starred_actors', 'Amy Adams'], ['Moonlight Serenade', 'starred_actors', 'Amy Adams'], ['Moonlight Serenade', 'has_genre', 'Romance'], ['Man of Steel', 'starred_actors', 'Michael Shannon']] 34 | Answer: 'Romance' 35 | 36 | 37 | Claim G: the films written by the writer of [French Kiss] starred who? 38 | Evidence set: [['French Kiss', 'written_by', 'Adam Brooks'], ['French Kiss', 'starred_actors', 'Kevin Kline'], ['The Invisible Circus', 'written_by', 'Adam Brooks'], ['The Invisible Circus', 'starred_actors', 'Cameron Diaz'], ['French Kiss', 'starred_actors', 'Meg Ryan']] 39 | Answer: 'Cameron Diaz' 40 | 41 | 42 | Claim H: who directed movies for the writer of [King Kelly]? 43 | Evidence set: [['King Kelly', 'written_by', 'Andrew Neel'], ['Darkon', 'directed_by', 'Andrew Neel'], ['Darkon', 'directed_by', 'Luke Meyer']] 44 | Answer: 'Luke Meyer' 45 | 46 | 47 | Claim I: what are the languages spoken in the films whose actors also appear in [Pascali's Island]? 48 | Evidence set: [["Pascali's Island", 'starred_actors', 'Ben Kingsley'], ['Bugsy', 'starred_actors', 'Ben Kingsley'], ['The Confession', 'starred_actors', 'Ben Kingsley'], ['The Confession', 'in_language', 'French']] 49 | Answer: 'French' 50 | 51 | 52 | Claim J: the films that share actors with the film [Santa Who?] were released in which years? 53 | Evidence set: [['Santa Who?', 'starred_actors', 'Leslie Nielsen'], ['Santa Who?', 'release_year', '2000'], ['Santa Who?', 'directed_by', 'William Dear'], ['Prom Night', 'starred_actors', 'Leslie Nielsen'], ['Prom Night', 'release_year', '2008']] 54 | Answer: '2008' 55 | 56 | 57 | Claim K: what were the release dates of movies starred by actors in [Charlie Chan at Monte Carlo]? 58 | Evidence set: [['Charlie Chan at Monte Carlo', 'starred_actors', 'Keye Luke'], ['The Feathered Serpent', 'starred_actors', 'Keye Luke'], ['The Feathered Serpent', 'release_year', '1948'], ['Charlie Chan at Monte Carlo', 'release_year', '1937']] 59 | Answer: '1948' 60 | 61 | 62 | Claim L: who is listed as screenwriter of the films directed by the [Blue Collar] director? 63 | Evidence set: [['Blue Collar', 'directed_by', 'Paul Schrader'], ['Cat People', 'directed_by', 'Paul Schrader'], ['Cat People', 'written_by', 'Alan Ormsby'], ['Blue Collar', 'written_by', 'Paul Schrader']] 64 | Answer: 'Alan Ormsby' 65 | 66 | 67 | Now let's verify the Claim based on the Evidence set. Please do not say there is no evdience, you must say the most related one entity from the evidence set. 68 | Claim: <<<>>> 69 | Evidence set: <<<>>> 70 | Answer: -------------------------------------------------------------------------------- /metaqa/meta_generated_prompts/1hop/1_final_evidence.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiho283/KG-GPT/ea443df2e9d81e4cbaaa6552cffc74fc54f62f6a/metaqa/meta_generated_prompts/1hop/1_final_evidence.pickle -------------------------------------------------------------------------------- /metaqa/meta_generated_prompts/1hop/1_sentence_divide.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiho283/KG-GPT/ea443df2e9d81e4cbaaa6552cffc74fc54f62f6a/metaqa/meta_generated_prompts/1hop/1_sentence_divide.pickle -------------------------------------------------------------------------------- /metaqa/meta_generated_prompts/1hop/1_verification_result.txt: -------------------------------------------------------------------------------- 1 | 'Before the Rain' -------------------------------------------------------------------------------- /metaqa/meta_generated_prompts/2hop/1_final_evidence.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiho283/KG-GPT/ea443df2e9d81e4cbaaa6552cffc74fc54f62f6a/metaqa/meta_generated_prompts/2hop/1_final_evidence.pickle -------------------------------------------------------------------------------- /metaqa/meta_generated_prompts/2hop/1_sentence_divide.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiho283/KG-GPT/ea443df2e9d81e4cbaaa6552cffc74fc54f62f6a/metaqa/meta_generated_prompts/2hop/1_sentence_divide.pickle -------------------------------------------------------------------------------- /metaqa/meta_generated_prompts/2hop/1_verification_result.txt: -------------------------------------------------------------------------------- 1 | George Clooney -------------------------------------------------------------------------------- /metaqa/meta_generated_prompts/3hop/1_final_evidence.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiho283/KG-GPT/ea443df2e9d81e4cbaaa6552cffc74fc54f62f6a/metaqa/meta_generated_prompts/3hop/1_final_evidence.pickle -------------------------------------------------------------------------------- /metaqa/meta_generated_prompts/3hop/1_sentence_divide.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiho283/KG-GPT/ea443df2e9d81e4cbaaa6552cffc74fc54f62f6a/metaqa/meta_generated_prompts/3hop/1_sentence_divide.pickle -------------------------------------------------------------------------------- /metaqa/meta_generated_prompts/3hop/1_verification_result.txt: -------------------------------------------------------------------------------- 1 | 'Japanese' -------------------------------------------------------------------------------- /metaqa/metaqa_1hop_test.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | import time 5 | import concurrent.futures 6 | import re 7 | import openai 8 | import tqdm 9 | import shortuuid 10 | import pickle 11 | 12 | def open_file(filepath): 13 | with open(filepath, 'r', encoding='utf-8') as infile: 14 | return infile.read() 15 | 16 | 17 | def save_file(filepath, content): 18 | with open(filepath, 'w', encoding='utf-8') as outfile: 19 | outfile.write(content) 20 | 21 | 22 | def claim_divider_parse_answer(answer, gt_entity): 23 | processed_answer_set = {} 24 | answer = answer.strip() 25 | splitted_answers = answer.split('\n') 26 | 27 | try: 28 | for nth_answer in splitted_answers: 29 | nth_answer = nth_answer.strip() 30 | for i in range(5): 31 | if str(i+1)+'. ' in nth_answer[:5]: 32 | temp_ans = nth_answer.split(str(i+1)+'. ')[1] 33 | temp_split = temp_ans.split(', Entity set: ') 34 | sentence = temp_split[0] 35 | 36 | entity_set = temp_split[1] 37 | entity_set = entity_set.split('[')[1] 38 | entity_set = entity_set.split(']')[0] 39 | new_entity_set = [] 40 | if len(entity_set) == 1 or len(entity_set) == 2: 41 | new_entity_set.append(entity_set) 42 | elif '"' in entity_set[0] or "'" in entity_set[0]: 43 | new_entity_set.append(entity_set[1:-1]) 44 | else: 45 | new_entity_set.append(entity_set) 46 | break 47 | processed_answer_set[sentence] = new_entity_set 48 | except: 49 | processed_answer_set[sentence] = [gt_entity] 50 | 51 | return processed_answer_set 52 | 53 | 54 | def retrieval_relation_parse_answer(answer): 55 | pattern = r'\[[^\]]+\]' 56 | 57 | matches = re.findall(pattern, answer) 58 | 59 | if len(matches) != 1: 60 | return None 61 | 62 | # Extract the components from the matches and flatten the list 63 | components = [component.strip() for match in matches for component in match.strip('[]').split(',')] 64 | 65 | components = [component.strip("''") if "'" in component else component for component in components ] 66 | return components 67 | 68 | 69 | def get_answer(qid: int, claim: str, gt_entity: str, KG: dict, max_tokens: int): 70 | ans = { 71 | "answer_id": shortuuid.uuid(), 72 | } 73 | entities = re.findall(r'\[(.*?)\]', claim) 74 | sentence_divide_query = open_file('./meta_1hop_prompts/sentence_divide_prompt.txt').replace('<<<>>>', claim).replace('<<<>>>', '['+entities[0]+']') 75 | 76 | #### 1. sentence divide 77 | for _ in range(5): 78 | try: 79 | response = openai.ChatCompletion.create( 80 | model="gpt-3.5-turbo-0613", 81 | messages=[ 82 | {"role": "system", "content": "You are a helpful assistant."}, 83 | { 84 | "role": "user", 85 | "content": sentence_divide_query, 86 | }, 87 | ], 88 | max_tokens=max_tokens, 89 | temperature=0.2, 90 | top_p = 0.1 91 | ) 92 | divide_claim_result = response["choices"][0]["message"]["content"] 93 | divided_claim_list = claim_divider_parse_answer(divide_claim_result, gt_entity) 94 | break 95 | except Exception as e: 96 | print("[ERROR]", e) 97 | time.sleep(5) 98 | with open(f'./meta_generated_prompts/1hop/{qid}_sentence_divide.pickle', 'wb') as f: 99 | pickle.dump(divided_claim_list, f) 100 | 101 | used_all_relations = [] 102 | #### 2. Relation Retrieval 103 | total_evidence = [] 104 | start_relation = [] 105 | extra_relation = [] 106 | for divided_claim, corresponding_entity_set in divided_claim_list.items(): 107 | one_hop_relation_set = list(KG[gt_entity]) 108 | all_relation_set = [] 109 | for one_rel in list(KG[gt_entity]): 110 | all_relation_set.append(one_rel) 111 | one_tails = KG[gt_entity][one_rel] 112 | for one_tail in one_tails: 113 | two_rels = list(KG[one_tail]) 114 | for two_rel in two_rels: 115 | all_relation_set.append(two_rel) 116 | two_tails = KG[one_tail][two_rel] 117 | for two_tail in two_tails: 118 | three_rels = list(KG[two_tail]) 119 | all_relation_set += three_rels 120 | all_relation_set = list(set(all_relation_set)) 121 | new_all_relation_set = [] 122 | 123 | for tar_rel in all_relation_set: 124 | if tar_rel in new_all_relation_set: 125 | continue 126 | if '~' in tar_rel: 127 | new_all_relation_set.append(tar_rel.split('~')[1]) 128 | else: 129 | new_all_relation_set.append(tar_rel) 130 | new_all_relation_set = list(set(new_all_relation_set)) 131 | 132 | if len(corresponding_entity_set) == 1: 133 | if len(list(KG[corresponding_entity_set[0]])) == 1: 134 | start_relation.append(one_hop_relation_set) 135 | continue 136 | else: 137 | relation_retrieval_query = open_file('./meta_1hop_prompts/relation_retrieval_prompt.txt').replace('<<<>>>', divided_claim).replace('<<<>>>', str(one_hop_relation_set)) 138 | else: 139 | relation_retrieval_query = open_file('./meta_1hop_prompts/relation_retrieval_prompt.txt').replace('<<<>>>', divided_claim).replace('<<<>>>', str(new_all_relation_set)) 140 | 141 | for _ in range(5): 142 | try: 143 | response = openai.ChatCompletion.create( 144 | model="gpt-3.5-turbo-0613", 145 | messages=[ 146 | {"role": "system", "content": "You are a helpful assistant."}, 147 | { 148 | "role": "user", 149 | "content": relation_retrieval_query, 150 | }, 151 | ], 152 | max_tokens=max_tokens, 153 | temperature=0.2, 154 | top_p = 0.1 155 | ) 156 | relation_retrieval_result = response["choices"][0]["message"]["content"] 157 | retrieved_relations = retrieval_relation_parse_answer(relation_retrieval_result) 158 | 159 | if retrieved_relations: 160 | break 161 | print(_, 'th try!') 162 | except Exception as e: 163 | print("[ERROR]", e) 164 | time.sleep(5) 165 | 166 | if len(corresponding_entity_set) == 1: 167 | start_relation.append(retrieved_relations) 168 | else: 169 | extra_relation.append(retrieved_relations) 170 | 171 | final_evidence = [] 172 | if len(extra_relation) == 0: 173 | for rel in start_relation[0]: 174 | try: 175 | tails = KG[gt_entity][rel] 176 | for tail in tails: 177 | final_evidence.append([gt_entity, rel, tail]) 178 | except: 179 | pass 180 | 181 | try: 182 | tails = KG[gt_entity]['~'+rel] 183 | for tail in tails: 184 | final_evidence.append([tail, rel, gt_entity]) 185 | except: 186 | pass 187 | 188 | elif len(extra_relation) == 1: 189 | for fir_rev in ['', '~']: 190 | for sec_rev in ['', '~']: 191 | for rel in start_relation[0]: 192 | for sec_rel in extra_relation[0]: 193 | try: 194 | tails = KG[gt_entity][fir_rev+rel] 195 | for tail in tails: 196 | try: 197 | hop_2_tails = KG[tail][sec_rev+sec_rel] 198 | for hop_2_tail in hop_2_tails: 199 | if fir_rev == '': 200 | final_evidence.append([gt_entity, rel, tail]) 201 | else: 202 | final_evidence.append([tail, rel, gt_entity]) 203 | if sec_rev == '': 204 | final_evidence.append([tail, sec_rel, hop_2_tail]) 205 | else: 206 | final_evidence.append([hop_2_tail, sec_rel, tail]) 207 | except: 208 | pass 209 | except: 210 | pass 211 | 212 | else: 213 | for fir_rev in ['', '~']: 214 | for sec_rev in ['', '~']: 215 | for thr_rev in ['', '~']: 216 | for rel in start_relation[0]: 217 | for sec_rel in extra_relation[0]: 218 | for thr_rel in extra_relation[1]: 219 | try: 220 | tails = KG[gt_entity][fir_rev+rel] 221 | for tail in tails: 222 | try: 223 | hop_2_tails = KG[tail][sec_rev+sec_rel] 224 | for hop_2_tail in hop_2_tails: 225 | try: 226 | hop_3_tails = KG[hop_2_tail][thr_rev+thr_rel] 227 | for hop_3_tail in hop_3_tails: 228 | if fir_rev == '': 229 | final_evidence.append([gt_entity, rel, tail]) 230 | else: 231 | final_evidence.append([tail, rel, gt_entity]) 232 | if sec_rev == '': 233 | final_evidence.append([tail, sec_rel, hop_2_tail]) 234 | else: 235 | final_evidence.append([hop_2_tail, sec_rel, tail]) 236 | if thr_rev == '': 237 | final_evidence.append([hop_2_tail, thr_rel, hop_3_tail]) 238 | else: 239 | final_evidence.append([hop_3_tail, thr_rel, hop_2_tail]) 240 | 241 | except: 242 | pass 243 | except: 244 | pass 245 | try: 246 | hop_2_tails = KG[tail][thr_rev+thr_rel] 247 | for hop_2_tail in hop_2_tails: 248 | try: 249 | hop_3_tails = KG[hop_2_tail][sec_rev+sec_rel] 250 | for hop_3_tail in hop_3_tails: 251 | if fir_rev == '': 252 | final_evidence.append([gt_entity, rel, tail]) 253 | else: 254 | final_evidence.append([tail, rel, gt_entity]) 255 | if thr_rev == '': 256 | final_evidence.append([tail, thr_rel, hop_2_tail]) 257 | else: 258 | final_evidence.append([hop_2_tail, thr_rel, tail]) 259 | if sec_rev == '': 260 | final_evidence.append([hop_2_tail, sec_rel, hop_3_tail]) 261 | else: 262 | final_evidence.append([hop_3_tail, sec_rel, hop_2_tail]) 263 | 264 | except: 265 | pass 266 | except: 267 | pass 268 | except: 269 | pass 270 | 271 | 272 | new_final_evidence = [] 273 | 274 | for evi in final_evidence: 275 | if evi in new_final_evidence: 276 | continue 277 | if '~' in evi[1]: 278 | if [evi[2], evi[1].split('~')[1], evi[0]] not in new_final_evidence: 279 | new_final_evidence.append([evi[2], evi[1].split('~')[1], evi[0]]) 280 | else: 281 | new_final_evidence.append([evi[0], evi[1], evi[2]]) 282 | with open(f'./meta_generated_prompts/1hop/{qid}_final_evidence.pickle', 'wb') as f: 283 | pickle.dump(new_final_evidence, f) 284 | 285 | #### Verification phase 286 | verify_prompt = open_file('./meta_1hop_prompts/verify_claim_with_evidence.txt').replace('<<<>>>', claim).replace('<<<>>>', str(new_final_evidence)) 287 | for _ in range(5): 288 | try: 289 | response = openai.ChatCompletion.create( 290 | model="gpt-3.5-turbo-0613", 291 | messages=[ 292 | {"role": "system", "content": "You are a helpful assistant."}, 293 | { 294 | "role": "user", 295 | "content": verify_prompt, 296 | }, 297 | ], 298 | max_tokens=max_tokens, 299 | temperature=0.2, 300 | top_p = 0.1 301 | ) 302 | verification = response["choices"][0]["message"]["content"] 303 | save_file(f'./meta_generated_prompts/1hop/{qid}_verification_result.txt', verification) 304 | 305 | return verification.lower() 306 | 307 | except Exception as e: 308 | print("[ERROR]", e) 309 | time.sleep(5) 310 | 311 | return 'Error' 312 | 313 | 314 | if __name__ == "__main__": 315 | 316 | openai.api_key = open_file('./openai_api_key.txt') 317 | 318 | print('Start!!!') 319 | with open('./data/metaqa_kg.pickle', 'rb') as f: 320 | metakg = pickle.load(f) 321 | 322 | print('Data loading done!!!') 323 | 324 | futures = [] 325 | start_token = 0 326 | 327 | ####For new experiment, use it. 328 | result = {} 329 | questions_dict = {} 330 | entity_set_dict = {} 331 | label_set_dict = {} 332 | with open(os.path.expanduser(f"./data/onehop_test_set.jsonl")) as f: 333 | for line in f: 334 | if not line: 335 | continue 336 | q = json.loads(line) 337 | questions_dict[q["question_id"]] = q["question"] 338 | entity_set_dict[q["question_id"]] = q["entity_set"] 339 | label_set_dict[q["question_id"]] = q["Label"] 340 | 341 | # with open(f'./onehop_result.pickle', 'rb') as f: 342 | # result = pickle.load(f) 343 | 344 | Correct = [] 345 | Wrong = [] 346 | Error = [] 347 | Another = [] 348 | 349 | for qid, question in questions_dict.items(): 350 | try: 351 | future = get_answer(qid, question, entity_set_dict[qid][0], KG=metakg, max_tokens=1024) 352 | futures.append(future) 353 | is_correct = 0 354 | 355 | lower_label = [] 356 | for lab in label_set_dict[qid]: 357 | if lab.lower() in future.lower(): 358 | is_correct += 1 359 | break 360 | 361 | if is_correct > 0: 362 | Correct.append(qid) 363 | print(qid, ': Correct!') 364 | result[qid] = 'Correct' 365 | else: 366 | Wrong.append(qid) 367 | print(qid, ': Wrong...') 368 | result[qid] = 'Wrong' 369 | except Exception as e: 370 | print(e) 371 | Error.append(qid) 372 | print(qid, ': Error...') 373 | result[qid] = 'Error' 374 | 375 | with open(f'./onehop_result.pickle', 'wb') as f: 376 | pickle.dump(result, f) 377 | 378 | tot_corr = 0 379 | for tot_id in list(result): 380 | if result[tot_id] == 'Correct': 381 | tot_corr += 1 382 | 383 | print('Accuracy: ', tot_corr/len(list(result))) 384 | -------------------------------------------------------------------------------- /metaqa/metaqa_2hop_test.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | import time 5 | import concurrent.futures 6 | import re 7 | import openai 8 | import tqdm 9 | import shortuuid 10 | import pickle 11 | 12 | 13 | def open_file(filepath): 14 | with open(filepath, 'r', encoding='utf-8') as infile: 15 | return infile.read() 16 | 17 | 18 | def save_file(filepath, content): 19 | with open(filepath, 'w', encoding='utf-8') as outfile: 20 | outfile.write(content) 21 | 22 | 23 | def claim_divider_parse_answer(answer): 24 | processed_answer_set = {} 25 | answer = answer.strip() 26 | splitted_answers = answer.split('\n') 27 | # print(splitted_answers) 28 | try: 29 | for nth_answer in splitted_answers: 30 | nth_answer = nth_answer.strip() 31 | for i in range(5): 32 | if str(i+1)+'. ' in nth_answer[:5]: 33 | temp_ans = nth_answer.split(str(i+1)+'. ')[1] 34 | temp_split = temp_ans.split(', Entity set: ') 35 | sentence = temp_split[0] 36 | 37 | entity_set = temp_split[1] 38 | entity_set = entity_set.split('[')[1] 39 | entity_set = entity_set.split(']')[0] 40 | # print(entity_set) 41 | new_entity_set = [] 42 | # print(entity_set) 43 | # print(len(entity_set)) 44 | if len(entity_set) == 0: 45 | new_entity_set.append(entity_set) 46 | elif '"' in entity_set[0] or "'" in entity_set[0]: 47 | new_entity_set.append(entity_set[1:-1]) 48 | else: 49 | new_entity_set.append(entity_set) 50 | # print(new_entity_set) 51 | # print(new_entity_set) 52 | break 53 | processed_answer_set[sentence] = new_entity_set 54 | except: 55 | # return None 56 | pass 57 | 58 | return processed_answer_set 59 | 60 | 61 | def retrieval_relation_parse_answer(answer): 62 | pattern = r'\[[^\]]+\]' 63 | 64 | matches = re.findall(pattern, answer) 65 | 66 | if len(matches) != 1: 67 | return None 68 | 69 | # Extract the components from the matches and flatten the list 70 | components = [component.strip() for match in matches for component in match.strip('[]').split(',')] 71 | 72 | components = [component.strip("''") if "'" in component else component for component in components ] 73 | return components 74 | 75 | 76 | def get_answer(qid: int, claim: str, gt_entity: str, KG: dict, max_tokens: int): 77 | ans = { 78 | "answer_id": shortuuid.uuid(), 79 | } 80 | entities = re.findall(r'\[(.*?)\]', claim) 81 | sentence_divide_query = open_file('./meta_2hop_prompts/sentence_divide_prompt.txt').replace('<<<>>>', claim).replace('<<<>>>', '['+entities[0]+']') 82 | 83 | #### 1. sentence divide 84 | for _ in range(5): 85 | try: 86 | response = openai.ChatCompletion.create( 87 | model="gpt-3.5-turbo-0613", 88 | messages=[ 89 | {"role": "system", "content": "You are a helpful assistant."}, 90 | { 91 | "role": "user", 92 | "content": sentence_divide_query, 93 | }, 94 | ], 95 | max_tokens=max_tokens, 96 | temperature=0.2, 97 | top_p = 0.1 98 | ) 99 | divide_claim_result = response["choices"][0]["message"]["content"] 100 | # print("gpt output is ", divide_claim_result) 101 | 102 | divided_claim_list = claim_divider_parse_answer(divide_claim_result) 103 | # print(divided_claim_list) 104 | break 105 | except Exception as e: 106 | print("[ERROR]", e) 107 | time.sleep(5) 108 | with open(f'./meta_generated_prompts/2hop/{qid}_sentence_divide.pickle', 'wb') as f: 109 | pickle.dump(divided_claim_list, f) 110 | # save_file(f'./generated_prompts/{qid}_sentence_divide.txt', str(divided_claim_list)) 111 | 112 | used_all_relations = [] 113 | #### 2. Relation Retrieval 114 | total_evidence = [] 115 | start_relation = [] 116 | extra_relation = [] 117 | all_relation_set = [] 118 | 119 | for one_rel in list(KG[gt_entity]): 120 | all_relation_set.append(one_rel) 121 | one_tails = KG[gt_entity][one_rel] 122 | for one_tail in one_tails: 123 | all_relation_set += list(KG[one_tail]) 124 | 125 | new_all_relation_set = [] 126 | for tar_rel in all_relation_set: 127 | if tar_rel in new_all_relation_set: 128 | continue 129 | if '~' in tar_rel: 130 | new_all_relation_set.append(tar_rel.split('~')[1]) 131 | else: 132 | new_all_relation_set.append(tar_rel) 133 | new_all_relation_set = list(set(new_all_relation_set)) 134 | 135 | for divided_claim, corresponding_entity_set in divided_claim_list.items(): 136 | one_hop_relation_set = list(KG[gt_entity]) 137 | all_relation_set = list(set(all_relation_set)) 138 | 139 | if len(corresponding_entity_set) == 1 and len(corresponding_entity_set[0]) > 0: 140 | if len(list(KG[corresponding_entity_set[0]])) == 1: 141 | start_relation.append(one_hop_relation_set) 142 | continue 143 | else: 144 | relation_retrieval_query = open_file('./meta_2hop_prompts/relation_retrieval_prompt.txt').replace('<<<>>>', divided_claim).replace('<<<>>>', str(one_hop_relation_set)) 145 | else: 146 | relation_retrieval_query = open_file('./meta_2hop_prompts/relation_retrieval_prompt.txt').replace('<<<>>>', divided_claim).replace('<<<>>>', str(new_all_relation_set)) 147 | 148 | for _ in range(5): 149 | try: 150 | response = openai.ChatCompletion.create( 151 | model="gpt-3.5-turbo-0613", 152 | messages=[ 153 | {"role": "system", "content": "You are a helpful assistant."}, 154 | { 155 | "role": "user", 156 | "content": relation_retrieval_query, 157 | }, 158 | ], 159 | max_tokens=max_tokens, 160 | temperature=0.2, 161 | top_p = 0.1 162 | ) 163 | relation_retrieval_result = response["choices"][0]["message"]["content"] 164 | retrieved_relations = retrieval_relation_parse_answer(relation_retrieval_result) 165 | 166 | if retrieved_relations: 167 | break 168 | print(_, 'th try!') 169 | except Exception as e: 170 | print("[ERROR]", e) 171 | time.sleep(5) 172 | 173 | if len(corresponding_entity_set) == 1 and len(corresponding_entity_set[0]) > 0: 174 | start_relation.append(retrieved_relations) 175 | else: 176 | extra_relation.append(retrieved_relations) 177 | # print('corresponding entity set is', corresponding_entity_set) 178 | # print('start is ', start_relation) 179 | # print('extra is ', extra_relation) 180 | final_evidence = [] 181 | if len(extra_relation) == 0: 182 | for rel in start_relation[0]: 183 | try: 184 | tails = KG[gt_entity][rel] 185 | for tail in tails: 186 | final_evidence.append([gt_entity, rel, tail]) 187 | except: 188 | pass 189 | 190 | try: 191 | tails = KG[gt_entity]['~'+rel] 192 | for tail in tails: 193 | final_evidence.append([tail, rel, gt_entity]) 194 | except: 195 | pass 196 | 197 | elif len(extra_relation) == 1: 198 | for fir_rev in ['', '~']: 199 | for sec_rev in ['', '~']: 200 | for rel in start_relation[0]: 201 | for sec_rel in extra_relation[0]: 202 | try: 203 | tails = KG[gt_entity][fir_rev+rel] 204 | for tail in tails: 205 | try: 206 | hop_2_tails = KG[tail][sec_rev+sec_rel] 207 | for hop_2_tail in hop_2_tails: 208 | if fir_rev == '': 209 | final_evidence.append([gt_entity, rel, tail]) 210 | else: 211 | final_evidence.append([tail, rel, gt_entity]) 212 | if gt_entity != hop_2_tail: 213 | if sec_rev == '': 214 | final_evidence.append([tail, sec_rel, hop_2_tail]) 215 | else: 216 | final_evidence.append([hop_2_tail, sec_rel, tail]) 217 | break 218 | except: 219 | pass 220 | except: 221 | pass 222 | 223 | else: 224 | for fir_rev in ['', '~']: 225 | for sec_rev in ['', '~']: 226 | for thr_rev in ['', '~']: 227 | for rel in start_relation[0]: 228 | for sec_rel in extra_relation[0]: 229 | for thr_rel in extra_relation[1]: 230 | try: 231 | tails = KG[gt_entity][fir_rev+rel] 232 | for tail in tails: 233 | try: 234 | hop_2_tails = KG[tail][sec_rev+sec_rel] 235 | for hop_2_tail in hop_2_tails: 236 | try: 237 | hop_3_tails = KG[hop_2_tail][thr_rev+thr_rel] 238 | for hop_3_tail in hop_3_tails: 239 | if fir_rev == '': 240 | final_evidence.append([gt_entity, rel, tail]) 241 | else: 242 | final_evidence.append([tail, rel, gt_entity]) 243 | if sec_rev == '': 244 | final_evidence.append([tail, sec_rel, hop_2_tail]) 245 | else: 246 | final_evidence.append([hop_2_tail, sec_rel, tail]) 247 | if thr_rev == '': 248 | final_evidence.append([hop_2_tail, thr_rel, hop_3_tail]) 249 | else: 250 | final_evidence.append([hop_3_tail, thr_rel, hop_2_tail]) 251 | 252 | except: 253 | pass 254 | except: 255 | pass 256 | try: 257 | hop_2_tails = KG[tail][thr_rev+thr_rel] 258 | for hop_2_tail in hop_2_tails: 259 | try: 260 | hop_3_tails = KG[hop_2_tail][sec_rev+sec_rel] 261 | for hop_3_tail in hop_3_tails: 262 | if fir_rev == '': 263 | final_evidence.append([gt_entity, rel, tail]) 264 | else: 265 | final_evidence.append([tail, rel, gt_entity]) 266 | if thr_rev == '': 267 | final_evidence.append([tail, thr_rel, hop_2_tail]) 268 | else: 269 | final_evidence.append([hop_2_tail, thr_rel, tail]) 270 | if sec_rev == '': 271 | final_evidence.append([hop_2_tail, sec_rel, hop_3_tail]) 272 | else: 273 | final_evidence.append([hop_3_tail, sec_rel, hop_2_tail]) 274 | 275 | except: 276 | pass 277 | except: 278 | pass 279 | except: 280 | pass 281 | 282 | 283 | new_final_evidence = [] 284 | 285 | for evi in final_evidence: 286 | if evi in new_final_evidence: 287 | continue 288 | if '~' in evi[1]: 289 | if [evi[2], evi[1].split('~')[1], evi[0]] not in new_final_evidence: 290 | new_final_evidence.append([evi[2], evi[1].split('~')[1], evi[0]]) 291 | else: 292 | new_final_evidence.append([evi[0], evi[1], evi[2]]) 293 | # print('final_evidence is ', new_final_evidence) 294 | with open(f'./meta_generated_prompts/2hop/{qid}_final_evidence.pickle', 'wb') as f: 295 | pickle.dump(new_final_evidence, f) 296 | # save_file(f'./generated_prompts/{qid}_final_evidence.txt', str(new_final_evidence)) 297 | 298 | #### Verification phase 299 | verify_prompt = open_file('./meta_2hop_prompts/verify_claim_with_evidence.txt').replace('<<<>>>', claim).replace('<<<>>>', str(new_final_evidence)) 300 | # print('Verifying...') 301 | for _ in range(5): 302 | try: 303 | response = openai.ChatCompletion.create( 304 | model="gpt-3.5-turbo-0613", 305 | messages=[ 306 | {"role": "system", "content": "You are a helpful assistant."}, 307 | { 308 | "role": "user", 309 | "content": verify_prompt, 310 | }, 311 | ], 312 | max_tokens=max_tokens, 313 | temperature=0.2, 314 | top_p = 0.1 315 | ) 316 | verification = response["choices"][0]["message"]["content"] 317 | save_file(f'./meta_generated_prompts/2hop/{qid}_verification_result.txt', verification) 318 | 319 | return verification.lower() 320 | 321 | except Exception as e: 322 | print("[ERROR]", e) 323 | time.sleep(5) 324 | 325 | return 'Error' 326 | 327 | 328 | if __name__ == "__main__": 329 | 330 | openai.api_key = open_file('./openai_api_key.txt') 331 | 332 | print('Start!!!') 333 | with open('./data/metaqa_kg.pickle', 'rb') as f: 334 | metakg = pickle.load(f) 335 | 336 | print('Data loading done!!!') 337 | 338 | futures = [] 339 | start_token = 0 340 | 341 | ####For new experiment, use it. 342 | result = {} 343 | questions_dict = {} 344 | entity_set_dict = {} 345 | label_set_dict = {} 346 | with open(os.path.expanduser(f"./data/twohop_test_set.jsonl")) as f: 347 | for line in f: 348 | if not line: 349 | continue 350 | q = json.loads(line) 351 | questions_dict[q["question_id"]] = q["question"] 352 | entity_set_dict[q["question_id"]] = q["entity_set"] 353 | label_set_dict[q["question_id"]] = q["Label"] 354 | 355 | # with open(f'./twohop_result.pickle', 'rb') as f: 356 | # result = pickle.load(f) 357 | 358 | Correct = [] 359 | Wrong = [] 360 | Error = [] 361 | Another = [] 362 | 363 | for qid, question in questions_dict.items(): 364 | try: 365 | future = get_answer(qid, question, entity_set_dict[qid][0], KG=metakg, max_tokens=1024) 366 | futures.append(future) 367 | is_correct = 0 368 | 369 | lower_label = [] 370 | for lab in label_set_dict[qid]: 371 | if lab.lower() in future.lower(): 372 | is_correct += 1 373 | break 374 | 375 | if is_correct > 0: 376 | Correct.append(qid) 377 | print(qid, ': Correct!') 378 | result[qid] = 'Correct' 379 | else: 380 | Wrong.append(qid) 381 | print(qid, ': Wrong...') 382 | result[qid] = 'Wrong' 383 | except Exception as e: 384 | print(e) 385 | Error.append(qid) 386 | print(qid, ': Error...') 387 | result[qid] = 'Error' 388 | 389 | with open(f'./twohop_result.pickle', 'wb') as f: 390 | pickle.dump(result, f) 391 | 392 | tot_corr = 0 393 | for tot_id in list(result): 394 | if result[tot_id] == 'Correct': 395 | tot_corr += 1 396 | 397 | print('Accuracy: ', tot_corr/len(list(result))) 398 | -------------------------------------------------------------------------------- /metaqa/metaqa_3hop_test.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | import time 5 | import concurrent.futures 6 | import re 7 | import openai 8 | import tqdm 9 | import shortuuid 10 | import pickle 11 | 12 | 13 | def open_file(filepath): 14 | with open(filepath, 'r', encoding='utf-8') as infile: 15 | return infile.read() 16 | 17 | 18 | def save_file(filepath, content): 19 | with open(filepath, 'w', encoding='utf-8') as outfile: 20 | outfile.write(content) 21 | 22 | 23 | def claim_divider_parse_answer(answer): 24 | processed_answer_set = {} 25 | answer = answer.strip() 26 | splitted_answers = answer.split('\n') 27 | # print(splitted_answers) 28 | try: 29 | for nth_answer in splitted_answers: 30 | nth_answer = nth_answer.strip() 31 | for i in range(5): 32 | if str(i+1)+'. ' in nth_answer[:5]: 33 | temp_ans = nth_answer.split(str(i+1)+'. ')[1] 34 | temp_split = temp_ans.split(', Entity set: ') 35 | sentence = temp_split[0] 36 | 37 | entity_set = temp_split[1] 38 | entity_set = entity_set.split('[')[1] 39 | entity_set = entity_set.split(']')[0] 40 | # print(entity_set) 41 | new_entity_set = [] 42 | # print(entity_set) 43 | # print(len(entity_set)) 44 | if len(entity_set) == 0: 45 | new_entity_set.append(entity_set) 46 | elif '"' in entity_set[0] or "'" in entity_set[0]: 47 | new_entity_set.append(entity_set[1:-1]) 48 | else: 49 | new_entity_set.append(entity_set) 50 | # print(new_entity_set) 51 | # print(new_entity_set) 52 | break 53 | processed_answer_set[sentence] = new_entity_set 54 | except: 55 | # return None 56 | pass 57 | 58 | return processed_answer_set 59 | 60 | 61 | def retrieval_relation_parse_answer(answer): 62 | pattern = r'\[[^\]]+\]' 63 | 64 | matches = re.findall(pattern, answer) 65 | 66 | if len(matches) != 1: 67 | return None 68 | 69 | # Extract the components from the matches and flatten the list 70 | components = [component.strip() for match in matches for component in match.strip('[]').split(',')] 71 | 72 | components = [component.strip("''") if "'" in component else component for component in components ] 73 | return components 74 | 75 | 76 | def get_answer(qid: int, claim: str, gt_entity: str, KG: dict, max_tokens: int): 77 | ans = { 78 | "answer_id": shortuuid.uuid(), 79 | } 80 | entities = re.findall(r'\[(.*?)\]', claim) 81 | sentence_divide_query = open_file('./meta_3hop_prompts/sentence_divide_prompt.txt').replace('<<<>>>', claim).replace('<<<>>>', '['+entities[0]+']') 82 | 83 | #### 1. sentence divide 84 | for _ in range(5): 85 | try: 86 | response = openai.ChatCompletion.create( 87 | model="gpt-3.5-turbo-0613", 88 | messages=[ 89 | {"role": "system", "content": "You are a helpful assistant."}, 90 | { 91 | "role": "user", 92 | "content": sentence_divide_query, 93 | }, 94 | ], 95 | max_tokens=max_tokens, 96 | temperature=0.2, 97 | top_p = 0.1 98 | ) 99 | divide_claim_result = response["choices"][0]["message"]["content"] 100 | # print("gpt output is ", divide_claim_result) 101 | 102 | divided_claim_list = claim_divider_parse_answer(divide_claim_result) 103 | # print(divided_claim_list) 104 | break 105 | except Exception as e: 106 | print("[ERROR]", e) 107 | time.sleep(5) 108 | with open(f'./meta_generated_prompts/3hop/{qid}_sentence_divide.pickle', 'wb') as f: 109 | pickle.dump(divided_claim_list, f) 110 | # save_file(f'./generated_prompts/{qid}_sentence_divide.txt', str(divided_claim_list)) 111 | 112 | used_all_relations = [] 113 | #### 2. Relation Retrieval 114 | total_evidence = [] 115 | start_relation = [] 116 | extra_relation = [] 117 | 118 | one_hop_relation_set = list(KG[gt_entity]) 119 | all_relation_set = [] 120 | for one_rel in list(KG[gt_entity]): 121 | all_relation_set.append(one_rel) 122 | one_tails = KG[gt_entity][one_rel] 123 | for one_tail in one_tails: 124 | two_rels = list(KG[one_tail]) 125 | for two_rel in two_rels: 126 | all_relation_set.append(two_rel) 127 | two_tails = KG[one_tail][two_rel] 128 | for two_tail in two_tails: 129 | three_rels = list(KG[two_tail]) 130 | all_relation_set += three_rels 131 | all_relation_set = list(set(all_relation_set)) 132 | new_all_relation_set = [] 133 | 134 | for tar_rel in all_relation_set: 135 | if tar_rel in new_all_relation_set: 136 | continue 137 | if '~' in tar_rel: 138 | new_all_relation_set.append(tar_rel.split('~')[1]) 139 | else: 140 | new_all_relation_set.append(tar_rel) 141 | new_all_relation_set = list(set(new_all_relation_set)) 142 | 143 | for divided_claim, corresponding_entity_set in divided_claim_list.items(): 144 | if len(corresponding_entity_set) == 1 and len(corresponding_entity_set[0]) > 0: 145 | if len(list(KG[corresponding_entity_set[0]])) == 1: 146 | start_relation.append(one_hop_relation_set) 147 | continue 148 | else: 149 | relation_retrieval_query = open_file('./meta_3hop_prompts/relation_retrieval_prompt.txt').replace('<<<>>>', divided_claim).replace('<<<>>>', str(one_hop_relation_set)) 150 | else: 151 | relation_retrieval_query = open_file('./meta_3hop_prompts/relation_retrieval_prompt.txt').replace('<<<>>>', divided_claim).replace('<<<>>>', str(new_all_relation_set)) 152 | 153 | for _ in range(5): 154 | try: 155 | response = openai.ChatCompletion.create( 156 | model="gpt-3.5-turbo-0613", 157 | messages=[ 158 | {"role": "system", "content": "You are a helpful assistant."}, 159 | { 160 | "role": "user", 161 | "content": relation_retrieval_query, 162 | }, 163 | ], 164 | max_tokens=max_tokens, 165 | temperature=0.2, 166 | top_p = 0.1 167 | ) 168 | relation_retrieval_result = response["choices"][0]["message"]["content"] 169 | retrieved_relations = retrieval_relation_parse_answer(relation_retrieval_result) 170 | 171 | if retrieved_relations: 172 | break 173 | print(_, 'th try!') 174 | except Exception as e: 175 | print("[ERROR]", e) 176 | time.sleep(5) 177 | 178 | if len(corresponding_entity_set) == 1 and len(corresponding_entity_set[0]) > 0: 179 | start_relation.append(retrieved_relations) 180 | else: 181 | extra_relation.append(retrieved_relations) 182 | # print('corresponding entity set is', corresponding_entity_set) 183 | # print('start is ', start_relation) 184 | # print('extra is ', extra_relation) 185 | final_evidence = [] 186 | if len(extra_relation) == 0: 187 | for rel in start_relation[0]: 188 | try: 189 | tails = KG[gt_entity][rel] 190 | for tail in tails: 191 | final_evidence.append([gt_entity, rel, tail]) 192 | except: 193 | pass 194 | 195 | try: 196 | tails = KG[gt_entity]['~'+rel] 197 | for tail in tails: 198 | final_evidence.append([tail, rel, gt_entity]) 199 | except: 200 | pass 201 | 202 | elif len(extra_relation) == 1: 203 | for fir_rev in ['', '~']: 204 | for sec_rev in ['', '~']: 205 | for rel in start_relation[0]: 206 | for sec_rel in extra_relation[0]: 207 | try: 208 | tails = KG[gt_entity][fir_rev+rel] 209 | for tail in tails: 210 | try: 211 | hop_2_tails = KG[tail][sec_rev+sec_rel] 212 | for hop_2_tail in hop_2_tails: 213 | if fir_rev == '': 214 | final_evidence.append([gt_entity, rel, tail]) 215 | else: 216 | final_evidence.append([tail, rel, gt_entity]) 217 | if sec_rev == '': 218 | final_evidence.append([tail, sec_rel, hop_2_tail]) 219 | else: 220 | final_evidence.append([hop_2_tail, sec_rel, tail]) 221 | except: 222 | pass 223 | except: 224 | pass 225 | 226 | else: 227 | for fir_rev in ['', '~']: 228 | for sec_rev in ['', '~']: 229 | for thr_rev in ['', '~']: 230 | for rel in start_relation[0]: 231 | for sec_rel in extra_relation[0]: 232 | for thr_rel in extra_relation[1]: 233 | try: 234 | stop = 0 235 | tails = KG[gt_entity][fir_rev+rel] 236 | for tail in tails: 237 | if stop == 1: 238 | break 239 | try: 240 | hop_2_tails = KG[tail][sec_rev+sec_rel] 241 | for hop_2_tail in hop_2_tails: 242 | if stop == 1: 243 | break 244 | if gt_entity == hop_2_tail: 245 | continue 246 | try: 247 | hop_3_tails = KG[hop_2_tail][thr_rev+thr_rel] 248 | for hop_3_tail in hop_3_tails: 249 | if stop == 1: 250 | break 251 | if fir_rev == '': 252 | final_evidence.append([gt_entity, rel, tail]) 253 | else: 254 | final_evidence.append([tail, rel, gt_entity]) 255 | if sec_rev == '': 256 | final_evidence.append([tail, sec_rel, hop_2_tail]) 257 | else: 258 | final_evidence.append([hop_2_tail, sec_rel, tail]) 259 | if thr_rev == '': 260 | final_evidence.append([hop_2_tail, thr_rel, hop_3_tail]) 261 | else: 262 | final_evidence.append([hop_3_tail, thr_rel, hop_2_tail]) 263 | stop = 1 264 | except: 265 | pass 266 | except: 267 | pass 268 | try: 269 | hop_2_tails = KG[tail][thr_rev+thr_rel] 270 | for hop_2_tail in hop_2_tails: 271 | if stop == 1: 272 | break 273 | if gt_entity == hop_2_tail: 274 | continue 275 | try: 276 | hop_3_tails = KG[hop_2_tail][sec_rev+sec_rel] 277 | for hop_3_tail in hop_3_tails: 278 | if stop == 1: 279 | break 280 | if fir_rev == '': 281 | final_evidence.append([gt_entity, rel, tail]) 282 | else: 283 | final_evidence.append([tail, rel, gt_entity]) 284 | if thr_rev == '': 285 | final_evidence.append([tail, thr_rel, hop_2_tail]) 286 | else: 287 | final_evidence.append([hop_2_tail, thr_rel, tail]) 288 | if sec_rev == '': 289 | final_evidence.append([hop_2_tail, sec_rel, hop_3_tail]) 290 | else: 291 | final_evidence.append([hop_3_tail, sec_rel, hop_2_tail]) 292 | stop = 1 293 | except: 294 | pass 295 | except: 296 | pass 297 | except: 298 | pass 299 | 300 | 301 | new_final_evidence = [] 302 | 303 | for evi in final_evidence: 304 | if evi in new_final_evidence: 305 | continue 306 | if '~' in evi[1]: 307 | if [evi[2], evi[1].split('~')[1], evi[0]] not in new_final_evidence: 308 | new_final_evidence.append([evi[2], evi[1].split('~')[1], evi[0]]) 309 | else: 310 | new_final_evidence.append([evi[0], evi[1], evi[2]]) 311 | # print('final_evidence is ', new_final_evidence) 312 | with open(f'./meta_generated_prompts/3hop/{qid}_final_evidence.pickle', 'wb') as f: 313 | pickle.dump(new_final_evidence, f) 314 | # save_file(f'./generated_prompts/{qid}_final_evidence.txt', str(new_final_evidence)) 315 | 316 | #### Verification phase 317 | verify_prompt = open_file('./meta_3hop_prompts/verify_claim_with_evidence.txt').replace('<<<>>>', claim).replace('<<<>>>', str(new_final_evidence)) 318 | # print('Verifying...') 319 | for _ in range(5): 320 | try: 321 | response = openai.ChatCompletion.create( 322 | model="gpt-3.5-turbo-0613", 323 | messages=[ 324 | {"role": "system", "content": "You are a helpful assistant."}, 325 | { 326 | "role": "user", 327 | "content": verify_prompt, 328 | }, 329 | ], 330 | max_tokens=max_tokens, 331 | temperature=0.2, 332 | top_p = 0.1 333 | ) 334 | verification = response["choices"][0]["message"]["content"] 335 | save_file(f'./meta_generated_prompts/3hop/{qid}_verification_result.txt', verification) 336 | 337 | return verification.lower() 338 | 339 | except Exception as e: 340 | print("[ERROR]", e) 341 | time.sleep(5) 342 | 343 | return 'Error' 344 | 345 | 346 | if __name__ == "__main__": 347 | 348 | openai.api_key = open_file('./openai_api_key.txt') 349 | 350 | print('Start!!!') 351 | with open('./data/metaqa_kg.pickle', 'rb') as f: 352 | metakg = pickle.load(f) 353 | 354 | print('Data loading done!!!') 355 | 356 | futures = [] 357 | start_token = 0 358 | 359 | ####For new experiment, use it. 360 | result = {} 361 | questions_dict = {} 362 | entity_set_dict = {} 363 | label_set_dict = {} 364 | with open(os.path.expanduser(f"./data/threehop_test_set.jsonl")) as f: 365 | for line in f: 366 | if not line: 367 | continue 368 | q = json.loads(line) 369 | questions_dict[q["question_id"]] = q["question"] 370 | entity_set_dict[q["question_id"]] = q["entity_set"] 371 | label_set_dict[q["question_id"]] = q["Label"] 372 | 373 | # with open(f'./threehop_result.pickle', 'rb') as f: 374 | # result = pickle.load(f) 375 | 376 | Correct = [] 377 | Wrong = [] 378 | Error = [] 379 | Another = [] 380 | 381 | for qid, question in questions_dict.items(): 382 | try: 383 | future = get_answer(qid, question, entity_set_dict[qid][0], KG=metakg, max_tokens=1024) 384 | futures.append(future) 385 | is_correct = 0 386 | 387 | lower_label = [] 388 | for lab in label_set_dict[qid]: 389 | if lab.lower() in future.lower(): 390 | is_correct += 1 391 | break 392 | 393 | if is_correct > 0: 394 | Correct.append(qid) 395 | print(qid, ': Correct!') 396 | result[qid] = 'Correct' 397 | else: 398 | Wrong.append(qid) 399 | print(qid, ': Wrong...') 400 | result[qid] = 'Wrong' 401 | except Exception as e: 402 | print(e) 403 | Error.append(qid) 404 | print(qid, ': Error...') 405 | result[qid] = 'Error' 406 | 407 | with open(f'./threehop_result.pickle', 'wb') as f: 408 | pickle.dump(result, f) 409 | 410 | tot_corr = 0 411 | for tot_id in list(result): 412 | if result[tot_id] == 'Correct': 413 | tot_corr += 1 414 | 415 | print('Accuracy: ', tot_corr/len(list(result))) 416 | -------------------------------------------------------------------------------- /metaqa/openai_api_key.txt: -------------------------------------------------------------------------------- 1 | <<>> --------------------------------------------------------------------------------