├── README.md ├── calc_metrics_with_check_and_match.py ├── data ├── tempquestions │ └── test.jsonl ├── timeqa │ ├── test_easy.jsonl │ └── test_hard.jsonl └── timequestions │ └── test.jsonl ├── images ├── intro_fig.png └── main_figure.png ├── main.py ├── prompts ├── timeqa.json └── timequestions.json ├── requirements.txt ├── search_wiki.py └── utils.py /README.md: -------------------------------------------------------------------------------- 1 | # Question Answering as Programming for Solving Time-Sensitive Questions (EMNLP 2023) 2 | 3 | Image Title 4 | 5 | # Setup 6 | 1. `pip install -r requirements.txt` 7 | 2. Set the openai api key in `main.py` 8 | 9 | # Quick Start 10 | Run TimeQA experiments 11 | ``` 12 | python main.py 13 | --prompt_file timeqa.json \ 14 | --given_context 1 \ 15 | --dataset timeqa \ 16 | --data_file test_hard.jsonl \ 17 | --max_slice_length 512 \ 18 | --slice_stride 384 \ 19 | --return_search_passage content \ 20 | --model_name gpt-3.5-turbo \ 21 | --resume_id -1 22 | ``` 23 | 24 | Run TempQuestions and TimeQuestions experiments 25 | ``` 26 | python main.py 27 | --prompt_file timequestions.json \ 28 | --given_context 0 \ 29 | --dataset tempquestion \ 30 | --data_file test.jsonl \ 31 | --max_slice_length 512 \ 32 | --slice_stride 384 \ 33 | --return_search_passage content \ 34 | --model_name gpt-3.5-turbo \ 35 | --resume_id -1 36 | ``` 37 | 38 | The output should look like 39 | ``` 40 | 0 Joachim Löw was the coach of which team between Jan 1997 and Aug 1997? 41 | '''python 42 | query = {"subject": "Joachim Löw", "relation": "coach of", "object": None, "time": {"start": datetime(1997, 1, 1), "end": datetime(1997, 8, 31)}} 43 | answer_key = "object" 44 | ''' 45 | Search: 46 | '''python 47 | entities_to_search = ["Joachim Löw"] 48 | ''' 49 | -------------------------------------------------- 50 | Generate a background document from Wikipedia to answer the given question:Joachim Löw is a German football coach and former player. He was the head coach of VfB Stuttgart from July 1996 to October 1998. 51 | Extract information relevant to the query: 52 | '''python 53 | information.append({"subject": "Joachim Löw", "relation": "coach of", "object": "VfB Stuttgart", "time": {"start": datetime(1996, 7, 1), "end": datetime(1998, 10, 31)}}) 54 | ''' 55 | ************************************************** 56 | Extract information relevant to the query: 57 | '''python 58 | information.append({"subject": "Joachim Löw", "relation": "coach of", "object": None, "time": {"start": datetime(1997, 1, 1), "end": datetime(1997, 8, 31)}}) 59 | ''' 60 | ... 61 | ``` 62 | 63 | # Evaluation 64 | Set the `file_path` in `calc_metrics_with_check_and_match.py` and execute it. 65 | 66 | # How to Generalize to Other Constrained-based Reasoning QA 67 | In this work, we focus on the time-constrained QA. However, our framework can be modified to generalize to other constrained-based QA tasks. The key is to define the constraint as a python class, which should be able to be measured how well the constraint is satisfied and redefine the `match` function in `calc_metrics_with_check_and_match.py`. 68 | 69 | # Note 70 | We run all the experiments with `gpt-3.5-turbo-0301`. However, we found the updated versions like `gpt-3.5-turbo-0613` and `gpt-3.5-turbo-1106` have a different behavior, their in-context learning ability become degraded and cannot correctly perform the task. 71 | 72 | # Citation 73 | Please cite the paper and star this repo if you find QAaP interesting or useful, thank you! Feel free to contact zhuxy21@mails.tsinghua.edu.cn or open an issue if you have any questions. 74 | ```bibtex 75 | @article{zhu2023qaap, 76 | title={Question Answering as Programming for Solving Time-Sensitive Questions}, 77 | author={Zhu, Xinyu and Yang, Cheng and Chen, Bei and Li, Siheng and Lou, Jian-Guang and Yang, Yujiu}, 78 | journal={arXiv preprint arXiv:2305.14221}, 79 | year={2023} 80 | } 81 | ``` 82 | -------------------------------------------------------------------------------- /calc_metrics_with_check_and_match.py: -------------------------------------------------------------------------------- 1 | import random 2 | import jsonlines 3 | from typing import List, Dict 4 | from datetime import datetime, timedelta 5 | import datetime as dt 6 | from utils import extract_answer, get_metrics, extract_code_from_string, normalize_answer 7 | 8 | 9 | def match(query, information, answer_key): 10 | time_type = None 11 | default_start = datetime(1, 1, 1) 12 | default_end = datetime(3000, 1, 1) 13 | if "time" not in query or query['time'] is None or (("start" in query['time'] and "end" in query['time']) and (query['time']['start'] is None and query['time']['end'] is None)): 14 | query['time'] = {'start': default_start, 'end': default_end} 15 | time_type = 'overlap' 16 | elif isinstance(query['time'], datetime): 17 | query['time'] = {'start': query['time'], 'end': query['time'] + dt.timedelta(365)} 18 | time_type = 'overlap' 19 | elif 'start' not in query['time'] or query['time']['start'] is None: 20 | time_type = 'before or end' 21 | elif 'end' not in query['time'] or query['time']['end'] is None: 22 | time_type = 'after or start' 23 | else: 24 | time_type = 'overlap' 25 | 26 | information = [x for x in information if 'subject' in x and 'object' in x and 'relation' in x] #and x['time'] is not None] 27 | information = [x for x in information if x[answer_key] is not None] 28 | if len(information) == 0: 29 | return [""] 30 | 31 | for idx, ex in enumerate(information): 32 | try: 33 | if "time" not in ex or ex['time'] is None or (("start" in ex['time'] and "end" in ex['time']) and (ex['time']['start'] is None and ex['time']['end'] is None)): 34 | ex['time'] = {'start': default_start, 'end': default_end} 35 | elif isinstance(ex['time'], datetime): 36 | ex['time'] = {'start': ex['time'], 'end': ex['time'] + dt.timedelta(365)} 37 | elif len(ex['time']) == 0: 38 | ex['time'] = {'start': default_start, 'end': default_end} 39 | if 'start' not in ex['time'] or ex['time']['start'] is None: 40 | ex['time'].update(start=default_start) 41 | if 'end' not in ex['time'] or ex['time']['end'] is None: 42 | ex['time'].update(end=default_end) 43 | except Exception as e: 44 | print("Error Type:", type(e)) 45 | 46 | overlapped = False 47 | information = [x for x in information if x['time'] is not None] 48 | # answer_count = defaultdict(int) 49 | if time_type == "overlap": 50 | for idx, ex in enumerate(information): 51 | latest_start = max(query['time']['start'], ex['time']['start']) 52 | earliest_end = min(query['time']['end'], ex['time']['end']) 53 | delta = (earliest_end - latest_start).days + 1 54 | overlap = max(0, delta) 55 | if overlap > 0: 56 | overlapped = True 57 | time_union = max((query['time']['end'] - query['time']['start']).days + (ex['time']['end'] - ex['time']['start']).days - overlap, 1) 58 | ex.update(overlap=overlap) 59 | ex.update(time_union=time_union) 60 | ex.update(time_iou=overlap / time_union) 61 | information = sorted(information, key=lambda x: (x['time_iou'], x['overlap']), reverse=True) 62 | elif time_type == "after or start": 63 | information = sorted(information, key=lambda x: abs((x['time']['start'] - query['time']['start']).days)) 64 | elif time_type == "before or end": 65 | information = sorted(information, key=lambda x: abs((x['time']['end'] - query['time']['end']).days)) 66 | 67 | return information 68 | 69 | 70 | def calc_origin_metrics(data): 71 | rs = [] 72 | f1_list = [] 73 | for ex in data: 74 | gt_answer = ex['gt_answer'] 75 | answer = ex['answer'] 76 | metrics = get_metrics(answer, gt_answer) 77 | rs.append(metrics['em']) 78 | f1_list.append(metrics['f1']) 79 | em_num = sum(rs) 80 | em_rate = sum(rs) / len(rs) 81 | avg_f1 = sum(f1_list) / len(f1_list) 82 | print(f"em number: {em_num}, em rate: {em_rate}, avg f1: {avg_f1:.5f}") 83 | 84 | 85 | def calc_metrics(data): 86 | rs = [] 87 | f1_list = [] 88 | for idx, ex in enumerate(data): 89 | traj_list = ex['traj_list'] 90 | parsed_query = traj_list[0] 91 | parsed_query = extract_code_from_string(parsed_query) 92 | query_ = {} 93 | try: 94 | exec(parsed_query, globals(), query_) 95 | query = query_['query'] 96 | answer_key = query_['answer_key'] 97 | except Exception as e: 98 | print(f"fail to get query:\n{parsed_query}") 99 | rs.append(0) 100 | f1_list.append(0) 101 | continue 102 | 103 | context_slices = ex['context_slices'] 104 | information_list = ex['information_list'] 105 | final_information_list = [] 106 | assert len(context_slices) == len(information_list) 107 | if len(context_slices) == 0: 108 | print(f"{idx} no context, Q: {ex['question']}") 109 | rs.append(0) 110 | f1_list.append(0) 111 | continue 112 | 113 | cot_context = context_slices[0] 114 | cot_infor = information_list[0] 115 | cot_instances = [] 116 | locals_ = {'information': []} 117 | try: 118 | cot_infor = extract_code_from_string(cot_infor) 119 | if cot_infor is not None: 120 | for line in cot_infor.split("\n"): 121 | try: 122 | exec(line, globals(), locals_) 123 | except Exception as e: 124 | continue 125 | except Exception as e: 126 | print(f"fail to execute code:\n{cot_infor}") 127 | 128 | if CHECK_TIME: 129 | for ins in locals_['information']: 130 | if 'time' not in ins or ins['time'] is None: 131 | cot_instances.append(ins) 132 | continue 133 | if 'start' in ins['time'] and ins['time']['start'] is not None: 134 | try: 135 | if str(ins['time']['start'].year) not in cot_context: 136 | ins['time']['start'] = None 137 | except Exception as e: 138 | pass 139 | if 'end' in ins['time'] and ins['time']['end'] is not None: 140 | try: 141 | if str(ins['time']['end'].year) not in cot_context: 142 | ins['time']['end'] = None 143 | except Exception as e: 144 | pass 145 | cot_instances.append(ins) 146 | else: 147 | cot_instances = locals_['information'] 148 | 149 | extract_instances = [] 150 | for text, infor in zip(context_slices[1:], information_list[1:]): 151 | if not infor: 152 | continue 153 | locals_ = {'information': []} 154 | try: 155 | infor = extract_code_from_string(infor) 156 | if infor is not None: 157 | for line in infor.split("\n"): 158 | try: 159 | exec(line, globals(), locals_) 160 | except Exception as e: 161 | continue 162 | except Exception as e: 163 | print(f"fail to execute code:\n{infor}") 164 | continue 165 | if CHECK_TIME: 166 | for ins in locals_['information']: 167 | if 'time' not in ins or ins['time'] is None: 168 | extract_instances.append(ins) 169 | continue 170 | if 'start' in ins['time'] and ins['time']['start'] is not None: 171 | try: 172 | if str(ins['time']['start'].year) not in text: 173 | ins['time']['start'] = None 174 | except Exception as e: 175 | pass 176 | elif 'end' in ins['time'] and ins['time']['end'] is not None: 177 | try: 178 | if str(ins['time']['end'].year) not in text: 179 | ins['time']['end'] = None 180 | except Exception as e: 181 | pass 182 | extract_instances.append(ins) 183 | else: 184 | extract_instances.extend(locals_['information']) 185 | 186 | if CHECK_APPEAR: 187 | try: 188 | possible_answer_from_extract = [normalize_answer(ins_e[answer_key]) for ins_e in extract_instances if answer_key in ins_e and isinstance(ins_e[answer_key], str)] 189 | if len(possible_answer_from_extract) != 0: 190 | for ins_c in cot_instances: 191 | if answer_key not in ins_c or ins_c[answer_key] is None: 192 | # print("No answer key ins_c", answer_key, ins_c) 193 | continue 194 | if normalize_answer(ins_c[answer_key]) in possible_answer_from_extract: 195 | final_information_list.append(ins_c) 196 | else: 197 | final_information_list += cot_instances 198 | final_information_list += extract_instances 199 | except Exception as e: 200 | final_information_list = cot_instances + extract_instances 201 | else: 202 | final_information_list = cot_instances + extract_instances 203 | 204 | for idy, ey in enumerate(final_information_list): 205 | if 'subject' not in query or 'object' not in query or 'subject' not in ey or 'object' not in ey: 206 | continue 207 | if ey[answer_key] == query['subject'] or ey[answer_key] == query['object']: 208 | if answer_key == "subject": 209 | final_information_list[idy][answer_key] = final_information_list[idy]['object'] 210 | else: 211 | final_information_list[idy][answer_key] = final_information_list[idy]['subject'] 212 | 213 | # Match 214 | try: 215 | final_information_list = match(query, final_information_list, answer_key) 216 | except Exception as e: 217 | pass 218 | try: 219 | answer = extract_answer(answer_key, final_information_list) 220 | except Exception as e: 221 | answer = [""] 222 | 223 | gt_answer = ex['gt_answer'] 224 | metrics = get_metrics(answer, gt_answer) 225 | 226 | rs.append(metrics['em']) 227 | f1_list.append(metrics['f1']) 228 | 229 | em_num = sum(rs) 230 | em_rate = sum(rs) / len(rs) 231 | avg_f1 = sum(f1_list) / len(f1_list) 232 | print(f"em number: {em_num}, em rate: {em_rate}, avg f1: {avg_f1:.5f}") 233 | 234 | 235 | if __name__ == "__main__": 236 | CHECK_TIME = True 237 | CHECK_APPEAR = True 238 | file_path = "outputs/" 239 | with jsonlines.open(file_path, "r") as f: 240 | data = list(f) 241 | random.seed(0) 242 | random.shuffle(data) 243 | print("data size: ", len(data)) 244 | 245 | print("metrics") 246 | calc_metrics(data) 247 | -------------------------------------------------------------------------------- /data/tempquestions/test.jsonl: -------------------------------------------------------------------------------- 1 | {"question": "What soviet leader came to power in the 1920s?", "answer": ["Joseph Stalin"], "question_idx": 6, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 2 | {"question": "What buildings were destroyed in september 11th?", "answer": [""], "question_idx": 8, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 3 | {"question": "What did italy invade in 1935?", "answer": ["Africa", "Ethiopia"], "question_idx": 14, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 4 | {"question": "Which movie did john rubinstein star in in 1996?", "answer": ["Norma Jean & Marilyn"], "question_idx": 23, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 5 | {"question": "Which album did neko case release in march 2006?", "answer": ["Fox Confessor Brings The Flood"], "question_idx": 25, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 6 | {"question": "What colonies did spain lose in 1898?", "answer": ["Guam", "Mariana Islands", "Caroline Islands", "Palau"], "question_idx": 35, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 7 | {"question": "What to do in seattle during the summer?", "answer": ["Woodland Park Zoo", "Henry Art Gallery", "Seattle Art Museum", "Seattle Aquarium", "Frye Art Museum", "Space Needle", "Pike Place Market", "Smith Tower", "Tillicum Village", "Museum of Flight"], "question_idx": 36, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 8 | {"question": "Where did huguenots settle in 1678?", "answer": ["New Paltz"], "question_idx": 37, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 9 | {"question": "Who won the fa cup in 2008?", "answer": ["Portsmouth F.C."], "question_idx": 38, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 10 | {"question": "Who did america gain independence from in 1776?", "answer": ["Britain"], "question_idx": 40, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 11 | {"question": "What flowers are in bloom in september?", "answer": ["the Cactus", "The tulips", "many roses"], "question_idx": 51, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 12 | {"question": "What nation did the united states invaded in 1983?", "answer": ["Grenada", "the tiny Caribbean Island of Grenada", "the island of Grenada", "the tiny island of Grenada"], "question_idx": 52, "Temporal signal": ["OVERPAL"], "Temporal question type": ["Explicit"]} 13 | {"question": "Who established a permanent settlement in north america before 1600?", "answer": ["Spain", "Portugal", "the French", "Drake", "England"], "question_idx": 55, "Temporal signal": ["BEFORE"], "Temporal question type": ["Explicit"]} 14 | {"question": "Which album did dillinger four release in 2000?", "answer": ["Versus God", "Dillinger Four / Pinhead Gunpowder"], "question_idx": 57, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 15 | {"question": "Who was the leader of the soviet union in 1945?", "answer": ["Joseph Stalin"], "question_idx": 59, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 16 | {"question": "Which film starring sam anderson was released in 1982?", "answer": ["Airplane II: The Sequel"], "question_idx": 60, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 17 | {"question": "In 1981 what award did danny devito win?", "answer": ["Primetime Emmy Award for Outstanding Supporting Actor - Comedy Series", "Primetime Emmy Award for Outstanding Supporting Actor in a Comedy, Variety or Music Series"], "question_idx": 61, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 18 | {"question": "Who does david beckham play for in 2012?", "answer": ["LA Galaxy"], "question_idx": 65, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 19 | {"question": "Where was the olympics held in 2000?", "answer": ["Sydney", "Australia", "Held In Sydney"], "question_idx": 67, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 20 | {"question": "Which album did tears for fears release in 1983?", "answer": ["The Hurting", "The Videosingles"], "question_idx": 75, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 21 | {"question": "Which album did x japan release in july 1994?", "answer": ["Rusty Nail"], "question_idx": 95, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 22 | {"question": "Which movie did ventura pons direct in 1990?", "answer": ["What's It All About"], "question_idx": 96, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 23 | {"question": "Who did mexico win its independence from in 1821?", "answer": ["Spain"], "question_idx": 99, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 24 | {"question": "What caused the cuban rebellion in 1868?", "answer": ["Ten Years' War"], "question_idx": 103, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 25 | {"question": "Who was the colts coach in 2011?", "answer": ["Jim Caldwell"], "question_idx": 108, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 26 | {"question": "What team does messi play for in 2011?", "answer": ["Argentina national football team", "FC Barcelona"], "question_idx": 117, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 27 | {"question": "Who was the newscaster in 1948 on cbs evening news?", "answer": ["Douglas Edwards"], "question_idx": 118, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 28 | {"question": "Which movie did barney martin star in in 1981?", "answer": ["Arthur"], "question_idx": 124, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 29 | {"question": "Who was invaded by iraq in 1990?", "answer": ["Kuwait"], "question_idx": 127, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 30 | {"question": "Which book did william golding write in 1995?", "answer": ["The Double Tongue"], "question_idx": 133, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 31 | {"question": "Who won fedex cup in 2012?", "answer": ["Brandt Snedeker"], "question_idx": 135, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 32 | {"question": "What did charles babbage create in the 1800s?", "answer": ["Analytical Engine"], "question_idx": 136, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 33 | {"question": "Who does ronaldinho play for now in 2011?", "answer": ["Clube de Regatas do Flamengo"], "question_idx": 137, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 34 | {"question": "Who is the current governor of the us in 2011?", "answer": ["Barack obama"], "question_idx": 145, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 35 | {"question": "Which album did boredoms release in september 1992?", "answer": ["Pop Tatari"], "question_idx": 146, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 36 | {"question": "What book did charles darwin wrote in 1859?", "answer": ["The Origin of Species"], "question_idx": 149, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 37 | {"question": "Which film starring gao yuanyuan was released in 2001?", "answer": ["Beijing Bicycle"], "question_idx": 151, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 38 | {"question": "Which movie did billy wilder direct in 1981?", "answer": ["Buddy Buddy"], "question_idx": 154, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 39 | {"question": "Which movie did alana austin star in in 1996?", "answer": ["A Mother's Instinct"], "question_idx": 156, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 40 | {"question": "Who did florida belong to in 1770?", "answer": ["Spain", "the Spaniards", "the Spanish"], "question_idx": 160, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 41 | {"question": "What is money used in korea before 1892?", "answer": ["the Korean mun"], "question_idx": 170, "Temporal signal": ["BEFORE"], "Temporal question type": ["Explicit"]} 42 | {"question": "Who plays captain kirk in 2009?", "answer": ["Chris Pine"], "question_idx": 177, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 43 | {"question": "Who won the grammy for best new artist in 2014?", "answer": ["Sam Smith"], "question_idx": 186, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 44 | {"question": "Which album did aretha franklin release in may 1976?", "answer": ["Sparkle"], "question_idx": 187, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 45 | {"question": "Who was the prime minister of australia in 1958?", "answer": ["Robert Menzies"], "question_idx": 188, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 46 | {"question": "Who seized power in italy in 1922?", "answer": ["Benito Mussolini"], "question_idx": 192, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 47 | {"question": "Which album did steve hackett release in 1999?", "answer": ["Dark Town"], "question_idx": 193, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 48 | {"question": "Who is the president of ecuador in 2012?", "answer": ["Rafael Correa"], "question_idx": 194, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 49 | {"question": "Which album did venus hum release in july 2006?", "answer": ["The Colors in the Wheel"], "question_idx": 198, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 50 | {"question": "Which album did billy currington release in october 2008?", "answer": ["Little Bit of Everything"], "question_idx": 204, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 51 | {"question": "Who is the quarterback for the new england patriots in 1985?", "answer": ["Tony Eason", "Steve Grogan"], "question_idx": 208, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 52 | {"question": "Where will tebow go in 2013?", "answer": ["New England Patriots"], "question_idx": 211, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 53 | {"question": "Which album did john scofield release in 1997?", "answer": ["A Go Go"], "question_idx": 214, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 54 | {"question": "What money do they use in spain before 2002?", "answer": ["Peseta"], "question_idx": 217, "Temporal signal": ["BEFORE"], "Temporal question type": ["Explicit"]} 55 | {"question": "What team is hank baskett playing for in 2010?", "answer": ["Philadelphia Eagles", "Minnesota Vikings"], "question_idx": 218, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 56 | {"question": "Who is the current president of the dominican republic in 2010?", "answer": ["Leonel Fernández"], "question_idx": 221, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 57 | {"question": "Which film starring katherine shindle was released in 2005?", "answer": ["Capote"], "question_idx": 225, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 58 | {"question": "Which album did shalamar release in august 1990?", "answer": ["Wake Up"], "question_idx": 230, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 59 | {"question": "Which album did billy paul release in 2000?", "answer": ["Live World Tour 1999"], "question_idx": 231, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 60 | {"question": "Who will sponsor kevin harvick in 2011?", "answer": ["Royal Dutch Shell", "Pennzoil-Quaker State"], "question_idx": 234, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 61 | {"question": "Which film starring zoe caldwell was released in 2004?", "answer": ["Birth"], "question_idx": 240, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 62 | {"question": "Who did taft defeated in 1908?", "answer": ["William Jennings Bryan"], "question_idx": 243, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 63 | {"question": "Which book did alan weisman write in 2007?", "answer": ["The World Without Us"], "question_idx": 248, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 64 | {"question": "Who will alex smith play for in 2011?", "answer": ["San Francisco 49ers"], "question_idx": 249, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 65 | {"question": "Who played in the ncaa football championship in 2013?", "answer": ["Kansas"], "question_idx": 252, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 66 | {"question": "Who did the chargers draft in 2011?", "answer": ["Frank Summers"], "question_idx": 258, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 67 | {"question": "Who was the leader of the nation of islam in 1996?", "answer": ["Louis Farrakhan"], "question_idx": 266, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 68 | {"question": "What superman actors birthday is on 31st july?", "answer": ["dean Cain"], "question_idx": 276, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 69 | {"question": "Which movie did paul greengrass direct in 2006?", "answer": ["United 93"], "question_idx": 286, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 70 | {"question": "What song did the beatles sing on the ed sullivan show in 1964?", "answer": ["the chorus of She Loves You", "All My Loving", "Till There Was You", "She Loves You"], "question_idx": 287, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 71 | {"question": "Which album did carter the unstoppable sex machine release in 1991?", "answer": ["30 Something", "Handbuilt by Perverts", "Bloodsport For All", "After the Watershed"], "question_idx": 290, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 72 | {"question": "Which book did edgar allan poe write in 1827?", "answer": ["Tamerlane and Other Poems"], "question_idx": 292, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 73 | {"question": "Which book did alasdair gray write in 1990?", "answer": ["Something Leather"], "question_idx": 299, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 74 | {"question": "Who did president obama run against in 2008?", "answer": ["John McCain"], "question_idx": 300, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 75 | {"question": "What team did messi play for in 2010?", "answer": ["Argentina national football team", "FC Barcelona"], "question_idx": 306, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 76 | {"question": "Who became king of england in 1910?", "answer": ["Edward VII", "Albert Edward"], "question_idx": 315, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 77 | {"question": "Which album did gang of four release in 1982?", "answer": ["Songs Of The Free", "Another Day/Another Dollar"], "question_idx": 319, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 78 | {"question": "Who became the wife of martin luther in 1525?", "answer": ["Katharina von Bora"], "question_idx": 322, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 79 | {"question": "Which country did eric of sweden conquered in 1157?", "answer": ["Finland"], "question_idx": 324, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 80 | {"question": "What club does cristiano ronaldo play for in 2010?", "answer": ["Real Madrid C.F."], "question_idx": 327, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 81 | {"question": "What country gained its independence from britain on july 10 1973?", "answer": ["The Bahamas"], "question_idx": 333, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 82 | {"question": "Which country gained its independence from spain in 1968?", "answer": ["Equatorial Guinea"], "question_idx": 337, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 83 | {"question": "Who did the detroit lions play on thanksgiving day in 1934?", "answer": ["Chicago Bears", "Dallas Cowboys"], "question_idx": 338, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 84 | {"question": "Who did cristiano ronaldo play for in 2010?", "answer": ["Real Madrid C.F."], "question_idx": 346, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 85 | {"question": "What land did the us acquire from spain in 1819?", "answer": ["Florida"], "question_idx": 347, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 86 | {"question": "Who is governor of ohio in 2011?", "answer": ["John Kasich", "republican john kasich", "john kasich", "John Richard Kasich"], "question_idx": 357, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 87 | {"question": "Who does michael oher play for in 2009?", "answer": ["Baltimore Ravens"], "question_idx": 367, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 88 | {"question": "Where does frida kahlo live in 1950?", "answer": ["Coyoacán"], "question_idx": 373, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 89 | {"question": "What vegetables can i plant in november in southern california?", "answer": ["Olallieberry", "Apple", "Phoenix dactylifera", "Common Fig", "Apricot", "Lemon", "Grapefruit", "Cherry", "Grape", "Boysenberry"], "question_idx": 375, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 90 | {"question": "Who won the league cup in 2002?", "answer": ["Liverpool F.C."], "question_idx": 381, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 91 | {"question": "What is there to do in gatlinburg in december?", "answer": ["Gatlinburg Space Needle", "Gatlinburg Falls Parkview Resort", "Gatlinburg Skylift", "Ober Gatlinburg", "Ripley's Haunted Adventure", "Baskins Creek Falls Trail", "Bent Creek Golf Village Resort", "Mysterious Mansion of Gatlinburg", "Westgate Smoky Mountain Resort", "Gatlinburg Whitewater Rafting"], "question_idx": 386, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 92 | {"question": "Who will coach carolina panthers in 2013?", "answer": ["Ron Rivera"], "question_idx": 392, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 93 | {"question": "What award did jack albertson win in 1968?", "answer": ["Academy Award for Actor in a Supporting Role"], "question_idx": 399, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 94 | {"question": "Who settled in massachusetts in 1620?", "answer": ["Pilgrim", "Puritan", "English people"], "question_idx": 401, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 95 | {"question": "Which movie did atom egoyan direct in 1991?", "answer": ["The Adjuster"], "question_idx": 402, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 96 | {"question": "Who is the american author who lived in japan during the 1800's?", "answer": ["Lafcadio Hearn"], "question_idx": 408, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 97 | {"question": "Who was the former male entertainment tonight host with mary hart?", "answer": ["John Tesh"], "question_idx": 410, "Temporal signal": ["ORDINAL"], "Temporal question type": ["Explicit"]} 98 | {"question": "What movie did danny devito win an award for in 1981?", "answer": ["Taxi"], "question_idx": 413, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 99 | {"question": "Who was in the national championship in 2013?", "answer": ["Alabama"], "question_idx": 415, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 100 | {"question": "Which album did merril bainbridge release in 1996?", "answer": ["The Garden"], "question_idx": 420, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 101 | {"question": "Who is the governor of virginia in 2011?", "answer": ["Bob McDonnell"], "question_idx": 421, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 102 | {"question": "Which album did humble pie release in march 1972?", "answer": ["Smokin'"], "question_idx": 429, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 103 | {"question": "Where did the massive allied invasion of northern france take place in june 1944?", "answer": ["Normandy"], "question_idx": 430, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 104 | {"question": "Which album did kreator release in july 1997?", "answer": ["Outcast"], "question_idx": 432, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 105 | {"question": "Where did monsanto rank on the forune 500 in 2000?", "answer": ["54", "70"], "question_idx": 443, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 106 | {"question": "Which book did greg bear write in 1999?", "answer": ["Darwin's Radio"], "question_idx": 446, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 107 | {"question": "What president was graduated from harvard in 1904?", "answer": ["Franklin Delano Roosevelt", "Roosevelt", "Franklin"], "question_idx": 449, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 108 | {"question": "What book was written by george orwell and published in 1945?", "answer": ["Animal Farm"], "question_idx": 456, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 109 | {"question": "Which movie did kevin lima star in in 1995?", "answer": ["A Goofy Movie"], "question_idx": 457, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 110 | {"question": "What to do in panama city beach in february?", "answer": ["Thunder Beach Motorcycle Rally", "Club La Vela", "Man in the Sea Museum", "Camp Helen State Park", "Pier Park", "Shipwreck Island", "Latimer Cabin", "St. Andrews State Park", "Miracle Strip at Pier Park"], "question_idx": 460, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 111 | {"question": "What is the currency of germany in 2010?", "answer": ["Euro"], "question_idx": 462, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 112 | {"question": "Who did mexico gain independence from in 1821?", "answer": ["Spain"], "question_idx": 468, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 113 | {"question": "What conservative walked out of the democratic convention in 1948?", "answer": ["Strom Thurmond"], "question_idx": 469, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 114 | {"question": "Which book did tobias wolff write in 2003?", "answer": ["Old School"], "question_idx": 470, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 115 | {"question": "When do we change the clocks in 2011 in australia?", "answer": ["04-01"], "question_idx": 480, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 116 | {"question": "Who was governor of texas in 2003?", "answer": ["Rick Perry"], "question_idx": 483, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 117 | {"question": "Which album did bennie green release in 1955?", "answer": ["Bennie Green Blows His Horn"], "question_idx": 486, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 118 | {"question": "What team will michael vick play for in 2011?", "answer": ["Philadelphia Eagles"], "question_idx": 488, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 119 | {"question": "What country did the mongols conquer in 1240?", "answer": ["Russia", "Tibet"], "question_idx": 491, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 120 | {"question": "Which album did clem snide release in 2007?", "answer": ["Lose Big"], "question_idx": 492, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 121 | {"question": "Who came to power in cuba in 1959?", "answer": ["Fidel Castro"], "question_idx": 493, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 122 | {"question": "Which album did ziggy marley release in may 1991?", "answer": ["Jahmekya"], "question_idx": 495, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 123 | {"question": "Which album did loudon wainwright iii release in 1973?", "answer": ["Attempted Mustache"], "question_idx": 501, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 124 | {"question": "Which album did eddie rabbitt release in march 1986?", "answer": ["Rabbitt Trax"], "question_idx": 503, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 125 | {"question": "What team did kaka play for in 2009?", "answer": ["A.C. Milan", "Real Madrid C.F."], "question_idx": 505, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 126 | {"question": "What to see in washington dc in 2 days?", "answer": ["White House", "The Phillips Collection", "United States Capitol", "Thomas Jefferson Memorial", "National Museum of the American Indian", "International Spy Museum", "National Portrait Gallery", "Washington Monument", "Lincoln Memorial", "Freer Gallery of Art"], "question_idx": 507, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 127 | {"question": "Who did nigeria gain independence from in 1960?", "answer": ["Britain", "its independence from the United Kingdom", "British Empire", "its independence from the British"], "question_idx": 513, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 128 | {"question": "Who was bill clinton's vice president during 1995?", "answer": ["Al Gore"], "question_idx": 517, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 129 | {"question": "Which album did boom boom satellites release in may 2006?", "answer": ["On"], "question_idx": 520, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 130 | {"question": "What team did david beckham play for in 2011?", "answer": ["LA Galaxy"], "question_idx": 521, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 131 | {"question": "What kind of government does iran have after 1979?", "answer": ["Islamic republic", "Theocracy", "Unitary state"], "question_idx": 527, "Temporal signal": ["AFTER"], "Temporal question type": ["Explicit"]} 132 | {"question": "Which book did mark twain write in 1909?", "answer": ["Letters from the Earth"], "question_idx": 533, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 133 | {"question": "Who will plaxico burress play for in 2011?", "answer": ["New York Jets"], "question_idx": 534, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 134 | {"question": "Who played the title role of superman in 1950?", "answer": ["George Reeves"], "question_idx": 547, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 135 | {"question": "Which movie did costa gavras direct in 2005?", "answer": ["Le Couperet"], "question_idx": 550, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 136 | {"question": "Who did brett gardner play for in 2008?", "answer": ["New York Yankees"], "question_idx": 565, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 137 | {"question": "What is celebrated on january 6th in france?", "answer": ["Epiphany"], "question_idx": 569, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 138 | {"question": "What countries did japan attack on december 7 1941?", "answer": ["the United States", "Pearl Harbor the United States"], "question_idx": 573, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 139 | {"question": "What national holiday is celebrated in france on july 14?", "answer": ["Bastille Day", "July 14 Bastille Day"], "question_idx": 574, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 140 | {"question": "Who arrived in jamestown in 1620?", "answer": ["German"], "question_idx": 576, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 141 | {"question": "Who does michael oher play for in 2010?", "answer": ["Baltimore Ravens"], "question_idx": 578, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 142 | {"question": "Which film starring mighty gabby was released in 2005?", "answer": ["500 Years Later"], "question_idx": 579, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 143 | {"question": "Which film starring thom hoffman was released in 2006?", "answer": ["Black Book"], "question_idx": 580, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 144 | {"question": "Which album did bonnie pink release in april 2000?", "answer": ["Let Go"], "question_idx": 582, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 145 | {"question": "What is the currency of sweden in 2010?", "answer": ["Swedish krona"], "question_idx": 590, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 146 | {"question": "Which movie did rob minkoff direct in 2002?", "answer": ["Stuart Little 2"], "question_idx": 591, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 147 | {"question": "Which movie did charles mcgraw star in in 1956?", "answer": ["The Cruel Tower", "Toward the Unknown", "Away All Boats"], "question_idx": 592, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 148 | {"question": "Who was the cohost in 1993 on cbs evening news?", "answer": ["Connie Chung"], "question_idx": 594, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 149 | {"question": "Which book did adeline yen mah write in 2004?", "answer": ["Chinese Cinderella and the Secret Dragon Society"], "question_idx": 603, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 150 | {"question": "What led to the split of the republican party in 1912?", "answer": ["William Howard Taft"], "question_idx": 609, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 151 | {"question": "Which book did cory doctorow write in 2001?", "answer": ["Frequency Audio"], "question_idx": 610, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 152 | {"question": "Which movie did zoe cassavetes direct in 2007?", "answer": ["Broken English"], "question_idx": 613, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 153 | {"question": "Where is the warmest weather in the united states in december?", "answer": ["Florida"], "question_idx": 619, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 154 | {"question": "Who did terrell owens play for in 2010?", "answer": ["Cincinnati Bengals"], "question_idx": 622, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 155 | {"question": "Which movie did eddie hodges star in in 1959?", "answer": ["A Hole in the Head"], "question_idx": 626, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 156 | {"question": "Who did abyssinia belong to in 1935?", "answer": ["Italy"], "question_idx": 630, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 157 | {"question": "What victories did labor unions win in the 1930s?", "answer": ["strike"], "question_idx": 637, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 158 | {"question": "Who is the governor of arizona in 2009?", "answer": ["Janet Napolitano", "Jan Brewer"], "question_idx": 638, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 159 | {"question": "Which album did peter cetera release in 1986?", "answer": ["Solitude/Solitaire"], "question_idx": 642, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 160 | {"question": "What did charles lindbergh became famous for in the 1920s?", "answer": ["Pilot"], "question_idx": 643, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 161 | {"question": "Which film starring aubrey morris was released in 1996?", "answer": ["Bordello of Blood"], "question_idx": 644, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 162 | {"question": "Who won the presidential election in egypt in 2012?", "answer": ["Mohamed Morsi"], "question_idx": 648, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 163 | {"question": "Who was the prime minister of canada in 1998?", "answer": ["Jean Chrétien"], "question_idx": 653, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 164 | {"question": "What is the capital of spain in 2010?", "answer": ["Madrid"], "question_idx": 654, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 165 | {"question": "What were the causes of the russian revolution in 1917?", "answer": ["World War I"], "question_idx": 662, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 166 | {"question": "Who did abraham lincoln run against for president in 1864?", "answer": ["George B. McClellan"], "question_idx": 670, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 167 | {"question": "What is the currency in germany in 2010?", "answer": ["Euro"], "question_idx": 672, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 168 | {"question": "Which album did ruff endz release in may 2002?", "answer": ["Someone to Love You"], "question_idx": 675, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 169 | {"question": "Who was the winner of dancing on ice in 2013?", "answer": ["Beth Tweddle", "Olympic gymnast Beth Tweddle", "Tweddle"], "question_idx": 681, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 170 | {"question": "Which album did the chordettes release in 1952?", "answer": ["Harmony Encores"], "question_idx": 684, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 171 | {"question": "Where to travel in africa in january?", "answer": ["Angola", "Côte d’Ivoire", "Central African Republic", "Burkina Faso", "Djibouti", "Congo", "Burundi", "Botswana", "Cape Verde", "Benin"], "question_idx": 695, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 172 | {"question": "Who did president bush run against in 2004?", "answer": ["John Kerry"], "question_idx": 696, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 173 | {"question": "Who did michael vick play for in 2008?", "answer": ["Atlanta Falcons"], "question_idx": 698, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 174 | {"question": "What is currency of czechoslovakia before 1939?", "answer": ["Czechoslovak koruna"], "question_idx": 701, "Temporal signal": ["BEFORE"], "Temporal question type": ["Explicit"]} 175 | {"question": "Which movie did alfred hitchcock direct in 1960?", "answer": ["Psycho"], "question_idx": 704, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 176 | {"question": "What to do in hong kong for a week?", "answer": ["Victoria Peak", "Hong Kong Museum of Art", "Dr Sun Yat-sen Museum", "Kowloon Walled City", "Po Lin Monastery", "Hong Kong Disneyland", "Ten Thousand Buddhas Monastery", "Lantau Island"], "question_idx": 708, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 177 | {"question": "What state does the simpsons live in 2012?", "answer": ["Springfield"], "question_idx": 714, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 178 | {"question": "Which album did they might be giants release in september 1994?", "answer": ["John Henry"], "question_idx": 717, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 179 | {"question": "Where did the chinese settle in america in the 1800s?", "answer": ["California", "American West"], "question_idx": 721, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 180 | {"question": "Which album did jan akkerman release in 1993?", "answer": ["Puccini's Cafe"], "question_idx": 727, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 181 | {"question": "Who will michael schumacher drive for in 2013?", "answer": ["Omega SA"], "question_idx": 728, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 182 | {"question": "What country was formed from eastern pakistan in 1971?", "answer": ["Bangladesh"], "question_idx": 729, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 183 | {"question": "Who was president in 1988 in the united states?", "answer": ["Ronald Reagan"], "question_idx": 730, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 184 | {"question": "What college did martin luther king graduated from in 1948?", "answer": ["Morehouse College"], "question_idx": 731, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 185 | {"question": "Which movie did eric sykes star in in 2005?", "answer": ["Harry Potter and the Goblet of Fire"], "question_idx": 739, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 186 | {"question": "Which movie did alan rafkin direct in 1965?", "answer": ["Ski Party"], "question_idx": 740, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 187 | {"question": "What team does kris humphries play for in 2011?", "answer": ["Brooklyn Nets"], "question_idx": 745, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 188 | {"question": "Who does cristiano ronaldo play for in 2010?", "answer": ["Real Madrid C.F."], "question_idx": 747, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 189 | {"question": "Which book did william faulkner write in 1942?", "answer": ["Go Down, Moses"], "question_idx": 753, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 190 | {"question": "What president resigned from office in 1974?", "answer": ["Richard Nixon"], "question_idx": 760, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 191 | {"question": "Who did libya gain independence from in 1951?", "answer": ["Italian Imperialists", "its independence from Italy", "Italy"], "question_idx": 765, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 192 | {"question": "Who settled in delaware in 1638?", "answer": ["Sweden", "Finns"], "question_idx": 767, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 193 | {"question": "Where did barack obama go to college in 1991?", "answer": ["Harvard Law School"], "question_idx": 770, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 194 | {"question": "What document was signed by the pilgrims in 1620?", "answer": ["Mayflower Compact"], "question_idx": 772, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 195 | {"question": "Which album did chris hillman release in 2005?", "answer": ["The Other Side"], "question_idx": 784, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 196 | {"question": "Which album did deep purple release in july 1993?", "answer": ["The Battle Rages On..."], "question_idx": 785, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 197 | {"question": "What country did united states purchase alaska from in 1867 for $7.2 million dollars?", "answer": ["Russia"], "question_idx": 798, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 198 | {"question": "Who was the president of pakistan in 1980?", "answer": ["Muhammad Zia-ul-Haq"], "question_idx": 810, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 199 | {"question": "Which album did professor griff release in 2001?", "answer": ["And the Word Became Flesh"], "question_idx": 813, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 200 | {"question": "What money do they have in greek before 2002?", "answer": ["Greek drachma"], "question_idx": 818, "Temporal signal": ["BEFORE"], "Temporal question type": ["Explicit"]} 201 | {"question": "Who was defeated at the battle of fallen timbers in 1794?", "answer": ["Confederate States of America", "British Indian Army", "Native Americans in the United States"], "question_idx": 822, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 202 | {"question": "Who won the american league east in 2012?", "answer": ["New York Yankees"], "question_idx": 830, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 203 | {"question": "Who coached the dallas cowboys in 1996?", "answer": ["Chan Gailey"], "question_idx": 832, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 204 | {"question": "Who graduated from west point in 1829?", "answer": ["Robert E. Lee"], "question_idx": 835, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 205 | {"question": "What did france lose to the british in the treaty of paris in 1763?", "answer": ["North America"], "question_idx": 837, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 206 | {"question": "Which book did ann martin write in 2003?", "answer": ["The Meanest Doll in the World"], "question_idx": 838, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 207 | {"question": "What team does david beckham play for in 2012?", "answer": ["LA Galaxy"], "question_idx": 839, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 208 | {"question": "Who won the nobel peace prize in 2007?", "answer": ["Intergovernmental Panel on Climate Change", "Al Gore"], "question_idx": 841, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 209 | {"question": "Who is the president of costa rica in 2012?", "answer": ["Laura Chinchilla"], "question_idx": 842, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 210 | {"question": "Which movie did deepak sareen direct in 1998?", "answer": ["Jab Pyaar Kisise Hota Hai"], "question_idx": 863, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 211 | {"question": "What country did italy conquer in 1939?", "answer": ["Albania"], "question_idx": 865, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 212 | {"question": "Which album did the octopus project release in october 2005?", "answer": ["One Ten Hundred Thousand Million"], "question_idx": 878, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 213 | {"question": "What country gained its independence from britain in 1960?", "answer": ["Nigeria", "Cyprus"], "question_idx": 881, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 214 | {"question": "Who founded the american federation of labor afl in 1886?", "answer": ["Samuel Gompers"], "question_idx": 882, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 215 | {"question": "Which album did zimmers hole release in 1999?", "answer": ["Bound By Fire"], "question_idx": 883, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 216 | {"question": "Which album did limbeck release in april 2007?", "answer": ["Limbeck"], "question_idx": 891, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 217 | {"question": "Who was nominated for the academy award for best director in 2011?", "answer": ["Martin Scorsese", "Michel Hazanavicius", "Woody Allen", "Terrence Malick", "Alexander Payne"], "question_idx": 895, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 218 | {"question": "Which movie did anita yuen star in in 2004?", "answer": ["Love Trilogy"], "question_idx": 896, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 219 | {"question": "Who was the prime minister of pakistan in 1999?", "answer": ["Nawaz Sharif"], "question_idx": 897, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 220 | {"question": "Who does david beckham play for in 2013?", "answer": ["Paris Saint-Germain F.C."], "question_idx": 903, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 221 | {"question": "Which album did dave vanian and the phantom chords release in 1995?", "answer": ["David Vanian and the Phantom Chords"], "question_idx": 908, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 222 | {"question": "Which book did katherine hannigan write in 2004?", "answer": ["Ida B."], "question_idx": 913, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 223 | {"question": "Who is the prime minister of spain in 2011?", "answer": ["Juan Carlos I of Spain", "José Luis Rodríguez Zapatero", "Mariano Rajoy"], "question_idx": 925, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 224 | {"question": "Who won the nobel peace prize in 2011 for physics?", "answer": ["Ellen Johnson Sirleaf", "Tawakkol Karman", "Leymah Gbowee"], "question_idx": 926, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 225 | {"question": "Which film starring vyacheslav tikhonov was released in 1994?", "answer": ["Burnt by the Sun"], "question_idx": 938, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 226 | {"question": "What contribution did hooke made to science in 1665?", "answer": ["Micrographia"], "question_idx": 941, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 227 | {"question": "In 1982 who were the primetieme emmy award for comedy series nominees?", "answer": ["John Rappaport", "Burt Metcalfe", "Howard Gewirtz", "Frank Dungan", "Thad Mumford", "James L. Brooks", "Stan Daniels", "Ed. Weinberger", "Richard Sakai", "Steve Marshall", "Glen Charles", "Les Charles", "Hugh Wilson", "Blake Hunter", "Ken Estin", "Jeff Stein", "April Kelly", "Dennis Koenig", "Mel Tolkin", "George Eckstein", "Ernest Chambers", "Jim Parker", "Bob Brunner", "Ken Hecht", "Peter Torokvei", "Danny Arnold", "Roland Kibbee", "Dan Guntzelman", "Gary Shaw", "Ian Praiser", "Dan Wilcox"], "question_idx": 944, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 228 | {"question": "Which album did the crystal method release in september 1997?", "answer": ["Vegas"], "question_idx": 950, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 229 | {"question": "Which album did dave seaman release in august 1999?", "answer": ["Global Underground 012"], "question_idx": 957, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 230 | {"question": "Who was manchuria invaded by in 1931?", "answer": ["Japan", "Imperial Japan", "the Japanese Army", "the Kwantung Army of the Empire of Japan", "the Japanese"], "question_idx": 961, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 231 | {"question": "What land did the us buy from france in 1803?", "answer": ["Louisiana"], "question_idx": 968, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 232 | {"question": "Which book did roald dahl write in 1990?", "answer": ["Esio Trot"], "question_idx": 974, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 233 | {"question": "Which album did jackie mclean release in 2005?", "answer": ["Consequence"], "question_idx": 978, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 234 | {"question": "Who won the state of texas in 2008?", "answer": ["John McCain"], "question_idx": 982, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 235 | {"question": "Which album did ramy ayach release in december 2006?", "answer": ["Habbaytak ana"], "question_idx": 983, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 236 | {"question": "Which album did toad the wet sprocket release in 1995?", "answer": ["In Light Syrup"], "question_idx": 986, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 237 | {"question": "Which movie did jaco van dormael direct in 2009?", "answer": ["Mr. Nobody"], "question_idx": 993, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 238 | {"question": "Which album did a band called pain release in 2005?", "answer": ["Broken Dreams"], "question_idx": 994, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 239 | {"question": "Which movie did giri babu star in in 2008?", "answer": ["Krishna", "Nee Sneham"], "question_idx": 995, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 240 | {"question": "What movies has taylor lautner been in 2011?", "answer": ["Field of Dreams 2: Lockout", "Abduction", "The Twilight Saga: Breaking Dawn – Part 1"], "question_idx": 1002, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 241 | {"question": "Who does jeremy shockey play for in 2012?", "answer": ["Carolina Panthers"], "question_idx": 1003, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 242 | {"question": "Which album did cosima de vito release in october 2004?", "answer": ["Cosima"], "question_idx": 1004, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 243 | {"question": "Who founded the pittsburgh steelers in 1933?", "answer": ["Rooney family"], "question_idx": 1006, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 244 | {"question": "Where to vacation in italy in august?", "answer": ["Corriere della Sera", "l'Unità", "Il Foglio", "la Repubblica", "Il Messaggero", "La Domenica del Corriere", "La Stampa", "Avanti!", "La Gazzetta dello Sport", "Liberazione"], "question_idx": 1008, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 245 | {"question": "What did dmitri mendeleev discover in 1869?", "answer": ["Periodic Table"], "question_idx": 1014, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 246 | {"question": "Which movie did matt leblanc star in in 2001?", "answer": ["All the Queen's Men"], "question_idx": 1016, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 247 | {"question": "Who is the current governor of arizona in 2010?", "answer": ["Jan Brewer"], "question_idx": 1024, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 248 | {"question": "Which album did the colourfield release in march 1987?", "answer": ["Deception"], "question_idx": 1029, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 249 | {"question": "What countries were occupied by germany in 1940?", "answer": ["France", "Denmark", "Norway", "Belgium", "the Netherlands", "Luxembourg"], "question_idx": 1033, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 250 | {"question": "Who managed liverpool f.c. from 2004 to june 2010?", "answer": ["Gérard Houllier"], "question_idx": 1036, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 251 | {"question": "Which film starring brooke theiss was released in 1988?", "answer": ["Little Nikita", "A Nightmare on Elm Street 4: The Dream Master"], "question_idx": 1039, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 252 | {"question": "Who performed at the super bowl in 1993?", "answer": ["Michael Jackson"], "question_idx": 1047, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 253 | {"question": "What did hans oersted discovered in 1819?", "answer": ["Aluminium"], "question_idx": 1063, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 254 | {"question": "Who are the 9 justices of the supreme court in 2011?", "answer": ["John Roberts", "Clarence Thomas", "David Souter", "John Paul Stevens", "Edward Douglass White", "david souter", "john paul steven", "john robert", "clarence thoma", "edward douglass white", "ricky polston"], "question_idx": 1069, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 255 | {"question": "Which film starring dhanush was released in 2008?", "answer": ["Yaaradi Nee Mohini"], "question_idx": 1085, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 256 | {"question": "Which album did hanin elias release in may 2004?", "answer": ["Future Noir"], "question_idx": 1089, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 257 | {"question": "What honor did agatha christie receive in 1971?", "answer": ["Dame Commander of the Order of the British Empire"], "question_idx": 1092, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 258 | {"question": "Which album did deep forest release in 2000?", "answer": ["Pacifique"], "question_idx": 1094, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 259 | {"question": "Who will david beckham play for in 2013?", "answer": ["Paris Saint-Germain F.C."], "question_idx": 1097, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 260 | {"question": "Which album did bardo pond release in 2001?", "answer": ["Cypher Documents I", "Dilate", "Vol. II", "U.S. Tour, Spring 2001 Split"], "question_idx": 1099, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 261 | {"question": "What to see in london in june 2012?", "answer": ["Regent's Park", "Tower of London", "Buckingham Palace", "Palace of Westminster", "London Eye", "Tower Bridge", "Hyde Park", "Westminster Abbey", "St Paul's Cathedral", "Trafalgar Square"], "question_idx": 1102, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 262 | {"question": "Who will run for president in 2016 democrat?", "answer": ["Clinton", "Chris Christie", "Hillary Clinton", "Hillary", "Hillary Rodham Clinton", "Hitlery Clinton", "Christie", "Mitt Romney"], "question_idx": 1105, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 263 | {"question": "Who ruled jerusalem in 1099?", "answer": ["The Crusaders", "First Crusade", "Crusader Kings"], "question_idx": 1114, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 264 | {"question": "Who is hugh hefner dating now in 2012?", "answer": ["Crystal Harris"], "question_idx": 1115, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 265 | {"question": "Which movie did hanna schygulla star in in 1980?", "answer": ["Berlin Alexanderplatz"], "question_idx": 1125, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 266 | {"question": "Which album did wayne wonder release in april 2006?", "answer": ["Original Bombshell (Remastered)"], "question_idx": 1128, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 267 | {"question": "Who are the virginia senators in 2013?", "answer": ["Mark Warner", "mark warner", "Mark Robert Warner (Democrat, start date: 2009)"], "question_idx": 1134, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 268 | {"question": "Who was the governor of missouri in 1996?", "answer": ["Mel Carnahan"], "question_idx": 1138, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 269 | {"question": "Who was canadian prime minister in 1993?", "answer": ["Brian Mulroney", "Kim Campbell", "Jean Chrétien"], "question_idx": 1139, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 270 | {"question": "Which movie did steve hanft direct in 2004?", "answer": ["Kill the Moonlight"], "question_idx": 1141, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 271 | {"question": "Which book did alasdair gray write in 1981?", "answer": ["Lanark: A Life in Four Books"], "question_idx": 1156, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 272 | {"question": "Who ran against abraham lincoln for president in 1860?", "answer": ["John C. Breckinridge"], "question_idx": 1157, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 273 | {"question": "Who did the baltimore ravens draft in 2011?", "answer": ["Haloti Ngata", "Mark Clayton", "Ben Grubbs", "Joe Flacco", "Terrell Suggs", "Kyle Boller", "Michael Oher"], "question_idx": 1163, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 274 | {"question": "What team did ronaldo play for in 2003?", "answer": ["Real Madrid C.F."], "question_idx": 1165, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 275 | {"question": "Who played in the superbowl in 2006?", "answer": ["the Steelers"], "question_idx": 1168, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 276 | {"question": "Who took control of rome in 46 bc and became dictator?", "answer": ["Julius Caesar"], "question_idx": 1176, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 277 | {"question": "What do muslims celebrate in december?", "answer": ["Christmas", "Ramadan", "Ramasan"], "question_idx": 1180, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 278 | {"question": "Who is the president of cuba in 2009?", "answer": ["Raúl Castro"], "question_idx": 1183, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 279 | {"question": "Who does justin bieber go out with on facebook in 2011?", "answer": ["selena gomez"], "question_idx": 1188, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 280 | {"question": "What is the average temperature in sydney in august?", "answer": ["17"], "question_idx": 1191, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 281 | {"question": "Who sang at the super bowl in 2013?", "answer": ["Alicia Keys", "Beyonce"], "question_idx": 1193, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 282 | {"question": "Which film starring adelle lutz was released in 1995?", "answer": ["Beyond Rangoon"], "question_idx": 1200, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 283 | {"question": "Which album did kathleen hanna release in 1997?", "answer": ["Julie Ruin"], "question_idx": 1202, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 284 | {"question": "Which film starring jamie waylett was released in 2005?", "answer": ["Harry Potter and the Goblet of Fire"], "question_idx": 1207, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 285 | {"question": "Who is lamar odom playing for in 2012?", "answer": ["Los Angeles Lakers", "Dallas Mavericks"], "question_idx": 1211, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 286 | {"question": "What ship did christopher columbus sail on in 1492?", "answer": ["Santa Maria"], "question_idx": 1213, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 287 | {"question": "Who was the leader of the ussr in 1948?", "answer": ["Joseph Stalin"], "question_idx": 1217, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 288 | {"question": "Which album did fleetwood mac release in april 2003?", "answer": ["Say You Will"], "question_idx": 1219, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 289 | {"question": "Which prize did elie wiesel win in 1986?", "answer": ["Nobel Peace Prize"], "question_idx": 1220, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 290 | {"question": "What teams did the patriots lose to in 2013?", "answer": ["Baltimore Ravens"], "question_idx": 1222, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 291 | {"question": "What countries were occupied by the soviet union in 1988?", "answer": ["Afghanistan"], "question_idx": 1225, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 292 | {"question": "Who defeated napoleon at the battle of trafalgar in 1805?", "answer": ["Nelson"], "question_idx": 1230, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 293 | {"question": "Who were the curators for renoir in the 20th century?", "answer": ["Sylvie Patry", "J.Patrice Marandel", "Claudia Einecke", "Joseph J. Rishel"], "question_idx": 1233, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 294 | {"question": "Who was the president of the us in 1971?", "answer": ["Richard Nixon"], "question_idx": 1244, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 295 | {"question": "Which album did mark pickerel release in may 2006?", "answer": ["Snake in the Radio"], "question_idx": 1246, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 296 | {"question": "Who plays batman in 2012?", "answer": ["Kevin Conroy", "Val Kilmer", "Michael Keaton", "Will Friedle", "Will Arnett", "Charles Stanley", "Peter Weller", "George Clooney", "Rino Romano", "William Baldwin", "Jason Sudeikis", "Benjamin McKenzie", "Sam Rockwell", "Christian Bale", "Bruce Greenwood", "Adam West", "Ike Barinholtz", "Jing Abalos", "Joe Krajcar"], "question_idx": 1257, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 297 | {"question": "Which film starring paul dillon was released in 1994?", "answer": ["Blink"], "question_idx": 1269, "Temporal signal": ["OVERLAP"], "Temporal question type": ["Explicit"]} 298 | -------------------------------------------------------------------------------- /images/intro_fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TianHongZXY/qaap/4fa31253a0bca3d42ac01aba5cad205ed20e8c05/images/intro_fig.png -------------------------------------------------------------------------------- /images/main_figure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TianHongZXY/qaap/4fa31253a0bca3d42ac01aba5cad205ed20e8c05/images/main_figure.png -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import json 4 | import random 5 | import argparse 6 | import jsonlines 7 | import openai 8 | import tiktoken 9 | import datetime as dt 10 | from datetime import datetime, timedelta 11 | import coloredlogs, logging 12 | import traceback 13 | from search_wiki import search 14 | from utils import create_context_slices, extract_answer, get_metrics, extract_code_from_string, calc_time_iou 15 | logger = logging.getLogger(__name__) 16 | coloredlogs.install(level='DEBUG', logger=logger) 17 | 18 | 19 | def post(prompt, stop, max_tokens=1600, model_name="gpt-3.5-turbo"): 20 | global TOKENIZER 21 | prompt_num_tokens = len(TOKENIZER.encode(prompt)) 22 | if prompt_num_tokens >= 4096: 23 | return "" 24 | params = { 25 | 'max_tokens': min(4097 - prompt_num_tokens, max_tokens), 26 | 'temperature': 0.0, 27 | 'top_p': 1, 28 | 'n': 1, 29 | 'stop': stop, 30 | } 31 | if model_name == "gpt-3.5-turbo": 32 | params["messages"] = [ 33 | {"role": "system", "content": "You are a helpful assistant."}, 34 | {"role": "user", "content": prompt} 35 | ] 36 | else: 37 | params["prompt"] = prompt 38 | 39 | res = None 40 | attempts = 0 41 | max_attempts = 3 42 | while res is None and attempts < max_attempts: 43 | try: 44 | if model_name == "gpt-3.5-turbo": 45 | response = openai.ChatCompletion.create( 46 | engine=model_name, 47 | **params, 48 | timeout=60 49 | ) 50 | res = response.choices[0].message["content"] 51 | else: 52 | response = openai.Completion.create( 53 | engine=model_name, 54 | **params, 55 | timeout=60 56 | ) 57 | res = response['choices'][0]['text'] 58 | except openai.error.InvalidRequestError as e: 59 | print(e) 60 | print(f"Traceback:\n{traceback.format_exc()}") 61 | return "" 62 | except Exception as e: 63 | print(e) 64 | print(f"Traceback:\n{traceback.format_exc()}") 65 | attempts += 1 66 | time.sleep(1) 67 | continue 68 | 69 | if res is None: 70 | return "" 71 | 72 | return res 73 | 74 | 75 | def qaap(init_prompt, question, info, args, passage=None): 76 | prompt = init_prompt + "\n\nQuestion:" + question + "\nQuestion parsing:\n" 77 | # parse 78 | parsed_query = post(prompt, stop=["\nSearch:"], max_tokens=200, model_name=args.model_name) 79 | print(parsed_query) 80 | time.sleep(1) 81 | info['traj_list'].append(parsed_query) 82 | info['context_slices'] = [] 83 | query = extract_code_from_string(parsed_query) 84 | 85 | prompt += parsed_query 86 | extracted_code_list = [query] 87 | 88 | # search 89 | parsed_search = post(prompt + '\nSearch:\n', stop=['\nGenerate'], max_tokens=200, model_name=args.model_name) 90 | parsed_search = parsed_search.split("Failed entities")[0] 91 | parsed_search = parsed_search.split("\nContext")[0] 92 | print('Search:\n' + parsed_search) 93 | info['traj_list'].append(parsed_search) 94 | prompt += parsed_search 95 | locals_ = {} 96 | tables_list = [] 97 | passages_list = [] 98 | if passage is None: 99 | # retrieve passage from wikipedia 100 | try: 101 | exec(extract_code_from_string(parsed_search), globals(), locals_) 102 | except Exception as e: 103 | print(e) 104 | logger.error('Failed to obtain search entities.') 105 | 106 | try: 107 | entities_to_search = locals_['entities_to_search'] 108 | except KeyError: 109 | entities_to_search = [] 110 | info['search_failed'] = True 111 | 112 | for et_idx, et in enumerate(entities_to_search): 113 | state, results = search(et, summary=args.return_search_passage == "summary") 114 | while not state: 115 | locals_temp = {} 116 | f_et = [et] 117 | new_search_str = \ 118 | f"""\nFailed entities: 119 | ```python 120 | failed_entities = {f_et} 121 | similar_entities = {results} 122 | ``` 123 | """ 124 | info['traj_list'].append(new_search_str) 125 | print(new_search_str) 126 | parsed_search = post(prompt + new_search_str + '\nSearch:\n', stop=['\nGenerate'], max_tokens=200, model_name=args.model_name) 127 | parsed_search = parsed_search.split("Failed entities")[0] 128 | parsed_search = parsed_search.split("\nContext")[0] 129 | print(f'\nSearch:\n{parsed_search}') 130 | info['traj_list'].append(parsed_search) 131 | try: 132 | exec(extract_code_from_string(parsed_search), globals(), locals_temp) 133 | except Exception as e: 134 | print(e) 135 | logger.error('Failed to obtain search entities.') 136 | info['search_failed'] = True 137 | break 138 | # return [""], info 139 | et_to_search = locals_temp['entities_to_search'] 140 | # state, results = search(et_to_search[0]) 141 | state, results = search(et_to_search[0], summary=args.return_search_passage == "summary") 142 | if state == False: 143 | info['search_failed'] = True 144 | break 145 | # return [""], info 146 | tables = results[0] 147 | passages = results[1] 148 | tables_list.append(tables) 149 | passages_list.append(passages) 150 | else: 151 | tables_list = [[]] 152 | passages_list = [[passage]] 153 | 154 | generated_document = post(prompt + "\nGenerate a background document from Wikipedia to answer the given question:", stop=["\nExtract"], max_tokens=200, model_name=args.model_name) 155 | info['context_slices'].append(generated_document) 156 | res = post(prompt + "\nGenerate a background document from Wikipedia to answer the given question:" + generated_document + '\nExtract information relevant to the query:\n', max_tokens=400, stop=['\nContext'], model_name=args.model_name) 157 | res = res.split("\nContext")[0] 158 | res = res.split("\nQuestion")[0] 159 | res = res.split("\nSearch")[0] 160 | info['information_list'].append(res) 161 | print("-" * 50) 162 | # print(context_slice) 163 | print("\nGenerate a background document from Wikipedia to answer the given question:" + generated_document + '\nExtract information relevant to the query:\n' + res) 164 | info['traj_list'].append("\nGenerate a background document from Wikipedia to answer the given question:" + generated_document + '\nExtract information relevant to the query:\n' + res) 165 | try: 166 | extracted_code = extract_code_from_string(res) 167 | if extracted_code is not None: 168 | for c in extracted_code.split("\n"): 169 | if c: 170 | if "query" in c or "information = " in c: 171 | continue 172 | extracted_code_list.append(c) 173 | except Exception as e: 174 | logger.error('Failed to obtain code from returned strings.') 175 | print("Error Type:", type(e)) 176 | print("Error Message:", e) 177 | print(f"Traceback:\n{traceback.format_exc()}") 178 | 179 | for tables, passage_list in zip(tables_list, passages_list): 180 | table_slices = [] 181 | for t in tables: 182 | table_slices += create_context_slices(t) 183 | context_slices = create_context_slices("\n".join(passage_list)) 184 | 185 | for context_slice in table_slices + context_slices: 186 | info['context_slices'].append(context_slice) 187 | res = post(prompt + "\nContext: " + context_slice + '\nExtract information relevant to the query:\n', stop=['\n\nQuestion:'], model_name=args.model_name) 188 | res = res.split("\nContext:")[0] 189 | res = res.split("\nSearch:")[0] 190 | info['information_list'].append(res) 191 | print("*" * 50) 192 | # print(context_slice) 193 | print('Extract information relevant to the query:\n' + res) 194 | info['traj_list'].append("Extract information relevant to the query:\n" + res) 195 | try: 196 | extracted_code = extract_code_from_string(res) 197 | if extracted_code is not None: 198 | for c in extracted_code.split("\n"): 199 | if c: 200 | if "query" in c or "information = " in c: 201 | continue 202 | extracted_code_list.append(c) 203 | # extracted_code_list.append(extracted_code) 204 | except Exception as e: 205 | logger.error('Failed to obtain code from returned strings.') 206 | print("Error Type:", type(e)) 207 | print("Error Message:", e) 208 | print(f"Traceback:\n{traceback.format_exc()}") 209 | continue 210 | try: 211 | answer_key, information = calc_time_iou(extracted_code_list) 212 | predictions = extract_answer(answer_key, information) 213 | except Exception as e: 214 | logger.error('Failed to obtain answer after code execution.') 215 | print("Error Type:", type(e)) 216 | print("Error Message:", e) 217 | print(f"Traceback:\n{traceback.format_exc()}") 218 | predictions = [""] 219 | 220 | assert len(info['information_list']) == len(info['context_slices']) 221 | 222 | return predictions, info 223 | 224 | 225 | if __name__ == "__main__": 226 | TIMESTAMP = time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime()) 227 | parser = argparse.ArgumentParser() 228 | parser.add_argument("--prompt_file", type=str) 229 | parser.add_argument("--given_context", type=int, choices=[0, 1]) 230 | parser.add_argument("--model_name", default="gpt-3.5-turbo") 231 | parser.add_argument("--dataset", type=str, choices=["timeqa", "timequestions", "tempquestions"]) 232 | parser.add_argument("--data_file", type=str) 233 | parser.add_argument("--resume_id", type=int, default=-1) 234 | parser.add_argument("--max_slice_length", type=int, default=512) 235 | parser.add_argument("--slice_stride", type=int, default=384) 236 | parser.add_argument("--return_search_passage", type=str, default="content") 237 | parser.add_argument("--comment", type=str, default="") 238 | args = parser.parse_args() 239 | ROOT_DIR = "~/qaap" 240 | DATA_DIR = os.path.join(ROOT_DIR, "data") 241 | PROMPT_DIR = os.path.join(ROOT_DIR, 'prompts') 242 | TOKENIZER = tiktoken.encoding_for_model(args.model_name) 243 | 244 | openai.api_key = "" 245 | 246 | prompt_file = os.path.join(PROMPT_DIR, args.prompt_file) 247 | with open(prompt_file, 'r', encoding="utf-8") as f: 248 | prompt_dict = json.load(f) 249 | init_prompt = prompt_dict['prompt_text'] 250 | 251 | with jsonlines.open(f"{DATA_DIR}/{args.dataset}/{args.data_file}") as f: 252 | data = list(f) 253 | idxs = list(range(len(data))) 254 | random.seed(0) 255 | random.shuffle(idxs) 256 | data = [data[i] for i in idxs[:100]] 257 | print("Data size: ", len(data)) 258 | 259 | rs = [] 260 | f1_list = [] 261 | old_time = time.time() 262 | with jsonlines.open(f'outputs/{args.dataset}.jsonl-' + args.comment + TIMESTAMP, mode='w', flush=True) as f: 263 | for idx, ex in enumerate(data): 264 | if idx < args.resume_id: 265 | continue 266 | question = ex['question'] 267 | info = {'question': question, 'gt_answer': ex['answer'], 'answer': None, 'traj_list': [], 'information_list': []} 268 | print("-" * 50) 269 | print(idx, question) 270 | 271 | if args.given_context: 272 | passage = ex['context'] 273 | else: 274 | passage = None 275 | 276 | predictions, info = qaap(init_prompt, question, info, args, passage) 277 | 278 | info['answer'] = predictions 279 | print("Predictions: ", predictions) 280 | print("Ground truth: ", ex['answer']) 281 | metrics = get_metrics(predictions, ex['answer']) 282 | info.update(metrics) 283 | rs.append(metrics['em']) 284 | f1_list.append(metrics['f1']) 285 | em_num = sum(rs) 286 | em_rate = sum(rs) / len(rs) 287 | avg_f1 = sum(f1_list) / len(f1_list) 288 | avg_time_per_ques = (time.time() - old_time) / len(rs) 289 | logger.info(f"idx: {idx}, em number: {em_num}, em rate: {em_rate}, avg f1: {avg_f1:.3f}, avg time per question {avg_time_per_ques:.1f}s.") 290 | f.write(info) 291 | -------------------------------------------------------------------------------- /prompts/timeqa.json: -------------------------------------------------------------------------------- 1 | {"prompt_text": "Solve a question answering task by first parsing the question, figuring out what the question is asking, representing the query as a python dictionary with the key where the answer should be stored specified as either the subject or object, with the initial value of None. Secondly, decide what entities to search to find the answer. Thirdly, generate a background document that contains information relevant to the question being asked. Then read the generated document and the searched passage, step by step extract the information that directly and explicitly relates to the question, place the answer as the value of the answer key in the dictionary. For example, 'XXX joined A team in 2017, ..., in 2019, B team signed a contract with XXX', it is easy to know that B team and A team are mutually exclusive, therefore the termination time of A team is in 2019. Represent the extracted information as a dictionary and adding it to a list. If the context does not tell any useful information, extract nothing.\nHere are some examples.\nQuestion:Which school did David Jolly go to in Jan 1989?\nQuestion parsing:\n```python\nquery = {\"subject\": \"David Jolly\", \"relation\": \"go to school\", \"object\": None, \"time\": {\"start\": datetime(1989, 1, 1), \"end\": datetime(1989, 1, 31)}}\nanswer_key = \"object\"\n```\nSearch:\n```python\nentities_to_search = [\"David Jolly\"]\n```\nGenerate a background document from Wikipedia to answer the given question: David Jolly is an American politician who served as the U.S. Representative for Florida's 13th congressional district from 2014 to 2017. He graduated from Indian Rocks Christian School in 1990.\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"David Jolly\", \"relation\": \"go to school\", \"object\": \"Indian Rocks Christian School\", \"time\": {\"start\": datetime(1986, 1, 1), \"end\": datetime(1990, 12, 31)}})\n```\nContext: David Jolly David Wilson Jolly ( born October 31 , 1972 ) is an American attorney , former lobbyist , and politician who served as the U.S . Representative for Floridas 13th congressional district , based in Pinellas County , from 2014 to 2017 . He was subsequently reelected in November 2014 , winning 75 percent of the vote , but was unseated in 2016 by former Governor Charlie Crist after court-ordered redistricting made his district more Democratic . In September 2018 , Jolly announced he had left the Republican Party .\nExtract information relevant to the query:\nThere is nothing relevant to the query.\nContext: Early life . Jolly was born in Dunedin , Florida , the son of Judith and Lawson Jolly , a Baptist pastor . He received his B.A . degree from Emory University in 1994 and his J.D . degree from the George Mason University School of Law in 2001 .\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"David Jolly\", \"relation\": \"go to school\", \"object\": \"Emory University\", \"time\": {\"start\": datetime(1990, 1, 1), \"end\": datetime(1994, 12, 31)}})\ninformation.append({\"subject\": \"David Jolly\", \"relation\": \"go to school\", \"object\": \"George Mason University\", \"time\": {\"start\": datetime(1995, 1, 1), \"end\": datetime(2001, 12, 31)}})\n```\n\nQuestion:Which position did Crispin Blunt hold in Feb 2011?\nQuestion parsing:\n```python\nquery = {\"subject\": \"Crispin Blunt\", \"relation\": \"hold position\", \"object\": None, \"time\": {\"start\": datetime(2011, 2, 1), \"end\": datetime(2011, 2, 28)}}\nanswer_key = \"object\"\n```\nSearch:\n```python\nentities_to_search = [\"Crispin Blunt\"]\n```\nGenerate a background document from Wikipedia to answer the given question: Crispin Blunt is a British Conservative Party politician who has been the Member of Parliament (MP) for Reigate since 1997. In February 2011, he was appointed as the Parliamentary Under-Secretary of State for Prisons and Youth Justice in the Ministry of Justice.\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Crispin Blunt\", \"relation\": \"hold position\", \"object\": \"Member of Parliament\", \"time\": {\"start\": datetime(1997, 1, 1), \"end\": None}})\ninformation.append({\"subject\": \"Crispin Blunt\", \"relation\": \"hold position\", \"object\": \"Parliamentary Under-Secretary of State for Prisons and Youth Justice\", \"time\": {\"start\": datetime(2011, 2, 1), \"end\": None}})\n```\nContext: Crispin Blunt Crispin Jeremy Rupert Blunt ( born 15 July 1960 ) is a British Conservative Party politician . He has served as Member of Parliament ( MP ) for Reigate since 1997 , and from May 2010 to September 2012 he was the Parliamentary Under-Secretary of State for Prisons and Youth Justice within the Ministry of Justice . Blunt first entered the House of Commons at the 1997 general election , when he replaced the then MP Sir George Gardiner who had been deselected by the Constituency Conservative Association Executive Council and joined the Referendum Party .\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Crispin Blunt\", \"relation\": \"hold position\", \"object\": \"Member of Parliament\", \"time\": {\"start\": datetime(1997, 1, 1), \"end\": None}})\ninformation.append({\"subject\": \"Crispin Blunt\", \"relation\": \"hold position\", \"object\": \"Parliamentary Under-Secretary of State for Prisons and Youth Justice\", \"time\": {\"start\": datetime(2010, 5, 1), \"end\": datetime(2012, 9, 30)}})\n```\n\nQuestion:What’s the capital of Klamath County, California between Aug 1854 and Jun 1855?\nQuestion parsing:\n```python\nquery = {\"subject\": None, \"relation\": \"capital of\", \"object\": \"Klamath County, California\", \"time\": {\"start\": datetime(1854, 8, 1), \"end\": datetime(1855, 6, 30)}}\nanswer_key = \"subject\"\n```\nSearch:\n```python\nentities_to_search = [\"Klamath County\"]\n```\nFailed entities:\n```python\nfailed_entities = [\"Klamath County\"]\nsimilar_entities = [\"Klamath County, California\", \"Klamath County, Oregon\", \"Klamath Falls, Oregon\"]\n```\nSearch:\n```python\nentities_to_search = [\"Klamath County, California\"]\n```\nGenerate a background document from Wikipedia to answer the given question: Klamath County, California was a county of California from 1851 to 1874. The county seat was Crescent City from 1851 to 1854 and then Yreka from 1854 to 1874.\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Crescent City\", \"relation\": \"capital of\", \"object\": \"Klamath County, California\", \"time\": {\"start\": datetime(1851, 1, 1), \"end\": datetime(1854, 12, 31)}})\ninformation.append({\"subject\": \"Yreka\", \"relation\": \"capital of\", \"object\": \"Klamath County, California\", \"time\": {\"start\": datetime(1854, 1, 1), \"end\": datetime(1874, 12, 31)}})\n```\nContext: Klamath County , California Klamath County was a county of California from 1851 to 1874 . During its existence , the county seat moved twice and ultimately portions of the territory it once had were carved up and added to nearby counties . The original county seat was Trinidad , on the countys southwestern coast . In 1854 the county seat was moved to Crescent City , because of its larger population . But the western portion of the county was unrepresentative of the mining interests in the eastern portion of the county , and so , in 1856 , the county seat was moved inland , to Orleans Bar , now Orleans . In 1857 , Del Norte County , including Crescent City , was split off from Klamath County . In 1874 Klamath County was finally abolished , divided between Siskiyou and Humboldt counties .\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Trinidad\", \"relation\": \"capital of\", \"object\": \"Klamath County, California\", \"time\": {\"start\": datetime(1851, 1, 1), \"end\": datetime(1853, 12, 31)}})\ninformation.append({\"subject\": \"Crescent City\", \"relation\": \"capital of\", \"object\": \"Klamath County, California\", \"time\": {\"start\": datetime(1854, 1, 1), \"end\": datetime(1855, 12, 31)}})\ninformation.append({\"subject\": \"Orleans\", \"relation\": \"capital of\", \"object\": \"Klamath County, California\", \"time\": {\"start\": datetime(1856, 1, 1), \"end\": datetime(1856, 12, 31)}})\n```"} 2 | -------------------------------------------------------------------------------- /prompts/timequestions.json: -------------------------------------------------------------------------------- 1 | {"prompt_text": "Solve a question answering task by first parsing the question, figuring out what the question is asking, representing the query as a python dictionary with the key where the answer should be stored specified as either the subject or object, with the initial value of None. Secondly, decide what entities to search to find the answer. Thirdly, generate a background document that contains information relevant to the question being asked. Then read the generated document and the searched passage, step by step extract the information that directly and explicitly relates to the question, place the answer as the value of the answer key in the dictionary. For example, 'XXX joined A team in 2017, ..., in 2019, B team signed a contract with XXX', it is easy to know that B team and A team are mutually exclusive, therefore the termination time of A team is in 2019. Represent the extracted information as a dictionary and adding it to a list. If the context does not tell any useful information, extract nothing.\nHere are some examples.\nQuestion:what award did sylvia chang win in 1986?\nQuestion parsing:\n```python\nquery = {\"subject\": \"Sylvia Chang\", \"relation\": \"win award\", \"object\": None, \"time\": {\"start\": datetime(1986, 1, 1), \"end\": datetime(1986, 12, 31)}}\nanswer_key = \"object\"\n```\nSearch:\n```python\nentities_to_search = [\"Sylvia Chang\"]\n```\nGenerate a background document from Wikipedia to answer the given question: Sylvia Chang is a Taiwanese actress, writer, singer, producer and director. She has won numerous awards throughout her career, including the Golden Horse Award for Best Leading Actress in 1986 for her role in the film \"Passion\".\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Sylvia Chang\", \"relation\": \"win award\", \"object\": \"Golden Horse Award for Best Leading Actress\", \"time\": {\"start\": datetime(1986, 1, 1), \"end\": datetime(1986, 12, 31)}})\n```\nContext: | Born | (1953-07-21) 21 July 1953 (age 69)Dalin, Chiayi County, Taiwan | \n | Other names | Zhang Aijia | \n | Occupation(s) | Actress, writer, director, singer, producer | \n | Spouse(s) | Bob Liu (m. 1978; div. 1984) Billy Wang (m. 1991) | \n\nExtract information relevant to the query:\nThere is nothing relevant to the query.\nContext: | | Year | Award | Category | Nominated work | Result | \n | 0 | 1976 | 13th Golden Horse Awards | Best Supporting Actress | Posterity and Perplexity | Won | \n | 1 | 1980 | 17th Golden Horse Awards | Best Actress | White Jasmine | Nominated | \n | 2 | 1981 | 18th Golden Horse Awards | Best Actress | My Grandfather | Won | \n | 3 | 1983 | 2nd Hong Kong Film Awards | Best Actress | Aces Go Places | Nominated | \n | 4 | 1985 | 4th Hong Kong Film Awards | Best Actress | Shanghai Blues | Nominated | \n | 5 | 1986 | 23rd Golden Horse Awards | Best Feature Film | Passion | Nominated | \n | 6 | 1986 | 23rd Golden Horse Awards | Best Director | Passion | Nominated | \n | 7 | 1986 | 23rd Golden Horse Awards | Best Actress | Passion | Won | \n\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Sylvia Chang\", \"relation\": \"win award\", \"object\": \"Golden Horse Award for Best Supporting Actress\", \"time\": {\"start\": datetime(1976, 1, 1), \"end\": datetime(1976, 12, 31)}})\ninformation.append({\"subject\": \"Sylvia Chang\", \"relation\": \"nominate award\", \"object\": \"Golden Horse Award for Best Actress\", \"time\": {\"start\": datetime(1980, 1, 1), \"end\": datetime(1980, 12, 31)}})\ninformation.append({\"subject\": \"Sylvia Chang\", \"relation\": \"win award\", \"object\": \"Golden Horse Award for Best Actress\", \"time\": {\"start\": datetime(1981, 1, 1), \"end\": datetime(1981, 12, 31)}})\ninformation.append({\"subject\": \"Sylvia Chang\", \"relation\": \"nominate award\", \"object\": \"Hong Kong Film Award for Best Actress\", \"time\": {\"start\": datetime(1983, 1, 1), \"end\": datetime(1983, 12, 31)}})\ninformation.append({\"subject\": \"Sylvia Chang\", \"relation\": \"nominate award\", \"object\": \"Hong Kong Film Award for Best Actress\", \"time\": {\"start\": datetime(1985, 1, 1), \"end\": datetime(1985, 12, 31)}})\ninformation.append({\"subject\": \"Sylvia Chang\", \"relation\": \"nominate award\", \"object\": \"Golden Horse Award for Best Feature Film\", \"time\": {\"start\": datetime(1986, 1, 1), \"end\": datetime(1986, 12, 31)}})\ninformation.append({\"subject\": \"Sylvia Chang\", \"relation\": \"nominate award\", \"object\": \"Golden Horse Award for Best Director\", \"time\": {\"start\": datetime(1986, 1, 1), \"end\": datetime(1986, 12, 31)}})\ninformation.append({\"subject\": \"Sylvia Chang\", \"relation\": \"win award\", \"object\": \"Golden Horse Award for Best Actress\", \"time\": {\"start\": datetime(1986, 1, 1), \"end\": datetime(1986, 12, 31)}})\n```\n\nQuestion:what is spouse of ivan pyryev that is start time is 1936?\nQuestion parsing:\n```python\nquery = {\"subject\": \"Ivan Pyryev\", \"relation\": \"spouse\", \"object\": None, \"time\": {\"start\": datetime(1936, 1, 1), \"end\": None}}\nanswer_key = \"object\"\n```\nSearch:\n```python\nentities_to_search = [\"Ivan Pyryev\"]\n```\nGenerate a background document from Wikipedia to answer the given question: Ivan Pyryev was a Soviet film director and screenwriter. He was married to actress Marina Ladynina from 1937 until his death in 1968. The relevant information from this document to answer the question is that Ivan Pyryev married Marina Ladynina in 1937.\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Ivan Pyryev\", \"relation\": \"spouse\", \"object\": \"Marina Ladynina\", \"time\": {\"start\": datetime(1937, 1, 1), \"end\": None}})\n```\nContext: | Born | (1901-11-17)17 November 1901Kamen-na-Obi, Tomsk Governorate, Russian Empire | \n | Died | 7 February 1968(1968-02-07) (aged 66)Moscow, Russian SFSR, Soviet Union | \n | Occupation(s) | Film directorScreenwriter | \n | Spouse(s) | Ada Wojcik Marina Ladynina Lionella Skirda | \n \nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Ivan Pyryev\", \"relation\": \"spouse\", \"object\": \"Ada Wojcik\", \"time\": {\"start\": None, \"end\": None}})\ninformation.append({\"subject\": \"Ivan Pyryev\", \"relation\": \"spouse\", \"object\": \"Marina Ladynina\", \"time\": {\"start\": None, \"end\": None}})\ninformation.append({\"subject\": \"Ivan Pyryev\", \"relation\": \"spouse\", \"object\": \"Lionella Skirda\", \"time\": {\"start\": None, \"end\": None}})\n```\n\nQuestion:what team did paulino alcantara play for in 1915?\nQuestion parsing:\n```python\nquery = {\"subject\": \"Paulino Alcantara\", \"relation\": \"play for team\", \"object\": None, \"time\": {\"start\": datetime(1915, 1, 1), \"end\": datetime(1915, 12, 31)}}\nanswer_key = \"object\"\n```\nSearch:\n```python\nentities_to_search = [\"Paulino Alcantara\"]\n```\nGenerate a background document from Wikipedia to answer the given question: Paulino Alcantara played for FC Barcelona in 1915. He was a prolific striker who scored 369 goals in 357 matches for FC Barcelona. The relevant information from this document to answer the question is that Paulino Alcántara played for FC Barcelona in 1915.\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Paulino Alcantara\", \"relation\": \"play for team\", \"object\": \"FC Barcelona\", \"time\": {\"start\": datetime(1915, 1, 1), \"end\": datetime(1915, 1, 1)}})\n```\nContext: Title: Personal information \n | Full name | Paulino Alcantara Riestra | \n | Date of birth | (1896-10-07)7 October 1896 | \n | Date of death | 13 February 1964(1964-02-13) (aged 67) | \n | Position(s) | Striker | \n \n Title: Senior career* \n | Years | Team | Apps | (Gls) | \n | 1912-1916 | Barcelona | 37 | (45) | \n | 1916-1918 | Bohemian | 23 | (24) | \n | 1918-1927 | Barcelona | 104 | (99) | \n \n Title: International career \n | 1915-1926 | Catalonia | 10 | (5) | \n | 1917 | Philippines | 5 | (15) | \n | 1921-1923 | Spain | 5 | (6) | \n\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Paulino Alcantara\", \"relation\": \"play for team\", \"object\": \"Barcelona\", \"time\": {\"start\": datetime(1912, 1, 1), \"end\": datetime(1916, 12, 31)}})\ninformation.append({\"subject\": \"Paulino Alcantara\", \"relation\": \"play for team\", \"object\": \"Bohemian\", \"time\": {\"start\": datetime(1916, 1, 1), \"end\": datetime(1918, 12, 31)}})\ninformation.append({\"subject\": \"Paulino Alcantara\", \"relation\": \"play for team\", \"object\": \"Barcelona\", \"time\": {\"start\": datetime(1918, 1, 1), \"end\": datetime(1927, 12, 31)}})\ninformation.append({\"subject\": \"Paulino Alcantara\", \"relation\": \"play for team\", \"object\": \"Catalonia\", \"time\": {\"start\": datetime(1915, 1, 1), \"end\": datetime(1926, 12, 31)}})\ninformation.append({\"subject\": \"Paulino Alcantara\", \"relation\": \"play for team\", \"object\": \"Philippines\", \"time\": {\"start\": datetime(1917, 1, 1), \"end\": datetime(1917, 12, 31)}})\ninformation.append({\"subject\": \"Paulino Alcantara\", \"relation\": \"play for team\", \"object\": \"Spain\", \"time\": {\"start\": datetime(1921, 1, 1), \"end\": datetime(1923, 12, 31)}})\n```\n\nQuestion:what is award received of hugh lofting that is point in time is 1923?\nQuestion parsing:\n```python\nquery = {\"subject\": None, \"relation\": \"received by\", \"object\": \"Hugh Lofting\", \"time\": {\"start\": datetime(1923, 1, 1), \"end\": datetime(1923, 12, 31)}}\nanswer_key = \"subject\"\n```\nSearch:\n```python\nentities_to_search = [\"Hugh Lofting\"]\n```\nGenerate a background document from Wikipedia to answer the given question: Hugh Lofting was a British author and illustrator who is best known for creating the character of Doctor Dolittle, a veterinarian who can talk to animals. In 1923, he was awarded the Newbery Medal for his book The Voyages of Doctor Dolittle.\nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Newbery Medal\", \"relation\": \"received by\", \"object\": \"Hugh Lofting\", \"time\": {\"start\": datetime(1923, 1, 1), \"end\": datetime(1923, 12, 31)}})\n```\nContext: Hugh John Lofting (14 January 1886 – 26 September 1947) was an English American writer trained as a civil engineer, who created the classic children's literature character Doctor Dolittle.\nExtract information relevant to the query:\nThere is nothing relevant to the query.\nContext: | Born | Hugh John Lofting(1886-01-14)14 January 1886Maidenhead, Berkshire, England, UK | \n | Died | 26 September 1947(1947-09-26) (aged 61)Topanga, California, USA | \n | Resting place | Evergreen Cemetery, Killingworth, Middlesex County, Connecticut, USA | \n | Occupation | Novelist, poet | \n | Genre | Children's literature, fantasy | \n | Notable works | Doctor Dolittle series | \n | Notable awards | Newbery Medal 1923 | \n | | Awards | Awards.1 | Awards.2 | \n | 0 | Preceded byHendrik Willem Van Loon | Newbery Medal winner 1923 | Succeeded byCharles Hawes | \nExtract information relevant to the query:\n```python\ninformation.append({\"subject\": \"Newbery Medal\", \"relation\": \"received by\", \"object\": \"Hugh Lofting\", \"time\": {\"start\": datetime(1923, 1, 1), \"end\": datetime(1923, 12, 31)}})\n```"} 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TianHongZXY/qaap/4fa31253a0bca3d42ac01aba5cad205ed20e8c05/requirements.txt -------------------------------------------------------------------------------- /search_wiki.py: -------------------------------------------------------------------------------- 1 | # ==================================================== 2 | # Copyright (C) 2023 All rights reserved. 3 | # 4 | # Author : Xinyu Zhu 5 | # Email : zhuxy21@mails.tsinghua.edu.cn 6 | # File Name : search_wiki.py 7 | # Last Modified : 2023-01-30 20:09 8 | # Describe : 9 | # 10 | # ==================================================== 11 | import re 12 | import time 13 | import requests 14 | import wikipedia 15 | from bs4 import BeautifulSoup 16 | from utils import clean_str 17 | from unidecode import unidecode 18 | import pandas as pd 19 | wikipedia.set_lang("en") 20 | 21 | 22 | def extract_info_table(soup): 23 | try: 24 | tags = soup.select_one(".mw-parser-output").find_all(recursive=False) 25 | tables = soup.find_all('table', {'class': 'infobox'}) 26 | if len(tables) == 0: 27 | tables = soup.find_all('table', {'class': 'infobox vcard'}) 28 | if len(tables) == 0: 29 | tables = soup.find_all('table', {'class': 'infobox vcard plainlist'}) 30 | if len(tables) == 0: 31 | tables = soup.find_all('table', {'class': 'infobox vevent'}) 32 | if len(tables) == 0: 33 | tables = soup.find_all('table', {'class': 'infobox biography vcard'}) 34 | if len(tables) == 0: 35 | return "" 36 | except Exception as e: 37 | return "" 38 | 39 | rh_list = [] 40 | rl_list = [] 41 | rd_list = [] 42 | flat_table = "" 43 | for table in tables: 44 | for row in table.find_all('tr'): 45 | if len(row) == 0: 46 | continue 47 | row_head = row.find_all('th', {'class': 'infobox-header'}) 48 | row_label = row.find_all('th', {'class': 'infobox-label'}) 49 | row_data = row.find_all('td', {'class': 'infobox-data'}) 50 | for rh in row_head: 51 | if rh.text.strip() == "": 52 | continue 53 | flat_table += "\nTitle: " + rh.text.replace('\n', ' ') 54 | if len(row_label) > 0: 55 | flat_table += " | " 56 | for rl in row_label: 57 | if rl.text.strip() == "": 58 | continue 59 | flat_table += rl.text.replace('\n', ' ') + " | " 60 | for rd in row_data: 61 | if rd.text.strip() == "": 62 | continue 63 | flat_table += rd.text.replace('\n', ' ') + " | " 64 | flat_table = flat_table.strip() 65 | flat_table += "\n" 66 | flat_table = flat_table.strip() + "\n" 67 | # print(flat_table) 68 | 69 | return unidecode(flat_table) 70 | 71 | 72 | def extract_wiki_table(soup): 73 | try: 74 | tags = soup.select_one(".mw-parser-output").find_all(recursive=False) 75 | tables = soup.find_all('table', {'class': 'wikitable'}) 76 | except Exception as e: 77 | return [] 78 | if len(tables) == 0: 79 | return [] 80 | 81 | results = [] 82 | pd_failed_tables = [] 83 | for t in tables: 84 | try: 85 | df = pd.read_html(str(t)) 86 | except Exception as e: 87 | pd_failed_tables.append(t) 88 | continue 89 | x = df[0].to_markdown() 90 | x = re.sub(' +', ' ', x) 91 | x = x.split("\n") 92 | x.pop(1) 93 | results.append("\n".join(x)) 94 | 95 | if len(pd_failed_tables) == 0: 96 | return results 97 | 98 | # current using pandas to read the table instead of the code below 99 | for table in pd_failed_tables: 100 | rh_list = [] 101 | rl_list = [] 102 | rd_list = [] 103 | flat_table = "" 104 | for row in table.find_all('tr'): 105 | if len(row) == 0: 106 | continue 107 | row_head = row.find_all('th') 108 | row_label = row.find_all('th') 109 | row_data = row.find_all('td') 110 | for rh in row_head: 111 | if rh.text.strip() == "": 112 | continue 113 | flat_table += "\n" + rh.text.replace('\n', ' ') 114 | if len(row_label) > 0: 115 | flat_table += " | " 116 | for rl in row_label: 117 | if rl.text.strip() == "": 118 | continue 119 | flat_table += rl.text.replace('\n', ' ') + " | " 120 | for rd in row_data: 121 | if rd.text.strip() == "": 122 | continue 123 | flat_table += rd.text.replace('\n', ' ') + " | " 124 | flat_table = flat_table.strip() 125 | flat_table += "\n" 126 | flat_table = flat_table.strip() + "\n" 127 | results.append(unidecode(flat_table)) 128 | # print(flat_table) 129 | 130 | return results 131 | 132 | 133 | def search(entity, summary=True): 134 | entity_ = entity.replace(" ", "+") 135 | search_url = f"https://en.wikipedia.org/w/index.php?search={entity_}" 136 | try: 137 | response_text = requests.get(search_url).text 138 | soup = BeautifulSoup(response_text, features="html.parser") 139 | except requests.exceptions.ProxyError: 140 | time.sleep(2) 141 | response_text = requests.get(search_url).text 142 | soup = BeautifulSoup(response_text, features="html.parser") 143 | try: 144 | result_divs = soup.find_all("div", {"class": "mw-search-result-heading"}) 145 | if result_divs: # mismatch 146 | result_titles = [div.get_text().strip() for div in result_divs] 147 | for i in range(len(result_titles)): 148 | result_titles[i] = result_titles[i].split(" ( redirect from ")[0] 149 | result_titles[i] = result_titles[i].split(" (redirect from ")[0] 150 | 151 | results = [x for x in result_titles if x.lower() != entity.lower() and x.lower() != entity[1:-1].lower()][:5] 152 | return False, results 153 | paragraphs = [] 154 | for p in soup.find_all("p") + soup.find_all("ul"): 155 | p = p.get_text().strip() 156 | if len(p.split()) > 2: 157 | p = re.sub(' +', ' ', p) 158 | p = re.sub(r'[\n]+', '\n', p) 159 | paragraphs.append(clean_str(p) + "\n") 160 | if any("may refer to" in p for p in paragraphs): 161 | return search("[" + entity + "]", summary) 162 | page = wikipedia.page(entity) 163 | if summary: 164 | main_text = page.summary 165 | else: 166 | main_text = page.content 167 | paragraphs = [] 168 | for p in soup.find_all("p") + soup.find_all("ul"): 169 | p = p.get_text().strip() 170 | if len(p.split()) > 2: 171 | p = re.sub(' +', ' ', p) 172 | p = re.sub(r'[\n]+', '\n', p) 173 | paragraphs.append(clean_str(p) + "\n") 174 | if any("may refer to:" in p for p in paragraphs): 175 | return search("[" + entity + "]") 176 | except wikipedia.exceptions.DisambiguationError as e: 177 | main_text = "" 178 | #NOTE if summary add exintro parameter 179 | if summary: 180 | r = requests.get(f"https://en.wikipedia.org/w/api.php?action=query&explaintext=&exsectionformat=plain&prop=extracts&exintro&redirects=&titles={entity}&format=json") 181 | else: 182 | r = requests.get(f"https://en.wikipedia.org/w/api.php?action=query&explaintext=&exsectionformat=plain&prop=extracts&redirects=&titles={entity}&format=json") 183 | x = r.json() 184 | extract = [] 185 | pages = x['query']['pages'] 186 | for key, val in pages.items(): 187 | if 'extract' not in val: 188 | return False, e.options[:5] 189 | main_text += val['extract'] 190 | except wikipedia.exceptions.PageError: 191 | try: 192 | script_content = soup.select_one("head > script:nth-of-type(1)").decode_contents() 193 | page_id = re.search(r".*wgArticleId..([0-9]+).*",script_content).group(1) 194 | page = wikipedia.page(pageid=page_id) 195 | if summary: 196 | main_text = page.summary 197 | else: 198 | main_text = page.content 199 | except Exception as e: 200 | return search(entity) 201 | except wikipedia.exceptions.WikipediaException: 202 | time.sleep(1) 203 | return search(entity, summary) 204 | except Exception as e: 205 | return search(entity) 206 | 207 | main_text = main_text.split("== References ==")[0] 208 | main_text = main_text.split("== See also ==")[0] 209 | main_text = [main_text] 210 | for sup in soup.find_all("sup", {'class':'reference'}): # delete all reference like [1][3][13] 211 | sup.decompose() 212 | info_table = extract_info_table(soup) 213 | wiki_tables = extract_wiki_table(soup) 214 | tables = [info_table] + wiki_tables 215 | 216 | return True, (tables, main_text) 217 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | import string 3 | from collections import Counter 4 | from unidecode import unidecode 5 | import traceback 6 | import coloredlogs, logging 7 | import datetime as dt 8 | from datetime import datetime 9 | logger = logging.getLogger(__name__) 10 | coloredlogs.install(level='DEBUG', logger=logger) 11 | 12 | 13 | def create_context_slices(context, max_length=512, stride=384): 14 | """ 15 | Splits a context into slices of length max_length, with a stride of stride. 16 | """ 17 | context_paras = context.split("\n") 18 | context_tokens = [] 19 | slices = [] 20 | for i, para in enumerate(context_paras): 21 | context_tokens += para.split(" ") 22 | context_tokens.append("\n") 23 | while len(context_tokens) > max_length: 24 | slices.append(" ".join(context_tokens[:max_length])) 25 | context_tokens = context_tokens[stride:] 26 | slices.append(" ".join(context_tokens)) 27 | 28 | return slices 29 | 30 | 31 | def extract_answer(answer_key, information): 32 | if information[0] != "": 33 | answer_infor = information[0] # only keep the first one 34 | if answer_infor[answer_key] is not None: 35 | if isinstance(answer_infor[answer_key], (list, tuple, frozenset)) and isinstance(answer_infor[answer_key][0], str): 36 | predictions = list(answer_infor[answer_key]) 37 | elif isinstance(answer_infor[answer_key], (list, tuple, frozenset)) and isinstance(answer_infor[answer_key][0], dict): 38 | predictions = list(answer_infor[answer_key][0].values()) 39 | elif isinstance(answer_infor[answer_key], dict): 40 | predictions = list(answer_infor[answer_key].values()) 41 | else: 42 | predictions = [answer_infor[answer_key]] 43 | else: 44 | predictions = [""] 45 | else: 46 | predictions = [""] 47 | 48 | assert isinstance(predictions, list), predictions 49 | 50 | return predictions 51 | 52 | 53 | def extract_code_from_string(string): 54 | # Written by chatgpt 55 | pattern = r"```python(.*?)```" 56 | matches = re.findall(pattern, string, re.DOTALL) 57 | if matches: 58 | return "\n".join(matches) 59 | 60 | return None 61 | 62 | 63 | def clean_str(p): 64 | p = unidecode(p) 65 | try: 66 | p = p.encode().decode("unicode-escape").encode("latin1").decode("utf-8") 67 | except: 68 | p = p.encode().decode("ISO-8859-1").encode("latin1").decode("utf-8") 69 | # p = re.sub('([a-zA-Z])([.,!?()])', r'\1\2 ', p) 70 | # p = re.sub('\s{2,}', ' ', p) 71 | p = re.sub(r'([a-zA-Z])([,.!?()])', r'\1 \2', p) 72 | p = re.sub(r'([,.!?()])([a-zA-Z])', r'\1 \2', p) 73 | 74 | return p 75 | 76 | 77 | def f1_score(prediction, ground_truth): 78 | ZERO_METRIC = (0, 0, 0) 79 | 80 | if prediction in ['yes', 'no', 'noanswer'] and prediction != ground_truth: 81 | return ZERO_METRIC 82 | if ground_truth in ['yes', 'no', 'noanswer'] and prediction != ground_truth: 83 | return ZERO_METRIC 84 | 85 | prediction_tokens = prediction.split() 86 | ground_truth_tokens = ground_truth.split() 87 | common = Counter(prediction_tokens) & Counter(ground_truth_tokens) 88 | num_same = sum(common.values()) 89 | if num_same == 0: 90 | return ZERO_METRIC 91 | precision = 1.0 * num_same / len(prediction_tokens) 92 | recall = 1.0 * num_same / len(ground_truth_tokens) 93 | f1 = (2 * precision * recall) / (precision + recall) 94 | 95 | return f1, precision, recall 96 | 97 | 98 | def normalize_answer(s): 99 | def remove_articles(text): 100 | return re.sub(r"\b(a|an|the)\b", " ", text) 101 | 102 | def white_space_fix(text): 103 | return " ".join(text.split()) 104 | 105 | def replace_dash_with_space(text): 106 | return " ".join(text.split("-")) 107 | 108 | def remove_punc(text): 109 | exclude = set(string.punctuation) 110 | return "".join([ch for ch in text if ch not in exclude]) 111 | 112 | def lower(text): 113 | if isinstance(text, int) or isinstance(text, float): 114 | text = str(text) 115 | return unidecode(text.lower()) 116 | 117 | return white_space_fix(remove_articles(remove_punc(replace_dash_with_space(lower(s))))) 118 | 119 | 120 | def get_metrics(preds, gt_answer): 121 | if isinstance(gt_answer, str): 122 | gt_answer = [gt_answer] 123 | if isinstance(preds, str): 124 | preds = [preds] 125 | if len(preds) == 0 and len(gt_answer) != 0: 126 | return {'reward': 0, 'em': 0, 'f1': 0} 127 | if len(preds) != 0 and len(gt_answer) == 0: 128 | return {'reward': 0, 'em': 0, 'f1': 0} 129 | em = 0 130 | f1 = 0 131 | for pred in preds: 132 | pred = normalize_answer(pred) 133 | if pred == "": 134 | if gt_answer[0] == "": 135 | return {'reward': 1, 'em': 1, 'f1': 1.} 136 | else: 137 | return {'reward': 0, 'em': 0, 'f1': 0} 138 | for gt in gt_answer: 139 | gt = normalize_answer(gt) 140 | em = max(em, int(pred == gt)) 141 | f1 = max(f1, f1_score(pred, gt)[0]) 142 | if em: 143 | return {'reward': 1, 'em': 1, 'f1': 1.} 144 | 145 | return {'reward': em, 'em': em, 'f1': f1} 146 | 147 | 148 | def calc_time_iou(code): 149 | time_type = None 150 | locals_ = {'information': []} 151 | try: 152 | exec(code[0], globals(), locals_) 153 | query = locals_.get('query') 154 | answer_key = locals_.get('answer_key') 155 | except Exception as e: 156 | logger.error("Failed to get origin query") 157 | query = None 158 | answer_key = None 159 | 160 | for c in code[1:]: 161 | try: 162 | exec(c, globals(), locals_) 163 | except Exception as e: 164 | logger.error(f'Failed to execute code:\n{c}') 165 | print("Error Type:", type(e)) 166 | print("Error Message:", e) 167 | print(f"Traceback:\n{traceback.format_exc()}") 168 | continue 169 | default_start = datetime(1, 1, 1) 170 | default_end = datetime(3000, 1, 1) 171 | # query = locals_.get('query') 172 | information = locals_.get('information') 173 | if query is None: 174 | return "object", information 175 | if "time" not in query or query['time'] is None or (("start" in query['time'] and "end" in query['time']) and (query['time']['start'] is None and query['time']['end'] is None)): 176 | query['time'] = {'start': default_start, 'end': default_end} 177 | time_type = 'overlap' 178 | elif isinstance(query['time'], datetime): 179 | query['time'] = {'start': query['time'], 'end': query['time'] + dt.timedelta(365)} 180 | time_type = 'overlap' 181 | elif 'start' not in query['time'] or query['time']['start'] is None: 182 | time_type = 'before or end' 183 | elif 'end' not in query['time'] or query['time']['end'] is None: 184 | time_type = 'after or start' 185 | else: 186 | time_type = 'overlap' 187 | 188 | information = [x for x in information if 'subject' in x and 'object' in x and 'relation' in x and x[answer_key] is not None] #and x['time'] is not None] 189 | if len(information) == 0: 190 | return "object", [""] 191 | 192 | for idx, ex in enumerate(information): 193 | try: 194 | if "time" not in ex or ex['time'] is None or (("start" in ex['time'] and "end" in ex['time']) and (ex['time']['start'] is None and ex['time']['end'] is None)): 195 | ex['time'] = {'start': default_start, 'end': default_end} 196 | elif isinstance(ex['time'], datetime): 197 | ex['time'] = {'start': ex['time'], 'end': ex['time'] + dt.timedelta(365)} 198 | elif len(ex['time']) == 0: 199 | ex['time'] = {'start': default_start, 'end': default_end} 200 | if 'start' not in ex['time'] or ex['time']['start'] is None: 201 | ex['time'].update(start=default_start) 202 | if 'end' not in ex['time'] or ex['time']['end'] is None: 203 | ex['time'].update(end=default_end) 204 | except Exception as e: 205 | print("Error Type:", type(e)) 206 | print("Error Message:", e) 207 | print(f"Traceback:\n{traceback.format_exc()}") 208 | print(ex) 209 | 210 | overlapped = False 211 | information = [x for x in information if x['time'] is not None] 212 | if time_type == "overlap": 213 | for idx, ex in enumerate(information): 214 | latest_start = max(query['time']['start'], ex['time']['start']) 215 | earliest_end = min(query['time']['end'], ex['time']['end']) 216 | delta = (earliest_end - latest_start).days + 1 217 | overlap = max(0, delta) 218 | if overlap > 0: 219 | overlapped = True 220 | time_union = max((query['time']['end'] - query['time']['start']).days + (ex['time']['end'] - ex['time']['start']).days - overlap, 1) 221 | ex.update(overlap=overlap) 222 | ex.update(time_union=time_union) 223 | ex.update(time_iou=overlap / time_union) 224 | information = sorted(information, key=lambda x: (x['time_iou'], x['overlap']), reverse=True) 225 | elif time_type == "after or start": 226 | information = sorted(information, key=lambda x: abs((x['time']['start'] - query['time']['start']).days)) 227 | elif time_type == "before or end": 228 | information = sorted(information, key=lambda x: abs((x['time']['end'] - query['time']['end']).days)) 229 | 230 | return answer_key, information --------------------------------------------------------------------------------