├── assets └── Adaptive-Note.png ├── config └── config.yaml ├── data ├── corpus │ └── .gitkeep └── eval │ ├── 2wikimultihopqa │ └── test.json │ ├── hotpotqa │ └── test.json │ └── musique │ └── test.json ├── prompts ├── crud │ └── quest_eval_answer.txt ├── en │ ├── base │ ├── base_asqa │ ├── base_wo_retri │ ├── base_wo_retri_asqa │ ├── chain-of-note │ ├── compare │ ├── first_gen_answer │ ├── gen_answer │ ├── gen_answer_asqa │ ├── gen_new_query │ ├── refine_note │ ├── self_refine │ └── self_rerank └── zh │ ├── base │ ├── base_wo_retri │ ├── chain-of-note │ ├── compare │ ├── first_gen_answer │ ├── gen_answer │ ├── gen_answer_raw │ ├── gen_new_query │ ├── refine_note │ ├── self_refine │ └── self_rerank ├── readme.md ├── requirements.txt └── src ├── build_index ├── emb │ └── index.py └── es │ ├── index_2wiki.py │ ├── index_hotpotqa.py │ └── index_musique.py ├── es_retrieve.py ├── eval.py ├── main.py ├── metrics_questeval_crud.py └── utils.py /assets/Adaptive-Note.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thunlp/Adaptive-Note/ab69094f84558ab3dcda9ac9d530222a91c2d5bc/assets/Adaptive-Note.png -------------------------------------------------------------------------------- /config/config.yaml: -------------------------------------------------------------------------------- 1 | model: 2 | OPENAI_API_KEY: "" 3 | OPENAI_BASE_URL: "" 4 | llama3-8b-instruct: "meta-llama/Meta-Llama-3-8B-Instruct" 5 | qwen2-7b-instruct: "Qwen/Qwen2-7B-Instruct" 6 | bge-base-zh-v1.5: "BAAI/bge-base-zh-v1.5" 7 | gtr-t5-xxl: "sentence-transformers/gtr-t5-xxl" 8 | es: 9 | url: "http://localhost:9200" -------------------------------------------------------------------------------- /data/corpus/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thunlp/Adaptive-Note/ab69094f84558ab3dcda9ac9d530222a91c2d5bc/data/corpus/.gitkeep -------------------------------------------------------------------------------- /data/eval/2wikimultihopqa/test.json: -------------------------------------------------------------------------------- 1 | [{"question": "Who is the father-in-law of Queen Hyojeong?", "answer": ["Crown Prince Hyomyeong"]}, {"question": "Which one was established first, Grouplogic or Inbios?", "answer": ["Grouplogic"]}, {"question": "Where was the place of death of the director of film The Decision Of Christopher Blake?", "answer": ["Hollywood"]}, {"question": "Who is the child of the director of film Mukhyamantri (1996 Film)?", "answer": ["Chumki Chowdhury"]}, {"question": "Which film was released earlier, The Devil'S Miner or Don'T Tell?", "answer": ["The Devil'S Miner"]}, {"question": "Who is younger, Halc\u00f3n Suriano Jr. or Pierre De Geyter?", "answer": ["Halc\u00f3n Suriano Jr."]}, {"question": "Where did Abe Laboriel Jr.'s father study?", "answer": ["Berklee College of Music"]}, {"question": "Where did Edward Hoby's father study?", "answer": ["St. John's College, Cambridge"]}, {"question": "Who is Lord Herbert Montagu Douglas Scott's maternal grandfather?", "answer": ["James Hamilton"]}, {"question": "Which film came out first, Pozna\u0144 '56 or The Last Day Of School Before Christmas?", "answer": ["The Last Day Of School Before Christmas"]}, {"question": "Who is the husband of Princess Elene Of Georgia?", "answer": ["Prince Archil of Imereti"]}, {"question": "Which country the director of film Snow Flower And The Secret Fan (Film) is from?", "answer": ["American"]}, {"question": "Which film has the director born first, Gopalapuranam or Here Is The Beauty?", "answer": ["Here Is The Beauty"]}, {"question": "Who is the father of the performer of song Lighter (Miley Cyrus Song)?", "answer": ["Billy Ray Cyrus"]}, {"question": "Where was the place of death of Suzanne Farrington's mother?", "answer": ["London"]}, {"question": "Where was the director of film Eisenstein In Guanajuato born?", "answer": ["Newport, Wales"]}, {"question": "Where did George Tryon, 1St Baron Tryon's father die?", "answer": ["Tripoli"]}, {"question": "What is the cause of death of director of film Destination: Dewsbury?", "answer": ["Parkinson's disease"]}, {"question": "Where was the place of death of the director of film Happy Go Lucky (1936 Film)?", "answer": ["Los Angeles"]}, {"question": "Which film was released earlier, Drive Hard or Make-Out With Violence?", "answer": ["Make-Out With Violence"]}, {"question": "Which album was released first, A Little Light Left or The Best Of Celly Cel?", "answer": ["The Best Of Celly Cel"]}, {"question": "What is the award that the performer of song In My Mind (Heather Headley Song) got?", "answer": ["Tony Award for Best Actress in a Musical"]}, {"question": "Which film has the director born earlier, The Adventures Of Priscilla, Queen Of The Desert or Harvest: 3,000 Years?", "answer": ["Harvest: 3,000 Years"]}, {"question": "Where did the director of film Man Detained graduate from?", "answer": ["Churcher's College"]}, {"question": "Where was the place of burial of Auguste Tessier's father?", "answer": ["Cimeti\u00e8re Notre-Dame-de-Belmont"]}, {"question": "Where was the place of death of the director of film The First Day Of Freedom?", "answer": ["Florida"]}, {"question": "Do both films Apache Woman and Dime With A Halo have the directors from the same country?", "answer": ["yes"]}, {"question": "Do both My Friend From The Park and Punks (Film) films have the directors from the same country?", "answer": ["no"]}, {"question": "What is the place of birth of the performer of song Svennebanan?", "answer": ["V\u00e4ster\u00e5s"]}, {"question": "Where did the performer of song Words Of Love die?", "answer": ["Clear Lake, Iowa"]}, {"question": "What nationality is the director of film Guns Of Diablo?", "answer": ["American"]}, {"question": "Who is the paternal grandfather of Konrad Vi The Dean?", "answer": ["Konrad II the Gray"]}, {"question": "Where was the place of death of Anna Of Pomerania's father?", "answer": ["Stettin"]}, {"question": "Who is Catherine Fitzcharles's paternal grandmother?", "answer": ["Henrietta Maria of France"]}, {"question": "Where was the founder of Douro Wine Company born?", "answer": ["Lisbon"]}, {"question": "Who is Thomas Lyon, 8Th Earl Of Strathmore And Kinghorne's paternal grandfather?", "answer": ["Patrick Lyon, 3rd Earl of Strathmore and Kinghorne"]}, {"question": "Who is the spouse of the performer of song Les Rois Du Monde?", "answer": ["Joy Esther"]}, {"question": "What is the place of birth of S. G. Kittappa's wife?", "answer": ["Kodumudi"]}, {"question": "Who is the mother-in-law of Blanche Of Navarre, Duchess Of Brittany?", "answer": ["Alix"]}, {"question": "Where was the performer of song Believe It (Meek Mill Song) detained?", "answer": ["State Correctional Institution \u2013 Chester"]}, {"question": "Which film has the director who died later, The Monkey Talks or Varakatnam?", "answer": ["Varakatnam"]}, {"question": "Who is the maternal grandfather of Philip Of Ibelin (1180\u20131227)?", "answer": ["John Doukas Komnenos"]}, {"question": "Where did Thep Kasattri's father die?", "answer": ["Ayutthaya"]}, {"question": "Where was the father of Frederick, Prince Of Hohenzollern born?", "answer": ["Benrath"]}, {"question": "Who is Titus Junius Brutus's paternal grandfather?", "answer": ["Marcus Junius"]}, {"question": "Who is older, Edward Bligh, 5Th Earl Of Darnley or Adolfo Costa Du Rels?", "answer": ["Edward Bligh, 5Th Earl Of Darnley"]}, {"question": "When was Edward Douglas-Pennant, 3Rd Baron Penrhyn's father born?", "answer": ["30 September 1836"]}, {"question": "Which film was released first, The Pirate'S Dream or Prem Mhanje Prem Mhanje Prem Asta?", "answer": ["The Pirate'S Dream"]}, {"question": "Which film has the director who was born later, Accident 703 or Habana Eva?", "answer": ["Habana Eva"]}, {"question": "Were Alan Stephenson Boyd and Charles Grandison Finney of the same nationality?", "answer": ["yes"]}, {"question": "Who lived longer, Jean Gr\u00e9millon or \u00c1ngel Recasens?", "answer": ["\u00c1ngel Recasens"]}, {"question": "Are Gaulier River and Hallowing Run located in the same country?", "answer": ["no"]}, {"question": "Where did William Henry Gregory's wife die?", "answer": ["Coole Park"]}, {"question": "Where was the place of death of the director of film Touki Bouki?", "answer": ["Paris"]}, {"question": "Did the movies The Hideout (Film) and Trieste Mia!, originate from the same country?", "answer": ["yes"]}, {"question": "Where did the director of film The Great Circus Mystery die?", "answer": ["Los Angeles County"]}, {"question": "What is the place of birth of Elijah Blue Allman's father?", "answer": ["Nashville, Tennessee"]}, {"question": "Which award the director of film Ang Padrino won?", "answer": ["National Artist of the Philippines"]}, {"question": "Who is Pepin Of Herstal's paternal grandfather?", "answer": ["Arnulf"]}, {"question": "Where was the father of Benedikte Shetelig Kruse born?", "answer": ["London"]}, {"question": "Who are William Cunningham, 9Th Earl Of Glencairn's aunt?", "answer": ["Lady Ann Cunningham and Margaret Cunningham"]}, {"question": "Are both directors of films Modern Mothers and The Slaughter Rule from the same country?", "answer": ["no"]}, {"question": "What is the award that the composer of film C.I.D. (1956 Film) got?", "answer": ["Filmfare Award"]}, {"question": "Who is James Franklin Perry's sibling-in-law?", "answer": ["Stephen F. Austin"]}, {"question": "What is the place of birth of the composer of film Raagam Thaanam Pallavi?", "answer": ["Kerala"]}, {"question": "Why did Elizabeth Bacon Custer's husband die?", "answer": ["Battle of the Little Bighorn"]}, {"question": "Which film has the director who was born first, Perd\u00f3n, Viejita or Smokey And The Bandit?", "answer": ["Perd\u00f3n, Viejita"]}, {"question": "Was Arcadio Padilla or Leon Duray born first?", "answer": ["Leon Duray"]}, {"question": "Who is the paternal grandfather of Euphemia Ii, Countess Of Ross?", "answer": ["Walter Leslie"]}, {"question": "Where was the place of burial of Henry Of Blois's father?", "answer": ["Ramla"]}, {"question": "Which film has the director who is older than the other, Spooks And Spirits or Cat Chaser? ", "answer": ["Spooks And Spirits"]}, {"question": "What is the place of birth of the director of film Love Between The Raindrops?", "answer": ["Vy\u0161kov"]}, {"question": "Which country the founder of Team Red Bull (Nascar Team) is from?", "answer": ["Austrian"]}, {"question": "Was Alan Haynes or Marie-Georges-G\u00e9rard-L\u00e9on Le Maire De Warz\u00e9e D'Hermalle born first?", "answer": ["Marie-Georges-G\u00e9rard-L\u00e9on Le Maire De Warz\u00e9e D'Hermalle"]}, {"question": "Where did the founder of university Escuela Superior Latinoamericana De Inform\u00e1tica graduate from?", "answer": ["University of Buenos Aires"]}, {"question": "Who is the paternal grandfather of Bruno Ii Von Berg?", "answer": ["Adolf II of Berg"]}, {"question": "Who was born earlier, Martin Stapleton or Gina Fulton?", "answer": ["Gina Fulton"]}, {"question": "Which film came out earlier, A Wall Of Silence or Agent 3S3: Massacre In The Sun?", "answer": ["Agent 3S3: Massacre In The Sun"]}, {"question": "What is the place of birth of the performer of song Afterglow (Tina Turner Song)?", "answer": ["Nutbush"]}, {"question": "Who is the paternal grandfather of Duchess Elisabeth Alexandrine Of W\u00fcrttemberg?", "answer": ["Frederick II Eugene, Duke of W\u00fcrttemberg"]}, {"question": "When is the founder of Apo Group 's birthday?", "answer": ["November 3, 1975"]}, {"question": "Where was the place of death of the performer of song Solhverv?", "answer": ["Sweden"]}, {"question": "Which film has the director who was born later, Tarzan'S Hidden Jungle or The Brink'S Job?", "answer": ["The Brink'S Job"]}, {"question": "Which film has the director who is older than the other, My Own Private Idaho or Bread, Love And Dreams? ", "answer": ["Bread, Love And Dreams"]}, {"question": "Which film was released earlier, Ontari (Film) or A Private'S Affair?", "answer": ["A Private'S Affair"]}, {"question": "Who is Adelaide Of Burgundy, Duchess Of Brabant's paternal grandfather?", "answer": ["Odo III, Duke of Burgundy"]}, {"question": "Who is the sibling-in-law of Giuditta Grisi?", "answer": ["Giovanni Matteo Mario"]}, {"question": "Who was born later, Allan Havey or Bradley Dredge?", "answer": ["Bradley Dredge"]}, {"question": "Which film was released more recently, Nearly A Deserter or Face \u00c0 L'Oc\u00e9an?", "answer": ["Face \u00c0 L'Oc\u00e9an"]}, {"question": "Who was born later, Franz Xaver Haberl or Sharon Richardson?", "answer": ["Sharon Richardson"]}, {"question": "Who is the spouse of the director of film The Ear?", "answer": ["Alena Mihulov\u00e1"]}, {"question": "Which film came out first, The Ex-Wife Of My Life or Wrangler'S Roost?", "answer": ["Wrangler'S Roost"]}, {"question": "What nationality is the director of film Astronauts Gone Wild?", "answer": ["American"]}, {"question": "Who is the father of the director of film Manasuku Nachindi?", "answer": ["Krishna"]}, {"question": "Which film whose director was born first, The Olsen Gang'S Big Score or To Be Or Not To Be (1983 Film)?", "answer": ["The Olsen Gang'S Big Score"]}, {"question": "Which film has the director who died first, The Wild Bull'S Lair or Devotion (1950 Film)?", "answer": ["The Wild Bull'S Lair"]}, {"question": "Are You Had To Be A Gypsy and Paul Blart: Mall Cop from the same country?", "answer": ["no"]}, {"question": "What is the place of birth of Mar\u00eda De Los Remedios De Escalada's husband?", "answer": ["Yapey\u00fa"]}, {"question": "Which country the performer of song Den Lilla F\u00e5geln is from?", "answer": ["Sweden"]}, {"question": "Which film was released earlier, Fast Five or The Testament Of Dr. Mabuse?", "answer": ["The Testament Of Dr. Mabuse"]}, {"question": "Where was the place of death of John Ii Of Luxembourg, Count Of Ligny's father?", "answer": ["Italy"]}, {"question": "Which film has the director died later, Arctic Blue or Jasper And The Haunted House?", "answer": ["Arctic Blue"]}, {"question": "Which film has the director who died later, Lukket Avdeling or Three Hearts For Julia?", "answer": ["Three Hearts For Julia"]}, {"question": "Who is Kaev Hua Ii's paternal grandfather?", "answer": ["Outey"]}, {"question": "Where was the husband of Frances Tupper born?", "answer": ["Amherst, Nova Scotia"]}, {"question": "What nationality is the director of film Little Papa?", "answer": ["American"]}, {"question": "Are Afterwards and The Whistle (Film) from the same country?", "answer": ["no"]}, {"question": "When did John Of Luxembourg, Count Of Soissons's father die?", "answer": ["19 December 1475"]}, {"question": "Who is Maud Of Gloucester, Countess Of Chester's father-in-law?", "answer": ["Ranulf le Meschin, 3rd Earl of Chester"]}, {"question": "Did the movies The Shadow Of Evil and The Lone Wolf Takes A Chance, originate from the same country?", "answer": ["no"]}, {"question": "When is the composer of film Hi-De-Ho (1947 Film) 's birthday?", "answer": ["December 25, 1907"]}, {"question": "Which film has the director who was born earlier, A Romance Of The Air or How Moscha Came Back?", "answer": ["How Moscha Came Back"]}, {"question": "Which one was established first, Colt'S Manufacturing Company or Banco Azteca?", "answer": ["Colt'S Manufacturing Company"]}, {"question": "Where was the father of Parthamasiris Of Armenia born?", "answer": ["Parthian Empire"]}, {"question": "Which film came out earlier, Hideout In The Sun or The Truth About Women?", "answer": ["The Truth About Women"]}, {"question": "What nationality is Reginald Ii, Count Of Bar's father?", "answer": ["France"]}, {"question": "What is the date of death of William I, Count Of Burgundy's mother?", "answer": ["1038"]}, {"question": "Which country Natalie Portman's husband is from?", "answer": ["French"]}, {"question": "Who is Archduchess Immaculata Of Austria's paternal grandmother?", "answer": ["Princess Maria Immaculata of Bourbon-Two Sicilies"]}, {"question": "Which film whose director is younger, My Baby'S Daddy or A Tale Of Winter?", "answer": ["My Baby'S Daddy"]}, {"question": "Are It\u2019S A Matter Of Love and The Trusted Outlaw from the same country?", "answer": ["no"]}, {"question": "Who is the paternal grandmother of Princess Elisabeth Of Prussia?", "answer": ["Frederika Louisa of Hesse-Darmstadt"]}, {"question": "Who was born first out of Li Yuyang and Adam Richman?", "answer": ["Adam Richman"]}, {"question": "Who is Philip, Prince Of Anhalt-K\u00f6then's paternal grandfather?", "answer": ["Albert IV, Prince of Anhalt-K\u00f6then"]}, {"question": "What is the place of birth of the composer of film Salome (1953 Film)?", "answer": ["Richmond, Indiana"]}, {"question": "Where did Oleg Kerensky's father die?", "answer": ["New York"]}, {"question": "Are both islands, Temagami Island and Kingak Island, located in the same country?", "answer": ["yes"]}, {"question": "Which film has the director who was born later, Walkover (Film) or Terry Tate: Office Linebacker?", "answer": ["Terry Tate: Office Linebacker"]}, {"question": "Which film was released more recently, The White Planet or The Glass Sphinx?", "answer": ["The White Planet"]}, {"question": "Where was the place of death of the director of film The Red Ace?", "answer": ["Los Angeles, California"]}, {"question": "Which film was released more recently, The Love God? or Under The Pavement Lies The Strand?", "answer": ["Under The Pavement Lies The Strand"]}, {"question": "Who was born earlier, Armando Ribeiro or Kostas Papoutsis?", "answer": ["Armando Ribeiro"]}, {"question": "Was Judith S. Beck or F\u00e9lix Erviti Barcelona born first?", "answer": ["F\u00e9lix Erviti Barcelona"]}, {"question": "Who is younger, Alfred Staines or George Sidney?", "answer": ["George Sidney"]}, {"question": "Where did Amytis Of Media's husband die?", "answer": ["Babylon"]}, {"question": "Which film has the director who died first, Long Shot: The Kevin Laue Story or The Law'S Lash?", "answer": ["The Law'S Lash"]}, {"question": "What nationality is the director of film Krantikaal?", "answer": ["Indian"]}, {"question": "Where was the wife of Vladimir Feldman born?", "answer": ["Kiev"]}, {"question": "What is the place of birth of the composer of film Inherent Vice (Film)?", "answer": ["Oxford"]}, {"question": "Which film has the director born first, Irapada or The Lone Wolf In Mexico?", "answer": ["The Lone Wolf In Mexico"]}, {"question": "Where did Margaret Simey's husband study at?", "answer": ["Balliol College"]}, {"question": "Which film whose director was born first, Apasionadamente or The Blue Bead?", "answer": ["Apasionadamente"]}, {"question": "Which film whose director is younger, The Tiger Makes Out or La V\u00e9rit\u00e9 Sur B\u00e9b\u00e9 Donge?", "answer": ["The Tiger Makes Out"]}, {"question": "Where was the husband of Wiktoryna Baka\u0142owiczowa born?", "answer": ["Chrzan\u00f3w"]}, {"question": "Which film came out first, Un Extra\u00f1o En La Escalera or Arlington West: The Film?", "answer": ["Un Extra\u00f1o En La Escalera"]}, {"question": "Where did the director of film 13 Fighting Men die?", "answer": ["Palm Springs, California"]}, {"question": "When was the performer of song Mademoiselle Chante Le Blues born?", "answer": ["5 December 1966"]}, {"question": "Which film has more directors, Duel Of Champions or Two Nights With Cleopatra?", "answer": ["Duel Of Champions"]}, {"question": "Where was the place of death of Geoffrey Plantagenet, Count Of Anjou's wife?", "answer": ["Rouen"]}, {"question": "What is the place of birth of the director of film Dawn Of Life?", "answer": ["San Juan, Puerto Rico"]}, {"question": "What is the date of birth of the performer of song Nawti?", "answer": ["1976"]}, {"question": "Where did the director of film Napoleon (1995 Film) graduate from?", "answer": ["Film and Television School"]}, {"question": "Where did the director of film Mani Madhava Chakyar: The Master At Work die?", "answer": ["Trivandrum"]}, {"question": "Where was the director of film Kon Khon born?", "answer": ["Bang Khonthi"]}, {"question": "Which film has the director who was born later, The Home Song Stories or The Sterile Cuckoo?", "answer": ["The Home Song Stories"]}, {"question": "Which film has the director born later, Occhio, Malocchio, Prezzemolo E Finocchio or Jailhouse Rock (Film)?", "answer": ["Occhio, Malocchio, Prezzemolo E Finocchio"]}, {"question": "Who was born later, Brahima Ciss\u00e9 or V\u00e4in\u00f6 Malmivaara?", "answer": ["Brahima Ciss\u00e9"]}, {"question": "Who is Nicolas Stern's sibling-in-law?", "answer": ["Brennan Brown"]}, {"question": "Who is older, Cary Devall Langhorne or Ji\u0159\u00ed Ma\u0161t\u00e1lka?", "answer": ["Cary Devall Langhorne"]}, {"question": "Which film was released more recently, Affectionately Yours or Captain America: The First Avenger?", "answer": ["Captain America: The First Avenger"]}, {"question": "Who died earlier, Alessio Perilli or Harry G. Day?", "answer": ["Alessio Perilli"]}, {"question": "Which film has the director who is older, Overland Mail Robbery or Hearts Of The World?", "answer": ["Hearts Of The World"]}, {"question": "Who is Joan, Lady Of Wales's paternal grandmother?", "answer": ["Eleanor of Aquitaine"]}, {"question": "Which film has the director born earlier, Santa Fe Uprising or Trainer And Temptress?", "answer": ["Trainer And Temptress"]}, {"question": "Which film has the director died earlier, Miss Liselott or Flight From Destiny?", "answer": ["Miss Liselott"]}, {"question": "Which film was released first, Ee Sanje or Zorawar?", "answer": ["Ee Sanje"]}, {"question": "Which film has the director who was born later, A Warm December or It Rains In My Village?", "answer": ["It Rains In My Village"]}, {"question": "Who is Teriimaevarua Iii's paternal grandmother?", "answer": ["P\u014dmare IV"]}, {"question": "Which film was released more recently, Hollywood Ending or Tales Of An Ancient Empire?", "answer": ["Tales Of An Ancient Empire"]}, {"question": "Which film has the director who died earlier, Il Contratto or The Fiddler Of Florence?", "answer": ["The Fiddler Of Florence"]}, {"question": "Where was the place of death of Lambert I, Count Of Louvain's father?", "answer": ["Bohemia"]}, {"question": "Where did Saw Thanda's husband die?", "answer": ["Mrauk U"]}, {"question": "Which country the founder of Osborne Computer Corporation is from?", "answer": ["American"]}, {"question": "Was Andreas P. Nielsen or Lydia Flood Jackson born first?", "answer": ["Lydia Flood Jackson"]}, {"question": "What is the place of birth of the director of film Marianela (1940 Film)?", "answer": ["Madrid"]}, {"question": "Which film came out first, Murder By Proxy or Rx Murder?", "answer": ["Murder By Proxy"]}, {"question": "What is the place of birth of the director of film It'S The Rage (Film)?", "answer": ["American"]}, {"question": "What is the place of birth of Princess Albertina Frederica Of Baden-Durlach's father?", "answer": ["Ueckerm\u00fcnde"]}, {"question": "Which film has the director born earlier, Badman'S Gold or Sugar, Honey And Pepper?", "answer": ["Badman'S Gold"]}, {"question": "Who is the stepfather of Lorenzo Strozzi?", "answer": ["Filippo Strozzi the Younger"]}, {"question": "Which film has the director born first, 84 Charing Cross Road (Film) or Emperor'S Ball?", "answer": ["Emperor'S Ball"]}, {"question": "Who is Ian Leslie, 21St Earl Of Rothes's paternal grandfather?", "answer": ["Norman Leslie, 19th Earl of Rothes"]}, {"question": "Who died later, N\u00fckhetsez\u00e2 Han\u0131m or Henry W. Oliver?", "answer": ["Henry W. Oliver"]}, {"question": "Which country Boghos Nubar's father is from?", "answer": ["Egypt"]}, {"question": "Where was the director of film The Green Fog born?", "answer": ["Winnipeg, Manitoba"]}, {"question": "Who is younger, Aneta Hlad\u00edkov\u00e1 or Ren\u00e9 Poupardin?", "answer": ["Aneta Hlad\u00edkov\u00e1"]}, {"question": "Who is older, Marius Mitu or Bea Palya?", "answer": ["Marius Mitu"]}, {"question": "Which film whose director is younger, Only The Lonely (Film) or A Brother'S Love?", "answer": ["A Brother'S Love"]}, {"question": "Who is the paternal grandfather of Nia Segamain?", "answer": ["Fer Corb"]}, {"question": "Who is Walter Maule's paternal grandfather?", "answer": ["Sir William Maule"]}, {"question": "Was Paul Antoine Brunel or George Markham Giffard born first?", "answer": ["George Markham Giffard"]}, {"question": "Where does the director of film Suzanne'S Career work at?", "answer": ["Cahiers du cin\u00e9ma"]}, {"question": "Do both films, The Big Attraction and All Women Have Secrets, have the directors who are from the same country?", "answer": ["yes"]}, {"question": "Where was the place of death of the director of film Dans La Nuit (Film)?", "answer": ["Cannes"]}, {"question": "What is the place of birth of the director of film Mongolian Ping Pong?", "answer": ["Taiyuan"]}, {"question": "Which film has the director who was born earlier, Posl\u00ed\u010dek L\u00e1sky or John Meade'S Woman?", "answer": ["John Meade'S Woman"]}, {"question": "Where was the performer of song My Mistakes born?", "answer": ["London"]}, {"question": "Where was the place of death of the director of film Holiday In Tyrol?", "answer": ["Berlin"]}, {"question": "Which film has the director born earlier, Zombi Kampung Pisang or Sign Of The Anchor?", "answer": ["Sign Of The Anchor"]}, {"question": "Which film has the director who died earlier, Pursued or The Last Manuscript?", "answer": ["Pursued"]}, {"question": "Who is the maternal grandmother of Alfred, Duke Of Saxe-Coburg And Gotha?", "answer": ["Princess Victoria of Saxe-Coburg-Saalfeld"]}, {"question": "Does Ryan Hamilton (Rugby Union) have the same nationality as Bob King (Children'S Musician)?", "answer": ["yes"]}, {"question": "Where did the composer of film Oru Swakaryam die?", "answer": ["Lakshadweep"]}, {"question": "What is the place of birth of the director of film Deathstalker Ii?", "answer": ["New York"]}, {"question": "Who is Dorelia Mcneill's father-in-law?", "answer": ["Edwin John"]}, {"question": "Who was born later, Patrick Janvier or Nicolas About?", "answer": ["Patrick Janvier"]}, {"question": "What nationality is the performer of song Six Strings Down?", "answer": ["United States"]}, {"question": "Who is Marjorie Barretto's father-in-law?", "answer": ["Dencio Padilla"]}, {"question": "Where was the wife of Jean-\u00c9douard Bommer born?", "answer": ["Laeken"]}, {"question": "Which film came out earlier, Thendral Veesum or On The Same Planet?", "answer": ["Thendral Veesum"]}, {"question": "Which film has the director who died later, Her Own People or Ven Mi Coraz\u00f3n Te Llama?", "answer": ["Ven Mi Coraz\u00f3n Te Llama"]}, {"question": "Are both Big Sound Authority and Bandana (Country Band) from the same country?", "answer": ["no"]}, {"question": "Which film has the director died later, Too Tough To Kill or Jhoothi?", "answer": ["Jhoothi"]}, {"question": "Which country Catherine Of York's mother is from?", "answer": ["England"]}, {"question": "Who is the child of the director of film Deadlines (Film)?", "answer": ["Julia Levy-Boeken"]}, {"question": "Who is the maternal grandfather of Hans Albrecht, Hereditary Prince Of Schleswig-Holstein?", "answer": ["Ernst II, Prince of Hohenlohe-Langenburg"]}, {"question": "Who was born later, Miko Mayama or Wangechi?", "answer": ["Wangechi"]}, {"question": "Who was born first out of Werner Hartenstein and Albin Francisco Schoepf?", "answer": ["Albin Francisco Schoepf"]}, {"question": "Which film has the director who was born earlier, Running Out Of Luck or The Babe?", "answer": ["The Babe"]}, {"question": "Which film has the director died later, People In The Net or Mad Youth?", "answer": ["People In The Net"]}, {"question": "Where was the husband of Florence Bligh, Countess Of Darnley born?", "answer": ["Westminster"]}, {"question": "Where was the place of death of the director of film Sensational Janine?", "answer": ["Gr\u00e4felfing"]}, {"question": "Which film has the director born earlier, Everyone Else or The Girl From The Chartreuse?", "answer": ["The Girl From The Chartreuse"]}, {"question": "Who is the paternal grandfather of Charles Bruce, 5Th Earl Of Elgin?", "answer": ["Thomas Bruce, 7th Earl of Kincardine"]}, {"question": "Which film was released earlier, Nicky'S Family or Tales Of A Long Journey?", "answer": ["Tales Of A Long Journey"]}, {"question": "Which film has the director who was born first, I, The Aboriginal or Ca\u00f3tica Ana?", "answer": ["I, The Aboriginal"]}, {"question": "Where was the director of film Munchie born?", "answer": ["New York"]}, {"question": "Who is the paternal grandfather of Connor O'Brien, 2Nd Viscount Clare?", "answer": ["Connor O'Brien, 3rd Earl of Thomond"]}, {"question": "Who is the maternal grandfather of Rudolph Of France?", "answer": ["Conrad II, Duke of Transjurane Burgundy"]}, {"question": "Where did Paul Cobb Methuen's father study?", "answer": ["Oriel College"]}, {"question": "Did Lenny Mcallister and Ken Xie have the same nationality?", "answer": ["yes"]}, {"question": "Which film was released more recently, Unseen Forces or The Twilight Saga: Breaking Dawn \u2013 Part 1?", "answer": ["The Twilight Saga: Breaking Dawn \u2013 Part 1"]}, {"question": "Which film has the director who was born later, Money On The Street or She-Devils On Wheels?", "answer": ["She-Devils On Wheels"]}, {"question": "Which film whose director was born first, The Badlanders or The Man From Monterey?", "answer": ["The Man From Monterey"]}, {"question": "Who was born earlier, Igor Golomstock or Ted Garry?", "answer": ["Ted Garry"]}, {"question": "Where was the place of death of Quentin Keynes's father?", "answer": ["Cambridge"]}, {"question": "Where was the place of death of the director of film Marianela (1940 Film)?", "answer": ["Madrid"]}, {"question": "Which film has the director who died later, Song Of Nevada or The Sign On The Door?", "answer": ["Song Of Nevada"]}, {"question": "Which film was released more recently, Life According To Agfa or Memoria Negra?", "answer": ["Memoria Negra"]}, {"question": "What is the date of birth of George Frederick, Count Of Erbach-Breuberg's father?", "answer": ["16 December 1597"]}, {"question": "Do the movies High Life (2009 Film) and Route 132 (Film), originate from the same country?", "answer": ["yes"]}, {"question": "Who is younger, Moritz Brosch or Branko Bela\u010di\u0107?", "answer": ["Branko Bela\u010di\u0107"]}, {"question": "Did Charles Robinson (Raf Airman) and Terri Hooley share the same nationality?", "answer": ["yes"]}, {"question": "Where was the place of death of the composer of film Dream Wife?", "answer": ["Pacific Palisades"]}, {"question": "Which film has the director who was born earlier, Time For Loving or Un'Altra Vita?", "answer": ["Time For Loving"]}, {"question": "Which film whose director is younger, Many Tanks Mr. Atkins or Do Musafir?", "answer": ["Do Musafir"]}, {"question": "Who was born first, Selahattin Seyhun or Nil Tun Maung?", "answer": ["Nil Tun Maung"]}, {"question": "When is Princess Jos\u00e9phine Marie Of Belgium's mother's birthday?", "answer": ["17 November 1845"]}, {"question": "Which film has the director who died later, A Girl... And A Million or Invitation To Happiness?", "answer": ["A Girl... And A Million"]}, {"question": "Who is the sibling-in-law of Bella Sidney Woolf?", "answer": ["Virginia Woolf"]}, {"question": "Which country the director of film My Blueberry Nights is from?", "answer": ["Hong Kong"]}, {"question": "Which film was released earlier, Sathyavan Savithri or Kids Of The Round Table?", "answer": ["Kids Of The Round Table"]}, {"question": "When was Prince Frederick William Of Solms-Braunfels's wife born?", "answer": ["3 March 1778"]}, {"question": "What is the place of birth of the director of film Bhagirathi (Film)?", "answer": ["Tumkur"]}, {"question": "What nationality is the director of film Estudantes?", "answer": ["America"]}, {"question": "Where was the place of death of Henry Frederick, Hereditary Prince Of The Palatinate's father?", "answer": ["Mainz"]}, {"question": "Who is the spouse of the director of film Married And In Love?", "answer": ["Maureen O'Sullivan"]}, {"question": "Who lived longer, Martino Finotto or Franklin Edson?", "answer": ["Martino Finotto"]}, {"question": "When did Thomas Tufton, 6Th Earl Of Thanet's father die?", "answer": ["7 May 1664"]}, {"question": "Which film has the director died first, The Mystic or The Hasty Heart?", "answer": ["The Mystic"]}, {"question": "Where does the director of film The Marquise Of O (Film) work at?", "answer": ["Cahiers du cin\u00e9ma"]}, {"question": "Where was the director of film Noose (1958 Film) born?", "answer": ["Krak\u00f3w"]}, {"question": "Who is Philip Iii Of Navarre's father-in-law?", "answer": ["Louis X of France"]}, {"question": "What nationality is the director of film Running Scared (1972 Film)?", "answer": ["English"]}, {"question": "Who was born earlier, Ola Stunes Isene or Ruby Helder?", "answer": ["Ruby Helder"]}, {"question": "Did the movies The Ritual (2017 Film) and Triple Crossed, originate from the same country?", "answer": ["no"]}, {"question": "Are both Ciudad Ju\u00e1rez International Airport and Canc\u00fan International Airport located in the same country?", "answer": ["yes"]}, {"question": "Where was the director of film The Song Of Kaprun born?", "answer": ["Biberach"]}, {"question": "Who is Swanandi Tikekar's mother?", "answer": ["Arati Ankalikar-Tikekar"]}, {"question": "Who died earlier, Theodor Kotsch or Herbert Walther?", "answer": ["Theodor Kotsch"]}, {"question": "Where was the composer of song By The Beautiful Sea (Song) born?", "answer": ["Atlantic City, New Jersey"]}, {"question": "Which film has the director who is older, Tarmina or Wrong Turn?", "answer": ["Tarmina"]}, {"question": "What is the place of birth of the composer of film The Quiet One (Film)?", "answer": ["Tucson, Arizona"]}, {"question": "Who is the paternal grandfather of Adolf Iv, Count Of Berg?", "answer": ["Adolf II of Berg"]}, {"question": "Who lived longer, Nina \u0160kottov\u00e1 or Henry Strutt, 2Nd Baron Belper?", "answer": ["Henry Strutt, 2Nd Baron Belper"]}, {"question": "Are Edmonton/Villeneuve Airport and Pembroke Airport located in the same country?", "answer": ["yes"]}, {"question": "Where was the place of death of the director of film Joan Of Arc At The Stake?", "answer": ["Rome"]}, {"question": "Where did Diane Meyer Simon's husband graduate from?", "answer": ["City College of New York"]}, {"question": "Which film has the director who was born later, Land Of Hunted Men or Black Bandit?", "answer": ["Land Of Hunted Men"]}, {"question": "Who is Axel Rosenkrantz's maternal grandfather?", "answer": ["Axel Mowat"]}, {"question": "Which film has the director who died later, The Black Duke or The Way To The Gold?", "answer": ["The Way To The Gold"]}, {"question": "Who is Ieuan Ab Owain Glynd\u0175r's paternal grandfather?", "answer": ["Gruffudd Fychan II"]}, {"question": "What is the place of birth of Joan Of Savoy's father?", "answer": ["Baug\u00e9"]}, {"question": "When is Jean-Baptiste Colbert, Marquis De Seignelay's father's birthday?", "answer": ["29 August 1619"]}, {"question": "Where was the director of film Carolina Blues born?", "answer": ["New York"]}, {"question": "Who is Busba Kitiyakara's paternal grandfather?", "answer": ["Kitiyakara Voralaksana"]}, {"question": "Do both films, Shadows Of The Metropolis and Jaider, Der Einsame J\u00e4ger, have the directors who are from the same country?", "answer": ["yes"]}, {"question": "Are Carnic Alps and Torre Del Gran San Pietro located in the same country?", "answer": ["yes"]}, {"question": "Where was the director of film Axiliad born?", "answer": ["\u0141\u00f3d\u017a"]}, {"question": "Are the movies High Roller: The Stu Ungar Story and Mr. Peek-A-Boo, from the same country?", "answer": ["no"]}, {"question": "When was Patrick Berg's father born?", "answer": ["20 August 1968"]}, {"question": "Which film was released more recently, Woman Without A Past or Il Seduttore?", "answer": ["Il Seduttore"]}, {"question": "Which film has the director who was born later, Toll Of The Desert or One Wild Ride?", "answer": ["Toll Of The Desert"]}, {"question": "Who is the child of the director of film The Savage Is Loose?", "answer": ["Campbell Scott"]}, {"question": "What nationality is Humehume's father?", "answer": ["Kingdom of Hawai\u02bbi"]}, {"question": "Who is the father-in-law of Ri Sol-Ju?", "answer": ["Kim Jong-il"]}, {"question": "Which film has the director who was born earlier, Gendernauts or I Am Guilty?", "answer": ["Gendernauts"]}, {"question": "Who is Conrad I, Count Of Oldenburg's paternal grandmother?", "answer": ["Hedwig von Oldenburg in Wildeshausen"]}, {"question": "What is the place of birth of the director of film The Boss Of Rustler'S Roost?", "answer": ["Santa Rosa"]}, {"question": "Where was the father of Bonne Of Armagnac born?", "answer": ["Paris"]}, {"question": "Which film has the director born earlier, Safety Not Guaranteed or Once On The Rhine?", "answer": ["Once On The Rhine"]}, {"question": "Which country Lu\u00eds Of Portugal, Duke Of Beja's mother is from?", "answer": ["Spanish"]}, {"question": "What is the place of birth of the director of film Mallrats?", "answer": ["Red Bank"]}, {"question": "Who is the paternal grandmother of John I Of Brienne, Count Of Eu?", "answer": ["Berengaria of Le\u00f3n"]}, {"question": "Which film was released more recently, Blue Demon Contra Las Diab\u00f3licas or The Kid Brother?", "answer": ["Blue Demon Contra Las Diab\u00f3licas"]}, {"question": "Which film has the director who was born later, A Woman'S Case or Memoirs Of A Film Actress?", "answer": ["A Woman'S Case"]}, {"question": "What is the place of birth of the director of film The Possession Of Hannah Grace?", "answer": ["Dutch"]}, {"question": "Where does the director of film Man Of God (Film) work at?", "answer": ["University of Southern California"]}, {"question": "Do the movies Sweet Mama (Film) and Soy Andina, originate from the same country?", "answer": ["yes"]}, {"question": "Which film has the director died earlier, The Mark Of The Renegade or Muchachas Que Estudian?", "answer": ["Muchachas Que Estudian"]}, {"question": "Where was the place of death of the director of film Family Parade?", "answer": ["K\u00f6nigstein im Taunus"]}, {"question": "Who is younger, Lo Hsiao-Ting or Basil Hoffman?", "answer": ["Lo Hsiao-Ting"]}, {"question": "Who was born first, Giorgia Campana or Arcadius?", "answer": ["Arcadius"]}, {"question": "Who is David Carnegie, 10Th Earl Of Northesk's paternal grandmother?", "answer": ["Georgiana Maria Elliot"]}, {"question": "Which award the performer of song Jihad (Song) received?", "answer": ["Grammy Award"]}, {"question": "Which film has the director who was born later, Breaking The Silence: Truth And Lies In The War On Terror or Ju-On: The Grudge?", "answer": ["Ju-On: The Grudge"]}, {"question": "Who is the spouse of the director of film Commandos Strike At Dawn?", "answer": ["Maureen O'Sullivan"]}, {"question": "Where was the director of film En Aasai Rasave born?", "answer": ["Theni"]}, {"question": "Who was the teacher of the composer of song Prometheus (Art Song)?", "answer": ["Antonio Salieri"]}, {"question": "Where was the composer of song Heart On Fire (Ruslana Song) born?", "answer": ["L'viv"]}, {"question": "Who was born later, Meghan O'Leary or Engin F\u0131rat?", "answer": ["Meghan O'Leary"]}, {"question": "Where was the place of death of the founder of Bloomberg Industry Group?", "answer": ["Sarasota, Florida"]}, {"question": "Who is Cecil Calvert, 2Nd Baron Baltimore's paternal grandfather?", "answer": ["Leonard Calvert"]}, {"question": "Who died later, George Cressman or Herbert Douglas Anthony?", "answer": ["George Cressman"]}, {"question": "Which film was released first, Lettre Ouverte or A Woman'S War?", "answer": ["Lettre Ouverte"]}, {"question": "Are both Paloalto (Band) and Showbread (Band) from the same country?", "answer": ["yes"]}, {"question": "Which film whose director is younger, Running For Grace or Irulum Oliyum?", "answer": ["Running For Grace"]}, {"question": "What nationality is Krisztina Ny\u00e1ry's husband?", "answer": ["Hungarian"]}, {"question": "Who is the paternal grandfather of Princess Elisabeth Of Waldeck And Pyrmont?", "answer": ["George II"]}, {"question": "Which film was released more recently, Boundaries Of The Heart or Truck Turner?", "answer": ["Boundaries Of The Heart"]}, {"question": "Did the bands Cardiacs and Tedeschi Trucks Band, originate from the same country?", "answer": ["no"]}, {"question": "Where was the place of death of the director of film Gitano (1970 Film)?", "answer": ["Buenos Aires"]}, {"question": "Which film has the director who died later, America America or Bees In Paradise?", "answer": ["Bees In Paradise"]}, {"question": "Which film was released more recently, Fine Dead Girls or Venussian Tabutasco?", "answer": ["Venussian Tabutasco"]}, {"question": "What is the place of birth of the director of film Time Out Of Mind (2014 Film)?", "answer": ["Israeli"]}, {"question": "Who is Agnes I, Countess Of Nevers's paternal grandfather?", "answer": ["William III, Count of Nevers"]}, {"question": "What is the award that the performer of song Little More (Royalty) got?", "answer": ["Grammy Award for Best R&B Album"]}, {"question": "Who is younger, Charles R. Nesbitt or Ruben Trumpelmann?", "answer": ["Ruben Trumpelmann"]}, {"question": "Which one was established first, Noerr or Korn Ferry?", "answer": ["Noerr"]}, {"question": "Which film has the director who died first, The Year Of The Rabbit or Monster On The Campus?", "answer": ["Monster On The Campus"]}, {"question": "Who was born first out of Jes\u00fas Alejandro Gallardo and Kaleigh Quennec?", "answer": ["Jes\u00fas Alejandro Gallardo"]}, {"question": "Which film has the director died later, Gharabaher or Esc\u00e1ndalo De Estrellas?", "answer": ["Gharabaher"]}, {"question": "Where did Adolph Of Cleves, Lord Of Ravenstein's mother die?", "answer": ["Kalkar"]}, {"question": "Where did George Poulett, 8Th Earl Poulett's father study?", "answer": ["Cheltenham College"]}, {"question": "Are Stealing Harvard and Between Day And Night from the same country?", "answer": ["no"]}, {"question": "Which film whose director is younger, Song Of The Prairie or Ramen Teh?", "answer": ["Ramen Teh"]}, {"question": "Who is Hr\u00f3lfr Gautreksson's paternal grandfather?", "answer": ["Gaut"]}, {"question": "When was the director of film Slap Her... She'S French born?", "answer": ["October 20, 1952"]}, {"question": "Which film has the director born later, The Canyon Of Light or Diary For My Mother And Father?", "answer": ["Diary For My Mother And Father"]}, {"question": "Which film has the director who died later, The Ghost Breakers or A Hill In Korea?", "answer": ["A Hill In Korea"]}, {"question": "Did the bands Malavsky Family and Survay Says!, originate from the same country?", "answer": ["yes"]}, {"question": "Which film was released more recently, Espoir: Sierra De Teruel or The Trygon Factor?", "answer": ["The Trygon Factor"]}, {"question": "Who is Geoffrey Iii, Count Of Perche's paternal grandfather?", "answer": ["Rotrou III, Count of Perche"]}, {"question": "Which film whose director is younger, Mr. Doodle Kicks Off or Boyz N The Hood?", "answer": ["Boyz N The Hood"]}, {"question": "Which film has the director died later, Fighting Through or In Folly'S Trail?", "answer": ["In Folly'S Trail"]}, {"question": "Where was the director of film See Naples And Die born?", "answer": ["Alexandria"]}, {"question": "Where was the director of film You'Ve Got To Walk It Like You Talk It Or You'Ll Lose That Beat born?", "answer": ["American"]}, {"question": "Who is Blanche Of Navarre, Queen Of France's maternal grandfather?", "answer": ["Louis X of France"]}, {"question": "What is the place of birth of the director of film Paris By Night (2012 Film)?", "answer": ["Algiers"]}, {"question": "Which film has the director born first, Big Happiness or Border Romance?", "answer": ["Big Happiness"]}, {"question": "Where did the composer of song Wiegenlied, D 498 (Schubert) die?", "answer": ["Vienna"]}, {"question": "When did Emperor Houshao Of Han's father die?", "answer": ["26 September 188"]}, {"question": "Who is the maternal grandfather of Sophie Of W\u00fcrttemberg, Duchess Of Saxe-Weimar?", "answer": ["George"]}, {"question": "Which film has the director born earlier, Only God Forgives or Doublecrossed?", "answer": ["Doublecrossed"]}, {"question": "Why did Edward Despenser (Died 1342)'s father die?", "answer": ["hanged, drawn and quartered"]}, {"question": "Which film whose director is younger, K\u00f6pekler Adas\u0131 or Twisted Mistress?", "answer": ["K\u00f6pekler Adas\u0131"]}, {"question": "Who is Magdalene Of J\u00fclich-Cleves-Berg's father-in-law?", "answer": ["Wolfgang, Count Palatine of Zweibr\u00fccken"]}, {"question": "Are Beer Ajam and Al-Zabadani both located in the same country?", "answer": ["yes"]}, {"question": "Who is Prince Jean Of Luxembourg's mother?", "answer": ["Princess Jos\u00e9phine Charlotte of Belgium"]}, {"question": "Who is Renaud Ii, Count Of Soissons's uncle?", "answer": ["Guy II, Count of Soissons"]}, {"question": "Which country Padmavati (Wife Of Ashoka)'s husband is from?", "answer": ["Maurya dynasty"]}, {"question": "Who is the paternal grandfather of Princess Yasmin Aga Khan?", "answer": ["Aga Khan III"]}, {"question": "Which film has the director born later, Ladro Lui, Ladra Lei or Blow Job (Soffio Erotico)?", "answer": ["Blow Job (Soffio Erotico)"]}, {"question": "Who is the paternal grandmother of Zo\u00eb Porphyrogenita?", "answer": ["Theophano"]}, {"question": "Do both directors of films Anne Of The Indies and One Hour Of Love share the same nationality?", "answer": ["yes"]}, {"question": "Are Leontari, Boeotia and Ardameri both located in the same country?", "answer": ["yes"]}, {"question": "Are both movies, Some Like It Hot (2016 Film) and Under The Dome (Film), from the same country?", "answer": ["yes"]}, {"question": "Which film has the director who was born later, The Silver Treasure or Taxi To Paradise?", "answer": ["Taxi To Paradise"]}, {"question": "What is the place of birth of the performer of song All Neon Like?", "answer": ["Reykjav\u00edk"]}, {"question": "Are To Kill A Dragon and Tannenberg (Film) from the same country?", "answer": ["yes"]}, {"question": "Where did Edward Of Middleham, Prince Of Wales's mother die?", "answer": ["Westminster"]}, {"question": "Where was the place of death of the director of film A Hero Of The Big Snows?", "answer": ["Long Island"]}, {"question": "Who is Louis D'Armagnac, Duke Of Nemours's paternal grandfather?", "answer": ["Bernard d'Armagnac, Count of Pardiac"]}, {"question": "Who is the founder of the company that manufactured Fraise Tagada?", "answer": ["Hans Riegel"]}, {"question": "Which film whose director was born first, Psychos In Love or Waves Of Lust?", "answer": ["Waves Of Lust"]}, {"question": "Which country the director of film A Man Called Hero is from?", "answer": ["China"]}, {"question": "Who is the paternal grandfather of Yi Hong?", "answer": ["Prince Yi Kang"]}, {"question": "Where was the composer of song Come On (Chuck Berry Song) born?", "answer": ["St. Louis, Missouri"]}, {"question": "Who was born earlier, Elmer W. Conti or Seth Joshua?", "answer": ["Seth Joshua"]}, {"question": "Which film has the director who was born later, Unsullied or Not Wanted On Voyage?", "answer": ["Unsullied"]}, {"question": "Where was the mother of Vittoria Farnese D'Este born?", "answer": ["Capodimonte"]}, {"question": "Where was the place of death of Caroline Of Hesse-Homburg's father?", "answer": ["Bad Homburg vor der H\u00f6he"]}, {"question": "Which film has the director died first, He Who Gets Slapped or Mysterious Mr. Parkes?", "answer": ["He Who Gets Slapped"]}, {"question": "When did Camillo Ruspoli, Duke Of Sueca's father die?", "answer": ["March 8, 1829"]}, {"question": "Which film has the director born first, 30 Days In Atlanta or Les Tuche 3?", "answer": ["30 Days In Atlanta"]}, {"question": "Which film was released first, An Acceptable Loss or Douro, Faina Fluvial?", "answer": ["Douro, Faina Fluvial"]}, {"question": "Which film has the director who is older than the other, Perdues Dans New York or Web Of The Spider? ", "answer": ["Web Of The Spider"]}, {"question": "Do both directors of films Old School New School and Midnight (1922 Film) share the same nationality?", "answer": ["no"]}, {"question": "Which film has the director who died earlier, The Fixer (1968 Film) or Midnight At Madame Tussaud'S?", "answer": ["Midnight At Madame Tussaud'S"]}, {"question": "Which album was released more recently, The Arockalypse or Because The Internet?", "answer": ["Because The Internet"]}, {"question": "Where was the place of death of Randi Monsen's father?", "answer": ["Kristiania"]}, {"question": "Which film has the director died first, Her Greatest Love or Beware Of Women?", "answer": ["Her Greatest Love"]}, {"question": "Which film whose director is younger, Porky'S Revenge! or Amici Miei \u2013 Atto Iii?", "answer": ["Amici Miei \u2013 Atto Iii"]}, {"question": "Which film has the director who died first, Le P\u00e9riscope or Churam?", "answer": ["Le P\u00e9riscope"]}, {"question": "Who is the spouse of the director of film You Talkin' To Me? (Film)?", "answer": ["Sandra Nelson"]}, {"question": "Where did Rachel Weisz's husband study at?", "answer": ["Guildhall School of Music and Drama"]}, {"question": "When did Princess St\u00e9phanie Of Monaco's mother die?", "answer": ["September 14, 1982"]}, {"question": "Who is Richard De Montfort's paternal grandfather?", "answer": ["Amaury I de Montfort"]}, {"question": "Who lived longer, Crockett Johnson or Sven-Olof Lundgren?", "answer": ["Crockett Johnson"]}, {"question": "Which film has the director who died earlier, Guns And Guitars or To The Victor?", "answer": ["Guns And Guitars"]}, {"question": "Who is Mieszko Mieszkowic's paternal grandfather?", "answer": ["Siemomys\u0142"]}, {"question": "Which country the director of film Partly Cloudy With Sunny Spells is from?", "answer": ["Italian"]}, {"question": "When is Bakaffa's father's birthday?", "answer": ["1654"]}, {"question": "Which film came out first, Ayat-Ayat Cinta 2 or Lost Kisses?", "answer": ["Lost Kisses"]}, {"question": "When did Princess Alexandra Of Saxe-Coburg And Gotha's mother die?", "answer": ["24 October 1920"]}, {"question": "Where does the director of film Die Abfahrer work at?", "answer": ["Dortmund University of Applied Sciences and Arts"]}, {"question": "Who is the paternal grandfather of Randolph Greene Pack?", "answer": ["George Willis Pack"]}, {"question": "Which film was released more recently, Jolly Roger: Massacre At Cutter'S Cove or Texas Lightning (Film)?", "answer": ["Jolly Roger: Massacre At Cutter'S Cove"]}, {"question": "Which film has the director who is older, Hangman'S Knot or Last Foxtrot In Burbank?", "answer": ["Hangman'S Knot"]}, {"question": "Who is the paternal grandfather of William Ii, Duke Of Aquitaine?", "answer": ["Oliba I of Carcassonne"]}, {"question": "Which country William Herbert, 18Th Earl Of Pembroke's father is from?", "answer": ["British"]}, {"question": "Which country William Neville (Poet)'s father is from?", "answer": ["English"]}, {"question": "Where does Tova Hartman's father work at?", "answer": ["Shalom Hartman Institute"]}, {"question": "What is the place of birth of the director of film Breed Of The Border (1924 Film)?", "answer": ["Rochester"]}, {"question": "Who is Charles D'Orl\u00e9ans, Duke Of Penthi\u00e8vre's paternal grandfather?", "answer": ["Louis Philippe II, Duke of Orl\u00e9ans"]}, {"question": "Who died later, Joachim Gasquet or T. J. Richards?", "answer": ["T. J. Richards"]}, {"question": "Which film whose director was born first, The Snake Brothers or Olympus Has Fallen?", "answer": ["Olympus Has Fallen"]}, {"question": "Which film has the director who is older, The Woman Next Door or La Estatua De Carne?", "answer": ["La Estatua De Carne"]}, {"question": "Where did the director of film Il... Belpaese die?", "answer": ["Rome"]}, {"question": "Which film came out first, Hotel Der Toten G\u00e4ste or The Relative Of His Excellency?", "answer": ["The Relative Of His Excellency"]}, {"question": "Who is the paternal grandfather of Sultan Al-Dawla?", "answer": ["'Adud al-Dawla"]}, {"question": "Who is the father of Archduchess Maria Johanna Gabriela Of Austria?", "answer": ["Francis I, Holy Roman Emperor"]}, {"question": "Who is Charles Hutchinson (Nottingham Mp)'s paternal grandmother?", "answer": ["Jane Sacheverell"]}, {"question": "Are the directors of both films Pretty Foe and Goin' Coconuts from the same country?", "answer": ["no"]}, {"question": "Where did Nesrin Kad\u0131n's husband die?", "answer": ["Constantinople"]}, {"question": "Are the bands Shirehorses and From First To Last, from the same country?", "answer": ["no"]}, {"question": "Which film was released first, Heaven Help Us or Ghar Ki Rani?", "answer": ["Ghar Ki Rani"]}, {"question": "Which film was released earlier, Drive, He Said or Ocean Of Pearls?", "answer": ["Drive, He Said"]}, {"question": "Where was the father of Dinanath Puthenchery born?", "answer": ["Kozhikode"]}, {"question": "Who is the paternal grandfather of Janusz Iii Of Masovia?", "answer": ["Boles\u0142aw IV of Warsaw"]}, {"question": "Who is the paternal grandfather of Mariamne (Daughter Of Herod Agrippa)?", "answer": ["Aristobulus IV"]}, {"question": "Where was the place of death of the director of film The Secret Of Wera Baranska?", "answer": ["Theresienstadt concentration camp"]}, {"question": "Which film whose director was born first, Curry And Pepper or End Of Watch?", "answer": ["Curry And Pepper"]}, {"question": "Where did Marie-Th\u00e9r\u00e8se Houphou\u00ebt-Boigny's husband die?", "answer": ["Yamoussoukro"]}, {"question": "What is the date of death of Doamna Chiajna's husband?", "answer": ["21 September 1559"]}, {"question": "Which film has the director who died earlier, Tangled Destinies or The Daltons' Women?", "answer": ["Tangled Destinies"]}, {"question": "Where did Marie Eleonore Of Cleves's father die?", "answer": ["D\u00fcsseldorf"]}, {"question": "Where was the place of burial of Monika Von Habsburg's father?", "answer": ["Imperial Crypt"]}, {"question": "Who is younger, Ismael \u00cd\u00f1iguez or Fred Kornet?", "answer": ["Ismael \u00cd\u00f1iguez"]}, {"question": "Who is the stepmother of Anne Lennard, Countess Of Sussex?", "answer": ["Catherine of Braganza"]}, {"question": "Which film has the director who died later, The Little Brother or National Red Cross Pageant?", "answer": ["National Red Cross Pageant"]}, {"question": "Who is the father-in-law of Countess Palatine Elisabeth Auguste Sofie Of Neuburg?", "answer": ["Theodore Eustace"]}, {"question": "What is the place of birth of the director of film Los Secretos Del Buz\u00f3n?", "answer": ["Citt\u00e0 di Castello"]}, {"question": "Which film has the director died later, Flat Number Three or Alla Mia Cara Mamma Nel Giorno Del Suo Compleanno?", "answer": ["Alla Mia Cara Mamma Nel Giorno Del Suo Compleanno"]}, {"question": "Which film was released more recently, Steel Dawn or Wake Up And Live?", "answer": ["Steel Dawn"]}, {"question": "Who is younger, Jack Mildren or Alf Hellevik?", "answer": ["Jack Mildren"]}, {"question": "What is the place of birth of Countess Johanna Magdalene Of Hanau-Lichtenberg's mother?", "answer": ["Strasbourg"]}, {"question": "Who is younger, Kathryn Thomas or Ole Bj\u00f8rnmose?", "answer": ["Kathryn Thomas"]}, {"question": "Which film has the director who is older than the other, Gas, Oil And Water or Monte Carlo Or Bust!? ", "answer": ["Gas, Oil And Water"]}, {"question": "Which film has the director who died first, It'S A Grand Old World or Rustlers' Rhapsody?", "answer": ["It'S A Grand Old World"]}, {"question": "Where was the director of film Rembrandt'S J'Accuse born?", "answer": ["Newport, Wales"]}, {"question": "When is Charles I, Count Of Nevers's father's birthday?", "answer": ["1389"]}, {"question": "What is the place of birth of the director of film Rozstanie?", "answer": ["Krak\u00f3w"]}, {"question": "Where was the director of film The Road Through The Dark born?", "answer": ["New York"]}, {"question": "Who lived longer, Eleni Lambiri or Douglas Wilmer?", "answer": ["Douglas Wilmer"]}, {"question": "Who was born first, Earl Alexander Olmsted or Pietro Ghislandi?", "answer": ["Earl Alexander Olmsted"]}, {"question": "Which film came out earlier, Rajapattai or Tars And Spars?", "answer": ["Tars And Spars"]}, {"question": "Which country the performer of song I Like Control is from?", "answer": ["American"]}, {"question": "Do both directors of films Sometimes Good and Sweet Sixteen (2002 Film) share the same nationality?", "answer": ["yes"]}, {"question": "Which film has the director died later, No Babies Wanted or Prey For The Shadows?", "answer": ["Prey For The Shadows"]}, {"question": "Which film has the director who died first, Everybody'S Hobby or One Sunday Afternoon (1948 Film)?", "answer": ["Everybody'S Hobby"]}, {"question": "Who lived longer, Aniela Pawlikowska or Erling Jensen?", "answer": ["Erling Jensen"]}, {"question": "What is the place of birth of the director of film Back Long Ago?", "answer": ["Buenos Aires"]}, {"question": "Do both films, Cuban Colony and Prathyartha, have the directors who are from the same country?", "answer": ["yes"]}, {"question": "Which film was released more recently, Drugstore Cowboy or When The Sea Rises?", "answer": ["When The Sea Rises"]}, {"question": "Who is Julia Cornelia Paula's father-in-law?", "answer": ["Sextus Varius Marcellus"]}, {"question": "Are both Cupid The Cowpuncher and Hashimura Togo from the same country?", "answer": ["yes"]}, {"question": "Which film whose director is younger, I Enjoy The World With You or Fear On Trial?", "answer": ["I Enjoy The World With You"]}, {"question": "Where was the place of death of the director of film Las Tecnolog\u00edas Pesqueras?", "answer": ["Houston, Texas"]}, {"question": "Which film came out earlier, The Wrong Box or Soft Shoes?", "answer": ["Soft Shoes"]}, {"question": "Where was the place of death of Sabiha G\u00f6k\u00e7en's father?", "answer": ["Dolmabah\u00e7e Palace"]}, {"question": "Who is Consort Duan's father-in-law?", "answer": ["Zhu Youyuan"]}, {"question": "Which country the director of film And The Spring Comes is from?", "answer": ["Chinese"]}, {"question": "What is the place of birth of the director of film My Divine Poverty?", "answer": ["Italy"]}, {"question": "Where was the place of death of Yuen Shun-Yi's father?", "answer": ["Hong Kong"]}, {"question": "Who is Agnes Of Austria (1281\u20131364)'s father-in-law?", "answer": ["Stephen the Posthumous"]}, {"question": "Who is the child of the director of film Reunion At Fairborough?", "answer": ["Susannah Wise"]}, {"question": "Where was the director of film The Heart Of The World born?", "answer": ["Winnipeg, Manitoba"]}, {"question": "Who is Infanta Adelgundes, Duchess Of Guimar\u00e3es's father?", "answer": ["Miguel I of Portugal"]}, {"question": "Where did Salin Mibaya's father die?", "answer": ["Pyay"]}, {"question": "Which film has the director born earlier, Speaking Of Sex or Being There?", "answer": ["Being There"]}, {"question": "Where did the director of film A Night In Paradise (1919 Film) go to prison?", "answer": ["Theresienstadt concentration camp"]}, {"question": "Who is Margaret Of Valois, Countess Of Blois's maternal grandfather?", "answer": ["Charles II of Naples"]}, {"question": "Are both directors of films Shadow Of The Law (1926 Film) and A Touch Of Class (Film) from the same country?", "answer": ["yes"]}, {"question": "Which film has the director who died later, The Fair Barbarian or Vortex, The Face Of Medusa?", "answer": ["Vortex, The Face Of Medusa"]}, {"question": "Which film has the director who was born later, Attack Of Life: The Bang Tango Movie or King Kong (2005 Film)?", "answer": ["Attack Of Life: The Bang Tango Movie"]}, {"question": "Which film has the director who died later, Lincoln In The White House or Shattered Idols?", "answer": ["Lincoln In The White House"]}, {"question": "What nationality is Amytis Of Media's husband?", "answer": ["Babylon"]}, {"question": "Who is Humphrey Of Montfort's paternal grandmother?", "answer": ["Helvis of Ibelin"]}, {"question": "Which film came out first, Kansas City Confidential or Men O' War?", "answer": ["Men O' War"]}] -------------------------------------------------------------------------------- /data/eval/hotpotqa/test.json: -------------------------------------------------------------------------------- 1 | [{"question": "What is the nationality of the foreign born victim of Singapore's caning punishment before Oliver Fricker experienced the same?", "answer": ["American"]}, {"question": "Dorian Gray is the main character of what philosophical novel whose editor feared the story was indecent, and deleted roughly five hundred words before publication?", "answer": ["The Picture of Dorian Gray"]}, {"question": "Who wrote the 1970 international hit song Murray Head is most recognized for?", "answer": ["Andrew Lloyd Webber and Tim Rice"]}, {"question": "Where was the match held where Chicharito scored goals along with Antonio Valencia and Dimitar Berbatov?", "answer": ["Wembley Stadium"]}, {"question": "Where was Nexon who was composed by Adam Gubman located at?", "answer": ["South Korea"]}, {"question": "What event in 1920 was established by the director of the play \"The Miracle?\"", "answer": ["Salzburg Festival"]}, {"question": "This annual event held in several major cities in Finland is one of many all-night arts festival whose name is given to areas of high latitude in the weeks around what?", "answer": ["the summer solstice"]}, {"question": " Chun Woo-hee's notable films include a South Korean horror film about a policeman who investigates a series of mysterious killings and what?", "answer": ["illnesses"]}, {"question": "What team had Tony Roberts replaced by an American sportscaster from Buffalo, New York when he retired?", "answer": ["Notre Dame Fighting Irish football"]}, {"question": "Are Catasetum and Origanum in the same family?", "answer": ["no"]}, {"question": "Cooperative Living Organization is located in a city that is the county seat of what county in Florida?", "answer": ["Alachua"]}, {"question": "The 2015 Austrian thriller film Jack tells the story of a serial killer who died in what year?", "answer": ["1994"]}, {"question": "Which musician played in the band Soulfly, Max Cavalera or Billy Idol?", "answer": ["Massimiliano Antonio \"Max\" Cavalera"]}, {"question": "A guitar player called Noodles is a longtime member of which band, The Smashing Pumpkins or The Offspring?", "answer": ["The Offspring"]}, {"question": "Which airport is located further north in the United States, San Angelo Regional Airport or Grand Junction Regional Airport?", "answer": ["Grand Junction Regional Airport"]}, {"question": "The 1978 NBA World Championship Series had as MVP which Hall of Fame class member of 1988?", "answer": ["Westley Sissel Unseld"]}, {"question": "What is the name of the actress who has received a Drama Desk award and was part of the cast album for the 2007 Broadway production of the hit musical, \"Grease\"?", "answer": ["Laura Ann Osnes"]}, {"question": "Are Steve Perry and Leslie West both singers?", "answer": ["yes"]}, {"question": "What pop band features an Argentine actress that appeared in the television series \"Chiquititas\"?", "answer": ["Erreway"]}, {"question": "In what city did the 1959 William & Mary Indians football team lose against the #13 Naval Academy?", "answer": ["Annapolis, Maryland"]}, {"question": "Which star in the Maltese Bippy died in 1987?", "answer": ["Dan Rowan"]}, {"question": "The 2005 film Remedy featured Frank Vincent from The Sopranos and several mob movies by which acclaimed director?", "answer": ["Martin Scorsese"]}, {"question": "What is the city in which Bauerfield International Ariport is based out of?", "answer": ["Port Vila"]}, {"question": "What is the name of the movie that stars Katrina Bowden and was directed by Sean Anders?", "answer": ["Sex Drive"]}, {"question": "A man who played in the 1986 FIFA world cup played for what team during the 1982 Scottish League Cup Final?", "answer": ["Celtic"]}, {"question": "Based on a True Story... is an album by which country music star, with the single My Eyes?", "answer": ["Blake Shelton"]}, {"question": "Little Brown Stein is a rivalry trophy that imitates a mug made out of what item?", "answer": ["stoneware"]}, {"question": "What was the island, on which Marinelli Glacier is located, formerly known as?", "answer": ["Isla de Xativa"]}, {"question": "In what county was Duffy Jackson born?", "answer": ["Nassau County"]}, {"question": "What sort of dog did the same man who cared for the living gift that Martin Bormann gave Hitler have?", "answer": ["dachshund"]}, {"question": "How large of the area is the nature reserve south of Seacroft Esplanade?", "answer": ["4.3 km2"]}, {"question": "Viking: The Ultimate Obstacle Course was a game show that aired in Europe with the British presenter who was an Olympic participant in what sport?", "answer": ["bobsledder"]}, {"question": "Chimaja Appawas the younger brother of a general of the Maratha Empire in India, who served as what to the fifth Maratha Chhatrapati Shahu from 1720 until his death?", "answer": ["\"Peshwa\" (Prime Minister)"]}, {"question": "When will Flicker, created by a member of One Direction, be released?", "answer": ["October 20, 2017"]}, {"question": "Who was born first, Kwok Kin Pong or Edison Chen?", "answer": ["Edison Koon-hei Chen"]}, {"question": "Luke Rockhold defeated the MMA fighter who was the first to earn a win against which champion in the UFC?", "answer": ["Anderson Silva"]}, {"question": "Which genus's family comes first alphabetically, Crepis or Isatis?", "answer": ["Crepis"]}, {"question": "What country does Haringey Heartlands and The Mall Wood Green have in common?", "answer": ["London"]}, {"question": "C. Bernard Jackson founded the center that started the career of which actor, who played the part of Marty Castillo in \"Miami Vice\"?", "answer": ["Edward James Olmos"]}, {"question": "The McKinley Tariff was framed by the representative who was later assassinated in what month?", "answer": ["September 1901"]}, {"question": "Stephen susco wrote a movie produced by which american film actor?", "answer": ["Sam Raimi"]}, {"question": "How many member universities are there in this conference of which the Connecticut Huskies baseball team is a competing member?", "answer": ["12 member universities"]}, {"question": "Who starred in My Dog Skip and Malcolm in the Middle?", "answer": ["Frankie Muniz"]}, {"question": "In the 1962 German Grand Prix, a racer came second. What did this racer found?", "answer": ["Surtees Racing Organisation team"]}, {"question": "Chandgi Ram won a gold medal in 1970 in a continental multi-sport event held at which city ?", "answer": ["Bangkok"]}, {"question": "Which film was released first, Home on the Range or Pete's Dragon?", "answer": ["Home on the Range"]}, {"question": "Who was born first, Umar Israilov or Ramzan Kadyrov?", "answer": ["Ramzan Kadyrov"]}, {"question": "Did Lewis Allen or Danny DeVito play Louie De Palma?", "answer": ["Daniel Michael DeVito, Jr."]}, {"question": "Emily Beecham is best known for her role in a televsion series whose second season premiered on what date?", "answer": ["March 19, 2017"]}, {"question": "How long did The Shire of Flinders, which encompassed the extremity of the Mornington Peninsula located south-east of Melbourne, Australia, exist?", "answer": ["1874 until 1994"]}, {"question": "What county in Minnesota holds a 202 lb slab of greywacke covered in runes?", "answer": ["Douglas"]}, {"question": "What was the capital of India when the Taj Mahal was commissioned?", "answer": ["Agra"]}, {"question": "Forest Hill, Texas is a suburb of which fifth-largest city in the state?", "answer": ["Fort Worth"]}, {"question": "At what school is the individual who was awarded the 2012 Nobel Prize in Physics, jointly with David J. Wineland, a professor?", "answer": ["Coll\u00e8ge de France"]}, {"question": "Which genus has the most number of species, Myrsine or Styrax?", "answer": ["Myrsine"]}, {"question": "How much was Ammon Bundy's son ordered to pay in grazing fees?", "answer": ["over $1 million"]}, {"question": "How many films were directed by the director of Wise Blood?", "answer": ["37"]}, {"question": "Rock Springs is a collection of short stories by an author born in what year?", "answer": ["1944"]}, {"question": "Which tennis player is older out of John Peers and Simone Bolelli?", "answer": ["Simone Bolelli"]}, {"question": "Who was the chief executive officer of the second largest US car rental company by sales?", "answer": ["Robert L. Stone"]}, {"question": "Don Holmes played for the football club that is based in what eastern suburb of Perth?", "answer": ["Bassendean"]}, {"question": "In what country did Gildo Siopaes win a bronze medal for bobsledding in the 1964 Winter Olympics?", "answer": ["Austria"]}, {"question": "Who was the narrator of the 26-episode series which featured the British national museum organization with branches at five locations in England?", "answer": ["Michael Redgrave"]}, {"question": "\"Lost!\" is a song by a British rock band formed in what year?", "answer": ["1996"]}, {"question": "Which sport has been played at the BayArena in Leverkusen, Germany, since 1958?", "answer": ["football"]}, {"question": "What member of the South Korean-Chinese boy group EXO stars in the upcoming movie \"The Underdog\"?", "answer": ["Do Kyung-soo"]}, {"question": "Was Pocahontas made before Born in China?", "answer": ["yes"]}, {"question": "The Simpsons episode that aired on February 19, 2012 entitled \"At Long Last Leave\" represented what milestone for the show? ", "answer": ["show's 500th episode"]}, {"question": "Where is the American Hockey League, that Dan Frawley played in, located?", "answer": ["United States and Canada"]}, {"question": "what type of media does Ratatouille and PlayStation 3 have in common?", "answer": ["video game"]}, {"question": "What number installment of the Ghost School and Ghost School Horror was the film Voice?", "answer": ["4"]}, {"question": "This team which is moving from Oakland, California to Las Vegas Nevada with share its stadium with what college?", "answer": ["University of Nevada, Las Vegas"]}, {"question": "Are both Forman Christian College and Purbanchal University public schools?", "answer": ["no"]}, {"question": "When was the Argentine former footballer which Dutch manager Ronald Koeman replaced in 2014\u201315 Southampton F.C. season born", "answer": ["2 March 1972"]}, {"question": "Were Paul \u00c9luard and Georges Bataille from the same country?", "answer": ["yes"]}, {"question": "Which comic series involves characters such as Nick Fury and Baron von Strucker?", "answer": ["Marvel"]}, {"question": "Which is bigger, Griffon Bruxellois or English Mastiff?", "answer": ["English Mastiff"]}, {"question": "Did both Slaughter-House Cases and United States v. Darby Lumber Co. involve civil rights law?", "answer": ["no"]}, {"question": "Firstar Center was the name for two buildings before the company changed its name to match which company based in Minneapolis?", "answer": ["U.S. Bancorp"]}, {"question": "Which current CBS NBA analyst played for the PORTLAND TRAILBLazers along with former UNLV teammate Stacey Augmon?", "answer": ["Greg Anthony"]}, {"question": "What was the company that was co-owned by the person who Abbe error was named after?", "answer": ["Carl Zeiss AG"]}, {"question": "Are Toshi and Warrel Dane both in the band Sanctuary?", "answer": ["no"]}, {"question": "what air force based named in honor of Carl Benjamin \"Ben\" Eielson is located 26 miles southeast of Fairbanks, Alaska?", "answer": ["Eielson Air Force Base"]}, {"question": "The 1996 film \"The Christmas Tree\" stars Andrew McCarthy and an actress that has won how any Emmy Awards?", "answer": ["three"]}, {"question": "Why is Minister Pool important to Black Country and the West Midlands in England?", "answer": ["defence of the Cathedral"]}, {"question": "What other country does the league that Taylor Eric Kemp played in represents the sport's highest level besides the United States?", "answer": ["Canada"]}, {"question": " Which American rock band, Sugar Ray or Against the Current gained mainstream fame in the 1990's?", "answer": ["rock"]}, {"question": "This Fox News Specialist Host also took over what Fox Business Channel news program in 2013?", "answer": ["Cashin' In"]}, {"question": "Was Atom Egoyans biggest commercial success on stage or on film?", "answer": ["erotic thriller film"]}, {"question": "What year did the movement that Tess Asplund protested in the viral image of her with her fist in the air disband?", "answer": ["2016"]}, {"question": "Are Macheng and Lianjiang, Guangdong both cities ?", "answer": ["yes"]}, {"question": "What term is the current chairman of Prince William County serving?", "answer": ["fourth term"]}, {"question": "what country are Mudvayne and Hellyeah both from?", "answer": ["American"]}, {"question": "Who was born first, Antonin Artaud or Elizabeth Taylor?", "answer": ["Antonin Artaud"]}, {"question": "Which movie is loosley based off the Brother Grimm's \"Iron Henry\"?", "answer": ["The Frog Prince"]}, {"question": "Which beverage is ignited prior to consumption, Flaming beverage or Blue Hawaii", "answer": ["Flaming beverages"]}, {"question": "Toby Scott worked with which American singer born on May 24, 1941?", "answer": ["Bob Dylan"]}, {"question": "Orson Scott Card first published his short story \"Killing Children\" in this American science-fiction magazine whose first issue was published when?", "answer": ["1930"]}, {"question": "Church of All Saints, Doddinghurst is part of the state church led by which person?", "answer": ["Archbishop of Canterbury"]}, {"question": "What ingredient is in both Cosmopolitan and Cuba Libre?", "answer": ["lime"]}, {"question": "The winner of the the London Riders' Championship in 1953 scored how many points in the 1952 Individual Speedway World Championship?", "answer": ["14"]}, {"question": "Typically how many pages were in the fiction magazine in which Robert E. Howard wrote in 1930s?", "answer": ["128 pages"]}, {"question": "In 2009 the OCA announced major changes to it event lists, and in what city did this multi-sport event take place?", "answer": ["New Delhi, India"]}, {"question": "This high school located in the Bronzeville neighborhood was named after a settler who was arrested by who?", "answer": ["the British military"]}, {"question": "Are Florida Atlantic University and Cleveland State University both colleges in the US?", "answer": ["yes"]}, {"question": "In what year did the Danish plant ecologist who assisted a Danish chemist, famous for the introduction of the concept of pH die?", "answer": ["1974"]}, {"question": "The Prodigal Daughter, though a book about an American, is by a novelist of what nationality?", "answer": ["English"]}, {"question": "Dulce Maria Garc\u00eda Rivas took part in a \"Mask vs. Hair\" match with which female wrestler at Guerra de Titanes?", "answer": ["Faby Apache"]}, {"question": "Were both Tom Gullikson and J\u00fcrgen Melzer professional Austrian tennis players?", "answer": ["no"]}, {"question": "What was the nationality of the author of \"The Sunlight on the Garden\"?", "answer": ["Irish"]}, {"question": "The Secret of Moonacre was directed by the founder of which animation studio?", "answer": ["Klasky Csupo"]}, {"question": "What is the familial relationship of the founder of the Gulf Cartel to Juan Garc\u00eda \u00c1brego?", "answer": ["uncle"]}, {"question": " Who was born first Burton Cummings or Sharleen Spiteri?", "answer": ["Burton Lorne Cummings"]}, {"question": "Which was published first Polarity or The Farming Game?", "answer": ["The Farming Game"]}, {"question": " Season of Migration to the North is a counter narrative to a novella by what Polish-British novelist?", "answer": ["Joseph Conrad"]}, {"question": "What legend that lends its name to a modern medical term, inspired the plot of the horror film Ghostkeeper?", "answer": ["Wendigo"]}, {"question": "H-63 is a highway that runs near the county seat of what Michigan county?", "answer": ["Chippewa County"]}, {"question": "Timothy J. Sloan was the CEO of Wells Fargo who succeeded the CEO who took the job in what month?", "answer": ["June 2007"]}, {"question": "Which American politician and lawyer contested in San Francisco mayoral election, 2003", "answer": ["Matthew Edward"]}, {"question": "The Pluralist school is said to have included what creator of the theory of atomism? ", "answer": ["Leucippus"]}, {"question": "Who was the running mate of the man about whose candidacy the 2016 book Insane Clown President: Dispatches from the 2016 Circus documents?", "answer": ["Mike Pence"]}, {"question": "which plant has the highest specie Panicum or Populus ", "answer": ["Panicum"]}, {"question": "Gene Sculatti write for CREEM sometime in the 60s or 70s, what famous term did CREEM coin for describing a genre of rock?", "answer": ["\"punk rock"]}, {"question": "When did the predecessor of The Eyes of Nye first air?", "answer": ["September 10, 1993"]}, {"question": "When was the movie adaption of Jennifer Weiner's novel \"In Her Shoes\" released?", "answer": ["2005"]}, {"question": "The 1994 \"The Shadow\" co-stars Lone, Penelope Ann Miller, Peter Boyle, Jonathan Winters, Tim Curry and an actor that has received how many Emmy Award nominations?", "answer": ["five"]}, {"question": "What date in 2010 was a South Korean film starring Kim Hyang-gi released?", "answer": ["January 14, 2010"]}, {"question": " Force India VJM10 is a Formula One racing car previous driven by Nico H\u00fclkenberg, and is now driven by which driver born 17 September 1996?", "answer": ["Esteban Ocon"]}, {"question": "\"The Chance of a Lunchtime\" is an episode of a BBC sitcom writtem by who?", "answer": ["John Sullivan"]}, {"question": "Which region of Austria means 'Estate of the Salt Chamber' and covers the village of Obertraun, a popular destination for skiing and snowboarding in the winter?", "answer": ["Salzkammergut"]}, {"question": " Lucas da Silva Carvalho was an unused reserve runner for a Brazilian athletics track event which at top class events is run in lanes for the first what?", "answer": ["500 metres"]}, {"question": "Seal Harris was born in what county?", "answer": ["Polk County"]}, {"question": "Lawerence Turman had produced a film, with a character named \"Johnny 5\" that had a follow up sequel released in what year?", "answer": ["1988"]}, {"question": "After partially sinking in 1994, Celestyal Crystal was rebuilt for the purpose of what Bermudian company?", "answer": ["Norwegian Cruise Line Holdings Ltd."]}, {"question": "Which former lead guitarist has a solo album named two disc compilation album?", "answer": ["Ted Nugent"]}, {"question": "Who was born first, Brooklyn Decker or Andy Roddick?", "answer": ["Andrew Stephen Roddick"]}, {"question": "Football at the 2006 Asian Games with Wisoon Wichaya as its coach was played how many days prior to the opening ceromony in Doha, Qatar in 2006? ", "answer": ["14"]}, {"question": "Charlotte Hatherley initially came to prominence in a band formed in what year?", "answer": ["1992"]}, {"question": "What song was number 4 on the charts when a song from FutureSex/LoveSounds was number 1?", "answer": ["Rudebox"]}, {"question": "Howard Van Doren Shaw was the designer of which 1917 Progressive Era community? ", "answer": ["Marktown"]}, {"question": "This man who was born on April 25th, 1947, was a member of this lower house of the Hawaii State Legislature.", "answer": ["Robert \"Bobby\" Bunda"]}, {"question": "What United Kingdom Prime Minister was succeeded in the 1990s by the Leader of the Conservative Party?", "answer": ["Margaret Thatcher"]}, {"question": "Andrei Ivanovich Gorchakov commanded the 1st Infanty Corps in what major engagement of the Napoleonic Wars?", "answer": ["Battle of Dresden"]}, {"question": "Which comic book was also written by the writer of Crossed?", "answer": ["the Vertigo series \"Preacher\""]}, {"question": "The father of Manning Marius Kimmel retired from the Navy in what year?", "answer": ["1942"]}, {"question": "What occupation do Henry Moore and John Wesley have in common?", "answer": ["minister"]}, {"question": "What was the 2010 population of the town where Lake George Avenue Historic District is located?", "answer": ["5,042"]}, {"question": "Of these two publication--B\u00e1iki and Sick--what type of publication is the one that was published most frequently? ", "answer": ["satirical-humor magazine"]}, {"question": "The 2000 ICC KnockOut Trophy was the debut of a Jamaican cricketer who is a right-handed what?", "answer": ["middle order batsman"]}, {"question": "Which novelist was German, David Guterson or Thomas Mann?", "answer": ["Paul Thomas Mann"]}, {"question": "Which 1962 film starring Tom Courtenay is based on a short story of the same name?", "answer": ["The Loneliness of the Long Distance Runner"]}, {"question": "Which Austrian writer and journalist introduced the term \"Journaille \"", "answer": ["Karl Kraus"]}, {"question": "What profession does Stephen Jay Gould and Ann Beattie have in common?", "answer": ["writer"]}, {"question": "Suicide's 1977 released album features a song based on what brand's comic character?", "answer": ["Marvel"]}, {"question": "My Neighbor Totoro was produced by a Japanese animation film studio founded in what year?", "answer": ["1985"]}, {"question": "In what year did both Orson Scott Card's novel \"Ender's Game\" and Keri Hulme's \"The Bone People\" come out?", "answer": ["1985"]}, {"question": "Operalia, The World Opera Competition helped launch the career of an operatic soprano of what nationality?", "answer": ["Bulgarian"]}, {"question": "How many members were in the South Korean boy group whose first official sub-unit is Bastarz?", "answer": ["seven members"]}, {"question": "Which casino closed first The Atlantic Club Casino Hotel or Trump Plaza Hotel and Casino ? ", "answer": ["The Atlantic Club Casino Hotel"]}, {"question": "What group of Elektra Records recording artists are known to be an indie pop band?", "answer": ["Saint Motel"]}, {"question": "Is Ordos City more west than Yangzhong?", "answer": ["yes"]}, {"question": "Between Valparaiso University and Old Dominion University, which institution has higher student enrollment?", "answer": ["Old Dominion University"]}, {"question": "What type of media does J. T. Petty and Outlast have in common?", "answer": ["video game"]}, {"question": "How many girls and boys are in the seventeen sibling family featured on Counting On?", "answer": ["nine girls and 10 boys"]}, {"question": "How many episodes were in the original run of the HBO show that Kristin Davis played the character Charlotte York Goldenblatt?", "answer": ["94"]}, {"question": "W41AP is a station owned by the daily newspaper serving what nearby city in Erie County?", "answer": ["Port Clinton"]}, {"question": "What English-American actor known for his role in \"Raiders of the Lost Ark\" starred in The Lodger?", "answer": ["Alfredo \"Alfred\" Molina"]}, {"question": "What is the birth name of the woman that \"A Woman Called Moses\" is based on?", "answer": ["Araminta Ross"]}, {"question": "What political party is the current Attorney General of Missouri associated with?", "answer": ["Republican"]}, {"question": "Who was the first Russian composer to make a lasting impression internationally, Alessandro Scarlatti or Pyotr Ilyich Tchaikovsky?", "answer": ["Pyotr Ilyich Tchaikovsky"]}, {"question": "What do a Flaming beverage and a Gin and tonic have in common?", "answer": ["cocktails"]}, {"question": "Which feature does the G-shock watch have that is tested by a leakage test?", "answer": ["water resistance"]}, {"question": "The man who currently serves as the wide receivers coach for the Dallas Cowboys used to coach at what college?", "answer": ["Louisiana Tech"]}, {"question": "What Canadian filmmaker created a film about Bettie Page in 2005?", "answer": ["Mary Harron (born January 12, 1953)"]}, {"question": "Kent Dairy Round Barn is near the city in what Montana county?", "answer": ["Carbon"]}, {"question": "Which Croatian professional basketball player (a shooting guard) did New Jersey Nets acquire during their 1990\u201391 season?", "answer": ["Dra\u017een Petrovi\u0107"]}, {"question": "What movie was released at a later date, My Dog Tulip or Monsters vs. Aliens?", "answer": ["Monsters vs. Aliens"]}, {"question": "An Austrian-British psychoanalyst born in 1882 and innovative in object relations theory was also rigorous about what other type of therapy?", "answer": ["Morita therapy"]}, {"question": "Who is the controlling shareholder of the world's fourth-largest tyre manufacturer?", "answer": ["J\u00fcrgen M. Geissinger"]}, {"question": "What type of the district is the Downtown Bentonville of the state in the southeastern region of the United States which is home to over 3 million people as of 2017?", "answer": ["business district"]}, {"question": "The View from the Bottom is the fifth studio album by an American rock band best known for what hit song?", "answer": ["My Own Worst Enemy"]}, {"question": "Who was attempting to regain the British throne until the Raids on Lachaber and Shiramore shut down his operations?", "answer": ["Charles Edward Stuart"]}, {"question": "Who starred as an American attorney in Grey Gardens?", "answer": ["Ken Howard"]}, {"question": "Triangle Brewing Company was started by a founder who had experience as what non-commercial alcohol producer?", "answer": ["homebrewer"]}, {"question": "Which magazine had more previous names, Watercolor Artist or The General?", "answer": ["The General"]}, {"question": "Where is the county seat for people living in Henderson Point, Mississippi?", "answer": ["Gulfport, Biloxi"]}, {"question": "What album did John Lennon release before the one that contained the song \"How?\"", "answer": ["John Lennon/Plastic Ono Band"]}, {"question": "In which town was the amusement park that is operated by Britannia Jinky Jersey Ltd located where the first Tomorrow's Parties Festival was held?.", "answer": ["Camber Sands"]}, {"question": "In which movie the Stephanie Kay Panabaker was alongside with her elder sister Danielle Nicole Panabaker?", "answer": ["\"Read It and Weep\" (2006)"]}, {"question": " Robert Smith founded the multinational company headquartered in what city?", "answer": ["Golden Valley, Minnesota,"]}, {"question": "What is the name of the company in Independent Hill, Virginia that is also headquartered in Houston Texas?", "answer": ["Citgo"]}, {"question": "Which characters are akin to goblins and were added to a Tilted Mill Entertainment video game in 2009?", "answer": ["Orc characters"]}, {"question": "The Bachelor was based on what novel written by an Austrian author?", "answer": ["\"Dr. Gr\u00e4sler, Badearzt\""]}, {"question": "Which industrialist established a museum in Heckscher Park that features \"First Friday\" concerts?", "answer": ["August Heckscher"]}, {"question": "Which film was released first, Jawbreaker or Havenhurst?", "answer": ["Jawbreaker"]}, {"question": "Did the the Battle of Stones River occur before the Battle of Saipan?", "answer": ["yes"]}, {"question": "What \"God Gave Me You\" artist wrote the song \"Imagine That\" for Diamond Rio?", "answer": ["Bryan Shelton White"]}, {"question": "Alex Haley wrote what novel about a man that was born in 1750?", "answer": ["Roots: The Saga of an American Family"]}, {"question": "Wind was directed by which American actress born on March 26, 1960?", "answer": ["Jennifer Grey"]}, {"question": "Which writer of \"Rewrite This Story\" are an American songwriting duo and composing team for musical theater, films, and television?", "answer": ["Pasek and Paul"]}, {"question": "Which song that John Kirby scored is often the final piece of music played during an evening of revelry?", "answer": ["\"Loch Lomond\""]}, {"question": "\"Lesbian Request Denied\" is the third episode of the first season of a series created by who?", "answer": ["Jenji Kohan"]}, {"question": "Where was one of Sergei Prokofiev's most well-known operas first performed in 1921?", "answer": ["Chicago, Illinois"]}, {"question": "California joined the Union due to the passage of a package of how many separate bills?", "answer": ["five"]}, {"question": "Which type of character is featured by the P.L. Travers's third book and last novel in the \"Mary Poppins\" series?", "answer": ["fictional character"]}, {"question": "What is the name of the singer who's song was released as the lead single from the album \"Confessions\", and that had popular song stuck behind for eight consecutive weeks?", "answer": ["Usher"]}, {"question": "Who founded gaming company which owns and operates the Red Rock Resort?", "answer": ["Frank Fertitta, Jr."]}, {"question": "Do Ray Wilson and Leo have the same nationality ?", "answer": ["no"]}, {"question": "The creator of \"Wallace and Gromit\" also created what animation comedy that matched animated zoo animals with a soundtrack of people talking about their homes? ", "answer": ["Creature Comforts"]}, {"question": "What type of engine powered both the Saab 36 and the Avro Vulcan?", "answer": ["British Bristol Olympus turbojet"]}, {"question": "When did the author of the book One More River is based on win the Nobel Prize?", "answer": ["1932"]}, {"question": "Who played the role of Nettie Harris in the 1985 film directed by Steven Spielberg?", "answer": ["Akosua Gyamama Busia"]}, {"question": "The head of the Foreign Relations Department of the Rastriya Janashakti Party holds a degree that can be abbreviated MS, M.S., or ScM, in what field?", "answer": ["Engineering"]}, {"question": "Who was the artist that recorded a duet in 1985 for the CBS TV series 'Dallas' with an artist that was born in 1951 and has twenty #1 hits?", "answer": ["Gary Morris"]}, {"question": "Are Freakonomics and In the Realm of the Hackers both American documentaries?", "answer": ["no"]}, {"question": "What was the code name of the campaign where Joe Foss received a Medal of Honor?", "answer": ["Operation Watchtower"]}, {"question": "What German state was Karl Julius Perleb born in?", "answer": ["Baden-W\u00fcrttemberg"]}, {"question": "Who was born first, Johan Leysen or George Clooney?", "answer": ["Johan Leysen"]}, {"question": "Alfred Balk served as the secretary of the Committee on the Employment of Minority Groups in the News Media under which United States Vice President?", "answer": ["Nelson Rockefeller"]}, {"question": "Who beacme a star as a comic book character created by Gerry Conway and Bob Oksner?", "answer": ["Megalyn Echikunwoke"]}, {"question": "Who has released more solo albums, Nick Carter or Brady Seals?", "answer": ["Brady Seals"]}, {"question": "Both Tsai Ming-liang and Giacomo Gentilomo were involved with which profession?", "answer": ["film director"]}, {"question": "Of the companies that sponsor the British League Riders Championship, which is owned and operated by Goodyear Tire and Rubber Company?", "answer": ["Dunlop"]}, {"question": "Connetquot High School is a public high school in what city in New York?", "answer": ["Oakdale"]}, {"question": "\"The Empire Builder\"'s Columbia River Subdivision was part of which line?", "answer": ["Great Northern Railway"]}, {"question": "The mass killing that took place at Oakland, California on April 2, 2012 was less deadly than the one that took place on October 1, 2015 in which state?", "answer": ["Oregon"]}, {"question": "Which Canadian rock band released a song called \"Counterparts\" and had a drummer who was inducted into the Modern Drummer Hall of Fame?", "answer": ["Rush"]}, {"question": "What profession did Willi Forst and Elmer Clifton share?", "answer": ["actor"]}, {"question": "What city is the women's Vans Triple Crown of Surfing event hosted at?", "answer": ["City and County of Honolulu"]}, {"question": " Lou Cutell appeared on the third episode of the ninth season of an CBS sitcom that was what number episode overall?", "answer": ["187th"]}, {"question": "Who was the New York City Fire Commissioner at the time of Providenza Panno's death?", "answer": ["Rhinelander Waldo"]}, {"question": "What was the name of the country ran by the president who maintained the mercenary unit, White Legion? ", "answer": ["Zaire"]}, {"question": "What cuisine is described as a cultural blending of Mediterranean influences (such as those created by Italian and Spanish populations)?", "answer": ["Argentine cuisine"]}, {"question": "In which year was the King who made the 1925 Birthday Honours born?", "answer": ["1865"]}, {"question": "What is the name of the largest shipping facility in the Northeastern United States, where the Boston Red Socks play?", "answer": ["Port of Boston"]}, {"question": "What American actress who died in 2011 starred in Fear Stalk?", "answer": ["Sada Thompson"]}, {"question": "The retired American professional basketball player Walt Williams attended this institution in 1988 that is in which conference today?", "answer": ["Big Ten Conference"]}, {"question": "What is the name of a member of the North Melbourne Football Club, whose brother Rob has starred in Bridgetowns Cricket Clubs Premiership in 2016/7?", "answer": ["Mark Andrew Brayshaw"]}, {"question": "which American soldier and serial killer was Death of a Soldier based on ", "answer": ["Edward Joseph Leonski"]}, {"question": "What state is home to the Lake Wobegon Trails and the recording location for the live radio show \"A Prairie Home Companion\"?", "answer": ["Minnesota"]}, {"question": "Which film stars more animals, The Jungle Book or The Lone Ranger?", "answer": ["The Jungle Book"]}, {"question": "Lucy Pevensie is a character in the series of fantasy novels that have sold more than how many copies?", "answer": ["100 million"]}, {"question": "What is the current job of the former partner in ice skating of Vladimir Fedorov?", "answer": ["She currently works as a coach and choreographer"]}, {"question": "The Nike Mercurial Vapor is endorsed by a Stoke City football player who also plays for what national team?", "answer": ["the Switzerland national team"]}, {"question": "When was Leeds no longer called Elmet?", "answer": ["early 7th century"]}, {"question": "What was the name of the team that represented an Arizona university in the conference that covers a broad expanse of the western United States, including Arizona, California, and New Mexico?", "answer": ["Arizona State Sun Devils"]}, {"question": "With whom was a notable partner of Elena Krykanova a two-time Olympic champion with?", "answer": ["Oksana Grishuk"]}, {"question": " What was the address of the building on U.S. Route 3 that was the tallest building in New Hampshire when completed in 1972?", "answer": ["1000 Elm Street"]}, {"question": "Which documentary is the oldest, The Private Life of Plants or Marx Reloaded?", "answer": ["The Private Life of Plants"]}, {"question": "What is the title of the 1979 film adaptation of William Shakespeare's play in which the English poet, actor, political activist and dramatist who wrote wrote a number of book-length polemical poems such as \"Autogeddon\", \"Falling for a Dolphin\" and \"Whale Nation\" played a main character?", "answer": ["The Tempest"]}, {"question": "Dance and Laugh Amongst the Rotten is a studio Album by a band from which country ?", "answer": ["the Netherlands"]}, {"question": "Tommy Swerdlow co-wrote the screenplay of what film directed by Jon Turteltaub?", "answer": ["Cool Runnings"]}, {"question": "Who died more recently, Ferdinand Anton Ernst Porsche or Ferdinand Porsche", "answer": ["Ferdinand Anton Ernst Porsche"]}, {"question": "The Declaration of Montreal was the founding document of which world event, lasting from July 26-August 5, 2006?", "answer": ["The 1st World Outgames"]}, {"question": "The third solo album by David Gates of Bread shares a name with a 1977 film directed by who?", "answer": ["Herbert Ross"]}, {"question": "What third film in a franchise did Camile Delamarre edit?", "answer": ["Transporter 3"]}, {"question": "Yau Ma Tei North is a district of a city with how many citizens?", "answer": ["7.2 million"]}, {"question": "How many episodes were made for the science fiction series with Gus Birney?", "answer": ["ten"]}, {"question": "What is the name of the daughter of the Irish entertainer in the girl band Maxi, Dick and Twink?", "answer": ["Chlo\u00eb Alexandra Adele Emily Agnew"]}, {"question": "What car did the German rally driver, who won victories for Fiat, Opel, Lancia, and Audi drive?", "answer": ["The Lancia Rally"]}, {"question": "Sian Gibson co-wrote which British sitcom, set around a supermarket and car share scheme?", "answer": ["Peter Kay's Car Share"]}, {"question": "Fiske O'Hara was born in what city Knox County, Maine, that had a population of 2,297 as of the 2010 census?", "answer": ["Rockland"]}, {"question": "When was the British economic historian who established \"The Economic History Review\" born? ", "answer": ["9 January 18898"]}, {"question": "Which breed, Chinese Chongqing Dog or Kanni, is used for hunting?", "answer": ["Kanni"]}, {"question": "H Pistorius & Co is part of the reason for the wealth of the family that includes which South African sprinter?", "answer": ["Oscar Leonard Carl Pistorius"]}, {"question": "Which city in Missouri is the Route 66 State Park closest to travel from, St.Louis or Eureka?", "answer": ["Eureka"]}, {"question": "Bill Bryson grew up in a city that is the seat of which county ?", "answer": ["Polk County"]}, {"question": "James Conlon directed the oldest outdoor Music Festival that is located and what state?", "answer": ["Illinois"]}, {"question": "Which countries forces fought in The Battle of H\u00fcrtgen Forest which was covered in the shooter video game Darkest Hour: Europe '44-'45?", "answer": ["American and German"]}, {"question": "How many beer brands are in the portfolio of the company that hired Leonardus (Leo) Petrus Beukeboom to decorate cafes?", "answer": ["over 170"]}, {"question": "What group of languages is the surname Gref, meaning \"Grave,\" taken from?", "answer": ["Germanic"]}, {"question": "Nguy\u1ec5n V\u0103n Tr\u1ed7i tried to assassinate a future ambassador from which US state ?", "answer": ["Massachusetts"]}, {"question": "When was the duo formed who's English language song is the \"Freaky Like Me\"?", "answer": ["1992"]}, {"question": "Who owns the broadcasting platform where Games Done Quick is broadcast?", "answer": ["Amazon.com"]}, {"question": "The star of \"The Inkwell\" also starred in what 1993 movie as O-Dog?", "answer": ["Menace II Society"]}, {"question": "Who composed music for this 1941 American animated film with the main character of Jumbo Jr.?", "answer": ["Dan Crow"]}, {"question": "Robert Earl Holding owned an oil company that was originally founded by who?", "answer": ["Harry F. Sinclair"]}, {"question": "Northland Mall was the first directionally-named shopping center in Columbus, 5 years before which shopping center at the intersection of US-40 and I-270?", "answer": ["Westland Mall"]}, {"question": "WFMT FM radio transmits from the second tallest building in the United States, which is located where?", "answer": ["Chicago, Illinois, United States"]}, {"question": "Where is the city and \"comune\" in Emilia-Romagna in which talian Renaissance painter ;Ludovico Mazzolino located ", "answer": ["northern Italy,"]}, {"question": "Is Bob Hewitt a citizen of a different country than Ray Ruffels?", "answer": ["yes"]}, {"question": "In the Library with the Lead Pipe, an academic journal on topics about libraries, is abstracted and indexed by this third largest private company in Birmingham, Alabama? ", "answer": ["EBSCO Information Services"]}, {"question": "Where will the student which pass the exam for which Le R\u00eave de Rico has to pass progress to? ", "answer": ["secondary school study"]}, {"question": "Are both Yixing and Shaoshan county-level cities in China?", "answer": ["yes"]}, {"question": "Are Buried Country and South of the Border both films turned into stage performances?", "answer": ["no"]}, {"question": "Which university is further North, University of New Hampshire or Stony Brook University?", "answer": ["Stony Brook University"]}, {"question": "When did an IndyCar race driver drive a Formula 1 car designed by Peter McCool?", "answer": ["2007 Formula One season"]}, {"question": "Cagliostro-Walzer op.370 was composed by a man born in what year?", "answer": ["1825"]}, {"question": "Which movie was filmed first \"The Guest\" or \"You're Next\"?", "answer": ["You're Next"]}, {"question": "Which shore does the castle, which Sir James Montgomery, 4th Baronet is the tenth laird of, stand?", "answer": ["Firth of Clyde"]}, {"question": "Who was the great grandfather of Franklin Seaver Pratt's wife?", "answer": ["Kalokuokamaile"]}, {"question": "Which director was also a writer, Noel M. Smith or Reza Parsa?", "answer": ["Noel Mason Smith"]}, {"question": "Who gave the mother of Alberic II of Spoleto the title \"patricia\" of Rome?", "answer": ["Pope John X"]}, {"question": " What did an English former footballer purchase in march 2009?", "answer": ["50 JJB Sports Fitness Clubs"]}, {"question": "Ross Pople worked with this French dominant figure of the post-war classical music world.", "answer": ["Pierre Louis Joseph Boulez"]}, {"question": "What line featured characters from a DC Comic creator by Bob Kane and Bill Finger?", "answer": ["action figure toyline"]}, {"question": "Why is Serena Williams more accomplished than Mariaan de Swardt in her tennis career?", "answer": ["the world No. 1 for the first time"]}, {"question": "Who was the eldest brother of the Mexican drug trafficker born 12 March 1952?", "answer": ["Francisco Rafael Arellano F\u00e9lix"]}, {"question": "Which US Supreme court case involved a waiver of right to counsel Planned Parenthood v. Casey or Montejo v. Louisiana?", "answer": ["Montejo v. Louisiana"]}, {"question": "Jennifer Jones captained the gold medalists for Canada at the Sochi Winter Olympics in which event?", "answer": ["Curling"]}, {"question": "Who directed the film in which Catherine Ann \"Kate\" Bosworth made her debut?", "answer": ["Robert Redford"]}, {"question": "In which year did Baek Bong-ki appear in the South Korean drama film set in a high school?", "answer": ["2004"]}, {"question": "Where was the company, of which David DeVoe is Chief Financial Officer, ranked in the world, in 2014, in terms of revenue, against similar types of company ? ", "answer": ["fourth-largest"]}, {"question": "What profession does H. L. Mencken and Albert Camus have in common?", "answer": ["journalist"]}, {"question": "This work of literature _______ , known by its Greek title Oedipus Tyrannus was argued by Sigmund Freud to be one of the greatest works of world literature in his 1928 article. ", "answer": ["Oedipus Rex"]}, {"question": "What year did the CEO of Tata Consultancy Services takeover as Chairman?", "answer": ["2017"]}, {"question": "What plant has about 40 species native to Asia, Manglietia or Abronia?", "answer": ["Manglietia"]}, {"question": "Substorm was described in qualitative terms by a scientist nominated for what seven times?", "answer": ["Nobel Prize"]}, {"question": "Which Australian city founded in 1838 contains a boarding school opened by a Prime Minister of Australia and named after a school in London of the same name.", "answer": ["Marion, South Australia"]}, {"question": "Van Andel Institute was founded in part by what American businessman, who was best known as co-founder of the Amway Corporation?", "answer": ["Jay Van Andel"]}, {"question": "Which Air Force member was behind enemy lines for 11 1/2 days and had the largest,longest and most complex rescue mission?", "answer": ["Lieutenant Colonel Iceal E. \"Gene\" Hambleton"]}, {"question": "Yakuza Kiwami is a remake of the first video game in what video game series that is an open world action-adventure beat 'em up video game franchise?", "answer": ["Yakuza"]}, {"question": "The scunthorpe telegraph and the Grimsby Telegraph used to service Scunthorpe which is in what country?", "answer": ["England"]}, {"question": "John Constantine Williams, Sr was cofounder of a Florida city located in what county?", "answer": ["Pinellas County"]}, {"question": "Which actor in The Lieutenant Wears Skirts is named Samuel?", "answer": ["Tom Ewell"]}, {"question": "When was the magazine Sam Duckworth got his name Get Cape. Wear Cape. Fly from launched?", "answer": ["January 2004"]}, {"question": "What color clothing do people of the Netherlands wear during Oranjegekte or to celebrate the national holiday Koningsdag? ", "answer": ["orange"]}, {"question": "What type of building does Jim Bartels and \u02bbIolani Palace have in common?", "answer": ["museum"]}, {"question": "Were both Life magazine and Strictly Slots magazine published monthly in 1998?", "answer": ["yes"]}, {"question": "What is the focus of the movie in which Nolan North played the role of Superboy?", "answer": ["focus on young superheroes"]}, {"question": "Sony/Philips Digital Interface Format can transmit over what Toshiba optical fiber connector system?", "answer": ["TOSLINK (from \"Toshiba Link\") is a standardized optical fiber connector system."]}, {"question": "The brewing company known for making Genny Light, a beer with only 100 calories per 12 ounce portion, and located along the Genesee River, was formerly known by what brewery name?", "answer": ["High Falls Brewery"]}, {"question": "In which city is the parent company of Sterlite Industries headquartered?", "answer": ["London"]}, {"question": "Edwyn Collins and Ian Gillan both have which profession in common?", "answer": ["singer"]}, {"question": "Which Outer Banks landmark and premiere BBQ restaurant has been featured in the magazine Southern Living?", "answer": ["Pigman's Bar-B-Que"]}, {"question": "Who portrays Rachel Zane on the American legal comedy-drama that airs on USA Network?", "answer": ["Rachel Meghan Markle"]}, {"question": "Ellie Parker is a 2005 American drama-comedy film whose title character is played by an English actress who made her screen debut in what Australian drama film?", "answer": ["For Love Alone"]}, {"question": "Between Parsifal and Saul og David which opera has more acts?", "answer": ["Saul og David"]}, {"question": "A Talk is an EP by which South Korean singer and dancer?", "answer": ["Hyuna"]}, {"question": "What is the name of the American author, songwriter, educator, lawyer, diplomat, civil rights activist and husband to Grace Nail Johnson who was the first African American executive secretary of the National Association for the Advancement of Colored People (NAACP)?", "answer": ["James Weldon Johnson"]}, {"question": "Which is the Oil Capital of China, Chaoyang District, Shantou, or Daqing?", "answer": ["Daqing"]}, {"question": "Over what time period was the motion picture studio which produced Marvel One-Shots known as \"Marvel Films\"?", "answer": ["from 1993 to 1996"]}, {"question": "What Nickelodeon movie stars the actress and YouTuber who appeared alongside Jon Voight in the 2016 film American Wrestler:The Wizard?", "answer": ["Terry the Tomboy"]}, {"question": "At which level of French football does the football team named after the manager of France national team play?", "answer": ["Championnat National 3"]}, {"question": "Kristine Moore Gebbie is a professor at a university founded in what year?", "answer": ["1966"]}, {"question": "A vehicle that uses hydrogen as its onboard fuel for motive power is cosidered which, Flex-fuel or Bi-fuel?", "answer": ["bi-fuel vehicles"]}, {"question": "The Ronkonkama railroad station and the town of Islip, New York are near?", "answer": ["Long Island"]}, {"question": "What career led Brandon James Routh to move to the city where Brian Ralston lives?", "answer": ["an acting career"]}, {"question": "Which of Damon Stoudamire's cousins once played college basketball for the University of Kentucky?", "answer": ["Terrence Jones"]}, {"question": "Where was the founder of the Trussed Concrete Steel Company raised?", "answer": ["Detroit, Michigan"]}, {"question": "Which star of Zork was also the voice of Pac-Man?", "answer": ["Martin Ingerman"]}, {"question": "What country of origin does Jerry Barrett and Florence Nightingale have in common?", "answer": ["English"]}, {"question": "Which composer, who was a crucial figure in the transition between the Classical and Romantic eras in Western art music, wrote Piano Sonata No. 16 ?", "answer": ["Ludwig van Beethoven"]}, {"question": "Which dog breed has more derivative breeds from the original, German Spitz or Norfolk Spaniel?", "answer": ["German Spitz"]}, {"question": "Who created the NBC sitcom that Johnny Pemberton appears in as the character Bo Thompson?", "answer": ["Justin Spitzer"]}, {"question": "Which band My Darkest Days or Human Drama had only one constant member?", "answer": ["Human Drama"]}, {"question": "Which Kentucky Writers Hall of Fame author has had works published by Dim Gray Bar Press?", "answer": ["Wendell Berry"]}, {"question": "Which Australian singer-songwriter wrote Cold Hard Bitch", "answer": ["Nicholas John \"Nic\" Cester"]}, {"question": "What singer who sang in church during her childhood did Stacy Barthe pen album tracks for?", "answer": ["Katy Perry"]}, {"question": "What band formed in 1995 and came out with an album called \"Welcome Interstate Managers'?", "answer": ["Fountains of Wayne"]}, {"question": "Sinch when has the rank held by Sir Charles Comyn Egertong been the highest in the British Army? ", "answer": ["1736"]}, {"question": " Hamilton Evans \"Tony\" James is the COO and director of the largest alternative investment firm in what?", "answer": ["the world"]}, {"question": "Which space mission did the man, in which the Onizuka Prop Wash Award is named after, perish?", "answer": ["STS-51-L"]}, {"question": "The pilot and author of Highest Duty: My Search for What Really Matters was hired by which news station in 2011?", "answer": ["CBS News"]}, {"question": "the first woman to be the chef de mission of an Australian Olympic team won gold medal in which winter Olympics ?", "answer": ["2002 Winter Olympics"]}, {"question": "What award did the actor in Tales from the Crypt win?", "answer": ["BAFTA TV Award Best Actor"]}, {"question": "What russian Marxist revolutionary born in 1875 was part of the Vpered organization?", "answer": ["Anatoly Vasilyevich Lunacharsky"]}, {"question": "Are Villa Paletti and Betrayal at House on the Hill both board games?", "answer": ["yes"]}, {"question": "Which client of Lindka Cierach played Queen Elizabeth II in \"The Queen\"?", "answer": ["Helen Mirren"]}, {"question": "Back Up n da Chevy was an album by the rap group from what city?", "answer": ["Atlanta, Georgia"]}, {"question": "Who was born first, Susan Oliver or W. S. Van Dyke?", "answer": ["Woodbridge Strong Van Dyke II"]}, {"question": "Were Enrique Carreras and Lionel Ngakane both filmakers during the 20th century? ", "answer": ["yes"]}, {"question": "Who has a wider scope of profession, Hrag Vartanian or Hovsep Pushman?", "answer": ["Hrag Vartanian"]}, {"question": "Were Halld\u00f3r Laxness and Timothy Leary from the same country?", "answer": ["no"]}, {"question": "What netflix series, produced by Joe Swanberg, had an actress best known for her role as Vanessa on \"Atlanta\"?", "answer": ["Easy"]}, {"question": "Brad Budde played professionally for a team in the NFL that was founded in 1960 as the Dallas Texans by who?", "answer": ["Lamar Hunt"]}, {"question": "Were Daniel Mann and Todd Solondz both directors?", "answer": ["yes"]}, {"question": "What was the arena, in which the 2017 Brit Awards were held, called during the 2012 Olympics ?", "answer": ["North Greenwich Arena"]}, {"question": "Were Chad Gray and Dickie Peterson both singers?", "answer": ["yes"]}, {"question": "Which of the steering committee was an American broadcast journalist, best known as anchorman for the \"CBS Evening News\" for 19 years?", "answer": ["Walter Cronkite"]}, {"question": "When did the 2005 Ladbrokes.com World Darts Championship begin?", "answer": ["the day after Christmas Day"]}, {"question": "In which county does the United States Navy air base located in which Attack Squadron 174 (VA-174) was based on?", "answer": ["Duval County"]}, {"question": "In recent decades American schools have included much more on ncient times tens of thousands of years ago with the settlement of the Americas by who?", "answer": ["Paleo-Indians"]}, {"question": "Jimmy Garcia lost by unanimous decision to a professional boxer that challenged for what title in 1995?", "answer": ["WBO lightweight title"]}, {"question": "What religion was the martyr to whom the Lady of the Valey statue is dedicated?", "answer": ["Christian"]}, {"question": "How many days did it take the Best New Artist of 2009 to sell 2 million copies of an album in the UK and break a twenty year old record?", "answer": ["29"]}, {"question": "Consider the band formed in 2015 by JYP Entertainment through the reality show \"Sixteen\". How many tracks are on the reissue of their third album?", "answer": ["13"]}, {"question": "\"Let's Get Ready to Rhumble\" is a 1994 song by a music duo from where?", "answer": ["Newcastle upon Tyne, England"]}, {"question": "The founder of the Ali Akbar College of Music was known for his virtuosity in what musical instrument?", "answer": ["the sarod"]}, {"question": "The Lang Labor Party was the group that aligned with the policies of the government that followed what system?", "answer": ["Westminster"]}, {"question": "What is the shared country of ancestry between Art Laboe and Scout Tufankjian?", "answer": ["Armenian"]}, {"question": "What British made dance competition television series franchise did Claudia Albertario appear on?", "answer": ["\"Dancing on Ice\" around the world"]}, {"question": "Are both Viken Babikian and Don Manoukian American ?", "answer": ["yes"]}, {"question": "The host of PokerFace is from where?", "answer": ["Newcastle upon Tyne, England"]}, {"question": "The Prussian General Carl von Clausewitz is associated with what type of realism?", "answer": ["Modern thinkers associated with classical realism are Carl von Clausewitz"]}, {"question": "As former head of Orange County Ironworks, what has his hit TLC show called?", "answer": ["American Chopper"]}, {"question": "On which station did the 2014 South Korean television series, starring the main rapper of boyband SS501, air?", "answer": ["KBS2"]}, {"question": "when was the retired Jamaican sprinter which Richard Thompson is, 0.99 seconds slower than born ?", "answer": ["born 21 August 1986"]}, {"question": "The Sh\u014dgitai were an elite corps of which shogunate?", "answer": ["Tokugawa shogunate"]}, {"question": "In what century did this Native warrior and chief, whose brother Tenskwatawa led the Tippecanoe order of battle, become the primary leader of a large, multi tribal confederacy?", "answer": ["nineteenth"]}, {"question": "Which band had more members, Semisonic or Cake?", "answer": ["Cake"]}, {"question": "Which Russian professional ice hockey player, known as the Magic Man, was the final recipient of the NHL Plus-Minus Award?", "answer": ["Pavel Valerievich Datsyuk"]}, {"question": "Which terrorist group was the cause of the battle in which Mark E. Michell won an award for his actions?", "answer": ["al-Qaeda"]}, {"question": "What ended the career of the German composer of Bunte Blatter?", "answer": ["hand injury"]}, {"question": "The magazine that nominated George Rainsford for their Best Actor award in 2017 comes out every week on what day of the week?", "answer": ["Tuesday"]}, {"question": "In what New York town did Glenn Martin Hughes attend high school?", "answer": ["Mineola, New York"]}, {"question": "Who created and mostly wrote the show that Idris Akuna Elba played Stringer Bell in?", "answer": ["David Simon"]}, {"question": "What British novelist born in November 1898 wrote Surprised by Joy: The Shape of My Early Life?", "answer": ["Clive Staples Lewis"]}, {"question": "What is the population of the city in which the first edition of the South American Youth Olympic Games were held?", "answer": ["more than 10 million"]}, {"question": "What number president was Annie Caputo nominated by to become a member of the Nuclear Regulatory Commission?", "answer": ["45th"]}, {"question": "What was the job of the character Jack Nicholson played in a 1992 French-American biographical crime film directed by Danny DeVito?", "answer": ["Teamsters leader"]}, {"question": "The Hindmarsh Stadium is sponsored by a brewery that sold how many liters of beer in 2013 ?", "answer": ["69.7 million litres"]}, {"question": "Which was secretary to President Richard Nixon: Rose Mary Woods or H. R. Haldeman?", "answer": ["Rose Mary Woods"]}, {"question": "Which American chain of bakery-caf\u00e9 fast casual restaurants sponsored Bill Steers Men's 4-Miler", "answer": ["Panera Bread Company"]}, {"question": "In what year was the Sayrevill, New Jersey rock band that Frankie LaRocka played drums for formed?", "answer": ["1983"]}, {"question": "What is the nationality of the actor who was a leading man of Mich\u00e8le Mercier and born on November 3, 1921?", "answer": ["American"]}, {"question": "What is the birthdate of this Uruguayan former footballer, who management team consists of Charlie Oatway?", "answer": ["15 November 1967"]}, {"question": "About how many people are employed by the company founded by Denise Coates?", "answer": ["3,000"]}, {"question": "Which team featured in both the 2012 and 2011 Cops del Rey Finals ?", "answer": ["Barcelona"]}, {"question": "What is the organized crime syndicate founded and led by Carlos Alberto Rosales Mendoza who is friends with Osiel C\u00e1rdenas Guill\u00e9n?", "answer": ["La Familia Michoacana"]}, {"question": "The 2017\u201318 Wigan Athletic F.C. season will be a year in which the team competes in the league cup known as what for sponsorship reasons?", "answer": ["Carabao Cup"]}, {"question": "Selena Quintanilla-P\u00e9rez's assistant and head of her fan club will be eligible for parole on what date?", "answer": ["March 30, 2025"]}, {"question": "Driven To Perform has a sponsor that includes the electronics store chain from what country?", "answer": ["Canada"]}, {"question": "Live Wire Radio will possibly be considered as a replacement for a variety show created in what year?", "answer": ["1974"]}, {"question": "What year was the early consumer co-operative, in which a 2012 British biographical feature film tells the story of, formed?", "answer": ["1844"]}, {"question": "In which University is Richard Williams' former assistant a member of the Athletics Hall of Fame?", "answer": ["Campbellsville University"]}, {"question": "What is the English translation of the name of the novel series which the Four III is the final installment of?", "answer": ["\"The Four Great Constables\""]}, {"question": "What American corporation owns Claim Jumper?", "answer": ["Landry's"]}, {"question": "Is Xingcheng or Ulan Hot located in the Inner Mongolia region of China?", "answer": ["Ulanhot"]}, {"question": "Spring Waltz starred the American actor who is known for what 2007 film?", "answer": ["My Father"]}, {"question": "On what date will the comedy film directed by Kevin Tent and starring the Bulgarian-Canadian actress known for portraying the role of Mia Jones be released on video on demand services?", "answer": ["September 25, 2017"]}, {"question": "Was Steve Sekely and Willian King Baggot known by multiple names or nicknames?", "answer": ["yes"]}, {"question": "Which aired first, Life After People or Werner Herzog Eats His Shoe?", "answer": ["Werner Herzog Eats His Shoe"]}, {"question": "What professional do both Marc Bolan and Ian MacKaye share?", "answer": ["musician"]}, {"question": "Which bi-weekly publication ranked Schreiber Foods as number 81 in 2016.", "answer": ["Forbes ranked Schreiber Foods"]}, {"question": "Which member of the research project conducted by the Creation Research Society and the Institute for Creation Research between 1997 and 2005 was Kevin R. Henke most critical of? ", "answer": ["Russell Humphreys"]}, {"question": "What location under Charing Cross railway station did G-A-Y move to?", "answer": ["venue Heaven"]}, {"question": "What county is Ron Teachworth from?", "answer": ["Oakland County"]}, {"question": "What actor from Northern Ireland born in 1983 starred in Whole Lotta Sole?", "answer": ["Martin \"Marty\" McCann"]}, {"question": "What were the nations after 1722 that composed a culture that was extensively studied by William N. Fenton?", "answer": ["Mohawk, Onondaga, Oneida, Cayuga, Seneca, and Tuscarora"]}, {"question": "Both Alexander Hall and Pierre Morel are involved in which profession?", "answer": ["film director"]}, {"question": "Piers Bizony has written articles for which magazine based in Bristol, UK?", "answer": ["BBC Focus"]}, {"question": "Are Ranger Rick and Tennis both sports magazines?", "answer": ["no"]}, {"question": "Which satellite suburb of Melbourne is partially included in the Electoral district of Bass?", "answer": ["Pakenham"]}, {"question": "Were Dee Dee Ramone and Ben Weasel ever in the same band?", "answer": ["no"]}, {"question": "Who starred in the 1991 American action comedy film The Hard Way and was featured on the roster of Rawkus Records?", "answer": ["Mos Def"]}, {"question": "Brad Elterman is a professional photographer who addressed the rock 'n' roll lifestyle of a band that became noted for what?", "answer": ["its eclectic mix of musical styles"]}, {"question": "A Big Daddy is a fictional character in a series that has sold more than how many copies? ", "answer": ["25 million copies"]}, {"question": "J.B. Scotts located on this 11 mile stretch in Albany New York and is also known by this name in Schenectady County?", "answer": ["State Street"]}, {"question": "Which bay in the Irish Sea is located Wales's first offshore wind farm?", "answer": ["Liverpool Bay"]}, {"question": "Who does the minister at The First Church of Springfield help make teaching evolution illegal?", "answer": ["Ned Flanders"]}, {"question": "The actress who played Laurie Keller in the episode \"Pilot\" of the show Cougartown won what award in 2011?", "answer": ["Critics' Choice Television Award for Best Supporting Actress in a Comedy Series"]}, {"question": "Are both Yangzhou and Jiangyan District considered coastal cities?", "answer": ["no"]}, {"question": "What was held in 2011 at the same stadium where the seventeenth edition of the IAAF World Championships is scheduled to be held in 2019?", "answer": ["The final of 2011 AFC Asian Cup"]}, {"question": "Where is the author of Killing Pablo: The Hunt for the World's Greatest Outlaw (2001) born?", "answer": ["St. Louis, Missouri"]}, {"question": "What Cantonese slang term can mean both \"ghost man\" and to refer to Westerners?", "answer": ["Gweilo"]}, {"question": "TMNT is a single-player action-adventure multi-platform action game based on a film which was the final film role for who?", "answer": ["Mako"]}, {"question": "Which university offers more campuses in the Philippines, Fairleigh Dickinson University or the Western Institute of Technology?", "answer": ["Western Institute of Technology"]}, {"question": "Michael Lent is best known as the co-writer of the memoirs of Hugh Rowland, a star on a reality television series that premiered on what date?", "answer": ["June 17, 2007"]}, {"question": "What Happens in Vegas, an American comedy film starring Lake Bell, was released in what year?", "answer": ["2008"]}, {"question": "Do both Adventist World and Girls' Life have a headquarter or editor based in Maryland?", "answer": ["yes"]}, {"question": "Which musician who first became famous in the late 1960s and early 1970s is Inner Secrets the ninth studio album by", "answer": ["Carlos Santana"]}, {"question": "Who was the head of the Imperial Family that was making an broadcast announcement of Japan's surrender in 1945?", "answer": ["Emperor of Japan"]}, {"question": "Which Lithuanian-Canadian anthropologist, primatologist, conservationist, ethologist, and author was chosen by Louis Leakey, along with Jane Goodall, Dian Fossey to study hominids in their natural environments?", "answer": ["Birut\u0117 Marija Filomena Galdikas"]}, {"question": "The Government Digital Service is overseen by the Chief Secretary to the Treasury, who is from which party?", "answer": ["Conservative"]}, {"question": "Which documentary was filmed first, Almost Sunrise or Hail! Hail! Rock 'n' Roll?", "answer": ["Hail! Hail! Rock 'n' Roll"]}, {"question": "Which battle took place first out of the Battle of the Ch'ongch'on River and the Battle of Tarawa?", "answer": ["The Battle of Tarawa"]}, {"question": "Back Home is a children's historical novel adapted into a TV drama starring what English actress born in 1946?", "answer": ["Hayley Catherine Rose Vivien Mills"]}, {"question": " \"I Will Not Say Goodbye\" is a song written in part by a music artist who first gained national attention as a winner of what talent competition?", "answer": ["You Can Be a Star"]}, {"question": "In between Thomas Anders and Micky Dolenz, who was the a vocalist of the Monkees?", "answer": ["George Michael Dolenz"]}, {"question": "Which is a university in Pennsylvania, University of New Haven or Carnegie Mellon University?", "answer": ["Carnegie Mellon University"]}, {"question": "Matt Doheny lost his primary race to what current youngest woman ever elected to Congress?", "answer": ["Elise Stefanik"]}, {"question": "Which composer created Night Ferry first performed in 2012 at Symphony Center in Chicago?", "answer": ["Anna Clyne"]}, {"question": "What are a Uruguayan professional footballer and Luz Pfannenstiel are best known as?", "answer": ["the best known globetrotters"]}, {"question": "What shallow water table aquifer is located near the county seat of Keith County, Nebraska?", "answer": ["Ogallala Aquifer"]}, {"question": "What award won by only twelve people has a man who Ted Kooshian has performed with won?", "answer": ["EGOT"]}, {"question": "The 53rd National Hockey League All-Star Game took place at the indoor arena that was completed in what year?", "answer": ["1998"]}, {"question": "Which legal, autonomous North American tribal government signed its constitution in Oklahoma on September 6, 1839?", "answer": ["The Cherokee Nation"]}, {"question": "Who was the winner of the season of The Rap Game where a contestant named Lady Leshurr as one of her influences?", "answer": ["Mani"]}, {"question": "What was the birth name of the start of Hwarang: The Poet Warrior Youth?", "answer": ["Park Yong-gyu"]}, {"question": "The tap dancing soloist in My Feet Keep Dancing also starred in what 1935 MGM musical?", "answer": ["\"An All-Colored Vaudeville Show\""]}, {"question": "What was the capacity of the stadium that Colecci\u00f3n Privada that was filmed on October 10, 2000?", "answer": ["15,500"]}, {"question": "What year did the musical premiere that was based on the 2007 Irish romantic musical film of the same name?", "answer": ["2011"]}, {"question": " The Nike Hoop Summit has had many current NBA players as former participants, including what German player from the Dallas Mavericks?", "answer": ["Dirk Werner Nowitzki"]}, {"question": "What lawyers were featured in The State of Marriage, with the lawyer nominated by Governor Peter Shumlin?", "answer": ["Mary Bonauto, Susan Murray"]}, {"question": "What building is opposite the ceremonial meeting place of the Accession Council in the United Kingdom?", "answer": ["Mark Masons' Hall"]}, {"question": "Gerd Neggo trained under the founder of which type of dance analysis?", "answer": ["Laban Movement Analysis"]}, {"question": "Do both Icehouse pieces and El Grande come from the same game?", "answer": ["no"]}, {"question": "Falling Stars is a role-playing game published by the company based out of what country?", "answer": ["Lithuania"]}, {"question": "What country of origin does Shawnee Trail and Great Indian Warpath have in common?", "answer": ["American"]}, {"question": "When was the designer of the Disneyland attraction with variants in California, France, Hong Kong, Tokyo, and the Tomorrowland Speedway born?", "answer": ["born October 25, 1931"]}, {"question": "Why does the City of Boston get their christmas tree from Ainslee Glen Nova Scotia?", "answer": ["for their assistance after the 1917 Halifax Explosion"]}, {"question": "Which refers to more breeds, Stabyhoun or Griffon Bruxellois?", "answer": ["Griffon Bruxellois"]}, {"question": "Marvel Classics Comics specialized in adaptations of literary classics, such as a novel by American writer Herman Melville, that was published in what year?", "answer": ["1851"]}, {"question": "In what year was the novel that Louren\u00e7o Mutarelli based \"Nina\" on based first published?", "answer": ["1866"]}, {"question": "Robert Jan van Pelt testified for the defense in this \"Churchill's War\" author's civil libel suit?", "answer": ["David Irving"]}, {"question": "Which is founded by a president, University of Washington or University of Virginia?", "answer": ["University of Virginia"]}, {"question": "Who co-starred in an episode of The ABC Comedy Hour with the man who played Duke Slater in \"Gomer Pyle, U.S.M.C.\"? ", "answer": ["Jack Cassidy"]}, {"question": "What Nobel Laureate teaches at Chapman University?", "answer": ["Vernon Smith"]}, {"question": "Are Floor Jansen and John McCrea from the same band?", "answer": ["no"]}, {"question": "How far from Buenos Aires is the birthplace of Ezequiel Lazaro? ", "answer": ["about 700 km"]}, {"question": "Many right wing WNP were connected to a clandestine operation ran by a \"stay-behind\" operation ran by what organization?", "answer": ["NATO"]}, {"question": "\"Ew!\" is a song by a television host born where?", "answer": ["Bay Ridge, Brooklyn"]}, {"question": "Snoop Dogg's Doggystyle and Snoop Dogg's Hustlaz: Diary of a Pimp are hip-hop music videos that feature what film genre?", "answer": ["hardcore pornography"]}, {"question": "In what city did the 23rd overall pick of the 2015 NHL Entry Draft helped the United States national junior team win a bronze medal?", "answer": ["Helsinki, Finland"]}, {"question": "What movie was the actor born on May 4, 1946 involved in in 1972?", "answer": ["Five Summer Stories"]}, {"question": "Party of Syrian Unity was established in the wake of a public statement that announced what ?", "answer": ["support for the establishment of a \"national home for the Jewish people\" in Palestine"]}, {"question": "Bobby Arora is the trading director of a retail chain that was formed in what year?", "answer": ["1978"]}, {"question": "What public radio network in New York has allowed Eric Plakun to dis his psychiatric work?", "answer": ["WAMC"]}, {"question": "Ckm wz. 25 Hotchkiss was a Polish derivative of what machine gun that became the standard machine gun of the French Army during World War I?", "answer": ["Mle 1914 Hotchkiss machine gun"]}] -------------------------------------------------------------------------------- /prompts/crud/quest_eval_answer.txt: -------------------------------------------------------------------------------- 1 | 你是一位新闻编辑，现在，你被提供了1篇新闻的摘要，和几个问题，请分别根据新闻的摘要回答这些问题。用list的格式输出答案。以下是个例子： 2 | 3 | 新闻摘要: 4 | 5 | 2023年7月27日，42位来自9个国家的华裔青年在云南腾冲的普洱茶研学基地参观学习，体验了采茶、压茶、包茶饼等传统制茶工艺，感受到浓厚的普洱茶历史文化。活动由云南省人民政府侨务办公室主办。腾冲市以其悠久的种茶历史和丰富的茶文化，吸引了众多海外华裔青年的参与。 6 | 7 | 问题： 8 | 9 | ["参观学习的华裔青年共有多少人？","这些华裔青年来自几个国家？","华裔青年们的领队是谁？","这次活动是由谁主办的？","腾冲市吸引海外华裔青年的特点是什么？"], 10 | 11 | 回答: 12 | 13 | 参观学习的华裔青年共有多少人？ 14 | 15 | 42人 16 | 17 | 18 | 这些华裔青年来自几个国家？ 19 | 20 | 9个 21 | 22 | 23 | 华裔青年们的领队是谁？ 24 | 25 | 无法推断 26 | 27 | 28 | 这次活动是由谁主办的？ 29 | 30 | 云南省人民政府侨务办公室 31 | 32 | 33 | 腾冲市吸引海外华裔青年的特点是什么？ 34 | 35 | 悠久的种茶历史和丰富的茶文化 36 | 37 | 38 | 39 | 现在新闻的摘要是： 40 | 41 | {context} 42 | 43 | 问题： 44 | 45 | {questions} 46 | 47 | 请给出根据新闻的摘要的回答（回答的文本**必须**写在之间。请注意，用一两个词或者非常简短的语句回答问题，不要添加多余的词。遇见无法回答的问题，请说：“无法推断”。） 48 | -------------------------------------------------------------------------------- /prompts/en/base: -------------------------------------------------------------------------------- 1 | Answer the question based on the given passages. Only give me the answer and do not output any other words. 2 | 3 | The following are given passages: 4 | {refs} 5 | 6 | Question: {query} 7 | Answer: -------------------------------------------------------------------------------- /prompts/en/base_asqa: -------------------------------------------------------------------------------- 1 | Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided passages. Use an unbiased and journalistic tone. 2 | 3 | 4 | Question: 5 | {query} 6 | 7 | 8 | Passages: 9 | {refs} 10 | 11 | -------------------------------------------------------------------------------- /prompts/en/base_wo_retri: -------------------------------------------------------------------------------- 1 | Only give me the answer and do not output any other words. 2 | 3 | Question: {query} 4 | Answer: -------------------------------------------------------------------------------- /prompts/en/base_wo_retri_asqa: -------------------------------------------------------------------------------- 1 | Instruction: Write an accurate, engaging, and concise answer for the given question. Use an unbiased and journalistic tone. 2 | 3 | Question: 4 | {query} 5 | -------------------------------------------------------------------------------- /prompts/en/chain-of-note: -------------------------------------------------------------------------------- 1 | Task Description: 2 | 1. Read the given question and a few passages to gather relevant information. 3 | 2. Write reading notes summarizing the key points from these passages. 4 | 3. Discuss the relevance of the given question to these passages. 5 | 4. If some passages are relevant to the given question, provide a brief answer based on the passages. 6 | 5. If no passage is relevant, directly provide the answer without considering the passages. 7 | Question: {query} 8 | Passage: {refs} -------------------------------------------------------------------------------- /prompts/en/compare: -------------------------------------------------------------------------------- 1 | Task: Please help me determine which note is better based on the following evaluation criteria: 2 | 1. Contains key information directly related to the question. 3 | 2. Completeness of Information: Does it cover all relevant aspects and details? 4 | 3. Level of Detail: Does it provide enough detail to understand the issue in depth? 5 | 4. Practicality: Does the note offer practical help and solutions? 6 | 7 | Please make your judgment adhering strictly to the following rules: 8 | - If Note 2 does not add new meaningful content on top of Note 1, or only adds redundant information, return {{"status":"False"}} directly. 9 | - If Note 2 has significant improvements over Note 1 based on the above criteria, return {{"status":"True"}} directly; otherwise, return {{"status":"False"}}. 10 | 11 | Question: {query} 12 | Provided Note 1: {best_note} 13 | Provided Note 2: {new_note} 14 | 15 | Based on the above information, make your judgment without explanation and return the result directly. 16 | -------------------------------------------------------------------------------- /prompts/en/first_gen_answer: -------------------------------------------------------------------------------- 1 | Based on the provided document content, write a note. The note should integrate all relevant information from the original text that can help answer the specified question and form a coherent paragraph. Please ensure that the note includes all original text information useful for answering the question. 2 | 3 | 4 | Question to be answered: {query} 5 | Document content: {refs} 6 | 7 | 8 | Please provide the note you wrote: -------------------------------------------------------------------------------- /prompts/en/gen_answer: -------------------------------------------------------------------------------- 1 | Answer the question based on the given note. Only give me the answer and do not output any other words. 2 | 3 | The following are given notes: 4 | {note} 5 | 6 | Question: {query} 7 | Answer: -------------------------------------------------------------------------------- /prompts/en/gen_answer_asqa: -------------------------------------------------------------------------------- 1 | Instruction: Write an accurate, engaging, and concise answer for the given question using only the provided notes. Use an unbiased and journalistic tone. 2 | 3 | 4 | Question: 5 | {query} 6 | 7 | 8 | Notes: 9 | {note} -------------------------------------------------------------------------------- /prompts/en/gen_new_query: -------------------------------------------------------------------------------- 1 | Task: Based on the notes, propose two new questions. These new questions will be used to retrieve documents to supplement the notes and help answer the original question. The new questions should be concise and include keywords that facilitate retrieval. The new questions should avoid duplication with the existing question list. 2 | 3 | original question: {query} 4 | notes: {note} 5 | 6 | existing question list: {query_log} 7 | 8 | Provide your two new questions: -------------------------------------------------------------------------------- /prompts/en/refine_note: -------------------------------------------------------------------------------- 1 | Task: Based on the retrieved documents, supplement the notes with content not yet included but useful for answering the question. The supplement should use the original text from the retrieved documents. The added content should include as much information from the retrieved documents as possible. 2 | 3 | question: {query} 4 | retrieved documents: {refs} 5 | 6 | notes: {note} 7 | 8 | Provide your supplemented notes: -------------------------------------------------------------------------------- /prompts/en/self_refine: -------------------------------------------------------------------------------- 1 | Complete the following task: 2 | The questions that need to be answered are: {query} 3 | To answer these questions, I retrieved some passages. 4 | Passages: {refs} 5 | Task: Based on the content of the questions to be answered, 6 | Please extract relevant and useful information from these passages then provide a summary and analysis of these passages. -------------------------------------------------------------------------------- /prompts/en/self_rerank: -------------------------------------------------------------------------------- 1 | Complete the following task: 2 | The questions that need to be answered are: {query} 3 | To answer these questions, I retrieved some passages. 4 | Passages: {refs} 5 | Based on the content of the questions to be answered, Please label each passage with two tags and give the reason, the first one is whether it is useful or not, and the second one is whether it is relevant or not, 6 | The four possible scenarios are as follows: 7 | 1., 2., 3., 4.. 8 | Generation Format: 9 | 1. passage:..., label:..., reason:... -------------------------------------------------------------------------------- /prompts/zh/base: -------------------------------------------------------------------------------- 1 | 你是一位新闻编辑，现在，你被提供了1个问题，和根据这些问题检索到的文档，请分别检索内容和你自身的知识回答这些问题。 2 | 问题：{query} 3 | 4 | 检索到的文档：{refs} 5 | 6 | 请给出你的回答： -------------------------------------------------------------------------------- /prompts/zh/base_wo_retri: -------------------------------------------------------------------------------- 1 | 你是一位新闻编辑，现在，你被提供了1个问题。 2 | 问题：{query} 3 | 4 | 请给出你的回答： -------------------------------------------------------------------------------- /prompts/zh/chain-of-note: -------------------------------------------------------------------------------- 1 | 任务描述： 2 | 1. 阅读给定的问题和和段落，收集相关信息。 3 | 2. 写阅读笔记，总结这些段落的要点。 4 | 3. 讨论给定问题与这些段落的相关性。 5 | 4. 如果某些段落与给定的问题相关，请根据这些段落给出答案。 6 | 5. 如果没有相关段落，不考虑段落，直接给出答案。 7 | 问题:{query} 8 | 段落:{refs} -------------------------------------------------------------------------------- /prompts/zh/compare: -------------------------------------------------------------------------------- 1 | 任务：请你帮我判断提供的哪个笔记更好，具体评价标准如下： 2 | 1. 包含与问题直接相关的关键信息。 3 | 2. 信息的全面性：是否涵盖了所有相关的方面和细节。 4 | 3. 信息的细节程度：是否提供了足够的细节来深入理解问题。 5 | 4. 实用性：笔记是否提供了实际的帮助和解决方案。 6 | 7 | 请严格按照以下要求进行判断： 8 | - 如果笔记2没有在笔记1的基础上增加新的有意义的内容，或者只是增加了多余的信息，请直接返回 {{"status":"False"}}。 9 | - 如果笔记2比笔记1在上述标准上有明显改进，请直接返回 {{"status":"True"}}，否则请直接返回 {{"status":"False"}}。 10 | 11 | 问题：{query} 12 | 提供的笔记1：{best_note} 13 | 提供的笔记2：{new_note} 14 | 15 | 16 | 请根据以上信息进行判断，不要解释，直接返回结果。 -------------------------------------------------------------------------------- /prompts/zh/first_gen_answer: -------------------------------------------------------------------------------- 1 | 根据以下提供的文档内容撰写一篇笔记。笔记应整合所有能够帮助回答指定问题的相关原文信息，并形成一段连贯的文本。请确保笔记包含所有对回答问题有用的原文信息。 2 | 3 | 需要回答的问题： {query} 4 | 文档内容： {refs} 5 | 6 | 请提供你撰写的笔记： -------------------------------------------------------------------------------- /prompts/zh/gen_answer: -------------------------------------------------------------------------------- 1 | 你是一位新闻编辑，现在，你被提供了1个问题，和与问题相关的笔记。请你结合笔记和你自身的知识回答这些问题。 2 | 问题：{query} 3 | 4 | 与问题相关的笔记：{note} 5 | 6 | 请给出你的回答： -------------------------------------------------------------------------------- /prompts/zh/gen_answer_raw: -------------------------------------------------------------------------------- 1 | 你是一位新闻编辑，请结合笔记和你自身的知识回答这个问题。 2 | 3 | 问题：{query} 4 | 笔记：{note} 5 | 6 | 提供您的答案： -------------------------------------------------------------------------------- /prompts/zh/gen_new_query: -------------------------------------------------------------------------------- 1 | 任务：根据笔记，提出两个新问题。这些新问题将用于检索文档，以补充笔记并帮助回答原始问题。新问题应简明扼要，并包括有利于检索的关键词。新问题应避免与现有问题列表重复。 2 | 3 | 原始问题：{query} 4 | 笔记：{note} 5 | 6 | 现有问题列表：{query_log} 7 | 8 | 提供两个新问题： -------------------------------------------------------------------------------- /prompts/zh/refine_note: -------------------------------------------------------------------------------- 1 | 任务：根据检索到的文档，用尚未包含但对回答问题有用的内容补充笔记。补充内容应使用检索到的文档中的原始文本，并尽可能多的包含检索到的文档的信息。 2 | 3 | 问题：{query} 4 | 检索到的文档：{refs} 5 | 6 | 笔记：{note} 7 | 8 | 提供补充后的笔记： -------------------------------------------------------------------------------- /prompts/zh/self_refine: -------------------------------------------------------------------------------- 1 | 完成以下任务: 2 | 需要回答的问题是:{query} 3 | 为了回答这些问题，我检索了一些段落。 4 | 段落:{refs} 5 | 任务:根据待回答问题的内容， 6 | 请从这些段落中提取相关和有用的信息，然后对这些信息进行总结和分析这些段落 -------------------------------------------------------------------------------- /prompts/zh/self_rerank: -------------------------------------------------------------------------------- 1 | 完成以下任务: 2 | 需要回答的问题是:{query} 3 | 为了回答这些问题，我检索了一些段落。 4 | 段落:{refs} 5 | 根据所要回答的问题的内容，给每一篇文章打上两个标签，并给出原因，第一个是有用与否，第二个是相关与否， 6 | 四种可能的情况如下: 7 | 1.<有用的> <有关>，2.<无用的><相关> ，3.<有用的><无关> ，4.<无用的><无关>。 8 | 生成的格式: 9 | 1. 段落:……，标签:……，原因是:…… -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 |

2 | Retriever-and-Memory: Towards Adaptive Note-Enhanced Retrieval-Augmented Generation 3 |

4 | 5 | 6 |
7 | 8 | [![Python 3.10](https://img.shields.io/badge/python-%E2%89%A53.10-blue)](https://www.python.org/downloads/release/python-3109/) 9 | [![Arxiv](https://img.shields.io/badge/arXiv-2408.01262-red)](https://arxiv.org/abs/2410.08821) 10 | 11 |
12 | 13 | We propose a generic RAG approach called Adaptive Note- 14 | Enhanced RAG (Adaptive-Note) for complex QA tasks, 15 | which includes the iterative information collector, adaptive 16 | memory reviewer, and task-oriented generator, while fol- 17 | lowing a new Retriever-and-Memory paradigm. 18 | 19 | ![RNote](assets/Adaptive-Note.png) 20 | 21 | # Prepare Datasets 22 | 23 | Please create a `data/` directory and place all the corpus and evaluation files within it. All experimental datasets can be found [here](https://drive.google.com/drive/folders/11EGpumXoMVpnBsiR5xeLGDD3Z8ikBdVT?usp=drive_link). For the ASQA corpus, please download it separately from [this link](https://dl.fbaipublicfiles.com/dpr/wikipedia_split/psgs_w100.tsv.gz) due to its large size. 24 | 25 | # Retrieval Services 26 | 27 | We offer three retrieval services as detailed in our paper: 28 | 29 | 1. **BM25 Retrieval Service** using ElasticSearch for 2WikiMQA, MuSiQue, and HotpotQA. 30 | 2. **BGE Retrieval Service** using FAISS for CRUD. 31 | 3. **GTR Retrieval Service** using FAISS for ASQA. 32 | 33 | ## Prepare Services for BM25 34 | 35 | First, install Elasticsearch 7.10 using the following commands: 36 | 37 | ```bash 38 | wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.10.2-linux-x86_64.tar.gz 39 | wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.10.2-linux-x86_64.tar.gz.sha512 40 | shasum -a 512 -c elasticsearch-7.10.2-linux-x86_64.tar.gz.sha512 41 | tar -xzf elasticsearch-7.10.2-linux-x86_64.tar.gz 42 | cd elasticsearch-7.10.2/ 43 | ./bin/elasticsearch # Start the server 44 | pkill -f elasticsearch # To stop the server 45 | ``` 46 | 47 | ## Indexing Corpora for the Datasets 48 | 49 | ### Build indices for 2WikiMQA, MuSiQue, and HotpotQA: 50 | 51 | ```bash 52 | cd src/build_index/es 53 | 54 | # 2WikiMQA 55 | python index_2wiki.py 56 | 57 | # MuSiQue 58 | python index_musique.py 59 | 60 | # HotpotQA 61 | python index_hotpotqa.py 62 | ``` 63 | 64 | ### ASQA Indexing 65 | 66 | Our corpus and retrieval settings follow [ALCE](https://github.com/princeton-nlp/ALCE.git). Since generating GTR embeddings is resource-intensive, you can download them and place them in `data/corpus/asqa/` as follows: 67 | 68 | ```bash 69 | wget https://huggingface.co/datasets/princeton-nlp/gtr-t5-xxl-wikipedia-psgs_w100-index/resolve/main/gtr_wikipedia_index.pkl 70 | ``` 71 | 72 | Build the index: 73 | ```bash 74 | cd src/build_index/emb 75 | python index.py --dataset asqa --model gtr-t5-xxl 76 | ``` 77 | 78 | ### CRUD Indexing 79 | 80 | Build the index for CRUD: 81 | ```bash 82 | cd src/build_index/emb 83 | python index.py --dataset crud --model bge-base-zh-v1.5 --chunk_size 512 84 | ``` 85 | 86 | # Configuration 87 | 88 | You can configure your API key, URL, and other settings in the `./config/config.yaml` file. 89 | 90 | # Running Note and Evaluation 91 | 92 | To run and evaluate for 2WikiMQA, MuSiQue, HotpotQA, and ASQA: 93 | ```bash 94 | python main.py --method note --retrieve_top_k 5 --dataset asqa --max_step 3 --max_fail_step 1 --MaxClients 10 --model gpt-3.5-turbo-0125 --device cuda:0 95 | ``` 96 | The predicted results and evaluation metrics (F1 and EM) will be automatically saved in the `output/{dataset}/{method}/{model}` directory. The evaluation results can be found at the end of the file. 97 | 98 | For CRUD (crud_1doc, crud_2doc, crud_3doc): 99 | ```bash 100 | python main.py --method note --retrieve_top_k 2 --dataset crud_1doc --max_step 3 --max_fail_step 1 --MaxClients 10 --model gpt-3.5-turbo-0125 --device cuda:0 101 | ``` 102 | The predicted results will be automatically saved in the `output/{dataset}/{method}/{model}` directory. 103 | 104 | ## Evaluating CRUD 105 | 106 | We follow [CRUD-RAG](https://github.com/IAAR-Shanghai/CRUD_RAG.git) and use the RAGQuestEval metric, which relies on GPT. Run the following code to perform the evaluation: 107 | 108 | ```bash 109 | python metrics_questeval_crud.py --eval_path {saved predict file} 110 | ``` 111 | 112 | The evaluation results will be automatically saved in the `output/{dataset}/{method}/{model}/metric_questeval` directory. 113 | 114 | 115 | ## Citation 116 | Please cite the following paper if you find Adaptive-Note helpful! 117 | ```bibtex 118 | @misc{wang2024retrieverandmemoryadaptivenoteenhancedretrievalaugmented, 119 | title={Retriever-and-Memory: Towards Adaptive Note-Enhanced Retrieval-Augmented Generation}, 120 | author={Ruobing Wang and Daren Zha and Shi Yu and Qingfei Zhao and Yuxuan Chen and Yixuan Wang and Shuo Wang and Yukun Yan and Zhenghao Liu and Xu Han and Zhiyuan Liu and Maosong Sun}, 121 | year={2024}, 122 | eprint={2410.08821}, 123 | archivePrefix={arXiv}, 124 | primaryClass={cs.CL}, 125 | url={https://arxiv.org/abs/2410.08821}, 126 | } 127 | ``` 128 | 129 | 130 |

131 | 132 | 133 | 134 | 135 | Star History Chart 136 | 137 | 138 |

-------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | backoff 2 | elasticsearch 3 | faiss_cpu 4 | jieba 5 | loguru 6 | numpy 7 | openai 8 | pandas 9 | sentence_transformers 10 | torch 11 | tqdm 12 | transformers 13 | pyyaml 14 | -------------------------------------------------------------------------------- /src/build_index/emb/index.py: -------------------------------------------------------------------------------- 1 | import time 2 | import os 3 | import json 4 | from llama_index import SimpleDirectoryReader 5 | from llama_index.node_parser import SimpleNodeParser 6 | import csv 7 | from sentence_transformers import SentenceTransformer 8 | import faiss 9 | import argparse 10 | import yaml 11 | 12 | with open("../../../config/config.yaml", "r") as f: 13 | config = yaml.safe_load(f) 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument( 16 | "--model", 17 | type=str, 18 | choices=["bge-base-zh-v1.5", "gtr-t5-xxl"], 19 | default="bge-base-zh-v1.5", 20 | help="Model to use", 21 | ) 22 | parser.add_argument( 23 | "--dataset", 24 | type=str, 25 | default="crud", 26 | choices=["crud", "asqa"], 27 | help="Dataset to use", 28 | ) 29 | parser.add_argument("--chunk_size", type=int, default=512, help="chunk size") 30 | parser.add_argument("--chunk_overlap", type=int, default=0, help="chunk overlap") 31 | parser.add_argument("--device", type=str, default="cuda", help="Device to use") 32 | args = parser.parse_args() 33 | 34 | 35 | def split_text(docs_path): 36 | documents = SimpleDirectoryReader(docs_path).load_data() 37 | 38 | node_parser = SimpleNodeParser.from_defaults( 39 | chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap) 40 | nodes = node_parser.get_nodes_from_documents(documents, show_progress=True) 41 | contens=[d.text for d in nodes] 42 | return contens 43 | 44 | 45 | def build_index(embeddings, vectorstore_path): 46 | dimension = embeddings.shape[1] 47 | index = faiss.IndexFlatIP(dimension) 48 | index.add(embeddings) 49 | faiss.write_index(index, vectorstore_path) 50 | 51 | 52 | if __name__ == "__main__": 53 | 54 | model = SentenceTransformer(config["model"][args.model], device=args.device) 55 | vectorstore_path = f"../../../data/corpus/{args.dataset}/{args.dataset}.index" 56 | print("loading document ...") 57 | start = time.time() 58 | if args.dataset == "asqa": 59 | if os.path.exists("../../../data/corpus/asqa/gtr_wikipedia_index.pkl"): 60 | import pickle 61 | 62 | with open( 63 | "../../../data/corpus/asqa/gtr_wikipedia_index.pkl", "rb" 64 | ) as file: 65 | embeddings = pickle.load(file) 66 | else: 67 | with open( 68 | "../../../data/corpus/asqa/psgs_w100.tsv", "r", encoding="utf-8" 69 | ) as file: 70 | tsv_data = csv.DictReader(file, delimiter="\t") 71 | raw_data = [row["title"] + "\n" + row["text"] for row in tsv_data] 72 | print("dataset length", len(raw_data)) 73 | embeddings = model.encode(raw_data, batch_size=100) 74 | elif args.dataset == "crud": 75 | contents=split_text("../../../data/corpus/crud/80000_docs") 76 | contents = list(set(contents)) 77 | print("Generating embeddings ...") 78 | embeddings = model.encode(contents, batch_size=800) 79 | print("Building index ...") 80 | build_index(embeddings, vectorstore_path) 81 | end = time.time() 82 | with open( 83 | f"../../../data/corpus/{args.dataset}/chunk.json", "w", encoding="utf-8" 84 | ) as fout: 85 | json.dump(contents, fout, ensure_ascii=False) 86 | -------------------------------------------------------------------------------- /src/build_index/es/index_2wiki.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from elasticsearch import Elasticsearch 3 | import html 4 | import json 5 | from tqdm import tqdm 6 | from itertools import chain 7 | 8 | 9 | def chunks(l, n): 10 | """Yield successive n-sized chunks from l.""" 11 | for i in range(0, len(l), n): 12 | yield l[i : i + n] 13 | 14 | 15 | INDEX_NAME = "2wikimultihopqa" 16 | 17 | 18 | def process_line(data): 19 | item = { 20 | "id": data["id"], 21 | "url": "empty", 22 | "title": data["title"], 23 | "title_unescape": html.unescape(data["title"]), 24 | "text": data["text"], 25 | "title_bigram": html.unescape(data["title"]), 26 | "title_unescape_bigram": html.unescape(data["title"]), 27 | "text_bigram": data["text"], 28 | "original_json": json.dumps(data), 29 | } 30 | return "{}\n{}".format( 31 | json.dumps({"index": {"_id": "wiki-{}".format(data["id"])}}), json.dumps(item) 32 | ) 33 | 34 | 35 | es = Elasticsearch(hosts="http://localhost:9200", timeout=100) 36 | 37 | 38 | def index_chunk(chunk): 39 | res = es.bulk(index=INDEX_NAME, body="\n".join(chunk), timeout="100s") 40 | assert not res["errors"], res 41 | 42 | 43 | def main(args): 44 | 45 | train = json.load(open("../../../data/corpus/2wikimultihopqa/train.json", "r")) 46 | dev = json.load(open("../../../data/corpus/2wikimultihopqa/dev.json", "r")) 47 | test = json.load(open("../../../data/corpus/2wikimultihopqa/test.json", "r")) 48 | 49 | data = {} 50 | for item in tqdm(chain(train, dev, test)): 51 | for title, sentences in item["context"]: 52 | para = " ".join(sentences) 53 | data[para] = title 54 | wikipedia_data = [ 55 | {"id": i, "text": text, "title": title} 56 | for i, (text, title) in enumerate(data.items()) 57 | ] 58 | 59 | # make index 60 | if not args.dry: 61 | # add 62 | if es.indices.exists(index=INDEX_NAME): 63 | es.indices.delete(index=INDEX_NAME, ignore=[400, 403]) 64 | es.indices.create( 65 | index=INDEX_NAME, 66 | ignore=400, 67 | body=json.dumps( 68 | { 69 | "mappings": { 70 | "doc": { 71 | "properties": { 72 | "id": {"type": "keyword"}, 73 | "url": {"type": "keyword"}, 74 | "title": { 75 | "type": "text", 76 | "analyzer": "simple", 77 | "copy_to": "title_all", 78 | }, 79 | "title_unescape": { 80 | "type": "text", 81 | "analyzer": "simple", 82 | "copy_to": "title_all", 83 | }, 84 | "text": { 85 | "type": "text", 86 | "analyzer": "my_english_analyzer", 87 | }, 88 | "anchortext": { 89 | "type": "text", 90 | "analyzer": "my_english_analyzer", 91 | }, 92 | "title_bigram": { 93 | "type": "text", 94 | "analyzer": "simple_bigram_analyzer", 95 | "copy_to": "title_all_bigram", 96 | }, 97 | "title_unescape_bigram": { 98 | "type": "text", 99 | "analyzer": "simple_bigram_analyzer", 100 | "copy_to": "title_all_bigram", 101 | }, 102 | "text_bigram": { 103 | "type": "text", 104 | "analyzer": "bigram_analyzer", 105 | }, 106 | "anchortext_bigram": { 107 | "type": "text", 108 | "analyzer": "bigram_analyzer", 109 | }, 110 | "original_json": {"type": "string"}, 111 | } 112 | } 113 | }, 114 | "settings": { 115 | "analysis": { 116 | "my_english_analyzer": { 117 | "type": "standard", 118 | "stopwords": "_english_", 119 | }, 120 | "simple_bigram_analyzer": { 121 | "tokenizer": "standard", 122 | "filter": ["lowercase", "shingle", "asciifolding"], 123 | }, 124 | "bigram_analyzer": { 125 | "tokenizer": "standard", 126 | "filter": [ 127 | "lowercase", 128 | "stop", 129 | "shingle", 130 | "asciifolding", 131 | ], 132 | }, 133 | }, 134 | }, 135 | } 136 | ), 137 | ) 138 | 139 | print("Making indexing queries...") 140 | all_queries = [] 141 | for item in tqdm(wikipedia_data): 142 | all_queries.append(process_line(item)) 143 | 144 | count = sum(len(queries.split("\n")) for queries in all_queries) // 2 145 | 146 | if not args.dry: 147 | print("Indexing...") 148 | chunksize = 100 149 | for chunk in tqdm( 150 | chunks(all_queries, chunksize), 151 | total=(len(all_queries) + chunksize - 1) // chunksize, 152 | ): 153 | res = es.bulk(index=INDEX_NAME, body="\n".join(chunk), timeout="100s") 154 | assert not res["errors"], res 155 | 156 | print(f"{count} documents indexed in total") 157 | 158 | 159 | if __name__ == "__main__": 160 | parser = ArgumentParser() 161 | 162 | parser.add_argument("--reindex", action="store_true", help="Reindex everything") 163 | parser.add_argument("--dry", action="store_true", help="Dry run") 164 | 165 | args = parser.parse_args() 166 | 167 | main(args) 168 | -------------------------------------------------------------------------------- /src/build_index/es/index_hotpotqa.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | import bz2 3 | from elasticsearch import Elasticsearch 4 | from glob import glob 5 | import html 6 | import json 7 | from multiprocessing import Pool 8 | from tqdm import tqdm 9 | 10 | WIKIPEDIA_INDEX_NAME = "hotpotqa" 11 | 12 | 13 | def chunks(l, n): 14 | for i in range(0, len(l), n): 15 | yield l[i : i + n] 16 | 17 | 18 | def process_line(line): 19 | data = json.loads(line) 20 | item = { 21 | "id": data["id"], 22 | "url": data["url"], 23 | "title": data["title"], 24 | "title_unescape": html.unescape(data["title"]), 25 | "text": "".join(data["text"]), 26 | "title_bigram": html.unescape(data["title"]), 27 | "title_unescape_bigram": html.unescape(data["title"]), 28 | "text_bigram": "".join(data["text"]), 29 | "original_json": line, 30 | } 31 | return "{}\n{}".format( 32 | json.dumps({"index": {"_id": "wiki-{}".format(data["id"])}}), json.dumps(item) 33 | ) 34 | 35 | 36 | def generate_indexing_queries_from_bz2(bz2file, dry=False): 37 | if dry: 38 | return 39 | 40 | with bz2.open(bz2file, "rt") as f: 41 | body = [process_line(line) for line in f] 42 | 43 | return "\n".join(body) 44 | 45 | 46 | es = Elasticsearch(hosts="http://localhost:9200", timeout=50) 47 | 48 | 49 | def index_chunk(chunk): 50 | res = es.bulk(index=WIKIPEDIA_INDEX_NAME, body="\n".join(chunk), timeout="100s") 51 | assert not res["errors"], res 52 | 53 | 54 | def main(args): 55 | if not args.dry: 56 | if es.indices.exists(index=WIKIPEDIA_INDEX_NAME): 57 | es.indices.delete(index=WIKIPEDIA_INDEX_NAME, ignore=[400, 403]) 58 | if not es.indices.exists(index=WIKIPEDIA_INDEX_NAME): 59 | es.indices.create( 60 | index=WIKIPEDIA_INDEX_NAME, 61 | ignore=400, 62 | body=json.dumps( 63 | { 64 | "mappings": { 65 | "doc": { 66 | "properties": { 67 | "id": {"type": "keyword"}, 68 | "url": {"type": "keyword"}, 69 | "title": { 70 | "type": "text", 71 | "analyzer": "simple", 72 | "copy_to": "title_all", 73 | }, 74 | "title_unescape": { 75 | "type": "text", 76 | "analyzer": "simple", 77 | "copy_to": "title_all", 78 | }, 79 | "text": { 80 | "type": "text", 81 | "analyzer": "my_english_analyzer", 82 | }, 83 | "anchortext": { 84 | "type": "text", 85 | "analyzer": "my_english_analyzer", 86 | }, 87 | "title_bigram": { 88 | "type": "text", 89 | "analyzer": "simple_bigram_analyzer", 90 | "copy_to": "title_all_bigram", 91 | }, 92 | "title_unescape_bigram": { 93 | "type": "text", 94 | "analyzer": "simple_bigram_analyzer", 95 | "copy_to": "title_all_bigram", 96 | }, 97 | "text_bigram": { 98 | "type": "text", 99 | "analyzer": "bigram_analyzer", 100 | }, 101 | "anchortext_bigram": { 102 | "type": "text", 103 | "analyzer": "bigram_analyzer", 104 | }, 105 | "original_json": {"type": "string"}, 106 | } 107 | } 108 | }, 109 | "settings": { 110 | "analysis": { 111 | "my_english_analyzer": { 112 | "type": "standard", 113 | "stopwords": "_english_", 114 | }, 115 | "simple_bigram_analyzer": { 116 | "tokenizer": "standard", 117 | "filter": ["lowercase", "shingle", "asciifolding"], 118 | }, 119 | "bigram_analyzer": { 120 | "tokenizer": "standard", 121 | "filter": [ 122 | "lowercase", 123 | "stop", 124 | "shingle", 125 | "asciifolding", 126 | ], 127 | }, 128 | }, 129 | }, 130 | } 131 | ), 132 | ) 133 | 134 | filelist = glob("../../../data/corpus/hotpotqa/*/wiki_*.bz2") 135 | 136 | print("Making indexing queries...") 137 | pool = Pool() 138 | all_queries = list( 139 | tqdm( 140 | pool.imap(generate_indexing_queries_from_bz2, filelist), total=len(filelist) 141 | ) 142 | ) 143 | 144 | count = sum(len(queries.split("\n")) for queries in all_queries) // 2 145 | 146 | if not args.dry: 147 | print("Indexing...") 148 | chunksize = 50 149 | for chunk in tqdm( 150 | chunks(all_queries, chunksize), 151 | total=(len(all_queries) + chunksize - 1) // chunksize, 152 | ): 153 | res = es.bulk( 154 | index=WIKIPEDIA_INDEX_NAME, body="\n".join(chunk), timeout="100s" 155 | ) 156 | assert not res["errors"], res 157 | 158 | print(f"{count} documents indexed in total") 159 | 160 | 161 | if __name__ == "__main__": 162 | parser = ArgumentParser() 163 | 164 | parser.add_argument("--reindex", action="store_true", help="Reindex everything") 165 | parser.add_argument("--dry", action="store_true", help="Dry run") 166 | 167 | args = parser.parse_args() 168 | 169 | main(args) 170 | -------------------------------------------------------------------------------- /src/build_index/es/index_musique.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from elasticsearch import Elasticsearch 3 | import html 4 | import json 5 | from tqdm import tqdm 6 | 7 | from itertools import chain 8 | 9 | 10 | def chunks(l, n): 11 | for i in range(0, len(l), n): 12 | yield l[i : i + n] 13 | 14 | 15 | INDEX_NAME = "musique" 16 | 17 | 18 | def process_line(data): 19 | item = { 20 | "id": data["id"], 21 | "url": "empty", 22 | "title": data["title"], 23 | "title_unescape": html.unescape(data["title"]), 24 | "text": data["text"], 25 | "title_bigram": html.unescape(data["title"]), 26 | "title_unescape_bigram": html.unescape(data["title"]), 27 | "text_bigram": data["text"], 28 | "original_json": json.dumps(data), 29 | } 30 | return "{}\n{}".format( 31 | json.dumps({"index": {"_id": "wiki-{}".format(data["id"])}}), json.dumps(item) 32 | ) 33 | 34 | 35 | es = Elasticsearch(hosts="http://localhost:9200", timeout=50) 36 | 37 | 38 | def index_chunk(chunk): 39 | res = es.bulk(index=INDEX_NAME, body="\n".join(chunk), timeout="100s") 40 | assert not res["errors"], res 41 | 42 | 43 | def main(args): 44 | # make index 45 | if not args.dry: 46 | if es.indices.exists(index=INDEX_NAME): 47 | es.indices.delete(index=INDEX_NAME, ignore=[400, 403]) 48 | es.indices.create( 49 | index=INDEX_NAME, 50 | ignore=400, 51 | body=json.dumps( 52 | { 53 | "mappings": { 54 | "doc": { 55 | "properties": { 56 | "id": {"type": "keyword"}, 57 | "url": {"type": "keyword"}, 58 | "title": { 59 | "type": "text", 60 | "analyzer": "simple", 61 | "copy_to": "title_all", 62 | }, 63 | "title_unescape": { 64 | "type": "text", 65 | "analyzer": "simple", 66 | "copy_to": "title_all", 67 | }, 68 | "text": { 69 | "type": "text", 70 | "analyzer": "my_english_analyzer", 71 | }, 72 | "anchortext": { 73 | "type": "text", 74 | "analyzer": "my_english_analyzer", 75 | }, 76 | "title_bigram": { 77 | "type": "text", 78 | "analyzer": "simple_bigram_analyzer", 79 | "copy_to": "title_all_bigram", 80 | }, 81 | "title_unescape_bigram": { 82 | "type": "text", 83 | "analyzer": "simple_bigram_analyzer", 84 | "copy_to": "title_all_bigram", 85 | }, 86 | "text_bigram": { 87 | "type": "text", 88 | "analyzer": "bigram_analyzer", 89 | }, 90 | "anchortext_bigram": { 91 | "type": "text", 92 | "analyzer": "bigram_analyzer", 93 | }, 94 | "original_json": {"type": "string"}, 95 | } 96 | } 97 | }, 98 | "settings": { 99 | "analysis": { 100 | "my_english_analyzer": { 101 | "type": "standard", 102 | "stopwords": "_english_", 103 | }, 104 | "simple_bigram_analyzer": { 105 | "tokenizer": "standard", 106 | "filter": ["lowercase", "shingle", "asciifolding"], 107 | }, 108 | "bigram_analyzer": { 109 | "tokenizer": "standard", 110 | "filter": [ 111 | "lowercase", 112 | "stop", 113 | "shingle", 114 | "asciifolding", 115 | ], 116 | }, 117 | }, 118 | }, 119 | } 120 | ), 121 | ) 122 | 123 | train = [ 124 | json.loads(line.strip()) 125 | for line in open("../../../data/corpus/musique/musique_ans_v1.0_train.jsonl") 126 | ] + [ 127 | json.loads(line.strip()) 128 | for line in open("../../../data/corpus/musique/musique_full_v1.0_train.jsonl") 129 | ] 130 | dev = [ 131 | json.loads(line.strip()) 132 | for line in open("../../../data/corpus/musique/musique_ans_v1.0_dev.jsonl") 133 | ] + [ 134 | json.loads(line.strip()) 135 | for line in open("../../../data/corpus/musique/musique_full_v1.0_dev.jsonl") 136 | ] 137 | test = [ 138 | json.loads(line.strip()) 139 | for line in open("../../../data/corpus/musique/musique_ans_v1.0_test.jsonl") 140 | ] + [ 141 | json.loads(line.strip()) 142 | for line in open("../../../data/corpus/musique/musique_full_v1.0_test.jsonl") 143 | ] 144 | 145 | tot = 0 146 | wikipedia_data = [] 147 | hist = set() 148 | for item in tqdm(chain(train, dev, test)): 149 | for p in item["paragraphs"]: 150 | stamp = p["title"] + " " + p["paragraph_text"] 151 | if not stamp in hist: 152 | wikipedia_data.append( 153 | {"id": tot, "text": p["paragraph_text"], "title": p["title"]} 154 | ) 155 | hist.add(stamp) 156 | tot += 1 157 | 158 | print("Making indexing queries...") 159 | all_queries = [] 160 | for item in tqdm(wikipedia_data): 161 | all_queries.append(process_line(item)) 162 | 163 | count = sum(len(queries.split("\n")) for queries in all_queries) // 2 164 | 165 | if not args.dry: 166 | print("Indexing...") 167 | chunksize = 100 168 | for chunk in tqdm( 169 | chunks(all_queries, chunksize), 170 | total=(len(all_queries) + chunksize - 1) // chunksize, 171 | ): 172 | res = es.bulk(index=INDEX_NAME, body="\n".join(chunk), timeout="100s") 173 | assert not res["errors"], res 174 | 175 | print(f"{count} documents indexed in total") 176 | 177 | 178 | if __name__ == "__main__": 179 | parser = ArgumentParser() 180 | 181 | parser.add_argument("--reindex", action="store_true", help="Reindex everything") 182 | parser.add_argument("--dry", action="store_true", help="Dry run") 183 | 184 | args = parser.parse_args() 185 | 186 | main(args) 187 | -------------------------------------------------------------------------------- /src/es_retrieve.py: -------------------------------------------------------------------------------- 1 | import json 2 | from elasticsearch import Elasticsearch 3 | import json 4 | import re 5 | import yaml 6 | 7 | with open("../config/config.yaml", "r") as f: 8 | config = yaml.safe_load(f) 9 | 10 | core_title_matcher = re.compile("([^()]+[^\s()])(?:\s*\(.+\))?") 11 | core_title_filter = lambda x: ( 12 | core_title_matcher.match(x).group(1) if core_title_matcher.match(x) else x 13 | ) 14 | 15 | 16 | class ElasticSearch: 17 | def __init__(self, index_name): 18 | self.index_name = index_name 19 | self.client = Elasticsearch(config["es"]["url"]) 20 | 21 | def _extract_one(self, item, lazy=False): 22 | res = { 23 | k: item["_source"][k] 24 | for k in ["id", "url", "title", "text", "title_unescape"] 25 | } 26 | res["_score"] = item["_score"] 27 | return res 28 | 29 | def rerank_with_query(self, query, results): 30 | def score_boost(item, query): 31 | score = item["_score"] 32 | core_title = core_title_filter(item["title_unescape"]) 33 | if query.startswith("The ") or query.startswith("the "): 34 | query1 = query[4:] 35 | else: 36 | query1 = query 37 | if query == item["title_unescape"] or query1 == item["title_unescape"]: 38 | score *= 1.5 39 | elif ( 40 | query.lower() == item["title_unescape"].lower() 41 | or query1.lower() == item["title_unescape"].lower() 42 | ): 43 | score *= 1.2 44 | elif item["title"].lower() in query: 45 | score *= 1.1 46 | elif query == core_title or query1 == core_title: 47 | score *= 1.2 48 | elif ( 49 | query.lower() == core_title.lower() 50 | or query1.lower() == core_title.lower() 51 | ): 52 | score *= 1.1 53 | elif core_title.lower() in query.lower(): 54 | score *= 1.05 55 | 56 | item["_score"] = score 57 | return item 58 | 59 | return list( 60 | sorted( 61 | [score_boost(item, query) for item in results], 62 | key=lambda item: -item["_score"], 63 | ) 64 | ) 65 | 66 | def single_text_query(self, query, topn=10, lazy=False, rerank_topn=50): 67 | 68 | constructed_query = { 69 | "multi_match": { 70 | "query": query, 71 | "fields": [ 72 | "title^1.25", 73 | "title_unescape^1.25", 74 | "text", 75 | "title_bigram^1.25", 76 | "title_unescape_bigram^1.25", 77 | "text_bigram", 78 | ], 79 | } 80 | } 81 | res = self.client.search( 82 | index=self.index_name, 83 | body={"query": constructed_query, "timeout": "100s"}, 84 | size=max(topn, rerank_topn), 85 | request_timeout=100, 86 | ) 87 | 88 | res = [self._extract_one(x, lazy=lazy) for x in res["hits"]["hits"]] 89 | res = self.rerank_with_query(query, res)[:topn] 90 | res = [{"title": _["title"], "paragraph_text": _["text"]} for _ in res] 91 | return res 92 | 93 | def search(self, question, k=10): 94 | try: 95 | res = self.single_text_query(query=question, topn=k) 96 | return json.dumps(res, ensure_ascii=False) 97 | except Exception as err: 98 | print(Exception, err) 99 | raise 100 | 101 | 102 | def retrieve(index_name, query, topk): 103 | ES = ElasticSearch(index_name) 104 | result = ES.search(query, topk) 105 | return json.loads(result) 106 | 107 | 108 | if __name__ == "__main__": 109 | 110 | print(retrieve("musique", "Bilbo Baggins", 2)) 111 | -------------------------------------------------------------------------------- /src/eval.py: -------------------------------------------------------------------------------- 1 | import json 2 | import string 3 | from collections import Counter 4 | import re 5 | import numpy as np 6 | 7 | 8 | def acc_score(predictions, answers): 9 | num_correct = 0 10 | for id, answer in enumerate(answers): 11 | pred = predictions[id] 12 | correctness = ( 13 | "True" if any(ans.lower() in pred.lower() for ans in answer) else "False" 14 | ) 15 | if correctness == "True": 16 | num_correct += 1 17 | else: 18 | pass 19 | acc = num_correct / len(answers) 20 | return acc 21 | 22 | 23 | def normalize_answer(s): 24 | """Lower text and remove punctuation, articles and extra whitespace.""" 25 | 26 | def remove_articles(text): 27 | return re.sub(r"\b(a|an|the)\b", " ", text) 28 | 29 | def white_space_fix(text): 30 | return " ".join(text.split()) 31 | 32 | def remove_punc(text): 33 | exclude = set(string.punctuation) 34 | return "".join(ch for ch in text if ch not in exclude) 35 | 36 | def lower(text): 37 | return text.lower() 38 | 39 | return white_space_fix(remove_articles(remove_punc(lower(s)))) 40 | 41 | 42 | def f1_score(prediction, ground_truth): 43 | common = Counter(prediction) & Counter(ground_truth) 44 | num_same = sum(common.values()) 45 | if num_same == 0: 46 | return 0 47 | precision = 1.0 * num_same / len(prediction) 48 | recall = 1.0 * num_same / len(ground_truth) 49 | f1 = (2 * precision * recall) / (precision + recall) 50 | return f1 51 | 52 | 53 | def qa_f1_score(prediction, ground_truth): 54 | normalized_prediction = normalize_answer(prediction) 55 | normalized_ground_truth = normalize_answer(ground_truth) 56 | 57 | prediction_tokens = normalized_prediction.split() 58 | ground_truth_tokens = normalized_ground_truth.split() 59 | return f1_score(prediction_tokens, ground_truth_tokens) 60 | 61 | 62 | def F1_scorer(predictions, answers): 63 | total_score = 0.0 64 | for prediction, ground_truths in zip(predictions, answers): 65 | score = 0.0 66 | for ground_truth in ground_truths: 67 | score = max(score, qa_f1_score(prediction, ground_truth)) 68 | total_score += score 69 | return round(100 * total_score / len(predictions), 2) 70 | 71 | 72 | def compute_exact(predictions, answers): 73 | total_score = 0.0 74 | for prediction, ground_truths in zip(predictions, answers): 75 | score = 0.0 76 | for ground_truth in ground_truths: 77 | score = max( 78 | score, 79 | int(normalize_answer(prediction) == normalize_answer(ground_truth)), 80 | ) 81 | total_score += score 82 | return round(100 * total_score / len(predictions), 2) 83 | 84 | 85 | def exact_presence(short_answers, context): 86 | """Verify if any of the answers is present in the given context. 87 | Args: 88 | short_answers: list of short answers to look for in the context 89 | context: a paragraph to search for short answers 90 | Returns: 91 | true if any of the short answers is present in the context 92 | """ 93 | 94 | n_short_answers = [normalize_answer(sa) for sa in short_answers] 95 | n_context = normalize_answer(context) 96 | 97 | for ans in n_short_answers: 98 | if ans in n_context: 99 | return True 100 | 101 | return False 102 | 103 | 104 | def compute_str_em(data): 105 | """Compute STR-EM metric (only for ASQA) 106 | Args: 107 | data: requires field `qa_pairs/short_answers` and `output` 108 | Returns: 109 | STR-EM and STR-EM-HIT () 110 | """ 111 | 112 | if "qa_pairs" not in data[0] or data[0]["qa_pairs"] is None: 113 | return 0, 0 114 | 115 | acc = [] 116 | hit = [] 117 | 118 | for item in data: 119 | loc_acc = [] 120 | for qa_pair in item["qa_pairs"]: 121 | loc_acc.append(exact_presence(qa_pair["short_answers"], item["output"])) 122 | acc.append(np.mean(loc_acc)) 123 | hit.append(int(np.mean(loc_acc) == 1)) 124 | 125 | return 100 * np.mean(acc), 100 * np.mean(hit) 126 | 127 | 128 | def eval_asqa(pred): 129 | raw_data_path = "../data/eval/asqa/asqa_eval_gtr_top100.json" 130 | with open(raw_data_path, "r") as f: 131 | raw_data = json.load(f) 132 | normalized_data = [] 133 | for i in range(len(pred)): 134 | pred[i]["answer"] = pred[i]["output"].strip().split("\n")[0] 135 | pred[i]["answer"] = pred[i]["output"].replace("<|im_end|>", "") 136 | result = {} 137 | for id, data in enumerate(pred): 138 | normalized_data.append( 139 | { 140 | "question": raw_data[id]["question"], 141 | "answer": raw_data[id]["answer"], 142 | "qa_pairs": raw_data[id]["qa_pairs"], 143 | "output": data["answer"], 144 | } 145 | ) 146 | result["str_em"], result["str_hit"] = compute_str_em(normalized_data) 147 | return result 148 | 149 | 150 | if __name__ == "__main__": 151 | 152 | from argparse import ArgumentParser 153 | 154 | parser = ArgumentParser() 155 | parser.add_argument( 156 | "--dataset", 157 | type=str, 158 | choices=[ 159 | "2wikimultihopqa", 160 | "hotpotqa", 161 | "musique", 162 | "asqa", 163 | "crud_1doc", 164 | "crud_2doc", 165 | "crud_3doc", 166 | ], 167 | default="hotpotqa", 168 | help="Dataset to use", 169 | ) 170 | parser.add_argument("--pred_path", type=str, help="Location of the prediction file") 171 | args = parser.parse_args() 172 | 173 | outputs = [] 174 | with open(args.pred_path, "r") as fin: 175 | for d in fin: 176 | pred = json.loads(d) 177 | if "id" in pred.keys(): 178 | outputs.append(pred) 179 | if "asqa" in args.pred_path: 180 | result = eval_asqa(outputs) 181 | print("eval result:", result) 182 | else: 183 | with open( 184 | f"../data/eval/{args.dataset}/test.json", "r", encoding="utf-8" 185 | ) as f: # dev_sample_10 186 | qa_data = json.load(f) 187 | 188 | predictions = [data["output"] for data in outputs] 189 | answers = [data["answer"] for data in outputs] 190 | acc = acc_score(predictions, answers) 191 | f1 = F1_scorer(predictions, answers) 192 | em = compute_exact(predictions, answers) 193 | 194 | print("ACC:", acc, "F1:", f1, "em:", em) 195 | -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import argparse 4 | import logging 5 | import datetime 6 | import yaml 7 | import faiss 8 | import torch 9 | from sentence_transformers import SentenceTransformer 10 | import pandas as pd 11 | from transformers import AutoModelForCausalLM, AutoTokenizer 12 | import backoff 13 | import re 14 | import time 15 | from multiprocessing.pool import ThreadPool 16 | from tqdm import tqdm 17 | from eval import acc_score, F1_scorer, compute_exact, eval_asqa 18 | from utils import seed_everything 19 | 20 | parser = argparse.ArgumentParser() 21 | parser.add_argument( 22 | "--model", 23 | type=str, 24 | choices=["gpt-3.5-turbo-0125", "llama3-8b-instruct", "qwen2-7b-instruct"], 25 | default="llama3-8b-instruct", 26 | help="Model to use", 27 | ) 28 | parser.add_argument( 29 | "--max_step", type=int, default=2, help="Maximum number of update steps" 30 | ) 31 | parser.add_argument( 32 | "--max_fail_step", type=int, default=2, help="Maximum number of failed steps" 33 | ) 34 | parser.add_argument( 35 | "--MaxClients", type=int, default=1, help="Number of concurrent clients" 36 | ) 37 | parser.add_argument( 38 | "--retrieve_top_k", type=int, default=5, help="Top K documents to retrieve" 39 | ) 40 | parser.add_argument( 41 | "--max_top_k", 42 | type=int, 43 | default=15, 44 | help="Total maximum number of documents to retrieve", 45 | ) 46 | parser.add_argument( 47 | "--dataset", 48 | type=str, 49 | choices=[ 50 | "2wikimultihopqa", 51 | "hotpotqa", 52 | "musique", 53 | "asqa", 54 | "crud_1doc", 55 | "crud_2doc", 56 | "crud_3doc", 57 | ], 58 | default="hotpotqa", 59 | help="Dataset to use", 60 | ) 61 | parser.add_argument( 62 | "--method", 63 | type=str, 64 | default="note", 65 | choices=["note", "base", "base_wo_retri"], 66 | help="Method to use", 67 | ) 68 | parser.add_argument( 69 | "--resume_path", 70 | type=str, 71 | default="", 72 | help="Path to the file for resuming generation", 73 | ) 74 | parser.add_argument("--device", type=str, default="cuda:0", help="Device to use") 75 | args = parser.parse_args() 76 | 77 | with open("../config/config.yaml", "r") as f: 78 | config = yaml.safe_load(f) 79 | 80 | 81 | @backoff.on_exception(backoff.expo, (Exception), max_time=100) 82 | def call_gpt(prompt_file, variable_dict): 83 | with open(prompt_file, "r") as fin: 84 | prompt = fin.read().format(**variable_dict) 85 | res = gpt_gen(args.model, prompt) 86 | assert res is not None 87 | return res 88 | 89 | 90 | def call_local(prompt_file, variable_dict): 91 | return local_gen(model, tokenizer, prompt_file, variable_dict) 92 | 93 | 94 | if "gpt" in args.model: 95 | from utils import gpt_gen 96 | 97 | call_llm = call_gpt 98 | elif "llama" in args.model: 99 | path = config["model"]["llama3-8b-instruct"] 100 | tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True) 101 | model = AutoModelForCausalLM.from_pretrained( 102 | path, trust_remote_code=True, torch_dtype=torch.bfloat16, device_map=args.device 103 | ) 104 | from utils import llama_gen as local_gen 105 | 106 | call_llm = call_local 107 | elif "qwen" in args.model: 108 | path = config["model"]["qwen2-7b-instruct"] 109 | model = AutoModelForCausalLM.from_pretrained( 110 | path, torch_dtype=torch.bfloat16, device_map=args.device 111 | ) 112 | tokenizer = AutoTokenizer.from_pretrained(path) 113 | from utils import qwen_gen as local_gen 114 | 115 | call_llm = call_local 116 | 117 | 118 | def get_context(data): 119 | 120 | if args.dataset == "asqa" or "crud" in args.dataset: 121 | text = "\n".join(data) 122 | else: 123 | text = "" 124 | for i in range(len(data)): 125 | text += f"Passage #{i+1} Title: {data[i]['title']}\nPassage #{i+1} Text: {data[i]['paragraph_text']} \n\n" 126 | return text 127 | 128 | 129 | def save_log_to_file(logger, log_file="my_log", log_folder="logs"): 130 | if not os.path.exists(log_folder): 131 | os.makedirs(log_folder) 132 | current_date = datetime.datetime.now().strftime("%Y%m%d-%H:%M:%S") 133 | log_file_name = f"{log_file}-{current_date}.log" 134 | file_handler = logging.FileHandler(os.path.join(log_folder, log_file_name)) 135 | logger.addHandler(file_handler) 136 | 137 | 138 | def is_directory_empty(directory_path: str) -> bool: 139 | try: 140 | return len(os.listdir(directory_path)) == 0 141 | except OSError: 142 | return False 143 | 144 | 145 | def call_llm_template(template, variables): 146 | return call_llm(f"../prompts/{LAUGUAGE}/{template}", variables) 147 | 148 | 149 | def first_gen_answer(query, refs): 150 | return call_llm_template("first_gen_answer", {"query": query, "refs": refs}) 151 | 152 | 153 | def gen_new_query(query, note, query_log): 154 | return call_llm_template( 155 | "gen_new_query", {"query": query, "note": note, "query_log": query_log} 156 | ) 157 | 158 | 159 | def refine_note(note, query, refs): 160 | return call_llm_template( 161 | "refine_note", {"note": note, "query": query, "refs": refs} 162 | ) 163 | 164 | 165 | def gen_answer(query, note_str): 166 | template = "gen_answer_asqa" if "asqa" in args.dataset else "gen_answer" 167 | return call_llm_template(template, {"query": query, "note": note_str}) 168 | 169 | 170 | def compare_note(query, best_note, new_note): 171 | response = call_llm_template( 172 | "compare", {"query": query, "best_note": best_note, "new_note": new_note} 173 | ) 174 | try: 175 | status = json.loads(response)["status"] 176 | return status.lower() == "true" 177 | except: 178 | return "true" in response.lower() 179 | 180 | 181 | def retrieve_note(doc_id, query, answer, top_k=2): 182 | ref_log, llm_times, note_log, query_log, query_list = [], 0, [], [], [] 183 | refs = retrieve(args.dataset, query=query, topk=top_k) 184 | note = best_note = first_gen_answer(query, get_context(refs)) 185 | ref_log.append({"refs": refs, "step": 0, "flag": "init_refs"}) 186 | note_log.append({"note": note, "step": 0, "flag": "init_note"}) 187 | llm_times += 1 188 | step, notes_status = 0, [] 189 | 190 | while step < args.max_step: 191 | all_refs = { 192 | ref 193 | for d in ref_log 194 | for ref in ( 195 | d["refs"] 196 | if "crud" in args.dataset or args.dataset == "asqa" 197 | else [d["title"] + d["paragraph_text"] for d in d["refs"]] 198 | ) 199 | } 200 | if len(all_refs) >= args.max_top_k: 201 | break 202 | max_ref = ( 203 | args.max_top_k - len(all_refs) 204 | if (len(all_refs) + top_k) > args.max_top_k 205 | else 0 206 | ) 207 | 208 | new_query = gen_new_query(query, best_note, str(query_list)) 209 | query_list.append(new_query) 210 | llm_times += 1 211 | 212 | refs = retrieve( 213 | args.dataset, query=(new_query + "\n" + query)[:500], topk=top_k 214 | ) 215 | if max_ref > 0: 216 | refs = ( 217 | [d for d in refs if d not in all_refs][:max_ref] 218 | if "crud" in args.dataset or args.dataset == "asqa" 219 | else [ 220 | d for d in refs if d["title"] + d["paragraph_text"] not in all_refs 221 | ][:max_ref] 222 | ) 223 | 224 | note = refine_note(best_note, query, get_context(refs)).replace("\n", "") 225 | llm_times += 1 226 | status = compare_note(query, best_note, note) 227 | flag = "True" if status else "False" 228 | 229 | ref_log.append({"refs": refs, "step": step, "flag": flag}) 230 | note_log.append({"note": note, "step": step, "flag": flag}) 231 | query_log.append({"query": new_query, "step": step, "flag": flag}) 232 | 233 | if status: 234 | best_note = note 235 | notes_status.append(status) 236 | if notes_status.count(False) >= args.max_fail_step: 237 | break 238 | step += 1 239 | 240 | llm_times += 1 241 | return { 242 | "id": doc_id, 243 | "question": query, 244 | "answer": answer, 245 | "output": gen_answer(query, best_note), 246 | "note": best_note, 247 | "query_log": query_log, 248 | "note_log": note_log, 249 | "ref_log": ref_log, 250 | } 251 | 252 | 253 | def process_doc_cell(idx, doc_cell, args): 254 | id_new, query, answer = idx, doc_cell["question"], doc_cell["answer"] 255 | 256 | if args.method != "note": 257 | prompt_path = f"../prompts/{LAUGUAGE}" 258 | gen_name = ( 259 | "base_wo_retri_asqa" 260 | if "wo_retri" in args.method and "asqa" in args.dataset 261 | else ( 262 | "base_wo_retri" 263 | if "wo_retri" in args.method 264 | else "base_asqa" if "asqa" in args.dataset else "base" 265 | ) 266 | ) 267 | 268 | output = ( 269 | call_llm(f"{prompt_path}/{gen_name}", {"query": query}) 270 | if "wo_retri" in args.method 271 | else call_llm( 272 | f"{prompt_path}/{gen_name}", 273 | { 274 | "query": query, 275 | "refs": get_context( 276 | retrieve(args.dataset, query=query, topk=args.retrieve_top_k) 277 | ), 278 | }, 279 | ) 280 | ) 281 | 282 | return {"id": id_new, "query": query, "answer": answer, "output": output} 283 | else: 284 | return retrieve_note(id_new, query, answer, top_k=args.retrieve_top_k) 285 | 286 | 287 | if __name__ == "__main__": 288 | 289 | LAUGUAGE = "en" 290 | seed_everything(66) 291 | 292 | logger = logging.getLogger(__name__) 293 | logging.basicConfig( 294 | format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", 295 | datefmt="%m/%d/%Y %H:%M:%S", 296 | level=logging.INFO, 297 | ) 298 | save_log_to_file( 299 | logger, 300 | log_file=f"{args.dataset}_{args.method}_{args.model}", 301 | log_folder="../log", 302 | ) 303 | logger.info(f"{'*' * 30} CONFIGURATION {'*' * 30}") 304 | for key, val in sorted(vars(args).items()): 305 | keystr = "{}".format(key) + (" " * (30 - len(key))) 306 | logger.info("%s --> %s", keystr, val) 307 | dataset_name=args.dataset 308 | if "crud" in args.dataset: 309 | dataset_name = "crud" 310 | vector_path = f"../data/corpus/{dataset_name}/{dataset_name}.index" 311 | if args.dataset == "asqa": 312 | vector = faiss.read_index(vector_path) 313 | emb_model = SentenceTransformer(config["model"]["gtr-t5-xxl"], device="cpu") 314 | raw_data = pd.read_csv("../data/corpus/asqa/psgs_w100.tsv", sep="\t") 315 | 316 | def retrieve(_, query, topk): 317 | feature = emb_model.encode([query]) 318 | _, match_id = vector.search(feature, topk) 319 | return [ 320 | raw_data.iloc[i]["title"] + "\n" + raw_data.iloc[i]["text"] 321 | for i in match_id[0] 322 | ] 323 | 324 | elif "crud" in args.dataset: 325 | LAUGUAGE = "zh" 326 | vector = faiss.read_index(vector_path) 327 | emb_model = SentenceTransformer( 328 | config["model"]["bge-base-zh-v1.5"], device=args.device 329 | ) 330 | with open(f"../data/corpus/crud/chunk.json", encoding="utf-8") as f: 331 | raw_data = json.load(f) 332 | 333 | def retrieve(_, query, topk): 334 | feature = emb_model.encode([query]) 335 | _, match_id = vector.search(feature, topk) 336 | return [raw_data[i] for i in match_id[0]] 337 | 338 | else: 339 | from es_retrieve import retrieve 340 | 341 | formatted_time = time.strftime("%Y%m%d-%H%M%S") 342 | dataset = re.sub(r"_es$", "", args.dataset) 343 | 344 | with open(f"../data/eval/{dataset}/test.json", encoding="utf-8") as f: 345 | qa_data = json.load(f) 346 | 347 | save_path = f"../output/{args.dataset}/{args.method}/{args.model}" 348 | os.makedirs(save_path, exist_ok=True) 349 | 350 | all_result = [] 351 | 352 | if args.resume_path: 353 | with open(args.resume_path, "r", encoding="utf-8") as fin: 354 | resume_data = [json.loads(i) for i in fin.readlines()] 355 | all_result = resume_data 356 | filepath = args.resume_path 357 | else: 358 | resume_data = [] 359 | filepath = ( 360 | f"{save_path}/topk-{args.retrieve_top_k}_{formatted_time}.jsonl" 361 | if args.method != "note" 362 | else f"{save_path}/topk-{args.retrieve_top_k}__max_step-{args.max_step}__max_fail_step-{args.max_fail_step}__{formatted_time}.jsonl" 363 | ) 364 | logger.info(f"The predicted results will be saved in '{filepath}'.") 365 | last_id = len(resume_data) 366 | batch_size = args.MaxClients 367 | start = time.time() 368 | logger.info("start predicting ...") 369 | for i in tqdm(range(last_id, len(qa_data), batch_size)): 370 | pool = ThreadPool(processes=args.MaxClients) 371 | current_batch = qa_data[i : i + batch_size] 372 | tasks = [ 373 | (idx + i, doc_cell, args) for idx, doc_cell in enumerate(current_batch) 374 | ] 375 | 376 | results = pool.starmap(process_doc_cell, tasks) 377 | pool.close() 378 | pool.join() 379 | 380 | for result in results: 381 | if result: 382 | all_result.append(result) 383 | with open(filepath, "a", buffering=1) as fout: 384 | fout.write(json.dumps(result, ensure_ascii=False) + "\n") 385 | 386 | logger.info("start evaluating ...") 387 | 388 | predictions = [data["output"] for data in all_result] 389 | answers = [data["answer"] for data in all_result] 390 | 391 | if "asqa" in args.dataset: 392 | eval_result = eval_asqa(all_result) 393 | elif "crud" in args.dataset: 394 | eval_result = None 395 | else: 396 | acc = acc_score(predictions, answers) 397 | f1 = F1_scorer(predictions, answers) 398 | em = compute_exact(predictions, answers) 399 | eval_result = {"Acc": acc, "F1": f1, "EM": em} 400 | 401 | if eval_result: 402 | with open(filepath, "a", buffering=1) as fout: 403 | fout.write(json.dumps(eval_result, ensure_ascii=False) + "\n") 404 | 405 | logger.info(f"eval result: {eval_result}") 406 | logger.info(f"Spend time: {time.time() - start}") 407 | -------------------------------------------------------------------------------- /src/metrics_questeval_crud.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import json 4 | import jieba 5 | import numpy as np 6 | from loguru import logger 7 | from collections import Counter 8 | import argparse 9 | import os 10 | import backoff 11 | from multiprocessing.pool import ThreadPool 12 | from utils import gpt_gen 13 | 14 | 15 | @backoff.on_exception(backoff.expo, (Exception), max_time=100) 16 | def call_gpt(prompt_file, variable_dict): 17 | with open(prompt_file, "r") as fin: 18 | prompt = fin.read().format(**variable_dict) 19 | res = gpt_gen(args.model, prompt) 20 | assert res is not None 21 | return res 22 | 23 | 24 | def compute_f1(a_gold, a_pred): 25 | gold_toks = list(jieba.cut(a_gold)) 26 | pred_toks = list(jieba.cut(a_pred)) 27 | common = Counter(gold_toks) & Counter(pred_toks) 28 | num_same = sum(common.values()) 29 | if len(gold_toks) == 0 or len(pred_toks) == 0: 30 | return ( 31 | int(gold_toks == pred_toks), 32 | int(gold_toks == pred_toks), 33 | int(gold_toks == pred_toks), 34 | ) 35 | if num_same == 0: 36 | return 0, 0, 0 37 | precision = 1.0 * num_same / len(pred_toks) 38 | recall = 1.0 * num_same / len(gold_toks) 39 | f1 = (2 * precision * recall) / (precision + recall) 40 | return f1, precision, recall 41 | 42 | 43 | def word_based_f1_score(a_gold_list, a_pred_list): 44 | f1_list, precision_list, recall_list = [], [], [] 45 | for a_gold, a_pred in zip(a_gold_list, a_pred_list): 46 | f1, precision, recall = compute_f1(a_gold, a_pred) 47 | f1_list.append(f1) 48 | precision_list.append(precision) 49 | recall_list.append(recall) 50 | return np.mean(f1_list), np.mean(precision_list), np.mean(recall_list) 51 | 52 | 53 | def question_answer(context, question): 54 | origin_answers = call_gpt( 55 | f"../prompts/crud/quest_eval_answer.txt", 56 | {"context": context, "questions": question}, 57 | ) 58 | answers = origin_answers 59 | pattern = r"\n(.*?)\n" 60 | real_answers = re.findall(pattern, answers, re.DOTALL) 61 | return real_answers 62 | 63 | 64 | def get_QA_pair(data_point: dict, quest_gt_save: dict): 65 | generated_text = data_point["output"] 66 | questions_gt = quest_gt_save["question"] 67 | answers_gt4gt = quest_gt_save["answers"] 68 | answers_gm4gt = question_answer(generated_text, questions_gt) 69 | return questions_gt, answers_gt4gt, answers_gm4gt 70 | 71 | 72 | def quest_eval(data_point: dict, quest_gt: dict): 73 | try: 74 | questions_gt, answers_gt4gt, answers_gm4gt = get_QA_pair(data_point, quest_gt) 75 | quest_eval_save = { 76 | "questions_gt": questions_gt, 77 | "answers_gt4gt": answers_gt4gt, 78 | "answers_gm4gt": answers_gm4gt, 79 | } 80 | 81 | indices = [i for i, x in enumerate(answers_gt4gt) if x != "无法推断"] 82 | answers_gm4gt = [answers_gm4gt[i] for i in indices] 83 | answers_gt4gt = [answers_gt4gt[i] for i in indices] 84 | 85 | if len(answers_gm4gt) == 0: 86 | return 0, 0, 0, 0, quest_eval_save 87 | 88 | undetermined_ratio = answers_gm4gt.count("无法推断") / len(answers_gm4gt) 89 | quest_recall = 1 - undetermined_ratio 90 | 91 | indices = [i for i, x in enumerate(answers_gm4gt) if x != "无法推断"] 92 | answers_gm4gt = [answers_gm4gt[i] for i in indices] 93 | answers_gt4gt = [answers_gt4gt[i] for i in indices] 94 | 95 | if answers_gm4gt == []: 96 | return 0, 0, 0, 0, quest_eval_save 97 | 98 | quest_avg_token_f1, quest_avg_token_precision, quest_avg_token_recall = ( 99 | word_based_f1_score(answers_gt4gt, answers_gm4gt) 100 | ) 101 | 102 | except Exception as e: 103 | logger.warning(repr(e)) 104 | quest_eval_save = {} 105 | quest_eval_save["questions_gt"] = [] 106 | quest_eval_save["answers_gt4gt"] = [] 107 | quest_eval_save["answers_gm4gt"] = [] 108 | return 0, 0, 0, 0, quest_eval_save 109 | 110 | return ( 111 | quest_avg_token_f1, 112 | quest_avg_token_precision, 113 | quest_avg_token_recall, 114 | quest_recall, 115 | quest_eval_save, 116 | ) 117 | 118 | 119 | if __name__ == "__main__": 120 | parser = argparse.ArgumentParser() 121 | parser.add_argument( 122 | "--model", 123 | type=str, 124 | choices=["gpt-3.5-turbo-0125"], 125 | default="gpt-3.5-turbo-0125", 126 | help="Model to use", 127 | ) 128 | parser.add_argument( 129 | "--MaxClients", type=int, default=10, help="Number of concurrent clients" 130 | ) 131 | parser.add_argument("--eval_path", type=str, help="eval path") 132 | args = parser.parse_args() 133 | save_path = os.path.join(os.path.dirname(args.eval_path), "metric_questeval") 134 | os.makedirs(save_path, exist_ok=True) 135 | with open(args.eval_path, "r") as fin: 136 | generate_answer = [json.loads(d) for d in fin] 137 | eval_name = args.eval_path.split("/")[-1].split(".")[0] 138 | pattern = r"crud_[123]doc" 139 | data_name = re.findall(pattern, args.eval_path)[0] 140 | with open(f"../data/eval/{data_name}/QuestAnswerQA_quest_gt.json", "r") as f: 141 | quest_gt = json.load(f) 142 | 143 | res_cell_list = [] 144 | all_quest_avg_token_f1, all_quest_recall = [], [] 145 | all_quest_avg_token_precision, all_quest_avg_token_recall = [], [] 146 | pool = ThreadPool(processes=args.MaxClients) 147 | tasks = [] 148 | ids = [] 149 | for idx, generate_cell in enumerate(generate_answer): 150 | tasks.append((generate_cell, quest_gt[idx])) 151 | results = pool.starmap(quest_eval, tasks) 152 | pool.close() 153 | pool.join() 154 | 155 | for index, result in enumerate(results): 156 | ( 157 | quest_avg_token_f1, 158 | quest_avg_token_precision, 159 | quest_avg_token_recall, 160 | quest_recall, 161 | quest_eval_save, 162 | ) = result 163 | all_quest_avg_token_f1.append(quest_avg_token_f1) 164 | all_quest_avg_token_precision.append(quest_avg_token_precision) 165 | all_quest_avg_token_recall.append(quest_avg_token_recall) 166 | all_quest_recall.append(quest_recall) 167 | 168 | res_cell = { 169 | "id": index, 170 | "quest_avg_token_f1": quest_avg_token_f1, 171 | "quest_avg_token_precision": quest_avg_token_precision, 172 | "quest_avg_token_recall": quest_avg_token_recall, 173 | "quest_recall": quest_recall, 174 | "quest_eval_save": quest_eval_save, 175 | } 176 | res_cell_list.append(res_cell) 177 | 178 | valid_qa_count = 0 179 | for res in res_cell_list: 180 | if res["quest_eval_save"]["questions_gt"]: 181 | valid_qa_count += 1 182 | avg_score = { 183 | "all_quest_avg_token_f1": sum(all_quest_avg_token_f1) / valid_qa_count, 184 | "all_quest_avg_token_precision": sum(all_quest_avg_token_precision) 185 | / valid_qa_count, 186 | "all_quest_avg_token_recall": sum(all_quest_avg_token_recall) / valid_qa_count, 187 | "all_quest_recall": sum(all_quest_recall) / valid_qa_count, 188 | } 189 | res_cell_list.append(avg_score) 190 | print("eval result", res_cell_list) 191 | with open(f"{save_path}/{eval_name}_questeval.json", "w", buffering=1) as f: 192 | f.write(json.dumps(res_cell_list, ensure_ascii=False, indent=2)) 193 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | import os, time 2 | import torch 3 | from openai import OpenAI 4 | import random 5 | import numpy as np 6 | import yaml 7 | 8 | with open("../config/config.yaml", "r") as f: 9 | config = yaml.safe_load(f) 10 | os.environ["OPENAI_API_KEY"] = config["model"]["OPENAI_API_KEY"] 11 | os.environ["OPENAI_BASE_URL"] = config["model"]["OPENAI_BASE_URL"] 12 | client = OpenAI() 13 | 14 | 15 | def gpt_gen(model, content): 16 | try: 17 | completion = client.chat.completions.create( 18 | model=model, 19 | temperature=0.1, 20 | top_p=0.9, 21 | max_tokens=1280, 22 | messages=[{"role": "user", "content": content}], 23 | ) 24 | return completion.choices[0].message.content 25 | 26 | except Exception as e: 27 | print(f"An error occurred: {e}") 28 | time.sleep(0.5) 29 | 30 | return None 31 | 32 | 33 | def llama_gen(model, tokenizer, prompt_file, variable_dict): 34 | with open(prompt_file, "r") as fin: 35 | prompt = fin.read() 36 | prompt = prompt.format(**variable_dict) 37 | messages = [{"role": "user", "content": prompt}] 38 | input_ids = tokenizer.apply_chat_template( 39 | messages, add_generation_prompt=True, return_tensors="pt" 40 | ).to(model.device) 41 | terminators = [ 42 | tokenizer.eos_token_id, 43 | tokenizer.convert_tokens_to_ids("<|eot_id|>"), 44 | ] 45 | outputs = model.generate( 46 | input_ids, 47 | eos_token_id=terminators, 48 | temperature=0.1, 49 | top_p=0.9, 50 | max_new_tokens=1280, 51 | ) 52 | response = tokenizer.decode( 53 | outputs[0][input_ids.shape[-1] :], skip_special_tokens=True 54 | ) 55 | return response 56 | 57 | 58 | def qwen_gen(model, tokenizer, prompt_file, variable_dict): 59 | with open(prompt_file, "r") as fin: 60 | prompt = fin.read() 61 | prompt = prompt.format(**variable_dict) 62 | messages = [ 63 | {"role": "system", "content": "You are a helpful assistant."}, 64 | {"role": "user", "content": prompt}, 65 | ] 66 | text = tokenizer.apply_chat_template( 67 | messages, tokenize=False, add_generation_prompt=True 68 | ) 69 | model_inputs = tokenizer([text], return_tensors="pt").to(model.device) 70 | 71 | generated_ids = model.generate( 72 | model_inputs.input_ids, temperature=0.1, top_p=0.9, max_new_tokens=1280 73 | ) 74 | generated_ids = [ 75 | output_ids[len(input_ids) :] 76 | for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) 77 | ] 78 | 79 | response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] 80 | 81 | return response 82 | 83 | 84 | def seed_everything(seed): 85 | torch.manual_seed(seed) 86 | torch.cuda.manual_seed(seed) 87 | np.random.seed(seed) 88 | random.seed(seed) 89 | torch.backends.cudnn.benchmark = False 90 | torch.backends.cudnn.deterministic = True 91 | torch.cuda.manual_seed_all(seed) 92 | --------------------------------------------------------------------------------