├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── baseline.py ├── data ├── dev(None).jsonl ├── dev.jsonl ├── prompts_natural_language │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── prompts_optimized │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── prompts_triple_based │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── test.jsonl └── train.jsonl ├── evaluate.py ├── failure_cases ├── baseline │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── baseline_factcheck │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── baseline_factcheck_wikiclean │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── gpt3(davinci-dev-language-empty) │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── gpt3(davinci-dev-language-none) │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── gpt3(davinci-dev-optimized) │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── gpt3(davinci-dev-optimized)_factcheck │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── gpt3(davinci-dev-optimized)_factcheck_wikiclean │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── gpt3(davinci-dev-triple-empty) │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt └── gpt3(davinci-dev-triple-none) │ ├── ChemicalCompoundElement.txt │ ├── CompanyParentOrganization.txt │ ├── CountryBordersWithCountry.txt │ ├── CountryOfficialLanguage.txt │ ├── PersonCauseOfDeath.txt │ ├── PersonEmployer.txt │ ├── PersonInstrument.txt │ ├── PersonLanguage.txt │ ├── PersonPlaceOfDeath.txt │ ├── PersonProfession.txt │ ├── RiverBasinsCountry.txt │ └── StateSharesBorderState.txt ├── gpt3_baseline.py ├── integrity_checking.py ├── notebooks └── Dataset Statistics.ipynb ├── opt_baseline.py ├── predictions ├── README.md ├── baseline.pred.jsonl ├── baseline_factcheck.pred.jsonl ├── baseline_factcheck_wikiclean.pred.jsonl ├── gpt3(davinci-dev-language-empty).pred.jsonl ├── gpt3(davinci-dev-language-none).pred.jsonl ├── gpt3(davinci-dev-optimized).pred.jsonl ├── gpt3(davinci-dev-optimized)_factcheck.pred.jsonl ├── gpt3(davinci-dev-optimized)_factcheck_wikiclean.pred.jsonl ├── gpt3(davinci-dev-triple-empty).pred.jsonl ├── gpt3(davinci-dev-triple-none).pred.jsonl ├── gpt3(davinci-test-optimized).pred.jsonl ├── gpt3(davinci-test-optimized)_factcheck.pred.jsonl ├── gpt3(davinci-test-optimized)_factcheck_wikiclean.pred.jsonl └── gpt3(davinci-test-triple-empty).pred.jsonl ├── requirements.txt ├── submission ├── predictions(davinci-optimized)_19_07_2022.zip ├── predictions(davinci-optimized)_21_07_2022.zip ├── predictions(davinci-optimized-factcheck)_19_07_2022.zip ├── predictions(davinci-optimized-factcheck)_21_07_2022.zip ├── predictions(davinci-optimized-factcheck-wikiclean)_21_07_2022.zip ├── predictions(davinci-triple-empty)_19_07_2022.zip └── predictions_14_07_2022.zip ├── utils ├── __init__.py ├── file_io.py ├── model.py └── wordnet_utils.py ├── wikidata_cleanup.py └── wikidata_extract_aliases.go /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Emile 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # iswc-challenge 2 | 3 | ## Getting started 4 | 5 | ### Prerequisites 6 | 7 | This repository uses Python >= 3.10 8 | 9 | Be sure to run in a virtual python environment (e.g. conda, venv, mkvirtualenv, etc.) 10 | 11 | ### Installation 12 | 13 | 1. In the root directory of this repo run 14 | 15 | ```bash 16 | pip install -r requirements.txt 17 | ``` 18 | 19 | ### Usage 20 | 21 | For running and evaluating the baseline, run : 22 | 23 | ```bash 24 | python baseline.py -i "data/dev.jsonl" -o "predictions/baseline.pred.jsonl" 25 | python evaluate.py -p "predictions/baseline.pred.jsonl" -g "data/dev.jsonl" 26 | ``` 27 | 28 | For running and evaluating our proposed GPT3 approach, make sure you set your `OPENAI_API_KEY` in the environmental 29 | variables. This will use the default values for training, i.e. `text-davinci-002` model, `data/dev.jsonl` as input 30 | and `predictions/gpt3.pred.jsonl` as output. Run : 31 | 32 | ```bash 33 | python gpt3_baseline 34 | python evaluate -p "predictions/gpt3.pred.jsonl" -g "data/dev.jsonl" 35 | ``` 36 | 37 | For the scaling experiment, you need to change the flag `model` to the respective model. The options 38 | include: `['text-davinci-002', 'text-curie-001', 'text-babbage-001', 'text-ada-001']` 39 | 40 | ```bash 41 | python gpt3_baseline -i "data/dev.jsonl" -o "predictions/gpt3-ada.pred.jsonl" --model "text-ada-001" 42 | python evaluate -p "predictions/gpt3.pred.jsonl" -g "data/dev.jsonl" 43 | ``` 44 | 45 | ## Tasks: 46 | 47 | - [X] Make changes that the competition organisers suggest [priority] 48 | - [X] Pull the changes from their repo 49 | - [X] Check our performance on the updated train/val dataset 50 | - [X] Dataset statistics (nice to include in the paper) 51 | - [X] The number of answers per relation 52 | - [X] Count the number of 'None' per relation 53 | - [X] Logic integrity 54 | - [X] Run for all prompts. 55 | - [ ] Report on performance difference. 56 | - [X] Submit current version to leadership board 57 | - [X] Look at failure cases 58 | - [X] Wrong formatting? :: We tried different formatting - no significant improvement. 59 | - [ ] Improve recall via 60 | - [ ] Reduce temperature and generate multiple samples (k=3?) 61 | - [ ] Rephrase 62 | prompts? :: [link to colab](https://colab.research.google.com/drive/180FCaZYRLEk0pPOWVYGsM_vn1UpIYy4N?usp=sharing#scrollTo=uuU4UPYDsSLP) 63 | - [ ] General improvements 64 | - [ ] Can we use the logprob? 65 | - [ ] Are we using other models? 66 | 67 | ## License 68 | 69 | Distributed under the MIT License. 70 | See [`LICENSE`]() for more information. 71 | 72 | ## Authors (Alphabetical order) 73 | 74 | * [Dimitrios Alivanistos](https://dimitrisalivas.github.io/) 75 | * [Selene Báez Santamaría](https://selbaez.github.io/) 76 | * [Michael Cochez](https://www.cochez.nl/) 77 | * [Jan-Christoph Kalo](https://github.com/JanKalo) 78 | * [Emile Van Krieken](https://emilevankrieken.com/) 79 | * [Thiviyan Thanapalasingam](https://thiviyansingam.com/) 80 | -------------------------------------------------------------------------------- /data/prompts_natural_language/ChemicalCompoundElement.txt: -------------------------------------------------------------------------------- 1 | What are all the atoms that make up the molecule Water? 2 | ['Hydrogen', 'Oxygen'] 3 | 4 | What are all the atoms that make up the molecule Bismuth subsalicylate? 5 | ['Bismuth'] 6 | 7 | What are all the atoms that make up the molecule Sodium Bicarbonate? 8 | ['Hydrogen', 'Oxygen', 'Sodium', 'Carbon'] 9 | 10 | What are all the atoms that make up the molecule Aspirin? 11 | ['Oxygen', 'Carbon', 'Hydrogen'] 12 | 13 | What are all the atoms that make up the molecule {subject_entity}? -------------------------------------------------------------------------------- /data/prompts_natural_language/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | What is the parent company of Microsoft? 2 | ['None'] 3 | 4 | What is the parent company of Sony? 5 | ['Sony Group'] 6 | 7 | What is the parent company of Saab? 8 | ['Saab Group', 'Saab-Scania', 'Spyker N.V.', 'National Electric Vehicle Sweden', 'General Motors'] 9 | 10 | What is the parent company of Max Motors? 11 | ['None'] 12 | 13 | What is the parent company of {subject_entity}? -------------------------------------------------------------------------------- /data/prompts_natural_language/CountryBordersWithCountry.txt: -------------------------------------------------------------------------------- 1 | Which countries neighbour Dominica? 2 | ['Venezuela'] 3 | 4 | Which countries neighbour North Korea? 5 | ['South Korea', 'China', 'Russia'] 6 | 7 | Which countries neighbour Serbia? 8 | ['Montenegro', 'Kosovo', 'Bosnia and Herzegovina', 'Hungary', 'Croatia', 'Bulgaria', 'Macedonia', 'Albania', 'Romania'] 9 | 10 | Which countries neighbour Fiji? 11 | ['None'] 12 | 13 | Which countries neighbour {subject_entity}? -------------------------------------------------------------------------------- /data/prompts_natural_language/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | Which are the official languages of Suriname? 2 | ['Dutch'] 3 | 4 | Which are the official languages of Canada? 5 | ['English', 'French'] 6 | 7 | Which are the official languages of Singapore? 8 | ['English', 'Malay', 'Mandarin', 'Tamil'] 9 | 10 | Which are the official languages of Sri Lanka? 11 | ['Sinhala', 'Tamil'] 12 | 13 | Which are the official languages of {subject_entity}? -------------------------------------------------------------------------------- /data/prompts_natural_language/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | How did André Leon Talley die? 2 | ['Infarction'] 3 | 4 | How did Angela Merkel die? 5 | ['None'] 6 | 7 | How did Bob Saget die? 8 | ['Injury', 'Blunt Trauma'] 9 | 10 | How did Jamal Khashoggi die? 11 | ['Murder'] 12 | 13 | How did {subject_entity} die? -------------------------------------------------------------------------------- /data/prompts_natural_language/PersonEmployer.txt: -------------------------------------------------------------------------------- 1 | Where is or was Susan Wojcicki employed? 2 | ['Google'] 3 | 4 | Where is or was Steve Wozniak employed? 5 | ['Apple Inc', 'Hewlett-Packard', 'University of Technology Sydney', 'Atari'] 6 | 7 | Where is or was Yukio Hatoyama employed? 8 | ['Senshu University','Tokyo Institute of Technology'] 9 | 10 | Where is or was Yahtzee Croshaw employed? 11 | ['PC Gamer', 'Hyper', 'Escapist'] 12 | 13 | Where is or was {subject_entity} employed? -------------------------------------------------------------------------------- /data/prompts_natural_language/PersonInstrument.txt: -------------------------------------------------------------------------------- 1 | Which instruments does Liam Gallagher play? 2 | ['Maraca', 'Guitar'] 3 | 4 | Which instruments does Jay Park play? 5 | ['None'] 6 | 7 | Which instruments does Axl Rose play? 8 | ['Guitar', 'Piano', 'Pander', 'Bass'] 9 | 10 | Which instruments does Neil Young play? 11 | ['Guitar'] 12 | 13 | Which instruments does {subject_entity} play? -------------------------------------------------------------------------------- /data/prompts_natural_language/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | Which languages does Aamir Khan speak? 2 | ['Hindi', 'English', 'Urdu'] 3 | 4 | Which languages does Pharrell Williams speak? 5 | ['English'] 6 | 7 | Which languages does Xabi Alonso speak? 8 | ['German', 'Basque', 'Spanish', 'English'] 9 | 10 | Which languages does Shakira speak? 11 | ['Catalan', 'English', 'Portuguese', 'Spanish', 'Italian', 'French'] 12 | 13 | Which languages does {subject_entity} speak? -------------------------------------------------------------------------------- /data/prompts_natural_language/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | What is the place of death of Barack Obama? 2 | ['None'] 3 | 4 | What is the place of death of Ennio Morricone? 5 | ['Rome'] 6 | 7 | What is the place of death of Elon Musk? 8 | ['None'] 9 | 10 | What is the place of death of Prince? 11 | ['Chanhassen'] 12 | 13 | What is the place of death of {subject_entity}? -------------------------------------------------------------------------------- /data/prompts_natural_language/PersonProfession.txt: -------------------------------------------------------------------------------- 1 | What is Danny DeVito's profession? 2 | ['Comedian', 'Film Director', 'Voice Actor', 'Actor', 'Film Producer', 'Film Actor', 'Dub Actor', 'Activist', 'Television Actor'] 3 | 4 | What is David Guetta's profession? 5 | ['DJ'] 6 | 7 | What is Gary Lineker's profession? 8 | ['Commentator', 'Association Football Player', 'Journalist', 'Broadcaster'] 9 | 10 | What is Gwyneth Paltrow's profession? 11 | ['Film Actor','Musician'] 12 | 13 | What is {subject_entity}'s profession? -------------------------------------------------------------------------------- /data/prompts_natural_language/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | What countries does the river Drava cross? 2 | ['Hungary', 'Italy', 'Austria', 'Slovenia', 'Croatia'] 3 | 4 | What countries does the river Huai river cross? 5 | ['China'] 6 | 7 | What countries does the river Paraná river cross? 8 | ['Bolivia', 'Paraguay', 'Argentina', 'Brazil'] 9 | 10 | What countries does the river Oise cross? 11 | ['Belgium', 'France'] 12 | 13 | What countries does the river {subject_entity} cross? -------------------------------------------------------------------------------- /data/prompts_natural_language/StateSharesBorderState.txt: -------------------------------------------------------------------------------- 1 | What states border San Marino? 2 | ['San Leo', 'Acquaviva', 'Borgo Maggiore', 'Chiesanuova', 'Fiorentino'] 3 | 4 | What states border Whales? 5 | ['England'] 6 | 7 | What states border Liguria? 8 | ['Tuscany', 'Auvergne-Rhoone-Alpes', 'Piedmont', 'Emilia-Romagna'] 9 | 10 | What states border Mecklenberg-Western Pomerania? 11 | ['Brandenburg', 'Pomeranian', 'Schleswig-Holstein', 'Lower Saxony'] 12 | 13 | What states border {subject_entity}? -------------------------------------------------------------------------------- /data/prompts_optimized/ChemicalCompoundElement.txt: -------------------------------------------------------------------------------- 1 | Water ChemicalCompoundElement: ['Hydrogen', 'Oxygen'] 2 | 3 | Bismuth subsalicylate ChemicalCompoundElement: ['Bismuth'] 4 | 5 | Sodium Bicarbonate ChemicalCompoundElement: ['Hydrogen', 'Oxygen', 'Sodium', 'Carbon'] 6 | 7 | Aspirin ChemicalCompoundElement: ['Oxygen', 'Carbon', 'Hydrogen'] 8 | 9 | {subject_entity} ChemicalCompoundElement: -------------------------------------------------------------------------------- /data/prompts_optimized/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | Microsoft CompanyParentOrganization: ['None'] 2 | 3 | Sony CompanyParentOrganization: ['Sony Group'] 4 | 5 | Saab CompanyParentOrganization: ['Saab Group', 'Saab-Scania', 'Spyker N.V.', 'National Electric Vehicle Sweden', 'General Motors'] 6 | 7 | Max Motors CompanyParentOrganization: ['None'] 8 | 9 | {subject_entity} CompanyParentOrganization: -------------------------------------------------------------------------------- /data/prompts_optimized/CountryBordersWithCountry.txt: -------------------------------------------------------------------------------- 1 | Which countries neighbour Dominica? 2 | ['Venezuela'] 3 | 4 | Which countries neighbour North Korea? 5 | ['South Korea', 'China', 'Russia'] 6 | 7 | Which countries neighbour Serbia? 8 | ['Montenegro', 'Kosovo', 'Bosnia and Herzegovina', 'Hungary', 'Croatia', 'Bulgaria', 'Macedonia', 'Albania', 'Romania'] 9 | 10 | Which countries neighbour Fiji? 11 | [] 12 | 13 | Which countries neighbour {subject_entity}? -------------------------------------------------------------------------------- /data/prompts_optimized/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | Suriname CountryOfficialLanguage: ['Dutch'] 2 | 3 | Canada CountryOfficialLanguage: ['English', 'French'] 4 | 5 | Singapore CountryOfficialLanguage: ['English', 'Malay', 'Mandarin', 'Tamil'] 6 | 7 | Sri Lanka CountryOfficialLanguage: ['Sinhala', 'Tamil'] 8 | 9 | {subject_entity} CountryOfficialLanguage: -------------------------------------------------------------------------------- /data/prompts_optimized/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | André Leon Talley PersonCauseOfDeath: ['Infarction'] 2 | 3 | Angela Merkel PersonCauseOfDeath: ['None'] 4 | 5 | Bob Saget PersonCauseOfDeath: ['Injury', 'Blunt Trauma'] 6 | 7 | Jamal Khashoggi PersonCauseOfDeath: ['Murder'] 8 | 9 | {subject_entity} PersonCauseOfDeath: -------------------------------------------------------------------------------- /data/prompts_optimized/PersonEmployer.txt: -------------------------------------------------------------------------------- 1 | Where is or was Susan Wojcicki employed? 2 | ['Google'] 3 | 4 | Where is or was Steve Wozniak employed? 5 | ['Apple Inc', 'Hewlett-Packard', 'University of Technology Sydney', 'Atari'] 6 | 7 | Where is or was Yukio Hatoyama employed? 8 | ['Senshu University','Tokyo Institute of Technology'] 9 | 10 | Where is or was Yahtzee Croshaw employed? 11 | ['PC Gamer', 'Hyper', 'Escapist'] 12 | 13 | Where is or was {subject_entity} employed? -------------------------------------------------------------------------------- /data/prompts_optimized/PersonInstrument.txt: -------------------------------------------------------------------------------- 1 | Liam Gallagher PersonInstrument: ['Maraca', 'Guitar'] 2 | 3 | Jay Park PersonInstrument: ['None'] 4 | 5 | Axl Rose PersonInstrument: ['Guitar', 'Piano', 'Pander', 'Bass'] 6 | 7 | Neil Young PersonInstrument: ['Guitar'] 8 | 9 | {subject_entity} PersonInstrument: -------------------------------------------------------------------------------- /data/prompts_optimized/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | Aamir Khan PersonLanguage: ['Hindi', 'English', 'Urdu'] 2 | 3 | Pharrell Williams PersonLanguage: ['English'] 4 | 5 | Xabi Alonso PersonLanguage: ['German', 'Basque', 'Spanish', 'English'] 6 | 7 | Shakira PersonLanguage: ['Catalan', 'English', 'Portuguese', 'Spanish', 'Italian', 'French'] 8 | 9 | {subject_entity} PersonLanguage: -------------------------------------------------------------------------------- /data/prompts_optimized/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | What is the place of death of Barack Obama? 2 | [] 3 | 4 | What is the place of death of Ennio Morricone? 5 | ['Rome'] 6 | 7 | What is the place of death of Elon Musk? 8 | [] 9 | 10 | What is the place of death of Prince? 11 | ['Chanhassen'] 12 | 13 | What is the place of death of {subject_entity}? -------------------------------------------------------------------------------- /data/prompts_optimized/PersonProfession.txt: -------------------------------------------------------------------------------- 1 | What is Danny DeVito's profession? 2 | ['Comedian', 'Film Director', 'Voice Actor', 'Actor', 'Film Producer', 'Film Actor', 'Dub Actor', 'Activist', 'Television Actor'] 3 | 4 | What is David Guetta's profession? 5 | ['DJ'] 6 | 7 | What is Gary Lineker's profession? 8 | ['Commentator', 'Association Football Player', 'Journalist', 'Broadcaster'] 9 | 10 | What is Gwyneth Paltrow's profession? 11 | ['Film Actor','Musician'] 12 | 13 | What is {subject_entity}'s profession? -------------------------------------------------------------------------------- /data/prompts_optimized/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | Drava RiverBasinsCountry: ['Hungary', 'Italy', 'Austria', 'Slovenia', 'Croatia'] 2 | 3 | Huai river RiverBasinsCountry: ['China'] 4 | 5 | Paraná river RiverBasinsCountry: ['Bolivia', 'Paraguay', 'Argentina', 'Brazil'] 6 | 7 | Oise RiverBasinsCountry: ['Belgium', 'France'] 8 | 9 | {subject_entity} RiverBasinsCountry: -------------------------------------------------------------------------------- /data/prompts_optimized/StateSharesBorderState.txt: -------------------------------------------------------------------------------- 1 | What states border San Marino? 2 | ['San Leo', 'Acquaviva', 'Borgo Maggiore', 'Chiesanuova', 'Fiorentino'] 3 | 4 | What states border Whales? 5 | ['England'] 6 | 7 | What states border Liguria? 8 | ['Tuscany', 'Auvergne-Rhoone-Alpes', 'Piedmont', 'Emilia-Romagna'] 9 | 10 | What states border Mecklenberg-Western Pomerania? 11 | ['Brandenburg', 'Pomeranian', 'Schleswig-Holstein', 'Lower Saxony'] 12 | 13 | What states border {subject_entity}? -------------------------------------------------------------------------------- /data/prompts_triple_based/ChemicalCompoundElement.txt: -------------------------------------------------------------------------------- 1 | Water ChemicalCompoundElement: ['Hydrogen', 'Oxygen'] 2 | 3 | Bismuth subsalicylate ChemicalCompoundElement: ['Bismuth'] 4 | 5 | Sodium Bicarbonate ChemicalCompoundElement: ['Hydrogen', 'Oxygen', 'Sodium', 'Carbon'] 6 | 7 | Aspirin ChemicalCompoundElement: ['Oxygen', 'Carbon', 'Hydrogen'] 8 | 9 | {subject_entity} ChemicalCompoundElement: -------------------------------------------------------------------------------- /data/prompts_triple_based/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | Microsoft CompanyParentOrganization: ['None'] 2 | 3 | Sony CompanyParentOrganization: ['Sony Group'] 4 | 5 | Saab CompanyParentOrganization: ['Saab Group', 'Saab-Scania', 'Spyker N.V.', 'National Electric Vehicle Sweden', 'General Motors'] 6 | 7 | Max Motors CompanyParentOrganization: ['None'] 8 | 9 | {subject_entity} CompanyParentOrganization: -------------------------------------------------------------------------------- /data/prompts_triple_based/CountryBordersWithCountry.txt: -------------------------------------------------------------------------------- 1 | Dominica CountryBordersWithCountry: ['Venezuela'] 2 | 3 | North Korea CountryBordersWithCountry: ['South Korea', 'China', 'Russia'] 4 | 5 | Serbia CountryBordersWithCountry: ['Montenegro', 'Kosovo', 'Bosnia and Herzegovina', 'Hungary', 'Croatia', 'Bulgaria', 'Macedonia', 'Albania', 'Romania'] 6 | 7 | Fiji CountryBordersWithCountry: ['None'] 8 | 9 | {subject_entity} CountryBordersWithCountry: -------------------------------------------------------------------------------- /data/prompts_triple_based/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | Suriname CountryOfficialLanguage: ['Dutch'] 2 | 3 | Canada CountryOfficialLanguage: ['English', 'French'] 4 | 5 | Singapore CountryOfficialLanguage: ['English', 'Malay', 'Mandarin', 'Tamil'] 6 | 7 | Sri Lanka CountryOfficialLanguage: ['Sinhala', 'Tamil'] 8 | 9 | {subject_entity} CountryOfficialLanguage: -------------------------------------------------------------------------------- /data/prompts_triple_based/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | André Leon Talley PersonCauseOfDeath: ['Infarction'] 2 | 3 | Angela Merkel PersonCauseOfDeath: ['None'] 4 | 5 | Bob Saget PersonCauseOfDeath: ['Injury', 'Blunt Trauma'] 6 | 7 | Jamal Khashoggi PersonCauseOfDeath: ['Murder'] 8 | 9 | {subject_entity} PersonCauseOfDeath: -------------------------------------------------------------------------------- /data/prompts_triple_based/PersonEmployer.txt: -------------------------------------------------------------------------------- 1 | Susan Wojcicki PersonEmployer: ['Google'] 2 | 3 | Steve Wozniak PersonEmployer: ['Apple Inc', 'Hewlett-Packard', 'University of Technology Sydney', 'Atari'] 4 | 5 | Yukio Hatoyama PersonEmployer: ['Senshu University','Tokyo Institute of Technology'] 6 | 7 | Yahtzee Croshaw PersonEmployer: ['PC Gamer', 'Hyper', 'Escapist'] 8 | 9 | {subject_entity} PersonEmployer: -------------------------------------------------------------------------------- /data/prompts_triple_based/PersonInstrument.txt: -------------------------------------------------------------------------------- 1 | Liam Gallagher PersonInstrument: ['Maraca', 'Guitar'] 2 | 3 | Jay Park PersonInstrument: ['None'] 4 | 5 | Axl Rose PersonInstrument: ['Guitar', 'Piano', 'Pander', 'Bass'] 6 | 7 | Neil Young PersonInstrument: ['Guitar'] 8 | 9 | {subject_entity} PersonInstrument: -------------------------------------------------------------------------------- /data/prompts_triple_based/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | Aamir Khan PersonLanguage: ['Hindi', 'English', 'Urdu'] 2 | 3 | Pharrell Williams PersonLanguage: ['English'] 4 | 5 | Xabi Alonso PersonLanguage: ['German', 'Basque', 'Spanish', 'English'] 6 | 7 | Shakira PersonLanguage: ['Catalan', 'English', 'Portuguese', 'Spanish', 'Italian', 'French'] 8 | 9 | {subject_entity} PersonLanguage: -------------------------------------------------------------------------------- /data/prompts_triple_based/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | Barack Obama PersonPlaceOfDeath: ['None'] 2 | 3 | Ennio Morricone PersonPlaceOfDeath: ['Rome'] 4 | 5 | Elon Musk PersonPlaceOfDeath: ['None'] 6 | 7 | Prince PersonPlaceOfDeath: ['Chanhassen'] 8 | 9 | {subject_entity} PersonPlaceOfDeath: -------------------------------------------------------------------------------- /data/prompts_triple_based/PersonProfession.txt: -------------------------------------------------------------------------------- 1 | Danny DeVito PersonProfession: ['Comedian', 'Film Director', 'Voice Actor', 'Actor', 'Film Producer', 'Film Actor', 'Dub Actor', 'Activist', 'Television Actor'] 2 | 3 | David Guetta PersonProfession: ['DJ'] 4 | 5 | Gary Lineker PersonProfession: ['Commentator', 'Association Football Player', 'Journalist', 'Broadcaster'] 6 | 7 | Gwyneth Paltrow PersonProfession: ['Film Actor','Musician'] 8 | 9 | {subject_entity} PersonProfession: -------------------------------------------------------------------------------- /data/prompts_triple_based/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | Drava RiverBasinsCountry: ['Hungary', 'Italy', 'Austria', 'Slovenia', 'Croatia'] 2 | 3 | Huai river RiverBasinsCountry: ['China'] 4 | 5 | Paraná river RiverBasinsCountry: ['Bolivia', 'Paraguay', 'Argentina', 'Brazil'] 6 | 7 | Oise RiverBasinsCountry: ['Belgium', 'France'] 8 | 9 | {subject_entity} RiverBasinsCountry: -------------------------------------------------------------------------------- /data/prompts_triple_based/StateSharesBorderState.txt: -------------------------------------------------------------------------------- 1 | San Marino StateSharesBorderState: ['San Leo', 'Acquaviva', 'Borgo Maggiore', 'Chiesanuova', 'Fiorentino'] 2 | 3 | Whales StateSharesBorderState: ['England'] 4 | 5 | Liguria StateSharesBorderState: ['Tuscany', 'Auvergne-Rhoone-Alpes', 'Piedmont', 'Emilia-Romagna'] 6 | 7 | Mecklenberg-Western Pomerania StateSharesBorderState: ['Brandenburg', 'Pomeranian', 'Schleswig-Holstein', 'Lower Saxony'] 8 | 9 | {subject_entity} StateSharesBorderState: -------------------------------------------------------------------------------- /failure_cases/baseline/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | CompanyParentOrganization (average f1: 0.68): 16 cases 2 | 3 | 4 | 5 | SubjectEntity: Cadillac 6 | Ground Truth: ['general motors'] 7 | GPT-3 Prediction: [] 8 | 9 | 10 | SubjectEntity: Changan Automobile 11 | Ground Truth: ['china south industries', 'china south industries group'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Gaz 16 | Ground Truth: ['gaz', 'gaz group'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Gucci 21 | Ground Truth: ['kering'] 22 | GPT-3 Prediction: [] 23 | 24 | 25 | SubjectEntity: Hanomag 26 | Ground Truth: ['komatsu', 'komatsu limited'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Hyundai Motor Company 31 | Ground Truth: ['hyundai'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Kia 36 | Ground Truth: ['hyundai', 'hyundai motor', 'hyundai motor group'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Lada Zapad Tolyatti 41 | Ground Truth: ['avtovaz'] 42 | GPT-3 Prediction: [] 43 | 44 | 45 | SubjectEntity: Lagonda 46 | Ground Truth: ['aston', 'aston martin', 'aston martin lagonda', 'martin'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Lotus Cars 51 | Ground Truth: ['geely'] 52 | GPT-3 Prediction: ['lotus'] 53 | 54 | 55 | SubjectEntity: NSU 56 | Ground Truth: ['volkswagen'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Peugeot 61 | Ground Truth: ['stellantis'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Scripps-Booth 66 | Ground Truth: ['general motors', 'gm'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: Simca 71 | Ground Truth: ['chrysler', 'chrysler europe'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Volkswagen 76 | Ground Truth: ['volkswagen', 'volkswagen group'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Yamaha Motor Company 81 | Ground Truth: ['yamaha', 'yamaha corporation'] 82 | GPT-3 Prediction: ['japan'] 83 | 84 | 85 | -------------------------------------------------------------------------------- /failure_cases/baseline/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | CountryOfficialLanguage (average f1: 0.752): 20 cases 2 | 3 | 4 | 5 | SubjectEntity: Afghanistan 6 | Ground Truth: ['arabic', 'baluchi', 'dari', 'nuristani', 'pamir', 'pashayi', 'pashto', 'turkmen', 'uzbek'] 7 | GPT-3 Prediction: ['persian'] 8 | 9 | 10 | SubjectEntity: Brunei 11 | Ground Truth: ['english', 'malay'] 12 | GPT-3 Prediction: ['english'] 13 | 14 | 15 | SubjectEntity: Burundi 16 | Ground Truth: ['english', 'french', 'kirundi'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Cameroon 21 | Ground Truth: ['english', 'french'] 22 | GPT-3 Prediction: ['french'] 23 | 24 | 25 | SubjectEntity: Central African Republic 26 | Ground Truth: ['french', 'sango'] 27 | GPT-3 Prediction: ['french'] 28 | 29 | 30 | SubjectEntity: East Timor 31 | Ground Truth: ['portuguese', 'tetum'] 32 | GPT-3 Prediction: ['indonesian'] 33 | 34 | 35 | SubjectEntity: Finland 36 | Ground Truth: ['finnish', 'swedish'] 37 | GPT-3 Prediction: ['finnish'] 38 | 39 | 40 | SubjectEntity: Grenada 41 | Ground Truth: ['creole', 'english', 'grenadian', 'grenadian creole english'] 42 | GPT-3 Prediction: ['english'] 43 | 44 | 45 | SubjectEntity: India 46 | Ground Truth: ['english', 'hindi'] 47 | GPT-3 Prediction: ['hindi'] 48 | 49 | 50 | SubjectEntity: Kazakhstan 51 | Ground Truth: ['kazakh', 'russian'] 52 | GPT-3 Prediction: ['russian'] 53 | 54 | 55 | SubjectEntity: Maldives 56 | Ground Truth: ['maldivian'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: North Macedonia 61 | Ground Truth: ['albanian', 'macedonian'] 62 | GPT-3 Prediction: ['macedonian'] 63 | 64 | 65 | SubjectEntity: Norway 66 | Ground Truth: ['bokmal', 'bokmål', 'norwegian', 'nynorsk', 'sami', 'sámi'] 67 | GPT-3 Prediction: ['norwegian'] 68 | 69 | 70 | SubjectEntity: Pakistan 71 | Ground Truth: ['arabic', 'english', 'urdu'] 72 | GPT-3 Prediction: ['urdu'] 73 | 74 | 75 | SubjectEntity: People's Republic of China 76 | Ground Truth: ['chinese', 'mandarin', 'standard mandarin'] 77 | GPT-3 Prediction: ['mandarin'] 78 | 79 | 80 | SubjectEntity: Philippines 81 | Ground Truth: ['english', 'filipino'] 82 | GPT-3 Prediction: ['filipino'] 83 | 84 | 85 | SubjectEntity: Rwanda 86 | Ground Truth: ['english', 'french', 'kinyarwanda', 'swahili'] 87 | GPT-3 Prediction: ['english'] 88 | 89 | 90 | SubjectEntity: Taiwan 91 | Ground Truth: ['amis', 'austronesian', 'formosan', 'hakka', 'hokkien', 'mandarin', 'paiwan', 'standard taiwanese mandarin', 'taiwanese', 'taiwanese', 'taiwanese hokkien'] 92 | GPT-3 Prediction: ['mandarin'] 93 | 94 | 95 | SubjectEntity: Tajikistan 96 | Ground Truth: ['russian', 'tajik'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Zimbabwe 101 | Ground Truth: ['barwe', 'chewa', 'english', 'kalanga', 'khoisan', 'nambya', 'ndau', 'ndebele', 'northern ndebele', 'sesotho', 'shona', 'tonga', 'tsonga', 'tswana', 'venda', 'xhosa'] 102 | GPT-3 Prediction: ['english'] 103 | 104 | 105 | -------------------------------------------------------------------------------- /failure_cases/baseline/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonCauseOfDeath (average f1: 0.42): 29 cases 2 | 3 | 4 | 5 | SubjectEntity: Ahmed Zewail 6 | Ground Truth: ['lymphoma', 'spinal cord lymphoma'] 7 | GPT-3 Prediction: [] 8 | 9 | 10 | SubjectEntity: Avicii 11 | Ground Truth: ['exsanguination'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Benjamin Netanyahu 16 | Ground Truth: [] 17 | GPT-3 Prediction: ['cancer'] 18 | 19 | 20 | SubjectEntity: Boris Johnson 21 | Ground Truth: [] 22 | GPT-3 Prediction: ['cancer'] 23 | 24 | 25 | SubjectEntity: Christina Grimmie 26 | Ground Truth: ['ballistic trauma', 'trauma'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Cory Monteith 31 | Ground Truth: ['drug overdose', 'overdose'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Dilip Kumar 36 | Ground Truth: ['cancer', 'prostate cancer'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Ennio Morricone 41 | Ground Truth: ['femoral fracture', 'fracture'] 42 | GPT-3 Prediction: [] 43 | 44 | 45 | SubjectEntity: Frank Sinatra 46 | Ground Truth: ['infarction', 'myocardial infarction'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: George H. W. Bush 51 | Ground Truth: ['parkinson', "parkinson's disease"] 52 | GPT-3 Prediction: ['cancer'] 53 | 54 | 55 | SubjectEntity: George Harrison 56 | Ground Truth: ['cancer', 'lung cancer'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Haruma Miura 61 | Ground Truth: ['hanging'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Hu Jintao 66 | Ground Truth: [] 67 | GPT-3 Prediction: ['illness'] 68 | 69 | 70 | SubjectEntity: Hugh Hefner 71 | Ground Truth: ['sepsis'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Idriss Déby 76 | Ground Truth: ['ballistic trauma', 'trauma'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Jim Rohn 81 | Ground Truth: ['fibrosis', 'pulmonary fibrosis'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | SubjectEntity: John Magufuli 86 | Ground Truth: ['arrhythmia', 'heart arrhythmia'] 87 | GPT-3 Prediction: [] 88 | 89 | 90 | SubjectEntity: Johnny Cash 91 | Ground Truth: ['diabetes'] 92 | GPT-3 Prediction: [] 93 | 94 | 95 | SubjectEntity: Larry King 96 | Ground Truth: ['sepsis'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Maurane 101 | Ground Truth: ['falling'] 102 | GPT-3 Prediction: [] 103 | 104 | 105 | SubjectEntity: Maya Angelou 106 | Ground Truth: ['disease'] 107 | GPT-3 Prediction: [] 108 | 109 | 110 | SubjectEntity: Michelle Obama 111 | Ground Truth: [] 112 | GPT-3 Prediction: ['cancer'] 113 | 114 | 115 | SubjectEntity: Mikhail Zadornov 116 | Ground Truth: ['brain cancer', 'cancer'] 117 | GPT-3 Prediction: [] 118 | 119 | 120 | SubjectEntity: Miles Davis 121 | Ground Truth: ['pneumonia', 'stroke'] 122 | GPT-3 Prediction: [] 123 | 124 | 125 | SubjectEntity: Nelson Mandela 126 | Ground Truth: ['respiratory disease'] 127 | GPT-3 Prediction: [] 128 | 129 | 130 | SubjectEntity: Rishi Kapoor 131 | Ground Truth: ['leukemia'] 132 | GPT-3 Prediction: [] 133 | 134 | 135 | SubjectEntity: Roddy Piper 136 | Ground Truth: ['cardiac arrest'] 137 | GPT-3 Prediction: [] 138 | 139 | 140 | SubjectEntity: Roy Orbison 141 | Ground Truth: ['infarction', 'myocardial infarction'] 142 | GPT-3 Prediction: [] 143 | 144 | 145 | SubjectEntity: Sandra Bullock 146 | Ground Truth: [] 147 | GPT-3 Prediction: ['cancer'] 148 | 149 | 150 | -------------------------------------------------------------------------------- /failure_cases/baseline/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | PersonLanguage (average f1: 0.431): 27 cases 2 | 3 | 4 | 5 | SubjectEntity: A. R. Rahman 6 | Ground Truth: ['hindi', 'tamil', 'telugu'] 7 | GPT-3 Prediction: ['english'] 8 | 9 | 10 | SubjectEntity: Adele 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Akon 16 | Ground Truth: ['english'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Avril Lavigne 21 | Ground Truth: ['english'] 22 | GPT-3 Prediction: ['french'] 23 | 24 | 25 | SubjectEntity: Barack Obama 26 | Ground Truth: ['english', 'indonesian'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Bill Clinton 31 | Ground Truth: ['english'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Donald Trump 36 | Ground Truth: ['english'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Donald Tusk 41 | Ground Truth: ['english', 'german', 'polish'] 42 | GPT-3 Prediction: [] 43 | 44 | 45 | SubjectEntity: Emma Watson 46 | Ground Truth: ['english', 'french'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Genelia D'Souza 51 | Ground Truth: ['hindi', 'malayalam', 'tamil', 'telugu'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Henrique Capriles Radonski 56 | Ground Truth: ['spanish'] 57 | GPT-3 Prediction: ['portuguese'] 58 | 59 | 60 | SubjectEntity: Hugo Chávez 61 | Ground Truth: ['spanish'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Jessie J 66 | Ground Truth: ['english'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: Joe Biden 71 | Ground Truth: ['english'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: John Cena 76 | Ground Truth: ['chinese', 'english', 'mandarin', 'mandarin chinese'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Kobe Bryant 81 | Ground Truth: ['english', 'italian', 'spanish'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | SubjectEntity: Novak Djokovic 86 | Ground Truth: ['english', 'french', 'german', 'italian', 'serbian'] 87 | GPT-3 Prediction: [] 88 | 89 | 90 | SubjectEntity: Paris Hilton 91 | Ground Truth: ['english'] 92 | GPT-3 Prediction: [] 93 | 94 | 95 | SubjectEntity: Pau Gasol 96 | Ground Truth: ['catalan', 'english', 'french', 'italian', 'spanish'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Paulo Coelho 101 | Ground Truth: ['portuguese'] 102 | GPT-3 Prediction: [] 103 | 104 | 105 | SubjectEntity: Sachin 106 | Ground Truth: ['english', 'hindi', 'marathi'] 107 | GPT-3 Prediction: [] 108 | 109 | 110 | SubjectEntity: Salman Khan 111 | Ground Truth: ['hindi'] 112 | GPT-3 Prediction: ['english'] 113 | 114 | 115 | SubjectEntity: Samir Nasri 116 | Ground Truth: ['french'] 117 | GPT-3 Prediction: [] 118 | 119 | 120 | SubjectEntity: Skrillex 121 | Ground Truth: ['english'] 122 | GPT-3 Prediction: [] 123 | 124 | 125 | SubjectEntity: Thiago Alcântara 126 | Ground Truth: ['spanish'] 127 | GPT-3 Prediction: ['portuguese'] 128 | 129 | 130 | SubjectEntity: Vladimir Putin 131 | Ground Truth: ['english', 'german', 'russian', 'swedish'] 132 | GPT-3 Prediction: ['russian'] 133 | 134 | 135 | SubjectEntity: Zlatan Ibrahimović 136 | Ground Truth: ['bosnian', 'english', 'italian', 'spanish', 'swedish'] 137 | GPT-3 Prediction: [] 138 | 139 | 140 | -------------------------------------------------------------------------------- /failure_cases/baseline/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonPlaceOfDeath (average f1: 0.5): 25 cases 2 | 3 | 4 | 5 | SubjectEntity: Abdul Rehman Malik 6 | Ground Truth: ['islamabad'] 7 | GPT-3 Prediction: [] 8 | 9 | 10 | SubjectEntity: Adam West 11 | Ground Truth: ['la', 'los angeles'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Alan García 16 | Ground Truth: ['lima'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Alfredo González Flores 21 | Ground Truth: ['heredia'] 22 | GPT-3 Prediction: [] 23 | 24 | 25 | SubjectEntity: Almudena Grandes 26 | Ground Truth: ['madrid'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Anne Rice 31 | Ground Truth: ['rancho mirage'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Aretha Franklin 36 | Ground Truth: ['detroit'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Avicii 41 | Ground Truth: ['muscat'] 42 | GPT-3 Prediction: ['rome'] 43 | 44 | 45 | SubjectEntity: Cameron Boyce 46 | Ground Truth: ['la', 'los angeles'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Chespirito 51 | Ground Truth: ['cancun'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Chick Corea 56 | Ground Truth: ['tampa', 'tampa bay area'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Chris Cornell 61 | Ground Truth: ['detroit'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Elijah Cummings 66 | Ground Truth: ['baltimore'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: George H. W. Bush 71 | Ground Truth: ['houston'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Glenn Fredly 76 | Ground Truth: ['jakarta'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Hugo Chávez 81 | Ground Truth: ['caracas', 'hospital militar de caracas'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | SubjectEntity: John Coltrane 86 | Ground Truth: ['huntington'] 87 | GPT-3 Prediction: [] 88 | 89 | 90 | SubjectEntity: John Lennon 91 | Ground Truth: ['new york city', 'nyc'] 92 | GPT-3 Prediction: [] 93 | 94 | 95 | SubjectEntity: Ken Robinson 96 | Ground Truth: ['london'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Kurt Vonnegut 101 | Ground Truth: ['new york city', 'nyc'] 102 | GPT-3 Prediction: [] 103 | 104 | 105 | SubjectEntity: Michael Collins 106 | Ground Truth: ['naples'] 107 | GPT-3 Prediction: [] 108 | 109 | 110 | SubjectEntity: Michael Jackson 111 | Ground Truth: ['la', 'los angeles'] 112 | GPT-3 Prediction: [] 113 | 114 | 115 | SubjectEntity: Paul G. Allen 116 | Ground Truth: ['seattle'] 117 | GPT-3 Prediction: [] 118 | 119 | 120 | SubjectEntity: Rachel Caine 121 | Ground Truth: ['texas'] 122 | GPT-3 Prediction: [] 123 | 124 | 125 | SubjectEntity: Roald Dahl 126 | Ground Truth: ['oxford'] 127 | GPT-3 Prediction: [] 128 | 129 | 130 | -------------------------------------------------------------------------------- /failure_cases/baseline/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | RiverBasinsCountry (average f1: 0.381): 27 cases 2 | 3 | 4 | 5 | SubjectEntity: Alagón river 6 | Ground Truth: ['spain'] 7 | GPT-3 Prediction: [] 8 | 9 | 10 | SubjectEntity: Apurímac river 11 | Ground Truth: ['peru'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Brenta 16 | Ground Truth: ['italy'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Dahme 21 | Ground Truth: ['germany'] 22 | GPT-3 Prediction: [] 23 | 24 | 25 | SubjectEntity: Danube 26 | Ground Truth: ['austria', 'bulgaria', 'croatia', 'germany', 'hungary', 'moldova', 'romania', 'serbia', 'slovakia', 'ukraine'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Didi Liakhvi river 31 | Ground Truth: ['georgia'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Gadar river 36 | Ground Truth: ['iran'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Gambia river 41 | Ground Truth: ['gambia', 'guinea', 'senegal', 'the gambia'] 42 | GPT-3 Prediction: [] 43 | 44 | 45 | SubjectEntity: Han river 46 | Ground Truth: ['korea', 'korea', 'north korea', 'south korea'] 47 | GPT-3 Prediction: ['china'] 48 | 49 | 50 | SubjectEntity: Inn 51 | Ground Truth: ['austria', 'germany', 'switzerland'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Kura river 56 | Ground Truth: ['azerbaijan', 'georgia', 'turkey'] 57 | GPT-3 Prediction: ['russia'] 58 | 59 | 60 | SubjectEntity: Lielupe 61 | Ground Truth: ['latvia', 'lithuania'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Limpopo river 66 | Ground Truth: ['africa', 'botswana', 'mozambique', 'south africa', 'zimbabwe'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: Morava 71 | Ground Truth: ['austria', 'czech republic', 'czechia', 'slovakia'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Neman 76 | Ground Truth: ['belarus', 'latvia', 'lithuania', 'russia'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Neris 81 | Ground Truth: ['belarus', 'latvia', 'lithuania'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | SubjectEntity: Niger river 86 | Ground Truth: ['benin', 'guinea', 'mali', 'niger', 'nigeria'] 87 | GPT-3 Prediction: [] 88 | 89 | 90 | SubjectEntity: Paraguay river 91 | Ground Truth: ['argentina', 'brazil', 'paraguay'] 92 | GPT-3 Prediction: [] 93 | 94 | 95 | SubjectEntity: Red river of the south 96 | Ground Truth: ['texas', 'united states of america', 'usa'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Sakarya river 101 | Ground Truth: ['turkey'] 102 | GPT-3 Prediction: [] 103 | 104 | 105 | SubjectEntity: San 106 | Ground Truth: ['poland', 'ukraine'] 107 | GPT-3 Prediction: [] 108 | 109 | 110 | SubjectEntity: Shatt al-Arab 111 | Ground Truth: ['iran', 'iraq', 'kuwait', 'saudi arabia', 'syria', 'turkey'] 112 | GPT-3 Prediction: [] 113 | 114 | 115 | SubjectEntity: Siret river 116 | Ground Truth: ['romania', 'ukraine'] 117 | GPT-3 Prediction: [] 118 | 119 | 120 | SubjectEntity: Tisza 121 | Ground Truth: ['hungary', 'romania', 'serbia', 'slovakia', 'ukraine'] 122 | GPT-3 Prediction: ['hungary'] 123 | 124 | 125 | SubjectEntity: Torne river 126 | Ground Truth: ['finland', 'norway', 'sweden'] 127 | GPT-3 Prediction: [] 128 | 129 | 130 | SubjectEntity: Uruguay river 131 | Ground Truth: ['argentina', 'brazil', 'uruguay'] 132 | GPT-3 Prediction: [] 133 | 134 | 135 | SubjectEntity: Yantra 136 | Ground Truth: ['bulgaria'] 137 | GPT-3 Prediction: [] 138 | 139 | 140 | -------------------------------------------------------------------------------- /failure_cases/baseline_factcheck/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | CompanyParentOrganization (average f1: 0.68): 16 cases 2 | 3 | 4 | 5 | SubjectEntity: Cadillac 6 | Ground Truth: ['general motors'] 7 | GPT-3 Prediction: [] 8 | 9 | 10 | SubjectEntity: Changan Automobile 11 | Ground Truth: ['china south industries', 'china south industries group'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Gaz 16 | Ground Truth: ['gaz', 'gaz group'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Gucci 21 | Ground Truth: ['kering'] 22 | GPT-3 Prediction: [] 23 | 24 | 25 | SubjectEntity: Hanomag 26 | Ground Truth: ['komatsu', 'komatsu limited'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Hyundai Motor Company 31 | Ground Truth: ['hyundai'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Kia 36 | Ground Truth: ['hyundai', 'hyundai motor', 'hyundai motor group'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Lada Zapad Tolyatti 41 | Ground Truth: ['avtovaz'] 42 | GPT-3 Prediction: [] 43 | 44 | 45 | SubjectEntity: Lagonda 46 | Ground Truth: ['aston', 'aston martin', 'aston martin lagonda', 'martin'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Lotus Cars 51 | Ground Truth: ['geely'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: NSU 56 | Ground Truth: ['volkswagen'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Peugeot 61 | Ground Truth: ['stellantis'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Scripps-Booth 66 | Ground Truth: ['general motors', 'gm'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: Simca 71 | Ground Truth: ['chrysler', 'chrysler europe'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Volkswagen 76 | Ground Truth: ['volkswagen', 'volkswagen group'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Yamaha Motor Company 81 | Ground Truth: ['yamaha', 'yamaha corporation'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | -------------------------------------------------------------------------------- /failure_cases/baseline_factcheck/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | CountryOfficialLanguage (average f1: 0.752): 20 cases 2 | 3 | 4 | 5 | SubjectEntity: Afghanistan 6 | Ground Truth: ['arabic', 'baluchi', 'dari', 'nuristani', 'pamir', 'pashayi', 'pashto', 'turkmen', 'uzbek'] 7 | GPT-3 Prediction: ['persian'] 8 | 9 | 10 | SubjectEntity: Brunei 11 | Ground Truth: ['english', 'malay'] 12 | GPT-3 Prediction: ['english'] 13 | 14 | 15 | SubjectEntity: Burundi 16 | Ground Truth: ['english', 'french', 'kirundi'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Cameroon 21 | Ground Truth: ['english', 'french'] 22 | GPT-3 Prediction: ['french'] 23 | 24 | 25 | SubjectEntity: Central African Republic 26 | Ground Truth: ['french', 'sango'] 27 | GPT-3 Prediction: ['french'] 28 | 29 | 30 | SubjectEntity: East Timor 31 | Ground Truth: ['portuguese', 'tetum'] 32 | GPT-3 Prediction: ['indonesian'] 33 | 34 | 35 | SubjectEntity: Finland 36 | Ground Truth: ['finnish', 'swedish'] 37 | GPT-3 Prediction: ['finnish'] 38 | 39 | 40 | SubjectEntity: Grenada 41 | Ground Truth: ['creole', 'english', 'grenadian', 'grenadian creole english'] 42 | GPT-3 Prediction: ['english'] 43 | 44 | 45 | SubjectEntity: India 46 | Ground Truth: ['english', 'hindi'] 47 | GPT-3 Prediction: ['hindi'] 48 | 49 | 50 | SubjectEntity: Kazakhstan 51 | Ground Truth: ['kazakh', 'russian'] 52 | GPT-3 Prediction: ['russian'] 53 | 54 | 55 | SubjectEntity: Maldives 56 | Ground Truth: ['maldivian'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: North Macedonia 61 | Ground Truth: ['albanian', 'macedonian'] 62 | GPT-3 Prediction: ['macedonian'] 63 | 64 | 65 | SubjectEntity: Norway 66 | Ground Truth: ['bokmal', 'bokmål', 'norwegian', 'nynorsk', 'sami', 'sámi'] 67 | GPT-3 Prediction: ['norwegian'] 68 | 69 | 70 | SubjectEntity: Pakistan 71 | Ground Truth: ['arabic', 'english', 'urdu'] 72 | GPT-3 Prediction: ['urdu'] 73 | 74 | 75 | SubjectEntity: People's Republic of China 76 | Ground Truth: ['chinese', 'mandarin', 'standard mandarin'] 77 | GPT-3 Prediction: ['mandarin'] 78 | 79 | 80 | SubjectEntity: Philippines 81 | Ground Truth: ['english', 'filipino'] 82 | GPT-3 Prediction: ['filipino'] 83 | 84 | 85 | SubjectEntity: Rwanda 86 | Ground Truth: ['english', 'french', 'kinyarwanda', 'swahili'] 87 | GPT-3 Prediction: ['english'] 88 | 89 | 90 | SubjectEntity: Taiwan 91 | Ground Truth: ['amis', 'austronesian', 'formosan', 'hakka', 'hokkien', 'mandarin', 'paiwan', 'standard taiwanese mandarin', 'taiwanese', 'taiwanese', 'taiwanese hokkien'] 92 | GPT-3 Prediction: ['mandarin'] 93 | 94 | 95 | SubjectEntity: Tajikistan 96 | Ground Truth: ['russian', 'tajik'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Zimbabwe 101 | Ground Truth: ['barwe', 'chewa', 'english', 'kalanga', 'khoisan', 'nambya', 'ndau', 'ndebele', 'northern ndebele', 'sesotho', 'shona', 'tonga', 'tsonga', 'tswana', 'venda', 'xhosa'] 102 | GPT-3 Prediction: ['english'] 103 | 104 | 105 | -------------------------------------------------------------------------------- /failure_cases/baseline_factcheck/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonCauseOfDeath (average f1: 0.52): 24 cases 2 | 3 | 4 | 5 | SubjectEntity: Ahmed Zewail 6 | Ground Truth: ['lymphoma', 'spinal cord lymphoma'] 7 | GPT-3 Prediction: [] 8 | 9 | 10 | SubjectEntity: Avicii 11 | Ground Truth: ['exsanguination'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Christina Grimmie 16 | Ground Truth: ['ballistic trauma', 'trauma'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Cory Monteith 21 | Ground Truth: ['drug overdose', 'overdose'] 22 | GPT-3 Prediction: [] 23 | 24 | 25 | SubjectEntity: Dilip Kumar 26 | Ground Truth: ['cancer', 'prostate cancer'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Ennio Morricone 31 | Ground Truth: ['femoral fracture', 'fracture'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Frank Sinatra 36 | Ground Truth: ['infarction', 'myocardial infarction'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: George H. W. Bush 41 | Ground Truth: ['parkinson', "parkinson's disease"] 42 | GPT-3 Prediction: ['cancer'] 43 | 44 | 45 | SubjectEntity: George Harrison 46 | Ground Truth: ['cancer', 'lung cancer'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Haruma Miura 51 | Ground Truth: ['hanging'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Hugh Hefner 56 | Ground Truth: ['sepsis'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Idriss Déby 61 | Ground Truth: ['ballistic trauma', 'trauma'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Jim Rohn 66 | Ground Truth: ['fibrosis', 'pulmonary fibrosis'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: John Magufuli 71 | Ground Truth: ['arrhythmia', 'heart arrhythmia'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Johnny Cash 76 | Ground Truth: ['diabetes'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Larry King 81 | Ground Truth: ['sepsis'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | SubjectEntity: Maurane 86 | Ground Truth: ['falling'] 87 | GPT-3 Prediction: [] 88 | 89 | 90 | SubjectEntity: Maya Angelou 91 | Ground Truth: ['disease'] 92 | GPT-3 Prediction: [] 93 | 94 | 95 | SubjectEntity: Mikhail Zadornov 96 | Ground Truth: ['brain cancer', 'cancer'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Miles Davis 101 | Ground Truth: ['pneumonia', 'stroke'] 102 | GPT-3 Prediction: [] 103 | 104 | 105 | SubjectEntity: Nelson Mandela 106 | Ground Truth: ['respiratory disease'] 107 | GPT-3 Prediction: [] 108 | 109 | 110 | SubjectEntity: Rishi Kapoor 111 | Ground Truth: ['leukemia'] 112 | GPT-3 Prediction: [] 113 | 114 | 115 | SubjectEntity: Roddy Piper 116 | Ground Truth: ['cardiac arrest'] 117 | GPT-3 Prediction: [] 118 | 119 | 120 | SubjectEntity: Roy Orbison 121 | Ground Truth: ['infarction', 'myocardial infarction'] 122 | GPT-3 Prediction: [] 123 | 124 | 125 | -------------------------------------------------------------------------------- /failure_cases/baseline_factcheck/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | PersonLanguage (average f1: 0.431): 27 cases 2 | 3 | 4 | 5 | SubjectEntity: A. R. Rahman 6 | Ground Truth: ['hindi', 'tamil', 'telugu'] 7 | GPT-3 Prediction: ['english'] 8 | 9 | 10 | SubjectEntity: Adele 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Akon 16 | Ground Truth: ['english'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Avril Lavigne 21 | Ground Truth: ['english'] 22 | GPT-3 Prediction: ['french'] 23 | 24 | 25 | SubjectEntity: Barack Obama 26 | Ground Truth: ['english', 'indonesian'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Bill Clinton 31 | Ground Truth: ['english'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Donald Trump 36 | Ground Truth: ['english'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Donald Tusk 41 | Ground Truth: ['english', 'german', 'polish'] 42 | GPT-3 Prediction: [] 43 | 44 | 45 | SubjectEntity: Emma Watson 46 | Ground Truth: ['english', 'french'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Genelia D'Souza 51 | Ground Truth: ['hindi', 'malayalam', 'tamil', 'telugu'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Henrique Capriles Radonski 56 | Ground Truth: ['spanish'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Hugo Chávez 61 | Ground Truth: ['spanish'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Jessie J 66 | Ground Truth: ['english'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: Joe Biden 71 | Ground Truth: ['english'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: John Cena 76 | Ground Truth: ['chinese', 'english', 'mandarin', 'mandarin chinese'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Kobe Bryant 81 | Ground Truth: ['english', 'italian', 'spanish'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | SubjectEntity: Novak Djokovic 86 | Ground Truth: ['english', 'french', 'german', 'italian', 'serbian'] 87 | GPT-3 Prediction: [] 88 | 89 | 90 | SubjectEntity: Paris Hilton 91 | Ground Truth: ['english'] 92 | GPT-3 Prediction: [] 93 | 94 | 95 | SubjectEntity: Pau Gasol 96 | Ground Truth: ['catalan', 'english', 'french', 'italian', 'spanish'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Paulo Coelho 101 | Ground Truth: ['portuguese'] 102 | GPT-3 Prediction: [] 103 | 104 | 105 | SubjectEntity: Sachin 106 | Ground Truth: ['english', 'hindi', 'marathi'] 107 | GPT-3 Prediction: [] 108 | 109 | 110 | SubjectEntity: Salman Khan 111 | Ground Truth: ['hindi'] 112 | GPT-3 Prediction: [] 113 | 114 | 115 | SubjectEntity: Samir Nasri 116 | Ground Truth: ['french'] 117 | GPT-3 Prediction: [] 118 | 119 | 120 | SubjectEntity: Skrillex 121 | Ground Truth: ['english'] 122 | GPT-3 Prediction: [] 123 | 124 | 125 | SubjectEntity: Thiago Alcântara 126 | Ground Truth: ['spanish'] 127 | GPT-3 Prediction: ['portuguese'] 128 | 129 | 130 | SubjectEntity: Vladimir Putin 131 | Ground Truth: ['english', 'german', 'russian', 'swedish'] 132 | GPT-3 Prediction: ['russian'] 133 | 134 | 135 | SubjectEntity: Zlatan Ibrahimović 136 | Ground Truth: ['bosnian', 'english', 'italian', 'spanish', 'swedish'] 137 | GPT-3 Prediction: [] 138 | 139 | 140 | -------------------------------------------------------------------------------- /failure_cases/baseline_factcheck/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonPlaceOfDeath (average f1: 0.5): 25 cases 2 | 3 | 4 | 5 | SubjectEntity: Abdul Rehman Malik 6 | Ground Truth: ['islamabad'] 7 | GPT-3 Prediction: [] 8 | 9 | 10 | SubjectEntity: Adam West 11 | Ground Truth: ['la', 'los angeles'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Alan García 16 | Ground Truth: ['lima'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Alfredo González Flores 21 | Ground Truth: ['heredia'] 22 | GPT-3 Prediction: [] 23 | 24 | 25 | SubjectEntity: Almudena Grandes 26 | Ground Truth: ['madrid'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Anne Rice 31 | Ground Truth: ['rancho mirage'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Aretha Franklin 36 | Ground Truth: ['detroit'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Avicii 41 | Ground Truth: ['muscat'] 42 | GPT-3 Prediction: ['rome'] 43 | 44 | 45 | SubjectEntity: Cameron Boyce 46 | Ground Truth: ['la', 'los angeles'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Chespirito 51 | Ground Truth: ['cancun'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Chick Corea 56 | Ground Truth: ['tampa', 'tampa bay area'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Chris Cornell 61 | Ground Truth: ['detroit'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Elijah Cummings 66 | Ground Truth: ['baltimore'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: George H. W. Bush 71 | Ground Truth: ['houston'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Glenn Fredly 76 | Ground Truth: ['jakarta'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Hugo Chávez 81 | Ground Truth: ['caracas', 'hospital militar de caracas'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | SubjectEntity: John Coltrane 86 | Ground Truth: ['huntington'] 87 | GPT-3 Prediction: [] 88 | 89 | 90 | SubjectEntity: John Lennon 91 | Ground Truth: ['new york city', 'nyc'] 92 | GPT-3 Prediction: [] 93 | 94 | 95 | SubjectEntity: Ken Robinson 96 | Ground Truth: ['london'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Kurt Vonnegut 101 | Ground Truth: ['new york city', 'nyc'] 102 | GPT-3 Prediction: [] 103 | 104 | 105 | SubjectEntity: Michael Collins 106 | Ground Truth: ['naples'] 107 | GPT-3 Prediction: [] 108 | 109 | 110 | SubjectEntity: Michael Jackson 111 | Ground Truth: ['la', 'los angeles'] 112 | GPT-3 Prediction: [] 113 | 114 | 115 | SubjectEntity: Paul G. Allen 116 | Ground Truth: ['seattle'] 117 | GPT-3 Prediction: [] 118 | 119 | 120 | SubjectEntity: Rachel Caine 121 | Ground Truth: ['texas'] 122 | GPT-3 Prediction: [] 123 | 124 | 125 | SubjectEntity: Roald Dahl 126 | Ground Truth: ['oxford'] 127 | GPT-3 Prediction: [] 128 | 129 | 130 | -------------------------------------------------------------------------------- /failure_cases/baseline_factcheck/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | RiverBasinsCountry (average f1: 0.381): 27 cases 2 | 3 | 4 | 5 | SubjectEntity: Alagón river 6 | Ground Truth: ['spain'] 7 | GPT-3 Prediction: [] 8 | 9 | 10 | SubjectEntity: Apurímac river 11 | Ground Truth: ['peru'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Brenta 16 | Ground Truth: ['italy'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Dahme 21 | Ground Truth: ['germany'] 22 | GPT-3 Prediction: [] 23 | 24 | 25 | SubjectEntity: Danube 26 | Ground Truth: ['austria', 'bulgaria', 'croatia', 'germany', 'hungary', 'moldova', 'romania', 'serbia', 'slovakia', 'ukraine'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Didi Liakhvi river 31 | Ground Truth: ['georgia'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Gadar river 36 | Ground Truth: ['iran'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Gambia river 41 | Ground Truth: ['gambia', 'guinea', 'senegal', 'the gambia'] 42 | GPT-3 Prediction: [] 43 | 44 | 45 | SubjectEntity: Han river 46 | Ground Truth: ['korea', 'korea', 'north korea', 'south korea'] 47 | GPT-3 Prediction: ['china'] 48 | 49 | 50 | SubjectEntity: Inn 51 | Ground Truth: ['austria', 'germany', 'switzerland'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Kura river 56 | Ground Truth: ['azerbaijan', 'georgia', 'turkey'] 57 | GPT-3 Prediction: ['russia'] 58 | 59 | 60 | SubjectEntity: Lielupe 61 | Ground Truth: ['latvia', 'lithuania'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Limpopo river 66 | Ground Truth: ['africa', 'botswana', 'mozambique', 'south africa', 'zimbabwe'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: Morava 71 | Ground Truth: ['austria', 'czech republic', 'czechia', 'slovakia'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Neman 76 | Ground Truth: ['belarus', 'latvia', 'lithuania', 'russia'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Neris 81 | Ground Truth: ['belarus', 'latvia', 'lithuania'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | SubjectEntity: Niger river 86 | Ground Truth: ['benin', 'guinea', 'mali', 'niger', 'nigeria'] 87 | GPT-3 Prediction: [] 88 | 89 | 90 | SubjectEntity: Paraguay river 91 | Ground Truth: ['argentina', 'brazil', 'paraguay'] 92 | GPT-3 Prediction: [] 93 | 94 | 95 | SubjectEntity: Red river of the south 96 | Ground Truth: ['texas', 'united states of america', 'usa'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Sakarya river 101 | Ground Truth: ['turkey'] 102 | GPT-3 Prediction: [] 103 | 104 | 105 | SubjectEntity: San 106 | Ground Truth: ['poland', 'ukraine'] 107 | GPT-3 Prediction: [] 108 | 109 | 110 | SubjectEntity: Shatt al-Arab 111 | Ground Truth: ['iran', 'iraq', 'kuwait', 'saudi arabia', 'syria', 'turkey'] 112 | GPT-3 Prediction: [] 113 | 114 | 115 | SubjectEntity: Siret river 116 | Ground Truth: ['romania', 'ukraine'] 117 | GPT-3 Prediction: [] 118 | 119 | 120 | SubjectEntity: Tisza 121 | Ground Truth: ['hungary', 'romania', 'serbia', 'slovakia', 'ukraine'] 122 | GPT-3 Prediction: ['hungary'] 123 | 124 | 125 | SubjectEntity: Torne river 126 | Ground Truth: ['finland', 'norway', 'sweden'] 127 | GPT-3 Prediction: [] 128 | 129 | 130 | SubjectEntity: Uruguay river 131 | Ground Truth: ['argentina', 'brazil', 'uruguay'] 132 | GPT-3 Prediction: [] 133 | 134 | 135 | SubjectEntity: Yantra 136 | Ground Truth: ['bulgaria'] 137 | GPT-3 Prediction: [] 138 | 139 | 140 | -------------------------------------------------------------------------------- /failure_cases/baseline_factcheck_wikiclean/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | CompanyParentOrganization (average f1: 0.68): 16 cases 2 | 3 | 4 | 5 | SubjectEntity: Cadillac 6 | Ground Truth: ['general motors'] 7 | GPT-3 Prediction: [] 8 | 9 | 10 | SubjectEntity: Changan Automobile 11 | Ground Truth: ['china south industries', 'china south industries group'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Gaz 16 | Ground Truth: ['gaz', 'gaz group'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Gucci 21 | Ground Truth: ['kering'] 22 | GPT-3 Prediction: [] 23 | 24 | 25 | SubjectEntity: Hanomag 26 | Ground Truth: ['komatsu', 'komatsu limited'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Hyundai Motor Company 31 | Ground Truth: ['hyundai'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Kia 36 | Ground Truth: ['hyundai', 'hyundai motor', 'hyundai motor group'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Lada Zapad Tolyatti 41 | Ground Truth: ['avtovaz'] 42 | GPT-3 Prediction: [] 43 | 44 | 45 | SubjectEntity: Lagonda 46 | Ground Truth: ['aston', 'aston martin', 'aston martin lagonda', 'martin'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Lotus Cars 51 | Ground Truth: ['geely'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: NSU 56 | Ground Truth: ['volkswagen'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Peugeot 61 | Ground Truth: ['stellantis'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Scripps-Booth 66 | Ground Truth: ['general motors', 'gm'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: Simca 71 | Ground Truth: ['chrysler', 'chrysler europe'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Volkswagen 76 | Ground Truth: ['volkswagen', 'volkswagen group'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Yamaha Motor Company 81 | Ground Truth: ['yamaha', 'yamaha corporation'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | -------------------------------------------------------------------------------- /failure_cases/baseline_factcheck_wikiclean/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | CountryOfficialLanguage (average f1: 0.752): 20 cases 2 | 3 | 4 | 5 | SubjectEntity: Afghanistan 6 | Ground Truth: ['arabic', 'baluchi', 'dari', 'nuristani', 'pamir', 'pashayi', 'pashto', 'turkmen', 'uzbek'] 7 | GPT-3 Prediction: ['persian'] 8 | 9 | 10 | SubjectEntity: Brunei 11 | Ground Truth: ['english', 'malay'] 12 | GPT-3 Prediction: ['english'] 13 | 14 | 15 | SubjectEntity: Burundi 16 | Ground Truth: ['english', 'french', 'kirundi'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Cameroon 21 | Ground Truth: ['english', 'french'] 22 | GPT-3 Prediction: ['french'] 23 | 24 | 25 | SubjectEntity: Central African Republic 26 | Ground Truth: ['french', 'sango'] 27 | GPT-3 Prediction: ['french'] 28 | 29 | 30 | SubjectEntity: East Timor 31 | Ground Truth: ['portuguese', 'tetum'] 32 | GPT-3 Prediction: ['indonesian'] 33 | 34 | 35 | SubjectEntity: Finland 36 | Ground Truth: ['finnish', 'swedish'] 37 | GPT-3 Prediction: ['finnish'] 38 | 39 | 40 | SubjectEntity: Grenada 41 | Ground Truth: ['creole', 'english', 'grenadian', 'grenadian creole english'] 42 | GPT-3 Prediction: ['english'] 43 | 44 | 45 | SubjectEntity: India 46 | Ground Truth: ['english', 'hindi'] 47 | GPT-3 Prediction: ['hindi'] 48 | 49 | 50 | SubjectEntity: Kazakhstan 51 | Ground Truth: ['kazakh', 'russian'] 52 | GPT-3 Prediction: ['russian'] 53 | 54 | 55 | SubjectEntity: Maldives 56 | Ground Truth: ['maldivian'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: North Macedonia 61 | Ground Truth: ['albanian', 'macedonian'] 62 | GPT-3 Prediction: ['macedonian'] 63 | 64 | 65 | SubjectEntity: Norway 66 | Ground Truth: ['bokmal', 'bokmål', 'norwegian', 'nynorsk', 'sami', 'sámi'] 67 | GPT-3 Prediction: ['norwegian'] 68 | 69 | 70 | SubjectEntity: Pakistan 71 | Ground Truth: ['arabic', 'english', 'urdu'] 72 | GPT-3 Prediction: ['urdu'] 73 | 74 | 75 | SubjectEntity: People's Republic of China 76 | Ground Truth: ['chinese', 'mandarin', 'standard mandarin'] 77 | GPT-3 Prediction: ['mandarin'] 78 | 79 | 80 | SubjectEntity: Philippines 81 | Ground Truth: ['english', 'filipino'] 82 | GPT-3 Prediction: ['filipino'] 83 | 84 | 85 | SubjectEntity: Rwanda 86 | Ground Truth: ['english', 'french', 'kinyarwanda', 'swahili'] 87 | GPT-3 Prediction: ['english'] 88 | 89 | 90 | SubjectEntity: Taiwan 91 | Ground Truth: ['amis', 'austronesian', 'formosan', 'hakka', 'hokkien', 'mandarin', 'paiwan', 'standard taiwanese mandarin', 'taiwanese', 'taiwanese', 'taiwanese hokkien'] 92 | GPT-3 Prediction: ['mandarin'] 93 | 94 | 95 | SubjectEntity: Tajikistan 96 | Ground Truth: ['russian', 'tajik'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Zimbabwe 101 | Ground Truth: ['barwe', 'chewa', 'english', 'kalanga', 'khoisan', 'nambya', 'ndau', 'ndebele', 'northern ndebele', 'sesotho', 'shona', 'tonga', 'tsonga', 'tswana', 'venda', 'xhosa'] 102 | GPT-3 Prediction: ['english'] 103 | 104 | 105 | -------------------------------------------------------------------------------- /failure_cases/baseline_factcheck_wikiclean/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonCauseOfDeath (average f1: 0.52): 24 cases 2 | 3 | 4 | 5 | SubjectEntity: Ahmed Zewail 6 | Ground Truth: ['lymphoma', 'spinal cord lymphoma'] 7 | GPT-3 Prediction: [] 8 | 9 | 10 | SubjectEntity: Avicii 11 | Ground Truth: ['exsanguination'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Christina Grimmie 16 | Ground Truth: ['ballistic trauma', 'trauma'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Cory Monteith 21 | Ground Truth: ['drug overdose', 'overdose'] 22 | GPT-3 Prediction: [] 23 | 24 | 25 | SubjectEntity: Dilip Kumar 26 | Ground Truth: ['cancer', 'prostate cancer'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Ennio Morricone 31 | Ground Truth: ['femoral fracture', 'fracture'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Frank Sinatra 36 | Ground Truth: ['infarction', 'myocardial infarction'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: George H. W. Bush 41 | Ground Truth: ['parkinson', "parkinson's disease"] 42 | GPT-3 Prediction: ['cancer'] 43 | 44 | 45 | SubjectEntity: George Harrison 46 | Ground Truth: ['cancer', 'lung cancer'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Haruma Miura 51 | Ground Truth: ['hanging'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Hugh Hefner 56 | Ground Truth: ['sepsis'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Idriss Déby 61 | Ground Truth: ['ballistic trauma', 'trauma'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Jim Rohn 66 | Ground Truth: ['fibrosis', 'pulmonary fibrosis'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: John Magufuli 71 | Ground Truth: ['arrhythmia', 'heart arrhythmia'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Johnny Cash 76 | Ground Truth: ['diabetes'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Larry King 81 | Ground Truth: ['sepsis'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | SubjectEntity: Maurane 86 | Ground Truth: ['falling'] 87 | GPT-3 Prediction: [] 88 | 89 | 90 | SubjectEntity: Maya Angelou 91 | Ground Truth: ['disease'] 92 | GPT-3 Prediction: [] 93 | 94 | 95 | SubjectEntity: Mikhail Zadornov 96 | Ground Truth: ['brain cancer', 'cancer'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Miles Davis 101 | Ground Truth: ['pneumonia', 'stroke'] 102 | GPT-3 Prediction: [] 103 | 104 | 105 | SubjectEntity: Nelson Mandela 106 | Ground Truth: ['respiratory disease'] 107 | GPT-3 Prediction: [] 108 | 109 | 110 | SubjectEntity: Rishi Kapoor 111 | Ground Truth: ['leukemia'] 112 | GPT-3 Prediction: [] 113 | 114 | 115 | SubjectEntity: Roddy Piper 116 | Ground Truth: ['cardiac arrest'] 117 | GPT-3 Prediction: [] 118 | 119 | 120 | SubjectEntity: Roy Orbison 121 | Ground Truth: ['infarction', 'myocardial infarction'] 122 | GPT-3 Prediction: [] 123 | 124 | 125 | -------------------------------------------------------------------------------- /failure_cases/baseline_factcheck_wikiclean/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | PersonLanguage (average f1: 0.431): 27 cases 2 | 3 | 4 | 5 | SubjectEntity: A. R. Rahman 6 | Ground Truth: ['hindi', 'tamil', 'telugu'] 7 | GPT-3 Prediction: ['english'] 8 | 9 | 10 | SubjectEntity: Adele 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Akon 16 | Ground Truth: ['english'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Avril Lavigne 21 | Ground Truth: ['english'] 22 | GPT-3 Prediction: ['french'] 23 | 24 | 25 | SubjectEntity: Barack Obama 26 | Ground Truth: ['english', 'indonesian'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Bill Clinton 31 | Ground Truth: ['english'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Donald Trump 36 | Ground Truth: ['english'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Donald Tusk 41 | Ground Truth: ['english', 'german', 'polish'] 42 | GPT-3 Prediction: [] 43 | 44 | 45 | SubjectEntity: Emma Watson 46 | Ground Truth: ['english', 'french'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Genelia D'Souza 51 | Ground Truth: ['hindi', 'malayalam', 'tamil', 'telugu'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Henrique Capriles Radonski 56 | Ground Truth: ['spanish'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Hugo Chávez 61 | Ground Truth: ['spanish'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Jessie J 66 | Ground Truth: ['english'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: Joe Biden 71 | Ground Truth: ['english'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: John Cena 76 | Ground Truth: ['chinese', 'english', 'mandarin', 'mandarin chinese'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Kobe Bryant 81 | Ground Truth: ['english', 'italian', 'spanish'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | SubjectEntity: Novak Djokovic 86 | Ground Truth: ['english', 'french', 'german', 'italian', 'serbian'] 87 | GPT-3 Prediction: [] 88 | 89 | 90 | SubjectEntity: Paris Hilton 91 | Ground Truth: ['english'] 92 | GPT-3 Prediction: [] 93 | 94 | 95 | SubjectEntity: Pau Gasol 96 | Ground Truth: ['catalan', 'english', 'french', 'italian', 'spanish'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Paulo Coelho 101 | Ground Truth: ['portuguese'] 102 | GPT-3 Prediction: [] 103 | 104 | 105 | SubjectEntity: Sachin 106 | Ground Truth: ['english', 'hindi', 'marathi'] 107 | GPT-3 Prediction: [] 108 | 109 | 110 | SubjectEntity: Salman Khan 111 | Ground Truth: ['hindi'] 112 | GPT-3 Prediction: [] 113 | 114 | 115 | SubjectEntity: Samir Nasri 116 | Ground Truth: ['french'] 117 | GPT-3 Prediction: [] 118 | 119 | 120 | SubjectEntity: Skrillex 121 | Ground Truth: ['english'] 122 | GPT-3 Prediction: [] 123 | 124 | 125 | SubjectEntity: Thiago Alcântara 126 | Ground Truth: ['spanish'] 127 | GPT-3 Prediction: ['portuguese'] 128 | 129 | 130 | SubjectEntity: Vladimir Putin 131 | Ground Truth: ['english', 'german', 'russian', 'swedish'] 132 | GPT-3 Prediction: ['russian'] 133 | 134 | 135 | SubjectEntity: Zlatan Ibrahimović 136 | Ground Truth: ['bosnian', 'english', 'italian', 'spanish', 'swedish'] 137 | GPT-3 Prediction: [] 138 | 139 | 140 | -------------------------------------------------------------------------------- /failure_cases/baseline_factcheck_wikiclean/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonPlaceOfDeath (average f1: 0.5): 25 cases 2 | 3 | 4 | 5 | SubjectEntity: Abdul Rehman Malik 6 | Ground Truth: ['islamabad'] 7 | GPT-3 Prediction: [] 8 | 9 | 10 | SubjectEntity: Adam West 11 | Ground Truth: ['la', 'los angeles'] 12 | GPT-3 Prediction: [] 13 | 14 | 15 | SubjectEntity: Alan García 16 | Ground Truth: ['lima'] 17 | GPT-3 Prediction: [] 18 | 19 | 20 | SubjectEntity: Alfredo González Flores 21 | Ground Truth: ['heredia'] 22 | GPT-3 Prediction: [] 23 | 24 | 25 | SubjectEntity: Almudena Grandes 26 | Ground Truth: ['madrid'] 27 | GPT-3 Prediction: [] 28 | 29 | 30 | SubjectEntity: Anne Rice 31 | Ground Truth: ['rancho mirage'] 32 | GPT-3 Prediction: [] 33 | 34 | 35 | SubjectEntity: Aretha Franklin 36 | Ground Truth: ['detroit'] 37 | GPT-3 Prediction: [] 38 | 39 | 40 | SubjectEntity: Avicii 41 | Ground Truth: ['muscat'] 42 | GPT-3 Prediction: ['rome'] 43 | 44 | 45 | SubjectEntity: Cameron Boyce 46 | Ground Truth: ['la', 'los angeles'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Chespirito 51 | Ground Truth: ['cancun'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Chick Corea 56 | Ground Truth: ['tampa', 'tampa bay area'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Chris Cornell 61 | Ground Truth: ['detroit'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Elijah Cummings 66 | Ground Truth: ['baltimore'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: George H. W. Bush 71 | Ground Truth: ['houston'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Glenn Fredly 76 | Ground Truth: ['jakarta'] 77 | GPT-3 Prediction: [] 78 | 79 | 80 | SubjectEntity: Hugo Chávez 81 | Ground Truth: ['caracas', 'hospital militar de caracas'] 82 | GPT-3 Prediction: [] 83 | 84 | 85 | SubjectEntity: John Coltrane 86 | Ground Truth: ['huntington'] 87 | GPT-3 Prediction: [] 88 | 89 | 90 | SubjectEntity: John Lennon 91 | Ground Truth: ['new york city', 'nyc'] 92 | GPT-3 Prediction: [] 93 | 94 | 95 | SubjectEntity: Ken Robinson 96 | Ground Truth: ['london'] 97 | GPT-3 Prediction: [] 98 | 99 | 100 | SubjectEntity: Kurt Vonnegut 101 | Ground Truth: ['new york city', 'nyc'] 102 | GPT-3 Prediction: [] 103 | 104 | 105 | SubjectEntity: Michael Collins 106 | Ground Truth: ['naples'] 107 | GPT-3 Prediction: [] 108 | 109 | 110 | SubjectEntity: Michael Jackson 111 | Ground Truth: ['la', 'los angeles'] 112 | GPT-3 Prediction: [] 113 | 114 | 115 | SubjectEntity: Paul G. Allen 116 | Ground Truth: ['seattle'] 117 | GPT-3 Prediction: [] 118 | 119 | 120 | SubjectEntity: Rachel Caine 121 | Ground Truth: ['texas'] 122 | GPT-3 Prediction: [] 123 | 124 | 125 | SubjectEntity: Roald Dahl 126 | Ground Truth: ['oxford'] 127 | GPT-3 Prediction: [] 128 | 129 | 130 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-empty)/ChemicalCompoundElement.txt: -------------------------------------------------------------------------------- 1 | ChemicalCompoundElement (average f1: 0.884): 16 cases 2 | 3 | 4 | 5 | SubjectEntity: Aluminum Iodide 6 | Ground Truth: ['aluminium', 'iodine'] 7 | GPT-3 Prediction: ['aluminum', 'iodine'] 8 | 9 | 10 | SubjectEntity: Atracurium Besylate 11 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 12 | GPT-3 Prediction: ['atracurium', 'hydrogen', 'oxygen'] 13 | 14 | 15 | SubjectEntity: Cetyl Alcohol 16 | Ground Truth: ['carbon', 'hydrogen', 'oxygen'] 17 | GPT-3 Prediction: ['carbon', 'hydrogen'] 18 | 19 | 20 | SubjectEntity: Chlorophyll F 21 | Ground Truth: ['carbon', 'hydrogen', 'magnesium', 'nitrogen', 'oxygen'] 22 | GPT-3 Prediction: ['carbon', 'chlorine', 'hydrogen', 'magnesium', 'oxygen'] 23 | 24 | 25 | SubjectEntity: Cupric Phosphate 26 | Ground Truth: ['copper', 'oxygen', 'phosphorus'] 27 | GPT-3 Prediction: ['copper', 'hydrogen', 'oxygen', 'phosphorus'] 28 | 29 | 30 | SubjectEntity: Ethyl Acetoacetate 31 | Ground Truth: ['carbon', 'hydrogen', 'oxygen'] 32 | GPT-3 Prediction: ['carbon', 'ethyl', 'hydrogen', 'oxygen'] 33 | 34 | 35 | SubjectEntity: Flavin Adenine Dinucleotide 36 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 37 | GPT-3 Prediction: ['adenine', 'dinucleotide', 'flavin'] 38 | 39 | 40 | SubjectEntity: Glutathione 41 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 42 | GPT-3 Prediction: ['cysteine', 'glutamic acid', 'glycine'] 43 | 44 | 45 | SubjectEntity: Heroin 46 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 47 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 48 | 49 | 50 | SubjectEntity: Hydroxocobalamin 51 | Ground Truth: ['carbon', 'cobalt', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 52 | GPT-3 Prediction: ['cobalamin', 'hydrogen', 'oxygen'] 53 | 54 | 55 | SubjectEntity: Metformin 56 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen'] 57 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 58 | 59 | 60 | SubjectEntity: Motilin 61 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 62 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 63 | 64 | 65 | SubjectEntity: Tabun 66 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 67 | GPT-3 Prediction: ['carbon', 'chlorine', 'hydrogen', 'phosphorus'] 68 | 69 | 70 | SubjectEntity: Tenoxicam 71 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 72 | GPT-3 Prediction: ['carbon', 'fluorine', 'nitrogen', 'oxygen', 'sulfur'] 73 | 74 | 75 | SubjectEntity: Triphenylphosphine 76 | Ground Truth: ['carbon', 'hydrogen', 'phosphorus'] 77 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'phosphorus'] 78 | 79 | 80 | SubjectEntity: Vinylidene Fluoride 81 | Ground Truth: ['carbon', 'fluorine', 'hydrogen'] 82 | GPT-3 Prediction: ['carbon', 'fluorine', 'hydrogen', 'oxygen'] 83 | 84 | 85 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-empty)/CountryBordersWithCountry.txt: -------------------------------------------------------------------------------- 1 | CountryBordersWithCountry (average f1: 0.785): 14 cases 2 | 3 | 4 | 5 | SubjectEntity: Bahrain 6 | Ground Truth: ['iran', 'saudi arabia'] 7 | GPT-3 Prediction: ['qatar', 'saudi arabia', 'united arab emirates'] 8 | 9 | 10 | SubjectEntity: Barbados 11 | Ground Truth: [] 12 | GPT-3 Prediction: ['guyana', 'saint lucia', 'saint vincent and the grenadines', 'trinidad and tobago', 'venezuela'] 13 | 14 | 15 | SubjectEntity: Comoros 16 | Ground Truth: ['france', 'madagascar', 'mozambique', 'seychelles', 'tanzania'] 17 | GPT-3 Prediction: ['madagascar', 'mayotte', 'mozambique', 'seychelles'] 18 | 19 | 20 | SubjectEntity: Cuba 21 | Ground Truth: ['united states of america', 'usa'] 22 | GPT-3 Prediction: ['bahamas', 'cayman islands', 'haiti', 'jamaica', 'mexico', 'united states'] 23 | 24 | 25 | SubjectEntity: Dominican Republic 26 | Ground Truth: ['haiti', 'united states of america', 'usa', 'venezuela'] 27 | GPT-3 Prediction: ['haiti', 'puerto rico'] 28 | 29 | 30 | SubjectEntity: Gambia 31 | Ground Truth: ['senegal'] 32 | GPT-3 Prediction: ['guinea', 'guinea-bissau', 'senegal'] 33 | 34 | 35 | SubjectEntity: Ghana 36 | Ground Truth: ['burkina faso', 'ivory coast', 'togo'] 37 | GPT-3 Prediction: ['benin', 'burkina faso', 'cote divoire', 'togo'] 38 | 39 | 40 | SubjectEntity: Grenada 41 | Ground Truth: ['venezuela'] 42 | GPT-3 Prediction: ['trinidad and tobago', 'venezuela'] 43 | 44 | 45 | SubjectEntity: Japan 46 | Ground Truth: ['china', 'korea', "people's republic of china", 'philippines', 'russia', 'south korea', 'taiwan', 'united states of america', 'usa'] 47 | GPT-3 Prediction: ['china', 'north korea', 'russia', 'south korea'] 48 | 49 | 50 | SubjectEntity: Kingdom of Denmark 51 | Ground Truth: ['canada', 'germany', 'iceland', 'norway', 'poland', 'sweden', 'uk', 'united kingdom'] 52 | GPT-3 Prediction: ['germany', 'norway', 'sweden'] 53 | 54 | 55 | SubjectEntity: Portugal 56 | Ground Truth: ['castile and leon', 'castile and león', 'spain'] 57 | GPT-3 Prediction: ['spain'] 58 | 59 | 60 | SubjectEntity: Saint Vincent and the Grenadines 61 | Ground Truth: ['venezuela'] 62 | GPT-3 Prediction: ['barbados', 'grenada', 'saint lucia'] 63 | 64 | 65 | SubjectEntity: Samoa 66 | Ground Truth: ['united states of america', 'usa'] 67 | GPT-3 Prediction: [''] 68 | 69 | 70 | SubjectEntity: Venezuela 71 | Ground Truth: ['brazil', 'colombia', 'dominica', 'dominican republic', 'france', 'grenada', 'guyana', 'kingdom of the netherlands', 'netherlands', 'saint kitts and nevis', 'saint lucia', 'saint vincent and the grenadines', 'trinidad and tobago', 'uk', 'united kingdom', 'united states of america', 'usa'] 72 | GPT-3 Prediction: ['brazil', 'colombia', 'guyana'] 73 | 74 | 75 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-empty)/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | CountryOfficialLanguage (average f1: 0.785): 22 cases 2 | 3 | 4 | 5 | SubjectEntity: Afghanistan 6 | Ground Truth: ['arabic', 'baluchi', 'dari', 'nuristani', 'pamir', 'pashayi', 'pashto', 'turkmen', 'uzbek'] 7 | GPT-3 Prediction: ['dari', 'pashto'] 8 | 9 | 10 | SubjectEntity: Botswana 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: ['english', 'setswana'] 13 | 14 | 15 | SubjectEntity: Burkina Faso 16 | Ground Truth: ['french'] 17 | GPT-3 Prediction: ['dioula', 'french', 'fula', 'mossi'] 18 | 19 | 20 | SubjectEntity: Burundi 21 | Ground Truth: ['english', 'french', 'kirundi'] 22 | GPT-3 Prediction: ['french', 'kirundi', 'swahili'] 23 | 24 | 25 | SubjectEntity: Croatia 26 | Ground Truth: ['croatian'] 27 | GPT-3 Prediction: ['croatian', 'serbian'] 28 | 29 | 30 | SubjectEntity: Grenada 31 | Ground Truth: ['creole', 'english', 'grenadian', 'grenadian creole english'] 32 | GPT-3 Prediction: ['english'] 33 | 34 | 35 | SubjectEntity: Israel 36 | Ground Truth: ['hebrew'] 37 | GPT-3 Prediction: ['arabic', 'hebrew'] 38 | 39 | 40 | SubjectEntity: Jamaica 41 | Ground Truth: ['english'] 42 | GPT-3 Prediction: ['english', 'patois'] 43 | 44 | 45 | SubjectEntity: Jordan 46 | Ground Truth: ['arabic'] 47 | GPT-3 Prediction: ['arabic', 'english'] 48 | 49 | 50 | SubjectEntity: Latvia 51 | Ground Truth: ['latvian'] 52 | GPT-3 Prediction: ['latvian', 'russian'] 53 | 54 | 55 | SubjectEntity: Liechtenstein 56 | Ground Truth: ['german'] 57 | GPT-3 Prediction: ['alemannic', 'german'] 58 | 59 | 60 | SubjectEntity: Maldives 61 | Ground Truth: ['maldivian'] 62 | GPT-3 Prediction: ['dhivehi'] 63 | 64 | 65 | SubjectEntity: Mozambique 66 | Ground Truth: ['portuguese'] 67 | GPT-3 Prediction: ['cisena', 'echuwabo', 'emakhuwa', 'portuguese', 'xichangana'] 68 | 69 | 70 | SubjectEntity: Niger 71 | Ground Truth: ['french'] 72 | GPT-3 Prediction: ['french', 'hausa', 'zarma'] 73 | 74 | 75 | SubjectEntity: North Macedonia 76 | Ground Truth: ['albanian', 'macedonian'] 77 | GPT-3 Prediction: ['albanian', 'macedonian', 'romani', 'serbian', 'turkish'] 78 | 79 | 80 | SubjectEntity: Norway 81 | Ground Truth: ['bokmal', 'bokmål', 'norwegian', 'nynorsk', 'sami', 'sámi'] 82 | GPT-3 Prediction: ['norwegian', 'sami'] 83 | 84 | 85 | SubjectEntity: Panama 86 | Ground Truth: ['spanish'] 87 | GPT-3 Prediction: ['english', 'spanish'] 88 | 89 | 90 | SubjectEntity: People's Republic of China 91 | Ground Truth: ['chinese', 'mandarin', 'standard mandarin'] 92 | GPT-3 Prediction: ['mandarin'] 93 | 94 | 95 | SubjectEntity: Qatar 96 | Ground Truth: ['arabic'] 97 | GPT-3 Prediction: ['arabic', 'english'] 98 | 99 | 100 | SubjectEntity: Sierra Leone 101 | Ground Truth: ['english'] 102 | GPT-3 Prediction: ['english', 'krio'] 103 | 104 | 105 | SubjectEntity: Taiwan 106 | Ground Truth: ['amis', 'austronesian', 'formosan', 'hakka', 'hokkien', 'mandarin', 'paiwan', 'standard taiwanese mandarin', 'taiwanese', 'taiwanese', 'taiwanese hokkien'] 107 | GPT-3 Prediction: ['hakka', 'mandarin', 'taiwanese'] 108 | 109 | 110 | SubjectEntity: Zimbabwe 111 | Ground Truth: ['barwe', 'chewa', 'english', 'kalanga', 'khoisan', 'nambya', 'ndau', 'ndebele', 'northern ndebele', 'sesotho', 'shona', 'tonga', 'tsonga', 'tswana', 'venda', 'xhosa'] 112 | GPT-3 Prediction: ['english', 'ndebele', 'shona'] 113 | 114 | 115 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-empty)/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonCauseOfDeath (average f1: 0.5): 25 cases 2 | 3 | 4 | 5 | SubjectEntity: Ahmed Zewail 6 | Ground Truth: ['lymphoma', 'spinal cord lymphoma'] 7 | GPT-3 Prediction: ['cancer'] 8 | 9 | 10 | SubjectEntity: Avicii 11 | Ground Truth: ['exsanguination'] 12 | GPT-3 Prediction: ['suicide'] 13 | 14 | 15 | SubjectEntity: Boris Johnson 16 | Ground Truth: [] 17 | GPT-3 Prediction: ['infarction'] 18 | 19 | 20 | SubjectEntity: Christina Grimmie 21 | Ground Truth: ['ballistic trauma', 'trauma'] 22 | GPT-3 Prediction: ['murder'] 23 | 24 | 25 | SubjectEntity: Dilip Kumar 26 | Ground Truth: ['cancer', 'prostate cancer'] 27 | GPT-3 Prediction: ['natural causes'] 28 | 29 | 30 | SubjectEntity: Ennio Morricone 31 | Ground Truth: ['femoral fracture', 'fracture'] 32 | GPT-3 Prediction: ['infarction'] 33 | 34 | 35 | SubjectEntity: Frank Sinatra 36 | Ground Truth: ['infarction', 'myocardial infarction'] 37 | GPT-3 Prediction: ['heart attack'] 38 | 39 | 40 | SubjectEntity: George H. W. Bush 41 | Ground Truth: ['parkinson', "parkinson's disease"] 42 | GPT-3 Prediction: ['infarction'] 43 | 44 | 45 | SubjectEntity: Haruma Miura 46 | Ground Truth: ['hanging'] 47 | GPT-3 Prediction: ['suicide'] 48 | 49 | 50 | SubjectEntity: Hugh Hefner 51 | Ground Truth: ['sepsis'] 52 | GPT-3 Prediction: ['natural causes'] 53 | 54 | 55 | SubjectEntity: Idriss Déby 56 | Ground Truth: ['ballistic trauma', 'trauma'] 57 | GPT-3 Prediction: ['injury'] 58 | 59 | 60 | SubjectEntity: Indra Nooyi 61 | Ground Truth: [] 62 | GPT-3 Prediction: ['infarction'] 63 | 64 | 65 | SubjectEntity: Jim Rohn 66 | Ground Truth: ['fibrosis', 'pulmonary fibrosis'] 67 | GPT-3 Prediction: ['pneumonia'] 68 | 69 | 70 | SubjectEntity: John Magufuli 71 | Ground Truth: ['arrhythmia', 'heart arrhythmia'] 72 | GPT-3 Prediction: ['covid-19'] 73 | 74 | 75 | SubjectEntity: Johnny Cash 76 | Ground Truth: ['diabetes'] 77 | GPT-3 Prediction: ['infarction'] 78 | 79 | 80 | SubjectEntity: Larry King 81 | Ground Truth: ['sepsis'] 82 | GPT-3 Prediction: ['infarction'] 83 | 84 | 85 | SubjectEntity: Maurane 86 | Ground Truth: ['falling'] 87 | GPT-3 Prediction: ['cancer'] 88 | 89 | 90 | SubjectEntity: Maya Angelou 91 | Ground Truth: ['disease'] 92 | GPT-3 Prediction: ['natural causes'] 93 | 94 | 95 | SubjectEntity: Miles Davis 96 | Ground Truth: ['pneumonia', 'stroke'] 97 | GPT-3 Prediction: ['infarction'] 98 | 99 | 100 | SubjectEntity: Nelson Mandela 101 | Ground Truth: ['respiratory disease'] 102 | GPT-3 Prediction: ['respiratory infection'] 103 | 104 | 105 | SubjectEntity: Rishi Kapoor 106 | Ground Truth: ['leukemia'] 107 | GPT-3 Prediction: ['cancer'] 108 | 109 | 110 | SubjectEntity: Roddy Piper 111 | Ground Truth: ['cardiac arrest'] 112 | GPT-3 Prediction: ['heart attack'] 113 | 114 | 115 | SubjectEntity: Ronaldinho 116 | Ground Truth: [] 117 | GPT-3 Prediction: ['drowning'] 118 | 119 | 120 | SubjectEntity: Roy Orbison 121 | Ground Truth: ['infarction', 'myocardial infarction'] 122 | GPT-3 Prediction: ['heart attack'] 123 | 124 | 125 | SubjectEntity: Russell Crowe 126 | Ground Truth: [] 127 | GPT-3 Prediction: ['blunt trauma', 'injury'] 128 | 129 | 130 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-empty)/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | PersonLanguage (average f1: 0.793): 21 cases 2 | 3 | 4 | 5 | SubjectEntity: A. R. Rahman 6 | Ground Truth: ['hindi', 'tamil', 'telugu'] 7 | GPT-3 Prediction: ['english', 'hindi', 'malayalam', 'tamil', 'telugu'] 8 | 9 | 10 | SubjectEntity: Abdullah Gül 11 | Ground Truth: ['turkish'] 12 | GPT-3 Prediction: ['english', 'french', 'turkish'] 13 | 14 | 15 | SubjectEntity: Akon 16 | Ground Truth: ['english'] 17 | GPT-3 Prediction: ['english', 'french', 'wolof'] 18 | 19 | 20 | SubjectEntity: Avril Lavigne 21 | Ground Truth: ['english'] 22 | GPT-3 Prediction: ['english', 'french'] 23 | 24 | 25 | SubjectEntity: Blaise Matuidi 26 | Ground Truth: ['french'] 27 | GPT-3 Prediction: ['english', 'french', 'italian', 'spanish'] 28 | 29 | 30 | SubjectEntity: Genelia D'Souza 31 | Ground Truth: ['hindi', 'malayalam', 'tamil', 'telugu'] 32 | GPT-3 Prediction: ['english', 'hindi', 'marathi', 'telugu'] 33 | 34 | 35 | SubjectEntity: Henrique Capriles Radonski 36 | Ground Truth: ['spanish'] 37 | GPT-3 Prediction: ['english', 'portuguese', 'spanish'] 38 | 39 | 40 | SubjectEntity: Hugo Chávez 41 | Ground Truth: ['spanish'] 42 | GPT-3 Prediction: ['english', 'spanish'] 43 | 44 | 45 | SubjectEntity: John Cena 46 | Ground Truth: ['chinese', 'english', 'mandarin', 'mandarin chinese'] 47 | GPT-3 Prediction: ['english'] 48 | 49 | 50 | SubjectEntity: Jose Michimani 51 | Ground Truth: ['english'] 52 | GPT-3 Prediction: ['aymara', 'quechua', 'spanish'] 53 | 54 | 55 | SubjectEntity: Mohammed bin Rashid Al Maktoum 56 | Ground Truth: ['arabic', 'english'] 57 | GPT-3 Prediction: ['arabic', 'english', 'hindi', 'persian', 'urdu'] 58 | 59 | 60 | SubjectEntity: Oscar dos Santos Emboaba Júnior 61 | Ground Truth: ['portuguese'] 62 | GPT-3 Prediction: ['english', 'portuguese'] 63 | 64 | 65 | SubjectEntity: Patrick Stewart 66 | Ground Truth: ['english'] 67 | GPT-3 Prediction: ['english', 'french'] 68 | 69 | 70 | SubjectEntity: Pau Gasol 71 | Ground Truth: ['catalan', 'english', 'french', 'italian', 'spanish'] 72 | GPT-3 Prediction: ['catalan', 'english', 'spanish'] 73 | 74 | 75 | SubjectEntity: Paulo Coelho 76 | Ground Truth: ['portuguese'] 77 | GPT-3 Prediction: ['english', 'french', 'portuguese', 'spanish'] 78 | 79 | 80 | SubjectEntity: Rihanna 81 | Ground Truth: ['english'] 82 | GPT-3 Prediction: ['creole', 'english'] 83 | 84 | 85 | SubjectEntity: Salman Khan 86 | Ground Truth: ['hindi'] 87 | GPT-3 Prediction: ['hindi', 'urdu'] 88 | 89 | 90 | SubjectEntity: Samir Nasri 91 | Ground Truth: ['french'] 92 | GPT-3 Prediction: ['arabic', 'english', 'french'] 93 | 94 | 95 | SubjectEntity: Thiago Alcântara 96 | Ground Truth: ['spanish'] 97 | GPT-3 Prediction: ['catalan', 'english', 'portuguese', 'spanish'] 98 | 99 | 100 | SubjectEntity: Vladimir Putin 101 | Ground Truth: ['english', 'german', 'russian', 'swedish'] 102 | GPT-3 Prediction: ['english', 'french', 'german', 'russian'] 103 | 104 | 105 | SubjectEntity: Zlatan Ibrahimović 106 | Ground Truth: ['bosnian', 'english', 'italian', 'spanish', 'swedish'] 107 | GPT-3 Prediction: ['bosnian', 'croatian', 'english', 'french', 'italian', 'montenegrin', 'portuguese', 'serbian', 'spanish', 'swahili'] 108 | 109 | 110 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-empty)/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonPlaceOfDeath (average f1: 0.84): 8 cases 2 | 3 | 4 | 5 | SubjectEntity: Abdul Rehman Malik 6 | Ground Truth: ['islamabad'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Alfredo González Flores 11 | Ground Truth: ['heredia'] 12 | GPT-3 Prediction: ['san salvador'] 13 | 14 | 15 | SubjectEntity: Almudena Grandes 16 | Ground Truth: ['madrid'] 17 | GPT-3 Prediction: [''] 18 | 19 | 20 | SubjectEntity: Anne Rice 21 | Ground Truth: ['rancho mirage'] 22 | GPT-3 Prediction: [''] 23 | 24 | 25 | SubjectEntity: Chick Corea 26 | Ground Truth: ['tampa', 'tampa bay area'] 27 | GPT-3 Prediction: ['burbank'] 28 | 29 | 30 | SubjectEntity: John Coltrane 31 | Ground Truth: ['huntington'] 32 | GPT-3 Prediction: ['new york'] 33 | 34 | 35 | SubjectEntity: Michael Collins 36 | Ground Truth: ['naples'] 37 | GPT-3 Prediction: ['cape canaveral'] 38 | 39 | 40 | SubjectEntity: Rachel Caine 41 | Ground Truth: ['texas'] 42 | GPT-3 Prediction: [''] 43 | 44 | 45 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-empty)/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | RiverBasinsCountry (average f1: 0.822): 18 cases 2 | 3 | 4 | 5 | SubjectEntity: Brenta 6 | Ground Truth: ['italy'] 7 | GPT-3 Prediction: ['austria', 'italy'] 8 | 9 | 10 | SubjectEntity: Didi Liakhvi river 11 | Ground Truth: ['georgia'] 12 | GPT-3 Prediction: ['georgia', 'russia'] 13 | 14 | 15 | SubjectEntity: Dniester 16 | Ground Truth: ['moldova', 'ukraine'] 17 | GPT-3 Prediction: ['moldova', 'romania', 'ukraine'] 18 | 19 | 20 | SubjectEntity: Gadar river 21 | Ground Truth: ['iran'] 22 | GPT-3 Prediction: ['iraq', 'syria', 'turkey'] 23 | 24 | 25 | SubjectEntity: Gambia river 26 | Ground Truth: ['gambia', 'guinea', 'senegal', 'the gambia'] 27 | GPT-3 Prediction: ['senegal', 'the gambia'] 28 | 29 | 30 | SubjectEntity: Han river 31 | Ground Truth: ['korea', 'korea', 'north korea', 'south korea'] 32 | GPT-3 Prediction: ['china'] 33 | 34 | 35 | SubjectEntity: Kura river 36 | Ground Truth: ['azerbaijan', 'georgia', 'turkey'] 37 | GPT-3 Prediction: ['armenia', 'azerbaijan', 'georgia', 'iran', 'turkey', 'turkmenistan', 'uzbekistan'] 38 | 39 | 40 | SubjectEntity: Narva river 41 | Ground Truth: ['belarus', 'estonia', 'latvia', 'russia'] 42 | GPT-3 Prediction: ['estonia', 'russia'] 43 | 44 | 45 | SubjectEntity: Neman 46 | Ground Truth: ['belarus', 'latvia', 'lithuania', 'russia'] 47 | GPT-3 Prediction: ['belarus', 'lithuania', 'poland', 'russia'] 48 | 49 | 50 | SubjectEntity: Neris 51 | Ground Truth: ['belarus', 'latvia', 'lithuania'] 52 | GPT-3 Prediction: ['belarus', 'lithuania', 'russia'] 53 | 54 | 55 | SubjectEntity: Ottawa river 56 | Ground Truth: ['canada'] 57 | GPT-3 Prediction: ['canada', 'united states'] 58 | 59 | 60 | SubjectEntity: Red river of the south 61 | Ground Truth: ['texas', 'united states of america', 'usa'] 62 | GPT-3 Prediction: ['mexico', 'united states'] 63 | 64 | 65 | SubjectEntity: San 66 | Ground Truth: ['poland', 'ukraine'] 67 | GPT-3 Prediction: ['mexico'] 68 | 69 | 70 | SubjectEntity: Shatt al-Arab 71 | Ground Truth: ['iran', 'iraq', 'kuwait', 'saudi arabia', 'syria', 'turkey'] 72 | GPT-3 Prediction: ['iran', 'iraq'] 73 | 74 | 75 | SubjectEntity: Siret river 76 | Ground Truth: ['romania', 'ukraine'] 77 | GPT-3 Prediction: ['moldova', 'romania', 'ukraine'] 78 | 79 | 80 | SubjectEntity: Torne river 81 | Ground Truth: ['finland', 'norway', 'sweden'] 82 | GPT-3 Prediction: ['finland', 'sweden'] 83 | 84 | 85 | SubjectEntity: Ussuri 86 | Ground Truth: ['russia'] 87 | GPT-3 Prediction: ['china', 'russia'] 88 | 89 | 90 | SubjectEntity: Yantra 91 | Ground Truth: ['bulgaria'] 92 | GPT-3 Prediction: ['bulgaria', 'romania'] 93 | 94 | 95 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-none)/ChemicalCompoundElement.txt: -------------------------------------------------------------------------------- 1 | ChemicalCompoundElement (average f1: 0.884): 16 cases 2 | 3 | 4 | 5 | SubjectEntity: Aluminum Iodide 6 | Ground Truth: ['aluminium', 'iodine'] 7 | GPT-3 Prediction: ['aluminum', 'iodine'] 8 | 9 | 10 | SubjectEntity: Atracurium Besylate 11 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 12 | GPT-3 Prediction: ['atracurium', 'hydrogen', 'oxygen'] 13 | 14 | 15 | SubjectEntity: Cetyl Alcohol 16 | Ground Truth: ['carbon', 'hydrogen', 'oxygen'] 17 | GPT-3 Prediction: ['carbon', 'hydrogen'] 18 | 19 | 20 | SubjectEntity: Chlorophyll F 21 | Ground Truth: ['carbon', 'hydrogen', 'magnesium', 'nitrogen', 'oxygen'] 22 | GPT-3 Prediction: ['carbon', 'chlorine', 'hydrogen', 'magnesium', 'oxygen'] 23 | 24 | 25 | SubjectEntity: Cupric Phosphate 26 | Ground Truth: ['copper', 'oxygen', 'phosphorus'] 27 | GPT-3 Prediction: ['copper', 'hydrogen', 'oxygen', 'phosphorus'] 28 | 29 | 30 | SubjectEntity: Ethyl Acetoacetate 31 | Ground Truth: ['carbon', 'hydrogen', 'oxygen'] 32 | GPT-3 Prediction: ['carbon', 'ethyl', 'hydrogen', 'oxygen'] 33 | 34 | 35 | SubjectEntity: Flavin Adenine Dinucleotide 36 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 37 | GPT-3 Prediction: ['adenine', 'dinucleotide', 'flavin'] 38 | 39 | 40 | SubjectEntity: Glutathione 41 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 42 | GPT-3 Prediction: ['cysteine', 'glutamic acid', 'glycine'] 43 | 44 | 45 | SubjectEntity: Heroin 46 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 47 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 48 | 49 | 50 | SubjectEntity: Hydroxocobalamin 51 | Ground Truth: ['carbon', 'cobalt', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 52 | GPT-3 Prediction: ['cobalamin', 'hydrogen', 'oxygen'] 53 | 54 | 55 | SubjectEntity: Metformin 56 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen'] 57 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 58 | 59 | 60 | SubjectEntity: Motilin 61 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 62 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 63 | 64 | 65 | SubjectEntity: Tabun 66 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 67 | GPT-3 Prediction: ['carbon', 'chlorine', 'hydrogen', 'phosphorus'] 68 | 69 | 70 | SubjectEntity: Tenoxicam 71 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 72 | GPT-3 Prediction: ['carbon', 'fluorine', 'nitrogen', 'oxygen', 'sulfur'] 73 | 74 | 75 | SubjectEntity: Triphenylphosphine 76 | Ground Truth: ['carbon', 'hydrogen', 'phosphorus'] 77 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'phosphorus'] 78 | 79 | 80 | SubjectEntity: Vinylidene Fluoride 81 | Ground Truth: ['carbon', 'fluorine', 'hydrogen'] 82 | GPT-3 Prediction: ['carbon', 'fluorine', 'hydrogen', 'oxygen'] 83 | 84 | 85 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-none)/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | CompanyParentOrganization (average f1: 0.688): 16 cases 2 | 3 | 4 | 5 | SubjectEntity: Aixam 6 | Ground Truth: [] 7 | GPT-3 Prediction: ['aixam-mega', 'groupe ligier'] 8 | 9 | 10 | SubjectEntity: Alvis Car and Engineering Company Ltd 11 | Ground Truth: [] 12 | GPT-3 Prediction: ['alvis plc'] 13 | 14 | 15 | SubjectEntity: Aston Martin lagonda 16 | Ground Truth: [] 17 | GPT-3 Prediction: ['investindustrial', 'kuwait investment authority', 'mercedes-benz'] 18 | 19 | 20 | SubjectEntity: Austro-Daimler 21 | Ground Truth: [] 22 | GPT-3 Prediction: ['daimler ag'] 23 | 24 | 25 | SubjectEntity: Bell 26 | Ground Truth: [] 27 | GPT-3 Prediction: ['bell aliant', 'bell aliant', 'bell canada', 'bell mts', 'bell mts', 'bell media', 'bell mobility', 'bell mobility', 'northertel', 'northerntel', 'northerntel', 'sogetel', 'sogetel', 'sogetel', 'télébec', 'télébec', 'télébec', 'virgin mobile canada', 'virgin mobile canada'] 28 | 29 | 30 | SubjectEntity: Changan Automobile 31 | Ground Truth: ['china south industries', 'china south industries group'] 32 | GPT-3 Prediction: ['changan automobile group'] 33 | 34 | 35 | SubjectEntity: Duesenberg 36 | Ground Truth: [] 37 | GPT-3 Prediction: ['duesenberg motors company'] 38 | 39 | 40 | SubjectEntity: Hanomag 41 | Ground Truth: ['komatsu', 'komatsu limited'] 42 | GPT-3 Prediction: [''] 43 | 44 | 45 | SubjectEntity: Hyundai Motor Company 46 | Ground Truth: ['hyundai'] 47 | GPT-3 Prediction: ['hyundai motor group'] 48 | 49 | 50 | SubjectEntity: Lotus Cars 51 | Ground Truth: ['geely'] 52 | GPT-3 Prediction: ['proton'] 53 | 54 | 55 | SubjectEntity: MG Car Company Limited 56 | Ground Truth: [] 57 | GPT-3 Prediction: ['saic motor corporation limited'] 58 | 59 | 60 | SubjectEntity: NSU 61 | Ground Truth: ['volkswagen'] 62 | GPT-3 Prediction: [''] 63 | 64 | 65 | SubjectEntity: Peugeot 66 | Ground Truth: ['stellantis'] 67 | GPT-3 Prediction: ['groupe psa'] 68 | 69 | 70 | SubjectEntity: Scripps-Booth 71 | Ground Truth: ['general motors', 'gm'] 72 | GPT-3 Prediction: [''] 73 | 74 | 75 | SubjectEntity: Simca 76 | Ground Truth: ['chrysler', 'chrysler europe'] 77 | GPT-3 Prediction: ['chrysler europe', 'fiat', 'peugeot', 'simca'] 78 | 79 | 80 | SubjectEntity: Valmet Automotive 81 | Ground Truth: [] 82 | GPT-3 Prediction: ['valmet'] 83 | 84 | 85 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-none)/CountryBordersWithCountry.txt: -------------------------------------------------------------------------------- 1 | CountryBordersWithCountry (average f1: 0.758): 15 cases 2 | 3 | 4 | 5 | SubjectEntity: Australia 6 | Ground Truth: ['east timor', 'indonesia', 'new zealand', 'papua new guinea', 'timor'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Bahrain 11 | Ground Truth: ['iran', 'saudi arabia'] 12 | GPT-3 Prediction: ['qatar', 'saudi arabia'] 13 | 14 | 15 | SubjectEntity: Comoros 16 | Ground Truth: ['france', 'madagascar', 'mozambique', 'seychelles', 'tanzania'] 17 | GPT-3 Prediction: [''] 18 | 19 | 20 | SubjectEntity: Cuba 21 | Ground Truth: ['united states of america', 'usa'] 22 | GPT-3 Prediction: ['bahamas', 'cayman islands', 'haiti', 'jamaica', 'united states'] 23 | 24 | 25 | SubjectEntity: Dominican Republic 26 | Ground Truth: ['haiti', 'united states of america', 'usa', 'venezuela'] 27 | GPT-3 Prediction: ['haiti', 'puerto rico'] 28 | 29 | 30 | SubjectEntity: Gambia 31 | Ground Truth: ['senegal'] 32 | GPT-3 Prediction: ['guinea', 'guinea-bissau', 'senegal'] 33 | 34 | 35 | SubjectEntity: Ghana 36 | Ground Truth: ['burkina faso', 'ivory coast', 'togo'] 37 | GPT-3 Prediction: ['benin', 'burkina faso', 'cote divoire', 'togo'] 38 | 39 | 40 | SubjectEntity: Grenada 41 | Ground Truth: ['venezuela'] 42 | GPT-3 Prediction: ['trinidad and tobago', 'venezuela'] 43 | 44 | 45 | SubjectEntity: Japan 46 | Ground Truth: ['china', 'korea', "people's republic of china", 'philippines', 'russia', 'south korea', 'taiwan', 'united states of america', 'usa'] 47 | GPT-3 Prediction: ['china', 'north korea', 'russia', 'south korea'] 48 | 49 | 50 | SubjectEntity: Kingdom of Denmark 51 | Ground Truth: ['canada', 'germany', 'iceland', 'norway', 'poland', 'sweden', 'uk', 'united kingdom'] 52 | GPT-3 Prediction: ['germany', 'norway', 'sweden'] 53 | 54 | 55 | SubjectEntity: Portugal 56 | Ground Truth: ['castile and leon', 'castile and león', 'spain'] 57 | GPT-3 Prediction: ['andorra', 'morocco', 'spain'] 58 | 59 | 60 | SubjectEntity: Saint Vincent and the Grenadines 61 | Ground Truth: ['venezuela'] 62 | GPT-3 Prediction: [''] 63 | 64 | 65 | SubjectEntity: Samoa 66 | Ground Truth: ['united states of america', 'usa'] 67 | GPT-3 Prediction: [''] 68 | 69 | 70 | SubjectEntity: United kingdom 71 | Ground Truth: ['ireland', 'republic of ireland'] 72 | GPT-3 Prediction: ['belgium', 'france', 'germany', 'ireland', 'netherlands'] 73 | 74 | 75 | SubjectEntity: Venezuela 76 | Ground Truth: ['brazil', 'colombia', 'dominica', 'dominican republic', 'france', 'grenada', 'guyana', 'kingdom of the netherlands', 'netherlands', 'saint kitts and nevis', 'saint lucia', 'saint vincent and the grenadines', 'trinidad and tobago', 'uk', 'united kingdom', 'united states of america', 'usa'] 77 | GPT-3 Prediction: ['brazil', 'colombia', 'guyana'] 78 | 79 | 80 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-none)/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | CountryOfficialLanguage (average f1: 0.785): 22 cases 2 | 3 | 4 | 5 | SubjectEntity: Afghanistan 6 | Ground Truth: ['arabic', 'baluchi', 'dari', 'nuristani', 'pamir', 'pashayi', 'pashto', 'turkmen', 'uzbek'] 7 | GPT-3 Prediction: ['dari', 'pashto'] 8 | 9 | 10 | SubjectEntity: Botswana 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: ['english', 'setswana'] 13 | 14 | 15 | SubjectEntity: Burkina Faso 16 | Ground Truth: ['french'] 17 | GPT-3 Prediction: ['dioula', 'french', 'fula', 'mossi'] 18 | 19 | 20 | SubjectEntity: Burundi 21 | Ground Truth: ['english', 'french', 'kirundi'] 22 | GPT-3 Prediction: ['french', 'kirundi', 'swahili'] 23 | 24 | 25 | SubjectEntity: Croatia 26 | Ground Truth: ['croatian'] 27 | GPT-3 Prediction: ['croatian', 'serbian'] 28 | 29 | 30 | SubjectEntity: Grenada 31 | Ground Truth: ['creole', 'english', 'grenadian', 'grenadian creole english'] 32 | GPT-3 Prediction: ['english'] 33 | 34 | 35 | SubjectEntity: Israel 36 | Ground Truth: ['hebrew'] 37 | GPT-3 Prediction: ['arabic', 'hebrew'] 38 | 39 | 40 | SubjectEntity: Jamaica 41 | Ground Truth: ['english'] 42 | GPT-3 Prediction: ['english', 'patois'] 43 | 44 | 45 | SubjectEntity: Jordan 46 | Ground Truth: ['arabic'] 47 | GPT-3 Prediction: ['arabic', 'english'] 48 | 49 | 50 | SubjectEntity: Latvia 51 | Ground Truth: ['latvian'] 52 | GPT-3 Prediction: ['latvian', 'russian'] 53 | 54 | 55 | SubjectEntity: Liechtenstein 56 | Ground Truth: ['german'] 57 | GPT-3 Prediction: ['alemannic', 'german'] 58 | 59 | 60 | SubjectEntity: Maldives 61 | Ground Truth: ['maldivian'] 62 | GPT-3 Prediction: ['dhivehi'] 63 | 64 | 65 | SubjectEntity: Mozambique 66 | Ground Truth: ['portuguese'] 67 | GPT-3 Prediction: ['cisena', 'echuwabo', 'emakhuwa', 'portuguese', 'xichangana'] 68 | 69 | 70 | SubjectEntity: Niger 71 | Ground Truth: ['french'] 72 | GPT-3 Prediction: ['french', 'hausa', 'zarma'] 73 | 74 | 75 | SubjectEntity: North Macedonia 76 | Ground Truth: ['albanian', 'macedonian'] 77 | GPT-3 Prediction: ['albanian', 'macedonian', 'romani', 'serbian', 'turkish'] 78 | 79 | 80 | SubjectEntity: Norway 81 | Ground Truth: ['bokmal', 'bokmål', 'norwegian', 'nynorsk', 'sami', 'sámi'] 82 | GPT-3 Prediction: ['norwegian', 'sami'] 83 | 84 | 85 | SubjectEntity: Panama 86 | Ground Truth: ['spanish'] 87 | GPT-3 Prediction: ['english', 'spanish'] 88 | 89 | 90 | SubjectEntity: People's Republic of China 91 | Ground Truth: ['chinese', 'mandarin', 'standard mandarin'] 92 | GPT-3 Prediction: ['mandarin'] 93 | 94 | 95 | SubjectEntity: Qatar 96 | Ground Truth: ['arabic'] 97 | GPT-3 Prediction: ['arabic', 'english'] 98 | 99 | 100 | SubjectEntity: Sierra Leone 101 | Ground Truth: ['english'] 102 | GPT-3 Prediction: ['english', 'krio'] 103 | 104 | 105 | SubjectEntity: Taiwan 106 | Ground Truth: ['amis', 'austronesian', 'formosan', 'hakka', 'hokkien', 'mandarin', 'paiwan', 'standard taiwanese mandarin', 'taiwanese', 'taiwanese', 'taiwanese hokkien'] 107 | GPT-3 Prediction: ['hakka', 'mandarin', 'taiwanese'] 108 | 109 | 110 | SubjectEntity: Zimbabwe 111 | Ground Truth: ['barwe', 'chewa', 'english', 'kalanga', 'khoisan', 'nambya', 'ndau', 'ndebele', 'northern ndebele', 'sesotho', 'shona', 'tonga', 'tsonga', 'tswana', 'venda', 'xhosa'] 112 | GPT-3 Prediction: ['english', 'ndebele', 'shona'] 113 | 114 | 115 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-none)/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonCauseOfDeath (average f1: 0.56): 22 cases 2 | 3 | 4 | 5 | SubjectEntity: Ahmed Zewail 6 | Ground Truth: ['lymphoma', 'spinal cord lymphoma'] 7 | GPT-3 Prediction: ['cancer'] 8 | 9 | 10 | SubjectEntity: Avicii 11 | Ground Truth: ['exsanguination'] 12 | GPT-3 Prediction: ['suicide'] 13 | 14 | 15 | SubjectEntity: Christina Grimmie 16 | Ground Truth: ['ballistic trauma', 'trauma'] 17 | GPT-3 Prediction: ['murder'] 18 | 19 | 20 | SubjectEntity: Dilip Kumar 21 | Ground Truth: ['cancer', 'prostate cancer'] 22 | GPT-3 Prediction: [''] 23 | 24 | 25 | SubjectEntity: Ennio Morricone 26 | Ground Truth: ['femoral fracture', 'fracture'] 27 | GPT-3 Prediction: [''] 28 | 29 | 30 | SubjectEntity: Frank Sinatra 31 | Ground Truth: ['infarction', 'myocardial infarction'] 32 | GPT-3 Prediction: ['heart attack'] 33 | 34 | 35 | SubjectEntity: George H. W. Bush 36 | Ground Truth: ['parkinson', "parkinson's disease"] 37 | GPT-3 Prediction: ['infarction'] 38 | 39 | 40 | SubjectEntity: Haruma Miura 41 | Ground Truth: ['hanging'] 42 | GPT-3 Prediction: ['suicide'] 43 | 44 | 45 | SubjectEntity: Hugh Hefner 46 | Ground Truth: ['sepsis'] 47 | GPT-3 Prediction: ['natural causes'] 48 | 49 | 50 | SubjectEntity: Idriss Déby 51 | Ground Truth: ['ballistic trauma', 'trauma'] 52 | GPT-3 Prediction: ['injury'] 53 | 54 | 55 | SubjectEntity: Jim Rohn 56 | Ground Truth: ['fibrosis', 'pulmonary fibrosis'] 57 | GPT-3 Prediction: ['pneumonia'] 58 | 59 | 60 | SubjectEntity: John Magufuli 61 | Ground Truth: ['arrhythmia', 'heart arrhythmia'] 62 | GPT-3 Prediction: [''] 63 | 64 | 65 | SubjectEntity: Johnny Cash 66 | Ground Truth: ['diabetes'] 67 | GPT-3 Prediction: ['infarction'] 68 | 69 | 70 | SubjectEntity: Larry King 71 | Ground Truth: ['sepsis'] 72 | GPT-3 Prediction: [''] 73 | 74 | 75 | SubjectEntity: Maurane 76 | Ground Truth: ['falling'] 77 | GPT-3 Prediction: [''] 78 | 79 | 80 | SubjectEntity: Maya Angelou 81 | Ground Truth: ['disease'] 82 | GPT-3 Prediction: [''] 83 | 84 | 85 | SubjectEntity: Miles Davis 86 | Ground Truth: ['pneumonia', 'stroke'] 87 | GPT-3 Prediction: [''] 88 | 89 | 90 | SubjectEntity: Nelson Mandela 91 | Ground Truth: ['respiratory disease'] 92 | GPT-3 Prediction: ['respiratory infection'] 93 | 94 | 95 | SubjectEntity: Pitbull 96 | Ground Truth: [] 97 | GPT-3 Prediction: ['infarction'] 98 | 99 | 100 | SubjectEntity: Rishi Kapoor 101 | Ground Truth: ['leukemia'] 102 | GPT-3 Prediction: ['cancer'] 103 | 104 | 105 | SubjectEntity: Roddy Piper 106 | Ground Truth: ['cardiac arrest'] 107 | GPT-3 Prediction: ['heart attack'] 108 | 109 | 110 | SubjectEntity: Roy Orbison 111 | Ground Truth: ['infarction', 'myocardial infarction'] 112 | GPT-3 Prediction: ['heart attack'] 113 | 114 | 115 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-none)/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | PersonLanguage (average f1: 0.793): 21 cases 2 | 3 | 4 | 5 | SubjectEntity: A. R. Rahman 6 | Ground Truth: ['hindi', 'tamil', 'telugu'] 7 | GPT-3 Prediction: ['english', 'hindi', 'malayalam', 'tamil', 'telugu'] 8 | 9 | 10 | SubjectEntity: Abdullah Gül 11 | Ground Truth: ['turkish'] 12 | GPT-3 Prediction: ['english', 'french', 'turkish'] 13 | 14 | 15 | SubjectEntity: Akon 16 | Ground Truth: ['english'] 17 | GPT-3 Prediction: ['english', 'french', 'wolof'] 18 | 19 | 20 | SubjectEntity: Avril Lavigne 21 | Ground Truth: ['english'] 22 | GPT-3 Prediction: ['english', 'french'] 23 | 24 | 25 | SubjectEntity: Blaise Matuidi 26 | Ground Truth: ['french'] 27 | GPT-3 Prediction: ['english', 'french', 'italian', 'spanish'] 28 | 29 | 30 | SubjectEntity: Genelia D'Souza 31 | Ground Truth: ['hindi', 'malayalam', 'tamil', 'telugu'] 32 | GPT-3 Prediction: ['english', 'hindi', 'marathi', 'telugu'] 33 | 34 | 35 | SubjectEntity: Henrique Capriles Radonski 36 | Ground Truth: ['spanish'] 37 | GPT-3 Prediction: ['english', 'portuguese', 'spanish'] 38 | 39 | 40 | SubjectEntity: Hugo Chávez 41 | Ground Truth: ['spanish'] 42 | GPT-3 Prediction: ['english', 'spanish'] 43 | 44 | 45 | SubjectEntity: John Cena 46 | Ground Truth: ['chinese', 'english', 'mandarin', 'mandarin chinese'] 47 | GPT-3 Prediction: ['english'] 48 | 49 | 50 | SubjectEntity: Jose Michimani 51 | Ground Truth: ['english'] 52 | GPT-3 Prediction: ['aymara', 'quechua', 'spanish'] 53 | 54 | 55 | SubjectEntity: Mohammed bin Rashid Al Maktoum 56 | Ground Truth: ['arabic', 'english'] 57 | GPT-3 Prediction: ['arabic', 'english', 'hindi', 'persian', 'urdu'] 58 | 59 | 60 | SubjectEntity: Oscar dos Santos Emboaba Júnior 61 | Ground Truth: ['portuguese'] 62 | GPT-3 Prediction: ['english', 'portuguese'] 63 | 64 | 65 | SubjectEntity: Patrick Stewart 66 | Ground Truth: ['english'] 67 | GPT-3 Prediction: ['english', 'french'] 68 | 69 | 70 | SubjectEntity: Pau Gasol 71 | Ground Truth: ['catalan', 'english', 'french', 'italian', 'spanish'] 72 | GPT-3 Prediction: ['catalan', 'english', 'spanish'] 73 | 74 | 75 | SubjectEntity: Paulo Coelho 76 | Ground Truth: ['portuguese'] 77 | GPT-3 Prediction: ['english', 'french', 'portuguese', 'spanish'] 78 | 79 | 80 | SubjectEntity: Rihanna 81 | Ground Truth: ['english'] 82 | GPT-3 Prediction: ['creole', 'english'] 83 | 84 | 85 | SubjectEntity: Salman Khan 86 | Ground Truth: ['hindi'] 87 | GPT-3 Prediction: ['hindi', 'urdu'] 88 | 89 | 90 | SubjectEntity: Samir Nasri 91 | Ground Truth: ['french'] 92 | GPT-3 Prediction: ['arabic', 'english', 'french'] 93 | 94 | 95 | SubjectEntity: Thiago Alcântara 96 | Ground Truth: ['spanish'] 97 | GPT-3 Prediction: ['catalan', 'english', 'portuguese', 'spanish'] 98 | 99 | 100 | SubjectEntity: Vladimir Putin 101 | Ground Truth: ['english', 'german', 'russian', 'swedish'] 102 | GPT-3 Prediction: ['english', 'french', 'german', 'russian'] 103 | 104 | 105 | SubjectEntity: Zlatan Ibrahimović 106 | Ground Truth: ['bosnian', 'english', 'italian', 'spanish', 'swedish'] 107 | GPT-3 Prediction: ['bosnian', 'croatian', 'english', 'french', 'italian', 'montenegrin', 'portuguese', 'serbian', 'spanish', 'swahili'] 108 | 109 | 110 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-none)/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonPlaceOfDeath (average f1: 0.78): 11 cases 2 | 3 | 4 | 5 | SubjectEntity: Abdul Rehman Malik 6 | Ground Truth: ['islamabad'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Alfredo González Flores 11 | Ground Truth: ['heredia'] 12 | GPT-3 Prediction: [''] 13 | 14 | 15 | SubjectEntity: Almudena Grandes 16 | Ground Truth: ['madrid'] 17 | GPT-3 Prediction: [''] 18 | 19 | 20 | SubjectEntity: Anne Rice 21 | Ground Truth: ['rancho mirage'] 22 | GPT-3 Prediction: ['new orleans'] 23 | 24 | 25 | SubjectEntity: Aretha Franklin 26 | Ground Truth: ['detroit'] 27 | GPT-3 Prediction: [''] 28 | 29 | 30 | SubjectEntity: Avicii 31 | Ground Truth: ['muscat'] 32 | GPT-3 Prediction: [''] 33 | 34 | 35 | SubjectEntity: Chick Corea 36 | Ground Truth: ['tampa', 'tampa bay area'] 37 | GPT-3 Prediction: [''] 38 | 39 | 40 | SubjectEntity: John Coltrane 41 | Ground Truth: ['huntington'] 42 | GPT-3 Prediction: ['new york city'] 43 | 44 | 45 | SubjectEntity: Ken Robinson 46 | Ground Truth: ['london'] 47 | GPT-3 Prediction: [''] 48 | 49 | 50 | SubjectEntity: Michael Collins 51 | Ground Truth: ['naples'] 52 | GPT-3 Prediction: [''] 53 | 54 | 55 | SubjectEntity: Rachel Caine 56 | Ground Truth: ['texas'] 57 | GPT-3 Prediction: [''] 58 | 59 | 60 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-language-none)/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | RiverBasinsCountry (average f1: 0.822): 18 cases 2 | 3 | 4 | 5 | SubjectEntity: Brenta 6 | Ground Truth: ['italy'] 7 | GPT-3 Prediction: ['austria', 'italy'] 8 | 9 | 10 | SubjectEntity: Didi Liakhvi river 11 | Ground Truth: ['georgia'] 12 | GPT-3 Prediction: ['georgia', 'russia'] 13 | 14 | 15 | SubjectEntity: Dniester 16 | Ground Truth: ['moldova', 'ukraine'] 17 | GPT-3 Prediction: ['moldova', 'romania', 'ukraine'] 18 | 19 | 20 | SubjectEntity: Gadar river 21 | Ground Truth: ['iran'] 22 | GPT-3 Prediction: ['iraq', 'syria', 'turkey'] 23 | 24 | 25 | SubjectEntity: Gambia river 26 | Ground Truth: ['gambia', 'guinea', 'senegal', 'the gambia'] 27 | GPT-3 Prediction: ['senegal', 'the gambia'] 28 | 29 | 30 | SubjectEntity: Han river 31 | Ground Truth: ['korea', 'korea', 'north korea', 'south korea'] 32 | GPT-3 Prediction: ['china'] 33 | 34 | 35 | SubjectEntity: Kura river 36 | Ground Truth: ['azerbaijan', 'georgia', 'turkey'] 37 | GPT-3 Prediction: ['armenia', 'azerbaijan', 'georgia', 'iran', 'turkey', 'turkmenistan', 'uzbekistan'] 38 | 39 | 40 | SubjectEntity: Narva river 41 | Ground Truth: ['belarus', 'estonia', 'latvia', 'russia'] 42 | GPT-3 Prediction: ['estonia', 'russia'] 43 | 44 | 45 | SubjectEntity: Neman 46 | Ground Truth: ['belarus', 'latvia', 'lithuania', 'russia'] 47 | GPT-3 Prediction: ['belarus', 'lithuania', 'poland', 'russia'] 48 | 49 | 50 | SubjectEntity: Neris 51 | Ground Truth: ['belarus', 'latvia', 'lithuania'] 52 | GPT-3 Prediction: ['belarus', 'lithuania', 'russia'] 53 | 54 | 55 | SubjectEntity: Ottawa river 56 | Ground Truth: ['canada'] 57 | GPT-3 Prediction: ['canada', 'united states'] 58 | 59 | 60 | SubjectEntity: Red river of the south 61 | Ground Truth: ['texas', 'united states of america', 'usa'] 62 | GPT-3 Prediction: ['mexico', 'united states'] 63 | 64 | 65 | SubjectEntity: San 66 | Ground Truth: ['poland', 'ukraine'] 67 | GPT-3 Prediction: ['mexico'] 68 | 69 | 70 | SubjectEntity: Shatt al-Arab 71 | Ground Truth: ['iran', 'iraq', 'kuwait', 'saudi arabia', 'syria', 'turkey'] 72 | GPT-3 Prediction: ['iran', 'iraq'] 73 | 74 | 75 | SubjectEntity: Siret river 76 | Ground Truth: ['romania', 'ukraine'] 77 | GPT-3 Prediction: ['moldova', 'romania', 'ukraine'] 78 | 79 | 80 | SubjectEntity: Torne river 81 | Ground Truth: ['finland', 'norway', 'sweden'] 82 | GPT-3 Prediction: ['finland', 'sweden'] 83 | 84 | 85 | SubjectEntity: Ussuri 86 | Ground Truth: ['russia'] 87 | GPT-3 Prediction: ['china', 'russia'] 88 | 89 | 90 | SubjectEntity: Yantra 91 | Ground Truth: ['bulgaria'] 92 | GPT-3 Prediction: ['bulgaria', 'romania'] 93 | 94 | 95 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)/ChemicalCompoundElement.txt: -------------------------------------------------------------------------------- 1 | ChemicalCompoundElement (average f1: 0.94): 17 cases 2 | 3 | 4 | 5 | SubjectEntity: Adenosine Diphosphate Ribose 6 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 7 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 8 | 9 | 10 | SubjectEntity: Aluminium Chlorohydrate 11 | Ground Truth: ['aluminium', 'chlorine', 'hydrogen', 'oxygen'] 12 | GPT-3 Prediction: ['aluminium', 'chlorine', 'hydrogen'] 13 | 14 | 15 | SubjectEntity: Aluminum Iodide 16 | Ground Truth: ['aluminium', 'iodine'] 17 | GPT-3 Prediction: ['aluminum', 'iodine'] 18 | 19 | 20 | SubjectEntity: Atracurium Besylate 21 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 22 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 23 | 24 | 25 | SubjectEntity: Cetyl Alcohol 26 | Ground Truth: ['carbon', 'hydrogen', 'oxygen'] 27 | GPT-3 Prediction: ['carbon', 'hydrogen'] 28 | 29 | 30 | SubjectEntity: Chlorophyll F 31 | Ground Truth: ['carbon', 'hydrogen', 'magnesium', 'nitrogen', 'oxygen'] 32 | GPT-3 Prediction: ['carbon', 'hydrogen', 'magnesium', 'oxygen'] 33 | 34 | 35 | SubjectEntity: Flavin Adenine Dinucleotide 36 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 37 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 38 | 39 | 40 | SubjectEntity: Glucagon 41 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 42 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 43 | 44 | 45 | SubjectEntity: Glutathione 46 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 47 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'sulfur'] 48 | 49 | 50 | SubjectEntity: Heroin 51 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 52 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 53 | 54 | 55 | SubjectEntity: Hydroxocobalamin 56 | Ground Truth: ['carbon', 'cobalt', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 57 | GPT-3 Prediction: ['cobalt', 'hydrogen', 'nitrogen', 'oxygen'] 58 | 59 | 60 | SubjectEntity: Metformin 61 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen'] 62 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 63 | 64 | 65 | SubjectEntity: Motilin 66 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 67 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 68 | 69 | 70 | SubjectEntity: Mustard Gas 71 | Ground Truth: ['carbon', 'chlorine', 'hydrogen', 'sulfur'] 72 | GPT-3 Prediction: ['carbon', 'chlorine', 'hydrogen', 'oxygen', 'sulfur'] 73 | 74 | 75 | SubjectEntity: Tabun 76 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 77 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 78 | 79 | 80 | SubjectEntity: Tenoxicam 81 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 82 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 83 | 84 | 85 | SubjectEntity: Triphenylphosphine 86 | Ground Truth: ['carbon', 'hydrogen', 'phosphorus'] 87 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'phosphorus'] 88 | 89 | 90 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | CompanyParentOrganization (average f1: 0.77): 12 cases 2 | 3 | 4 | 5 | SubjectEntity: Alex von Falkenhausen Motorenbau 6 | Ground Truth: [] 7 | GPT-3 Prediction: ['bmw'] 8 | 9 | 10 | SubjectEntity: Aston Martin lagonda 11 | Ground Truth: [] 12 | GPT-3 Prediction: ['ford motor company', 'investindustrial', 'kuwait investment authority'] 13 | 14 | 15 | SubjectEntity: Austro-Daimler 16 | Ground Truth: [] 17 | GPT-3 Prediction: ['daimler ag'] 18 | 19 | 20 | SubjectEntity: Changan Automobile 21 | Ground Truth: ['china south industries', 'china south industries group'] 22 | GPT-3 Prediction: ['changan group'] 23 | 24 | 25 | SubjectEntity: Hanomag 26 | Ground Truth: ['komatsu', 'komatsu limited'] 27 | GPT-3 Prediction: [''] 28 | 29 | 30 | SubjectEntity: Hyundai Motor Company 31 | Ground Truth: ['hyundai'] 32 | GPT-3 Prediction: ['hyundai motor group'] 33 | 34 | 35 | SubjectEntity: Lotus Cars 36 | Ground Truth: ['geely'] 37 | GPT-3 Prediction: ['geely holding group', 'proton holdings berhad'] 38 | 39 | 40 | SubjectEntity: MG Car Company Limited 41 | Ground Truth: [] 42 | GPT-3 Prediction: ['saic motor corporation limited'] 43 | 44 | 45 | SubjectEntity: NSU 46 | Ground Truth: ['volkswagen'] 47 | GPT-3 Prediction: ['audi ag', 'volkswagen group'] 48 | 49 | 50 | SubjectEntity: Peugeot 51 | Ground Truth: ['stellantis'] 52 | GPT-3 Prediction: ['psa peugeot citroën', 'peugeot s.a.'] 53 | 54 | 55 | SubjectEntity: Scripps-Booth 56 | Ground Truth: ['general motors', 'gm'] 57 | GPT-3 Prediction: [''] 58 | 59 | 60 | SubjectEntity: Simca 61 | Ground Truth: ['chrysler', 'chrysler europe'] 62 | GPT-3 Prediction: ['chrysler europe', 'psa peugeot citroën', 'peugeot'] 63 | 64 | 65 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)/CountryBordersWithCountry.txt: -------------------------------------------------------------------------------- 1 | CountryBordersWithCountry (average f1: 0.785): 14 cases 2 | 3 | 4 | 5 | SubjectEntity: Bahrain 6 | Ground Truth: ['iran', 'saudi arabia'] 7 | GPT-3 Prediction: ['qatar', 'saudi arabia', 'united arab emirates'] 8 | 9 | 10 | SubjectEntity: Barbados 11 | Ground Truth: [] 12 | GPT-3 Prediction: ['guyana', 'saint lucia', 'saint vincent and the grenadines', 'trinidad and tobago', 'venezuela'] 13 | 14 | 15 | SubjectEntity: Comoros 16 | Ground Truth: ['france', 'madagascar', 'mozambique', 'seychelles', 'tanzania'] 17 | GPT-3 Prediction: ['madagascar', 'mayotte', 'mozambique', 'seychelles'] 18 | 19 | 20 | SubjectEntity: Cuba 21 | Ground Truth: ['united states of america', 'usa'] 22 | GPT-3 Prediction: ['bahamas', 'cayman islands', 'haiti', 'jamaica', 'mexico', 'united states'] 23 | 24 | 25 | SubjectEntity: Dominican Republic 26 | Ground Truth: ['haiti', 'united states of america', 'usa', 'venezuela'] 27 | GPT-3 Prediction: ['haiti', 'puerto rico'] 28 | 29 | 30 | SubjectEntity: Gambia 31 | Ground Truth: ['senegal'] 32 | GPT-3 Prediction: ['guinea', 'guinea-bissau', 'senegal'] 33 | 34 | 35 | SubjectEntity: Ghana 36 | Ground Truth: ['burkina faso', 'ivory coast', 'togo'] 37 | GPT-3 Prediction: ['benin', 'burkina faso', 'cote divoire', 'togo'] 38 | 39 | 40 | SubjectEntity: Grenada 41 | Ground Truth: ['venezuela'] 42 | GPT-3 Prediction: ['trinidad and tobago', 'venezuela'] 43 | 44 | 45 | SubjectEntity: Japan 46 | Ground Truth: ['china', 'korea', "people's republic of china", 'philippines', 'russia', 'south korea', 'taiwan', 'united states of america', 'usa'] 47 | GPT-3 Prediction: ['china', 'north korea', 'russia', 'south korea'] 48 | 49 | 50 | SubjectEntity: Kingdom of Denmark 51 | Ground Truth: ['canada', 'germany', 'iceland', 'norway', 'poland', 'sweden', 'uk', 'united kingdom'] 52 | GPT-3 Prediction: ['germany', 'norway', 'sweden'] 53 | 54 | 55 | SubjectEntity: Portugal 56 | Ground Truth: ['castile and leon', 'castile and león', 'spain'] 57 | GPT-3 Prediction: ['spain'] 58 | 59 | 60 | SubjectEntity: Saint Vincent and the Grenadines 61 | Ground Truth: ['venezuela'] 62 | GPT-3 Prediction: ['barbados', 'grenada', 'saint lucia'] 63 | 64 | 65 | SubjectEntity: Samoa 66 | Ground Truth: ['united states of america', 'usa'] 67 | GPT-3 Prediction: [''] 68 | 69 | 70 | SubjectEntity: Venezuela 71 | Ground Truth: ['brazil', 'colombia', 'dominica', 'dominican republic', 'france', 'grenada', 'guyana', 'kingdom of the netherlands', 'netherlands', 'saint kitts and nevis', 'saint lucia', 'saint vincent and the grenadines', 'trinidad and tobago', 'uk', 'united kingdom', 'united states of america', 'usa'] 72 | GPT-3 Prediction: ['brazil', 'colombia', 'guyana'] 73 | 74 | 75 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | CountryOfficialLanguage (average f1: 0.833): 15 cases 2 | 3 | 4 | 5 | SubjectEntity: Afghanistan 6 | Ground Truth: ['arabic', 'baluchi', 'dari', 'nuristani', 'pamir', 'pashayi', 'pashto', 'turkmen', 'uzbek'] 7 | GPT-3 Prediction: ['dari', 'pashto'] 8 | 9 | 10 | SubjectEntity: Botswana 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: ['tswana'] 13 | 14 | 15 | SubjectEntity: Brunei 16 | Ground Truth: ['english', 'malay'] 17 | GPT-3 Prediction: ['malay'] 18 | 19 | 20 | SubjectEntity: Burundi 21 | Ground Truth: ['english', 'french', 'kirundi'] 22 | GPT-3 Prediction: ['french', 'kirundi', 'swahili'] 23 | 24 | 25 | SubjectEntity: East Timor 26 | Ground Truth: ['portuguese', 'tetum'] 27 | GPT-3 Prediction: ['indonesian', 'portuguese', 'tetum'] 28 | 29 | 30 | SubjectEntity: Grenada 31 | Ground Truth: ['creole', 'english', 'grenadian', 'grenadian creole english'] 32 | GPT-3 Prediction: ['english'] 33 | 34 | 35 | SubjectEntity: Israel 36 | Ground Truth: ['hebrew'] 37 | GPT-3 Prediction: ['arabic', 'hebrew'] 38 | 39 | 40 | SubjectEntity: Maldives 41 | Ground Truth: ['maldivian'] 42 | GPT-3 Prediction: ['dhivehi'] 43 | 44 | 45 | SubjectEntity: North Macedonia 46 | Ground Truth: ['albanian', 'macedonian'] 47 | GPT-3 Prediction: ['macedonian'] 48 | 49 | 50 | SubjectEntity: Norway 51 | Ground Truth: ['bokmal', 'bokmål', 'norwegian', 'nynorsk', 'sami', 'sámi'] 52 | GPT-3 Prediction: ['norwegian'] 53 | 54 | 55 | SubjectEntity: Pakistan 56 | Ground Truth: ['arabic', 'english', 'urdu'] 57 | GPT-3 Prediction: ['urdu'] 58 | 59 | 60 | SubjectEntity: People's Republic of China 61 | Ground Truth: ['chinese', 'mandarin', 'standard mandarin'] 62 | GPT-3 Prediction: ['mandarin'] 63 | 64 | 65 | SubjectEntity: Taiwan 66 | Ground Truth: ['amis', 'austronesian', 'formosan', 'hakka', 'hokkien', 'mandarin', 'paiwan', 'standard taiwanese mandarin', 'taiwanese', 'taiwanese', 'taiwanese hokkien'] 67 | GPT-3 Prediction: ['mandarin'] 68 | 69 | 70 | SubjectEntity: Tajikistan 71 | Ground Truth: ['russian', 'tajik'] 72 | GPT-3 Prediction: ['tajik'] 73 | 74 | 75 | SubjectEntity: Zimbabwe 76 | Ground Truth: ['barwe', 'chewa', 'english', 'kalanga', 'khoisan', 'nambya', 'ndau', 'ndebele', 'northern ndebele', 'sesotho', 'shona', 'tonga', 'tsonga', 'tswana', 'venda', 'xhosa'] 77 | GPT-3 Prediction: ['english', 'ndebele', 'shona'] 78 | 79 | 80 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonCauseOfDeath (average f1: 0.593): 20 cases 2 | 3 | 4 | 5 | SubjectEntity: Ahmed Zewail 6 | Ground Truth: ['lymphoma', 'spinal cord lymphoma'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Avicii 11 | Ground Truth: ['exsanguination'] 12 | GPT-3 Prediction: ['suicide'] 13 | 14 | 15 | SubjectEntity: Christina Grimmie 16 | Ground Truth: ['ballistic trauma', 'trauma'] 17 | GPT-3 Prediction: ['murder'] 18 | 19 | 20 | SubjectEntity: Cory Monteith 21 | Ground Truth: ['drug overdose', 'overdose'] 22 | GPT-3 Prediction: ['mixed drug toxicity'] 23 | 24 | 25 | SubjectEntity: Dilip Kumar 26 | Ground Truth: ['cancer', 'prostate cancer'] 27 | GPT-3 Prediction: [''] 28 | 29 | 30 | SubjectEntity: Ennio Morricone 31 | Ground Truth: ['femoral fracture', 'fracture'] 32 | GPT-3 Prediction: [''] 33 | 34 | 35 | SubjectEntity: Frank Sinatra 36 | Ground Truth: ['infarction', 'myocardial infarction'] 37 | GPT-3 Prediction: ['heart attack'] 38 | 39 | 40 | SubjectEntity: George H. W. Bush 41 | Ground Truth: ['parkinson', "parkinson's disease"] 42 | GPT-3 Prediction: ['septic shock'] 43 | 44 | 45 | SubjectEntity: Haruma Miura 46 | Ground Truth: ['hanging'] 47 | GPT-3 Prediction: [''] 48 | 49 | 50 | SubjectEntity: Hugh Hefner 51 | Ground Truth: ['sepsis'] 52 | GPT-3 Prediction: ['septicemia'] 53 | 54 | 55 | SubjectEntity: Idriss Déby 56 | Ground Truth: ['ballistic trauma', 'trauma'] 57 | GPT-3 Prediction: ['blunt trauma', 'injury'] 58 | 59 | 60 | SubjectEntity: Jim Rohn 61 | Ground Truth: ['fibrosis', 'pulmonary fibrosis'] 62 | GPT-3 Prediction: ['pneumonia'] 63 | 64 | 65 | SubjectEntity: John Magufuli 66 | Ground Truth: ['arrhythmia', 'heart arrhythmia'] 67 | GPT-3 Prediction: ['covid-19'] 68 | 69 | 70 | SubjectEntity: Johnny Cash 71 | Ground Truth: ['diabetes'] 72 | GPT-3 Prediction: ['infarction'] 73 | 74 | 75 | SubjectEntity: Larry King 76 | Ground Truth: ['sepsis'] 77 | GPT-3 Prediction: [''] 78 | 79 | 80 | SubjectEntity: Maurane 81 | Ground Truth: ['falling'] 82 | GPT-3 Prediction: ['cancer'] 83 | 84 | 85 | SubjectEntity: Maya Angelou 86 | Ground Truth: ['disease'] 87 | GPT-3 Prediction: [''] 88 | 89 | 90 | SubjectEntity: Nelson Mandela 91 | Ground Truth: ['respiratory disease'] 92 | GPT-3 Prediction: ['respiratory infection'] 93 | 94 | 95 | SubjectEntity: Roddy Piper 96 | Ground Truth: ['cardiac arrest'] 97 | GPT-3 Prediction: ['aortic aneurysm'] 98 | 99 | 100 | SubjectEntity: Roy Orbison 101 | Ground Truth: ['infarction', 'myocardial infarction'] 102 | GPT-3 Prediction: ['heart attack'] 103 | 104 | 105 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)/PersonInstrument.txt: -------------------------------------------------------------------------------- 1 | PersonInstrument (average f1: 0.568): 22 cases 2 | 3 | 4 | 5 | SubjectEntity: A. J. McLean 6 | Ground Truth: ['lyric spinto tenor', 'tenor'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Arnel Pineda 11 | Ground Truth: ['guitar'] 12 | GPT-3 Prediction: [''] 13 | 14 | 15 | SubjectEntity: Billie Joe Armstrong 16 | Ground Truth: ['guitar', 'harmonica', 'mandolin', 'percussion', 'percussion instrument', 'piano', 'saxophone'] 17 | GPT-3 Prediction: ['bass', 'guitar', 'piano'] 18 | 19 | 20 | SubjectEntity: Björk 21 | Ground Truth: ['celesta', 'clarinet', 'clavichord', 'flute', 'harp', 'keyboard', 'keyboard instrument', 'piano', 'pipe organ', 'reeds', 'saxophone', 'sets of free reeds'] 22 | GPT-3 Prediction: ['flute', 'guitar', 'piano'] 23 | 24 | 25 | SubjectEntity: China Anne McClain 26 | Ground Truth: ['bass', 'guitar', 'piano'] 27 | GPT-3 Prediction: [''] 28 | 29 | 30 | SubjectEntity: Cho Kyuhyun 31 | Ground Truth: ['piano'] 32 | GPT-3 Prediction: [''] 33 | 34 | 35 | SubjectEntity: Duff McKagan 36 | Ground Truth: ['bass guitar', 'guitar', 'guitar'] 37 | GPT-3 Prediction: ['bass'] 38 | 39 | 40 | SubjectEntity: Gary Barlow 41 | Ground Truth: ['piano'] 42 | GPT-3 Prediction: [''] 43 | 44 | 45 | SubjectEntity: Gerard Way 46 | Ground Truth: ['electric guitar', 'guitar'] 47 | GPT-3 Prediction: [''] 48 | 49 | 50 | SubjectEntity: Heo Young-saeng 51 | Ground Truth: ['piano'] 52 | GPT-3 Prediction: [''] 53 | 54 | 55 | SubjectEntity: Joseph Hahn 56 | Ground Truth: ['keyboard', 'keyboard instrument'] 57 | GPT-3 Prediction: [''] 58 | 59 | 60 | SubjectEntity: Kangin 61 | Ground Truth: ['piano'] 62 | GPT-3 Prediction: [''] 63 | 64 | 65 | SubjectEntity: Lay 66 | Ground Truth: ['cello', 'drum', 'electric guitar', 'guitar', 'hulusi', 'ney', 'piano'] 67 | GPT-3 Prediction: [''] 68 | 69 | 70 | SubjectEntity: Lee Jung-shin 71 | Ground Truth: ['bass guitar', 'guitar'] 72 | GPT-3 Prediction: [''] 73 | 74 | 75 | SubjectEntity: Logan Henderson 76 | Ground Truth: ['piano'] 77 | GPT-3 Prediction: [''] 78 | 79 | 80 | SubjectEntity: Morrissey 81 | Ground Truth: ['piano'] 82 | GPT-3 Prediction: [''] 83 | 84 | 85 | SubjectEntity: Nichkhun 86 | Ground Truth: ['keyboard', 'keyboard instrument'] 87 | GPT-3 Prediction: [''] 88 | 89 | 90 | SubjectEntity: Paul McCartney 91 | Ground Truth: ['bass', 'bass guitar', 'cello', 'double bass', 'guitar', 'guitar', 'keyboard', 'keyboard instrument', 'mandolin', 'melodica', 'percussion', 'percussion instrument', 'piano', 'recorder', 'sitar', 'trumpet', 'ukulele', 'violin'] 92 | GPT-3 Prediction: ['bass', 'drums', 'guitar', 'piano'] 93 | 94 | 95 | SubjectEntity: Robert Plant 96 | Ground Truth: ['harmonica'] 97 | GPT-3 Prediction: ['guitar'] 98 | 99 | 100 | SubjectEntity: Roger Taylor 101 | Ground Truth: ['bass guitar', 'drum', 'drum', 'drum kit', 'drum machine', 'guitar', 'guitar', 'keyboard', 'keyboard instrument'] 102 | GPT-3 Prediction: ['drums', 'percussion'] 103 | 104 | 105 | SubjectEntity: Shindong 106 | Ground Truth: ['piano'] 107 | GPT-3 Prediction: [''] 108 | 109 | 110 | SubjectEntity: Yoshiki 111 | Ground Truth: ['arp', 'arp 2500', 'drum', 'drum kit', 'guitar', 'keyboard', 'keyboard instrument', 'piano'] 112 | GPT-3 Prediction: ['drums', 'piano'] 113 | 114 | 115 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | PersonLanguage (average f1: 0.83): 19 cases 2 | 3 | 4 | 5 | SubjectEntity: A. R. Rahman 6 | Ground Truth: ['hindi', 'tamil', 'telugu'] 7 | GPT-3 Prediction: ['english', 'hindi', 'kannada', 'malayalam', 'tamil', 'telugu'] 8 | 9 | 10 | SubjectEntity: Abdullah Gül 11 | Ground Truth: ['turkish'] 12 | GPT-3 Prediction: ['english', 'turkish'] 13 | 14 | 15 | SubjectEntity: Akon 16 | Ground Truth: ['english'] 17 | GPT-3 Prediction: ['english', 'french', 'wolof'] 18 | 19 | 20 | SubjectEntity: Akshay Kumar 21 | Ground Truth: ['hindi', 'punjabi'] 22 | GPT-3 Prediction: ['english', 'hindi'] 23 | 24 | 25 | SubjectEntity: Barack Obama 26 | Ground Truth: ['english', 'indonesian'] 27 | GPT-3 Prediction: ['english'] 28 | 29 | 30 | SubjectEntity: Blaise Matuidi 31 | Ground Truth: ['french'] 32 | GPT-3 Prediction: ['english', 'french', 'italian', 'spanish'] 33 | 34 | 35 | SubjectEntity: Emma Watson 36 | Ground Truth: ['english', 'french'] 37 | GPT-3 Prediction: ['english'] 38 | 39 | 40 | SubjectEntity: Genelia D'Souza 41 | Ground Truth: ['hindi', 'malayalam', 'tamil', 'telugu'] 42 | GPT-3 Prediction: ['english', 'hindi', 'marathi', 'telugu'] 43 | 44 | 45 | SubjectEntity: John Cena 46 | Ground Truth: ['chinese', 'english', 'mandarin', 'mandarin chinese'] 47 | GPT-3 Prediction: ['english'] 48 | 49 | 50 | SubjectEntity: Jose Michimani 51 | Ground Truth: ['english'] 52 | GPT-3 Prediction: ['quechua', 'spanish'] 53 | 54 | 55 | SubjectEntity: Kobe Bryant 56 | Ground Truth: ['english', 'italian', 'spanish'] 57 | GPT-3 Prediction: ['english'] 58 | 59 | 60 | SubjectEntity: Oscar dos Santos Emboaba Júnior 61 | Ground Truth: ['portuguese'] 62 | GPT-3 Prediction: ['portuguese', 'spanish'] 63 | 64 | 65 | SubjectEntity: Pau Gasol 66 | Ground Truth: ['catalan', 'english', 'french', 'italian', 'spanish'] 67 | GPT-3 Prediction: ['catalan', 'english', 'spanish'] 68 | 69 | 70 | SubjectEntity: Paulo Coelho 71 | Ground Truth: ['portuguese'] 72 | GPT-3 Prediction: ['english', 'french', 'portuguese', 'spanish'] 73 | 74 | 75 | SubjectEntity: Salman Khan 76 | Ground Truth: ['hindi'] 77 | GPT-3 Prediction: ['hindi', 'urdu'] 78 | 79 | 80 | SubjectEntity: Samir Nasri 81 | Ground Truth: ['french'] 82 | GPT-3 Prediction: ['arabic', 'french'] 83 | 84 | 85 | SubjectEntity: Selena Gomez 86 | Ground Truth: ['english', 'spanish'] 87 | GPT-3 Prediction: ['english'] 88 | 89 | 90 | SubjectEntity: Thiago Alcântara 91 | Ground Truth: ['spanish'] 92 | GPT-3 Prediction: ['catalan', 'english', 'german', 'portuguese', 'spanish'] 93 | 94 | 95 | SubjectEntity: Zlatan Ibrahimović 96 | Ground Truth: ['bosnian', 'english', 'italian', 'spanish', 'swedish'] 97 | GPT-3 Prediction: ['bosnian', 'croatian', 'english', 'french', 'italian', 'serbian', 'spanish', 'swedish'] 98 | 99 | 100 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonPlaceOfDeath (average f1: 0.84): 8 cases 2 | 3 | 4 | 5 | SubjectEntity: Abdul Rehman Malik 6 | Ground Truth: ['islamabad'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Alfredo González Flores 11 | Ground Truth: ['heredia'] 12 | GPT-3 Prediction: ['san salvador'] 13 | 14 | 15 | SubjectEntity: Almudena Grandes 16 | Ground Truth: ['madrid'] 17 | GPT-3 Prediction: [''] 18 | 19 | 20 | SubjectEntity: Anne Rice 21 | Ground Truth: ['rancho mirage'] 22 | GPT-3 Prediction: [''] 23 | 24 | 25 | SubjectEntity: Chick Corea 26 | Ground Truth: ['tampa', 'tampa bay area'] 27 | GPT-3 Prediction: ['burbank'] 28 | 29 | 30 | SubjectEntity: John Coltrane 31 | Ground Truth: ['huntington'] 32 | GPT-3 Prediction: ['new york'] 33 | 34 | 35 | SubjectEntity: Michael Collins 36 | Ground Truth: ['naples'] 37 | GPT-3 Prediction: ['cape canaveral'] 38 | 39 | 40 | SubjectEntity: Rachel Caine 41 | Ground Truth: ['texas'] 42 | GPT-3 Prediction: [''] 43 | 44 | 45 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | RiverBasinsCountry (average f1: 0.832): 20 cases 2 | 3 | 4 | 5 | SubjectEntity: Didi Liakhvi river 6 | Ground Truth: ['georgia'] 7 | GPT-3 Prediction: ['georgia', 'russia'] 8 | 9 | 10 | SubjectEntity: Dniester 11 | Ground Truth: ['moldova', 'ukraine'] 12 | GPT-3 Prediction: ['moldova', 'romania', 'ukraine'] 13 | 14 | 15 | SubjectEntity: Gadar river 16 | Ground Truth: ['iran'] 17 | GPT-3 Prediction: ['iran', 'iraq'] 18 | 19 | 20 | SubjectEntity: Gambia river 21 | Ground Truth: ['gambia', 'guinea', 'senegal', 'the gambia'] 22 | GPT-3 Prediction: ['senegal', 'the gambia'] 23 | 24 | 25 | SubjectEntity: Han river 26 | Ground Truth: ['korea', 'korea', 'north korea', 'south korea'] 27 | GPT-3 Prediction: ['china'] 28 | 29 | 30 | SubjectEntity: Japurá river 31 | Ground Truth: ['brazil', 'colombia'] 32 | GPT-3 Prediction: ['brazil', 'colombia', 'peru'] 33 | 34 | 35 | SubjectEntity: Kura river 36 | Ground Truth: ['azerbaijan', 'georgia', 'turkey'] 37 | GPT-3 Prediction: ['armenia', 'azerbaijan', 'georgia', 'iran', 'russia', 'turkey'] 38 | 39 | 40 | SubjectEntity: Narew 41 | Ground Truth: ['belarus', 'poland'] 42 | GPT-3 Prediction: ['poland'] 43 | 44 | 45 | SubjectEntity: Narva river 46 | Ground Truth: ['belarus', 'estonia', 'latvia', 'russia'] 47 | GPT-3 Prediction: ['estonia', 'russia'] 48 | 49 | 50 | SubjectEntity: Neman 51 | Ground Truth: ['belarus', 'latvia', 'lithuania', 'russia'] 52 | GPT-3 Prediction: ['belarus', 'lithuania', 'poland', 'russia'] 53 | 54 | 55 | SubjectEntity: Neris 56 | Ground Truth: ['belarus', 'latvia', 'lithuania'] 57 | GPT-3 Prediction: ['belarus', 'lithuania'] 58 | 59 | 60 | SubjectEntity: Ottawa river 61 | Ground Truth: ['canada'] 62 | GPT-3 Prediction: ['canada', 'united states'] 63 | 64 | 65 | SubjectEntity: Red river of the south 66 | Ground Truth: ['texas', 'united states of america', 'usa'] 67 | GPT-3 Prediction: ['united states'] 68 | 69 | 70 | SubjectEntity: San 71 | Ground Truth: ['poland', 'ukraine'] 72 | GPT-3 Prediction: ['argentina', 'bolivia', 'chile', 'paraguay'] 73 | 74 | 75 | SubjectEntity: Shatt al-Arab 76 | Ground Truth: ['iran', 'iraq', 'kuwait', 'saudi arabia', 'syria', 'turkey'] 77 | GPT-3 Prediction: ['iran', 'iraq'] 78 | 79 | 80 | SubjectEntity: Someș river 81 | Ground Truth: ['hungary', 'romania'] 82 | GPT-3 Prediction: ['romania'] 83 | 84 | 85 | SubjectEntity: Torne river 86 | Ground Truth: ['finland', 'norway', 'sweden'] 87 | GPT-3 Prediction: ['finland', 'sweden'] 88 | 89 | 90 | SubjectEntity: Ural river 91 | Ground Truth: ['kazakhstan', 'russia'] 92 | GPT-3 Prediction: ['kazakhstan', 'russia', 'turkmenistan', 'uzbekistan'] 93 | 94 | 95 | SubjectEntity: Ussuri 96 | Ground Truth: ['russia'] 97 | GPT-3 Prediction: ['china', 'russia'] 98 | 99 | 100 | SubjectEntity: Yantra 101 | Ground Truth: ['bulgaria'] 102 | GPT-3 Prediction: ['bulgaria', 'romania'] 103 | 104 | 105 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck/ChemicalCompoundElement.txt: -------------------------------------------------------------------------------- 1 | ChemicalCompoundElement (average f1: 0.94): 17 cases 2 | 3 | 4 | 5 | SubjectEntity: Adenosine Diphosphate Ribose 6 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 7 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 8 | 9 | 10 | SubjectEntity: Aluminium Chlorohydrate 11 | Ground Truth: ['aluminium', 'chlorine', 'hydrogen', 'oxygen'] 12 | GPT-3 Prediction: ['aluminium', 'chlorine', 'hydrogen'] 13 | 14 | 15 | SubjectEntity: Aluminum Iodide 16 | Ground Truth: ['aluminium', 'iodine'] 17 | GPT-3 Prediction: ['aluminum', 'iodine'] 18 | 19 | 20 | SubjectEntity: Atracurium Besylate 21 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 22 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 23 | 24 | 25 | SubjectEntity: Cetyl Alcohol 26 | Ground Truth: ['carbon', 'hydrogen', 'oxygen'] 27 | GPT-3 Prediction: ['carbon', 'hydrogen'] 28 | 29 | 30 | SubjectEntity: Chlorophyll F 31 | Ground Truth: ['carbon', 'hydrogen', 'magnesium', 'nitrogen', 'oxygen'] 32 | GPT-3 Prediction: ['carbon', 'hydrogen', 'magnesium', 'oxygen'] 33 | 34 | 35 | SubjectEntity: Flavin Adenine Dinucleotide 36 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 37 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 38 | 39 | 40 | SubjectEntity: Glucagon 41 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 42 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 43 | 44 | 45 | SubjectEntity: Glutathione 46 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 47 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'sulfur'] 48 | 49 | 50 | SubjectEntity: Heroin 51 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 52 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 53 | 54 | 55 | SubjectEntity: Hydroxocobalamin 56 | Ground Truth: ['carbon', 'cobalt', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 57 | GPT-3 Prediction: ['cobalt', 'hydrogen', 'nitrogen', 'oxygen'] 58 | 59 | 60 | SubjectEntity: Metformin 61 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen'] 62 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 63 | 64 | 65 | SubjectEntity: Motilin 66 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 67 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 68 | 69 | 70 | SubjectEntity: Mustard Gas 71 | Ground Truth: ['carbon', 'chlorine', 'hydrogen', 'sulfur'] 72 | GPT-3 Prediction: ['carbon', 'chlorine', 'hydrogen', 'oxygen', 'sulfur'] 73 | 74 | 75 | SubjectEntity: Tabun 76 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 77 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 78 | 79 | 80 | SubjectEntity: Tenoxicam 81 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 82 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 83 | 84 | 85 | SubjectEntity: Triphenylphosphine 86 | Ground Truth: ['carbon', 'hydrogen', 'phosphorus'] 87 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'phosphorus'] 88 | 89 | 90 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | CompanyParentOrganization (average f1: 0.8): 10 cases 2 | 3 | 4 | 5 | SubjectEntity: Aston Martin lagonda 6 | Ground Truth: [] 7 | GPT-3 Prediction: ['ford motor company', 'investindustrial'] 8 | 9 | 10 | SubjectEntity: Changan Automobile 11 | Ground Truth: ['china south industries', 'china south industries group'] 12 | GPT-3 Prediction: ['changan group'] 13 | 14 | 15 | SubjectEntity: Hanomag 16 | Ground Truth: ['komatsu', 'komatsu limited'] 17 | GPT-3 Prediction: [''] 18 | 19 | 20 | SubjectEntity: Hyundai Motor Company 21 | Ground Truth: ['hyundai'] 22 | GPT-3 Prediction: ['hyundai motor group'] 23 | 24 | 25 | SubjectEntity: Lotus Cars 26 | Ground Truth: ['geely'] 27 | GPT-3 Prediction: ['geely holding group'] 28 | 29 | 30 | SubjectEntity: MG Car Company Limited 31 | Ground Truth: [] 32 | GPT-3 Prediction: ['saic motor corporation limited'] 33 | 34 | 35 | SubjectEntity: NSU 36 | Ground Truth: ['volkswagen'] 37 | GPT-3 Prediction: ['volkswagen group'] 38 | 39 | 40 | SubjectEntity: Peugeot 41 | Ground Truth: ['stellantis'] 42 | GPT-3 Prediction: ['psa peugeot citroën', 'peugeot s.a.'] 43 | 44 | 45 | SubjectEntity: Scripps-Booth 46 | Ground Truth: ['general motors', 'gm'] 47 | GPT-3 Prediction: [''] 48 | 49 | 50 | SubjectEntity: Simca 51 | Ground Truth: ['chrysler', 'chrysler europe'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck/CountryBordersWithCountry.txt: -------------------------------------------------------------------------------- 1 | CountryBordersWithCountry (average f1: 0.785): 14 cases 2 | 3 | 4 | 5 | SubjectEntity: Bahrain 6 | Ground Truth: ['iran', 'saudi arabia'] 7 | GPT-3 Prediction: ['qatar', 'saudi arabia', 'united arab emirates'] 8 | 9 | 10 | SubjectEntity: Barbados 11 | Ground Truth: [] 12 | GPT-3 Prediction: ['guyana', 'saint lucia', 'saint vincent and the grenadines', 'trinidad and tobago', 'venezuela'] 13 | 14 | 15 | SubjectEntity: Comoros 16 | Ground Truth: ['france', 'madagascar', 'mozambique', 'seychelles', 'tanzania'] 17 | GPT-3 Prediction: ['madagascar', 'mayotte', 'mozambique', 'seychelles'] 18 | 19 | 20 | SubjectEntity: Cuba 21 | Ground Truth: ['united states of america', 'usa'] 22 | GPT-3 Prediction: ['bahamas', 'cayman islands', 'haiti', 'jamaica', 'mexico', 'united states'] 23 | 24 | 25 | SubjectEntity: Dominican Republic 26 | Ground Truth: ['haiti', 'united states of america', 'usa', 'venezuela'] 27 | GPT-3 Prediction: ['haiti', 'puerto rico'] 28 | 29 | 30 | SubjectEntity: Gambia 31 | Ground Truth: ['senegal'] 32 | GPT-3 Prediction: ['guinea', 'guinea-bissau', 'senegal'] 33 | 34 | 35 | SubjectEntity: Ghana 36 | Ground Truth: ['burkina faso', 'ivory coast', 'togo'] 37 | GPT-3 Prediction: ['benin', 'burkina faso', 'cote divoire', 'togo'] 38 | 39 | 40 | SubjectEntity: Grenada 41 | Ground Truth: ['venezuela'] 42 | GPT-3 Prediction: ['trinidad and tobago', 'venezuela'] 43 | 44 | 45 | SubjectEntity: Japan 46 | Ground Truth: ['china', 'korea', "people's republic of china", 'philippines', 'russia', 'south korea', 'taiwan', 'united states of america', 'usa'] 47 | GPT-3 Prediction: ['china', 'north korea', 'russia', 'south korea'] 48 | 49 | 50 | SubjectEntity: Kingdom of Denmark 51 | Ground Truth: ['canada', 'germany', 'iceland', 'norway', 'poland', 'sweden', 'uk', 'united kingdom'] 52 | GPT-3 Prediction: ['germany', 'norway', 'sweden'] 53 | 54 | 55 | SubjectEntity: Portugal 56 | Ground Truth: ['castile and leon', 'castile and león', 'spain'] 57 | GPT-3 Prediction: ['spain'] 58 | 59 | 60 | SubjectEntity: Saint Vincent and the Grenadines 61 | Ground Truth: ['venezuela'] 62 | GPT-3 Prediction: ['barbados', 'grenada', 'saint lucia'] 63 | 64 | 65 | SubjectEntity: Samoa 66 | Ground Truth: ['united states of america', 'usa'] 67 | GPT-3 Prediction: [''] 68 | 69 | 70 | SubjectEntity: Venezuela 71 | Ground Truth: ['brazil', 'colombia', 'dominica', 'dominican republic', 'france', 'grenada', 'guyana', 'kingdom of the netherlands', 'netherlands', 'saint kitts and nevis', 'saint lucia', 'saint vincent and the grenadines', 'trinidad and tobago', 'uk', 'united kingdom', 'united states of america', 'usa'] 72 | GPT-3 Prediction: ['brazil', 'colombia', 'guyana'] 73 | 74 | 75 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | CountryOfficialLanguage (average f1: 0.833): 15 cases 2 | 3 | 4 | 5 | SubjectEntity: Afghanistan 6 | Ground Truth: ['arabic', 'baluchi', 'dari', 'nuristani', 'pamir', 'pashayi', 'pashto', 'turkmen', 'uzbek'] 7 | GPT-3 Prediction: ['dari', 'pashto'] 8 | 9 | 10 | SubjectEntity: Botswana 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: ['tswana'] 13 | 14 | 15 | SubjectEntity: Brunei 16 | Ground Truth: ['english', 'malay'] 17 | GPT-3 Prediction: ['malay'] 18 | 19 | 20 | SubjectEntity: Burundi 21 | Ground Truth: ['english', 'french', 'kirundi'] 22 | GPT-3 Prediction: ['french', 'kirundi', 'swahili'] 23 | 24 | 25 | SubjectEntity: East Timor 26 | Ground Truth: ['portuguese', 'tetum'] 27 | GPT-3 Prediction: ['indonesian', 'portuguese', 'tetum'] 28 | 29 | 30 | SubjectEntity: Grenada 31 | Ground Truth: ['creole', 'english', 'grenadian', 'grenadian creole english'] 32 | GPT-3 Prediction: ['english'] 33 | 34 | 35 | SubjectEntity: Israel 36 | Ground Truth: ['hebrew'] 37 | GPT-3 Prediction: ['arabic', 'hebrew'] 38 | 39 | 40 | SubjectEntity: Maldives 41 | Ground Truth: ['maldivian'] 42 | GPT-3 Prediction: ['dhivehi'] 43 | 44 | 45 | SubjectEntity: North Macedonia 46 | Ground Truth: ['albanian', 'macedonian'] 47 | GPT-3 Prediction: ['macedonian'] 48 | 49 | 50 | SubjectEntity: Norway 51 | Ground Truth: ['bokmal', 'bokmål', 'norwegian', 'nynorsk', 'sami', 'sámi'] 52 | GPT-3 Prediction: ['norwegian'] 53 | 54 | 55 | SubjectEntity: Pakistan 56 | Ground Truth: ['arabic', 'english', 'urdu'] 57 | GPT-3 Prediction: ['urdu'] 58 | 59 | 60 | SubjectEntity: People's Republic of China 61 | Ground Truth: ['chinese', 'mandarin', 'standard mandarin'] 62 | GPT-3 Prediction: ['mandarin'] 63 | 64 | 65 | SubjectEntity: Taiwan 66 | Ground Truth: ['amis', 'austronesian', 'formosan', 'hakka', 'hokkien', 'mandarin', 'paiwan', 'standard taiwanese mandarin', 'taiwanese', 'taiwanese', 'taiwanese hokkien'] 67 | GPT-3 Prediction: ['mandarin'] 68 | 69 | 70 | SubjectEntity: Tajikistan 71 | Ground Truth: ['russian', 'tajik'] 72 | GPT-3 Prediction: ['tajik'] 73 | 74 | 75 | SubjectEntity: Zimbabwe 76 | Ground Truth: ['barwe', 'chewa', 'english', 'kalanga', 'khoisan', 'nambya', 'ndau', 'ndebele', 'northern ndebele', 'sesotho', 'shona', 'tonga', 'tsonga', 'tswana', 'venda', 'xhosa'] 77 | GPT-3 Prediction: ['english', 'ndebele', 'shona'] 78 | 79 | 80 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonCauseOfDeath (average f1: 0.593): 20 cases 2 | 3 | 4 | 5 | SubjectEntity: Ahmed Zewail 6 | Ground Truth: ['lymphoma', 'spinal cord lymphoma'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Avicii 11 | Ground Truth: ['exsanguination'] 12 | GPT-3 Prediction: ['suicide'] 13 | 14 | 15 | SubjectEntity: Christina Grimmie 16 | Ground Truth: ['ballistic trauma', 'trauma'] 17 | GPT-3 Prediction: ['murder'] 18 | 19 | 20 | SubjectEntity: Cory Monteith 21 | Ground Truth: ['drug overdose', 'overdose'] 22 | GPT-3 Prediction: ['mixed drug toxicity'] 23 | 24 | 25 | SubjectEntity: Dilip Kumar 26 | Ground Truth: ['cancer', 'prostate cancer'] 27 | GPT-3 Prediction: [''] 28 | 29 | 30 | SubjectEntity: Ennio Morricone 31 | Ground Truth: ['femoral fracture', 'fracture'] 32 | GPT-3 Prediction: [''] 33 | 34 | 35 | SubjectEntity: Frank Sinatra 36 | Ground Truth: ['infarction', 'myocardial infarction'] 37 | GPT-3 Prediction: ['heart attack'] 38 | 39 | 40 | SubjectEntity: George H. W. Bush 41 | Ground Truth: ['parkinson', "parkinson's disease"] 42 | GPT-3 Prediction: ['septic shock'] 43 | 44 | 45 | SubjectEntity: Haruma Miura 46 | Ground Truth: ['hanging'] 47 | GPT-3 Prediction: [''] 48 | 49 | 50 | SubjectEntity: Hugh Hefner 51 | Ground Truth: ['sepsis'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Idriss Déby 56 | Ground Truth: ['ballistic trauma', 'trauma'] 57 | GPT-3 Prediction: [] 58 | 59 | 60 | SubjectEntity: Jim Rohn 61 | Ground Truth: ['fibrosis', 'pulmonary fibrosis'] 62 | GPT-3 Prediction: ['pneumonia'] 63 | 64 | 65 | SubjectEntity: John Magufuli 66 | Ground Truth: ['arrhythmia', 'heart arrhythmia'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: Johnny Cash 71 | Ground Truth: ['diabetes'] 72 | GPT-3 Prediction: [] 73 | 74 | 75 | SubjectEntity: Larry King 76 | Ground Truth: ['sepsis'] 77 | GPT-3 Prediction: [''] 78 | 79 | 80 | SubjectEntity: Maurane 81 | Ground Truth: ['falling'] 82 | GPT-3 Prediction: ['cancer'] 83 | 84 | 85 | SubjectEntity: Maya Angelou 86 | Ground Truth: ['disease'] 87 | GPT-3 Prediction: [''] 88 | 89 | 90 | SubjectEntity: Nelson Mandela 91 | Ground Truth: ['respiratory disease'] 92 | GPT-3 Prediction: ['respiratory infection'] 93 | 94 | 95 | SubjectEntity: Roddy Piper 96 | Ground Truth: ['cardiac arrest'] 97 | GPT-3 Prediction: ['aortic aneurysm'] 98 | 99 | 100 | SubjectEntity: Roy Orbison 101 | Ground Truth: ['infarction', 'myocardial infarction'] 102 | GPT-3 Prediction: ['heart attack'] 103 | 104 | 105 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | PersonLanguage (average f1: 0.842): 19 cases 2 | 3 | 4 | 5 | SubjectEntity: A. R. Rahman 6 | Ground Truth: ['hindi', 'tamil', 'telugu'] 7 | GPT-3 Prediction: ['english', 'hindi', 'malayalam', 'tamil'] 8 | 9 | 10 | SubjectEntity: Akon 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: ['english', 'wolof'] 13 | 14 | 15 | SubjectEntity: Akshay Kumar 16 | Ground Truth: ['hindi', 'punjabi'] 17 | GPT-3 Prediction: ['hindi'] 18 | 19 | 20 | SubjectEntity: Barack Obama 21 | Ground Truth: ['english', 'indonesian'] 22 | GPT-3 Prediction: ['english'] 23 | 24 | 25 | SubjectEntity: Donald Tusk 26 | Ground Truth: ['english', 'german', 'polish'] 27 | GPT-3 Prediction: ['english', 'polish'] 28 | 29 | 30 | SubjectEntity: Emma Watson 31 | Ground Truth: ['english', 'french'] 32 | GPT-3 Prediction: ['english'] 33 | 34 | 35 | SubjectEntity: Genelia D'Souza 36 | Ground Truth: ['hindi', 'malayalam', 'tamil', 'telugu'] 37 | GPT-3 Prediction: ['english', 'hindi', 'marathi', 'telugu'] 38 | 39 | 40 | SubjectEntity: John Cena 41 | Ground Truth: ['chinese', 'english', 'mandarin', 'mandarin chinese'] 42 | GPT-3 Prediction: ['english'] 43 | 44 | 45 | SubjectEntity: Jose Michimani 46 | Ground Truth: ['english'] 47 | GPT-3 Prediction: ['quechua', 'spanish'] 48 | 49 | 50 | SubjectEntity: Kobe Bryant 51 | Ground Truth: ['english', 'italian', 'spanish'] 52 | GPT-3 Prediction: ['english'] 53 | 54 | 55 | SubjectEntity: Novak Djokovic 56 | Ground Truth: ['english', 'french', 'german', 'italian', 'serbian'] 57 | GPT-3 Prediction: ['english', 'serbian'] 58 | 59 | 60 | SubjectEntity: Pau Gasol 61 | Ground Truth: ['catalan', 'english', 'french', 'italian', 'spanish'] 62 | GPT-3 Prediction: ['catalan', 'english', 'spanish'] 63 | 64 | 65 | SubjectEntity: Rafael Nadal 66 | Ground Truth: ['catalan', 'english', 'spanish'] 67 | GPT-3 Prediction: ['catalan', 'spanish'] 68 | 69 | 70 | SubjectEntity: Sachin 71 | Ground Truth: ['english', 'hindi', 'marathi'] 72 | GPT-3 Prediction: ['hindi', 'marathi'] 73 | 74 | 75 | SubjectEntity: Salman Khan 76 | Ground Truth: ['hindi'] 77 | GPT-3 Prediction: ['hindi', 'urdu'] 78 | 79 | 80 | SubjectEntity: Selena Gomez 81 | Ground Truth: ['english', 'spanish'] 82 | GPT-3 Prediction: ['english'] 83 | 84 | 85 | SubjectEntity: Thiago Alcântara 86 | Ground Truth: ['spanish'] 87 | GPT-3 Prediction: ['catalan', 'portuguese', 'spanish'] 88 | 89 | 90 | SubjectEntity: Vladimir Putin 91 | Ground Truth: ['english', 'german', 'russian', 'swedish'] 92 | GPT-3 Prediction: ['russian'] 93 | 94 | 95 | SubjectEntity: Zlatan Ibrahimović 96 | Ground Truth: ['bosnian', 'english', 'italian', 'spanish', 'swedish'] 97 | GPT-3 Prediction: ['french', 'serbian', 'swedish'] 98 | 99 | 100 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonPlaceOfDeath (average f1: 0.84): 8 cases 2 | 3 | 4 | 5 | SubjectEntity: Abdul Rehman Malik 6 | Ground Truth: ['islamabad'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Alfredo González Flores 11 | Ground Truth: ['heredia'] 12 | GPT-3 Prediction: ['san salvador'] 13 | 14 | 15 | SubjectEntity: Almudena Grandes 16 | Ground Truth: ['madrid'] 17 | GPT-3 Prediction: [''] 18 | 19 | 20 | SubjectEntity: Anne Rice 21 | Ground Truth: ['rancho mirage'] 22 | GPT-3 Prediction: [''] 23 | 24 | 25 | SubjectEntity: Chick Corea 26 | Ground Truth: ['tampa', 'tampa bay area'] 27 | GPT-3 Prediction: ['burbank'] 28 | 29 | 30 | SubjectEntity: John Coltrane 31 | Ground Truth: ['huntington'] 32 | GPT-3 Prediction: ['new york'] 33 | 34 | 35 | SubjectEntity: Michael Collins 36 | Ground Truth: ['naples'] 37 | GPT-3 Prediction: ['cape canaveral'] 38 | 39 | 40 | SubjectEntity: Rachel Caine 41 | Ground Truth: ['texas'] 42 | GPT-3 Prediction: [''] 43 | 44 | 45 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | RiverBasinsCountry (average f1: 0.832): 20 cases 2 | 3 | 4 | 5 | SubjectEntity: Didi Liakhvi river 6 | Ground Truth: ['georgia'] 7 | GPT-3 Prediction: ['georgia', 'russia'] 8 | 9 | 10 | SubjectEntity: Dniester 11 | Ground Truth: ['moldova', 'ukraine'] 12 | GPT-3 Prediction: ['moldova', 'romania', 'ukraine'] 13 | 14 | 15 | SubjectEntity: Gadar river 16 | Ground Truth: ['iran'] 17 | GPT-3 Prediction: ['iran', 'iraq'] 18 | 19 | 20 | SubjectEntity: Gambia river 21 | Ground Truth: ['gambia', 'guinea', 'senegal', 'the gambia'] 22 | GPT-3 Prediction: ['senegal', 'the gambia'] 23 | 24 | 25 | SubjectEntity: Han river 26 | Ground Truth: ['korea', 'korea', 'north korea', 'south korea'] 27 | GPT-3 Prediction: ['china'] 28 | 29 | 30 | SubjectEntity: Japurá river 31 | Ground Truth: ['brazil', 'colombia'] 32 | GPT-3 Prediction: ['brazil', 'colombia', 'peru'] 33 | 34 | 35 | SubjectEntity: Kura river 36 | Ground Truth: ['azerbaijan', 'georgia', 'turkey'] 37 | GPT-3 Prediction: ['armenia', 'azerbaijan', 'georgia', 'iran', 'russia', 'turkey'] 38 | 39 | 40 | SubjectEntity: Narew 41 | Ground Truth: ['belarus', 'poland'] 42 | GPT-3 Prediction: ['poland'] 43 | 44 | 45 | SubjectEntity: Narva river 46 | Ground Truth: ['belarus', 'estonia', 'latvia', 'russia'] 47 | GPT-3 Prediction: ['estonia', 'russia'] 48 | 49 | 50 | SubjectEntity: Neman 51 | Ground Truth: ['belarus', 'latvia', 'lithuania', 'russia'] 52 | GPT-3 Prediction: ['belarus', 'lithuania', 'poland', 'russia'] 53 | 54 | 55 | SubjectEntity: Neris 56 | Ground Truth: ['belarus', 'latvia', 'lithuania'] 57 | GPT-3 Prediction: ['belarus', 'lithuania'] 58 | 59 | 60 | SubjectEntity: Ottawa river 61 | Ground Truth: ['canada'] 62 | GPT-3 Prediction: ['canada', 'united states'] 63 | 64 | 65 | SubjectEntity: Red river of the south 66 | Ground Truth: ['texas', 'united states of america', 'usa'] 67 | GPT-3 Prediction: ['united states'] 68 | 69 | 70 | SubjectEntity: San 71 | Ground Truth: ['poland', 'ukraine'] 72 | GPT-3 Prediction: ['argentina', 'bolivia', 'chile', 'paraguay'] 73 | 74 | 75 | SubjectEntity: Shatt al-Arab 76 | Ground Truth: ['iran', 'iraq', 'kuwait', 'saudi arabia', 'syria', 'turkey'] 77 | GPT-3 Prediction: ['iran', 'iraq'] 78 | 79 | 80 | SubjectEntity: Someș river 81 | Ground Truth: ['hungary', 'romania'] 82 | GPT-3 Prediction: ['romania'] 83 | 84 | 85 | SubjectEntity: Torne river 86 | Ground Truth: ['finland', 'norway', 'sweden'] 87 | GPT-3 Prediction: ['finland', 'sweden'] 88 | 89 | 90 | SubjectEntity: Ural river 91 | Ground Truth: ['kazakhstan', 'russia'] 92 | GPT-3 Prediction: ['kazakhstan', 'russia', 'turkmenistan', 'uzbekistan'] 93 | 94 | 95 | SubjectEntity: Ussuri 96 | Ground Truth: ['russia'] 97 | GPT-3 Prediction: ['china', 'russia'] 98 | 99 | 100 | SubjectEntity: Yantra 101 | Ground Truth: ['bulgaria'] 102 | GPT-3 Prediction: ['bulgaria', 'romania'] 103 | 104 | 105 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck_wikiclean/ChemicalCompoundElement.txt: -------------------------------------------------------------------------------- 1 | ChemicalCompoundElement (average f1: 0.95): 16 cases 2 | 3 | 4 | 5 | SubjectEntity: Adenosine Diphosphate Ribose 6 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 7 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 8 | 9 | 10 | SubjectEntity: Aluminium Chlorohydrate 11 | Ground Truth: ['aluminium', 'chlorine', 'hydrogen', 'oxygen'] 12 | GPT-3 Prediction: ['aluminium', 'chlorine', 'hydrogen'] 13 | 14 | 15 | SubjectEntity: Atracurium Besylate 16 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 17 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 18 | 19 | 20 | SubjectEntity: Cetyl Alcohol 21 | Ground Truth: ['carbon', 'hydrogen', 'oxygen'] 22 | GPT-3 Prediction: ['carbon', 'hydrogen'] 23 | 24 | 25 | SubjectEntity: Chlorophyll F 26 | Ground Truth: ['carbon', 'hydrogen', 'magnesium', 'nitrogen', 'oxygen'] 27 | GPT-3 Prediction: ['carbon', 'hydrogen', 'magnesium', 'oxygen'] 28 | 29 | 30 | SubjectEntity: Flavin Adenine Dinucleotide 31 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 32 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 33 | 34 | 35 | SubjectEntity: Glucagon 36 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 37 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 38 | 39 | 40 | SubjectEntity: Glutathione 41 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 42 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'sulfur'] 43 | 44 | 45 | SubjectEntity: Heroin 46 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 47 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 48 | 49 | 50 | SubjectEntity: Hydroxocobalamin 51 | Ground Truth: ['carbon', 'cobalt', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 52 | GPT-3 Prediction: ['cobalt', 'hydrogen', 'nitrogen', 'oxygen'] 53 | 54 | 55 | SubjectEntity: Metformin 56 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen'] 57 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 58 | 59 | 60 | SubjectEntity: Motilin 61 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 62 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 63 | 64 | 65 | SubjectEntity: Mustard Gas 66 | Ground Truth: ['carbon', 'chlorine', 'hydrogen', 'sulfur'] 67 | GPT-3 Prediction: ['carbon', 'chlorine', 'hydrogen', 'oxygen', 'sulfur'] 68 | 69 | 70 | SubjectEntity: Tabun 71 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 72 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 73 | 74 | 75 | SubjectEntity: Tenoxicam 76 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 77 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 78 | 79 | 80 | SubjectEntity: Triphenylphosphine 81 | Ground Truth: ['carbon', 'hydrogen', 'phosphorus'] 82 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'phosphorus'] 83 | 84 | 85 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck_wikiclean/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | CompanyParentOrganization (average f1: 0.82): 9 cases 2 | 3 | 4 | 5 | SubjectEntity: Aston Martin lagonda 6 | Ground Truth: [] 7 | GPT-3 Prediction: ['ford motor company', 'investindustrial'] 8 | 9 | 10 | SubjectEntity: Changan Automobile 11 | Ground Truth: ['china south industries', 'china south industries group'] 12 | GPT-3 Prediction: ['changan group'] 13 | 14 | 15 | SubjectEntity: Hanomag 16 | Ground Truth: ['komatsu', 'komatsu limited'] 17 | GPT-3 Prediction: [''] 18 | 19 | 20 | SubjectEntity: Hyundai Motor Company 21 | Ground Truth: ['hyundai'] 22 | GPT-3 Prediction: ['hyundai motor group'] 23 | 24 | 25 | SubjectEntity: MG Car Company Limited 26 | Ground Truth: [] 27 | GPT-3 Prediction: ['saic motor'] 28 | 29 | 30 | SubjectEntity: NSU 31 | Ground Truth: ['volkswagen'] 32 | GPT-3 Prediction: ['volkswagen group'] 33 | 34 | 35 | SubjectEntity: Peugeot 36 | Ground Truth: ['stellantis'] 37 | GPT-3 Prediction: ['groupe psa'] 38 | 39 | 40 | SubjectEntity: Scripps-Booth 41 | Ground Truth: ['general motors', 'gm'] 42 | GPT-3 Prediction: [''] 43 | 44 | 45 | SubjectEntity: Simca 46 | Ground Truth: ['chrysler', 'chrysler europe'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck_wikiclean/CountryBordersWithCountry.txt: -------------------------------------------------------------------------------- 1 | CountryBordersWithCountry (average f1: 0.79): 15 cases 2 | 3 | 4 | 5 | SubjectEntity: Angola 6 | Ground Truth: ['congo', 'congo', 'democratic republic of the congo', 'gabon', 'namibia', 'republic of the congo', 'zambia'] 7 | GPT-3 Prediction: ['botswana', 'cape verde', 'democratic republic of the congo', 'namibia', 'zambia'] 8 | 9 | 10 | SubjectEntity: Bahrain 11 | Ground Truth: ['iran', 'saudi arabia'] 12 | GPT-3 Prediction: ['qatar', 'saudi arabia', 'united arab emirates'] 13 | 14 | 15 | SubjectEntity: Barbados 16 | Ground Truth: [] 17 | GPT-3 Prediction: ['guyana', 'saint lucia', 'saint vincent and the grenadines', 'trinidad and tobago', 'venezuela'] 18 | 19 | 20 | SubjectEntity: Comoros 21 | Ground Truth: ['france', 'madagascar', 'mozambique', 'seychelles', 'tanzania'] 22 | GPT-3 Prediction: ['madagascar', 'mayotte', 'mozambique', 'seychelles'] 23 | 24 | 25 | SubjectEntity: Cuba 26 | Ground Truth: ['united states of america', 'usa'] 27 | GPT-3 Prediction: ['cayman islands', 'haiti', 'jamaica', 'mexico', 'the bahamas', 'united states of america'] 28 | 29 | 30 | SubjectEntity: Dominican Republic 31 | Ground Truth: ['haiti', 'united states of america', 'usa', 'venezuela'] 32 | GPT-3 Prediction: ['haiti', 'puerto rico'] 33 | 34 | 35 | SubjectEntity: Gambia 36 | Ground Truth: ['senegal'] 37 | GPT-3 Prediction: ['guinea', 'guinea-bissau', 'senegal'] 38 | 39 | 40 | SubjectEntity: Ghana 41 | Ground Truth: ['burkina faso', 'ivory coast', 'togo'] 42 | GPT-3 Prediction: ['benin', 'burkina faso', 'cote divoire', 'togo'] 43 | 44 | 45 | SubjectEntity: Grenada 46 | Ground Truth: ['venezuela'] 47 | GPT-3 Prediction: ['trinidad and tobago', 'venezuela'] 48 | 49 | 50 | SubjectEntity: Japan 51 | Ground Truth: ['china', 'korea', "people's republic of china", 'philippines', 'russia', 'south korea', 'taiwan', 'united states of america', 'usa'] 52 | GPT-3 Prediction: ['north korea', "people's republic of china", 'russia', 'south korea'] 53 | 54 | 55 | SubjectEntity: Kingdom of Denmark 56 | Ground Truth: ['canada', 'germany', 'iceland', 'norway', 'poland', 'sweden', 'uk', 'united kingdom'] 57 | GPT-3 Prediction: ['germany', 'norway', 'sweden'] 58 | 59 | 60 | SubjectEntity: Portugal 61 | Ground Truth: ['castile and leon', 'castile and león', 'spain'] 62 | GPT-3 Prediction: ['spain'] 63 | 64 | 65 | SubjectEntity: Saint Vincent and the Grenadines 66 | Ground Truth: ['venezuela'] 67 | GPT-3 Prediction: ['barbados', 'grenada', 'saint lucia'] 68 | 69 | 70 | SubjectEntity: Samoa 71 | Ground Truth: ['united states of america', 'usa'] 72 | GPT-3 Prediction: [''] 73 | 74 | 75 | SubjectEntity: Venezuela 76 | Ground Truth: ['brazil', 'colombia', 'dominica', 'dominican republic', 'france', 'grenada', 'guyana', 'kingdom of the netherlands', 'netherlands', 'saint kitts and nevis', 'saint lucia', 'saint vincent and the grenadines', 'trinidad and tobago', 'uk', 'united kingdom', 'united states of america', 'usa'] 77 | GPT-3 Prediction: ['brazil', 'colombia', 'guyana'] 78 | 79 | 80 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck_wikiclean/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | CountryOfficialLanguage (average f1: 0.833): 15 cases 2 | 3 | 4 | 5 | SubjectEntity: Afghanistan 6 | Ground Truth: ['arabic', 'baluchi', 'dari', 'nuristani', 'pamir', 'pashayi', 'pashto', 'turkmen', 'uzbek'] 7 | GPT-3 Prediction: ['dari', 'pashto'] 8 | 9 | 10 | SubjectEntity: Botswana 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: ['tswana'] 13 | 14 | 15 | SubjectEntity: Brunei 16 | Ground Truth: ['english', 'malay'] 17 | GPT-3 Prediction: ['malay'] 18 | 19 | 20 | SubjectEntity: Burundi 21 | Ground Truth: ['english', 'french', 'kirundi'] 22 | GPT-3 Prediction: ['french', 'kirundi', 'swahili'] 23 | 24 | 25 | SubjectEntity: East Timor 26 | Ground Truth: ['portuguese', 'tetum'] 27 | GPT-3 Prediction: ['indonesian', 'portuguese', 'tetum'] 28 | 29 | 30 | SubjectEntity: Grenada 31 | Ground Truth: ['creole', 'english', 'grenadian', 'grenadian creole english'] 32 | GPT-3 Prediction: ['english'] 33 | 34 | 35 | SubjectEntity: Israel 36 | Ground Truth: ['hebrew'] 37 | GPT-3 Prediction: ['arabic', 'hebrew'] 38 | 39 | 40 | SubjectEntity: Maldives 41 | Ground Truth: ['maldivian'] 42 | GPT-3 Prediction: ['dhivehi'] 43 | 44 | 45 | SubjectEntity: North Macedonia 46 | Ground Truth: ['albanian', 'macedonian'] 47 | GPT-3 Prediction: ['macedonian'] 48 | 49 | 50 | SubjectEntity: Norway 51 | Ground Truth: ['bokmal', 'bokmål', 'norwegian', 'nynorsk', 'sami', 'sámi'] 52 | GPT-3 Prediction: ['norwegian'] 53 | 54 | 55 | SubjectEntity: Pakistan 56 | Ground Truth: ['arabic', 'english', 'urdu'] 57 | GPT-3 Prediction: ['urdu'] 58 | 59 | 60 | SubjectEntity: People's Republic of China 61 | Ground Truth: ['chinese', 'mandarin', 'standard mandarin'] 62 | GPT-3 Prediction: ['mandarin'] 63 | 64 | 65 | SubjectEntity: Taiwan 66 | Ground Truth: ['amis', 'austronesian', 'formosan', 'hakka', 'hokkien', 'mandarin', 'paiwan', 'standard taiwanese mandarin', 'taiwanese', 'taiwanese', 'taiwanese hokkien'] 67 | GPT-3 Prediction: ['mandarin'] 68 | 69 | 70 | SubjectEntity: Tajikistan 71 | Ground Truth: ['russian', 'tajik'] 72 | GPT-3 Prediction: ['tajik'] 73 | 74 | 75 | SubjectEntity: Zimbabwe 76 | Ground Truth: ['barwe', 'chewa', 'english', 'kalanga', 'khoisan', 'nambya', 'ndau', 'ndebele', 'northern ndebele', 'sesotho', 'shona', 'tonga', 'tsonga', 'tswana', 'venda', 'xhosa'] 77 | GPT-3 Prediction: ['english', 'ndebele', 'shona'] 78 | 79 | 80 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck_wikiclean/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonCauseOfDeath (average f1: 0.633): 18 cases 2 | 3 | 4 | 5 | SubjectEntity: Ahmed Zewail 6 | Ground Truth: ['lymphoma', 'spinal cord lymphoma'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Avicii 11 | Ground Truth: ['exsanguination'] 12 | GPT-3 Prediction: ['suicide'] 13 | 14 | 15 | SubjectEntity: Christina Grimmie 16 | Ground Truth: ['ballistic trauma', 'trauma'] 17 | GPT-3 Prediction: ['murder'] 18 | 19 | 20 | SubjectEntity: Cory Monteith 21 | Ground Truth: ['drug overdose', 'overdose'] 22 | GPT-3 Prediction: ['mixed drug toxicity'] 23 | 24 | 25 | SubjectEntity: Dilip Kumar 26 | Ground Truth: ['cancer', 'prostate cancer'] 27 | GPT-3 Prediction: [''] 28 | 29 | 30 | SubjectEntity: Ennio Morricone 31 | Ground Truth: ['femoral fracture', 'fracture'] 32 | GPT-3 Prediction: [''] 33 | 34 | 35 | SubjectEntity: George H. W. Bush 36 | Ground Truth: ['parkinson', "parkinson's disease"] 37 | GPT-3 Prediction: ['septic shock'] 38 | 39 | 40 | SubjectEntity: Haruma Miura 41 | Ground Truth: ['hanging'] 42 | GPT-3 Prediction: [''] 43 | 44 | 45 | SubjectEntity: Hugh Hefner 46 | Ground Truth: ['sepsis'] 47 | GPT-3 Prediction: [] 48 | 49 | 50 | SubjectEntity: Idriss Déby 51 | Ground Truth: ['ballistic trauma', 'trauma'] 52 | GPT-3 Prediction: [] 53 | 54 | 55 | SubjectEntity: Jim Rohn 56 | Ground Truth: ['fibrosis', 'pulmonary fibrosis'] 57 | GPT-3 Prediction: ['pneumonia'] 58 | 59 | 60 | SubjectEntity: John Magufuli 61 | Ground Truth: ['arrhythmia', 'heart arrhythmia'] 62 | GPT-3 Prediction: [] 63 | 64 | 65 | SubjectEntity: Johnny Cash 66 | Ground Truth: ['diabetes'] 67 | GPT-3 Prediction: [] 68 | 69 | 70 | SubjectEntity: Larry King 71 | Ground Truth: ['sepsis'] 72 | GPT-3 Prediction: [''] 73 | 74 | 75 | SubjectEntity: Maurane 76 | Ground Truth: ['falling'] 77 | GPT-3 Prediction: ['cancer'] 78 | 79 | 80 | SubjectEntity: Maya Angelou 81 | Ground Truth: ['disease'] 82 | GPT-3 Prediction: [''] 83 | 84 | 85 | SubjectEntity: Nelson Mandela 86 | Ground Truth: ['respiratory disease'] 87 | GPT-3 Prediction: ['respiratory infection'] 88 | 89 | 90 | SubjectEntity: Roddy Piper 91 | Ground Truth: ['cardiac arrest'] 92 | GPT-3 Prediction: ['aortic aneurysm'] 93 | 94 | 95 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck_wikiclean/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | PersonLanguage (average f1: 0.842): 19 cases 2 | 3 | 4 | 5 | SubjectEntity: A. R. Rahman 6 | Ground Truth: ['hindi', 'tamil', 'telugu'] 7 | GPT-3 Prediction: ['english', 'hindi', 'malayalam', 'tamil'] 8 | 9 | 10 | SubjectEntity: Akon 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: ['english', 'wolof'] 13 | 14 | 15 | SubjectEntity: Akshay Kumar 16 | Ground Truth: ['hindi', 'punjabi'] 17 | GPT-3 Prediction: ['hindi'] 18 | 19 | 20 | SubjectEntity: Barack Obama 21 | Ground Truth: ['english', 'indonesian'] 22 | GPT-3 Prediction: ['english'] 23 | 24 | 25 | SubjectEntity: Donald Tusk 26 | Ground Truth: ['english', 'german', 'polish'] 27 | GPT-3 Prediction: ['english', 'polish'] 28 | 29 | 30 | SubjectEntity: Emma Watson 31 | Ground Truth: ['english', 'french'] 32 | GPT-3 Prediction: ['english'] 33 | 34 | 35 | SubjectEntity: Genelia D'Souza 36 | Ground Truth: ['hindi', 'malayalam', 'tamil', 'telugu'] 37 | GPT-3 Prediction: ['english', 'hindi', 'marathi', 'telugu'] 38 | 39 | 40 | SubjectEntity: John Cena 41 | Ground Truth: ['chinese', 'english', 'mandarin', 'mandarin chinese'] 42 | GPT-3 Prediction: ['english'] 43 | 44 | 45 | SubjectEntity: Jose Michimani 46 | Ground Truth: ['english'] 47 | GPT-3 Prediction: ['quechua', 'spanish'] 48 | 49 | 50 | SubjectEntity: Kobe Bryant 51 | Ground Truth: ['english', 'italian', 'spanish'] 52 | GPT-3 Prediction: ['english'] 53 | 54 | 55 | SubjectEntity: Novak Djokovic 56 | Ground Truth: ['english', 'french', 'german', 'italian', 'serbian'] 57 | GPT-3 Prediction: ['english', 'serbian'] 58 | 59 | 60 | SubjectEntity: Pau Gasol 61 | Ground Truth: ['catalan', 'english', 'french', 'italian', 'spanish'] 62 | GPT-3 Prediction: ['catalan', 'english', 'spanish'] 63 | 64 | 65 | SubjectEntity: Rafael Nadal 66 | Ground Truth: ['catalan', 'english', 'spanish'] 67 | GPT-3 Prediction: ['catalan', 'spanish'] 68 | 69 | 70 | SubjectEntity: Sachin 71 | Ground Truth: ['english', 'hindi', 'marathi'] 72 | GPT-3 Prediction: ['hindi', 'marathi'] 73 | 74 | 75 | SubjectEntity: Salman Khan 76 | Ground Truth: ['hindi'] 77 | GPT-3 Prediction: ['hindi', 'urdu'] 78 | 79 | 80 | SubjectEntity: Selena Gomez 81 | Ground Truth: ['english', 'spanish'] 82 | GPT-3 Prediction: ['english'] 83 | 84 | 85 | SubjectEntity: Thiago Alcântara 86 | Ground Truth: ['spanish'] 87 | GPT-3 Prediction: ['catalan', 'portuguese', 'spanish'] 88 | 89 | 90 | SubjectEntity: Vladimir Putin 91 | Ground Truth: ['english', 'german', 'russian', 'swedish'] 92 | GPT-3 Prediction: ['russian'] 93 | 94 | 95 | SubjectEntity: Zlatan Ibrahimović 96 | Ground Truth: ['bosnian', 'english', 'italian', 'spanish', 'swedish'] 97 | GPT-3 Prediction: ['french', 'serbian', 'swedish'] 98 | 99 | 100 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck_wikiclean/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonPlaceOfDeath (average f1: 0.84): 8 cases 2 | 3 | 4 | 5 | SubjectEntity: Abdul Rehman Malik 6 | Ground Truth: ['islamabad'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Alfredo González Flores 11 | Ground Truth: ['heredia'] 12 | GPT-3 Prediction: ['san salvador'] 13 | 14 | 15 | SubjectEntity: Almudena Grandes 16 | Ground Truth: ['madrid'] 17 | GPT-3 Prediction: [''] 18 | 19 | 20 | SubjectEntity: Anne Rice 21 | Ground Truth: ['rancho mirage'] 22 | GPT-3 Prediction: [''] 23 | 24 | 25 | SubjectEntity: Chick Corea 26 | Ground Truth: ['tampa', 'tampa bay area'] 27 | GPT-3 Prediction: ['burbank'] 28 | 29 | 30 | SubjectEntity: John Coltrane 31 | Ground Truth: ['huntington'] 32 | GPT-3 Prediction: ['new york'] 33 | 34 | 35 | SubjectEntity: Michael Collins 36 | Ground Truth: ['naples'] 37 | GPT-3 Prediction: ['cape canaveral'] 38 | 39 | 40 | SubjectEntity: Rachel Caine 41 | Ground Truth: ['texas'] 42 | GPT-3 Prediction: [''] 43 | 44 | 45 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-optimized)_factcheck_wikiclean/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | RiverBasinsCountry (average f1: 0.846): 20 cases 2 | 3 | 4 | 5 | SubjectEntity: Didi Liakhvi river 6 | Ground Truth: ['georgia'] 7 | GPT-3 Prediction: ['georgia', 'russia'] 8 | 9 | 10 | SubjectEntity: Dniester 11 | Ground Truth: ['moldova', 'ukraine'] 12 | GPT-3 Prediction: ['moldova', 'romania', 'ukraine'] 13 | 14 | 15 | SubjectEntity: Gadar river 16 | Ground Truth: ['iran'] 17 | GPT-3 Prediction: ['iran', 'iraq'] 18 | 19 | 20 | SubjectEntity: Gambia river 21 | Ground Truth: ['gambia', 'guinea', 'senegal', 'the gambia'] 22 | GPT-3 Prediction: ['senegal', 'the gambia'] 23 | 24 | 25 | SubjectEntity: Han river 26 | Ground Truth: ['korea', 'korea', 'north korea', 'south korea'] 27 | GPT-3 Prediction: ["people's republic of china"] 28 | 29 | 30 | SubjectEntity: Japurá river 31 | Ground Truth: ['brazil', 'colombia'] 32 | GPT-3 Prediction: ['brazil', 'colombia', 'peru'] 33 | 34 | 35 | SubjectEntity: Kura river 36 | Ground Truth: ['azerbaijan', 'georgia', 'turkey'] 37 | GPT-3 Prediction: ['armenia', 'azerbaijan', 'georgia', 'iran', 'russia', 'turkey'] 38 | 39 | 40 | SubjectEntity: Narew 41 | Ground Truth: ['belarus', 'poland'] 42 | GPT-3 Prediction: ['poland'] 43 | 44 | 45 | SubjectEntity: Narva river 46 | Ground Truth: ['belarus', 'estonia', 'latvia', 'russia'] 47 | GPT-3 Prediction: ['estonia', 'russia'] 48 | 49 | 50 | SubjectEntity: Neman 51 | Ground Truth: ['belarus', 'latvia', 'lithuania', 'russia'] 52 | GPT-3 Prediction: ['belarus', 'lithuania', 'poland', 'russia'] 53 | 54 | 55 | SubjectEntity: Neris 56 | Ground Truth: ['belarus', 'latvia', 'lithuania'] 57 | GPT-3 Prediction: ['belarus', 'lithuania'] 58 | 59 | 60 | SubjectEntity: Ottawa river 61 | Ground Truth: ['canada'] 62 | GPT-3 Prediction: ['canada', 'united states of america'] 63 | 64 | 65 | SubjectEntity: Red river of the south 66 | Ground Truth: ['texas', 'united states of america', 'usa'] 67 | GPT-3 Prediction: ['united states of america'] 68 | 69 | 70 | SubjectEntity: San 71 | Ground Truth: ['poland', 'ukraine'] 72 | GPT-3 Prediction: ['argentina', 'bolivia', 'chile', 'paraguay'] 73 | 74 | 75 | SubjectEntity: Shatt al-Arab 76 | Ground Truth: ['iran', 'iraq', 'kuwait', 'saudi arabia', 'syria', 'turkey'] 77 | GPT-3 Prediction: ['iran', 'iraq'] 78 | 79 | 80 | SubjectEntity: Someș river 81 | Ground Truth: ['hungary', 'romania'] 82 | GPT-3 Prediction: ['romania'] 83 | 84 | 85 | SubjectEntity: Torne river 86 | Ground Truth: ['finland', 'norway', 'sweden'] 87 | GPT-3 Prediction: ['finland', 'sweden'] 88 | 89 | 90 | SubjectEntity: Ural river 91 | Ground Truth: ['kazakhstan', 'russia'] 92 | GPT-3 Prediction: ['kazakhstan', 'russia', 'turkmenistan', 'uzbekistan'] 93 | 94 | 95 | SubjectEntity: Ussuri 96 | Ground Truth: ['russia'] 97 | GPT-3 Prediction: ["people's republic of china", 'russia'] 98 | 99 | 100 | SubjectEntity: Yantra 101 | Ground Truth: ['bulgaria'] 102 | GPT-3 Prediction: ['bulgaria', 'romania'] 103 | 104 | 105 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-empty)/ChemicalCompoundElement.txt: -------------------------------------------------------------------------------- 1 | ChemicalCompoundElement (average f1: 0.94): 17 cases 2 | 3 | 4 | 5 | SubjectEntity: Adenosine Diphosphate Ribose 6 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 7 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 8 | 9 | 10 | SubjectEntity: Aluminium Chlorohydrate 11 | Ground Truth: ['aluminium', 'chlorine', 'hydrogen', 'oxygen'] 12 | GPT-3 Prediction: ['aluminium', 'chlorine', 'hydrogen'] 13 | 14 | 15 | SubjectEntity: Aluminum Iodide 16 | Ground Truth: ['aluminium', 'iodine'] 17 | GPT-3 Prediction: ['aluminum', 'iodine'] 18 | 19 | 20 | SubjectEntity: Atracurium Besylate 21 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 22 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 23 | 24 | 25 | SubjectEntity: Cetyl Alcohol 26 | Ground Truth: ['carbon', 'hydrogen', 'oxygen'] 27 | GPT-3 Prediction: ['carbon', 'hydrogen'] 28 | 29 | 30 | SubjectEntity: Chlorophyll F 31 | Ground Truth: ['carbon', 'hydrogen', 'magnesium', 'nitrogen', 'oxygen'] 32 | GPT-3 Prediction: ['carbon', 'hydrogen', 'magnesium', 'oxygen'] 33 | 34 | 35 | SubjectEntity: Flavin Adenine Dinucleotide 36 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 37 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 38 | 39 | 40 | SubjectEntity: Glucagon 41 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 42 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 43 | 44 | 45 | SubjectEntity: Glutathione 46 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 47 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'sulfur'] 48 | 49 | 50 | SubjectEntity: Heroin 51 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 52 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 53 | 54 | 55 | SubjectEntity: Hydroxocobalamin 56 | Ground Truth: ['carbon', 'cobalt', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 57 | GPT-3 Prediction: ['cobalt', 'hydrogen', 'nitrogen', 'oxygen'] 58 | 59 | 60 | SubjectEntity: Metformin 61 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen'] 62 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 63 | 64 | 65 | SubjectEntity: Motilin 66 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 67 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 68 | 69 | 70 | SubjectEntity: Mustard Gas 71 | Ground Truth: ['carbon', 'chlorine', 'hydrogen', 'sulfur'] 72 | GPT-3 Prediction: ['carbon', 'chlorine', 'hydrogen', 'oxygen', 'sulfur'] 73 | 74 | 75 | SubjectEntity: Tabun 76 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 77 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 78 | 79 | 80 | SubjectEntity: Tenoxicam 81 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 82 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 83 | 84 | 85 | SubjectEntity: Triphenylphosphine 86 | Ground Truth: ['carbon', 'hydrogen', 'phosphorus'] 87 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'phosphorus'] 88 | 89 | 90 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-empty)/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | CompanyParentOrganization (average f1: 0.59): 21 cases 2 | 3 | 4 | 5 | SubjectEntity: Aixam 6 | Ground Truth: [] 7 | GPT-3 Prediction: ['groupe ligier'] 8 | 9 | 10 | SubjectEntity: Alvis Car and Engineering Company Ltd 11 | Ground Truth: [] 12 | GPT-3 Prediction: ['alvis plc'] 13 | 14 | 15 | SubjectEntity: Aston Martin lagonda 16 | Ground Truth: [] 17 | GPT-3 Prediction: ['ford motor company'] 18 | 19 | 20 | SubjectEntity: Austro-Daimler 21 | Ground Truth: [] 22 | GPT-3 Prediction: ['daimler ag'] 23 | 24 | 25 | SubjectEntity: Automobilwerk Eisenach 26 | Ground Truth: [] 27 | GPT-3 Prediction: ['bmw'] 28 | 29 | 30 | SubjectEntity: Bell 31 | Ground Truth: [] 32 | GPT-3 Prediction: ['bell textron inc.'] 33 | 34 | 35 | SubjectEntity: Bertone 36 | Ground Truth: [] 37 | GPT-3 Prediction: ['nuovo industriale', 'nuovo industriale', 'nuovo industriale'] 38 | 39 | 40 | SubjectEntity: Changan Automobile 41 | Ground Truth: ['china south industries', 'china south industries group'] 42 | GPT-3 Prediction: ['changan group'] 43 | 44 | 45 | SubjectEntity: Hanomag 46 | Ground Truth: ['komatsu', 'komatsu limited'] 47 | GPT-3 Prediction: ['volkswagen group'] 48 | 49 | 50 | SubjectEntity: Hyundai Motor Company 51 | Ground Truth: ['hyundai'] 52 | GPT-3 Prediction: ['hyundai motor group'] 53 | 54 | 55 | SubjectEntity: Lotus Cars 56 | Ground Truth: ['geely'] 57 | GPT-3 Prediction: ['proton'] 58 | 59 | 60 | SubjectEntity: MG Car Company Limited 61 | Ground Truth: [] 62 | GPT-3 Prediction: ['morris garages', 'saic motor'] 63 | 64 | 65 | SubjectEntity: Moskvitch 66 | Ground Truth: [] 67 | GPT-3 Prediction: ['ofai'] 68 | 69 | 70 | SubjectEntity: NSU 71 | Ground Truth: ['volkswagen'] 72 | GPT-3 Prediction: ['volkswagen group'] 73 | 74 | 75 | SubjectEntity: Peugeot 76 | Ground Truth: ['stellantis'] 77 | GPT-3 Prediction: ['peugeot citroën', 'peugeot s.a.'] 78 | 79 | 80 | SubjectEntity: Saipa 81 | Ground Truth: [] 82 | GPT-3 Prediction: ['iran khodro'] 83 | 84 | 85 | SubjectEntity: Scripps-Booth 86 | Ground Truth: ['general motors', 'gm'] 87 | GPT-3 Prediction: [''] 88 | 89 | 90 | SubjectEntity: Simca 91 | Ground Truth: ['chrysler', 'chrysler europe'] 92 | GPT-3 Prediction: ['chrysler europe', 'psa peugeot citroën', 'peugeot'] 93 | 94 | 95 | SubjectEntity: Società Torinese Automobili Elettrici 96 | Ground Truth: [] 97 | GPT-3 Prediction: ['staer s.p.a.'] 98 | 99 | 100 | SubjectEntity: Valmet Automotive 101 | Ground Truth: [] 102 | GPT-3 Prediction: ['valmet'] 103 | 104 | 105 | SubjectEntity: Élan Motorsport Technologies 106 | Ground Truth: [] 107 | GPT-3 Prediction: ['élan motorsports group'] 108 | 109 | 110 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-empty)/CountryBordersWithCountry.txt: -------------------------------------------------------------------------------- 1 | CountryBordersWithCountry (average f1: 0.766): 14 cases 2 | 3 | 4 | 5 | SubjectEntity: Bahrain 6 | Ground Truth: ['iran', 'saudi arabia'] 7 | GPT-3 Prediction: ['qatar'] 8 | 9 | 10 | SubjectEntity: Comoros 11 | Ground Truth: ['france', 'madagascar', 'mozambique', 'seychelles', 'tanzania'] 12 | GPT-3 Prediction: ['madagascar'] 13 | 14 | 15 | SubjectEntity: Cuba 16 | Ground Truth: ['united states of america', 'usa'] 17 | GPT-3 Prediction: ['united states'] 18 | 19 | 20 | SubjectEntity: Dominican Republic 21 | Ground Truth: ['haiti', 'united states of america', 'usa', 'venezuela'] 22 | GPT-3 Prediction: ['haiti'] 23 | 24 | 25 | SubjectEntity: Ghana 26 | Ground Truth: ['burkina faso', 'ivory coast', 'togo'] 27 | GPT-3 Prediction: ['burkina faso', 'cote divoire', 'la cote divoire', 'togo'] 28 | 29 | 30 | SubjectEntity: Grenada 31 | Ground Truth: ['venezuela'] 32 | GPT-3 Prediction: [''] 33 | 34 | 35 | SubjectEntity: Japan 36 | Ground Truth: ['china', 'korea', "people's republic of china", 'philippines', 'russia', 'south korea', 'taiwan', 'united states of america', 'usa'] 37 | GPT-3 Prediction: ['north korea', 'russia'] 38 | 39 | 40 | SubjectEntity: Kingdom of Denmark 41 | Ground Truth: ['canada', 'germany', 'iceland', 'norway', 'poland', 'sweden', 'uk', 'united kingdom'] 42 | GPT-3 Prediction: ['germany'] 43 | 44 | 45 | SubjectEntity: Kingdom of the Netherlands 46 | Ground Truth: ['belgium', 'france', 'germany', 'venezuela'] 47 | GPT-3 Prediction: ['germany'] 48 | 49 | 50 | SubjectEntity: Morocco 51 | Ground Truth: ['algeria', 'mauritania', 'sahara', 'spain', 'western sahara'] 52 | GPT-3 Prediction: ['algeria', 'western sahara'] 53 | 54 | 55 | SubjectEntity: Portugal 56 | Ground Truth: ['castile and leon', 'castile and león', 'spain'] 57 | GPT-3 Prediction: ['spain'] 58 | 59 | 60 | SubjectEntity: Saint Vincent and the Grenadines 61 | Ground Truth: ['venezuela'] 62 | GPT-3 Prediction: ['grenada'] 63 | 64 | 65 | SubjectEntity: Samoa 66 | Ground Truth: ['united states of america', 'usa'] 67 | GPT-3 Prediction: [''] 68 | 69 | 70 | SubjectEntity: Venezuela 71 | Ground Truth: ['brazil', 'colombia', 'dominica', 'dominican republic', 'france', 'grenada', 'guyana', 'kingdom of the netherlands', 'netherlands', 'saint kitts and nevis', 'saint lucia', 'saint vincent and the grenadines', 'trinidad and tobago', 'uk', 'united kingdom', 'united states of america', 'usa'] 72 | GPT-3 Prediction: ['brazil', 'colombia', 'guyana'] 73 | 74 | 75 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-empty)/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | CountryOfficialLanguage (average f1: 0.833): 15 cases 2 | 3 | 4 | 5 | SubjectEntity: Afghanistan 6 | Ground Truth: ['arabic', 'baluchi', 'dari', 'nuristani', 'pamir', 'pashayi', 'pashto', 'turkmen', 'uzbek'] 7 | GPT-3 Prediction: ['dari', 'pashto'] 8 | 9 | 10 | SubjectEntity: Botswana 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: ['tswana'] 13 | 14 | 15 | SubjectEntity: Brunei 16 | Ground Truth: ['english', 'malay'] 17 | GPT-3 Prediction: ['malay'] 18 | 19 | 20 | SubjectEntity: Burundi 21 | Ground Truth: ['english', 'french', 'kirundi'] 22 | GPT-3 Prediction: ['french', 'kirundi', 'swahili'] 23 | 24 | 25 | SubjectEntity: East Timor 26 | Ground Truth: ['portuguese', 'tetum'] 27 | GPT-3 Prediction: ['indonesian', 'portuguese', 'tetum'] 28 | 29 | 30 | SubjectEntity: Grenada 31 | Ground Truth: ['creole', 'english', 'grenadian', 'grenadian creole english'] 32 | GPT-3 Prediction: ['english'] 33 | 34 | 35 | SubjectEntity: Israel 36 | Ground Truth: ['hebrew'] 37 | GPT-3 Prediction: ['arabic', 'hebrew'] 38 | 39 | 40 | SubjectEntity: Maldives 41 | Ground Truth: ['maldivian'] 42 | GPT-3 Prediction: ['dhivehi'] 43 | 44 | 45 | SubjectEntity: North Macedonia 46 | Ground Truth: ['albanian', 'macedonian'] 47 | GPT-3 Prediction: ['macedonian'] 48 | 49 | 50 | SubjectEntity: Norway 51 | Ground Truth: ['bokmal', 'bokmål', 'norwegian', 'nynorsk', 'sami', 'sámi'] 52 | GPT-3 Prediction: ['norwegian'] 53 | 54 | 55 | SubjectEntity: Pakistan 56 | Ground Truth: ['arabic', 'english', 'urdu'] 57 | GPT-3 Prediction: ['urdu'] 58 | 59 | 60 | SubjectEntity: People's Republic of China 61 | Ground Truth: ['chinese', 'mandarin', 'standard mandarin'] 62 | GPT-3 Prediction: ['mandarin'] 63 | 64 | 65 | SubjectEntity: Taiwan 66 | Ground Truth: ['amis', 'austronesian', 'formosan', 'hakka', 'hokkien', 'mandarin', 'paiwan', 'standard taiwanese mandarin', 'taiwanese', 'taiwanese', 'taiwanese hokkien'] 67 | GPT-3 Prediction: ['mandarin'] 68 | 69 | 70 | SubjectEntity: Tajikistan 71 | Ground Truth: ['russian', 'tajik'] 72 | GPT-3 Prediction: ['tajik'] 73 | 74 | 75 | SubjectEntity: Zimbabwe 76 | Ground Truth: ['barwe', 'chewa', 'english', 'kalanga', 'khoisan', 'nambya', 'ndau', 'ndebele', 'northern ndebele', 'sesotho', 'shona', 'tonga', 'tsonga', 'tswana', 'venda', 'xhosa'] 77 | GPT-3 Prediction: ['english', 'ndebele', 'shona'] 78 | 79 | 80 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-empty)/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonCauseOfDeath (average f1: 0.553): 22 cases 2 | 3 | 4 | 5 | SubjectEntity: Ahmed Zewail 6 | Ground Truth: ['lymphoma', 'spinal cord lymphoma'] 7 | GPT-3 Prediction: ['cancer'] 8 | 9 | 10 | SubjectEntity: Avicii 11 | Ground Truth: ['exsanguination'] 12 | GPT-3 Prediction: ['suicide'] 13 | 14 | 15 | SubjectEntity: Bill Gates 16 | Ground Truth: [] 17 | GPT-3 Prediction: ['blunt trauma', 'injury'] 18 | 19 | 20 | SubjectEntity: Christina Grimmie 21 | Ground Truth: ['ballistic trauma', 'trauma'] 22 | GPT-3 Prediction: ['murder'] 23 | 24 | 25 | SubjectEntity: Cory Monteith 26 | Ground Truth: ['drug overdose', 'overdose'] 27 | GPT-3 Prediction: ['mixed drug toxicity'] 28 | 29 | 30 | SubjectEntity: Dilip Kumar 31 | Ground Truth: ['cancer', 'prostate cancer'] 32 | GPT-3 Prediction: ['renal failure'] 33 | 34 | 35 | SubjectEntity: Ennio Morricone 36 | Ground Truth: ['femoral fracture', 'fracture'] 37 | GPT-3 Prediction: ['infarction'] 38 | 39 | 40 | SubjectEntity: Frank Sinatra 41 | Ground Truth: ['infarction', 'myocardial infarction'] 42 | GPT-3 Prediction: ['heart attack'] 43 | 44 | 45 | SubjectEntity: George H. W. Bush 46 | Ground Truth: ['parkinson', "parkinson's disease"] 47 | GPT-3 Prediction: ['infarction'] 48 | 49 | 50 | SubjectEntity: Haruma Miura 51 | Ground Truth: ['hanging'] 52 | GPT-3 Prediction: ['suicide'] 53 | 54 | 55 | SubjectEntity: Hugh Hefner 56 | Ground Truth: ['sepsis'] 57 | GPT-3 Prediction: ['septicemia'] 58 | 59 | 60 | SubjectEntity: Idriss Déby 61 | Ground Truth: ['ballistic trauma', 'trauma'] 62 | GPT-3 Prediction: ['blunt trauma', 'injury'] 63 | 64 | 65 | SubjectEntity: Jim Rohn 66 | Ground Truth: ['fibrosis', 'pulmonary fibrosis'] 67 | GPT-3 Prediction: ['pneumonia'] 68 | 69 | 70 | SubjectEntity: John Magufuli 71 | Ground Truth: ['arrhythmia', 'heart arrhythmia'] 72 | GPT-3 Prediction: ['covid-19'] 73 | 74 | 75 | SubjectEntity: Johnny Cash 76 | Ground Truth: ['diabetes'] 77 | GPT-3 Prediction: ['infarction'] 78 | 79 | 80 | SubjectEntity: Larry King 81 | Ground Truth: ['sepsis'] 82 | GPT-3 Prediction: ['cerebrovascular disease'] 83 | 84 | 85 | SubjectEntity: Maurane 86 | Ground Truth: ['falling'] 87 | GPT-3 Prediction: ['cancer'] 88 | 89 | 90 | SubjectEntity: Maya Angelou 91 | Ground Truth: ['disease'] 92 | GPT-3 Prediction: ['heart failure'] 93 | 94 | 95 | SubjectEntity: Nelson Mandela 96 | Ground Truth: ['respiratory disease'] 97 | GPT-3 Prediction: ['respiratory infection'] 98 | 99 | 100 | SubjectEntity: Roddy Piper 101 | Ground Truth: ['cardiac arrest'] 102 | GPT-3 Prediction: ['heart attack'] 103 | 104 | 105 | SubjectEntity: Ronaldinho 106 | Ground Truth: [] 107 | GPT-3 Prediction: ['infarction'] 108 | 109 | 110 | SubjectEntity: Roy Orbison 111 | Ground Truth: ['infarction', 'myocardial infarction'] 112 | GPT-3 Prediction: ['heart attack'] 113 | 114 | 115 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-empty)/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | PersonLanguage (average f1: 0.827): 19 cases 2 | 3 | 4 | 5 | SubjectEntity: A. R. Rahman 6 | Ground Truth: ['hindi', 'tamil', 'telugu'] 7 | GPT-3 Prediction: ['english', 'hindi', 'kannada', 'malayalam', 'tamil', 'telugu'] 8 | 9 | 10 | SubjectEntity: Abdullah Gül 11 | Ground Truth: ['turkish'] 12 | GPT-3 Prediction: ['english', 'french', 'turkish'] 13 | 14 | 15 | SubjectEntity: Akon 16 | Ground Truth: ['english'] 17 | GPT-3 Prediction: ['english', 'french', 'wolof'] 18 | 19 | 20 | SubjectEntity: Akshay Kumar 21 | Ground Truth: ['hindi', 'punjabi'] 22 | GPT-3 Prediction: ['english', 'hindi'] 23 | 24 | 25 | SubjectEntity: Barack Obama 26 | Ground Truth: ['english', 'indonesian'] 27 | GPT-3 Prediction: ['english'] 28 | 29 | 30 | SubjectEntity: Blaise Matuidi 31 | Ground Truth: ['french'] 32 | GPT-3 Prediction: ['english', 'french', 'italian', 'spanish'] 33 | 34 | 35 | SubjectEntity: Emma Watson 36 | Ground Truth: ['english', 'french'] 37 | GPT-3 Prediction: ['english'] 38 | 39 | 40 | SubjectEntity: Genelia D'Souza 41 | Ground Truth: ['hindi', 'malayalam', 'tamil', 'telugu'] 42 | GPT-3 Prediction: ['english', 'hindi', 'marathi', 'telugu'] 43 | 44 | 45 | SubjectEntity: John Cena 46 | Ground Truth: ['chinese', 'english', 'mandarin', 'mandarin chinese'] 47 | GPT-3 Prediction: ['english'] 48 | 49 | 50 | SubjectEntity: Jose Michimani 51 | Ground Truth: ['english'] 52 | GPT-3 Prediction: ['quechua', 'spanish'] 53 | 54 | 55 | SubjectEntity: Kobe Bryant 56 | Ground Truth: ['english', 'italian', 'spanish'] 57 | GPT-3 Prediction: ['english'] 58 | 59 | 60 | SubjectEntity: Oscar dos Santos Emboaba Júnior 61 | Ground Truth: ['portuguese'] 62 | GPT-3 Prediction: ['portuguese', 'spanish'] 63 | 64 | 65 | SubjectEntity: Pau Gasol 66 | Ground Truth: ['catalan', 'english', 'french', 'italian', 'spanish'] 67 | GPT-3 Prediction: ['catalan', 'english', 'spanish'] 68 | 69 | 70 | SubjectEntity: Paulo Coelho 71 | Ground Truth: ['portuguese'] 72 | GPT-3 Prediction: ['english', 'french', 'portuguese', 'spanish'] 73 | 74 | 75 | SubjectEntity: Salman Khan 76 | Ground Truth: ['hindi'] 77 | GPT-3 Prediction: ['hindi', 'urdu'] 78 | 79 | 80 | SubjectEntity: Samir Nasri 81 | Ground Truth: ['french'] 82 | GPT-3 Prediction: ['arabic', 'french'] 83 | 84 | 85 | SubjectEntity: Selena Gomez 86 | Ground Truth: ['english', 'spanish'] 87 | GPT-3 Prediction: ['english'] 88 | 89 | 90 | SubjectEntity: Thiago Alcântara 91 | Ground Truth: ['spanish'] 92 | GPT-3 Prediction: ['catalan', 'english', 'german', 'portuguese', 'spanish'] 93 | 94 | 95 | SubjectEntity: Zlatan Ibrahimović 96 | Ground Truth: ['bosnian', 'english', 'italian', 'spanish', 'swedish'] 97 | GPT-3 Prediction: ['bosnian', 'croatian', 'english', 'french', 'italian', 'serbian', 'spanish', 'swedish'] 98 | 99 | 100 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-empty)/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonPlaceOfDeath (average f1: 0.82): 9 cases 2 | 3 | 4 | 5 | SubjectEntity: Abdul Rehman Malik 6 | Ground Truth: ['islamabad'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Alfredo González Flores 11 | Ground Truth: ['heredia'] 12 | GPT-3 Prediction: ['mexico city'] 13 | 14 | 15 | SubjectEntity: Almudena Grandes 16 | Ground Truth: ['madrid'] 17 | GPT-3 Prediction: [''] 18 | 19 | 20 | SubjectEntity: Anne Rice 21 | Ground Truth: ['rancho mirage'] 22 | GPT-3 Prediction: [''] 23 | 24 | 25 | SubjectEntity: Chick Corea 26 | Ground Truth: ['tampa', 'tampa bay area'] 27 | GPT-3 Prediction: ['beverly hills'] 28 | 29 | 30 | SubjectEntity: Ken Robinson 31 | Ground Truth: ['london'] 32 | GPT-3 Prediction: [''] 33 | 34 | 35 | SubjectEntity: Michael Collins 36 | Ground Truth: ['naples'] 37 | GPT-3 Prediction: ['cape canaveral'] 38 | 39 | 40 | SubjectEntity: Rachel Caine 41 | Ground Truth: ['texas'] 42 | GPT-3 Prediction: [''] 43 | 44 | 45 | SubjectEntity: Roald Dahl 46 | Ground Truth: ['oxford'] 47 | GPT-3 Prediction: ['great missenden'] 48 | 49 | 50 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-empty)/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | RiverBasinsCountry (average f1: 0.832): 20 cases 2 | 3 | 4 | 5 | SubjectEntity: Didi Liakhvi river 6 | Ground Truth: ['georgia'] 7 | GPT-3 Prediction: ['georgia', 'russia'] 8 | 9 | 10 | SubjectEntity: Dniester 11 | Ground Truth: ['moldova', 'ukraine'] 12 | GPT-3 Prediction: ['moldova', 'romania', 'ukraine'] 13 | 14 | 15 | SubjectEntity: Gadar river 16 | Ground Truth: ['iran'] 17 | GPT-3 Prediction: ['iran', 'iraq'] 18 | 19 | 20 | SubjectEntity: Gambia river 21 | Ground Truth: ['gambia', 'guinea', 'senegal', 'the gambia'] 22 | GPT-3 Prediction: ['senegal', 'the gambia'] 23 | 24 | 25 | SubjectEntity: Han river 26 | Ground Truth: ['korea', 'korea', 'north korea', 'south korea'] 27 | GPT-3 Prediction: ['china'] 28 | 29 | 30 | SubjectEntity: Japurá river 31 | Ground Truth: ['brazil', 'colombia'] 32 | GPT-3 Prediction: ['brazil', 'colombia', 'peru'] 33 | 34 | 35 | SubjectEntity: Kura river 36 | Ground Truth: ['azerbaijan', 'georgia', 'turkey'] 37 | GPT-3 Prediction: ['armenia', 'azerbaijan', 'georgia', 'iran', 'russia', 'turkey'] 38 | 39 | 40 | SubjectEntity: Narew 41 | Ground Truth: ['belarus', 'poland'] 42 | GPT-3 Prediction: ['poland'] 43 | 44 | 45 | SubjectEntity: Narva river 46 | Ground Truth: ['belarus', 'estonia', 'latvia', 'russia'] 47 | GPT-3 Prediction: ['estonia', 'russia'] 48 | 49 | 50 | SubjectEntity: Neman 51 | Ground Truth: ['belarus', 'latvia', 'lithuania', 'russia'] 52 | GPT-3 Prediction: ['belarus', 'lithuania', 'poland', 'russia'] 53 | 54 | 55 | SubjectEntity: Neris 56 | Ground Truth: ['belarus', 'latvia', 'lithuania'] 57 | GPT-3 Prediction: ['belarus', 'lithuania'] 58 | 59 | 60 | SubjectEntity: Ottawa river 61 | Ground Truth: ['canada'] 62 | GPT-3 Prediction: ['canada', 'united states'] 63 | 64 | 65 | SubjectEntity: Red river of the south 66 | Ground Truth: ['texas', 'united states of america', 'usa'] 67 | GPT-3 Prediction: ['united states'] 68 | 69 | 70 | SubjectEntity: San 71 | Ground Truth: ['poland', 'ukraine'] 72 | GPT-3 Prediction: ['argentina', 'bolivia', 'chile', 'paraguay'] 73 | 74 | 75 | SubjectEntity: Shatt al-Arab 76 | Ground Truth: ['iran', 'iraq', 'kuwait', 'saudi arabia', 'syria', 'turkey'] 77 | GPT-3 Prediction: ['iran', 'iraq'] 78 | 79 | 80 | SubjectEntity: Someș river 81 | Ground Truth: ['hungary', 'romania'] 82 | GPT-3 Prediction: ['romania'] 83 | 84 | 85 | SubjectEntity: Torne river 86 | Ground Truth: ['finland', 'norway', 'sweden'] 87 | GPT-3 Prediction: ['finland', 'sweden'] 88 | 89 | 90 | SubjectEntity: Ural river 91 | Ground Truth: ['kazakhstan', 'russia'] 92 | GPT-3 Prediction: ['kazakhstan', 'russia', 'turkmenistan', 'uzbekistan'] 93 | 94 | 95 | SubjectEntity: Ussuri 96 | Ground Truth: ['russia'] 97 | GPT-3 Prediction: ['china', 'russia'] 98 | 99 | 100 | SubjectEntity: Yantra 101 | Ground Truth: ['bulgaria'] 102 | GPT-3 Prediction: ['bulgaria', 'romania'] 103 | 104 | 105 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-none)/ChemicalCompoundElement.txt: -------------------------------------------------------------------------------- 1 | ChemicalCompoundElement (average f1: 0.94): 17 cases 2 | 3 | 4 | 5 | SubjectEntity: Adenosine Diphosphate Ribose 6 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 7 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 8 | 9 | 10 | SubjectEntity: Aluminium Chlorohydrate 11 | Ground Truth: ['aluminium', 'chlorine', 'hydrogen', 'oxygen'] 12 | GPT-3 Prediction: ['aluminium', 'chlorine', 'hydrogen'] 13 | 14 | 15 | SubjectEntity: Aluminum Iodide 16 | Ground Truth: ['aluminium', 'iodine'] 17 | GPT-3 Prediction: ['aluminum', 'iodine'] 18 | 19 | 20 | SubjectEntity: Atracurium Besylate 21 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 22 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 23 | 24 | 25 | SubjectEntity: Cetyl Alcohol 26 | Ground Truth: ['carbon', 'hydrogen', 'oxygen'] 27 | GPT-3 Prediction: ['carbon', 'hydrogen'] 28 | 29 | 30 | SubjectEntity: Chlorophyll F 31 | Ground Truth: ['carbon', 'hydrogen', 'magnesium', 'nitrogen', 'oxygen'] 32 | GPT-3 Prediction: ['carbon', 'hydrogen', 'magnesium', 'oxygen'] 33 | 34 | 35 | SubjectEntity: Flavin Adenine Dinucleotide 36 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 37 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 38 | 39 | 40 | SubjectEntity: Glucagon 41 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 42 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 43 | 44 | 45 | SubjectEntity: Glutathione 46 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 47 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'sulfur'] 48 | 49 | 50 | SubjectEntity: Heroin 51 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 52 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 53 | 54 | 55 | SubjectEntity: Hydroxocobalamin 56 | Ground Truth: ['carbon', 'cobalt', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 57 | GPT-3 Prediction: ['cobalt', 'hydrogen', 'nitrogen', 'oxygen'] 58 | 59 | 60 | SubjectEntity: Metformin 61 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen'] 62 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 63 | 64 | 65 | SubjectEntity: Motilin 66 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 67 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen'] 68 | 69 | 70 | SubjectEntity: Mustard Gas 71 | Ground Truth: ['carbon', 'chlorine', 'hydrogen', 'sulfur'] 72 | GPT-3 Prediction: ['carbon', 'chlorine', 'hydrogen', 'oxygen', 'sulfur'] 73 | 74 | 75 | SubjectEntity: Tabun 76 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'phosphorus'] 77 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 78 | 79 | 80 | SubjectEntity: Tenoxicam 81 | Ground Truth: ['carbon', 'hydrogen', 'nitrogen', 'oxygen', 'sulfur'] 82 | GPT-3 Prediction: ['carbon', 'hydrogen', 'nitrogen', 'oxygen'] 83 | 84 | 85 | SubjectEntity: Triphenylphosphine 86 | Ground Truth: ['carbon', 'hydrogen', 'phosphorus'] 87 | GPT-3 Prediction: ['carbon', 'hydrogen', 'oxygen', 'phosphorus'] 88 | 89 | 90 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-none)/CompanyParentOrganization.txt: -------------------------------------------------------------------------------- 1 | CompanyParentOrganization (average f1: 0.77): 12 cases 2 | 3 | 4 | 5 | SubjectEntity: Alex von Falkenhausen Motorenbau 6 | Ground Truth: [] 7 | GPT-3 Prediction: ['bmw'] 8 | 9 | 10 | SubjectEntity: Aston Martin lagonda 11 | Ground Truth: [] 12 | GPT-3 Prediction: ['ford motor company', 'investindustrial', 'kuwait investment authority'] 13 | 14 | 15 | SubjectEntity: Austro-Daimler 16 | Ground Truth: [] 17 | GPT-3 Prediction: ['daimler ag'] 18 | 19 | 20 | SubjectEntity: Changan Automobile 21 | Ground Truth: ['china south industries', 'china south industries group'] 22 | GPT-3 Prediction: ['changan group'] 23 | 24 | 25 | SubjectEntity: Hanomag 26 | Ground Truth: ['komatsu', 'komatsu limited'] 27 | GPT-3 Prediction: [''] 28 | 29 | 30 | SubjectEntity: Hyundai Motor Company 31 | Ground Truth: ['hyundai'] 32 | GPT-3 Prediction: ['hyundai motor group'] 33 | 34 | 35 | SubjectEntity: Lotus Cars 36 | Ground Truth: ['geely'] 37 | GPT-3 Prediction: ['geely holding group', 'proton holdings berhad'] 38 | 39 | 40 | SubjectEntity: MG Car Company Limited 41 | Ground Truth: [] 42 | GPT-3 Prediction: ['saic motor corporation limited'] 43 | 44 | 45 | SubjectEntity: NSU 46 | Ground Truth: ['volkswagen'] 47 | GPT-3 Prediction: ['audi ag', 'volkswagen group'] 48 | 49 | 50 | SubjectEntity: Peugeot 51 | Ground Truth: ['stellantis'] 52 | GPT-3 Prediction: ['psa peugeot citroën', 'peugeot s.a.'] 53 | 54 | 55 | SubjectEntity: Scripps-Booth 56 | Ground Truth: ['general motors', 'gm'] 57 | GPT-3 Prediction: [''] 58 | 59 | 60 | SubjectEntity: Simca 61 | Ground Truth: ['chrysler', 'chrysler europe'] 62 | GPT-3 Prediction: ['chrysler europe', 'psa peugeot citroën', 'peugeot'] 63 | 64 | 65 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-none)/CountryBordersWithCountry.txt: -------------------------------------------------------------------------------- 1 | CountryBordersWithCountry (average f1: 0.749): 14 cases 2 | 3 | 4 | 5 | SubjectEntity: Australia 6 | Ground Truth: ['east timor', 'indonesia', 'new zealand', 'papua new guinea', 'timor'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Bahrain 11 | Ground Truth: ['iran', 'saudi arabia'] 12 | GPT-3 Prediction: ['qatar'] 13 | 14 | 15 | SubjectEntity: Comoros 16 | Ground Truth: ['france', 'madagascar', 'mozambique', 'seychelles', 'tanzania'] 17 | GPT-3 Prediction: [''] 18 | 19 | 20 | SubjectEntity: Cuba 21 | Ground Truth: ['united states of america', 'usa'] 22 | GPT-3 Prediction: [''] 23 | 24 | 25 | SubjectEntity: Dominican Republic 26 | Ground Truth: ['haiti', 'united states of america', 'usa', 'venezuela'] 27 | GPT-3 Prediction: ['haiti'] 28 | 29 | 30 | SubjectEntity: Ghana 31 | Ground Truth: ['burkina faso', 'ivory coast', 'togo'] 32 | GPT-3 Prediction: ['burkina faso', 'cote divoire', 'la cote divoire', 'togo'] 33 | 34 | 35 | SubjectEntity: Grenada 36 | Ground Truth: ['venezuela'] 37 | GPT-3 Prediction: [''] 38 | 39 | 40 | SubjectEntity: Japan 41 | Ground Truth: ['china', 'korea', "people's republic of china", 'philippines', 'russia', 'south korea', 'taiwan', 'united states of america', 'usa'] 42 | GPT-3 Prediction: ['north korea', 'russia'] 43 | 44 | 45 | SubjectEntity: Kingdom of Denmark 46 | Ground Truth: ['canada', 'germany', 'iceland', 'norway', 'poland', 'sweden', 'uk', 'united kingdom'] 47 | GPT-3 Prediction: ['germany'] 48 | 49 | 50 | SubjectEntity: Kingdom of the Netherlands 51 | Ground Truth: ['belgium', 'france', 'germany', 'venezuela'] 52 | GPT-3 Prediction: ['belgium', 'germany'] 53 | 54 | 55 | SubjectEntity: Portugal 56 | Ground Truth: ['castile and leon', 'castile and león', 'spain'] 57 | GPT-3 Prediction: ['spain'] 58 | 59 | 60 | SubjectEntity: Saint Vincent and the Grenadines 61 | Ground Truth: ['venezuela'] 62 | GPT-3 Prediction: [''] 63 | 64 | 65 | SubjectEntity: Samoa 66 | Ground Truth: ['united states of america', 'usa'] 67 | GPT-3 Prediction: [''] 68 | 69 | 70 | SubjectEntity: Venezuela 71 | Ground Truth: ['brazil', 'colombia', 'dominica', 'dominican republic', 'france', 'grenada', 'guyana', 'kingdom of the netherlands', 'netherlands', 'saint kitts and nevis', 'saint lucia', 'saint vincent and the grenadines', 'trinidad and tobago', 'uk', 'united kingdom', 'united states of america', 'usa'] 72 | GPT-3 Prediction: ['brazil', 'colombia', 'guyana'] 73 | 74 | 75 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-none)/CountryOfficialLanguage.txt: -------------------------------------------------------------------------------- 1 | CountryOfficialLanguage (average f1: 0.833): 15 cases 2 | 3 | 4 | 5 | SubjectEntity: Afghanistan 6 | Ground Truth: ['arabic', 'baluchi', 'dari', 'nuristani', 'pamir', 'pashayi', 'pashto', 'turkmen', 'uzbek'] 7 | GPT-3 Prediction: ['dari', 'pashto'] 8 | 9 | 10 | SubjectEntity: Botswana 11 | Ground Truth: ['english'] 12 | GPT-3 Prediction: ['tswana'] 13 | 14 | 15 | SubjectEntity: Brunei 16 | Ground Truth: ['english', 'malay'] 17 | GPT-3 Prediction: ['malay'] 18 | 19 | 20 | SubjectEntity: Burundi 21 | Ground Truth: ['english', 'french', 'kirundi'] 22 | GPT-3 Prediction: ['french', 'kirundi', 'swahili'] 23 | 24 | 25 | SubjectEntity: East Timor 26 | Ground Truth: ['portuguese', 'tetum'] 27 | GPT-3 Prediction: ['indonesian', 'portuguese', 'tetum'] 28 | 29 | 30 | SubjectEntity: Grenada 31 | Ground Truth: ['creole', 'english', 'grenadian', 'grenadian creole english'] 32 | GPT-3 Prediction: ['english'] 33 | 34 | 35 | SubjectEntity: Israel 36 | Ground Truth: ['hebrew'] 37 | GPT-3 Prediction: ['arabic', 'hebrew'] 38 | 39 | 40 | SubjectEntity: Maldives 41 | Ground Truth: ['maldivian'] 42 | GPT-3 Prediction: ['dhivehi'] 43 | 44 | 45 | SubjectEntity: North Macedonia 46 | Ground Truth: ['albanian', 'macedonian'] 47 | GPT-3 Prediction: ['macedonian'] 48 | 49 | 50 | SubjectEntity: Norway 51 | Ground Truth: ['bokmal', 'bokmål', 'norwegian', 'nynorsk', 'sami', 'sámi'] 52 | GPT-3 Prediction: ['norwegian'] 53 | 54 | 55 | SubjectEntity: Pakistan 56 | Ground Truth: ['arabic', 'english', 'urdu'] 57 | GPT-3 Prediction: ['urdu'] 58 | 59 | 60 | SubjectEntity: People's Republic of China 61 | Ground Truth: ['chinese', 'mandarin', 'standard mandarin'] 62 | GPT-3 Prediction: ['mandarin'] 63 | 64 | 65 | SubjectEntity: Taiwan 66 | Ground Truth: ['amis', 'austronesian', 'formosan', 'hakka', 'hokkien', 'mandarin', 'paiwan', 'standard taiwanese mandarin', 'taiwanese', 'taiwanese', 'taiwanese hokkien'] 67 | GPT-3 Prediction: ['mandarin'] 68 | 69 | 70 | SubjectEntity: Tajikistan 71 | Ground Truth: ['russian', 'tajik'] 72 | GPT-3 Prediction: ['tajik'] 73 | 74 | 75 | SubjectEntity: Zimbabwe 76 | Ground Truth: ['barwe', 'chewa', 'english', 'kalanga', 'khoisan', 'nambya', 'ndau', 'ndebele', 'northern ndebele', 'sesotho', 'shona', 'tonga', 'tsonga', 'tswana', 'venda', 'xhosa'] 77 | GPT-3 Prediction: ['english', 'ndebele', 'shona'] 78 | 79 | 80 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-none)/PersonCauseOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonCauseOfDeath (average f1: 0.593): 20 cases 2 | 3 | 4 | 5 | SubjectEntity: Ahmed Zewail 6 | Ground Truth: ['lymphoma', 'spinal cord lymphoma'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Avicii 11 | Ground Truth: ['exsanguination'] 12 | GPT-3 Prediction: ['suicide'] 13 | 14 | 15 | SubjectEntity: Christina Grimmie 16 | Ground Truth: ['ballistic trauma', 'trauma'] 17 | GPT-3 Prediction: ['murder'] 18 | 19 | 20 | SubjectEntity: Cory Monteith 21 | Ground Truth: ['drug overdose', 'overdose'] 22 | GPT-3 Prediction: ['mixed drug toxicity'] 23 | 24 | 25 | SubjectEntity: Dilip Kumar 26 | Ground Truth: ['cancer', 'prostate cancer'] 27 | GPT-3 Prediction: [''] 28 | 29 | 30 | SubjectEntity: Ennio Morricone 31 | Ground Truth: ['femoral fracture', 'fracture'] 32 | GPT-3 Prediction: [''] 33 | 34 | 35 | SubjectEntity: Frank Sinatra 36 | Ground Truth: ['infarction', 'myocardial infarction'] 37 | GPT-3 Prediction: ['heart attack'] 38 | 39 | 40 | SubjectEntity: George H. W. Bush 41 | Ground Truth: ['parkinson', "parkinson's disease"] 42 | GPT-3 Prediction: ['septic shock'] 43 | 44 | 45 | SubjectEntity: Haruma Miura 46 | Ground Truth: ['hanging'] 47 | GPT-3 Prediction: [''] 48 | 49 | 50 | SubjectEntity: Hugh Hefner 51 | Ground Truth: ['sepsis'] 52 | GPT-3 Prediction: ['septicemia'] 53 | 54 | 55 | SubjectEntity: Idriss Déby 56 | Ground Truth: ['ballistic trauma', 'trauma'] 57 | GPT-3 Prediction: [''] 58 | 59 | 60 | SubjectEntity: Jim Rohn 61 | Ground Truth: ['fibrosis', 'pulmonary fibrosis'] 62 | GPT-3 Prediction: ['pneumonia'] 63 | 64 | 65 | SubjectEntity: John Magufuli 66 | Ground Truth: ['arrhythmia', 'heart arrhythmia'] 67 | GPT-3 Prediction: ['covid-19'] 68 | 69 | 70 | SubjectEntity: Johnny Cash 71 | Ground Truth: ['diabetes'] 72 | GPT-3 Prediction: ['infarction'] 73 | 74 | 75 | SubjectEntity: Larry King 76 | Ground Truth: ['sepsis'] 77 | GPT-3 Prediction: [''] 78 | 79 | 80 | SubjectEntity: Maurane 81 | Ground Truth: ['falling'] 82 | GPT-3 Prediction: ['cancer'] 83 | 84 | 85 | SubjectEntity: Maya Angelou 86 | Ground Truth: ['disease'] 87 | GPT-3 Prediction: [''] 88 | 89 | 90 | SubjectEntity: Nelson Mandela 91 | Ground Truth: ['respiratory disease'] 92 | GPT-3 Prediction: ['respiratory infection'] 93 | 94 | 95 | SubjectEntity: Roddy Piper 96 | Ground Truth: ['cardiac arrest'] 97 | GPT-3 Prediction: ['aortic aneurysm'] 98 | 99 | 100 | SubjectEntity: Roy Orbison 101 | Ground Truth: ['infarction', 'myocardial infarction'] 102 | GPT-3 Prediction: ['heart attack'] 103 | 104 | 105 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-none)/PersonInstrument.txt: -------------------------------------------------------------------------------- 1 | PersonInstrument (average f1: 0.568): 22 cases 2 | 3 | 4 | 5 | SubjectEntity: A. J. McLean 6 | Ground Truth: ['lyric spinto tenor', 'tenor'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Arnel Pineda 11 | Ground Truth: ['guitar'] 12 | GPT-3 Prediction: [''] 13 | 14 | 15 | SubjectEntity: Billie Joe Armstrong 16 | Ground Truth: ['guitar', 'harmonica', 'mandolin', 'percussion', 'percussion instrument', 'piano', 'saxophone'] 17 | GPT-3 Prediction: ['bass', 'guitar', 'piano'] 18 | 19 | 20 | SubjectEntity: Björk 21 | Ground Truth: ['celesta', 'clarinet', 'clavichord', 'flute', 'harp', 'keyboard', 'keyboard instrument', 'piano', 'pipe organ', 'reeds', 'saxophone', 'sets of free reeds'] 22 | GPT-3 Prediction: ['flute', 'guitar', 'piano'] 23 | 24 | 25 | SubjectEntity: China Anne McClain 26 | Ground Truth: ['bass', 'guitar', 'piano'] 27 | GPT-3 Prediction: [''] 28 | 29 | 30 | SubjectEntity: Cho Kyuhyun 31 | Ground Truth: ['piano'] 32 | GPT-3 Prediction: [''] 33 | 34 | 35 | SubjectEntity: Duff McKagan 36 | Ground Truth: ['bass guitar', 'guitar', 'guitar'] 37 | GPT-3 Prediction: ['bass'] 38 | 39 | 40 | SubjectEntity: Gary Barlow 41 | Ground Truth: ['piano'] 42 | GPT-3 Prediction: [''] 43 | 44 | 45 | SubjectEntity: Gerard Way 46 | Ground Truth: ['electric guitar', 'guitar'] 47 | GPT-3 Prediction: [''] 48 | 49 | 50 | SubjectEntity: Heo Young-saeng 51 | Ground Truth: ['piano'] 52 | GPT-3 Prediction: [''] 53 | 54 | 55 | SubjectEntity: Joseph Hahn 56 | Ground Truth: ['keyboard', 'keyboard instrument'] 57 | GPT-3 Prediction: [''] 58 | 59 | 60 | SubjectEntity: Kangin 61 | Ground Truth: ['piano'] 62 | GPT-3 Prediction: [''] 63 | 64 | 65 | SubjectEntity: Lay 66 | Ground Truth: ['cello', 'drum', 'electric guitar', 'guitar', 'hulusi', 'ney', 'piano'] 67 | GPT-3 Prediction: [''] 68 | 69 | 70 | SubjectEntity: Lee Jung-shin 71 | Ground Truth: ['bass guitar', 'guitar'] 72 | GPT-3 Prediction: [''] 73 | 74 | 75 | SubjectEntity: Logan Henderson 76 | Ground Truth: ['piano'] 77 | GPT-3 Prediction: [''] 78 | 79 | 80 | SubjectEntity: Morrissey 81 | Ground Truth: ['piano'] 82 | GPT-3 Prediction: [''] 83 | 84 | 85 | SubjectEntity: Nichkhun 86 | Ground Truth: ['keyboard', 'keyboard instrument'] 87 | GPT-3 Prediction: [''] 88 | 89 | 90 | SubjectEntity: Paul McCartney 91 | Ground Truth: ['bass', 'bass guitar', 'cello', 'double bass', 'guitar', 'guitar', 'keyboard', 'keyboard instrument', 'mandolin', 'melodica', 'percussion', 'percussion instrument', 'piano', 'recorder', 'sitar', 'trumpet', 'ukulele', 'violin'] 92 | GPT-3 Prediction: ['bass', 'drums', 'guitar', 'piano'] 93 | 94 | 95 | SubjectEntity: Robert Plant 96 | Ground Truth: ['harmonica'] 97 | GPT-3 Prediction: ['guitar'] 98 | 99 | 100 | SubjectEntity: Roger Taylor 101 | Ground Truth: ['bass guitar', 'drum', 'drum', 'drum kit', 'drum machine', 'guitar', 'guitar', 'keyboard', 'keyboard instrument'] 102 | GPT-3 Prediction: ['drums', 'piano'] 103 | 104 | 105 | SubjectEntity: Shindong 106 | Ground Truth: ['piano'] 107 | GPT-3 Prediction: [''] 108 | 109 | 110 | SubjectEntity: Yoshiki 111 | Ground Truth: ['arp', 'arp 2500', 'drum', 'drum kit', 'guitar', 'keyboard', 'keyboard instrument', 'piano'] 112 | GPT-3 Prediction: ['drums', 'piano'] 113 | 114 | 115 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-none)/PersonLanguage.txt: -------------------------------------------------------------------------------- 1 | PersonLanguage (average f1: 0.827): 19 cases 2 | 3 | 4 | 5 | SubjectEntity: A. R. Rahman 6 | Ground Truth: ['hindi', 'tamil', 'telugu'] 7 | GPT-3 Prediction: ['english', 'hindi', 'kannada', 'malayalam', 'tamil', 'telugu'] 8 | 9 | 10 | SubjectEntity: Abdullah Gül 11 | Ground Truth: ['turkish'] 12 | GPT-3 Prediction: ['english', 'french', 'turkish'] 13 | 14 | 15 | SubjectEntity: Akon 16 | Ground Truth: ['english'] 17 | GPT-3 Prediction: ['english', 'french', 'wolof'] 18 | 19 | 20 | SubjectEntity: Akshay Kumar 21 | Ground Truth: ['hindi', 'punjabi'] 22 | GPT-3 Prediction: ['english', 'hindi'] 23 | 24 | 25 | SubjectEntity: Barack Obama 26 | Ground Truth: ['english', 'indonesian'] 27 | GPT-3 Prediction: ['english'] 28 | 29 | 30 | SubjectEntity: Blaise Matuidi 31 | Ground Truth: ['french'] 32 | GPT-3 Prediction: ['english', 'french', 'italian', 'spanish'] 33 | 34 | 35 | SubjectEntity: Emma Watson 36 | Ground Truth: ['english', 'french'] 37 | GPT-3 Prediction: ['english'] 38 | 39 | 40 | SubjectEntity: Genelia D'Souza 41 | Ground Truth: ['hindi', 'malayalam', 'tamil', 'telugu'] 42 | GPT-3 Prediction: ['english', 'hindi', 'marathi', 'telugu'] 43 | 44 | 45 | SubjectEntity: John Cena 46 | Ground Truth: ['chinese', 'english', 'mandarin', 'mandarin chinese'] 47 | GPT-3 Prediction: ['english'] 48 | 49 | 50 | SubjectEntity: Jose Michimani 51 | Ground Truth: ['english'] 52 | GPT-3 Prediction: ['quechua', 'spanish'] 53 | 54 | 55 | SubjectEntity: Kobe Bryant 56 | Ground Truth: ['english', 'italian', 'spanish'] 57 | GPT-3 Prediction: ['english'] 58 | 59 | 60 | SubjectEntity: Oscar dos Santos Emboaba Júnior 61 | Ground Truth: ['portuguese'] 62 | GPT-3 Prediction: ['portuguese', 'spanish'] 63 | 64 | 65 | SubjectEntity: Pau Gasol 66 | Ground Truth: ['catalan', 'english', 'french', 'italian', 'spanish'] 67 | GPT-3 Prediction: ['catalan', 'english', 'spanish'] 68 | 69 | 70 | SubjectEntity: Paulo Coelho 71 | Ground Truth: ['portuguese'] 72 | GPT-3 Prediction: ['english', 'french', 'portuguese', 'spanish'] 73 | 74 | 75 | SubjectEntity: Salman Khan 76 | Ground Truth: ['hindi'] 77 | GPT-3 Prediction: ['hindi', 'urdu'] 78 | 79 | 80 | SubjectEntity: Samir Nasri 81 | Ground Truth: ['french'] 82 | GPT-3 Prediction: ['arabic', 'french'] 83 | 84 | 85 | SubjectEntity: Selena Gomez 86 | Ground Truth: ['english', 'spanish'] 87 | GPT-3 Prediction: ['english'] 88 | 89 | 90 | SubjectEntity: Thiago Alcântara 91 | Ground Truth: ['spanish'] 92 | GPT-3 Prediction: ['catalan', 'english', 'german', 'portuguese', 'spanish'] 93 | 94 | 95 | SubjectEntity: Zlatan Ibrahimović 96 | Ground Truth: ['bosnian', 'english', 'italian', 'spanish', 'swedish'] 97 | GPT-3 Prediction: ['bosnian', 'croatian', 'english', 'french', 'italian', 'serbian', 'spanish', 'swedish'] 98 | 99 | 100 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-none)/PersonPlaceOfDeath.txt: -------------------------------------------------------------------------------- 1 | PersonPlaceOfDeath (average f1: 0.78): 11 cases 2 | 3 | 4 | 5 | SubjectEntity: Abdul Rehman Malik 6 | Ground Truth: ['islamabad'] 7 | GPT-3 Prediction: [''] 8 | 9 | 10 | SubjectEntity: Alfredo González Flores 11 | Ground Truth: ['heredia'] 12 | GPT-3 Prediction: [''] 13 | 14 | 15 | SubjectEntity: Almudena Grandes 16 | Ground Truth: ['madrid'] 17 | GPT-3 Prediction: [''] 18 | 19 | 20 | SubjectEntity: Anne Rice 21 | Ground Truth: ['rancho mirage'] 22 | GPT-3 Prediction: ['new orleans'] 23 | 24 | 25 | SubjectEntity: Aretha Franklin 26 | Ground Truth: ['detroit'] 27 | GPT-3 Prediction: [''] 28 | 29 | 30 | SubjectEntity: Cameron Boyce 31 | Ground Truth: ['la', 'los angeles'] 32 | GPT-3 Prediction: [''] 33 | 34 | 35 | SubjectEntity: Chick Corea 36 | Ground Truth: ['tampa', 'tampa bay area'] 37 | GPT-3 Prediction: [''] 38 | 39 | 40 | SubjectEntity: Glenn Fredly 41 | Ground Truth: ['jakarta'] 42 | GPT-3 Prediction: [''] 43 | 44 | 45 | SubjectEntity: Ken Robinson 46 | Ground Truth: ['london'] 47 | GPT-3 Prediction: [''] 48 | 49 | 50 | SubjectEntity: Michael Collins 51 | Ground Truth: ['naples'] 52 | GPT-3 Prediction: [''] 53 | 54 | 55 | SubjectEntity: Rachel Caine 56 | Ground Truth: ['texas'] 57 | GPT-3 Prediction: [''] 58 | 59 | 60 | -------------------------------------------------------------------------------- /failure_cases/gpt3(davinci-dev-triple-none)/RiverBasinsCountry.txt: -------------------------------------------------------------------------------- 1 | RiverBasinsCountry (average f1: 0.832): 20 cases 2 | 3 | 4 | 5 | SubjectEntity: Didi Liakhvi river 6 | Ground Truth: ['georgia'] 7 | GPT-3 Prediction: ['georgia', 'russia'] 8 | 9 | 10 | SubjectEntity: Dniester 11 | Ground Truth: ['moldova', 'ukraine'] 12 | GPT-3 Prediction: ['moldova', 'romania', 'ukraine'] 13 | 14 | 15 | SubjectEntity: Gadar river 16 | Ground Truth: ['iran'] 17 | GPT-3 Prediction: ['iran', 'iraq'] 18 | 19 | 20 | SubjectEntity: Gambia river 21 | Ground Truth: ['gambia', 'guinea', 'senegal', 'the gambia'] 22 | GPT-3 Prediction: ['senegal', 'the gambia'] 23 | 24 | 25 | SubjectEntity: Han river 26 | Ground Truth: ['korea', 'korea', 'north korea', 'south korea'] 27 | GPT-3 Prediction: ['china'] 28 | 29 | 30 | SubjectEntity: Japurá river 31 | Ground Truth: ['brazil', 'colombia'] 32 | GPT-3 Prediction: ['brazil', 'colombia', 'peru'] 33 | 34 | 35 | SubjectEntity: Kura river 36 | Ground Truth: ['azerbaijan', 'georgia', 'turkey'] 37 | GPT-3 Prediction: ['armenia', 'azerbaijan', 'georgia', 'iran', 'russia', 'turkey'] 38 | 39 | 40 | SubjectEntity: Narew 41 | Ground Truth: ['belarus', 'poland'] 42 | GPT-3 Prediction: ['poland'] 43 | 44 | 45 | SubjectEntity: Narva river 46 | Ground Truth: ['belarus', 'estonia', 'latvia', 'russia'] 47 | GPT-3 Prediction: ['estonia', 'russia'] 48 | 49 | 50 | SubjectEntity: Neman 51 | Ground Truth: ['belarus', 'latvia', 'lithuania', 'russia'] 52 | GPT-3 Prediction: ['belarus', 'lithuania', 'poland', 'russia'] 53 | 54 | 55 | SubjectEntity: Neris 56 | Ground Truth: ['belarus', 'latvia', 'lithuania'] 57 | GPT-3 Prediction: ['belarus', 'lithuania'] 58 | 59 | 60 | SubjectEntity: Ottawa river 61 | Ground Truth: ['canada'] 62 | GPT-3 Prediction: ['canada', 'united states'] 63 | 64 | 65 | SubjectEntity: Red river of the south 66 | Ground Truth: ['texas', 'united states of america', 'usa'] 67 | GPT-3 Prediction: ['united states'] 68 | 69 | 70 | SubjectEntity: San 71 | Ground Truth: ['poland', 'ukraine'] 72 | GPT-3 Prediction: ['argentina', 'bolivia', 'chile', 'paraguay'] 73 | 74 | 75 | SubjectEntity: Shatt al-Arab 76 | Ground Truth: ['iran', 'iraq', 'kuwait', 'saudi arabia', 'syria', 'turkey'] 77 | GPT-3 Prediction: ['iran', 'iraq'] 78 | 79 | 80 | SubjectEntity: Someș river 81 | Ground Truth: ['hungary', 'romania'] 82 | GPT-3 Prediction: ['romania'] 83 | 84 | 85 | SubjectEntity: Torne river 86 | Ground Truth: ['finland', 'norway', 'sweden'] 87 | GPT-3 Prediction: ['finland', 'sweden'] 88 | 89 | 90 | SubjectEntity: Ural river 91 | Ground Truth: ['kazakhstan', 'russia'] 92 | GPT-3 Prediction: ['kazakhstan', 'russia', 'turkmenistan', 'uzbekistan'] 93 | 94 | 95 | SubjectEntity: Ussuri 96 | Ground Truth: ['russia'] 97 | GPT-3 Prediction: ['china', 'russia'] 98 | 99 | 100 | SubjectEntity: Yantra 101 | Ground Truth: ['bulgaria'] 102 | GPT-3 Prediction: ['bulgaria', 'romania'] 103 | 104 | 105 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | jupyter==1.0.0 2 | pandas==1.4.2 3 | torch==1.12.0 4 | transformers==4.20.1 5 | openai==0.20.0 6 | tqdm==4.64.0 7 | openai==0.20.0 -------------------------------------------------------------------------------- /submission/predictions(davinci-optimized)_19_07_2022.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HEmile/iswc-challenge/9172f32af838edee6139dc00cba8dfa3f8b862a9/submission/predictions(davinci-optimized)_19_07_2022.zip -------------------------------------------------------------------------------- /submission/predictions(davinci-optimized)_21_07_2022.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HEmile/iswc-challenge/9172f32af838edee6139dc00cba8dfa3f8b862a9/submission/predictions(davinci-optimized)_21_07_2022.zip -------------------------------------------------------------------------------- /submission/predictions(davinci-optimized-factcheck)_19_07_2022.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HEmile/iswc-challenge/9172f32af838edee6139dc00cba8dfa3f8b862a9/submission/predictions(davinci-optimized-factcheck)_19_07_2022.zip -------------------------------------------------------------------------------- /submission/predictions(davinci-optimized-factcheck)_21_07_2022.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HEmile/iswc-challenge/9172f32af838edee6139dc00cba8dfa3f8b862a9/submission/predictions(davinci-optimized-factcheck)_21_07_2022.zip -------------------------------------------------------------------------------- /submission/predictions(davinci-optimized-factcheck-wikiclean)_21_07_2022.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HEmile/iswc-challenge/9172f32af838edee6139dc00cba8dfa3f8b862a9/submission/predictions(davinci-optimized-factcheck-wikiclean)_21_07_2022.zip -------------------------------------------------------------------------------- /submission/predictions(davinci-triple-empty)_19_07_2022.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HEmile/iswc-challenge/9172f32af838edee6139dc00cba8dfa3f8b862a9/submission/predictions(davinci-triple-empty)_19_07_2022.zip -------------------------------------------------------------------------------- /submission/predictions_14_07_2022.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HEmile/iswc-challenge/9172f32af838edee6139dc00cba8dfa3f8b862a9/submission/predictions_14_07_2022.zip -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HEmile/iswc-challenge/9172f32af838edee6139dc00cba8dfa3f8b862a9/utils/__init__.py -------------------------------------------------------------------------------- /utils/file_io.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pathlib import Path 3 | from typing import List, Dict, Union 4 | 5 | import pandas as pd 6 | 7 | 8 | def read_lm_kbc_jsonl(file_path: Union[str, Path]) -> List[Dict]: 9 | """ 10 | Reads a LM-KBC jsonl file and returns a list of dictionaries. 11 | 12 | Args: 13 | file_path: path to the jsonl file 14 | 15 | Returns: 16 | list of dictionaries, each possibly has the following keys: 17 | - "SubjectEntity": str 18 | - "Relation": str 19 | - "ObjectEntities": 20 | None or List[List[str]] (can be omitted for the test input) 21 | """ 22 | rows = [] 23 | with open(file_path, "r") as f: 24 | for line in f: 25 | row = json.loads(line) 26 | rows.append(row) 27 | return rows 28 | 29 | 30 | def read_lm_kbc_jsonl_to_df(file_path: Union[str, Path]) -> pd.DataFrame: 31 | """ 32 | Reads a LM-KBC jsonl file and returns a dataframe. 33 | """ 34 | rows = read_lm_kbc_jsonl(file_path) 35 | df = pd.DataFrame(rows) 36 | return df 37 | 38 | def df_to_jsonl(df: pd.DataFrame) -> List[Dict]: 39 | """ 40 | Converts a dataframe to a list of dictionaries. 41 | """ 42 | return df.to_dict("records") 43 | -------------------------------------------------------------------------------- /utils/model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import openai 3 | 4 | openai.api_key = os.getenv("OPENAI_API_KEY") 5 | 6 | 7 | def gpt3(prompts, model="text-davinci-002"): 8 | """ functions to call GPT3 predictions """ 9 | response = openai.Completion.create( 10 | model=model, 11 | prompt=prompts, 12 | temperature=0, 13 | max_tokens=100, 14 | top_p=1, 15 | frequency_penalty=0, 16 | presence_penalty=0, 17 | logprobs=1 18 | ) 19 | return [ 20 | { 21 | 'prompt': prompt, 22 | 'text': response['text'], 23 | 'tokens': response['logprobs']['tokens'], 24 | 'logprob': response['logprobs']['token_logprobs'] 25 | } 26 | for response, prompt in zip(response.choices, prompts) 27 | ] 28 | 29 | 30 | def clean_up(probe_outputs): 31 | """ functions to clean up api output """ 32 | probe_outputs = probe_outputs.strip() 33 | probe_outputs = probe_outputs.split('\n')[0] 34 | probe_outputs = probe_outputs[2:-2].split("', '") 35 | return probe_outputs 36 | 37 | 38 | def convert_nan(probe_outputs): 39 | new_probe_outputs = [] 40 | for item in probe_outputs: 41 | if item == 'None': 42 | new_probe_outputs.append('') 43 | else: 44 | new_probe_outputs.append(item) 45 | return new_probe_outputs 46 | -------------------------------------------------------------------------------- /utils/wordnet_utils.py: -------------------------------------------------------------------------------- 1 | from nltk.corpus import wordnet as wn 2 | import nltk 3 | import random 4 | 5 | 6 | # nltk.download('wordnet') 7 | # nltk.download('omw-1.4') 8 | 9 | 10 | def distort_sentence(sentence: str) -> str: 11 | """ 12 | Distorts the input sentence with by replacing random tokens with synonyms. 13 | :param sentence: 14 | :return: 15 | """ 16 | words = sentence.split(" ") 17 | for i in range(5): 18 | replace_index = random.randint(0, len(words) - 2) 19 | word = words[replace_index] 20 | synonyms = wn.synsets(word) 21 | if len(synonyms) > 0: 22 | synonym = synonyms[0].lemmas()[0].name() 23 | words[replace_index] = synonym 24 | return " ".join(words) 25 | 26 | 27 | def augment_sentence(sentence: str) -> str: 28 | """ 29 | WIP: Maybe we will switch this component out for HuggingFace paraphrasing. 30 | 31 | Augments the input sentence with Hypernymy and Hyponymy information. 32 | Might need some kind of NER to find nouns... 33 | :param sentence: 34 | :return: 35 | """ 36 | words = sentence.split(" ") 37 | for i in range(5): 38 | replace_index = random.randint(0, len(words) - 2) 39 | word = words[replace_index] 40 | 41 | for synset in wn.synsets(word): 42 | for hyper in synset.hypernyms(): 43 | print(synset, hyper) 44 | words[replace_index] = hyper 45 | return " ".join(words) 46 | 47 | 48 | test_sentence = """Which countries share a border with Morocco?""" 49 | 50 | # Uncomment to test 51 | # print(distort_sentence(test_sentence)) 52 | # print(augment_sentence(test_sentence)) --------------------------------------------------------------------------------