├── data ├── .DS_Store └── memorizing │ ├── range.jsonl │ ├── domain.jsonl │ └── subPropertyOf.jsonl ├── docs └── overview.png ├── scripts ├── .DS_Store ├── memorizing │ ├── .DS_Store │ ├── baseline.sh │ ├── range.sh │ ├── domain.sh │ ├── type.sh │ ├── subPropertyOf.sh │ └── subClassOf.sh └── reasoning │ ├── rdfs11.sh │ ├── rdfs5.sh │ ├── rdfs7.sh │ ├── rdfs9.sh │ ├── rdfs3.sh │ └── rdfs2.sh ├── test-gpt ├── .DS_Store ├── data-gpt │ ├── .DS_Store │ ├── reasoning │ │ ├── .DS_Store │ │ ├── rdfs2-02.jsonl │ │ ├── rdfs3-02.jsonl │ │ ├── rdfs5-02.jsonl │ │ ├── rdfs7-02.jsonl │ │ ├── rdfs3-12.jsonl │ │ ├── rdfs2-12.jsonl │ │ ├── rdfs5-12.jsonl │ │ ├── rdfs7-12.jsonl │ │ ├── rdfs3-22.jsonl │ │ ├── rdfs2-22.jsonl │ │ ├── rdfs5-22.jsonl │ │ └── rdfs7-22.jsonl │ └── memorizing │ │ └── range.jsonl └── probing_gpt.py ├── README.md ├── get_memorized.py ├── metric.py ├── utils.py └── main.py /data/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vickywu1022/OntoProbe-PLMs/HEAD/data/.DS_Store -------------------------------------------------------------------------------- /docs/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vickywu1022/OntoProbe-PLMs/HEAD/docs/overview.png -------------------------------------------------------------------------------- /scripts/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vickywu1022/OntoProbe-PLMs/HEAD/scripts/.DS_Store -------------------------------------------------------------------------------- /test-gpt/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vickywu1022/OntoProbe-PLMs/HEAD/test-gpt/.DS_Store -------------------------------------------------------------------------------- /scripts/memorizing/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vickywu1022/OntoProbe-PLMs/HEAD/scripts/memorizing/.DS_Store -------------------------------------------------------------------------------- /test-gpt/data-gpt/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vickywu1022/OntoProbe-PLMs/HEAD/test-gpt/data-gpt/.DS_Store -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vickywu1022/OntoProbe-PLMs/HEAD/test-gpt/data-gpt/reasoning/.DS_Store -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OntoProbe-PLMs 2 | This repository contains the data and codes for our paper *[Do PLMs Know and Understand Ontological Knowledge?](https://www.aclanthology.org/2023.acl-long.173.pdf)*. 3 | 4 | ![overview](docs/overview.png) 5 | -------------------------------------------------------------------------------- /scripts/memorizing/baseline.sh: -------------------------------------------------------------------------------- 1 | python main.py --knowledge 'type' --model_class 'baseline' --use_cuda --mode "zs" 2 | python main.py --knowledge 'subClassOf' --model_class 'baseline' --use_cuda False --mode "zs" 3 | python main.py --knowledge 'domain' --model_class 'baseline' --use_cuda False --mode "zs" 4 | python main.py --knowledge 'range' --model_class 'baseline' --use_cuda False --mode "zs" 5 | python main.py --knowledge 'subPropertyOf' --model_class 'baseline' --use_cuda False --mode "zs" -------------------------------------------------------------------------------- /get_memorized.py: -------------------------------------------------------------------------------- 1 | def split_by_MRR(f_path): 2 | with open(f_path) as fr: 3 | data = fr.readlines() 4 | mrr_sort = [] 5 | for d in data[1::2]: 6 | [index, _, mrr] = d.split(" ") 7 | mrr_sort.append([int(index), float(mrr)]) 8 | mrr_sort.sort(key=lambda x: x[1], reverse=True) 9 | split_size = int(len(mrr_sort) / 2) 10 | memorized_index = [x[0] for x in mrr_sort[:split_size]] 11 | not_memorized_index = [x[0] for x in mrr_sort[-split_size:]] 12 | all_index = memorized_index + not_memorized_index 13 | return all_index , memorized_index, not_memorized_index 14 | 15 | def split_by_Acc(f_path): 16 | ''' 17 | Count as memorized only when the gold is ranked first / chosen 18 | ''' 19 | with open(f_path) as fr: 20 | data = fr.readlines() 21 | memorized_index = [x for x in range(len(data)) if int(data[x].strip()) == 1] 22 | not_memorized_index = [x for x in range(len(data)) if int(data[x].strip()) != 1] 23 | all_index = memorized_index + not_memorized_index 24 | return all_index , memorized_index, not_memorized_index -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs2-02.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "X played for Y . Therefore, What is the type of X? (a) Lawyer, (b) field hockey league, (c) military structure, (d) volleyball player, (e) parliament, (f) Producer, (g) philosopher, (h) politician, (i) road tunnel, (j) Capital of region, (k) video game, (l) brewery, (m) national soccer club, (n) railway tunnel, (o) murderer, (p) Globular Swarm, (q) Protocol, (r) person, (s) Nuclear Power plant, (t) geopolitical organisation", "gold": "(r) person"} 2 | {"prompt": "X is educated at Y . Therefore, What is the type of X? (a) arrondissement, (b) mountain pass, (c) chancellor, (d) sports team, (e) person, (f) egyptologist, (g) Rebbe, (h) digital camera, (i) classical music artist, (j) population, (k) subway station, (l) sea, (m) lock, (n) company, (o) light novel, (p) professor, (q) person function, (r) tennis tournament, (s) locomotive, (t) businessperson", "gold": "(e) person"} 3 | {"prompt": "X belongs to the party Y . Therefore, What is the type of X? (a) soccer player, (b) castle, (c) Sculpture, (d) military service, (e) road, (f) music genre, (g) canadian football Player, (h) hot spring, (i) person, (j) parliament, (k) race track, (l) businessperson, (m) painter, (n) term of office, (o) theatre, (p) blood vessel, (q) image, (r) soccer manager, (s) venue, (t) given name", "gold": "(i) person"} 4 | {"prompt": "X's highest point is Y . Therefore, What is the type of X? (a) canadian football Player, (b) Hormone, (c) embryology, (d) sports team, (e) district, (f) Satellite, (g) cultivated variety, (h) animanga character, (i) software, (j) rally driver, (k) earthquake, (l) protected area, (m) cycling team, (n) settlement, (o) shopping mall, (p) radio host, (q) train, (r) amusement park attraction, (s) painter, (t) Cipher", "gold": "(n) settlement"} 5 | -------------------------------------------------------------------------------- /scripts/reasoning/rdfs11.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BERT_LIST="bert-base-cased bert-base-uncased bert-large-cased bert-large-uncased" 3 | MODE="zs-f fs" 4 | KNOW="rdfs11-" 5 | PREM1="0 1 2" 6 | PREM2="0 1 2" 7 | SOFT="none load" 8 | 9 | for B in $BERT_LIST; 10 | do 11 | for m in $MODE; 12 | do 13 | python main.py --knowledge 'subClassOf' --model_class 'bert' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 14 | python main.py --knowledge 'subClassOf-type' --model_class 'bert' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 15 | done 16 | done 17 | 18 | for B in $BERT_LIST; 19 | do 20 | for m in $MODE; 21 | do 22 | for s in $SOFT; 23 | do 24 | for p1 in $PREM1; 25 | do 26 | for p2 in $PREM2; 27 | do 28 | python main.py --knowledge $KNOW$p1$p2 --model_class 'bert' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True 29 | done 30 | done 31 | done 32 | done 33 | done 34 | 35 | 36 | 37 | ROBERTA_LIST="roberta-base roberta-large" 38 | 39 | for B in $ROBERTA_LIST; 40 | do 41 | for m in $MODE; 42 | do 43 | python main.py --knowledge 'subClassOf' --model_class 'roberta' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 44 | python main.py --knowledge 'subClassOf-type' --model_class 'roberta' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 45 | done 46 | done 47 | 48 | for B in $ROBERTA_LIST; 49 | do 50 | for m in $MODE; 51 | do 52 | for s in $SOFT; 53 | do 54 | for p1 in $PREM1; 55 | do 56 | for p2 in $PREM2; 57 | do 58 | python main.py --knowledge $KNOW$p1$p2 --model_class 'roberta' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True 59 | done 60 | done 61 | done 62 | done 63 | done -------------------------------------------------------------------------------- /scripts/reasoning/rdfs5.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BERT_LIST="bert-base-cased bert-base-uncased bert-large-cased bert-large-uncased" 3 | MODE="zs-f fs" 4 | KNOW="rdfs5-" 5 | PREM1="0 1 2" 6 | PREM2="0 1 2" 7 | SOFT="none load" 8 | 9 | for B in $BERT_LIST; 10 | do 11 | for m in $MODE; 12 | do 13 | python main.py --knowledge 'subPropertyOf' --model_class 'bert' --model_path $B --template " implies" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 14 | python main.py --knowledge 'subPropertyOf-sub' --model_class 'bert' --model_path $B --template " implies" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 15 | done 16 | done 17 | 18 | for B in $BERT_LIST; 19 | do 20 | for m in $MODE; 21 | do 22 | for s in $SOFT; 23 | do 24 | for p1 in $PREM1; 25 | do 26 | for p2 in $PREM2; 27 | do 28 | python main.py --knowledge $KNOW$p1$p2 --model_class 'bert' --model_path $B --template " implies" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True --save_path "result1208.csv" 29 | done 30 | done 31 | done 32 | done 33 | done 34 | 35 | 36 | 37 | ROBERTA_LIST="roberta-base roberta-large" 38 | 39 | for B in $ROBERTA_LIST; 40 | do 41 | for m in $MODE; 42 | do 43 | python main.py --knowledge 'subPropertyOf' --model_class 'roberta' --model_path $B --template " implies" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 44 | python main.py --knowledge 'subPropertyOf-sub' --model_class 'roberta' --model_path $B --template " implies" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 45 | done 46 | done 47 | 48 | for B in $ROBERTA_LIST; 49 | do 50 | for m in $MODE; 51 | do 52 | for s in $SOFT; 53 | do 54 | for p1 in $PREM1; 55 | do 56 | for p2 in $PREM2; 57 | do 58 | python main.py --knowledge $KNOW$p1$p2 --model_class 'roberta' --model_path $B --template " implies" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True --save_path "result1208.csv" 59 | done 60 | done 61 | done 62 | done 63 | done -------------------------------------------------------------------------------- /scripts/reasoning/rdfs7.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BERT_LIST="bert-base-cased bert-base-uncased bert-large-cased bert-large-uncased" 3 | MODE="zs-f fs" 4 | KNOW="rdfs7-" 5 | PREM1="0 1 2" 6 | PREM2="0 1 2" 7 | SOFT="none load" 8 | 9 | for B in $BERT_LIST; 10 | do 11 | for m in $MODE; 12 | do 13 | python main.py --knowledge 'subPropertyOf' --model_class 'bert' --model_path $B --template " implies" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 14 | done 15 | python main.py --knowledge 'subPropertyOf-prop' --model_class 'bert' --model_path $B --template " implies" --multi_token_handler "mean" --mode "zs" --multi_mask "m" --is_premise True --use_cuda True 16 | done 17 | 18 | for B in $BERT_LIST; 19 | do 20 | for m in $MODE; 21 | do 22 | for s in $SOFT; 23 | do 24 | for p1 in $PREM1; 25 | do 26 | for p2 in $PREM2; 27 | do 28 | python main.py --knowledge $KNOW$p1$p2 --model_class 'bert' --model_path $B --template " implies" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True --save_path "result1208.csv" 29 | done 30 | done 31 | done 32 | done 33 | done 34 | 35 | 36 | 37 | ROBERTA_LIST="roberta-base roberta-large" 38 | 39 | for B in $ROBERTA_LIST; 40 | do 41 | for m in $MODE; 42 | do 43 | python main.py --knowledge 'subPropertyOf' --model_class 'roberta' --model_path $B --template " implies" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 44 | done 45 | python main.py --knowledge 'subPropertyOf-prop' --model_class 'roberta' --model_path $B --template " implies" --multi_token_handler "mean" --mode "zs" --multi_mask "m" --is_premise True --use_cuda True 46 | done 47 | 48 | for B in $ROBERTA_LIST; 49 | do 50 | for m in $MODE; 51 | do 52 | for s in $SOFT; 53 | do 54 | for p1 in $PREM1; 55 | do 56 | for p2 in $PREM2; 57 | do 58 | python main.py --knowledge $KNOW$p1$p2 --model_class 'roberta' --model_path $B --template " implies" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True --save_path "result1208.csv" 59 | done 60 | done 61 | done 62 | done 63 | done -------------------------------------------------------------------------------- /scripts/reasoning/rdfs9.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BERT_LIST="bert-base-cased bert-base-uncased bert-large-cased bert-large-uncased" 3 | MODE="zs-f fs" 4 | KNOW="rdfs9-" 5 | PREM1="0 1 2" 6 | PREM2="0 1 2" 7 | SOFT="none load" 8 | 9 | for B in $BERT_LIST; 10 | do 11 | for m in $MODE; 12 | do 13 | python main.py --knowledge 'subClassOf' --model_class 'bert' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 14 | python main.py --knowledge 'subClassOf-type' --model_class 'bert' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 15 | done 16 | done 17 | 18 | for B in $BERT_LIST; 19 | do 20 | for m in $MODE; 21 | do 22 | for s in $SOFT; 23 | do 24 | for p1 in $PREM1; 25 | do 26 | for p2 in $PREM2; 27 | do 28 | python main.py --knowledge $KNOW$p1$p2 --model_class 'bert' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True --save_path "result1208.csv" 29 | done 30 | done 31 | done 32 | done 33 | done 34 | 35 | 36 | 37 | ROBERTA_LIST="roberta-base roberta-large" 38 | 39 | for B in $ROBERTA_LIST; 40 | do 41 | for m in $MODE; 42 | do 43 | python main.py --knowledge 'subClassOf' --model_class 'roberta' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 44 | python main.py --knowledge 'subClassOf-type' --model_class 'roberta' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 45 | done 46 | done 47 | 48 | for B in $ROBERTA_LIST; 49 | do 50 | for m in $MODE; 51 | do 52 | for s in $SOFT; 53 | do 54 | for p1 in $PREM1; 55 | do 56 | for p2 in $PREM2; 57 | do 58 | python main.py --knowledge $KNOW$p1$p2 --model_class 'roberta' --model_path $B --template " is a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True --save_path "result1208.csv" 59 | done 60 | done 61 | done 62 | done 63 | done -------------------------------------------------------------------------------- /scripts/reasoning/rdfs3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BERT_LIST="bert-base-cased bert-base-uncased bert-large-cased bert-large-uncased" 3 | MODE="zs-f fs" 4 | KNOW="rdfs3-" 5 | PREM1="0 1 2" 6 | PREM2="0 1 2" 7 | SOFT="none load" 8 | 9 | for B in $BERT_LIST; 10 | do 11 | for m in $MODE; 12 | do 13 | python main.py --knowledge 'range' --model_class 'bert' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 14 | done 15 | python main.py --knowledge 'range-prop' --model_class 'bert' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode "zs" --multi_mask "m" --is_premise True --use_cuda True 16 | done 17 | 18 | for B in $BERT_LIST; 19 | do 20 | for m in $MODE; 21 | do 22 | for s in $SOFT; 23 | do 24 | for p1 in $PREM1; 25 | do 26 | for p2 in $PREM2; 27 | do 28 | python main.py --knowledge $KNOW$p1$p2 --model_class 'bert' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True --save_path "result1208.csv" 29 | done 30 | done 31 | done 32 | done 33 | done 34 | 35 | 36 | 37 | ROBERTA_LIST="roberta-base roberta-large" 38 | 39 | for B in $ROBERTA_LIST; 40 | do 41 | for m in $MODE; 42 | do 43 | python main.py --knowledge 'range' --model_class 'roberta' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 44 | done 45 | python main.py --knowledge 'range-prop' --model_class 'roberta' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode "zs" --multi_mask "m" --is_premise True --use_cuda True 46 | done 47 | 48 | for B in $ROBERTA_LIST; 49 | do 50 | for m in $MODE; 51 | do 52 | for s in $SOFT; 53 | do 54 | for p1 in $PREM1; 55 | do 56 | for p2 in $PREM2; 57 | do 58 | python main.py --knowledge $KNOW$p1$p2 --model_class 'roberta' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True --save_path "result1208.csv" 59 | done 60 | done 61 | done 62 | done 63 | done -------------------------------------------------------------------------------- /scripts/reasoning/rdfs2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BERT_LIST="bert-base-cased bert-base-uncased bert-large-cased bert-large-uncased" 3 | MODE="zs-f fs" 4 | KNOW="rdfs2-" 5 | PREM1="0 1 2" 6 | PREM2="0 1 2" 7 | SOFT="none load" 8 | 9 | for B in $BERT_LIST; 10 | do 11 | for m in $MODE; 12 | do 13 | python main.py --knowledge 'domain' --model_class 'bert' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 14 | done 15 | python main.py --knowledge 'domain-prop' --model_class 'bert' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode "zs" --multi_mask "m" --is_premise True --use_cuda True 16 | done 17 | 18 | for B in $BERT_LIST; 19 | do 20 | for m in $MODE; 21 | do 22 | for s in $SOFT; 23 | do 24 | for p1 in $PREM1; 25 | do 26 | for p2 in $PREM2; 27 | do 28 | python main.py --knowledge $KNOW$p1$p2 --model_class 'bert' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True --save_path "result1208.csv" 29 | done 30 | done 31 | done 32 | done 33 | done 34 | 35 | 36 | 37 | ROBERTA_LIST="roberta-base roberta-large" 38 | 39 | for B in $ROBERTA_LIST; 40 | do 41 | for m in $MODE; 42 | do 43 | python main.py --knowledge 'domain' --model_class 'roberta' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --is_premise True --use_cuda True 44 | done 45 | python main.py --knowledge 'domain-prop' --model_class 'roberta' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode "zs" --multi_mask "m" --is_premise True --use_cuda True 46 | done 47 | 48 | for B in $ROBERTA_LIST; 49 | do 50 | for m in $MODE; 51 | do 52 | for s in $SOFT; 53 | do 54 | for p1 in $PREM1; 55 | do 56 | for p2 in $PREM2; 57 | do 58 | python main.py --knowledge $KNOW$p1$p2 --model_class 'roberta' --model_path $B --template "One has to be a particular" --multi_token_handler "mean" --mode $m --multi_mask "m" --use_cuda True --trained_soft $s --is_premise True --save_path "result1208.csv" 59 | done 60 | done 61 | done 62 | done 63 | done 64 | -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs3-02.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "X was founded by Y . Therefore, What is the type of Y? (a) congressman, (b) Manor, (c) television episode, (d) martial artist, (e) farmer, (f) person, (g) Port, (h) decoration, (i) road junction, (j) basketball league, (k) saint, (l) HumanGene, (m) sports club, (n) Gated community, (o) Youtuber, (p) lacrosse player, (q) american football league, (r) wine, (s) government agency, (t) Pandemic", "gold": "(f) person"} 2 | {"prompt": "X used to work in Y . Therefore, What is the type of Y? (a) Quote, (b) space station, (c) hot spring, (d) paintball league, (e) drug, (f) criminal, (g) monarch, (h) chemical substance, (i) competition, (j) casino, (k) sport, (l) academic conference, (m) racecourse, (n) National anthem, (o) place, (p) atoll, (q) novel, (r) year, (s) soccer tournoment, (t) career station", "gold": "(o) place"} 3 | {"prompt": "X was born in Y . Therefore, What is the type of Y? (a) place, (b) hockey club, (c) battery, (d) journalist, (e) award, (f) golf player, (g) Archer Player, (h) lipid, (i) man, (j) motocycle racer, (k) arachnid, (l) Cipher, (m) Lawyer, (n) penalty shoot out, (o) speedway team, (p) murderer, (q) agglomeration, (r) poem, (s) mountain range, (t) football league season", "gold": "(a) place"} 4 | {"prompt": "X's author is Y . Therefore, What is the type of Y? (a) on site mean of transportation, (b) lock, (c) overseas department, (d) zoo, (e) television show, (f) person, (g) minister, (h) engine, (i) Reference, (j) office holder, (k) adult pornographic actor, (l) murderer, (m) volleyball player, (n) book, (o) asteroid, (p) Member of a Resistance Movement, (q) basketball team, (r) formula one racing, (s) Employer, (t) law firm", "gold": "(f) person"} 5 | {"prompt": "X inflows Y . Therefore, What is the type of Y? (a) body of water, (b) cross country skier, (c) manga, (d) canoeist, (e) soap character, (f) chemical compound, (g) ice hockey player, (h) Theatre director, (i) GeneLocation, (j) drug, (k) national football league event, (l) prison, (m) Cardinal direction, (n) digital camera, (o) still image, (p) Controlled designation of origin wine, (q) woman, (r) castle, (s) lake, (t) egyptologist", "gold": "(a) body of water"} 6 | {"prompt": "X's chief executive officer is Y . Therefore, What is the type of Y? (a) bone, (b) photographer, (c) constellation, (d) embryology, (e) football match, (f) linguist, (g) skater, (h) noble, (i) Browser, (j) person, (k) regency, (l) motocycle racer, (m) historical event, (n) magazine, (o) deputy, (p) cycling competition, (q) canoeist, (r) Employer, (s) team sport, (t) sports league", "gold": "(j) person"} 7 | -------------------------------------------------------------------------------- /scripts/memorizing/range.sh: -------------------------------------------------------------------------------- 1 | MODE="zs" 2 | HANDLER="mean max first" 3 | MASK="m s" 4 | 5 | for m in $MODE; 6 | do 7 | for MM in $MASK; 8 | do 9 | for h in $HANDLER; 10 | do 11 | python main.py --knowledge 'range' --model_class 'bert' --model_path 'bert-base-cased' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 12 | python main.py --knowledge 'range' --model_class 'bert' --model_path 'bert-base-uncased' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 13 | python main.py --knowledge 'range' --model_class 'bert' --model_path 'bert-large-cased' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 14 | python main.py --knowledge 'range' --model_class 'bert' --model_path 'bert-large-uncased' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 15 | python main.py --knowledge 'range' --model_class 'roberta' --model_path 'roberta-base' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 16 | python main.py --knowledge 'range' --model_class 'roberta' --model_path 'roberta-large' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 17 | done 18 | done 19 | done 20 | 21 | MODE="fs" 22 | LOSS="log NLL" 23 | for m in $MODE; 24 | do 25 | for MM in $MASK; 26 | do 27 | for h in $HANDLER; 28 | do 29 | for l in $LOSS; 30 | do 31 | python main.py --knowledge 'range' --model_class 'bert' --model_path 'bert-base-cased' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 32 | python main.py --knowledge 'range' --model_class 'bert' --model_path 'bert-base-uncased' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 33 | python main.py --knowledge 'range' --model_class 'bert' --model_path 'bert-large-cased' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 34 | python main.py --knowledge 'range' --model_class 'bert' --model_path 'bert-large-uncased' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 35 | python main.py --knowledge 'range' --model_class 'roberta' --model_path 'roberta-base' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 36 | python main.py --knowledge 'range' --model_class 'roberta' --model_path 'roberta-large' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 37 | done 38 | done 39 | done 40 | done 41 | -------------------------------------------------------------------------------- /scripts/memorizing/domain.sh: -------------------------------------------------------------------------------- 1 | MODE="zs" 2 | HANDLER="mean max first" 3 | MASK="m s" 4 | 5 | for m in $MODE; 6 | do 7 | for MM in $MASK; 8 | do 9 | for h in $HANDLER; 10 | do 11 | python main.py --knowledge 'domain' --model_class 'bert' --model_path 'bert-base-cased' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 12 | python main.py --knowledge 'domain' --model_class 'bert' --model_path 'bert-base-uncased' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 13 | python main.py --knowledge 'domain' --model_class 'bert' --model_path 'bert-large-cased' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 14 | python main.py --knowledge 'domain' --model_class 'bert' --model_path 'bert-large-uncased' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 15 | python main.py --knowledge 'domain' --model_class 'roberta' --model_path 'roberta-base' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 16 | python main.py --knowledge 'domain' --model_class 'roberta' --model_path 'roberta-large' --template "One has to be a particular" --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM 17 | done 18 | done 19 | done 20 | 21 | MODE="fs" 22 | LOSS="log NLL" 23 | for m in $MODE; 24 | do 25 | for MM in $MASK; 26 | do 27 | for h in $HANDLER; 28 | do 29 | for l in $LOSS; 30 | do 31 | python main.py --knowledge 'domain' --model_class 'bert' --model_path 'bert-base-cased' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 32 | python main.py --knowledge 'domain' --model_class 'bert' --model_path 'bert-base-uncased' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 33 | python main.py --knowledge 'domain' --model_class 'bert' --model_path 'bert-large-cased' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 34 | python main.py --knowledge 'domain' --model_class 'bert' --model_path 'bert-large-uncased' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 35 | python main.py --knowledge 'domain' --model_class 'roberta' --model_path 'roberta-base' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 36 | python main.py --knowledge 'domain' --model_class 'roberta' --model_path 'roberta-large' --template " " --use_cuda True --multi_token_handler $h --mode $m --multi_mask $MM --loss $l --soft_init "init" 37 | done 38 | done 39 | done 40 | done -------------------------------------------------------------------------------- /metric.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import jsonlines 3 | from get_memorized import split_by_Acc 4 | 5 | def evaluate_batch(Recall_1, Recall_5, Recall_10, Recall_25, Recall_50, mrr_avg, mrr, logits, batch_gt): 6 | Recall_1 += Recall_at_k(1, logits, batch_gt) 7 | Recall_5 += Recall_at_k(5, logits, batch_gt) 8 | Recall_10 += Recall_at_k(10, logits, batch_gt) 9 | Recall_25 += Recall_at_k(25, logits, batch_gt) 10 | Recall_50 += Recall_at_k(50, logits, batch_gt) 11 | mrr_avg += MRR_avg(logits, batch_gt) 12 | mrr += MRR(logits, batch_gt) 13 | 14 | return Recall_1, Recall_5, Recall_10, Recall_25, Recall_50, mrr_avg, mrr 15 | 16 | def MRR_avg(logits, ground_truth): 17 | score = 0.0 18 | result = torch.argsort(logits, dim=1, descending=True) 19 | result = torch.argsort(result, dim=1) 20 | for i in range(len(ground_truth)): 21 | indices = ground_truth[i] 22 | score += 1 / (torch.mean(result[i][indices].float()) + 1) 23 | 24 | return score 25 | 26 | def MRR(logits, ground_truth): 27 | score = 0.0 28 | result = torch.argsort(logits, dim=1, descending=True) 29 | result = torch.argsort(result, dim=1) 30 | for i in range(len(ground_truth)): 31 | indices = ground_truth[i] 32 | score += 1 / (torch.min(result[i][indices].float()) + 1) 33 | 34 | return score 35 | 36 | def Recall_at_k(k, logits, ground_truth): 37 | score = 0.0 38 | pred = torch.topk(logits, k, dim=1) 39 | indices = pred.indices.tolist() 40 | for i in range(len(ground_truth)): 41 | pred_cand = [indices[i][j] for j in range(k)] 42 | gts = ground_truth[i] 43 | for gt in gts: 44 | if gt in pred_cand: 45 | score += 1.0 46 | break 47 | return score 48 | 49 | def evaluate_batch_mul_choice(acc, cands_file, logits, batch_gt, cnt, batch_size, cand_ids, save_name): 50 | ''' 51 | Use a subset of candidates. 52 | Take the finest-grained gold as ground truth. 53 | Calcaulate Accuracy. 54 | ''' 55 | with jsonlines.open(cands_file, "r") as reader: 56 | data = list(reader) 57 | 58 | if '02' in save_name: 59 | selected = [data[i] for i in split_by_Acc("memorized_gpt/{}/{}.txt".format(cands_file.split('/')[-1].replace(".jsonl",""), save_name.split('/')[1]))[2]] 60 | data = selected 61 | elif '12' in save_name: 62 | selected = [data[i] for i in split_by_Acc("memorized_gpt/{}/{}.txt".format(cands_file.split('/')[-1].replace(".jsonl",""), save_name.split('/')[1]))[1]] 63 | data = selected 64 | 65 | for i in range(len(batch_gt)): 66 | index = [cand_ids[c] for c in data[cnt * batch_size + i]["cands"]] 67 | pred_cand = logits[i][index] 68 | max_index = torch.argmax(pred_cand) 69 | gt = batch_gt[i][0] 70 | # print(gt, index[max_index], index) 71 | if gt == index[max_index]: 72 | acc += 1.0 73 | return acc -------------------------------------------------------------------------------- /scripts/memorizing/type.sh: -------------------------------------------------------------------------------- 1 | IFS='|' 2 | TEMPLATE=" is a| has class| is a particular" 3 | HANDLER="mean|max|first" 4 | MASK="m|s" 5 | MODE="zs" 6 | for m in $MODE; 7 | do 8 | for MM in $MASK; 9 | do 10 | for h in $HANDLER; 11 | do 12 | for TEMP in $TEMPLATE; 13 | do 14 | python main.py --knowledge 'type' --model_class 'bert' --model_path 'bert-base-cased' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 15 | python main.py --knowledge 'type' --model_class 'bert' --model_path 'bert-base-uncased' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 16 | python main.py --knowledge 'type' --model_class 'bert' --model_path 'bert-large-cased' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 17 | python main.py --knowledge 'type' --model_class 'bert' --model_path 'bert-large-uncased' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 18 | python main.py --knowledge 'type' --model_class 'roberta' --model_path 'roberta-base' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 19 | python main.py --knowledge 'type' --model_class 'roberta' --model_path 'roberta-large' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 20 | done 21 | done 22 | done 23 | done 24 | 25 | MODE="fs" 26 | LOSS="log|NLL" 27 | for m in $MODE; 28 | do 29 | for MM in $MASK; 30 | do 31 | for h in $HANDLER; 32 | do 33 | for l in $LOSS; 34 | do 35 | python main.py --knowledge 'type' --model_class 'bert' --model_path 'bert-base-cased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 36 | python main.py --knowledge 'type' --model_class 'bert' --model_path 'bert-base-uncased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 37 | python main.py --knowledge 'type' --model_class 'bert' --model_path 'bert-large-cased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 38 | python main.py --knowledge 'type' --model_class 'bert' --model_path 'bert-large-uncased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 39 | python main.py --knowledge 'type' --model_class 'roberta' --model_path 'roberta-base' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 40 | python main.py --knowledge 'type' --model_class 'roberta' --model_path 'roberta-large' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 41 | done 42 | done 43 | done 44 | done -------------------------------------------------------------------------------- /scripts/memorizing/subPropertyOf.sh: -------------------------------------------------------------------------------- 1 | MODE="zs" 2 | HANDLER="mean max first" 3 | MASK="m s" 4 | 5 | for m in $MODE; 6 | do 7 | for MM in $MASK; 8 | do 9 | for h in $HANDLER; 10 | do 11 | python main.py --knowledge 'subPropertyOf' --model_class 'bert' --model_path 'bert-base-cased' --template " implies" --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 12 | python main.py --knowledge 'subPropertyOf' --model_class 'bert' --model_path 'bert-base-uncased' --template " implies" --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 13 | python main.py --knowledge 'subPropertyOf' --model_class 'bert' --model_path 'bert-large-cased' --template " implies" --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 14 | python main.py --knowledge 'subPropertyOf' --model_class 'bert' --model_path 'bert-large-uncased' --template " implies" --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 15 | python main.py --knowledge 'subPropertyOf' --model_class 'roberta' --model_path 'roberta-base' --template " implies" --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 16 | python main.py --knowledge 'subPropertyOf' --model_class 'roberta' --model_path 'roberta-large' --template " implies" --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 17 | done 18 | done 19 | done 20 | 21 | MODE="fs" 22 | LOSS="log NLL" 23 | for m in $MODE; 24 | do 25 | for MM in $MASK; 26 | do 27 | for h in $HANDLER; 28 | do 29 | for l in $LOSS; 30 | do 31 | python main.py --knowledge 'subPropertyOf' --model_class 'bert' --model_path 'bert-base-cased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 32 | python main.py --knowledge 'subPropertyOf' --model_class 'bert' --model_path 'bert-base-uncased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 33 | python main.py --knowledge 'subPropertyOf' --model_class 'bert' --model_path 'bert-large-cased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 34 | python main.py --knowledge 'subPropertyOf' --model_class 'bert' --model_path 'bert-large-uncased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 35 | python main.py --knowledge 'subPropertyOf' --model_class 'roberta' --model_path 'roberta-base' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 36 | python main.py --knowledge 'subPropertyOf' --model_class 'roberta' --model_path 'roberta-large' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 37 | done 38 | done 39 | done 40 | done -------------------------------------------------------------------------------- /scripts/memorizing/subClassOf.sh: -------------------------------------------------------------------------------- 1 | IFS='|' 2 | TEMPLATE=" is a| has class| is a particular" 3 | HANDLER="mean|max|first" 4 | MASK="m|s" 5 | MODE="zs" 6 | 7 | for m in $MODE; 8 | do 9 | for MM in $MASK; 10 | do 11 | for h in $HANDLER; 12 | do 13 | for TEMP in $TEMPLATE; 14 | do 15 | python main.py --knowledge 'subClassOf' --model_class 'bert' --model_path 'bert-base-cased' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 16 | python main.py --knowledge 'subClassOf' --model_class 'bert' --model_path 'bert-base-uncased' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 17 | python main.py --knowledge 'subClassOf' --model_class 'bert' --model_path 'bert-large-cased' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 18 | python main.py --knowledge 'subClassOf' --model_class 'bert' --model_path 'bert-large-uncased' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 19 | python main.py --knowledge 'subClassOf' --model_class 'roberta' --model_path 'roberta-base' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 20 | python main.py --knowledge 'subClassOf' --model_class 'roberta' --model_path 'roberta-large' --template $TEMP --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM 21 | done 22 | done 23 | done 24 | done 25 | 26 | MODE="fs" 27 | LOSS="log|NLL" 28 | for m in $MODE; 29 | do 30 | for MM in $MASK; 31 | do 32 | for h in $HANDLER; 33 | do 34 | for l in $LOSS; 35 | do 36 | python main.py --knowledge 'subClassOf' --model_class 'bert' --model_path 'bert-base-cased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 37 | python main.py --knowledge 'subClassOf' --model_class 'bert' --model_path 'bert-base-uncased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 38 | python main.py --knowledge 'subClassOf' --model_class 'bert' --model_path 'bert-large-cased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 39 | python main.py --knowledge 'subClassOf' --model_class 'bert' --model_path 'bert-large-uncased' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 40 | python main.py --knowledge 'subClassOf' --model_class 'roberta' --model_path 'roberta-base' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 41 | python main.py --knowledge 'subClassOf' --model_class 'roberta' --model_path 'roberta-large' --template " " --use_cuda True --multi_token_handler $h --save_path "result1205.csv" --mode $m --multi_mask $MM --loss $l --soft_init "init" 42 | done 43 | done 44 | done 45 | done -------------------------------------------------------------------------------- /data/memorizing/range.jsonl: -------------------------------------------------------------------------------- 1 | {"uuu": {"place of death": "[X] died in [Y] ."}, "xxx": ["place"]} 2 | {"uuu": {"architect": "[X]'s architect is [Y] ."}, "xxx": ["architect"]} 3 | {"uuu": {"highest point": "[X]'s highest point is [Y] ."}, "xxx": ["place"]} 4 | {"uuu": {"has subsidiary": "[X]'s subsidiary is [Y] ."}, "xxx": ["company"]} 5 | {"uuu": {"member of sports team": "[X] played for [Y] ."}, "xxx": ["sports team"]} 6 | {"uuu": {"rector": "[X]'s rector is [Y] ."}, "xxx": ["person"]} 7 | {"uuu": {"based on": "[X] is based on [Y] ."}, "xxx": ["work"]} 8 | {"uuu": {"continent": "[X] is located on the continent of [Y] ."}, "xxx": ["continent"]} 9 | {"uuu": {"country of citizenship": "[X] is a citizen of [Y] ."}, "xxx": ["country"]} 10 | {"uuu": {"head of government": "[X]'s government is headed by [Y] ."}, "xxx": ["person"]} 11 | {"uuu": {"judge": "[X]'s judge is [Y] ."}, "xxx": ["judge"]} 12 | {"uuu": {"residence": "[X] lives in [Y] ."}, "xxx": ["place"]} 13 | {"uuu": {"place of burial": "[X] is buried in [Y] ."}, "xxx": ["place"]} 14 | {"uuu": {"educated at": "[X] is educated at [Y] ."}, "xxx": ["educational institution"]} 15 | {"uuu": {"illustrator": "[X] is illustrated by [Y] ."}, "xxx": ["person"]} 16 | {"uuu": {"conflict": "[X] participated in conflict [Y] ."}, "xxx": ["military conflict"]} 17 | {"uuu": {"headquarters location": "[X]'s headquarter is in [Y] ."}, "xxx": ["populated place"]} 18 | {"uuu": {"curator": "[X]'s curator is [Y] ."}, "xxx": ["person"]} 19 | {"uuu": {"military branch": "[X]'s service branch is [Y] ."}, "xxx": ["military unit"]} 20 | {"uuu": {"official language": "[X]'s offical language is [Y] ."}, "xxx": ["language"]} 21 | {"uuu": {"employer": "[X] is an employer of [Y] ."}, "xxx": ["organisation"]} 22 | {"uuu": {"composer": "[X]'s composer is [Y] ."}, "xxx": ["person"]} 23 | {"uuu": {"doctoral student": "[X]'s doctoral student is [Y] ."}, "xxx": ["person"]} 24 | {"uuu": {"founded by": "[X] was founded by [Y] ."}, "xxx": ["person"]} 25 | {"uuu": {"successful candidate": "[X]'s successful candidate is [Y] ."}, "xxx": ["person"]} 26 | {"uuu": {"father": "[X]'s father is [Y] ."}, "xxx": ["man"]} 27 | {"uuu": {"work location": "[X] used to work in [Y] ."}, "xxx": ["place"]} 28 | {"uuu": {"mother": "[X]'s mother is [Y] ."}, "xxx": ["woman"]} 29 | {"uuu": {"student": "[X]'s student is [Y] ."}, "xxx": ["person"]} 30 | {"uuu": {"presenter": "[X] is hosted by [Y] ."}, "xxx": ["person"]} 31 | {"uuu": {"killed by": "[X] was killed by [Y] ."}, "xxx": ["person"]} 32 | {"uuu": {"place of birth": "[X] was born in [Y] ."}, "xxx": ["place"]} 33 | {"uuu": {"author": "[X]'s author is [Y] ."}, "xxx": ["person"]} 34 | {"uuu": {"dialect of": "[X] is a dialect of [Y] ."}, "xxx": ["language"]} 35 | {"uuu": {"has melody": "[X] has the melody of [Y] ."}, "xxx": ["musical work"]} 36 | {"uuu": {"constellation": "[X] is part of constellation [Y] ."}, "xxx": ["constellation"]} 37 | {"uuu": {"parliamentary group": "[X] belongs to the party [Y] ."}, "xxx": ["political party"]} 38 | {"uuu": {"located in the administrative territorial entity": "[X] is located in the administrative territory of [Y] ."}, "xxx": ["administrative region"]} 39 | {"uuu": {"child": "[X]'s child is [Y] ."}, "xxx": ["person"]} 40 | {"uuu": {"developer": "[X]'s developer is [Y] ."}, "xxx": ["agent"]} 41 | {"uuu": {"participating team": "[X]'s participating teams contain [Y] ."}, "xxx": ["sports team"]} 42 | {"uuu": {"inflows": "[X] inflows [Y] ."}, "xxx": ["body of water"]} 43 | {"uuu": {"home venue": "[X]'s home stadium is [Y] ."}, "xxx": ["architectural structure"]} 44 | {"uuu": {"cast member": "[X] has cast member [Y] ."}, "xxx": ["actor"]} 45 | {"uuu": {"spouse": "[X]'s spouse is [Y] ."}, "xxx": ["person"]} 46 | {"uuu": {"original language": "[X] was originally created in [Y] ."}, "xxx": ["language"]} 47 | {"uuu": {"chief executive officer": "[X]'s chief executive officer is [Y] ."}, "xxx": ["person"]} 48 | {"uuu": {"home port": "[X]'s home port is [Y] ."}, "xxx": ["place"]} -------------------------------------------------------------------------------- /data/memorizing/domain.jsonl: -------------------------------------------------------------------------------- 1 | {"uuu": {"spouse": "[X]'s spouse is [Y] ."}, "xxx": ["person"]} 2 | {"uuu": {"inflows": "[X] inflows [Y] ."}, "xxx": ["body of water"]} 3 | {"uuu": {"father": "[X]'s father is [Y] ."}, "xxx": ["person"]} 4 | {"uuu": {"head of government": "[X]'s government is headed by [Y] ."}, "xxx": ["country"]} 5 | {"uuu": {"home venue": "[X]'s home stadium is [Y] ."}, "xxx": ["sports team"]} 6 | {"uuu": {"architect": "[X]'s architect is [Y] ."}, "xxx": ["architectural structure"]} 7 | {"uuu": {"developer": "[X]'s developer is [Y] ."}, "xxx": ["work"]} 8 | {"uuu": {"home port": "[X]'s home port is [Y] ."}, "xxx": ["ship"]} 9 | {"uuu": {"author": "[X]'s author is [Y] ."}, "xxx": ["work"]} 10 | {"uuu": {"judge": "[X]'s judge is [Y] ."}, "xxx": ["Legal Case"]} 11 | {"uuu": {"based on": "[X] is based on [Y] ."}, "xxx": ["work"]} 12 | {"uuu": {"family": "[X]'s family is [Y] ."}, "xxx": ["person"]} 13 | {"uuu": {"presenter": "[X] is hosted by [Y] ."}, "xxx": ["television show"]} 14 | {"uuu": {"chief executive officer": "[X]'s chief executive officer is [Y] ."}, "xxx": ["organisation"]} 15 | {"uuu": {"has subsidiary": "[X]'s subsidiary is [Y] ."}, "xxx": ["company"]} 16 | {"uuu": {"doctoral student": "[X]'s doctoral student is [Y] ."}, "xxx": ["scientist"]} 17 | {"uuu": {"place of death": "[X] died in [Y] ."}, "xxx": ["animal"]} 18 | {"uuu": {"industry": "[X]'s specific industry is [Y] ."}, "xxx": ["company"]} 19 | {"uuu": {"constellation": "[X] is part of constellation [Y] ."}, "xxx": ["celestial body"]} 20 | {"uuu": {"illustrator": "[X] is illustrated by [Y] ."}, "xxx": ["written work"]} 21 | {"uuu": {"composer": "[X]'s composer is [Y] ."}, "xxx": ["work"]} 22 | {"uuu": {"employer": "[X] is an employer of [Y] ."}, "xxx": ["person"]} 23 | {"uuu": {"curator": "[X]'s curator is [Y] ."}, "xxx": ["museum"]} 24 | {"uuu": {"place of burial": "[X] is buried in [Y] ."}, "xxx": ["person"]} 25 | {"uuu": {"country of citizenship": "[X] is a citizen of [Y] ."}, "xxx": ["person"]} 26 | {"uuu": {"member of sports team": "[X] played for [Y] ."}, "xxx": ["person"]} 27 | {"uuu": {"original language": "[X] was originally created in [Y] ."}, "xxx": ["work"]} 28 | {"uuu": {"mother": "[X]'s mother is [Y] ."}, "xxx": ["person"]} 29 | {"uuu": {"dialect of": "[X] is a dialect of [Y] ."}, "xxx": ["language"]} 30 | {"uuu": {"place of birth": "[X] was born in [Y] ."}, "xxx": ["animal"]} 31 | {"uuu": {"rector": "[X]'s rector is [Y] ."}, "xxx": ["educational institution"]} 32 | {"uuu": {"headquarters location": "[X]'s headquarter is in [Y] ."}, "xxx": ["organisation"]} 33 | {"uuu": {"residence": "[X] lives in [Y] ."}, "xxx": ["person"]} 34 | {"uuu": {"has melody": "[X] has the melody of [Y] ."}, "xxx": ["work"]} 35 | {"uuu": {"military branch": "[X]'s service branch is [Y] ."}, "xxx": ["military person"]} 36 | {"uuu": {"educated at": "[X] is educated at [Y] ."}, "xxx": ["person"]} 37 | {"uuu": {"successful candidate": "[X]'s successful candidate is [Y] ."}, "xxx": ["election"]} 38 | {"uuu": {"continent": "[X] is located on the continent of [Y] ."}, "xxx": ["country"]} 39 | {"uuu": {"official language": "[X]'s offical language is [Y] ."}, "xxx": ["populated place"]} 40 | {"uuu": {"conflict": "[X] participated in conflict [Y] ."}, "xxx": ["person"]} 41 | {"uuu": {"parliamentary group": "[X] belongs to the party [Y] ."}, "xxx": ["person"]} 42 | {"uuu": {"killed by": "[X] was killed by [Y] ."}, "xxx": ["person"]} 43 | {"uuu": {"founded by": "[X] was founded by [Y] ."}, "xxx": ["organisation"]} 44 | {"uuu": {"streak color": "[X]'s streak color is [Y] ."}, "xxx": ["mineral"]} 45 | {"uuu": {"student": "[X]'s student is [Y] ."}, "xxx": ["person"]} 46 | {"uuu": {"child": "[X]'s child is [Y] ."}, "xxx": ["person"]} 47 | {"uuu": {"work location": "[X] used to work in [Y] ."}, "xxx": ["person"]} 48 | {"uuu": {"highest point": "[X]'s highest point is [Y] ."}, "xxx": ["settlement"]} 49 | {"uuu": {"cast member": "[X] has cast member [Y] ."}, "xxx": ["work"]} 50 | {"uuu": {"participating team": "[X]'s participating teams contain [Y] ."}, "xxx": ["sports event"]} -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs5-02.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "X implies successful candidate. Therefore, What does X imply? (a) continent, (b) founded by, (c) inspired by, (d) father, (e) home venue, (f) participant in, (g) country, (h) based on, (i) has subsidiary, (j) corporate officer, (k) participating team, (l) head of government, (m) place of death, (n) winner, (o) owner of, (p) candidate, (q) highest point, (r) participant, (s) location, (t) parent", "gold": "(n) winner"} 2 | {"prompt": "X implies dialect of. Therefore, What does X imply? (a) derivative work, (b) references work, tradition or theory, (c) head of government, (d) curator, (e) significant event, (f) developer, (g) rector, (h) judge, (i) chairperson, (j) student, (k) employer, (l) chief executive officer, (m) place of burial, (n) headquarters location, (o) work location, (p) continent, (q) official language, (r) parliamentary group, (s) residence, (t) conflict", "gold": "(a) derivative work"} 3 | {"prompt": "X implies designed by. Therefore, What does X imply? (a) creator, (b) parliamentary group, (c) member of, (d) developer, (e) relative, (f) located in on physical feature, (g) successful candidate, (h) conflict, (i) killed by, (j) affiliation, (k) author, (l) work location, (m) original language, (n) participant in, (o) judge, (p) location, (q) family, (r) place of burial, (s) headquarters location, (t) industry", "gold": "(a) creator"} 4 | {"prompt": "X implies rector. Therefore, What does X imply? (a) located in on physical feature, (b) streak color, (c) original language, (d) director manager, (e) illustrator, (f) conflict, (g) residence, (h) color, (i) head of government, (j) participating team, (k) place of burial, (l) military branch, (m) original broadcaster, (n) industry, (o) curator, (p) relative, (q) chief executive officer, (r) presenter, (s) significant event, (t) developer", "gold": "(d) director manager"} 5 | {"prompt": "X implies educated at. Therefore, What does X imply? (a) successful candidate, (b) based on, (c) creator, (d) place of burial, (e) participant, (f) affiliation, (g) architect, (h) member of, (i) parent, (j) has parts, (k) owner of, (l) has melody, (m) references work, tradition or theory, (n) cast member, (o) employer, (p) parliamentary group, (q) constellation, (r) student, (s) winner, (t) rector", "gold": "(f) affiliation"} 6 | {"prompt": "X implies cast member. Therefore, What does X imply? (a) significant event, (b) parent, (c) headquarters location, (d) location, (e) chief executive officer, (f) significant person, (g) language used, (h) located in the administrative territorial entity, (i) author, (j) located in on physical feature, (k) candidate, (l) head of government, (m) inflows, (n) chairperson, (o) place of burial, (p) educated at, (q) contributor to the creative work or subject, (r) designed by, (s) conflict, (t) has subsidiary", "gold": "(f) significant person"} 7 | {"prompt": "X implies curator. Therefore, What does X imply? (a) corporate officer, (b) affiliation, (c) original language, (d) location, (e) spouse, (f) official language, (g) member of sports team, (h) rector, (i) language of work or name, (j) successful candidate, (k) architect, (l) dialect of, (m) industry, (n) significant person, (o) original broadcaster, (p) part of, (q) participating team, (r) color, (s) employer, (t) mother", "gold": "(n) significant person"} 8 | {"prompt": "X implies participant. Therefore, What does X imply? (a) child, (b) father, (c) corporate officer, (d) significant place, (e) located in on physical feature, (f) relative, (g) has subsidiary, (h) owner of, (i) significant person, (j) employer, (k) place of birth, (l) mother, (m) official language, (n) winner, (o) performer, (p) original language, (q) language of work or name, (r) significant event, (s) judge, (t) member of", "gold": "(i) significant person"} 9 | {"prompt": "X implies architect. Therefore, What does X imply? (a) father, (b) located in on physical feature, (c) references work, tradition or theory, (d) significant event, (e) member of sports team, (f) military branch, (g) place of birth, (h) cast member, (i) designed by, (j) significant person, (k) affiliation, (l) performer, (m) relative, (n) successful candidate, (o) place of burial, (p) conflict, (q) curator, (r) mother, (s) language used, (t) parliamentary group", "gold": "(i) designed by"} 10 | {"prompt": "X implies contributor. Therefore, What does X imply? (a) has parts, (b) participant, (c) founded by, (d) affiliation, (e) color, (f) significant person, (g) creator, (h) author, (i) place of death, (j) director manager, (k) corporate officer, (l) head of government, (m) family, (n) student, (o) derivative work, (p) employer, (q) constellation, (r) home venue, (s) cast member, (t) owner of", "gold": "(b) participant"} 11 | {"prompt": "X implies constellation. Therefore, What does X imply? (a) highest point, (b) chairperson, (c) headquarters location, (d) continent, (e) student, (f) performer, (g) significant place, (h) child, (i) successful candidate, (j) has parts, (k) developer, (l) author, (m) references work, tradition or theory, (n) part of, (o) inspired by, (p) owner of, (q) residence, (r) parliamentary group, (s) director manager, (t) participant", "gold": "(n) part of"} 12 | -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs7-02.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "X's successful candidate is Y . Therefore, What is the relation between X and Y? (a) continent, (b) founded by, (c) inspired by, (d) father, (e) home venue, (f) participant in, (g) country, (h) based on, (i) has subsidiary, (j) corporate officer, (k) participating team, (l) head of government, (m) place of death, (n) winner, (o) owner of, (p) candidate, (q) highest point, (r) participant, (s) location, (t) parent", "gold": "(n) winner"} 2 | {"prompt": "X is a dialect of Y . Therefore, What is the relation between X and Y? (a) derivative work, (b) references work, tradition or theory, (c) head of government, (d) curator, (e) significant event, (f) developer, (g) rector, (h) judge, (i) chairperson, (j) student, (k) employer, (l) chief executive officer, (m) place of burial, (n) headquarters location, (o) work location, (p) continent, (q) official language, (r) parliamentary group, (s) residence, (t) conflict", "gold": "(a) derivative work"} 3 | {"prompt": "X is designed by Y . Therefore, What is the relation between X and Y? (a) creator, (b) parliamentary group, (c) member of, (d) developer, (e) relative, (f) located in on physical feature, (g) successful candidate, (h) conflict, (i) killed by, (j) affiliation, (k) author, (l) work location, (m) original language, (n) participant in, (o) judge, (p) location, (q) family, (r) place of burial, (s) headquarters location, (t) industry", "gold": "(a) creator"} 4 | {"prompt": "X's rector is Y . Therefore, What is the relation between X and Y? (a) located in on physical feature, (b) streak color, (c) original language, (d) director manager, (e) illustrator, (f) conflict, (g) residence, (h) color, (i) head of government, (j) participating team, (k) place of burial, (l) military branch, (m) original broadcaster, (n) industry, (o) curator, (p) relative, (q) chief executive officer, (r) presenter, (s) significant event, (t) developer", "gold": "(d) director manager"} 5 | {"prompt": "X is educated at Y . Therefore, What is the relation between X and Y? (a) successful candidate, (b) based on, (c) creator, (d) place of burial, (e) participant, (f) affiliation, (g) architect, (h) member of, (i) parent, (j) has parts, (k) owner of, (l) has melody, (m) references work, tradition or theory, (n) cast member, (o) employer, (p) parliamentary group, (q) constellation, (r) student, (s) winner, (t) rector", "gold": "(f) affiliation"} 6 | {"prompt": "X's cast member is Y . Therefore, What is the relation between X and Y? (a) significant event, (b) parent, (c) headquarters location, (d) location, (e) chief executive officer, (f) significant person, (g) language used, (h) located in the administrative territorial entity, (i) author, (j) located in on physical feature, (k) candidate, (l) head of government, (m) inflows, (n) chairperson, (o) place of burial, (p) educated at, (q) contributor to the creative work or subject, (r) designed by, (s) conflict, (t) has subsidiary", "gold": "(f) significant person"} 7 | {"prompt": "X's curator is Y . Therefore, What is the relation between X and Y? (a) corporate officer, (b) affiliation, (c) original language, (d) location, (e) spouse, (f) official language, (g) member of sports team, (h) rector, (i) language of work or name, (j) successful candidate, (k) architect, (l) dialect of, (m) industry, (n) significant person, (o) original broadcaster, (p) part of, (q) participating team, (r) color, (s) employer, (t) mother", "gold": "(n) significant person"} 8 | {"prompt": "X's participant is Y . Therefore, What is the relation between X and Y? (a) child, (b) father, (c) corporate officer, (d) significant place, (e) located in on physical feature, (f) relative, (g) has subsidiary, (h) owner of, (i) significant person, (j) employer, (k) place of birth, (l) mother, (m) official language, (n) winner, (o) performer, (p) original language, (q) language of work or name, (r) significant event, (s) judge, (t) member of", "gold": "(i) significant person"} 9 | {"prompt": "X's architect is Y . Therefore, What is the relation between X and Y? (a) father, (b) located in on physical feature, (c) references work, tradition or theory, (d) significant event, (e) member of sports team, (f) military branch, (g) place of birth, (h) cast member, (i) designed by, (j) significant person, (k) affiliation, (l) performer, (m) relative, (n) successful candidate, (o) place of burial, (p) conflict, (q) curator, (r) mother, (s) language used, (t) parliamentary group", "gold": "(i) designed by"} 10 | {"prompt": "X's contributor is Y . Therefore, What is the relation between X and Y? (a) has parts, (b) participant, (c) founded by, (d) affiliation, (e) color, (f) significant person, (g) creator, (h) author, (i) place of death, (j) director manager, (k) corporate officer, (l) head of government, (m) family, (n) student, (o) derivative work, (p) employer, (q) constellation, (r) home venue, (s) cast member, (t) owner of", "gold": "(b) participant"} 11 | {"prompt": "X is part of constellation Y . Therefore, What is the relation between X and Y? (a) highest point, (b) chairperson, (c) headquarters location, (d) continent, (e) student, (f) performer, (g) significant place, (h) child, (i) successful candidate, (j) has parts, (k) developer, (l) author, (m) references work, tradition or theory, (n) part of, (o) inspired by, (p) owner of, (q) residence, (r) parliamentary group, (s) director manager, (t) participant", "gold": "(n) part of"} 12 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def load_soft_emb_without_conj(template, save_name, device): 4 | premise_mode = (save_name.split('/')[0]).split("-")[1] 5 | emb_arg = save_name.split('/')[1].split(".") 6 | emb_path = ".".join([emb_arg[0], "fs", emb_arg[2], emb_arg[3], "log"]) 7 | 8 | soft_type = torch.load('soft_embedding/type/{}.pth'.format(emb_path), map_location=device)[1:4] 9 | soft_subclass = torch.load('soft_embedding/subClassOf/{}.pth'.format(emb_path), map_location=device)[1:4] 10 | soft_subprop = torch.load('soft_embedding/subPropertyOf/{}.pth'.format(emb_path), map_location=device)[1:4] 11 | soft_domain = torch.load('soft_embedding/domain/{}.pth'.format(emb_path), map_location=device)[1:4] 12 | soft_range = torch.load('soft_embedding/range/{}.pth'.format(emb_path), map_location=device)[1:4] 13 | 14 | if "rdfs2" in save_name: 15 | if premise_mode[0] == '2': 16 | template.soft_embedding.weight.data[1:4] = soft_domain 17 | template.soft_embedding.weight.data[4:7] = soft_type 18 | else: 19 | template.soft_embedding.weight.data[1:4] = soft_type 20 | elif "rdfs3" in save_name: 21 | if premise_mode[0] == '2': 22 | template.soft_embedding.weight.data[1:4] = soft_range 23 | template.soft_embedding.weight.data[4:7] = soft_type 24 | else: 25 | template.soft_embedding.weight.data[1:4] = soft_type 26 | elif "rdfs5" in save_name: 27 | if premise_mode[0] == '2' and premise_mode[1] == '2': 28 | template.soft_embedding.weight.data[1:4] = soft_subprop 29 | template.soft_embedding.weight.data[4:7] = soft_subprop 30 | template.soft_embedding.weight.data[7:10] = soft_subprop 31 | elif premise_mode[0] == '2' and premise_mode[1] != '2': 32 | template.soft_embedding.weight.data[1:4] = soft_subprop 33 | template.soft_embedding.weight.data[4:7] = soft_subprop 34 | else: 35 | template.soft_embedding.weight.data[1:4] = soft_subprop 36 | elif "rdfs7" in save_name: 37 | if premise_mode[0] == '2': 38 | template.soft_embedding.weight.data[1:4] = soft_subprop 39 | elif "rdfs9" in save_name: 40 | if premise_mode[0] == '2' and premise_mode[1] == '2': 41 | template.soft_embedding.weight.data[1:4] = soft_subclass 42 | template.soft_embedding.weight.data[4:7] = soft_type 43 | template.soft_embedding.weight.data[7:10] = soft_type 44 | elif premise_mode[0] == '2' and premise_mode[1] != '2': 45 | template.soft_embedding.weight.data[1:4] = soft_subclass 46 | template.soft_embedding.weight.data[4:7] = soft_type 47 | elif premise_mode[0] != '2' and premise_mode[1] == '2': 48 | template.soft_embedding.weight.data[1:4] = soft_type 49 | template.soft_embedding.weight.data[4:7] = soft_type 50 | else: 51 | template.soft_embedding.weight.data[1:4] = soft_type 52 | elif "rdfs11" in save_name: 53 | if premise_mode[0] == '2' and premise_mode[1] == '2': 54 | template.soft_embedding.weight.data[1:4] = soft_subclass 55 | template.soft_embedding.weight.data[4:7] = soft_subclass 56 | template.soft_embedding.weight.data[7:10] = soft_subclass 57 | elif premise_mode[0] == '2' and premise_mode[1] != '2': 58 | template.soft_embedding.weight.data[1:4] = soft_subclass 59 | template.soft_embedding.weight.data[4:7] = soft_subclass 60 | else: 61 | template.soft_embedding.weight.data[1:4] = soft_subclass 62 | return template 63 | 64 | def load_soft_emb_with_conj(template, save_name, device): 65 | premise_mode = (save_name.split('/')[0]).split("-")[1] 66 | emb_arg = save_name.split('/')[1].split(".") 67 | emb_path = ".".join([emb_arg[0], "fs", emb_arg[2], emb_arg[3], "log"]) 68 | 69 | soft_type = torch.load('soft_embedding/type/{}.pth'.format(emb_path), map_location=device)[1:4] 70 | soft_subclass = torch.load('soft_embedding/subClassOf/{}.pth'.format(emb_path), map_location=device)[1:4] 71 | soft_subprop = torch.load('soft_embedding/subPropertyOf/{}.pth'.format(emb_path), map_location=device)[1:4] 72 | soft_domain = torch.load('soft_embedding/domain/{}.pth'.format(emb_path), map_location=device)[1:4] 73 | soft_range = torch.load('soft_embedding/range/{}.pth'.format(emb_path), map_location=device)[1:4] 74 | 75 | if "rdfs2" in save_name: 76 | if premise_mode[0] == '2' and premise_mode[1] == '2': 77 | template.soft_embedding.weight.data[1:4] = soft_domain 78 | template.soft_embedding.weight.data[6:9] = soft_type 79 | elif premise_mode[0] == '2' and premise_mode[1] != '2': 80 | template.soft_embedding.weight.data[1:4] = soft_domain 81 | template.soft_embedding.weight.data[5:8] = soft_type 82 | else: 83 | template.soft_embedding.weight.data[2:5] = soft_type 84 | elif "rdfs3" in save_name: 85 | if premise_mode[0] == '2' and premise_mode[1] == '2': 86 | template.soft_embedding.weight.data[1:4] = soft_range 87 | template.soft_embedding.weight.data[6:9] = soft_type 88 | elif premise_mode[0] == '2' and premise_mode[1] != '2': 89 | template.soft_embedding.weight.data[1:4] = soft_range 90 | template.soft_embedding.weight.data[5:8] = soft_type 91 | else: 92 | template.soft_embedding.weight.data[2:5] = soft_type 93 | elif "rdfs5" in save_name: 94 | if premise_mode[0] == '2' and premise_mode[1] == '2': 95 | template.soft_embedding.weight.data[1:4] = soft_subprop 96 | template.soft_embedding.weight.data[5:8] = soft_subprop 97 | template.soft_embedding.weight.data[9:12] = soft_subprop 98 | elif premise_mode[0] == '2' and premise_mode[1] != '2': 99 | template.soft_embedding.weight.data[1:4] = soft_subprop 100 | template.soft_embedding.weight.data[5:8] = soft_subprop 101 | else: 102 | template.soft_embedding.weight.data[2:5] = soft_subprop 103 | elif "rdfs7" in save_name: 104 | if premise_mode[0] == '2': 105 | template.soft_embedding.weight.data[1:4] = soft_subprop 106 | elif "rdfs9" in save_name: 107 | if premise_mode[0] == '2' and premise_mode[1] == '2': 108 | template.soft_embedding.weight.data[1:4] = soft_subclass 109 | template.soft_embedding.weight.data[5:8] = soft_type 110 | template.soft_embedding.weight.data[9:12] = soft_type 111 | elif premise_mode[0] == '2' and premise_mode[1] != '2': 112 | template.soft_embedding.weight.data[1:4] = soft_subclass 113 | template.soft_embedding.weight.data[5:8] = soft_type 114 | elif premise_mode[0] != '2' and premise_mode[1] == '2': 115 | template.soft_embedding.weight.data[1:4] = soft_type 116 | template.soft_embedding.weight.data[5:8] = soft_type 117 | else: 118 | template.soft_embedding.weight.data[2:5] = soft_type 119 | elif "rdfs11" in save_name: 120 | if premise_mode[0] == '2' and premise_mode[1] == '2': 121 | template.soft_embedding.weight.data[1:4] = soft_subclass 122 | template.soft_embedding.weight.data[5:8] = soft_subclass 123 | template.soft_embedding.weight.data[9:12] = soft_subclass 124 | elif premise_mode[0] == '2' and premise_mode[1] != '2': 125 | template.soft_embedding.weight.data[1:4] = soft_subclass 126 | template.soft_embedding.weight.data[5:8] = soft_subclass 127 | else: 128 | template.soft_embedding.weight.data[2:5] = soft_subclass 129 | return template -------------------------------------------------------------------------------- /data/memorizing/subPropertyOf.jsonl: -------------------------------------------------------------------------------- 1 | {"uuu": {"child": "[X]'s child is [Y] ."}, "xxx": {"relative": "[X]'s relative is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 2 | {"uuu": {"presenter": "[X] is hosted by [Y] ."}, "xxx": {"cast member": "[X]'s cast member is [Y] .", "performer": "[X]'s performer is [Y] .", "contributor to the creative work or subject": "[X]'s contributor is [Y] .", "participant": "[X]'s participant is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 3 | {"uuu": {"relative": "[X]'s relative is [Y] ."}, "xxx": {"significant person": "[X]'s significant person is [Y] ."}} 4 | {"uuu": {"has subsidiary": "[X]'s subsidiary is [Y] ."}, "xxx": {"owner of": "[X] is the owner of [Y] .", "has parts": "[X] has part [Y] ."}} 5 | {"uuu": {"place of death": "[X] died in [Y] ."}, "xxx": {"location": "[X] is located in [Y] ."}} 6 | {"uuu": {"cast member": "[X] has cast member [Y] ."}, "xxx": {"performer": "[X]'s performer is [Y] .", "contributor to the creative work or subject": "[X]'s contributor is [Y] .", "participant": "[X]'s participant is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 7 | {"uuu": {"parent": "[X]'s parent is [Y] ."}, "xxx": {"relative": "[X]'s relative is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 8 | {"uuu": {"student": "[X]'s student is [Y] ."}, "xxx": {"significant person": "[X]'s significant person is [Y] ."}} 9 | {"uuu": {"family": "[X]'s family is [Y] ."}, "xxx": {"affiliation": "[X] is affiliated with [Y] .", "member of": "[X] is member of [Y] .", "part of": "[X] is part of [Y] ."}} 10 | {"uuu": {"highest point": "[X]'s highest point is [Y] ."}, "xxx": {"has parts": "[X] has part [Y] ."}} 11 | {"uuu": {"father": "[X]'s father is [Y] ."}, "xxx": {"parent": "[X]'s parent is [Y] .", "relative": "[X]'s relative is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 12 | {"uuu": {"killed by": "[X] was killed by [Y] ."}, "xxx": {"significant person": "[X]'s significant person is [Y] ."}} 13 | {"uuu": {"judge": "[X]'s judge is [Y] ."}, "xxx": {"participant": "[X]'s participant is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 14 | {"uuu": {"developer": "[X]'s developer is [Y] ."}, "xxx": {"creator": "[X] is created by [Y] ."}} 15 | {"uuu": {"military branch": "[X]'s service branch is [Y] ."}, "xxx": {"part of": "[X] is part of [Y] ."}} 16 | {"uuu": {"place of birth": "[X] was born in [Y] ."}, "xxx": {"location": "[X] is located in [Y] ."}} 17 | {"uuu": {"author": "[X]'s author is [Y] ."}, "xxx": {"creator": "[X] is created by [Y] ."}} 18 | {"uuu": {"home port": "[X]'s home port is [Y] ."}, "xxx": {"location": "[X] is located in [Y] .", "significant place": "[X]'s significant place is [Y] ."}} 19 | {"uuu": {"industry": "[X]'s specific industry is [Y] ."}, "xxx": {"part of": "[X] is part of [Y] ."}} 20 | {"uuu": {"official language": "[X]'s offical language is [Y] ."}, "xxx": {"language used": "[X]'s widely used language is [Y] ."}} 21 | {"uuu": {"chief executive officer": "[X]'s chief executive officer is [Y] ."}, "xxx": {"corporate officer": "[X]'s corporate officer is [Y] .", "director / manager": "[X] is managed by [Y] ."}} 22 | {"uuu": {"successful candidate": "[X]'s successful candidate is [Y] ."}, "xxx": {"winner": "[X]'s winner is [Y] .", "candidate": "[X]'s candidate is [Y] .", "participant": "[X]'s participant is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 23 | {"uuu": {"located in the administrative territorial entity": "[X] is located in the administrative territory of [Y] ."}, "xxx": {"location": "[X] is located in [Y] .", "part of": "[X] is part of [Y] ."}} 24 | {"uuu": {"original language": "[X] was originally created in [Y] ."}, "xxx": {"language of work or name": "[X]'s language is [Y] ."}} 25 | {"uuu": {"affiliation": "[X] is affiliated with [Y] ."}, "xxx": {"part of": "[X] is part of [Y] ."}} 26 | {"uuu": {"headquarters location": "[X]'s headquarter is in [Y] ."}, "xxx": {"location": "[X] is located in [Y] .", "significant place": "[X]'s significant place is [Y] ."}} 27 | {"uuu": {"parliamentary group": "[X] belongs to the party [Y] ."}, "xxx": {"affiliation": "[X] is affiliated with [Y] .", "member of": "[X] is member of [Y] .", "part of": "[X] is part of [Y] ."}} 28 | {"uuu": {"located in/on physical feature": "[X] is located on landform [Y] ."}, "xxx": {"location": "[X] is located in [Y] .", "part of": "[X] is part of [Y] ."}} 29 | {"uuu": {"dialect of": "[X] is a dialect of [Y] ."}, "xxx": {"derivative work": "[X] is derived from [Y] ."}} 30 | {"uuu": {"based on": "[X] is based on [Y] ."}, "xxx": {"references work, tradition or theory": "[X] references [Y] .", "inspired by": "[X] is inspired by [Y] ."}} 31 | {"uuu": {"head of government": "[X]'s government is headed by [Y] ."}, "xxx": {"chairperson": "[X]'s chairperson is [Y] .", "director / manager": "[X] is managed by [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 32 | {"uuu": {"streak color": "[X]'s streak color is [Y] ."}, "xxx": {"color": "[X]'s color is [Y] ."}} 33 | {"uuu": {"composer": "[X]'s composer is [Y] ."}, "xxx": {"creator": "[X] is created by [Y] ."}} 34 | {"uuu": {"mother": "[X]'s mother is [Y] ."}, "xxx": {"parent": "[X]'s parent is [Y] .", "relative": "[X]'s relative is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 35 | {"uuu": {"residence": "[X] lives in [Y] ."}, "xxx": {"location": "[X] is located in [Y] ."}} 36 | {"uuu": {"founded by": "[X] was founded by [Y] ."}, "xxx": {"creator": "[X] is created by [Y] ."}} 37 | {"uuu": {"performer": "[X]'s performer is [Y] ."}, "xxx": {"contributor to the creative work or subject": "[X]'s contributor is [Y] .", "participant": "[X]'s participant is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 38 | {"uuu": {"home stadium": "[X]'s home stadium is [Y] ."}, "xxx": {"location": "[X] is located in [Y] ."}} 39 | {"uuu": {"member of sports team": "[X] played for [Y] ."}, "xxx": {"member of": "[X] is member of [Y] .", "affiliation": "[X] is affiliated with [Y] .", "part of": "[X] is part of [Y] ."}} 40 | {"uuu": {"designed by": "[X] is designed by [Y] ."}, "xxx": {"creator": "[X] is created by [Y] ."}} 41 | {"uuu": {"conflict": "[X] participated in conflict [Y] ."}, "xxx": {"participant in": "[X] participated in [Y] .", "part of": "[X] is part of [Y] .", "significant event": "[X]'s significant event is [Y] ."}} 42 | {"uuu": {"doctoral student": "[X]'s doctoral student is [Y] ."}, "xxx": {"student": "[X]'s student is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 43 | {"uuu": {"place of burial": "[X] is buried in [Y] ."}, "xxx": {"location": "[X] is located in [Y] ."}} 44 | {"uuu": {"rector": "[X]'s rector is [Y] ."}, "xxx": {"director / manager": "[X] is managed by [Y] ."}} 45 | {"uuu": {"has melody": "[X] has the melody of [Y] ."}, "xxx": {"references work, tradition or theory": "[X] references [Y] ."}} 46 | {"uuu": {"educated at": "[X] is educated at [Y] ."}, "xxx": {"affiliation": "[X] is affiliated with [Y] .", "part of": "[X] is part of [Y] ."}} 47 | {"uuu": {"work location": "[X] used to work in [Y] ."}, "xxx": {"location": "[X] is located in [Y] .", "significant place": "[X]'s significant place is [Y] ."}} 48 | {"uuu": {"cast member": "[X]'s cast member is [Y] ."}, "xxx": {"significant person": "[X]'s significant person is [Y] ."}} 49 | {"uuu": {"curator": "[X]'s curator is [Y] ."}, "xxx": {"significant person": "[X]'s significant person is [Y] ."}} 50 | {"uuu": {"participant": "[X]'s participant is [Y] ."}, "xxx": {"significant person": "[X]'s significant person is [Y] ."}} 51 | {"uuu": {"participating team": "[X]'s participating teams contain [Y] ."}, "xxx": {"participant": "[X]'s participant is [Y] ."}} 52 | {"uuu": {"architect": "[X]'s architect is [Y] ."}, "xxx": {"designed by": "[X] is designed by [Y] .", "creator": "[X] is created by [Y] ."}} 53 | {"uuu": {"continent": "[X] is located on the continent of [Y] ."}, "xxx": {"located in/on physical feature": "[X] is located on landform [Y] .", "location": "[X] is located in [Y] .", "part of": "[X] is part of [Y] ."}} 54 | {"uuu": {"spouse": "[X]'s spouse is [Y] ."}, "xxx": {"relative": "[X]'s relative is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 55 | {"uuu": {"illustrator": "[X] is illustrated by [Y] ."}, "xxx": {"creator": "[X] is created by [Y] ."}} 56 | {"uuu": {"contributor": "[X]'s contributor is [Y] ."}, "xxx": {"participant": "[X]'s participant is [Y] .", "significant person": "[X]'s significant person is [Y] ."}} 57 | {"uuu": {"employer": "[X] is an employer of [Y] ."}, "xxx": {"affiliation": "[X] is affiliated with [Y] .", "part of": "[X] is part of [Y] ."}} 58 | {"uuu": {"constellation": "[X] is part of constellation [Y] ."}, "xxx": {"part of": "[X] is part of [Y] ."}} 59 | {"uuu": {"country of citizenship": "[X] is a citizen of [Y] ."}, "xxx": {"country": "[X]'s country is [Y] .", "located in the administrative territorial entity": "[X] is located in the administrative territory of [Y] .", "location": "[X] is located in [Y] .", "part of": "[X] is part of [Y] ."}} -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from bdb import Breakpoint 2 | from probing import hard_prompt_probing, soft_prompt_probing 3 | from typing import Counter 4 | import argparse 5 | from openprompt.plms import load_plm 6 | from data import load_candidates, DatasetLoader 7 | import torch 8 | import torch.nn as nn 9 | import csv 10 | 11 | if __name__ == "__main__": 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument('--knowledge', type=str, default='type') 14 | parser.add_argument('--model_class', type=str, default='bert') 15 | parser.add_argument('--model_path', type=str, default='bert-base-cased') 16 | parser.add_argument('--template', type=str, default=' is a') 17 | parser.add_argument('--special_token', type=str, default='[MASK]') # 18 | parser.add_argument('--mode', type=str, default="zs-f") 19 | parser.add_argument('--multi_mask', type=str, default="m") 20 | parser.add_argument('--multi_token_handler', type=str, default="mean") 21 | parser.add_argument('--seed', type=int, default=0) 22 | parser.add_argument('--threshold', type=float, default=0.5) 23 | parser.add_argument('--mem_at_k', type=int, default=10) 24 | parser.add_argument('--save_path', type=str, default="result.csv") 25 | parser.add_argument('--is_demo', type=bool, default=False) 26 | parser.add_argument('--use_cuda', type=bool, default=False) 27 | parser.add_argument('--loss', type=str, default="log") 28 | parser.add_argument('--soft_init', type=str, default="rand") 29 | parser.add_argument('--trained_soft', type=str, default="none") 30 | 31 | args = parser.parse_args() 32 | assert args.model_class in ['bert', 'roberta', 'albert', 'gpt', 'gpt2', 't5', 'baseline'] 33 | assert args.trained_soft in ["none", "load"] 34 | print(args) 35 | 36 | if args.trained_soft == "none": 37 | save_name = "{}/{}.{}.{}.{}.{}".format(args.knowledge, args.model_path, "zs-f", args.multi_mask, args.multi_token_handler,args.template) 38 | else: 39 | save_name = "{}/{}.{}.{}.{}.{}".format(args.knowledge, args.model_path, "fs", args.multi_mask, args.multi_token_handler,args.loss) 40 | 41 | if args.model_class == "baseline": 42 | plm, tokenizer, model_config, WrapperClass = load_plm("bert", "bert-base-cased") 43 | candidates = load_candidates(args.knowledge, tokenizer) 44 | K = [1, 5] # recall at k 45 | DatasetLoader = DatasetLoader(args.knowledge, candidates, args.template, args.mode, save_name, args.trained_soft) 46 | _, gt, special_tokens, template_text = DatasetLoader.load_data() 47 | import itertools 48 | import random 49 | gt_all = list(itertools.chain.from_iterable(gt["train"])) # [["a"], ["b", "c"]] -> ["a", "b", "c"] 50 | pred_all = sorted(Counter(gt_all).items(), key=lambda x: x[1], reverse=True) 51 | 52 | for k in K: 53 | pred = [l[0] for l in pred_all[:k]] 54 | recall = 0.0 55 | for golds in gt["test"]: 56 | for l in golds: 57 | if l in pred: 58 | recall += 1.0 59 | break 60 | recall /= len(gt["test"]) 61 | print("Recall@{}: {}".format(k, recall)) 62 | 63 | mrr = 0 64 | pred = [l[0] for l in pred_all] 65 | other = list(set(candidates.values()) - set(pred)) 66 | random.shuffle(other) 67 | pred = pred + other 68 | 69 | for golds in gt["test"]: 70 | rank_avg = 0 71 | for l in golds: 72 | rank_avg += 1 + pred.index(l) 73 | rank_avg /= len(golds) 74 | mrr += 1 / rank_avg 75 | rank_avg = 0 76 | print("MRR_avg: {}".format(mrr / len(gt["test"]))) 77 | 78 | mrr = 0 79 | for golds in gt["test"]: 80 | ranks = [] 81 | for l in golds: 82 | ranks.append(1 + pred.index(l)) 83 | mrr += 1 / min(ranks) 84 | print("MRR: {}".format(mrr / len(gt["test"]))) 85 | exit() 86 | 87 | plm, tokenizer, model_config, WrapperClass = load_plm(args.model_class, args.model_path) 88 | candidates = load_candidates(args.knowledge, tokenizer) 89 | 90 | DatasetLoader = DatasetLoader(args.knowledge, candidates, args.template, args.mode, save_name, args.trained_soft) 91 | dataset, gt, special_tokens, template_text = DatasetLoader.load_data() 92 | 93 | special_tokens_dict = {'additional_special_tokens': special_tokens} 94 | tokenizer.add_special_tokens(special_tokens_dict) 95 | plm.resize_token_embeddings(len(tokenizer)) 96 | 97 | ######### Create Pseudoword for Reasoning Task ####### 98 | if "rdfs" in args.knowledge: 99 | emb = torch.Tensor() 100 | if args.model_class == "bert": 101 | emb = plm.bert.embeddings.word_embeddings.weight 102 | elif args.model_class == "roberta": 103 | emb = plm.roberta.embeddings.word_embeddings.weight 104 | token_id = tokenizer.convert_tokens_to_ids(args.special_token) 105 | emb = torch.Tensor(emb).detach() 106 | spec_token_emb = emb[token_id] 107 | other_emb = torch.cat((emb[:token_id], emb[token_id+1:-len(special_tokens)]),dim=0) # compare to all other words except args.special_token and added special tokens with randomized embeddings to be replaced 108 | pairwise_dist = nn.PairwiseDistance(p=2) # Euclidian Distance 109 | dist = pairwise_dist(spec_token_emb.repeat(other_emb.size()[0],1), other_emb) 110 | min_dist = torch.min(dist) 111 | if int(torch.argmin(dist)) < token_id: 112 | print("Distance between [MASK] and other words in {}: Min=".format(args.model_path), min_dist, tokenizer.convert_ids_to_tokens(int(torch.argmin(dist)))) # nearest word from [mask] 113 | else: 114 | print("Distance between [MASK] and other words in {}: Min=".format(args.model_path), min_dist, tokenizer.convert_ids_to_tokens(int(torch.argmin(dist)) + 1)) # nearest word from [mask] 115 | sample_dist = min_dist * args.threshold 116 | torch.manual_seed(args.seed) 117 | pwords = [] 118 | while len(pwords) < len(special_tokens): 119 | new_emb = torch.randn_like(spec_token_emb) 120 | new_emb = spec_token_emb + sample_dist * new_emb / new_emb.norm(p=2) 121 | print("Distance between pseudoword and [MASK]: ", pairwise_dist(new_emb.unsqueeze(0), spec_token_emb.unsqueeze(0))) 122 | dist = pairwise_dist(new_emb.repeat(other_emb.size()[0],1), other_emb) 123 | if int(torch.argmin(dist)) < token_id: 124 | print("Distance between pseudoword and other words in {}: Min=".format(args.model_path), torch.min(dist), tokenizer.convert_ids_to_tokens(int(torch.argmin(dist)))) # nearest word from [mask] 125 | else: 126 | print("Distance between pseudoword and other words in {}: Min=".format(args.model_path), torch.min(dist), tokenizer.convert_ids_to_tokens(int(torch.argmin(dist)) + 1)) # nearest word from [mask] 127 | for w in pwords: 128 | if pairwise_dist(w.unsqueeze(0), new_emb.unsqueeze(0)) < sample_dist: 129 | break 130 | else: 131 | t_id = tokenizer.convert_tokens_to_ids(special_tokens[len(pwords)]) 132 | emb[t_id] = new_emb 133 | pwords.append(new_emb) 134 | 135 | if args.model_class == "bert": 136 | plm.bert.embeddings.word_embeddings = nn.Embedding(emb.size()[0],emb.size()[1], _weight=emb) 137 | elif args.model_class == "roberta": 138 | plm.roberta.embeddings.word_embeddings = nn.Embedding(emb.size()[0],emb.size()[1], _weight=emb) 139 | 140 | metric = tuple() 141 | if "zs" in args.mode: 142 | save_name = "{}/{}.{}.{}.{}.{}".format(args.knowledge, args.model_path, args.mode, args.multi_mask, args.multi_token_handler,args.template) 143 | metric = hard_prompt_probing(plm, tokenizer, WrapperClass, dataset, candidates, gt, save_name, args.multi_mask, args.multi_token_handler, args.is_demo, args.use_cuda, args.mem_at_k, args.trained_soft, template_text) 144 | else: 145 | save_name = "{}/{}.{}.{}.{}.{}".format(args.knowledge, args.model_path, args.mode, args.multi_mask, args.multi_token_handler,args.loss) 146 | metric = soft_prompt_probing(plm, tokenizer, WrapperClass, dataset, candidates, gt, save_name, args.multi_mask, args.multi_token_handler, args.is_demo, args.use_cuda, args.mem_at_k, args.loss, args.trained_soft, template_text, args.soft_init) 147 | 148 | if len(metric) == 1: 149 | print("Acc with subset candidates: {:.4f}".format(metric[0])) 150 | else: 151 | print("R@1: {:.4f} R@5: {:.4f} R@10: {:.4f} R@25: {:.4f} R@50: {:.4f} MRR_avg: {:.4f} MRR: {:.4f}".format(metric[0], metric[1], metric[2], metric[3], metric[4], metric[5], metric[6])) 152 | print("===========================================") 153 | 154 | with open(args.save_path, "a", newline='') as f: 155 | writer = csv.writer(f) 156 | cont = [args.knowledge, args.model_path, args.mode, args.template, args.seed, args.multi_mask, args.multi_token_handler, args.loss, args.mem_at_k, args.threshold, args.trained_soft, args.soft_init] 157 | for m in metric: 158 | cont.append(float(m)) 159 | writer.writerow(cont) 160 | -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs3-12.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "X is an employer of Y . Therefore, What is the type of Y? (a) launch pad, (b) bridge, (c) national football league event, (d) curling league, (e) snooker player, (f) BobsleighAthlete, (g) on site mean of transportation, (h) overseas department, (i) baseball season, (j) soccer player, (k) organisation, (l) artist, (m) Port, (n) profession, (o) darts player, (p) periodical literature, (q) subway station, (r) racecourse, (s) shrine, (t) artistic genre", "gold": "(k) organisation"} 2 | {"prompt": "X's composer is Y . Therefore, What is the type of Y? (a) educational institution, (b) reptile, (c) racecourse, (d) Food, (e) Controlled designation of origin wine, (f) cricket league, (g) memorial, (h) album, (i) Wind motor, (j) chemical substance, (k) programming language, (l) architect, (m) baseball player, (n) person, (o) football match, (p) Nobel Prize, (q) sculptor, (r) tax, (s) river, (t) infrastructure", "gold": "(n) person"} 3 | {"prompt": "X's doctoral student is Y . Therefore, What is the type of Y? (a) poet, (b) military person, (c) bridge, (d) single, (e) mountain, (f) Community, (g) comics character, (h) deanery, (i) restaurant, (j) Member of a Resistance Movement, (k) cat, (l) Database, (m) historian, (n) rest area, (o) train carriage, (p) parish, (q) Sailor, (r) Theatre director, (s) person, (t) reign", "gold": "(s) person"} 4 | {"prompt": "X's successful candidate is Y . Therefore, What is the type of Y? (a) Christian Patriarch, (b) manhua, (c) Producer, (d) cape, (e) tram station, (f) clerical order, (g) gross domestic product per capita, (h) military conflict, (i) tennis league, (j) golf tournament, (k) boxing league, (l) natural place, (m) cardinal, (n) mineral, (o) board game, (p) sculptor, (q) fictional character, (r) beauty queen, (s) person, (t) parliament", "gold": "(s) person"} 5 | {"prompt": "X's father is Y . Therefore, What is the type of Y? (a) electrical substation, (b) gene, (c) man, (d) manhua, (e) artistic genre, (f) escalator, (g) Globular Swarm, (h) Political concept, (i) programming language, (j) valley, (k) archbishop, (l) old territory, (m) Reference, (n) automobile, (o) Spreadsheet, (p) radio station, (q) Star сluster, (r) collection of valuables, (s) high diver, (t) musical", "gold": "(c) man"} 6 | {"prompt": "X's mother is Y . Therefore, What is the type of Y? (a) speedway team, (b) Browser, (c) district, (d) year, (e) Spy, (f) chancellor, (g) castle, (h) australian football league, (i) sport, (j) arrondissement, (k) field hockey league, (l) high diver, (m) top level domain, (n) psychologist, (o) hockey club, (p) school, (q) forest, (r) guitarist, (s) businessperson, (t) woman", "gold": "(t) woman"} 7 | {"prompt": "X's student is Y . Therefore, What is the type of Y? (a) tournament, (b) arena, (c) person, (d) chancellor, (e) political function, (f) lacrosse league, (g) atoll, (h) bacteria, (i) american football player, (j) beer, (k) religious, (l) ski jumper, (m) venue, (n) national collegiate athletic association team season, (o) politician spouse, (p) guitarist, (q) competition, (r) Annotation, (s) meeting, (t) jockey", "gold": "(c) person"} 8 | {"prompt": "X is hosted by Y . Therefore, What is the type of Y? (a) fort, (b) country, (c) man, (d) locality, (e) Open Swarm, (f) state, (g) tennis player, (h) roller coaster, (i) rest area, (j) bone, (k) Resume, (l) moving image, (m) conifer, (n) Pandemic, (o) Medicine, (p) Organisation member, (q) military unit, (r) muscle, (s) rebellion, (t) person", "gold": "(t) person"} 9 | {"prompt": "X was killed by Y . Therefore, What is the type of Y? (a) stadium, (b) BobsleighAthlete, (c) coach, (d) political party, (e) Satellite, (f) motorsport racer, (g) Playwright, (h) Sculpture, (i) movie, (j) Instrument, (k) baseball season, (l) Historical settlement, (m) horse rider, (n) Algorithm, (o) borough, (p) results of a sport competition, (q) person, (r) cat, (s) lighthouse, (t) horse", "gold": "(q) person"} 10 | {"prompt": "X is a dialect of Y . Therefore, What is the type of Y? (a) poet, (b) clerical administrative region, (c) garden, (d) historic place, (e) Watermill, (f) vodka, (g) language, (h) golf player, (i) museum, (j) province, (k) city, (l) Globular Swarm, (m) project, (n) Painting, (o) musical work, (p) music composer, (q) brewery, (r) sport, (s) prehistorical period, (t) gymnast", "gold": "(g) language"} 11 | {"prompt": "X has the melody of Y . Therefore, What is the type of Y? (a) road tunnel, (b) chemical compound, (c) square, (d) coach, (e) volleyball player, (f) university, (g) museum, (h) province, (i) entomologist, (j) musical work, (k) monument, (l) constellation, (m) area, (n) COVID 19 pandemic, (o) basketball player, (p) comic, (q) sound, (r) wine region, (s) swimmer, (t) dam", "gold": "(j) musical work"} 12 | {"prompt": "X is part of constellation Y . Therefore, What is the type of Y? (a) GeneLocation, (b) galaxy, (c) constellation, (d) type, (e) bus company, (f) college, (g) standard, (h) tennis tournament, (i) JewishLeader, (j) racecourse, (k) time period, (l) race, (m) politician, (n) underground journal, (o) territory, (p) jockey, (q) sports league, (r) badminton player, (s) school, (t) surfer", "gold": "(c) constellation"} 13 | {"prompt": "X belongs to the party Y . Therefore, What is the type of Y? (a) newspaper, (b) chemical substance, (c) celestial body, (d) parliament, (e) top level domain, (f) volleyball league, (g) World Heritage Site, (h) songwriter, (i) law, (j) senator, (k) political party, (l) Anime, (m) cricket team, (n) olympics, (o) cycling competition, (p) soccer league, (q) multi volume publication, (r) race track, (s) engineer, (t) street", "gold": "(k) political party"} 14 | {"prompt": "X is located in the administrative territory of Y . Therefore, What is the type of Y? (a) american football league, (b) launch pad, (c) trade union, (d) space shuttle, (e) conveyor system, (f) river, (g) guitarist, (h) musical, (i) Historical settlement, (j) bridge, (k) Resume, (l) racecourse, (m) administrative region, (n) name, (o) speed skater, (p) time period, (q) fashion designer, (r) results of a sport competition, (s) soap character, (t) province", "gold": "(m) administrative region"} 15 | {"prompt": "X's child is Y . Therefore, What is the type of Y? (a) speedway league, (b) route stop, (c) nascar driver, (d) academic journal, (e) natural region, (f) member of parliament, (g) Protocol, (h) Swarm, (i) philosopher, (j) cycad, (k) opera, (l) vaccine, (m) person, (n) disease, (o) language, (p) reign, (q) water polo Player, (r) ski resort, (s) Team member, (t) polysaccharide", "gold": "(m) person"} 16 | {"prompt": "X's developer is Y . Therefore, What is the type of Y? (a) Browser, (b) Formula One racer, (c) australian football league, (d) brain, (e) handball player, (f) videogames league, (g) pope, (h) Reference, (i) political function, (j) amusement park attraction, (k) Singer, (l) member of parliament, (m) geological period, (n) governor, (o) launch pad, (p) lymph, (q) cheese, (r) agent, (s) racecourse, (t) space shuttle", "gold": "(r) agent"} 17 | {"prompt": "X's participating teams contain Y . Therefore, What is the type of Y? (a) Quote, (b) artery, (c) cabinet of ministers, (d) chess player, (e) sports team, (f) rugby league, (g) jockey, (h) monument, (i) geological period, (j) contest, (k) cycling league, (l) MouseGeneLocation, (m) Biomolecule, (n) linguist, (o) soccer tournoment, (p) volcano, (q) songwriter, (r) academic subject, (s) government agency, (t) newspaper", "gold": "(e) sports team"} 18 | {"prompt": "X's home stadium is Y . Therefore, What is the type of Y? (a) Community, (b) Vaccination Statistics, (c) street, (d) hockey club, (e) road junction, (f) Cipher, (g) Gated community, (h) tenure, (i) engine, (j) manga, (k) Pretender, (l) convention, (m) MouseGene, (n) amateur boxer, (o) music composer, (p) ambassador, (q) entomologist, (r) architectural structure, (s) ligament, (t) image", "gold": "(r) architectural structure"} 19 | {"prompt": "X has cast member Y . Therefore, What is the type of Y? (a) classical music composition, (b) speedway league, (c) sports event, (d) motorsport racer, (e) videogames league, (f) water ride, (g) actor, (h) protohistorical period, (i) natural place, (j) team sport, (k) chemical compound, (l) religious organisation, (m) casino, (n) crustacean, (o) list, (p) Place in the Music Charts, (q) district, (r) island, (s) international football league event, (t) canadian football Player", "gold": "(g) actor"} 20 | {"prompt": "X's spouse is Y . Therefore, What is the type of Y? (a) road tunnel, (b) animal, (c) Archer Player, (d) election, (e) sports club, (f) Vaccination Statistics, (g) on site mean of transportation, (h) File system, (i) Christian Patriarch, (j) model, (k) bowling league, (l) camera, (m) president, (n) given name, (o) person, (p) temple, (q) period of artistic style, (r) multi volume publication, (s) samba school, (t) Artificial Satellite", "gold": "(o) person"} 21 | {"prompt": "X was originally created in Y . Therefore, What is the type of Y? (a) canoeist, (b) convention, (c) BobsleighAthlete, (d) netball player, (e) Scientific concept, (f) publisher, (g) mineral, (h) language, (i) Controlled designation of origin wine, (j) archbishop, (k) cleric, (l) Wind motor, (m) artistic genre, (n) squash player, (o) synagogue, (p) hockey club, (q) populated place, (r) Wikimedia template, (s) standard, (t) multi volume publication", "gold": "(h) language"} 22 | {"prompt": "X's home port is Y . Therefore, What is the type of Y? (a) eukaryote, (b) space mission, (c) Supreme Court of the United States case, (d) cabinet of ministers, (e) archbishop, (f) national football league event, (g) period of artistic style, (h) Open Swarm, (i) Controlled designation of origin wine, (j) member of parliament, (k) ski area, (l) rugby club, (m) cycling league, (n) television show, (o) place, (p) streetcar, (q) Capital, (r) pope, (s) Historical settlement, (t) golf tournament", "gold": "(o) place"} 23 | -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs2-12.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "X's composer is Y . Therefore, What is the type of X? (a) Rebbe, (b) sales, (c) basketball player, (d) temple, (e) work, (f) Capital, (g) muscle, (h) Robot, (i) american football player, (j) law, (k) golf league, (l) unit of work, (m) tenure, (n) american football Team, (o) cricketer, (p) deanery, (q) record label, (r) tunnel, (s) standard, (t) Comedy Group", "gold": "(e) work"} 2 | {"prompt": "X is an employer of Y . Therefore, What is the type of X? (a) hockey team, (b) music genre, (c) person, (d) demographics, (e) Water tower, (f) cycling team, (g) football match, (h) ginkgo, (i) taxonomic group, (j) factory, (k) library, (l) penalty shoot out, (m) college coach, (n) route stop, (o) castle, (p) clerical administrative region, (q) mineral, (r) COVID 19 pandemic, (s) cape, (t) case", "gold": "(c) person"} 3 | {"prompt": "X's curator is Y . Therefore, What is the type of X? (a) wrestler, (b) mine, (c) article, (d) governmental administrative region, (e) Pyramid, (f) grave stone or grave monument, (g) museum, (h) flag, (i) Quote, (j) American Leader, (k) micro region, (l) System of law, (m) hockey club, (n) academic conference, (o) monoclonal antibody, (p) Television director, (q) canoeist, (r) basketball player, (s) Pilot, (t) group", "gold": "(g) museum"} 4 | {"prompt": "X is buried in Y . Therefore, What is the type of X? (a) agent, (b) ski area, (c) formula one racing, (d) military unit, (e) digital camera, (f) university, (g) lymph, (h) casino, (i) person, (j) old territory, (k) office holder, (l) decoration, (m) canton, (n) murderer, (o) speed skater, (p) ligament, (q) solar eclipse, (r) game, (s) military service, (t) vice president", "gold": "(i) person"} 5 | {"prompt": "X is a citizen of Y . Therefore, What is the type of X? (a) cycling competition, (b) natural event, (c) garden, (d) monastery, (e) animanga character, (f) Record Office, (g) deity, (h) motorcycle rider, (i) roller coaster, (j) coach, (k) demographics, (l) monarch, (m) topical concept, (n) Blazon, (o) national football league event, (p) mountain range, (q) soccer league season, (r) academic subject, (s) engine, (t) person", "gold": "(t) person"} 6 | {"prompt": "X was originally created in Y . Therefore, What is the type of X? (a) overseas department, (b) Gaelic games player, (c) Producer, (d) square, (e) national football league season, (f) asteroid, (g) movie genre, (h) Nebula, (i) lunar crater, (j) work, (k) congressman, (l) Biathlete, (m) church, (n) lock, (o) route of transportation, (p) Vaccination Statistics, (q) personal event, (r) minister, (s) light novel, (t) rebellion", "gold": "(j) work"} 7 | {"prompt": "X's mother is Y . Therefore, What is the type of X? (a) academic conference, (b) volcano, (c) Latter Day Saint, (d) person, (e) religious building, (f) Swarm, (g) ship, (h) Historical region, (i) bone, (j) Band, (k) Food, (l) eukaryote, (m) arachnid, (n) race track, (o) diocese, (p) lieutenant, (q) record label, (r) still image, (s) holiday, (t) DTM racer", "gold": "(d) person"} 8 | {"prompt": "X is a dialect of Y . Therefore, What is the type of X? (a) cabinet of ministers, (b) language, (c) golf course, (d) comic, (e) manhwa, (f) poem, (g) battery, (h) Hormone, (i) volleyball league, (j) fungus, (k) publisher, (l) gross domestic product, (m) volleyball player, (n) Latter Day Saint, (o) memorial, (p) music festival, (q) psychologist, (r) historical event, (s) life cycle event, (t) medical specialty", "gold": "(b) language"} 9 | {"prompt": "X was born in Y . Therefore, What is the type of X? (a) historic building, (b) winter sport Player, (c) ambassador, (d) rower, (e) tunnel, (f) music composer, (g) football match, (h) animal, (i) Rebbe, (j) parliament, (k) Organisation member, (l) Police Officer, (m) guitar, (n) Satellite, (o) Annotation, (p) cycling race, (q) flag, (r) term of office, (s) speedway league, (t) National anthem", "gold": "(h) animal"} 10 | {"prompt": "X's rector is Y . Therefore, What is the type of X? (a) sports club, (b) populated place, (c) baseball season, (d) professor, (e) educational institution, (f) Team member, (g) volleyball player, (h) videogames league, (i) volleyball coach, (j) Playboy Playmate, (k) bullfighter, (l) international organisation, (m) Stated Resolution, (n) election, (o) automobile, (p) literary genre, (q) Government Type, (r) route of transportation, (s) solar eclipse, (t) Biological database", "gold": "(e) educational institution"} 11 | {"prompt": "X's headquarter is in Y . Therefore, What is the type of X? (a) australian football league, (b) person, (c) organisation, (d) parish, (e) tenure, (f) subway station, (g) dam, (h) borough, (i) Pyramid, (j) deputy, (k) town, (l) trade union, (m) songwriter, (n) enzyme, (o) bowling league, (p) valley, (q) poet, (r) Nebula, (s) motorsport racer, (t) Organisation member", "gold": "(c) organisation"} 12 | {"prompt": "X lives in Y . Therefore, What is the type of X? (a) darts player, (b) politician, (c) amusement park attraction, (d) unit of work, (e) line of fashion, (f) snooker player, (g) Latter Day Saint, (h) tournament, (i) cleric, (j) tennis league, (k) minister, (l) international football league event, (m) bird, (n) Biathlete, (o) soap character, (p) lipid, (q) person, (r) pope, (s) tax, (t) Blazon", "gold": "(q) person"} 13 | {"prompt": "X has the melody of Y . Therefore, What is the type of X? (a) Cardinal direction, (b) professor, (c) prefecture, (d) Relationship, (e) law, (f) competition, (g) multi volume publication, (h) Resume, (i) Employers' Organisation, (j) Manor, (k) military aircraft, (l) flowering plant, (m) vaccine, (n) work, (o) song, (p) football league season, (q) lymph, (r) Wikimedia template, (s) office holder, (t) horse race", "gold": "(n) work"} 14 | {"prompt": "X's service branch is Y . Therefore, What is the type of X? (a) military person, (b) novel, (c) skyscraper, (d) deity, (e) artist discography, (f) vodka, (g) cycling competition, (h) streetcar, (i) cyclist, (j) Vaccination Statistics, (k) military unit, (l) archeologist, (m) American Leader, (n) Eurovision song contest entry, (o) amateur boxer, (p) Painting, (q) cultivated variety, (r) wine region, (s) enzyme, (t) american football player", "gold": "(a) military person"} 15 | {"prompt": "X's successful candidate is Y . Therefore, What is the type of X? (a) temple, (b) periodical literature, (c) software, (d) station, (e) deputy, (f) canadian football league, (g) Sculpture, (h) HumanGeneLocation, (i) award, (j) deanery, (k) sculptor, (l) stream, (m) roller coaster, (n) synagogue, (o) election, (p) Pandemic, (q) tower, (r) brewery, (s) monoclonal antibody, (t) mixed martial arts event", "gold": "(o) election"} 16 | {"prompt": "X is located on the continent of Y . Therefore, What is the type of X? (a) baseball team, (b) Satellite, (c) GeneLocation, (d) movie genre, (e) automobile, (f) voice actor, (g) country, (h) stadium, (i) Artificial Satellite, (j) inline hockey league, (k) television episode, (l) newspaper, (m) rugby club, (n) population, (o) gross domestic product per capita, (p) surfer, (q) organisation, (r) district water board, (s) canadian football Team, (t) library", "gold": "(g) country"} 17 | {"prompt": "X's offical language is Y . Therefore, What is the type of X? (a) parliament, (b) military structure, (c) shopping mall, (d) manhwa, (e) speed skater, (f) auto racing league, (g) squash player, (h) Ocean, (i) ice hockey league, (j) lipid, (k) conifer, (l) amateur boxer, (m) person function, (n) airport, (o) moving image, (p) radio station, (q) businessperson, (r) brewery, (s) infrastructure, (t) populated place", "gold": "(t) populated place"} 18 | {"prompt": "X participated in conflict Y . Therefore, What is the type of X? (a) artery, (b) chef, (c) mammal, (d) cycling competition, (e) soccer club, (f) egyptologist, (g) mollusca, (h) person, (i) horse trainer, (j) engineer, (k) military service, (l) martial artist, (m) reptile, (n) Historical region, (o) Singer, (p) rugby league, (q) comics character, (r) softball league, (s) mixed martial arts league, (t) basketball team", "gold": "(h) person"} 19 | {"prompt": "X was killed by Y . Therefore, What is the type of X? (a) battery, (b) trade union, (c) anatomical structure, (d) opera, (e) collection of valuables, (f) sport, (g) classical music artist, (h) gross domestic product per capita, (i) JewishLeader, (j) manhua, (k) Lawyer, (l) person, (m) animal, (n) medician, (o) earthquake, (p) geological period, (q) senator, (r) beverage, (s) formula 1 team, (t) fort", "gold": "(l) person"} 20 | {"prompt": "X was founded by Y . Therefore, What is the type of X? (a) medician, (b) reptile, (c) space shuttle, (d) cycling competition, (e) cheese, (f) document, (g) comic strip, (h) casino, (i) baronet, (j) BobsleighAthlete, (k) sculptor, (l) netball player, (m) military person, (n) Cardinal direction, (o) settlement, (p) department, (q) cycad, (r) anatomical structure, (s) stream, (t) organisation", "gold": "(t) organisation"} 21 | {"prompt": "X's streak color is Y . Therefore, What is the type of X? (a) place, (b) college coach, (c) Political concept, (d) ginkgo, (e) mineral, (f) rebellion, (g) castle, (h) Star сluster, (i) Comedy Group, (j) atoll, (k) brown dwarf, (l) archbishop, (m) soccer manager, (n) historic place, (o) device, (p) constellation, (q) National anthem, (r) cartoon, (s) sport, (t) politician", "gold": "(e) mineral"} 22 | {"prompt": "X's student is Y . Therefore, What is the type of X? (a) Document Type, (b) trade union, (c) artery, (d) research project, (e) poet, (f) sports club, (g) name, (h) contest, (i) Gated community, (j) literary genre, (k) insect, (l) Rebbe, (m) curler, (n) agglomeration, (o) legislature, (p) state, (q) person, (r) Desert, (s) Intercommunality, (t) military unit", "gold": "(q) person"} 23 | {"prompt": "X's child is Y . Therefore, What is the type of X? (a) National anthem, (b) person, (c) screenwriter, (d) golf course, (e) tennis player, (f) archeologist, (g) mammal, (h) sports season, (i) ancient area of jurisdiction of a person feudal or of a governmental body, (j) monoclonal antibody, (k) Concentration camp, (l) sport, (m) Team member, (n) judge, (o) light novel, (p) System of law, (q) amusement park attraction, (r) team sport, (s) DTM racer, (t) olympic event", "gold": "(b) person"} 24 | {"prompt": "X used to work in Y . Therefore, What is the type of X? (a) moss, (b) mountain range, (c) crater, (d) research project, (e) basketball player, (f) locomotive, (g) race track, (h) written work, (i) Biathlete, (j) file, (k) Election Diagram, (l) rugby player, (m) medician, (n) chemical element, (o) conifer, (p) plant, (q) college coach, (r) archipelago, (s) Stated Resolution, (t) person", "gold": "(t) person"} 25 | {"prompt": "X has cast member Y . Therefore, What is the type of X? (a) work, (b) Robot, (c) artery, (d) casino, (e) national collegiate athletic association athlete, (f) protein, (g) broadcast network, (h) storm surge, (i) radio program, (j) organ, (k) train carriage, (l) Theatre director, (m) Pretender, (n) DBpedian, (o) body of water, (p) dam, (q) brown dwarf, (r) ligament, (s) Windmill, (t) adult pornographic actor", "gold": "(a) work"} 26 | {"prompt": "X's participating teams contain Y . Therefore, What is the type of X? (a) meeting, (b) written work, (c) voice actor, (d) lipid, (e) video game, (f) card game, (g) Document Type, (h) archeologist, (i) Biomolecule, (j) natural region, (k) currency, (l) musical, (m) model, (n) television host, (o) woman, (p) sports event, (q) event, (r) streetcar, (s) sports manager, (t) Television director", "gold": "(p) sports event"} 27 | -------------------------------------------------------------------------------- /test-gpt/probing_gpt.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | import jsonlines 4 | import random 5 | 6 | SAMPLE = 20 7 | class_list, prop_list = [] ,[] 8 | r1, r2 = {}, {} 9 | 10 | import openai 11 | import jsonlines 12 | from tenacity import ( 13 | retry, 14 | stop_after_attempt, 15 | wait_random_exponential, 16 | ) 17 | 18 | 19 | @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) 20 | def completion_with_backoff(**kwargs): 21 | return openai.ChatCompletion.create(**kwargs) 22 | 23 | # OpenAI API credentials 24 | openai.api_key = 'YOUR_API_KEY' 25 | 26 | def test_memorization(knowledge): 27 | acc = 0 28 | responses = list() 29 | with jsonlines.open("{}.jsonl".format(knowledge)) as reader: 30 | for obj in reader: 31 | query = obj["prompt"] 32 | gold = obj["gold"] 33 | response = completion_with_backoff(model="gpt-3.5-turbo", messages=[{"role": "user", "content": query}]) 34 | response_text = response.choices[0].message.content.strip() 35 | if gold.lower() in response_text.lower(): 36 | acc += 1 37 | responses.append(response_text + " 1!") 38 | else: 39 | responses.append(response_text + " 0!") 40 | print(responses[-1]) 41 | 42 | with open("{}-response.txt".format(knowledge), "w") as f: 43 | f.write("\n".join(responses)) 44 | 45 | return acc / len(list(responses)) 46 | 47 | def domain_manual_textb(d, r): 48 | label = list(d["uuu"].keys())[0] 49 | template = list(d["uuu"].values())[0] 50 | w = template.strip("[X]").split(" ")[0] 51 | exp_know = "" 52 | if w == "'s": 53 | if "contain" in template: 54 | exp_know = "if one has" + template.strip("[X]'s").split("contain ")[0] 55 | else: 56 | exp_know = "if one has" + template.strip("[X]'s").split("is ")[0] 57 | else: 58 | exp_know = "if one {}".format(template.strip("[X] ")) 59 | if label in r.keys(): 60 | if r[label][0] in 'aeiouAEIOU': 61 | exp_know = exp_know.replace(r[label],"").replace("[Y]", "an "+ r[label]) 62 | else: 63 | exp_know = exp_know.replace("[Y]", "a "+ r[label]) 64 | else: 65 | exp_know = exp_know.replace("[Y]", "something") 66 | return exp_know.strip(".").strip() 67 | 68 | def range_manual_textb(d, r): 69 | label = list(d["uuu"].keys())[0] 70 | template = list(d["uuu"].values())[0] 71 | w = template.strip(" [Y] .").split(" ")[-1] 72 | exp_know = "" 73 | if w == "is": 74 | exp_know = "if one is ".format(d["xxx"][0]) + template.replace("is [Y] ","") 75 | elif w == "contain": 76 | exp_know = "if one is ".format(d["xxx"][0]) + template.replace("contain [Y] ","") 77 | else: 78 | exp_know = "if {}".format(template.replace("[Y]", "that")) 79 | if label in r.keys(): 80 | if r[label][0] in 'aeiouAEIOU': 81 | exp_know = exp_know.replace("[X]", "an "+ r[label]) 82 | else: 83 | exp_know = exp_know.replace("[X]", "a "+ r[label]) 84 | else: 85 | exp_know = exp_know.replace("[X]", "something") 86 | return exp_know.strip(".").strip() 87 | 88 | def generate_memorization_data(knowledge): 89 | with jsonlines.open("../data/memorizing/{}.jsonl".format(knowledge)) as reader: 90 | data = list(reader) 91 | golds = list() 92 | prompts = list() 93 | cands = list() 94 | MAX = min(520, len(data)) 95 | for d in data[20:MAX]: 96 | if knowledge == "subPropertyOf": 97 | query, gold = " ".join(re.sub(r'[-()/]',' ', next(iter(d["uuu"]))).split()), " ".join(re.sub(r'[-()/]',' ', next(iter(d["xxx"]))).split()) 98 | options = random.sample(set(prop_list) - {gold} - {query}, SAMPLE - 1) 99 | elif knowledge == "type" or knowledge == "subClassOf": 100 | query, gold = " ".join(re.sub(r'[-()/]',' ', d["uuu"]).split()), " ".join(re.sub(r'[-()/]',' ', d["xxx"][0]).split()) 101 | options = random.sample(set(class_list) - {gold} - {query}, SAMPLE - 1) 102 | else: 103 | query, gold = " ".join(re.sub(r'[-()/]',' ', next(iter(d["uuu"]))).split()), " ".join(re.sub(r'[-()/]',' ', d["xxx"][0]).split()) 104 | options = random.sample(set(class_list) - {gold} - {query}, SAMPLE - 1) 105 | 106 | options.append(gold) 107 | random.shuffle(options) 108 | 109 | formatted_options = [] 110 | for i, option in enumerate(options): 111 | formatted_option = "({}) {}".format(chr(ord('a') + i), option) 112 | formatted_options.append(formatted_option) 113 | if gold == option: 114 | golds.append(formatted_option) 115 | 116 | if knowledge == "type" : 117 | prompt = "What is the type of {}? {}".format(query, ", ".join(formatted_options)) 118 | elif knowledge == "subClassOf": 119 | prompt = "What is the superclass of {}? {}".format(query, ", ".join(formatted_options)) 120 | elif knowledge == "subPropertyOf": 121 | prompt = "What does {} imply? {}".format(query, ", ".join(formatted_options)) 122 | elif knowledge == "domain": 123 | prompt = "What is the type of it {}? {}".format(domain_manual_textb(d, r1), ", ".join(formatted_options)) 124 | elif knowledge == "range": 125 | prompt = "What is the type of it {}? {}".format(range_manual_textb(d, r2), ", ".join(formatted_options)) 126 | else: 127 | pass 128 | prompts.append(prompt) 129 | cands.append(options) 130 | # print(prompt) 131 | 132 | with jsonlines.open("data-gpt/memorizing/{}.jsonl".format(knowledge), "w") as f: 133 | for p, g, i in zip(prompts, golds, cands): 134 | f.write({"prompt": p, "gold": g, "cands": i}) 135 | 136 | def generate_reasoning_data(knowledge, rule, index, mode): 137 | with jsonlines.open("data-gpt/memorizing/{}.jsonl".format(knowledge)) as reader: 138 | mem_data = list(reader) 139 | candidates = [d["cands"] for d in mem_data] 140 | 141 | with jsonlines.open("../data/memorizing/{}.jsonl".format(knowledge)) as reader: 142 | data = list(reader) 143 | golds = list() 144 | prompts = list() 145 | MAX = min(520, len(data)) 146 | data = data[20:MAX] 147 | index = [j-1 for j in index] ########## 148 | if mode == 1: 149 | index = list(set(range(len(data))) - set(index)) 150 | if mode == 2: 151 | index= list(range(len(data))) 152 | index.sort() 153 | 154 | data = [data[i] for i in index] 155 | candidates = [candidates[i] for i in index] 156 | cnt = 0 157 | 158 | for d in data: 159 | if knowledge == "subPropertyOf": 160 | if "rdfs7" in rule: 161 | prefix, gold = " ".join(re.sub(r'[-()/]',' ', next(iter(d["uuu"].values()))).split()), " ".join(re.sub(r'[-()/]',' ', next(iter(d["xxx"].values()))).split()) 162 | else: 163 | prefix, gold = "X implies {}. ".format(" ".join(re.sub(r'[-()/]',' ', next(iter(d["uuu"]))).split())), " ".join(re.sub(r'[-()/]',' ', next(iter(d["xxx"]))).split()) 164 | if mode == 2: 165 | prefix = "{} implies {}. ".format(" ".join(re.sub(r'[-()/]',' ', next(iter(d["uuu"]))).split()), gold) + prefix 166 | 167 | elif knowledge == "subClassOf": 168 | prefix, gold = "X is a " + " ".join(re.sub(r'[-()/]',' ', d["uuu"]).split()), " ".join(re.sub(r'[-()/]',' ', d["xxx"][0]).split()) 169 | if mode == 2: 170 | prefix = "{} is a {}. ".format(" ".join(re.sub(r'[-()/]',' ', d["uuu"]).split()), gold) + prefix 171 | 172 | elif knowledge == "domain": 173 | prefix, gold = " ".join(re.sub(r'[-()/]',' ', next(iter(d["uuu"].values()))).split()), " ".join(re.sub(r'[-()/]',' ', d["xxx"][0]).split()) 174 | if mode == 2: 175 | prefix = "One has to be a {} {}. ".format(gold, domain_manual_textb(d, r1)) + prefix 176 | 177 | elif knowledge == "range": 178 | prefix, gold = " ".join(re.sub(r'[-()/]',' ', next(iter(d["uuu"].values()))).split()), " ".join(re.sub(r'[-()/]',' ', d["xxx"][0]).split()) 179 | if mode == 2: 180 | prefix = "One has to be a {} {}. ".format(gold, range_manual_textb(d, r2)) + prefix 181 | 182 | options = candidates[cnt] 183 | cnt += 1 184 | formatted_options = ["({}) {}".format(chr(ord('a') + i), option) for i, option in enumerate(options)] 185 | 186 | if "rdfs2" in rule or "rdfs9" in rule: 187 | prompt = "{} Therefore, What is the type of X? {}".format(prefix, ", ".join(formatted_options)) 188 | elif "rdfs3" in rule: 189 | prompt = "{} Therefore, What is the type of Y? {}".format(prefix, ", ".join(formatted_options)) 190 | elif "rdfs5" in rule: 191 | prompt = "{} Therefore, What does X imply? {}".format(prefix, ", ".join(formatted_options)) 192 | elif "rdfs7" in rule: 193 | prompt = "{} Therefore, What is the relation between X and Y? {}".format(prefix, ", ".join(formatted_options)) 194 | elif "rdfs11" in rule: 195 | prompt = "{} Therefore, What is the superclass of X? {}".format(prefix, ", ".join(formatted_options)) 196 | else: 197 | pass 198 | 199 | prompts.append(prompt.replace("[X]", "X").replace("[Y]", "Y")) 200 | # print(prompts[-1]) 201 | golds.append(gold) 202 | 203 | with jsonlines.open("data-gpt/reasoning/{}-{}.jsonl".format(rule, str(mode)+"2"), "w") as f: 204 | for p, g in zip(prompts, golds): 205 | f.write({"prompt": p, "gold": g}) 206 | 207 | if __name__ == "__main__": 208 | 209 | with open("../data/ontology/class.json") as f: 210 | data = json.loads(f.read()) 211 | for d in data: 212 | class_list.append(re.sub(r'[-()/]',' ', d["rdfs:label"])) 213 | class_list += [re.sub(r'[-()/]',' ', word) for word in d["rdfs:subClassOf"]] 214 | 215 | with open("../data/ontology/property.json") as f: 216 | data = json.loads(f.read()) 217 | for d in data: 218 | prop_list.append(re.sub(r'[()/]',' ', d["rdfs:label"])) 219 | if d["rdfs:subPropertyOf"]: 220 | prop_list += [re.sub(r'[()/]',' ', word) for word in d["rdfs:subPropertyOf"].keys()] 221 | 222 | class_list = [' '.join(w.split()) for w in class_list] # remove redundant space 223 | class_list = list(dict.fromkeys(class_list)) # remove duplicate candidates 224 | 225 | prop_list = [' '.join(w.split()) for w in prop_list] # remove redundant space 226 | prop_list = list(dict.fromkeys(prop_list)) # remove duplicate candidates 227 | 228 | with jsonlines.open("../data/memorizing/range.jsonl") as ranges: # only [Y]->range needs 229 | for d in ranges: 230 | r1[list(d["uuu"].keys())[0]] = d["xxx"][0] 231 | 232 | with jsonlines.open("../data/memorizing/domain.jsonl") as domains: 233 | for d in domains: 234 | r2[list(d["uuu"].keys())[0]] = d["xxx"][0] 235 | 236 | random.seed(0) 237 | 238 | for k in ["type", "subClassOf", "subPropertyOf", "domain", "range"]: 239 | generate_memorization_data(k) 240 | acc = test_memorization("data-gpt/memorizing/{}".format(k)) 241 | print("Acc: ", acc) 242 | 243 | with open("Accuracy.txt", "a") as f: 244 | f.write("{}: {}\n".format(k, acc)) 245 | 246 | 247 | 248 | 249 | 250 | -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs5-12.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "X implies chief executive officer. Therefore, What does X imply? (a) presenter, (b) official language, (c) highest point, (d) judge, (e) part of, (f) has parts, (g) chairperson, (h) significant event, (i) architect, (j) father, (k) corporate officer, (l) military branch, (m) significant person, (n) cast member, (o) contributor to the creative work or subject, (p) original language, (q) inspired by, (r) country of citizenship, (s) head of government, (t) member of sports team", "gold": "(k) corporate officer"} 2 | {"prompt": "X implies located in the administrative territorial entity. Therefore, What does X imply? (a) headquarters location, (b) inspired by, (c) references work, tradition or theory, (d) language used, (e) significant event, (f) country, (g) official language, (h) killed by, (i) dialect of, (j) conflict, (k) candidate, (l) illustrator, (m) director manager, (n) participant, (o) member of, (p) architect, (q) constellation, (r) participating team, (s) location, (t) place of birth", "gold": "(s) location"} 3 | {"prompt": "X implies original language. Therefore, What does X imply? (a) head of government, (b) original broadcaster, (c) judge, (d) illustrator, (e) architect, (f) member of sports team, (g) participant in, (h) relative, (i) curator, (j) has parts, (k) winner, (l) language of work or name, (m) place of burial, (n) home venue, (o) composer, (p) part of, (q) child, (r) participant, (s) highest point, (t) constellation", "gold": "(l) language of work or name"} 4 | {"prompt": "X implies affiliation. Therefore, What does X imply? (a) designed by, (b) home venue, (c) owner of, (d) derivative work, (e) successful candidate, (f) doctoral student, (g) place of birth, (h) part of, (i) language used, (j) parent, (k) judge, (l) original language, (m) author, (n) founded by, (o) creator, (p) continent, (q) constellation, (r) student, (s) rector, (t) work location", "gold": "(h) part of"} 5 | {"prompt": "X implies headquarters location. Therefore, What does X imply? (a) inflows, (b) child, (c) located in on physical feature, (d) continent, (e) based on, (f) architect, (g) family, (h) curator, (i) chief executive officer, (j) military branch, (k) student, (l) significant event, (m) place of burial, (n) place of birth, (o) language used, (p) member of sports team, (q) developer, (r) mother, (s) location, (t) founded by", "gold": "(s) location"} 6 | {"prompt": "X implies parliamentary group. Therefore, What does X imply? (a) mother, (b) located in on physical feature, (c) presenter, (d) significant event, (e) home venue, (f) participating team, (g) has subsidiary, (h) architect, (i) country, (j) derivative work, (k) affiliation, (l) father, (m) country of citizenship, (n) curator, (o) head of government, (p) color, (q) inspired by, (r) has melody, (s) location, (t) highest point", "gold": "(k) affiliation"} 7 | {"prompt": "X implies located in on physical feature. Therefore, What does X imply? (a) student, (b) country, (c) references work, tradition or theory, (d) designed by, (e) affiliation, (f) father, (g) parliamentary group, (h) contributor to the creative work or subject, (i) composer, (j) highest point, (k) place of death, (l) developer, (m) location, (n) part of, (o) inflows, (p) color, (q) streak color, (r) based on, (s) spouse, (t) residence", "gold": "(m) location"} 8 | {"prompt": "X implies based on. Therefore, What does X imply? (a) killed by, (b) residence, (c) presenter, (d) designed by, (e) participating team, (f) family, (g) home venue, (h) participant in, (i) continent, (j) references work, tradition or theory, (k) place of death, (l) mother, (m) has subsidiary, (n) located in the administrative territorial entity, (o) significant person, (p) derivative work, (q) parent, (r) country, (s) highest point, (t) corporate officer", "gold": "(j) references work, tradition or theory"} 9 | {"prompt": "X implies head of government. Therefore, What does X imply? (a) employer, (b) chairperson, (c) killed by, (d) student, (e) color, (f) owner of, (g) home port, (h) country of citizenship, (i) derivative work, (j) military branch, (k) successful candidate, (l) cast member, (m) judge, (n) performer, (o) participating team, (p) significant event, (q) part of, (r) architect, (s) location, (t) located in the administrative territorial entity", "gold": "(b) chairperson"} 10 | {"prompt": "X implies streak color. Therefore, What does X imply? (a) child, (b) creator, (c) head of government, (d) composer, (e) candidate, (f) participant, (g) student, (h) relative, (i) part of, (j) judge, (k) industry, (l) headquarters location, (m) conflict, (n) inspired by, (o) rector, (p) color, (q) father, (r) work location, (s) constellation, (t) significant event", "gold": "(p) color"} 11 | {"prompt": "X implies composer. Therefore, What does X imply? (a) affiliation, (b) has melody, (c) educated at, (d) references work, tradition or theory, (e) relative, (f) corporate officer, (g) candidate, (h) developer, (i) designed by, (j) spouse, (k) judge, (l) significant place, (m) creator, (n) dialect of, (o) place of death, (p) country of citizenship, (q) rector, (r) parent, (s) member of sports team, (t) participant", "gold": "(m) creator"} 12 | {"prompt": "X implies mother. Therefore, What does X imply? (a) affiliation, (b) original language, (c) derivative work, (d) official language, (e) military branch, (f) home venue, (g) highest point, (h) dialect of, (i) language used, (j) participant in, (k) significant place, (l) located in on physical feature, (m) country, (n) judge, (o) parent, (p) chief executive officer, (q) inspired by, (r) illustrator, (s) corporate officer, (t) place of burial", "gold": "(o) parent"} 13 | {"prompt": "X implies residence. Therefore, What does X imply? (a) place of burial, (b) cast member, (c) location, (d) relative, (e) candidate, (f) language used, (g) member of sports team, (h) place of death, (i) chairperson, (j) developer, (k) illustrator, (l) original language, (m) work location, (n) rector, (o) founded by, (p) author, (q) architect, (r) original broadcaster, (s) creator, (t) parent", "gold": "(c) location"} 14 | {"prompt": "X implies founded by. Therefore, What does X imply? (a) candidate, (b) owner of, (c) creator, (d) chief executive officer, (e) spouse, (f) located in the administrative territorial entity, (g) participant in, (h) presenter, (i) located in on physical feature, (j) has parts, (k) significant person, (l) winner, (m) dialect of, (n) parent, (o) corporate officer, (p) language of work or name, (q) doctoral student, (r) place of burial, (s) residence, (t) family", "gold": "(c) creator"} 15 | {"prompt": "X implies performer. Therefore, What does X imply? (a) parliamentary group, (b) chairperson, (c) has subsidiary, (d) located in the administrative territorial entity, (e) rector, (f) family, (g) developer, (h) home port, (i) head of government, (j) place of death, (k) inspired by, (l) employer, (m) affiliation, (n) father, (o) member of sports team, (p) contributor to the creative work or subject, (q) member of, (r) curator, (s) parent, (t) home venue", "gold": "(p) contributor to the creative work or subject"} 16 | {"prompt": "X implies home stadium. Therefore, What does X imply? (a) father, (b) participating team, (c) place of burial, (d) place of death, (e) illustrator, (f) constellation, (g) relative, (h) color, (i) language used, (j) parliamentary group, (k) home port, (l) has parts, (m) location, (n) continent, (o) designed by, (p) child, (q) candidate, (r) has melody, (s) work location, (t) located in on physical feature", "gold": "(m) location"} 17 | {"prompt": "X implies member of sports team. Therefore, What does X imply? (a) member of, (b) significant event, (c) military branch, (d) conflict, (e) director manager, (f) continent, (g) has melody, (h) curator, (i) killed by, (j) family, (k) relative, (l) cast member, (m) doctoral student, (n) color, (o) student, (p) located in on physical feature, (q) headquarters location, (r) winner, (s) developer, (t) highest point", "gold": "(a) member of"} 18 | {"prompt": "X implies conflict. Therefore, What does X imply? (a) child, (b) killed by, (c) educated at, (d) illustrator, (e) doctoral student, (f) place of birth, (g) cast member, (h) home port, (i) participant in, (j) student, (k) military branch, (l) significant person, (m) located in the administrative territorial entity, (n) family, (o) member of, (p) original language, (q) developer, (r) founded by, (s) place of burial, (t) inflows", "gold": "(i) participant in"} 19 | {"prompt": "X implies doctoral student. Therefore, What does X imply? (a) relative, (b) headquarters location, (c) language of work or name, (d) home port, (e) inflows, (f) contributor to the creative work or subject, (g) father, (h) language used, (i) significant person, (j) has subsidiary, (k) spouse, (l) derivative work, (m) streak color, (n) part of, (o) original broadcaster, (p) head of government, (q) member of, (r) cast member, (s) student, (t) successful candidate", "gold": "(s) student"} 20 | {"prompt": "X implies place of burial. Therefore, What does X imply? (a) head of government, (b) creator, (c) location, (d) has subsidiary, (e) student, (f) performer, (g) participating team, (h) spouse, (i) parent, (j) industry, (k) illustrator, (l) original broadcaster, (m) successful candidate, (n) country, (o) references work, tradition or theory, (p) continent, (q) language used, (r) based on, (s) developer, (t) significant event", "gold": "(c) location"} 21 | {"prompt": "X implies has melody. Therefore, What does X imply? (a) founded by, (b) part of, (c) winner, (d) doctoral student, (e) place of burial, (f) spouse, (g) has subsidiary, (h) inflows, (i) place of death, (j) participant in, (k) chief executive officer, (l) judge, (m) language of work or name, (n) original language, (o) references work, tradition or theory, (p) place of birth, (q) rector, (r) presenter, (s) killed by, (t) highest point", "gold": "(o) references work, tradition or theory"} 22 | {"prompt": "X implies work location. Therefore, What does X imply? (a) place of burial, (b) inspired by, (c) participant, (d) participating team, (e) affiliation, (f) original broadcaster, (g) director manager, (h) composer, (i) founded by, (j) has melody, (k) color, (l) constellation, (m) location, (n) student, (o) author, (p) language used, (q) spouse, (r) streak color, (s) candidate, (t) conflict", "gold": "(m) location"} 23 | {"prompt": "X implies participating team. Therefore, What does X imply? (a) presenter, (b) doctoral student, (c) derivative work, (d) winner, (e) language of work or name, (f) participant in, (g) composer, (h) affiliation, (i) military branch, (j) participant, (k) corporate officer, (l) chairperson, (m) language used, (n) home port, (o) parliamentary group, (p) employer, (q) headquarters location, (r) head of government, (s) successful candidate, (t) student", "gold": "(j) participant"} 24 | {"prompt": "X implies continent. Therefore, What does X imply? (a) member of sports team, (b) headquarters location, (c) significant place, (d) chairperson, (e) located in on physical feature, (f) family, (g) derivative work, (h) highest point, (i) designed by, (j) residence, (k) doctoral student, (l) home port, (m) contributor to the creative work or subject, (n) parliamentary group, (o) dialect of, (p) inflows, (q) head of government, (r) language used, (s) student, (t) relative", "gold": "(e) located in on physical feature"} 25 | {"prompt": "X implies spouse. Therefore, What does X imply? (a) relative, (b) family, (c) place of death, (d) original broadcaster, (e) composer, (f) industry, (g) located in on physical feature, (h) candidate, (i) father, (j) country of citizenship, (k) author, (l) constellation, (m) presenter, (n) inspired by, (o) home port, (p) participant in, (q) color, (r) headquarters location, (s) founded by, (t) parliamentary group", "gold": "(a) relative"} 26 | {"prompt": "X implies illustrator. Therefore, What does X imply? (a) derivative work, (b) has melody, (c) highest point, (d) child, (e) creator, (f) inspired by, (g) head of government, (h) color, (i) place of birth, (j) spouse, (k) home venue, (l) family, (m) corporate officer, (n) developer, (o) participant, (p) parliamentary group, (q) member of sports team, (r) employer, (s) curator, (t) continent", "gold": "(e) creator"} 27 | {"prompt": "X implies employer. Therefore, What does X imply? (a) color, (b) continent, (c) chief executive officer, (d) language used, (e) place of death, (f) family, (g) place of birth, (h) inspired by, (i) rector, (j) affiliation, (k) participant, (l) owner of, (m) country, (n) significant person, (o) industry, (p) spouse, (q) member of sports team, (r) significant event, (s) member of, (t) killed by", "gold": "(j) affiliation"} 28 | {"prompt": "X implies country of citizenship. Therefore, What does X imply? (a) spouse, (b) educated at, (c) has melody, (d) work location, (e) derivative work, (f) presenter, (g) owner of, (h) part of, (i) successful candidate, (j) has subsidiary, (k) located in on physical feature, (l) member of, (m) parent, (n) relative, (o) judge, (p) location, (q) country, (r) developer, (s) residence, (t) place of burial", "gold": "(q) country"} 29 | -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs7-12.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "X's chief executive officer is Y . Therefore, What is the relation between X and Y? (a) presenter, (b) official language, (c) highest point, (d) judge, (e) part of, (f) has parts, (g) chairperson, (h) significant event, (i) architect, (j) father, (k) corporate officer, (l) military branch, (m) significant person, (n) cast member, (o) contributor to the creative work or subject, (p) original language, (q) inspired by, (r) country of citizenship, (s) head of government, (t) member of sports team", "gold": "(k) corporate officer"} 2 | {"prompt": "X is located in the administrative territory of Y . Therefore, What is the relation between X and Y? (a) headquarters location, (b) inspired by, (c) references work, tradition or theory, (d) language used, (e) significant event, (f) country, (g) official language, (h) killed by, (i) dialect of, (j) conflict, (k) candidate, (l) illustrator, (m) director manager, (n) participant, (o) member of, (p) architect, (q) constellation, (r) participating team, (s) location, (t) place of birth", "gold": "(s) location"} 3 | {"prompt": "X was originally created in Y . Therefore, What is the relation between X and Y? (a) head of government, (b) original broadcaster, (c) judge, (d) illustrator, (e) architect, (f) member of sports team, (g) participant in, (h) relative, (i) curator, (j) has parts, (k) winner, (l) language of work or name, (m) place of burial, (n) home venue, (o) composer, (p) part of, (q) child, (r) participant, (s) highest point, (t) constellation", "gold": "(l) language of work or name"} 4 | {"prompt": "X is affiliated with Y . Therefore, What is the relation between X and Y? (a) designed by, (b) home venue, (c) owner of, (d) derivative work, (e) successful candidate, (f) doctoral student, (g) place of birth, (h) part of, (i) language used, (j) parent, (k) judge, (l) original language, (m) author, (n) founded by, (o) creator, (p) continent, (q) constellation, (r) student, (s) rector, (t) work location", "gold": "(h) part of"} 5 | {"prompt": "X's headquarter is in Y . Therefore, What is the relation between X and Y? (a) inflows, (b) child, (c) located in on physical feature, (d) continent, (e) based on, (f) architect, (g) family, (h) curator, (i) chief executive officer, (j) military branch, (k) student, (l) significant event, (m) place of burial, (n) place of birth, (o) language used, (p) member of sports team, (q) developer, (r) mother, (s) location, (t) founded by", "gold": "(s) location"} 6 | {"prompt": "X belongs to the party Y . Therefore, What is the relation between X and Y? (a) mother, (b) located in on physical feature, (c) presenter, (d) significant event, (e) home venue, (f) participating team, (g) has subsidiary, (h) architect, (i) country, (j) derivative work, (k) affiliation, (l) father, (m) country of citizenship, (n) curator, (o) head of government, (p) color, (q) inspired by, (r) has melody, (s) location, (t) highest point", "gold": "(k) affiliation"} 7 | {"prompt": "X is located on landform Y . Therefore, What is the relation between X and Y? (a) student, (b) country, (c) references work, tradition or theory, (d) designed by, (e) affiliation, (f) father, (g) parliamentary group, (h) contributor to the creative work or subject, (i) composer, (j) highest point, (k) place of death, (l) developer, (m) location, (n) part of, (o) inflows, (p) color, (q) streak color, (r) based on, (s) spouse, (t) residence", "gold": "(m) location"} 8 | {"prompt": "X is based on Y . Therefore, What is the relation between X and Y? (a) killed by, (b) residence, (c) presenter, (d) designed by, (e) participating team, (f) family, (g) home venue, (h) participant in, (i) continent, (j) references work, tradition or theory, (k) place of death, (l) mother, (m) has subsidiary, (n) located in the administrative territorial entity, (o) significant person, (p) derivative work, (q) parent, (r) country, (s) highest point, (t) corporate officer", "gold": "(j) references work, tradition or theory"} 9 | {"prompt": "X's government is headed by Y . Therefore, What is the relation between X and Y? (a) employer, (b) chairperson, (c) killed by, (d) student, (e) color, (f) owner of, (g) home port, (h) country of citizenship, (i) derivative work, (j) military branch, (k) successful candidate, (l) cast member, (m) judge, (n) performer, (o) participating team, (p) significant event, (q) part of, (r) architect, (s) location, (t) located in the administrative territorial entity", "gold": "(b) chairperson"} 10 | {"prompt": "X's streak color is Y . Therefore, What is the relation between X and Y? (a) child, (b) creator, (c) head of government, (d) composer, (e) candidate, (f) participant, (g) student, (h) relative, (i) part of, (j) judge, (k) industry, (l) headquarters location, (m) conflict, (n) inspired by, (o) rector, (p) color, (q) father, (r) work location, (s) constellation, (t) significant event", "gold": "(p) color"} 11 | {"prompt": "X's composer is Y . Therefore, What is the relation between X and Y? (a) affiliation, (b) has melody, (c) educated at, (d) references work, tradition or theory, (e) relative, (f) corporate officer, (g) candidate, (h) developer, (i) designed by, (j) spouse, (k) judge, (l) significant place, (m) creator, (n) dialect of, (o) place of death, (p) country of citizenship, (q) rector, (r) parent, (s) member of sports team, (t) participant", "gold": "(m) creator"} 12 | {"prompt": "X's mother is Y . Therefore, What is the relation between X and Y? (a) affiliation, (b) original language, (c) derivative work, (d) official language, (e) military branch, (f) home venue, (g) highest point, (h) dialect of, (i) language used, (j) participant in, (k) significant place, (l) located in on physical feature, (m) country, (n) judge, (o) parent, (p) chief executive officer, (q) inspired by, (r) illustrator, (s) corporate officer, (t) place of burial", "gold": "(o) parent"} 13 | {"prompt": "X lives in Y . Therefore, What is the relation between X and Y? (a) place of burial, (b) cast member, (c) location, (d) relative, (e) candidate, (f) language used, (g) member of sports team, (h) place of death, (i) chairperson, (j) developer, (k) illustrator, (l) original language, (m) work location, (n) rector, (o) founded by, (p) author, (q) architect, (r) original broadcaster, (s) creator, (t) parent", "gold": "(c) location"} 14 | {"prompt": "X was founded by Y . Therefore, What is the relation between X and Y? (a) candidate, (b) owner of, (c) creator, (d) chief executive officer, (e) spouse, (f) located in the administrative territorial entity, (g) participant in, (h) presenter, (i) located in on physical feature, (j) has parts, (k) significant person, (l) winner, (m) dialect of, (n) parent, (o) corporate officer, (p) language of work or name, (q) doctoral student, (r) place of burial, (s) residence, (t) family", "gold": "(c) creator"} 15 | {"prompt": "X's performer is Y . Therefore, What is the relation between X and Y? (a) parliamentary group, (b) chairperson, (c) has subsidiary, (d) located in the administrative territorial entity, (e) rector, (f) family, (g) developer, (h) home port, (i) head of government, (j) place of death, (k) inspired by, (l) employer, (m) affiliation, (n) father, (o) member of sports team, (p) contributor to the creative work or subject, (q) member of, (r) curator, (s) parent, (t) home venue", "gold": "(p) contributor to the creative work or subject"} 16 | {"prompt": "X's home stadium is Y . Therefore, What is the relation between X and Y? (a) father, (b) participating team, (c) place of burial, (d) place of death, (e) illustrator, (f) constellation, (g) relative, (h) color, (i) language used, (j) parliamentary group, (k) home port, (l) has parts, (m) location, (n) continent, (o) designed by, (p) child, (q) candidate, (r) has melody, (s) work location, (t) located in on physical feature", "gold": "(m) location"} 17 | {"prompt": "X played for Y . Therefore, What is the relation between X and Y? (a) member of, (b) significant event, (c) military branch, (d) conflict, (e) director manager, (f) continent, (g) has melody, (h) curator, (i) killed by, (j) family, (k) relative, (l) cast member, (m) doctoral student, (n) color, (o) student, (p) located in on physical feature, (q) headquarters location, (r) winner, (s) developer, (t) highest point", "gold": "(a) member of"} 18 | {"prompt": "X participated in conflict Y . Therefore, What is the relation between X and Y? (a) child, (b) killed by, (c) educated at, (d) illustrator, (e) doctoral student, (f) place of birth, (g) cast member, (h) home port, (i) participant in, (j) student, (k) military branch, (l) significant person, (m) located in the administrative territorial entity, (n) family, (o) member of, (p) original language, (q) developer, (r) founded by, (s) place of burial, (t) inflows", "gold": "(i) participant in"} 19 | {"prompt": "X's doctoral student is Y . Therefore, What is the relation between X and Y? (a) relative, (b) headquarters location, (c) language of work or name, (d) home port, (e) inflows, (f) contributor to the creative work or subject, (g) father, (h) language used, (i) significant person, (j) has subsidiary, (k) spouse, (l) derivative work, (m) streak color, (n) part of, (o) original broadcaster, (p) head of government, (q) member of, (r) cast member, (s) student, (t) successful candidate", "gold": "(s) student"} 20 | {"prompt": "X is buried in Y . Therefore, What is the relation between X and Y? (a) head of government, (b) creator, (c) location, (d) has subsidiary, (e) student, (f) performer, (g) participating team, (h) spouse, (i) parent, (j) industry, (k) illustrator, (l) original broadcaster, (m) successful candidate, (n) country, (o) references work, tradition or theory, (p) continent, (q) language used, (r) based on, (s) developer, (t) significant event", "gold": "(c) location"} 21 | {"prompt": "X has the melody of Y . Therefore, What is the relation between X and Y? (a) founded by, (b) part of, (c) winner, (d) doctoral student, (e) place of burial, (f) spouse, (g) has subsidiary, (h) inflows, (i) place of death, (j) participant in, (k) chief executive officer, (l) judge, (m) language of work or name, (n) original language, (o) references work, tradition or theory, (p) place of birth, (q) rector, (r) presenter, (s) killed by, (t) highest point", "gold": "(o) references work, tradition or theory"} 22 | {"prompt": "X used to work in Y . Therefore, What is the relation between X and Y? (a) place of burial, (b) inspired by, (c) participant, (d) participating team, (e) affiliation, (f) original broadcaster, (g) director manager, (h) composer, (i) founded by, (j) has melody, (k) color, (l) constellation, (m) location, (n) student, (o) author, (p) language used, (q) spouse, (r) streak color, (s) candidate, (t) conflict", "gold": "(m) location"} 23 | {"prompt": "X's participating teams contain Y . Therefore, What is the relation between X and Y? (a) presenter, (b) doctoral student, (c) derivative work, (d) winner, (e) language of work or name, (f) participant in, (g) composer, (h) affiliation, (i) military branch, (j) participant, (k) corporate officer, (l) chairperson, (m) language used, (n) home port, (o) parliamentary group, (p) employer, (q) headquarters location, (r) head of government, (s) successful candidate, (t) student", "gold": "(j) participant"} 24 | {"prompt": "X is located on the continent of Y . Therefore, What is the relation between X and Y? (a) member of sports team, (b) headquarters location, (c) significant place, (d) chairperson, (e) located in on physical feature, (f) family, (g) derivative work, (h) highest point, (i) designed by, (j) residence, (k) doctoral student, (l) home port, (m) contributor to the creative work or subject, (n) parliamentary group, (o) dialect of, (p) inflows, (q) head of government, (r) language used, (s) student, (t) relative", "gold": "(e) located in on physical feature"} 25 | {"prompt": "X's spouse is Y . Therefore, What is the relation between X and Y? (a) relative, (b) family, (c) place of death, (d) original broadcaster, (e) composer, (f) industry, (g) located in on physical feature, (h) candidate, (i) father, (j) country of citizenship, (k) author, (l) constellation, (m) presenter, (n) inspired by, (o) home port, (p) participant in, (q) color, (r) headquarters location, (s) founded by, (t) parliamentary group", "gold": "(a) relative"} 26 | {"prompt": "X is illustrated by Y . Therefore, What is the relation between X and Y? (a) derivative work, (b) has melody, (c) highest point, (d) child, (e) creator, (f) inspired by, (g) head of government, (h) color, (i) place of birth, (j) spouse, (k) home venue, (l) family, (m) corporate officer, (n) developer, (o) participant, (p) parliamentary group, (q) member of sports team, (r) employer, (s) curator, (t) continent", "gold": "(e) creator"} 27 | {"prompt": "X is an employer of Y . Therefore, What is the relation between X and Y? (a) color, (b) continent, (c) chief executive officer, (d) language used, (e) place of death, (f) family, (g) place of birth, (h) inspired by, (i) rector, (j) affiliation, (k) participant, (l) owner of, (m) country, (n) significant person, (o) industry, (p) spouse, (q) member of sports team, (r) significant event, (s) member of, (t) killed by", "gold": "(j) affiliation"} 28 | {"prompt": "X is a citizen of Y . Therefore, What is the relation between X and Y? (a) spouse, (b) educated at, (c) has melody, (d) work location, (e) derivative work, (f) presenter, (g) owner of, (h) part of, (i) successful candidate, (j) has subsidiary, (k) located in on physical feature, (l) member of, (m) parent, (n) relative, (o) judge, (p) location, (q) country, (r) developer, (s) residence, (t) place of burial", "gold": "(q) country"} 29 | -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs3-22.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "One has to be a organisation if a person is an employer of that. X is an employer of Y . Therefore, What is the type of Y? (a) launch pad, (b) bridge, (c) national football league event, (d) curling league, (e) snooker player, (f) BobsleighAthlete, (g) on site mean of transportation, (h) overseas department, (i) baseball season, (j) soccer player, (k) organisation, (l) artist, (m) Port, (n) profession, (o) darts player, (p) periodical literature, (q) subway station, (r) racecourse, (s) shrine, (t) artistic genre", "gold": "(k) organisation"} 2 | {"prompt": "One has to be a person if one is a work's composer. X's composer is Y . Therefore, What is the type of Y? (a) educational institution, (b) reptile, (c) racecourse, (d) Food, (e) Controlled designation of origin wine, (f) cricket league, (g) memorial, (h) album, (i) Wind motor, (j) chemical substance, (k) programming language, (l) architect, (m) baseball player, (n) person, (o) football match, (p) Nobel Prize, (q) sculptor, (r) tax, (s) river, (t) infrastructure", "gold": "(n) person"} 3 | {"prompt": "One has to be a person if one is a scientist's doctoral student. X's doctoral student is Y . Therefore, What is the type of Y? (a) poet, (b) military person, (c) bridge, (d) single, (e) mountain, (f) Community, (g) comics character, (h) deanery, (i) restaurant, (j) Member of a Resistance Movement, (k) cat, (l) Database, (m) historian, (n) rest area, (o) train carriage, (p) parish, (q) Sailor, (r) Theatre director, (s) person, (t) reign", "gold": "(s) person"} 4 | {"prompt": "One has to be a person if an organisation was founded by that. X was founded by Y . Therefore, What is the type of Y? (a) congressman, (b) Manor, (c) television episode, (d) martial artist, (e) farmer, (f) person, (g) Port, (h) decoration, (i) road junction, (j) basketball league, (k) saint, (l) HumanGene, (m) sports club, (n) Gated community, (o) Youtuber, (p) lacrosse player, (q) american football league, (r) wine, (s) government agency, (t) Pandemic", "gold": "(f) person"} 5 | {"prompt": "One has to be a person if one is an election's successful candidate. X's successful candidate is Y . Therefore, What is the type of Y? (a) Christian Patriarch, (b) manhua, (c) Producer, (d) cape, (e) tram station, (f) clerical order, (g) gross domestic product per capita, (h) military conflict, (i) tennis league, (j) golf tournament, (k) boxing league, (l) natural place, (m) cardinal, (n) mineral, (o) board game, (p) sculptor, (q) fictional character, (r) beauty queen, (s) person, (t) parliament", "gold": "(s) person"} 6 | {"prompt": "One has to be a man if one is a person's father. X's father is Y . Therefore, What is the type of Y? (a) electrical substation, (b) gene, (c) man, (d) manhua, (e) artistic genre, (f) escalator, (g) Globular Swarm, (h) Political concept, (i) programming language, (j) valley, (k) archbishop, (l) old territory, (m) Reference, (n) automobile, (o) Spreadsheet, (p) radio station, (q) Star сluster, (r) collection of valuables, (s) high diver, (t) musical", "gold": "(c) man"} 7 | {"prompt": "One has to be a place if a person used to work in that. X used to work in Y . Therefore, What is the type of Y? (a) Quote, (b) space station, (c) hot spring, (d) paintball league, (e) drug, (f) criminal, (g) monarch, (h) chemical substance, (i) competition, (j) casino, (k) sport, (l) academic conference, (m) racecourse, (n) National anthem, (o) place, (p) atoll, (q) novel, (r) year, (s) soccer tournoment, (t) career station", "gold": "(o) place"} 8 | {"prompt": "One has to be a woman if one is a person's mother. X's mother is Y . Therefore, What is the type of Y? (a) speedway team, (b) Browser, (c) district, (d) year, (e) Spy, (f) chancellor, (g) castle, (h) australian football league, (i) sport, (j) arrondissement, (k) field hockey league, (l) high diver, (m) top level domain, (n) psychologist, (o) hockey club, (p) school, (q) forest, (r) guitarist, (s) businessperson, (t) woman", "gold": "(t) woman"} 9 | {"prompt": "One has to be a person if one is a person's student. X's student is Y . Therefore, What is the type of Y? (a) tournament, (b) arena, (c) person, (d) chancellor, (e) political function, (f) lacrosse league, (g) atoll, (h) bacteria, (i) american football player, (j) beer, (k) religious, (l) ski jumper, (m) venue, (n) national collegiate athletic association team season, (o) politician spouse, (p) guitarist, (q) competition, (r) Annotation, (s) meeting, (t) jockey", "gold": "(c) person"} 10 | {"prompt": "One has to be a person if a television show is hosted by that. X is hosted by Y . Therefore, What is the type of Y? (a) fort, (b) country, (c) man, (d) locality, (e) Open Swarm, (f) state, (g) tennis player, (h) roller coaster, (i) rest area, (j) bone, (k) Resume, (l) moving image, (m) conifer, (n) Pandemic, (o) Medicine, (p) Organisation member, (q) military unit, (r) muscle, (s) rebellion, (t) person", "gold": "(t) person"} 11 | {"prompt": "One has to be a person if a person was killed by that. X was killed by Y . Therefore, What is the type of Y? (a) stadium, (b) BobsleighAthlete, (c) coach, (d) political party, (e) Satellite, (f) motorsport racer, (g) Playwright, (h) Sculpture, (i) movie, (j) Instrument, (k) baseball season, (l) Historical settlement, (m) horse rider, (n) Algorithm, (o) borough, (p) results of a sport competition, (q) person, (r) cat, (s) lighthouse, (t) horse", "gold": "(q) person"} 12 | {"prompt": "One has to be a place if an animal was born in that. X was born in Y . Therefore, What is the type of Y? (a) place, (b) hockey club, (c) battery, (d) journalist, (e) award, (f) golf player, (g) Archer Player, (h) lipid, (i) man, (j) motocycle racer, (k) arachnid, (l) Cipher, (m) Lawyer, (n) penalty shoot out, (o) speedway team, (p) murderer, (q) agglomeration, (r) poem, (s) mountain range, (t) football league season", "gold": "(a) place"} 13 | {"prompt": "One has to be a person if one is a work's author. X's author is Y . Therefore, What is the type of Y? (a) on site mean of transportation, (b) lock, (c) overseas department, (d) zoo, (e) television show, (f) person, (g) minister, (h) engine, (i) Reference, (j) office holder, (k) adult pornographic actor, (l) murderer, (m) volleyball player, (n) book, (o) asteroid, (p) Member of a Resistance Movement, (q) basketball team, (r) formula one racing, (s) Employer, (t) law firm", "gold": "(f) person"} 14 | {"prompt": "One has to be a language if a language is a dialect of that. X is a dialect of Y . Therefore, What is the type of Y? (a) poet, (b) clerical administrative region, (c) garden, (d) historic place, (e) Watermill, (f) vodka, (g) language, (h) golf player, (i) museum, (j) province, (k) city, (l) Globular Swarm, (m) project, (n) Painting, (o) musical work, (p) music composer, (q) brewery, (r) sport, (s) prehistorical period, (t) gymnast", "gold": "(g) language"} 15 | {"prompt": "One has to be a musical work if a work has the melody of that. X has the melody of Y . Therefore, What is the type of Y? (a) road tunnel, (b) chemical compound, (c) square, (d) coach, (e) volleyball player, (f) university, (g) museum, (h) province, (i) entomologist, (j) musical work, (k) monument, (l) constellation, (m) area, (n) COVID 19 pandemic, (o) basketball player, (p) comic, (q) sound, (r) wine region, (s) swimmer, (t) dam", "gold": "(j) musical work"} 16 | {"prompt": "One has to be a constellation if a celestial body is part of constellation that. X is part of constellation Y . Therefore, What is the type of Y? (a) GeneLocation, (b) galaxy, (c) constellation, (d) type, (e) bus company, (f) college, (g) standard, (h) tennis tournament, (i) JewishLeader, (j) racecourse, (k) time period, (l) race, (m) politician, (n) underground journal, (o) territory, (p) jockey, (q) sports league, (r) badminton player, (s) school, (t) surfer", "gold": "(c) constellation"} 17 | {"prompt": "One has to be a political party if a person belongs to the party that. X belongs to the party Y . Therefore, What is the type of Y? (a) newspaper, (b) chemical substance, (c) celestial body, (d) parliament, (e) top level domain, (f) volleyball league, (g) World Heritage Site, (h) songwriter, (i) law, (j) senator, (k) political party, (l) Anime, (m) cricket team, (n) olympics, (o) cycling competition, (p) soccer league, (q) multi volume publication, (r) race track, (s) engineer, (t) street", "gold": "(k) political party"} 18 | {"prompt": "One has to be a administrative region if something is located in the administrative territory of that. X is located in the administrative territory of Y . Therefore, What is the type of Y? (a) american football league, (b) launch pad, (c) trade union, (d) space shuttle, (e) conveyor system, (f) river, (g) guitarist, (h) musical, (i) Historical settlement, (j) bridge, (k) Resume, (l) racecourse, (m) administrative region, (n) name, (o) speed skater, (p) time period, (q) fashion designer, (r) results of a sport competition, (s) soap character, (t) province", "gold": "(m) administrative region"} 19 | {"prompt": "One has to be a person if one is a person's child. X's child is Y . Therefore, What is the type of Y? (a) speedway league, (b) route stop, (c) nascar driver, (d) academic journal, (e) natural region, (f) member of parliament, (g) Protocol, (h) Swarm, (i) philosopher, (j) cycad, (k) opera, (l) vaccine, (m) person, (n) disease, (o) language, (p) reign, (q) water polo Player, (r) ski resort, (s) Team member, (t) polysaccharide", "gold": "(m) person"} 20 | {"prompt": "One has to be a agent if one is a work's developer. X's developer is Y . Therefore, What is the type of Y? (a) Browser, (b) Formula One racer, (c) australian football league, (d) brain, (e) handball player, (f) videogames league, (g) pope, (h) Reference, (i) political function, (j) amusement park attraction, (k) Singer, (l) member of parliament, (m) geological period, (n) governor, (o) launch pad, (p) lymph, (q) cheese, (r) agent, (s) racecourse, (t) space shuttle", "gold": "(r) agent"} 21 | {"prompt": "One has to be a sports team if one is a sports event's participating teams. X's participating teams contain Y . Therefore, What is the type of Y? (a) Quote, (b) artery, (c) cabinet of ministers, (d) chess player, (e) sports team, (f) rugby league, (g) jockey, (h) monument, (i) geological period, (j) contest, (k) cycling league, (l) MouseGeneLocation, (m) Biomolecule, (n) linguist, (o) soccer tournoment, (p) volcano, (q) songwriter, (r) academic subject, (s) government agency, (t) newspaper", "gold": "(e) sports team"} 22 | {"prompt": "One has to be a body of water if a body of water inflows that. X inflows Y . Therefore, What is the type of Y? (a) body of water, (b) cross country skier, (c) manga, (d) canoeist, (e) soap character, (f) chemical compound, (g) ice hockey player, (h) Theatre director, (i) GeneLocation, (j) drug, (k) national football league event, (l) prison, (m) Cardinal direction, (n) digital camera, (o) still image, (p) Controlled designation of origin wine, (q) woman, (r) castle, (s) lake, (t) egyptologist", "gold": "(a) body of water"} 23 | {"prompt": "One has to be a architectural structure if one is a sports team's home stadium. X's home stadium is Y . Therefore, What is the type of Y? (a) Community, (b) Vaccination Statistics, (c) street, (d) hockey club, (e) road junction, (f) Cipher, (g) Gated community, (h) tenure, (i) engine, (j) manga, (k) Pretender, (l) convention, (m) MouseGene, (n) amateur boxer, (o) music composer, (p) ambassador, (q) entomologist, (r) architectural structure, (s) ligament, (t) image", "gold": "(r) architectural structure"} 24 | {"prompt": "One has to be a actor if a work has cast member that. X has cast member Y . Therefore, What is the type of Y? (a) classical music composition, (b) speedway league, (c) sports event, (d) motorsport racer, (e) videogames league, (f) water ride, (g) actor, (h) protohistorical period, (i) natural place, (j) team sport, (k) chemical compound, (l) religious organisation, (m) casino, (n) crustacean, (o) list, (p) Place in the Music Charts, (q) district, (r) island, (s) international football league event, (t) canadian football Player", "gold": "(g) actor"} 25 | {"prompt": "One has to be a person if one is a person's spouse. X's spouse is Y . Therefore, What is the type of Y? (a) road tunnel, (b) animal, (c) Archer Player, (d) election, (e) sports club, (f) Vaccination Statistics, (g) on site mean of transportation, (h) File system, (i) Christian Patriarch, (j) model, (k) bowling league, (l) camera, (m) president, (n) given name, (o) person, (p) temple, (q) period of artistic style, (r) multi volume publication, (s) samba school, (t) Artificial Satellite", "gold": "(o) person"} 26 | {"prompt": "One has to be a language if a work was originally created in that. X was originally created in Y . Therefore, What is the type of Y? (a) canoeist, (b) convention, (c) BobsleighAthlete, (d) netball player, (e) Scientific concept, (f) publisher, (g) mineral, (h) language, (i) Controlled designation of origin wine, (j) archbishop, (k) cleric, (l) Wind motor, (m) artistic genre, (n) squash player, (o) synagogue, (p) hockey club, (q) populated place, (r) Wikimedia template, (s) standard, (t) multi volume publication", "gold": "(h) language"} 27 | {"prompt": "One has to be a person if one is an organisation's chief executive officer. X's chief executive officer is Y . Therefore, What is the type of Y? (a) bone, (b) photographer, (c) constellation, (d) embryology, (e) football match, (f) linguist, (g) skater, (h) noble, (i) Browser, (j) person, (k) regency, (l) motocycle racer, (m) historical event, (n) magazine, (o) deputy, (p) cycling competition, (q) canoeist, (r) Employer, (s) team sport, (t) sports league", "gold": "(j) person"} 28 | {"prompt": "One has to be a place if one is a ship's home port. X's home port is Y . Therefore, What is the type of Y? (a) eukaryote, (b) space mission, (c) Supreme Court of the United States case, (d) cabinet of ministers, (e) archbishop, (f) national football league event, (g) period of artistic style, (h) Open Swarm, (i) Controlled designation of origin wine, (j) member of parliament, (k) ski area, (l) rugby club, (m) cycling league, (n) television show, (o) place, (p) streetcar, (q) Capital, (r) pope, (s) Historical settlement, (t) golf tournament", "gold": "(o) place"} 29 | -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs2-22.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "One has to be a work if one has composer. X's composer is Y . Therefore, What is the type of X? (a) Rebbe, (b) sales, (c) basketball player, (d) temple, (e) work, (f) Capital, (g) muscle, (h) Robot, (i) american football player, (j) law, (k) golf league, (l) unit of work, (m) tenure, (n) american football Team, (o) cricketer, (p) deanery, (q) record label, (r) tunnel, (s) standard, (t) Comedy Group", "gold": "(e) work"} 2 | {"prompt": "One has to be a person if one is an employer of an organisation. X is an employer of Y . Therefore, What is the type of X? (a) hockey team, (b) music genre, (c) person, (d) demographics, (e) Water tower, (f) cycling team, (g) football match, (h) ginkgo, (i) taxonomic group, (j) factory, (k) library, (l) penalty shoot out, (m) college coach, (n) route stop, (o) castle, (p) clerical administrative region, (q) mineral, (r) COVID 19 pandemic, (s) cape, (t) case", "gold": "(c) person"} 3 | {"prompt": "One has to be a museum if one has curator. X's curator is Y . Therefore, What is the type of X? (a) wrestler, (b) mine, (c) article, (d) governmental administrative region, (e) Pyramid, (f) grave stone or grave monument, (g) museum, (h) flag, (i) Quote, (j) American Leader, (k) micro region, (l) System of law, (m) hockey club, (n) academic conference, (o) monoclonal antibody, (p) Television director, (q) canoeist, (r) basketball player, (s) Pilot, (t) group", "gold": "(g) museum"} 4 | {"prompt": "One has to be a person if one is buried in a place. X is buried in Y . Therefore, What is the type of X? (a) agent, (b) ski area, (c) formula one racing, (d) military unit, (e) digital camera, (f) university, (g) lymph, (h) casino, (i) person, (j) old territory, (k) office holder, (l) decoration, (m) canton, (n) murderer, (o) speed skater, (p) ligament, (q) solar eclipse, (r) game, (s) military service, (t) vice president", "gold": "(i) person"} 5 | {"prompt": "One has to be a person if one is a citizen of a country. X is a citizen of Y . Therefore, What is the type of X? (a) cycling competition, (b) natural event, (c) garden, (d) monastery, (e) animanga character, (f) Record Office, (g) deity, (h) motorcycle rider, (i) roller coaster, (j) coach, (k) demographics, (l) monarch, (m) topical concept, (n) Blazon, (o) national football league event, (p) mountain range, (q) soccer league season, (r) academic subject, (s) engine, (t) person", "gold": "(t) person"} 6 | {"prompt": "One has to be a person if one played for a sports team. X played for Y . Therefore, What is the type of X? (a) Lawyer, (b) field hockey league, (c) military structure, (d) volleyball player, (e) parliament, (f) Producer, (g) philosopher, (h) politician, (i) road tunnel, (j) Capital of region, (k) video game, (l) brewery, (m) national soccer club, (n) railway tunnel, (o) murderer, (p) Globular Swarm, (q) Protocol, (r) person, (s) Nuclear Power plant, (t) geopolitical organisation", "gold": "(r) person"} 7 | {"prompt": "One has to be a work if one was originally created in a language. X was originally created in Y . Therefore, What is the type of X? (a) overseas department, (b) Gaelic games player, (c) Producer, (d) square, (e) national football league season, (f) asteroid, (g) movie genre, (h) Nebula, (i) lunar crater, (j) work, (k) congressman, (l) Biathlete, (m) church, (n) lock, (o) route of transportation, (p) Vaccination Statistics, (q) personal event, (r) minister, (s) light novel, (t) rebellion", "gold": "(j) work"} 8 | {"prompt": "One has to be a person if one has mother. X's mother is Y . Therefore, What is the type of X? (a) academic conference, (b) volcano, (c) Latter Day Saint, (d) person, (e) religious building, (f) Swarm, (g) ship, (h) Historical region, (i) bone, (j) Band, (k) Food, (l) eukaryote, (m) arachnid, (n) race track, (o) diocese, (p) lieutenant, (q) record label, (r) still image, (s) holiday, (t) DTM racer", "gold": "(d) person"} 9 | {"prompt": "One has to be a language if one is a dialect of a language. X is a dialect of Y . Therefore, What is the type of X? (a) cabinet of ministers, (b) language, (c) golf course, (d) comic, (e) manhwa, (f) poem, (g) battery, (h) Hormone, (i) volleyball league, (j) fungus, (k) publisher, (l) gross domestic product, (m) volleyball player, (n) Latter Day Saint, (o) memorial, (p) music festival, (q) psychologist, (r) historical event, (s) life cycle event, (t) medical specialty", "gold": "(b) language"} 10 | {"prompt": "One has to be a animal if one was born in a place. X was born in Y . Therefore, What is the type of X? (a) historic building, (b) winter sport Player, (c) ambassador, (d) rower, (e) tunnel, (f) music composer, (g) football match, (h) animal, (i) Rebbe, (j) parliament, (k) Organisation member, (l) Police Officer, (m) guitar, (n) Satellite, (o) Annotation, (p) cycling race, (q) flag, (r) term of office, (s) speedway league, (t) National anthem", "gold": "(h) animal"} 11 | {"prompt": "One has to be a educational institution if one has rector. X's rector is Y . Therefore, What is the type of X? (a) sports club, (b) populated place, (c) baseball season, (d) professor, (e) educational institution, (f) Team member, (g) volleyball player, (h) videogames league, (i) volleyball coach, (j) Playboy Playmate, (k) bullfighter, (l) international organisation, (m) Stated Resolution, (n) election, (o) automobile, (p) literary genre, (q) Government Type, (r) route of transportation, (s) solar eclipse, (t) Biological database", "gold": "(e) educational institution"} 12 | {"prompt": "One has to be a organisation if one has headquarter. X's headquarter is in Y . Therefore, What is the type of X? (a) australian football league, (b) person, (c) organisation, (d) parish, (e) tenure, (f) subway station, (g) dam, (h) borough, (i) Pyramid, (j) deputy, (k) town, (l) trade union, (m) songwriter, (n) enzyme, (o) bowling league, (p) valley, (q) poet, (r) Nebula, (s) motorsport racer, (t) Organisation member", "gold": "(c) organisation"} 13 | {"prompt": "One has to be a person if one lives in a place. X lives in Y . Therefore, What is the type of X? (a) darts player, (b) politician, (c) amusement park attraction, (d) unit of work, (e) line of fashion, (f) snooker player, (g) Latter Day Saint, (h) tournament, (i) cleric, (j) tennis league, (k) minister, (l) international football league event, (m) bird, (n) Biathlete, (o) soap character, (p) lipid, (q) person, (r) pope, (s) tax, (t) Blazon", "gold": "(q) person"} 14 | {"prompt": "One has to be a work if one has the melody of a musical work. X has the melody of Y . Therefore, What is the type of X? (a) Cardinal direction, (b) professor, (c) prefecture, (d) Relationship, (e) law, (f) competition, (g) multi volume publication, (h) Resume, (i) Employers' Organisation, (j) Manor, (k) military aircraft, (l) flowering plant, (m) vaccine, (n) work, (o) song, (p) football league season, (q) lymph, (r) Wikimedia template, (s) office holder, (t) horse race", "gold": "(n) work"} 15 | {"prompt": "One has to be a military person if one has service branch. X's service branch is Y . Therefore, What is the type of X? (a) military person, (b) novel, (c) skyscraper, (d) deity, (e) artist discography, (f) vodka, (g) cycling competition, (h) streetcar, (i) cyclist, (j) Vaccination Statistics, (k) military unit, (l) archeologist, (m) American Leader, (n) Eurovision song contest entry, (o) amateur boxer, (p) Painting, (q) cultivated variety, (r) wine region, (s) enzyme, (t) american football player", "gold": "(a) military person"} 16 | {"prompt": "One has to be a person if one is educated at an educational institution. X is educated at Y . Therefore, What is the type of X? (a) arrondissement, (b) mountain pass, (c) chancellor, (d) sports team, (e) person, (f) egyptologist, (g) Rebbe, (h) digital camera, (i) classical music artist, (j) population, (k) subway station, (l) sea, (m) lock, (n) company, (o) light novel, (p) professor, (q) person function, (r) tennis tournament, (s) locomotive, (t) businessperson", "gold": "(e) person"} 17 | {"prompt": "One has to be a election if one has successful candidate. X's successful candidate is Y . Therefore, What is the type of X? (a) temple, (b) periodical literature, (c) software, (d) station, (e) deputy, (f) canadian football league, (g) Sculpture, (h) HumanGeneLocation, (i) award, (j) deanery, (k) sculptor, (l) stream, (m) roller coaster, (n) synagogue, (o) election, (p) Pandemic, (q) tower, (r) brewery, (s) monoclonal antibody, (t) mixed martial arts event", "gold": "(o) election"} 18 | {"prompt": "One has to be a country if one is located on the continent of a continent. X is located on the continent of Y . Therefore, What is the type of X? (a) baseball team, (b) Satellite, (c) GeneLocation, (d) movie genre, (e) automobile, (f) voice actor, (g) country, (h) stadium, (i) Artificial Satellite, (j) inline hockey league, (k) television episode, (l) newspaper, (m) rugby club, (n) population, (o) gross domestic product per capita, (p) surfer, (q) organisation, (r) district water board, (s) canadian football Team, (t) library", "gold": "(g) country"} 19 | {"prompt": "One has to be a populated place if one has offical language. X's offical language is Y . Therefore, What is the type of X? (a) parliament, (b) military structure, (c) shopping mall, (d) manhwa, (e) speed skater, (f) auto racing league, (g) squash player, (h) Ocean, (i) ice hockey league, (j) lipid, (k) conifer, (l) amateur boxer, (m) person function, (n) airport, (o) moving image, (p) radio station, (q) businessperson, (r) brewery, (s) infrastructure, (t) populated place", "gold": "(t) populated place"} 20 | {"prompt": "One has to be a person if one participated in conflict a military conflict. X participated in conflict Y . Therefore, What is the type of X? (a) artery, (b) chef, (c) mammal, (d) cycling competition, (e) soccer club, (f) egyptologist, (g) mollusca, (h) person, (i) horse trainer, (j) engineer, (k) military service, (l) martial artist, (m) reptile, (n) Historical region, (o) Singer, (p) rugby league, (q) comics character, (r) softball league, (s) mixed martial arts league, (t) basketball team", "gold": "(h) person"} 21 | {"prompt": "One has to be a person if one belongs to the party a political party. X belongs to the party Y . Therefore, What is the type of X? (a) soccer player, (b) castle, (c) Sculpture, (d) military service, (e) road, (f) music genre, (g) canadian football Player, (h) hot spring, (i) person, (j) parliament, (k) race track, (l) businessperson, (m) painter, (n) term of office, (o) theatre, (p) blood vessel, (q) image, (r) soccer manager, (s) venue, (t) given name", "gold": "(i) person"} 22 | {"prompt": "One has to be a person if one was killed by a person. X was killed by Y . Therefore, What is the type of X? (a) battery, (b) trade union, (c) anatomical structure, (d) opera, (e) collection of valuables, (f) sport, (g) classical music artist, (h) gross domestic product per capita, (i) JewishLeader, (j) manhua, (k) Lawyer, (l) person, (m) animal, (n) medician, (o) earthquake, (p) geological period, (q) senator, (r) beverage, (s) formula 1 team, (t) fort", "gold": "(l) person"} 23 | {"prompt": "One has to be a organisation if one was founded by a person. X was founded by Y . Therefore, What is the type of X? (a) medician, (b) reptile, (c) space shuttle, (d) cycling competition, (e) cheese, (f) document, (g) comic strip, (h) casino, (i) baronet, (j) BobsleighAthlete, (k) sculptor, (l) netball player, (m) military person, (n) Cardinal direction, (o) settlement, (p) department, (q) cycad, (r) anatomical structure, (s) stream, (t) organisation", "gold": "(t) organisation"} 24 | {"prompt": "One has to be a mineral if one has streak color. X's streak color is Y . Therefore, What is the type of X? (a) place, (b) college coach, (c) Political concept, (d) ginkgo, (e) mineral, (f) rebellion, (g) castle, (h) Star сluster, (i) Comedy Group, (j) atoll, (k) brown dwarf, (l) archbishop, (m) soccer manager, (n) historic place, (o) device, (p) constellation, (q) National anthem, (r) cartoon, (s) sport, (t) politician", "gold": "(e) mineral"} 25 | {"prompt": "One has to be a person if one has student. X's student is Y . Therefore, What is the type of X? (a) Document Type, (b) trade union, (c) artery, (d) research project, (e) poet, (f) sports club, (g) name, (h) contest, (i) Gated community, (j) literary genre, (k) insect, (l) Rebbe, (m) curler, (n) agglomeration, (o) legislature, (p) state, (q) person, (r) Desert, (s) Intercommunality, (t) military unit", "gold": "(q) person"} 26 | {"prompt": "One has to be a person if one has child. X's child is Y . Therefore, What is the type of X? (a) National anthem, (b) person, (c) screenwriter, (d) golf course, (e) tennis player, (f) archeologist, (g) mammal, (h) sports season, (i) ancient area of jurisdiction of a person feudal or of a governmental body, (j) monoclonal antibody, (k) Concentration camp, (l) sport, (m) Team member, (n) judge, (o) light novel, (p) System of law, (q) amusement park attraction, (r) team sport, (s) DTM racer, (t) olympic event", "gold": "(b) person"} 27 | {"prompt": "One has to be a person if one used to work in a place. X used to work in Y . Therefore, What is the type of X? (a) moss, (b) mountain range, (c) crater, (d) research project, (e) basketball player, (f) locomotive, (g) race track, (h) written work, (i) Biathlete, (j) file, (k) Election Diagram, (l) rugby player, (m) medician, (n) chemical element, (o) conifer, (p) plant, (q) college coach, (r) archipelago, (s) Stated Resolution, (t) person", "gold": "(t) person"} 28 | {"prompt": "One has to be a settlement if one has highest point. X's highest point is Y . Therefore, What is the type of X? (a) canadian football Player, (b) Hormone, (c) embryology, (d) sports team, (e) district, (f) Satellite, (g) cultivated variety, (h) animanga character, (i) software, (j) rally driver, (k) earthquake, (l) protected area, (m) cycling team, (n) settlement, (o) shopping mall, (p) radio host, (q) train, (r) amusement park attraction, (s) painter, (t) Cipher", "gold": "(n) settlement"} 29 | {"prompt": "One has to be a work if one has cast member an actor. X has cast member Y . Therefore, What is the type of X? (a) work, (b) Robot, (c) artery, (d) casino, (e) national collegiate athletic association athlete, (f) protein, (g) broadcast network, (h) storm surge, (i) radio program, (j) organ, (k) train carriage, (l) Theatre director, (m) Pretender, (n) DBpedian, (o) body of water, (p) dam, (q) brown dwarf, (r) ligament, (s) Windmill, (t) adult pornographic actor", "gold": "(a) work"} 30 | {"prompt": "One has to be a sports event if one has participating teams. X's participating teams contain Y . Therefore, What is the type of X? (a) meeting, (b) written work, (c) voice actor, (d) lipid, (e) video game, (f) card game, (g) Document Type, (h) archeologist, (i) Biomolecule, (j) natural region, (k) currency, (l) musical, (m) model, (n) television host, (o) woman, (p) sports event, (q) event, (r) streetcar, (s) sports manager, (t) Television director", "gold": "(p) sports event"} 31 | -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs5-22.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "chief executive officer implies corporate officer. X implies chief executive officer. Therefore, What does X imply? (a) presenter, (b) official language, (c) highest point, (d) judge, (e) part of, (f) has parts, (g) chairperson, (h) significant event, (i) architect, (j) father, (k) corporate officer, (l) military branch, (m) significant person, (n) cast member, (o) contributor to the creative work or subject, (p) original language, (q) inspired by, (r) country of citizenship, (s) head of government, (t) member of sports team", "gold": "(k) corporate officer"} 2 | {"prompt": "successful candidate implies winner. X implies successful candidate. Therefore, What does X imply? (a) continent, (b) founded by, (c) inspired by, (d) father, (e) home venue, (f) participant in, (g) country, (h) based on, (i) has subsidiary, (j) corporate officer, (k) participating team, (l) head of government, (m) place of death, (n) winner, (o) owner of, (p) candidate, (q) highest point, (r) participant, (s) location, (t) parent", "gold": "(n) winner"} 3 | {"prompt": "located in the administrative territorial entity implies location. X implies located in the administrative territorial entity. Therefore, What does X imply? (a) headquarters location, (b) inspired by, (c) references work, tradition or theory, (d) language used, (e) significant event, (f) country, (g) official language, (h) killed by, (i) dialect of, (j) conflict, (k) candidate, (l) illustrator, (m) director manager, (n) participant, (o) member of, (p) architect, (q) constellation, (r) participating team, (s) location, (t) place of birth", "gold": "(s) location"} 4 | {"prompt": "original language implies language of work or name. X implies original language. Therefore, What does X imply? (a) head of government, (b) original broadcaster, (c) judge, (d) illustrator, (e) architect, (f) member of sports team, (g) participant in, (h) relative, (i) curator, (j) has parts, (k) winner, (l) language of work or name, (m) place of burial, (n) home venue, (o) composer, (p) part of, (q) child, (r) participant, (s) highest point, (t) constellation", "gold": "(l) language of work or name"} 5 | {"prompt": "affiliation implies part of. X implies affiliation. Therefore, What does X imply? (a) designed by, (b) home venue, (c) owner of, (d) derivative work, (e) successful candidate, (f) doctoral student, (g) place of birth, (h) part of, (i) language used, (j) parent, (k) judge, (l) original language, (m) author, (n) founded by, (o) creator, (p) continent, (q) constellation, (r) student, (s) rector, (t) work location", "gold": "(h) part of"} 6 | {"prompt": "headquarters location implies location. X implies headquarters location. Therefore, What does X imply? (a) inflows, (b) child, (c) located in on physical feature, (d) continent, (e) based on, (f) architect, (g) family, (h) curator, (i) chief executive officer, (j) military branch, (k) student, (l) significant event, (m) place of burial, (n) place of birth, (o) language used, (p) member of sports team, (q) developer, (r) mother, (s) location, (t) founded by", "gold": "(s) location"} 7 | {"prompt": "parliamentary group implies affiliation. X implies parliamentary group. Therefore, What does X imply? (a) mother, (b) located in on physical feature, (c) presenter, (d) significant event, (e) home venue, (f) participating team, (g) has subsidiary, (h) architect, (i) country, (j) derivative work, (k) affiliation, (l) father, (m) country of citizenship, (n) curator, (o) head of government, (p) color, (q) inspired by, (r) has melody, (s) location, (t) highest point", "gold": "(k) affiliation"} 8 | {"prompt": "located in on physical feature implies location. X implies located in on physical feature. Therefore, What does X imply? (a) student, (b) country, (c) references work, tradition or theory, (d) designed by, (e) affiliation, (f) father, (g) parliamentary group, (h) contributor to the creative work or subject, (i) composer, (j) highest point, (k) place of death, (l) developer, (m) location, (n) part of, (o) inflows, (p) color, (q) streak color, (r) based on, (s) spouse, (t) residence", "gold": "(m) location"} 9 | {"prompt": "dialect of implies derivative work. X implies dialect of. Therefore, What does X imply? (a) derivative work, (b) references work, tradition or theory, (c) head of government, (d) curator, (e) significant event, (f) developer, (g) rector, (h) judge, (i) chairperson, (j) student, (k) employer, (l) chief executive officer, (m) place of burial, (n) headquarters location, (o) work location, (p) continent, (q) official language, (r) parliamentary group, (s) residence, (t) conflict", "gold": "(a) derivative work"} 10 | {"prompt": "based on implies references work, tradition or theory. X implies based on. Therefore, What does X imply? (a) killed by, (b) residence, (c) presenter, (d) designed by, (e) participating team, (f) family, (g) home venue, (h) participant in, (i) continent, (j) references work, tradition or theory, (k) place of death, (l) mother, (m) has subsidiary, (n) located in the administrative territorial entity, (o) significant person, (p) derivative work, (q) parent, (r) country, (s) highest point, (t) corporate officer", "gold": "(j) references work, tradition or theory"} 11 | {"prompt": "head of government implies chairperson. X implies head of government. Therefore, What does X imply? (a) employer, (b) chairperson, (c) killed by, (d) student, (e) color, (f) owner of, (g) home port, (h) country of citizenship, (i) derivative work, (j) military branch, (k) successful candidate, (l) cast member, (m) judge, (n) performer, (o) participating team, (p) significant event, (q) part of, (r) architect, (s) location, (t) located in the administrative territorial entity", "gold": "(b) chairperson"} 12 | {"prompt": "streak color implies color. X implies streak color. Therefore, What does X imply? (a) child, (b) creator, (c) head of government, (d) composer, (e) candidate, (f) participant, (g) student, (h) relative, (i) part of, (j) judge, (k) industry, (l) headquarters location, (m) conflict, (n) inspired by, (o) rector, (p) color, (q) father, (r) work location, (s) constellation, (t) significant event", "gold": "(p) color"} 13 | {"prompt": "composer implies creator. X implies composer. Therefore, What does X imply? (a) affiliation, (b) has melody, (c) educated at, (d) references work, tradition or theory, (e) relative, (f) corporate officer, (g) candidate, (h) developer, (i) designed by, (j) spouse, (k) judge, (l) significant place, (m) creator, (n) dialect of, (o) place of death, (p) country of citizenship, (q) rector, (r) parent, (s) member of sports team, (t) participant", "gold": "(m) creator"} 14 | {"prompt": "mother implies parent. X implies mother. Therefore, What does X imply? (a) affiliation, (b) original language, (c) derivative work, (d) official language, (e) military branch, (f) home venue, (g) highest point, (h) dialect of, (i) language used, (j) participant in, (k) significant place, (l) located in on physical feature, (m) country, (n) judge, (o) parent, (p) chief executive officer, (q) inspired by, (r) illustrator, (s) corporate officer, (t) place of burial", "gold": "(o) parent"} 15 | {"prompt": "residence implies location. X implies residence. Therefore, What does X imply? (a) place of burial, (b) cast member, (c) location, (d) relative, (e) candidate, (f) language used, (g) member of sports team, (h) place of death, (i) chairperson, (j) developer, (k) illustrator, (l) original language, (m) work location, (n) rector, (o) founded by, (p) author, (q) architect, (r) original broadcaster, (s) creator, (t) parent", "gold": "(c) location"} 16 | {"prompt": "founded by implies creator. X implies founded by. Therefore, What does X imply? (a) candidate, (b) owner of, (c) creator, (d) chief executive officer, (e) spouse, (f) located in the administrative territorial entity, (g) participant in, (h) presenter, (i) located in on physical feature, (j) has parts, (k) significant person, (l) winner, (m) dialect of, (n) parent, (o) corporate officer, (p) language of work or name, (q) doctoral student, (r) place of burial, (s) residence, (t) family", "gold": "(c) creator"} 17 | {"prompt": "performer implies contributor to the creative work or subject. X implies performer. Therefore, What does X imply? (a) parliamentary group, (b) chairperson, (c) has subsidiary, (d) located in the administrative territorial entity, (e) rector, (f) family, (g) developer, (h) home port, (i) head of government, (j) place of death, (k) inspired by, (l) employer, (m) affiliation, (n) father, (o) member of sports team, (p) contributor to the creative work or subject, (q) member of, (r) curator, (s) parent, (t) home venue", "gold": "(p) contributor to the creative work or subject"} 18 | {"prompt": "home stadium implies location. X implies home stadium. Therefore, What does X imply? (a) father, (b) participating team, (c) place of burial, (d) place of death, (e) illustrator, (f) constellation, (g) relative, (h) color, (i) language used, (j) parliamentary group, (k) home port, (l) has parts, (m) location, (n) continent, (o) designed by, (p) child, (q) candidate, (r) has melody, (s) work location, (t) located in on physical feature", "gold": "(m) location"} 19 | {"prompt": "member of sports team implies member of. X implies member of sports team. Therefore, What does X imply? (a) member of, (b) significant event, (c) military branch, (d) conflict, (e) director manager, (f) continent, (g) has melody, (h) curator, (i) killed by, (j) family, (k) relative, (l) cast member, (m) doctoral student, (n) color, (o) student, (p) located in on physical feature, (q) headquarters location, (r) winner, (s) developer, (t) highest point", "gold": "(a) member of"} 20 | {"prompt": "designed by implies creator. X implies designed by. Therefore, What does X imply? (a) creator, (b) parliamentary group, (c) member of, (d) developer, (e) relative, (f) located in on physical feature, (g) successful candidate, (h) conflict, (i) killed by, (j) affiliation, (k) author, (l) work location, (m) original language, (n) participant in, (o) judge, (p) location, (q) family, (r) place of burial, (s) headquarters location, (t) industry", "gold": "(a) creator"} 21 | {"prompt": "conflict implies participant in. X implies conflict. Therefore, What does X imply? (a) child, (b) killed by, (c) educated at, (d) illustrator, (e) doctoral student, (f) place of birth, (g) cast member, (h) home port, (i) participant in, (j) student, (k) military branch, (l) significant person, (m) located in the administrative territorial entity, (n) family, (o) member of, (p) original language, (q) developer, (r) founded by, (s) place of burial, (t) inflows", "gold": "(i) participant in"} 22 | {"prompt": "doctoral student implies student. X implies doctoral student. Therefore, What does X imply? (a) relative, (b) headquarters location, (c) language of work or name, (d) home port, (e) inflows, (f) contributor to the creative work or subject, (g) father, (h) language used, (i) significant person, (j) has subsidiary, (k) spouse, (l) derivative work, (m) streak color, (n) part of, (o) original broadcaster, (p) head of government, (q) member of, (r) cast member, (s) student, (t) successful candidate", "gold": "(s) student"} 23 | {"prompt": "place of burial implies location. X implies place of burial. Therefore, What does X imply? (a) head of government, (b) creator, (c) location, (d) has subsidiary, (e) student, (f) performer, (g) participating team, (h) spouse, (i) parent, (j) industry, (k) illustrator, (l) original broadcaster, (m) successful candidate, (n) country, (o) references work, tradition or theory, (p) continent, (q) language used, (r) based on, (s) developer, (t) significant event", "gold": "(c) location"} 24 | {"prompt": "rector implies director manager. X implies rector. Therefore, What does X imply? (a) located in on physical feature, (b) streak color, (c) original language, (d) director manager, (e) illustrator, (f) conflict, (g) residence, (h) color, (i) head of government, (j) participating team, (k) place of burial, (l) military branch, (m) original broadcaster, (n) industry, (o) curator, (p) relative, (q) chief executive officer, (r) presenter, (s) significant event, (t) developer", "gold": "(d) director manager"} 25 | {"prompt": "has melody implies references work, tradition or theory. X implies has melody. Therefore, What does X imply? (a) founded by, (b) part of, (c) winner, (d) doctoral student, (e) place of burial, (f) spouse, (g) has subsidiary, (h) inflows, (i) place of death, (j) participant in, (k) chief executive officer, (l) judge, (m) language of work or name, (n) original language, (o) references work, tradition or theory, (p) place of birth, (q) rector, (r) presenter, (s) killed by, (t) highest point", "gold": "(o) references work, tradition or theory"} 26 | {"prompt": "educated at implies affiliation. X implies educated at. Therefore, What does X imply? (a) successful candidate, (b) based on, (c) creator, (d) place of burial, (e) participant, (f) affiliation, (g) architect, (h) member of, (i) parent, (j) has parts, (k) owner of, (l) has melody, (m) references work, tradition or theory, (n) cast member, (o) employer, (p) parliamentary group, (q) constellation, (r) student, (s) winner, (t) rector", "gold": "(f) affiliation"} 27 | {"prompt": "work location implies location. X implies work location. Therefore, What does X imply? (a) place of burial, (b) inspired by, (c) participant, (d) participating team, (e) affiliation, (f) original broadcaster, (g) director manager, (h) composer, (i) founded by, (j) has melody, (k) color, (l) constellation, (m) location, (n) student, (o) author, (p) language used, (q) spouse, (r) streak color, (s) candidate, (t) conflict", "gold": "(m) location"} 28 | {"prompt": "cast member implies significant person. X implies cast member. Therefore, What does X imply? (a) significant event, (b) parent, (c) headquarters location, (d) location, (e) chief executive officer, (f) significant person, (g) language used, (h) located in the administrative territorial entity, (i) author, (j) located in on physical feature, (k) candidate, (l) head of government, (m) inflows, (n) chairperson, (o) place of burial, (p) educated at, (q) contributor to the creative work or subject, (r) designed by, (s) conflict, (t) has subsidiary", "gold": "(f) significant person"} 29 | {"prompt": "curator implies significant person. X implies curator. Therefore, What does X imply? (a) corporate officer, (b) affiliation, (c) original language, (d) location, (e) spouse, (f) official language, (g) member of sports team, (h) rector, (i) language of work or name, (j) successful candidate, (k) architect, (l) dialect of, (m) industry, (n) significant person, (o) original broadcaster, (p) part of, (q) participating team, (r) color, (s) employer, (t) mother", "gold": "(n) significant person"} 30 | {"prompt": "participant implies significant person. X implies participant. Therefore, What does X imply? (a) child, (b) father, (c) corporate officer, (d) significant place, (e) located in on physical feature, (f) relative, (g) has subsidiary, (h) owner of, (i) significant person, (j) employer, (k) place of birth, (l) mother, (m) official language, (n) winner, (o) performer, (p) original language, (q) language of work or name, (r) significant event, (s) judge, (t) member of", "gold": "(i) significant person"} 31 | {"prompt": "participating team implies participant. X implies participating team. Therefore, What does X imply? (a) presenter, (b) doctoral student, (c) derivative work, (d) winner, (e) language of work or name, (f) participant in, (g) composer, (h) affiliation, (i) military branch, (j) participant, (k) corporate officer, (l) chairperson, (m) language used, (n) home port, (o) parliamentary group, (p) employer, (q) headquarters location, (r) head of government, (s) successful candidate, (t) student", "gold": "(j) participant"} 32 | {"prompt": "architect implies designed by. X implies architect. Therefore, What does X imply? (a) father, (b) located in on physical feature, (c) references work, tradition or theory, (d) significant event, (e) member of sports team, (f) military branch, (g) place of birth, (h) cast member, (i) designed by, (j) significant person, (k) affiliation, (l) performer, (m) relative, (n) successful candidate, (o) place of burial, (p) conflict, (q) curator, (r) mother, (s) language used, (t) parliamentary group", "gold": "(i) designed by"} 33 | {"prompt": "continent implies located in on physical feature. X implies continent. Therefore, What does X imply? (a) member of sports team, (b) headquarters location, (c) significant place, (d) chairperson, (e) located in on physical feature, (f) family, (g) derivative work, (h) highest point, (i) designed by, (j) residence, (k) doctoral student, (l) home port, (m) contributor to the creative work or subject, (n) parliamentary group, (o) dialect of, (p) inflows, (q) head of government, (r) language used, (s) student, (t) relative", "gold": "(e) located in on physical feature"} 34 | {"prompt": "spouse implies relative. X implies spouse. Therefore, What does X imply? (a) relative, (b) family, (c) place of death, (d) original broadcaster, (e) composer, (f) industry, (g) located in on physical feature, (h) candidate, (i) father, (j) country of citizenship, (k) author, (l) constellation, (m) presenter, (n) inspired by, (o) home port, (p) participant in, (q) color, (r) headquarters location, (s) founded by, (t) parliamentary group", "gold": "(a) relative"} 35 | {"prompt": "illustrator implies creator. X implies illustrator. Therefore, What does X imply? (a) derivative work, (b) has melody, (c) highest point, (d) child, (e) creator, (f) inspired by, (g) head of government, (h) color, (i) place of birth, (j) spouse, (k) home venue, (l) family, (m) corporate officer, (n) developer, (o) participant, (p) parliamentary group, (q) member of sports team, (r) employer, (s) curator, (t) continent", "gold": "(e) creator"} 36 | {"prompt": "contributor implies participant. X implies contributor. Therefore, What does X imply? (a) has parts, (b) participant, (c) founded by, (d) affiliation, (e) color, (f) significant person, (g) creator, (h) author, (i) place of death, (j) director manager, (k) corporate officer, (l) head of government, (m) family, (n) student, (o) derivative work, (p) employer, (q) constellation, (r) home venue, (s) cast member, (t) owner of", "gold": "(b) participant"} 37 | {"prompt": "employer implies affiliation. X implies employer. Therefore, What does X imply? (a) color, (b) continent, (c) chief executive officer, (d) language used, (e) place of death, (f) family, (g) place of birth, (h) inspired by, (i) rector, (j) affiliation, (k) participant, (l) owner of, (m) country, (n) significant person, (o) industry, (p) spouse, (q) member of sports team, (r) significant event, (s) member of, (t) killed by", "gold": "(j) affiliation"} 38 | {"prompt": "constellation implies part of. X implies constellation. Therefore, What does X imply? (a) highest point, (b) chairperson, (c) headquarters location, (d) continent, (e) student, (f) performer, (g) significant place, (h) child, (i) successful candidate, (j) has parts, (k) developer, (l) author, (m) references work, tradition or theory, (n) part of, (o) inspired by, (p) owner of, (q) residence, (r) parliamentary group, (s) director manager, (t) participant", "gold": "(n) part of"} 39 | {"prompt": "country of citizenship implies country. X implies country of citizenship. Therefore, What does X imply? (a) spouse, (b) educated at, (c) has melody, (d) work location, (e) derivative work, (f) presenter, (g) owner of, (h) part of, (i) successful candidate, (j) has subsidiary, (k) located in on physical feature, (l) member of, (m) parent, (n) relative, (o) judge, (p) location, (q) country, (r) developer, (s) residence, (t) place of burial", "gold": "(q) country"} 40 | -------------------------------------------------------------------------------- /test-gpt/data-gpt/reasoning/rdfs7-22.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "chief executive officer implies X's corporate officer is Y .. X's chief executive officer is Y . Therefore, What is the relation between X and Y? (a) presenter, (b) official language, (c) highest point, (d) judge, (e) part of, (f) has parts, (g) chairperson, (h) significant event, (i) architect, (j) father, (k) corporate officer, (l) military branch, (m) significant person, (n) cast member, (o) contributor to the creative work or subject, (p) original language, (q) inspired by, (r) country of citizenship, (s) head of government, (t) member of sports team", "gold": "(k) corporate officer"} 2 | {"prompt": "successful candidate implies X's winner is Y .. X's successful candidate is Y . Therefore, What is the relation between X and Y? (a) continent, (b) founded by, (c) inspired by, (d) father, (e) home venue, (f) participant in, (g) country, (h) based on, (i) has subsidiary, (j) corporate officer, (k) participating team, (l) head of government, (m) place of death, (n) winner, (o) owner of, (p) candidate, (q) highest point, (r) participant, (s) location, (t) parent", "gold": "(n) winner"} 3 | {"prompt": "located in the administrative territorial entity implies X is located in Y .. X is located in the administrative territory of Y . Therefore, What is the relation between X and Y? (a) headquarters location, (b) inspired by, (c) references work, tradition or theory, (d) language used, (e) significant event, (f) country, (g) official language, (h) killed by, (i) dialect of, (j) conflict, (k) candidate, (l) illustrator, (m) director manager, (n) participant, (o) member of, (p) architect, (q) constellation, (r) participating team, (s) location, (t) place of birth", "gold": "(s) location"} 4 | {"prompt": "original language implies X's language is Y .. X was originally created in Y . Therefore, What is the relation between X and Y? (a) head of government, (b) original broadcaster, (c) judge, (d) illustrator, (e) architect, (f) member of sports team, (g) participant in, (h) relative, (i) curator, (j) has parts, (k) winner, (l) language of work or name, (m) place of burial, (n) home venue, (o) composer, (p) part of, (q) child, (r) participant, (s) highest point, (t) constellation", "gold": "(l) language of work or name"} 5 | {"prompt": "affiliation implies X is part of Y .. X is affiliated with Y . Therefore, What is the relation between X and Y? (a) designed by, (b) home venue, (c) owner of, (d) derivative work, (e) successful candidate, (f) doctoral student, (g) place of birth, (h) part of, (i) language used, (j) parent, (k) judge, (l) original language, (m) author, (n) founded by, (o) creator, (p) continent, (q) constellation, (r) student, (s) rector, (t) work location", "gold": "(h) part of"} 6 | {"prompt": "headquarters location implies X is located in Y .. X's headquarter is in Y . Therefore, What is the relation between X and Y? (a) inflows, (b) child, (c) located in on physical feature, (d) continent, (e) based on, (f) architect, (g) family, (h) curator, (i) chief executive officer, (j) military branch, (k) student, (l) significant event, (m) place of burial, (n) place of birth, (o) language used, (p) member of sports team, (q) developer, (r) mother, (s) location, (t) founded by", "gold": "(s) location"} 7 | {"prompt": "parliamentary group implies X is affiliated with Y .. X belongs to the party Y . Therefore, What is the relation between X and Y? (a) mother, (b) located in on physical feature, (c) presenter, (d) significant event, (e) home venue, (f) participating team, (g) has subsidiary, (h) architect, (i) country, (j) derivative work, (k) affiliation, (l) father, (m) country of citizenship, (n) curator, (o) head of government, (p) color, (q) inspired by, (r) has melody, (s) location, (t) highest point", "gold": "(k) affiliation"} 8 | {"prompt": "located in on physical feature implies X is located in Y .. X is located on landform Y . Therefore, What is the relation between X and Y? (a) student, (b) country, (c) references work, tradition or theory, (d) designed by, (e) affiliation, (f) father, (g) parliamentary group, (h) contributor to the creative work or subject, (i) composer, (j) highest point, (k) place of death, (l) developer, (m) location, (n) part of, (o) inflows, (p) color, (q) streak color, (r) based on, (s) spouse, (t) residence", "gold": "(m) location"} 9 | {"prompt": "dialect of implies X is derived from Y .. X is a dialect of Y . Therefore, What is the relation between X and Y? (a) derivative work, (b) references work, tradition or theory, (c) head of government, (d) curator, (e) significant event, (f) developer, (g) rector, (h) judge, (i) chairperson, (j) student, (k) employer, (l) chief executive officer, (m) place of burial, (n) headquarters location, (o) work location, (p) continent, (q) official language, (r) parliamentary group, (s) residence, (t) conflict", "gold": "(a) derivative work"} 10 | {"prompt": "based on implies X references Y .. X is based on Y . Therefore, What is the relation between X and Y? (a) killed by, (b) residence, (c) presenter, (d) designed by, (e) participating team, (f) family, (g) home venue, (h) participant in, (i) continent, (j) references work, tradition or theory, (k) place of death, (l) mother, (m) has subsidiary, (n) located in the administrative territorial entity, (o) significant person, (p) derivative work, (q) parent, (r) country, (s) highest point, (t) corporate officer", "gold": "(j) references work, tradition or theory"} 11 | {"prompt": "head of government implies X's chairperson is Y .. X's government is headed by Y . Therefore, What is the relation between X and Y? (a) employer, (b) chairperson, (c) killed by, (d) student, (e) color, (f) owner of, (g) home port, (h) country of citizenship, (i) derivative work, (j) military branch, (k) successful candidate, (l) cast member, (m) judge, (n) performer, (o) participating team, (p) significant event, (q) part of, (r) architect, (s) location, (t) located in the administrative territorial entity", "gold": "(b) chairperson"} 12 | {"prompt": "streak color implies X's color is Y .. X's streak color is Y . Therefore, What is the relation between X and Y? (a) child, (b) creator, (c) head of government, (d) composer, (e) candidate, (f) participant, (g) student, (h) relative, (i) part of, (j) judge, (k) industry, (l) headquarters location, (m) conflict, (n) inspired by, (o) rector, (p) color, (q) father, (r) work location, (s) constellation, (t) significant event", "gold": "(p) color"} 13 | {"prompt": "composer implies X is created by Y .. X's composer is Y . Therefore, What is the relation between X and Y? (a) affiliation, (b) has melody, (c) educated at, (d) references work, tradition or theory, (e) relative, (f) corporate officer, (g) candidate, (h) developer, (i) designed by, (j) spouse, (k) judge, (l) significant place, (m) creator, (n) dialect of, (o) place of death, (p) country of citizenship, (q) rector, (r) parent, (s) member of sports team, (t) participant", "gold": "(m) creator"} 14 | {"prompt": "mother implies X's parent is Y .. X's mother is Y . Therefore, What is the relation between X and Y? (a) affiliation, (b) original language, (c) derivative work, (d) official language, (e) military branch, (f) home venue, (g) highest point, (h) dialect of, (i) language used, (j) participant in, (k) significant place, (l) located in on physical feature, (m) country, (n) judge, (o) parent, (p) chief executive officer, (q) inspired by, (r) illustrator, (s) corporate officer, (t) place of burial", "gold": "(o) parent"} 15 | {"prompt": "residence implies X is located in Y .. X lives in Y . Therefore, What is the relation between X and Y? (a) place of burial, (b) cast member, (c) location, (d) relative, (e) candidate, (f) language used, (g) member of sports team, (h) place of death, (i) chairperson, (j) developer, (k) illustrator, (l) original language, (m) work location, (n) rector, (o) founded by, (p) author, (q) architect, (r) original broadcaster, (s) creator, (t) parent", "gold": "(c) location"} 16 | {"prompt": "founded by implies X is created by Y .. X was founded by Y . Therefore, What is the relation between X and Y? (a) candidate, (b) owner of, (c) creator, (d) chief executive officer, (e) spouse, (f) located in the administrative territorial entity, (g) participant in, (h) presenter, (i) located in on physical feature, (j) has parts, (k) significant person, (l) winner, (m) dialect of, (n) parent, (o) corporate officer, (p) language of work or name, (q) doctoral student, (r) place of burial, (s) residence, (t) family", "gold": "(c) creator"} 17 | {"prompt": "performer implies X's contributor is Y .. X's performer is Y . Therefore, What is the relation between X and Y? (a) parliamentary group, (b) chairperson, (c) has subsidiary, (d) located in the administrative territorial entity, (e) rector, (f) family, (g) developer, (h) home port, (i) head of government, (j) place of death, (k) inspired by, (l) employer, (m) affiliation, (n) father, (o) member of sports team, (p) contributor to the creative work or subject, (q) member of, (r) curator, (s) parent, (t) home venue", "gold": "(p) contributor to the creative work or subject"} 18 | {"prompt": "home stadium implies X is located in Y .. X's home stadium is Y . Therefore, What is the relation between X and Y? (a) father, (b) participating team, (c) place of burial, (d) place of death, (e) illustrator, (f) constellation, (g) relative, (h) color, (i) language used, (j) parliamentary group, (k) home port, (l) has parts, (m) location, (n) continent, (o) designed by, (p) child, (q) candidate, (r) has melody, (s) work location, (t) located in on physical feature", "gold": "(m) location"} 19 | {"prompt": "member of sports team implies X is member of Y .. X played for Y . Therefore, What is the relation between X and Y? (a) member of, (b) significant event, (c) military branch, (d) conflict, (e) director manager, (f) continent, (g) has melody, (h) curator, (i) killed by, (j) family, (k) relative, (l) cast member, (m) doctoral student, (n) color, (o) student, (p) located in on physical feature, (q) headquarters location, (r) winner, (s) developer, (t) highest point", "gold": "(a) member of"} 20 | {"prompt": "designed by implies X is created by Y .. X is designed by Y . Therefore, What is the relation between X and Y? (a) creator, (b) parliamentary group, (c) member of, (d) developer, (e) relative, (f) located in on physical feature, (g) successful candidate, (h) conflict, (i) killed by, (j) affiliation, (k) author, (l) work location, (m) original language, (n) participant in, (o) judge, (p) location, (q) family, (r) place of burial, (s) headquarters location, (t) industry", "gold": "(a) creator"} 21 | {"prompt": "conflict implies X participated in Y .. X participated in conflict Y . Therefore, What is the relation between X and Y? (a) child, (b) killed by, (c) educated at, (d) illustrator, (e) doctoral student, (f) place of birth, (g) cast member, (h) home port, (i) participant in, (j) student, (k) military branch, (l) significant person, (m) located in the administrative territorial entity, (n) family, (o) member of, (p) original language, (q) developer, (r) founded by, (s) place of burial, (t) inflows", "gold": "(i) participant in"} 22 | {"prompt": "doctoral student implies X's student is Y .. X's doctoral student is Y . Therefore, What is the relation between X and Y? (a) relative, (b) headquarters location, (c) language of work or name, (d) home port, (e) inflows, (f) contributor to the creative work or subject, (g) father, (h) language used, (i) significant person, (j) has subsidiary, (k) spouse, (l) derivative work, (m) streak color, (n) part of, (o) original broadcaster, (p) head of government, (q) member of, (r) cast member, (s) student, (t) successful candidate", "gold": "(s) student"} 23 | {"prompt": "place of burial implies X is located in Y .. X is buried in Y . Therefore, What is the relation between X and Y? (a) head of government, (b) creator, (c) location, (d) has subsidiary, (e) student, (f) performer, (g) participating team, (h) spouse, (i) parent, (j) industry, (k) illustrator, (l) original broadcaster, (m) successful candidate, (n) country, (o) references work, tradition or theory, (p) continent, (q) language used, (r) based on, (s) developer, (t) significant event", "gold": "(c) location"} 24 | {"prompt": "rector implies X is managed by Y .. X's rector is Y . Therefore, What is the relation between X and Y? (a) located in on physical feature, (b) streak color, (c) original language, (d) director manager, (e) illustrator, (f) conflict, (g) residence, (h) color, (i) head of government, (j) participating team, (k) place of burial, (l) military branch, (m) original broadcaster, (n) industry, (o) curator, (p) relative, (q) chief executive officer, (r) presenter, (s) significant event, (t) developer", "gold": "(d) director manager"} 25 | {"prompt": "has melody implies X references Y .. X has the melody of Y . Therefore, What is the relation between X and Y? (a) founded by, (b) part of, (c) winner, (d) doctoral student, (e) place of burial, (f) spouse, (g) has subsidiary, (h) inflows, (i) place of death, (j) participant in, (k) chief executive officer, (l) judge, (m) language of work or name, (n) original language, (o) references work, tradition or theory, (p) place of birth, (q) rector, (r) presenter, (s) killed by, (t) highest point", "gold": "(o) references work, tradition or theory"} 26 | {"prompt": "educated at implies X is affiliated with Y .. X is educated at Y . Therefore, What is the relation between X and Y? (a) successful candidate, (b) based on, (c) creator, (d) place of burial, (e) participant, (f) affiliation, (g) architect, (h) member of, (i) parent, (j) has parts, (k) owner of, (l) has melody, (m) references work, tradition or theory, (n) cast member, (o) employer, (p) parliamentary group, (q) constellation, (r) student, (s) winner, (t) rector", "gold": "(f) affiliation"} 27 | {"prompt": "work location implies X is located in Y .. X used to work in Y . Therefore, What is the relation between X and Y? (a) place of burial, (b) inspired by, (c) participant, (d) participating team, (e) affiliation, (f) original broadcaster, (g) director manager, (h) composer, (i) founded by, (j) has melody, (k) color, (l) constellation, (m) location, (n) student, (o) author, (p) language used, (q) spouse, (r) streak color, (s) candidate, (t) conflict", "gold": "(m) location"} 28 | {"prompt": "cast member implies X's significant person is Y .. X's cast member is Y . Therefore, What is the relation between X and Y? (a) significant event, (b) parent, (c) headquarters location, (d) location, (e) chief executive officer, (f) significant person, (g) language used, (h) located in the administrative territorial entity, (i) author, (j) located in on physical feature, (k) candidate, (l) head of government, (m) inflows, (n) chairperson, (o) place of burial, (p) educated at, (q) contributor to the creative work or subject, (r) designed by, (s) conflict, (t) has subsidiary", "gold": "(f) significant person"} 29 | {"prompt": "curator implies X's significant person is Y .. X's curator is Y . Therefore, What is the relation between X and Y? (a) corporate officer, (b) affiliation, (c) original language, (d) location, (e) spouse, (f) official language, (g) member of sports team, (h) rector, (i) language of work or name, (j) successful candidate, (k) architect, (l) dialect of, (m) industry, (n) significant person, (o) original broadcaster, (p) part of, (q) participating team, (r) color, (s) employer, (t) mother", "gold": "(n) significant person"} 30 | {"prompt": "participant implies X's significant person is Y .. X's participant is Y . Therefore, What is the relation between X and Y? (a) child, (b) father, (c) corporate officer, (d) significant place, (e) located in on physical feature, (f) relative, (g) has subsidiary, (h) owner of, (i) significant person, (j) employer, (k) place of birth, (l) mother, (m) official language, (n) winner, (o) performer, (p) original language, (q) language of work or name, (r) significant event, (s) judge, (t) member of", "gold": "(i) significant person"} 31 | {"prompt": "participating team implies X's participant is Y .. X's participating teams contain Y . Therefore, What is the relation between X and Y? (a) presenter, (b) doctoral student, (c) derivative work, (d) winner, (e) language of work or name, (f) participant in, (g) composer, (h) affiliation, (i) military branch, (j) participant, (k) corporate officer, (l) chairperson, (m) language used, (n) home port, (o) parliamentary group, (p) employer, (q) headquarters location, (r) head of government, (s) successful candidate, (t) student", "gold": "(j) participant"} 32 | {"prompt": "architect implies X is designed by Y .. X's architect is Y . Therefore, What is the relation between X and Y? (a) father, (b) located in on physical feature, (c) references work, tradition or theory, (d) significant event, (e) member of sports team, (f) military branch, (g) place of birth, (h) cast member, (i) designed by, (j) significant person, (k) affiliation, (l) performer, (m) relative, (n) successful candidate, (o) place of burial, (p) conflict, (q) curator, (r) mother, (s) language used, (t) parliamentary group", "gold": "(i) designed by"} 33 | {"prompt": "continent implies X is located on landform Y .. X is located on the continent of Y . Therefore, What is the relation between X and Y? (a) member of sports team, (b) headquarters location, (c) significant place, (d) chairperson, (e) located in on physical feature, (f) family, (g) derivative work, (h) highest point, (i) designed by, (j) residence, (k) doctoral student, (l) home port, (m) contributor to the creative work or subject, (n) parliamentary group, (o) dialect of, (p) inflows, (q) head of government, (r) language used, (s) student, (t) relative", "gold": "(e) located in on physical feature"} 34 | {"prompt": "spouse implies X's relative is Y .. X's spouse is Y . Therefore, What is the relation between X and Y? (a) relative, (b) family, (c) place of death, (d) original broadcaster, (e) composer, (f) industry, (g) located in on physical feature, (h) candidate, (i) father, (j) country of citizenship, (k) author, (l) constellation, (m) presenter, (n) inspired by, (o) home port, (p) participant in, (q) color, (r) headquarters location, (s) founded by, (t) parliamentary group", "gold": "(a) relative"} 35 | {"prompt": "illustrator implies X is created by Y .. X is illustrated by Y . Therefore, What is the relation between X and Y? (a) derivative work, (b) has melody, (c) highest point, (d) child, (e) creator, (f) inspired by, (g) head of government, (h) color, (i) place of birth, (j) spouse, (k) home venue, (l) family, (m) corporate officer, (n) developer, (o) participant, (p) parliamentary group, (q) member of sports team, (r) employer, (s) curator, (t) continent", "gold": "(e) creator"} 36 | {"prompt": "contributor implies X's participant is Y .. X's contributor is Y . Therefore, What is the relation between X and Y? (a) has parts, (b) participant, (c) founded by, (d) affiliation, (e) color, (f) significant person, (g) creator, (h) author, (i) place of death, (j) director manager, (k) corporate officer, (l) head of government, (m) family, (n) student, (o) derivative work, (p) employer, (q) constellation, (r) home venue, (s) cast member, (t) owner of", "gold": "(b) participant"} 37 | {"prompt": "employer implies X is affiliated with Y .. X is an employer of Y . Therefore, What is the relation between X and Y? (a) color, (b) continent, (c) chief executive officer, (d) language used, (e) place of death, (f) family, (g) place of birth, (h) inspired by, (i) rector, (j) affiliation, (k) participant, (l) owner of, (m) country, (n) significant person, (o) industry, (p) spouse, (q) member of sports team, (r) significant event, (s) member of, (t) killed by", "gold": "(j) affiliation"} 38 | {"prompt": "constellation implies X is part of Y .. X is part of constellation Y . Therefore, What is the relation between X and Y? (a) highest point, (b) chairperson, (c) headquarters location, (d) continent, (e) student, (f) performer, (g) significant place, (h) child, (i) successful candidate, (j) has parts, (k) developer, (l) author, (m) references work, tradition or theory, (n) part of, (o) inspired by, (p) owner of, (q) residence, (r) parliamentary group, (s) director manager, (t) participant", "gold": "(n) part of"} 39 | {"prompt": "country of citizenship implies X's country is Y .. X is a citizen of Y . Therefore, What is the relation between X and Y? (a) spouse, (b) educated at, (c) has melody, (d) work location, (e) derivative work, (f) presenter, (g) owner of, (h) part of, (i) successful candidate, (j) has subsidiary, (k) located in on physical feature, (l) member of, (m) parent, (n) relative, (o) judge, (p) location, (q) country, (r) developer, (s) residence, (t) place of burial", "gold": "(q) country"} 40 | -------------------------------------------------------------------------------- /test-gpt/data-gpt/memorizing/range.jsonl: -------------------------------------------------------------------------------- 1 | {"prompt": "What is the type of it if a person is an employer of it? (a) launch pad, (b) bridge, (c) national football league event, (d) curling league, (e) snooker player, (f) BobsleighAthlete, (g) on site mean of transportation, (h) overseas department, (i) baseball season, (j) soccer player, (k) organisation, (l) artist, (m) Port, (n) profession, (o) darts player, (p) periodical literature, (q) subway station, (r) racecourse, (s) shrine, (t) artistic genre", "gold": "(k) organisation", "cands": ["launch pad", "bridge", "national football league event", "curling league", "snooker player", "BobsleighAthlete", "on site mean of transportation", "overseas department", "baseball season", "soccer player", "organisation", "artist", "Port", "profession", "darts player", "periodical literature", "subway station", "racecourse", "shrine", "artistic genre"]} 2 | {"prompt": "What is the type of it if one is a work's composer? (a) educational institution, (b) reptile, (c) racecourse, (d) Food, (e) Controlled designation of origin wine, (f) cricket league, (g) memorial, (h) album, (i) Wind motor, (j) chemical substance, (k) programming language, (l) architect, (m) baseball player, (n) person, (o) football match, (p) Nobel Prize, (q) sculptor, (r) tax, (s) river, (t) infrastructure", "gold": "(n) person", "cands": ["educational institution", "reptile", "racecourse", "Food", "Controlled designation of origin wine", "cricket league", "memorial", "album", "Wind motor", "chemical substance", "programming language", "architect", "baseball player", "person", "football match", "Nobel Prize", "sculptor", "tax", "river", "infrastructure"]} 3 | {"prompt": "What is the type of it if one is a scientist's doctoral student? (a) poet, (b) military person, (c) bridge, (d) single, (e) mountain, (f) Community, (g) comics character, (h) deanery, (i) restaurant, (j) Member of a Resistance Movement, (k) cat, (l) Database, (m) historian, (n) rest area, (o) train carriage, (p) parish, (q) Sailor, (r) Theatre director, (s) person, (t) reign", "gold": "(s) person", "cands": ["poet", "military person", "bridge", "single", "mountain", "Community", "comics character", "deanery", "restaurant", "Member of a Resistance Movement", "cat", "Database", "historian", "rest area", "train carriage", "parish", "Sailor", "Theatre director", "person", "reign"]} 4 | {"prompt": "What is the type of it if an organisation was founded by it? (a) congressman, (b) Manor, (c) television episode, (d) martial artist, (e) farmer, (f) person, (g) Port, (h) decoration, (i) road junction, (j) basketball league, (k) saint, (l) HumanGene, (m) sports club, (n) Gated community, (o) Youtuber, (p) lacrosse player, (q) american football league, (r) wine, (s) government agency, (t) Pandemic", "gold": "(f) person", "cands": ["congressman", "Manor", "television episode", "martial artist", "farmer", "person", "Port", "decoration", "road junction", "basketball league", "saint", "HumanGene", "sports club", "Gated community", "Youtuber", "lacrosse player", "american football league", "wine", "government agency", "Pandemic"]} 5 | {"prompt": "What is the type of it if one is an election's successful candidate? (a) Christian Patriarch, (b) manhua, (c) Producer, (d) cape, (e) tram station, (f) clerical order, (g) gross domestic product per capita, (h) military conflict, (i) tennis league, (j) golf tournament, (k) boxing league, (l) natural place, (m) cardinal, (n) mineral, (o) board game, (p) sculptor, (q) fictional character, (r) beauty queen, (s) person, (t) parliament", "gold": "(s) person", "cands": ["Christian Patriarch", "manhua", "Producer", "cape", "tram station", "clerical order", "gross domestic product per capita", "military conflict", "tennis league", "golf tournament", "boxing league", "natural place", "cardinal", "mineral", "board game", "sculptor", "fictional character", "beauty queen", "person", "parliament"]} 6 | {"prompt": "What is the type of it if one is a person's father? (a) electrical substation, (b) gene, (c) man, (d) manhua, (e) artistic genre, (f) escalator, (g) Globular Swarm, (h) Political concept, (i) programming language, (j) valley, (k) archbishop, (l) old territory, (m) Reference, (n) automobile, (o) Spreadsheet, (p) radio station, (q) Star сluster, (r) collection of valuables, (s) high diver, (t) musical", "gold": "(c) man", "cands": ["electrical substation", "gene", "man", "manhua", "artistic genre", "escalator", "Globular Swarm", "Political concept", "programming language", "valley", "archbishop", "old territory", "Reference", "automobile", "Spreadsheet", "radio station", "Star сluster", "collection of valuables", "high diver", "musical"]} 7 | {"prompt": "What is the type of it if a person used to work in it? (a) Quote, (b) space station, (c) hot spring, (d) paintball league, (e) drug, (f) criminal, (g) monarch, (h) chemical substance, (i) competition, (j) casino, (k) sport, (l) academic conference, (m) racecourse, (n) National anthem, (o) place, (p) atoll, (q) novel, (r) year, (s) soccer tournoment, (t) career station", "gold": "(o) place", "cands": ["Quote", "space station", "hot spring", "paintball league", "drug", "criminal", "monarch", "chemical substance", "competition", "casino", "sport", "academic conference", "racecourse", "National anthem", "place", "atoll", "novel", "year", "soccer tournoment", "career station"]} 8 | {"prompt": "What is the type of it if one is a person's mother? (a) speedway team, (b) Browser, (c) district, (d) year, (e) Spy, (f) chancellor, (g) castle, (h) australian football league, (i) sport, (j) arrondissement, (k) field hockey league, (l) high diver, (m) top level domain, (n) psychologist, (o) hockey club, (p) school, (q) forest, (r) guitarist, (s) businessperson, (t) woman", "gold": "(t) woman", "cands": ["speedway team", "Browser", "district", "year", "Spy", "chancellor", "castle", "australian football league", "sport", "arrondissement", "field hockey league", "high diver", "top level domain", "psychologist", "hockey club", "school", "forest", "guitarist", "businessperson", "woman"]} 9 | {"prompt": "What is the type of it if one is a person's student? (a) tournament, (b) arena, (c) person, (d) chancellor, (e) political function, (f) lacrosse league, (g) atoll, (h) bacteria, (i) american football player, (j) beer, (k) religious, (l) ski jumper, (m) venue, (n) national collegiate athletic association team season, (o) politician spouse, (p) guitarist, (q) competition, (r) Annotation, (s) meeting, (t) jockey", "gold": "(c) person", "cands": ["tournament", "arena", "person", "chancellor", "political function", "lacrosse league", "atoll", "bacteria", "american football player", "beer", "religious", "ski jumper", "venue", "national collegiate athletic association team season", "politician spouse", "guitarist", "competition", "Annotation", "meeting", "jockey"]} 10 | {"prompt": "What is the type of it if a television show is hosted by it? (a) fort, (b) country, (c) man, (d) locality, (e) Open Swarm, (f) state, (g) tennis player, (h) roller coaster, (i) rest area, (j) bone, (k) Resume, (l) moving image, (m) conifer, (n) Pandemic, (o) Medicine, (p) Organisation member, (q) military unit, (r) muscle, (s) rebellion, (t) person", "gold": "(t) person", "cands": ["fort", "country", "man", "locality", "Open Swarm", "state", "tennis player", "roller coaster", "rest area", "bone", "Resume", "moving image", "conifer", "Pandemic", "Medicine", "Organisation member", "military unit", "muscle", "rebellion", "person"]} 11 | {"prompt": "What is the type of it if a person was killed by it? (a) stadium, (b) BobsleighAthlete, (c) coach, (d) political party, (e) Satellite, (f) motorsport racer, (g) Playwright, (h) Sculpture, (i) movie, (j) Instrument, (k) baseball season, (l) Historical settlement, (m) horse rider, (n) Algorithm, (o) borough, (p) results of a sport competition, (q) person, (r) cat, (s) lighthouse, (t) horse", "gold": "(q) person", "cands": ["stadium", "BobsleighAthlete", "coach", "political party", "Satellite", "motorsport racer", "Playwright", "Sculpture", "movie", "Instrument", "baseball season", "Historical settlement", "horse rider", "Algorithm", "borough", "results of a sport competition", "person", "cat", "lighthouse", "horse"]} 12 | {"prompt": "What is the type of it if an animal was born in it? (a) place, (b) hockey club, (c) battery, (d) journalist, (e) award, (f) golf player, (g) Archer Player, (h) lipid, (i) man, (j) motocycle racer, (k) arachnid, (l) Cipher, (m) Lawyer, (n) penalty shoot out, (o) speedway team, (p) murderer, (q) agglomeration, (r) poem, (s) mountain range, (t) football league season", "gold": "(a) place", "cands": ["place", "hockey club", "battery", "journalist", "award", "golf player", "Archer Player", "lipid", "man", "motocycle racer", "arachnid", "Cipher", "Lawyer", "penalty shoot out", "speedway team", "murderer", "agglomeration", "poem", "mountain range", "football league season"]} 13 | {"prompt": "What is the type of it if one is a work's author? (a) on site mean of transportation, (b) lock, (c) overseas department, (d) zoo, (e) television show, (f) person, (g) minister, (h) engine, (i) Reference, (j) office holder, (k) adult pornographic actor, (l) murderer, (m) volleyball player, (n) book, (o) asteroid, (p) Member of a Resistance Movement, (q) basketball team, (r) formula one racing, (s) Employer, (t) law firm", "gold": "(f) person", "cands": ["on site mean of transportation", "lock", "overseas department", "zoo", "television show", "person", "minister", "engine", "Reference", "office holder", "adult pornographic actor", "murderer", "volleyball player", "book", "asteroid", "Member of a Resistance Movement", "basketball team", "formula one racing", "Employer", "law firm"]} 14 | {"prompt": "What is the type of it if a language is a dialect of it? (a) poet, (b) clerical administrative region, (c) garden, (d) historic place, (e) Watermill, (f) vodka, (g) language, (h) golf player, (i) museum, (j) province, (k) city, (l) Globular Swarm, (m) project, (n) Painting, (o) musical work, (p) music composer, (q) brewery, (r) sport, (s) prehistorical period, (t) gymnast", "gold": "(g) language", "cands": ["poet", "clerical administrative region", "garden", "historic place", "Watermill", "vodka", "language", "golf player", "museum", "province", "city", "Globular Swarm", "project", "Painting", "musical work", "music composer", "brewery", "sport", "prehistorical period", "gymnast"]} 15 | {"prompt": "What is the type of it if a work has the melody of it? (a) road tunnel, (b) chemical compound, (c) square, (d) coach, (e) volleyball player, (f) university, (g) museum, (h) province, (i) entomologist, (j) musical work, (k) monument, (l) constellation, (m) area, (n) COVID 19 pandemic, (o) basketball player, (p) comic, (q) sound, (r) wine region, (s) swimmer, (t) dam", "gold": "(j) musical work", "cands": ["road tunnel", "chemical compound", "square", "coach", "volleyball player", "university", "museum", "province", "entomologist", "musical work", "monument", "constellation", "area", "COVID 19 pandemic", "basketball player", "comic", "sound", "wine region", "swimmer", "dam"]} 16 | {"prompt": "What is the type of it if a celestial body is part of constellation it? (a) GeneLocation, (b) galaxy, (c) constellation, (d) type, (e) bus company, (f) college, (g) standard, (h) tennis tournament, (i) JewishLeader, (j) racecourse, (k) time period, (l) race, (m) politician, (n) underground journal, (o) territory, (p) jockey, (q) sports league, (r) badminton player, (s) school, (t) surfer", "gold": "(c) constellation", "cands": ["GeneLocation", "galaxy", "constellation", "type", "bus company", "college", "standard", "tennis tournament", "JewishLeader", "racecourse", "time period", "race", "politician", "underground journal", "territory", "jockey", "sports league", "badminton player", "school", "surfer"]} 17 | {"prompt": "What is the type of it if a person belongs to the party it? (a) newspaper, (b) chemical substance, (c) celestial body, (d) parliament, (e) top level domain, (f) volleyball league, (g) World Heritage Site, (h) songwriter, (i) law, (j) senator, (k) political party, (l) Anime, (m) cricket team, (n) olympics, (o) cycling competition, (p) soccer league, (q) multi volume publication, (r) race track, (s) engineer, (t) street", "gold": "(k) political party", "cands": ["newspaper", "chemical substance", "celestial body", "parliament", "top level domain", "volleyball league", "World Heritage Site", "songwriter", "law", "senator", "political party", "Anime", "cricket team", "olympics", "cycling competition", "soccer league", "multi volume publication", "race track", "engineer", "street"]} 18 | {"prompt": "What is the type of it if something is located in the administrative territory of it? (a) american football league, (b) launch pad, (c) trade union, (d) space shuttle, (e) conveyor system, (f) river, (g) guitarist, (h) musical, (i) Historical settlement, (j) bridge, (k) Resume, (l) racecourse, (m) administrative region, (n) name, (o) speed skater, (p) time period, (q) fashion designer, (r) results of a sport competition, (s) soap character, (t) province", "gold": "(m) administrative region", "cands": ["american football league", "launch pad", "trade union", "space shuttle", "conveyor system", "river", "guitarist", "musical", "Historical settlement", "bridge", "Resume", "racecourse", "administrative region", "name", "speed skater", "time period", "fashion designer", "results of a sport competition", "soap character", "province"]} 19 | {"prompt": "What is the type of it if one is a person's child? (a) speedway league, (b) route stop, (c) nascar driver, (d) academic journal, (e) natural region, (f) member of parliament, (g) Protocol, (h) Swarm, (i) philosopher, (j) cycad, (k) opera, (l) vaccine, (m) person, (n) disease, (o) language, (p) reign, (q) water polo Player, (r) ski resort, (s) Team member, (t) polysaccharide", "gold": "(m) person", "cands": ["speedway league", "route stop", "nascar driver", "academic journal", "natural region", "member of parliament", "Protocol", "Swarm", "philosopher", "cycad", "opera", "vaccine", "person", "disease", "language", "reign", "water polo Player", "ski resort", "Team member", "polysaccharide"]} 20 | {"prompt": "What is the type of it if one is a work's developer? (a) Browser, (b) Formula One racer, (c) australian football league, (d) brain, (e) handball player, (f) videogames league, (g) pope, (h) Reference, (i) political function, (j) amusement park attraction, (k) Singer, (l) member of parliament, (m) geological period, (n) governor, (o) launch pad, (p) lymph, (q) cheese, (r) agent, (s) racecourse, (t) space shuttle", "gold": "(r) agent", "cands": ["Browser", "Formula One racer", "australian football league", "brain", "handball player", "videogames league", "pope", "Reference", "political function", "amusement park attraction", "Singer", "member of parliament", "geological period", "governor", "launch pad", "lymph", "cheese", "agent", "racecourse", "space shuttle"]} 21 | {"prompt": "What is the type of it if one is a sports event's participating teams? (a) Quote, (b) artery, (c) cabinet of ministers, (d) chess player, (e) sports team, (f) rugby league, (g) jockey, (h) monument, (i) geological period, (j) contest, (k) cycling league, (l) MouseGeneLocation, (m) Biomolecule, (n) linguist, (o) soccer tournoment, (p) volcano, (q) songwriter, (r) academic subject, (s) government agency, (t) newspaper", "gold": "(e) sports team", "cands": ["Quote", "artery", "cabinet of ministers", "chess player", "sports team", "rugby league", "jockey", "monument", "geological period", "contest", "cycling league", "MouseGeneLocation", "Biomolecule", "linguist", "soccer tournoment", "volcano", "songwriter", "academic subject", "government agency", "newspaper"]} 22 | {"prompt": "What is the type of it if a body of water inflows it? (a) body of water, (b) cross country skier, (c) manga, (d) canoeist, (e) soap character, (f) chemical compound, (g) ice hockey player, (h) Theatre director, (i) GeneLocation, (j) drug, (k) national football league event, (l) prison, (m) Cardinal direction, (n) digital camera, (o) still image, (p) Controlled designation of origin wine, (q) woman, (r) castle, (s) lake, (t) egyptologist", "gold": "(a) body of water", "cands": ["body of water", "cross country skier", "manga", "canoeist", "soap character", "chemical compound", "ice hockey player", "Theatre director", "GeneLocation", "drug", "national football league event", "prison", "Cardinal direction", "digital camera", "still image", "Controlled designation of origin wine", "woman", "castle", "lake", "egyptologist"]} 23 | {"prompt": "What is the type of it if one is a sports team's home stadium? (a) Community, (b) Vaccination Statistics, (c) street, (d) hockey club, (e) road junction, (f) Cipher, (g) Gated community, (h) tenure, (i) engine, (j) manga, (k) Pretender, (l) convention, (m) MouseGene, (n) amateur boxer, (o) music composer, (p) ambassador, (q) entomologist, (r) architectural structure, (s) ligament, (t) image", "gold": "(r) architectural structure", "cands": ["Community", "Vaccination Statistics", "street", "hockey club", "road junction", "Cipher", "Gated community", "tenure", "engine", "manga", "Pretender", "convention", "MouseGene", "amateur boxer", "music composer", "ambassador", "entomologist", "architectural structure", "ligament", "image"]} 24 | {"prompt": "What is the type of it if a work has cast member it? (a) classical music composition, (b) speedway league, (c) sports event, (d) motorsport racer, (e) videogames league, (f) water ride, (g) actor, (h) protohistorical period, (i) natural place, (j) team sport, (k) chemical compound, (l) religious organisation, (m) casino, (n) crustacean, (o) list, (p) Place in the Music Charts, (q) district, (r) island, (s) international football league event, (t) canadian football Player", "gold": "(g) actor", "cands": ["classical music composition", "speedway league", "sports event", "motorsport racer", "videogames league", "water ride", "actor", "protohistorical period", "natural place", "team sport", "chemical compound", "religious organisation", "casino", "crustacean", "list", "Place in the Music Charts", "district", "island", "international football league event", "canadian football Player"]} 25 | {"prompt": "What is the type of it if one is a person's spouse? (a) road tunnel, (b) animal, (c) Archer Player, (d) election, (e) sports club, (f) Vaccination Statistics, (g) on site mean of transportation, (h) File system, (i) Christian Patriarch, (j) model, (k) bowling league, (l) camera, (m) president, (n) given name, (o) person, (p) temple, (q) period of artistic style, (r) multi volume publication, (s) samba school, (t) Artificial Satellite", "gold": "(o) person", "cands": ["road tunnel", "animal", "Archer Player", "election", "sports club", "Vaccination Statistics", "on site mean of transportation", "File system", "Christian Patriarch", "model", "bowling league", "camera", "president", "given name", "person", "temple", "period of artistic style", "multi volume publication", "samba school", "Artificial Satellite"]} 26 | {"prompt": "What is the type of it if a work was originally created in it? (a) canoeist, (b) convention, (c) BobsleighAthlete, (d) netball player, (e) Scientific concept, (f) publisher, (g) mineral, (h) language, (i) Controlled designation of origin wine, (j) archbishop, (k) cleric, (l) Wind motor, (m) artistic genre, (n) squash player, (o) synagogue, (p) hockey club, (q) populated place, (r) Wikimedia template, (s) standard, (t) multi volume publication", "gold": "(h) language", "cands": ["canoeist", "convention", "BobsleighAthlete", "netball player", "Scientific concept", "publisher", "mineral", "language", "Controlled designation of origin wine", "archbishop", "cleric", "Wind motor", "artistic genre", "squash player", "synagogue", "hockey club", "populated place", "Wikimedia template", "standard", "multi volume publication"]} 27 | {"prompt": "What is the type of it if one is an organisation's chief executive officer? (a) bone, (b) photographer, (c) constellation, (d) embryology, (e) football match, (f) linguist, (g) skater, (h) noble, (i) Browser, (j) person, (k) regency, (l) motocycle racer, (m) historical event, (n) magazine, (o) deputy, (p) cycling competition, (q) canoeist, (r) Employer, (s) team sport, (t) sports league", "gold": "(j) person", "cands": ["bone", "photographer", "constellation", "embryology", "football match", "linguist", "skater", "noble", "Browser", "person", "regency", "motocycle racer", "historical event", "magazine", "deputy", "cycling competition", "canoeist", "Employer", "team sport", "sports league"]} 28 | {"prompt": "What is the type of it if one is a ship's home port? (a) eukaryote, (b) space mission, (c) Supreme Court of the United States case, (d) cabinet of ministers, (e) archbishop, (f) national football league event, (g) period of artistic style, (h) Open Swarm, (i) Controlled designation of origin wine, (j) member of parliament, (k) ski area, (l) rugby club, (m) cycling league, (n) television show, (o) place, (p) streetcar, (q) Capital, (r) pope, (s) Historical settlement, (t) golf tournament", "gold": "(o) place", "cands": ["eukaryote", "space mission", "Supreme Court of the United States case", "cabinet of ministers", "archbishop", "national football league event", "period of artistic style", "Open Swarm", "Controlled designation of origin wine", "member of parliament", "ski area", "rugby club", "cycling league", "television show", "place", "streetcar", "Capital", "pope", "Historical settlement", "golf tournament"]} --------------------------------------------------------------------------------