├── README.md
├── ckpt_eval.py
├── constants.py
├── data
├── query_templates
│ ├── ace2004.json
│ ├── ace2005.json
│ └── generate_query.py
└── raw_data
│ ├── ace2004.zip
│ └── ace2005.zip
├── dataloader.py
├── evaluation.py
├── model.py
├── preprocess.py
├── requirements.txt
├── scripts
├── ckpt_eval.sh
├── prepare_ace2004.sh
├── prepare_ace2005.sh
├── train_ace2004.sh
└── train_ace2005.sh
└── train.py
/README.md:
--------------------------------------------------------------------------------
1 | # Entity-Relation Extraction as Multi-Turn Question Answering
2 | The repository contains the code of the recent research advances in [Shannon.AI](http://www.shannonai.com).
3 |
4 | **Entity-Relation Extraction as Multi-turn Question Answering (ACL 2019)**
5 | Xiaoya Li, Fan Yin, Zijun Sun, Xiayu Li, Arianna Yuan, Duo Chai, Mingxin Zhou and Jiwei Li
6 | Accepted by [ACL 2019](https://arxiv.org/pdf/1905.05529.pdf)
7 | If you find this repo helpful, please cite the following:
8 | ```text
9 | @inproceedings{li-etal-2019-entity,
10 | title = "Entity-Relation Extraction as Multi-Turn Question Answering",
11 | author = "Li, Xiaoya and
12 | Yin, Fan and
13 | Sun, Zijun and
14 | Li, Xiayu and
15 | Yuan, Arianna and
16 | Chai, Duo and
17 | Zhou, Mingxin and
18 | Li, Jiwei",
19 | booktitle = "Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics",
20 | month = jul,
21 | year = "2019",
22 | address = "Florence, Italy",
23 | publisher = "Association for Computational Linguistics",
24 | url = "https://www.aclweb.org/anthology/P19-1129",
25 | doi = "10.18653/v1/P19-1129",
26 | pages = "1340--1350"
27 | }
28 | ```
29 |
30 |
31 | ## Install Requirements
32 |
33 | `pip install -r requirements.txt`
34 |
35 | ## Prepare Data
36 | This project currently only support ace2004 and ace2005 dataset.
37 |
38 | For data processing:
39 | 1. We use moving sliding window with overlap to split the passage,
40 | 2. We use a threshold to filter impossible relations with small frequency,
41 | 3. We use max distance to further filter impossible relations that have end entity type that doesn't in a small range from the head entity.
42 | 4. We use two special tokens to mark the head entity in the context to avoid coreference problem in the window.
43 |
44 | Following these steps to obtain training data:
45 |
46 | 1. Unzip ace2004.zip and ace2005.zip into data/raw_data
47 | 2. Preprocess the data with `preprocess.py`.The example used can be found in `scripts/prepare_ace2005.sh`
48 |
49 | ## Pretrained Model
50 |
51 | We use [BERT-Base-Uncased](https://huggingface.co/bert-base-uncased)
52 |
53 | ## Train
54 |
55 | Use `train.py` to train the model.
56 |
57 | The example used can be found in `scripts/train_ace2005.sh`.
58 |
59 | ## Evaluate checkpoints:
60 |
61 | Use `ckpt_eval.py` to evaluate the saved model.
62 |
63 | The example used can be found in `scripts/ckpt_eval.sh`.
--------------------------------------------------------------------------------
/ckpt_eval.py:
--------------------------------------------------------------------------------
1 | import os
2 | import argparse
3 | import pickle
4 | import torch
5 |
6 | from evaluation import test_evaluation
7 | from model import MyModel
8 | from dataloader import load_t1_data
9 |
10 |
11 | if __name__=='__main__':
12 | parser = argparse.ArgumentParser()
13 | parser.add_argument("--window_size",type=int,default=300)
14 | parser.add_argument("--overlap",type=int,default=45)
15 | parser.add_argument("--checkpoint_path",required=True,default="./checkpoints/model_create_date/checkpoint_i.cpt")
16 | parser.add_argument("--test_path")
17 | parser.add_argument("--test_batch",default=10,type=int)
18 | parser.add_argument('--max_len',default=512,type=int)
19 | parser.add_argument("--threshold",type=int,default=-1)
20 | parser.add_argument("--pretrained_model_path",default="")
21 | parser.add_argument("--amp",action='store_true')
22 | args = parser.parse_args()
23 | model_dir,file = os.path.split(args.checkpoint_path)
24 | config = pickle.load(open(os.path.join(model_dir,'args'),'rb'))
25 | checkpoint = torch.load(os.path.join(model_dir,file),map_location=torch.device("cpu"))
26 | model_state_dict = checkpoint['model_state_dict']
27 | #replace default vaule if given
28 | config.pretrained_model_path = args.pretrained_model_path if args.pretrained_model_path else config.pretrained_model_path
29 | config.threshold = args.threshold if args.threshold==-1 else config.threshold
30 | mymodel = MyModel(config)
31 | mymodel.load_state_dict(model_state_dict,strict=False)
32 | device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
33 | mymodel.to(device)
34 |
35 | test_dataloader = load_t1_data(config.dataset_tag,args.test_path,config.pretrained_model_path,args.window_size,args.overlap,args.test_batch,args.max_len)
36 | (p1,r1,f1),(p2,r2,f2) = test_evaluation(mymodel,test_dataloader,config.threshold,args.amp)
37 | print("Turn 1: precision:{:.4f} recall:{:.4f} f1:{:.4f}".format(p1,r1,f1))
38 | print("Turn 2: precision:{:.4f} recall:{:.4f} f1:{:.4f}".format(p2,r2,f2))
--------------------------------------------------------------------------------
/constants.py:
--------------------------------------------------------------------------------
1 | import json
2 |
3 | question_templates_path = "./data/query_templates/"
4 | ace2004_question_templates = json.load(
5 | open(question_templates_path+'ace2004.json'))
6 | ace2005_question_templates = json.load(
7 | open(question_templates_path+'ace2005.json'))
8 |
9 | tag_idxs = {'B': 0, 'M': 1, 'E': 2, 'S': 3, 'O': 4}
10 |
11 | ace2004_entities = ['FAC', 'GPE', 'LOC', 'ORG', 'PER', 'VEH', 'WEA']
12 | ace2004_entities_full = ["facility", "geo political",
13 | "location", "organization", "person", "vehicle", "weapon"]
14 | ace2004_relations = ['ART', 'EMP-ORG',
15 | 'GPE-AFF', 'OTHER-AFF', 'PER-SOC', 'PHYS']
16 | ace2004_relations_full = ['artifact', 'employment, membership or subsidiary',
17 | 'geo political affiliation', 'person or organization affiliation', 'personal or social', 'physical']
18 |
19 | ace2005_entities = ['FAC', 'GPE', 'LOC', 'ORG', 'PER', 'VEH', 'WEA']
20 | ace2005_entities_full = ["facility", "geo political",
21 | "location", "organization", "person", "vehicle", "weapon"]
22 | ace2005_relations = ['ART', 'GEN-AFF',
23 | 'ORG-AFF', 'PART-WHOLE', 'PER-SOC', 'PHYS']
24 | ace2005_relations_full = ["artifact", "gen affilliation",
25 | 'organization affiliation', 'part whole', 'person social', 'physical']
26 |
27 | # index of ace2004 and ace2005 frequency matrix
28 | ace2004_idx1 = {'FAC': 0, 'GPE': 1, 'LOC': 2,
29 | 'ORG': 3, 'PER': 4, 'VEH': 5, 'WEA': 6}
30 | ace2005_idx1 = {'FAC': 0, 'GPE': 1, 'LOC': 2,
31 | 'ORG': 3, 'PER': 4, 'VEH': 5, 'WEA': 6}
32 | ace2005_idx2t = {}
33 | for i, rel in enumerate(ace2005_relations):
34 | for j, ent in enumerate(ace2005_entities):
35 | ace2005_idx2t[(rel, ent)] = i*len(ace2005_relations)+j+i
36 | ace2005_idx2 = ace2005_idx2t
37 | ace2004_idx2t = {}
38 | for i, rel in enumerate(ace2004_relations):
39 | for j, ent in enumerate(ace2004_entities):
40 | ace2004_idx2t[(rel, ent)] = i*len(ace2004_relations)+j+i
41 | ace2004_idx2 = ace2004_idx2t
42 | # statistics on the training set
43 | ace2005_dist = [[0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0,
44 | 0, 0, 0, 0, 0, 0, 0, 0, 33, 116, 39, 2, 0,
45 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 22, 11, 0,
46 | 0, 0, 0],
47 | [30, 0, 0, 0, 0, 60, 61, 0, 0, 0, 0, 0, 0,
48 | 0, 0, 0, 0, 19, 0, 0, 0, 1, 143, 47, 0, 0,
49 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 14, 9, 0,
50 | 0, 0, 0],
51 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
52 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 120, 31, 0, 0,
53 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 18, 8, 0,
54 | 0, 0, 0],
55 | [35, 0, 0, 0, 0, 35, 10, 0, 149, 20, 0, 0, 0,
56 | 0, 0, 5, 0, 12, 0, 0, 0, 0, 147, 1, 81, 0,
57 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 0,
58 | 0, 0, 0],
59 | [67, 1, 0, 2, 0, 113, 77, 0, 270, 27, 10, 32, 0,
60 | 0, 0, 587, 0, 844, 5, 0, 0, 0, 0, 0, 4, 0,
61 | 0, 0, 0, 0, 0, 4, 434, 0, 0, 281, 494, 213, 4,
62 | 0, 1, 0],
63 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
64 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
65 | 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
66 | 0, 0, 0],
67 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
68 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
69 | 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
70 | 0, 0, 0]]
71 | # statistics on the entire data set
72 | ace2004_dist = [[0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0,
73 | 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
74 | 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
75 | 0, 0, 31, 104, 28, 0, 0, 0, 0],
76 | [8, 1, 1, 0, 0, 27, 8, 0, 8, 0, 3,
77 | 0, 0, 0, 1, 12, 5, 0, 0, 0, 0, 0,
78 | 1, 0, 3, 20, 0, 0, 0, 0, 0, 0, 1,
79 | 0, 0, 0, 236, 55, 0, 0, 0, 0],
80 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
81 | 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0,
82 | 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0,
83 | 0, 0, 0, 65, 38, 0, 0, 0, 0],
84 | [28, 0, 0, 0, 0, 24, 2, 0, 132, 0, 120,
85 | 1, 0, 0, 0, 203, 16, 0, 0, 0, 0, 0,
86 | 4, 1, 6, 6, 0, 0, 0, 0, 0, 0, 1,
87 | 0, 0, 4, 19, 2, 1, 0, 0, 0],
88 | [55, 0, 0, 0, 0, 29, 25, 3, 311, 1, 1035,
89 | 8, 0, 0, 0, 276, 9, 0, 1, 0, 0, 1,
90 | 5, 4, 18, 69, 0, 0, 0, 0, 0, 0, 363,
91 | 0, 0, 168, 328, 55, 8, 9, 16, 0],
92 | [0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0,
93 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
94 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
95 | 0, 0, 8, 19, 5, 0, 0, 3, 0],
96 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
97 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
98 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
99 | 0, 0, 3, 1, 3, 0, 0, 2, 3]]
100 |
--------------------------------------------------------------------------------
/data/query_templates/ace2004.json:
--------------------------------------------------------------------------------
1 | {"qa_turn1": {"FAC": "find all facility entities in the context.", "GPE": "find all geo political entities in the context.", "LOC": "find all location entities in the context.", "ORG": "find all organization entities in the context.", "PER": "find all person entities in the context.", "VEH": "find all vehicle entities in the context.", "WEA": "find all weapon entities in the context."}, "qa_turn2": {"('FAC', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with facility entity XXX.", "('FAC', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with facility entity XXX.", "('FAC', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with facility entity XXX.", "('FAC', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with facility entity XXX.", "('FAC', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with facility entity XXX.", "('FAC', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with facility entity XXX.", "('FAC', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with facility entity XXX.", "('FAC', 'EMP-ORG', 'FAC')": "find all facility entities in the context that have a employment, membership or subsidiary relationship with facility entity XXX.", "('FAC', 'EMP-ORG', 'GPE')": "find all geo political entities in the context that have a employment, membership or subsidiary relationship with facility entity XXX.", "('FAC', 'EMP-ORG', 'LOC')": "find all location entities in the context that have a employment, membership or subsidiary relationship with facility entity XXX.", "('FAC', 'EMP-ORG', 'ORG')": "find all organization entities in the context that have a employment, membership or subsidiary relationship with facility entity XXX.", "('FAC', 'EMP-ORG', 'PER')": "find all person entities in the context that have a employment, membership or subsidiary relationship with facility entity XXX.", "('FAC', 'EMP-ORG', 'VEH')": "find all vehicle entities in the context that have a employment, membership or subsidiary relationship with facility entity XXX.", "('FAC', 'EMP-ORG', 'WEA')": "find all weapon entities in the context that have a employment, membership or subsidiary relationship with facility entity XXX.", "('FAC', 'GPE-AFF', 'FAC')": "find all facility entities in the context that have a geo political affiliation relationship with facility entity XXX.", "('FAC', 'GPE-AFF', 'GPE')": "find all geo political entities in the context that have a geo political affiliation relationship with facility entity XXX.", "('FAC', 'GPE-AFF', 'LOC')": "find all location entities in the context that have a geo political affiliation relationship with facility entity XXX.", "('FAC', 'GPE-AFF', 'ORG')": "find all organization entities in the context that have a geo political affiliation relationship with facility entity XXX.", "('FAC', 'GPE-AFF', 'PER')": "find all person entities in the context that have a geo political affiliation relationship with facility entity XXX.", "('FAC', 'GPE-AFF', 'VEH')": "find all vehicle entities in the context that have a geo political affiliation relationship with facility entity XXX.", "('FAC', 'GPE-AFF', 'WEA')": "find all weapon entities in the context that have a geo political affiliation relationship with facility entity XXX.", "('FAC', 'OTHER-AFF', 'FAC')": "find all facility entities in the context that have a person or organization affiliation relationship with facility entity XXX.", "('FAC', 'OTHER-AFF', 'GPE')": "find all geo political entities in the context that have a person or organization affiliation relationship with facility entity XXX.", "('FAC', 'OTHER-AFF', 'LOC')": "find all location entities in the context that have a person or organization affiliation relationship with facility entity XXX.", "('FAC', 'OTHER-AFF', 'ORG')": "find all organization entities in the context that have a person or organization affiliation relationship with facility entity XXX.", "('FAC', 'OTHER-AFF', 'PER')": "find all person entities in the context that have a person or organization affiliation relationship with facility entity XXX.", "('FAC', 'OTHER-AFF', 'VEH')": "find all vehicle entities in the context that have a person or organization affiliation relationship with facility entity XXX.", "('FAC', 'OTHER-AFF', 'WEA')": "find all weapon entities in the context that have a person or organization affiliation relationship with facility entity XXX.", "('FAC', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a personal or social relationship with facility entity XXX.", "('FAC', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a personal or social relationship with facility entity XXX.", "('FAC', 'PER-SOC', 'LOC')": "find all location entities in the context that have a personal or social relationship with facility entity XXX.", "('FAC', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a personal or social relationship with facility entity XXX.", "('FAC', 'PER-SOC', 'PER')": "find all person entities in the context that have a personal or social relationship with facility entity XXX.", "('FAC', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a personal or social relationship with facility entity XXX.", "('FAC', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a personal or social relationship with facility entity XXX.", "('FAC', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with facility entity XXX.", "('FAC', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with facility entity XXX.", "('FAC', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with facility entity XXX.", "('FAC', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with facility entity XXX.", "('FAC', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with facility entity XXX.", "('FAC', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with facility entity XXX.", "('FAC', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with facility entity XXX.", "('GPE', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with geo political entity XXX.", "('GPE', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with geo political entity XXX.", "('GPE', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with geo political entity XXX.", "('GPE', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with geo political entity XXX.", "('GPE', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with geo political entity XXX.", "('GPE', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with geo political entity XXX.", "('GPE', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with geo political entity XXX.", "('GPE', 'EMP-ORG', 'FAC')": "find all facility entities in the context that have a employment, membership or subsidiary relationship with geo political entity XXX.", "('GPE', 'EMP-ORG', 'GPE')": "find all geo political entities in the context that have a employment, membership or subsidiary relationship with geo political entity XXX.", "('GPE', 'EMP-ORG', 'LOC')": "find all location entities in the context that have a employment, membership or subsidiary relationship with geo political entity XXX.", "('GPE', 'EMP-ORG', 'ORG')": "find all organization entities in the context that have a employment, membership or subsidiary relationship with geo political entity XXX.", "('GPE', 'EMP-ORG', 'PER')": "find all person entities in the context that have a employment, membership or subsidiary relationship with geo political entity XXX.", "('GPE', 'EMP-ORG', 'VEH')": "find all vehicle entities in the context that have a employment, membership or subsidiary relationship with geo political entity XXX.", "('GPE', 'EMP-ORG', 'WEA')": "find all weapon entities in the context that have a employment, membership or subsidiary relationship with geo political entity XXX.", "('GPE', 'GPE-AFF', 'FAC')": "find all facility entities in the context that have a geo political affiliation relationship with geo political entity XXX.", "('GPE', 'GPE-AFF', 'GPE')": "find all geo political entities in the context that have a geo political affiliation relationship with geo political entity XXX.", "('GPE', 'GPE-AFF', 'LOC')": "find all location entities in the context that have a geo political affiliation relationship with geo political entity XXX.", "('GPE', 'GPE-AFF', 'ORG')": "find all organization entities in the context that have a geo political affiliation relationship with geo political entity XXX.", "('GPE', 'GPE-AFF', 'PER')": "find all person entities in the context that have a geo political affiliation relationship with geo political entity XXX.", "('GPE', 'GPE-AFF', 'VEH')": "find all vehicle entities in the context that have a geo political affiliation relationship with geo political entity XXX.", "('GPE', 'GPE-AFF', 'WEA')": "find all weapon entities in the context that have a geo political affiliation relationship with geo political entity XXX.", "('GPE', 'OTHER-AFF', 'FAC')": "find all facility entities in the context that have a person or organization affiliation relationship with geo political entity XXX.", "('GPE', 'OTHER-AFF', 'GPE')": "find all geo political entities in the context that have a person or organization affiliation relationship with geo political entity XXX.", "('GPE', 'OTHER-AFF', 'LOC')": "find all location entities in the context that have a person or organization affiliation relationship with geo political entity XXX.", "('GPE', 'OTHER-AFF', 'ORG')": "find all organization entities in the context that have a person or organization affiliation relationship with geo political entity XXX.", "('GPE', 'OTHER-AFF', 'PER')": "find all person entities in the context that have a person or organization affiliation relationship with geo political entity XXX.", "('GPE', 'OTHER-AFF', 'VEH')": "find all vehicle entities in the context that have a person or organization affiliation relationship with geo political entity XXX.", "('GPE', 'OTHER-AFF', 'WEA')": "find all weapon entities in the context that have a person or organization affiliation relationship with geo political entity XXX.", "('GPE', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a personal or social relationship with geo political entity XXX.", "('GPE', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a personal or social relationship with geo political entity XXX.", "('GPE', 'PER-SOC', 'LOC')": "find all location entities in the context that have a personal or social relationship with geo political entity XXX.", "('GPE', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a personal or social relationship with geo political entity XXX.", "('GPE', 'PER-SOC', 'PER')": "find all person entities in the context that have a personal or social relationship with geo political entity XXX.", "('GPE', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a personal or social relationship with geo political entity XXX.", "('GPE', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a personal or social relationship with geo political entity XXX.", "('GPE', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with geo political entity XXX.", "('GPE', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with geo political entity XXX.", "('GPE', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with geo political entity XXX.", "('GPE', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with geo political entity XXX.", "('GPE', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with geo political entity XXX.", "('GPE', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with geo political entity XXX.", "('GPE', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with geo political entity XXX.", "('LOC', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with location entity XXX.", "('LOC', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with location entity XXX.", "('LOC', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with location entity XXX.", "('LOC', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with location entity XXX.", "('LOC', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with location entity XXX.", "('LOC', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with location entity XXX.", "('LOC', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with location entity XXX.", "('LOC', 'EMP-ORG', 'FAC')": "find all facility entities in the context that have a employment, membership or subsidiary relationship with location entity XXX.", "('LOC', 'EMP-ORG', 'GPE')": "find all geo political entities in the context that have a employment, membership or subsidiary relationship with location entity XXX.", "('LOC', 'EMP-ORG', 'LOC')": "find all location entities in the context that have a employment, membership or subsidiary relationship with location entity XXX.", "('LOC', 'EMP-ORG', 'ORG')": "find all organization entities in the context that have a employment, membership or subsidiary relationship with location entity XXX.", "('LOC', 'EMP-ORG', 'PER')": "find all person entities in the context that have a employment, membership or subsidiary relationship with location entity XXX.", "('LOC', 'EMP-ORG', 'VEH')": "find all vehicle entities in the context that have a employment, membership or subsidiary relationship with location entity XXX.", "('LOC', 'EMP-ORG', 'WEA')": "find all weapon entities in the context that have a employment, membership or subsidiary relationship with location entity XXX.", "('LOC', 'GPE-AFF', 'FAC')": "find all facility entities in the context that have a geo political affiliation relationship with location entity XXX.", "('LOC', 'GPE-AFF', 'GPE')": "find all geo political entities in the context that have a geo political affiliation relationship with location entity XXX.", "('LOC', 'GPE-AFF', 'LOC')": "find all location entities in the context that have a geo political affiliation relationship with location entity XXX.", "('LOC', 'GPE-AFF', 'ORG')": "find all organization entities in the context that have a geo political affiliation relationship with location entity XXX.", "('LOC', 'GPE-AFF', 'PER')": "find all person entities in the context that have a geo political affiliation relationship with location entity XXX.", "('LOC', 'GPE-AFF', 'VEH')": "find all vehicle entities in the context that have a geo political affiliation relationship with location entity XXX.", "('LOC', 'GPE-AFF', 'WEA')": "find all weapon entities in the context that have a geo political affiliation relationship with location entity XXX.", "('LOC', 'OTHER-AFF', 'FAC')": "find all facility entities in the context that have a person or organization affiliation relationship with location entity XXX.", "('LOC', 'OTHER-AFF', 'GPE')": "find all geo political entities in the context that have a person or organization affiliation relationship with location entity XXX.", "('LOC', 'OTHER-AFF', 'LOC')": "find all location entities in the context that have a person or organization affiliation relationship with location entity XXX.", "('LOC', 'OTHER-AFF', 'ORG')": "find all organization entities in the context that have a person or organization affiliation relationship with location entity XXX.", "('LOC', 'OTHER-AFF', 'PER')": "find all person entities in the context that have a person or organization affiliation relationship with location entity XXX.", "('LOC', 'OTHER-AFF', 'VEH')": "find all vehicle entities in the context that have a person or organization affiliation relationship with location entity XXX.", "('LOC', 'OTHER-AFF', 'WEA')": "find all weapon entities in the context that have a person or organization affiliation relationship with location entity XXX.", "('LOC', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a personal or social relationship with location entity XXX.", "('LOC', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a personal or social relationship with location entity XXX.", "('LOC', 'PER-SOC', 'LOC')": "find all location entities in the context that have a personal or social relationship with location entity XXX.", "('LOC', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a personal or social relationship with location entity XXX.", "('LOC', 'PER-SOC', 'PER')": "find all person entities in the context that have a personal or social relationship with location entity XXX.", "('LOC', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a personal or social relationship with location entity XXX.", "('LOC', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a personal or social relationship with location entity XXX.", "('LOC', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with location entity XXX.", "('LOC', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with location entity XXX.", "('LOC', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with location entity XXX.", "('LOC', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with location entity XXX.", "('LOC', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with location entity XXX.", "('LOC', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with location entity XXX.", "('LOC', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with location entity XXX.", "('ORG', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with organization entity XXX.", "('ORG', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with organization entity XXX.", "('ORG', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with organization entity XXX.", "('ORG', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with organization entity XXX.", "('ORG', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with organization entity XXX.", "('ORG', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with organization entity XXX.", "('ORG', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with organization entity XXX.", "('ORG', 'EMP-ORG', 'FAC')": "find all facility entities in the context that have a employment, membership or subsidiary relationship with organization entity XXX.", "('ORG', 'EMP-ORG', 'GPE')": "find all geo political entities in the context that have a employment, membership or subsidiary relationship with organization entity XXX.", "('ORG', 'EMP-ORG', 'LOC')": "find all location entities in the context that have a employment, membership or subsidiary relationship with organization entity XXX.", "('ORG', 'EMP-ORG', 'ORG')": "find all organization entities in the context that have a employment, membership or subsidiary relationship with organization entity XXX.", "('ORG', 'EMP-ORG', 'PER')": "find all person entities in the context that have a employment, membership or subsidiary relationship with organization entity XXX.", "('ORG', 'EMP-ORG', 'VEH')": "find all vehicle entities in the context that have a employment, membership or subsidiary relationship with organization entity XXX.", "('ORG', 'EMP-ORG', 'WEA')": "find all weapon entities in the context that have a employment, membership or subsidiary relationship with organization entity XXX.", "('ORG', 'GPE-AFF', 'FAC')": "find all facility entities in the context that have a geo political affiliation relationship with organization entity XXX.", "('ORG', 'GPE-AFF', 'GPE')": "find all geo political entities in the context that have a geo political affiliation relationship with organization entity XXX.", "('ORG', 'GPE-AFF', 'LOC')": "find all location entities in the context that have a geo political affiliation relationship with organization entity XXX.", "('ORG', 'GPE-AFF', 'ORG')": "find all organization entities in the context that have a geo political affiliation relationship with organization entity XXX.", "('ORG', 'GPE-AFF', 'PER')": "find all person entities in the context that have a geo political affiliation relationship with organization entity XXX.", "('ORG', 'GPE-AFF', 'VEH')": "find all vehicle entities in the context that have a geo political affiliation relationship with organization entity XXX.", "('ORG', 'GPE-AFF', 'WEA')": "find all weapon entities in the context that have a geo political affiliation relationship with organization entity XXX.", "('ORG', 'OTHER-AFF', 'FAC')": "find all facility entities in the context that have a person or organization affiliation relationship with organization entity XXX.", "('ORG', 'OTHER-AFF', 'GPE')": "find all geo political entities in the context that have a person or organization affiliation relationship with organization entity XXX.", "('ORG', 'OTHER-AFF', 'LOC')": "find all location entities in the context that have a person or organization affiliation relationship with organization entity XXX.", "('ORG', 'OTHER-AFF', 'ORG')": "find all organization entities in the context that have a person or organization affiliation relationship with organization entity XXX.", "('ORG', 'OTHER-AFF', 'PER')": "find all person entities in the context that have a person or organization affiliation relationship with organization entity XXX.", "('ORG', 'OTHER-AFF', 'VEH')": "find all vehicle entities in the context that have a person or organization affiliation relationship with organization entity XXX.", "('ORG', 'OTHER-AFF', 'WEA')": "find all weapon entities in the context that have a person or organization affiliation relationship with organization entity XXX.", "('ORG', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a personal or social relationship with organization entity XXX.", "('ORG', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a personal or social relationship with organization entity XXX.", "('ORG', 'PER-SOC', 'LOC')": "find all location entities in the context that have a personal or social relationship with organization entity XXX.", "('ORG', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a personal or social relationship with organization entity XXX.", "('ORG', 'PER-SOC', 'PER')": "find all person entities in the context that have a personal or social relationship with organization entity XXX.", "('ORG', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a personal or social relationship with organization entity XXX.", "('ORG', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a personal or social relationship with organization entity XXX.", "('ORG', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with organization entity XXX.", "('ORG', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with organization entity XXX.", "('ORG', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with organization entity XXX.", "('ORG', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with organization entity XXX.", "('ORG', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with organization entity XXX.", "('ORG', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with organization entity XXX.", "('ORG', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with organization entity XXX.", "('PER', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with person entity XXX.", "('PER', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with person entity XXX.", "('PER', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with person entity XXX.", "('PER', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with person entity XXX.", "('PER', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with person entity XXX.", "('PER', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with person entity XXX.", "('PER', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with person entity XXX.", "('PER', 'EMP-ORG', 'FAC')": "find all facility entities in the context that have a employment, membership or subsidiary relationship with person entity XXX.", "('PER', 'EMP-ORG', 'GPE')": "find all geo political entities in the context that have a employment, membership or subsidiary relationship with person entity XXX.", "('PER', 'EMP-ORG', 'LOC')": "find all location entities in the context that have a employment, membership or subsidiary relationship with person entity XXX.", "('PER', 'EMP-ORG', 'ORG')": "find all organization entities in the context that have a employment, membership or subsidiary relationship with person entity XXX.", "('PER', 'EMP-ORG', 'PER')": "find all person entities in the context that have a employment, membership or subsidiary relationship with person entity XXX.", "('PER', 'EMP-ORG', 'VEH')": "find all vehicle entities in the context that have a employment, membership or subsidiary relationship with person entity XXX.", "('PER', 'EMP-ORG', 'WEA')": "find all weapon entities in the context that have a employment, membership or subsidiary relationship with person entity XXX.", "('PER', 'GPE-AFF', 'FAC')": "find all facility entities in the context that have a geo political affiliation relationship with person entity XXX.", "('PER', 'GPE-AFF', 'GPE')": "find all geo political entities in the context that have a geo political affiliation relationship with person entity XXX.", "('PER', 'GPE-AFF', 'LOC')": "find all location entities in the context that have a geo political affiliation relationship with person entity XXX.", "('PER', 'GPE-AFF', 'ORG')": "find all organization entities in the context that have a geo political affiliation relationship with person entity XXX.", "('PER', 'GPE-AFF', 'PER')": "find all person entities in the context that have a geo political affiliation relationship with person entity XXX.", "('PER', 'GPE-AFF', 'VEH')": "find all vehicle entities in the context that have a geo political affiliation relationship with person entity XXX.", "('PER', 'GPE-AFF', 'WEA')": "find all weapon entities in the context that have a geo political affiliation relationship with person entity XXX.", "('PER', 'OTHER-AFF', 'FAC')": "find all facility entities in the context that have a person or organization affiliation relationship with person entity XXX.", "('PER', 'OTHER-AFF', 'GPE')": "find all geo political entities in the context that have a person or organization affiliation relationship with person entity XXX.", "('PER', 'OTHER-AFF', 'LOC')": "find all location entities in the context that have a person or organization affiliation relationship with person entity XXX.", "('PER', 'OTHER-AFF', 'ORG')": "find all organization entities in the context that have a person or organization affiliation relationship with person entity XXX.", "('PER', 'OTHER-AFF', 'PER')": "find all person entities in the context that have a person or organization affiliation relationship with person entity XXX.", "('PER', 'OTHER-AFF', 'VEH')": "find all vehicle entities in the context that have a person or organization affiliation relationship with person entity XXX.", "('PER', 'OTHER-AFF', 'WEA')": "find all weapon entities in the context that have a person or organization affiliation relationship with person entity XXX.", "('PER', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a personal or social relationship with person entity XXX.", "('PER', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a personal or social relationship with person entity XXX.", "('PER', 'PER-SOC', 'LOC')": "find all location entities in the context that have a personal or social relationship with person entity XXX.", "('PER', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a personal or social relationship with person entity XXX.", "('PER', 'PER-SOC', 'PER')": "find all person entities in the context that have a personal or social relationship with person entity XXX.", "('PER', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a personal or social relationship with person entity XXX.", "('PER', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a personal or social relationship with person entity XXX.", "('PER', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with person entity XXX.", "('PER', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with person entity XXX.", "('PER', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with person entity XXX.", "('PER', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with person entity XXX.", "('PER', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with person entity XXX.", "('PER', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with person entity XXX.", "('PER', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with person entity XXX.", "('VEH', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with vehicle entity XXX.", "('VEH', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with vehicle entity XXX.", "('VEH', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with vehicle entity XXX.", "('VEH', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with vehicle entity XXX.", "('VEH', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with vehicle entity XXX.", "('VEH', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with vehicle entity XXX.", "('VEH', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with vehicle entity XXX.", "('VEH', 'EMP-ORG', 'FAC')": "find all facility entities in the context that have a employment, membership or subsidiary relationship with vehicle entity XXX.", "('VEH', 'EMP-ORG', 'GPE')": "find all geo political entities in the context that have a employment, membership or subsidiary relationship with vehicle entity XXX.", "('VEH', 'EMP-ORG', 'LOC')": "find all location entities in the context that have a employment, membership or subsidiary relationship with vehicle entity XXX.", "('VEH', 'EMP-ORG', 'ORG')": "find all organization entities in the context that have a employment, membership or subsidiary relationship with vehicle entity XXX.", "('VEH', 'EMP-ORG', 'PER')": "find all person entities in the context that have a employment, membership or subsidiary relationship with vehicle entity XXX.", "('VEH', 'EMP-ORG', 'VEH')": "find all vehicle entities in the context that have a employment, membership or subsidiary relationship with vehicle entity XXX.", "('VEH', 'EMP-ORG', 'WEA')": "find all weapon entities in the context that have a employment, membership or subsidiary relationship with vehicle entity XXX.", "('VEH', 'GPE-AFF', 'FAC')": "find all facility entities in the context that have a geo political affiliation relationship with vehicle entity XXX.", "('VEH', 'GPE-AFF', 'GPE')": "find all geo political entities in the context that have a geo political affiliation relationship with vehicle entity XXX.", "('VEH', 'GPE-AFF', 'LOC')": "find all location entities in the context that have a geo political affiliation relationship with vehicle entity XXX.", "('VEH', 'GPE-AFF', 'ORG')": "find all organization entities in the context that have a geo political affiliation relationship with vehicle entity XXX.", "('VEH', 'GPE-AFF', 'PER')": "find all person entities in the context that have a geo political affiliation relationship with vehicle entity XXX.", "('VEH', 'GPE-AFF', 'VEH')": "find all vehicle entities in the context that have a geo political affiliation relationship with vehicle entity XXX.", "('VEH', 'GPE-AFF', 'WEA')": "find all weapon entities in the context that have a geo political affiliation relationship with vehicle entity XXX.", "('VEH', 'OTHER-AFF', 'FAC')": "find all facility entities in the context that have a person or organization affiliation relationship with vehicle entity XXX.", "('VEH', 'OTHER-AFF', 'GPE')": "find all geo political entities in the context that have a person or organization affiliation relationship with vehicle entity XXX.", "('VEH', 'OTHER-AFF', 'LOC')": "find all location entities in the context that have a person or organization affiliation relationship with vehicle entity XXX.", "('VEH', 'OTHER-AFF', 'ORG')": "find all organization entities in the context that have a person or organization affiliation relationship with vehicle entity XXX.", "('VEH', 'OTHER-AFF', 'PER')": "find all person entities in the context that have a person or organization affiliation relationship with vehicle entity XXX.", "('VEH', 'OTHER-AFF', 'VEH')": "find all vehicle entities in the context that have a person or organization affiliation relationship with vehicle entity XXX.", "('VEH', 'OTHER-AFF', 'WEA')": "find all weapon entities in the context that have a person or organization affiliation relationship with vehicle entity XXX.", "('VEH', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a personal or social relationship with vehicle entity XXX.", "('VEH', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a personal or social relationship with vehicle entity XXX.", "('VEH', 'PER-SOC', 'LOC')": "find all location entities in the context that have a personal or social relationship with vehicle entity XXX.", "('VEH', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a personal or social relationship with vehicle entity XXX.", "('VEH', 'PER-SOC', 'PER')": "find all person entities in the context that have a personal or social relationship with vehicle entity XXX.", "('VEH', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a personal or social relationship with vehicle entity XXX.", "('VEH', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a personal or social relationship with vehicle entity XXX.", "('VEH', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with vehicle entity XXX.", "('VEH', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with vehicle entity XXX.", "('VEH', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with vehicle entity XXX.", "('VEH', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with vehicle entity XXX.", "('VEH', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with vehicle entity XXX.", "('VEH', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with vehicle entity XXX.", "('VEH', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with vehicle entity XXX.", "('WEA', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with weapon entity XXX.", "('WEA', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with weapon entity XXX.", "('WEA', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with weapon entity XXX.", "('WEA', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with weapon entity XXX.", "('WEA', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with weapon entity XXX.", "('WEA', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with weapon entity XXX.", "('WEA', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with weapon entity XXX.", "('WEA', 'EMP-ORG', 'FAC')": "find all facility entities in the context that have a employment, membership or subsidiary relationship with weapon entity XXX.", "('WEA', 'EMP-ORG', 'GPE')": "find all geo political entities in the context that have a employment, membership or subsidiary relationship with weapon entity XXX.", "('WEA', 'EMP-ORG', 'LOC')": "find all location entities in the context that have a employment, membership or subsidiary relationship with weapon entity XXX.", "('WEA', 'EMP-ORG', 'ORG')": "find all organization entities in the context that have a employment, membership or subsidiary relationship with weapon entity XXX.", "('WEA', 'EMP-ORG', 'PER')": "find all person entities in the context that have a employment, membership or subsidiary relationship with weapon entity XXX.", "('WEA', 'EMP-ORG', 'VEH')": "find all vehicle entities in the context that have a employment, membership or subsidiary relationship with weapon entity XXX.", "('WEA', 'EMP-ORG', 'WEA')": "find all weapon entities in the context that have a employment, membership or subsidiary relationship with weapon entity XXX.", "('WEA', 'GPE-AFF', 'FAC')": "find all facility entities in the context that have a geo political affiliation relationship with weapon entity XXX.", "('WEA', 'GPE-AFF', 'GPE')": "find all geo political entities in the context that have a geo political affiliation relationship with weapon entity XXX.", "('WEA', 'GPE-AFF', 'LOC')": "find all location entities in the context that have a geo political affiliation relationship with weapon entity XXX.", "('WEA', 'GPE-AFF', 'ORG')": "find all organization entities in the context that have a geo political affiliation relationship with weapon entity XXX.", "('WEA', 'GPE-AFF', 'PER')": "find all person entities in the context that have a geo political affiliation relationship with weapon entity XXX.", "('WEA', 'GPE-AFF', 'VEH')": "find all vehicle entities in the context that have a geo political affiliation relationship with weapon entity XXX.", "('WEA', 'GPE-AFF', 'WEA')": "find all weapon entities in the context that have a geo political affiliation relationship with weapon entity XXX.", "('WEA', 'OTHER-AFF', 'FAC')": "find all facility entities in the context that have a person or organization affiliation relationship with weapon entity XXX.", "('WEA', 'OTHER-AFF', 'GPE')": "find all geo political entities in the context that have a person or organization affiliation relationship with weapon entity XXX.", "('WEA', 'OTHER-AFF', 'LOC')": "find all location entities in the context that have a person or organization affiliation relationship with weapon entity XXX.", "('WEA', 'OTHER-AFF', 'ORG')": "find all organization entities in the context that have a person or organization affiliation relationship with weapon entity XXX.", "('WEA', 'OTHER-AFF', 'PER')": "find all person entities in the context that have a person or organization affiliation relationship with weapon entity XXX.", "('WEA', 'OTHER-AFF', 'VEH')": "find all vehicle entities in the context that have a person or organization affiliation relationship with weapon entity XXX.", "('WEA', 'OTHER-AFF', 'WEA')": "find all weapon entities in the context that have a person or organization affiliation relationship with weapon entity XXX.", "('WEA', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a personal or social relationship with weapon entity XXX.", "('WEA', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a personal or social relationship with weapon entity XXX.", "('WEA', 'PER-SOC', 'LOC')": "find all location entities in the context that have a personal or social relationship with weapon entity XXX.", "('WEA', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a personal or social relationship with weapon entity XXX.", "('WEA', 'PER-SOC', 'PER')": "find all person entities in the context that have a personal or social relationship with weapon entity XXX.", "('WEA', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a personal or social relationship with weapon entity XXX.", "('WEA', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a personal or social relationship with weapon entity XXX.", "('WEA', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with weapon entity XXX.", "('WEA', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with weapon entity XXX.", "('WEA', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with weapon entity XXX.", "('WEA', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with weapon entity XXX.", "('WEA', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with weapon entity XXX.", "('WEA', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with weapon entity XXX.", "('WEA', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with weapon entity XXX."}}
--------------------------------------------------------------------------------
/data/query_templates/ace2005.json:
--------------------------------------------------------------------------------
1 | {
2 | "qa_turn1": {
3 | "FAC": "find all facility entities in the context.",
4 | "GPE": "find all geo political entities in the context.",
5 | "LOC": "find all location entities in the context.",
6 | "ORG": "find all organization entities in the context.",
7 | "PER": "find all person entities in the context.",
8 | "VEH": "find all vehicle entities in the context.",
9 | "WEA": "find all weapon entities in the context."
10 | },
11 | "qa_turn2": {
12 | "('FAC', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with facility entity XXX.",
13 | "('FAC', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with facility entity XXX.",
14 | "('FAC', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with facility entity XXX.",
15 | "('FAC', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with facility entity XXX.",
16 | "('FAC', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with facility entity XXX.",
17 | "('FAC', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with facility entity XXX.",
18 | "('FAC', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with facility entity XXX.",
19 | "('FAC', 'GEN-AFF', 'FAC')": "find all facility entities in the context that have a gen affilliation relationship with facility entity XXX.",
20 | "('FAC', 'GEN-AFF', 'GPE')": "find all geo political entities in the context that have a gen affilliation relationship with facility entity XXX.",
21 | "('FAC', 'GEN-AFF', 'LOC')": "find all location entities in the context that have a gen affilliation relationship with facility entity XXX.",
22 | "('FAC', 'GEN-AFF', 'ORG')": "find all organization entities in the context that have a gen affilliation relationship with facility entity XXX.",
23 | "('FAC', 'GEN-AFF', 'PER')": "find all person entities in the context that have a gen affilliation relationship with facility entity XXX.",
24 | "('FAC', 'GEN-AFF', 'VEH')": "find all vehicle entities in the context that have a gen affilliation relationship with facility entity XXX.",
25 | "('FAC', 'GEN-AFF', 'WEA')": "find all weapon entities in the context that have a gen affilliation relationship with facility entity XXX.",
26 | "('FAC', 'ORG-AFF', 'FAC')": "find all facility entities in the context that have an organization affiliation relationship with facility entity XXX.",
27 | "('FAC', 'ORG-AFF', 'GPE')": "find all geo political entities in the context that have an organization affiliation relationship with facility entity XXX.",
28 | "('FAC', 'ORG-AFF', 'LOC')": "find all location entities in the context that have an organization affiliation relationship with facility entity XXX.",
29 | "('FAC', 'ORG-AFF', 'ORG')": "find all organization entities in the context that have an organization affiliation relationship with facility entity XXX.",
30 | "('FAC', 'ORG-AFF', 'PER')": "find all person entities in the context that have an organization affiliation relationship with facility entity XXX.",
31 | "('FAC', 'ORG-AFF', 'VEH')": "find all vehicle entities in the context that have an organization affiliation relationship with facility entity XXX.",
32 | "('FAC', 'ORG-AFF', 'WEA')": "find all weapon entities in the context that have an organization affiliation relationship with facility entity XXX.",
33 | "('FAC', 'PART-WHOLE', 'FAC')": "find all facility entities in the context that have a part whole relationship with facility entity XXX.",
34 | "('FAC', 'PART-WHOLE', 'GPE')": "find all geo political entities in the context that have a part whole relationship with facility entity XXX.",
35 | "('FAC', 'PART-WHOLE', 'LOC')": "find all location entities in the context that have a part whole relationship with facility entity XXX.",
36 | "('FAC', 'PART-WHOLE', 'ORG')": "find all organization entities in the context that have a part whole relationship with facility entity XXX.",
37 | "('FAC', 'PART-WHOLE', 'PER')": "find all person entities in the context that have a part whole relationship with facility entity XXX.",
38 | "('FAC', 'PART-WHOLE', 'VEH')": "find all vehicle entities in the context that have a part whole relationship with facility entity XXX.",
39 | "('FAC', 'PART-WHOLE', 'WEA')": "find all weapon entities in the context that have a part whole relationship with facility entity XXX.",
40 | "('FAC', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a person social relationship with facility entity XXX.",
41 | "('FAC', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a person social relationship with facility entity XXX.",
42 | "('FAC', 'PER-SOC', 'LOC')": "find all location entities in the context that have a person social relationship with facility entity XXX.",
43 | "('FAC', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a person social relationship with facility entity XXX.",
44 | "('FAC', 'PER-SOC', 'PER')": "find all person entities in the context that have a person social relationship with facility entity XXX.",
45 | "('FAC', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a person social relationship with facility entity XXX.",
46 | "('FAC', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a person social relationship with facility entity XXX.",
47 | "('FAC', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with facility entity XXX.",
48 | "('FAC', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with facility entity XXX.",
49 | "('FAC', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with facility entity XXX.",
50 | "('FAC', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with facility entity XXX.",
51 | "('FAC', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with facility entity XXX.",
52 | "('FAC', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with facility entity XXX.",
53 | "('FAC', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with facility entity XXX.",
54 | "('GPE', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with geo political entity XXX.",
55 | "('GPE', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with geo political entity XXX.",
56 | "('GPE', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with geo political entity XXX.",
57 | "('GPE', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with geo political entity XXX.",
58 | "('GPE', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with geo political entity XXX.",
59 | "('GPE', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with geo political entity XXX.",
60 | "('GPE', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with geo political entity XXX.",
61 | "('GPE', 'GEN-AFF', 'FAC')": "find all facility entities in the context that have a gen affilliation relationship with geo political entity XXX.",
62 | "('GPE', 'GEN-AFF', 'GPE')": "find all geo political entities in the context that have a gen affilliation relationship with geo political entity XXX.",
63 | "('GPE', 'GEN-AFF', 'LOC')": "find all location entities in the context that have a gen affilliation relationship with geo political entity XXX.",
64 | "('GPE', 'GEN-AFF', 'ORG')": "find all organization entities in the context that have a gen affilliation relationship with geo political entity XXX.",
65 | "('GPE', 'GEN-AFF', 'PER')": "find all person entities in the context that have a gen affilliation relationship with geo political entity XXX.",
66 | "('GPE', 'GEN-AFF', 'VEH')": "find all vehicle entities in the context that have a gen affilliation relationship with geo political entity XXX.",
67 | "('GPE', 'GEN-AFF', 'WEA')": "find all weapon entities in the context that have a gen affilliation relationship with geo political entity XXX.",
68 | "('GPE', 'ORG-AFF', 'FAC')": "find all facility entities in the context that have an organization affiliation relationship with geo political entity XXX.",
69 | "('GPE', 'ORG-AFF', 'GPE')": "find all geo political entities in the context that have an organization affiliation relationship with geo political entity XXX.",
70 | "('GPE', 'ORG-AFF', 'LOC')": "find all location entities in the context that have an organization affiliation relationship with geo political entity XXX.",
71 | "('GPE', 'ORG-AFF', 'ORG')": "find all organization entities in the context that have an organization affiliation relationship with geo political entity XXX.",
72 | "('GPE', 'ORG-AFF', 'PER')": "find all person entities in the context that have an organization affiliation relationship with geo political entity XXX.",
73 | "('GPE', 'ORG-AFF', 'VEH')": "find all vehicle entities in the context that have an organization affiliation relationship with geo political entity XXX.",
74 | "('GPE', 'ORG-AFF', 'WEA')": "find all weapon entities in the context that have an organization affiliation relationship with geo political entity XXX.",
75 | "('GPE', 'PART-WHOLE', 'FAC')": "find all facility entities in the context that have a part whole relationship with geo political entity XXX.",
76 | "('GPE', 'PART-WHOLE', 'GPE')": "find all geo political entities in the context that have a part whole relationship with geo political entity XXX.",
77 | "('GPE', 'PART-WHOLE', 'LOC')": "find all location entities in the context that have a part whole relationship with geo political entity XXX.",
78 | "('GPE', 'PART-WHOLE', 'ORG')": "find all organization entities in the context that have a part whole relationship with geo political entity XXX.",
79 | "('GPE', 'PART-WHOLE', 'PER')": "find all person entities in the context that have a part whole relationship with geo political entity XXX.",
80 | "('GPE', 'PART-WHOLE', 'VEH')": "find all vehicle entities in the context that have a part whole relationship with geo political entity XXX.",
81 | "('GPE', 'PART-WHOLE', 'WEA')": "find all weapon entities in the context that have a part whole relationship with geo political entity XXX.",
82 | "('GPE', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a person social relationship with geo political entity XXX.",
83 | "('GPE', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a person social relationship with geo political entity XXX.",
84 | "('GPE', 'PER-SOC', 'LOC')": "find all location entities in the context that have a person social relationship with geo political entity XXX.",
85 | "('GPE', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a person social relationship with geo political entity XXX.",
86 | "('GPE', 'PER-SOC', 'PER')": "find all person entities in the context that have a person social relationship with geo political entity XXX.",
87 | "('GPE', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a person social relationship with geo political entity XXX.",
88 | "('GPE', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a person social relationship with geo political entity XXX.",
89 | "('GPE', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with geo political entity XXX.",
90 | "('GPE', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with geo political entity XXX.",
91 | "('GPE', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with geo political entity XXX.",
92 | "('GPE', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with geo political entity XXX.",
93 | "('GPE', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with geo political entity XXX.",
94 | "('GPE', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with geo political entity XXX.",
95 | "('GPE', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with geo political entity XXX.",
96 | "('LOC', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with location entity XXX.",
97 | "('LOC', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with location entity XXX.",
98 | "('LOC', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with location entity XXX.",
99 | "('LOC', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with location entity XXX.",
100 | "('LOC', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with location entity XXX.",
101 | "('LOC', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with location entity XXX.",
102 | "('LOC', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with location entity XXX.",
103 | "('LOC', 'GEN-AFF', 'FAC')": "find all facility entities in the context that have a gen affilliation relationship with location entity XXX.",
104 | "('LOC', 'GEN-AFF', 'GPE')": "find all geo political entities in the context that have a gen affilliation relationship with location entity XXX.",
105 | "('LOC', 'GEN-AFF', 'LOC')": "find all location entities in the context that have a gen affilliation relationship with location entity XXX.",
106 | "('LOC', 'GEN-AFF', 'ORG')": "find all organization entities in the context that have a gen affilliation relationship with location entity XXX.",
107 | "('LOC', 'GEN-AFF', 'PER')": "find all person entities in the context that have a gen affilliation relationship with location entity XXX.",
108 | "('LOC', 'GEN-AFF', 'VEH')": "find all vehicle entities in the context that have a gen affilliation relationship with location entity XXX.",
109 | "('LOC', 'GEN-AFF', 'WEA')": "find all weapon entities in the context that have a gen affilliation relationship with location entity XXX.",
110 | "('LOC', 'ORG-AFF', 'FAC')": "find all facility entities in the context that have an organization affiliation relationship with location entity XXX.",
111 | "('LOC', 'ORG-AFF', 'GPE')": "find all geo political entities in the context that have an organization affiliation relationship with location entity XXX.",
112 | "('LOC', 'ORG-AFF', 'LOC')": "find all location entities in the context that have an organization affiliation relationship with location entity XXX.",
113 | "('LOC', 'ORG-AFF', 'ORG')": "find all organization entities in the context that have an organization affiliation relationship with location entity XXX.",
114 | "('LOC', 'ORG-AFF', 'PER')": "find all person entities in the context that have an organization affiliation relationship with location entity XXX.",
115 | "('LOC', 'ORG-AFF', 'VEH')": "find all vehicle entities in the context that have an organization affiliation relationship with location entity XXX.",
116 | "('LOC', 'ORG-AFF', 'WEA')": "find all weapon entities in the context that have an organization affiliation relationship with location entity XXX.",
117 | "('LOC', 'PART-WHOLE', 'FAC')": "find all facility entities in the context that have a part whole relationship with location entity XXX.",
118 | "('LOC', 'PART-WHOLE', 'GPE')": "find all geo political entities in the context that have a part whole relationship with location entity XXX.",
119 | "('LOC', 'PART-WHOLE', 'LOC')": "find all location entities in the context that have a part whole relationship with location entity XXX.",
120 | "('LOC', 'PART-WHOLE', 'ORG')": "find all organization entities in the context that have a part whole relationship with location entity XXX.",
121 | "('LOC', 'PART-WHOLE', 'PER')": "find all person entities in the context that have a part whole relationship with location entity XXX.",
122 | "('LOC', 'PART-WHOLE', 'VEH')": "find all vehicle entities in the context that have a part whole relationship with location entity XXX.",
123 | "('LOC', 'PART-WHOLE', 'WEA')": "find all weapon entities in the context that have a part whole relationship with location entity XXX.",
124 | "('LOC', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a person social relationship with location entity XXX.",
125 | "('LOC', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a person social relationship with location entity XXX.",
126 | "('LOC', 'PER-SOC', 'LOC')": "find all location entities in the context that have a person social relationship with location entity XXX.",
127 | "('LOC', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a person social relationship with location entity XXX.",
128 | "('LOC', 'PER-SOC', 'PER')": "find all person entities in the context that have a person social relationship with location entity XXX.",
129 | "('LOC', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a person social relationship with location entity XXX.",
130 | "('LOC', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a person social relationship with location entity XXX.",
131 | "('LOC', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with location entity XXX.",
132 | "('LOC', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with location entity XXX.",
133 | "('LOC', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with location entity XXX.",
134 | "('LOC', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with location entity XXX.",
135 | "('LOC', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with location entity XXX.",
136 | "('LOC', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with location entity XXX.",
137 | "('LOC', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with location entity XXX.",
138 | "('ORG', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with organization entity XXX.",
139 | "('ORG', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with organization entity XXX.",
140 | "('ORG', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with organization entity XXX.",
141 | "('ORG', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with organization entity XXX.",
142 | "('ORG', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with organization entity XXX.",
143 | "('ORG', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with organization entity XXX.",
144 | "('ORG', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with organization entity XXX.",
145 | "('ORG', 'GEN-AFF', 'FAC')": "find all facility entities in the context that have a gen affilliation relationship with organization entity XXX.",
146 | "('ORG', 'GEN-AFF', 'GPE')": "find all geo political entities in the context that have a gen affilliation relationship with organization entity XXX.",
147 | "('ORG', 'GEN-AFF', 'LOC')": "find all location entities in the context that have a gen affilliation relationship with organization entity XXX.",
148 | "('ORG', 'GEN-AFF', 'ORG')": "find all organization entities in the context that have a gen affilliation relationship with organization entity XXX.",
149 | "('ORG', 'GEN-AFF', 'PER')": "find all person entities in the context that have a gen affilliation relationship with organization entity XXX.",
150 | "('ORG', 'GEN-AFF', 'VEH')": "find all vehicle entities in the context that have a gen affilliation relationship with organization entity XXX.",
151 | "('ORG', 'GEN-AFF', 'WEA')": "find all weapon entities in the context that have a gen affilliation relationship with organization entity XXX.",
152 | "('ORG', 'ORG-AFF', 'FAC')": "find all facility entities in the context that have an organization affiliation relationship with organization entity XXX.",
153 | "('ORG', 'ORG-AFF', 'GPE')": "find all geo political entities in the context that have an organization affiliation relationship with organization entity XXX.",
154 | "('ORG', 'ORG-AFF', 'LOC')": "find all location entities in the context that have an organization affiliation relationship with organization entity XXX.",
155 | "('ORG', 'ORG-AFF', 'ORG')": "find all organization entities in the context that have an organization affiliation relationship with organization entity XXX.",
156 | "('ORG', 'ORG-AFF', 'PER')": "find all person entities in the context that have an organization affiliation relationship with organization entity XXX.",
157 | "('ORG', 'ORG-AFF', 'VEH')": "find all vehicle entities in the context that have an organization affiliation relationship with organization entity XXX.",
158 | "('ORG', 'ORG-AFF', 'WEA')": "find all weapon entities in the context that have an organization affiliation relationship with organization entity XXX.",
159 | "('ORG', 'PART-WHOLE', 'FAC')": "find all facility entities in the context that have a part whole relationship with organization entity XXX.",
160 | "('ORG', 'PART-WHOLE', 'GPE')": "find all geo political entities in the context that have a part whole relationship with organization entity XXX.",
161 | "('ORG', 'PART-WHOLE', 'LOC')": "find all location entities in the context that have a part whole relationship with organization entity XXX.",
162 | "('ORG', 'PART-WHOLE', 'ORG')": "find all organization entities in the context that have a part whole relationship with organization entity XXX.",
163 | "('ORG', 'PART-WHOLE', 'PER')": "find all person entities in the context that have a part whole relationship with organization entity XXX.",
164 | "('ORG', 'PART-WHOLE', 'VEH')": "find all vehicle entities in the context that have a part whole relationship with organization entity XXX.",
165 | "('ORG', 'PART-WHOLE', 'WEA')": "find all weapon entities in the context that have a part whole relationship with organization entity XXX.",
166 | "('ORG', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a person social relationship with organization entity XXX.",
167 | "('ORG', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a person social relationship with organization entity XXX.",
168 | "('ORG', 'PER-SOC', 'LOC')": "find all location entities in the context that have a person social relationship with organization entity XXX.",
169 | "('ORG', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a person social relationship with organization entity XXX.",
170 | "('ORG', 'PER-SOC', 'PER')": "find all person entities in the context that have a person social relationship with organization entity XXX.",
171 | "('ORG', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a person social relationship with organization entity XXX.",
172 | "('ORG', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a person social relationship with organization entity XXX.",
173 | "('ORG', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with organization entity XXX.",
174 | "('ORG', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with organization entity XXX.",
175 | "('ORG', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with organization entity XXX.",
176 | "('ORG', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with organization entity XXX.",
177 | "('ORG', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with organization entity XXX.",
178 | "('ORG', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with organization entity XXX.",
179 | "('ORG', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with organization entity XXX.",
180 | "('PER', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with person entity XXX.",
181 | "('PER', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with person entity XXX.",
182 | "('PER', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with person entity XXX.",
183 | "('PER', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with person entity XXX.",
184 | "('PER', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with person entity XXX.",
185 | "('PER', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with person entity XXX.",
186 | "('PER', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with person entity XXX.",
187 | "('PER', 'GEN-AFF', 'FAC')": "find all facility entities in the context that have a gen affilliation relationship with person entity XXX.",
188 | "('PER', 'GEN-AFF', 'GPE')": "find all geo political entities in the context that have a gen affilliation relationship with person entity XXX.",
189 | "('PER', 'GEN-AFF', 'LOC')": "find all location entities in the context that have a gen affilliation relationship with person entity XXX.",
190 | "('PER', 'GEN-AFF', 'ORG')": "find all organization entities in the context that have a gen affilliation relationship with person entity XXX.",
191 | "('PER', 'GEN-AFF', 'PER')": "find all person entities in the context that have a gen affilliation relationship with person entity XXX.",
192 | "('PER', 'GEN-AFF', 'VEH')": "find all vehicle entities in the context that have a gen affilliation relationship with person entity XXX.",
193 | "('PER', 'GEN-AFF', 'WEA')": "find all weapon entities in the context that have a gen affilliation relationship with person entity XXX.",
194 | "('PER', 'ORG-AFF', 'FAC')": "find all facility entities in the context that have an organization affiliation relationship with person entity XXX.",
195 | "('PER', 'ORG-AFF', 'GPE')": "find all geo political entities in the context that have an organization affiliation relationship with person entity XXX.",
196 | "('PER', 'ORG-AFF', 'LOC')": "find all location entities in the context that have an organization affiliation relationship with person entity XXX.",
197 | "('PER', 'ORG-AFF', 'ORG')": "find all organization entities in the context that have an organization affiliation relationship with person entity XXX.",
198 | "('PER', 'ORG-AFF', 'PER')": "find all person entities in the context that have an organization affiliation relationship with person entity XXX.",
199 | "('PER', 'ORG-AFF', 'VEH')": "find all vehicle entities in the context that have an organization affiliation relationship with person entity XXX.",
200 | "('PER', 'ORG-AFF', 'WEA')": "find all weapon entities in the context that have an organization affiliation relationship with person entity XXX.",
201 | "('PER', 'PART-WHOLE', 'FAC')": "find all facility entities in the context that have a part whole relationship with person entity XXX.",
202 | "('PER', 'PART-WHOLE', 'GPE')": "find all geo political entities in the context that have a part whole relationship with person entity XXX.",
203 | "('PER', 'PART-WHOLE', 'LOC')": "find all location entities in the context that have a part whole relationship with person entity XXX.",
204 | "('PER', 'PART-WHOLE', 'ORG')": "find all organization entities in the context that have a part whole relationship with person entity XXX.",
205 | "('PER', 'PART-WHOLE', 'PER')": "find all person entities in the context that have a part whole relationship with person entity XXX.",
206 | "('PER', 'PART-WHOLE', 'VEH')": "find all vehicle entities in the context that have a part whole relationship with person entity XXX.",
207 | "('PER', 'PART-WHOLE', 'WEA')": "find all weapon entities in the context that have a part whole relationship with person entity XXX.",
208 | "('PER', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a person social relationship with person entity XXX.",
209 | "('PER', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a person social relationship with person entity XXX.",
210 | "('PER', 'PER-SOC', 'LOC')": "find all location entities in the context that have a person social relationship with person entity XXX.",
211 | "('PER', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a person social relationship with person entity XXX.",
212 | "('PER', 'PER-SOC', 'PER')": "find all person entities in the context that have a person social relationship with person entity XXX.",
213 | "('PER', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a person social relationship with person entity XXX.",
214 | "('PER', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a person social relationship with person entity XXX.",
215 | "('PER', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with person entity XXX.",
216 | "('PER', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with person entity XXX.",
217 | "('PER', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with person entity XXX.",
218 | "('PER', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with person entity XXX.",
219 | "('PER', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with person entity XXX.",
220 | "('PER', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with person entity XXX.",
221 | "('PER', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with person entity XXX.",
222 | "('VEH', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with vehicle entity XXX.",
223 | "('VEH', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with vehicle entity XXX.",
224 | "('VEH', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with vehicle entity XXX.",
225 | "('VEH', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with vehicle entity XXX.",
226 | "('VEH', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with vehicle entity XXX.",
227 | "('VEH', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with vehicle entity XXX.",
228 | "('VEH', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with vehicle entity XXX.",
229 | "('VEH', 'GEN-AFF', 'FAC')": "find all facility entities in the context that have a gen affilliation relationship with vehicle entity XXX.",
230 | "('VEH', 'GEN-AFF', 'GPE')": "find all geo political entities in the context that have a gen affilliation relationship with vehicle entity XXX.",
231 | "('VEH', 'GEN-AFF', 'LOC')": "find all location entities in the context that have a gen affilliation relationship with vehicle entity XXX.",
232 | "('VEH', 'GEN-AFF', 'ORG')": "find all organization entities in the context that have a gen affilliation relationship with vehicle entity XXX.",
233 | "('VEH', 'GEN-AFF', 'PER')": "find all person entities in the context that have a gen affilliation relationship with vehicle entity XXX.",
234 | "('VEH', 'GEN-AFF', 'VEH')": "find all vehicle entities in the context that have a gen affilliation relationship with vehicle entity XXX.",
235 | "('VEH', 'GEN-AFF', 'WEA')": "find all weapon entities in the context that have a gen affilliation relationship with vehicle entity XXX.",
236 | "('VEH', 'ORG-AFF', 'FAC')": "find all facility entities in the context that have an organization affiliation relationship with vehicle entity XXX.",
237 | "('VEH', 'ORG-AFF', 'GPE')": "find all geo political entities in the context that have an organization affiliation relationship with vehicle entity XXX.",
238 | "('VEH', 'ORG-AFF', 'LOC')": "find all location entities in the context that have an organization affiliation relationship with vehicle entity XXX.",
239 | "('VEH', 'ORG-AFF', 'ORG')": "find all organization entities in the context that have an organization affiliation relationship with vehicle entity XXX.",
240 | "('VEH', 'ORG-AFF', 'PER')": "find all person entities in the context that have an organization affiliation relationship with vehicle entity XXX.",
241 | "('VEH', 'ORG-AFF', 'VEH')": "find all vehicle entities in the context that have an organization affiliation relationship with vehicle entity XXX.",
242 | "('VEH', 'ORG-AFF', 'WEA')": "find all weapon entities in the context that have an organization affiliation relationship with vehicle entity XXX.",
243 | "('VEH', 'PART-WHOLE', 'FAC')": "find all facility entities in the context that have a part whole relationship with vehicle entity XXX.",
244 | "('VEH', 'PART-WHOLE', 'GPE')": "find all geo political entities in the context that have a part whole relationship with vehicle entity XXX.",
245 | "('VEH', 'PART-WHOLE', 'LOC')": "find all location entities in the context that have a part whole relationship with vehicle entity XXX.",
246 | "('VEH', 'PART-WHOLE', 'ORG')": "find all organization entities in the context that have a part whole relationship with vehicle entity XXX.",
247 | "('VEH', 'PART-WHOLE', 'PER')": "find all person entities in the context that have a part whole relationship with vehicle entity XXX.",
248 | "('VEH', 'PART-WHOLE', 'VEH')": "find all vehicle entities in the context that have a part whole relationship with vehicle entity XXX.",
249 | "('VEH', 'PART-WHOLE', 'WEA')": "find all weapon entities in the context that have a part whole relationship with vehicle entity XXX.",
250 | "('VEH', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a person social relationship with vehicle entity XXX.",
251 | "('VEH', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a person social relationship with vehicle entity XXX.",
252 | "('VEH', 'PER-SOC', 'LOC')": "find all location entities in the context that have a person social relationship with vehicle entity XXX.",
253 | "('VEH', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a person social relationship with vehicle entity XXX.",
254 | "('VEH', 'PER-SOC', 'PER')": "find all person entities in the context that have a person social relationship with vehicle entity XXX.",
255 | "('VEH', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a person social relationship with vehicle entity XXX.",
256 | "('VEH', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a person social relationship with vehicle entity XXX.",
257 | "('VEH', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with vehicle entity XXX.",
258 | "('VEH', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with vehicle entity XXX.",
259 | "('VEH', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with vehicle entity XXX.",
260 | "('VEH', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with vehicle entity XXX.",
261 | "('VEH', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with vehicle entity XXX.",
262 | "('VEH', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with vehicle entity XXX.",
263 | "('VEH', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with vehicle entity XXX.",
264 | "('WEA', 'ART', 'FAC')": "find all facility entities in the context that have an artifact relationship with weapon entity XXX.",
265 | "('WEA', 'ART', 'GPE')": "find all geo political entities in the context that have an artifact relationship with weapon entity XXX.",
266 | "('WEA', 'ART', 'LOC')": "find all location entities in the context that have an artifact relationship with weapon entity XXX.",
267 | "('WEA', 'ART', 'ORG')": "find all organization entities in the context that have an artifact relationship with weapon entity XXX.",
268 | "('WEA', 'ART', 'PER')": "find all person entities in the context that have an artifact relationship with weapon entity XXX.",
269 | "('WEA', 'ART', 'VEH')": "find all vehicle entities in the context that have an artifact relationship with weapon entity XXX.",
270 | "('WEA', 'ART', 'WEA')": "find all weapon entities in the context that have an artifact relationship with weapon entity XXX.",
271 | "('WEA', 'GEN-AFF', 'FAC')": "find all facility entities in the context that have a gen affilliation relationship with weapon entity XXX.",
272 | "('WEA', 'GEN-AFF', 'GPE')": "find all geo political entities in the context that have a gen affilliation relationship with weapon entity XXX.",
273 | "('WEA', 'GEN-AFF', 'LOC')": "find all location entities in the context that have a gen affilliation relationship with weapon entity XXX.",
274 | "('WEA', 'GEN-AFF', 'ORG')": "find all organization entities in the context that have a gen affilliation relationship with weapon entity XXX.",
275 | "('WEA', 'GEN-AFF', 'PER')": "find all person entities in the context that have a gen affilliation relationship with weapon entity XXX.",
276 | "('WEA', 'GEN-AFF', 'VEH')": "find all vehicle entities in the context that have a gen affilliation relationship with weapon entity XXX.",
277 | "('WEA', 'GEN-AFF', 'WEA')": "find all weapon entities in the context that have a gen affilliation relationship with weapon entity XXX.",
278 | "('WEA', 'ORG-AFF', 'FAC')": "find all facility entities in the context that have an organization affiliation relationship with weapon entity XXX.",
279 | "('WEA', 'ORG-AFF', 'GPE')": "find all geo political entities in the context that have an organization affiliation relationship with weapon entity XXX.",
280 | "('WEA', 'ORG-AFF', 'LOC')": "find all location entities in the context that have an organization affiliation relationship with weapon entity XXX.",
281 | "('WEA', 'ORG-AFF', 'ORG')": "find all organization entities in the context that have an organization affiliation relationship with weapon entity XXX.",
282 | "('WEA', 'ORG-AFF', 'PER')": "find all person entities in the context that have an organization affiliation relationship with weapon entity XXX.",
283 | "('WEA', 'ORG-AFF', 'VEH')": "find all vehicle entities in the context that have an organization affiliation relationship with weapon entity XXX.",
284 | "('WEA', 'ORG-AFF', 'WEA')": "find all weapon entities in the context that have an organization affiliation relationship with weapon entity XXX.",
285 | "('WEA', 'PART-WHOLE', 'FAC')": "find all facility entities in the context that have a part whole relationship with weapon entity XXX.",
286 | "('WEA', 'PART-WHOLE', 'GPE')": "find all geo political entities in the context that have a part whole relationship with weapon entity XXX.",
287 | "('WEA', 'PART-WHOLE', 'LOC')": "find all location entities in the context that have a part whole relationship with weapon entity XXX.",
288 | "('WEA', 'PART-WHOLE', 'ORG')": "find all organization entities in the context that have a part whole relationship with weapon entity XXX.",
289 | "('WEA', 'PART-WHOLE', 'PER')": "find all person entities in the context that have a part whole relationship with weapon entity XXX.",
290 | "('WEA', 'PART-WHOLE', 'VEH')": "find all vehicle entities in the context that have a part whole relationship with weapon entity XXX.",
291 | "('WEA', 'PART-WHOLE', 'WEA')": "find all weapon entities in the context that have a part whole relationship with weapon entity XXX.",
292 | "('WEA', 'PER-SOC', 'FAC')": "find all facility entities in the context that have a person social relationship with weapon entity XXX.",
293 | "('WEA', 'PER-SOC', 'GPE')": "find all geo political entities in the context that have a person social relationship with weapon entity XXX.",
294 | "('WEA', 'PER-SOC', 'LOC')": "find all location entities in the context that have a person social relationship with weapon entity XXX.",
295 | "('WEA', 'PER-SOC', 'ORG')": "find all organization entities in the context that have a person social relationship with weapon entity XXX.",
296 | "('WEA', 'PER-SOC', 'PER')": "find all person entities in the context that have a person social relationship with weapon entity XXX.",
297 | "('WEA', 'PER-SOC', 'VEH')": "find all vehicle entities in the context that have a person social relationship with weapon entity XXX.",
298 | "('WEA', 'PER-SOC', 'WEA')": "find all weapon entities in the context that have a person social relationship with weapon entity XXX.",
299 | "('WEA', 'PHYS', 'FAC')": "find all facility entities in the context that have a physical relationship with weapon entity XXX.",
300 | "('WEA', 'PHYS', 'GPE')": "find all geo political entities in the context that have a physical relationship with weapon entity XXX.",
301 | "('WEA', 'PHYS', 'LOC')": "find all location entities in the context that have a physical relationship with weapon entity XXX.",
302 | "('WEA', 'PHYS', 'ORG')": "find all organization entities in the context that have a physical relationship with weapon entity XXX.",
303 | "('WEA', 'PHYS', 'PER')": "find all person entities in the context that have a physical relationship with weapon entity XXX.",
304 | "('WEA', 'PHYS', 'VEH')": "find all vehicle entities in the context that have a physical relationship with weapon entity XXX.",
305 | "('WEA', 'PHYS', 'WEA')": "find all weapon entities in the context that have a physical relationship with weapon entity XXX."
306 | }
307 | }
--------------------------------------------------------------------------------
/data/query_templates/generate_query.py:
--------------------------------------------------------------------------------
1 | import json
2 |
3 |
4 | ace2004_entities = ['FAC', 'GPE', 'LOC', 'ORG', 'PER', 'VEH', 'WEA']
5 | ace2004_entities_full = ["facility","geo political","location","organization","person","vehicle","weapon"]
6 | ace2004_relations = ['ART', 'EMP-ORG', 'GPE-AFF', 'OTHER-AFF', 'PER-SOC', 'PHYS']
7 | ace2004_relations_full = ['artifact','employment, membership or subsidiary','geo political affiliation','person or organization affiliation','personal or social','physical']
8 |
9 | ace2005_entities = ['FAC', 'GPE', 'LOC', 'ORG', 'PER', 'VEH', 'WEA']
10 | ace2005_entities_full = ["facility","geo political","location","organization","person","vehicle","weapon"]
11 | ace2005_relations = ['ART', 'GEN-AFF', 'ORG-AFF', 'PART-WHOLE', 'PER-SOC', 'PHYS']
12 | ace2005_relations_full = ["artifact","gen affilliation",'organization affiliation','part whole','person social','physical']
13 |
14 |
15 | if __name__=="__main__":
16 | #这里的模板应该保证我们的模型的query和对应的entity type或者(head_entity relation_type, end_entity_type)存在一一对应关系
17 | templates = {"qa_turn1":{},"qa_turn2":{}}
18 | for ent1,ent1f in zip(ace2005_entities,ace2005_entities_full):
19 | templates['qa_turn1'][ent1]="find all {} entities in the context.".format(ent1f)
20 | for rel,relf in zip(ace2005_relations,ace2005_relations_full):
21 | for ent2,ent2f in zip(ace2005_entities,ace2005_entities_full):
22 | templates['qa_turn2'][str((ent1,rel,ent2))]="find all {} entities in the context that have {} {} relationship with {} entity XXX.".format(ent2f,'an' if relf[0] in ['a','o'] else 'a',relf,ent1f)
23 | with open("ace2005.json",'w') as f:
24 | json.dump(templates,f)
25 | for ent1,ent1f in zip(ace2004_entities,ace2004_entities_full):
26 | templates['qa_turn1'][ent1]="find all {} entities in the context.".format(ent1f)
27 | for rel,relf in zip(ace2004_relations,ace2004_relations_full):
28 | for ent2,ent2f in zip(ace2004_entities,ace2004_entities_full):
29 | templates['qa_turn2'][str((ent1,rel,ent2))]="find all {} entities in the context that have {} {} relationship with {} entity XXX.".format(ent2f,'an' if relf[0] in ['a','o'] else 'a',relf,ent1f)
30 | with open("ace2004.json",'w') as f:
31 | json.dump(templates,f)
--------------------------------------------------------------------------------
/data/raw_data/ace2004.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ShannonAI/Entity-Relation-As-Multi-Turn-QA/30e1ba83fe4b2e3a6bfa17682193d4b8ecd691d8/data/raw_data/ace2004.zip
--------------------------------------------------------------------------------
/data/raw_data/ace2005.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ShannonAI/Entity-Relation-As-Multi-Turn-QA/30e1ba83fe4b2e3a6bfa17682193d4b8ecd691d8/data/raw_data/ace2005.zip
--------------------------------------------------------------------------------
/dataloader.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import numpy as np
3 | from tqdm import tqdm
4 | from torch.nn.utils.rnn import pad_sequence
5 | from torch.utils.data import DataLoader, DistributedSampler
6 | from transformers import BertTokenizer
7 |
8 | from preprocess import passage_blocks, get_question
9 | from constants import *
10 |
11 |
12 | def collate_fn(batch):
13 | # for training
14 | nbatch = {}
15 | for b in batch:
16 | for k, v in b.items():
17 | nbatch[k] = nbatch.get(k, []) + [torch.tensor(v)]
18 | txt_ids = nbatch['txt_ids']
19 | tags = nbatch['tags']
20 | context_mask = nbatch['context_mask']
21 | token_type_ids = nbatch['token_type_ids']
22 | turn_mask = nbatch['turn_mask']
23 | ntxt_ids = pad_sequence(txt_ids, batch_first=True,
24 | padding_value=0)
25 | ntags = pad_sequence(tags, batch_first=True, padding_value=-1)
26 | ncontext_mask = pad_sequence(
27 | context_mask, batch_first=True, padding_value=0)
28 | ntoken_type_ids = pad_sequence(
29 | token_type_ids, batch_first=True, padding_value=1)
30 | attention_mask = torch.zeros(ntxt_ids.shape)
31 | for i in range(len(ntxt_ids)):
32 | txt_len = len(txt_ids[i])
33 | attention_mask[i, :txt_len] = 1
34 | nbatch['txt_ids'] = ntxt_ids
35 | nbatch['tags'] = ntags
36 | nbatch['context_mask'] = ncontext_mask
37 | nbatch['token_type_ids'] = ntoken_type_ids
38 | nbatch['attention_mask'] = attention_mask
39 | nbatch['turn_mask'] = torch.tensor(turn_mask, dtype=torch.uint8)
40 | return nbatch
41 |
42 |
43 | def collate_fn1(batch):
44 | # for testing
45 | nbatch = {}
46 | for b in batch:
47 | for k, v in b.items():
48 | nbatch[k] = nbatch.get(k, []) + [torch.tensor(v)]
49 | txt_ids = nbatch['txt_ids']
50 | context_mask = nbatch['context_mask']
51 | token_type_ids = nbatch['token_type_ids']
52 | ntxt_ids = pad_sequence(txt_ids, batch_first=True,
53 | padding_value=0)
54 | ncontext_mask = pad_sequence(
55 | context_mask, batch_first=True, padding_value=0)
56 | ntoken_type_ids = pad_sequence(
57 | token_type_ids, batch_first=True, padding_value=1)
58 | attention_mask = torch.zeros(ntxt_ids.shape)
59 | for i in range(len(ntxt_ids)):
60 | txt_len = len(txt_ids[i])
61 | attention_mask[i, :txt_len] = 1
62 | nbatch['txt_ids'] = ntxt_ids
63 | nbatch['context_mask'] = ncontext_mask
64 | nbatch['token_type_ids'] = ntoken_type_ids
65 | nbatch['attention_mask'] = attention_mask
66 | return nbatch
67 |
68 |
69 | def get_inputs(context, q, tokenizer, title="", max_len=200, ans=[], head_entity=None):
70 | query = tokenizer.tokenize(q)
71 | tags = [tag_idxs['O']]*len(context)
72 | for i, an in enumerate(ans):
73 | start, end = an[1:-1]
74 | end = end-1
75 | if start != end:
76 | tags[start] = tag_idxs['B']
77 | tags[end] = tag_idxs['E']
78 | for i in range(start+1, end):
79 | tags[i] = tag_idxs['M']
80 | else:
81 | tags[start] = tag_idxs['S']
82 | if head_entity:
83 | h_start, h_end = head_entity[1], head_entity[2]
84 | context = context[:h_start]+['[unused0]'] + \
85 | context[h_start:h_end]+["[unused1]"]+context[h_end:]
86 | assert len(context) == len(tags)+2
87 | tags = tags[:h_start]+[tag_idxs['O']] + \
88 | tags[h_start:h_end]+[tag_idxs['O']]+tags[h_end:]
89 | txt_len = len(query)+len(title)+len(context) + \
90 | 4 if title else len(query)+len(context)+3
91 | if txt_len > max_len:
92 | context = context[:max_len -
93 | len(query) - 3] if not title else context[:max_len-len(query)-len(title)-4]
94 | tags = tags[:max_len -
95 | len(query) - 3] if not title else tags[:max_len-len(query)-len(title)-4]
96 | if title:
97 | txt = ['[CLS]'] + query+['[SEP]'] + \
98 | title + ['[SEP]'] + context + ['[SEP]']
99 | else:
100 | txt = ['[CLS]'] + query + ['[SEP]'] + context + ['[SEP]']
101 | txt_ids = tokenizer.convert_tokens_to_ids(txt)
102 | # [CLS] is used to judge whether there is an answe
103 | if not title:
104 | tags1 = [tag_idxs['O'] if len(
105 | ans) > 0 else tag_idxs['S']] + [-1] * (len(query) + 1) + tags + [-1]
106 | context_mask = [1] + [0] * (len(query) + 1) + [1] * len(context) + [0]
107 | token_type_ids = [0] * (len(query) + 2) + [1] * (len(context) + 1)
108 | else:
109 | tags1 = [tag_idxs['O'] if len(
110 | ans) > 0 else tag_idxs['S']] + [-1]*(len(query)+len(title)+2) + tags + [-1]
111 | context_mask = [1] + [0] * \
112 | (len(query)+len(title)+2) + [1] * len(context) + [0]
113 | token_type_ids = [0]*(len(query)+len(title)+3)+[1]*(len(context) + 1)
114 | return txt_ids, tags1, context_mask, token_type_ids
115 |
116 |
117 | def query2relation(question, question_templates):
118 | '''
119 | query ->
120 | '''
121 | turn2_questions = question_templates['qa_turn2']
122 | turn2_questions = {v: k for k, v in turn2_questions.items()}
123 | for k, v in turn2_questions.items():
124 | k1 = k.replace("XXX.", "")
125 | if question.startswith(k1):
126 | return eval(v)
127 | raise Exception("cannot find the relation type corresponding to the query, if the \
128 | query template is changed, please re-implement this function according to the new template")
129 |
130 |
131 | class MyDataset:
132 | def __init__(self, dataset_tag, path, tokenizer, max_len=512, threshold=5):
133 | """
134 | Args:
135 | dataset_tag: type of dataset
136 | path: path to training set file
137 | tokenizer： tokenizer of pretrained model
138 | max_len: max length of input
139 | threshold: only consider relationships where the frequency is greater than or equal to the threshold
140 | """
141 | with open(path, encoding='utf-8') as f:
142 | data = json.load(f)
143 | self.data = data
144 | self.tokenizer = tokenizer
145 | self.max_len = max_len
146 | self.threshold = threshold
147 | self.dataset_tag = dataset_tag
148 | self.init_data()
149 |
150 | def init_data(self):
151 | self.all_t1 = []
152 | self.all_t2 = []
153 | if self.dataset_tag.lower() == "ace2004":
154 | idx1s = ace2004_idx1
155 | idx2s = ace2004_idx2
156 | dist = ace2004_dist
157 | question_templates = ace2004_question_templates
158 | elif self.dataset_tag.lower() == 'ace2005':
159 | idx1s = ace2005_idx1
160 | idx2s = ace2005_idx2
161 | dist = ace2005_dist
162 | question_templates = ace2005_question_templates
163 | else:
164 | raise Exception("this dataset is not yet supported")
165 | for d in tqdm(self.data, desc="dataset"):
166 | context = d['context']
167 | title = d['title']
168 | qa_pairs = d['qa_pairs']
169 | t1 = qa_pairs[0]
170 | t2 = qa_pairs[1]
171 | t1_qas = []
172 | t2_qas = []
173 | for i, (q, ans) in enumerate(t1.items()):
174 | txt_ids, tags, context_mask, token_type_ids = get_inputs(
175 | context, q, self.tokenizer, title, self.max_len, ans)
176 | t1_qas.append(
177 | {"txt_ids": txt_ids, "tags": tags, "context_mask": context_mask, "token_type_ids": token_type_ids, 'turn_mask': 0})
178 | for t in t2:
179 | head_entity = t['head_entity']
180 | for q, ans in t['qas'].items():
181 | rel = query2relation(q, question_templates)
182 | idx1, idx2 = rel[0], rel[1:]
183 | idx1, idx2 = idx1s[idx1], idx2s[idx2]
184 | if dist[idx1][idx2] >= self.threshold:
185 | txt_ids, tags, context_mask, token_type_ids = get_inputs(
186 | context, q, self.tokenizer, title, self.max_len, ans, head_entity)
187 | t2_qas.append({"txt_ids": txt_ids, "tags": tags, "context_mask": context_mask,
188 | "token_type_ids": token_type_ids, 'turn_mask': 1})
189 | self.all_t1.extend(t1_qas)
190 | self.all_t2.extend(t2_qas)
191 | self.all_qas = self.all_t1+self.all_t2
192 |
193 | def __len__(self):
194 | return len(self.all_qas)
195 |
196 | def __getitem__(self, i):
197 | return self.all_qas[i]
198 |
199 |
200 | class T1Dataset:
201 | def __init__(self, dataset_tag, test_path, tokenizer, window_size, overlap, max_len=512):
202 | """
203 | Args:
204 | dataset_tag: type of dataset
205 | test_path: path to test set file
206 | tokenizer: tokenizer of pretrained model
207 | window_size: sliding window size
208 | overlap: overlap between two adjacent windows
209 | max_len: max length of input
210 | """
211 | with open(test_path, encoding="utf=8") as f:
212 | data = json.load(f)
213 | self.dataset_tag = dataset_tag
214 | if dataset_tag.lower() == 'ace2004':
215 | dataset_entities = ace2004_entities
216 | question_templates = ace2004_question_templates
217 | elif dataset_tag.lower() == 'ace2005':
218 | dataset_entities = ace2005_entities
219 | question_templates = ace2005_question_templates
220 | else:
221 | raise Exception("this data set is not yet supported")
222 | self.t1_qas = []
223 | self.passages = []
224 | self.entities = []
225 | self.relations = []
226 | self.titles = []
227 | self.window_size = window_size
228 | self.overlap = overlap
229 | self.t1_querys = []
230 | self.t1_ids = []
231 | self.t1_gold = []
232 | self.tokenizer = tokenizer
233 | self.max_len = max_len
234 | # passage_windows[i][j] represents the j-th window of the i-th passage
235 | self.passage_windows = []
236 | self.query_offset1 = []
237 | self.window_offset_base = window_size-overlap
238 | for ent_type in dataset_entities:
239 | query = get_question(question_templates, ent_type)
240 | self.t1_querys.append(query)
241 | for p_id, d in enumerate(tqdm(data, desc="t1_dataset")):
242 | passage = d["passage"]
243 | entities = d['entities']
244 | relations = d['relations']
245 | title = d['title']
246 | self.passages.append(passage)
247 | self.entities.append(entities)
248 | self.relations.append(relations)
249 | self.titles.append(title)
250 | blocks, _ = passage_blocks(passage, window_size, overlap)
251 | self.passage_windows.append(blocks)
252 | for ent in entities:
253 | self.t1_gold.append((p_id, tuple(ent[:-1])))
254 | for b_id, block in enumerate(blocks):
255 | for q_id, q in enumerate(self.t1_querys):
256 | txt_ids, _, context_mask, token_type_ids = get_inputs(
257 | block, q, tokenizer, title, max_len)
258 | self.t1_qas.append({"txt_ids": txt_ids, "context_mask": context_mask,
259 | "token_type_ids": token_type_ids})
260 | self.t1_ids.append((p_id, b_id, dataset_entities[q_id]))
261 | ofs = len(title)+len(tokenizer.tokenize(q))+3
262 | self.query_offset1.append(ofs)
263 |
264 | def __len__(self):
265 | return len(self.t1_qas)
266 |
267 | def __getitem__(self, i):
268 | return self.t1_qas[i]
269 |
270 |
271 | class T2Dataset:
272 | def __init__(self, t1_dataset, t1_predict, threshold=5):
273 | '''
274 | Args:
275 | t1_dataset: an instance of T1Dataset
276 | t1_predict: predictions of the first turn QA
277 | threshold: only consider relationships where the frequency is greater than or equal to the threshold
278 | '''
279 | if t1_dataset.dataset_tag.lower() == "ace2004":
280 | idx1s = ace2004_idx1
281 | idx2s = ace2004_idx2
282 | dist = ace2004_dist
283 | dataset_entities = ace2004_entities
284 | dataset_relations = ace2004_relations
285 | question_templates = ace2004_question_templates
286 | elif t1_dataset.dataset_tag.lower() == 'ace2005':
287 | idx1s = ace2005_idx1
288 | idx2s = ace2005_idx2
289 | dist = ace2005_dist
290 | dataset_entities = ace2005_entities
291 | dataset_relations = ace2005_relations
292 | question_templates = ace2005_question_templates
293 | else:
294 | raise Exception("this data set is not yet supported")
295 | tokenizer = t1_dataset.tokenizer
296 | max_len = t1_dataset.max_len
297 | t1_ids = t1_dataset.t1_ids
298 | passages = t1_dataset.passages
299 | titles = t1_dataset.titles
300 | passage_windows = t1_dataset.passage_windows
301 | self.t2_qas = []
302 | self.t2_ids = []
303 | self.t2_gold = []
304 | self.query_offset2 = []
305 | relations = t1_dataset.relations
306 | entities = t1_dataset.entities
307 | query_offset1 = t1_dataset.query_offset1
308 | window_offset_base = t1_dataset.window_offset_base
309 | for passage_id, (ents, rels) in enumerate(zip(entities, relations)):
310 | for re in rels:
311 | head, rel, end = ents[re[1]], re[0], ents[re[2]]
312 | self.t2_gold.append(
313 | (passage_id, (tuple(head[:-1]), rel, tuple(end[:-1]))))
314 | for i, (_id, pre) in enumerate(zip(tqdm(t1_ids, desc="t2 dataset"), t1_predict)):
315 | passage_id, window_id, head_entity_type = _id
316 | window_offset = window_offset_base*window_id
317 | context = passage_windows[passage_id][window_id]
318 | title = titles[passage_id]
319 | head_entities = []
320 | for start, end in pre:
321 | start1, end1 = start - \
322 | query_offset1[i]+window_offset, end - \
323 | query_offset1[i]+window_offset
324 | ent_str = tokenizer.convert_tokens_to_string(
325 | passages[passage_id][start1:end1])
326 | head_entity = (head_entity_type, start1, end1, ent_str)
327 | head_entities.append(head_entity)
328 | for head_entity in head_entities:
329 | for rel in dataset_relations:
330 | for end_ent_type in dataset_entities:
331 | idx1, idx2 = idx1s[head_entity[0]
332 | ], idx2s[(rel, end_ent_type)]
333 | if dist[idx1][idx2] >= threshold:
334 | query = get_question(
335 | question_templates, head_entity, rel, end_ent_type)
336 | window_head_entity = (
337 | head_entity[0], head_entity[1]-window_offset, head_entity[2]-window_offset, head_entity[3])
338 | txt_ids, _, context_mask, token_type_ids = get_inputs(
339 | context, query, tokenizer, title, max_len, [], window_head_entity)
340 | self.t2_qas.append({"txt_ids": txt_ids, "context_mask": context_mask,
341 | "token_type_ids": token_type_ids})
342 | self.t2_ids.append(
343 | (passage_id, window_id, head_entity[:-1], rel, end_ent_type))
344 | ofs = len(title) + \
345 | len(tokenizer.tokenize(query)) + 3
346 | self.query_offset2.append(ofs)
347 |
348 | def __len__(self):
349 | return len(self.t2_qas)
350 |
351 | def __getitem__(self, i):
352 | return self.t2_qas[i]
353 |
354 |
355 | def load_data(dataset_tag, file_path, batch_size, max_len, pretrained_model_path, dist=False, shuffle=False, threshold=5):
356 | tokenizer = BertTokenizer.from_pretrained(pretrained_model_path)
357 | dataset = MyDataset(dataset_tag, file_path, tokenizer,
358 | max_len, threshold)
359 | sampler = DistributedSampler(dataset) if dist else None
360 | dataloader = DataLoader(dataset, batch_size, sampler=sampler, shuffle=shuffle if not sampler else False,
361 | collate_fn=collate_fn)
362 | return dataloader
363 |
364 |
365 | def reload_data(old_dataloader, batch_size, max_len, threshold, local_rank, shuffle=True):
366 | dataset = old_dataloader.dataset
367 | old_max_len, old_threshold = dataset.max_len, dataset.threshold
368 | if not(old_max_len == max_len and old_threshold == threshold):
369 | dataset.max_len = max_len
370 | dataset.threshold = threshold
371 | dataset.init_data()
372 | sampler = DistributedSampler(
373 | dataset, rank=local_rank) if local_rank != -1 else None
374 | dataloader = DataLoader(
375 | dataset, batch_size, sampler=sampler, shuffle=shuffle, collate_fn=collate_fn)
376 | return dataloader
377 |
378 |
379 | def load_t1_data(dataset_tag, test_path, pretrained_model_path, window_size, overlap, batch_size=10, max_len=512):
380 | tokenizer = BertTokenizer.from_pretrained(pretrained_model_path)
381 | t1_dataset = T1Dataset(dataset_tag, test_path,
382 | tokenizer, window_size, overlap, max_len)
383 | dataloader = DataLoader(t1_dataset, batch_size, collate_fn=collate_fn1)
384 | return dataloader
385 |
386 |
387 | def load_t2_data(t1_dataset, t1_predict, batch_size=10, threshold=5):
388 | t2_dataset = T2Dataset(t1_dataset, t1_predict, threshold)
389 | dataloader = DataLoader(t2_dataset, batch_size, collate_fn=collate_fn1)
390 | return dataloader
391 |
--------------------------------------------------------------------------------
/evaluation.py:
--------------------------------------------------------------------------------
1 | import torch
2 | from tqdm import tqdm
3 |
4 | from dataloader import tag_idxs, load_t2_data
5 |
6 |
7 | def get_score(gold_set, predict_set):
8 | TP = len(set.intersection(gold_set, predict_set))
9 | print("#TP:", TP, "#Gold:", len(gold_set), "#Predict:", len(predict_set))
10 | precision = TP/(len(predict_set)+1e-6)
11 | recall = TP/(len(gold_set)+1e-6)
12 | f1 = 2*precision*recall/(precision+recall+1e-6)
13 | return precision, recall, f1
14 |
15 |
16 | def test_evaluation(model, t1_dataloader, threshold, amp=False):
17 | if hasattr(model, 'module'):
18 | model = model.module
19 | model.eval()
20 | t1_predict = []
21 | t2_predict = []
22 | device = torch.device(
23 | "cuda") if torch.cuda.is_available() else torch.device("cpu")
24 | # turn 1
25 | with (torch.no_grad() if not amp else torch.cuda.amp.autocast()):
26 | for i, batch in enumerate(tqdm(t1_dataloader, desc="t1 predict")):
27 | txt_ids, attention_mask, token_type_ids, context_mask = batch['txt_ids'], batch[
28 | 'attention_mask'], batch['token_type_ids'], batch['context_mask']
29 | tag_idxs = model(txt_ids.to(device), attention_mask.to(
30 | device), token_type_ids.to(device))
31 | predict_spans = tag_decode(tag_idxs, context_mask)
32 | t1_predict.extend(predict_spans)
33 | # turn 2
34 | t2_dataloader = load_t2_data(
35 | t1_dataloader.dataset, t1_predict, 10, threshold)
36 | with (torch.no_grad() if not amp else torch.cuda.amp.autocast()):
37 | for i, batch in enumerate(tqdm(t2_dataloader, desc="t2 predict")):
38 | txt_ids, attention_mask, token_type_ids, context_mask = batch['txt_ids'], batch[
39 | 'attention_mask'], batch['token_type_ids'], batch['context_mask']
40 | tag_idxs = model(txt_ids.to(device), attention_mask.to(
41 | device), token_type_ids.to(device))
42 | predict_spans = tag_decode(tag_idxs, context_mask)
43 | t2_predict.extend(predict_spans)
44 | # get basic information
45 | t1_ids = t1_dataloader.dataset.t1_ids
46 | t2_ids = t2_dataloader.dataset.t2_ids
47 | window_offset_base = t1_dataloader.dataset.window_offset_base
48 | query_offset1 = t1_dataloader.dataset.query_offset1
49 | query_offset2 = t2_dataloader.dataset.query_offset2
50 | t1_gold = t1_dataloader.dataset.t1_gold
51 | t2_gold = t2_dataloader.dataset.t2_gold
52 | p1, r1, f1 = eval_t1(t1_predict, t1_gold, t1_ids,
53 | query_offset1, window_offset_base)
54 | p2, r2, f2 = eval_t2(t2_predict, t2_gold, t2_ids,
55 | query_offset2, window_offset_base)
56 | return (p1, r1, f1), (p2, r2, f2)
57 |
58 |
59 | def eval_t1(predict, gold, ids, query_offset, window_offset_base):
60 | """
61 | Args:
62 | predict: [(s1,e1),(s2,e2),(s3,e3),...]
63 | gold: (passage_id,(entity_type,start_idx,end_idx,entity_str))
64 | ids: (passage_id, window_id,entity_type)
65 | query_offset: offset of [CLS]+title+[SEP]+query+[SEP]
66 | window_offset_base: value of window_size-overlap的值
67 | """
68 | predict1 = []
69 | for i, (_id, pre) in enumerate(zip(ids, predict)):
70 | passage_id, window_id, entity_type = _id
71 | window_offset = window_offset_base*window_id
72 | for start, end in pre:
73 | start1, end1 = start - query_offset[i]+window_offset, \
74 | end - query_offset[i]+window_offset
75 | new = (passage_id, (entity_type, start1, end1))
76 | predict1.append(new)
77 | return get_score(set(gold), set(predict1))
78 |
79 |
80 | def eval_t2(predict, gold, ids, query_offset, window_offset_base):
81 | """
82 | Args:
83 | predict: [(s1,e1),(s2,e2),(s3,e3),...]
84 | gold: (passage_id,(head_entity,relation_type,end_entity))
85 | ids: (passage_id,window_id,head_entity,relation_type,end_entity_type)
86 | query_offset: offset of [CLS]+title+[SEP]+query+[SEP]
87 | window_offset_base: value of window_size-overlap
88 | """
89 | predict1 = []
90 | for i, (_id, pre) in enumerate(zip(ids, predict)):
91 | passage_id, window_id, head_entity, relation_type, end_entity_type = _id
92 | window_offset = window_offset_base*window_id
93 | head_start = head_entity[1]
94 | for start, end in pre:
95 | #since we added two special tokens around the head entity for identification, there is a correction of 1.
96 | if head_start+query_offset[i]-window_offset+1 < start:
97 | start1, end1 = start - \
98 | query_offset[i]+window_offset-2, end - \
99 | query_offset[i]+window_offset-2
100 | else:
101 | start1, end1 = start - \
102 | query_offset[i]+window_offset, end - \
103 | query_offset[i]+window_offset
104 | new = (passage_id, (head_entity, relation_type,
105 | (end_entity_type, start1, end1)))
106 | predict1.append(new)
107 | return get_score(set(gold), set(predict1))
108 |
109 |
110 | def tag_decode(tags, context_mask=None):
111 | spans = [[]]*tags.shape[0]
112 | tags = tags.tolist()
113 | if not context_mask is None:
114 | context_mask = context_mask.tolist()
115 | has_answer = []
116 | start_idxs = []
117 | end_idxs = []
118 | for i, t in enumerate(tags):
119 | if t[0] != tag_idxs['S']:
120 | has_answer.append(i)
121 | if context_mask is None:
122 | mask = [1 if i != -1 else 0 for i in t]
123 | else:
124 | mask = context_mask[i]
125 | s = mask.index(1, 1)
126 | e = mask.index(0, s)
127 | start_idxs.append(s)
128 | end_idxs.append(e)
129 | for i, s, e in zip(has_answer, start_idxs, end_idxs):
130 | span = []
131 | j = s
132 | while j < e:
133 | if tags[i][j] == tag_idxs['S']:
134 | span.append([j, j+1])
135 | j += 1
136 | elif tags[i][j] == tag_idxs['B'] and j < e-1:
137 | for k in range(j+1, e):
138 | if tags[i][k] in [tag_idxs['B'], tag_idxs['S']]:
139 | j = k
140 | break
141 | elif tags[i][k] == tag_idxs["E"]:
142 | span.append([j, k+1])
143 | j = k+1
144 | break
145 | elif k == e-1:
146 | j = k+1
147 | else:
148 | j += 1
149 | spans[i] = span
150 | return spans
151 |
--------------------------------------------------------------------------------
/model.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | from transformers import BertModel
4 |
5 |
6 | class MyModel(nn.Module):
7 | def __init__(self, config):
8 | super(MyModel, self).__init__()
9 | self.config = config
10 | self.bert = BertModel.from_pretrained(config.pretrained_model_path)
11 | self.tag_linear = nn.Linear(self.bert.config.hidden_size, 5)
12 | self.dropout = nn.Dropout(config.dropout_prob)
13 | self.loss_func = nn.CrossEntropyLoss()
14 | self.theta = config.theta
15 |
16 | def forward(self, input, attention_mask, token_type_ids, context_mask=None, turn_mask=None, target_tags=None):
17 | """
18 | Args:
19 | input: （batch,seq_len）
20 | attention_mask: (batch,seq_len)
21 | token_type_ids: (batch,seq_len)
22 | context_mask: (batch,seq_len), used to identify labeled tokens
23 | target_tags: (batch,seq_len)
24 | turn_mask: (batch,) turn_mask[i]=0 for turn 1，turn_mask[i]=1 for turn 2
25 | """
26 | rep, _ = self.bert(input, attention_mask, token_type_ids)
27 | rep = self.dropout(rep)
28 | tag_logits = self.tag_linear(rep) # (batch,seq_len,num_tag)
29 | if not target_tags is None:
30 | tag_logits_t1 = tag_logits[turn_mask == 0] # (n1,seq_len,num_tag)
31 | target_tags_t1 = target_tags[turn_mask == 0] # (n1,seq_len)
32 | context_mask_t1 = context_mask[turn_mask == 0] # (n1,seq_len)
33 |
34 | tag_logits_t2 = tag_logits[turn_mask == 1] # (n2,seq_len,num_tag)
35 | target_tags_t2 = target_tags[turn_mask == 1] # (n2,seq_len)
36 | context_mask_t2 = context_mask[turn_mask == 1] # (n2,seq_len)
37 |
38 | tag_logits_t1 = tag_logits_t1[context_mask_t1 == 1] # (N1,num_tag)
39 | target_tags_t1 = target_tags_t1[context_mask_t1 == 1] # (N1)
40 |
41 | tag_logits_t2 = tag_logits_t2[context_mask_t2 == 1] # (N2,num_tag)
42 | target_tags_t2 = target_tags_t2[context_mask_t2 == 1] # (N2)
43 |
44 | loss_t1 = self.loss_func(tag_logits_t1, target_tags_t1) if len(
45 | target_tags_t1) != 0 else torch.tensor(0).type_as(input)
46 | loss_t2 = self.loss_func(tag_logits_t2, target_tags_t2) if len(
47 | target_tags_t2) != 0 else torch.tensor(0).type_as(input)
48 | loss = self.theta*loss_t1+(1-self.theta)*loss_t2
49 | return loss, (loss_t1.item(), loss_t2.item())
50 | else:
51 | # for prediction
52 | tag_idxs = torch.argmax(
53 | tag_logits, dim=-1).squeeze(-1) # (batch,seq_len)
54 | return tag_idxs
55 |
--------------------------------------------------------------------------------
/preprocess.py:
--------------------------------------------------------------------------------
1 | import os
2 | import json
3 | import re
4 | import argparse
5 |
6 | from tqdm import tqdm
7 | from constants import *
8 |
9 |
10 | def aligment_ann(original, newtext, ann_file, offset):
11 | # Ensure that uncased tokenizers can also be aligned
12 | original = original.lower()
13 | newtext = newtext.lower()
14 | annotation = []
15 | terms = {}
16 | ends = {}
17 | for line in open(ann_file):
18 | if line.startswith('T'):
19 | annots = line.rstrip().split("\t", 2)
20 | typeregion = annots[1].split(" ")
21 | start = int(typeregion[1]) - offset
22 | end = int(typeregion[2]) - offset
23 | if not start in terms:
24 | terms[start] = []
25 | if not end in ends:
26 | ends[end] = []
27 | if len(annots) == 3:
28 | terms[start].append(
29 | [start, end, annots[0], typeregion[0], annots[2]])
30 | else:
31 | terms[start].append([start, end, annots[0], typeregion[0], ""])
32 | ends[end].append(start)
33 | else:
34 | annotation.append(line)
35 | orgidx = 0
36 | newidx = 0
37 | orglen = len(original)
38 | newlen = len(newtext)
39 |
40 | while orgidx < orglen and newidx < newlen:
41 | if original[orgidx] == newtext[newidx]:
42 | orgidx += 1
43 | newidx += 1
44 | elif newtext[newidx] == '\n':
45 | newidx += 1
46 | elif original[orgidx] == '\n':
47 | orgidx += 1
48 | elif newtext[newidx] == ' ':
49 | newidx += 1
50 | elif original[orgidx] == ' ':
51 | orgidx += 1
52 | elif newtext[newidx] == '\t':
53 | newidx += 1
54 | elif original[orgidx] == '\t':
55 | orgidx += 1
56 | elif newtext[newidx] == '.':
57 | # ignore extra "." for stanford
58 | newidx += 1
59 | else:
60 | assert False, "%d\t$%s$\t$%s$" % (
61 | orgidx, original[orgidx:orgidx + 20], newtext[newidx:newidx + 20])
62 | if orgidx in terms:
63 | for l in terms[orgidx]:
64 | l[0] = newidx
65 | if orgidx in ends:
66 | for start in ends[orgidx]:
67 | for l in terms[start]:
68 | if l[1] == orgidx:
69 | l[1] = newidx
70 | del ends[orgidx]
71 | entities = []
72 | relations = []
73 | dict1 = {}
74 | i = 0
75 | for ts in terms.values():
76 | for term in ts:
77 | if term[4] == "":
78 | entities.append([term[2], term[3], term[0],
79 | term[1], newtext[term[0]:term[1]]])
80 | else:
81 | # assert newtext[term[0]:term[1]].replace("&", "&").replace("&", "&").replace(" ", "").replace(
82 | # "\n", "") == term[4].replace(" ", "").lower(), newtext[term[0]:term[1]] + "<=>" + term[4]
83 | assert newtext[term[0]:term[1]].replace(" ", "").replace('\n', "").replace("&", "&").replace("&", "&") == \
84 | term[4].replace(" ", "").lower(
85 | ), newtext[term[0]:term[1]] + "<=>" + term[4]
86 | entities.append([term[2], term[3], term[0], term[1],
87 | newtext[term[0]:term[1]].replace("\n", " ")])
88 | dict1[term[2]] = i
89 | i += 1
90 | for rel in annotation:
91 | rel_id, rel_type, rel_e1, rel_e2 = rel.strip().split()
92 | rel_e1 = rel_e1[5:]
93 | rel_e2 = rel_e2[5:]
94 | relations.append([rel_id, rel_type, rel_e1, rel_e2])
95 | relations1 = []
96 | for rel in relations:
97 | _, rel_type, rel_e1, rel_e2 = rel
98 | rel_e1_idx = dict1[rel_e1]
99 | rel_e2_idx = dict1[rel_e2]
100 | relations1.append([rel_type, rel_e1_idx, rel_e2_idx])
101 | entities1 = [[ent[1], ent[2], ent[3], ent[4]] for ent in entities]
102 | return entities1, relations1
103 |
104 |
105 | def passage_blocks(txt, window_size, overlap):
106 | blocks = []
107 | regions = []
108 | for i in range(0, len(txt), window_size-overlap):
109 | b = txt[i:i+window_size]
110 | blocks.append(b)
111 | regions.append((i, i+window_size))
112 | return blocks, regions
113 |
114 |
115 | def get_block_er(txt, entities, relations, window_size, overlap, tokenizer):
116 | """
117 | Get the block level annotation
118 | Args:
119 | txt: text to be processed, list of token
120 | entities: list of (entity_type, start, end,entity)
121 | relations: list of (relation_type,entity1_idx,entity2_idx)
122 | window_siez: sliding window size
123 | overlap: overlap between two adjacent windows
124 | Returns:
125 | ber: list of [block，entities, relations]
126 | """
127 | blocks, block_range = passage_blocks(txt, window_size, overlap)
128 | ber = [[[], [], []] for i in range(len(block_range))]
129 | e_dict = {}
130 | for i, (s, e) in enumerate(block_range):
131 | es = []
132 | for j, (entity_type, start, end, entity_str) in enumerate(entities):
133 | if start >= s and end <= e:
134 | nstart, nend = start-s, end-s
135 | if tokenizer.convert_tokens_to_string(blocks[i][nstart:nend]) == entity_str:
136 | es.append((entity_type, nstart, nend, entity_str))
137 | e_dict[j] = e_dict.get(j, [])+[i]
138 | else:
139 | print(
140 | "The entity string and its corresponding index are inconsistent")
141 | ber[i][0] = blocks[i]
142 | ber[i][1].extend(es)
143 | for r, e1i, e2i in relations:
144 | if e1i not in e_dict or e2i not in e_dict:
145 | print("Entity lost due to sliding window")
146 | continue
147 | i1s, i2s = e_dict[e1i], e_dict[e2i]
148 | intersec = set.intersection(set(i1s), set(i2s))
149 | if intersec:
150 | for i in intersec:
151 | t1, s1, e1, es1 = entities[e1i][0], entities[e1i][1] - \
152 | block_range[i][0], entities[e1i][2] - \
153 | block_range[i][0], entities[e1i][3]
154 | t2, s2, e2, es2 = entities[e2i][0], entities[e2i][1] - \
155 | block_range[i][0], entities[e2i][2] - \
156 | block_range[i][0], entities[e2i][3]
157 | ber[i][2].append((r, (t1, s1, e1, es1), (t2, s2, e2, es2)))
158 | else:
159 | print("The two entities of the relationship are not on the same sentence")
160 | return ber
161 |
162 |
163 | def get_question(question_templates, head_entity, relation_type=None, end_entity_type=None):
164 | """
165 | Args:
166 | head_entity: (entity_type,start_idx,end_idx,entity_string) or entity_type
167 | """
168 | if relation_type == None:
169 | question = question_templates['qa_turn1'][head_entity[0]] if isinstance(
170 | head_entity, tuple) else question_templates['qa_turn1'][head_entity]
171 | else:
172 | question = question_templates['qa_turn2'][str(
173 | (head_entity[0], relation_type, end_entity_type))]
174 | question = question.replace('XXX', head_entity[3])
175 | return question
176 |
177 |
178 | def block2qas(ber, dataset_tag, title="", threshold=1, max_distance=45):
179 | """
180 | Args:
181 | ber: (block,entities,relations)
182 | dataset_tag: type of dataset
183 | title: title corresponding to the passage to which the block belongs
184 | threshold: only consider relationships where the frequency is greater than or equal to the threshold
185 | max_distance: used to filter relations by distance from the head entity
186 | """
187 | if dataset_tag.lower() == "ace2004":
188 | entities = ace2004_entities
189 | relations = ace2004_relations
190 | idx1s = ace2004_idx1
191 | idx2s = ace2004_idx2
192 | dist = ace2004_dist
193 | question_templates = ace2004_question_templates
194 | elif dataset_tag.lower() == 'ace2005':
195 | entities = ace2005_entities
196 | relations = ace2005_relations
197 | idx1s = ace2005_idx1
198 | idx2s = ace2005_idx2
199 | dist = ace2005_dist
200 | question_templates = ace2005_question_templates
201 | else:
202 | raise Exception("this data set is not yet supported")
203 | block, ents, relas = ber
204 | res = {'context': block, 'title': title}
205 | # QA turn 1
206 | dict1 = {k: get_question(question_templates, k) for k in entities}
207 | qat1 = {dict1[k]: [] for k in dict1}
208 | for en in ents:
209 | q = dict1[en[0]]
210 | qat1[q].append(en)
211 | # QA turn 2
212 | if max_distance > 0:
213 | dict2 = {(rel[1], rel[0], rel[2][0]): [] for rel in relas}
214 | for rel in relas:
215 | dict2[(rel[1], rel[0], rel[2][0])].append(rel[2])
216 | qat2 = []
217 | ents1 = sorted(ents, key=lambda x: x[1])
218 | for i, ent1 in enumerate(ents1):
219 | start = ent1[1]
220 | qas = {}
221 | for j, ent2 in enumerate(ents1[i+1:], i+1):
222 | if ent2[1] > start+max_distance:
223 | break
224 | else:
225 | head_type, end_type = ent1[0], ent2[0]
226 | for rel_type in relations:
227 | idx1, idx2 = idx1s[head_type], idx2s[(
228 | rel_type, end_type)]
229 | if dist[idx1][idx2] >= threshold:
230 | k = (ent1, rel_type, end_type)
231 | q = get_question(
232 | question_templates, ent1, rel_type, end_type)
233 | qas[q] = dict2.get(k, [])
234 | qat2.append({"head_entity": ent1, "qas": qas})
235 | else:
236 | dict2 = {(rel[1], rel[0], rel[2][0]): [] for rel in relas}
237 | for rel in relas:
238 | dict2[(rel[1], rel[0], rel[2][0])].append(rel[2])
239 | qat2 = []
240 | for ent in ents:
241 | qas = {}
242 | for rel_type in relations:
243 | for ent_type in entities:
244 | k = (ent, rel_type, ent_type)
245 | idx1, idx2 = idx1s[ent[0]], idx2s[(rel_type, ent_type)]
246 | if dist[idx1][idx2] >= threshold:
247 | q = get_question(question_templates,
248 | ent, rel_type, ent_type)
249 | qas[q] = dict2.get(k, [])
250 | qat2.append({'head_entity': ent, "qas": qas})
251 | qas = [qat1, qat2]
252 | res["qa_pairs"] = qas
253 | return res
254 |
255 |
256 | def char_to_wordpiece(passage, entities, tokenizer):
257 | entities1 = []
258 | tpassage = tokenizer.tokenize(passage)
259 | for ent in entities:
260 | ent_type, start, end, ent_str = ent
261 | s = tokenizer.tokenize(passage[:start])
262 | start1 = len(s)
263 | ent_str1 = tokenizer.tokenize(ent_str)
264 | end1 = start1 + len(ent_str1)
265 | ent_str2 = tokenizer.convert_tokens_to_string(ent_str1)
266 | assert tpassage[start1:end1] == ent_str1
267 | entities1.append((ent_type, start1, end1, ent_str2))
268 | return entities1
269 |
270 |
271 | def process(data_dir, output_dir, tokenizer, is_test, window_size, overlap, dataset_tag, threshold=1, max_distance=45):
272 | """
273 | Args:
274 | data_dir: data directory
275 | output_dir: output directory
276 | tokenizer: tokenizer of pretrained model
277 | is_test: whether it is test data
278 | window_size: sliding window size
279 | overlap: overlap between two adjacent windows
280 | dataset_tag: type of dataset
281 | threshold： only consider relationships where the frequency is greater than or equal to the threshold
282 | max_distance: used to filter relations by distance from the head entity
283 | """
284 | ann_files = []
285 | txt_files = []
286 | data = []
287 | for f in os.listdir(data_dir):
288 | if f.endswith('.txt'):
289 | txt_files.append(os.path.join(data_dir, f))
290 | elif f.endswith('.ann'):
291 | ann_files.append(os.path.join(data_dir, f))
292 | ann_files = sorted(ann_files)
293 | txt_files = sorted(txt_files)
294 | for ann_path, txt_path in tqdm(zip(ann_files, txt_files), total=len(ann_files)):
295 | with open(txt_path, encoding='utf-8') as f:
296 | raw_txt = f.read()
297 | txt = [t for t in raw_txt.split('\n') if t.strip()]
298 | # get the title information, the title will be added to all windows of a passage
299 | title = re.search('[A-Za-z_]+[A-Za-z]', txt[0]
300 | ).group().split('-')+txt[1].strip().split()
301 | title = " ".join(title)
302 | title = tokenizer.tokenize(title)
303 | ntxt = ' '.join(txt[3:])
304 | ntxt1 = tokenizer.tokenize(ntxt)
305 | ntxt2 = tokenizer.convert_tokens_to_string(ntxt1)
306 | offset = raw_txt.index(txt[3])
307 | entities, relations = aligment_ann(
308 | raw_txt[offset:], ntxt2, ann_path, offset)
309 | # convert entitiy index from char level to wordpiece level
310 | entities = char_to_wordpiece(ntxt2, entities, tokenizer)
311 | if is_test:
312 | data.append({"title": title, "passage": ntxt1,
313 | "entities": entities, "relations": relations})
314 | else:
315 | block_er = get_block_er(
316 | ntxt1, entities, relations, window_size, overlap, tokenizer)
317 | for ber in block_er:
318 | data.append(block2qas(ber, dataset_tag,
319 | title, threshold, max_distance))
320 | if not os.path.exists(output_dir):
321 | os.makedirs(output_dir)
322 | save_path = os.path.join(output_dir, os.path.split(data_dir)[-1]+".json")
323 | with open(save_path, 'w', encoding='utf-8') as f:
324 | json.dump(data, f, sort_keys=True, indent=4)
325 |
326 |
327 | if __name__ == "__main__":
328 | parser = argparse.ArgumentParser()
329 | parser.add_argument("--data_dir", default='./data/raw_data/ACE2005/train')
330 | parser.add_argument(
331 | "--dataset_tag", choices=["ace2004", 'ace2005'], default='ace2005')
332 | parser.add_argument("--window_size", type=int, default=300)
333 | parser.add_argument("--overlap", type=int, default=15)
334 | parser.add_argument("--is_test", action="store_true")
335 | parser.add_argument("--threshold", type=int, default=5)
336 | parser.add_argument("--output_base_dir",
337 | default="./data/cleaned_data/ACE2005")
338 | parser.add_argument("--pretrained_model_path",
339 | default='/home/wangnan/pretrained_models/bert-base-uncased')
340 | parser.add_argument("--max_distance", type=int, default=45,
341 | help="used to filter relations by distance from the head entity")
342 | args = parser.parse_args()
343 | if not args.is_test:
344 | output_dir = "{}/{}_overlap_{}_window_{}_threshold_{}_max_distance_{}".format(args.output_base_dir, os.path.split(
345 | args.pretrained_model_path)[-1], args.overlap, args.window_size, args.threshold, args.max_distance)
346 | else:
347 | output_dir = args.output_base_dir
348 | from transformers import BertTokenizer
349 | tokenizer = BertTokenizer.from_pretrained(args.pretrained_model_path)
350 | process(args.data_dir, output_dir, tokenizer, args.is_test,
351 | args.window_size, args.overlap, args.dataset_tag, args.threshold, args.max_distance)
352 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | tqdm
2 | transformers==3.2.0
3 | torch==1.6.0
--------------------------------------------------------------------------------
/scripts/ckpt_eval.sh:
--------------------------------------------------------------------------------
1 | HOME=/home/wangnan
2 | REPO=$HOME/ERE-MQA
3 |
4 |
5 | python $REPO/ckpt_eval.py \
6 | --window_size 300 \
7 | --overlap 45 \
8 | --checkpoint_path $REPO/checkpoints/ace2004/2020_10_10_00_09_21/checkpoint_0.cpt \
9 | --test_path $REPO/data/cleaned_data/ACE2004/test0.json \
10 | --test_batch 20 \
11 | --threshold 3
--------------------------------------------------------------------------------
/scripts/prepare_ace2004.sh:
--------------------------------------------------------------------------------
1 | HOME=/home/wangnan
2 | REPO=$HOME/ERE-MQA
3 | PRETRAINED_MODEL_PATH=$HOME/pretrained_models/bert-base-uncased
4 |
5 | #training data
6 | python $REPO/preprocess.py \
7 | --data_dir $REPO/data/raw_data/ACE2004/train0 \
8 | --dataset_tag ace2004 \
9 | --window_size 300 \
10 | --overlap 15 \
11 | --threshold 1 \
12 | --max_distance 45 \
13 | --output_base_dir $REPO/data/cleaned_data/ACE2004 \
14 | --pretrained_model_path $PRETRAINED_MODEL_PATH
15 |
16 | #test data
17 | python $REPO/preprocess.py \
18 | --data_dir $REPO/data/raw_data/ACE2004/test0 \
19 | --dataset_tag ace2004 \
20 | --output_base_dir $REPO/data/cleaned_data/ACE2004 \
21 | --pretrained_model_path $PRETRAINED_MODEL_PATH \
22 | --is_test
--------------------------------------------------------------------------------
/scripts/prepare_ace2005.sh:
--------------------------------------------------------------------------------
1 | HOME=/home/wangnan
2 | REPO=$HOME/ERE-MQA
3 | PRETRAINED_MODEL_PATH=$HOME/pretrained_models/bert-base-uncased
4 |
5 |
6 | #training data
7 | python $REPO/preprocess.py \
8 | --data_dir $REPO/data/raw_data/ACE2005/train \
9 | --dataset_tag ace2005 \
10 | --window_size 300 \
11 | --overlap 15 \
12 | --threshold 1 \
13 | --max_distance 45 \
14 | --output_base_dir $REPO/data/cleaned_data/ACE2005 \
15 | --pretrained_model_path $PRETRAINED_MODEL_PATH
16 |
17 | #development data
18 | python $REPO/preprocess.py \
19 | --data_dir $REPO/data/raw_data/ACE2005/dev \
20 | --dataset_tag ace2005 \
21 | --output_base_dir $REPO/data/cleaned_data/ACE2005 \
22 | --pretrained_model_path $PRETRAINED_MODEL_PATH \
23 | --is_test
24 |
25 | #test data
26 | python $REPO/preprocess.py \
27 | --data_dir $REPO/data/raw_data/ACE2005/test \
28 | --dataset_tag ace2005 \
29 | --output_base_dir $REPO/data/cleaned_data/ACE2005 \
30 | --pretrained_model_path $PRETRAINED_MODEL_PATH \
31 | --is_test
--------------------------------------------------------------------------------
/scripts/train_ace2004.sh:
--------------------------------------------------------------------------------
1 | HOME=/home/wangnan
2 | REPO=$HOME/ERE-MQA
3 | PRETRAINED_MODEL=/home/wangnan/pretrained_models/bert-base-uncased
4 |
5 |
6 | python $REPO/train.py \
7 | --dataset_tag ace2004 \
8 | --train_path $REPO/data/cleaned_data/ACE2004/bert-base-uncased_overlap_15_window_300_threshold_1_max_distance_45/train0.json \
9 | --train_batch 20 \
10 | --test_path $REPO/data/cleaned_data/ACE2004/test0.json \
11 | --test_batch 20 \
12 | --pretrained_model_path $PRETRAINED_MODEL \
13 | --max_epochs 10 \
14 | --warmup_ratio 0.1 \
15 | --lr 2e-5 \
16 | --theta 0.25 \
17 | --window_size 300 \
18 | --overlap 45 \
19 | --threshold 3 \
20 | --max_grad_norm 1 \
21 | --test_eval \
22 | --seed 0 \
23 | --amp
--------------------------------------------------------------------------------
/scripts/train_ace2005.sh:
--------------------------------------------------------------------------------
1 | HOME=/home/wangnan
2 | REPO=$HOME/ERE-MQA
3 | PRETRAINED_MODEL=/home/wangnan/pretrained_models/bert-base-uncased
4 |
5 |
6 | python $REPO/train.py \
7 | --dataset_tag ace2005 \
8 | --train_path $REPO/data/cleaned_data/ACE2005/bert-base-uncased_overlap_15_window_300_threshold_1_max_distance_45/train.json \
9 | --train_batch 20 \
10 | --test_path $REPO/data/cleaned_data/ACE2005/test.json \
11 | --test_batch 20 \
12 | --pretrained_model_path $PRETRAINED_MODEL \
13 | --max_epochs 10 \
14 | --warmup_ratio 0.1 \
15 | --lr 2e-5 \
16 | --theta 0.25 \
17 | --window_size 300 \
18 | --overlap 45 \
19 | --threshold 5 \
20 | --max_grad_norm 1 \
21 | --test_eval \
22 | --seed 0 \
23 | --amp
--------------------------------------------------------------------------------
/train.py:
--------------------------------------------------------------------------------
1 | import os
2 | import time
3 | import random
4 | import argparse
5 |
6 | import pickle
7 | from transformers.optimization import get_linear_schedule_with_warmup
8 | from torch.utils.tensorboard import SummaryWriter
9 | from torch.cuda.amp import autocast, GradScaler
10 | from tqdm import trange, tqdm
11 | import numpy as np
12 | import torch
13 | from torch.nn.utils import clip_grad_norm_
14 |
15 | from model import MyModel
16 | from evaluation import test_evaluation
17 | from dataloader import load_data, load_t1_data, reload_data
18 |
19 |
20 | def set_seed(seed):
21 | random.seed(seed)
22 | np.random.seed(seed)
23 | torch.manual_seed(seed)
24 | torch.cuda.manual_seed(seed)
25 |
26 |
27 | def args_parser():
28 | parser = argparse.ArgumentParser()
29 | parser.add_argument("--dataset_tag", default='ace2005',
30 | choices=['ace2005', 'ace2004'])
31 | parser.add_argument("--train_path")
32 | parser.add_argument("--train_batch", type=int, default=10)
33 | parser.add_argument("--test_path")
34 | parser.add_argument("--test_batch", type=int, default=10)
35 | parser.add_argument("--max_len", default=512, type=int,
36 | help="maximum length of input")
37 | parser.add_argument("--pretrained_model_path")
38 | parser.add_argument("--max_epochs", default=10, type=int)
39 | parser.add_argument("--warmup_ratio", type=float, default=-1)
40 | parser.add_argument("--lr", type=float, default=2e-5)
41 | parser.add_argument("--dropout_prob", type=float, default=0.1)
42 | parser.add_argument("--weight_decay", type=float, default=0.01)
43 | parser.add_argument("--theta", type=float,
44 | help="weight of two tasks", default=0.25)
45 | parser.add_argument("--window_size", type=int,
46 | default=100, help="size of the sliding window")
47 | parser.add_argument("--overlap", type=int, default=50,
48 | help="overlap size of the two sliding windows")
49 | parser.add_argument("--threshold", type=int, default=5,
50 | help="At least the number of times a possible relationship should appear in the training set (should be greater than or equal to the threshold in the data preprocessing stage)")
51 | parser.add_argument("--local_rank", type=int, default=-
52 | 1, help="用于DistributedDataParallel")
53 | parser.add_argument("--max_grad_norm", type=float, default=1)
54 | parser.add_argument("--seed", type=int, default=0)
55 | parser.add_argument("--amp", action="store_true",
56 | help="whether to enable mixed precision")
57 | parser.add_argument("--not_save", action="store_true",
58 | help="whether to save the model")
59 | parser.add_argument("--reload", action="store_true",
60 | help="whether to reload data")
61 | parser.add_argument("--test_eval", action="store_true")
62 | args = parser.parse_args()
63 | return args
64 |
65 |
66 | def train(args, train_dataloader):
67 | model = MyModel(args)
68 | model.train()
69 | if args.amp:
70 | scaler = GradScaler()
71 | device = args.local_rank if args.local_rank != -1 \
72 | else (torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu'))
73 | if args.local_rank != -1:
74 | torch.cuda.set_device(args.local_rank)
75 | model.to(device)
76 | if args.local_rank != -1:
77 | model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[
78 | args.local_rank], output_device=args.local_rank, find_unused_parameters=True)
79 | no_decay = ['bias', 'LayerNorm.weight']
80 | optimizer_grouped_parameters = [
81 | {"params": [p for n, p in model.named_parameters() if not any(
82 | nd in n for nd in no_decay)], "weight_decay":args.weight_decay},
83 | {"params": [p for n, p in model.named_parameters() if any(
84 | nd in n for nd in no_decay)], "weight_decay":0.0}
85 | ]
86 | optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=args.lr)
87 | if args.warmup_ratio > 0:
88 | num_training_steps = len(train_dataloader)*args.max_epochs
89 | warmup_steps = args.warmup_ratio*num_training_steps
90 | scheduler = get_linear_schedule_with_warmup(
91 | optimizer, warmup_steps, num_training_steps)
92 | if args.local_rank < 1:
93 | mid = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime(time.time()))
94 | for epoch in range(args.max_epochs):
95 | if args.local_rank != -1:
96 | train_dataloader.sampler.set_epoch(epoch)
97 | tqdm_train_dataloader = tqdm(
98 | train_dataloader, desc="epoch:%d" % epoch, ncols=150)
99 | for i, batch in enumerate(tqdm_train_dataloader):
100 | torch.cuda.empty_cache()
101 | optimizer.zero_grad()
102 | txt_ids, attention_mask, token_type_ids, context_mask, turn_mask, tags = batch['txt_ids'], batch['attention_mask'], batch['token_type_ids'],\
103 | batch['context_mask'], batch['turn_mask'], batch['tags']
104 | txt_ids, attention_mask, token_type_ids, context_mask, turn_mask, tags = txt_ids.to(device), attention_mask.to(device), token_type_ids.to(device),\
105 | context_mask.to(device), turn_mask.to(device), tags.to(device)
106 | if args.amp:
107 | with autocast():
108 | loss, (loss_t1, loss_t2) = model(
109 | txt_ids, attention_mask, token_type_ids, context_mask, turn_mask, tags)
110 | scaler.scale(loss).backward()
111 | if args.max_grad_norm > 0:
112 | scaler.unscale_(optimizer)
113 | torch.nn.utils.clip_grad_norm_(
114 | model.parameters(), args.max_grad_norm)
115 | scaler.step(optimizer)
116 | scaler.update()
117 | else:
118 | loss, (loss_t1, loss_t2) = model(txt_ids, attention_mask,
119 | token_type_ids, context_mask, turn_mask, tags)
120 | loss.backward()
121 | if args.max_grad_norm > 0:
122 | clip_grad_norm_(model.parameters(), args.max_grad_norm)
123 | optimizer.step()
124 | lr = optimizer.param_groups[0]['lr']
125 | named_parameters = [
126 | (n, p) for n, p in model.named_parameters() if not p.grad is None]
127 | grad_norm = torch.norm(torch.stack(
128 | [torch.norm(p.grad) for n, p in named_parameters])).item()
129 | if args.warmup_ratio > 0:
130 | scheduler.step()
131 | postfix_str = "norm:{:.2f},lr:{:.1e},loss:{:.2e},t1:{:.2e},t2:{:.2e}".format(
132 | grad_norm, lr, loss.item(), loss_t1, loss_t2)
133 | tqdm_train_dataloader.set_postfix_str(postfix_str)
134 | if args.local_rank in [-1, 0] and not args.not_save:
135 | if hasattr(model, 'module'):
136 | model_state_dict = model.module.state_dict()
137 | else:
138 | model_state_dict = model.state_dict()
139 | checkpoint = {"model_state_dict": model_state_dict}
140 | save_dir = './checkpoints/%s/%s/' % (args.dataset_tag, mid)
141 | if not os.path.exists(save_dir):
142 | os.makedirs(save_dir)
143 | pickle.dump(args, open(save_dir+'args', 'wb'))
144 | save_path = save_dir+"checkpoint_%d.cpt" % epoch
145 | torch.save(checkpoint, save_path)
146 | print("model saved at:", save_path)
147 | if args.test_eval and args.local_rank in [-1, 0]:
148 | test_dataloader = load_t1_data(args.dataset_tag, args.test_path, args.pretrained_model_path,
149 | args.window_size, args.overlap, args.test_batch, args.max_len) # test_dataloader是第一轮问答的dataloder
150 | (p1, r1, f1), (p2, r2, f2) = test_evaluation(
151 | model, test_dataloader, args.threshold, args.amp)
152 | print(
153 | "Turn 1: precision:{:.4f} recall:{:.4f} f1:{:.4f}".format(p1, r1, f1))
154 | print(
155 | "Turn 2: precision:{:.4f} recall:{:.4f} f1:{:.4f}".format(p2, r2, f2))
156 | model.train()
157 | if args.local_rank != -1:
158 | torch.distributed.barrier()
159 |
160 |
161 | if __name__ == "__main__":
162 | args = args_parser()
163 | set_seed(args.seed)
164 | print(args)
165 | if args.local_rank != -1:
166 | torch.distributed.init_process_group(backend='nccl')
167 | p = '{}_{}_{}'.format(args.dataset_tag, os.path.split(
168 | args.train_path)[-1].split('.')[0], os.path.split(args.pretrained_model_path)[-1])
169 | p1 = os.path.join(os.path.split(args.train_path)[0], p)
170 | if not os.path.exists(p1) or args.reload:
171 | train_dataloader = load_data(args.dataset_tag, args.train_path, args.train_batch, args.max_len, args.pretrained_model_path,
172 | args.local_rank != -1, shuffle=True, threshold=args.threshold)
173 | pickle.dump(train_dataloader, open(p1, 'wb'))
174 | print("training data saved at ", p1)
175 | else:
176 | print("reload training data from ", p1)
177 | train_dataloader = pickle.load(open(p1, 'rb'))
178 | train_dataloader = reload_data(train_dataloader, args.train_batch, args.max_len,
179 | args.threshold, args.local_rank, True)
180 | pickle.dump(train_dataloader, open(p1, 'wb'))
181 | train(args, train_dataloader)
182 |
--------------------------------------------------------------------------------