├── .gitignore ├── README.md ├── bert.py ├── config.ini ├── config.py ├── data ├── dev.tsv ├── relation2id.tsv ├── test.tsv └── train.tsv ├── eval ├── bootstrap_resampling.py ├── res.txt ├── sem_res.txt ├── semeval2010_task8_format_checker.pl ├── semeval2010_task8_scorer-v1.2.pl ├── test.sh └── test_keys.txt ├── model.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | !/.gitignore 3 | *.pyc 4 | scripts 5 | __pycache__/* 6 | __pycache__ 7 | .vscode/* 8 | .vscode 9 | .data/* 10 | .data/* 11 | .data/* 12 | .datset 13 | .datset/* 14 | .datset1 15 | .datset1/* -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A Pytorch Implementation of BERT-based Relation Classification 2 | 3 | This is a stable pytorch implementation of ``Enriching Pre-trained Language Model with Entity Information for Relation Classification`` https://arxiv.org/abs/1905.08284. 4 | ### Requirements: 5 | 6 | Python version >= 3.6 (recommended) 7 | 8 | Pytorch version >= 1.1 (recommended) 9 | 10 | pytorch-transformers: https://github.com/huggingface/pytorch-transformers 11 | !!! pytorch-transformers = 1.1 12 | 13 | 14 | 15 | 16 | ### Tutorial of the code 17 | 18 | 1. Download the project and prepare the data 19 | 20 | ``` 21 | > git clone https://github.com/wang-h/bert-relation-classification 22 | > cd bert-relation-classification 23 | ``` 24 | 25 | 2. Train the bert-based classification model 26 | 27 | ``` 28 | > python bert.py --config config.ini 29 | ``` 30 | 31 | ``` 32 | ... 33 | 09/11/2019 16:36:31 - INFO - pytorch_transformers.modeling_utils - loading weights file /tmp/semeval/pytorch_model.bin 34 | 09/11/2019 16:36:33 - INFO - __main__ - Loading features from cached file ./dataset/cached_dev_bert-large-uncased_128_semeval 35 | 09/11/2019 16:36:33 - INFO - __main__ - Saving features into cached file ./dataset/cached_dev_bert-large-uncased_128_semeval 36 | 09/11/2019 16:36:34 - INFO - __main__ - ***** Running evaluation ***** 37 | 09/11/2019 16:36:34 - INFO - __main__ - Num examples = 2717 38 | 09/11/2019 16:36:34 - INFO - __main__ - Batch size = 8 39 | Evaluating: 100%|████████████████████████████████████████████████████| 340/340 [00:46<00:00, 7.24it/s] 40 | 09/11/2019 16:37:21 - INFO - __main__ - ***** Eval results ***** 41 | 10/07/2019 10:02:23 - INFO - __main__ - acc = 0.8579315421420685 42 | 10/07/2019 10:02:23 - INFO - __main__ - acc_and_f1 = 0.8579315421420685 43 | 10/07/2019 10:02:23 - INFO - __main__ - f1 = 0.8579315421420685 44 | ``` 45 | 46 | 3. Evaluate using the official script for SemEval task-8 47 | 48 | ``` 49 | > cd eval 50 | > bash test.sh 51 | > cat res.txt 52 | ``` 53 | 54 | ``` 55 | (the result reported in the paper, tensorflow) MACRO-averaged result (excluding Other, uncased-large-model): 89.25 56 | (this pytorch implementation) MACRO-averaged result (excluding Other, uncased-large-model): 89.25 (same) 57 | ``` 58 | 59 | I also have the source code written in tensorflow. Feel free to contact me if you need it. 60 | 61 | We also appreciate if you could cite our recent paper with the best result (90.36). 62 | 63 | Enhancing Relation Extraction Using Syntactic Indicators and Sentential Contexts 64 | 65 | https://arxiv.org/abs/1912.01858 66 | 67 | or 68 | 69 | An Extensible Framework of Leveraging Syntactic Skeleton for Semantic Relation Classification, ACM TALLIP, September 2020 70 | 71 | https://dl.acm.org/doi/10.1145/3402885 72 | -------------------------------------------------------------------------------- /bert.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import glob 3 | import logging 4 | import os 5 | import sys 6 | import random 7 | import torch.nn as nn 8 | import numpy as np 9 | import torch 10 | import socket 11 | # wss 12 | # import ptvsd 13 | # Allow other computers to attach to ptvsd at this IP address and port. 14 | # ptvsd.enable_attach(address=('192.168.11.2', 3000), redirect_output=True) 15 | # Pause the program until a remote debugger is attached 16 | # ptvsd.wait_for_attach() 17 | 18 | 19 | from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, 20 | TensorDataset) 21 | from torch.utils.data.distributed import DistributedSampler 22 | # from tensorboardX import SummaryWriter 23 | from tqdm import tqdm, trange 24 | 25 | from pytorch_transformers import (WEIGHTS_NAME, BertConfig, BertTokenizer) 26 | 27 | from pytorch_transformers import AdamW, WarmupLinearSchedule 28 | 29 | from utils import (RELATION_LABELS, compute_metrics, convert_examples_to_features, 30 | output_modes, data_processors) 31 | import torch.nn.functional as F 32 | 33 | from argparse import ArgumentParser 34 | from config import Config 35 | from model import BertForSequenceClassification 36 | logger = logging.getLogger(__name__) 37 | #additional_special_tokens = ["[E11]", "[E12]", "[E21]", "[E22]"] 38 | additional_special_tokens = [] 39 | #additional_special_tokens = ["e11", "e12", "e21", "e22"] 40 | 41 | 42 | def set_seed(seed): 43 | random.seed(seed) 44 | np.random.seed(seed) 45 | torch.manual_seed(seed) 46 | torch.cuda.manual_seed_all(seed) 47 | 48 | 49 | def train(config, train_dataset, model, tokenizer): 50 | """ Train the model """ 51 | config.train_batch_size = config.per_gpu_train_batch_size * \ 52 | max(1, config.n_gpu) 53 | if config.local_rank == -1: 54 | train_sampler = RandomSampler(train_dataset) 55 | else: 56 | DistributedSampler(train_dataset) 57 | 58 | train_dataloader = DataLoader( 59 | train_dataset, sampler=train_sampler, batch_size=config.train_batch_size) 60 | 61 | if config.max_steps > 0: 62 | t_total = config.max_steps 63 | config.num_train_epochs = config.max_steps // ( 64 | len(train_dataloader) // config.gradient_accumulation_steps) + 1 65 | else: 66 | t_total = len( 67 | train_dataloader) // config.gradient_accumulation_steps * config.num_train_epochs 68 | 69 | # Prepare optimizer and schedule (linear warmup and decay) 70 | no_decay = ['bias', 'LayerNorm.weight'] 71 | optimizer_grouped_parameters = [ 72 | {'params': [p for n, p in model.named_parameters() 73 | if not any(nd in n for nd in no_decay)], 'weight_decay': config.weight_decay}, 74 | {'params': [p for n, p in model.named_parameters() 75 | if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} 76 | ] 77 | optimizer = AdamW(optimizer_grouped_parameters, 78 | lr=config.learning_rate, eps=config.adam_epsilon) 79 | scheduler = WarmupLinearSchedule( 80 | optimizer, warmup_steps=config.warmup_steps, t_total=t_total) 81 | if config.n_gpu > 1: 82 | model = torch.nn.DataParallel(model) 83 | 84 | # Distributed training (should be after apex fp16 initialization) 85 | if config.local_rank != -1: 86 | model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[config.local_rank], 87 | output_device=config.local_rank, 88 | find_unused_parameters=True) 89 | 90 | # Train! 91 | logger.info("***** Running training *****") 92 | logger.info(" Num examples = %d", len(train_dataset)) 93 | logger.info(" Num Epochs = %d", config.num_train_epochs) 94 | logger.info(" Instantaneous batch size per GPU = %d", 95 | config.per_gpu_train_batch_size) 96 | logger.info(" Total train batch size (w. parallel, distributed & accumulation) = %d", 97 | config.train_batch_size * config.gradient_accumulation_steps 98 | * (torch.distributed.get_world_size() if config.local_rank != -1 else 1)) 99 | logger.info(" Gradient Accumulation steps = %d", 100 | config.gradient_accumulation_steps) 101 | logger.info(" Total optimization steps = %d", t_total) 102 | 103 | global_step = 0 104 | tr_loss, logging_loss = 0.0, 0.0 105 | model.zero_grad() 106 | train_iterator = trange(int(config.num_train_epochs), 107 | desc="Epoch", disable=config.local_rank not in [-1, 0]) 108 | # Added here for reproductibility (even between python 2 and 3) 109 | set_seed(config.seed) 110 | for _ in train_iterator: 111 | epoch_iterator = tqdm(train_dataloader, desc="Iteration", 112 | disable=config.local_rank not in [-1, 0]) 113 | for step, batch in enumerate(epoch_iterator): 114 | model.train() 115 | batch = tuple(t.to(config.device) for t in batch) 116 | inputs = {'input_ids': batch[0], 117 | 'attention_mask': batch[1], 118 | # XLM and RoBERTa don't use segment_ids 119 | 'token_type_ids': batch[2], 120 | 'labels': batch[3], 121 | 'e1_mask': batch[4], 122 | 'e2_mask': batch[5], 123 | } 124 | 125 | outputs = model(**inputs) 126 | # model outputs are always tuple in pytorch-transformers (see doc) 127 | loss = outputs[0] 128 | if config.n_gpu > 1: 129 | loss = loss.mean() # mean() to average on multi-gpu parallel training 130 | if config.gradient_accumulation_steps > 1: 131 | loss = loss / config.gradient_accumulation_steps 132 | 133 | loss.backward() 134 | torch.nn.utils.clip_grad_norm_( 135 | model.parameters(), config.max_grad_norm) 136 | 137 | tr_loss += loss.item() 138 | if (step + 1) % config.gradient_accumulation_steps == 0: 139 | 140 | optimizer.step() 141 | scheduler.step() # Update learning rate schedule 142 | model.zero_grad() 143 | global_step += 1 144 | 145 | if config.local_rank in [-1, 0] and config.logging_steps > 0 and global_step % config.logging_steps == 0: 146 | # Log metrics 147 | # Only evaluate when single GPU otherwise metrics may not average well 148 | if config.local_rank == -1 and config.evaluate_during_training: 149 | results = evaluate(config, model, tokenizer) 150 | logging_loss = tr_loss 151 | # if config.local_rank in [-1, 0] and config.save_steps > 0 and global_step % config.save_steps == 0: 152 | # # Save model checkpoint 153 | # output_dir = os.path.join( 154 | # config.output_dir, 'checkpoint-{}'.format(global_step)) 155 | # if not os.path.exists(output_dir): 156 | # os.makedirs(output_dir) 157 | # # Take care of distributed/parallel training 158 | # model_to_save = model.module if hasattr( 159 | # model, 'module') else model 160 | # model_to_save.save_pretrained(output_dir) 161 | # torch.save(config, os.path.join( 162 | # output_dir, 'training_config.bin')) 163 | # logger.info("Saving model checkpoint to %s", output_dir) 164 | 165 | if config.max_steps > 0 and global_step > config.max_steps: 166 | epoch_iterator.close() 167 | break 168 | if config.max_steps > 0 and global_step > config.max_steps: 169 | train_iterator.close() 170 | break 171 | return global_step, tr_loss / global_step 172 | 173 | 174 | def evaluate(config, model, tokenizer, prefix=""): 175 | # Loop to handle MNLI double evaluation (matched, mis-matched) 176 | eval_task = config.task_name 177 | eval_output_dir = config.output_dir 178 | 179 | results = {} 180 | eval_dataset = load_and_cache_examples( 181 | config, eval_task, tokenizer, evaluate=True) 182 | 183 | if not os.path.exists(eval_output_dir) and config.local_rank in [-1, 0]: 184 | os.makedirs(eval_output_dir) 185 | 186 | config.eval_batch_size = config.per_gpu_eval_batch_size * \ 187 | max(1, config.n_gpu) 188 | # Note that DistributedSampler samples randomly 189 | eval_sampler = SequentialSampler( 190 | eval_dataset) if config.local_rank == -1 else DistributedSampler(eval_dataset) 191 | eval_dataloader = DataLoader( 192 | eval_dataset, sampler=eval_sampler, batch_size=config.eval_batch_size) 193 | 194 | # Eval! 195 | logger.info("***** Running evaluation {} *****".format(prefix)) 196 | logger.info(" Num examples = %d", len(eval_dataset)) 197 | logger.info(" Batch size = %d", config.eval_batch_size) 198 | eval_loss = 0.0 199 | nb_eval_steps = 0 200 | preds = None 201 | out_label_ids = None 202 | for batch in tqdm(eval_dataloader, desc="Evaluating"): 203 | model.eval() 204 | batch = tuple(t.to(config.device) for t in batch) 205 | 206 | with torch.no_grad(): 207 | inputs = {'input_ids': batch[0], 208 | 'attention_mask': batch[1], 209 | # XLM and RoBERTa don't use segment_ids 210 | 'token_type_ids': batch[2], 211 | 'labels': batch[3], 212 | 'e1_mask': batch[4], 213 | 'e2_mask': batch[5], 214 | } 215 | outputs = model(**inputs) 216 | tmp_eval_loss, logits = outputs[:2] 217 | 218 | eval_loss += tmp_eval_loss.mean().item() 219 | nb_eval_steps += 1 220 | if preds is None: 221 | preds = logits.detach().cpu().numpy() 222 | out_label_ids = inputs['labels'].detach().cpu().numpy() 223 | else: 224 | preds = np.append(preds, logits.detach().cpu().numpy(), axis=0) 225 | out_label_ids = np.append( 226 | out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0) 227 | 228 | eval_loss = eval_loss / nb_eval_steps 229 | preds = np.argmax(preds, axis=1) 230 | result = compute_metrics(eval_task, preds, out_label_ids) 231 | results.update(result) 232 | logger.info("***** Eval results {} *****".format(prefix)) 233 | for key in sorted(result.keys()): 234 | logger.info(" %s = %s", key, str(result[key])) 235 | output_eval_file = "eval/sem_res.txt" 236 | with open(output_eval_file, "w") as writer: 237 | for key in range(len(preds)): 238 | writer.write("%d\t%s\n" % 239 | (key+8001, str(RELATION_LABELS[preds[key]]))) 240 | return result 241 | 242 | 243 | def load_and_cache_examples(config, task, tokenizer, evaluate=False): 244 | if config.local_rank not in [-1, 0] and not evaluate: 245 | # Make sure only the first process in distributed training process the dataset, and the others will use the cache 246 | torch.distributed.barrier() 247 | 248 | processor = data_processors[config.task_name]() 249 | # Load data features from cache or dataset file 250 | # cached_features_file = os.path.join(config.data_dir, 'cached_{}_{}_{}_{}'.format( 251 | # 'dev' if evaluate else 'train', 252 | # list(filter(None, 'bert-large-uncased'.split('/'))).pop(), 253 | # str(config.max_seq_len), 254 | # str(task))) 255 | # if os.path.exists(cached_features_file): 256 | # logger.info("Loading features from cached file %s", 257 | # cached_features_file) 258 | # features = torch.load(cached_features_file) 259 | # else: 260 | logger.info("Creating features from dataset file at %s", 261 | config.data_dir) 262 | label_list = processor.get_labels() 263 | examples = processor.get_dev_examples( 264 | config.data_dir) if evaluate else processor.get_train_examples(config.data_dir) 265 | features = convert_examples_to_features( 266 | examples, label_list, config.max_seq_len, tokenizer, "classification", use_entity_indicator=config.use_entity_indicator) 267 | # if config.local_rank in [-1, 0]: 268 | # logger.info("Saving features into cached file %s", 269 | # cached_features_file) 270 | # torch.save(features, cached_features_file) 271 | 272 | if config.local_rank == 0 and not evaluate: 273 | # Make sure only the first process in distributed training process the dataset, and the others will use the cache 274 | torch.distributed.barrier() 275 | output_mode = "classification" 276 | # Convert to Tensors and build dataset 277 | all_input_ids = torch.tensor( 278 | [f.input_ids for f in features], dtype=torch.long) 279 | all_input_mask = torch.tensor( 280 | [f.input_mask for f in features], dtype=torch.long) 281 | all_segment_ids = torch.tensor( 282 | [f.segment_ids for f in features], dtype=torch.long) 283 | all_e1_mask = torch.tensor( 284 | [f.e1_mask for f in features], dtype=torch.long) # add e1 mask 285 | all_e2_mask = torch.tensor( 286 | [f.e2_mask for f in features], dtype=torch.long) # add e2 mask 287 | if output_mode == "classification": 288 | all_label_ids = torch.tensor( 289 | [f.label_id for f in features], dtype=torch.long) 290 | elif output_mode == "regression": 291 | all_label_ids = torch.tensor( 292 | [f.label_id for f in features], dtype=torch.float) 293 | dataset = TensorDataset(all_input_ids, all_input_mask, 294 | all_segment_ids, all_label_ids, all_e1_mask, all_e2_mask) 295 | return dataset 296 | 297 | 298 | def main(): 299 | parser = ArgumentParser( 300 | description="BERT for relation extraction (classification)") 301 | parser.add_argument('--config', dest='config') 302 | args = parser.parse_args() 303 | config = Config(args.config) 304 | 305 | if os.path.exists(config.output_dir) and os.listdir(config.output_dir) and config.train and not config.overwrite_output_dir: 306 | raise ValueError( 307 | "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(config.output_dir)) 308 | 309 | # Setup CUDA, GPU & distributed training 310 | if config.local_rank == -1 or config.no_cuda: 311 | device = torch.device( 312 | "cuda" if torch.cuda.is_available() and not config.no_cuda else "cpu") 313 | config.n_gpu = torch.cuda.device_count() 314 | else: 315 | # Initializes the distributed backend which will take care of sychronizing nodes/GPUs 316 | torch.cuda.set_device(config.local_rank) 317 | device = torch.device("cuda", args.local_rank) 318 | torch.distributed.init_process_group(backend='nccl') 319 | config.n_gpu = 1 320 | config.device = device 321 | 322 | # Setup logging 323 | logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', 324 | datefmt='%m/%d/%Y %H:%M:%S', 325 | level=logging.INFO if config.local_rank in [-1, 0] else logging.WARN) 326 | logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s", 327 | config.local_rank, device, config.n_gpu, bool(config.local_rank != -1)) 328 | 329 | # Set seed 330 | set_seed(config.seed) 331 | 332 | # Prepare GLUE task 333 | processor = data_processors["semeval"]() 334 | output_mode = output_modes["semeval"] 335 | label_list = processor.get_labels() 336 | num_labels = len(label_list) 337 | 338 | # Load pretrained model and tokenizer 339 | if config.local_rank not in [-1, 0]: 340 | torch.distributed.barrier() 341 | # Make sure only the first process in distributed training will download model & vocab 342 | bertconfig = BertConfig.from_pretrained( 343 | config.pretrained_model_name, num_labels=num_labels, finetuning_task=config.task_name) 344 | # './large-uncased-model', num_labels=num_labels, finetuning_task=config.task_name) 345 | bertconfig.l2_reg_lambda = config.l2_reg_lambda 346 | bertconfig.latent_entity_typing = config.latent_entity_typing 347 | if config.l2_reg_lambda > 0: 348 | logger.info("using L2 regularization with lambda %.5f", 349 | config.l2_reg_lambda) 350 | if config.latent_entity_typing: 351 | logger.info("adding the component of latent entity typing: %s", 352 | str(config.latent_entity_typing)) 353 | tokenizer = BertTokenizer.from_pretrained( 354 | 'bert-base-uncased', do_lower_case=True, additional_special_tokens=additional_special_tokens) 355 | # 'bert-large-uncased', do_lower_case=True, additional_special_tokens=additional_special_tokens) 356 | model = BertForSequenceClassification.from_pretrained( 357 | config.pretrained_model_name, config=bertconfig) 358 | # './large-uncased-model', config=bertconfig) 359 | 360 | if config.local_rank == 0: 361 | # Make sure only the first process in distributed training will download model & vocab 362 | torch.distributed.barrier() 363 | 364 | model.to(config.device) 365 | 366 | # logger.info("Training/evaluation parameters %s", config) 367 | 368 | # Training 369 | if config.train: 370 | train_dataset = load_and_cache_examples( 371 | config, config.task_name, tokenizer, evaluate=False) 372 | global_step, tr_loss = train(config, train_dataset, model, tokenizer) 373 | logger.info(" global_step = %s, average loss = %s", 374 | global_step, tr_loss) 375 | 376 | # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() 377 | if config.train and (config.local_rank == -1 or torch.distributed.get_rank() == 0): 378 | # Create output directory if needed 379 | if not os.path.exists(config.output_dir) and config.local_rank in [-1, 0]: 380 | os.makedirs(config.output_dir) 381 | 382 | logger.info("Saving model checkpoint to %s", config.output_dir) 383 | # Save a trained model, configuration and tokenizer using `save_pretrained()`. 384 | # They can then be reloaded using `from_pretrained()` 385 | # Take care of distributed/parallel training 386 | model_to_save = model.module if hasattr(model, 'module') else model 387 | model_to_save.save_pretrained(config.output_dir) 388 | tokenizer.save_pretrained(config.output_dir) 389 | 390 | # Good practice: save your training arguments together with the trained model 391 | torch.save(config, os.path.join( 392 | config.output_dir, 'training_config.bin')) 393 | 394 | # Load a trained model and vocabulary that you have fine-tuned 395 | model = BertForSequenceClassification.from_pretrained( 396 | config.output_dir) 397 | tokenizer = BertTokenizer.from_pretrained( 398 | config.output_dir, do_lower_case=True, additional_special_tokens=additional_special_tokens) 399 | model.to(config.device) 400 | 401 | # Evaluation 402 | results = {} 403 | if config.eval and config.local_rank in [-1, 0]: 404 | tokenizer = BertTokenizer.from_pretrained( 405 | config.output_dir, do_lower_case=True, additional_special_tokens=additional_special_tokens) 406 | checkpoints = [config.output_dir] 407 | if config.eval_all_checkpoints: 408 | checkpoints = list(os.path.dirname(c) for c in sorted( 409 | glob.glob(config.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) 410 | logging.getLogger("pytorch_transformers.modeling_utils").setLevel( 411 | logging.WARN) # Reduce logging 412 | logger.info("Evaluate the following checkpoints: %s", checkpoints) 413 | for checkpoint in checkpoints: 414 | global_step = checkpoint.split( 415 | '-')[-1] if len(checkpoints) > 1 else "" 416 | model = BertForSequenceClassification.from_pretrained(checkpoint) 417 | model.to(config.device) 418 | result = evaluate(config, model, tokenizer, prefix=global_step) 419 | result = dict((k + '_{}'.format(global_step), v) 420 | for k, v in result.items()) 421 | results.update(result) 422 | 423 | return results 424 | 425 | 426 | if __name__ == "__main__": 427 | main() 428 | -------------------------------------------------------------------------------- /config.ini: -------------------------------------------------------------------------------- 1 | ################################### 2 | # configuration # 3 | ################################### 4 | [DEFAULT] 5 | task_name = semeval 6 | output_dir=/tmp/%(task_name)s 7 | use_entity_indicator=True 8 | 9 | [MODEL] 10 | seed = 12345 11 | #pretrained_model_name=bert-base-uncased 12 | pretrained_model_name=bert-large-uncased 13 | #pretrained_model_name=./large-uncased-model 14 | 15 | [Train] 16 | num_train_epochs=5.0 17 | # Total number of training epochs to perform. 18 | learning_rate=2e-5 19 | # The initial learning rate for Adam. 20 | per_gpu_train_batch_size=16 21 | # Batch size per GPU/CPU for training. 22 | per_gpu_eval_batch_size=8 23 | # Batch size per GPU/CPU for evaluation. 24 | no_cuda=False 25 | # Avoid using CUDA when available 26 | 27 | [Dataset] 28 | data_dir= ./data 29 | 30 | max_seq_len=128 31 | 32 | train=True 33 | eval=True 34 | evaluate_during_training=True 35 | 36 | gradient_accumulation_steps=1 37 | # Number of updates steps to accumulate before performing a backward/update pass. 38 | 39 | weight_decay=0.9 40 | # Weight deay if we apply some. 41 | adam_epsilon=1e-8 42 | # Epsilon for Adam optimizer. 43 | max_grad_norm=1.0 44 | # Max gradient norm. 45 | 46 | max_steps=-1 47 | # If > 0: set total number of training steps to perform. Override num_train_epochs. 48 | warmup_steps=0 49 | # Linear warmup over warmup_steps. 50 | logging_steps=200 51 | # Log every X updates steps. 52 | 53 | save_steps=200 54 | # help="Save checkpoint every X updates steps. 55 | 56 | eval_all_checkpoints=False 57 | # Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number 58 | l2_reg_lambda=5e-3 59 | #l2_reg_lambda=0 60 | 61 | overwrite_output_dir=True 62 | # Overwrite the content of the output directory 63 | overwrite_cache=True 64 | # Overwrite the cached training and evaluation sets 65 | local_rank=-1 66 | # For distributed training: local_rank 67 | 68 | latent_entity_typing=False 69 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | from configparser import ConfigParser 2 | 3 | 4 | class Config(ConfigParser): 5 | def __init__(self, config_file): 6 | raw_config = ConfigParser() 7 | raw_config.read(config_file) 8 | self.cast_values(raw_config) 9 | 10 | def cast_values(self, raw_config): 11 | for section in raw_config.sections(): 12 | for key, value in raw_config.items(section): 13 | val = None 14 | if type(value) is str and value.startswith("[") and value.endswith("]"): 15 | val = eval(value) 16 | setattr(self, key, val) 17 | continue 18 | for attr in ["getint", "getfloat", "getboolean"]: 19 | try: 20 | val = getattr(raw_config[section], attr)(key) 21 | break 22 | except: 23 | val = value 24 | setattr(self, key, val) 25 | -------------------------------------------------------------------------------- /data/relation2id.tsv: -------------------------------------------------------------------------------- 1 | Other 2 | Message-Topic(e1,e2) 3 | Message-Topic(e2,e1) 4 | Product-Producer(e1,e2) 5 | Product-Producer(e2,e1) 6 | Instrument-Agency(e1,e2) 7 | Instrument-Agency(e2,e1) 8 | Entity-Destination(e1,e2) 9 | Entity-Destination(e2,e1) 10 | Cause-Effect(e1,e2) 11 | Cause-Effect(e2,e1) 12 | Component-Whole(e1,e2) 13 | Component-Whole(e2,e1) 14 | Entity-Origin(e1,e2) 15 | Entity-Origin(e2,e1) 16 | Member-Collection(e1,e2) 17 | Member-Collection(e2,e1) 18 | Content-Container(e1,e2) 19 | Content-Container(e2,e1) -------------------------------------------------------------------------------- /eval/bootstrap_resampling.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Author: Hao WANG 3 | ############################################### 4 | # An implementation of paired bootstrap resampling for testing the statistical 5 | # significance of the difference between two systems from (Koehn 2004 @ EMNLP) 6 | # Specified for Categorical F1 Scores. 7 | # Usage: ./bootstrap-resampling.py hypothesis_1 hypothesis_2 reference_1 [ reference_2 ... ] 8 | ############################################### 9 | 10 | 11 | from sklearn.metrics import matthews_corrcoef, f1_score 12 | import sys 13 | import numpy as np 14 | from tqdm import tqdm 15 | # constants 16 | TIMES_TO_REPEAT_SUBSAMPLING = 1000 17 | SUBSAMPLE_SIZE = 0 18 | # if 0 then subsample size is equal to the whole set 19 | 20 | 21 | def read_order(line): 22 | """Read one example from the order file.""" 23 | if not line: 24 | return None 25 | line = line[:-1] 26 | order = line.split() 27 | order = [int(item) for item in order] 28 | return order 29 | 30 | 31 | def getAcc(refs, hypos, indices=None): 32 | return _CalculateAccScores(refs, hypos, indices) 33 | 34 | 35 | def _CalculateAccScores(refs, hypos, indices=None): 36 | num = 0 37 | skipped = 0 38 | sum_ = 0 39 | if indices is None: 40 | candidates = list(range(len(refs))) 41 | else: 42 | candidates = indices 43 | candidates = [ 44 | idx for idx in candidates if not refs[idx].startswith("Other")] 45 | 46 | sample_refs = [refs[idx] for idx in candidates] 47 | sample_hypos = [hypos[idx] for idx in candidates] 48 | 49 | return f1_score(sample_refs, sample_hypos, average='macro') 50 | 51 | 52 | def main(argv): 53 | # checking cmdline argument consistency 54 | if len(argv) != 4: 55 | print("Usage: ./bootstrap-hypothesis-difference-significance.py hypothesis_1 hypothesis_2 reference\n", file=sys.stderr) 56 | sys.exit(1) 57 | print("reading data", file=sys.stderr) 58 | # read all data 59 | data = readAllData(argv) 60 | # #calculate each sentence's contribution to BP and ngram precision 61 | # print("rperforming preliminary calculations (hypothesis 1); ", file=sys.stderr) 62 | # preEvalHypo(data, "hyp1") 63 | 64 | # print("rperforming preliminary calculations (hypothesis 2); ", file=sys.stderr) 65 | # preEvalHypo(data, "hyp2") 66 | 67 | # start comparing 68 | print("comparing hypotheses -- this may take some time; ", file=sys.stderr) 69 | 70 | # bootstrap_report(data, "Fuzzy Reordering Scores", getFRS) 71 | # bootstrap_report(data, "Normalized Kendall's Tau", getNKT) 72 | bootstrap_report(data, "ACC", getAcc) 73 | 74 | ##### 75 | 76 | 77 | def bootstrap_report(data, title, func): 78 | subSampleIndices = np.random.choice( 79 | data["size"], SUBSAMPLE_SIZE if SUBSAMPLE_SIZE > 0 else data["size"], replace=True) 80 | print(f1_score(data["refs"], data["hyp1"], average='macro')) 81 | print(f1_score(data["refs"], data["hyp2"], average='macro')) 82 | 83 | realScore1 = func(data["refs"], data["hyp1"], subSampleIndices) 84 | realScore2 = func(data["refs"], data["hyp2"], subSampleIndices) 85 | subSampleScoreDiffArr, subSampleScore1Arr, subSampleScore2Arr = bootstrap_pass( 86 | data, func) 87 | 88 | scorePValue = bootstrap_pvalue( 89 | subSampleScoreDiffArr, realScore1, realScore2) 90 | 91 | (scoreAvg1, scoreVar1) = bootstrap_interval(subSampleScore1Arr) 92 | (scoreAvg2, scoreVar2) = bootstrap_interval(subSampleScore2Arr) 93 | 94 | print("\n---=== %s score ===---\n" % title) 95 | print("actual score of hypothesis 1: %f" % realScore1) 96 | print("95/100 confidence interval for hypothesis 1 score: %f +- %f" % 97 | (scoreAvg1, scoreVar1) + "\n-----\n") 98 | print("actual score of hypothesis 1: %f" % realScore2) 99 | print("95/100 confidence interval for hypothesis 2 score: %f +- %f" % 100 | (scoreAvg2, scoreVar2) + "\n-----\n") 101 | print("Assuming that essentially the same system generated the two hypothesis translations (null-hypothesis),\n") 102 | print("the probability of actually getting them (p-value) is: %f\n" % 103 | scorePValue) 104 | 105 | 106 | ##### 107 | def bootstrap_pass(data, scoreFunc): 108 | subSampleDiffArr = [] 109 | subSample1Arr = [] 110 | subSample2Arr = [] 111 | 112 | # applying sampling 113 | for idx in tqdm(range(TIMES_TO_REPEAT_SUBSAMPLING), ncols=80, postfix="Subsampling"): 114 | subSampleIndices = np.random.choice( 115 | data["size"], SUBSAMPLE_SIZE if SUBSAMPLE_SIZE > 0 else data["size"], replace=True) 116 | score1 = scoreFunc(data["refs"], data["hyp1"], subSampleIndices) 117 | score2 = scoreFunc(data["refs"], data["hyp2"], subSampleIndices) 118 | subSampleDiffArr.append(abs(score2 - score1)) 119 | subSample1Arr.append(score1) 120 | subSample2Arr.append(score2) 121 | return np.array(subSampleDiffArr), np.array(subSample1Arr), np.array(subSample2Arr) 122 | ##### 123 | # 124 | ##### 125 | 126 | 127 | def bootstrap_pvalue(subSampleDiffArr, realScore1, realScore2): 128 | realDiff = abs(realScore2 - realScore1) 129 | 130 | # get subsample difference mean 131 | averageSubSampleDiff = subSampleDiffArr.mean() 132 | 133 | # calculating p-value 134 | count = 0.0 135 | 136 | for subSampleDiff in subSampleDiffArr: 137 | if subSampleDiff - averageSubSampleDiff >= realDiff: 138 | count += 1 139 | return count / TIMES_TO_REPEAT_SUBSAMPLING 140 | 141 | ##### 142 | # 143 | ##### 144 | 145 | 146 | def bootstrap_interval(subSampleArr): 147 | sortedArr = sorted(subSampleArr, reverse=False) 148 | lowerIdx = int(TIMES_TO_REPEAT_SUBSAMPLING / 40) 149 | higherIdx = TIMES_TO_REPEAT_SUBSAMPLING - lowerIdx - 1 150 | 151 | lower = sortedArr[lowerIdx] 152 | higher = sortedArr[higherIdx] 153 | diff = higher - lower 154 | return (lower + 0.5 * diff, 0.5 * diff) 155 | 156 | ##### 157 | # read 2 hyp and 1 to \infty ref data files 158 | ##### 159 | 160 | 161 | def readAllData(argv): 162 | print(argv) 163 | assert len(argv[1:]) == 3 164 | hypFile1, hypFile2 = argv[1:3] 165 | refFile = argv[3] 166 | result = {} 167 | # reading hypotheses and checking for matching sizes 168 | result["hyp1"] = [line for line in open(hypFile1)] 169 | result["size"] = len(result["hyp1"]) 170 | 171 | result["hyp2"] = [line for line in open(hypFile2)] 172 | assert len(result["hyp2"]) == len(result["hyp1"]) 173 | 174 | refDataX = [line for line in open(refFile)] 175 | # updateCounts($result{ngramCounts}, $refDataX); 176 | result["refs"] = refDataX 177 | # print(result) 178 | return result 179 | 180 | 181 | if __name__ == '__main__': 182 | main(sys.argv) 183 | -------------------------------------------------------------------------------- /eval/res.txt: -------------------------------------------------------------------------------- 1 | <<< (2*9+1)-WAY EVALUATION (USING DIRECTIONALITY)>>>: 2 | 3 | Confusion matrix: 4 | C-E1 C-E2 C-W1 C-W2 C-C1 C-C2 E-D1 E-D2 E-O1 E-O2 I-A1 I-A2 M-C1 M-C2 M-T1 M-T2 P-P1 P-P2 _O_ <-- classified as 5 | +-----------------------------------------------------------------------------------------------+ -SUM- skip ACTUAL 6 | C-E1 | 127 2 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 4 | 134 0 134 7 | C-E2 | 2 183 0 0 0 0 0 0 2 0 0 0 0 0 0 0 2 0 5 | 194 0 194 8 | C-W1 | 0 0 140 3 0 0 0 0 0 0 3 1 4 1 1 0 0 0 9 | 162 0 162 9 | C-W2 | 0 0 3 125 0 1 0 0 1 0 0 4 0 2 2 0 0 1 11 | 150 0 150 10 | C-C1 | 0 0 2 0 141 0 7 0 0 0 0 0 0 0 0 0 1 0 2 | 153 0 153 11 | C-C2 | 0 0 0 1 0 35 0 0 0 0 0 0 0 1 0 0 0 0 2 | 39 0 39 12 | E-D1 | 0 0 1 0 4 0 276 0 0 0 0 0 0 0 0 0 0 0 10 | 291 0 291 13 | E-D2 | 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 | 1 0 1 14 | E-O1 | 0 7 2 1 0 0 2 0 182 0 0 0 1 0 0 0 5 0 11 | 211 0 211 15 | E-O2 | 1 0 0 0 0 0 0 0 1 40 0 1 0 0 0 0 0 1 3 | 47 0 47 16 | I-A1 | 0 0 1 0 0 0 0 0 0 0 15 0 0 0 0 0 1 0 5 | 22 0 22 17 | I-A2 | 0 0 0 4 0 0 0 0 0 0 0 108 0 0 1 0 0 1 20 | 134 0 134 18 | M-C1 | 0 0 1 0 0 0 1 0 1 0 0 0 26 0 0 0 0 0 3 | 32 0 32 19 | M-C2 | 0 0 1 1 0 0 0 0 0 0 0 0 0 179 0 0 1 0 19 | 201 0 201 20 | M-T1 | 0 0 0 1 0 0 0 0 0 0 0 0 0 1 199 0 0 0 9 | 210 0 210 21 | M-T2 | 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 47 0 0 3 | 51 0 51 22 | P-P1 | 0 4 0 0 0 0 0 0 3 0 2 0 0 0 0 0 92 1 6 | 108 0 108 23 | P-P2 | 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 114 6 | 123 0 123 24 | _O_ | 8 6 17 11 12 2 14 0 20 2 1 15 8 20 19 5 11 10 273 | 454 0 454 25 | +-----------------------------------------------------------------------------------------------+ 26 | -SUM- 139 202 168 148 157 39 301 0 210 42 21 129 39 204 223 53 113 128 401 2717 0 2717 27 | 28 | Coverage = 2717/2717 = 100.00% 29 | Accuracy (calculated for the above confusion matrix) = 2302/2717 = 84.73% 30 | Accuracy (considering all skipped examples as Wrong) = 2302/2717 = 84.73% 31 | Accuracy (considering all skipped examples as Other) = 2302/2717 = 84.73% 32 | 33 | Results for the individual relations: 34 | Cause-Effect(e1,e2) : P = 127/ 139 = 91.37% R = 127/ 134 = 94.78% F1 = 93.04% 35 | Cause-Effect(e2,e1) : P = 183/ 202 = 90.59% R = 183/ 194 = 94.33% F1 = 92.42% 36 | Component-Whole(e1,e2) : P = 140/ 168 = 83.33% R = 140/ 162 = 86.42% F1 = 84.85% 37 | Component-Whole(e2,e1) : P = 125/ 148 = 84.46% R = 125/ 150 = 83.33% F1 = 83.89% 38 | Content-Container(e1,e2) : P = 141/ 157 = 89.81% R = 141/ 153 = 92.16% F1 = 90.97% 39 | Content-Container(e2,e1) : P = 35/ 39 = 89.74% R = 35/ 39 = 89.74% F1 = 89.74% 40 | Entity-Destination(e1,e2) : P = 276/ 301 = 91.69% R = 276/ 291 = 94.85% F1 = 93.24% 41 | Entity-Destination(e2,e1) : P = 0/ 0 = 0.00% R = 0/ 1 = 0.00% F1 = 0.00% 42 | Entity-Origin(e1,e2) : P = 182/ 210 = 86.67% R = 182/ 211 = 86.26% F1 = 86.46% 43 | Entity-Origin(e2,e1) : P = 40/ 42 = 95.24% R = 40/ 47 = 85.11% F1 = 89.89% 44 | Instrument-Agency(e1,e2) : P = 15/ 21 = 71.43% R = 15/ 22 = 68.18% F1 = 69.77% 45 | Instrument-Agency(e2,e1) : P = 108/ 129 = 83.72% R = 108/ 134 = 80.60% F1 = 82.13% 46 | Member-Collection(e1,e2) : P = 26/ 39 = 66.67% R = 26/ 32 = 81.25% F1 = 73.24% 47 | Member-Collection(e2,e1) : P = 179/ 204 = 87.75% R = 179/ 201 = 89.05% F1 = 88.40% 48 | Message-Topic(e1,e2) : P = 199/ 223 = 89.24% R = 199/ 210 = 94.76% F1 = 91.92% 49 | Message-Topic(e2,e1) : P = 47/ 53 = 88.68% R = 47/ 51 = 92.16% F1 = 90.38% 50 | Product-Producer(e1,e2) : P = 92/ 113 = 81.42% R = 92/ 108 = 85.19% F1 = 83.26% 51 | Product-Producer(e2,e1) : P = 114/ 128 = 89.06% R = 114/ 123 = 92.68% F1 = 90.84% 52 | _Other : P = 273/ 401 = 68.08% R = 273/ 454 = 60.13% F1 = 63.86% 53 | 54 | Micro-averaged result (excluding Other): 55 | P = 2029/2316 = 87.61% R = 2029/2263 = 89.66% F1 = 88.62% 56 | 57 | MACRO-averaged result (excluding Other): 58 | P = 81.16% R = 82.82% F1 = 81.91% 59 | 60 | 61 | 62 | <<< (9+1)-WAY EVALUATION IGNORING DIRECTIONALITY >>>: 63 | 64 | Confusion matrix: 65 | C-E C-W C-C E-D E-O I-A M-C M-T P-P _O_ <-- classified as 66 | +--------------------------------------------------+ -SUM- skip ACTUAL 67 | C-E | 314 0 0 0 2 0 0 1 2 9 | 328 0 328 68 | C-W | 0 271 1 0 1 8 7 3 1 20 | 312 0 312 69 | C-C | 0 3 176 7 0 0 1 0 1 4 | 192 0 192 70 | E-D | 0 1 5 276 0 0 0 0 0 10 | 292 0 292 71 | E-O | 8 3 0 2 223 1 1 0 6 14 | 258 0 258 72 | I-A | 0 5 0 0 0 123 0 1 2 25 | 156 0 156 73 | M-C | 0 3 0 1 1 0 205 0 1 22 | 233 0 233 74 | M-T | 0 2 0 0 0 0 1 246 0 12 | 261 0 261 75 | P-P | 5 0 0 1 3 2 0 1 207 12 | 231 0 231 76 | _O_ | 14 28 14 14 22 16 28 24 21 273 | 454 0 454 77 | +--------------------------------------------------+ 78 | -SUM- 341 316 196 301 252 150 243 276 241 401 2717 0 2717 79 | 80 | Coverage = 2717/2717 = 100.00% 81 | Accuracy (calculated for the above confusion matrix) = 2314/2717 = 85.17% 82 | Accuracy (considering all skipped examples as Wrong) = 2314/2717 = 85.17% 83 | Accuracy (considering all skipped examples as Other) = 2314/2717 = 85.17% 84 | 85 | Results for the individual relations: 86 | Cause-Effect : P = 314/ 341 = 92.08% R = 314/ 328 = 95.73% F1 = 93.87% 87 | Component-Whole : P = 271/ 316 = 85.76% R = 271/ 312 = 86.86% F1 = 86.31% 88 | Content-Container : P = 176/ 196 = 89.80% R = 176/ 192 = 91.67% F1 = 90.72% 89 | Entity-Destination : P = 276/ 301 = 91.69% R = 276/ 292 = 94.52% F1 = 93.09% 90 | Entity-Origin : P = 223/ 252 = 88.49% R = 223/ 258 = 86.43% F1 = 87.45% 91 | Instrument-Agency : P = 123/ 150 = 82.00% R = 123/ 156 = 78.85% F1 = 80.39% 92 | Member-Collection : P = 205/ 243 = 84.36% R = 205/ 233 = 87.98% F1 = 86.13% 93 | Message-Topic : P = 246/ 276 = 89.13% R = 246/ 261 = 94.25% F1 = 91.62% 94 | Product-Producer : P = 207/ 241 = 85.89% R = 207/ 231 = 89.61% F1 = 87.71% 95 | _Other : P = 273/ 401 = 68.08% R = 273/ 454 = 60.13% F1 = 63.86% 96 | 97 | Micro-averaged result (excluding Other): 98 | P = 2041/2316 = 88.13% R = 2041/2263 = 90.19% F1 = 89.15% 99 | 100 | MACRO-averaged result (excluding Other): 101 | P = 87.69% R = 89.54% F1 = 88.59% 102 | 103 | 104 | 105 | <<< (9+1)-WAY EVALUATION TAKING DIRECTIONALITY INTO ACCOUNT -- OFFICIAL >>>: 106 | 107 | Confusion matrix: 108 | C-E C-W C-C E-D E-O I-A M-C M-T P-P _O_ <-- classified as 109 | +--------------------------------------------------+ -SUM- xDIRx skip ACTUAL 110 | C-E | 310 0 0 0 2 0 0 1 2 9 | 324 4 0 328 111 | C-W | 0 265 1 0 1 8 7 3 1 20 | 306 6 0 312 112 | C-C | 0 3 176 7 0 0 1 0 1 4 | 192 0 0 192 113 | E-D | 0 1 5 276 0 0 0 0 0 10 | 292 0 0 292 114 | E-O | 8 3 0 2 222 1 1 0 6 14 | 257 1 0 258 115 | I-A | 0 5 0 0 0 123 0 1 2 25 | 156 0 0 156 116 | M-C | 0 3 0 1 1 0 205 0 1 22 | 233 0 0 233 117 | M-T | 0 2 0 0 0 0 1 246 0 12 | 261 0 0 261 118 | P-P | 5 0 0 1 3 2 0 1 206 12 | 230 1 0 231 119 | _O_ | 14 28 14 14 22 16 28 24 21 273 | 454 0 0 454 120 | +--------------------------------------------------+ 121 | -SUM- 337 310 196 301 251 150 243 276 240 401 2705 12 0 2717 122 | 123 | Coverage = 2717/2717 = 100.00% 124 | Accuracy (calculated for the above confusion matrix) = 2302/2717 = 84.73% 125 | Accuracy (considering all skipped examples as Wrong) = 2302/2717 = 84.73% 126 | Accuracy (considering all skipped examples as Other) = 2302/2717 = 84.73% 127 | 128 | Results for the individual relations: 129 | Cause-Effect : P = 310/( 337 + 4) = 90.91% R = 310/ 328 = 94.51% F1 = 92.68% 130 | Component-Whole : P = 265/( 310 + 6) = 83.86% R = 265/ 312 = 84.94% F1 = 84.39% 131 | Content-Container : P = 176/( 196 + 0) = 89.80% R = 176/ 192 = 91.67% F1 = 90.72% 132 | Entity-Destination : P = 276/( 301 + 0) = 91.69% R = 276/ 292 = 94.52% F1 = 93.09% 133 | Entity-Origin : P = 222/( 251 + 1) = 88.10% R = 222/ 258 = 86.05% F1 = 87.06% 134 | Instrument-Agency : P = 123/( 150 + 0) = 82.00% R = 123/ 156 = 78.85% F1 = 80.39% 135 | Member-Collection : P = 205/( 243 + 0) = 84.36% R = 205/ 233 = 87.98% F1 = 86.13% 136 | Message-Topic : P = 246/( 276 + 0) = 89.13% R = 246/ 261 = 94.25% F1 = 91.62% 137 | Product-Producer : P = 206/( 240 + 1) = 85.48% R = 206/ 231 = 89.18% F1 = 87.29% 138 | _Other : P = 273/( 401 + 0) = 68.08% R = 273/ 454 = 60.13% F1 = 63.86% 139 | 140 | Micro-averaged result (excluding Other): 141 | P = 2029/2316 = 87.61% R = 2029/2263 = 89.66% F1 = 88.62% 142 | 143 | MACRO-averaged result (excluding Other): 144 | P = 87.26% R = 89.10% F1 = 88.15% 145 | 146 | 147 | 148 | <<< The official score is (9+1)-way evaluation with directionality taken into account: macro-averaged F1 = 88.15% >>> 149 | -------------------------------------------------------------------------------- /eval/semeval2010_task8_format_checker.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # 3 | # 4 | # Author: Preslav Nakov 5 | # nakov@comp.nus.edu.sg 6 | # National University of Singapore 7 | # 8 | # WHAT: This is an official output file format checker for SemEval-2010 Task #8. 9 | # 10 | # Use: 11 | # semeval2010_task8_format_checker.pl 12 | # 13 | # Examples: 14 | # semeval2010_task8_format_checker.pl proposed_answer1.txt 15 | # semeval2010_task8_format_checker.pl proposed_answer2.txt 16 | # semeval2010_task8_format_checker.pl proposed_answer3.txt 17 | # semeval2010_task8_format_checker.pl proposed_answer4.txt 18 | # 19 | # In the examples above, the first three files are OK, while the last one contains four errors. 20 | # And answer_key2.txt contains the true labels for the *training* dataset. 21 | # 22 | # Description: 23 | # The scorer takes as input a proposed classification file, 24 | # which should contain one prediction per line in the format " " 25 | # with a TAB as a separator, e.g., 26 | # 1 Component-Whole(e2,e1) 27 | # 2 Other 28 | # 3 Instrument-Agency(e2,e1) 29 | # ... 30 | # The file does not have to be sorted in any way. 31 | # Repetitions of IDs are not allowed. 32 | # 33 | # In case of problems, the checker outputs the problemtic line and its number. 34 | # Finally, the total number of problems found is reported 35 | # or a message is output saying that the file format is OK. 36 | # 37 | # Participants are expected to check their output using this checker before submission. 38 | # 39 | # Last modified: March 10, 2010 40 | # 41 | # 42 | 43 | use strict; 44 | 45 | ############### 46 | ### I/O ### 47 | ############### 48 | 49 | if ($#ARGV != 0) { 50 | die "Usage:\nsemeval2010_task8_format_checker.pl \n"; 51 | } 52 | 53 | my $INPUT_FILE_NAME = $ARGV[0]; 54 | 55 | ################ 56 | ### MAIN ### 57 | ################ 58 | my %ids = (); 59 | 60 | my $errCnt = 0; 61 | open(INPUT, $INPUT_FILE_NAME) or die "Failed to open $INPUT_FILE_NAME for text reading.\n"; 62 | for (my $lineNo = 1; ; $lineNo++) { 63 | my ($id, $label) = &getIDandLabel($_); 64 | if ($id < 0) { 65 | s/[\n\r]*$//; 66 | print "Bad file format on line $lineNo: '$_'\n"; 67 | $errCnt++; 68 | } 69 | elsif (defined $ids{$id}) { 70 | s/[\n\r]*$//; 71 | print "Bad file format on line $lineNo (ID $id is already defined): '$_'\n"; 72 | $errCnt++; 73 | } 74 | $ids{$id}++; 75 | } 76 | close(INPUT) or die "Failed to close $INPUT_FILE_NAME.\n"; 77 | 78 | if (0 == $errCnt) { 79 | print "\n<<< The file format is OK.\n"; 80 | } 81 | else { 82 | print "\n<<< The format is INCORRECT: $errCnt problematic line(s) found!\n"; 83 | } 84 | 85 | 86 | ################ 87 | ### SUBS ### 88 | ################ 89 | 90 | sub getIDandLabel() { 91 | my $line = shift; 92 | 93 | return (-1,()) if ($line !~ /^([0-9]+)\t([^\r]+)\r?\n$/); 94 | my ($id, $label) = ($1, $2); 95 | 96 | return ($id, '_Other') if ($label eq 'Other'); 97 | 98 | return ($id, $label) 99 | if (($label eq 'Cause-Effect(e1,e2)') || ($label eq 'Cause-Effect(e2,e1)') || 100 | ($label eq 'Component-Whole(e1,e2)') || ($label eq 'Component-Whole(e2,e1)') || 101 | ($label eq 'Content-Container(e1,e2)') || ($label eq 'Content-Container(e2,e1)') || 102 | ($label eq 'Entity-Destination(e1,e2)') || ($label eq 'Entity-Destination(e2,e1)') || 103 | ($label eq 'Entity-Origin(e1,e2)') || ($label eq 'Entity-Origin(e2,e1)') || 104 | ($label eq 'Instrument-Agency(e1,e2)') || ($label eq 'Instrument-Agency(e2,e1)') || 105 | ($label eq 'Member-Collection(e1,e2)') || ($label eq 'Member-Collection(e2,e1)') || 106 | ($label eq 'Message-Topic(e1,e2)') || ($label eq 'Message-Topic(e2,e1)') || 107 | ($label eq 'Product-Producer(e1,e2)') || ($label eq 'Product-Producer(e2,e1)')); 108 | 109 | return (-1, ()); 110 | } 111 | -------------------------------------------------------------------------------- /eval/semeval2010_task8_scorer-v1.2.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | # 3 | # 4 | # Author: Preslav Nakov 5 | # nakov@comp.nus.edu.sg 6 | # National University of Singapore 7 | # 8 | # WHAT: This is the official scorer for SemEval-2010 Task #8. 9 | # 10 | # 11 | # Last modified: March 22, 2010 12 | # 13 | # Current version: 1.2 14 | # 15 | # Revision history: 16 | # - Version 1.2 (fixed a bug in the precision for the scoring of (iii)) 17 | # - Version 1.1 (fixed a bug in the calculation of accuracy) 18 | # 19 | # 20 | # Use: 21 | # semeval2010_task8_scorer-v1.1.pl 22 | # 23 | # Example2: 24 | # semeval2010_task8_scorer-v1.1.pl proposed_answer1.txt answer_key1.txt > result_scores1.txt 25 | # semeval2010_task8_scorer-v1.1.pl proposed_answer2.txt answer_key2.txt > result_scores2.txt 26 | # semeval2010_task8_scorer-v1.1.pl proposed_answer3.txt answer_key3.txt > result_scores3.txt 27 | # 28 | # Description: 29 | # The scorer takes as input a proposed classification file and an answer key file. 30 | # Both files should contain one prediction per line in the format " " 31 | # with a TAB as a separator, e.g., 32 | # 1 Component-Whole(e2,e1) 33 | # 2 Other 34 | # 3 Instrument-Agency(e2,e1) 35 | # ... 36 | # The files do not have to be sorted in any way and the first file can have predictions 37 | # for a subset of the IDs in the second file only, e.g., because hard examples have been skipped. 38 | # Repetitions of IDs are not allowed in either of the files. 39 | # 40 | # The scorer calculates and outputs the following statistics: 41 | # (1) confusion matrix, which shows 42 | # - the sums for each row/column: -SUM- 43 | # - the number of skipped examples: skip 44 | # - the number of examples with correct relation, but wrong directionality: xDIRx 45 | # - the number of examples in the answer key file: ACTUAL ( = -SUM- + skip + xDIRx ) 46 | # (2) accuracy and coverage 47 | # (3) precision (P), recall (R), and F1-score for each relation 48 | # (4) micro-averaged P, R, F1, where the calculations ignore the Other category. 49 | # (5) macro-averaged P, R, F1, where the calculations ignore the Other category. 50 | # 51 | # Note that in scores (4) and (5), skipped examples are equivalent to those classified as Other. 52 | # So are examples classified as relations that do not exist in the key file (which is probably not optimal). 53 | # 54 | # The scoring is done three times: 55 | # (i) as a (2*9+1)-way classification 56 | # (ii) as a (9+1)-way classification, with directionality ignored 57 | # (iii) as a (9+1)-way classification, with directionality taken into account. 58 | # 59 | # The official score is the macro-averaged F1-score for (iii). 60 | # 61 | 62 | use strict; 63 | 64 | 65 | ############### 66 | ### I/O ### 67 | ############### 68 | 69 | if ($#ARGV != 1) { 70 | die "Usage:\nsemeval2010_task8_scorer.pl \n"; 71 | } 72 | 73 | my $PROPOSED_ANSWERS_FILE_NAME = $ARGV[0]; 74 | my $ANSWER_KEYS_FILE_NAME = $ARGV[1]; 75 | 76 | 77 | ################ 78 | ### MAIN ### 79 | ################ 80 | 81 | my (%confMatrix19way, %confMatrix10wayNoDir, %confMatrix10wayWithDir) = (); 82 | my (%idsProposed, %idsAnswer) = (); 83 | my (%allLabels19waylAnswer, %allLabels10wayAnswer) = (); 84 | my (%allLabels19wayProposed, %allLabels10wayNoDirProposed, %allLabels10wayWithDirProposed) = (); 85 | 86 | ### 1. Read the file contents 87 | my $totalProposed = &readFileIntoHash($PROPOSED_ANSWERS_FILE_NAME, \%idsProposed); 88 | my $totalAnswer = &readFileIntoHash($ANSWER_KEYS_FILE_NAME, \%idsAnswer); 89 | 90 | ### 2. Calculate the confusion matrices 91 | foreach my $id (keys %idsProposed) { 92 | 93 | ### 2.1. Unexpected IDs are not allowed 94 | die "File $PROPOSED_ANSWERS_FILE_NAME contains a bad ID: '$id'" 95 | if (!defined($idsAnswer{$id})); 96 | 97 | ### 2.2. Update the 19-way confusion matrix 98 | my $labelProposed = $idsProposed{$id}; 99 | my $labelAnswer = $idsAnswer{$id}; 100 | $confMatrix19way{$labelProposed}{$labelAnswer}++; 101 | $allLabels19wayProposed{$labelProposed}++; 102 | 103 | ### 2.3. Update the 10-way confusion matrix *without* direction 104 | my $labelProposedNoDir = $labelProposed; 105 | my $labelAnswerNoDir = $labelAnswer; 106 | $labelProposedNoDir =~ s/\(e[12],e[12]\)[\n\r]*$//; 107 | $labelAnswerNoDir =~ s/\(e[12],e[12]\)[\n\r]*$//; 108 | $confMatrix10wayNoDir{$labelProposedNoDir}{$labelAnswerNoDir}++; 109 | $allLabels10wayNoDirProposed{$labelProposedNoDir}++; 110 | 111 | ### 2.4. Update the 10-way confusion matrix *with* direction 112 | if ($labelProposed eq $labelAnswer) { ## both relation and direction match 113 | $confMatrix10wayWithDir{$labelProposedNoDir}{$labelAnswerNoDir}++; 114 | $allLabels10wayWithDirProposed{$labelProposedNoDir}++; 115 | } 116 | elsif ($labelProposedNoDir eq $labelAnswerNoDir) { ## the relations match, but the direction is wrong 117 | $confMatrix10wayWithDir{'WRONG_DIR'}{$labelAnswerNoDir}++; 118 | $allLabels10wayWithDirProposed{'WRONG_DIR'}++; 119 | } 120 | else { ### Wrong relation 121 | $confMatrix10wayWithDir{$labelProposedNoDir}{$labelAnswerNoDir}++; 122 | $allLabels10wayWithDirProposed{$labelProposedNoDir}++; 123 | } 124 | } 125 | 126 | ### 3. Calculate the ground truth distributions 127 | foreach my $id (keys %idsAnswer) { 128 | 129 | ### 3.1. Update the 19-way answer distribution 130 | my $labelAnswer = $idsAnswer{$id}; 131 | $allLabels19waylAnswer{$labelAnswer}++; 132 | 133 | ### 3.2. Update the 10-way answer distribution 134 | my $labelAnswerNoDir = $labelAnswer; 135 | $labelAnswerNoDir =~ s/\(e[12],e[12]\)[\n\r]*$//; 136 | $allLabels10wayAnswer{$labelAnswerNoDir}++; 137 | } 138 | 139 | ### 4. Check for proposed classes that are not contained in the answer key file: this may happen in cross-validation 140 | foreach my $labelProposed (sort keys %allLabels19wayProposed) { 141 | if (!defined($allLabels19waylAnswer{$labelProposed})) { 142 | print "!!!WARNING!!! The proposed file contains $allLabels19wayProposed{$labelProposed} label(s) of type '$labelProposed', which is NOT present in the key file.\n\n"; 143 | } 144 | } 145 | 146 | ### 4. 19-way evaluation with directionality 147 | print "<<< (2*9+1)-WAY EVALUATION (USING DIRECTIONALITY)>>>:\n\n"; 148 | &evaluate(\%confMatrix19way, \%allLabels19wayProposed, \%allLabels19waylAnswer, $totalProposed, $totalAnswer, 0); 149 | 150 | ### 5. Evaluate without directionality 151 | print "<<< (9+1)-WAY EVALUATION IGNORING DIRECTIONALITY >>>:\n\n"; 152 | &evaluate(\%confMatrix10wayNoDir, \%allLabels10wayNoDirProposed, \%allLabels10wayAnswer, $totalProposed, $totalAnswer, 0); 153 | 154 | ### 6. Evaluate without directionality 155 | print "<<< (9+1)-WAY EVALUATION TAKING DIRECTIONALITY INTO ACCOUNT -- OFFICIAL >>>:\n\n"; 156 | my $officialScore = &evaluate(\%confMatrix10wayWithDir, \%allLabels10wayWithDirProposed, \%allLabels10wayAnswer, $totalProposed, $totalAnswer, 1); 157 | 158 | ### 7. Output the official score 159 | printf "<<< The official score is (9+1)-way evaluation with directionality taken into account: macro-averaged F1 = %0.2f%s >>>\n", $officialScore, '%'; 160 | 161 | 162 | ################ 163 | ### SUBS ### 164 | ################ 165 | 166 | sub getIDandLabel() { 167 | my $line = shift; 168 | return (-1,()) if ($line !~ /^([0-9]+)\t([^\r]+)\r?\n$/); 169 | 170 | my ($id, $label) = ($1, $2); 171 | 172 | return ($id, '_Other') if ($label eq 'Other'); 173 | 174 | return ($id, $label) 175 | if (($label eq 'Cause-Effect(e1,e2)') || ($label eq 'Cause-Effect(e2,e1)') || 176 | ($label eq 'Component-Whole(e1,e2)') || ($label eq 'Component-Whole(e2,e1)') || 177 | ($label eq 'Content-Container(e1,e2)') || ($label eq 'Content-Container(e2,e1)') || 178 | ($label eq 'Entity-Destination(e1,e2)') || ($label eq 'Entity-Destination(e2,e1)') || 179 | ($label eq 'Entity-Origin(e1,e2)') || ($label eq 'Entity-Origin(e2,e1)') || 180 | ($label eq 'Instrument-Agency(e1,e2)') || ($label eq 'Instrument-Agency(e2,e1)') || 181 | ($label eq 'Member-Collection(e1,e2)') || ($label eq 'Member-Collection(e2,e1)') || 182 | ($label eq 'Message-Topic(e1,e2)') || ($label eq 'Message-Topic(e2,e1)') || 183 | ($label eq 'Product-Producer(e1,e2)') || ($label eq 'Product-Producer(e2,e1)')); 184 | 185 | return (-1, ()); 186 | } 187 | 188 | 189 | sub readFileIntoHash() { 190 | my ($fname, $ids) = @_; 191 | open(INPUT, $fname) or die "Failed to open $fname for text reading.\n"; 192 | my $lineNo = 0; 193 | while () { 194 | $lineNo++; 195 | my ($id, $label) = &getIDandLabel($_); 196 | die "Bad file format on line $lineNo: '$_'\n" if ($id < 0); 197 | if (defined $$ids{$id}) { 198 | s/[\n\r]*$//; 199 | die "Bad file format on line $lineNo (ID $id is already defined): '$_'\n"; 200 | } 201 | $$ids{$id} = $label; 202 | } 203 | close(INPUT) or die "Failed to close $fname.\n"; 204 | return $lineNo; 205 | } 206 | 207 | 208 | sub evaluate() { 209 | my ($confMatrix, $allLabelsProposed, $allLabelsAnswer, $totalProposed, $totalAnswer, $useWrongDir) = @_; 210 | 211 | ### 0. Create a merged list for the confusion matrix 212 | my @allLabels = (); 213 | &mergeLabelLists($allLabelsAnswer, $allLabelsProposed, \@allLabels); 214 | 215 | ### 1. Print the confusion matrix heading 216 | print "Confusion matrix:\n"; 217 | print " "; 218 | foreach my $label (@allLabels) { 219 | printf " %4s", &getShortRelName($label, $allLabelsAnswer); 220 | } 221 | print " <-- classified as\n"; 222 | print " +"; 223 | foreach my $label (@allLabels) { 224 | print "-----"; 225 | } 226 | if ($useWrongDir) { 227 | print "+ -SUM- xDIRx skip ACTUAL\n"; 228 | } 229 | else { 230 | print "+ -SUM- skip ACTUAL\n"; 231 | } 232 | 233 | ### 2. Print the rest of the confusion matrix 234 | my $freqCorrect = 0; 235 | my $ind = 1; 236 | my $otherSkipped = 0; 237 | foreach my $labelAnswer (sort keys %{$allLabelsAnswer}) { 238 | 239 | ### 2.1. Output the short relation label 240 | printf " %4s |", &getShortRelName($labelAnswer, $allLabelsAnswer); 241 | 242 | ### 2.2. Output a row of the confusion matrix 243 | my $sumProposed = 0; 244 | foreach my $labelProposed (@allLabels) { 245 | $$confMatrix{$labelProposed}{$labelAnswer} = 0 246 | if (!defined($$confMatrix{$labelProposed}{$labelAnswer})); 247 | printf "%4d ", $$confMatrix{$labelProposed}{$labelAnswer}; 248 | $sumProposed += $$confMatrix{$labelProposed}{$labelAnswer}; 249 | } 250 | 251 | ### 2.3. Output the horizontal sums 252 | if ($useWrongDir) { 253 | my $ans = defined($$allLabelsAnswer{$labelAnswer}) ? $$allLabelsAnswer{$labelAnswer} : 0; 254 | $$confMatrix{'WRONG_DIR'}{$labelAnswer} = 0 if (!defined $$confMatrix{'WRONG_DIR'}{$labelAnswer}); 255 | printf "| %4d %4d %4d %6d\n", $sumProposed, $$confMatrix{'WRONG_DIR'}{$labelAnswer}, $ans - $sumProposed - $$confMatrix{'WRONG_DIR'}{$labelAnswer}, $ans; 256 | if ($labelAnswer eq '_Other') { 257 | $otherSkipped = $ans - $sumProposed - $$confMatrix{'WRONG_DIR'}{$labelAnswer}; 258 | } 259 | } 260 | else { 261 | my $ans = defined($$allLabelsAnswer{$labelAnswer}) ? $$allLabelsAnswer{$labelAnswer} : 0; 262 | printf "| %4d %4d %4d\n", $sumProposed, $ans - $sumProposed, $ans; 263 | if ($labelAnswer eq '_Other') { 264 | $otherSkipped = $ans - $sumProposed; 265 | } 266 | } 267 | 268 | $ind++; 269 | 270 | $$confMatrix{$labelAnswer}{$labelAnswer} = 0 271 | if (!defined($$confMatrix{$labelAnswer}{$labelAnswer})); 272 | $freqCorrect += $$confMatrix{$labelAnswer}{$labelAnswer}; 273 | } 274 | print " +"; 275 | foreach (@allLabels) { 276 | print "-----"; 277 | } 278 | print "+\n"; 279 | 280 | ### 3. Print the vertical sums 281 | print " -SUM- "; 282 | foreach my $labelProposed (@allLabels) { 283 | $$allLabelsProposed{$labelProposed} = 0 284 | if (!defined $$allLabelsProposed{$labelProposed}); 285 | printf "%4d ", $$allLabelsProposed{$labelProposed}; 286 | } 287 | if ($useWrongDir) { 288 | printf " %4d %4d %4d %6d\n\n", $totalProposed - $$allLabelsProposed{'WRONG_DIR'}, $$allLabelsProposed{'WRONG_DIR'}, $totalAnswer - $totalProposed, $totalAnswer; 289 | } 290 | else { 291 | printf " %4d %4d %4d\n\n", $totalProposed, $totalAnswer - $totalProposed, $totalAnswer; 292 | } 293 | 294 | ### 4. Output the coverage 295 | my $coverage = 100.0 * $totalProposed / $totalAnswer; 296 | printf "%s%d%s%d%s%5.2f%s", 'Coverage = ', $totalProposed, '/', $totalAnswer, ' = ', $coverage, "\%\n"; 297 | 298 | ### 5. Output the accuracy 299 | my $accuracy = 100.0 * $freqCorrect / $totalProposed; 300 | printf "%s%d%s%d%s%5.2f%s", 'Accuracy (calculated for the above confusion matrix) = ', $freqCorrect, '/', $totalProposed, ' = ', $accuracy, "\%\n"; 301 | 302 | ### 6. Output the accuracy considering all skipped to be wrong 303 | $accuracy = 100.0 * $freqCorrect / $totalAnswer; 304 | printf "%s%d%s%d%s%5.2f%s", 'Accuracy (considering all skipped examples as Wrong) = ', $freqCorrect, '/', $totalAnswer, ' = ', $accuracy, "\%\n"; 305 | 306 | ### 7. Calculate accuracy with all skipped examples considered Other 307 | my $accuracyWithOther = 100.0 * ($freqCorrect + $otherSkipped) / $totalAnswer; 308 | printf "%s%d%s%d%s%5.2f%s", 'Accuracy (considering all skipped examples as Other) = ', ($freqCorrect + $otherSkipped), '/', $totalAnswer, ' = ', $accuracyWithOther, "\%\n"; 309 | 310 | ### 8. Output P, R, F1 for each relation 311 | my ($macroP, $macroR, $macroF1) = (0, 0, 0); 312 | my ($microCorrect, $microProposed, $microAnswer) = (0, 0, 0); 313 | print "\nResults for the individual relations:\n"; 314 | foreach my $labelAnswer (sort keys %{$allLabelsAnswer}) { 315 | 316 | ### 8.1. Consider all wrong directionalities as wrong classification decisions 317 | my $wrongDirectionCnt = 0; 318 | if ($useWrongDir && defined $$confMatrix{'WRONG_DIR'}{$labelAnswer}) { 319 | $wrongDirectionCnt = $$confMatrix{'WRONG_DIR'}{$labelAnswer}; 320 | } 321 | 322 | ### 8.2. Prevent Perl complains about unintialized values 323 | if (!defined($$allLabelsProposed{$labelAnswer})) { 324 | $$allLabelsProposed{$labelAnswer} = 0; 325 | } 326 | 327 | ### 8.3. Calculate P/R/F1 328 | my $P = (0 == $$allLabelsProposed{$labelAnswer}) ? 0 329 | : 100.0 * $$confMatrix{$labelAnswer}{$labelAnswer} / ($$allLabelsProposed{$labelAnswer} + $wrongDirectionCnt); 330 | my $R = (0 == $$allLabelsAnswer{$labelAnswer}) ? 0 331 | : 100.0 * $$confMatrix{$labelAnswer}{$labelAnswer} / $$allLabelsAnswer{$labelAnswer}; 332 | my $F1 = (0 == $P + $R) ? 0 : 2 * $P * $R / ($P + $R); 333 | 334 | ### 8.4. Output P/R/F1 335 | if ($useWrongDir) { 336 | printf "%25s%s%4d%s(%4d +%4d)%s%6.2f", $labelAnswer, 337 | " : P = ", $$confMatrix{$labelAnswer}{$labelAnswer}, '/', $$allLabelsProposed{$labelAnswer}, $wrongDirectionCnt, ' = ', $P; 338 | } 339 | else { 340 | printf "%25s%s%4d%s%4d%s%6.2f", $labelAnswer, 341 | " : P = ", $$confMatrix{$labelAnswer}{$labelAnswer}, '/', ($$allLabelsProposed{$labelAnswer} + $wrongDirectionCnt), ' = ', $P; 342 | } 343 | printf"%s%4d%s%4d%s%6.2f%s%6.2f%s\n", 344 | "% R = ", $$confMatrix{$labelAnswer}{$labelAnswer}, '/', $$allLabelsAnswer{$labelAnswer}, ' = ', $R, 345 | "% F1 = ", $F1, '%'; 346 | 347 | ### 8.5. Accumulate statistics for micro/macro-averaging 348 | if ($labelAnswer ne '_Other') { 349 | $macroP += $P; 350 | $macroR += $R; 351 | $macroF1 += $F1; 352 | $microCorrect += $$confMatrix{$labelAnswer}{$labelAnswer}; 353 | $microProposed += $$allLabelsProposed{$labelAnswer} + $wrongDirectionCnt; 354 | $microAnswer += $$allLabelsAnswer{$labelAnswer}; 355 | } 356 | } 357 | 358 | ### 9. Output the micro-averaged P, R, F1 359 | my $microP = (0 == $microProposed) ? 0 : 100.0 * $microCorrect / $microProposed; 360 | my $microR = (0 == $microAnswer) ? 0 : 100.0 * $microCorrect / $microAnswer; 361 | my $microF1 = (0 == $microP + $microR) ? 0 : 2.0 * $microP * $microR / ($microP + $microR); 362 | print "\nMicro-averaged result (excluding Other):\n"; 363 | printf "%s%4d%s%4d%s%6.2f%s%4d%s%4d%s%6.2f%s%6.2f%s\n", 364 | "P = ", $microCorrect, '/', $microProposed, ' = ', $microP, 365 | "% R = ", $microCorrect, '/', $microAnswer, ' = ', $microR, 366 | "% F1 = ", $microF1, '%'; 367 | 368 | ### 10. Output the macro-averaged P, R, F1 369 | my $distinctLabelsCnt = keys %{$allLabelsAnswer}; 370 | ## -1, if '_Other' exists 371 | $distinctLabelsCnt-- if (defined $$allLabelsAnswer{'_Other'}); 372 | 373 | $macroP /= $distinctLabelsCnt; # first divide by the number of non-Other categories 374 | $macroR /= $distinctLabelsCnt; 375 | $macroF1 /= $distinctLabelsCnt; 376 | print "\nMACRO-averaged result (excluding Other):\n"; 377 | printf "%s%6.2f%s%6.2f%s%6.2f%s\n\n\n\n", "P = ", $macroP, "%\tR = ", $macroR, "%\tF1 = ", $macroF1, '%'; 378 | 379 | ### 11. Return the official score 380 | return $macroF1; 381 | } 382 | 383 | 384 | sub getShortRelName() { 385 | my ($relName, $hashToCheck) = @_; 386 | return '_O_' if ($relName eq '_Other'); 387 | die "relName='$relName'" if ($relName !~ /^(.)[^\-]+\-(.)/); 388 | my $result = (defined $$hashToCheck{$relName}) ? "$1\-$2" : "*$1$2"; 389 | if ($relName =~ /\(e([12])/) { 390 | $result .= $1; 391 | } 392 | return $result; 393 | } 394 | 395 | sub mergeLabelLists() { 396 | my ($hash1, $hash2, $mergedList) = @_; 397 | foreach my $key (sort keys %{$hash1}) { 398 | push @{$mergedList}, $key if ($key ne 'WRONG_DIR'); 399 | } 400 | foreach my $key (sort keys %{$hash2}) { 401 | push @{$mergedList}, $key if (($key ne 'WRONG_DIR') && !defined($$hash1{$key})); 402 | } 403 | } 404 | -------------------------------------------------------------------------------- /eval/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | perl ./semeval2010_task8_scorer-v1.2.pl sem_res.txt test_keys.txt > res.txt 4 | echo "the results save in res.txt" 5 | -------------------------------------------------------------------------------- /eval/test_keys.txt: -------------------------------------------------------------------------------- 1 | 8001 Message-Topic(e1,e2) 2 | 8002 Product-Producer(e2,e1) 3 | 8003 Instrument-Agency(e2,e1) 4 | 8004 Entity-Destination(e1,e2) 5 | 8005 Cause-Effect(e2,e1) 6 | 8006 Component-Whole(e1,e2) 7 | 8007 Product-Producer(e1,e2) 8 | 8008 Member-Collection(e2,e1) 9 | 8009 Component-Whole(e1,e2) 10 | 8010 Message-Topic(e1,e2) 11 | 8011 Entity-Destination(e1,e2) 12 | 8012 Other 13 | 8013 Entity-Destination(e1,e2) 14 | 8014 Product-Producer(e1,e2) 15 | 8015 Entity-Origin(e1,e2) 16 | 8016 Entity-Origin(e1,e2) 17 | 8017 Entity-Destination(e1,e2) 18 | 8018 Other 19 | 8019 Member-Collection(e2,e1) 20 | 8020 Product-Producer(e1,e2) 21 | 8021 Message-Topic(e1,e2) 22 | 8022 Content-Container(e1,e2) 23 | 8023 Product-Producer(e1,e2) 24 | 8024 Other 25 | 8025 Entity-Origin(e2,e1) 26 | 8026 Product-Producer(e1,e2) 27 | 8027 Cause-Effect(e2,e1) 28 | 8028 Other 29 | 8029 Other 30 | 8030 Entity-Origin(e1,e2) 31 | 8031 Cause-Effect(e1,e2) 32 | 8032 Message-Topic(e1,e2) 33 | 8033 Component-Whole(e1,e2) 34 | 8034 Product-Producer(e1,e2) 35 | 8035 Component-Whole(e1,e2) 36 | 8036 Component-Whole(e2,e1) 37 | 8037 Member-Collection(e2,e1) 38 | 8038 Content-Container(e2,e1) 39 | 8039 Member-Collection(e2,e1) 40 | 8040 Product-Producer(e1,e2) 41 | 8041 Cause-Effect(e1,e2) 42 | 8042 Component-Whole(e2,e1) 43 | 8043 Cause-Effect(e1,e2) 44 | 8044 Entity-Destination(e1,e2) 45 | 8045 Entity-Origin(e1,e2) 46 | 8046 Content-Container(e1,e2) 47 | 8047 Other 48 | 8048 Entity-Destination(e1,e2) 49 | 8049 Message-Topic(e1,e2) 50 | 8050 Other 51 | 8051 Entity-Destination(e1,e2) 52 | 8052 Other 53 | 8053 Member-Collection(e2,e1) 54 | 8054 Other 55 | 8055 Cause-Effect(e1,e2) 56 | 8056 Entity-Origin(e1,e2) 57 | 8057 Other 58 | 8058 Cause-Effect(e1,e2) 59 | 8059 Other 60 | 8060 Component-Whole(e2,e1) 61 | 8061 Entity-Origin(e2,e1) 62 | 8062 Product-Producer(e1,e2) 63 | 8063 Instrument-Agency(e2,e1) 64 | 8064 Component-Whole(e1,e2) 65 | 8065 Entity-Destination(e1,e2) 66 | 8066 Product-Producer(e2,e1) 67 | 8067 Other 68 | 8068 Other 69 | 8069 Message-Topic(e1,e2) 70 | 8070 Product-Producer(e1,e2) 71 | 8071 Other 72 | 8072 Entity-Origin(e1,e2) 73 | 8073 Cause-Effect(e2,e1) 74 | 8074 Entity-Origin(e1,e2) 75 | 8075 Other 76 | 8076 Product-Producer(e1,e2) 77 | 8077 Other 78 | 8078 Instrument-Agency(e2,e1) 79 | 8079 Entity-Destination(e1,e2) 80 | 8080 Product-Producer(e2,e1) 81 | 8081 Component-Whole(e1,e2) 82 | 8082 Component-Whole(e1,e2) 83 | 8083 Cause-Effect(e1,e2) 84 | 8084 Component-Whole(e1,e2) 85 | 8085 Message-Topic(e1,e2) 86 | 8086 Instrument-Agency(e2,e1) 87 | 8087 Message-Topic(e1,e2) 88 | 8088 Product-Producer(e2,e1) 89 | 8089 Entity-Origin(e2,e1) 90 | 8090 Message-Topic(e1,e2) 91 | 8091 Entity-Origin(e1,e2) 92 | 8092 Other 93 | 8093 Component-Whole(e1,e2) 94 | 8094 Component-Whole(e1,e2) 95 | 8095 Other 96 | 8096 Entity-Destination(e1,e2) 97 | 8097 Message-Topic(e1,e2) 98 | 8098 Component-Whole(e1,e2) 99 | 8099 Entity-Destination(e1,e2) 100 | 8100 Message-Topic(e1,e2) 101 | 8101 Message-Topic(e1,e2) 102 | 8102 Component-Whole(e2,e1) 103 | 8103 Entity-Origin(e1,e2) 104 | 8104 Message-Topic(e1,e2) 105 | 8105 Cause-Effect(e2,e1) 106 | 8106 Other 107 | 8107 Cause-Effect(e2,e1) 108 | 8108 Cause-Effect(e1,e2) 109 | 8109 Component-Whole(e2,e1) 110 | 8110 Member-Collection(e2,e1) 111 | 8111 Other 112 | 8112 Content-Container(e1,e2) 113 | 8113 Other 114 | 8114 Product-Producer(e2,e1) 115 | 8115 Other 116 | 8116 Cause-Effect(e2,e1) 117 | 8117 Product-Producer(e1,e2) 118 | 8118 Cause-Effect(e1,e2) 119 | 8119 Member-Collection(e2,e1) 120 | 8120 Component-Whole(e2,e1) 121 | 8121 Entity-Destination(e1,e2) 122 | 8122 Instrument-Agency(e2,e1) 123 | 8123 Other 124 | 8124 Other 125 | 8125 Message-Topic(e1,e2) 126 | 8126 Entity-Origin(e2,e1) 127 | 8127 Entity-Origin(e2,e1) 128 | 8128 Other 129 | 8129 Component-Whole(e2,e1) 130 | 8130 Content-Container(e1,e2) 131 | 8131 Instrument-Agency(e1,e2) 132 | 8132 Message-Topic(e1,e2) 133 | 8133 Component-Whole(e1,e2) 134 | 8134 Other 135 | 8135 Content-Container(e1,e2) 136 | 8136 Instrument-Agency(e2,e1) 137 | 8137 Component-Whole(e1,e2) 138 | 8138 Member-Collection(e2,e1) 139 | 8139 Entity-Origin(e1,e2) 140 | 8140 Member-Collection(e2,e1) 141 | 8141 Instrument-Agency(e2,e1) 142 | 8142 Entity-Origin(e1,e2) 143 | 8143 Other 144 | 8144 Entity-Origin(e1,e2) 145 | 8145 Member-Collection(e2,e1) 146 | 8146 Instrument-Agency(e2,e1) 147 | 8147 Content-Container(e1,e2) 148 | 8148 Message-Topic(e2,e1) 149 | 8149 Other 150 | 8150 Product-Producer(e2,e1) 151 | 8151 Product-Producer(e1,e2) 152 | 8152 Member-Collection(e2,e1) 153 | 8153 Member-Collection(e2,e1) 154 | 8154 Message-Topic(e1,e2) 155 | 8155 Message-Topic(e1,e2) 156 | 8156 Product-Producer(e2,e1) 157 | 8157 Other 158 | 8158 Component-Whole(e1,e2) 159 | 8159 Cause-Effect(e1,e2) 160 | 8160 Message-Topic(e2,e1) 161 | 8161 Message-Topic(e1,e2) 162 | 8162 Entity-Origin(e1,e2) 163 | 8163 Entity-Origin(e1,e2) 164 | 8164 Product-Producer(e2,e1) 165 | 8165 Entity-Destination(e1,e2) 166 | 8166 Content-Container(e1,e2) 167 | 8167 Member-Collection(e2,e1) 168 | 8168 Component-Whole(e2,e1) 169 | 8169 Entity-Origin(e1,e2) 170 | 8170 Instrument-Agency(e2,e1) 171 | 8171 Entity-Destination(e1,e2) 172 | 8172 Member-Collection(e1,e2) 173 | 8173 Other 174 | 8174 Other 175 | 8175 Cause-Effect(e2,e1) 176 | 8176 Product-Producer(e1,e2) 177 | 8177 Entity-Destination(e1,e2) 178 | 8178 Entity-Origin(e1,e2) 179 | 8179 Instrument-Agency(e2,e1) 180 | 8180 Message-Topic(e1,e2) 181 | 8181 Entity-Destination(e1,e2) 182 | 8182 Content-Container(e1,e2) 183 | 8183 Other 184 | 8184 Product-Producer(e2,e1) 185 | 8185 Other 186 | 8186 Member-Collection(e2,e1) 187 | 8187 Entity-Destination(e1,e2) 188 | 8188 Product-Producer(e1,e2) 189 | 8189 Message-Topic(e2,e1) 190 | 8190 Instrument-Agency(e2,e1) 191 | 8191 Cause-Effect(e1,e2) 192 | 8192 Other 193 | 8193 Message-Topic(e1,e2) 194 | 8194 Component-Whole(e2,e1) 195 | 8195 Message-Topic(e2,e1) 196 | 8196 Other 197 | 8197 Entity-Origin(e2,e1) 198 | 8198 Entity-Destination(e1,e2) 199 | 8199 Entity-Destination(e1,e2) 200 | 8200 Product-Producer(e1,e2) 201 | 8201 Component-Whole(e1,e2) 202 | 8202 Content-Container(e1,e2) 203 | 8203 Other 204 | 8204 Cause-Effect(e2,e1) 205 | 8205 Entity-Destination(e1,e2) 206 | 8206 Component-Whole(e1,e2) 207 | 8207 Component-Whole(e2,e1) 208 | 8208 Content-Container(e2,e1) 209 | 8209 Member-Collection(e2,e1) 210 | 8210 Member-Collection(e2,e1) 211 | 8211 Component-Whole(e1,e2) 212 | 8212 Entity-Origin(e1,e2) 213 | 8213 Content-Container(e1,e2) 214 | 8214 Instrument-Agency(e2,e1) 215 | 8215 Entity-Origin(e2,e1) 216 | 8216 Content-Container(e2,e1) 217 | 8217 Content-Container(e1,e2) 218 | 8218 Other 219 | 8219 Cause-Effect(e2,e1) 220 | 8220 Message-Topic(e1,e2) 221 | 8221 Content-Container(e1,e2) 222 | 8222 Entity-Origin(e1,e2) 223 | 8223 Message-Topic(e1,e2) 224 | 8224 Message-Topic(e2,e1) 225 | 8225 Other 226 | 8226 Other 227 | 8227 Content-Container(e1,e2) 228 | 8228 Member-Collection(e2,e1) 229 | 8229 Product-Producer(e1,e2) 230 | 8230 Other 231 | 8231 Entity-Origin(e1,e2) 232 | 8232 Component-Whole(e2,e1) 233 | 8233 Message-Topic(e1,e2) 234 | 8234 Cause-Effect(e2,e1) 235 | 8235 Component-Whole(e1,e2) 236 | 8236 Cause-Effect(e2,e1) 237 | 8237 Other 238 | 8238 Component-Whole(e1,e2) 239 | 8239 Cause-Effect(e1,e2) 240 | 8240 Cause-Effect(e1,e2) 241 | 8241 Product-Producer(e1,e2) 242 | 8242 Entity-Destination(e1,e2) 243 | 8243 Component-Whole(e1,e2) 244 | 8244 Other 245 | 8245 Other 246 | 8246 Product-Producer(e2,e1) 247 | 8247 Content-Container(e1,e2) 248 | 8248 Component-Whole(e1,e2) 249 | 8249 Entity-Origin(e1,e2) 250 | 8250 Entity-Destination(e1,e2) 251 | 8251 Component-Whole(e1,e2) 252 | 8252 Entity-Origin(e1,e2) 253 | 8253 Cause-Effect(e1,e2) 254 | 8254 Component-Whole(e1,e2) 255 | 8255 Other 256 | 8256 Other 257 | 8257 Cause-Effect(e2,e1) 258 | 8258 Product-Producer(e1,e2) 259 | 8259 Component-Whole(e2,e1) 260 | 8260 Instrument-Agency(e2,e1) 261 | 8261 Message-Topic(e1,e2) 262 | 8262 Entity-Destination(e1,e2) 263 | 8263 Entity-Origin(e2,e1) 264 | 8264 Message-Topic(e2,e1) 265 | 8265 Cause-Effect(e2,e1) 266 | 8266 Entity-Destination(e1,e2) 267 | 8267 Message-Topic(e1,e2) 268 | 8268 Component-Whole(e2,e1) 269 | 8269 Other 270 | 8270 Entity-Destination(e1,e2) 271 | 8271 Other 272 | 8272 Other 273 | 8273 Message-Topic(e2,e1) 274 | 8274 Member-Collection(e2,e1) 275 | 8275 Other 276 | 8276 Entity-Destination(e1,e2) 277 | 8277 Message-Topic(e1,e2) 278 | 8278 Instrument-Agency(e2,e1) 279 | 8279 Product-Producer(e2,e1) 280 | 8280 Product-Producer(e1,e2) 281 | 8281 Member-Collection(e1,e2) 282 | 8282 Entity-Destination(e1,e2) 283 | 8283 Member-Collection(e2,e1) 284 | 8284 Other 285 | 8285 Message-Topic(e1,e2) 286 | 8286 Content-Container(e1,e2) 287 | 8287 Member-Collection(e2,e1) 288 | 8288 Cause-Effect(e2,e1) 289 | 8289 Other 290 | 8290 Message-Topic(e1,e2) 291 | 8291 Content-Container(e1,e2) 292 | 8292 Message-Topic(e1,e2) 293 | 8293 Component-Whole(e1,e2) 294 | 8294 Other 295 | 8295 Entity-Origin(e1,e2) 296 | 8296 Entity-Origin(e1,e2) 297 | 8297 Entity-Destination(e1,e2) 298 | 8298 Entity-Destination(e1,e2) 299 | 8299 Entity-Destination(e1,e2) 300 | 8300 Product-Producer(e2,e1) 301 | 8301 Other 302 | 8302 Instrument-Agency(e2,e1) 303 | 8303 Component-Whole(e2,e1) 304 | 8304 Other 305 | 8305 Product-Producer(e2,e1) 306 | 8306 Message-Topic(e1,e2) 307 | 8307 Product-Producer(e1,e2) 308 | 8308 Other 309 | 8309 Message-Topic(e1,e2) 310 | 8310 Product-Producer(e2,e1) 311 | 8311 Other 312 | 8312 Cause-Effect(e2,e1) 313 | 8313 Message-Topic(e1,e2) 314 | 8314 Product-Producer(e1,e2) 315 | 8315 Message-Topic(e2,e1) 316 | 8316 Member-Collection(e2,e1) 317 | 8317 Content-Container(e1,e2) 318 | 8318 Content-Container(e1,e2) 319 | 8319 Entity-Destination(e1,e2) 320 | 8320 Instrument-Agency(e2,e1) 321 | 8321 Entity-Destination(e1,e2) 322 | 8322 Member-Collection(e2,e1) 323 | 8323 Member-Collection(e1,e2) 324 | 8324 Entity-Destination(e1,e2) 325 | 8325 Content-Container(e2,e1) 326 | 8326 Other 327 | 8327 Message-Topic(e2,e1) 328 | 8328 Message-Topic(e1,e2) 329 | 8329 Message-Topic(e1,e2) 330 | 8330 Product-Producer(e1,e2) 331 | 8331 Member-Collection(e2,e1) 332 | 8332 Message-Topic(e1,e2) 333 | 8333 Message-Topic(e2,e1) 334 | 8334 Cause-Effect(e2,e1) 335 | 8335 Member-Collection(e2,e1) 336 | 8336 Other 337 | 8337 Other 338 | 8338 Message-Topic(e1,e2) 339 | 8339 Other 340 | 8340 Content-Container(e1,e2) 341 | 8341 Message-Topic(e1,e2) 342 | 8342 Other 343 | 8343 Instrument-Agency(e2,e1) 344 | 8344 Entity-Destination(e1,e2) 345 | 8345 Content-Container(e1,e2) 346 | 8346 Content-Container(e2,e1) 347 | 8347 Other 348 | 8348 Other 349 | 8349 Member-Collection(e2,e1) 350 | 8350 Component-Whole(e2,e1) 351 | 8351 Content-Container(e1,e2) 352 | 8352 Member-Collection(e2,e1) 353 | 8353 Message-Topic(e1,e2) 354 | 8354 Message-Topic(e2,e1) 355 | 8355 Content-Container(e1,e2) 356 | 8356 Other 357 | 8357 Cause-Effect(e1,e2) 358 | 8358 Instrument-Agency(e2,e1) 359 | 8359 Member-Collection(e2,e1) 360 | 8360 Component-Whole(e2,e1) 361 | 8361 Cause-Effect(e2,e1) 362 | 8362 Other 363 | 8363 Entity-Origin(e1,e2) 364 | 8364 Instrument-Agency(e2,e1) 365 | 8365 Product-Producer(e1,e2) 366 | 8366 Message-Topic(e1,e2) 367 | 8367 Entity-Destination(e1,e2) 368 | 8368 Entity-Destination(e1,e2) 369 | 8369 Member-Collection(e1,e2) 370 | 8370 Other 371 | 8371 Component-Whole(e1,e2) 372 | 8372 Other 373 | 8373 Cause-Effect(e2,e1) 374 | 8374 Product-Producer(e2,e1) 375 | 8375 Entity-Destination(e1,e2) 376 | 8376 Entity-Destination(e1,e2) 377 | 8377 Cause-Effect(e1,e2) 378 | 8378 Product-Producer(e2,e1) 379 | 8379 Other 380 | 8380 Other 381 | 8381 Instrument-Agency(e1,e2) 382 | 8382 Cause-Effect(e2,e1) 383 | 8383 Entity-Destination(e1,e2) 384 | 8384 Other 385 | 8385 Entity-Origin(e1,e2) 386 | 8386 Component-Whole(e2,e1) 387 | 8387 Product-Producer(e2,e1) 388 | 8388 Component-Whole(e1,e2) 389 | 8389 Message-Topic(e1,e2) 390 | 8390 Other 391 | 8391 Other 392 | 8392 Component-Whole(e2,e1) 393 | 8393 Entity-Origin(e1,e2) 394 | 8394 Entity-Origin(e1,e2) 395 | 8395 Component-Whole(e1,e2) 396 | 8396 Other 397 | 8397 Other 398 | 8398 Entity-Destination(e1,e2) 399 | 8399 Instrument-Agency(e2,e1) 400 | 8400 Other 401 | 8401 Entity-Destination(e1,e2) 402 | 8402 Cause-Effect(e2,e1) 403 | 8403 Cause-Effect(e2,e1) 404 | 8404 Cause-Effect(e2,e1) 405 | 8405 Cause-Effect(e2,e1) 406 | 8406 Component-Whole(e1,e2) 407 | 8407 Other 408 | 8408 Entity-Origin(e2,e1) 409 | 8409 Cause-Effect(e2,e1) 410 | 8410 Entity-Destination(e1,e2) 411 | 8411 Entity-Origin(e1,e2) 412 | 8412 Content-Container(e2,e1) 413 | 8413 Component-Whole(e1,e2) 414 | 8414 Entity-Destination(e1,e2) 415 | 8415 Member-Collection(e2,e1) 416 | 8416 Component-Whole(e2,e1) 417 | 8417 Cause-Effect(e1,e2) 418 | 8418 Entity-Destination(e1,e2) 419 | 8419 Content-Container(e2,e1) 420 | 8420 Message-Topic(e1,e2) 421 | 8421 Component-Whole(e1,e2) 422 | 8422 Component-Whole(e2,e1) 423 | 8423 Entity-Destination(e1,e2) 424 | 8424 Instrument-Agency(e2,e1) 425 | 8425 Other 426 | 8426 Other 427 | 8427 Component-Whole(e1,e2) 428 | 8428 Product-Producer(e1,e2) 429 | 8429 Component-Whole(e1,e2) 430 | 8430 Entity-Origin(e1,e2) 431 | 8431 Component-Whole(e2,e1) 432 | 8432 Other 433 | 8433 Member-Collection(e2,e1) 434 | 8434 Other 435 | 8435 Other 436 | 8436 Other 437 | 8437 Message-Topic(e2,e1) 438 | 8438 Component-Whole(e2,e1) 439 | 8439 Cause-Effect(e2,e1) 440 | 8440 Message-Topic(e1,e2) 441 | 8441 Product-Producer(e2,e1) 442 | 8442 Component-Whole(e1,e2) 443 | 8443 Component-Whole(e1,e2) 444 | 8444 Component-Whole(e2,e1) 445 | 8445 Content-Container(e1,e2) 446 | 8446 Product-Producer(e1,e2) 447 | 8447 Other 448 | 8448 Other 449 | 8449 Entity-Origin(e1,e2) 450 | 8450 Other 451 | 8451 Other 452 | 8452 Member-Collection(e2,e1) 453 | 8453 Entity-Origin(e1,e2) 454 | 8454 Product-Producer(e2,e1) 455 | 8455 Cause-Effect(e2,e1) 456 | 8456 Entity-Destination(e1,e2) 457 | 8457 Entity-Destination(e1,e2) 458 | 8458 Product-Producer(e1,e2) 459 | 8459 Instrument-Agency(e2,e1) 460 | 8460 Entity-Destination(e1,e2) 461 | 8461 Other 462 | 8462 Product-Producer(e2,e1) 463 | 8463 Entity-Destination(e1,e2) 464 | 8464 Entity-Destination(e1,e2) 465 | 8465 Content-Container(e1,e2) 466 | 8466 Other 467 | 8467 Entity-Destination(e1,e2) 468 | 8468 Entity-Destination(e1,e2) 469 | 8469 Entity-Origin(e1,e2) 470 | 8470 Component-Whole(e1,e2) 471 | 8471 Cause-Effect(e1,e2) 472 | 8472 Component-Whole(e1,e2) 473 | 8473 Cause-Effect(e2,e1) 474 | 8474 Content-Container(e1,e2) 475 | 8475 Other 476 | 8476 Cause-Effect(e1,e2) 477 | 8477 Other 478 | 8478 Entity-Origin(e1,e2) 479 | 8479 Message-Topic(e1,e2) 480 | 8480 Message-Topic(e1,e2) 481 | 8481 Entity-Destination(e1,e2) 482 | 8482 Other 483 | 8483 Product-Producer(e1,e2) 484 | 8484 Other 485 | 8485 Product-Producer(e1,e2) 486 | 8486 Cause-Effect(e1,e2) 487 | 8487 Other 488 | 8488 Product-Producer(e1,e2) 489 | 8489 Cause-Effect(e2,e1) 490 | 8490 Content-Container(e1,e2) 491 | 8491 Other 492 | 8492 Member-Collection(e2,e1) 493 | 8493 Cause-Effect(e2,e1) 494 | 8494 Cause-Effect(e2,e1) 495 | 8495 Message-Topic(e2,e1) 496 | 8496 Entity-Destination(e1,e2) 497 | 8497 Entity-Origin(e1,e2) 498 | 8498 Cause-Effect(e1,e2) 499 | 8499 Component-Whole(e1,e2) 500 | 8500 Cause-Effect(e1,e2) 501 | 8501 Message-Topic(e2,e1) 502 | 8502 Content-Container(e1,e2) 503 | 8503 Cause-Effect(e2,e1) 504 | 8504 Entity-Origin(e1,e2) 505 | 8505 Content-Container(e1,e2) 506 | 8506 Entity-Destination(e1,e2) 507 | 8507 Member-Collection(e2,e1) 508 | 8508 Other 509 | 8509 Cause-Effect(e2,e1) 510 | 8510 Other 511 | 8511 Instrument-Agency(e2,e1) 512 | 8512 Cause-Effect(e1,e2) 513 | 8513 Other 514 | 8514 Message-Topic(e1,e2) 515 | 8515 Other 516 | 8516 Other 517 | 8517 Entity-Origin(e1,e2) 518 | 8518 Entity-Origin(e2,e1) 519 | 8519 Product-Producer(e2,e1) 520 | 8520 Cause-Effect(e2,e1) 521 | 8521 Cause-Effect(e2,e1) 522 | 8522 Other 523 | 8523 Cause-Effect(e2,e1) 524 | 8524 Instrument-Agency(e2,e1) 525 | 8525 Entity-Origin(e1,e2) 526 | 8526 Entity-Destination(e1,e2) 527 | 8527 Component-Whole(e1,e2) 528 | 8528 Content-Container(e1,e2) 529 | 8529 Entity-Destination(e1,e2) 530 | 8530 Product-Producer(e2,e1) 531 | 8531 Component-Whole(e2,e1) 532 | 8532 Other 533 | 8533 Product-Producer(e1,e2) 534 | 8534 Cause-Effect(e1,e2) 535 | 8535 Cause-Effect(e2,e1) 536 | 8536 Cause-Effect(e1,e2) 537 | 8537 Other 538 | 8538 Member-Collection(e2,e1) 539 | 8539 Member-Collection(e1,e2) 540 | 8540 Other 541 | 8541 Product-Producer(e1,e2) 542 | 8542 Cause-Effect(e1,e2) 543 | 8543 Entity-Origin(e1,e2) 544 | 8544 Message-Topic(e1,e2) 545 | 8545 Instrument-Agency(e2,e1) 546 | 8546 Entity-Origin(e1,e2) 547 | 8547 Component-Whole(e1,e2) 548 | 8548 Component-Whole(e2,e1) 549 | 8549 Component-Whole(e1,e2) 550 | 8550 Other 551 | 8551 Message-Topic(e2,e1) 552 | 8552 Entity-Destination(e1,e2) 553 | 8553 Message-Topic(e1,e2) 554 | 8554 Content-Container(e1,e2) 555 | 8555 Entity-Origin(e2,e1) 556 | 8556 Cause-Effect(e1,e2) 557 | 8557 Entity-Origin(e1,e2) 558 | 8558 Entity-Destination(e1,e2) 559 | 8559 Product-Producer(e2,e1) 560 | 8560 Other 561 | 8561 Component-Whole(e2,e1) 562 | 8562 Entity-Origin(e1,e2) 563 | 8563 Message-Topic(e1,e2) 564 | 8564 Message-Topic(e1,e2) 565 | 8565 Other 566 | 8566 Entity-Destination(e1,e2) 567 | 8567 Instrument-Agency(e2,e1) 568 | 8568 Other 569 | 8569 Entity-Origin(e1,e2) 570 | 8570 Member-Collection(e2,e1) 571 | 8571 Other 572 | 8572 Member-Collection(e2,e1) 573 | 8573 Other 574 | 8574 Component-Whole(e1,e2) 575 | 8575 Entity-Destination(e1,e2) 576 | 8576 Content-Container(e1,e2) 577 | 8577 Member-Collection(e2,e1) 578 | 8578 Member-Collection(e2,e1) 579 | 8579 Message-Topic(e1,e2) 580 | 8580 Message-Topic(e1,e2) 581 | 8581 Other 582 | 8582 Other 583 | 8583 Member-Collection(e2,e1) 584 | 8584 Component-Whole(e2,e1) 585 | 8585 Message-Topic(e2,e1) 586 | 8586 Component-Whole(e2,e1) 587 | 8587 Entity-Origin(e1,e2) 588 | 8588 Message-Topic(e1,e2) 589 | 8589 Message-Topic(e1,e2) 590 | 8590 Member-Collection(e2,e1) 591 | 8591 Cause-Effect(e2,e1) 592 | 8592 Other 593 | 8593 Product-Producer(e1,e2) 594 | 8594 Entity-Origin(e1,e2) 595 | 8595 Product-Producer(e1,e2) 596 | 8596 Cause-Effect(e1,e2) 597 | 8597 Message-Topic(e1,e2) 598 | 8598 Entity-Destination(e1,e2) 599 | 8599 Component-Whole(e2,e1) 600 | 8600 Member-Collection(e2,e1) 601 | 8601 Product-Producer(e1,e2) 602 | 8602 Cause-Effect(e2,e1) 603 | 8603 Cause-Effect(e2,e1) 604 | 8604 Message-Topic(e1,e2) 605 | 8605 Component-Whole(e1,e2) 606 | 8606 Entity-Destination(e1,e2) 607 | 8607 Other 608 | 8608 Cause-Effect(e2,e1) 609 | 8609 Component-Whole(e2,e1) 610 | 8610 Other 611 | 8611 Message-Topic(e1,e2) 612 | 8612 Entity-Origin(e1,e2) 613 | 8613 Content-Container(e2,e1) 614 | 8614 Entity-Origin(e1,e2) 615 | 8615 Other 616 | 8616 Component-Whole(e1,e2) 617 | 8617 Entity-Origin(e1,e2) 618 | 8618 Other 619 | 8619 Entity-Destination(e1,e2) 620 | 8620 Entity-Origin(e1,e2) 621 | 8621 Cause-Effect(e2,e1) 622 | 8622 Component-Whole(e1,e2) 623 | 8623 Cause-Effect(e1,e2) 624 | 8624 Component-Whole(e1,e2) 625 | 8625 Message-Topic(e1,e2) 626 | 8626 Other 627 | 8627 Member-Collection(e2,e1) 628 | 8628 Other 629 | 8629 Message-Topic(e1,e2) 630 | 8630 Entity-Destination(e1,e2) 631 | 8631 Entity-Destination(e1,e2) 632 | 8632 Component-Whole(e2,e1) 633 | 8633 Cause-Effect(e1,e2) 634 | 8634 Instrument-Agency(e2,e1) 635 | 8635 Entity-Origin(e1,e2) 636 | 8636 Content-Container(e2,e1) 637 | 8637 Instrument-Agency(e2,e1) 638 | 8638 Member-Collection(e2,e1) 639 | 8639 Entity-Destination(e1,e2) 640 | 8640 Entity-Origin(e1,e2) 641 | 8641 Cause-Effect(e2,e1) 642 | 8642 Product-Producer(e2,e1) 643 | 8643 Entity-Destination(e1,e2) 644 | 8644 Product-Producer(e2,e1) 645 | 8645 Other 646 | 8646 Other 647 | 8647 Other 648 | 8648 Cause-Effect(e2,e1) 649 | 8649 Member-Collection(e1,e2) 650 | 8650 Message-Topic(e1,e2) 651 | 8651 Message-Topic(e1,e2) 652 | 8652 Other 653 | 8653 Entity-Origin(e1,e2) 654 | 8654 Content-Container(e1,e2) 655 | 8655 Cause-Effect(e1,e2) 656 | 8656 Member-Collection(e2,e1) 657 | 8657 Component-Whole(e1,e2) 658 | 8658 Message-Topic(e1,e2) 659 | 8659 Cause-Effect(e1,e2) 660 | 8660 Message-Topic(e1,e2) 661 | 8661 Product-Producer(e1,e2) 662 | 8662 Message-Topic(e2,e1) 663 | 8663 Entity-Destination(e1,e2) 664 | 8664 Product-Producer(e1,e2) 665 | 8665 Component-Whole(e2,e1) 666 | 8666 Component-Whole(e2,e1) 667 | 8667 Component-Whole(e1,e2) 668 | 8668 Other 669 | 8669 Member-Collection(e2,e1) 670 | 8670 Entity-Destination(e1,e2) 671 | 8671 Content-Container(e1,e2) 672 | 8672 Message-Topic(e1,e2) 673 | 8673 Product-Producer(e2,e1) 674 | 8674 Message-Topic(e1,e2) 675 | 8675 Component-Whole(e1,e2) 676 | 8676 Message-Topic(e1,e2) 677 | 8677 Component-Whole(e2,e1) 678 | 8678 Other 679 | 8679 Component-Whole(e2,e1) 680 | 8680 Other 681 | 8681 Cause-Effect(e2,e1) 682 | 8682 Message-Topic(e1,e2) 683 | 8683 Member-Collection(e2,e1) 684 | 8684 Component-Whole(e2,e1) 685 | 8685 Content-Container(e1,e2) 686 | 8686 Member-Collection(e1,e2) 687 | 8687 Other 688 | 8688 Entity-Origin(e1,e2) 689 | 8689 Content-Container(e1,e2) 690 | 8690 Cause-Effect(e2,e1) 691 | 8691 Message-Topic(e1,e2) 692 | 8692 Component-Whole(e1,e2) 693 | 8693 Content-Container(e1,e2) 694 | 8694 Other 695 | 8695 Content-Container(e1,e2) 696 | 8696 Member-Collection(e1,e2) 697 | 8697 Other 698 | 8698 Entity-Destination(e1,e2) 699 | 8699 Entity-Origin(e1,e2) 700 | 8700 Product-Producer(e2,e1) 701 | 8701 Member-Collection(e2,e1) 702 | 8702 Component-Whole(e1,e2) 703 | 8703 Component-Whole(e2,e1) 704 | 8704 Entity-Origin(e2,e1) 705 | 8705 Cause-Effect(e1,e2) 706 | 8706 Other 707 | 8707 Content-Container(e2,e1) 708 | 8708 Cause-Effect(e2,e1) 709 | 8709 Entity-Origin(e1,e2) 710 | 8710 Entity-Destination(e1,e2) 711 | 8711 Message-Topic(e1,e2) 712 | 8712 Member-Collection(e1,e2) 713 | 8713 Member-Collection(e2,e1) 714 | 8714 Member-Collection(e2,e1) 715 | 8715 Content-Container(e1,e2) 716 | 8716 Other 717 | 8717 Product-Producer(e2,e1) 718 | 8718 Other 719 | 8719 Entity-Destination(e1,e2) 720 | 8720 Cause-Effect(e2,e1) 721 | 8721 Other 722 | 8722 Product-Producer(e2,e1) 723 | 8723 Product-Producer(e2,e1) 724 | 8724 Component-Whole(e2,e1) 725 | 8725 Message-Topic(e1,e2) 726 | 8726 Other 727 | 8727 Product-Producer(e2,e1) 728 | 8728 Content-Container(e1,e2) 729 | 8729 Member-Collection(e2,e1) 730 | 8730 Component-Whole(e1,e2) 731 | 8731 Cause-Effect(e2,e1) 732 | 8732 Instrument-Agency(e2,e1) 733 | 8733 Entity-Origin(e1,e2) 734 | 8734 Entity-Origin(e1,e2) 735 | 8735 Component-Whole(e1,e2) 736 | 8736 Cause-Effect(e1,e2) 737 | 8737 Instrument-Agency(e2,e1) 738 | 8738 Content-Container(e1,e2) 739 | 8739 Cause-Effect(e2,e1) 740 | 8740 Cause-Effect(e1,e2) 741 | 8741 Member-Collection(e2,e1) 742 | 8742 Entity-Destination(e1,e2) 743 | 8743 Entity-Destination(e1,e2) 744 | 8744 Product-Producer(e2,e1) 745 | 8745 Cause-Effect(e2,e1) 746 | 8746 Component-Whole(e1,e2) 747 | 8747 Entity-Origin(e1,e2) 748 | 8748 Cause-Effect(e1,e2) 749 | 8749 Entity-Origin(e1,e2) 750 | 8750 Instrument-Agency(e2,e1) 751 | 8751 Member-Collection(e2,e1) 752 | 8752 Cause-Effect(e1,e2) 753 | 8753 Other 754 | 8754 Cause-Effect(e2,e1) 755 | 8755 Entity-Destination(e1,e2) 756 | 8756 Product-Producer(e1,e2) 757 | 8757 Entity-Destination(e1,e2) 758 | 8758 Entity-Destination(e1,e2) 759 | 8759 Other 760 | 8760 Entity-Destination(e1,e2) 761 | 8761 Entity-Origin(e1,e2) 762 | 8762 Entity-Origin(e1,e2) 763 | 8763 Other 764 | 8764 Cause-Effect(e1,e2) 765 | 8765 Product-Producer(e2,e1) 766 | 8766 Product-Producer(e1,e2) 767 | 8767 Message-Topic(e2,e1) 768 | 8768 Product-Producer(e1,e2) 769 | 8769 Product-Producer(e1,e2) 770 | 8770 Content-Container(e1,e2) 771 | 8771 Other 772 | 8772 Entity-Destination(e1,e2) 773 | 8773 Member-Collection(e2,e1) 774 | 8774 Cause-Effect(e2,e1) 775 | 8775 Cause-Effect(e2,e1) 776 | 8776 Component-Whole(e2,e1) 777 | 8777 Content-Container(e1,e2) 778 | 8778 Component-Whole(e2,e1) 779 | 8779 Component-Whole(e2,e1) 780 | 8780 Content-Container(e1,e2) 781 | 8781 Cause-Effect(e1,e2) 782 | 8782 Instrument-Agency(e2,e1) 783 | 8783 Product-Producer(e2,e1) 784 | 8784 Entity-Origin(e1,e2) 785 | 8785 Other 786 | 8786 Other 787 | 8787 Entity-Origin(e2,e1) 788 | 8788 Message-Topic(e1,e2) 789 | 8789 Message-Topic(e1,e2) 790 | 8790 Instrument-Agency(e2,e1) 791 | 8791 Entity-Destination(e1,e2) 792 | 8792 Other 793 | 8793 Entity-Destination(e1,e2) 794 | 8794 Other 795 | 8795 Member-Collection(e2,e1) 796 | 8796 Member-Collection(e2,e1) 797 | 8797 Product-Producer(e1,e2) 798 | 8798 Member-Collection(e2,e1) 799 | 8799 Entity-Origin(e1,e2) 800 | 8800 Entity-Destination(e1,e2) 801 | 8801 Other 802 | 8802 Component-Whole(e2,e1) 803 | 8803 Member-Collection(e2,e1) 804 | 8804 Instrument-Agency(e2,e1) 805 | 8805 Entity-Origin(e2,e1) 806 | 8806 Content-Container(e1,e2) 807 | 8807 Component-Whole(e1,e2) 808 | 8808 Component-Whole(e1,e2) 809 | 8809 Other 810 | 8810 Entity-Origin(e2,e1) 811 | 8811 Instrument-Agency(e1,e2) 812 | 8812 Cause-Effect(e2,e1) 813 | 8813 Instrument-Agency(e2,e1) 814 | 8814 Member-Collection(e1,e2) 815 | 8815 Entity-Destination(e1,e2) 816 | 8816 Content-Container(e1,e2) 817 | 8817 Member-Collection(e2,e1) 818 | 8818 Other 819 | 8819 Component-Whole(e1,e2) 820 | 8820 Component-Whole(e1,e2) 821 | 8821 Product-Producer(e2,e1) 822 | 8822 Member-Collection(e2,e1) 823 | 8823 Instrument-Agency(e2,e1) 824 | 8824 Member-Collection(e2,e1) 825 | 8825 Entity-Destination(e1,e2) 826 | 8826 Message-Topic(e1,e2) 827 | 8827 Entity-Destination(e1,e2) 828 | 8828 Product-Producer(e2,e1) 829 | 8829 Cause-Effect(e1,e2) 830 | 8830 Message-Topic(e1,e2) 831 | 8831 Component-Whole(e1,e2) 832 | 8832 Entity-Origin(e1,e2) 833 | 8833 Content-Container(e1,e2) 834 | 8834 Entity-Origin(e1,e2) 835 | 8835 Instrument-Agency(e2,e1) 836 | 8836 Entity-Origin(e1,e2) 837 | 8837 Component-Whole(e2,e1) 838 | 8838 Instrument-Agency(e2,e1) 839 | 8839 Member-Collection(e2,e1) 840 | 8840 Product-Producer(e2,e1) 841 | 8841 Cause-Effect(e1,e2) 842 | 8842 Other 843 | 8843 Content-Container(e1,e2) 844 | 8844 Message-Topic(e1,e2) 845 | 8845 Other 846 | 8846 Entity-Destination(e1,e2) 847 | 8847 Other 848 | 8848 Message-Topic(e1,e2) 849 | 8849 Entity-Destination(e1,e2) 850 | 8850 Entity-Destination(e1,e2) 851 | 8851 Cause-Effect(e2,e1) 852 | 8852 Content-Container(e1,e2) 853 | 8853 Entity-Origin(e1,e2) 854 | 8854 Member-Collection(e2,e1) 855 | 8855 Cause-Effect(e2,e1) 856 | 8856 Content-Container(e1,e2) 857 | 8857 Cause-Effect(e2,e1) 858 | 8858 Cause-Effect(e1,e2) 859 | 8859 Cause-Effect(e2,e1) 860 | 8860 Other 861 | 8861 Message-Topic(e1,e2) 862 | 8862 Entity-Destination(e1,e2) 863 | 8863 Other 864 | 8864 Component-Whole(e2,e1) 865 | 8865 Component-Whole(e1,e2) 866 | 8866 Other 867 | 8867 Entity-Destination(e1,e2) 868 | 8868 Component-Whole(e2,e1) 869 | 8869 Product-Producer(e1,e2) 870 | 8870 Entity-Destination(e1,e2) 871 | 8871 Member-Collection(e2,e1) 872 | 8872 Instrument-Agency(e1,e2) 873 | 8873 Component-Whole(e1,e2) 874 | 8874 Other 875 | 8875 Cause-Effect(e1,e2) 876 | 8876 Other 877 | 8877 Member-Collection(e1,e2) 878 | 8878 Entity-Origin(e1,e2) 879 | 8879 Cause-Effect(e2,e1) 880 | 8880 Entity-Origin(e1,e2) 881 | 8881 Content-Container(e1,e2) 882 | 8882 Entity-Origin(e2,e1) 883 | 8883 Product-Producer(e2,e1) 884 | 8884 Component-Whole(e2,e1) 885 | 8885 Cause-Effect(e2,e1) 886 | 8886 Entity-Origin(e1,e2) 887 | 8887 Message-Topic(e2,e1) 888 | 8888 Other 889 | 8889 Cause-Effect(e2,e1) 890 | 8890 Entity-Origin(e1,e2) 891 | 8891 Content-Container(e1,e2) 892 | 8892 Product-Producer(e1,e2) 893 | 8893 Component-Whole(e2,e1) 894 | 8894 Entity-Origin(e1,e2) 895 | 8895 Product-Producer(e1,e2) 896 | 8896 Other 897 | 8897 Member-Collection(e2,e1) 898 | 8898 Entity-Destination(e1,e2) 899 | 8899 Entity-Origin(e2,e1) 900 | 8900 Message-Topic(e1,e2) 901 | 8901 Message-Topic(e1,e2) 902 | 8902 Member-Collection(e2,e1) 903 | 8903 Entity-Destination(e1,e2) 904 | 8904 Instrument-Agency(e2,e1) 905 | 8905 Other 906 | 8906 Member-Collection(e2,e1) 907 | 8907 Entity-Origin(e2,e1) 908 | 8908 Message-Topic(e1,e2) 909 | 8909 Other 910 | 8910 Other 911 | 8911 Member-Collection(e1,e2) 912 | 8912 Message-Topic(e1,e2) 913 | 8913 Product-Producer(e2,e1) 914 | 8914 Cause-Effect(e2,e1) 915 | 8915 Component-Whole(e2,e1) 916 | 8916 Product-Producer(e2,e1) 917 | 8917 Other 918 | 8918 Instrument-Agency(e2,e1) 919 | 8919 Message-Topic(e2,e1) 920 | 8920 Product-Producer(e1,e2) 921 | 8921 Entity-Origin(e2,e1) 922 | 8922 Product-Producer(e1,e2) 923 | 8923 Component-Whole(e1,e2) 924 | 8924 Product-Producer(e1,e2) 925 | 8925 Other 926 | 8926 Component-Whole(e1,e2) 927 | 8927 Product-Producer(e2,e1) 928 | 8928 Component-Whole(e2,e1) 929 | 8929 Component-Whole(e2,e1) 930 | 8930 Entity-Destination(e1,e2) 931 | 8931 Other 932 | 8932 Component-Whole(e1,e2) 933 | 8933 Other 934 | 8934 Member-Collection(e2,e1) 935 | 8935 Component-Whole(e1,e2) 936 | 8936 Component-Whole(e1,e2) 937 | 8937 Cause-Effect(e2,e1) 938 | 8938 Content-Container(e1,e2) 939 | 8939 Entity-Destination(e1,e2) 940 | 8940 Cause-Effect(e2,e1) 941 | 8941 Component-Whole(e2,e1) 942 | 8942 Other 943 | 8943 Product-Producer(e2,e1) 944 | 8944 Member-Collection(e2,e1) 945 | 8945 Other 946 | 8946 Entity-Destination(e1,e2) 947 | 8947 Instrument-Agency(e1,e2) 948 | 8948 Message-Topic(e1,e2) 949 | 8949 Cause-Effect(e1,e2) 950 | 8950 Content-Container(e1,e2) 951 | 8951 Component-Whole(e2,e1) 952 | 8952 Member-Collection(e2,e1) 953 | 8953 Cause-Effect(e2,e1) 954 | 8954 Cause-Effect(e1,e2) 955 | 8955 Product-Producer(e1,e2) 956 | 8956 Other 957 | 8957 Member-Collection(e2,e1) 958 | 8958 Instrument-Agency(e2,e1) 959 | 8959 Component-Whole(e1,e2) 960 | 8960 Entity-Destination(e1,e2) 961 | 8961 Other 962 | 8962 Component-Whole(e1,e2) 963 | 8963 Content-Container(e1,e2) 964 | 8964 Other 965 | 8965 Member-Collection(e2,e1) 966 | 8966 Member-Collection(e2,e1) 967 | 8967 Other 968 | 8968 Entity-Destination(e1,e2) 969 | 8969 Product-Producer(e1,e2) 970 | 8970 Instrument-Agency(e2,e1) 971 | 8971 Product-Producer(e2,e1) 972 | 8972 Cause-Effect(e2,e1) 973 | 8973 Entity-Destination(e1,e2) 974 | 8974 Cause-Effect(e2,e1) 975 | 8975 Message-Topic(e2,e1) 976 | 8976 Product-Producer(e1,e2) 977 | 8977 Instrument-Agency(e2,e1) 978 | 8978 Entity-Destination(e1,e2) 979 | 8979 Message-Topic(e1,e2) 980 | 8980 Message-Topic(e2,e1) 981 | 8981 Instrument-Agency(e2,e1) 982 | 8982 Instrument-Agency(e2,e1) 983 | 8983 Entity-Destination(e1,e2) 984 | 8984 Component-Whole(e1,e2) 985 | 8985 Message-Topic(e1,e2) 986 | 8986 Member-Collection(e2,e1) 987 | 8987 Cause-Effect(e2,e1) 988 | 8988 Product-Producer(e1,e2) 989 | 8989 Cause-Effect(e2,e1) 990 | 8990 Entity-Destination(e1,e2) 991 | 8991 Other 992 | 8992 Cause-Effect(e2,e1) 993 | 8993 Message-Topic(e1,e2) 994 | 8994 Message-Topic(e2,e1) 995 | 8995 Other 996 | 8996 Content-Container(e2,e1) 997 | 8997 Instrument-Agency(e2,e1) 998 | 8998 Member-Collection(e2,e1) 999 | 8999 Message-Topic(e1,e2) 1000 | 9000 Content-Container(e1,e2) 1001 | 9001 Content-Container(e1,e2) 1002 | 9002 Other 1003 | 9003 Component-Whole(e1,e2) 1004 | 9004 Content-Container(e1,e2) 1005 | 9005 Cause-Effect(e2,e1) 1006 | 9006 Component-Whole(e1,e2) 1007 | 9007 Content-Container(e1,e2) 1008 | 9008 Member-Collection(e2,e1) 1009 | 9009 Other 1010 | 9010 Content-Container(e1,e2) 1011 | 9011 Product-Producer(e2,e1) 1012 | 9012 Cause-Effect(e1,e2) 1013 | 9013 Component-Whole(e1,e2) 1014 | 9014 Cause-Effect(e2,e1) 1015 | 9015 Cause-Effect(e2,e1) 1016 | 9016 Entity-Destination(e1,e2) 1017 | 9017 Entity-Origin(e1,e2) 1018 | 9018 Cause-Effect(e1,e2) 1019 | 9019 Other 1020 | 9020 Other 1021 | 9021 Member-Collection(e1,e2) 1022 | 9022 Other 1023 | 9023 Content-Container(e1,e2) 1024 | 9024 Content-Container(e1,e2) 1025 | 9025 Cause-Effect(e2,e1) 1026 | 9026 Entity-Origin(e1,e2) 1027 | 9027 Entity-Origin(e1,e2) 1028 | 9028 Other 1029 | 9029 Component-Whole(e2,e1) 1030 | 9030 Message-Topic(e2,e1) 1031 | 9031 Product-Producer(e2,e1) 1032 | 9032 Member-Collection(e1,e2) 1033 | 9033 Product-Producer(e2,e1) 1034 | 9034 Other 1035 | 9035 Content-Container(e1,e2) 1036 | 9036 Instrument-Agency(e1,e2) 1037 | 9037 Entity-Destination(e1,e2) 1038 | 9038 Entity-Destination(e1,e2) 1039 | 9039 Entity-Destination(e1,e2) 1040 | 9040 Component-Whole(e1,e2) 1041 | 9041 Entity-Origin(e1,e2) 1042 | 9042 Instrument-Agency(e2,e1) 1043 | 9043 Content-Container(e1,e2) 1044 | 9044 Content-Container(e2,e1) 1045 | 9045 Content-Container(e2,e1) 1046 | 9046 Product-Producer(e2,e1) 1047 | 9047 Product-Producer(e1,e2) 1048 | 9048 Entity-Destination(e1,e2) 1049 | 9049 Product-Producer(e1,e2) 1050 | 9050 Message-Topic(e1,e2) 1051 | 9051 Entity-Origin(e2,e1) 1052 | 9052 Product-Producer(e2,e1) 1053 | 9053 Other 1054 | 9054 Other 1055 | 9055 Cause-Effect(e2,e1) 1056 | 9056 Product-Producer(e2,e1) 1057 | 9057 Cause-Effect(e2,e1) 1058 | 9058 Product-Producer(e2,e1) 1059 | 9059 Message-Topic(e1,e2) 1060 | 9060 Entity-Destination(e1,e2) 1061 | 9061 Entity-Destination(e1,e2) 1062 | 9062 Cause-Effect(e1,e2) 1063 | 9063 Message-Topic(e2,e1) 1064 | 9064 Member-Collection(e2,e1) 1065 | 9065 Entity-Origin(e1,e2) 1066 | 9066 Member-Collection(e2,e1) 1067 | 9067 Entity-Origin(e1,e2) 1068 | 9068 Cause-Effect(e1,e2) 1069 | 9069 Member-Collection(e2,e1) 1070 | 9070 Entity-Destination(e1,e2) 1071 | 9071 Product-Producer(e2,e1) 1072 | 9072 Other 1073 | 9073 Cause-Effect(e2,e1) 1074 | 9074 Other 1075 | 9075 Member-Collection(e2,e1) 1076 | 9076 Entity-Destination(e1,e2) 1077 | 9077 Other 1078 | 9078 Entity-Destination(e1,e2) 1079 | 9079 Member-Collection(e2,e1) 1080 | 9080 Other 1081 | 9081 Cause-Effect(e1,e2) 1082 | 9082 Content-Container(e1,e2) 1083 | 9083 Cause-Effect(e2,e1) 1084 | 9084 Member-Collection(e2,e1) 1085 | 9085 Product-Producer(e1,e2) 1086 | 9086 Component-Whole(e2,e1) 1087 | 9087 Cause-Effect(e1,e2) 1088 | 9088 Other 1089 | 9089 Product-Producer(e2,e1) 1090 | 9090 Component-Whole(e1,e2) 1091 | 9091 Entity-Destination(e1,e2) 1092 | 9092 Entity-Origin(e1,e2) 1093 | 9093 Other 1094 | 9094 Other 1095 | 9095 Message-Topic(e1,e2) 1096 | 9096 Product-Producer(e2,e1) 1097 | 9097 Instrument-Agency(e2,e1) 1098 | 9098 Product-Producer(e2,e1) 1099 | 9099 Other 1100 | 9100 Component-Whole(e1,e2) 1101 | 9101 Other 1102 | 9102 Member-Collection(e2,e1) 1103 | 9103 Content-Container(e1,e2) 1104 | 9104 Member-Collection(e2,e1) 1105 | 9105 Component-Whole(e2,e1) 1106 | 9106 Entity-Destination(e1,e2) 1107 | 9107 Entity-Destination(e1,e2) 1108 | 9108 Message-Topic(e1,e2) 1109 | 9109 Message-Topic(e1,e2) 1110 | 9110 Cause-Effect(e1,e2) 1111 | 9111 Cause-Effect(e2,e1) 1112 | 9112 Content-Container(e2,e1) 1113 | 9113 Component-Whole(e1,e2) 1114 | 9114 Product-Producer(e2,e1) 1115 | 9115 Entity-Origin(e1,e2) 1116 | 9116 Instrument-Agency(e2,e1) 1117 | 9117 Entity-Destination(e1,e2) 1118 | 9118 Entity-Destination(e1,e2) 1119 | 9119 Cause-Effect(e1,e2) 1120 | 9120 Product-Producer(e1,e2) 1121 | 9121 Product-Producer(e1,e2) 1122 | 9122 Entity-Destination(e1,e2) 1123 | 9123 Entity-Origin(e1,e2) 1124 | 9124 Instrument-Agency(e2,e1) 1125 | 9125 Entity-Origin(e1,e2) 1126 | 9126 Member-Collection(e2,e1) 1127 | 9127 Entity-Origin(e1,e2) 1128 | 9128 Cause-Effect(e1,e2) 1129 | 9129 Content-Container(e1,e2) 1130 | 9130 Other 1131 | 9131 Cause-Effect(e1,e2) 1132 | 9132 Instrument-Agency(e2,e1) 1133 | 9133 Instrument-Agency(e2,e1) 1134 | 9134 Component-Whole(e1,e2) 1135 | 9135 Instrument-Agency(e2,e1) 1136 | 9136 Cause-Effect(e2,e1) 1137 | 9137 Other 1138 | 9138 Component-Whole(e2,e1) 1139 | 9139 Cause-Effect(e2,e1) 1140 | 9140 Entity-Destination(e1,e2) 1141 | 9141 Message-Topic(e1,e2) 1142 | 9142 Entity-Destination(e1,e2) 1143 | 9143 Member-Collection(e2,e1) 1144 | 9144 Product-Producer(e2,e1) 1145 | 9145 Message-Topic(e1,e2) 1146 | 9146 Cause-Effect(e2,e1) 1147 | 9147 Cause-Effect(e2,e1) 1148 | 9148 Cause-Effect(e2,e1) 1149 | 9149 Other 1150 | 9150 Entity-Origin(e1,e2) 1151 | 9151 Product-Producer(e1,e2) 1152 | 9152 Component-Whole(e2,e1) 1153 | 9153 Content-Container(e1,e2) 1154 | 9154 Other 1155 | 9155 Entity-Origin(e1,e2) 1156 | 9156 Other 1157 | 9157 Other 1158 | 9158 Content-Container(e1,e2) 1159 | 9159 Content-Container(e1,e2) 1160 | 9160 Member-Collection(e2,e1) 1161 | 9161 Cause-Effect(e1,e2) 1162 | 9162 Entity-Destination(e1,e2) 1163 | 9163 Cause-Effect(e1,e2) 1164 | 9164 Other 1165 | 9165 Message-Topic(e1,e2) 1166 | 9166 Component-Whole(e2,e1) 1167 | 9167 Cause-Effect(e2,e1) 1168 | 9168 Cause-Effect(e2,e1) 1169 | 9169 Message-Topic(e1,e2) 1170 | 9170 Other 1171 | 9171 Cause-Effect(e1,e2) 1172 | 9172 Cause-Effect(e1,e2) 1173 | 9173 Entity-Origin(e1,e2) 1174 | 9174 Component-Whole(e2,e1) 1175 | 9175 Entity-Origin(e1,e2) 1176 | 9176 Entity-Origin(e1,e2) 1177 | 9177 Product-Producer(e2,e1) 1178 | 9178 Entity-Origin(e1,e2) 1179 | 9179 Cause-Effect(e1,e2) 1180 | 9180 Entity-Origin(e1,e2) 1181 | 9181 Cause-Effect(e2,e1) 1182 | 9182 Entity-Origin(e1,e2) 1183 | 9183 Component-Whole(e2,e1) 1184 | 9184 Content-Container(e2,e1) 1185 | 9185 Component-Whole(e2,e1) 1186 | 9186 Message-Topic(e1,e2) 1187 | 9187 Other 1188 | 9188 Entity-Origin(e2,e1) 1189 | 9189 Entity-Destination(e1,e2) 1190 | 9190 Cause-Effect(e2,e1) 1191 | 9191 Message-Topic(e1,e2) 1192 | 9192 Other 1193 | 9193 Other 1194 | 9194 Member-Collection(e1,e2) 1195 | 9195 Instrument-Agency(e1,e2) 1196 | 9196 Content-Container(e1,e2) 1197 | 9197 Entity-Destination(e1,e2) 1198 | 9198 Member-Collection(e2,e1) 1199 | 9199 Message-Topic(e1,e2) 1200 | 9200 Entity-Destination(e1,e2) 1201 | 9201 Entity-Origin(e1,e2) 1202 | 9202 Message-Topic(e1,e2) 1203 | 9203 Component-Whole(e1,e2) 1204 | 9204 Entity-Origin(e1,e2) 1205 | 9205 Instrument-Agency(e2,e1) 1206 | 9206 Entity-Origin(e2,e1) 1207 | 9207 Component-Whole(e1,e2) 1208 | 9208 Other 1209 | 9209 Entity-Origin(e1,e2) 1210 | 9210 Component-Whole(e1,e2) 1211 | 9211 Member-Collection(e2,e1) 1212 | 9212 Content-Container(e1,e2) 1213 | 9213 Cause-Effect(e1,e2) 1214 | 9214 Component-Whole(e2,e1) 1215 | 9215 Instrument-Agency(e2,e1) 1216 | 9216 Member-Collection(e2,e1) 1217 | 9217 Other 1218 | 9218 Entity-Destination(e1,e2) 1219 | 9219 Other 1220 | 9220 Entity-Origin(e1,e2) 1221 | 9221 Cause-Effect(e2,e1) 1222 | 9222 Entity-Destination(e1,e2) 1223 | 9223 Product-Producer(e1,e2) 1224 | 9224 Cause-Effect(e1,e2) 1225 | 9225 Entity-Origin(e1,e2) 1226 | 9226 Cause-Effect(e1,e2) 1227 | 9227 Other 1228 | 9228 Cause-Effect(e1,e2) 1229 | 9229 Member-Collection(e2,e1) 1230 | 9230 Component-Whole(e2,e1) 1231 | 9231 Entity-Destination(e1,e2) 1232 | 9232 Other 1233 | 9233 Member-Collection(e2,e1) 1234 | 9234 Cause-Effect(e2,e1) 1235 | 9235 Other 1236 | 9236 Entity-Origin(e2,e1) 1237 | 9237 Component-Whole(e1,e2) 1238 | 9238 Component-Whole(e2,e1) 1239 | 9239 Product-Producer(e2,e1) 1240 | 9240 Entity-Origin(e1,e2) 1241 | 9241 Component-Whole(e2,e1) 1242 | 9242 Member-Collection(e2,e1) 1243 | 9243 Content-Container(e1,e2) 1244 | 9244 Entity-Destination(e1,e2) 1245 | 9245 Other 1246 | 9246 Other 1247 | 9247 Entity-Destination(e1,e2) 1248 | 9248 Other 1249 | 9249 Other 1250 | 9250 Component-Whole(e2,e1) 1251 | 9251 Other 1252 | 9252 Other 1253 | 9253 Product-Producer(e1,e2) 1254 | 9254 Member-Collection(e1,e2) 1255 | 9255 Content-Container(e1,e2) 1256 | 9256 Other 1257 | 9257 Component-Whole(e2,e1) 1258 | 9258 Message-Topic(e1,e2) 1259 | 9259 Cause-Effect(e2,e1) 1260 | 9260 Content-Container(e2,e1) 1261 | 9261 Message-Topic(e1,e2) 1262 | 9262 Member-Collection(e2,e1) 1263 | 9263 Member-Collection(e2,e1) 1264 | 9264 Component-Whole(e2,e1) 1265 | 9265 Component-Whole(e1,e2) 1266 | 9266 Entity-Origin(e1,e2) 1267 | 9267 Component-Whole(e2,e1) 1268 | 9268 Member-Collection(e2,e1) 1269 | 9269 Message-Topic(e2,e1) 1270 | 9270 Instrument-Agency(e2,e1) 1271 | 9271 Entity-Origin(e1,e2) 1272 | 9272 Component-Whole(e1,e2) 1273 | 9273 Content-Container(e2,e1) 1274 | 9274 Entity-Origin(e1,e2) 1275 | 9275 Entity-Destination(e1,e2) 1276 | 9276 Component-Whole(e1,e2) 1277 | 9277 Product-Producer(e2,e1) 1278 | 9278 Entity-Origin(e1,e2) 1279 | 9279 Entity-Origin(e1,e2) 1280 | 9280 Cause-Effect(e2,e1) 1281 | 9281 Other 1282 | 9282 Member-Collection(e2,e1) 1283 | 9283 Other 1284 | 9284 Instrument-Agency(e1,e2) 1285 | 9285 Content-Container(e2,e1) 1286 | 9286 Member-Collection(e1,e2) 1287 | 9287 Entity-Origin(e2,e1) 1288 | 9288 Component-Whole(e2,e1) 1289 | 9289 Cause-Effect(e2,e1) 1290 | 9290 Message-Topic(e1,e2) 1291 | 9291 Instrument-Agency(e2,e1) 1292 | 9292 Content-Container(e1,e2) 1293 | 9293 Component-Whole(e2,e1) 1294 | 9294 Member-Collection(e2,e1) 1295 | 9295 Entity-Destination(e1,e2) 1296 | 9296 Entity-Origin(e1,e2) 1297 | 9297 Entity-Destination(e1,e2) 1298 | 9298 Message-Topic(e1,e2) 1299 | 9299 Entity-Origin(e1,e2) 1300 | 9300 Entity-Destination(e1,e2) 1301 | 9301 Other 1302 | 9302 Component-Whole(e1,e2) 1303 | 9303 Member-Collection(e2,e1) 1304 | 9304 Message-Topic(e2,e1) 1305 | 9305 Entity-Origin(e1,e2) 1306 | 9306 Entity-Destination(e1,e2) 1307 | 9307 Product-Producer(e1,e2) 1308 | 9308 Instrument-Agency(e2,e1) 1309 | 9309 Cause-Effect(e2,e1) 1310 | 9310 Other 1311 | 9311 Cause-Effect(e2,e1) 1312 | 9312 Other 1313 | 9313 Component-Whole(e2,e1) 1314 | 9314 Content-Container(e1,e2) 1315 | 9315 Message-Topic(e1,e2) 1316 | 9316 Component-Whole(e1,e2) 1317 | 9317 Instrument-Agency(e2,e1) 1318 | 9318 Entity-Destination(e1,e2) 1319 | 9319 Cause-Effect(e2,e1) 1320 | 9320 Other 1321 | 9321 Message-Topic(e1,e2) 1322 | 9322 Product-Producer(e2,e1) 1323 | 9323 Cause-Effect(e2,e1) 1324 | 9324 Content-Container(e1,e2) 1325 | 9325 Member-Collection(e2,e1) 1326 | 9326 Entity-Origin(e1,e2) 1327 | 9327 Message-Topic(e1,e2) 1328 | 9328 Cause-Effect(e1,e2) 1329 | 9329 Component-Whole(e2,e1) 1330 | 9330 Product-Producer(e2,e1) 1331 | 9331 Instrument-Agency(e2,e1) 1332 | 9332 Content-Container(e1,e2) 1333 | 9333 Component-Whole(e2,e1) 1334 | 9334 Content-Container(e2,e1) 1335 | 9335 Entity-Destination(e1,e2) 1336 | 9336 Member-Collection(e1,e2) 1337 | 9337 Component-Whole(e2,e1) 1338 | 9338 Entity-Destination(e1,e2) 1339 | 9339 Message-Topic(e1,e2) 1340 | 9340 Product-Producer(e2,e1) 1341 | 9341 Content-Container(e1,e2) 1342 | 9342 Cause-Effect(e1,e2) 1343 | 9343 Entity-Origin(e1,e2) 1344 | 9344 Member-Collection(e2,e1) 1345 | 9345 Content-Container(e2,e1) 1346 | 9346 Component-Whole(e1,e2) 1347 | 9347 Entity-Origin(e1,e2) 1348 | 9348 Product-Producer(e1,e2) 1349 | 9349 Instrument-Agency(e2,e1) 1350 | 9350 Other 1351 | 9351 Entity-Destination(e1,e2) 1352 | 9352 Cause-Effect(e1,e2) 1353 | 9353 Instrument-Agency(e1,e2) 1354 | 9354 Member-Collection(e2,e1) 1355 | 9355 Member-Collection(e2,e1) 1356 | 9356 Instrument-Agency(e2,e1) 1357 | 9357 Cause-Effect(e2,e1) 1358 | 9358 Other 1359 | 9359 Entity-Origin(e1,e2) 1360 | 9360 Entity-Destination(e1,e2) 1361 | 9361 Component-Whole(e1,e2) 1362 | 9362 Member-Collection(e2,e1) 1363 | 9363 Message-Topic(e1,e2) 1364 | 9364 Message-Topic(e1,e2) 1365 | 9365 Content-Container(e2,e1) 1366 | 9366 Cause-Effect(e1,e2) 1367 | 9367 Product-Producer(e2,e1) 1368 | 9368 Member-Collection(e2,e1) 1369 | 9369 Other 1370 | 9370 Cause-Effect(e2,e1) 1371 | 9371 Message-Topic(e1,e2) 1372 | 9372 Cause-Effect(e2,e1) 1373 | 9373 Cause-Effect(e1,e2) 1374 | 9374 Entity-Destination(e1,e2) 1375 | 9375 Entity-Destination(e1,e2) 1376 | 9376 Component-Whole(e2,e1) 1377 | 9377 Instrument-Agency(e2,e1) 1378 | 9378 Cause-Effect(e2,e1) 1379 | 9379 Cause-Effect(e1,e2) 1380 | 9380 Entity-Destination(e1,e2) 1381 | 9381 Message-Topic(e1,e2) 1382 | 9382 Component-Whole(e1,e2) 1383 | 9383 Entity-Origin(e2,e1) 1384 | 9384 Instrument-Agency(e2,e1) 1385 | 9385 Content-Container(e2,e1) 1386 | 9386 Other 1387 | 9387 Component-Whole(e2,e1) 1388 | 9388 Other 1389 | 9389 Entity-Destination(e1,e2) 1390 | 9390 Entity-Origin(e1,e2) 1391 | 9391 Component-Whole(e2,e1) 1392 | 9392 Other 1393 | 9393 Component-Whole(e2,e1) 1394 | 9394 Cause-Effect(e2,e1) 1395 | 9395 Entity-Origin(e2,e1) 1396 | 9396 Other 1397 | 9397 Other 1398 | 9398 Instrument-Agency(e2,e1) 1399 | 9399 Entity-Destination(e1,e2) 1400 | 9400 Other 1401 | 9401 Message-Topic(e2,e1) 1402 | 9402 Other 1403 | 9403 Cause-Effect(e1,e2) 1404 | 9404 Component-Whole(e2,e1) 1405 | 9405 Component-Whole(e1,e2) 1406 | 9406 Other 1407 | 9407 Content-Container(e2,e1) 1408 | 9408 Other 1409 | 9409 Instrument-Agency(e2,e1) 1410 | 9410 Message-Topic(e1,e2) 1411 | 9411 Component-Whole(e2,e1) 1412 | 9412 Member-Collection(e2,e1) 1413 | 9413 Instrument-Agency(e2,e1) 1414 | 9414 Other 1415 | 9415 Cause-Effect(e1,e2) 1416 | 9416 Entity-Destination(e1,e2) 1417 | 9417 Other 1418 | 9418 Other 1419 | 9419 Component-Whole(e1,e2) 1420 | 9420 Component-Whole(e2,e1) 1421 | 9421 Entity-Origin(e2,e1) 1422 | 9422 Product-Producer(e2,e1) 1423 | 9423 Member-Collection(e1,e2) 1424 | 9424 Other 1425 | 9425 Message-Topic(e1,e2) 1426 | 9426 Entity-Destination(e1,e2) 1427 | 9427 Cause-Effect(e2,e1) 1428 | 9428 Product-Producer(e1,e2) 1429 | 9429 Entity-Destination(e1,e2) 1430 | 9430 Message-Topic(e1,e2) 1431 | 9431 Other 1432 | 9432 Message-Topic(e1,e2) 1433 | 9433 Member-Collection(e1,e2) 1434 | 9434 Cause-Effect(e2,e1) 1435 | 9435 Instrument-Agency(e2,e1) 1436 | 9436 Content-Container(e1,e2) 1437 | 9437 Entity-Destination(e1,e2) 1438 | 9438 Cause-Effect(e1,e2) 1439 | 9439 Other 1440 | 9440 Entity-Origin(e1,e2) 1441 | 9441 Component-Whole(e1,e2) 1442 | 9442 Message-Topic(e1,e2) 1443 | 9443 Instrument-Agency(e2,e1) 1444 | 9444 Other 1445 | 9445 Component-Whole(e2,e1) 1446 | 9446 Member-Collection(e2,e1) 1447 | 9447 Content-Container(e1,e2) 1448 | 9448 Component-Whole(e2,e1) 1449 | 9449 Component-Whole(e2,e1) 1450 | 9450 Product-Producer(e1,e2) 1451 | 9451 Member-Collection(e2,e1) 1452 | 9452 Cause-Effect(e1,e2) 1453 | 9453 Entity-Origin(e1,e2) 1454 | 9454 Entity-Origin(e1,e2) 1455 | 9455 Member-Collection(e1,e2) 1456 | 9456 Message-Topic(e1,e2) 1457 | 9457 Instrument-Agency(e2,e1) 1458 | 9458 Product-Producer(e1,e2) 1459 | 9459 Other 1460 | 9460 Entity-Origin(e1,e2) 1461 | 9461 Other 1462 | 9462 Member-Collection(e2,e1) 1463 | 9463 Entity-Origin(e1,e2) 1464 | 9464 Cause-Effect(e2,e1) 1465 | 9465 Other 1466 | 9466 Product-Producer(e1,e2) 1467 | 9467 Cause-Effect(e1,e2) 1468 | 9468 Member-Collection(e2,e1) 1469 | 9469 Cause-Effect(e2,e1) 1470 | 9470 Message-Topic(e2,e1) 1471 | 9471 Content-Container(e1,e2) 1472 | 9472 Entity-Destination(e1,e2) 1473 | 9473 Entity-Origin(e1,e2) 1474 | 9474 Member-Collection(e1,e2) 1475 | 9475 Content-Container(e1,e2) 1476 | 9476 Message-Topic(e1,e2) 1477 | 9477 Instrument-Agency(e1,e2) 1478 | 9478 Member-Collection(e2,e1) 1479 | 9479 Component-Whole(e1,e2) 1480 | 9480 Other 1481 | 9481 Product-Producer(e2,e1) 1482 | 9482 Cause-Effect(e1,e2) 1483 | 9483 Content-Container(e1,e2) 1484 | 9484 Component-Whole(e1,e2) 1485 | 9485 Component-Whole(e2,e1) 1486 | 9486 Instrument-Agency(e2,e1) 1487 | 9487 Instrument-Agency(e2,e1) 1488 | 9488 Instrument-Agency(e2,e1) 1489 | 9489 Cause-Effect(e2,e1) 1490 | 9490 Cause-Effect(e2,e1) 1491 | 9491 Instrument-Agency(e2,e1) 1492 | 9492 Other 1493 | 9493 Entity-Origin(e2,e1) 1494 | 9494 Cause-Effect(e1,e2) 1495 | 9495 Message-Topic(e1,e2) 1496 | 9496 Content-Container(e1,e2) 1497 | 9497 Component-Whole(e1,e2) 1498 | 9498 Message-Topic(e1,e2) 1499 | 9499 Message-Topic(e2,e1) 1500 | 9500 Content-Container(e1,e2) 1501 | 9501 Content-Container(e1,e2) 1502 | 9502 Entity-Origin(e1,e2) 1503 | 9503 Other 1504 | 9504 Message-Topic(e1,e2) 1505 | 9505 Other 1506 | 9506 Entity-Destination(e1,e2) 1507 | 9507 Other 1508 | 9508 Cause-Effect(e1,e2) 1509 | 9509 Member-Collection(e2,e1) 1510 | 9510 Other 1511 | 9511 Other 1512 | 9512 Instrument-Agency(e2,e1) 1513 | 9513 Product-Producer(e2,e1) 1514 | 9514 Entity-Origin(e1,e2) 1515 | 9515 Cause-Effect(e2,e1) 1516 | 9516 Other 1517 | 9517 Other 1518 | 9518 Member-Collection(e2,e1) 1519 | 9519 Cause-Effect(e2,e1) 1520 | 9520 Other 1521 | 9521 Message-Topic(e1,e2) 1522 | 9522 Content-Container(e1,e2) 1523 | 9523 Other 1524 | 9524 Cause-Effect(e1,e2) 1525 | 9525 Message-Topic(e1,e2) 1526 | 9526 Message-Topic(e1,e2) 1527 | 9527 Component-Whole(e1,e2) 1528 | 9528 Content-Container(e1,e2) 1529 | 9529 Entity-Origin(e1,e2) 1530 | 9530 Member-Collection(e2,e1) 1531 | 9531 Entity-Destination(e1,e2) 1532 | 9532 Other 1533 | 9533 Entity-Destination(e1,e2) 1534 | 9534 Content-Container(e1,e2) 1535 | 9535 Component-Whole(e1,e2) 1536 | 9536 Message-Topic(e1,e2) 1537 | 9537 Other 1538 | 9538 Message-Topic(e1,e2) 1539 | 9539 Entity-Destination(e1,e2) 1540 | 9540 Component-Whole(e1,e2) 1541 | 9541 Other 1542 | 9542 Cause-Effect(e1,e2) 1543 | 9543 Message-Topic(e1,e2) 1544 | 9544 Entity-Destination(e1,e2) 1545 | 9545 Other 1546 | 9546 Other 1547 | 9547 Component-Whole(e2,e1) 1548 | 9548 Entity-Origin(e1,e2) 1549 | 9549 Other 1550 | 9550 Member-Collection(e2,e1) 1551 | 9551 Instrument-Agency(e2,e1) 1552 | 9552 Other 1553 | 9553 Product-Producer(e1,e2) 1554 | 9554 Entity-Destination(e1,e2) 1555 | 9555 Instrument-Agency(e2,e1) 1556 | 9556 Cause-Effect(e2,e1) 1557 | 9557 Component-Whole(e2,e1) 1558 | 9558 Other 1559 | 9559 Cause-Effect(e2,e1) 1560 | 9560 Entity-Origin(e1,e2) 1561 | 9561 Component-Whole(e1,e2) 1562 | 9562 Component-Whole(e2,e1) 1563 | 9563 Entity-Destination(e1,e2) 1564 | 9564 Message-Topic(e2,e1) 1565 | 9565 Component-Whole(e1,e2) 1566 | 9566 Message-Topic(e1,e2) 1567 | 9567 Message-Topic(e2,e1) 1568 | 9568 Entity-Destination(e1,e2) 1569 | 9569 Other 1570 | 9570 Member-Collection(e2,e1) 1571 | 9571 Entity-Origin(e1,e2) 1572 | 9572 Instrument-Agency(e2,e1) 1573 | 9573 Cause-Effect(e1,e2) 1574 | 9574 Other 1575 | 9575 Instrument-Agency(e1,e2) 1576 | 9576 Cause-Effect(e2,e1) 1577 | 9577 Other 1578 | 9578 Entity-Destination(e1,e2) 1579 | 9579 Component-Whole(e2,e1) 1580 | 9580 Component-Whole(e2,e1) 1581 | 9581 Entity-Destination(e1,e2) 1582 | 9582 Cause-Effect(e1,e2) 1583 | 9583 Component-Whole(e2,e1) 1584 | 9584 Member-Collection(e2,e1) 1585 | 9585 Entity-Destination(e1,e2) 1586 | 9586 Entity-Destination(e1,e2) 1587 | 9587 Product-Producer(e1,e2) 1588 | 9588 Other 1589 | 9589 Cause-Effect(e1,e2) 1590 | 9590 Instrument-Agency(e2,e1) 1591 | 9591 Entity-Origin(e2,e1) 1592 | 9592 Member-Collection(e2,e1) 1593 | 9593 Entity-Destination(e1,e2) 1594 | 9594 Instrument-Agency(e2,e1) 1595 | 9595 Member-Collection(e2,e1) 1596 | 9596 Message-Topic(e1,e2) 1597 | 9597 Cause-Effect(e2,e1) 1598 | 9598 Entity-Destination(e1,e2) 1599 | 9599 Other 1600 | 9600 Component-Whole(e1,e2) 1601 | 9601 Cause-Effect(e2,e1) 1602 | 9602 Member-Collection(e2,e1) 1603 | 9603 Component-Whole(e1,e2) 1604 | 9604 Content-Container(e2,e1) 1605 | 9605 Instrument-Agency(e2,e1) 1606 | 9606 Other 1607 | 9607 Other 1608 | 9608 Member-Collection(e2,e1) 1609 | 9609 Content-Container(e2,e1) 1610 | 9610 Other 1611 | 9611 Entity-Origin(e1,e2) 1612 | 9612 Component-Whole(e2,e1) 1613 | 9613 Component-Whole(e1,e2) 1614 | 9614 Member-Collection(e1,e2) 1615 | 9615 Message-Topic(e1,e2) 1616 | 9616 Other 1617 | 9617 Component-Whole(e1,e2) 1618 | 9618 Cause-Effect(e1,e2) 1619 | 9619 Instrument-Agency(e2,e1) 1620 | 9620 Member-Collection(e2,e1) 1621 | 9621 Entity-Destination(e1,e2) 1622 | 9622 Message-Topic(e1,e2) 1623 | 9623 Other 1624 | 9624 Cause-Effect(e1,e2) 1625 | 9625 Component-Whole(e1,e2) 1626 | 9626 Entity-Origin(e2,e1) 1627 | 9627 Other 1628 | 9628 Instrument-Agency(e2,e1) 1629 | 9629 Message-Topic(e1,e2) 1630 | 9630 Other 1631 | 9631 Other 1632 | 9632 Component-Whole(e2,e1) 1633 | 9633 Entity-Destination(e1,e2) 1634 | 9634 Component-Whole(e2,e1) 1635 | 9635 Content-Container(e1,e2) 1636 | 9636 Component-Whole(e1,e2) 1637 | 9637 Entity-Destination(e1,e2) 1638 | 9638 Other 1639 | 9639 Other 1640 | 9640 Content-Container(e1,e2) 1641 | 9641 Other 1642 | 9642 Other 1643 | 9643 Other 1644 | 9644 Product-Producer(e1,e2) 1645 | 9645 Content-Container(e1,e2) 1646 | 9646 Other 1647 | 9647 Cause-Effect(e2,e1) 1648 | 9648 Cause-Effect(e2,e1) 1649 | 9649 Instrument-Agency(e1,e2) 1650 | 9650 Other 1651 | 9651 Member-Collection(e2,e1) 1652 | 9652 Other 1653 | 9653 Message-Topic(e2,e1) 1654 | 9654 Instrument-Agency(e1,e2) 1655 | 9655 Entity-Destination(e1,e2) 1656 | 9656 Entity-Origin(e1,e2) 1657 | 9657 Entity-Origin(e1,e2) 1658 | 9658 Other 1659 | 9659 Cause-Effect(e1,e2) 1660 | 9660 Member-Collection(e2,e1) 1661 | 9661 Message-Topic(e1,e2) 1662 | 9662 Content-Container(e1,e2) 1663 | 9663 Other 1664 | 9664 Member-Collection(e2,e1) 1665 | 9665 Entity-Destination(e1,e2) 1666 | 9666 Component-Whole(e1,e2) 1667 | 9667 Product-Producer(e2,e1) 1668 | 9668 Component-Whole(e1,e2) 1669 | 9669 Entity-Origin(e1,e2) 1670 | 9670 Entity-Origin(e1,e2) 1671 | 9671 Component-Whole(e2,e1) 1672 | 9672 Component-Whole(e2,e1) 1673 | 9673 Cause-Effect(e2,e1) 1674 | 9674 Other 1675 | 9675 Message-Topic(e1,e2) 1676 | 9676 Entity-Destination(e1,e2) 1677 | 9677 Product-Producer(e1,e2) 1678 | 9678 Member-Collection(e2,e1) 1679 | 9679 Component-Whole(e1,e2) 1680 | 9680 Other 1681 | 9681 Member-Collection(e2,e1) 1682 | 9682 Cause-Effect(e1,e2) 1683 | 9683 Entity-Destination(e1,e2) 1684 | 9684 Cause-Effect(e2,e1) 1685 | 9685 Component-Whole(e1,e2) 1686 | 9686 Other 1687 | 9687 Instrument-Agency(e2,e1) 1688 | 9688 Cause-Effect(e1,e2) 1689 | 9689 Cause-Effect(e2,e1) 1690 | 9690 Cause-Effect(e2,e1) 1691 | 9691 Message-Topic(e1,e2) 1692 | 9692 Product-Producer(e1,e2) 1693 | 9693 Entity-Origin(e2,e1) 1694 | 9694 Content-Container(e1,e2) 1695 | 9695 Cause-Effect(e1,e2) 1696 | 9696 Instrument-Agency(e2,e1) 1697 | 9697 Component-Whole(e1,e2) 1698 | 9698 Cause-Effect(e1,e2) 1699 | 9699 Cause-Effect(e2,e1) 1700 | 9700 Other 1701 | 9701 Other 1702 | 9702 Member-Collection(e2,e1) 1703 | 9703 Cause-Effect(e2,e1) 1704 | 9704 Instrument-Agency(e2,e1) 1705 | 9705 Message-Topic(e1,e2) 1706 | 9706 Other 1707 | 9707 Component-Whole(e2,e1) 1708 | 9708 Cause-Effect(e2,e1) 1709 | 9709 Member-Collection(e1,e2) 1710 | 9710 Entity-Origin(e1,e2) 1711 | 9711 Entity-Origin(e1,e2) 1712 | 9712 Product-Producer(e2,e1) 1713 | 9713 Component-Whole(e1,e2) 1714 | 9714 Other 1715 | 9715 Component-Whole(e2,e1) 1716 | 9716 Product-Producer(e2,e1) 1717 | 9717 Member-Collection(e2,e1) 1718 | 9718 Other 1719 | 9719 Cause-Effect(e1,e2) 1720 | 9720 Cause-Effect(e2,e1) 1721 | 9721 Instrument-Agency(e2,e1) 1722 | 9722 Cause-Effect(e2,e1) 1723 | 9723 Component-Whole(e1,e2) 1724 | 9724 Entity-Origin(e1,e2) 1725 | 9725 Cause-Effect(e1,e2) 1726 | 9726 Other 1727 | 9727 Cause-Effect(e1,e2) 1728 | 9728 Message-Topic(e1,e2) 1729 | 9729 Instrument-Agency(e1,e2) 1730 | 9730 Message-Topic(e1,e2) 1731 | 9731 Cause-Effect(e1,e2) 1732 | 9732 Cause-Effect(e1,e2) 1733 | 9733 Entity-Origin(e1,e2) 1734 | 9734 Message-Topic(e2,e1) 1735 | 9735 Message-Topic(e1,e2) 1736 | 9736 Component-Whole(e2,e1) 1737 | 9737 Component-Whole(e1,e2) 1738 | 9738 Other 1739 | 9739 Cause-Effect(e2,e1) 1740 | 9740 Cause-Effect(e2,e1) 1741 | 9741 Other 1742 | 9742 Message-Topic(e2,e1) 1743 | 9743 Component-Whole(e2,e1) 1744 | 9744 Message-Topic(e2,e1) 1745 | 9745 Member-Collection(e2,e1) 1746 | 9746 Member-Collection(e2,e1) 1747 | 9747 Other 1748 | 9748 Member-Collection(e2,e1) 1749 | 9749 Message-Topic(e1,e2) 1750 | 9750 Other 1751 | 9751 Member-Collection(e1,e2) 1752 | 9752 Entity-Origin(e1,e2) 1753 | 9753 Member-Collection(e2,e1) 1754 | 9754 Content-Container(e1,e2) 1755 | 9755 Entity-Origin(e2,e1) 1756 | 9756 Message-Topic(e1,e2) 1757 | 9757 Message-Topic(e1,e2) 1758 | 9758 Cause-Effect(e2,e1) 1759 | 9759 Component-Whole(e2,e1) 1760 | 9760 Content-Container(e1,e2) 1761 | 9761 Other 1762 | 9762 Other 1763 | 9763 Component-Whole(e2,e1) 1764 | 9764 Message-Topic(e1,e2) 1765 | 9765 Member-Collection(e2,e1) 1766 | 9766 Entity-Destination(e1,e2) 1767 | 9767 Entity-Origin(e1,e2) 1768 | 9768 Instrument-Agency(e2,e1) 1769 | 9769 Cause-Effect(e2,e1) 1770 | 9770 Entity-Origin(e1,e2) 1771 | 9771 Component-Whole(e1,e2) 1772 | 9772 Entity-Destination(e1,e2) 1773 | 9773 Component-Whole(e2,e1) 1774 | 9774 Entity-Destination(e1,e2) 1775 | 9775 Entity-Destination(e1,e2) 1776 | 9776 Other 1777 | 9777 Message-Topic(e1,e2) 1778 | 9778 Product-Producer(e2,e1) 1779 | 9779 Other 1780 | 9780 Cause-Effect(e2,e1) 1781 | 9781 Entity-Destination(e1,e2) 1782 | 9782 Other 1783 | 9783 Entity-Origin(e1,e2) 1784 | 9784 Product-Producer(e2,e1) 1785 | 9785 Product-Producer(e2,e1) 1786 | 9786 Content-Container(e1,e2) 1787 | 9787 Entity-Destination(e1,e2) 1788 | 9788 Message-Topic(e1,e2) 1789 | 9789 Other 1790 | 9790 Member-Collection(e2,e1) 1791 | 9791 Cause-Effect(e1,e2) 1792 | 9792 Entity-Destination(e1,e2) 1793 | 9793 Other 1794 | 9794 Component-Whole(e1,e2) 1795 | 9795 Other 1796 | 9796 Other 1797 | 9797 Cause-Effect(e2,e1) 1798 | 9798 Entity-Destination(e1,e2) 1799 | 9799 Other 1800 | 9800 Message-Topic(e1,e2) 1801 | 9801 Entity-Origin(e1,e2) 1802 | 9802 Entity-Destination(e1,e2) 1803 | 9803 Other 1804 | 9804 Member-Collection(e2,e1) 1805 | 9805 Entity-Destination(e1,e2) 1806 | 9806 Content-Container(e1,e2) 1807 | 9807 Entity-Origin(e1,e2) 1808 | 9808 Other 1809 | 9809 Entity-Destination(e1,e2) 1810 | 9810 Content-Container(e1,e2) 1811 | 9811 Other 1812 | 9812 Cause-Effect(e1,e2) 1813 | 9813 Instrument-Agency(e1,e2) 1814 | 9814 Member-Collection(e2,e1) 1815 | 9815 Other 1816 | 9816 Instrument-Agency(e2,e1) 1817 | 9817 Message-Topic(e1,e2) 1818 | 9818 Member-Collection(e2,e1) 1819 | 9819 Cause-Effect(e2,e1) 1820 | 9820 Other 1821 | 9821 Product-Producer(e1,e2) 1822 | 9822 Product-Producer(e1,e2) 1823 | 9823 Entity-Origin(e2,e1) 1824 | 9824 Instrument-Agency(e2,e1) 1825 | 9825 Member-Collection(e2,e1) 1826 | 9826 Member-Collection(e2,e1) 1827 | 9827 Member-Collection(e2,e1) 1828 | 9828 Product-Producer(e1,e2) 1829 | 9829 Cause-Effect(e1,e2) 1830 | 9830 Entity-Origin(e1,e2) 1831 | 9831 Cause-Effect(e1,e2) 1832 | 9832 Entity-Origin(e1,e2) 1833 | 9833 Other 1834 | 9834 Component-Whole(e1,e2) 1835 | 9835 Content-Container(e1,e2) 1836 | 9836 Product-Producer(e2,e1) 1837 | 9837 Instrument-Agency(e1,e2) 1838 | 9838 Member-Collection(e2,e1) 1839 | 9839 Other 1840 | 9840 Message-Topic(e1,e2) 1841 | 9841 Member-Collection(e2,e1) 1842 | 9842 Other 1843 | 9843 Other 1844 | 9844 Entity-Origin(e1,e2) 1845 | 9845 Component-Whole(e2,e1) 1846 | 9846 Product-Producer(e2,e1) 1847 | 9847 Other 1848 | 9848 Cause-Effect(e2,e1) 1849 | 9849 Other 1850 | 9850 Product-Producer(e2,e1) 1851 | 9851 Member-Collection(e2,e1) 1852 | 9852 Entity-Origin(e1,e2) 1853 | 9853 Other 1854 | 9854 Member-Collection(e2,e1) 1855 | 9855 Entity-Destination(e1,e2) 1856 | 9856 Content-Container(e1,e2) 1857 | 9857 Component-Whole(e2,e1) 1858 | 9858 Product-Producer(e1,e2) 1859 | 9859 Content-Container(e2,e1) 1860 | 9860 Entity-Origin(e1,e2) 1861 | 9861 Cause-Effect(e2,e1) 1862 | 9862 Entity-Origin(e1,e2) 1863 | 9863 Product-Producer(e2,e1) 1864 | 9864 Product-Producer(e2,e1) 1865 | 9865 Entity-Destination(e1,e2) 1866 | 9866 Member-Collection(e2,e1) 1867 | 9867 Other 1868 | 9868 Cause-Effect(e2,e1) 1869 | 9869 Other 1870 | 9870 Product-Producer(e1,e2) 1871 | 9871 Entity-Destination(e1,e2) 1872 | 9872 Other 1873 | 9873 Entity-Destination(e1,e2) 1874 | 9874 Entity-Destination(e1,e2) 1875 | 9875 Member-Collection(e1,e2) 1876 | 9876 Cause-Effect(e2,e1) 1877 | 9877 Other 1878 | 9878 Member-Collection(e2,e1) 1879 | 9879 Other 1880 | 9880 Content-Container(e1,e2) 1881 | 9881 Member-Collection(e2,e1) 1882 | 9882 Entity-Origin(e1,e2) 1883 | 9883 Entity-Destination(e1,e2) 1884 | 9884 Instrument-Agency(e2,e1) 1885 | 9885 Message-Topic(e1,e2) 1886 | 9886 Other 1887 | 9887 Member-Collection(e2,e1) 1888 | 9888 Member-Collection(e1,e2) 1889 | 9889 Instrument-Agency(e2,e1) 1890 | 9890 Member-Collection(e2,e1) 1891 | 9891 Member-Collection(e2,e1) 1892 | 9892 Other 1893 | 9893 Component-Whole(e2,e1) 1894 | 9894 Entity-Destination(e1,e2) 1895 | 9895 Product-Producer(e2,e1) 1896 | 9896 Content-Container(e1,e2) 1897 | 9897 Other 1898 | 9898 Entity-Destination(e1,e2) 1899 | 9899 Cause-Effect(e2,e1) 1900 | 9900 Entity-Destination(e1,e2) 1901 | 9901 Cause-Effect(e2,e1) 1902 | 9902 Cause-Effect(e1,e2) 1903 | 9903 Other 1904 | 9904 Entity-Origin(e1,e2) 1905 | 9905 Other 1906 | 9906 Component-Whole(e2,e1) 1907 | 9907 Product-Producer(e1,e2) 1908 | 9908 Other 1909 | 9909 Product-Producer(e2,e1) 1910 | 9910 Member-Collection(e1,e2) 1911 | 9911 Message-Topic(e1,e2) 1912 | 9912 Instrument-Agency(e1,e2) 1913 | 9913 Content-Container(e1,e2) 1914 | 9914 Content-Container(e1,e2) 1915 | 9915 Other 1916 | 9916 Other 1917 | 9917 Product-Producer(e2,e1) 1918 | 9918 Member-Collection(e2,e1) 1919 | 9919 Cause-Effect(e2,e1) 1920 | 9920 Product-Producer(e2,e1) 1921 | 9921 Component-Whole(e1,e2) 1922 | 9922 Entity-Origin(e2,e1) 1923 | 9923 Member-Collection(e2,e1) 1924 | 9924 Other 1925 | 9925 Component-Whole(e1,e2) 1926 | 9926 Product-Producer(e2,e1) 1927 | 9927 Component-Whole(e1,e2) 1928 | 9928 Component-Whole(e1,e2) 1929 | 9929 Content-Container(e1,e2) 1930 | 9930 Other 1931 | 9931 Entity-Destination(e1,e2) 1932 | 9932 Content-Container(e1,e2) 1933 | 9933 Product-Producer(e2,e1) 1934 | 9934 Component-Whole(e1,e2) 1935 | 9935 Product-Producer(e2,e1) 1936 | 9936 Entity-Destination(e1,e2) 1937 | 9937 Member-Collection(e2,e1) 1938 | 9938 Member-Collection(e2,e1) 1939 | 9939 Entity-Destination(e1,e2) 1940 | 9940 Content-Container(e2,e1) 1941 | 9941 Entity-Destination(e1,e2) 1942 | 9942 Content-Container(e2,e1) 1943 | 9943 Other 1944 | 9944 Message-Topic(e1,e2) 1945 | 9945 Component-Whole(e2,e1) 1946 | 9946 Message-Topic(e2,e1) 1947 | 9947 Product-Producer(e2,e1) 1948 | 9948 Entity-Destination(e1,e2) 1949 | 9949 Entity-Origin(e1,e2) 1950 | 9950 Other 1951 | 9951 Message-Topic(e1,e2) 1952 | 9952 Entity-Destination(e1,e2) 1953 | 9953 Entity-Destination(e1,e2) 1954 | 9954 Entity-Origin(e1,e2) 1955 | 9955 Content-Container(e2,e1) 1956 | 9956 Cause-Effect(e2,e1) 1957 | 9957 Component-Whole(e2,e1) 1958 | 9958 Entity-Origin(e1,e2) 1959 | 9959 Instrument-Agency(e2,e1) 1960 | 9960 Member-Collection(e2,e1) 1961 | 9961 Product-Producer(e2,e1) 1962 | 9962 Entity-Origin(e1,e2) 1963 | 9963 Entity-Destination(e1,e2) 1964 | 9964 Entity-Destination(e1,e2) 1965 | 9965 Cause-Effect(e2,e1) 1966 | 9966 Other 1967 | 9967 Cause-Effect(e1,e2) 1968 | 9968 Message-Topic(e2,e1) 1969 | 9969 Entity-Destination(e1,e2) 1970 | 9970 Instrument-Agency(e2,e1) 1971 | 9971 Component-Whole(e2,e1) 1972 | 9972 Component-Whole(e1,e2) 1973 | 9973 Message-Topic(e1,e2) 1974 | 9974 Cause-Effect(e2,e1) 1975 | 9975 Cause-Effect(e2,e1) 1976 | 9976 Other 1977 | 9977 Product-Producer(e2,e1) 1978 | 9978 Other 1979 | 9979 Cause-Effect(e1,e2) 1980 | 9980 Component-Whole(e1,e2) 1981 | 9981 Member-Collection(e2,e1) 1982 | 9982 Entity-Destination(e1,e2) 1983 | 9983 Content-Container(e1,e2) 1984 | 9984 Member-Collection(e2,e1) 1985 | 9985 Cause-Effect(e2,e1) 1986 | 9986 Other 1987 | 9987 Product-Producer(e2,e1) 1988 | 9988 Content-Container(e1,e2) 1989 | 9989 Other 1990 | 9990 Other 1991 | 9991 Message-Topic(e1,e2) 1992 | 9992 Component-Whole(e1,e2) 1993 | 9993 Content-Container(e1,e2) 1994 | 9994 Component-Whole(e1,e2) 1995 | 9995 Other 1996 | 9996 Message-Topic(e1,e2) 1997 | 9997 Component-Whole(e1,e2) 1998 | 9998 Entity-Origin(e1,e2) 1999 | 9999 Entity-Destination(e1,e2) 2000 | 10000 Instrument-Agency(e2,e1) 2001 | 10001 Instrument-Agency(e2,e1) 2002 | 10002 Message-Topic(e1,e2) 2003 | 10003 Cause-Effect(e2,e1) 2004 | 10004 Entity-Destination(e1,e2) 2005 | 10005 Instrument-Agency(e2,e1) 2006 | 10006 Member-Collection(e2,e1) 2007 | 10007 Entity-Origin(e1,e2) 2008 | 10008 Entity-Destination(e1,e2) 2009 | 10009 Cause-Effect(e1,e2) 2010 | 10010 Entity-Origin(e1,e2) 2011 | 10011 Other 2012 | 10012 Cause-Effect(e2,e1) 2013 | 10013 Member-Collection(e2,e1) 2014 | 10014 Entity-Destination(e1,e2) 2015 | 10015 Other 2016 | 10016 Content-Container(e1,e2) 2017 | 10017 Entity-Destination(e1,e2) 2018 | 10018 Entity-Origin(e1,e2) 2019 | 10019 Other 2020 | 10020 Entity-Destination(e1,e2) 2021 | 10021 Other 2022 | 10022 Other 2023 | 10023 Message-Topic(e1,e2) 2024 | 10024 Message-Topic(e1,e2) 2025 | 10025 Other 2026 | 10026 Instrument-Agency(e2,e1) 2027 | 10027 Entity-Destination(e1,e2) 2028 | 10028 Message-Topic(e1,e2) 2029 | 10029 Member-Collection(e2,e1) 2030 | 10030 Other 2031 | 10031 Member-Collection(e2,e1) 2032 | 10032 Member-Collection(e2,e1) 2033 | 10033 Other 2034 | 10034 Content-Container(e1,e2) 2035 | 10035 Component-Whole(e2,e1) 2036 | 10036 Other 2037 | 10037 Entity-Destination(e1,e2) 2038 | 10038 Cause-Effect(e2,e1) 2039 | 10039 Entity-Destination(e1,e2) 2040 | 10040 Cause-Effect(e2,e1) 2041 | 10041 Cause-Effect(e2,e1) 2042 | 10042 Message-Topic(e2,e1) 2043 | 10043 Entity-Destination(e1,e2) 2044 | 10044 Component-Whole(e2,e1) 2045 | 10045 Component-Whole(e2,e1) 2046 | 10046 Entity-Destination(e1,e2) 2047 | 10047 Cause-Effect(e1,e2) 2048 | 10048 Instrument-Agency(e2,e1) 2049 | 10049 Message-Topic(e1,e2) 2050 | 10050 Content-Container(e2,e1) 2051 | 10051 Component-Whole(e2,e1) 2052 | 10052 Member-Collection(e2,e1) 2053 | 10053 Content-Container(e1,e2) 2054 | 10054 Cause-Effect(e2,e1) 2055 | 10055 Entity-Destination(e1,e2) 2056 | 10056 Entity-Destination(e1,e2) 2057 | 10057 Instrument-Agency(e2,e1) 2058 | 10058 Member-Collection(e1,e2) 2059 | 10059 Cause-Effect(e2,e1) 2060 | 10060 Other 2061 | 10061 Other 2062 | 10062 Content-Container(e1,e2) 2063 | 10063 Component-Whole(e2,e1) 2064 | 10064 Cause-Effect(e1,e2) 2065 | 10065 Content-Container(e1,e2) 2066 | 10066 Other 2067 | 10067 Entity-Origin(e1,e2) 2068 | 10068 Entity-Destination(e1,e2) 2069 | 10069 Other 2070 | 10070 Component-Whole(e1,e2) 2071 | 10071 Entity-Origin(e1,e2) 2072 | 10072 Content-Container(e2,e1) 2073 | 10073 Other 2074 | 10074 Entity-Origin(e1,e2) 2075 | 10075 Entity-Origin(e1,e2) 2076 | 10076 Product-Producer(e1,e2) 2077 | 10077 Entity-Destination(e1,e2) 2078 | 10078 Entity-Destination(e1,e2) 2079 | 10079 Product-Producer(e2,e1) 2080 | 10080 Entity-Origin(e2,e1) 2081 | 10081 Entity-Destination(e1,e2) 2082 | 10082 Entity-Origin(e1,e2) 2083 | 10083 Component-Whole(e1,e2) 2084 | 10084 Entity-Origin(e1,e2) 2085 | 10085 Entity-Destination(e1,e2) 2086 | 10086 Cause-Effect(e1,e2) 2087 | 10087 Entity-Destination(e1,e2) 2088 | 10088 Instrument-Agency(e2,e1) 2089 | 10089 Product-Producer(e2,e1) 2090 | 10090 Cause-Effect(e1,e2) 2091 | 10091 Entity-Origin(e2,e1) 2092 | 10092 Entity-Origin(e1,e2) 2093 | 10093 Other 2094 | 10094 Content-Container(e1,e2) 2095 | 10095 Entity-Destination(e1,e2) 2096 | 10096 Component-Whole(e2,e1) 2097 | 10097 Other 2098 | 10098 Message-Topic(e1,e2) 2099 | 10099 Entity-Destination(e1,e2) 2100 | 10100 Entity-Destination(e1,e2) 2101 | 10101 Entity-Origin(e2,e1) 2102 | 10102 Cause-Effect(e1,e2) 2103 | 10103 Message-Topic(e1,e2) 2104 | 10104 Member-Collection(e2,e1) 2105 | 10105 Member-Collection(e2,e1) 2106 | 10106 Component-Whole(e2,e1) 2107 | 10107 Content-Container(e1,e2) 2108 | 10108 Message-Topic(e1,e2) 2109 | 10109 Other 2110 | 10110 Message-Topic(e1,e2) 2111 | 10111 Other 2112 | 10112 Other 2113 | 10113 Product-Producer(e2,e1) 2114 | 10114 Message-Topic(e2,e1) 2115 | 10115 Message-Topic(e1,e2) 2116 | 10116 Entity-Origin(e2,e1) 2117 | 10117 Product-Producer(e2,e1) 2118 | 10118 Cause-Effect(e1,e2) 2119 | 10119 Member-Collection(e2,e1) 2120 | 10120 Component-Whole(e2,e1) 2121 | 10121 Entity-Destination(e1,e2) 2122 | 10122 Entity-Origin(e1,e2) 2123 | 10123 Message-Topic(e1,e2) 2124 | 10124 Other 2125 | 10125 Other 2126 | 10126 Member-Collection(e2,e1) 2127 | 10127 Other 2128 | 10128 Instrument-Agency(e1,e2) 2129 | 10129 Other 2130 | 10130 Other 2131 | 10131 Product-Producer(e1,e2) 2132 | 10132 Component-Whole(e2,e1) 2133 | 10133 Instrument-Agency(e2,e1) 2134 | 10134 Cause-Effect(e2,e1) 2135 | 10135 Component-Whole(e2,e1) 2136 | 10136 Entity-Origin(e1,e2) 2137 | 10137 Message-Topic(e1,e2) 2138 | 10138 Entity-Origin(e1,e2) 2139 | 10139 Entity-Origin(e1,e2) 2140 | 10140 Product-Producer(e2,e1) 2141 | 10141 Other 2142 | 10142 Product-Producer(e2,e1) 2143 | 10143 Other 2144 | 10144 Instrument-Agency(e2,e1) 2145 | 10145 Instrument-Agency(e1,e2) 2146 | 10146 Product-Producer(e1,e2) 2147 | 10147 Component-Whole(e2,e1) 2148 | 10148 Product-Producer(e2,e1) 2149 | 10149 Instrument-Agency(e2,e1) 2150 | 10150 Component-Whole(e1,e2) 2151 | 10151 Product-Producer(e2,e1) 2152 | 10152 Instrument-Agency(e1,e2) 2153 | 10153 Product-Producer(e2,e1) 2154 | 10154 Member-Collection(e2,e1) 2155 | 10155 Message-Topic(e1,e2) 2156 | 10156 Cause-Effect(e1,e2) 2157 | 10157 Component-Whole(e1,e2) 2158 | 10158 Entity-Destination(e1,e2) 2159 | 10159 Other 2160 | 10160 Other 2161 | 10161 Component-Whole(e1,e2) 2162 | 10162 Entity-Origin(e1,e2) 2163 | 10163 Entity-Origin(e2,e1) 2164 | 10164 Entity-Origin(e1,e2) 2165 | 10165 Entity-Destination(e1,e2) 2166 | 10166 Component-Whole(e1,e2) 2167 | 10167 Entity-Origin(e1,e2) 2168 | 10168 Content-Container(e1,e2) 2169 | 10169 Member-Collection(e2,e1) 2170 | 10170 Entity-Origin(e1,e2) 2171 | 10171 Content-Container(e2,e1) 2172 | 10172 Message-Topic(e2,e1) 2173 | 10173 Other 2174 | 10174 Member-Collection(e2,e1) 2175 | 10175 Entity-Destination(e1,e2) 2176 | 10176 Product-Producer(e2,e1) 2177 | 10177 Cause-Effect(e2,e1) 2178 | 10178 Entity-Destination(e1,e2) 2179 | 10179 Product-Producer(e1,e2) 2180 | 10180 Instrument-Agency(e2,e1) 2181 | 10181 Other 2182 | 10182 Cause-Effect(e2,e1) 2183 | 10183 Message-Topic(e1,e2) 2184 | 10184 Entity-Destination(e1,e2) 2185 | 10185 Entity-Origin(e1,e2) 2186 | 10186 Other 2187 | 10187 Entity-Destination(e1,e2) 2188 | 10188 Other 2189 | 10189 Message-Topic(e1,e2) 2190 | 10190 Product-Producer(e1,e2) 2191 | 10191 Entity-Destination(e1,e2) 2192 | 10192 Product-Producer(e2,e1) 2193 | 10193 Component-Whole(e1,e2) 2194 | 10194 Entity-Origin(e1,e2) 2195 | 10195 Instrument-Agency(e2,e1) 2196 | 10196 Other 2197 | 10197 Product-Producer(e1,e2) 2198 | 10198 Entity-Origin(e1,e2) 2199 | 10199 Entity-Origin(e1,e2) 2200 | 10200 Entity-Origin(e1,e2) 2201 | 10201 Instrument-Agency(e2,e1) 2202 | 10202 Entity-Destination(e1,e2) 2203 | 10203 Instrument-Agency(e2,e1) 2204 | 10204 Message-Topic(e1,e2) 2205 | 10205 Product-Producer(e2,e1) 2206 | 10206 Product-Producer(e2,e1) 2207 | 10207 Entity-Destination(e1,e2) 2208 | 10208 Component-Whole(e1,e2) 2209 | 10209 Cause-Effect(e2,e1) 2210 | 10210 Component-Whole(e2,e1) 2211 | 10211 Message-Topic(e1,e2) 2212 | 10212 Component-Whole(e1,e2) 2213 | 10213 Other 2214 | 10214 Component-Whole(e1,e2) 2215 | 10215 Entity-Origin(e1,e2) 2216 | 10216 Message-Topic(e1,e2) 2217 | 10217 Other 2218 | 10218 Entity-Origin(e2,e1) 2219 | 10219 Content-Container(e1,e2) 2220 | 10220 Message-Topic(e1,e2) 2221 | 10221 Entity-Origin(e1,e2) 2222 | 10222 Entity-Origin(e1,e2) 2223 | 10223 Member-Collection(e2,e1) 2224 | 10224 Product-Producer(e2,e1) 2225 | 10225 Member-Collection(e2,e1) 2226 | 10226 Entity-Destination(e1,e2) 2227 | 10227 Content-Container(e1,e2) 2228 | 10228 Cause-Effect(e2,e1) 2229 | 10229 Member-Collection(e1,e2) 2230 | 10230 Cause-Effect(e2,e1) 2231 | 10231 Entity-Destination(e1,e2) 2232 | 10232 Content-Container(e1,e2) 2233 | 10233 Other 2234 | 10234 Product-Producer(e2,e1) 2235 | 10235 Instrument-Agency(e2,e1) 2236 | 10236 Message-Topic(e1,e2) 2237 | 10237 Product-Producer(e2,e1) 2238 | 10238 Member-Collection(e2,e1) 2239 | 10239 Member-Collection(e2,e1) 2240 | 10240 Entity-Destination(e1,e2) 2241 | 10241 Instrument-Agency(e2,e1) 2242 | 10242 Message-Topic(e2,e1) 2243 | 10243 Instrument-Agency(e2,e1) 2244 | 10244 Other 2245 | 10245 Entity-Destination(e1,e2) 2246 | 10246 Cause-Effect(e2,e1) 2247 | 10247 Message-Topic(e1,e2) 2248 | 10248 Content-Container(e2,e1) 2249 | 10249 Instrument-Agency(e2,e1) 2250 | 10250 Product-Producer(e1,e2) 2251 | 10251 Other 2252 | 10252 Instrument-Agency(e2,e1) 2253 | 10253 Message-Topic(e1,e2) 2254 | 10254 Cause-Effect(e2,e1) 2255 | 10255 Entity-Destination(e1,e2) 2256 | 10256 Content-Container(e2,e1) 2257 | 10257 Cause-Effect(e1,e2) 2258 | 10258 Cause-Effect(e2,e1) 2259 | 10259 Message-Topic(e2,e1) 2260 | 10260 Entity-Origin(e1,e2) 2261 | 10261 Other 2262 | 10262 Other 2263 | 10263 Entity-Destination(e1,e2) 2264 | 10264 Component-Whole(e2,e1) 2265 | 10265 Message-Topic(e1,e2) 2266 | 10266 Product-Producer(e2,e1) 2267 | 10267 Cause-Effect(e1,e2) 2268 | 10268 Member-Collection(e2,e1) 2269 | 10269 Message-Topic(e1,e2) 2270 | 10270 Product-Producer(e2,e1) 2271 | 10271 Entity-Origin(e1,e2) 2272 | 10272 Component-Whole(e1,e2) 2273 | 10273 Entity-Origin(e1,e2) 2274 | 10274 Component-Whole(e2,e1) 2275 | 10275 Cause-Effect(e1,e2) 2276 | 10276 Entity-Destination(e1,e2) 2277 | 10277 Component-Whole(e1,e2) 2278 | 10278 Product-Producer(e1,e2) 2279 | 10279 Cause-Effect(e2,e1) 2280 | 10280 Entity-Destination(e1,e2) 2281 | 10281 Cause-Effect(e2,e1) 2282 | 10282 Other 2283 | 10283 Entity-Origin(e2,e1) 2284 | 10284 Entity-Destination(e1,e2) 2285 | 10285 Cause-Effect(e2,e1) 2286 | 10286 Content-Container(e1,e2) 2287 | 10287 Content-Container(e1,e2) 2288 | 10288 Component-Whole(e2,e1) 2289 | 10289 Member-Collection(e2,e1) 2290 | 10290 Content-Container(e1,e2) 2291 | 10291 Other 2292 | 10292 Message-Topic(e1,e2) 2293 | 10293 Entity-Destination(e1,e2) 2294 | 10294 Instrument-Agency(e1,e2) 2295 | 10295 Message-Topic(e2,e1) 2296 | 10296 Cause-Effect(e2,e1) 2297 | 10297 Entity-Origin(e1,e2) 2298 | 10298 Entity-Origin(e2,e1) 2299 | 10299 Entity-Origin(e1,e2) 2300 | 10300 Other 2301 | 10301 Member-Collection(e2,e1) 2302 | 10302 Message-Topic(e1,e2) 2303 | 10303 Entity-Destination(e1,e2) 2304 | 10304 Instrument-Agency(e2,e1) 2305 | 10305 Component-Whole(e1,e2) 2306 | 10306 Component-Whole(e2,e1) 2307 | 10307 Component-Whole(e1,e2) 2308 | 10308 Other 2309 | 10309 Message-Topic(e1,e2) 2310 | 10310 Message-Topic(e1,e2) 2311 | 10311 Component-Whole(e2,e1) 2312 | 10312 Content-Container(e2,e1) 2313 | 10313 Product-Producer(e1,e2) 2314 | 10314 Content-Container(e1,e2) 2315 | 10315 Component-Whole(e1,e2) 2316 | 10316 Content-Container(e1,e2) 2317 | 10317 Other 2318 | 10318 Other 2319 | 10319 Member-Collection(e2,e1) 2320 | 10320 Instrument-Agency(e2,e1) 2321 | 10321 Entity-Destination(e1,e2) 2322 | 10322 Component-Whole(e1,e2) 2323 | 10323 Other 2324 | 10324 Message-Topic(e1,e2) 2325 | 10325 Content-Container(e1,e2) 2326 | 10326 Other 2327 | 10327 Content-Container(e1,e2) 2328 | 10328 Product-Producer(e1,e2) 2329 | 10329 Instrument-Agency(e2,e1) 2330 | 10330 Entity-Destination(e1,e2) 2331 | 10331 Instrument-Agency(e2,e1) 2332 | 10332 Content-Container(e1,e2) 2333 | 10333 Other 2334 | 10334 Other 2335 | 10335 Cause-Effect(e2,e1) 2336 | 10336 Entity-Origin(e1,e2) 2337 | 10337 Content-Container(e1,e2) 2338 | 10338 Entity-Origin(e1,e2) 2339 | 10339 Other 2340 | 10340 Entity-Origin(e1,e2) 2341 | 10341 Other 2342 | 10342 Entity-Destination(e1,e2) 2343 | 10343 Instrument-Agency(e2,e1) 2344 | 10344 Cause-Effect(e2,e1) 2345 | 10345 Component-Whole(e2,e1) 2346 | 10346 Instrument-Agency(e2,e1) 2347 | 10347 Content-Container(e2,e1) 2348 | 10348 Entity-Destination(e1,e2) 2349 | 10349 Member-Collection(e2,e1) 2350 | 10350 Cause-Effect(e1,e2) 2351 | 10351 Entity-Destination(e1,e2) 2352 | 10352 Message-Topic(e2,e1) 2353 | 10353 Product-Producer(e2,e1) 2354 | 10354 Entity-Destination(e1,e2) 2355 | 10355 Content-Container(e1,e2) 2356 | 10356 Entity-Origin(e1,e2) 2357 | 10357 Entity-Origin(e1,e2) 2358 | 10358 Component-Whole(e1,e2) 2359 | 10359 Other 2360 | 10360 Message-Topic(e1,e2) 2361 | 10361 Instrument-Agency(e1,e2) 2362 | 10362 Entity-Destination(e1,e2) 2363 | 10363 Entity-Destination(e1,e2) 2364 | 10364 Product-Producer(e2,e1) 2365 | 10365 Message-Topic(e1,e2) 2366 | 10366 Member-Collection(e2,e1) 2367 | 10367 Product-Producer(e2,e1) 2368 | 10368 Instrument-Agency(e2,e1) 2369 | 10369 Instrument-Agency(e2,e1) 2370 | 10370 Other 2371 | 10371 Product-Producer(e1,e2) 2372 | 10372 Product-Producer(e1,e2) 2373 | 10373 Cause-Effect(e1,e2) 2374 | 10374 Content-Container(e1,e2) 2375 | 10375 Member-Collection(e2,e1) 2376 | 10376 Entity-Destination(e1,e2) 2377 | 10377 Message-Topic(e1,e2) 2378 | 10378 Entity-Origin(e2,e1) 2379 | 10379 Cause-Effect(e1,e2) 2380 | 10380 Component-Whole(e2,e1) 2381 | 10381 Message-Topic(e1,e2) 2382 | 10382 Cause-Effect(e2,e1) 2383 | 10383 Cause-Effect(e1,e2) 2384 | 10384 Entity-Origin(e1,e2) 2385 | 10385 Instrument-Agency(e2,e1) 2386 | 10386 Component-Whole(e2,e1) 2387 | 10387 Component-Whole(e2,e1) 2388 | 10388 Product-Producer(e1,e2) 2389 | 10389 Component-Whole(e1,e2) 2390 | 10390 Other 2391 | 10391 Instrument-Agency(e2,e1) 2392 | 10392 Message-Topic(e1,e2) 2393 | 10393 Entity-Origin(e1,e2) 2394 | 10394 Other 2395 | 10395 Message-Topic(e1,e2) 2396 | 10396 Cause-Effect(e2,e1) 2397 | 10397 Entity-Origin(e1,e2) 2398 | 10398 Cause-Effect(e1,e2) 2399 | 10399 Entity-Destination(e1,e2) 2400 | 10400 Component-Whole(e1,e2) 2401 | 10401 Member-Collection(e2,e1) 2402 | 10402 Other 2403 | 10403 Entity-Origin(e1,e2) 2404 | 10404 Member-Collection(e2,e1) 2405 | 10405 Entity-Destination(e1,e2) 2406 | 10406 Other 2407 | 10407 Product-Producer(e2,e1) 2408 | 10408 Member-Collection(e2,e1) 2409 | 10409 Product-Producer(e1,e2) 2410 | 10410 Other 2411 | 10411 Other 2412 | 10412 Product-Producer(e2,e1) 2413 | 10413 Entity-Destination(e1,e2) 2414 | 10414 Message-Topic(e2,e1) 2415 | 10415 Entity-Destination(e1,e2) 2416 | 10416 Member-Collection(e2,e1) 2417 | 10417 Cause-Effect(e2,e1) 2418 | 10418 Entity-Destination(e1,e2) 2419 | 10419 Cause-Effect(e2,e1) 2420 | 10420 Other 2421 | 10421 Entity-Destination(e1,e2) 2422 | 10422 Message-Topic(e1,e2) 2423 | 10423 Entity-Origin(e1,e2) 2424 | 10424 Instrument-Agency(e2,e1) 2425 | 10425 Cause-Effect(e2,e1) 2426 | 10426 Cause-Effect(e2,e1) 2427 | 10427 Other 2428 | 10428 Component-Whole(e1,e2) 2429 | 10429 Message-Topic(e1,e2) 2430 | 10430 Member-Collection(e2,e1) 2431 | 10431 Content-Container(e1,e2) 2432 | 10432 Content-Container(e1,e2) 2433 | 10433 Component-Whole(e2,e1) 2434 | 10434 Cause-Effect(e1,e2) 2435 | 10435 Component-Whole(e1,e2) 2436 | 10436 Entity-Destination(e1,e2) 2437 | 10437 Message-Topic(e1,e2) 2438 | 10438 Other 2439 | 10439 Other 2440 | 10440 Product-Producer(e1,e2) 2441 | 10441 Member-Collection(e1,e2) 2442 | 10442 Entity-Destination(e1,e2) 2443 | 10443 Content-Container(e1,e2) 2444 | 10444 Instrument-Agency(e2,e1) 2445 | 10445 Content-Container(e1,e2) 2446 | 10446 Entity-Destination(e1,e2) 2447 | 10447 Other 2448 | 10448 Product-Producer(e1,e2) 2449 | 10449 Member-Collection(e2,e1) 2450 | 10450 Other 2451 | 10451 Component-Whole(e2,e1) 2452 | 10452 Other 2453 | 10453 Entity-Destination(e1,e2) 2454 | 10454 Message-Topic(e1,e2) 2455 | 10455 Product-Producer(e1,e2) 2456 | 10456 Entity-Destination(e1,e2) 2457 | 10457 Message-Topic(e1,e2) 2458 | 10458 Other 2459 | 10459 Other 2460 | 10460 Component-Whole(e2,e1) 2461 | 10461 Product-Producer(e2,e1) 2462 | 10462 Content-Container(e1,e2) 2463 | 10463 Entity-Destination(e1,e2) 2464 | 10464 Product-Producer(e2,e1) 2465 | 10465 Message-Topic(e1,e2) 2466 | 10466 Cause-Effect(e2,e1) 2467 | 10467 Entity-Destination(e1,e2) 2468 | 10468 Cause-Effect(e1,e2) 2469 | 10469 Component-Whole(e1,e2) 2470 | 10470 Content-Container(e1,e2) 2471 | 10471 Entity-Origin(e1,e2) 2472 | 10472 Message-Topic(e1,e2) 2473 | 10473 Product-Producer(e1,e2) 2474 | 10474 Entity-Origin(e1,e2) 2475 | 10475 Member-Collection(e2,e1) 2476 | 10476 Content-Container(e1,e2) 2477 | 10477 Content-Container(e1,e2) 2478 | 10478 Entity-Destination(e1,e2) 2479 | 10479 Content-Container(e1,e2) 2480 | 10480 Entity-Origin(e2,e1) 2481 | 10481 Cause-Effect(e2,e1) 2482 | 10482 Product-Producer(e1,e2) 2483 | 10483 Component-Whole(e1,e2) 2484 | 10484 Component-Whole(e1,e2) 2485 | 10485 Other 2486 | 10486 Message-Topic(e1,e2) 2487 | 10487 Other 2488 | 10488 Entity-Destination(e1,e2) 2489 | 10489 Component-Whole(e2,e1) 2490 | 10490 Entity-Origin(e2,e1) 2491 | 10491 Instrument-Agency(e2,e1) 2492 | 10492 Other 2493 | 10493 Cause-Effect(e1,e2) 2494 | 10494 Other 2495 | 10495 Content-Container(e2,e1) 2496 | 10496 Product-Producer(e2,e1) 2497 | 10497 Component-Whole(e2,e1) 2498 | 10498 Content-Container(e2,e1) 2499 | 10499 Other 2500 | 10500 Cause-Effect(e2,e1) 2501 | 10501 Cause-Effect(e2,e1) 2502 | 10502 Component-Whole(e1,e2) 2503 | 10503 Component-Whole(e1,e2) 2504 | 10504 Cause-Effect(e1,e2) 2505 | 10505 Cause-Effect(e1,e2) 2506 | 10506 Instrument-Agency(e2,e1) 2507 | 10507 Entity-Origin(e2,e1) 2508 | 10508 Product-Producer(e1,e2) 2509 | 10509 Entity-Destination(e1,e2) 2510 | 10510 Component-Whole(e1,e2) 2511 | 10511 Product-Producer(e1,e2) 2512 | 10512 Other 2513 | 10513 Other 2514 | 10514 Entity-Origin(e1,e2) 2515 | 10515 Member-Collection(e2,e1) 2516 | 10516 Product-Producer(e2,e1) 2517 | 10517 Other 2518 | 10518 Message-Topic(e1,e2) 2519 | 10519 Entity-Destination(e1,e2) 2520 | 10520 Member-Collection(e2,e1) 2521 | 10521 Other 2522 | 10522 Other 2523 | 10523 Cause-Effect(e2,e1) 2524 | 10524 Cause-Effect(e2,e1) 2525 | 10525 Member-Collection(e2,e1) 2526 | 10526 Component-Whole(e1,e2) 2527 | 10527 Member-Collection(e2,e1) 2528 | 10528 Cause-Effect(e2,e1) 2529 | 10529 Component-Whole(e1,e2) 2530 | 10530 Content-Container(e1,e2) 2531 | 10531 Message-Topic(e2,e1) 2532 | 10532 Entity-Origin(e1,e2) 2533 | 10533 Message-Topic(e1,e2) 2534 | 10534 Other 2535 | 10535 Message-Topic(e1,e2) 2536 | 10536 Component-Whole(e1,e2) 2537 | 10537 Product-Producer(e2,e1) 2538 | 10538 Entity-Origin(e1,e2) 2539 | 10539 Product-Producer(e1,e2) 2540 | 10540 Entity-Destination(e1,e2) 2541 | 10541 Entity-Origin(e1,e2) 2542 | 10542 Component-Whole(e2,e1) 2543 | 10543 Entity-Origin(e1,e2) 2544 | 10544 Cause-Effect(e1,e2) 2545 | 10545 Cause-Effect(e2,e1) 2546 | 10546 Other 2547 | 10547 Component-Whole(e2,e1) 2548 | 10548 Component-Whole(e1,e2) 2549 | 10549 Product-Producer(e1,e2) 2550 | 10550 Instrument-Agency(e2,e1) 2551 | 10551 Cause-Effect(e2,e1) 2552 | 10552 Cause-Effect(e1,e2) 2553 | 10553 Product-Producer(e2,e1) 2554 | 10554 Product-Producer(e2,e1) 2555 | 10555 Content-Container(e1,e2) 2556 | 10556 Component-Whole(e2,e1) 2557 | 10557 Entity-Destination(e1,e2) 2558 | 10558 Message-Topic(e1,e2) 2559 | 10559 Entity-Destination(e1,e2) 2560 | 10560 Member-Collection(e2,e1) 2561 | 10561 Other 2562 | 10562 Other 2563 | 10563 Product-Producer(e2,e1) 2564 | 10564 Entity-Destination(e1,e2) 2565 | 10565 Product-Producer(e1,e2) 2566 | 10566 Entity-Destination(e1,e2) 2567 | 10567 Other 2568 | 10568 Other 2569 | 10569 Product-Producer(e1,e2) 2570 | 10570 Message-Topic(e1,e2) 2571 | 10571 Other 2572 | 10572 Other 2573 | 10573 Entity-Origin(e1,e2) 2574 | 10574 Other 2575 | 10575 Content-Container(e1,e2) 2576 | 10576 Product-Producer(e2,e1) 2577 | 10577 Cause-Effect(e1,e2) 2578 | 10578 Cause-Effect(e2,e1) 2579 | 10579 Content-Container(e1,e2) 2580 | 10580 Member-Collection(e2,e1) 2581 | 10581 Component-Whole(e2,e1) 2582 | 10582 Member-Collection(e2,e1) 2583 | 10583 Instrument-Agency(e2,e1) 2584 | 10584 Cause-Effect(e1,e2) 2585 | 10585 Product-Producer(e1,e2) 2586 | 10586 Component-Whole(e1,e2) 2587 | 10587 Entity-Origin(e2,e1) 2588 | 10588 Member-Collection(e2,e1) 2589 | 10589 Other 2590 | 10590 Entity-Destination(e1,e2) 2591 | 10591 Component-Whole(e2,e1) 2592 | 10592 Component-Whole(e1,e2) 2593 | 10593 Other 2594 | 10594 Entity-Origin(e1,e2) 2595 | 10595 Other 2596 | 10596 Message-Topic(e1,e2) 2597 | 10597 Cause-Effect(e1,e2) 2598 | 10598 Other 2599 | 10599 Cause-Effect(e2,e1) 2600 | 10600 Product-Producer(e1,e2) 2601 | 10601 Other 2602 | 10602 Entity-Destination(e1,e2) 2603 | 10603 Other 2604 | 10604 Component-Whole(e2,e1) 2605 | 10605 Cause-Effect(e1,e2) 2606 | 10606 Cause-Effect(e1,e2) 2607 | 10607 Component-Whole(e2,e1) 2608 | 10608 Entity-Origin(e1,e2) 2609 | 10609 Instrument-Agency(e2,e1) 2610 | 10610 Other 2611 | 10611 Entity-Destination(e1,e2) 2612 | 10612 Other 2613 | 10613 Entity-Destination(e1,e2) 2614 | 10614 Cause-Effect(e2,e1) 2615 | 10615 Other 2616 | 10616 Message-Topic(e1,e2) 2617 | 10617 Entity-Destination(e1,e2) 2618 | 10618 Product-Producer(e2,e1) 2619 | 10619 Entity-Origin(e1,e2) 2620 | 10620 Other 2621 | 10621 Other 2622 | 10622 Cause-Effect(e1,e2) 2623 | 10623 Entity-Origin(e1,e2) 2624 | 10624 Content-Container(e1,e2) 2625 | 10625 Member-Collection(e2,e1) 2626 | 10626 Component-Whole(e2,e1) 2627 | 10627 Cause-Effect(e2,e1) 2628 | 10628 Message-Topic(e1,e2) 2629 | 10629 Cause-Effect(e1,e2) 2630 | 10630 Other 2631 | 10631 Content-Container(e1,e2) 2632 | 10632 Entity-Destination(e1,e2) 2633 | 10633 Entity-Destination(e1,e2) 2634 | 10634 Member-Collection(e2,e1) 2635 | 10635 Content-Container(e1,e2) 2636 | 10636 Content-Container(e1,e2) 2637 | 10637 Cause-Effect(e2,e1) 2638 | 10638 Other 2639 | 10639 Cause-Effect(e2,e1) 2640 | 10640 Component-Whole(e1,e2) 2641 | 10641 Cause-Effect(e2,e1) 2642 | 10642 Cause-Effect(e1,e2) 2643 | 10643 Cause-Effect(e2,e1) 2644 | 10644 Entity-Origin(e1,e2) 2645 | 10645 Cause-Effect(e2,e1) 2646 | 10646 Cause-Effect(e2,e1) 2647 | 10647 Message-Topic(e2,e1) 2648 | 10648 Product-Producer(e1,e2) 2649 | 10649 Entity-Origin(e1,e2) 2650 | 10650 Cause-Effect(e2,e1) 2651 | 10651 Member-Collection(e2,e1) 2652 | 10652 Other 2653 | 10653 Message-Topic(e1,e2) 2654 | 10654 Other 2655 | 10655 Content-Container(e1,e2) 2656 | 10656 Entity-Origin(e1,e2) 2657 | 10657 Component-Whole(e1,e2) 2658 | 10658 Message-Topic(e1,e2) 2659 | 10659 Member-Collection(e1,e2) 2660 | 10660 Message-Topic(e1,e2) 2661 | 10661 Product-Producer(e2,e1) 2662 | 10662 Content-Container(e2,e1) 2663 | 10663 Content-Container(e1,e2) 2664 | 10664 Other 2665 | 10665 Component-Whole(e2,e1) 2666 | 10666 Entity-Origin(e1,e2) 2667 | 10667 Other 2668 | 10668 Member-Collection(e2,e1) 2669 | 10669 Message-Topic(e1,e2) 2670 | 10670 Content-Container(e1,e2) 2671 | 10671 Other 2672 | 10672 Other 2673 | 10673 Product-Producer(e1,e2) 2674 | 10674 Other 2675 | 10675 Entity-Destination(e1,e2) 2676 | 10676 Component-Whole(e2,e1) 2677 | 10677 Message-Topic(e2,e1) 2678 | 10678 Other 2679 | 10679 Product-Producer(e1,e2) 2680 | 10680 Instrument-Agency(e2,e1) 2681 | 10681 Entity-Destination(e1,e2) 2682 | 10682 Other 2683 | 10683 Entity-Destination(e1,e2) 2684 | 10684 Entity-Origin(e1,e2) 2685 | 10685 Product-Producer(e2,e1) 2686 | 10686 Component-Whole(e1,e2) 2687 | 10687 Other 2688 | 10688 Entity-Destination(e1,e2) 2689 | 10689 Component-Whole(e2,e1) 2690 | 10690 Entity-Origin(e1,e2) 2691 | 10691 Entity-Origin(e1,e2) 2692 | 10692 Cause-Effect(e1,e2) 2693 | 10693 Content-Container(e1,e2) 2694 | 10694 Entity-Destination(e1,e2) 2695 | 10695 Message-Topic(e2,e1) 2696 | 10696 Instrument-Agency(e2,e1) 2697 | 10697 Message-Topic(e1,e2) 2698 | 10698 Other 2699 | 10699 Message-Topic(e2,e1) 2700 | 10700 Member-Collection(e2,e1) 2701 | 10701 Entity-Destination(e1,e2) 2702 | 10702 Instrument-Agency(e2,e1) 2703 | 10703 Cause-Effect(e1,e2) 2704 | 10704 Cause-Effect(e2,e1) 2705 | 10705 Entity-Destination(e1,e2) 2706 | 10706 Other 2707 | 10707 Component-Whole(e2,e1) 2708 | 10708 Entity-Destination(e1,e2) 2709 | 10709 Other 2710 | 10710 Member-Collection(e2,e1) 2711 | 10711 Entity-Origin(e1,e2) 2712 | 10712 Entity-Origin(e1,e2) 2713 | 10713 Instrument-Agency(e2,e1) 2714 | 10714 Product-Producer(e1,e2) 2715 | 10715 Component-Whole(e2,e1) 2716 | 10716 Product-Producer(e1,e2) 2717 | 10717 Entity-Destination(e2,e1) 2718 | -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from pytorch_transformers import ( 5 | WEIGHTS_NAME, BertConfig, BertModel, BertPreTrainedModel, BertTokenizer) 6 | from torch.nn import MSELoss, CrossEntropyLoss 7 | 8 | 9 | def l2_loss(parameters): 10 | return torch.sum( 11 | torch.tensor([ 12 | torch.sum(p ** 2) / 2 for p in parameters if p.requires_grad 13 | ])) 14 | 15 | 16 | class BertForSequenceClassification(BertPreTrainedModel): 17 | r""" 18 | **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: 19 | Labels for computing the sequence classification/regression loss. 20 | Indices should be in ``[0, ..., config.num_labels - 1]``. 21 | If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), 22 | If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). 23 | 24 | Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: 25 | **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: 26 | Classification (or regression if config.num_labels==1) loss. 27 | **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` 28 | Classification (or regression if config.num_labels==1) scores (before SoftMax). 29 | **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) 30 | list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) 31 | of shape ``(batch_size, sequence_length, hidden_size)``: 32 | Hidden-states of the model at the output of each layer plus the initial embedding outputs. 33 | **attentions**: (`optional`, returned when ``config.output_attentions=True``) 34 | list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: 35 | Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. 36 | 37 | Examples:: 38 | 39 | tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') 40 | model = BertForSequenceClassification.from_pretrained( 41 | 'bert-base-uncased') 42 | input_ids = torch.tensor(tokenizer.encode( 43 | "Hello, my dog is cute")).unsqueeze(0) # Batch size 1 44 | labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 45 | outputs = model(input_ids, labels=labels) 46 | loss, logits = outputs[:2] 47 | 48 | """ 49 | 50 | def __init__(self, config): 51 | super(BertForSequenceClassification, self).__init__(config) 52 | self.num_labels = config.num_labels 53 | self.l2_reg_lambda = config.l2_reg_lambda 54 | self.bert = BertModel(config) 55 | self.latent_entity_typing = config.latent_entity_typing 56 | self.dropout = nn.Dropout(config.hidden_dropout_prob) 57 | classifier_size = config.hidden_size*3 58 | if self.latent_entity_typing: 59 | classifier_size += config.hidden_size*2 60 | self.classifier = nn.Linear( 61 | classifier_size, self.config.num_labels) 62 | self.latent_size = config.hidden_size 63 | self.latent_type = nn.Parameter(torch.FloatTensor( 64 | 3, config.hidden_size), requires_grad=True) 65 | 66 | self.apply(self.init_weights) 67 | 68 | def forward(self, input_ids, token_type_ids=None, attention_mask=None, e1_mask=None, e2_mask=None, labels=None, 69 | position_ids=None, head_mask=None): 70 | outputs = self.bert(input_ids, position_ids=position_ids, token_type_ids=token_type_ids, 71 | attention_mask=attention_mask, head_mask=head_mask) 72 | # for details, see the document of pytorch-transformer 73 | pooled_output = outputs[1] 74 | sequence_output = outputs[0] 75 | #pooled_output = self.dropout(pooled_output) 76 | 77 | def extract_entity(sequence_output, e_mask): 78 | extended_e_mask = e_mask.unsqueeze(1) 79 | extended_e_mask = torch.bmm( 80 | extended_e_mask.float(), sequence_output).squeeze(1) 81 | return extended_e_mask.float() 82 | e1_h = extract_entity(sequence_output, e1_mask) 83 | e2_h = extract_entity(sequence_output, e2_mask) 84 | context = self.dropout(pooled_output) 85 | pooled_output = torch.cat([context, e1_h, e2_h], dim=-1) 86 | 87 | # 88 | # second_pre = - tf.reduce_max(rc_probabilities[:, 1:], axis=-1) + 1 89 | # rc_loss = - tf.math.log(second_pre)#+ tf.math.log(second_pre) * log_probs[:,0] 90 | # print(pooled_output.size()) 91 | logits = self.classifier(pooled_output) 92 | 93 | # add hidden states and attention if they are here 94 | outputs = (logits,) + outputs[2:] 95 | 96 | device = logits.get_device() 97 | l2 = l2_loss(self.parameters()) 98 | # print(l2) 99 | if device >= 0: 100 | l2 = l2.to(device) 101 | loss = l2 * self.l2_reg_lambda 102 | if labels is not None: 103 | if self.num_labels == 1: 104 | # We are doing regression 105 | loss_fct = MSELoss() 106 | loss += loss_fct(logits.view(-1), labels.view(-1)) 107 | else: 108 | # loss_fct = CrossEntropyLoss() 109 | # loss += loss_fct( 110 | # logits.view(-1, self.num_labels), labels.view(-1)) 111 | # I thought that using Gumbel softmax should be better than the following code. 112 | 113 | probabilities = F.softmax(logits, dim=-1) 114 | log_probs = F.log_softmax(logits, dim=-1) 115 | one_hot_labels = F.one_hot(labels, num_classes=self.num_labels) 116 | if device >= 0: 117 | one_hot_labels = one_hot_labels.to(device) 118 | 119 | dist = one_hot_labels[:, 1:].float() * log_probs[:, 1:] 120 | example_loss_except_other, _ = dist.min(dim=-1) 121 | per_example_loss = - example_loss_except_other.mean() 122 | 123 | rc_probabilities = probabilities - probabilities * one_hot_labels.float() 124 | second_pre, _ = rc_probabilities[:, 1:].max(dim=-1) 125 | rc_loss = - (1 - second_pre).log().mean() 126 | 127 | #print(loss, per_example_loss, rc_loss) 128 | loss += per_example_loss + 5 * rc_loss 129 | 130 | outputs = (loss,) + outputs 131 | 132 | return outputs # (loss), logits, (hidden_states), (attentions) 133 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2019 Hao WANG, Shanghai University, KB-NLP team. 3 | # Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. 4 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | """ BERT classification fine-tuning: utilities to work with GLUE tasks """ 18 | 19 | from __future__ import absolute_import, division, print_function 20 | 21 | import csv 22 | import re 23 | import logging 24 | import os 25 | import sys 26 | from io import open 27 | 28 | from scipy.stats import pearsonr, spearmanr 29 | from sklearn.metrics import matthews_corrcoef, f1_score 30 | 31 | logger = logging.getLogger(__name__) 32 | 33 | 34 | def clean_str(text): 35 | text = text.lower() 36 | # Clean the text 37 | text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text) 38 | text = re.sub(r"what's", "what is ", text) 39 | text = re.sub(r"that's", "that is ", text) 40 | text = re.sub(r"there's", "there is ", text) 41 | text = re.sub(r"it's", "it is ", text) 42 | text = re.sub(r"\'s", " ", text) 43 | text = re.sub(r"\'ve", " have ", text) 44 | text = re.sub(r"can't", "can not ", text) 45 | text = re.sub(r"n't", " not ", text) 46 | text = re.sub(r"i'm", "i am ", text) 47 | text = re.sub(r"\'re", " are ", text) 48 | text = re.sub(r"\'d", " would ", text) 49 | text = re.sub(r"\'ll", " will ", text) 50 | text = re.sub(r",", " ", text) 51 | text = re.sub(r"\.", " ", text) 52 | text = re.sub(r"!", " ! ", text) 53 | text = re.sub(r"\/", " ", text) 54 | text = re.sub(r"\^", " ^ ", text) 55 | text = re.sub(r"\+", " + ", text) 56 | text = re.sub(r"\-", " - ", text) 57 | text = re.sub(r"\=", " = ", text) 58 | text = re.sub(r"'", " ", text) 59 | text = re.sub(r"(\d+)(k)", r"\g<1>000", text) 60 | text = re.sub(r":", " : ", text) 61 | text = re.sub(r" e g ", " eg ", text) 62 | text = re.sub(r" b g ", " bg ", text) 63 | text = re.sub(r" u s ", " american ", text) 64 | text = re.sub(r"\0s", "0", text) 65 | text = re.sub(r" 9 11 ", "911", text) 66 | text = re.sub(r"e - mail", "email", text) 67 | text = re.sub(r"j k", "jk", text) 68 | text = re.sub(r"\s{2,}", " ", text) 69 | 70 | 71 | RELATION_LABELS = ['Other', 'Message-Topic(e1,e2)', 'Message-Topic(e2,e1)', 72 | 'Product-Producer(e1,e2)', 'Product-Producer(e2,e1)', 73 | 'Instrument-Agency(e1,e2)', 'Instrument-Agency(e2,e1)', 74 | 'Entity-Destination(e1,e2)', 'Entity-Destination(e2,e1)', 75 | 'Cause-Effect(e1,e2)', 'Cause-Effect(e2,e1)', 76 | 'Component-Whole(e1,e2)', 'Component-Whole(e2,e1)', 77 | 'Entity-Origin(e1,e2)', 'Entity-Origin(e2,e1)', 78 | 'Member-Collection(e1,e2)', 'Member-Collection(e2,e1)', 79 | 'Content-Container(e1,e2)', 'Content-Container(e2,e1)'] 80 | 81 | 82 | class InputExample(object): 83 | """A single training/test example for simple sequence classification.""" 84 | 85 | def __init__(self, guid, text_a, text_b=None, label=None): 86 | """Constructs a InputExample. 87 | 88 | Args: 89 | guid: Unique id for the example. 90 | text_a: string. The untokenized text of the first sequence. For single 91 | sequence tasks, only this sequence must be specified. 92 | text_b: (Optional) string. The untokenized text of the second sequence. 93 | Only must be specified for sequence pair tasks. 94 | label: (Optional) string. The label of the example. This should be 95 | specified for train and dev examples, but not for test examples. 96 | """ 97 | self.guid = guid 98 | self.text_a = text_a 99 | self.text_b = text_b 100 | self.label = label 101 | 102 | 103 | # class InputFeatures(object): 104 | # """A single set of features of data.""" 105 | 106 | # def __init__(self, 107 | # input_ids, 108 | # input_mask, 109 | # segment_ids, 110 | # label_id): 111 | # self.input_ids = input_ids 112 | # self.input_mask = input_mask 113 | # self.segment_ids = segment_ids 114 | # self.label_id = label_id 115 | 116 | class InputFeatures(object): 117 | """A single set of features of data.""" 118 | 119 | def __init__(self, 120 | input_ids, 121 | input_mask, 122 | e11_p, e12_p, e21_p, e22_p, 123 | e1_mask, e2_mask, 124 | segment_ids, 125 | label_id): 126 | self.input_ids = input_ids 127 | self.input_mask = input_mask 128 | self.segment_ids = segment_ids 129 | self.label_id = label_id 130 | self.e11_p = e11_p 131 | self.e12_p = e12_p 132 | self.e21_p = e21_p 133 | self.e22_p = e22_p 134 | self.e1_mask = e1_mask 135 | self.e2_mask = e2_mask 136 | 137 | 138 | class DataProcessor(object): 139 | """Base class for data converters for sequence classification data sets.""" 140 | 141 | def get_train_examples(self, data_dir): 142 | """Gets a collection of `InputExample`s for the train set.""" 143 | raise NotImplementedError() 144 | 145 | def get_dev_examples(self, data_dir): 146 | """Gets a collection of `InputExample`s for the dev set.""" 147 | raise NotImplementedError() 148 | 149 | def get_labels(self): 150 | """Gets the list of labels for this data set.""" 151 | raise NotImplementedError() 152 | 153 | @classmethod 154 | def _read_tsv(cls, input_file, quotechar=None): 155 | """Reads a tab separated value file.""" 156 | with open(input_file, "r", encoding="utf-8-sig") as f: 157 | reader = csv.reader(f, delimiter="\t", quotechar=quotechar) 158 | lines = [] 159 | for line in reader: 160 | if sys.version_info[0] == 2: 161 | line = list(cell for cell in line) 162 | lines.append(line) 163 | return lines 164 | 165 | 166 | class MrpcProcessor(DataProcessor): 167 | """Processor for the MRPC data set (GLUE version).""" 168 | 169 | def get_train_examples(self, data_dir): 170 | """See base class.""" 171 | logger.info("LOOKING AT {}".format( 172 | os.path.join(data_dir, "train.tsv"))) 173 | return self._create_examples( 174 | self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") 175 | 176 | def get_dev_examples(self, data_dir): 177 | """See base class.""" 178 | return self._create_examples( 179 | self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") 180 | 181 | def get_labels(self): 182 | """See base class.""" 183 | return ["0", "1"] 184 | 185 | def _create_examples(self, lines, set_type): 186 | """Creates examples for the training and dev sets.""" 187 | examples = [] 188 | for (i, line) in enumerate(lines): 189 | logger.info(line) 190 | if i == 0: 191 | continue 192 | guid = "%s-%s" % (set_type, i) 193 | text_a = line[4] 194 | text_b = line[5] 195 | label = RELATION_LABELS.index(line[0]) 196 | examples.append( 197 | InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) 198 | return examples 199 | 200 | 201 | class SemEvalProcessor(DataProcessor): 202 | """Processor for the MRPC data set (GLUE version).""" 203 | 204 | def get_train_examples(self, data_dir): 205 | """See base class.""" 206 | logger.info("LOOKING AT {}".format( 207 | os.path.join(data_dir, "train.tsv"))) 208 | return self._create_examples( 209 | self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") 210 | 211 | def get_dev_examples(self, data_dir): 212 | """See base class.""" 213 | return self._create_examples( 214 | self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") 215 | 216 | def get_labels(self): 217 | """See base class.""" 218 | return [str(i) for i in range(19)] 219 | 220 | def _create_examples(self, lines, set_type): 221 | """Creates examples for the training and dev sets. 222 | e.g.,: 223 | 2 the [E11] author [E12] of a keygen uses a [E21] disassembler [E22] to look at the raw assembly code . 6 224 | """ 225 | examples = [] 226 | for (i, line) in enumerate(lines): 227 | # if i == 0: 228 | # continue 229 | guid = "%s-%s" % (set_type, i) 230 | logger.info(line) 231 | text_a = line[1] 232 | text_b = None 233 | #label = RELATION_LABELS.index(int(line[2])) 234 | label = line[2] 235 | examples.append( 236 | InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) 237 | return examples 238 | 239 | 240 | def convert_examples_to_features(examples, label_list, max_seq_len, 241 | tokenizer, output_mode, 242 | cls_token='[CLS]', 243 | cls_token_segment_id=1, 244 | sep_token='[SEP]', 245 | pad_token=0, 246 | pad_token_segment_id=0, 247 | sequence_a_segment_id=0, 248 | sequence_b_segment_id=1, 249 | mask_padding_with_zero=True, 250 | use_entity_indicator=True): 251 | """ Loads a data file into a list of `InputBatch`s 252 | Default, BERT/XLM pattern: [CLS] + A + [SEP] + B + [SEP] 253 | `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet) 254 | """ 255 | 256 | label_map = {label: i for i, label in enumerate(label_list)} 257 | 258 | features = [] 259 | for (ex_index, example) in enumerate(examples): 260 | if ex_index % 10000 == 0: 261 | logger.info("Writing example %d of %d" % (ex_index, len(examples))) 262 | 263 | tokens_a = tokenizer.tokenize(example.text_a) 264 | if use_entity_indicator: 265 | # e11_p = tokens_a.index("e11")+1 # the start position of entity1 266 | # e12_p = tokens_a.index("e12")+2 # the end position ofentity1 267 | # e21_p = tokens_a.index("e21")+1 # the start position ofentity2 268 | # e22_p = tokens_a.index("e22")+2 # the end position of entity2 269 | # e11_p = tokens_a.index("[E11]")+2 # the start position of entity1 270 | # e12_p = tokens_a.index("[E12]")+1 # the end position ofentity1 271 | # e21_p = tokens_a.index("[E21]")+2 # the start position ofentity2 272 | # e22_p = tokens_a.index("[E22]")+1 # the end position of entity2 273 | l = len(tokens_a) 274 | e11_p = tokens_a.index("#")+1 # the start position of entity1 275 | e12_p = l-tokens_a[::-1].index("#")+1 # the end position ofentity1 276 | e21_p = tokens_a.index("$")+1 # the start position ofentity2 277 | # the end position of entity2 278 | e22_p = l-tokens_a[::-1].index("$")+1 279 | 280 | tokens_b = None 281 | if example.text_b: 282 | tokens_b = tokenizer.tokenize(example.text_b) 283 | # Modifies `tokens_a` and `tokens_b` in place so that the total 284 | # length is less than the specified length. 285 | # Account for [CLS], [SEP], [SEP] with "- 3". " -4" for RoBERTa. 286 | special_tokens_count = 3 287 | _truncate_seq_pair(tokens_a, tokens_b, 288 | max_seq_len - special_tokens_count) 289 | else: 290 | # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa. 291 | special_tokens_count = 2 292 | if len(tokens_a) > max_seq_len - special_tokens_count: 293 | tokens_a = tokens_a[:(max_seq_len - special_tokens_count)] 294 | 295 | # The convention in BERT is: 296 | # (a) For sequence pairs: 297 | # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] 298 | # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 299 | # (b) For single sequences: 300 | # tokens: [CLS] the dog is hairy . [SEP] 301 | # type_ids: 0 0 0 0 0 0 0 302 | # 303 | # Where "type_ids" are used to indicate whether this is the first 304 | # sequence or the second sequence. The embedding vectors for `type=0` and 305 | # `type=1` were learned during pre-training and are added to the wordpiece 306 | # embedding vector (and position vector). This is not *strictly* necessary 307 | # since the [SEP] token unambiguously separates the sequences, but it makes 308 | # it easier for the model to learn the concept of sequences. 309 | # 310 | # For classification tasks, the first vector (corresponding to [CLS]) is 311 | # used as as the "sentence vector". Note that this only makes sense because 312 | # the entire model is fine-tuned. 313 | tokens = tokens_a + [sep_token] 314 | segment_ids = [sequence_a_segment_id] * len(tokens) 315 | 316 | if tokens_b: 317 | tokens += tokens_b + [sep_token] 318 | segment_ids += [sequence_b_segment_id] * (len(tokens_b) + 1) 319 | 320 | tokens = [cls_token] + tokens 321 | segment_ids = [cls_token_segment_id] + segment_ids 322 | 323 | input_ids = tokenizer.convert_tokens_to_ids(tokens) 324 | 325 | # The mask has 1 for real tokens and 0 for padding tokens. Only real 326 | # tokens are attended to. 327 | input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) 328 | 329 | # Zero-pad up to the sequence length. 330 | padding_length = max_seq_len - len(input_ids) 331 | input_ids = input_ids + ([pad_token] * padding_length) 332 | input_mask = input_mask + \ 333 | ([0 if mask_padding_with_zero else 1] * padding_length) 334 | segment_ids = segment_ids + \ 335 | ([pad_token_segment_id] * padding_length) 336 | if use_entity_indicator: 337 | e1_mask = [0 for i in range(len(input_mask))] 338 | 339 | e2_mask = [0 for i in range(len(input_mask))] 340 | for i in range(e11_p, e12_p): 341 | e1_mask[i] = 1 342 | for i in range(e21_p, e22_p): 343 | e2_mask[i] = 1 344 | 345 | assert len(input_ids) == max_seq_len 346 | assert len(input_mask) == max_seq_len 347 | assert len(segment_ids) == max_seq_len 348 | 349 | if output_mode == "classification": 350 | # label_id = label_map[example.label] 351 | label_id = int(example.label) 352 | elif output_mode == "regression": 353 | label_id = float(example.label) 354 | else: 355 | raise KeyError(output_mode) 356 | 357 | if ex_index < 5: 358 | logger.info("*** Example ***") 359 | logger.info("guid: %s" % (example.guid)) 360 | logger.info("tokens: %s" % " ".join( 361 | [str(x) for x in tokens])) 362 | logger.info("input_ids: %s" % 363 | " ".join([str(x) for x in input_ids])) 364 | logger.info("input_mask: %s" % 365 | " ".join([str(x) for x in input_mask])) 366 | if use_entity_indicator: 367 | logger.info("e11_p: %s" % e11_p) 368 | logger.info("e12_p: %s" % e12_p) 369 | logger.info("e21_p: %s" % e21_p) 370 | logger.info("e22_p: %s" % e22_p) 371 | logger.info("e1_mask: %s" % 372 | " ".join([str(x) for x in e1_mask])) 373 | logger.info("e2_mask: %s" % 374 | " ".join([str(x) for x in e2_mask])) 375 | logger.info("segment_ids: %s" % 376 | " ".join([str(x) for x in segment_ids])) 377 | logger.info("label: %s (id = %d)" % (example.label, label_id)) 378 | 379 | features.append( 380 | InputFeatures(input_ids=input_ids, 381 | input_mask=input_mask, 382 | e11_p=e11_p, 383 | e12_p=e12_p, 384 | e21_p=e21_p, 385 | e22_p=e22_p, 386 | e1_mask=e1_mask, 387 | e2_mask=e2_mask, 388 | segment_ids=segment_ids, 389 | label_id=label_id)) 390 | 391 | return features 392 | 393 | 394 | def _truncate_seq_pair(tokens_a, tokens_b, max_length): 395 | """Truncates a sequence pair in place to the maximum length.""" 396 | 397 | # This is a simple heuristic which will always truncate the longer sequence 398 | # one token at a time. This makes more sense than truncating an equal percent 399 | # of tokens from each, since if one sequence is very short then each token 400 | # that's truncated likely contains more information than a longer sequence. 401 | while True: 402 | total_length = len(tokens_a) + len(tokens_b) 403 | if total_length <= max_length: 404 | break 405 | if len(tokens_a) > len(tokens_b): 406 | tokens_a.pop() 407 | else: 408 | tokens_b.pop() 409 | 410 | 411 | def simple_accuracy(preds, labels): 412 | return (preds == labels).mean() 413 | 414 | 415 | def acc_and_f1(preds, labels, average='micro'): 416 | acc = simple_accuracy(preds, labels) 417 | f1 = f1_score(y_true=labels, y_pred=preds, average='micro') 418 | return { 419 | "acc": acc, 420 | "f1": f1, 421 | "acc_and_f1": (acc + f1) / 2, 422 | } 423 | 424 | 425 | def compute_metrics(task_name, preds, labels): 426 | assert len(preds) == len(labels) 427 | return acc_and_f1(preds, labels) 428 | 429 | 430 | data_processors = { 431 | "semeval": SemEvalProcessor, 432 | "mrpc": MrpcProcessor, 433 | } 434 | 435 | output_modes = { 436 | "mrpc": "classification", 437 | "semeval": "classification" 438 | } 439 | 440 | GLUE_TASKS_NUM_LABELS = { 441 | "mrpc": 2, 442 | "semeval": 19, 443 | } 444 | --------------------------------------------------------------------------------