├── .gitignore ├── CoBERT.py ├── CoBERT_config.json ├── LICENSE ├── README.md └── util.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /CoBERT.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import itertools 3 | import json 4 | import multiprocessing as mp 5 | import os 6 | import pickle 7 | import random 8 | import re 9 | import string 10 | import sys 11 | import time 12 | import json 13 | import math 14 | import copy 15 | from collections import Counter, OrderedDict 16 | 17 | import numpy as np 18 | import pandas as pd 19 | import torch 20 | from torch.utils.data import DataLoader, RandomSampler, TensorDataset 21 | import torch.nn as nn 22 | import torch.nn.functional as F 23 | from tqdm import tqdm 24 | from transformers import AdamW, BertModel, BertTokenizer, get_linear_schedule_with_warmup 25 | 26 | from util import load_pickle, save_pickle, count_parameters, compute_metrics, compute_metrics_from_logits 27 | import logging 28 | 29 | logging.basicConfig(level = logging.INFO, \ 30 | format = '%(asctime)s %(levelname)-5s %(message)s', \ 31 | datefmt = "%Y-%m-%d-%H-%M-%S") 32 | 33 | 34 | def cprint(*args): 35 | text = "" 36 | for arg in args: 37 | text += "{0} ".format(arg) 38 | logging.info(text) 39 | 40 | def tokenize_conversations(data, tokenizer, max_sent_len): 41 | new_data = [] 42 | for conv in tqdm(data): 43 | new_conv = [] 44 | for i, (speaker, sents) in enumerate(conv): 45 | # each utterance has been segmented into multiple sentences 46 | if i==0: 47 | word_limit = 90 48 | else: 49 | word_limit = max_sent_len 50 | 51 | tokenized_sent = [] 52 | for sent in sents: 53 | tokenized = tokenizer.tokenize(sent) 54 | if len(tokenized_sent) + len(tokenized) <= word_limit: 55 | tokenized_sent.extend(tokenized) 56 | else: 57 | break 58 | if len(tokenized_sent) == 0: 59 | tokenized_sent = tokenized[:word_limit] 60 | new_conv.append((speaker, tokenized_sent)) 61 | new_data.append(new_conv) 62 | return new_data 63 | 64 | def tokenize_personas(data, tokenizer, all_speakers, num_personas): 65 | # average: each speaker corresponds to a list of tokens, separated by [SEP] between sents 66 | # memnet: each speaker corresponds to a 2D list of tokens 67 | new_data = {} 68 | for k, sents in tqdm(data.items()): 69 | if k in all_speakers: 70 | tokenized_words = [] 71 | for sent in sents[:num_personas]: 72 | tokenized_words.extend(tokenizer.tokenize(" ".join(sent))[:22] + ["[SEP]"]) 73 | if len(tokenized_words) > 1: 74 | tokenized_words.pop() # remove the last [SEP] 75 | new_data[k] = tokenized_words 76 | else: 77 | new_data[k] = ["."] 78 | return new_data 79 | 80 | def create_context_and_response(data): 81 | new_data = [] 82 | for conv in tqdm(data): 83 | context = [] 84 | for s, ts in conv[:-1]: 85 | context.extend(ts + ["[SEP]"]) 86 | context.pop() # pop the last [SEP] 87 | response = conv[-1][1] 88 | if len(context) > 0 and len(response) > 0: 89 | new_data.append((context, response, conv[-1][0])) 90 | return new_data 91 | 92 | 93 | def convert_conversations_to_ids(data, persona, tokenizer, max_seq_len, max_sent_len, num_personas): 94 | def pad_tokens(tokens, max_len, sentence_type, num_personas=0, response_ids=None): 95 | # note token_type_ids to differentiate context utterances 96 | # speaker A has 0, speaker B has 1, response is speaker B and has 1, persona has 1 97 | # persona does not have positional embedding 98 | if sentence_type == "persona" and num_personas > 0: 99 | # filter persona sentences that appeared in response_ids 100 | if response_ids is not None: 101 | response_sent = " ".join(tokenizer.convert_ids_to_tokens(response_ids, skip_special_tokens=True)) 102 | 103 | all_persona_sent_ids = [] 104 | for t_id in tokens: 105 | if t_id in [101]: 106 | sent_ids = [] 107 | if t_id in [102]: 108 | all_persona_sent_ids.append(sent_ids) 109 | sent_ids = [] 110 | if t_id not in tokenizer.all_special_ids: 111 | sent_ids.append(t_id) 112 | 113 | # convert ids to tokens 114 | filtered_tokens = [] 115 | for sent_ids in all_persona_sent_ids: 116 | sent = " ".join(tokenizer.convert_ids_to_tokens(sent_ids)) 117 | if sent not in response_sent: 118 | filtered_tokens.extend(sent_ids + [tokenizer.convert_tokens_to_ids("[SEP]")]) 119 | filtered_tokens.insert(0, tokenizer.convert_tokens_to_ids("[CLS]")) 120 | 121 | tokens = filtered_tokens 122 | 123 | # remove additional persona sentences 124 | persona_sent_count = 0 125 | truncated_tokens = [] 126 | for token_id in tokens: 127 | if token_id == tokenizer.convert_tokens_to_ids("[SEP]"): 128 | persona_sent_count += 1 129 | if persona_sent_count == num_personas: 130 | break 131 | truncated_tokens.append(token_id) 132 | tokens = truncated_tokens 133 | 134 | assert max_len >= len(tokens) 135 | attention_mask = [1]*len(tokens) 136 | padding_length = max_len - len(tokens) 137 | attention_mask = attention_mask + ([0] * padding_length) 138 | 139 | if sentence_type == "context": 140 | token_type_ids = [] 141 | token_type = 0 142 | for token_id in tokens: 143 | token_type_ids.append(token_type) 144 | if token_id == tokenizer.convert_tokens_to_ids("[SEP]"): 145 | token_type = int(1-token_type) 146 | token_type_ids = token_type_ids + [0] * padding_length 147 | else: 148 | token_type_ids = [0] * max_len 149 | 150 | tokens = tokens + [0] * padding_length 151 | return tokens, attention_mask, token_type_ids 152 | 153 | all_context_ids = [] 154 | all_context_attention_mask = [] 155 | all_context_token_type_ids = [] 156 | all_response_ids = [] 157 | all_response_attention_mask = [] 158 | all_response_token_type_ids = [] 159 | all_persona_ids = [] 160 | all_persona_attention_mask = [] 161 | all_persona_token_type_ids = [] 162 | max_persona_len = 23*num_personas+1 163 | context_lens = [] 164 | for context, response, speaker in tqdm(data): 165 | context_ids = tokenizer.encode(context, add_special_tokens=True) # convert to token ids, add [cls] and [sep] at beginning and end 166 | response_ids = tokenizer.encode(response, add_special_tokens=True) 167 | context_lens.append(len(context_ids)) 168 | 169 | context_ids, context_attention_mask, context_token_type_ids = pad_tokens(context_ids, max_seq_len, "context") 170 | response_ids, response_attention_mask, response_token_type_ids = pad_tokens(response_ids, max_sent_len+2, "response") 171 | 172 | all_context_ids.append(context_ids) 173 | all_context_attention_mask.append(context_attention_mask) 174 | all_context_token_type_ids.append(context_token_type_ids) 175 | all_response_ids.append(response_ids) 176 | all_response_attention_mask.append(response_attention_mask) 177 | all_response_token_type_ids.append(response_token_type_ids) 178 | 179 | if persona is not None: 180 | persona_ids = tokenizer.encode(persona[speaker], add_special_tokens=True) 181 | persona_ids, persona_attention_mask, persona_token_type_ids = pad_tokens(persona_ids, max_persona_len, "persona", num_personas, response_ids) 182 | # persona_ids, persona_attention_mask, persona_token_type_ids = pad_tokens(persona_ids, max_persona_len, "persona", num_personas) 183 | all_persona_ids.append(persona_ids) 184 | all_persona_attention_mask.append(persona_attention_mask) 185 | all_persona_token_type_ids.append(persona_token_type_ids) 186 | 187 | # (num_examples, max_seq_len) 188 | all_context_ids = torch.tensor(all_context_ids, dtype=torch.long) 189 | all_context_attention_mask = torch.tensor(all_context_attention_mask, dtype=torch.long) 190 | all_context_token_type_ids = torch.tensor(all_context_token_type_ids, dtype=torch.long) 191 | 192 | # (num_examples, max_sent_len) 193 | all_response_ids = torch.tensor(all_response_ids, dtype=torch.long) 194 | all_response_attention_mask = torch.tensor(all_response_attention_mask, dtype=torch.long) 195 | all_response_token_type_ids = torch.tensor(all_response_token_type_ids, dtype=torch.long) 196 | 197 | if persona is not None: 198 | # (num_examples, max_persona_len) 199 | all_persona_ids = torch.tensor(all_persona_ids, dtype=torch.long) 200 | all_persona_attention_mask = torch.tensor(all_persona_attention_mask, dtype=torch.long) 201 | all_persona_token_type_ids = torch.tensor(all_persona_token_type_ids, dtype=torch.long) 202 | 203 | cprint(all_context_ids.shape, all_context_attention_mask.shape, all_context_token_type_ids.shape) 204 | cprint(all_response_ids.shape, all_response_attention_mask.shape, all_response_token_type_ids.shape) 205 | 206 | if persona is not None: 207 | cprint(all_persona_ids.shape, all_persona_attention_mask.shape, all_persona_token_type_ids.shape) 208 | dataset = TensorDataset(all_context_ids, all_context_attention_mask, all_context_token_type_ids, \ 209 | all_response_ids, all_response_attention_mask, all_response_token_type_ids, \ 210 | all_persona_ids, all_persona_attention_mask, all_persona_token_type_ids) 211 | else: 212 | dataset = TensorDataset(all_context_ids, all_context_attention_mask, all_context_token_type_ids, \ 213 | all_response_ids, all_response_attention_mask, all_response_token_type_ids) 214 | 215 | cprint("context lens stats: ", min(context_lens), max(context_lens), \ 216 | np.mean(context_lens), np.std(context_lens)) 217 | return dataset 218 | 219 | 220 | def match(model, matching_method, x, y, x_mask, y_mask): 221 | # Multi-hop Co-Attention 222 | # x: (batch_size, m, hidden_size) 223 | # y: (batch_size, n, hidden_size) 224 | # x_mask: (batch_size, m) 225 | # y_mask: (batch_size, n) 226 | assert x.dim() == 3 and y.dim() == 3 227 | assert x_mask.dim() == 2 and y_mask.dim() == 2 228 | assert x_mask.shape == x.shape[:2] and y_mask.shape == y.shape[:2] 229 | m = x.shape[1] 230 | n = y.shape[1] 231 | 232 | attn_mask = torch.bmm(x_mask.unsqueeze(-1), y_mask.unsqueeze(1)) # (batch_size, m, n) 233 | attn = torch.bmm(x, y.transpose(1,2)) # (batch_size, m, n) 234 | model.attn = attn 235 | model.attn_mask = attn_mask 236 | 237 | x_to_y = torch.softmax(attn * attn_mask + (-5e4) * (1-attn_mask), dim=2) # (batch_size, m, n) 238 | y_to_x = torch.softmax(attn * attn_mask + (-5e4) * (1-attn_mask), dim=1).transpose(1,2) # # (batch_size, n, m) 239 | 240 | # x_attended, y_attended = None, None # no hop-1 241 | x_attended = torch.bmm(x_to_y, y) # (batch_size, m, hidden_size) 242 | y_attended = torch.bmm(y_to_x, x) # (batch_size, n, hidden_size) 243 | 244 | # x_attended_2hop, y_attended_2hop = None, None # no hop-2 245 | y_attn = torch.bmm(y_to_x.mean(dim=1, keepdim=True), x_to_y) # (batch_size, 1, n) # true important attention over y 246 | x_attn = torch.bmm(x_to_y.mean(dim=1, keepdim=True), y_to_x) # (batch_size, 1, m) # true important attention over x 247 | 248 | # truly attended representation 249 | x_attended_2hop = torch.bmm(x_attn, x).squeeze(1) # (batch_size, hidden_size) 250 | y_attended_2hop = torch.bmm(y_attn, y).squeeze(1) # (batch_size, hidden_size) 251 | 252 | # # hop-3 253 | # y_attn, x_attn = torch.bmm(x_attn, x_to_y), torch.bmm(y_attn, y_to_x) # (batch_size, 1, n) # true important attention over y 254 | # x_attended_3hop = torch.bmm(x_attn, x).squeeze(1) # (batch_size, hidden_size) 255 | # y_attended_3hop = torch.bmm(y_attn, y).squeeze(1) # (batch_size, hidden_size) 256 | # x_attended_2hop = torch.cat([x_attended_2hop, x_attended_3hop], dim=-1) 257 | # y_attended_2hop = torch.cat([y_attended_2hop, y_attended_3hop], dim=-1) 258 | 259 | x_attended = x_attended, x_attended_2hop 260 | y_attended = y_attended, y_attended_2hop 261 | 262 | return x_attended, y_attended 263 | 264 | 265 | def aggregate(model, aggregation_method, x, x_mask): 266 | # x: (batch_size, seq_len, emb_size) 267 | # x_mask: (batch_size, seq_len) 268 | assert x.dim() == 3 and x_mask.dim() == 2 269 | assert x.shape[:2] == x_mask.shape 270 | # batch_size, seq_len, emb_size = x.shape 271 | 272 | if aggregation_method == "mean": 273 | return (x * x_mask.unsqueeze(-1)).sum(dim=1)/x_mask.sum(dim=-1, keepdim=True).clamp(min=1) # (batch_size, emb_size) 274 | 275 | if aggregation_method == "max": 276 | return x.masked_fill(x_mask.unsqueeze(-1)==0, -5e4).max(dim=1)[0] # (batch_size, emb_size) 277 | 278 | if aggregation_method == "mean_max": 279 | return torch.cat([(x * x_mask.unsqueeze(-1)).sum(dim=1)/x_mask.sum(dim=-1, keepdim=True).clamp(min=1), \ 280 | x.masked_fill(x_mask.unsqueeze(-1)==0, -5e4).max(dim=1)[0]], dim=-1) # (batch_size, 2*emb_size) 281 | 282 | 283 | def fuse(model, matching_method, aggregation_method, batch_x_emb, batch_y_emb, batch_persona_emb, \ 284 | batch_x_mask, batch_y_mask, batch_persona_mask, batch_size, num_candidates): 285 | 286 | batch_x_emb, batch_y_emb_context = match(model, matching_method, batch_x_emb, batch_y_emb, batch_x_mask, batch_y_mask) 287 | # batch_x_emb: ((batch_size*num_candidates, m, emb_size), (batch_size*num_candidates, emb_size)) 288 | # batch_y_emb_context: (batch_size*num_candidates, n, emb_size), (batch_size*num_candidates, emb_size) 289 | 290 | # hop 2 results 291 | batch_x_emb_2hop = batch_x_emb[1] 292 | batch_y_emb_context_2hop = batch_y_emb_context[1] 293 | 294 | # mean_max aggregation for the 1st hop result 295 | batch_x_emb = aggregate(model, aggregation_method, batch_x_emb[0], batch_x_mask) # batch_x_emb: (batch_size*num_candidates, 2*emb_size) 296 | batch_y_emb_context = aggregate(model, aggregation_method, batch_y_emb_context[0], batch_y_mask) # batch_y_emb_context: (batch_size*num_candidates, 2*emb_size) 297 | 298 | if batch_persona_emb is not None: 299 | batch_persona_emb, batch_y_emb_persona = match(model, matching_method, batch_persona_emb, batch_y_emb, batch_persona_mask, batch_y_mask) 300 | # batch_persona_emb: (batch_size*num_candidates, m, emb_size), (batch_size*num_candidates, emb_size) 301 | # batch_y_emb_persona: (batch_size*num_candidates, n, emb_size), (batch_size*num_candidates, emb_size) 302 | 303 | batch_persona_emb_2hop = batch_persona_emb[1] 304 | batch_y_emb_persona_2hop = batch_y_emb_persona[1] 305 | 306 | # # no hop-1 307 | # return torch.bmm(torch.cat([batch_x_emb_2hop, batch_persona_emb_2hop], dim=-1).unsqueeze(1), \ 308 | # torch.cat([batch_y_emb_context_2hop, batch_y_emb_persona_2hop], dim=-1)\ 309 | # .unsqueeze(-1)).reshape(batch_size, num_candidates) 310 | 311 | 312 | batch_persona_emb = aggregate(model, aggregation_method, batch_persona_emb[0], batch_persona_mask) # batch_persona_emb: (batch_size*num_candidates, 2*emb_size) 313 | batch_y_emb_persona = aggregate(model, aggregation_method, batch_y_emb_persona[0], batch_y_mask) # batch_y_emb_persona: (batch_size*num_candidates, 2*emb_size) 314 | 315 | # # no hop-2 316 | # return torch.bmm(torch.cat([batch_x_emb, batch_persona_emb], dim=-1).unsqueeze(1), \ 317 | # torch.cat([batch_y_emb_context, batch_y_emb_persona], dim=-1)\ 318 | # .unsqueeze(-1)).reshape(batch_size, num_candidates) 319 | return torch.bmm(torch.cat([batch_x_emb, batch_x_emb_2hop, batch_persona_emb, batch_persona_emb_2hop], dim=-1).unsqueeze(1), \ 320 | torch.cat([batch_y_emb_context, batch_y_emb_context_2hop, batch_y_emb_persona, batch_y_emb_persona_2hop], dim=-1)\ 321 | .unsqueeze(-1)).reshape(batch_size, num_candidates) 322 | else: 323 | return torch.bmm(torch.cat([batch_x_emb, batch_x_emb_2hop], dim=-1).unsqueeze(1), \ 324 | torch.cat([batch_y_emb_context, batch_y_emb_context_2hop], dim=-1)\ 325 | .unsqueeze(-1)).reshape(batch_size, num_candidates) 326 | 327 | 328 | def dot_product_loss(batch_x_emb, batch_y_emb): 329 | """ 330 | if batch_x_emb.dim() == 2: 331 | # batch_x_emb: (batch_size, emb_size) 332 | # batch_y_emb: (batch_size, emb_size) 333 | 334 | if batch_x_emb.dim() == 3: 335 | # batch_x_emb: (batch_size, batch_size, emb_size), the 1st dim is along examples and the 2nd dim is along candidates 336 | # batch_y_emb: (batch_size, emb_size) 337 | """ 338 | batch_size = batch_x_emb.size(0) 339 | targets = torch.arange(batch_size, device=batch_x_emb.device) 340 | 341 | if batch_x_emb.dim() == 2: 342 | dot_products = batch_x_emb.mm(batch_y_emb.t()) 343 | elif batch_x_emb.dim() == 3: 344 | dot_products = torch.bmm(batch_x_emb, batch_y_emb.unsqueeze(0).repeat(batch_size, 1, 1).transpose(1,2))[:, targets, targets] # (batch_size, batch_size) 345 | 346 | # dot_products: [batch, batch] 347 | log_prob = F.log_softmax(dot_products, dim=1) 348 | loss = F.nll_loss(log_prob, targets) 349 | nb_ok = (log_prob.max(dim=1)[1] == targets).float().sum() 350 | return loss, nb_ok 351 | 352 | def train_epoch(data_iter, models, num_personas, optimizers, schedulers, gradient_accumulation_steps, device, fp16, amp, \ 353 | apply_interaction, matching_method, aggregation_method): 354 | epoch_loss = [] 355 | ok = 0 356 | total = 0 357 | print_every = 1000 358 | if len(models) == 1: 359 | if num_personas == 0: 360 | context_model, response_model = models[0], models[0] 361 | else: 362 | context_model, response_model, persona_model = models[0], models[0], models[0] 363 | if len(models) == 2: 364 | context_model, response_model = models 365 | if len(models) == 3: 366 | context_model, response_model, persona_model = models 367 | 368 | for optimizer in optimizers: 369 | optimizer.zero_grad() 370 | for i, batch in enumerate(data_iter): 371 | batch = tuple(t.to(device) for t in batch) 372 | batch_x = {"input_ids": batch[0], "attention_mask": batch[1], "token_type_ids": batch[2]} 373 | batch_y = {"input_ids": batch[3], "attention_mask": batch[4], "token_type_ids": batch[5]} 374 | has_persona = len(batch) > 6 375 | if i==0: 376 | cprint(batch[0].shape, batch[3].shape) 377 | 378 | if has_persona: 379 | batch_persona = { 380 | "input_ids": batch[6], 381 | "attention_mask": batch[7], 382 | "token_type_ids": batch[8] 383 | } 384 | 385 | 386 | output_x = context_model(**batch_x) 387 | output_y = response_model(**batch_y) 388 | 389 | if apply_interaction: 390 | # batch_x_mask = batch[0].ne(0).float() 391 | # batch_y_mask = batch[3].ne(0).float() 392 | batch_x_mask = batch[1].float() 393 | batch_y_mask = batch[4].float() 394 | batch_x_emb = output_x[0] # (batch_size, context_len, emb_size) 395 | batch_y_emb = output_y[0] # (batch_size, sent_len, emb_size) 396 | batch_size, sent_len, emb_size = batch_y_emb.shape 397 | 398 | batch_persona_emb = None 399 | batch_persona_mask = None 400 | num_candidates = batch_size 401 | if has_persona: 402 | # batch_persona_mask = batch[6].ne(0).float() 403 | batch_persona_mask = batch[7].float() 404 | output_persona = persona_model(**batch_persona) 405 | batch_persona_emb = output_persona[0] # (batch_size, persona_len, emb_size) 406 | 407 | batch_persona_emb = batch_persona_emb.repeat_interleave(num_candidates, dim=0) 408 | batch_persona_mask = batch_persona_mask.repeat_interleave(num_candidates, dim=0) 409 | 410 | batch_x_emb = batch_x_emb.repeat_interleave(num_candidates, dim=0) # (batch_size*num_candidates, context_len, emb_size) 411 | batch_x_mask = batch_x_mask.repeat_interleave(num_candidates, dim=0) # (batch_size*num_candidates, context_len) 412 | 413 | # interaction 414 | # context-response attention 415 | batch_y_emb = batch_y_emb.unsqueeze(0).repeat(batch_size, 1, 1, 1).reshape(-1, sent_len, emb_size) # (batch_size*num_candidates, sent_len, emb_size) 416 | batch_y_mask = batch_y_mask.unsqueeze(0).repeat(batch_size, 1, 1).reshape(-1, sent_len) # (batch_size*num_candidates, sent_len) 417 | logits = fuse(context_model, matching_method, aggregation_method, \ 418 | batch_x_emb, batch_y_emb, batch_persona_emb, batch_x_mask, batch_y_mask, batch_persona_mask, batch_size, num_candidates) 419 | 420 | # compute loss 421 | targets = torch.arange(batch_size, dtype=torch.long, device=batch[0].device) 422 | loss = F.cross_entropy(logits, targets) 423 | num_ok = (targets.long() == logits.float().argmax(dim=1)).sum() 424 | else: 425 | batch_x_emb = output_x[0].mean(dim=1) # batch_x_emb: (batch_size, emb_size) 426 | batch_y_emb = output_y[0].mean(dim=1) 427 | 428 | if has_persona: 429 | output_persona = persona_model(**batch_persona) 430 | batch_persona_emb = output_persona[0].mean(dim=1) 431 | batch_x_emb = (batch_x_emb + batch_persona_emb)/2 432 | 433 | # compute loss 434 | loss, num_ok = dot_product_loss(batch_x_emb, batch_y_emb) 435 | 436 | ok += num_ok.item() 437 | total += batch[0].shape[0] 438 | 439 | if gradient_accumulation_steps > 1: 440 | loss = loss / gradient_accumulation_steps 441 | if fp16: 442 | with amp.scale_loss(loss, optimizer) as scaled_loss: 443 | scaled_loss.backward() 444 | else: 445 | loss.backward() 446 | 447 | if (i+1) % gradient_accumulation_steps == 0: 448 | for model, optimizer, scheduler in zip(models, optimizers, schedulers): 449 | if fp16: 450 | torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), 1) 451 | else: 452 | torch.nn.utils.clip_grad_norm_(model.parameters(), 1) 453 | # torch.nn.utils.clip_grad_norm_(model.parameters(), 1) 454 | optimizer.step() 455 | scheduler.step() 456 | 457 | # clear grads here 458 | for optimizer in optimizers: 459 | optimizer.zero_grad() 460 | epoch_loss.append(loss.item()) 461 | 462 | if i%print_every == 0: 463 | cprint("loss: ", np.mean(epoch_loss[-print_every:])) 464 | cprint("accuracy: ", ok/total) 465 | 466 | acc = ok/total 467 | return np.mean(epoch_loss), (acc, 0, 0) 468 | 469 | def evaluate_epoch(data_iter, models, num_personas, gradient_accumulation_steps, device, dataset, epoch, \ 470 | apply_interaction, matching_method, aggregation_method): 471 | epoch_loss = [] 472 | ok = 0 473 | total = 0 474 | recall = [] 475 | MRR = [] 476 | print_every = 1000 477 | if len(models) == 1: 478 | if num_personas == 0: 479 | context_model, response_model = models[0], models[0] 480 | else: 481 | context_model, response_model, persona_model = models[0], models[0], models[0] 482 | if len(models) == 2: 483 | context_model, response_model = models 484 | if len(models) == 3: 485 | context_model, response_model, persona_model = models 486 | 487 | for batch_idx, batch in enumerate(data_iter): 488 | batch = tuple(t.to(device) for t in batch) 489 | batch_y = {"input_ids": batch[3], "attention_mask": batch[4], "token_type_ids": batch[5]} 490 | has_persona = len(batch) > 6 491 | 492 | # get context embeddings in chunks due to memory constraint 493 | batch_size = batch[0].shape[0] 494 | chunk_size = 20 495 | num_chunks = math.ceil(batch_size/chunk_size) 496 | 497 | if apply_interaction: 498 | # batch_x_mask = batch[0].ne(0).float() 499 | # batch_y_mask = batch[3].ne(0).float() 500 | batch_x_mask = batch[1].float() 501 | batch_y_mask = batch[4].float() 502 | 503 | batch_x_emb = [] 504 | batch_x_pooled_emb = [] 505 | with torch.no_grad(): 506 | for i in range(num_chunks): 507 | mini_batch_x = { 508 | "input_ids": batch[0][i*chunk_size: (i+1)*chunk_size], 509 | "attention_mask": batch[1][i*chunk_size: (i+1)*chunk_size], 510 | "token_type_ids": batch[2][i*chunk_size: (i+1)*chunk_size] 511 | } 512 | mini_output_x = context_model(**mini_batch_x) 513 | batch_x_emb.append(mini_output_x[0]) # [(chunk_size, seq_len, emb_size), ...] 514 | batch_x_pooled_emb.append(mini_output_x[1]) 515 | batch_x_emb = torch.cat(batch_x_emb, dim=0) # (batch_size, seq_len, emb_size) 516 | batch_x_pooled_emb = torch.cat(batch_x_pooled_emb, dim=0) 517 | emb_size = batch_x_emb.shape[-1] 518 | 519 | if has_persona: 520 | # batch_persona_mask = batch[6].ne(0).float() 521 | batch_persona_mask = batch[7].float() 522 | batch_persona_emb = [] 523 | batch_persona_pooled_emb = [] 524 | with torch.no_grad(): 525 | for i in range(num_chunks): 526 | mini_batch_persona = { 527 | "input_ids": batch[6][i*chunk_size: (i+1)*chunk_size], 528 | "attention_mask": batch[7][i*chunk_size: (i+1)*chunk_size], 529 | "token_type_ids": batch[8][i*chunk_size: (i+1)*chunk_size] 530 | } 531 | mini_output_persona = persona_model(**mini_batch_persona) 532 | 533 | # [(chunk_size, emb_size), ...] 534 | batch_persona_emb.append(mini_output_persona[0]) 535 | batch_persona_pooled_emb.append(mini_output_persona[1]) 536 | 537 | batch_persona_emb = torch.cat(batch_persona_emb, dim=0) 538 | batch_persona_pooled_emb = torch.cat(batch_persona_pooled_emb, dim=0) 539 | 540 | with torch.no_grad(): 541 | output_y = response_model(**batch_y) 542 | batch_y_emb = output_y[0] 543 | batch_size, sent_len, emb_size = batch_y_emb.shape 544 | 545 | # interaction 546 | # context-response attention 547 | num_candidates = batch_size 548 | 549 | with torch.no_grad(): 550 | # evaluate per example 551 | logits = [] 552 | for i in range(batch_size): 553 | x_emb = batch_x_emb[i:i+1].repeat_interleave(num_candidates, dim=0) # (num_candidates, context_len, emb_size) 554 | x_mask = batch_x_mask[i:i+1].repeat_interleave(num_candidates, dim=0) # (batch_size*num_candidates, context_len) 555 | persona_emb, persona_mask = None, None 556 | if has_persona: 557 | persona_emb = batch_persona_emb[i:i+1].repeat_interleave(num_candidates, dim=0) 558 | persona_mask = batch_persona_mask[i:i+1].repeat_interleave(num_candidates, dim=0) 559 | 560 | logits_single = fuse(context_model, matching_method, aggregation_method, \ 561 | x_emb, batch_y_emb, persona_emb, x_mask, batch_y_mask, persona_mask, 1, num_candidates).reshape(-1) 562 | 563 | logits.append(logits_single) 564 | logits = torch.stack(logits, dim=0) 565 | 566 | # compute loss 567 | targets = torch.arange(batch_size, dtype=torch.long, device=batch[0].device) 568 | loss = F.cross_entropy(logits, targets) 569 | 570 | num_ok = (targets.long() == logits.float().argmax(dim=1)).sum() 571 | valid_recall, valid_MRR = compute_metrics_from_logits(logits, targets) 572 | else: 573 | batch_x_emb = [] 574 | with torch.no_grad(): 575 | for i in range(num_chunks): 576 | mini_batch_x = { 577 | "input_ids": batch[0][i*chunk_size: (i+1)*chunk_size], 578 | "attention_mask": batch[1][i*chunk_size: (i+1)*chunk_size], 579 | "token_type_ids": batch[2][i*chunk_size: (i+1)*chunk_size] 580 | } 581 | mini_output_x = context_model(**mini_batch_x) 582 | batch_x_emb.append(mini_output_x[0].mean(dim=1)) # [(chunk_size, emb_size), ...] 583 | batch_x_emb = torch.cat(batch_x_emb, dim=0) # (batch_size, emb_size) 584 | emb_size = batch_x_emb.shape[-1] 585 | 586 | if has_persona: 587 | batch_persona_emb = [] 588 | with torch.no_grad(): 589 | for i in range(num_chunks): 590 | mini_batch_persona = { 591 | "input_ids": batch[6][i*chunk_size: (i+1)*chunk_size], 592 | "attention_mask": batch[7][i*chunk_size: (i+1)*chunk_size], 593 | "token_type_ids": batch[8][i*chunk_size: (i+1)*chunk_size] 594 | } 595 | mini_output_persona = persona_model(**mini_batch_persona) 596 | 597 | # [(chunk_size, emb_size), ...] 598 | batch_persona_emb.append(mini_output_persona[0].mean(dim=1)) 599 | 600 | with torch.no_grad(): 601 | batch_persona_emb = torch.cat(batch_persona_emb, dim=0) 602 | batch_x_emb = (batch_x_emb + batch_persona_emb)/2 603 | 604 | output_y = response_model(**batch_y) 605 | batch_y_emb = output_y[0].mean(dim=1) 606 | 607 | # compute loss 608 | loss, num_ok = dot_product_loss(batch_x_emb, batch_y_emb) 609 | valid_recall, valid_MRR = compute_metrics(batch_x_emb, batch_y_emb) 610 | 611 | ok += num_ok.item() 612 | total += batch[0].shape[0] 613 | 614 | # compute valid recall 615 | recall.append(valid_recall) 616 | MRR.append(valid_MRR) 617 | 618 | if gradient_accumulation_steps > 1: 619 | loss = loss / gradient_accumulation_steps 620 | epoch_loss.append(loss.item()) 621 | 622 | if batch_idx%print_every == 0: 623 | cprint("loss: ", np.mean(epoch_loss[-print_every:])) 624 | cprint("valid recall: ", np.mean(recall[-print_every:], axis=0)) 625 | cprint("valid MRR: ", np.mean(MRR[-print_every:], axis=0)) 626 | 627 | acc = ok/total 628 | # compute recall for validation dataset 629 | recall = np.mean(recall, axis=0) 630 | MRR = np.mean(MRR) 631 | return np.mean(epoch_loss), (acc, recall, MRR) 632 | 633 | 634 | def main(config, progress): 635 | # save config 636 | with open("./log/configs.json", "a") as f: 637 | json.dump(config, f) 638 | f.write("\n") 639 | cprint("*"*80) 640 | cprint("Experiment progress: {0:.2f}%".format(progress*100)) 641 | cprint("*"*80) 642 | metrics = {} 643 | 644 | # data hyper-params 645 | train_path = config["train_path"] 646 | valid_path = config["valid_path"] 647 | test_path = config["test_path"] 648 | dataset = train_path.split("/")[3] 649 | test_mode = bool(config["test_mode"]) 650 | load_model_path = config["load_model_path"] 651 | save_model_path = config["save_model_path"] 652 | num_candidates = config["num_candidates"] 653 | num_personas = config["num_personas"] 654 | persona_path = config["persona_path"] 655 | max_sent_len = config["max_sent_len"] 656 | max_seq_len = config["max_seq_len"] 657 | PEC_ratio = config["PEC_ratio"] 658 | train_ratio = config["train_ratio"] 659 | if PEC_ratio != 0 and train_ratio != 1: 660 | raise ValueError("PEC_ratio or train_ratio not qualified!") 661 | 662 | # model hyper-params 663 | config_id = config["config_id"] 664 | model = config["model"] 665 | shared = bool(config["shared"]) 666 | apply_interaction = bool(config["apply_interaction"]) 667 | matching_method = config["matching_method"] 668 | aggregation_method = config["aggregation_method"] 669 | output_hidden_states = False 670 | 671 | # training hyper-params 672 | batch_size = config["batch_size"] 673 | epochs = config["epochs"] 674 | warmup_steps = config["warmup_steps"] 675 | gradient_accumulation_steps = config["gradient_accumulation_steps"] 676 | lr = config["lr"] 677 | weight_decay = 0 678 | seed = config["seed"] 679 | device = torch.device(config["device"]) 680 | fp16 = bool(config["fp16"]) 681 | fp16_opt_level = config["fp16_opt_level"] 682 | 683 | # set seed 684 | random.seed(seed) 685 | torch.manual_seed(seed) 686 | torch.cuda.manual_seed(seed) 687 | 688 | if test_mode and load_model_path == "": 689 | raise ValueError("Must specify test model path when in test mode!") 690 | 691 | # load data 692 | cprint("Loading conversation data...") 693 | train = load_pickle(train_path) 694 | valid = load_pickle(valid_path) 695 | if test_mode: 696 | test = load_pickle(test_path) 697 | valid_path = test_path 698 | valid = test 699 | 700 | cprint("sample train data: ", train[0]) 701 | cprint("sample valid data: ", valid[0]) 702 | 703 | # tokenization 704 | cprint("Tokenizing ...") 705 | tokenizer = BertTokenizer.from_pretrained(model) 706 | cached_tokenized_train_path = train_path.replace(".pkl", "_tokenized.pkl") 707 | cached_tokenized_valid_path = valid_path.replace(".pkl", "_tokenized.pkl") 708 | if os.path.exists(cached_tokenized_train_path): 709 | cprint("Loading tokenized dataset from ", cached_tokenized_train_path) 710 | train = load_pickle(cached_tokenized_train_path) 711 | else: 712 | train = tokenize_conversations(train, tokenizer, max_sent_len) 713 | cprint("Saving tokenized dataset to ", cached_tokenized_train_path) 714 | save_pickle(train, cached_tokenized_train_path) 715 | 716 | if os.path.exists(cached_tokenized_valid_path): 717 | cprint("Loading tokenized dataset from ", cached_tokenized_valid_path) 718 | valid = load_pickle(cached_tokenized_valid_path) 719 | else: 720 | valid = tokenize_conversations(valid, tokenizer, max_sent_len) 721 | cprint("Saving tokenized dataset to ", cached_tokenized_valid_path) 722 | save_pickle(valid, cached_tokenized_valid_path) 723 | 724 | persona = None 725 | if num_personas > 0: 726 | cprint("Tokenizing persona sentences...") 727 | cached_tokenized_persona_path = persona_path.replace(".pkl", "_tokenized.pkl") 728 | if os.path.exists(cached_tokenized_persona_path): 729 | cprint("Loading tokenized persona from file...") 730 | persona = load_pickle(cached_tokenized_persona_path) 731 | else: 732 | cprint("Loading persona data...") 733 | persona = load_pickle(persona_path) 734 | all_speakers = set([s for conv in load_pickle(config["train_path"]) + \ 735 | load_pickle(config["valid_path"]) + load_pickle(config["test_path"]) for s, sent in conv]) 736 | cprint("Tokenizing persona data...") 737 | persona = tokenize_personas(persona, tokenizer, all_speakers, num_personas) 738 | cprint("Saving tokenized persona to file...") 739 | save_pickle(persona, cached_tokenized_persona_path) 740 | cprint("Persona dataset statistics (after tokenization):", len(persona)) 741 | cprint("Sample tokenized persona:", list(persona.values())[0]) 742 | 743 | cprint("Sample tokenized data: ") 744 | cprint(train[0]) 745 | cprint(valid[0]) 746 | 747 | # select subsets of training and validation data for casualconversation 748 | cprint(dataset) 749 | if dataset == "casualconversation_v3": 750 | cprint("reducing dataset size...") 751 | train = train[:150000] 752 | valid = valid[:20000] 753 | 754 | if train_ratio != 1: 755 | num_train_examples = int(len(train) * train_ratio) 756 | cprint("reducing training set size to {0}...".format(num_train_examples)) 757 | train = train[:num_train_examples] 758 | 759 | if PEC_ratio != 0: 760 | cprint("Replacing {0} of casual to PEC...".format(PEC_ratio)) 761 | cprint(len(train)) 762 | 763 | PEC_train_path = "./data/reddit_empathetic/combined_v3/train_cleaned_bert.pkl" 764 | PEC_persona_path = "./data/reddit_empathetic/combined_v3/persona_10.pkl" 765 | 766 | # load cached casual conversations and persona 767 | num_PEC_examples = int(len(train) * PEC_ratio) 768 | train[:num_PEC_examples] = load_pickle(PEC_train_path.replace(".pkl", "_tokenized.pkl"))[:num_PEC_examples] 769 | cprint(num_PEC_examples, len(train)) 770 | 771 | if num_personas > 0: 772 | cprint("number of speakers before merging PEC and casual: ", len(persona)) 773 | # merge persona 774 | PEC_persona = load_pickle(PEC_persona_path.replace(".pkl", "_tokenized.pkl")) 775 | for k,v in PEC_persona.items(): 776 | if k not in persona: 777 | persona[k] = v 778 | cprint("number of speakers after merging PEC and casual: ", len(persona)) 779 | 780 | # create context and response 781 | train = create_context_and_response(train) 782 | valid = create_context_and_response(valid) 783 | cprint("Sample context and response: ") 784 | cprint(train[0]) 785 | cprint(valid[0]) 786 | 787 | # convert to token ids 788 | cprint("Converting conversations to ids: ") 789 | if not test_mode: 790 | train_dataset = convert_conversations_to_ids(train, persona, tokenizer, \ 791 | max_seq_len, max_sent_len, num_personas) 792 | train_sampler = RandomSampler(train_dataset) 793 | train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=batch_size, drop_last=True) 794 | t_total = len(train_dataloader) // gradient_accumulation_steps * epochs 795 | cprint(train_dataset[0]) 796 | valid_dataset = convert_conversations_to_ids(valid, persona, tokenizer, \ 797 | max_seq_len, max_sent_len, num_personas) 798 | valid_sampler = RandomSampler(valid_dataset) 799 | valid_dataloader = DataLoader(valid_dataset, sampler=valid_sampler, batch_size=num_candidates) 800 | 801 | # create model 802 | cprint("Building model...") 803 | model = BertModel.from_pretrained(model, output_hidden_states=output_hidden_states) 804 | cprint(model) 805 | cprint("number of parameters: ", count_parameters(model)) 806 | 807 | if shared: 808 | cprint("number of encoders: 1") 809 | models = [model] 810 | else: 811 | if num_personas == 0: 812 | cprint("number of encoders: 2") 813 | # models = [model, copy.deepcopy(model)] 814 | models = [model, pickle.loads(pickle.dumps(model))] 815 | else: 816 | cprint("number of encoders: 3") 817 | # models = [model, copy.deepcopy(model), copy.deepcopy(model)] 818 | models = [model, pickle.loads(pickle.dumps(model)), pickle.loads(pickle.dumps(model))] 819 | 820 | if test_mode: 821 | cprint("Loading weights from ", load_model_path) 822 | model.load_state_dict(torch.load(load_model_path)) 823 | models = [model] 824 | 825 | for i, model in enumerate(models): 826 | cprint("model {0} number of parameters: ".format(i), count_parameters(model)) 827 | model.to(device) 828 | 829 | # optimization 830 | amp = None 831 | if fp16: 832 | from apex import amp 833 | 834 | no_decay = ["bias", "LayerNorm.weight"] 835 | optimizers = [] 836 | schedulers = [] 837 | for i, model in enumerate(models): 838 | optimizer_grouped_parameters = [ 839 | { 840 | "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 841 | "weight_decay": weight_decay, 842 | }, 843 | {"params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], "weight_decay": 0.0}, 844 | ] 845 | optimizer = AdamW(optimizer_grouped_parameters, lr=lr, eps=1e-8) 846 | 847 | if fp16: 848 | model, optimizer = amp.initialize(model, optimizer, opt_level=fp16_opt_level) 849 | models[i] = model 850 | optimizers.append(optimizer) 851 | 852 | if not test_mode: 853 | scheduler = get_linear_schedule_with_warmup( 854 | optimizer, num_warmup_steps=warmup_steps, num_training_steps=t_total 855 | ) 856 | schedulers.append(scheduler) 857 | 858 | if test_mode: 859 | # evaluation 860 | for model in models: 861 | model.eval() 862 | valid_iterator = tqdm(valid_dataloader, desc="Iteration") 863 | valid_loss, (valid_acc, valid_recall, valid_MRR) = evaluate_epoch(valid_iterator, models, \ 864 | num_personas, gradient_accumulation_steps, device, dataset, 0, apply_interaction, matching_method, aggregation_method) 865 | cprint("test loss: {0:.4f}, test acc: {1:.4f}, test recall: {2}, test MRR: {3:.4f}" 866 | .format(valid_loss, valid_acc, valid_recall, valid_MRR)) 867 | sys.exit() 868 | 869 | # training 870 | epoch_train_losses = [] 871 | epoch_valid_losses = [] 872 | epoch_valid_accs = [] 873 | epoch_valid_recalls = [] 874 | epoch_valid_MRRs = [] 875 | cprint("***** Running training *****") 876 | cprint("Num examples =", len(train_dataset)) 877 | cprint("Num Epochs =", epochs) 878 | cprint("Total optimization steps =", t_total) 879 | best_model_statedict = {} 880 | for epoch in range(epochs): 881 | cprint("Epoch", epoch+1) 882 | # training 883 | for model in models: 884 | model.train() 885 | train_iterator = tqdm(train_dataloader, desc="Iteration") 886 | train_loss, (train_acc, _, _) = train_epoch(train_iterator, models, num_personas, optimizers, \ 887 | schedulers, gradient_accumulation_steps, device, fp16, amp, apply_interaction, matching_method, aggregation_method) 888 | epoch_train_losses.append(train_loss) 889 | 890 | # evaluation 891 | for model in models: 892 | model.eval() 893 | valid_iterator = tqdm(valid_dataloader, desc="Iteration") 894 | valid_loss, (valid_acc, valid_recall, valid_MRR) = evaluate_epoch(valid_iterator, models, \ 895 | num_personas, gradient_accumulation_steps, device, dataset, epoch, apply_interaction, matching_method, aggregation_method) 896 | 897 | cprint("Config id: {7}, Epoch {0}: train loss: {1:.4f}, valid loss: {2:.4f}, train_acc: {3:.4f}, valid acc: {4:.4f}, valid recall: {5}, valid_MRR: {6:.4f}" 898 | .format(epoch+1, train_loss, valid_loss, train_acc, valid_acc, valid_recall, valid_MRR, config_id)) 899 | 900 | epoch_valid_losses.append(valid_loss) 901 | epoch_valid_accs.append(valid_acc) 902 | epoch_valid_recalls.append(valid_recall) 903 | epoch_valid_MRRs.append(valid_MRR) 904 | 905 | if save_model_path != "": 906 | if epoch == 0: 907 | for k, v in models[0].state_dict().items(): 908 | best_model_statedict[k] = v.cpu() 909 | else: 910 | if epoch_valid_recalls[-1][0] == max([recall1 for recall1, _, _ in epoch_valid_recalls]): 911 | for k, v in models[0].state_dict().items(): 912 | best_model_statedict[k] = v.cpu() 913 | 914 | 915 | config.pop("seed") 916 | config.pop("config_id") 917 | metrics["config"] = config 918 | metrics["score"] = max(epoch_valid_accs) 919 | metrics["epoch"] = np.argmax(epoch_valid_accs).item() 920 | metrics["recall"] = epoch_valid_recalls 921 | metrics["MRR"] = epoch_valid_MRRs 922 | 923 | if save_model_path: 924 | cprint("Saving model to ", save_model_path) 925 | torch.save(best_model_statedict, save_model_path) 926 | 927 | return metrics 928 | 929 | 930 | def clean_config(configs): 931 | cleaned_configs = [] 932 | for config in configs: 933 | if config not in cleaned_configs: 934 | cleaned_configs.append(config) 935 | return cleaned_configs 936 | 937 | 938 | def merge_metrics(metrics): 939 | avg_metrics = {"score" : 0} 940 | num_metrics = len(metrics) 941 | for metric in metrics: 942 | for k in metric: 943 | if k != "config": 944 | avg_metrics[k] += np.array(metric[k]) 945 | 946 | for k, v in avg_metrics.items(): 947 | avg_metrics[k] = (v/num_metrics).tolist() 948 | 949 | return avg_metrics 950 | 951 | 952 | if __name__ == "__main__": 953 | mp.set_start_method('spawn', force=True) 954 | parser = argparse.ArgumentParser(description="Model for Transformer-based Dialogue Generation with Controlled Emotion") 955 | parser.add_argument('--config', help='Config to read details', required=True) 956 | parser.add_argument('--note', help='Experiment note', default="") 957 | args = parser.parse_args() 958 | cprint("Experiment note: ", args.note) 959 | with open(args.config) as configfile: 960 | config = json.load(configfile) # config is now a python dict 961 | 962 | # pass experiment config to main 963 | parameters_to_search = OrderedDict() # keep keys in order 964 | other_parameters = {} 965 | keys_to_omit = ["kernel_sizes"] # keys that allow a list of values 966 | for k, v in config.items(): 967 | # if value is a list provided that key is not device, or kernel_sizes is a nested list 968 | if isinstance(v, list) and k not in keys_to_omit: 969 | parameters_to_search[k] = v 970 | elif k in keys_to_omit and isinstance(config[k], list) and isinstance(config[k][0], list): 971 | parameters_to_search[k] = v 972 | else: 973 | other_parameters[k] = v 974 | 975 | if len(parameters_to_search) == 0: 976 | config_id = time.perf_counter() 977 | config["config_id"] = config_id 978 | cprint(config) 979 | output = main(config, progress=1) 980 | cprint("-"*80) 981 | cprint(output["config"]) 982 | cprint(output["epoch"]) 983 | cprint(output["score"]) 984 | cprint(output["recall"]) 985 | cprint(output["MRR"]) 986 | else: 987 | all_configs = [] 988 | for i, r in enumerate(itertools.product(*parameters_to_search.values())): 989 | specific_config = {} 990 | for idx, k in enumerate(parameters_to_search.keys()): 991 | specific_config[k] = r[idx] 992 | 993 | # merge with other parameters 994 | merged_config = {**other_parameters, **specific_config} 995 | all_configs.append(merged_config) 996 | 997 | # cprint all configs 998 | for config in all_configs: 999 | config_id = time.perf_counter() 1000 | config["config_id"] = config_id 1001 | logging.critical("config id: {0}".format(config_id)) 1002 | cprint(config) 1003 | cprint("\n") 1004 | 1005 | # multiprocessing 1006 | num_configs = len(all_configs) 1007 | # mp.set_start_method('spawn') 1008 | pool = mp.Pool(processes=config["processes"]) 1009 | results = [pool.apply_async(main, args=(x,i/num_configs)) for i,x in enumerate(all_configs)] 1010 | outputs = [p.get() for p in results] 1011 | 1012 | # if run multiple models using different seed and get the averaged result 1013 | if "seed" in parameters_to_search: 1014 | all_metrics = [] 1015 | all_cleaned_configs = clean_config([output["config"] for output in outputs]) 1016 | for config in all_cleaned_configs: 1017 | metrics_per_config = [] 1018 | for output in outputs: 1019 | if output["config"] == config: 1020 | metrics_per_config.append(output) 1021 | avg_metrics = merge_metrics(metrics_per_config) 1022 | all_metrics.append((config, avg_metrics)) 1023 | # log metrics 1024 | cprint("Average evaluation result across different seeds: ") 1025 | for config, metric in all_metrics: 1026 | cprint("-"*80) 1027 | cprint(config) 1028 | cprint(metric) 1029 | 1030 | # save to log 1031 | with open("./log/{0}.txt".format(time.perf_counter()), "a+") as f: 1032 | for config, metric in all_metrics: 1033 | f.write(json.dumps("-"*80) + "\n") 1034 | f.write(json.dumps(config) + "\n") 1035 | f.write(json.dumps(metric) + "\n") 1036 | 1037 | else: 1038 | for output in outputs: 1039 | cprint("-"*80) 1040 | cprint(output["config"]) 1041 | cprint(output["score"]) 1042 | cprint(output["recall"]) 1043 | cprint(output["MRR"]) 1044 | cprint("Best result at epoch {0}: ".format(output["epoch"])) 1045 | cprint(output["recall"][output["epoch"]], output["MRR"][output["epoch"]]) 1046 | 1047 | # save to log 1048 | with open("./log/{0}.txt".format(time.perf_counter()), "a+") as f: 1049 | for output in outputs: 1050 | f.write(json.dumps("-"*80) + "\n") 1051 | f.write(json.dumps(output) + "\n") 1052 | -------------------------------------------------------------------------------- /CoBERT_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "test_mode": 0, 3 | "num_candidates": 100, 4 | "num_personas": 10, 5 | "persona_path": "./data/reddit_empathetic/offmychest_v3/persona_20.pkl", 6 | "train_path": "./data/reddit_empathetic/offmychest_v3/train_cleaned_bert.pkl", 7 | "valid_path": "./data/reddit_empathetic/offmychest_v3/valid_cleaned_bert.pkl", 8 | "test_path": "./data/reddit_empathetic/offmychest_v3/test_cleaned_bert.pkl", 9 | "load_model_path": "./saved_model/offmychest_v3/CoBERT_max.pt", 10 | "save_model_path": "./saved_model/offmychest_v3/CoBERT_max.pt", 11 | "PEC_ratio": 0, 12 | "train_ratio": 1, 13 | "max_sent_len": 30, 14 | "max_seq_len": 256, 15 | "model": "bert-base-uncased", 16 | "shared": 1, 17 | "apply_interaction": 1, 18 | "matching_method": "CoBERT", 19 | "aggregation_method": "max", 20 | "epochs": 10, 21 | "warmup_steps": 0, 22 | "batch_size": 16, 23 | "gradient_accumulation_steps": 4, 24 | "lr": 2e-5, 25 | "fp16": 1, 26 | "fp16_opt_level": "O1", 27 | "device": 0, 28 | "seed": 1, 29 | "processes": 1 30 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Towards Persona-Based Empathetic Conversational Models (PEC) 2 | This is the repo for our work "[Towards Persona-Based Empathetic Conversational Models](https://arxiv.org/abs/2004.12316)" (EMNLP 2020). The code depends on PyTorch (>=v1.0) and [transformers]((https://github.com/huggingface/transformers)) (>=v2.3). 3 | 4 | 5 | ### Data 6 | The PEC dataset is available [here](https://www.dropbox.com/s/9lhdf6iwv61xiao/cleaned.zip?dl=0). 7 | 8 | The persona dataset with 100 persona sentences each is available [here](https://www.dropbox.com/s/enrsqee0obucddf/PEC_persona_100.zip?dl=0) 9 | 10 | You can refer to the sample files here to preprocess the datasets: [valid_cleaned_bert.pkl](https://www.dropbox.com/s/urb6kfcmuhxbs4k/valid_cleaned_bert.pkl?dl=0) and [persona_20.pkl](https://www.dropbox.com/s/q8ihrutg28jxyl8/persona_20.pkl?dl=0) 11 | 12 | **Our dataset is available on [Huggingface Datasets](https://huggingface.co/datasets/viewer/?dataset=pec&config=all) for ease of use.** 13 | 14 | ### Model 15 | This repo includes our implementation of CoBERT. 16 | 17 | ### Training 18 | 19 | ```python CoBERT.py --config CoBERT_config.json``` 20 | 21 | ### Evaluation 22 | Set test_mode=1 and load_model_path to a saved model in CoBERT_config.json, and then run 23 | 24 | ```python CoBERT.py --config CoBERT_config.json``` 25 | -------------------------------------------------------------------------------- /util.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import torch 3 | import numpy as np 4 | 5 | def save_pickle(obj, path): 6 | with open(path, "wb") as f: 7 | pickle.dump(obj, f) 8 | 9 | def load_pickle(path): 10 | with open(path, "rb") as f: 11 | return pickle.load(f) 12 | 13 | def count_parameters(model): 14 | return sum(p.numel() for p in model.parameters() if p.requires_grad) 15 | 16 | def compute_metrics(batch_x_emb, batch_y_emb): 17 | """ 18 | recall@k for N candidates 19 | if batch_x_emb.dim() == 2: 20 | # batch_x_emb: (batch_size, emb_size) 21 | # batch_y_emb: (batch_size, emb_size) 22 | 23 | if batch_x_emb.dim() == 3: 24 | # batch_x_emb: (batch_size, batch_size, emb_size), the 1st dim is along examples and the 2nd dim is along candidates 25 | # batch_y_emb: (batch_size, emb_size) 26 | 27 | """ 28 | batch_size = batch_x_emb.size(0) 29 | targets = torch.arange(batch_size, device=batch_x_emb.device) 30 | if batch_x_emb.dim() == 2: 31 | dot_products = batch_x_emb.mm(batch_y_emb.t()) # (batch_size, batch_size) 32 | elif batch_x_emb.dim() == 3: 33 | dot_products = torch.bmm(batch_x_emb, batch_y_emb.unsqueeze(0).repeat(batch_size, 1, 1).transpose(1,2))[:, targets, targets] 34 | 35 | # dot_products: (batch_size, batch_size) 36 | sorted_indices = dot_products.sort(descending=True)[1] 37 | targets = np.arange(batch_size).tolist() 38 | recall_k = [] 39 | if batch_size <= 10: 40 | ks = [1, max(1, round(batch_size*0.2)), max(1, round(batch_size*0.5))] 41 | elif batch_size <= 100: 42 | ks = [1, max(1, round(batch_size*0.1)), max(1, round(batch_size*0.5))] 43 | else: 44 | raise ValueError("batch_size: {0} is not proper".format(batch_size)) 45 | for k in ks: 46 | # sorted_indices[:,:k]: (batch_size, k) 47 | num_ok = 0 48 | for tgt, topk in zip(targets, sorted_indices[:,:k].tolist()): 49 | if tgt in topk: 50 | num_ok += 1 51 | recall_k.append(num_ok/batch_size) 52 | 53 | # MRR 54 | MRR = 0 55 | for tgt, topk in zip(targets, sorted_indices.tolist()): 56 | rank = topk.index(tgt)+1 57 | MRR += 1/rank 58 | MRR = MRR/batch_size 59 | return recall_k, MRR 60 | 61 | 62 | def compute_metrics_from_logits(logits, targets): 63 | """ 64 | recall@k for N candidates 65 | 66 | logits: (batch_size, num_candidates) 67 | targets: (batch_size, ) 68 | 69 | """ 70 | batch_size, num_candidates = logits.shape 71 | 72 | sorted_indices = logits.sort(descending=True)[1] 73 | targets = targets.tolist() 74 | 75 | recall_k = [] 76 | if num_candidates <= 10: 77 | ks = [1, max(1, round(num_candidates*0.2)), max(1, round(num_candidates*0.5))] 78 | elif num_candidates <= 100: 79 | ks = [1, max(1, round(num_candidates*0.1)), max(1, round(num_candidates*0.5))] 80 | else: 81 | raise ValueError("num_candidates: {0} is not proper".format(num_candidates)) 82 | for k in ks: 83 | # sorted_indices[:,:k]: (batch_size, k) 84 | num_ok = 0 85 | for tgt, topk in zip(targets, sorted_indices[:,:k].tolist()): 86 | if tgt in topk: 87 | num_ok += 1 88 | recall_k.append(num_ok/batch_size) 89 | 90 | # MRR 91 | MRR = 0 92 | for tgt, topk in zip(targets, sorted_indices.tolist()): 93 | rank = topk.index(tgt)+1 94 | MRR += 1/rank 95 | MRR = MRR/batch_size 96 | return recall_k, MRR 97 | --------------------------------------------------------------------------------