7 |
7 | SimCSE: Simple Contrastive Learning of Sentence Embeddings | Github Repo
" 45 | examples = [ 46 | ["There's a kid on a skateboard.", 47 | "A kid is skateboarding.", 48 | "A kid is inside the house."] 49 | ] 50 | 51 | gr.Interface(simcse, inputs, outputs, title=title, description=description, article=article, examples=examples).launch() -------------------------------------------------------------------------------- /SentEval/examples/skipthought.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017-present, Facebook, Inc. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | 8 | from __future__ import absolute_import, division, unicode_literals 9 | 10 | """ 11 | Example of file for SkipThought in SentEval 12 | """ 13 | import logging 14 | import sys 15 | sys.setdefaultencoding('utf8') 16 | 17 | 18 | # Set PATHs 19 | PATH_TO_SENTEVAL = '../' 20 | PATH_TO_DATA = '../data/senteval_data/' 21 | PATH_TO_SKIPTHOUGHT = '' 22 | 23 | assert PATH_TO_SKIPTHOUGHT != '', 'Download skipthought and set correct PATH' 24 | 25 | # import skipthought and Senteval 26 | sys.path.insert(0, PATH_TO_SKIPTHOUGHT) 27 | import skipthoughts 28 | sys.path.insert(0, PATH_TO_SENTEVAL) 29 | import senteval 30 | 31 | 32 | def prepare(params, samples): 33 | return 34 | 35 | def batcher(params, batch): 36 | batch = [str(' '.join(sent), errors="ignore") if sent != [] else '.' for sent in batch] 37 | embeddings = skipthoughts.encode(params['encoder'], batch, 38 | verbose=False, use_eos=True) 39 | return embeddings 40 | 41 | 42 | # Set params for SentEval 43 | params_senteval = {'task_path': PATH_TO_DATA, 'usepytorch': True, 'kfold': 10, 'batch_size': 512} 44 | params_senteval['classifier'] = {'nhid': 0, 'optim': 'adam', 'batch_size': 64, 45 | 'tenacity': 5, 'epoch_size': 4} 46 | # Set up logger 47 | logging.basicConfig(format='%(asctime)s : %(message)s', level=logging.DEBUG) 48 | 49 | if __name__ == "__main__": 50 | # Load SkipThought model 51 | params_senteval['encoder'] = skipthoughts.load_model() 52 | 53 | se = senteval.engine.SE(params_senteval, batcher, prepare) 54 | transfer_tasks = ['STS12', 'STS13', 'STS14', 'STS15', 'STS16', 55 | 'MR', 'CR', 'MPQA', 'SUBJ', 'SST2', 'SST5', 'TREC', 'MRPC', 56 | 'SICKEntailment', 'SICKRelatedness', 'STSBenchmark', 57 | 'Length', 'WordContent', 'Depth', 'TopConstituents', 58 | 'BigramShift', 'Tense', 'SubjNumber', 'ObjNumber', 59 | 'OddManOut', 'CoordinationInversion'] 60 | results = se.eval(transfer_tasks) 61 | print(results) 62 | -------------------------------------------------------------------------------- /SentEval/examples/googleuse.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017-present, Facebook, Inc. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | 8 | from __future__ import absolute_import, division 9 | 10 | import os 11 | import sys 12 | import logging 13 | import tensorflow as tf 14 | import tensorflow_hub as hub 15 | tf.logging.set_verbosity(0) 16 | 17 | # Set PATHs 18 | PATH_TO_SENTEVAL = '../' 19 | PATH_TO_DATA = '../data' 20 | 21 | # import SentEval 22 | sys.path.insert(0, PATH_TO_SENTEVAL) 23 | import senteval 24 | 25 | # tensorflow session 26 | session = tf.Session() 27 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 28 | 29 | # SentEval prepare and batcher 30 | def prepare(params, samples): 31 | return 32 | 33 | def batcher(params, batch): 34 | batch = [' '.join(sent) if sent != [] else '.' for sent in batch] 35 | embeddings = params['google_use'](batch) 36 | return embeddings 37 | 38 | def make_embed_fn(module): 39 | with tf.Graph().as_default(): 40 | sentences = tf.placeholder(tf.string) 41 | embed = hub.Module(module) 42 | embeddings = embed(sentences) 43 | session = tf.train.MonitoredSession() 44 | return lambda x: session.run(embeddings, {sentences: x}) 45 | 46 | # Start TF session and load Google Universal Sentence Encoder 47 | encoder = make_embed_fn("https://tfhub.dev/google/universal-sentence-encoder-large/2") 48 | 49 | # Set params for SentEval 50 | params_senteval = {'task_path': PATH_TO_DATA, 'usepytorch': True, 'kfold': 5} 51 | params_senteval['classifier'] = {'nhid': 0, 'optim': 'rmsprop', 'batch_size': 128, 52 | 'tenacity': 3, 'epoch_size': 2} 53 | params_senteval['google_use'] = encoder 54 | 55 | # Set up logger 56 | logging.basicConfig(format='%(asctime)s : %(message)s', level=logging.DEBUG) 57 | 58 | if __name__ == "__main__": 59 | se = senteval.engine.SE(params_senteval, batcher, prepare) 60 | transfer_tasks = ['STS12', 'STS13', 'STS14', 'STS15', 'STS16', 61 | 'MR', 'CR', 'MPQA', 'SUBJ', 'SST2', 'SST5', 'TREC', 'MRPC', 62 | 'SICKEntailment', 'SICKRelatedness', 'STSBenchmark', 63 | 'Length', 'WordContent', 'Depth', 'TopConstituents', 64 | 'BigramShift', 'Tense', 'SubjNumber', 'ObjNumber', 65 | 'OddManOut', 'CoordinationInversion'] 66 | results = se.eval(transfer_tasks) 67 | print(results) 68 | -------------------------------------------------------------------------------- /SentEval/examples/gensen.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017-present, Facebook, Inc. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | 8 | """ 9 | Clone GenSen repo here: https://github.com/Maluuba/gensen.git 10 | And follow instructions for loading the model used in batcher 11 | """ 12 | 13 | from __future__ import absolute_import, division, unicode_literals 14 | 15 | import sys 16 | import logging 17 | # import GenSen package 18 | from gensen import GenSen, GenSenSingle 19 | 20 | # Set PATHs 21 | PATH_TO_SENTEVAL = '../' 22 | PATH_TO_DATA = '../data' 23 | 24 | # import SentEval 25 | sys.path.insert(0, PATH_TO_SENTEVAL) 26 | import senteval 27 | 28 | # SentEval prepare and batcher 29 | def prepare(params, samples): 30 | return 31 | 32 | def batcher(params, batch): 33 | batch = [' '.join(sent) if sent != [] else '.' for sent in batch] 34 | _, reps_h_t = gensen.get_representation( 35 | sentences, pool='last', return_numpy=True, tokenize=True 36 | ) 37 | embeddings = reps_h_t 38 | return embeddings 39 | 40 | # Load GenSen model 41 | gensen_1 = GenSenSingle( 42 | model_folder='../data/models', 43 | filename_prefix='nli_large_bothskip', 44 | pretrained_emb='../data/embedding/glove.840B.300d.h5' 45 | ) 46 | gensen_2 = GenSenSingle( 47 | model_folder='../data/models', 48 | filename_prefix='nli_large_bothskip_parse', 49 | pretrained_emb='../data/embedding/glove.840B.300d.h5' 50 | ) 51 | gensen_encoder = GenSen(gensen_1, gensen_2) 52 | reps_h, reps_h_t = gensen.get_representation( 53 | sentences, pool='last', return_numpy=True, tokenize=True 54 | ) 55 | 56 | # Set params for SentEval 57 | params_senteval = {'task_path': PATH_TO_DATA, 'usepytorch': True, 'kfold': 5} 58 | params_senteval['classifier'] = {'nhid': 0, 'optim': 'rmsprop', 'batch_size': 128, 59 | 'tenacity': 3, 'epoch_size': 2} 60 | params_senteval['gensen'] = gensen_encoder 61 | 62 | # Set up logger 63 | logging.basicConfig(format='%(asctime)s : %(message)s', level=logging.DEBUG) 64 | 65 | if __name__ == "__main__": 66 | se = senteval.engine.SE(params_senteval, batcher, prepare) 67 | transfer_tasks = ['STS12', 'STS13', 'STS14', 'STS15', 'STS16', 68 | 'MR', 'CR', 'MPQA', 'SUBJ', 'SST2', 'SST5', 'TREC', 'MRPC', 69 | 'SICKEntailment', 'SICKRelatedness', 'STSBenchmark', 70 | 'Length', 'WordContent', 'Depth', 'TopConstituents', 71 | 'BigramShift', 'Tense', 'SubjNumber', 'ObjNumber', 72 | 'OddManOut', 'CoordinationInversion'] 73 | results = se.eval(transfer_tasks) 74 | print(results) 75 | -------------------------------------------------------------------------------- /SentEval/examples/infersent.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017-present, Facebook, Inc. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | 8 | """ 9 | InferSent models. See https://github.com/facebookresearch/InferSent. 10 | """ 11 | 12 | from __future__ import absolute_import, division, unicode_literals 13 | 14 | import sys 15 | import os 16 | import torch 17 | import logging 18 | 19 | # get models.py from InferSent repo 20 | from models import InferSent 21 | 22 | # Set PATHs 23 | PATH_SENTEVAL = '../' 24 | PATH_TO_DATA = '../data' 25 | PATH_TO_W2V = 'PATH/TO/glove.840B.300d.txt' # or crawl-300d-2M.vec for V2 26 | MODEL_PATH = 'infersent1.pkl' 27 | V = 1 # version of InferSent 28 | 29 | assert os.path.isfile(MODEL_PATH) and os.path.isfile(PATH_TO_W2V), \ 30 | 'Set MODEL and GloVe PATHs' 31 | 32 | # import senteval 33 | sys.path.insert(0, PATH_SENTEVAL) 34 | import senteval 35 | 36 | 37 | def prepare(params, samples): 38 | params.infersent.build_vocab([' '.join(s) for s in samples], tokenize=False) 39 | 40 | 41 | def batcher(params, batch): 42 | sentences = [' '.join(s) for s in batch] 43 | embeddings = params.infersent.encode(sentences, bsize=params.batch_size, tokenize=False) 44 | return embeddings 45 | 46 | 47 | """ 48 | Evaluation of trained model on Transfer Tasks (SentEval) 49 | """ 50 | 51 | # define senteval params 52 | params_senteval = {'task_path': PATH_TO_DATA, 'usepytorch': True, 'kfold': 5} 53 | params_senteval['classifier'] = {'nhid': 0, 'optim': 'rmsprop', 'batch_size': 128, 54 | 'tenacity': 3, 'epoch_size': 2} 55 | # Set up logger 56 | logging.basicConfig(format='%(asctime)s : %(message)s', level=logging.DEBUG) 57 | 58 | if __name__ == "__main__": 59 | # Load InferSent model 60 | params_model = {'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048, 61 | 'pool_type': 'max', 'dpout_model': 0.0, 'version': V} 62 | model = InferSent(params_model) 63 | model.load_state_dict(torch.load(MODEL_PATH)) 64 | model.set_w2v_path(PATH_TO_W2V) 65 | 66 | params_senteval['infersent'] = model.cuda() 67 | 68 | se = senteval.engine.SE(params_senteval, batcher, prepare) 69 | transfer_tasks = ['STS12', 'STS13', 'STS14', 'STS15', 'STS16', 70 | 'MR', 'CR', 'MPQA', 'SUBJ', 'SST2', 'SST5', 'TREC', 'MRPC', 71 | 'SICKEntailment', 'SICKRelatedness', 'STSBenchmark', 72 | 'Length', 'WordContent', 'Depth', 'TopConstituents', 73 | 'BigramShift', 'Tense', 'SubjNumber', 'ObjNumber', 74 | 'OddManOut', 'CoordinationInversion'] 75 | results = se.eval(transfer_tasks) 76 | print(results) 77 | -------------------------------------------------------------------------------- /SentEval/senteval/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017-present, Facebook, Inc. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the license found in the 5 | # LICENSE file in the root directory of this source tree. 6 | # 7 | 8 | from __future__ import absolute_import, division, unicode_literals 9 | 10 | import numpy as np 11 | import re 12 | import inspect 13 | from torch import optim 14 | 15 | 16 | def create_dictionary(sentences): 17 | words = {} 18 | for s in sentences: 19 | for word in s: 20 | if word in words: 21 | words[word] += 1 22 | else: 23 | words[word] = 1 24 | words[''] = 1e9 + 2
27 | # words[' '] = 1e9 + 2
43 |
44 | sorted_words = sorted(words.items(), key=lambda x: -x[1]) # inverse sort
45 | id2word = []
46 | word2id = {}
47 | for i, (w, _) in enumerate(sorted_words):
48 | id2word.append(w)
49 | word2id[w] = i
50 |
51 | return id2word, word2id
52 |
53 | # Get word vectors from vocabulary (glove, word2vec, fasttext ..)
54 | def get_wordvec(path_to_vec, word2id):
55 | word_vec = {}
56 |
57 | with io.open(path_to_vec, 'r', encoding='utf-8') as f:
58 | # if word2vec or fasttext file : skip first line "next(f)"
59 | for line in f:
60 | word, vec = line.split(' ', 1)
61 | if word in word2id:
62 | word_vec[word] = np.fromstring(vec, sep=' ')
63 |
64 | logging.info('Found {0} words with word vectors, out of \
65 | {1} words'.format(len(word_vec), len(word2id)))
66 | return word_vec
67 |
68 |
69 | # SentEval prepare and batcher
70 | def prepare(params, samples):
71 | _, params.word2id = create_dictionary(samples)
72 | params.word_vec = get_wordvec(PATH_TO_VEC, params.word2id)
73 | params.wvec_dim = 300
74 | return
75 |
76 | def batcher(params, batch):
77 | batch = [sent if sent != [] else ['.'] for sent in batch]
78 | embeddings = []
79 |
80 | for sent in batch:
81 | sentvec = []
82 | for word in sent:
83 | if word in params.word_vec:
84 | sentvec.append(params.word_vec[word])
85 | if not sentvec:
86 | vec = np.zeros(params.wvec_dim)
87 | sentvec.append(vec)
88 | sentvec = np.mean(sentvec, 0)
89 | embeddings.append(sentvec)
90 |
91 | embeddings = np.vstack(embeddings)
92 | return embeddings
93 |
94 |
95 | # Set params for SentEval
96 | params_senteval = {'task_path': PATH_TO_DATA, 'usepytorch': True, 'kfold': 5}
97 | params_senteval['classifier'] = {'nhid': 0, 'optim': 'rmsprop', 'batch_size': 128,
98 | 'tenacity': 3, 'epoch_size': 2}
99 |
100 | # Set up logger
101 | logging.basicConfig(format='%(asctime)s : %(message)s', level=logging.DEBUG)
102 |
103 | if __name__ == "__main__":
104 | se = senteval.engine.SE(params_senteval, batcher, prepare)
105 | transfer_tasks = ['STS12', 'STS13', 'STS14', 'STS15', 'STS16',
106 | 'MR', 'CR', 'MPQA', 'SUBJ', 'SST2', 'SST5', 'TREC', 'MRPC',
107 | 'SICKEntailment', 'SICKRelatedness', 'STSBenchmark',
108 | 'Length', 'WordContent', 'Depth', 'TopConstituents',
109 | 'BigramShift', 'Tense', 'SubjNumber', 'ObjNumber',
110 | 'OddManOut', 'CoordinationInversion']
111 | results = se.eval(transfer_tasks)
112 | print(results)
113 |
--------------------------------------------------------------------------------
/SentEval/senteval/binary.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | '''
9 | Binary classifier and corresponding datasets : MR, CR, SUBJ, MPQA
10 | '''
11 | from __future__ import absolute_import, division, unicode_literals
12 |
13 | import io
14 | import os
15 | import numpy as np
16 | import logging
17 |
18 | from senteval.tools.validation import InnerKFoldClassifier
19 |
20 |
21 | class BinaryClassifierEval(object):
22 | def __init__(self, pos, neg, seed=1111):
23 | self.seed = seed
24 | self.samples, self.labels = pos + neg, [1] * len(pos) + [0] * len(neg)
25 | self.n_samples = len(self.samples)
26 |
27 | def do_prepare(self, params, prepare):
28 | # prepare is given the whole text
29 | return prepare(params, self.samples)
30 | # prepare puts everything it outputs in "params" : params.word2id etc
31 | # Those output will be further used by "batcher".
32 |
33 | def loadFile(self, fpath):
34 | with io.open(fpath, 'r', encoding='latin-1') as f:
35 | return [line.split() for line in f.read().splitlines()]
36 |
37 | def run(self, params, batcher):
38 | enc_input = []
39 | # Sort to reduce padding
40 | sorted_corpus = sorted(zip(self.samples, self.labels),
41 | key=lambda z: (len(z[0]), z[1]))
42 | sorted_samples = [x for (x, y) in sorted_corpus]
43 | sorted_labels = [y for (x, y) in sorted_corpus]
44 | logging.info('Generating sentence embeddings')
45 | for ii in range(0, self.n_samples, params.batch_size):
46 | batch = sorted_samples[ii:ii + params.batch_size]
47 | embeddings = batcher(params, batch)
48 | enc_input.append(embeddings)
49 | enc_input = np.vstack(enc_input)
50 | logging.info('Generated sentence embeddings')
51 |
52 | config = {'nclasses': 2, 'seed': self.seed,
53 | 'usepytorch': params.usepytorch,
54 | 'classifier': params.classifier,
55 | 'nhid': params.nhid, 'kfold': params.kfold}
56 | clf = InnerKFoldClassifier(enc_input, np.array(sorted_labels), config)
57 | devacc, testacc = clf.run()
58 | logging.debug('Dev acc : {0} Test acc : {1}\n'.format(devacc, testacc))
59 | return {'devacc': devacc, 'acc': testacc, 'ndev': self.n_samples,
60 | 'ntest': self.n_samples}
61 |
62 |
63 | class CREval(BinaryClassifierEval):
64 | def __init__(self, task_path, seed=1111):
65 | logging.debug('***** Transfer task : CR *****\n\n')
66 | pos = self.loadFile(os.path.join(task_path, 'custrev.pos'))
67 | neg = self.loadFile(os.path.join(task_path, 'custrev.neg'))
68 | super(self.__class__, self).__init__(pos, neg, seed)
69 |
70 |
71 | class MREval(BinaryClassifierEval):
72 | def __init__(self, task_path, seed=1111):
73 | logging.debug('***** Transfer task : MR *****\n\n')
74 | pos = self.loadFile(os.path.join(task_path, 'rt-polarity.pos'))
75 | neg = self.loadFile(os.path.join(task_path, 'rt-polarity.neg'))
76 | super(self.__class__, self).__init__(pos, neg, seed)
77 |
78 |
79 | class SUBJEval(BinaryClassifierEval):
80 | def __init__(self, task_path, seed=1111):
81 | logging.debug('***** Transfer task : SUBJ *****\n\n')
82 | obj = self.loadFile(os.path.join(task_path, 'subj.objective'))
83 | subj = self.loadFile(os.path.join(task_path, 'subj.subjective'))
84 | super(self.__class__, self).__init__(obj, subj, seed)
85 |
86 |
87 | class MPQAEval(BinaryClassifierEval):
88 | def __init__(self, task_path, seed=1111):
89 | logging.debug('***** Transfer task : MPQA *****\n\n')
90 | pos = self.loadFile(os.path.join(task_path, 'mpqa.pos'))
91 | neg = self.loadFile(os.path.join(task_path, 'mpqa.neg'))
92 | super(self.__class__, self).__init__(pos, neg, seed)
93 |
--------------------------------------------------------------------------------
/SentEval/senteval/sst.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | '''
9 | SST - binary classification
10 | '''
11 |
12 | from __future__ import absolute_import, division, unicode_literals
13 |
14 | import os
15 | import io
16 | import logging
17 | import numpy as np
18 |
19 | from senteval.tools.validation import SplitClassifier
20 |
21 |
22 | class SSTEval(object):
23 | def __init__(self, task_path, nclasses=2, seed=1111):
24 | self.seed = seed
25 |
26 | # binary of fine-grained
27 | assert nclasses in [2, 5]
28 | self.nclasses = nclasses
29 | self.task_name = 'Binary' if self.nclasses == 2 else 'Fine-Grained'
30 | logging.debug('***** Transfer task : SST %s classification *****\n\n', self.task_name)
31 |
32 | train = self.loadFile(os.path.join(task_path, 'sentiment-train'))
33 | dev = self.loadFile(os.path.join(task_path, 'sentiment-dev'))
34 | test = self.loadFile(os.path.join(task_path, 'sentiment-test'))
35 | self.sst_data = {'train': train, 'dev': dev, 'test': test}
36 |
37 | def do_prepare(self, params, prepare):
38 | samples = self.sst_data['train']['X'] + self.sst_data['dev']['X'] + \
39 | self.sst_data['test']['X']
40 | return prepare(params, samples)
41 |
42 | def loadFile(self, fpath):
43 | sst_data = {'X': [], 'y': []}
44 | with io.open(fpath, 'r', encoding='utf-8') as f:
45 | for line in f:
46 | if self.nclasses == 2:
47 | sample = line.strip().split('\t')
48 | sst_data['y'].append(int(sample[1]))
49 | sst_data['X'].append(sample[0].split())
50 | elif self.nclasses == 5:
51 | sample = line.strip().split(' ', 1)
52 | sst_data['y'].append(int(sample[0]))
53 | sst_data['X'].append(sample[1].split())
54 | assert max(sst_data['y']) == self.nclasses - 1
55 | return sst_data
56 |
57 | def run(self, params, batcher):
58 | sst_embed = {'train': {}, 'dev': {}, 'test': {}}
59 | bsize = params.batch_size
60 |
61 | for key in self.sst_data:
62 | logging.info('Computing embedding for {0}'.format(key))
63 | # Sort to reduce padding
64 | sorted_data = sorted(zip(self.sst_data[key]['X'],
65 | self.sst_data[key]['y']),
66 | key=lambda z: (len(z[0]), z[1]))
67 | self.sst_data[key]['X'], self.sst_data[key]['y'] = map(list, zip(*sorted_data))
68 |
69 | sst_embed[key]['X'] = []
70 | for ii in range(0, len(self.sst_data[key]['y']), bsize):
71 | batch = self.sst_data[key]['X'][ii:ii + bsize]
72 | embeddings = batcher(params, batch)
73 | sst_embed[key]['X'].append(embeddings)
74 | sst_embed[key]['X'] = np.vstack(sst_embed[key]['X'])
75 | sst_embed[key]['y'] = np.array(self.sst_data[key]['y'])
76 | logging.info('Computed {0} embeddings'.format(key))
77 |
78 | config_classifier = {'nclasses': self.nclasses, 'seed': self.seed,
79 | 'usepytorch': params.usepytorch,
80 | 'classifier': params.classifier}
81 |
82 | clf = SplitClassifier(X={'train': sst_embed['train']['X'],
83 | 'valid': sst_embed['dev']['X'],
84 | 'test': sst_embed['test']['X']},
85 | y={'train': sst_embed['train']['y'],
86 | 'valid': sst_embed['dev']['y'],
87 | 'test': sst_embed['test']['y']},
88 | config=config_classifier)
89 |
90 | devacc, testacc = clf.run()
91 | logging.debug('\nDev acc : {0} Test acc : {1} for \
92 | SST {2} classification\n'.format(devacc, testacc, self.task_name))
93 |
94 | return {'devacc': devacc, 'acc': testacc,
95 | 'ndev': len(sst_embed['dev']['X']),
96 | 'ntest': len(sst_embed['test']['X'])}
97 |
--------------------------------------------------------------------------------
/SentEval/senteval/mrpc.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | '''
9 | MRPC : Microsoft Research Paraphrase (detection) Corpus
10 | '''
11 | from __future__ import absolute_import, division, unicode_literals
12 |
13 | import os
14 | import logging
15 | import numpy as np
16 | import io
17 |
18 | from senteval.tools.validation import KFoldClassifier
19 |
20 | from sklearn.metrics import f1_score
21 |
22 |
23 | class MRPCEval(object):
24 | def __init__(self, task_path, seed=1111):
25 | logging.info('***** Transfer task : MRPC *****\n\n')
26 | self.seed = seed
27 | train = self.loadFile(os.path.join(task_path,
28 | 'msr_paraphrase_train.txt'))
29 | test = self.loadFile(os.path.join(task_path,
30 | 'msr_paraphrase_test.txt'))
31 | self.mrpc_data = {'train': train, 'test': test}
32 |
33 | def do_prepare(self, params, prepare):
34 | # TODO : Should we separate samples in "train, test"?
35 | samples = self.mrpc_data['train']['X_A'] + \
36 | self.mrpc_data['train']['X_B'] + \
37 | self.mrpc_data['test']['X_A'] + self.mrpc_data['test']['X_B']
38 | return prepare(params, samples)
39 |
40 | def loadFile(self, fpath):
41 | mrpc_data = {'X_A': [], 'X_B': [], 'y': []}
42 | with io.open(fpath, 'r', encoding='utf-8') as f:
43 | for line in f:
44 | text = line.strip().split('\t')
45 | mrpc_data['X_A'].append(text[3].split())
46 | mrpc_data['X_B'].append(text[4].split())
47 | mrpc_data['y'].append(text[0])
48 |
49 | mrpc_data['X_A'] = mrpc_data['X_A'][1:]
50 | mrpc_data['X_B'] = mrpc_data['X_B'][1:]
51 | mrpc_data['y'] = [int(s) for s in mrpc_data['y'][1:]]
52 | return mrpc_data
53 |
54 | def run(self, params, batcher):
55 | mrpc_embed = {'train': {}, 'test': {}}
56 |
57 | for key in self.mrpc_data:
58 | logging.info('Computing embedding for {0}'.format(key))
59 | # Sort to reduce padding
60 | text_data = {}
61 | sorted_corpus = sorted(zip(self.mrpc_data[key]['X_A'],
62 | self.mrpc_data[key]['X_B'],
63 | self.mrpc_data[key]['y']),
64 | key=lambda z: (len(z[0]), len(z[1]), z[2]))
65 |
66 | text_data['A'] = [x for (x, y, z) in sorted_corpus]
67 | text_data['B'] = [y for (x, y, z) in sorted_corpus]
68 | text_data['y'] = [z for (x, y, z) in sorted_corpus]
69 |
70 | for txt_type in ['A', 'B']:
71 | mrpc_embed[key][txt_type] = []
72 | for ii in range(0, len(text_data['y']), params.batch_size):
73 | batch = text_data[txt_type][ii:ii + params.batch_size]
74 | embeddings = batcher(params, batch)
75 | mrpc_embed[key][txt_type].append(embeddings)
76 | mrpc_embed[key][txt_type] = np.vstack(mrpc_embed[key][txt_type])
77 | mrpc_embed[key]['y'] = np.array(text_data['y'])
78 | logging.info('Computed {0} embeddings'.format(key))
79 |
80 | # Train
81 | trainA = mrpc_embed['train']['A']
82 | trainB = mrpc_embed['train']['B']
83 | trainF = np.c_[np.abs(trainA - trainB), trainA * trainB]
84 | trainY = mrpc_embed['train']['y']
85 |
86 | # Test
87 | testA = mrpc_embed['test']['A']
88 | testB = mrpc_embed['test']['B']
89 | testF = np.c_[np.abs(testA - testB), testA * testB]
90 | testY = mrpc_embed['test']['y']
91 |
92 | config = {'nclasses': 2, 'seed': self.seed,
93 | 'usepytorch': params.usepytorch,
94 | 'classifier': params.classifier,
95 | 'nhid': params.nhid, 'kfold': params.kfold}
96 | clf = KFoldClassifier(train={'X': trainF, 'y': trainY},
97 | test={'X': testF, 'y': testY}, config=config)
98 |
99 | devacc, testacc, yhat = clf.run()
100 | testf1 = round(100*f1_score(testY, yhat), 2)
101 | logging.debug('Dev acc : {0} Test acc {1}; Test F1 {2} for MRPC.\n'
102 | .format(devacc, testacc, testf1))
103 | return {'devacc': devacc, 'acc': testacc, 'f1': testf1,
104 | 'ndev': len(trainA), 'ntest': len(testA)}
105 |
--------------------------------------------------------------------------------
/SentEval/senteval/snli.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | '''
9 | SNLI - Entailment
10 | '''
11 | from __future__ import absolute_import, division, unicode_literals
12 |
13 | import codecs
14 | import os
15 | import io
16 | import copy
17 | import logging
18 | import numpy as np
19 |
20 | from senteval.tools.validation import SplitClassifier
21 |
22 |
23 | class SNLIEval(object):
24 | def __init__(self, taskpath, seed=1111):
25 | logging.debug('***** Transfer task : SNLI Entailment*****\n\n')
26 | self.seed = seed
27 | train1 = self.loadFile(os.path.join(taskpath, 's1.train'))
28 | train2 = self.loadFile(os.path.join(taskpath, 's2.train'))
29 |
30 | trainlabels = io.open(os.path.join(taskpath, 'labels.train'),
31 | encoding='utf-8').read().splitlines()
32 |
33 | valid1 = self.loadFile(os.path.join(taskpath, 's1.dev'))
34 | valid2 = self.loadFile(os.path.join(taskpath, 's2.dev'))
35 | validlabels = io.open(os.path.join(taskpath, 'labels.dev'),
36 | encoding='utf-8').read().splitlines()
37 |
38 | test1 = self.loadFile(os.path.join(taskpath, 's1.test'))
39 | test2 = self.loadFile(os.path.join(taskpath, 's2.test'))
40 | testlabels = io.open(os.path.join(taskpath, 'labels.test'),
41 | encoding='utf-8').read().splitlines()
42 |
43 | # sort data (by s2 first) to reduce padding
44 | sorted_train = sorted(zip(train2, train1, trainlabels),
45 | key=lambda z: (len(z[0]), len(z[1]), z[2]))
46 | train2, train1, trainlabels = map(list, zip(*sorted_train))
47 |
48 | sorted_valid = sorted(zip(valid2, valid1, validlabels),
49 | key=lambda z: (len(z[0]), len(z[1]), z[2]))
50 | valid2, valid1, validlabels = map(list, zip(*sorted_valid))
51 |
52 | sorted_test = sorted(zip(test2, test1, testlabels),
53 | key=lambda z: (len(z[0]), len(z[1]), z[2]))
54 | test2, test1, testlabels = map(list, zip(*sorted_test))
55 |
56 | self.samples = train1 + train2 + valid1 + valid2 + test1 + test2
57 | self.data = {'train': (train1, train2, trainlabels),
58 | 'valid': (valid1, valid2, validlabels),
59 | 'test': (test1, test2, testlabels)
60 | }
61 |
62 | def do_prepare(self, params, prepare):
63 | return prepare(params, self.samples)
64 |
65 | def loadFile(self, fpath):
66 | with codecs.open(fpath, 'rb', 'latin-1') as f:
67 | return [line.split() for line in
68 | f.read().splitlines()]
69 |
70 | def run(self, params, batcher):
71 | self.X, self.y = {}, {}
72 | dico_label = {'entailment': 0, 'neutral': 1, 'contradiction': 2}
73 | for key in self.data:
74 | if key not in self.X:
75 | self.X[key] = []
76 | if key not in self.y:
77 | self.y[key] = []
78 |
79 | input1, input2, mylabels = self.data[key]
80 | enc_input = []
81 | n_labels = len(mylabels)
82 | for ii in range(0, n_labels, params.batch_size):
83 | batch1 = input1[ii:ii + params.batch_size]
84 | batch2 = input2[ii:ii + params.batch_size]
85 |
86 | if len(batch1) == len(batch2) and len(batch1) > 0:
87 | enc1 = batcher(params, batch1)
88 | enc2 = batcher(params, batch2)
89 | enc_input.append(np.hstack((enc1, enc2, enc1 * enc2,
90 | np.abs(enc1 - enc2))))
91 | if (ii*params.batch_size) % (20000*params.batch_size) == 0:
92 | logging.info("PROGRESS (encoding): %.2f%%" %
93 | (100 * ii / n_labels))
94 | self.X[key] = np.vstack(enc_input)
95 | self.y[key] = [dico_label[y] for y in mylabels]
96 |
97 | config = {'nclasses': 3, 'seed': self.seed,
98 | 'usepytorch': params.usepytorch,
99 | 'cudaEfficient': True,
100 | 'nhid': params.nhid, 'noreg': True}
101 |
102 | config_classifier = copy.deepcopy(params.classifier)
103 | config_classifier['max_epoch'] = 15
104 | config_classifier['epoch_size'] = 1
105 | config['classifier'] = config_classifier
106 |
107 | clf = SplitClassifier(self.X, self.y, config)
108 | devacc, testacc = clf.run()
109 | logging.debug('Dev acc : {0} Test acc : {1} for SNLI\n'
110 | .format(devacc, testacc))
111 | return {'devacc': devacc, 'acc': testacc,
112 | 'ndev': len(self.data['valid'][0]),
113 | 'ntest': len(self.data['test'][0])}
114 |
--------------------------------------------------------------------------------
/SentEval/senteval/rank.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | '''
9 | Image-Caption Retrieval with COCO dataset
10 | '''
11 | from __future__ import absolute_import, division, unicode_literals
12 |
13 | import os
14 | import sys
15 | import logging
16 | import numpy as np
17 |
18 | try:
19 | import cPickle as pickle
20 | except ImportError:
21 | import pickle
22 |
23 | from senteval.tools.ranking import ImageSentenceRankingPytorch
24 |
25 |
26 | class ImageCaptionRetrievalEval(object):
27 | def __init__(self, task_path, seed=1111):
28 | logging.debug('***** Transfer task: Image Caption Retrieval *****\n\n')
29 |
30 | # Get captions and image features
31 | self.seed = seed
32 | train, dev, test = self.loadFile(task_path)
33 | self.coco_data = {'train': train, 'dev': dev, 'test': test}
34 |
35 | def do_prepare(self, params, prepare):
36 | samples = self.coco_data['train']['sent'] + \
37 | self.coco_data['dev']['sent'] + \
38 | self.coco_data['test']['sent']
39 | prepare(params, samples)
40 |
41 | def loadFile(self, fpath):
42 | coco = {}
43 |
44 | for split in ['train', 'valid', 'test']:
45 | list_sent = []
46 | list_img_feat = []
47 | if sys.version_info < (3, 0):
48 | with open(os.path.join(fpath, split + '.pkl')) as f:
49 | cocodata = pickle.load(f)
50 | else:
51 | with open(os.path.join(fpath, split + '.pkl'), 'rb') as f:
52 | cocodata = pickle.load(f, encoding='latin1')
53 |
54 | for imgkey in range(len(cocodata['features'])):
55 | assert len(cocodata['image_to_caption_ids'][imgkey]) >= 5, \
56 | cocodata['image_to_caption_ids'][imgkey]
57 | for captkey in cocodata['image_to_caption_ids'][imgkey][0:5]:
58 | sent = cocodata['captions'][captkey]['cleaned_caption']
59 | sent += ' .' # add punctuation to end of sentence in COCO
60 | list_sent.append(sent.encode('utf-8').split())
61 | list_img_feat.append(cocodata['features'][imgkey])
62 | assert len(list_sent) == len(list_img_feat) and \
63 | len(list_sent) % 5 == 0
64 | list_img_feat = np.array(list_img_feat).astype('float32')
65 | coco[split] = {'sent': list_sent, 'imgfeat': list_img_feat}
66 | return coco['train'], coco['valid'], coco['test']
67 |
68 | def run(self, params, batcher):
69 | coco_embed = {'train': {'sentfeat': [], 'imgfeat': []},
70 | 'dev': {'sentfeat': [], 'imgfeat': []},
71 | 'test': {'sentfeat': [], 'imgfeat': []}}
72 |
73 | for key in self.coco_data:
74 | logging.info('Computing embedding for {0}'.format(key))
75 | # Sort to reduce padding
76 | self.coco_data[key]['sent'] = np.array(self.coco_data[key]['sent'])
77 | self.coco_data[key]['sent'], idx_sort = np.sort(self.coco_data[key]['sent']), np.argsort(self.coco_data[key]['sent'])
78 | idx_unsort = np.argsort(idx_sort)
79 |
80 | coco_embed[key]['X'] = []
81 | nsent = len(self.coco_data[key]['sent'])
82 | for ii in range(0, nsent, params.batch_size):
83 | batch = self.coco_data[key]['sent'][ii:ii + params.batch_size]
84 | embeddings = batcher(params, batch)
85 | coco_embed[key]['sentfeat'].append(embeddings)
86 | coco_embed[key]['sentfeat'] = np.vstack(coco_embed[key]['sentfeat'])[idx_unsort]
87 | coco_embed[key]['imgfeat'] = np.array(self.coco_data[key]['imgfeat'])
88 | logging.info('Computed {0} embeddings'.format(key))
89 |
90 | config = {'seed': self.seed, 'projdim': 1000, 'margin': 0.2}
91 | clf = ImageSentenceRankingPytorch(train=coco_embed['train'],
92 | valid=coco_embed['dev'],
93 | test=coco_embed['test'],
94 | config=config)
95 |
96 | bestdevscore, r1_i2t, r5_i2t, r10_i2t, medr_i2t, \
97 | r1_t2i, r5_t2i, r10_t2i, medr_t2i = clf.run()
98 |
99 | logging.debug("\nTest scores | Image to text: \
100 | {0}, {1}, {2}, {3}".format(r1_i2t, r5_i2t, r10_i2t, medr_i2t))
101 | logging.debug("Test scores | Text to image: \
102 | {0}, {1}, {2}, {3}\n".format(r1_t2i, r5_t2i, r10_t2i, medr_t2i))
103 |
104 | return {'devacc': bestdevscore,
105 | 'acc': [(r1_i2t, r5_i2t, r10_i2t, medr_i2t),
106 | (r1_t2i, r5_t2i, r10_t2i, medr_t2i)],
107 | 'ndev': len(coco_embed['dev']['sentfeat']),
108 | 'ntest': len(coco_embed['test']['sentfeat'])}
109 |
--------------------------------------------------------------------------------
/SentEval/senteval/tools/relatedness.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | """
9 | Semantic Relatedness (supervised) with Pytorch
10 | """
11 | from __future__ import absolute_import, division, unicode_literals
12 |
13 | import copy
14 | import numpy as np
15 |
16 | import torch
17 | from torch import nn
18 | import torch.optim as optim
19 |
20 | from scipy.stats import pearsonr, spearmanr
21 |
22 |
23 | class RelatednessPytorch(object):
24 | # Can be used for SICK-Relatedness, and STS14
25 | def __init__(self, train, valid, test, devscores, config):
26 | # fix seed
27 | np.random.seed(config['seed'])
28 | torch.manual_seed(config['seed'])
29 | assert torch.cuda.is_available(), 'torch.cuda required for Relatedness'
30 | torch.cuda.manual_seed(config['seed'])
31 |
32 | self.train = train
33 | self.valid = valid
34 | self.test = test
35 | self.devscores = devscores
36 |
37 | self.inputdim = train['X'].shape[1]
38 | self.nclasses = config['nclasses']
39 | self.seed = config['seed']
40 | self.l2reg = 0.
41 | self.batch_size = 64
42 | self.maxepoch = 1000
43 | self.early_stop = True
44 |
45 | self.model = nn.Sequential(
46 | nn.Linear(self.inputdim, self.nclasses),
47 | nn.Softmax(dim=-1),
48 | )
49 | self.loss_fn = nn.MSELoss()
50 |
51 | if torch.cuda.is_available():
52 | self.model = self.model.cuda()
53 | self.loss_fn = self.loss_fn.cuda()
54 |
55 | self.loss_fn.size_average = False
56 | self.optimizer = optim.Adam(self.model.parameters(),
57 | weight_decay=self.l2reg)
58 |
59 | def prepare_data(self, trainX, trainy, devX, devy, testX, testy):
60 | # Transform probs to log-probs for KL-divergence
61 | trainX = torch.from_numpy(trainX).float().cuda()
62 | trainy = torch.from_numpy(trainy).float().cuda()
63 | devX = torch.from_numpy(devX).float().cuda()
64 | devy = torch.from_numpy(devy).float().cuda()
65 | testX = torch.from_numpy(testX).float().cuda()
66 | testY = torch.from_numpy(testy).float().cuda()
67 |
68 | return trainX, trainy, devX, devy, testX, testy
69 |
70 | def run(self):
71 | self.nepoch = 0
72 | bestpr = -1
73 | early_stop_count = 0
74 | r = np.arange(1, 6)
75 | stop_train = False
76 |
77 | # Preparing data
78 | trainX, trainy, devX, devy, testX, testy = self.prepare_data(
79 | self.train['X'], self.train['y'],
80 | self.valid['X'], self.valid['y'],
81 | self.test['X'], self.test['y'])
82 |
83 | # Training
84 | while not stop_train and self.nepoch <= self.maxepoch:
85 | self.trainepoch(trainX, trainy, nepoches=50)
86 | yhat = np.dot(self.predict_proba(devX), r)
87 | pr = spearmanr(yhat, self.devscores)[0]
88 | pr = 0 if pr != pr else pr # if NaN bc std=0
89 | # early stop on Pearson
90 | if pr > bestpr:
91 | bestpr = pr
92 | bestmodel = copy.deepcopy(self.model)
93 | elif self.early_stop:
94 | if early_stop_count >= 3:
95 | stop_train = True
96 | early_stop_count += 1
97 | self.model = bestmodel
98 |
99 | yhat = np.dot(self.predict_proba(testX), r)
100 |
101 | return bestpr, yhat
102 |
103 | def trainepoch(self, X, y, nepoches=1):
104 | self.model.train()
105 | for _ in range(self.nepoch, self.nepoch + nepoches):
106 | permutation = np.random.permutation(len(X))
107 | all_costs = []
108 | for i in range(0, len(X), self.batch_size):
109 | # forward
110 | idx = torch.from_numpy(permutation[i:i + self.batch_size]).long().cuda()
111 | Xbatch = X[idx]
112 | ybatch = y[idx]
113 | output = self.model(Xbatch)
114 | # loss
115 | loss = self.loss_fn(output, ybatch)
116 | all_costs.append(loss.item())
117 | # backward
118 | self.optimizer.zero_grad()
119 | loss.backward()
120 | # Update parameters
121 | self.optimizer.step()
122 | self.nepoch += nepoches
123 |
124 | def predict_proba(self, devX):
125 | self.model.eval()
126 | probas = []
127 | with torch.no_grad():
128 | for i in range(0, len(devX), self.batch_size):
129 | Xbatch = devX[i:i + self.batch_size]
130 | if len(probas) == 0:
131 | probas = self.model(Xbatch).data.cpu().numpy()
132 | else:
133 | probas = np.concatenate((probas, self.model(Xbatch).data.cpu().numpy()), axis=0)
134 | return probas
135 |
--------------------------------------------------------------------------------
/SentEval/senteval/engine.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | '''
9 |
10 | Generic sentence evaluation scripts wrapper
11 |
12 | '''
13 | from __future__ import absolute_import, division, unicode_literals
14 |
15 | from senteval import utils
16 | from senteval.binary import CREval, MREval, MPQAEval, SUBJEval
17 | from senteval.snli import SNLIEval
18 | from senteval.trec import TRECEval
19 | from senteval.sick import SICKEntailmentEval, SICKEval
20 | from senteval.mrpc import MRPCEval
21 | from senteval.sts import STS12Eval, STS13Eval, STS14Eval, STS15Eval, STS16Eval, STSBenchmarkEval, SICKRelatednessEval, STSBenchmarkFinetune
22 | from senteval.sst import SSTEval
23 | from senteval.rank import ImageCaptionRetrievalEval
24 | from senteval.probing import *
25 |
26 | class SE(object):
27 | def __init__(self, params, batcher, prepare=None):
28 | # parameters
29 | params = utils.dotdict(params)
30 | params.usepytorch = True if 'usepytorch' not in params else params.usepytorch
31 | params.seed = 1111 if 'seed' not in params else params.seed
32 |
33 | params.batch_size = 128 if 'batch_size' not in params else params.batch_size
34 | params.nhid = 0 if 'nhid' not in params else params.nhid
35 | params.kfold = 5 if 'kfold' not in params else params.kfold
36 |
37 | if 'classifier' not in params or not params['classifier']:
38 | params.classifier = {'nhid': 0}
39 |
40 | assert 'nhid' in params.classifier, 'Set number of hidden units in classifier config!!'
41 |
42 | self.params = params
43 |
44 | # batcher and prepare
45 | self.batcher = batcher
46 | self.prepare = prepare if prepare else lambda x, y: None
47 |
48 | self.list_tasks = ['CR', 'MR', 'MPQA', 'SUBJ', 'SST2', 'SST5', 'TREC', 'MRPC',
49 | 'SICKRelatedness', 'SICKEntailment', 'STSBenchmark',
50 | 'SNLI', 'ImageCaptionRetrieval', 'STS12', 'STS13',
51 | 'STS14', 'STS15', 'STS16',
52 | 'Length', 'WordContent', 'Depth', 'TopConstituents',
53 | 'BigramShift', 'Tense', 'SubjNumber', 'ObjNumber',
54 | 'OddManOut', 'CoordinationInversion', 'SICKRelatedness-finetune', 'STSBenchmark-finetune', 'STSBenchmark-fix']
55 |
56 | def eval(self, name):
57 | # evaluate on evaluation [name], either takes string or list of strings
58 | if (isinstance(name, list)):
59 | self.results = {x: self.eval(x) for x in name}
60 | return self.results
61 |
62 | tpath = self.params.task_path
63 | assert name in self.list_tasks, str(name) + ' not in ' + str(self.list_tasks)
64 |
65 | # Original SentEval tasks
66 | if name == 'CR':
67 | self.evaluation = CREval(tpath + '/downstream/CR', seed=self.params.seed)
68 | elif name == 'MR':
69 | self.evaluation = MREval(tpath + '/downstream/MR', seed=self.params.seed)
70 | elif name == 'MPQA':
71 | self.evaluation = MPQAEval(tpath + '/downstream/MPQA', seed=self.params.seed)
72 | elif name == 'SUBJ':
73 | self.evaluation = SUBJEval(tpath + '/downstream/SUBJ', seed=self.params.seed)
74 | elif name == 'SST2':
75 | self.evaluation = SSTEval(tpath + '/downstream/SST/binary', nclasses=2, seed=self.params.seed)
76 | elif name == 'SST5':
77 | self.evaluation = SSTEval(tpath + '/downstream/SST/fine', nclasses=5, seed=self.params.seed)
78 | elif name == 'TREC':
79 | self.evaluation = TRECEval(tpath + '/downstream/TREC', seed=self.params.seed)
80 | elif name == 'MRPC':
81 | self.evaluation = MRPCEval(tpath + '/downstream/MRPC', seed=self.params.seed)
82 | elif name == 'SICKRelatedness':
83 | self.evaluation = SICKRelatednessEval(tpath + '/downstream/SICK', seed=self.params.seed)
84 | elif name == 'STSBenchmark':
85 | self.evaluation = STSBenchmarkEval(tpath + '/downstream/STS/STSBenchmark', seed=self.params.seed)
86 | elif name == 'STSBenchmark-fix':
87 | self.evaluation = STSBenchmarkEval(tpath + '/downstream/STS/STSBenchmark-fix', seed=self.params.seed)
88 | elif name == 'STSBenchmark-finetune':
89 | self.evaluation = STSBenchmarkFinetune(tpath + '/downstream/STS/STSBenchmark', seed=self.params.seed)
90 | elif name == 'SICKRelatedness-finetune':
91 | self.evaluation = SICKEval(tpath + '/downstream/SICK', seed=self.params.seed)
92 | elif name == 'SICKEntailment':
93 | self.evaluation = SICKEntailmentEval(tpath + '/downstream/SICK', seed=self.params.seed)
94 | elif name == 'SNLI':
95 | self.evaluation = SNLIEval(tpath + '/downstream/SNLI', seed=self.params.seed)
96 | elif name in ['STS12', 'STS13', 'STS14', 'STS15', 'STS16']:
97 | fpath = name + '-en-test'
98 | self.evaluation = eval(name + 'Eval')(tpath + '/downstream/STS/' + fpath, seed=self.params.seed)
99 | elif name == 'ImageCaptionRetrieval':
100 | self.evaluation = ImageCaptionRetrievalEval(tpath + '/downstream/COCO', seed=self.params.seed)
101 |
102 | # Probing Tasks
103 | elif name == 'Length':
104 | self.evaluation = LengthEval(tpath + '/probing', seed=self.params.seed)
105 | elif name == 'WordContent':
106 | self.evaluation = WordContentEval(tpath + '/probing', seed=self.params.seed)
107 | elif name == 'Depth':
108 | self.evaluation = DepthEval(tpath + '/probing', seed=self.params.seed)
109 | elif name == 'TopConstituents':
110 | self.evaluation = TopConstituentsEval(tpath + '/probing', seed=self.params.seed)
111 | elif name == 'BigramShift':
112 | self.evaluation = BigramShiftEval(tpath + '/probing', seed=self.params.seed)
113 | elif name == 'Tense':
114 | self.evaluation = TenseEval(tpath + '/probing', seed=self.params.seed)
115 | elif name == 'SubjNumber':
116 | self.evaluation = SubjNumberEval(tpath + '/probing', seed=self.params.seed)
117 | elif name == 'ObjNumber':
118 | self.evaluation = ObjNumberEval(tpath + '/probing', seed=self.params.seed)
119 | elif name == 'OddManOut':
120 | self.evaluation = OddManOutEval(tpath + '/probing', seed=self.params.seed)
121 | elif name == 'CoordinationInversion':
122 | self.evaluation = CoordinationInversionEval(tpath + '/probing', seed=self.params.seed)
123 |
124 | self.params.current_task = name
125 | self.evaluation.do_prepare(self.params, self.prepare)
126 |
127 | self.results = self.evaluation.run(self.params, self.batcher)
128 |
129 | return self.results
130 |
--------------------------------------------------------------------------------
/demo/static/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | '
41 | self.eos = ''] = 1e9 + 4
41 | words[''] = 1e9 + 3
42 | words['
30 |
31 |
32 |
49 |
50 |
33 |
37 | Simple Contrastive Learning of Sentence Embeddings
34 | Tianyu Gao Xingcheng Yao Danqi Chen
35 | Princeton University Tsinghua University
36 |
38 |
41 |
39 | SimCSE is a novel framework for contrastive learning of sentence embeddings. This demo shows how our pre-trained sentence embeddings can be directly applied to sentence retrieval tasks. You can type any natural language sentences and click the search button to see which sentences in the example database are semantically similar to the provided sentence. Here are some details about this demo:
40 |
42 |
48 |
43 |
47 |
51 |
52 |
53 |
94 |
95 |
169 |
170 |
171 |
172 |
--------------------------------------------------------------------------------
/SentEval/senteval/probing.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | '''
9 | probing tasks
10 | '''
11 |
12 | from __future__ import absolute_import, division, unicode_literals
13 |
14 | import os
15 | import io
16 | import copy
17 | import logging
18 | import numpy as np
19 |
20 | from senteval.tools.validation import SplitClassifier
21 |
22 |
23 | class PROBINGEval(object):
24 | def __init__(self, task, task_path, seed=1111):
25 | self.seed = seed
26 | self.task = task
27 | logging.debug('***** (Probing) Transfer task : %s classification *****', self.task.upper())
28 | self.task_data = {'train': {'X': [], 'y': []},
29 | 'dev': {'X': [], 'y': []},
30 | 'test': {'X': [], 'y': []}}
31 | self.loadFile(task_path)
32 | logging.info('Loaded %s train - %s dev - %s test for %s' %
33 | (len(self.task_data['train']['y']), len(self.task_data['dev']['y']),
34 | len(self.task_data['test']['y']), self.task))
35 |
36 | def do_prepare(self, params, prepare):
37 | samples = self.task_data['train']['X'] + self.task_data['dev']['X'] + \
38 | self.task_data['test']['X']
39 | return prepare(params, samples)
40 |
41 | def loadFile(self, fpath):
42 | self.tok2split = {'tr': 'train', 'va': 'dev', 'te': 'test'}
43 | with io.open(fpath, 'r', encoding='utf-8') as f:
44 | for line in f:
45 | line = line.rstrip().split('\t')
46 | self.task_data[self.tok2split[line[0]]]['X'].append(line[-1].split())
47 | self.task_data[self.tok2split[line[0]]]['y'].append(line[1])
48 |
49 | labels = sorted(np.unique(self.task_data['train']['y']))
50 | self.tok2label = dict(zip(labels, range(len(labels))))
51 | self.nclasses = len(self.tok2label)
52 |
53 | for split in self.task_data:
54 | for i, y in enumerate(self.task_data[split]['y']):
55 | self.task_data[split]['y'][i] = self.tok2label[y]
56 |
57 | def run(self, params, batcher):
58 | task_embed = {'train': {}, 'dev': {}, 'test': {}}
59 | bsize = params.batch_size
60 | logging.info('Computing embeddings for train/dev/test')
61 | for key in self.task_data:
62 | # Sort to reduce padding
63 | sorted_data = sorted(zip(self.task_data[key]['X'],
64 | self.task_data[key]['y']),
65 | key=lambda z: (len(z[0]), z[1]))
66 | self.task_data[key]['X'], self.task_data[key]['y'] = map(list, zip(*sorted_data))
67 |
68 | task_embed[key]['X'] = []
69 | for ii in range(0, len(self.task_data[key]['y']), bsize):
70 | batch = self.task_data[key]['X'][ii:ii + bsize]
71 | embeddings = batcher(params, batch)
72 | task_embed[key]['X'].append(embeddings)
73 | task_embed[key]['X'] = np.vstack(task_embed[key]['X'])
74 | task_embed[key]['y'] = np.array(self.task_data[key]['y'])
75 | logging.info('Computed embeddings')
76 |
77 | config_classifier = {'nclasses': self.nclasses, 'seed': self.seed,
78 | 'usepytorch': params.usepytorch,
79 | 'classifier': params.classifier}
80 |
81 | if self.task == "WordContent" and params.classifier['nhid'] > 0:
82 | config_classifier = copy.deepcopy(config_classifier)
83 | config_classifier['classifier']['nhid'] = 0
84 | print(params.classifier['nhid'])
85 |
86 | clf = SplitClassifier(X={'train': task_embed['train']['X'],
87 | 'valid': task_embed['dev']['X'],
88 | 'test': task_embed['test']['X']},
89 | y={'train': task_embed['train']['y'],
90 | 'valid': task_embed['dev']['y'],
91 | 'test': task_embed['test']['y']},
92 | config=config_classifier)
93 |
94 | devacc, testacc = clf.run()
95 | logging.debug('\nDev acc : %.1f Test acc : %.1f for %s classification\n' % (devacc, testacc, self.task.upper()))
96 |
97 | return {'devacc': devacc, 'acc': testacc,
98 | 'ndev': len(task_embed['dev']['X']),
99 | 'ntest': len(task_embed['test']['X'])}
100 |
101 | """
102 | Surface Information
103 | """
104 | class LengthEval(PROBINGEval):
105 | def __init__(self, task_path, seed=1111):
106 | task_path = os.path.join(task_path, 'sentence_length.txt')
107 | # labels: bins
108 | PROBINGEval.__init__(self, 'Length', task_path, seed)
109 |
110 | class WordContentEval(PROBINGEval):
111 | def __init__(self, task_path, seed=1111):
112 | task_path = os.path.join(task_path, 'word_content.txt')
113 | # labels: 200 target words
114 | PROBINGEval.__init__(self, 'WordContent', task_path, seed)
115 |
116 | """
117 | Latent Structural Information
118 | """
119 | class DepthEval(PROBINGEval):
120 | def __init__(self, task_path, seed=1111):
121 | task_path = os.path.join(task_path, 'tree_depth.txt')
122 | # labels: bins
123 | PROBINGEval.__init__(self, 'Depth', task_path, seed)
124 |
125 | class TopConstituentsEval(PROBINGEval):
126 | def __init__(self, task_path, seed=1111):
127 | task_path = os.path.join(task_path, 'top_constituents.txt')
128 | # labels: 'PP_NP_VP_.' .. (20 classes)
129 | PROBINGEval.__init__(self, 'TopConstituents', task_path, seed)
130 |
131 | class BigramShiftEval(PROBINGEval):
132 | def __init__(self, task_path, seed=1111):
133 | task_path = os.path.join(task_path, 'bigram_shift.txt')
134 | # labels: 0 or 1
135 | PROBINGEval.__init__(self, 'BigramShift', task_path, seed)
136 |
137 | # TODO: Voice?
138 |
139 | """
140 | Latent Semantic Information
141 | """
142 |
143 | class TenseEval(PROBINGEval):
144 | def __init__(self, task_path, seed=1111):
145 | task_path = os.path.join(task_path, 'past_present.txt')
146 | # labels: 'PRES', 'PAST'
147 | PROBINGEval.__init__(self, 'Tense', task_path, seed)
148 |
149 | class SubjNumberEval(PROBINGEval):
150 | def __init__(self, task_path, seed=1111):
151 | task_path = os.path.join(task_path, 'subj_number.txt')
152 | # labels: 'NN', 'NNS'
153 | PROBINGEval.__init__(self, 'SubjNumber', task_path, seed)
154 |
155 | class ObjNumberEval(PROBINGEval):
156 | def __init__(self, task_path, seed=1111):
157 | task_path = os.path.join(task_path, 'obj_number.txt')
158 | # labels: 'NN', 'NNS'
159 | PROBINGEval.__init__(self, 'ObjNumber', task_path, seed)
160 |
161 | class OddManOutEval(PROBINGEval):
162 | def __init__(self, task_path, seed=1111):
163 | task_path = os.path.join(task_path, 'odd_man_out.txt')
164 | # labels: 'O', 'C'
165 | PROBINGEval.__init__(self, 'OddManOut', task_path, seed)
166 |
167 | class CoordinationInversionEval(PROBINGEval):
168 | def __init__(self, task_path, seed=1111):
169 | task_path = os.path.join(task_path, 'coordination_inversion.txt')
170 | # labels: 'O', 'I'
171 | PROBINGEval.__init__(self, 'CoordinationInversion', task_path, seed)
172 |
--------------------------------------------------------------------------------
/SentEval/senteval/tools/classifier.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | """
9 | Pytorch Classifier class in the style of scikit-learn
10 | Classifiers include Logistic Regression and MLP
11 | """
12 |
13 | from __future__ import absolute_import, division, unicode_literals
14 |
15 | import numpy as np
16 | import copy
17 | from senteval import utils
18 |
19 | import torch
20 | from torch import nn
21 | import torch.nn.functional as F
22 |
23 |
24 | class PyTorchClassifier(object):
25 | def __init__(self, inputdim, nclasses, l2reg=0., batch_size=64, seed=1111,
26 | cudaEfficient=False):
27 | # fix seed
28 | np.random.seed(seed)
29 | torch.manual_seed(seed)
30 | torch.cuda.manual_seed(seed)
31 |
32 | self.inputdim = inputdim
33 | self.nclasses = nclasses
34 | self.l2reg = l2reg
35 | self.batch_size = batch_size
36 | self.cudaEfficient = cudaEfficient
37 |
38 | def prepare_split(self, X, y, validation_data=None, validation_split=None):
39 | # Preparing validation data
40 | assert validation_split or validation_data
41 | if validation_data is not None:
42 | trainX, trainy = X, y
43 | devX, devy = validation_data
44 | else:
45 | permutation = np.random.permutation(len(X))
46 | trainidx = permutation[int(validation_split * len(X)):]
47 | devidx = permutation[0:int(validation_split * len(X))]
48 | trainX, trainy = X[trainidx], y[trainidx]
49 | devX, devy = X[devidx], y[devidx]
50 |
51 | device = torch.device('cpu') if self.cudaEfficient else torch.device('cuda')
52 |
53 | trainX = torch.from_numpy(trainX).to(device, dtype=torch.float32)
54 | trainy = torch.from_numpy(trainy).to(device, dtype=torch.int64)
55 | devX = torch.from_numpy(devX).to(device, dtype=torch.float32)
56 | devy = torch.from_numpy(devy).to(device, dtype=torch.int64)
57 |
58 | return trainX, trainy, devX, devy
59 |
60 | def fit(self, X, y, validation_data=None, validation_split=None,
61 | early_stop=True):
62 | self.nepoch = 0
63 | bestaccuracy = -1
64 | stop_train = False
65 | early_stop_count = 0
66 |
67 | # Preparing validation data
68 | trainX, trainy, devX, devy = self.prepare_split(X, y, validation_data,
69 | validation_split)
70 |
71 | # Training
72 | while not stop_train and self.nepoch <= self.max_epoch:
73 | self.trainepoch(trainX, trainy, epoch_size=self.epoch_size)
74 | accuracy = self.score(devX, devy)
75 | if accuracy > bestaccuracy:
76 | bestaccuracy = accuracy
77 | bestmodel = copy.deepcopy(self.model)
78 | elif early_stop:
79 | if early_stop_count >= self.tenacity:
80 | stop_train = True
81 | early_stop_count += 1
82 | self.model = bestmodel
83 | return bestaccuracy
84 |
85 | def trainepoch(self, X, y, epoch_size=1):
86 | self.model.train()
87 | for _ in range(self.nepoch, self.nepoch + epoch_size):
88 | permutation = np.random.permutation(len(X))
89 | all_costs = []
90 | for i in range(0, len(X), self.batch_size):
91 | # forward
92 | idx = torch.from_numpy(permutation[i:i + self.batch_size]).long().to(X.device)
93 |
94 | Xbatch = X[idx]
95 | ybatch = y[idx]
96 |
97 | if self.cudaEfficient:
98 | Xbatch = Xbatch.cuda()
99 | ybatch = ybatch.cuda()
100 | output = self.model(Xbatch)
101 | # loss
102 | loss = self.loss_fn(output, ybatch)
103 | all_costs.append(loss.data.item())
104 | # backward
105 | self.optimizer.zero_grad()
106 | loss.backward()
107 | # Update parameters
108 | self.optimizer.step()
109 | self.nepoch += epoch_size
110 |
111 | def score(self, devX, devy):
112 | self.model.eval()
113 | correct = 0
114 | if not isinstance(devX, torch.cuda.FloatTensor) or self.cudaEfficient:
115 | devX = torch.FloatTensor(devX).cuda()
116 | devy = torch.LongTensor(devy).cuda()
117 | with torch.no_grad():
118 | for i in range(0, len(devX), self.batch_size):
119 | Xbatch = devX[i:i + self.batch_size]
120 | ybatch = devy[i:i + self.batch_size]
121 | if self.cudaEfficient:
122 | Xbatch = Xbatch.cuda()
123 | ybatch = ybatch.cuda()
124 | output = self.model(Xbatch)
125 | pred = output.data.max(1)[1]
126 | correct += pred.long().eq(ybatch.data.long()).sum().item()
127 | accuracy = 1.0 * correct / len(devX)
128 | return accuracy
129 |
130 | def predict(self, devX):
131 | self.model.eval()
132 | if not isinstance(devX, torch.cuda.FloatTensor):
133 | devX = torch.FloatTensor(devX).cuda()
134 | yhat = np.array([])
135 | with torch.no_grad():
136 | for i in range(0, len(devX), self.batch_size):
137 | Xbatch = devX[i:i + self.batch_size]
138 | output = self.model(Xbatch)
139 | yhat = np.append(yhat,
140 | output.data.max(1)[1].cpu().numpy())
141 | yhat = np.vstack(yhat)
142 | return yhat
143 |
144 | def predict_proba(self, devX):
145 | self.model.eval()
146 | probas = []
147 | with torch.no_grad():
148 | for i in range(0, len(devX), self.batch_size):
149 | Xbatch = devX[i:i + self.batch_size]
150 | vals = F.softmax(self.model(Xbatch).data.cpu().numpy())
151 | if not probas:
152 | probas = vals
153 | else:
154 | probas = np.concatenate(probas, vals, axis=0)
155 | return probas
156 |
157 |
158 | """
159 | MLP with Pytorch (nhid=0 --> Logistic Regression)
160 | """
161 |
162 | class MLP(PyTorchClassifier):
163 | def __init__(self, params, inputdim, nclasses, l2reg=0., batch_size=64,
164 | seed=1111, cudaEfficient=False):
165 | super(self.__class__, self).__init__(inputdim, nclasses, l2reg,
166 | batch_size, seed, cudaEfficient)
167 | """
168 | PARAMETERS:
169 | -nhid: number of hidden units (0: Logistic Regression)
170 | -optim: optimizer ("sgd,lr=0.1", "adam", "rmsprop" ..)
171 | -tenacity: how many times dev acc does not increase before stopping
172 | -epoch_size: each epoch corresponds to epoch_size pass on the train set
173 | -max_epoch: max number of epoches
174 | -dropout: dropout for MLP
175 | """
176 |
177 | self.nhid = 0 if "nhid" not in params else params["nhid"]
178 | self.optim = "adam" if "optim" not in params else params["optim"]
179 | self.tenacity = 5 if "tenacity" not in params else params["tenacity"]
180 | self.epoch_size = 4 if "epoch_size" not in params else params["epoch_size"]
181 | self.max_epoch = 200 if "max_epoch" not in params else params["max_epoch"]
182 | self.dropout = 0. if "dropout" not in params else params["dropout"]
183 | self.batch_size = 64 if "batch_size" not in params else params["batch_size"]
184 |
185 | if params["nhid"] == 0:
186 | self.model = nn.Sequential(
187 | nn.Linear(self.inputdim, self.nclasses),
188 | ).cuda()
189 | else:
190 | self.model = nn.Sequential(
191 | nn.Linear(self.inputdim, params["nhid"]),
192 | nn.Dropout(p=self.dropout),
193 | nn.Sigmoid(),
194 | nn.Linear(params["nhid"], self.nclasses),
195 | ).cuda()
196 |
197 | self.loss_fn = nn.CrossEntropyLoss().cuda()
198 | self.loss_fn.size_average = False
199 |
200 | optim_fn, optim_params = utils.get_optimizer(self.optim)
201 | self.optimizer = optim_fn(self.model.parameters(), **optim_params)
202 | self.optimizer.param_groups[0]['weight_decay'] = self.l2reg
203 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Alleviating Over-smoothing for Unsupervised Sentence Representation
2 | This is the project of our ACL 2023 paper: [Alleviating Over-smoothing for Unsupervised Sentence Representation](https://arxiv.org/pdf/2305.06154).
3 |
4 | Our work mainly based on [SimCSE project](https://github.com/princeton-nlp/SimCSE), thanks to SimCSE!
5 |
6 | ## Quick Links
7 |
8 | - [Overview](#overview)
9 | - [Getting Started](##setup)
10 | - [Train SSCL](#training)
11 | - [Requirements](#requirements)
12 | - [Evaluation](#evaluation)
13 | - [Training](#training)
14 | - [Citation](#citation)
15 |
16 | ## Overview
17 |
18 | we present a new training paradigm based on contrastive learning: Simple contrastive method named Self-Contrastive Learning (SSCL), which can significantly improve the performance of learned sentence representations while alleviating the over-smoothing issue. Simply Said, we utilize hidden representations from intermediate PLMs layers as negative samples which the final sentence representations should be away from. Generally, our SSCL has several advantages: (1) It is fairly straightforward and does not require complex data augmentation techniques; (2) It can be seen as a contrastive framework that focuses on mining negatives effectively, and can be easily extended into different sentence encoders that aim for building positive pairs; (3) It can further be viewed as a plug-and-play framework for enhancing sentence representations.
19 | 
20 |
21 | ## Setup
22 | First, install PyTorch by following the instructions from the [official website](https://pytorch.org/get-started/previous-versions/). To faithfully reproduce our results, please use the correct 1.9.1 version corresponding to your platforms/CUDA versions. Install PyTorch by the following command,
23 |
24 | ```
25 | pip install torch==1.9.1+cu111 torchvision==0.10.1+cu111 torchaudio==0.9.1 -f https://download.pytorch.org/whl/torch_stable.html
26 | ```
27 | Then run the following script to install the remaining dependencies,
28 |
29 | ```bash
30 | pip install -r requirements.txt
31 | ```
32 |
33 | ### Evaluation
34 | Our evaluation code for sentence embeddings is based on a modified version of [SentEval](https://github.com/facebookresearch/SentEval). It evaluates sentence embeddings on semantic textual similarity (STS) tasks and downstream transfer tasks. For STS tasks, our evaluation takes the "all" setting, and report Spearman's correlation.
35 |
36 | Before evaluation, please download the evaluation datasets by running
37 | ```bash
38 | cd SentEval/data/downstream/
39 | bash download_dataset.sh
40 | ```
41 |
42 | Then come back to the root directory, you can evaluate any `transformers`-based pre-trained models using our evaluation code. For example,
43 | ```bash
44 | python evaluation.py \
45 | --model_name_or_path sscl-bert-base-uncased \
46 | --pooler cls \
47 | --task_set sts \
48 | --mode test
49 | ```
50 |
51 | Arguments for the evaluation script are as follows,
52 |
53 | * `--model_name_or_path`: The name or path of a `transformers`-based pre-trained checkpoint. You can directly use the models in the above table.
54 | * `--pooler`: Pooling method. Now we support
55 | * `cls` (default): Use the representation of `[CLS]` token. A linear+activation layer is applied after the representation (it's in the standard BERT implementation).
56 | * `cls_before_pooler`: Use the representation of `[CLS]` token without the extra linear+activation.
57 | * `avg`: Average embeddings of the last layer. If you use checkpoints of SBERT/SRoBERTa ([paper](https://arxiv.org/abs/1908.10084)), you should use this option.
58 | * `avg_top2`: Average embeddings of the last two layers.
59 | * `avg_first_last`: Average embeddings of the first and last layers. If you use vanilla BERT or RoBERTa, this works the best.
60 | * `--mode`: Evaluation mode
61 | * `test` (default): The default test mode. To faithfully reproduce our results, you should use this option.
62 | * `dev`: Report the development set results. Note that in STS tasks, only `STS-B` and `SICK-R` have development sets, so we only report their numbers. It also takes a fast mode for transfer tasks, so the running time is much shorter than the `test` mode (though numbers are slightly lower).
63 | * `fasttest`: It is the same as `test`, but with a fast mode so the running time is much shorter, but the reported numbers may be lower (only for transfer tasks).
64 | * `--task_set`: What set of tasks to evaluate on (if set, it will override `--tasks`)
65 | * `sts` (default): Evaluate on STS tasks, including `STS 12~16`, `STS-B` and `SICK-R`. This is the most commonly-used set of tasks to evaluate the quality of sentence embeddings.
66 | * `transfer`: Evaluate on transfer tasks.
67 | * `full`: Evaluate on both STS and transfer tasks.
68 | * `na`: Manually set tasks by `--tasks`.
69 | * `--tasks`: Specify which dataset(s) to evaluate on. Will be overridden if `--task_set` is not `na`. See the code for a full list of tasks.
70 |
71 | ### Training
72 |
73 | **Data**
74 |
75 | You can run `data/download_wiki.sh` and `data/download_nli.sh` to download the two datasets.
76 |
77 | **Training scripts**
78 | In `run_unsup_example.sh`, we provide a single-GPU (or CPU) example for the unsupervised version. We explain the arguments in following:
79 | * `--train_file`: Training file path. We support "txt" files (one line for one sentence) and "csv" files (2-column: pair data with no hard negative; 3-column: pair data with one corresponding hard negative instance). You can use our provided Wikipedia or NLI data, or you can use your own data with the same format.
80 | * `--model_name_or_path`: Pre-trained checkpoints to start with. For now we support BERT-based models (`bert-base-uncased`, `bert-large-uncased`, etc.).
81 | * `--temp`: Temperature for the contrastive loss.
82 | * `--pooler_type`: Pooling method. It's the same as the `--pooler_type` in the [evaluation part](#evaluation).
83 | * `--mlp_only_train`: You should use this argument when training SSCL models.
84 | * `--hard_negative_weight`: If using hard negatives (i.e., there are 3 columns in the training file), this is the logarithm of the weight. For example, if the weight is 1, then this argument should be set as 0 (default value).
85 | * `--do_mlm`: Whether to use the MLM auxiliary objective. If True:
86 | * `--mlm_weight`: Weight for the MLM objective.
87 | * `--mlm_probability`: Masking rate for the MLM objective.
88 | * `--do_neg`: Whether to use negatives in SSCL.
89 | * `--hard_negative_layers`: How many previous layers to construct negative layers.
90 |
91 | All the other arguments are standard Huggingface's `transformers` training arguments. Some of the often-used arguments are: `--output_dir`, `--learning_rate`, `--per_device_train_batch_size`. In our example scripts, we also set to evaluate the model on the STS-B development set (need to download the dataset following the [evaluation](#evaluation) section) and save the best checkpoint.
92 |
93 | For results in the paper, we use Nvidia A100 GPUs with CUDA 11. Using different types of devices or different versions of CUDA/other softwares may lead to slightly different performance.
94 |
95 |
96 |
97 | **Convert models**
98 |
99 | Our saved checkpoints are slightly different from Huggingface's pre-trained checkpoints. Run `python sscl_to_huggingface.py --path {PATH_TO_CHECKPOINT_FOLDER}` to convert it. After that, you can evaluate it by our [evaluation](#evaluation) code or directly use it [out of the box](#use-our-models-out-of-the-box).
100 |
101 | ## Citation
102 |
103 | Please cite our paper if you use SSCL in your work:
104 |
105 | ```bibtex
106 | @inproceedings{chen-etal-2023-alleviating,
107 | title = "Alleviating Over-smoothing for Unsupervised Sentence Representation",
108 | author = "Chen, Nuo and
109 | Shou, Linjun and
110 | Pei, Jian and
111 | Gong, Ming and
112 | Cao, Bowen and
113 | Chang, Jianhui and
114 | Li, Jia and
115 | Jiang, Daxin",
116 | booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
117 | month = jul,
118 | year = "2023",
119 | address = "Toronto, Canada",
120 | publisher = "Association for Computational Linguistics",
121 | url = "https://aclanthology.org/2023.acl-long.197",
122 | pages = "3552--3566",
123 | abstract = "Currently, learning better unsupervised sentence representations is the pursuit of many natural language processing communities. Lots of approaches based on pre-trained language models (PLMs) and contrastive learning have achieved promising results on this task. Experimentally, we observe that the over-smoothing problem reduces the capacity of these powerful PLMs, leading to sub-optimal sentence representations. In this paper, we present a Simple method named Self-Contrastive Learning (SSCL) to alleviate this issue, which samples negatives from PLMs intermediate layers, improving the quality of the sentence representation. Our proposed method is quite simple and can be easily extended to various state-of-the-art models for performance boosting, which can be seen as a plug-and-play contrastive framework for learning unsupervised sentence representation. Extensive results prove that SSCL brings the superior performance improvements of different strong baselines (e.g., BERT and SimCSE) on Semantic Textual Similarity and Transfer datasets",
124 | }
125 | ```
126 |
--------------------------------------------------------------------------------
/evaluation.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import io, os
3 | import numpy as np
4 | import logging
5 | import argparse
6 | from prettytable import PrettyTable
7 | import torch
8 | import transformers
9 | from transformers import AutoModel, AutoTokenizer
10 | import xlrd
11 | from openpyxl import Workbook
12 |
13 | # Set up logger
14 | logging.basicConfig(format='%(asctime)s : %(message)s', level=logging.DEBUG)
15 |
16 | # Set PATHs
17 | PATH_TO_SENTEVAL = './SentEval'
18 | PATH_TO_DATA = './SentEval/data'
19 | workbook = Workbook()
20 |
21 | # Import SentEval
22 | sys.path.insert(0, PATH_TO_SENTEVAL)
23 | import senteval
24 |
25 | def print_table(task_names, scores):
26 | tb = PrettyTable()
27 | tb.field_names = task_names
28 | tb.add_row(scores)
29 | print(tb)
30 |
31 | def main():
32 | parser = argparse.ArgumentParser()
33 | parser.add_argument("--model_name_or_path", type=str,
34 | help="Transformers' model name or path")
35 | parser.add_argument("--pooler", type=str,
36 | choices=['cls', 'cls_before_pooler', 'avg', 'avg_top2', 'avg_first_last'],
37 | default='cls',
38 | help="Which pooler to use")
39 | parser.add_argument("--mode", type=str,
40 | choices=['dev', 'test', 'fasttest'],
41 | default='test',
42 | help="What evaluation mode to use (dev: fast mode, dev results; test: full mode, test results); fasttest: fast mode, test results")
43 | parser.add_argument("--task_set", type=str,
44 | choices=['sts', 'transfer', 'full', 'na'],
45 | default='sts',
46 | help="What set of tasks to evaluate on. If not 'na', this will override '--tasks'")
47 | parser.add_argument("--tasks", type=str, nargs='+',
48 | default=['STS12', 'STS13', 'STS14', 'STS15', 'STS16',
49 | 'MR', 'CR', 'MPQA', 'SUBJ', 'SST2', 'TREC', 'MRPC',
50 | 'SICKRelatedness', 'STSBenchmark'],
51 | help="Tasks to evaluate on. If '--task_set' is specified, this will be overridden")
52 |
53 | args = parser.parse_args()
54 |
55 | # Load transformers' model checkpoint
56 | model = AutoModel.from_pretrained(args.model_name_or_path)
57 | tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
58 | device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
59 | model = model.to(device)
60 |
61 | # Set up the tasks
62 | if args.task_set == 'sts':
63 | args.tasks = ['STS12', 'STS13', 'STS14', 'STS15', 'STS16', 'STSBenchmark', 'SICKRelatedness']
64 | elif args.task_set == 'transfer':
65 | args.tasks = ['MR', 'CR', 'MPQA', 'SUBJ', 'SST2', 'TREC', 'MRPC']
66 | elif args.task_set == 'full':
67 | args.tasks = ['STS12', 'STS13', 'STS14', 'STS15', 'STS16', 'STSBenchmark', 'SICKRelatedness']
68 | args.tasks += ['MR', 'CR', 'MPQA', 'SUBJ', 'SST2', 'TREC', 'MRPC']
69 |
70 | # Set params for SentEval
71 | if args.mode == 'dev' or args.mode == 'fasttest':
72 | # Fast mode
73 | params = {'task_path': PATH_TO_DATA, 'usepytorch': True, 'kfold': 5}
74 | params['classifier'] = {'nhid': 0, 'optim': 'rmsprop', 'batch_size': 128,
75 | 'tenacity': 3, 'epoch_size': 2}
76 | elif args.mode == 'test':
77 | # Full mode
78 | params = {'task_path': PATH_TO_DATA, 'usepytorch': True, 'kfold': 10}
79 | params['classifier'] = {'nhid': 0, 'optim': 'adam', 'batch_size': 64,
80 | 'tenacity': 5, 'epoch_size': 4}
81 | else:
82 | raise NotImplementedError
83 |
84 | # SentEval prepare and batcher
85 | def prepare(params, samples):
86 | return
87 |
88 | def batcher(params, batch, max_length=None):
89 | # Handle rare token encoding issues in the dataset
90 | if len(batch) >= 1 and len(batch[0]) >= 1 and isinstance(batch[0][0], bytes):
91 | batch = [[word.decode('utf-8') for word in s] for s in batch]
92 |
93 | sentences = [' '.join(s) for s in batch]
94 |
95 | # Tokenization
96 | if max_length is not None:
97 | batch = tokenizer.batch_encode_plus(
98 | sentences,
99 | return_tensors='pt',
100 | padding=True,
101 | max_length=max_length,
102 | truncation=True
103 | )
104 | else:
105 | batch = tokenizer.batch_encode_plus(
106 | sentences,
107 | return_tensors='pt',
108 | padding=True,
109 | )
110 |
111 | # Move to the correct device
112 | for k in batch:
113 | batch[k] = batch[k].to(device)
114 |
115 | # Get raw embeddings
116 | with torch.no_grad():
117 | outputs = model(**batch, output_hidden_states=True, return_dict=True)
118 | last_hidden = outputs.last_hidden_state
119 | pooler_output = outputs.pooler_output
120 | hidden_states = outputs.hidden_states
121 |
122 | # Apply different poolers
123 | if args.pooler == 'cls':
124 | # There is a linear+activation layer after CLS representation
125 | return pooler_output.cpu()
126 | elif args.pooler == 'cls_before_pooler':
127 | return last_hidden[:, 0].cpu()
128 | elif args.pooler == "avg":
129 | return ((last_hidden * batch['attention_mask'].unsqueeze(-1)).sum(1) / batch['attention_mask'].sum(-1).unsqueeze(-1)).cpu()
130 | elif args.pooler == "avg_first_last":
131 | first_hidden = hidden_states[0]
132 | last_hidden = hidden_states[-1]
133 | pooled_result = ((first_hidden + last_hidden) / 2.0 * batch['attention_mask'].unsqueeze(-1)).sum(1) / batch['attention_mask'].sum(-1).unsqueeze(-1)
134 | return pooled_result.cpu()
135 | elif args.pooler == "avg_top2":
136 | second_last_hidden = hidden_states[-2]
137 | last_hidden = hidden_states[-1]
138 | pooled_result = ((last_hidden + second_last_hidden) / 2.0 * batch['attention_mask'].unsqueeze(-1)).sum(1) / batch['attention_mask'].sum(-1).unsqueeze(-1)
139 | return pooled_result.cpu()
140 | else:
141 | raise NotImplementedError
142 |
143 | results = {}
144 |
145 | for task in args.tasks:
146 | se = senteval.engine.SE(params, batcher, prepare)
147 | result = se.eval(task)
148 | results[task] = result
149 |
150 | # Print evaluation results
151 | if args.mode == 'dev':
152 | print("------ %s ------" % (args.mode))
153 |
154 | task_names = []
155 | scores = []
156 | for task in ['STSBenchmark', 'SICKRelatedness']:
157 | task_names.append(task)
158 | if task in results:
159 | scores.append("%.2f" % (results[task]['dev']['spearman'][0] * 100))
160 | else:
161 | scores.append("0.00")
162 | print_table(task_names, scores)
163 |
164 | task_names = []
165 | scores = []
166 | for task in ['MR', 'CR', 'SUBJ', 'MPQA', 'SST2', 'TREC', 'MRPC']:
167 | task_names.append(task)
168 | if task in results:
169 | scores.append("%.2f" % (results[task]['devacc']))
170 | else:
171 | scores.append("0.00")
172 | task_names.append("Avg.")
173 | scores.append("%.2f" % (sum([float(score) for score in scores]) / len(scores)))
174 | print_table(task_names, scores)
175 |
176 | elif args.mode == 'test' or args.mode == 'fasttest':
177 | print("------ %s ------" % (args.mode))
178 | save_file = os.path.join(args.model_name_or_path, 'results.xlsx')
179 | task_names = []
180 | scores = []
181 | sheet_name = ''
182 | for task in ['STS12', 'STS13', 'STS14', 'STS15', 'STS16', 'STSBenchmark', 'SICKRelatedness']:
183 | task_names.append(task)
184 | if task in results:
185 | if task in ['STS12', 'STS13', 'STS14', 'STS15', 'STS16']:
186 | scores.append("%.2f" % (results[task]['all']['spearman']['all'] * 100))
187 | else:
188 | scores.append("%.2f" % (results[task]['test']['spearman'].correlation * 100))
189 | else:
190 | scores.append("0.00")
191 | sheet_name = 'STS'
192 | task_names.append("Avg.")
193 | scores.append("%.2f" % (sum([float(score) for score in scores]) / len(scores)))
194 | # print(task_names)
195 | # print(scores)
196 | worksheet = workbook.create_sheet(sheet_name)
197 | worksheet.append(task_names)
198 | worksheet.append(scores)
199 | print_table(task_names, scores)
200 |
201 | task_names = []
202 | scores = []
203 | for task in ['MR', 'CR', 'SUBJ', 'MPQA', 'SST2', 'TREC', 'MRPC']:
204 | task_names.append(task)
205 | if task in results:
206 | scores.append("%.2f" % (results[task]['acc']))
207 | else:
208 | scores.append("0.00")
209 | sheet_name = 'Transfer'
210 | task_names.append("Avg.")
211 | scores.append("%.2f" % (sum([float(score) for score in scores]) / len(scores)))
212 | print_table(task_names, scores)
213 | worksheet = workbook.create_sheet(sheet_name)
214 | worksheet.append(task_names)
215 | worksheet.append(scores)
216 | workbook.save(filename=save_file)
217 |
218 |
219 | if __name__ == "__main__":
220 | main()
221 |
--------------------------------------------------------------------------------
/SentEval/senteval/sick.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | '''
9 | SICK Relatedness and Entailment
10 | '''
11 | from __future__ import absolute_import, division, unicode_literals
12 |
13 | import os
14 | import io
15 | import logging
16 | import numpy as np
17 |
18 | from sklearn.metrics import mean_squared_error
19 | from scipy.stats import pearsonr, spearmanr
20 |
21 | from senteval.tools.relatedness import RelatednessPytorch
22 | from senteval.tools.validation import SplitClassifier
23 |
24 | class SICKEval(object):
25 | def __init__(self, task_path, seed=1111):
26 | logging.debug('***** Transfer task : SICK-Relatedness*****\n\n')
27 | self.seed = seed
28 | train = self.loadFile(os.path.join(task_path, 'SICK_train.txt'))
29 | dev = self.loadFile(os.path.join(task_path, 'SICK_trial.txt'))
30 | test = self.loadFile(os.path.join(task_path, 'SICK_test_annotated.txt'))
31 | self.sick_data = {'train': train, 'dev': dev, 'test': test}
32 |
33 | def do_prepare(self, params, prepare):
34 | samples = self.sick_data['train']['X_A'] + \
35 | self.sick_data['train']['X_B'] + \
36 | self.sick_data['dev']['X_A'] + \
37 | self.sick_data['dev']['X_B'] + \
38 | self.sick_data['test']['X_A'] + self.sick_data['test']['X_B']
39 | return prepare(params, samples)
40 |
41 | def loadFile(self, fpath):
42 | skipFirstLine = True
43 | sick_data = {'X_A': [], 'X_B': [], 'y': []}
44 | with io.open(fpath, 'r', encoding='utf-8') as f:
45 | for line in f:
46 | if skipFirstLine:
47 | skipFirstLine = False
48 | else:
49 | text = line.strip().split('\t')
50 | sick_data['X_A'].append(text[1].split())
51 | sick_data['X_B'].append(text[2].split())
52 | sick_data['y'].append(text[3])
53 |
54 | sick_data['y'] = [float(s) for s in sick_data['y']]
55 | return sick_data
56 |
57 | def run(self, params, batcher):
58 | sick_embed = {'train': {}, 'dev': {}, 'test': {}}
59 | bsize = params.batch_size
60 |
61 | for key in self.sick_data:
62 | logging.info('Computing embedding for {0}'.format(key))
63 | # Sort to reduce padding
64 | sorted_corpus = sorted(zip(self.sick_data[key]['X_A'],
65 | self.sick_data[key]['X_B'],
66 | self.sick_data[key]['y']),
67 | key=lambda z: (len(z[0]), len(z[1]), z[2]))
68 |
69 | self.sick_data[key]['X_A'] = [x for (x, y, z) in sorted_corpus]
70 | self.sick_data[key]['X_B'] = [y for (x, y, z) in sorted_corpus]
71 | self.sick_data[key]['y'] = [z for (x, y, z) in sorted_corpus]
72 |
73 | for txt_type in ['X_A', 'X_B']:
74 | sick_embed[key][txt_type] = []
75 | for ii in range(0, len(self.sick_data[key]['y']), bsize):
76 | batch = self.sick_data[key][txt_type][ii:ii + bsize]
77 | embeddings = batcher(params, batch)
78 | sick_embed[key][txt_type].append(embeddings)
79 | sick_embed[key][txt_type] = np.vstack(sick_embed[key][txt_type])
80 | sick_embed[key]['y'] = np.array(self.sick_data[key]['y'])
81 | logging.info('Computed {0} embeddings'.format(key))
82 |
83 | # Train
84 | trainA = sick_embed['train']['X_A']
85 | trainB = sick_embed['train']['X_B']
86 | trainF = np.c_[np.abs(trainA - trainB), trainA * trainB]
87 | trainY = self.encode_labels(self.sick_data['train']['y'])
88 |
89 | # Dev
90 | devA = sick_embed['dev']['X_A']
91 | devB = sick_embed['dev']['X_B']
92 | devF = np.c_[np.abs(devA - devB), devA * devB]
93 | devY = self.encode_labels(self.sick_data['dev']['y'])
94 |
95 | # Test
96 | testA = sick_embed['test']['X_A']
97 | testB = sick_embed['test']['X_B']
98 | testF = np.c_[np.abs(testA - testB), testA * testB]
99 | testY = self.encode_labels(self.sick_data['test']['y'])
100 |
101 | config = {'seed': self.seed, 'nclasses': 5}
102 | clf = RelatednessPytorch(train={'X': trainF, 'y': trainY},
103 | valid={'X': devF, 'y': devY},
104 | test={'X': testF, 'y': testY},
105 | devscores=self.sick_data['dev']['y'],
106 | config=config)
107 |
108 | devspr, yhat = clf.run()
109 |
110 | pr = pearsonr(yhat, self.sick_data['test']['y'])[0]
111 | sr = spearmanr(yhat, self.sick_data['test']['y'])[0]
112 | pr = 0 if pr != pr else pr
113 | sr = 0 if sr != sr else sr
114 | se = mean_squared_error(yhat, self.sick_data['test']['y'])
115 | logging.debug('Dev : Spearman {0}'.format(devspr))
116 | logging.debug('Test : Pearson {0} Spearman {1} MSE {2} \
117 | for SICK Relatedness\n'.format(pr, sr, se))
118 |
119 | return {'devspearman': devspr, 'pearson': pr, 'spearman': sr, 'mse': se,
120 | 'yhat': yhat, 'ndev': len(devA), 'ntest': len(testA)}
121 |
122 | def encode_labels(self, labels, nclass=5):
123 | """
124 | Label encoding from Tree LSTM paper (Tai, Socher, Manning)
125 | """
126 | Y = np.zeros((len(labels), nclass)).astype('float32')
127 | for j, y in enumerate(labels):
128 | for i in range(nclass):
129 | if i+1 == np.floor(y) + 1:
130 | Y[j, i] = y - np.floor(y)
131 | if i+1 == np.floor(y):
132 | Y[j, i] = np.floor(y) - y + 1
133 | return Y
134 |
135 |
136 | class SICKEntailmentEval(SICKEval):
137 | def __init__(self, task_path, seed=1111):
138 | logging.debug('***** Transfer task : SICK-Entailment*****\n\n')
139 | self.seed = seed
140 | train = self.loadFile(os.path.join(task_path, 'SICK_train.txt'))
141 | dev = self.loadFile(os.path.join(task_path, 'SICK_trial.txt'))
142 | test = self.loadFile(os.path.join(task_path, 'SICK_test_annotated.txt'))
143 | self.sick_data = {'train': train, 'dev': dev, 'test': test}
144 |
145 | def loadFile(self, fpath):
146 | label2id = {'CONTRADICTION': 0, 'NEUTRAL': 1, 'ENTAILMENT': 2}
147 | skipFirstLine = True
148 | sick_data = {'X_A': [], 'X_B': [], 'y': []}
149 | with io.open(fpath, 'r', encoding='utf-8') as f:
150 | for line in f:
151 | if skipFirstLine:
152 | skipFirstLine = False
153 | else:
154 | text = line.strip().split('\t')
155 | sick_data['X_A'].append(text[1].split())
156 | sick_data['X_B'].append(text[2].split())
157 | sick_data['y'].append(text[4])
158 | sick_data['y'] = [label2id[s] for s in sick_data['y']]
159 | return sick_data
160 |
161 | def run(self, params, batcher):
162 | sick_embed = {'train': {}, 'dev': {}, 'test': {}}
163 | bsize = params.batch_size
164 |
165 | for key in self.sick_data:
166 | logging.info('Computing embedding for {0}'.format(key))
167 | # Sort to reduce padding
168 | sorted_corpus = sorted(zip(self.sick_data[key]['X_A'],
169 | self.sick_data[key]['X_B'],
170 | self.sick_data[key]['y']),
171 | key=lambda z: (len(z[0]), len(z[1]), z[2]))
172 |
173 | self.sick_data[key]['X_A'] = [x for (x, y, z) in sorted_corpus]
174 | self.sick_data[key]['X_B'] = [y for (x, y, z) in sorted_corpus]
175 | self.sick_data[key]['y'] = [z for (x, y, z) in sorted_corpus]
176 |
177 | for txt_type in ['X_A', 'X_B']:
178 | sick_embed[key][txt_type] = []
179 | for ii in range(0, len(self.sick_data[key]['y']), bsize):
180 | batch = self.sick_data[key][txt_type][ii:ii + bsize]
181 | embeddings = batcher(params, batch)
182 | sick_embed[key][txt_type].append(embeddings)
183 | sick_embed[key][txt_type] = np.vstack(sick_embed[key][txt_type])
184 | logging.info('Computed {0} embeddings'.format(key))
185 |
186 | # Train
187 | trainA = sick_embed['train']['X_A']
188 | trainB = sick_embed['train']['X_B']
189 | trainF = np.c_[np.abs(trainA - trainB), trainA * trainB]
190 | trainY = np.array(self.sick_data['train']['y'])
191 |
192 | # Dev
193 | devA = sick_embed['dev']['X_A']
194 | devB = sick_embed['dev']['X_B']
195 | devF = np.c_[np.abs(devA - devB), devA * devB]
196 | devY = np.array(self.sick_data['dev']['y'])
197 |
198 | # Test
199 | testA = sick_embed['test']['X_A']
200 | testB = sick_embed['test']['X_B']
201 | testF = np.c_[np.abs(testA - testB), testA * testB]
202 | testY = np.array(self.sick_data['test']['y'])
203 |
204 | config = {'nclasses': 3, 'seed': self.seed,
205 | 'usepytorch': params.usepytorch,
206 | 'classifier': params.classifier,
207 | 'nhid': params.nhid}
208 | clf = SplitClassifier(X={'train': trainF, 'valid': devF, 'test': testF},
209 | y={'train': trainY, 'valid': devY, 'test': testY},
210 | config=config)
211 |
212 | devacc, testacc = clf.run()
213 | logging.debug('\nDev acc : {0} Test acc : {1} for \
214 | SICK entailment\n'.format(devacc, testacc))
215 | return {'devacc': devacc, 'acc': testacc,
216 | 'ndev': len(devA), 'ntest': len(testA)}
217 |
--------------------------------------------------------------------------------
/SentEval/senteval/sts.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | '''
9 | STS-{2012,2013,2014,2015,2016} (unsupervised) and
10 | STS-benchmark (supervised) tasks
11 | '''
12 |
13 | from __future__ import absolute_import, division, unicode_literals
14 |
15 | import os
16 | import io
17 | import numpy as np
18 | import logging
19 |
20 | from scipy.stats import spearmanr, pearsonr
21 |
22 | from senteval.utils import cosine
23 | from senteval.sick import SICKEval
24 |
25 |
26 | class STSEval(object):
27 | def loadFile(self, fpath):
28 | self.data = {}
29 | self.samples = []
30 |
31 | for dataset in self.datasets:
32 | sent1, sent2 = zip(*[l.split("\t") for l in
33 | io.open(fpath + '/STS.input.%s.txt' % dataset,
34 | encoding='utf8').read().splitlines()])
35 | raw_scores = np.array([x for x in
36 | io.open(fpath + '/STS.gs.%s.txt' % dataset,
37 | encoding='utf8')
38 | .read().splitlines()])
39 | not_empty_idx = raw_scores != ''
40 |
41 | gs_scores = [float(x) for x in raw_scores[not_empty_idx]]
42 | sent1 = np.array([s.split() for s in sent1])[not_empty_idx]
43 | sent2 = np.array([s.split() for s in sent2])[not_empty_idx]
44 | # sort data by length to minimize padding in batcher
45 | sorted_data = sorted(zip(sent1, sent2, gs_scores),
46 | key=lambda z: (len(z[0]), len(z[1]), z[2]))
47 | sent1, sent2, gs_scores = map(list, zip(*sorted_data))
48 |
49 | self.data[dataset] = (sent1, sent2, gs_scores)
50 | self.samples += sent1 + sent2
51 |
52 | def do_prepare(self, params, prepare):
53 | if 'similarity' in params:
54 | self.similarity = params.similarity
55 | else: # Default similarity is cosine
56 | self.similarity = lambda s1, s2: np.nan_to_num(cosine(np.nan_to_num(s1), np.nan_to_num(s2)))
57 | return prepare(params, self.samples)
58 |
59 | def run(self, params, batcher):
60 | results = {}
61 | all_sys_scores = []
62 | all_gs_scores = []
63 | for dataset in self.datasets:
64 | sys_scores = []
65 | input1, input2, gs_scores = self.data[dataset]
66 | for ii in range(0, len(gs_scores), params.batch_size):
67 | batch1 = input1[ii:ii + params.batch_size]
68 | batch2 = input2[ii:ii + params.batch_size]
69 |
70 | # we assume get_batch already throws out the faulty ones
71 | if len(batch1) == len(batch2) and len(batch1) > 0:
72 | enc1 = batcher(params, batch1)
73 | enc2 = batcher(params, batch2)
74 |
75 | for kk in range(enc2.shape[0]):
76 | sys_score = self.similarity(enc1[kk], enc2[kk])
77 | sys_scores.append(sys_score)
78 | all_sys_scores.extend(sys_scores)
79 | all_gs_scores.extend(gs_scores)
80 |
81 | results[dataset] = {'pearson': pearsonr(sys_scores, gs_scores),
82 | 'spearman': spearmanr(sys_scores, gs_scores),
83 | 'nsamples': len(sys_scores)}
84 | logging.debug('%s : pearson = %.4f, spearman = %.4f' %
85 | (dataset, results[dataset]['pearson'][0],
86 | results[dataset]['spearman'][0]))
87 |
88 | weights = [results[dset]['nsamples'] for dset in results.keys()]
89 | list_prs = np.array([results[dset]['pearson'][0] for
90 | dset in results.keys()])
91 | list_spr = np.array([results[dset]['spearman'][0] for
92 | dset in results.keys()])
93 |
94 | avg_pearson = np.average(list_prs)
95 | avg_spearman = np.average(list_spr)
96 | wavg_pearson = np.average(list_prs, weights=weights)
97 | wavg_spearman = np.average(list_spr, weights=weights)
98 | all_pearson = pearsonr(all_sys_scores, all_gs_scores)
99 | all_spearman = spearmanr(all_sys_scores, all_gs_scores)
100 | results['all'] = {'pearson': {'all': all_pearson[0],
101 | 'mean': avg_pearson,
102 | 'wmean': wavg_pearson},
103 | 'spearman': {'all': all_spearman[0],
104 | 'mean': avg_spearman,
105 | 'wmean': wavg_spearman}}
106 | logging.debug('ALL : Pearson = %.4f, \
107 | Spearman = %.4f' % (all_pearson[0], all_spearman[0]))
108 | logging.debug('ALL (weighted average) : Pearson = %.4f, \
109 | Spearman = %.4f' % (wavg_pearson, wavg_spearman))
110 | logging.debug('ALL (average) : Pearson = %.4f, \
111 | Spearman = %.4f\n' % (avg_pearson, avg_spearman))
112 |
113 | return results
114 |
115 |
116 | class STS12Eval(STSEval):
117 | def __init__(self, taskpath, seed=1111):
118 | logging.debug('***** Transfer task : STS12 *****\n\n')
119 | self.seed = seed
120 | self.datasets = ['MSRpar', 'MSRvid', 'SMTeuroparl',
121 | 'surprise.OnWN', 'surprise.SMTnews']
122 | self.loadFile(taskpath)
123 |
124 |
125 | class STS13Eval(STSEval):
126 | # STS13 here does not contain the "SMT" subtask due to LICENSE issue
127 | def __init__(self, taskpath, seed=1111):
128 | logging.debug('***** Transfer task : STS13 (-SMT) *****\n\n')
129 | self.seed = seed
130 | self.datasets = ['FNWN', 'headlines', 'OnWN']
131 | self.loadFile(taskpath)
132 |
133 |
134 | class STS14Eval(STSEval):
135 | def __init__(self, taskpath, seed=1111):
136 | logging.debug('***** Transfer task : STS14 *****\n\n')
137 | self.seed = seed
138 | self.datasets = ['deft-forum', 'deft-news', 'headlines',
139 | 'images', 'OnWN', 'tweet-news']
140 | self.loadFile(taskpath)
141 |
142 |
143 | class STS15Eval(STSEval):
144 | def __init__(self, taskpath, seed=1111):
145 | logging.debug('***** Transfer task : STS15 *****\n\n')
146 | self.seed = seed
147 | self.datasets = ['answers-forums', 'answers-students',
148 | 'belief', 'headlines', 'images']
149 | self.loadFile(taskpath)
150 |
151 |
152 | class STS16Eval(STSEval):
153 | def __init__(self, taskpath, seed=1111):
154 | logging.debug('***** Transfer task : STS16 *****\n\n')
155 | self.seed = seed
156 | self.datasets = ['answer-answer', 'headlines', 'plagiarism',
157 | 'postediting', 'question-question']
158 | self.loadFile(taskpath)
159 |
160 |
161 | class STSBenchmarkEval(STSEval):
162 | def __init__(self, task_path, seed=1111):
163 | logging.debug('\n\n***** Transfer task : STSBenchmark*****\n\n')
164 | self.seed = seed
165 | self.samples = []
166 | train = self.loadFile(os.path.join(task_path, 'sts-train.csv'))
167 | dev = self.loadFile(os.path.join(task_path, 'sts-dev.csv'))
168 | test = self.loadFile(os.path.join(task_path, 'sts-test.csv'))
169 | self.datasets = ['train', 'dev', 'test']
170 | self.data = {'train': train, 'dev': dev, 'test': test}
171 |
172 | def loadFile(self, fpath):
173 | sick_data = {'X_A': [], 'X_B': [], 'y': []}
174 | with io.open(fpath, 'r', encoding='utf-8') as f:
175 | for line in f:
176 | text = line.strip().split('\t')
177 | sick_data['X_A'].append(text[5].split())
178 | sick_data['X_B'].append(text[6].split())
179 | sick_data['y'].append(text[4])
180 |
181 | sick_data['y'] = [float(s) for s in sick_data['y']]
182 | self.samples += sick_data['X_A'] + sick_data["X_B"]
183 | return (sick_data['X_A'], sick_data["X_B"], sick_data['y'])
184 |
185 | class STSBenchmarkFinetune(SICKEval):
186 | def __init__(self, task_path, seed=1111):
187 | logging.debug('\n\n***** Transfer task : STSBenchmark*****\n\n')
188 | self.seed = seed
189 | train = self.loadFile(os.path.join(task_path, 'sts-train.csv'))
190 | dev = self.loadFile(os.path.join(task_path, 'sts-dev.csv'))
191 | test = self.loadFile(os.path.join(task_path, 'sts-test.csv'))
192 | self.sick_data = {'train': train, 'dev': dev, 'test': test}
193 |
194 | def loadFile(self, fpath):
195 | sick_data = {'X_A': [], 'X_B': [], 'y': []}
196 | with io.open(fpath, 'r', encoding='utf-8') as f:
197 | for line in f:
198 | text = line.strip().split('\t')
199 | sick_data['X_A'].append(text[5].split())
200 | sick_data['X_B'].append(text[6].split())
201 | sick_data['y'].append(text[4])
202 |
203 | sick_data['y'] = [float(s) for s in sick_data['y']]
204 | return sick_data
205 |
206 | class SICKRelatednessEval(STSEval):
207 | def __init__(self, task_path, seed=1111):
208 | logging.debug('\n\n***** Transfer task : SICKRelatedness*****\n\n')
209 | self.seed = seed
210 | self.samples = []
211 | train = self.loadFile(os.path.join(task_path, 'SICK_train.txt'))
212 | dev = self.loadFile(os.path.join(task_path, 'SICK_trial.txt'))
213 | test = self.loadFile(os.path.join(task_path, 'SICK_test_annotated.txt'))
214 | self.datasets = ['train', 'dev', 'test']
215 | self.data = {'train': train, 'dev': dev, 'test': test}
216 |
217 | def loadFile(self, fpath):
218 | skipFirstLine = True
219 | sick_data = {'X_A': [], 'X_B': [], 'y': []}
220 | with io.open(fpath, 'r', encoding='utf-8') as f:
221 | for line in f:
222 | if skipFirstLine:
223 | skipFirstLine = False
224 | else:
225 | text = line.strip().split('\t')
226 | sick_data['X_A'].append(text[1].split())
227 | sick_data['X_B'].append(text[2].split())
228 | sick_data['y'].append(text[3])
229 |
230 | sick_data['y'] = [float(s) for s in sick_data['y']]
231 | self.samples += sick_data['X_A'] + sick_data["X_B"]
232 | return (sick_data['X_A'], sick_data["X_B"], sick_data['y'])
233 |
--------------------------------------------------------------------------------
/SentEval/senteval/.ipynb_checkpoints/sts-checkpoint.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | '''
9 | STS-{2012,2013,2014,2015,2016} (unsupervised) and
10 | STS-benchmark (supervised) tasks
11 | '''
12 |
13 | from __future__ import absolute_import, division, unicode_literals
14 |
15 | import os
16 | import io
17 | import numpy as np
18 | import logging
19 |
20 | from scipy.stats import spearmanr, pearsonr
21 |
22 | from senteval.utils import cosine
23 | from senteval.sick import SICKEval
24 |
25 |
26 | class STSEval(object):
27 | def loadFile(self, fpath):
28 | self.data = {}
29 | self.samples = []
30 |
31 | for dataset in self.datasets:
32 | sent1, sent2 = zip(*[l.split("\t") for l in
33 | io.open(fpath + '/STS.input.%s.txt' % dataset,
34 | encoding='utf8').read().splitlines()])
35 | raw_scores = np.array([x for x in
36 | io.open(fpath + '/STS.gs.%s.txt' % dataset,
37 | encoding='utf8')
38 | .read().splitlines()])
39 | not_empty_idx = raw_scores != ''
40 |
41 | gs_scores = [float(x) for x in raw_scores[not_empty_idx]]
42 | sent1 = np.array([s.split() for s in sent1])[not_empty_idx]
43 | sent2 = np.array([s.split() for s in sent2])[not_empty_idx]
44 | # sort data by length to minimize padding in batcher
45 | sorted_data = sorted(zip(sent1, sent2, gs_scores),
46 | key=lambda z: (len(z[0]), len(z[1]), z[2]))
47 | sent1, sent2, gs_scores = map(list, zip(*sorted_data))
48 |
49 | self.data[dataset] = (sent1, sent2, gs_scores)
50 | self.samples += sent1 + sent2
51 |
52 | def do_prepare(self, params, prepare):
53 | if 'similarity' in params:
54 | self.similarity = params.similarity
55 | else: # Default similarity is cosine
56 | self.similarity = lambda s1, s2: np.nan_to_num(cosine(np.nan_to_num(s1), np.nan_to_num(s2)))
57 | return prepare(params, self.samples)
58 |
59 | def run(self, params, batcher):
60 | results = {}
61 | all_sys_scores = []
62 | all_gs_scores = []
63 | for dataset in self.datasets:
64 | sys_scores = []
65 | input1, input2, gs_scores = self.data[dataset]
66 | for ii in range(0, len(gs_scores), params.batch_size):
67 | batch1 = input1[ii:ii + params.batch_size]
68 | batch2 = input2[ii:ii + params.batch_size]
69 |
70 | # we assume get_batch already throws out the faulty ones
71 | if len(batch1) == len(batch2) and len(batch1) > 0:
72 | enc1 = batcher(params, batch1)
73 | enc2 = batcher(params, batch2)
74 |
75 | for kk in range(enc2.shape[0]):
76 | sys_score = self.similarity(enc1[kk], enc2[kk])
77 | sys_scores.append(sys_score)
78 | all_sys_scores.extend(sys_scores)
79 | all_gs_scores.extend(gs_scores)
80 |
81 | results[dataset] = {'pearson': pearsonr(sys_scores, gs_scores),
82 | 'spearman': spearmanr(sys_scores, gs_scores),
83 | 'nsamples': len(sys_scores)}
84 | logging.debug('%s : pearson = %.4f, spearman = %.4f' %
85 | (dataset, results[dataset]['pearson'][0],
86 | results[dataset]['spearman'][0]))
87 |
88 | weights = [results[dset]['nsamples'] for dset in results.keys()]
89 | list_prs = np.array([results[dset]['pearson'][0] for
90 | dset in results.keys()])
91 | list_spr = np.array([results[dset]['spearman'][0] for
92 | dset in results.keys()])
93 |
94 | avg_pearson = np.average(list_prs)
95 | avg_spearman = np.average(list_spr)
96 | wavg_pearson = np.average(list_prs, weights=weights)
97 | wavg_spearman = np.average(list_spr, weights=weights)
98 | all_pearson = pearsonr(all_sys_scores, all_gs_scores)
99 | all_spearman = spearmanr(all_sys_scores, all_gs_scores)
100 | results['all'] = {'pearson': {'all': all_pearson[0],
101 | 'mean': avg_pearson,
102 | 'wmean': wavg_pearson},
103 | 'spearman': {'all': all_spearman[0],
104 | 'mean': avg_spearman,
105 | 'wmean': wavg_spearman}}
106 | logging.debug('ALL : Pearson = %.4f, \
107 | Spearman = %.4f' % (all_pearson[0], all_spearman[0]))
108 | logging.debug('ALL (weighted average) : Pearson = %.4f, \
109 | Spearman = %.4f' % (wavg_pearson, wavg_spearman))
110 | logging.debug('ALL (average) : Pearson = %.4f, \
111 | Spearman = %.4f\n' % (avg_pearson, avg_spearman))
112 |
113 | return results
114 |
115 |
116 | class STS12Eval(STSEval):
117 | def __init__(self, taskpath, seed=1111):
118 | logging.debug('***** Transfer task : STS12 *****\n\n')
119 | self.seed = seed
120 | self.datasets = ['MSRpar', 'MSRvid', 'SMTeuroparl',
121 | 'surprise.OnWN', 'surprise.SMTnews']
122 | self.loadFile(taskpath)
123 |
124 |
125 | class STS13Eval(STSEval):
126 | # STS13 here does not contain the "SMT" subtask due to LICENSE issue
127 | def __init__(self, taskpath, seed=1111):
128 | logging.debug('***** Transfer task : STS13 (-SMT) *****\n\n')
129 | self.seed = seed
130 | self.datasets = ['FNWN', 'headlines', 'OnWN']
131 | self.loadFile(taskpath)
132 |
133 |
134 | class STS14Eval(STSEval):
135 | def __init__(self, taskpath, seed=1111):
136 | logging.debug('***** Transfer task : STS14 *****\n\n')
137 | self.seed = seed
138 | self.datasets = ['deft-forum', 'deft-news', 'headlines',
139 | 'images', 'OnWN', 'tweet-news']
140 | self.loadFile(taskpath)
141 |
142 |
143 | class STS15Eval(STSEval):
144 | def __init__(self, taskpath, seed=1111):
145 | logging.debug('***** Transfer task : STS15 *****\n\n')
146 | self.seed = seed
147 | self.datasets = ['answers-forums', 'answers-students',
148 | 'belief', 'headlines', 'images']
149 | self.loadFile(taskpath)
150 |
151 |
152 | class STS16Eval(STSEval):
153 | def __init__(self, taskpath, seed=1111):
154 | logging.debug('***** Transfer task : STS16 *****\n\n')
155 | self.seed = seed
156 | self.datasets = ['answer-answer', 'headlines', 'plagiarism',
157 | 'postediting', 'question-question']
158 | self.loadFile(taskpath)
159 |
160 |
161 | class STSBenchmarkEval(STSEval):
162 | def __init__(self, task_path, seed=1111):
163 | logging.debug('\n\n***** Transfer task : STSBenchmark*****\n\n')
164 | self.seed = seed
165 | self.samples = []
166 | train = self.loadFile(os.path.join(task_path, 'sts-train.csv'))
167 | dev = self.loadFile(os.path.join(task_path, 'sts-dev.csv'))
168 | test = self.loadFile(os.path.join(task_path, 'sts-test.csv'))
169 | self.datasets = ['train', 'dev', 'test']
170 | self.data = {'train': train, 'dev': dev, 'test': test}
171 |
172 | def loadFile(self, fpath):
173 | sick_data = {'X_A': [], 'X_B': [], 'y': []}
174 | with io.open(fpath, 'r', encoding='utf-8') as f:
175 | for line in f:
176 | text = line.strip().split('\t')
177 | sick_data['X_A'].append(text[5].split())
178 | sick_data['X_B'].append(text[6].split())
179 | sick_data['y'].append(text[4])
180 |
181 | sick_data['y'] = [float(s) for s in sick_data['y']]
182 | self.samples += sick_data['X_A'] + sick_data["X_B"]
183 | return (sick_data['X_A'], sick_data["X_B"], sick_data['y'])
184 |
185 | class STSBenchmarkFinetune(SICKEval):
186 | def __init__(self, task_path, seed=1111):
187 | logging.debug('\n\n***** Transfer task : STSBenchmark*****\n\n')
188 | self.seed = seed
189 | train = self.loadFile(os.path.join(task_path, 'sts-train.csv'))
190 | dev = self.loadFile(os.path.join(task_path, 'sts-dev.csv'))
191 | test = self.loadFile(os.path.join(task_path, 'sts-test.csv'))
192 | self.sick_data = {'train': train, 'dev': dev, 'test': test}
193 |
194 | def loadFile(self, fpath):
195 | sick_data = {'X_A': [], 'X_B': [], 'y': []}
196 | with io.open(fpath, 'r', encoding='utf-8') as f:
197 | for line in f:
198 | text = line.strip().split('\t')
199 | sick_data['X_A'].append(text[5].split())
200 | sick_data['X_B'].append(text[6].split())
201 | sick_data['y'].append(text[4])
202 |
203 | sick_data['y'] = [float(s) for s in sick_data['y']]
204 | return sick_data
205 |
206 | class SICKRelatednessEval(STSEval):
207 | def __init__(self, task_path, seed=1111):
208 | logging.debug('\n\n***** Transfer task : SICKRelatedness*****\n\n')
209 | self.seed = seed
210 | self.samples = []
211 | train = self.loadFile(os.path.join(task_path, 'SICK_train.txt'))
212 | dev = self.loadFile(os.path.join(task_path, 'SICK_trial.txt'))
213 | test = self.loadFile(os.path.join(task_path, 'SICK_test_annotated.txt'))
214 | self.datasets = ['train', 'dev', 'test']
215 | self.data = {'train': train, 'dev': dev, 'test': test}
216 |
217 | def loadFile(self, fpath):
218 | skipFirstLine = True
219 | sick_data = {'X_A': [], 'X_B': [], 'y': []}
220 | with io.open(fpath, 'r', encoding='utf-8') as f:
221 | for line in f:
222 | if skipFirstLine:
223 | skipFirstLine = False
224 | else:
225 | text = line.strip().split('\t')
226 | sick_data['X_A'].append(text[1].split())
227 | sick_data['X_B'].append(text[2].split())
228 | sick_data['y'].append(text[3])
229 |
230 | sick_data['y'] = [float(s) for s in sick_data['y']]
231 | self.samples += sick_data['X_A'] + sick_data["X_B"]
232 | return (sick_data['X_A'], sick_data["X_B"], sick_data['y'])
233 |
--------------------------------------------------------------------------------
/SentEval/examples/models.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2017-present, Facebook, Inc.
2 | # All rights reserved.
3 | #
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 | #
7 |
8 | """
9 | This file contains the definition of encoders used in https://arxiv.org/pdf/1705.02364.pdf
10 | """
11 |
12 | import numpy as np
13 | import time
14 |
15 | import torch
16 | import torch.nn as nn
17 |
18 |
19 | class InferSent(nn.Module):
20 |
21 | def __init__(self, config):
22 | super(InferSent, self).__init__()
23 | self.bsize = config['bsize']
24 | self.word_emb_dim = config['word_emb_dim']
25 | self.enc_lstm_dim = config['enc_lstm_dim']
26 | self.pool_type = config['pool_type']
27 | self.dpout_model = config['dpout_model']
28 | self.version = 1 if 'version' not in config else config['version']
29 |
30 | self.enc_lstm = nn.LSTM(self.word_emb_dim, self.enc_lstm_dim, 1,
31 | bidirectional=True, dropout=self.dpout_model)
32 |
33 | assert self.version in [1, 2]
34 | if self.version == 1:
35 | self.bos = '
54 |
71 |
72 |
55 |
58 |
60 |
61 |
62 |
65 |
66 |
69 |
70 |
73 |
74 |
83 |
84 |
75 |
78 | Top K: 5
76 |
77 |
79 |
82 | Threshold: 0.6
80 |
81 |
85 |
86 |
87 |
88 |
92 |
93 |
89 |
90 |
91 | '
36 | self.eos = ''
37 | self.max_pad = True
38 | self.moses_tok = False
39 | elif self.version == 2:
40 | self.bos = '