├── AttModel.py
├── README.md
├── data_load.py
├── eval.py
├── fig
├── accuracy.png
└── loss.png
├── hyperparams.py
├── modules.py
├── prepro.py
├── requirements.txt
├── results
└── model18.txt
└── train.py
/AttModel.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | '''
4 | Janurary 2018 by Wei Li
5 | liweihfyz@sjtu.edu.cn
6 | https://www.github.cim/leviswind/transformer-pytorch
7 | '''
8 |
9 | from __future__ import print_function
10 | import torch
11 | import torch.nn as nn
12 | import torch.nn.functional as F
13 | from torch.autograd import *
14 | import numpy as np
15 | from modules import *
16 | from hyperparams import Hyperparams as hp
17 |
18 |
19 | class AttModel(nn.Module):
20 | def __init__(self, hp_, enc_voc, dec_voc):
21 | '''Attention is all you nedd. https://arxiv.org/abs/1706.03762
22 | Args:
23 | hp: Hyper Parameters
24 | enc_voc: vocabulary size of encoder language
25 | dec_voc: vacabulary size of decoder language
26 | '''
27 | super(AttModel, self).__init__()
28 | self.hp = hp_
29 |
30 | self.enc_voc = enc_voc
31 | self.dec_voc = dec_voc
32 |
33 | # encoder
34 | self.enc_emb = embedding(self.enc_voc, self.hp.hidden_units, scale=True)
35 |
36 | if self.hp.sinusoid:
37 | self.enc_positional_encoding = positional_encoding(num_units=self.hp.hidden_units,
38 | zeros_pad=False,
39 | scale=False)
40 | else:
41 | self.enc_positional_encoding = embedding(self.hp.maxlen, self.hp.hidden_units, zeros_pad=False, scale=False)
42 | self.enc_dropout = nn.Dropout(self.hp.dropout_rate)
43 | for i in range(self.hp.num_blocks):
44 | self.__setattr__('enc_self_attention_%d' % i, multihead_attention(num_units=self.hp.hidden_units,
45 | num_heads=self.hp.num_heads,
46 | dropout_rate=self.hp.dropout_rate,
47 | causality=False))
48 | self.__setattr__('enc_feed_forward_%d' % i, feedforward(self.hp.hidden_units,
49 | [4 * self.hp.hidden_units,
50 | self.hp.hidden_units]))
51 |
52 | # decoder
53 | self.dec_emb = embedding(self.dec_voc, self.hp.hidden_units, scale=True)
54 | if self.hp.sinusoid:
55 | self.dec_positional_encoding = positional_encoding(num_units=self.hp.hidden_units,
56 | zeros_pad=False,
57 | scale=False)
58 | else:
59 | self.dec_positional_encoding = embedding(self.hp.maxlen, self.hp.hidden_units, zeros_pad=False, scale=False)
60 |
61 | self.dec_dropout = nn.Dropout(self.hp.dropout_rate)
62 | for i in range(self.hp.num_blocks):
63 | self.__setattr__('dec_self_attention_%d' % i,
64 | multihead_attention(num_units=self.hp.hidden_units,
65 | num_heads=self.hp.num_heads,
66 | dropout_rate=self.hp.dropout_rate,
67 | causality=True))
68 | self.__setattr__('dec_vanilla_attention_%d' % i,
69 | multihead_attention(num_units=self.hp.hidden_units,
70 | num_heads=self.hp.num_heads,
71 | dropout_rate=self.hp.dropout_rate,
72 | causality=False))
73 | self.__setattr__('dec_feed_forward_%d' % i, feedforward(self.hp.hidden_units,
74 | [4 * self.hp.hidden_units,
75 | self.hp.hidden_units]))
76 | self.logits_layer = nn.Linear(self.hp.hidden_units, self.dec_voc)
77 | self.label_smoothing = label_smoothing()
78 | # self.losslayer = nn.CrossEntropyLoss(reduce=False)
79 |
80 | def forward(self, x, y):
81 | # define decoder inputs
82 | self.decoder_inputs = torch.cat([Variable(torch.ones(y[:, :1].size()).cuda() * 2).long(), y[:, :-1]], dim=-1) # 2:
83 |
84 | # Encoder
85 | self.enc = self.enc_emb(x)
86 | # Positional Encoding
87 | if self.hp.sinusoid:
88 | self.enc += self.enc_positional_encoding(x)
89 | else:
90 | self.enc += self.enc_positional_encoding(
91 | Variable(torch.unsqueeze(torch.arange(0, x.size()[1]), 0).repeat(x.size(0), 1).long().cuda()))
92 | self.enc = self.enc_dropout(self.enc)
93 | # Blocks
94 | for i in range(self.hp.num_blocks):
95 | self.enc = self.__getattr__('enc_self_attention_%d' % i)(self.enc, self.enc, self.enc)
96 | # Feed Forward
97 | self.enc = self.__getattr__('enc_feed_forward_%d' % i)(self.enc)
98 | # Decoder
99 | self.dec = self.dec_emb(self.decoder_inputs)
100 | # Positional Encoding
101 | if self.hp.sinusoid:
102 | self.dec += self.dec_positional_encoding(self.decoder_inputs)
103 | else:
104 | self.dec += self.dec_positional_encoding(
105 | Variable(torch.unsqueeze(torch.arange(0, self.decoder_inputs.size()[1]), 0).repeat(self.decoder_inputs.size(0), 1).long().cuda()))
106 |
107 | # Dropout
108 | self.dec = self.dec_dropout(self.dec)
109 | # Blocks
110 | for i in range(self.hp.num_blocks):
111 | # self-attention
112 | self.dec = self.__getattr__('dec_self_attention_%d' % i)(self.dec, self.dec, self.dec)
113 | # vanilla attention
114 | self.dec = self.__getattr__('dec_vanilla_attention_%d' % i)(self.dec, self.enc, self.enc)
115 | # feed forward
116 | self.dec = self.__getattr__('dec_feed_forward_%d' % i)(self.dec)
117 |
118 | # Final linear projection
119 | self.logits = self.logits_layer(self.dec)
120 | self.probs = F.softmax(self.logits, dim=-1).view(-1, self.dec_voc)
121 | _, self.preds = torch.max(self.logits, -1)
122 | self.istarget = (1. - y.eq(0.).float()).view(-1)
123 | self.acc = torch.sum(self.preds.eq(y).float().view(-1) * self.istarget) / torch.sum(self.istarget)
124 |
125 | # Loss
126 | self.y_onehot = torch.zeros(self.logits.size()[0] * self.logits.size()[1], self.dec_voc).cuda()
127 | self.y_onehot = Variable(self.y_onehot.scatter_(1, y.view(-1, 1).data, 1))
128 |
129 | self.y_smoothed = self.label_smoothing(self.y_onehot)
130 |
131 | # self.loss = self.losslayer(self.probs, self.y_smoothed)
132 | self.loss = - torch.sum(self.y_smoothed * torch.log(self.probs), dim=-1)
133 | # print(self.loss)
134 |
135 | self.mean_loss = torch.sum(self.loss * self.istarget) / torch.sum(self.istarget)
136 |
137 | return self.mean_loss, self.preds, self.acc
138 |
139 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # A Pytorch Implementation of the Transformer: Attention Is All You Need
2 | Our implementation is largely based on [Tensorflow implementation](https://github.com/Kyubyong/transformer)
3 |
4 | ## Requirements
5 | * NumPy >= 1.11.1
6 | * Pytorch >= 0.3.0
7 | * nltk
8 | * [tensorboard-pytorch](https://github.com/lanpa/tensorboard-pytorch) (build from source)
9 |
10 | ## Why This Project?
11 | I'm a freshman of pytorch. So I tried to implement some projects by pytorch. Recently, I read the paper [Attention is all you need](https://arxiv.org/abs/1706.03762) and impressed by the idea. So that's it. I got similar result compared with the original tensorflow implementation.
12 |
13 | ## Differences with the original paper
14 | I don't intend to replicate the paper exactly. Rather, I aim to implement the main ideas in the paper and verify them in a SIMPLE and QUICK way. In this respect, some parts in my code are different than those in the paper. Among them are
15 |
16 | * I used the IWSLT 2016 de-en dataset, not the wmt dataset because the former is much smaller, and requires no special preprocessing.
17 | * I constructed vocabulary with words, not subwords for simplicity. Of course, you can try bpe or word-piece if you want.
18 | * I parameterized positional encoding. The paper used some sinusoidal formula, but Noam, one of the authors, says they both work. See the [discussion in reddit](https://www.reddit.com/r/MachineLearning/comments/6gwqiw/r_170603762_attention_is_all_you_need_sota_nmt/)
19 | * The paper adjusted the learning rate to global steps. I fixed the learning to a small number, 0.0001 simply because training was reasonably fast enough with the small dataset (Only a couple of hours on a single GTX 1060!!).
20 |
21 | ## File description
22 | * `hyperparams.py` includes all hyper parameters that are needed.
23 | * `prepro.py` creates vocabulary files for the source and the target.
24 | * `data_load.py` contains functions regarding loading and batching data.
25 | * `modules.py` has all building blocks for encoder/decoder networks.
26 | * `train.py` has the model.
27 | * `eval.py` is for evaluation.
28 |
29 | ## Training
30 | * STEP 1. Download [IWSLT 2016 German–English parallel corpus](https://wit3.fbk.eu/download.php?release=2016-01&type=texts&slang=de&tlang=en) and extract it to `corpora/` folder.
31 | ```sh
32 | wget -qO- https://wit3.fbk.eu/archive/2016-01//texts/de/en/de-en.tgz | tar xz; mv de-en corpora
33 | ```
34 | * STEP 2. Adjust hyper parameters in `hyperparams.py` if necessary.
35 | * STEP 3. Run `prepro.py` to generate vocabulary files to the `preprocessed` folder.
36 | * STEP 4. Run `train.py` or download [pretrained weights](https://www.dropbox.com/s/iqjiuw3hkdqa6td/model_epoch_18.pth?dl=0), put it into folder './models/' and change the `eval_epoch` in `hpyerparams.py` to 18
37 | * STEP 5. Show loss and accuracy in tensorboard
38 | ```sh
39 | tensorboard --logdir runs
40 | ```
41 |
42 | ## Evaluation
43 | * Run `eval.py`.
44 |
45 | ## Results
46 | I got a BLEU score of 16.7.(tensorflow implementation 17.14) (Recollect I trained with a small dataset, limited vocabulary) Some of the evaluation results are as follows. Details are available in the `results` folder.
47 |
48 |
49 | source: Ich bin nicht sicher was ich antworten soll
50 | expected: I'm not really sure about the answer
51 | got: I'm not sure what I'm going to answer
52 |
53 | source: Was macht den Unterschied aus
54 | expected: What makes his story different
55 | got: What makes a difference
56 |
57 | source: Vielen Dank
58 | expected: Thank you
59 | got: Thank you
60 |
61 | source: Das ist ein Baum
62 | expected: This is a tree
63 | got: So this is a tree
64 |
65 |
--------------------------------------------------------------------------------
/data_load.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | #/usr/bin/python2
3 | '''
4 | June 2017 by kyubyong park.
5 | kbpark.linguist@gmail.com.
6 | https://www.github.com/kyubyong/transformer
7 | '''
8 | from __future__ import print_function
9 | from hyperparams import Hyperparams as hp
10 |
11 | import numpy as np
12 | import codecs
13 | import regex
14 | import random
15 | import torch
16 |
17 | try:
18 | xrange # Python 2
19 | except NameError:
20 | xrange = range # Python 3
21 |
22 | def load_de_vocab():
23 | vocab = [line.split()[0] for line in codecs.open('preprocessed/de.vocab.tsv', 'r', 'utf-8').read().splitlines() if int(line.split()[1])>=hp.min_cnt]
24 | word2idx = {word: idx for idx, word in enumerate(vocab)}
25 | idx2word = {idx: word for idx, word in enumerate(vocab)}
26 | return word2idx, idx2word
27 |
28 | def load_en_vocab():
29 | vocab = [line.split()[0] for line in codecs.open('preprocessed/en.vocab.tsv', 'r', 'utf-8').read().splitlines() if int(line.split()[1])>=hp.min_cnt]
30 | word2idx = {word: idx for idx, word in enumerate(vocab)}
31 | idx2word = {idx: word for idx, word in enumerate(vocab)}
32 | return word2idx, idx2word
33 |
34 | def create_data(source_sents, target_sents):
35 | de2idx, idx2de = load_de_vocab()
36 | en2idx, idx2en = load_en_vocab()
37 |
38 | # Index
39 | x_list, y_list, Sources, Targets = [], [], [], []
40 | for source_sent, target_sent in zip(source_sents, target_sents):
41 | x = [de2idx.get(word, 1) for word in (source_sent + u" ").split()] # 1: OOV, : End of Text
42 | y = [en2idx.get(word, 1) for word in (target_sent + u" ").split()]
43 | if max(len(x), len(y)) <=hp.maxlen:
44 | x_list.append(np.array(x))
45 | y_list.append(np.array(y))
46 | Sources.append(source_sent)
47 | Targets.append(target_sent)
48 |
49 | # Pad
50 | X = np.zeros([len(x_list), hp.maxlen], np.int32)
51 | Y = np.zeros([len(y_list), hp.maxlen], np.int32)
52 | for i, (x, y) in enumerate(zip(x_list, y_list)):
53 | X[i] = np.lib.pad(x, [0, hp.maxlen-len(x)], 'constant', constant_values=(0, 0))
54 | Y[i] = np.lib.pad(y, [0, hp.maxlen-len(y)], 'constant', constant_values=(0, 0))
55 |
56 | return X, Y, Sources, Targets
57 |
58 | def load_train_data():
59 | de_sents = [regex.sub("[^\s\p{Latin}']", "", line) for line in codecs.open(hp.source_train, 'r', 'utf-8').read().split("\n") if line and line[0] != "<"]
60 | en_sents = [regex.sub("[^\s\p{Latin}']", "", line) for line in codecs.open(hp.target_train, 'r', 'utf-8').read().split("\n") if line and line[0] != "<"]
61 |
62 | X, Y, Sources, Targets = create_data(de_sents, en_sents)
63 | return X, Y
64 |
65 | def load_test_data():
66 | def _refine(line):
67 | line = regex.sub("<[^>]+>", "", line)
68 | line = regex.sub("[^\s\p{Latin}']", "", line)
69 | return line.strip()
70 |
71 | de_sents = [_refine(line) for line in codecs.open(hp.source_test, 'r', 'utf-8').read().split("\n") if line and line[:4] == "= total_length:
84 | break
85 | current_index += batch_size
86 | yield indexs[current_index: current_index + batch_size], current_index
87 |
88 |
89 |
--------------------------------------------------------------------------------
/eval.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | '''
4 | Janurary 2018 by Wei Li
5 | liweihfyz@sjtu.edu.cn
6 | https://www.github.cim/leviswind/transformer-pytorch
7 | '''
8 |
9 | from __future__ import print_function
10 | import codecs
11 | import os
12 |
13 | import numpy as np
14 |
15 | from hyperparams import Hyperparams as hp
16 | from data_load import load_test_data, load_de_vocab, load_en_vocab
17 | from nltk.translate.bleu_score import corpus_bleu
18 | from AttModel import AttModel
19 | from torch.autograd import Variable
20 | import torch
21 |
22 |
23 | def eval():
24 | # Load data
25 | X, Sources, Targets = load_test_data()
26 | de2idx, idx2de = load_de_vocab()
27 | en2idx, idx2en = load_en_vocab()
28 | enc_voc = len(de2idx)
29 | dec_voc = len(en2idx)
30 |
31 | # load model
32 | model = AttModel(hp, enc_voc, dec_voc)
33 | model.load_state_dict(torch.load(hp.model_dir + '/model_epoch_%02d' % hp.eval_epoch + '.pth'))
34 | print('Model Loaded.')
35 | model.eval()
36 | model.cuda()
37 | # Inference
38 | if not os.path.exists('results'):
39 | os.mkdir('results')
40 | with codecs.open('results/model%d.txt' % hp.eval_epoch, 'w', 'utf-8') as fout:
41 | list_of_refs, hypotheses = [], []
42 | for i in range(len(X) // hp.batch_size):
43 | # Get mini-batches
44 | x = X[i * hp.batch_size: (i + 1) * hp.batch_size]
45 | sources = Sources[i * hp.batch_size: (i + 1) * hp.batch_size]
46 | targets = Targets[i * hp.batch_size: (i + 1) * hp.batch_size]
47 |
48 | # Autoregressive inference
49 | x_ = Variable(torch.LongTensor(x).cuda())
50 | preds_t = torch.LongTensor(np.zeros((hp.batch_size, hp.maxlen), np.int32)).cuda()
51 | preds = Variable(preds_t)
52 | for j in range(hp.maxlen):
53 |
54 | _, _preds, _ = model(x_, preds)
55 | preds_t[:, j] = _preds.data[:, j]
56 | preds = Variable(preds_t.long())
57 | preds = preds.data.cpu().numpy()
58 |
59 | # Write to file
60 | for source, target, pred in zip(sources, targets, preds): # sentence-wise
61 | got = " ".join(idx2en[idx] for idx in pred).split("")[0].strip()
62 | fout.write("- source: " + source + "\n")
63 | fout.write("- expected: " + target + "\n")
64 | fout.write("- got: " + got + "\n\n")
65 | fout.flush()
66 |
67 | # bleu score
68 | ref = target.split()
69 | hypothesis = got.split()
70 | if len(ref) > 3 and len(hypothesis) > 3:
71 | list_of_refs.append([ref])
72 | hypotheses.append(hypothesis)
73 | # Calculate bleu score
74 | score = corpus_bleu(list_of_refs, hypotheses)
75 | fout.write("Bleu Score = " + str(100 * score))
76 |
77 |
78 | if __name__ == '__main__':
79 | eval()
80 | print('Done')
81 |
82 |
83 |
--------------------------------------------------------------------------------
/fig/accuracy.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/leviswind/pytorch-transformer/9aaf4cd5a5f88671a06ef1d7fe617984027c586f/fig/accuracy.png
--------------------------------------------------------------------------------
/fig/loss.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/leviswind/pytorch-transformer/9aaf4cd5a5f88671a06ef1d7fe617984027c586f/fig/loss.png
--------------------------------------------------------------------------------
/hyperparams.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | #/usr/bin/python2
3 | '''
4 | June 2017 by kyubyong park.
5 | kbpark.linguist@gmail.com.
6 | https://www.github.com/kyubyong/transformer
7 | '''
8 | class Hyperparams:
9 | '''Hyperparameters'''
10 | # data
11 | source_train = 'corpora/train.tags.de-en.de'
12 | target_train = 'corpora/train.tags.de-en.en'
13 | source_test = 'corpora/IWSLT16.TED.tst2014.de-en.de.xml'
14 | target_test = 'corpora/IWSLT16.TED.tst2014.de-en.en.xml'
15 |
16 | # training
17 | batch_size = 32 # alias = N
18 | lr = 0.0001 # learning rate. In paper, learning rate is adjusted to the global step.
19 | logdir = 'logdir' # log directory
20 |
21 | model_dir = './models/' # saving directory
22 |
23 | # model
24 | maxlen = 10 # Maximum number of words in a sentence. alias = T.
25 | # Feel free to increase this if you are ambitious.
26 | min_cnt = 20 # words whose occurred less than min_cnt are encoded as .
27 | hidden_units = 512 # alias = C
28 | num_blocks = 6 # number of encoder/decoder blocks
29 | num_epochs = 20
30 | num_heads = 8
31 | dropout_rate = 0.1
32 | sinusoid = False # If True, use sinusoid. If false, positional embedding.
33 | eval_epoch = 20 # epoch of model for eval
34 | preload = None # epcho of preloaded model for resuming training
35 |
36 |
37 |
38 |
39 |
--------------------------------------------------------------------------------
/modules.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | '''
4 | Janurary 2018 by Wei Li
5 | liweihfyz@sjtu.edu.cn
6 | https://www.github.cim/leviswind/transformer-pytorch
7 | '''
8 |
9 | from __future__ import print_function
10 | import torch
11 | import torch.nn as nn
12 | import torch.nn.functional as F
13 | from torch.autograd import *
14 | import numpy as np
15 | from torch.nn.parameter import Parameter
16 |
17 |
18 | class embedding(nn.Module):
19 |
20 | def __init__(self, vocab_size, num_units, zeros_pad=True, scale=True):
21 | '''Embeds a given Variable.
22 | Args:
23 | vocab_size: An int. Vocabulary size.
24 | num_units: An int. Number of embedding hidden units.
25 | zero_pad: A boolean. If True, all the values of the fist row (id 0)
26 | should be constant zeros.
27 | scale: A boolean. If True. the outputs is multiplied by sqrt num_units.
28 | '''
29 | super(embedding, self).__init__()
30 | self.vocab_size = vocab_size
31 | self.num_units = num_units
32 | self.zeros_pad = zeros_pad
33 | self.scale = scale
34 | self.lookup_table = Parameter(torch.Tensor(vocab_size, num_units))
35 | nn.init.xavier_normal(self.lookup_table.data)
36 | if self.zeros_pad:
37 | self.lookup_table.data[0, :].fill_(0)
38 |
39 | def forward(self, inputs):
40 | if self.zeros_pad:
41 | self.padding_idx = 0
42 | else:
43 | self.padding_idx = -1
44 | outputs = self._backend.Embedding.apply(
45 | inputs, self.lookup_table, self.padding_idx, None, 2, False, False) # copied from torch.nn.modules.sparse.py
46 |
47 | if self.scale:
48 | outputs = outputs * (self.num_units ** 0.5)
49 |
50 | return outputs
51 |
52 |
53 | class layer_normalization(nn.Module):
54 |
55 | def __init__(self, features, epsilon=1e-8):
56 | '''Applies layer normalization.
57 |
58 | Args:
59 | epsilon: A floating number. A very small number for preventing ZeroDivision Error.
60 | '''
61 | super(layer_normalization, self).__init__()
62 | self.epsilon = epsilon
63 | self.gamma = nn.Parameter(torch.ones(features))
64 | self.beta = nn.Parameter(torch.zeros(features))
65 |
66 | def forward(self, x):
67 | mean = x.mean(-1, keepdim=True)
68 | std = x.std(-1, keepdim=True)
69 | return self.gamma * (x - mean) / (std + self.epsilon) + self.beta
70 |
71 |
72 | class positional_encoding(nn.Module):
73 |
74 | def __init__(self, num_units, zeros_pad=True, scale=True):
75 | '''Sinusoidal Positional_Encoding.
76 |
77 | Args:
78 | num_units: Output dimensionality
79 | zero_pad: Boolean. If True, all the values of the first row (id = 0) should be constant zero
80 | scale: Boolean. If True, the output will be multiplied by sqrt num_units(check details from paper)
81 | '''
82 | super(positional_encoding, self).__init__()
83 | self.num_units = num_units
84 | self.zeros_pad = zeros_pad
85 | self.scale = scale
86 |
87 | def forward(self, inputs):
88 | # inputs: A 2d Tensor with shape of (N, T).
89 | N, T = inputs.size()[0: 2]
90 |
91 | # First part of the PE function: sin and cos argument
92 | position_ind = Variable(torch.unsqueeze(torch.arange(0, T), 0).repeat(N, 1).long())
93 | position_enc = torch.Tensor([
94 | [pos / np.power(10000, 2. * i / self.num_units) for i in range(self.num_units)]
95 | for pos in range(T)])
96 |
97 | # Second part, apply the cosine to even columns and sin to odds.
98 | position_enc[:, 0::2] = torch.sin(position_enc[:, 0::2]) # dim 2i
99 | position_enc[:, 1::2] = torch.cos(position_enc[:, 1::2]) # dim 2i+1
100 |
101 | # Convert to a Variable
102 | lookup_table = Variable(position_enc)
103 |
104 | if self.zeros_pad:
105 | lookup_table = torch.cat((Variable(torch.zeros(1, self.num_units)),
106 | lookup_table[1:, :]), 0)
107 | padding_idx = 0
108 | else:
109 | padding_idx = -1
110 |
111 | outputs = self._backend.Embedding.apply(
112 | position_ind, lookup_table, padding_idx, None, 2, False, False) # copied from torch.nn.modules.sparse.py
113 |
114 | if self.scale:
115 | outputs = outputs * self.num_units ** 0.5
116 |
117 | return outputs
118 |
119 |
120 | class multihead_attention(nn.Module):
121 |
122 | def __init__(self, num_units, num_heads=8, dropout_rate=0, causality=False):
123 | '''Applies multihead attention.
124 |
125 | Args:
126 | num_units: A scalar. Attention size.
127 | dropout_rate: A floating point number.
128 | causality: Boolean. If true, units that reference the future are masked.
129 | num_heads: An int. Number of heads.
130 | '''
131 | super(multihead_attention, self).__init__()
132 | self.num_units = num_units
133 | self.num_heads = num_heads
134 | self.dropout_rate = dropout_rate
135 | self.causality = causality
136 | self.Q_proj = nn.Sequential(nn.Linear(self.num_units, self.num_units), nn.ReLU())
137 | self.K_proj = nn.Sequential(nn.Linear(self.num_units, self.num_units), nn.ReLU())
138 | self.V_proj = nn.Sequential(nn.Linear(self.num_units, self.num_units), nn.ReLU())
139 |
140 | self.output_dropout = nn.Dropout(p=self.dropout_rate)
141 |
142 | self.normalization = layer_normalization(self.num_units)
143 |
144 | def forward(self, queries, keys, values):
145 | # keys, values: same shape of [N, T_k, C_k]
146 | # queries: A 3d Variable with shape of [N, T_q, C_q]
147 |
148 | # Linear projections
149 | Q = self.Q_proj(queries) # (N, T_q, C)
150 | K = self.K_proj(keys) # (N, T_q, C)
151 | V = self.V_proj(values) # (N, T_q, C)
152 |
153 | # Split and concat
154 | Q_ = torch.cat(torch.chunk(Q, self.num_heads, dim=2), dim=0) # (h*N, T_q, C/h)
155 | K_ = torch.cat(torch.chunk(K, self.num_heads, dim=2), dim=0) # (h*N, T_q, C/h)
156 | V_ = torch.cat(torch.chunk(V, self.num_heads, dim=2), dim=0) # (h*N, T_q, C/h)
157 |
158 | # Multiplication
159 | outputs = torch.bmm(Q_, K_.permute(0, 2, 1)) # (h*N, T_q, T_k)
160 |
161 | # Scale
162 | outputs = outputs / (K_.size()[-1] ** 0.5)
163 |
164 | # Key Masking
165 | key_masks = torch.sign(torch.abs(torch.sum(keys, dim=-1))) # (N, T_k)
166 | key_masks = key_masks.repeat(self.num_heads, 1) # (h*N, T_k)
167 | key_masks = torch.unsqueeze(key_masks, 1).repeat(1, queries.size()[1], 1) # (h*N, T_q, T_k)
168 |
169 | padding = Variable(torch.ones(*outputs.size()).cuda() * (-2 ** 32 + 1))
170 | condition = key_masks.eq(0.).float()
171 | outputs = padding * condition + outputs * (1. - condition)
172 |
173 | # Causality = Future blinding
174 | if self.causality:
175 | diag_vals = torch.ones(*outputs[0, :, :].size()).cuda() # (T_q, T_k)
176 | tril = torch.tril(diag_vals, diagonal=0) # (T_q, T_k)
177 | # print(tril)
178 | masks = Variable(torch.unsqueeze(tril, 0).repeat(outputs.size()[0], 1, 1)) # (h*N, T_q, T_k)
179 |
180 | padding = Variable(torch.ones(*masks.size()).cuda() * (-2 ** 32 + 1))
181 | condition = masks.eq(0.).float()
182 | outputs = padding * condition + outputs * (1. - condition)
183 |
184 | # Activation
185 | outputs = F.softmax(outputs, dim=-1) # (h*N, T_q, T_k)
186 |
187 | # Query Masking
188 | query_masks = torch.sign(torch.abs(torch.sum(queries, dim=-1))) # (N, T_q)
189 | query_masks = query_masks.repeat(self.num_heads, 1) # (h*N, T_q)
190 | query_masks = torch.unsqueeze(query_masks, 2).repeat(1, 1, keys.size()[1]) # (h*N, T_q, T_k)
191 | outputs = outputs * query_masks
192 |
193 | # Dropouts
194 | outputs = self.output_dropout(outputs) # (h*N, T_q, T_k)
195 |
196 | # Weighted sum
197 | outputs = torch.bmm(outputs, V_) # (h*N, T_q, C/h)
198 |
199 | # Restore shape
200 | outputs = torch.cat(torch.chunk(outputs, self.num_heads, dim=0), dim=2) # (N, T_q, C)
201 |
202 | # Residual connection
203 | outputs += queries
204 |
205 | # Normalize
206 | outputs = self.normalization(outputs) # (N, T_q, C)
207 |
208 | return outputs
209 |
210 |
211 | class feedforward(nn.Module):
212 |
213 | def __init__(self, in_channels, num_units=[2048, 512]):
214 | '''Point-wise feed forward net.
215 |
216 | Args:
217 | in_channels: a number of channels of inputs
218 | num_units: A list of two integers.
219 | '''
220 | super(feedforward, self).__init__()
221 | self.in_channels = in_channels
222 | self.num_units = num_units
223 |
224 | # nn.Linear is faster than nn.Conv1d
225 | self.conv = False
226 | if self.conv:
227 | params = {'in_channels': self.in_channels, 'out_channels': self.num_units[0],
228 | 'kernel_size': 1, 'stride': 1, 'bias': True}
229 | self.conv1 = nn.Sequential(nn.Conv1d(**params), nn.ReLU())
230 | params = {'in_channels': self.num_units[0], 'out_channels': self.num_units[1],
231 | 'kernel_size': 1, 'stride': 1, 'bias': True}
232 | self.conv2 = nn.Conv1d(**params)
233 | else:
234 | self.conv1 = nn.Sequential(nn.Linear(self.in_channels, self.num_units[0]), nn.ReLU())
235 | self.conv2 = nn.Linear(self.num_units[0], self.num_units[1])
236 | self.normalization = layer_normalization(self.in_channels)
237 |
238 | def forward(self, inputs):
239 | if self.conv:
240 | inputs = inputs.permute(0, 2, 1)
241 | outputs = self.conv1(inputs)
242 | outputs = self.conv2(outputs)
243 |
244 | # Residual connection
245 | outputs += inputs
246 |
247 | # Layer normalization
248 | if self.conv:
249 | outputs = self.normalization(outputs.permute(0, 2, 1))
250 | else:
251 | outputs = self.normalization(outputs)
252 |
253 | return outputs
254 |
255 |
256 | class label_smoothing(nn.Module):
257 |
258 | def __init__(self, epsilon=0.1):
259 | '''Applies label smoothing. See https://arxiv.org/abs/1512.00567.
260 |
261 | Args:
262 | epsilon: Smoothing rate.
263 | '''
264 | super(label_smoothing, self).__init__()
265 | self.epsilon = epsilon
266 |
267 | def forward(self, inputs):
268 | K = inputs.size()[-1]
269 | return ((1 - self.epsilon) * inputs) + (self.epsilon / K)
270 |
271 |
272 | if __name__ == '__main__':
273 | num_units = 512
274 | inputs = Variable(torch.randn((100, 10)))
275 | outputs = position_encoding(num_units)(inputs)
276 | outputs = multihead_attention(num_units)(outputs, outputs, outputs)
277 | outputs = feedforward(num_units)(outputs)
278 |
279 | print(outputs)
280 |
--------------------------------------------------------------------------------
/prepro.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | #/usr/bin/python2
3 | '''
4 | June 2017 by kyubyong park.
5 | kbpark.linguist@gmail.com.
6 | https://www.github.com/kyubyong/transformer
7 | '''
8 | from __future__ import print_function
9 | from hyperparams import Hyperparams as hp
10 | import tensorflow as tf
11 | import numpy as np
12 | import codecs
13 | import os
14 | import regex
15 | from collections import Counter
16 |
17 | def make_vocab(fpath, fname):
18 | '''Constructs vocabulary.
19 |
20 | Args:
21 | fpath: A string. Input file path.
22 | fname: A string. Output file name.
23 |
24 | Writes vocabulary line by line to `preprocessed/fname`
25 | '''
26 | text = codecs.open(fpath, 'r', 'utf-8').read()
27 | text = regex.sub("[^\s\p{Latin}']", "", text)
28 | words = text.split()
29 | word2cnt = Counter(words)
30 | if not os.path.exists('preprocessed'): os.mkdir('preprocessed')
31 | with codecs.open('preprocessed/{}'.format(fname), 'w', 'utf-8') as fout:
32 | fout.write("{}\t1000000000\n{}\t1000000000\n{}\t1000000000\n{}\t1000000000\n".format("", "", "", ""))
33 | for word, cnt in word2cnt.most_common(len(word2cnt)):
34 | fout.write(u"{}\t{}\n".format(word, cnt))
35 |
36 | if __name__ == '__main__':
37 | make_vocab(hp.source_train, "de.vocab.tsv")
38 | make_vocab(hp.target_train, "en.vocab.tsv")
39 | print("Done")
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | NumPy >= 1.11.1
2 | Pytorch >= 0.3.0
3 | nltk
4 | tensorboard-pytorch(https://github.com/lanpa/tensorboard-pytorch) (build from source)
--------------------------------------------------------------------------------
/results/model18.txt:
--------------------------------------------------------------------------------
1 | - source: Sie war eine jhrige Frau namens Alex
2 | - expected: She was a yearold woman named Alex
3 | - got: She was a yearold woman called about begin with
4 |
5 | - source: Und als ich das hrte war ich erleichtert
6 | - expected: Now when I heard this I was so relieved
7 | - got: And as I heard it was I heard
8 |
9 | - source: Meine Kommilitonin bekam nmlich einen Brandstifter als ersten Patienten
10 | - expected: My classmate got an arsonist for her first client
11 | - got: Because got an first than patients
12 |
13 | - source: Das kriege ich hin dachte ich mir
14 | - expected: This I thought I could handle
15 | - got: That I'll go along I thought
16 |
17 | - source: Aber ich habe es nicht hingekriegt
18 | - expected: But I didn't handle it
19 | - got: But I didn't need it up
20 |
21 | - source: Ich hielt dagegen
22 | - expected: I pushed back
23 | - got: I thought about it
24 |
25 | - source: Das ist es was Psychologen einen AhaMoment nennen
26 | - expected: That's what psychologists call an Aha moment
27 | - got: This is what the
28 |
29 | - source: Meldet euch wenn ihr in euren ern seid
30 | - expected: Raise your hand if you're in your s
31 | - got: Get when you're in your own s
32 |
33 | - source: Ich mchte ein paar von euch sehen
34 | - expected: I really want to see some twentysomethings here
35 | - got: I want to see a few
36 |
37 | - source: Oh yeah Ihr seid alle unglaublich
38 | - expected: Oh yay Y'all's awesome
39 | - got: Oh yeah You're all incredibly
40 |
41 | - source: Dies ist nicht meine Meinung Das sind Fakten
42 | - expected: This is not my opinion These are the facts
43 | - got: This is not my fact that's what I think
44 |
45 | - source: Werdet nicht panisch wenn ihr ber seid
46 | - expected: People who are over don't panic
47 | - got: Don't worry about being over her
48 |
49 | - source: Dieser Gruppe wird es gut gehen glaube ich
50 | - expected: This crowd is going to be fine I think
51 | - got: This group is going to go good
52 |
53 | - source: Die Presse redet ber die Zeitverschiebung des Erwachsenwerdens
54 | - expected: Newspapers talk about the changing timetable of adulthood
55 | - got: The press started talking about the press society
56 |
57 | - source: Wissenschaftler nennen die er eine verlngerte Pubertt
58 | - expected: Researchers call the s an extended adolescence
59 | - got: Scientists call it the hit out
60 |
61 | - source: Das stimmt
62 | - expected: It's true
63 | - got: That's true
64 |
65 | - source: Stimmt das nicht
66 | - expected: Isn't that true
67 | - got: Does it happen
68 |
69 | - source: Gar nichts
70 | - expected: Nothing happens
71 | - got: Nothing else
72 |
73 | - source: Wo sind die Leute in den ern hier
74 | - expected: Where are the twentysomethings here
75 | - got: Where are people in the s here
76 |
77 | - source: Macht das nicht
78 | - expected: Do not do that
79 | - got: Does that happen
80 |
81 | - source: Es ist die Geschichte von Emma
82 | - expected: It's a story about a woman named Emma
83 | - got: It's the story of power
84 |
85 | - source: Das ist Prokastination
86 | - expected: That's procrastination
87 | - got: This is
88 |
89 | - source: Was geschah also mit Emma
90 | - expected: So what happened to Emma
91 | - got: So what happened to of
92 |
93 | - source: Es ist so einfach ihnen zu helfen
94 | - expected: They are so easy to help
95 | - got: It's so simple to help them
96 |
97 | - source: Ihr entscheidet gerade jetzt ber euer Leben
98 | - expected: You're deciding your life right now
99 | - got: You're just saying your lives now
100 |
101 | - source: Vielen Dank
102 | - expected: Thank you
103 | - got: Thank you
104 |
105 | - source: Ich habe mir vorgestellt wie das aussehen wrde
106 | - expected: I started to imagine what this would look like
107 | - got: I would also like to look at this
108 |
109 | - source: Mit diesem Werkzeug wurde die Grenze also durchbrochen
110 | - expected: So with this tool this boundary has been broken
111 | - got: So with that tool the frontier was
112 |
113 | - source: Aber unsere Hnde bleiben immer noch auerhalb des Bildschirms
114 | - expected: But our two hands still remain outside the screen
115 | - got: But our hands still remain outside the
116 |
117 | - source: Wie wrde eine solche Zukunft aussehen
118 | - expected: What would such a future look like
119 | - got: What would that future look like
120 |
121 | - source: Vielen Dank
122 | - expected: Thank you
123 | - got: Thank you
124 |
125 | - source: Aber meine Mutter sagte Nein
126 | - expected: But my tiger mother said No
127 | - got: But my mom said No
128 |
129 | Bleu Score = 17.282999562- source: Aber die Erwachsenen mochten meine Idee nicht
130 | - expected: But no adults liked the idea
131 | - got: But the adults never do my idea
132 |
133 | - source: Mein Traum wrde also nie wahr werden
134 | - expected: My dream would never come true
135 | - got: So my dream would never be true
136 |
137 | - source: Aber das ist so ungerecht
138 | - expected: But that's so unfair
139 | - got: But this is the thing
140 |
141 | - source: Keiner da um mich etwas zu lehren Auch gut
142 | - expected: Nobody around to teach me Fine
143 | - got: Nobody in there was too good to teach me
144 |
145 | - source: Also wendete ich mich Bchern zu
146 | - expected: I turned to books
147 | - got: So I went to books on to me
148 |
149 | - source: Natrlich Bcher die in China verboten sind
150 | - expected: Books banned in China of course
151 | - got: Of books books are being there in China
152 |
153 | - source: Die gute Erde handelt vom Leben der chinesischen Bauern
154 | - expected: The Good Earth is about Chinese peasant life
155 | - got: The good is the Earth of the farmers
156 |
157 | - source: Die Bibel ist interessant aber seltsam
158 | - expected: The Bible is interesting but strange
159 | - got: The Bible is interesting but weird
160 |
161 | - source: Aber das ist ein Thema fr ein anderes Mal
162 | - expected: That's a topic for a different day
163 | - got: But it's a topic for another time
164 |
165 | - source: Dies gewhrt viele Einblicke
166 | - expected: It offers many insights
167 | - got: This many individuals have a lot of
168 |
169 | - source: Das vergleichende Lesen ist eigentlich nichts Neues
170 | - expected: Comparative reading actually is nothing new
171 | - got: The reading is just nothing new actually
172 |
173 | - source: In der akademischen Welt ist es ein Standardverfahren
174 | - expected: It's a standard practice in the academic world
175 | - got: In the academic world it's a or a
176 |
177 | - source: So begann ich immer zwei Bcher gleichzeitig zu lesen
178 | - expected: So I started reading books in pairs
179 | - got: So I started reading two books
180 |
181 | - source: Braut im Chinesischen bedeutet wrtlich bersetzt neue Mutter Ohoh
182 | - expected: Bride in Chinese literally means new mother Uhoh
183 | - got: of means new to
184 |
185 | - source: Sogar ein zerplatzter Traum kann diesen Zweck erfllen
186 | - expected: Even a shattered dream can do that for you
187 | - got: Even a dream can be that dream
188 |
189 | - source: Mgen Bcher immer mit Ihnen sein
190 | - expected: So may books be always with you
191 | - got: Do books always been working with you
192 |
193 | - source: Vielen Dank
194 | - expected: Thank you
195 | - got: Thank you
196 |
197 | - source: Vielen Dank
198 | - expected: Thank you
199 | - got: Thank you
200 |
201 | - source: Vielen Dank
202 | - expected: Thank you
203 | - got: Thank you
204 |
205 | - source: Und dann tat ich es
206 | - expected: And then I did it
207 | - got: And then I did
208 |
209 | - source: Alle Einser
210 | - expected: All of those A's
211 | - got: And everybody was quiet
212 |
213 | - source: Ich bin immer noch verdutzt ber mein Verhalten
214 | - expected: And I am still baffled by my behavior
215 | - got: I'm still on my behavior
216 |
217 | - source: Ich verstehe nicht woher der Gedanke kam
218 | - expected: I don't understand where the idea came from
219 | - got: I don't get how I came from
220 |
221 | - source: Ich fhlte mich richtig gut
222 | - expected: I felt great
223 | - got: I felt really good to me
224 |
225 | - source: Es war doch so offensichtlich
226 | - expected: I mean it should have been so blatantly obvious
227 | - got: It was so obvious
228 |
229 | - source: Ich wurde nie erwischt
230 | - expected: I was never caught
231 | - got: I never got caught in
232 |
233 | - source: Eifersucht verblfft mich
234 | - expected: Jealousy baffles me
235 | - got: Just my mind
236 |
237 | - source: Sie ist so geheimnisvoll so tiefgreifend
238 | - expected: It's so mysterious and it's so pervasive
239 | - got: It's so uncomfortable
240 |
241 | - source: Wir wissen dass Babys unter Eifersucht leiden
242 | - expected: We know babies suffer from jealousy
243 | - got: We know that under some kinds of
244 |
245 | - source: Also keine untreue Helena keine Odyssee
246 | - expected: Well no faithless Helen no Odyssey
247 | - got: So not no energy plan
248 |
249 | - source: Kein eiferschtiger Knig keine Abenteuer aus Nacht
250 | - expected: No jealous king no Arabian Nights
251 | - got: No king was not night
252 |
253 | - source: Kein Shakespeare
254 | - expected: No Shakespeare
255 | - got: No that love
256 |
257 | Bleu Score = 15.9470382004- source: Wir denken an ein Stck Kuchen in LavendelTee eingetaucht
258 | - expected: We think about a madeleine moistened in lavender tea
259 | - got: We think about a bit of your piece
260 |
261 | - source: Wir vergessen wie rau seine Vision war
262 | - expected: We forget how harsh his vision was
263 | - got: We forget how vision was
264 |
265 | - source: Wir vergessen wie erbarmungslos er ist
266 | - expected: We forget how pitiless he is
267 | - got: We forget how they are him about is
268 |
269 | - source: Wenn wir eiferschtig sind erzhlen wir uns eine Geschichte
270 | - expected: When we feel jealous we tell ourselves a story
271 | - got: If we have we're just a story
272 |
273 | - source: Eifersucht ist ermdend
274 | - expected: Jealousy is exhausting
275 | - got: So an example of is
276 |
277 | - source: Es ist ein hungriges Gefhl das genhrt werden muss
278 | - expected: It's a hungry emotion It must be fed
279 | - got: It's a feeling of
280 |
281 | - source: Was mag die Eifersucht
282 | - expected: And what does jealousy like
283 | - got: What does the like
284 |
285 | - source: Eifersucht mag Informationen
286 | - expected: Jealousy likes information
287 | - got: information really
288 |
289 | - source: Eifersucht mag Details
290 | - expected: Jealousy likes details
291 | - got: like details
292 |
293 | - source: Eifersucht mag Fotos
294 | - expected: Jealousy likes photos
295 | - got: And so it likes pictures
296 |
297 | - source: Deshalb ist Instagram auch so erfolgreich
298 | - expected: That's why Instagram is such a hit
299 | - got: And so the question is why did
300 |
301 | - source: Freud sollte spter darber schreiben
302 | - expected: Freud would write about this later
303 | - got: should write about this a little bit
304 |
305 | - source: Weil sie unschuldig ist und jeder wei das
306 | - expected: Because she's blameless everybody knows it
307 | - got: Because it is power and everyone knows
308 |
309 | - source: Der Roman ist sehr gut in diesem Punkt
310 | - expected: The novel is very good on this point
311 | - got: The novel is very good at this point
312 |
313 | - source: Ein ganzes Jahr lang
314 | - expected: For a year A year
315 | - got: for a year
316 |
317 | - source: Weil Lassen Sie uns mal ehrlich sein
318 | - expected: Because I mean let's be real
319 | - got: Because let's be
320 |
321 | - source: Welch ungeheure wenn auch deplatzierte Kreativitt Stimmt's
322 | - expected: What immense if misplaced creativity Right
323 | - got: The also when creativity right
324 |
325 | - source: Das ist etwas aus einem Roman
326 | - expected: This is something from a novel
327 | - got: That's something from a novel
328 |
329 | - source: Das ist etwas aus einem Roman von Patricia Highsmith
330 | - expected: This is something from a Patricia Highsmith novel
331 | - got: This is something from a novel from a novel
332 |
333 | - source: Highsmith ist eine meiner Lieblingsschriftstellerinnen
334 | - expected: Now Highsmith is a particular favorite of mine
335 | - got: is one of my
336 |
337 | - source: Nehmen wir Tom Ripley ihre berhmteste Figur
338 | - expected: Take Tom Ripley her most famous character
339 | - got: Take Tom
340 |
341 | - source: Das ist eine Methode
342 | - expected: That's one way to go
343 | - got: This is a way of technology
344 |
345 | - source: Wir leben in einer eifersuchtsanflligen Zeit
346 | - expected: We live in jealous times
347 | - got: We live in a time
348 |
349 | - source: Das ist mir so vertraut
350 | - expected: Oh it's so familiar to me
351 | - got: That me
352 |
353 | - source: Wir sind stolz auf Sie
354 | - expected: We're proud of you
355 | - got: We are proud of you
356 |
357 | - source: Und im nchsten Moment liegen sie auf einer Wellenlnge
358 | - expected: The next minute they're on the same side
359 | - got: And the next moment they are
360 |
361 | - source: Ist es denn wirklich so einfach
362 | - expected: Could it be so simple though
363 | - got: Is it that really that
364 |
365 | - source: Wir knnten uns aneinander ausrichten
366 | - expected: We could align ourselves with it
367 | - got: We could Let's go with what he really do
368 |
369 | - source: Aber ich mag Notfallplne
370 | - expected: But I like contingency plans
371 | - got: But I just like rock
372 |
373 | - source: Literatur allein entrtselt die Eifersucht
374 | - expected: Fiction alone demystifies jealousy
375 | - got: alone the
376 |
377 | - source: Literatur allein kann sie zhmen ldt sie ein
378 | - expected: Fiction alone domesticates it invites it to the table
379 | - got: alone can put them in the
380 |
381 | - source: Wir befinden uns in exzellenter Gesellschaft
382 | - expected: We are in excellent company
383 | - got: We live in the society
384 |
385 | Bleu Score = 16.1113294767- source: Vielen Dank
386 | - expected: Thank you
387 | - got: Thank you
388 |
389 | - source: Es war ein wunderschner Tag
390 | - expected: It was a beautiful day
391 | - got: It was a beautiful day
392 |
393 | - source: Der Wald funkelte
394 | - expected: The forest was sparkling
395 | - got: The forest
396 |
397 | - source: Wir kamen zu unserem Zeltplatz
398 | - expected: We got to our campsite
399 | - got: We showed up to our women's issues
400 |
401 | - source: Wer rumt nach uns auf
402 | - expected: Who cleans up after us
403 | - got: Who
404 |
405 | - source: Ich wollte sie kennenlernen als Individuen
406 | - expected: I wanted to get to know them as individuals
407 | - got: I wanted to meet them as individuals
408 |
409 | - source: Ich wollte verstehen wer den Job macht
410 | - expected: I wanted to understand who takes the job
411 | - got: I wanted to understand who that job
412 |
413 | - source: Also startete ich ein Forschungsprojekt mit ihnen
414 | - expected: So I started a research project with them
415 | - got: So I started a
416 |
417 | - source: Ich musste noch tiefer gehen
418 | - expected: I needed to go deeper
419 | - got: I had to go further and going to
420 |
421 | - source: Also nahm ich einen Job als Reinigungskraft an
422 | - expected: So I took the job as a sanitation worker
423 | - got: So I took a job as a picture
424 |
425 | - source: Es war ein bemerkenswertes Privileg und eine aufregende Erfahrung
426 | - expected: It was a remarkable privilege and an amazing education
427 | - got: It was an amazing privilege and an exciting experience
428 |
429 | - source: Jeder fragt nach dem Gestank
430 | - expected: Everyone asks about the smell
431 | - got: Each one makes it
432 |
433 | - source: Dann ist da die Gefahr
434 | - expected: Then there's the danger
435 | - got: There is the danger going on
436 |
437 | - source: Das ist wirklich schlecht fr die Arbeiter
438 | - expected: That's really bad for the worker
439 | - got: It's actually bad for the workers
440 |
441 | - source: Ich lernte auch ber die Unbarmherzigkeit des Mlls
442 | - expected: I also learned about the relentlessness of trash
443 | - got: I learned also about the of the
444 |
445 | - source: Er hrt nicht auf zu entstehen
446 | - expected: It never stops coming
447 | - got: He keeps not
448 |
449 | - source: Er muss immer in Bewegung sein
450 | - expected: It must always be in motion
451 | - got: He's always must be in movement
452 |
453 | - source: Und dann ist da das Stigma
454 | - expected: And then there's the stigma
455 | - got: And then there's the the So the the the
456 |
457 | - source: Sie sind die Wchter der ffentlichen Gesundheit
458 | - expected: They are the first guardians of public health
459 | - got: They're the of public health
460 |
461 | - source: Die Wirtschaft braucht sie
462 | - expected: The economy needs them
463 | - got: The economy needs them
464 |
465 | - source: Ihre Arbeit so denke ich ist irgendwie liturgisch
466 | - expected: Their work I think is kind of liturgical
467 | - got: And the work I think is kind of
468 |
469 | - source: Sie sehen die Straen jeden Tag regelmig
470 | - expected: They're on the streets every day rhythmically
471 | - got: You see the streets of it go away
472 |
473 | - source: Sie tragen in vielen Stdten Uniformen
474 | - expected: They wear a uniform in many cities
475 | - got: She in many places
476 |
477 | - source: Man wei wann man sie erwarten kann
478 | - expected: You know when to expect them
479 | - got: You know when you can expect them
480 |
481 | - source: Und ihre Arbeit lsst uns unsere tun
482 | - expected: And their work lets us do our work
483 | - got: And their work done to us
484 |
485 | - source: Sie sind wie eine Rckversicherung
486 | - expected: They are almost a form of reassurance
487 | - got: They're like a
488 |
489 | - source: Alles wird gut
490 | - expected: We're going to be okay
491 | - got: going to be good
492 |
493 | - source: Ich wollte glauben dass Paulie recht hatte
494 | - expected: I want to believe that Paulie was right
495 | - got: I wanted to believe that was right
496 |
497 | - source: Alles wird gut
498 | - expected: We are going to be okay
499 | - got: going to be good
500 |
501 | - source: Es ist eine bemerkenswerte Statistik
502 | - expected: It's a remarkable statistic
503 | - got: It's a remarkable Because
504 |
505 | - source: Nehmen Sie sich einen Moment sich zu bedanken
506 | - expected: Take a moment to say thank you
507 | - got: Take a thank you for coming
508 |
509 | - source: Hallo TEDWomen was geht
510 | - expected: Hello TEDWomen what's up
511 | - got: Hi what is
512 |
513 | Bleu Score = 19.0778555279- source: Das reicht nicht
514 | - expected: Not good enough
515 | - got: That's not enough
516 |
517 | - source: Hallo TEDWomen was geht
518 | - expected: Hello TEDWomen what is up
519 | - got: Hi what is
520 |
521 | - source: Schaut mal
522 | - expected: Look
523 | - got: Look at that
524 |
525 | - source: Zerebralparese ist nicht genetisch
526 | - expected: CP is not genetic
527 | - got: There's no in this way
528 |
529 | - source: Sie ist kein Geburtsfehler Man kann sie nicht kriegen
530 | - expected: It's not a birth defect You can't catch it
531 | - got: It's not a You can't get them
532 |
533 | - source: Kommt mal kurz mit
534 | - expected: Come on a journey with me
535 | - got: Come on for a moment
536 |
537 | - source: Sechzehn leere Behindertenparkpltze
538 | - expected: Sixteen empty handicapped spaces
539 | - got: empty That means of empty
540 |
541 | - source: Cliffside Park in New Jersey ist meine Heimatstadt
542 | - expected: Cliffside Park New Jersey is my hometown
543 | - got: in New York is my home
544 |
545 | - source: Ich habe am Broadway gesteppt
546 | - expected: I have tapdanced on Broadway
547 | - got: I have the
548 |
549 | - source: Ja am Broadway Total verrckt
550 | - expected: Yeah on Broadway It's crazy
551 | - got: Yes in the crazy
552 |
553 | - source: Aber eine Wunderkur fanden wir Yoga
554 | - expected: But one miracle cure we did find was yoga
555 | - got: But we found we found
556 |
557 | - source: Jetzt kann ich auch auf dem Kopf stehen
558 | - expected: And now I can stand on my head
559 | - got: Now I can also stand on my head
560 |
561 | - source: Alle liebten mich
562 | - expected: Everybody loved me
563 | - got: He loved me
564 |
565 | - source: Aber ich bekam nie eine Rolle
566 | - expected: But I never got cast
567 | - got: But I never got an role to be a
568 |
569 | - source: Ein Stck ber ein Mdchen mit CP
570 | - expected: It's a play about a girl with CP
571 | - got: A piece of girls a play with an What
572 |
573 | - source: Ich war ein Mdchen mit CP
574 | - expected: I was a girl with CP
575 | - got: I was a girl with
576 |
577 | - source: Ich habe Zerebralparese
578 | - expected: I have cerebral palsy
579 | - got: I have
580 |
581 | - source: Endlich frei Endlich frei
582 | - expected: Free at last Free at last
583 | - got: Finally
584 |
585 | - source: Gott sei Dank ich bin endlich frei
586 | - expected: Thank God almighty I'm free at last
587 | - got: Thank God I finally did it
588 |
589 | - source: Ich bekam die Rolle nicht
590 | - expected: I didn't get the part
591 | - got: I didn't get the role to
592 |
593 | - source: Sherry Brown bekam die Rolle
594 | - expected: Sherry Brown got the part
595 | - got: Brown got the role to be the role
596 |
597 | - source: Die Uni imitierte das Leben
598 | - expected: College was imitating life
599 | - got: the life
600 |
601 | - source: Mein Traum wurde wahr
602 | - expected: My dream was coming true
603 | - got: It's been my dream actually
604 |
605 | - source: Sie stellten nur perfekte Leute ein
606 | - expected: They only hired perfect people
607 | - got: They just found perfect people
608 |
609 | - source: Aber Ausnahmen besttigen ja die Regel
610 | - expected: But there were exceptions to the rule
611 | - got: But exceptions yes
612 |
613 | - source: Also wurde ich Komikerin
614 | - expected: So I became a comic
615 | - got: So I became a school math
616 |
617 | - source: Mein groer Durchbruch war
618 | - expected: My big break came in
619 | - got: My big breakthrough was
620 |
621 | - source: Und wir waren live
622 | - expected: And we were live right
623 | - got: And we were live
624 |
625 | - source: Ey ist die bekloppt
626 | - expected: Yo is she retarded
627 | - got: Now the story is the
628 |
629 | - source: Und mein Favorit Armer HngelippenTerrorist
630 | - expected: And my favorite Poor Gumbymouth terrorist
631 | - got: And my
632 |
633 | - source: Hat sie ne Krankheit
634 | - expected: What does she suffer from
635 | - got: Does she have disease
636 |
637 | - source: Wir sollten fr sie beten
638 | - expected: We should really pray for her
639 | - got: We should be pray for them
640 |
641 | Bleu Score = 19.1724450034- source: Eine Behinderung ist so sichtbar wie die Herkunft
642 | - expected: Disability is as visual as race
643 | - got: One of the visible is as the background
644 |
645 | - source: Oder vielleicht nicht
646 | - expected: Or maybe not
647 | - got: Or maybe not
648 |
649 | - source: Menschen starben darunter auch die Besatzung der Apollo
650 | - expected: People died including the crew of Apollo
651 | - got: People died so the Apollo did the Apollo
652 |
653 | - source: Doch warum gingen sie
654 | - expected: So why did they go
655 | - got: But why did they do
656 |
657 | - source: Ich erinnere mich noch daran wie ich als Fnfjhriger
658 | - expected: I remember watching the liftoff of Apollo
659 | - got: So I remember it so I remember it
660 |
661 | - source: Man hrt es berall
662 | - expected: You hear it all the time
663 | - got: You hear it everywhere
664 |
665 | - source: Sie haben unser Leben verlngert und bereichert
666 | - expected: They've expanded and enriched our lives
667 | - got: This is what our lives and our
668 |
669 | - source: Doch sie lsen nicht die groen Probleme der Menschheit
670 | - expected: But they don't solve humanity's big problems
671 | - got: But they don't solve the big problems
672 |
673 | - source: Was ist passiert
674 | - expected: What happened
675 | - got: What happened
676 |
677 | - source: Aber diese Erklrung ist mir nicht gut genug
678 | - expected: But I don't think that explanation is good enough
679 | - got: But that explanation for me is not good enough
680 |
681 | - source: Es erklrt hauptschlich was mit Silicon Valley nicht stimmt
682 | - expected: It mostly explains what's wrong with Silicon Valley
683 | - got: It's what makes it with the same Valley
684 |
685 | - source: Wir knnten zum Mars fliegen wenn wir wollen
686 | - expected: We could go to Mars if we want
687 | - got: We could fly Mars if we want
688 |
689 | - source: NASA hat sogar schon einen Plan entworfen
690 | - expected: NASA even has the outline of a plan
691 | - got: NASA has even designed a plan
692 |
693 | - source: Schwierige Probleme sind schwierig
694 | - expected: Hard problems are hard
695 | - got: problems are hard
696 |
697 | - source: Es nicht das Jahr
698 | - expected: It is not
699 | - got: It wasn't the year
700 |
701 | - source: Er war nur drei Tage entfernt
702 | - expected: It was just three days away
703 | - got: He was only three days
704 |
705 | - source: Wei Gott es fehlt uns jedenfalls nicht an Herausforderungen
706 | - expected: God knows we don't lack for the challenges
707 | - got: Does God there will not be believe in challenges
708 |
709 | - source: Vielen Dank
710 | - expected: Thank you very much
711 | - got: Thank you
712 |
713 | - source: Ich mchte euch einladen eure Augen zu schlieen
714 | - expected: I'd like to invite you to close your eyes
715 | - got: I just want to close you to close
716 |
717 | - source: Du musst dir das wirklich vorstellen
718 | - expected: I need you to actually see this
719 | - got: You've got to introduce you to it
720 |
721 | - source: Das Licht scheint auf das Krmelmonster
722 | - expected: The light is shining down on Cookie Monster
723 | - got: And the light seems to be on the
724 |
725 | - source: Einem sprechenden Pferd
726 | - expected: It's a talking horse
727 | - got: CC horse
728 |
729 | - source: Jetzt folge mir in deine Kche
730 | - expected: And then follow me into your kitchen
731 | - got: Now just me in your kitchen
732 |
733 | - source: Okay Jetzt ffne die Augen
734 | - expected: Okay Open your eyes
735 | - got: Okay It's now supposed to look at the eyes
736 |
737 | - source: Er heit United States Memory Championship
738 | - expected: It's called the United States Memory Championship
739 | - got: It's called
740 |
741 | - source: Das ist verrckt
742 | - expected: I was like this is unbelievable
743 | - got: This is crazy
744 |
745 | - source: Diese Leute mssen bernatrliche Fhigkeiten haben
746 | - expected: These people must be freaks of nature
747 | - got: These people have to have their skills
748 |
749 | - source: Und Ed sagte Ich bin kein Inselbegabter
750 | - expected: And Ed was like I'm not a savant
751 | - got: And I said I can't get a
752 |
753 | - source: Ich habe nur ein durchschnittliches Gedchtnis
754 | - expected: In fact I have just an average memory
755 | - got: I have just a memory
756 |
757 | - source: Kennst du Britney Spears
758 | - expected: Do you know Britney Spears
759 | - got: Do you feel much
760 |
761 | - source: Ich Was Nee Wieso
762 | - expected: I'm like What No Why
763 | - got: I said What
764 |
765 | - source: Ich meine du musst ja irgendwo anfangen nicht wahr
766 | - expected: I mean you've got to start somewhere right
767 | - got: I mean you don't have to start right yes
768 |
769 | Bleu Score = 17.750355415- source: Ich habe eine Menge wirklich interessanter Leute getroffen
770 | - expected: I met a host of really interesting people
771 | - got: And I've met a lot of really interesting people
772 |
773 | - source: Das hier ist zum Beispiel EP
774 | - expected: This is a guy called EP
775 | - got: So this for example is
776 |
777 | - source: Kim Peek kennen
778 | - expected: This is Kim Peek
779 | - got: are know
780 |
781 | - source: Die Antwort war Nein
782 | - expected: The answer was no
783 | - got: The answer was no
784 |
785 | - source: Sind sie klger als wir
786 | - expected: Are they smarter than the rest of us
787 | - got: Are they a time than we can
788 |
789 | - source: Ja
790 | - expected: Yeah
791 | - got: Yes
792 |
793 | - source: Das ist sein Name
794 | - expected: That's his name
795 | - got: This is his name
796 |
797 | - source: Weit du noch welches Wort es war
798 | - expected: Do you remember what it was
799 | - got: Do you know what word was it
800 |
801 | - source: Gleiches Wort unterschiedliches Erinnerungsvermgen das ist seltsam
802 | - expected: Same word different amount of remembering that's weird
803 | - got: Now word is strange
804 |
805 | - source: Was geht hier vor sich
806 | - expected: What's going on here
807 | - got: What is going on
808 |
809 | - source: Aber der Beruf des Bckers wir kennen Bcker
810 | - expected: But the common noun baker we know bakers
811 | - got: But the job of the we know
812 |
813 | - source: Bcker tragen komische weie Mtzen
814 | - expected: Bakers wear funny white hats
815 | - got: funny white white
816 |
817 | - source: Bcker haben Mehl an ihren Hnden
818 | - expected: Bakers have flour on their hands
819 | - got: have with their hands
820 |
821 | - source: Vielleicht kennen wir sogar einen Bcker
822 | - expected: Maybe we even know a baker
823 | - got: Maybe we even know a
824 |
825 | - source: Sie wurde als der Gedchtnispalast bekannt
826 | - expected: It came to be known as the memory palace
827 | - got: It became known as
828 |
829 | - source: Die Krper knnen nicht richtig bestattet werden
830 | - expected: The bodies can't be properly buried
831 | - got: The body can't be right
832 |
833 | - source: Es ist eine Tragdie nach der anderen
834 | - expected: It's one tragedy compounding another
835 | - got: It's a tragedy to others
836 |
837 | - source: Wie funktioniert es also
838 | - expected: So how does this work
839 | - got: So how does it work
840 |
841 | - source: Aber es gab da ein Problem
842 | - expected: But there was a problem
843 | - got: But there was just this one problem
844 |
845 | - source: Vielleicht ein Gedicht oder
846 | - expected: Maybe it was a poem
847 | - got: Maybe a poem right
848 |
849 | - source: Ich musste feststellen dass es erstaunlich viel Spa macht
850 | - expected: And I found that this was shockingly fun
851 | - got: I tell you it's
852 |
853 | - source: Ich konnte mir das niemals vorstellen
854 | - expected: I never would have expected that
855 | - got: I could never imagine it
856 |
857 | - source: Ich vertiefte mich darin
858 | - expected: And I got pretty into it
859 | - got: I myself
860 |
861 | - source: Das sind nur Tricks
862 | - expected: These are just tricks
863 | - got: These are just tricks
864 |
865 | - source: Gute Gedchtnisse sind antrainiert
866 | - expected: Great memories are learned
867 | - got: Good news are good
868 |
869 | - source: Wir erinnern uns wenn wir engagiert sind
870 | - expected: We remember when we are deeply engaged
871 | - got: We remember we're dedicated and dedicated
872 |
873 | - source: Der Gedchtnispalast diese Gedchtnistechniken sie sind alle nur Abkrzungen
874 | - expected: The memory palace these memory techniques they're just shortcuts
875 | - got: The that they're all
876 |
877 | - source: Tatschlich sind es nicht mal wirkliche Abkrzungen
878 | - expected: In fact they're not even really shortcuts
879 | - got: It's not really real business
880 |
881 | - source: Sie funktionieren weil sie uns zwingen zu arbeiten
882 | - expected: They work because they make you work
883 | - got: They work because they works to us
884 |
885 | - source: Aber es sind keine Abkrzungen
886 | - expected: But there actually are no shortcuts
887 | - got: But it's not a very
888 |
889 | - source: So werden Dinge im Gedchtnis verankert
890 | - expected: This is how stuff is made memorable
891 | - got: It will happen in things
892 |
893 | - source: Danke
894 | - expected: Thank you
895 | - got: Thank you
896 |
897 | Bleu Score = 17.8227667174- source: Wie jeder Lehrer fhrte ich Tests und Prfungen durch
898 | - expected: And like any teacher I made quizzes and tests
899 | - got: How every teachers I and test
900 |
901 | - source: Ich gab Hausaufgaben auf
902 | - expected: I gave out homework assignments
903 | - got: I was
904 |
905 | - source: Als die Arbeiten zurckkamen berechnete ich Noten
906 | - expected: When the work came back I calculated grades
907 | - got: As the work I works on notes
908 |
909 | - source: Einige meiner schlauesten Kinder schnitten nicht besonders gut ab
910 | - expected: Some of my smartest kids weren't doing so well
911 | - got: Some of my kids not particularly good
912 |
913 | - source: Das lie mich nachdenken
914 | - expected: And that got me thinking
915 | - got: It would think
916 |
917 | - source: und wer das meiste Geld verdienen wrde
918 | - expected: And who's going to earn the most money
919 | - got: and who would make the money
920 |
921 | - source: Uns es war nicht soziale Intelligenz
922 | - expected: And it wasn't social intelligence
923 | - got: We weren't social intelligence with some intelligence
924 |
925 | - source: Es war Durchhaltevermgen
926 | - expected: It was grit
927 | - got: It was
928 |
929 | - source: Durchhaltevermgen ist Leidenschaft und Ausdauer fr sehr langfristige Ziele
930 | - expected: Grit is passion and perseverance for very longterm goals
931 | - got: is passion and are very goals
932 |
933 | - source: Durchhaltevermgen ist Stehvermgen
934 | - expected: Grit is having stamina
935 | - got: So an example of is
936 |
937 | - source: Die ehrliche Antwort Ich wei es nicht
938 | - expected: The honest answer is I don't know
939 | - got: The answer I don't know
940 |
941 | - source: Aber wir brauchen mehr
942 | - expected: But we need more
943 | - got: But we need more
944 |
945 | - source: Das ist die Arbeit die vor uns liegt
946 | - expected: That's the work that stands before us
947 | - got: And that's the work to us
948 |
949 | - source: Vielen Dank
950 | - expected: Thank you
951 | - got: Thank you
952 |
953 | - source: Ich berlegte auch Wie schlieen wir das Gebude
954 | - expected: I also thought how to close the building
955 | - got: I tried this How do we building
956 |
957 | - source: Es verstrkt auch das Konzept der Tore
958 | - expected: And it also reinforces this idea of the gates
959 | - got: It the concept of
960 |
961 | - source: Vielen Dank
962 | - expected: Thank you very much
963 | - got: Thank you
964 |
965 | - source: Davon gibt es meiner Ansicht nach drei
966 | - expected: And I think there are three
967 | - got: You have about my view three
968 |
969 | - source: Mir fehlt der berblick
970 | - expected: I don't have an overview
971 | - got: I was missing the whole lot of them
972 |
973 | - source: Aber ist das ein guter Beweis
974 | - expected: But is that good evidence
975 | - got: But that's a good thing
976 |
977 | - source: Meinungsumfragen erheben klar Meinungen
978 | - expected: What opinion polls record is of course opinions
979 | - got: For sure that's what opinions
980 |
981 | - source: Was knnten sie sonst erheben
982 | - expected: What else can they record
983 | - got: What else could they raise their hands
984 |
985 | - source: Vertrauen Sie Politikern Vertrauen Sie Lehrern
986 | - expected: Do you trust politicians Do you trust teachers
987 | - got: And you can trust teachers
988 |
989 | - source: Vertrauen Sie Fischhndlern
990 | - expected: Do you trust fishmongers
991 | - got: And you can trust it
992 |
993 | - source: Vertrauen Sie Grundschullehrern
994 | - expected: Do you trust elementary school teachers
995 | - got: And you can trust it
996 |
997 | - source: dann wrden Sie wohl fragen Inwiefern genau
998 | - expected: you would probably begin by saying To do what
999 | - got: So you would ask if you the
1000 |
1001 | - source: Und das wre eine vernnftige Reaktion
1002 | - expected: And that would be a perfectly sensible response
1003 | - got: And it would be a response
1004 |
1005 | - source: Vllig rational
1006 | - expected: That's a perfectly rational thing
1007 | - got: rational
1008 |
1009 | - source: Einfach
1010 | - expected: Simple
1011 | - got: Simple
1012 |
1013 | - source: Zweitens kommen wir zum Ziel
1014 | - expected: Secondly what about the aim
1015 | - got: Secondly we get to the goal
1016 |
1017 | - source: Das Ziel ist es mehr zu vertrauen
1018 | - expected: The aim is to have more trust
1019 | - got: The goal is to trust anymore
1020 |
1021 | - source: Ehrlich gesagt finde ich das Ziel dumm
1022 | - expected: Well frankly I think that's a stupid aim
1023 | - got: You know I find the goal
1024 |
1025 | Bleu Score = 16.5807640399- source: Darauf wrde ich nicht abzielen
1026 | - expected: It's not what I would aim at
1027 | - got: That's not what I'm looking for
1028 |
1029 | - source: Die Vertrauenswrdigkeit von Menschen einschtzen zu knnen
1030 | - expected: It's judging how trustworthy people are in particular respects
1031 | - got: The can from people to color
1032 |
1033 | - source: Sind sie kompetent Sind sie ehrlich Sind sie verlsslich
1034 | - expected: Are they competent Are they honest Are they reliable
1035 | - got: Are they Are they honest Are they
1036 |
1037 | - source: Falls doch habe ich es noch nicht erkannt
1038 | - expected: If so I haven't yet spotted it
1039 | - got: If yet I haven't got the change
1040 |
1041 | - source: Aber danach suchen wir erst Vertrauenswrdigkeit dann Vertrauen
1042 | - expected: But that's what we're looking for trustworthiness before trust
1043 | - got: But after that we're only looking for trust
1044 |
1045 | - source: Vertrauen ist die Reaktion
1046 | - expected: Trust is the response
1047 | - got: And it's And the reaction that he did
1048 |
1049 | - source: Viele dieser Systeme haben den gegenteiligen Effekt
1050 | - expected: A lot of these systems have the converse effect
1051 | - got: Many of these systems have the effect
1052 |
1053 | - source: Sie funktionieren nicht wie beabsichtigt
1054 | - expected: They don't work as they're supposed to
1055 | - got: They don't work like
1056 |
1057 | - source: Hierzu kann sicherlich jeder Beispiele beitragen
1058 | - expected: You can all give your own examples there
1059 | - got: can do everyone have a examples
1060 |
1061 | - source: So viel also zum Ziel
1062 | - expected: So so much for the aim
1063 | - got: So much for the goal
1064 |
1065 | - source: Und nun zur Aufgabe
1066 | - expected: Now thirdly the task
1067 | - got: And now for the task
1068 |
1069 | - source: Wir knnen das natrlich fr uns selbst tun
1070 | - expected: Well we can do that for ourselves
1071 | - got: We can do that for ourselves up to itself
1072 |
1073 | - source: Wir knnen ein bisschen Vertrauenswrdigkeit gewinnen
1074 | - expected: We can rebuild a bit of trustworthiness
1075 | - got: We can make a little win
1076 |
1077 | - source: Man muss also vertrauenswrdig sein
1078 | - expected: So you have to I think be trustworthy
1079 | - got: So you have to be
1080 |
1081 | - source: Wie geht das
1082 | - expected: How to do it
1083 | - got: How does this happen
1084 |
1085 | - source: Hier ist ein einfaches kommerzielles Beispiel
1086 | - expected: Let me give you a simple commercial example
1087 | - got: Here's a real simple example
1088 |
1089 | - source: Darin steckt eine wichtige Moral
1090 | - expected: I think there's a big lesson in that
1091 | - got: It makes an important work in morality
1092 |
1093 | - source: Danke
1094 | - expected: Thanks
1095 | - got: Thank you
1096 |
1097 | - source: Der Film hat sie monatelang nicht mehr losgelassen
1098 | - expected: It totally dominated her imagination for months
1099 | - got: He didn't follow it the film the story
1100 |
1101 | - source: Ihr Lieblingscharakter war natrlich Glinda
1102 | - expected: Her favorite character was Glinda of course
1103 | - got: Her was natural
1104 |
1105 | - source: Aber Der Zauberer von Oz war anders
1106 | - expected: But The Wizard of Oz stood alone
1107 | - got: But the magic was different
1108 |
1109 | - source: Er hat diesen Trend nicht gestartet
1110 | - expected: It did not start that trend
1111 | - got: He didn't stop that trend
1112 |
1113 | - source: Sie wissen wovon ich rede oder
1114 | - expected: Do you know what I'm talking about
1115 | - got: You know what I mean
1116 |
1117 | - source: Genau
1118 | - expected: Yeah
1119 | - got: Right
1120 |
1121 | - source: Aber so passiert es nicht
1122 | - expected: But that's not how it happens
1123 | - got: It doesn't mean that way
1124 |
1125 | - source: Zu jenem Zeitpunkt hatte ich ebenfalls einen Sohn
1126 | - expected: At that point I also had a son
1127 | - got: That's the time I had a son
1128 |
1129 | - source: Vergleichen Sie das mit Der Zauberer von Oz von
1130 | - expected: Compare this to with The Wizard of Oz
1131 | - got: the magic with of
1132 |
1133 | - source: Wie bringt Dorothy ihren Film zu einem guten Ende
1134 | - expected: How does Dorothy win her movie
1135 | - got: How does their movie not good to a
1136 |
1137 | - source: Warum stehst du da noch rum
1138 | - expected: Why are you still standing there
1139 | - got: Why are you there around there
1140 |
1141 | - source: Ich wei nicht was ich machen soll
1142 | - expected: I don't know what I'm supposed to do
1143 | - got: I don't know what should I do
1144 |
1145 | - source: Es gibt keine Vorbilder fr sie
1146 | - expected: There's no models for them
1147 | - got: There are no examples for them
1148 |
1149 | - source: Ich wei nicht ob Sie schon davon gehrt haben
1150 | - expected: I don't know if you've heard of this
1151 | - got: I don't know if you've heard about it already
1152 |
1153 | Bleu Score = 16.8497927571- source: Versuchen Sie einmal diese Voraussetzung zu erfllen
1154 | - expected: So try to meet that bar
1155 | - got: Try not to realize this value
1156 |
1157 | - source: Verstanden Dankeschn
1158 | - expected: Right Thank you
1159 | - got: Thank you
1160 |
1161 | - source: Vielen Dank
1162 | - expected: Thank you very much
1163 | - got: Thank you
1164 |
1165 | - source: Er fllt mit wehenden Fahnen durch den BechdelTest
1166 | - expected: It pretty much flunks the Bechdel test
1167 | - got: It falls with that by the
1168 |
1169 | - source: So stellen sich manche Hollywood vor
1170 | - expected: That's Hollywood for you
1171 | - got: That's how some Hollywood learning about
1172 |
1173 | - source: Elf Das ist nicht bel
1174 | - expected: Eleven It's not bad
1175 | - got: This is not nice
1176 |
1177 | - source: Sie besagt dass
1178 | - expected: Here's what it said
1179 | - got: It's saying that
1180 |
1181 | - source: Wer sind diese Typen Was lernen sie
1182 | - expected: Who are these guys What are they learning
1183 | - got: Who are these guys learning What are they learning
1184 |
1185 | - source: Was schaffen sie nicht zu lernen
1186 | - expected: What are they failing to learn
1187 | - got: What do they not learn to learn
1188 |
1189 | - source: Nehmen wir diese Geschichte so auf
1190 | - expected: Are we soaking up that story
1191 | - got: Well let's think of this story
1192 |
1193 | - source: Sie werden durcheinander gewirbelt
1194 | - expected: They're throwing it up in the air
1195 | - got: They're the way
1196 |
1197 | - source: ObiWan
1198 | - expected: ObiWan
1199 | - got:
1200 |
1201 | - source: ObiWan Kenobi und Glinda
1202 | - expected: ObiWan Kenobi and Glinda
1203 | - got: and
1204 |
1205 | - source: Was haben diese beiden gemeinsam
1206 | - expected: What do these two have in common
1207 | - got: What do these two of these
1208 |
1209 | - source: Vielleicht ist es nicht nur das glitzernde Kleid
1210 | - expected: Maybe it's not just the sparkly dress
1211 | - got: Maybe it's not just the and
1212 |
1213 | - source: Ich denke diese beiden Figuren sind Experten
1214 | - expected: I think these people are experts
1215 | - got: I think these two are the two experts
1216 |
1217 | - source: Auerdem sind sie Anfhrer
1218 | - expected: Now they are leaders
1219 | - got: And they're that leaders
1220 |
1221 | - source: Ich mchte mehr Aufgaben wie diese
1222 | - expected: I want more quests like that
1223 | - got: I want to share more like this
1224 |
1225 | - source: Vielen Dank
1226 | - expected: Thank you
1227 | - got: Thank you
1228 |
1229 | - source: Er lebt in einem kleinen Dorf im Sden Somalias
1230 | - expected: He lives in a small village in southern Somalia
1231 | - got: He lives in a little village in
1232 |
1233 | - source: Sein Dorf liegt in der Nhe von Mogadischu
1234 | - expected: His village is near Mogadishu
1235 | - got: His village is by the village of
1236 |
1237 | - source: Es vergeht vielleicht ein Jahr aber nichts
1238 | - expected: Maybe a year passes nothing
1239 | - got: It perhaps an year but nothing
1240 |
1241 | - source: Ihm wird diese junge Frau vorgestellt
1242 | - expected: He is introduced to this young woman
1243 | - got: He is going to introduced this young woman
1244 |
1245 | - source: Schlielich heiratet er
1246 | - expected: He eventually gets married
1247 | - got: He finally he was
1248 |
1249 | - source: Er beginnt ein neues Leben
1250 | - expected: He starts this new life
1251 | - got: He's starting to make a new life
1252 |
1253 | - source: Sein Leben hat einen Sinn
1254 | - expected: He has a purpose in life
1255 | - got: His life has the point
1256 |
1257 | - source: Er hatte gewartet
1258 | - expected: He was waiting
1259 | - got: He had a look
1260 |
1261 | - source: haben keine Arbeit gehen nicht zur Schule
1262 | - expected: percent don't work don't go to school
1263 | - got: They're not working for school
1264 |
1265 | - source: Sie machen eigentlich gar nichts
1266 | - expected: They pretty much do nothing
1267 | - got: They don't actually do anything
1268 |
1269 | - source: Wre ich Terrorist geworden
1270 | - expected: Would I have become a terrorist
1271 | - got: I have become
1272 |
1273 | - source: Ich bin nicht sicher was ich antworten soll
1274 | - expected: I'm not really sure about the answer
1275 | - got: I'm not sure what I'm going to answer
1276 |
1277 | - source: Ich brachte rund junge somalische Fhrungskrfte zusammen
1278 | - expected: I brought together about young Somali leaders
1279 | - got: I got about young young men together
1280 |
1281 | Bleu Score = 17.0130422572- source: Es gab keine Arbeit und keine Mglichkeiten
1282 | - expected: There were no jobs no opportunities
1283 | - got: There was no work and no possibilities
1284 |
1285 | - source: Sie suchten Leute wie ihn
1286 | - expected: They sought people like him out
1287 | - got: They were looking like people
1288 |
1289 | - source: Aber diese Geschichte nimmt einen anderen Lauf
1290 | - expected: But his story takes a different route
1291 | - got: But this story takes a different way
1292 |
1293 | - source: Aden erkannte die Gelegenheit und ergriff sie
1294 | - expected: Aden saw an opportunity and seized it
1295 | - got: realized they realized and opportunity it
1296 |
1297 | - source: Er erffnete eine Motorradgeschft
1298 | - expected: He started a motorbike company
1299 | - got: He a He's an
1300 |
1301 | - source: Wie unterscheidet sich diese Geschichte von der anderen
1302 | - expected: How is this story different
1303 | - got: How is it different from the other
1304 |
1305 | - source: Was macht den Unterschied aus
1306 | - expected: What makes his story different
1307 | - got: What makes a difference
1308 |
1309 | - source: Man kann jungen Leuten beibringen Unternehmer zu sein
1310 | - expected: And you can train young people to be entrepreneurs
1311 | - got: You can teach young people to teach entrepreneurs
1312 |
1313 | - source: Also sah Mohamed eine Gelegenheit
1314 | - expected: So Mohamed saw an opportunity
1315 | - got: So I saw an opportunity
1316 |
1317 | - source: Er grndete eine Landschaftsgrtnerei mit Blumenladen
1318 | - expected: He started a landscaping and design floral company
1319 | - got: He set up a with
1320 |
1321 | - source: Es gibt keine ffentlichen Parks in Mogadischu
1322 | - expected: There's no public park in Mogadishu
1323 | - got: There's no public service in public spaces
1324 |
1325 | - source: Sie hatten unternehmerischen Lsungen fr lokale Probleme
1326 | - expected: They came up with entrepreneurial solutions to local problems
1327 | - got: They had solutions for local problems
1328 |
1329 | - source: Es geht um die Erschaffung sozialer Effekte
1330 | - expected: It's about creating a social impact
1331 | - got: It's about the social media
1332 |
1333 | - source: Mohamed verkauft nicht einfach nur Blumen
1334 | - expected: Mohamed is not simply selling flowers
1335 | - got: just doesn't sell flowers
1336 |
1337 | - source: Ich glaube er verkauft Hoffnung
1338 | - expected: I believe he is selling hope
1339 | - got: And I think he sell hope
1340 |
1341 | - source: Vielen Dank
1342 | - expected: Thank you
1343 | - got: Thank you
1344 |
1345 | - source: Aber diese Muscheln sind schwer zu finden
1346 | - expected: But you know these shells they're hard to find
1347 | - got: But these are hard to find
1348 |
1349 | - source: Sie sind von Sand umhllt kaum zu sehen
1350 | - expected: They're covered in sand They're difficult to see
1351 | - got: They're very much from sand to see
1352 |
1353 | - source: Wissen zu teilen ist die grte aller Berufungen
1354 | - expected: Sharing knowledge is the greatest of all callings
1355 | - got: share space is completely space to most just
1356 |
1357 | - source: Es gibt nichts dergleichen im Lande
1358 | - expected: There's nothing like it in the land
1359 | - got: There is nothing in there were of that
1360 |
1361 | - source: Danke vielmals
1362 | - expected: Thank you very much
1363 | - got: Thank you very much
1364 |
1365 | - source: Danke
1366 | - expected: Thank you
1367 | - got: Thank you
1368 |
1369 | - source: Ein chinesischer Gelehrter versteht ca Schriftzeichen
1370 | - expected: A Chinese scholar would understand characters
1371 | - got: It's a understand that about
1372 |
1373 | - source: Sie brauchen nur Schriftzeichen fr ein einfaches Schriftverstndnis
1374 | - expected: You only need to understand the basic literacy
1375 | - got: They need a simple It's a simple time
1376 |
1377 | - source: Sind Sie bereit
1378 | - expected: You are ready
1379 | - got:
1380 |
1381 | - source: Daraus wird ein Mund
1382 | - expected: You get a mouth
1383 | - got: This is going to happen
1384 |
1385 | - source: Dieser Mensch geht spazieren
1386 | - expected: This is a person going for a walk
1387 | - got: This person goes on that for a walk
1388 |
1389 | - source: Mensch
1390 | - expected: Person
1391 | - got:
1392 |
1393 | - source: Das ist ein Baum
1394 | - expected: This is a tree
1395 | - got: So this is a tree
1396 |
1397 | - source: Baum
1398 | - expected: Tree
1399 | - got:
1400 |
1401 | - source: Das ist ein Berg
1402 | - expected: This is a mountain
1403 | - got: This is a mountain
1404 |
1405 | - source: Die Sonne
1406 | - expected: The sun
1407 | - got: The sun is
1408 |
1409 | Bleu Score = 16.7033642402
--------------------------------------------------------------------------------
/train.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | '''
4 | Janurary 2018 by Wei Li
5 | liweihfyz@sjtu.edu.cn
6 | https://www.github.cim/leviswind/transformer-pytorch
7 | '''
8 |
9 | from __future__ import print_function
10 |
11 | from hyperparams import Hyperparams as hp
12 | from data_load import get_batch_indices, load_de_vocab, load_en_vocab
13 |
14 | from torch.autograd import Variable
15 | import os
16 | from AttModel import AttModel
17 | import torch
18 | import torch.optim as optim
19 | from data_load import load_train_data
20 | import time
21 | import cPickle as pickle
22 | from tensorboardX import SummaryWriter
23 |
24 |
25 | def train():
26 | current_batches = 0
27 | de2idx, idx2de = load_de_vocab()
28 | en2idx, idx2en = load_en_vocab()
29 | enc_voc = len(de2idx)
30 | dec_voc = len(en2idx)
31 | writer = SummaryWriter()
32 | # Load data
33 | X, Y = load_train_data()
34 | # calc total batch count
35 | num_batch = len(X) // hp.batch_size
36 | model = AttModel(hp, enc_voc, dec_voc)
37 | model.train()
38 | model.cuda()
39 | torch.backends.cudnn.benchmark = True
40 | if not os.path.exists(hp.model_dir):
41 | os.makedirs(hp.model_dir)
42 | if hp.preload is not None and os.path.exists(hp.model_dir + '/history.pkl'):
43 | with open(hp.model_dir + '/history.pkl') as in_file:
44 | history = pickle.load(in_file)
45 | else:
46 | history = {'current_batches': 0}
47 | current_batches = history['current_batches']
48 | optimizer = optim.Adam(model.parameters(), lr=hp.lr, betas=[0.9, 0.98], eps=1e-8)
49 | if hp.preload is not None and os.path.exists(hp.model_dir + '/optimizer.pth'):
50 | optimizer.load_state_dict(torch.load(hp.model_dir + '/optimizer.pth'))
51 | if hp.preload is not None and os.path.exists(hp.model_dir + '/model_epoch_%02d.pth' % hp.preload):
52 | model.load_state_dict(torch.load(hp.model_dir + '/model_epoch_%02d.pth' % hp.preload))
53 |
54 | startepoch = int(hp.preload) if hp.preload is not None else 1
55 | for epoch in range(startepoch, hp.num_epochs + 1):
56 | current_batch = 0
57 | for index, current_index in get_batch_indices(len(X), hp.batch_size):
58 | tic = time.time()
59 | x_batch = Variable(torch.LongTensor(X[index]).cuda())
60 | y_batch = Variable(torch.LongTensor(Y[index]).cuda())
61 | toc = time.time()
62 | tic_r = time.time()
63 | torch.cuda.synchronize()
64 | optimizer.zero_grad()
65 | loss, _, acc = model(x_batch, y_batch)
66 | loss.backward()
67 | optimizer.step()
68 | torch.cuda.synchronize()
69 | toc_r = time.time()
70 | current_batches += 1
71 | current_batch += 1
72 | if current_batches % 10 == 0:
73 | writer.add_scalar('./loss', loss.data.cpu().numpy()[0], current_batches)
74 | writer.add_scalar('./acc', acc.data.cpu().numpy()[0], current_batches)
75 | if current_batches % 5 == 0:
76 | print('epoch %d, batch %d/%d, loss %f, acc %f' % (epoch, current_batch, num_batch, loss.data[0], acc.data[0]))
77 | print('batch loading used time %f, model forward used time %f' % (toc - tic, toc_r - tic_r))
78 | if current_batches % 100 == 0:
79 | writer.export_scalars_to_json(hp.model_dir + '/all_scalars.json')
80 | with open(hp.model_dir + '/history.pkl', 'w') as out_file:
81 | pickle.dump(history, out_file)
82 | checkpoint_path = hp.model_dir + '/model_epoch_%02d' % epoch + '.pth'
83 | torch.save(model.state_dict(), checkpoint_path)
84 | torch.save(optimizer.state_dict(), hp.model_dir + '/optimizer.pth')
85 |
86 |
87 | if __name__ == '__main__':
88 | train()
89 |
--------------------------------------------------------------------------------