├── README.md
├── RecursiveNN.py
├── SenTree.py
├── TreeLSTM.py
└── trees
    ├── dev.txt
    ├── small.txt
    ├── test.txt
    └── train.txt


/README.md:
--------------------------------------------------------------------------------
 1 | PyTorch implementation of <br>
 2 | 1) RNN: Recursive Neural Network from https://nlp.stanford.edu/~socherr/EMNLP2013_RNTN.pdf<br>
 3 |    python RecursiveNN.py<br>
 4 | 2) TreeLSTM from https://arxiv.org/abs/1503.00075 <br>
 5 |    python TreeLSTM.py<br>
 6 | 
 7 | add cuda as an argument to run it using cuda<br>
 8 | 
 9 | Requirements:<br>
10 | nltk <br>
11 | pytorch <br>
12 | progressbar <br>
13 | 
14 | Mixed code from:<br>
15 | Socher's cs224d class (see for e.g. https://github.com/kingtaurus/cs224d/tree/master/assignment3)<br>
16 | and https://gist.github.com/wolet/1b49c03968b2c83897a4a15c78980b18<br>
17 | 


--------------------------------------------------------------------------------
/RecursiveNN.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | import random
 3 | import progressbar
 4 | import torch
 5 | import torch.nn as nn
 6 | from torch.autograd import Variable
 7 | import torch.nn.functional as F
 8 | from torch.nn.utils import clip_grad_norm
 9 | from SenTree import *
10 | 
11 | class RecursiveNN(nn.Module):
12 |     def __init__(self, vocabSize, embedSize=100, numClasses=5):
13 |         super(RecursiveNN, self).__init__()
14 |         self.embedding = nn.Embedding(int(vocabSize), embedSize)
15 |         self.W = nn.Linear(2*embedSize, embedSize, bias=True)
16 |         self.projection = nn.Linear(embedSize, numClasses, bias=True)
17 |         self.activation = F.relu
18 |         self.nodeProbList = []
19 |         self.labelList = []
20 | 
21 |     def traverse(self, node):
22 |         if node.isLeaf(): currentNode = self.activation(self.embedding(Var(torch.LongTensor([node.getLeafWord()]))))
23 |         else: currentNode = self.activation(self.W(torch.cat((self.traverse(node.left()),self.traverse(node.right())),1)))
24 |         self.nodeProbList.append(self.projection(currentNode))
25 |         self.labelList.append(torch.LongTensor([node.label()]))
26 |         return currentNode
27 | 
28 |     def forward(self, x):
29 |         self.nodeProbList = []
30 |         self.labelList = []
31 |         self.traverse(x)
32 |         self.labelList = Var(torch.cat(self.labelList))
33 |         return torch.cat(self.nodeProbList)
34 | 
35 |     def getLoss(self, tree):
36 |         nodes = self.forward(tree)
37 |         predictions = nodes.max(dim=1)[1]
38 |         loss = F.cross_entropy(input=nodes, target=self.labelList)
39 |         return predictions,loss
40 | 
41 |     def evaluate(self, trees):
42 |         pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(trees)).start()
43 |         n = nAll = correctRoot = correctAll = 0.0
44 |         for j, tree in enumerate(trees):
45 |             predictions,loss = self.getLoss(tree)
46 |             correct = (predictions.data==self.labelList.data)
47 |             correctAll += correct.sum()
48 |             nAll += correct.squeeze().size()[0]
49 |             correctRoot += correct.squeeze()[-1]
50 |             n += 1
51 |             pbar.update(j)
52 |         pbar.finish()
53 |         return correctRoot / n, correctAll/nAll
54 | 
55 | def Var(v):
56 |     if CUDA: return Variable(v.cuda())
57 |     else: return Variable(v)
58 | 
59 | CUDA=False
60 | if len(sys.argv)>1:
61 |     if sys.argv[1].lower()=="cuda": CUDA=True
62 | 
63 | print("Reading and parsing trees")
64 | trn = SenTree.getTrees("./trees/train.txt","train.vocab")
65 | dev = SenTree.getTrees("./trees/dev.txt",vocabIndicesMapFile="train.vocab")
66 | 
67 | if CUDA: model = RecursiveNN(SenTree.vocabSize).cuda()
68 | else: model = RecursiveNN(SenTree.vocabSize)
69 | max_epochs = 100
70 | widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA()]
71 | 
72 | optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9, dampening=0.0)
73 | bestAll=bestRoot=0.0
74 | for epoch in range(max_epochs):
75 |   print("Epoch %d" % epoch)
76 |   pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(trn)).start()
77 |   for step, tree in enumerate(trn):
78 |     predictions, loss = model.getLoss(tree)
79 |     optimizer.zero_grad()
80 |     loss.backward()
81 |     clip_grad_norm(model.parameters(), 5, norm_type=2.)
82 |     optimizer.step()
83 |     pbar.update(step)
84 |   pbar.finish()
85 |   correctRoot, correctAll = model.evaluate(dev)
86 |   if bestAll<correctAll: bestAll=correctAll
87 |   if bestRoot<correctRoot: bestRoot=correctRoot
88 |   print("\nValidation All-nodes accuracy:"+str(round(correctAll,2))+"(best:"+str(round(bestAll,2))+")")
89 |   print("Validation Root accuracy:" + str(round(correctRoot,2))+"(best:"+str(round(bestRoot,2))+")")
90 |   random.shuffle(trn)


--------------------------------------------------------------------------------
/SenTree.py:
--------------------------------------------------------------------------------
 1 | from nltk.tree import ParentedTree
 2 | import _pickle as cPickle
 3 | 
 4 | class SenTree(ParentedTree):
 5 |     def __init__(self, node, children=None):
 6 |         super(SenTree,self).__init__(node, children)
 7 | 
 8 |     def left(self):
 9 |         return self[0]
10 | 
11 |     def right(self):
12 |         return self[1]
13 | 
14 |     def isLeaf(self):
15 |         return self.height()==2
16 | 
17 |     def getLeafWord(self):
18 |         return self[0]
19 | 
20 | 
21 |     @staticmethod
22 |     def getTrees(file='./trees/small.txt',vocabOutFile=None, vocabIndicesMapFile=None):
23 |          if vocabIndicesMapFile is None:
24 |             return SenTree.constructVocabAndGetTrees(file, vocabOutFile=vocabOutFile)
25 |          else:
26 |             return SenTree.getTreesGivenVocab(file, vocabIndicesMapFile)
27 | 
28 |     @staticmethod
29 |     def getTreesGivenVocab(file, vocabIndicesMapFile):
30 |         trees = []
31 |         vocabIndicesMap=cPickle.load(open(vocabIndicesMapFile,'rb'))
32 |         with open(file, "r") as f:
33 |             for line in f:
34 |                 tree = SenTree.fromstring(line)
35 |                 SenTree.mapTreeNodes(tree,vocabIndicesMap)
36 |                 SenTree.castLabelsToInt(tree)
37 |                 trees.append(tree)
38 |         SenTree.vocabSize=len(vocabIndicesMap)
39 |         return trees
40 | 
41 |     @staticmethod
42 |     def constructVocabAndGetTrees(file, vocabOutFile=None):
43 |         trees = []
44 |         vocab = set()
45 |         with open(file, "r") as f:
46 |             for line in f:
47 |                 tree = SenTree.fromstring(line)
48 |                 trees.append(tree)
49 |                 vocab.update(tree.leaves())
50 |         vocabIndicesMap = dict(zip(vocab,range(len(vocab))))
51 |         vocabIndicesMap['UNK'] = len(vocab)
52 |         if vocabOutFile is not None:
53 |             with open(vocabOutFile,'wb') as fp: cPickle.dump(vocabIndicesMap,fp)
54 |         for tree in trees:
55 |             SenTree.mapTreeNodes(tree,vocabIndicesMap)
56 |             SenTree.castLabelsToInt(tree)
57 |         SenTree.vocabSize=len(vocabIndicesMap)
58 |         return trees
59 | 
60 |     @staticmethod
61 |     def mapTreeNodes(tree, vocabIndicesMap):
62 |         for leafPos in tree.treepositions('leaves'):
63 |             if tree[leafPos] in vocabIndicesMap: tree[leafPos] = vocabIndicesMap[tree[leafPos]]
64 |             else: tree[leafPos]= vocabIndicesMap['UNK']
65 | 
66 |     @staticmethod
67 |     def castLabelsToInt(tree):
68 |         for subtree in tree.subtrees():
69 |             subtree.set_label(int(subtree.label()))
70 | 
71 | #trees = SenTree.getTrees()
72 | 
73 | 
74 | 
75 | 
76 | 
77 | 


--------------------------------------------------------------------------------
/TreeLSTM.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import random
  3 | import progressbar
  4 | import torch
  5 | import torch.nn as nn
  6 | from torch.autograd import Variable
  7 | import torch.nn.functional as F
  8 | from torch.nn.utils import clip_grad_norm
  9 | from SenTree import *
 10 | 
 11 | class TreeLSTM(nn.Module):
 12 |     def __init__(self, vocabSize, hdim=100, numClasses=5):
 13 |         super(TreeLSTM, self).__init__()
 14 |         self.embedding = nn.Embedding(int(vocabSize), hdim)
 15 |         self.Wi = nn.Linear(hdim, hdim, bias=True)
 16 |         self.Wo = nn.Linear(hdim, hdim, bias=True)
 17 |         self.Wu = nn.Linear(hdim, hdim, bias=True)
 18 |         self.Ui = nn.Linear(2 * hdim, hdim, bias=True)
 19 |         self.Uo = nn.Linear(2 * hdim, hdim, bias=True)
 20 |         self.Uu = nn.Linear(2 * hdim, hdim, bias=True)
 21 |         self.Uf1 = nn.Linear(hdim, hdim, bias=True)
 22 |         self.Uf2 = nn.Linear(hdim, hdim, bias=True)
 23 |         self.projection = nn.Linear(hdim, numClasses, bias=True)
 24 |         self.activation = F.relu
 25 |         self.nodeProbList = []
 26 |         self.labelList = []
 27 | 
 28 |     def traverse(self, node):
 29 |         if node.isLeaf():
 30 |             e = self.embedding(Var(torch.LongTensor([node.getLeafWord()])))
 31 |             i = F.sigmoid(self.Wi(e))
 32 |             o = F.sigmoid(self.Wo(e))
 33 |             u = self.activation(self.Wu(e))
 34 |             c = i * u
 35 |         else:
 36 |             leftH,leftC = self.traverse(node.left())
 37 |             rightH,rightC = self.traverse(node.right())
 38 |             e = torch.cat((leftH, rightH), 1)
 39 |             i = F.sigmoid(self.Ui(e))
 40 |             o = F.sigmoid(self.Uo(e))
 41 |             u = self.activation(self.Uu(e))
 42 |             c = i * u + F.sigmoid(self.Uf1(leftH)) * leftC + F.sigmoid(self.Uf2(rightH)) * rightC
 43 |         h = o * self.activation(c)
 44 |         self.nodeProbList.append(self.projection(h))
 45 |         self.labelList.append(torch.LongTensor([node.label()]))
 46 |         return h,c
 47 | 
 48 |     def forward(self, x):
 49 |         self.nodeProbList = []
 50 |         self.labelList = []
 51 |         self.traverse(x)
 52 |         self.labelList = Var(torch.cat(self.labelList))
 53 |         return torch.cat(self.nodeProbList)
 54 | 
 55 |     def getLoss(self, tree):
 56 |         nodes = self.forward(tree)
 57 |         predictions = nodes.max(dim=1)[1]
 58 |         loss = F.cross_entropy(input=nodes, target=self.labelList)
 59 |         return predictions,loss
 60 | 
 61 |     def evaluate(self, trees):
 62 |         pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(trees)).start()
 63 |         n = nAll = correctRoot = correctAll = 0.0
 64 |         for j, tree in enumerate(trees):
 65 |             predictions,loss = self.getLoss(tree)
 66 |             correct = (predictions.data==self.labelList.data)
 67 |             correctAll += correct.sum()
 68 |             nAll += correct.squeeze().size()[0]
 69 |             correctRoot += correct.squeeze()[-1]
 70 |             n += 1
 71 |             pbar.update(j)
 72 |         pbar.finish()
 73 |         return correctRoot / n, correctAll/nAll
 74 | 
 75 | def Var(v):
 76 |     if CUDA: return Variable(v.cuda())
 77 |     else: return Variable(v)
 78 | 
 79 | CUDA=False
 80 | if len(sys.argv)>1:
 81 |   if sys.argv[1].lower()=="cuda": CUDA=True
 82 | 
 83 | print("Reading and parsing trees")
 84 | trn = SenTree.getTrees("./trees/train.txt","train.vocab")
 85 | dev = SenTree.getTrees("./trees/dev.txt",vocabIndicesMapFile="train.vocab")
 86 | 
 87 | if CUDA: model = TreeLSTM(SenTree.vocabSize).cuda()
 88 | else: model = TreeLSTM(SenTree.vocabSize)
 89 | max_epochs = 100
 90 | widgets = [progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA()]
 91 | optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9, dampening=0.0)
 92 | bestAll=bestRoot=0.0
 93 | for epoch in range(max_epochs):
 94 |   print("Epoch %d" % epoch)
 95 |   pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(trn)).start()
 96 |   for step, tree in enumerate(trn):
 97 |      predictions, loss = model.getLoss(tree)
 98 |      optimizer.zero_grad()
 99 |      loss.backward()
100 |      clip_grad_norm(model.parameters(), 5, norm_type=2.)
101 |      optimizer.step()
102 |      pbar.update(step)
103 |   pbar.finish()
104 |   correctRoot, correctAll = model.evaluate(dev)
105 |   if bestAll<correctAll: bestAll=correctAll
106 |   if bestRoot<correctRoot: bestRoot=correctRoot
107 |   print("\nValidation All-nodes accuracy:"+str(correctAll)+"(best:"+str(bestAll)+")")
108 |   print("Validation Root accuracy:" + str(correctRoot)+"(best:"+str(bestRoot)+")")
109 |   random.shuffle(trn)
110 | 


--------------------------------------------------------------------------------
/trees/small.txt:
--------------------------------------------------------------------------------
 1 | (3 (2 It) (4 (4 (2 's) (4 (3 (2 a) (4 (3 lovely) (2 film))) (3 (2 with) (4 (3 (3 lovely) (2 performances)) (2 (2 by) (2 (2 (2 Buy) (2 and)) (2 Accorsi))))))) (2 .)))
 2 | (2 (2 (1 No) (2 one)) (1 (1 (2 goes) (2 (1 (2 (2 unindicted) (2 here)) (2 ,)) (2 (2 which) (3 (2 (2 is) (2 probably)) (3 (2 for) (4 (2 the) (4 best))))))) (2 .)))
 3 | (3 (2 And) (4 (3 (2 if) (1 (2 you) (1 (2 (2 (2 're) (1 not)) (2 nearly)) (4 (3 (3 moved) (2 (2 to) (1 tears))) (2 (2 by) (2 (2 (2 a) (2 couple)) (2 (2 of) (2 scenes)))))))) (2 (2 ,) (2 (2 you) (2 (2 (2 've) (1 (2 got) (2 (3 (2 ice) (2 water)) (2 (2 in) (2 (2 your) (2 veins)))))) (2 .))))))
 4 | (4 (4 (2 A) (4 (3 (3 warm) (2 ,)) (3 funny))) (3 (2 ,) (3 (4 (4 engaging) (2 film)) (2 .))))
 5 | (4 (3 (2 Uses) (3 (3 (4 (3 sharp) (4 (3 (4 humor) (2 and)) (2 insight))) (2 (2 into) (3 (2 human) (2 nature)))) (2 (2 to) (2 (2 examine) (2 (2 class) (1 conflict)))))) (2 (2 ,) (2 (2 (2 adolescent) (2 (2 (2 yearning) (2 ,)) (3 (2 (2 the) (2 roots)) (3 (2 of) (2 (2 friendship) (2 (2 and) (2 (2 sexual) (2 identity)))))))) (2 .))))
 6 | (2 (2 (2 Half) (1 (2 (2 (2 (2 (2 Submarine) (2 flick)) (2 ,)) (2 (2 Half) (2 (2 Ghost) (2 Story)))) (2 ,)) (2 (2 All) (2 (2 in) (2 (2 one) (2 criminally)))))) (1 (1 neglected) (2 film)))
 7 | (3 (3 (3 Entertains) (3 (2 by) (3 (2 providing) (3 (3 good) (3 (2 ,) (2 (3 lively) (2 company))))))) (2 .))
 8 | (4 (4 (4 Dazzles) (4 (2 with) (4 (3 (4 (4 (3 (3 (2 its) (3 (2 fully-written) (2 characters))) (2 ,)) (2 (2 its) (2 (2 determined) (3 stylishness)))) (2 (1 -LRB-) (3 (2 (2 which) (3 (2 always) (3 (2 relates) (2 (2 to) (2 (2 (2 characters) (2 and)) (2 story)))))) (3 -RRB-)))) (2 and)) (4 (3 (2 (2 Johnny) (2 (2 Dankworth) (2 's))) (4 (4 best) (2 soundtrack))) (2 (2 in) (2 years)))))) (2 .))
 9 | (4 (4 (3 (3 (3 (3 (3 Visually) (4 imaginative)) (2 ,)) (3 (3 thematically) (3 instructive))) (2 and)) (4 (2 thoroughly) (4 delightful))) (3 (2 ,) (3 (2 it) (4 (3 (4 (2 (2 takes) (3 us)) (3 (2 on) (3 (3 (2 a) (3 (2 roller-coaster) (2 ride))) (2 (2 from) (2 innocence))))) (2 (2 to) (2 (2 experience) (3 (2 without) (2 (2 (2 even) (3 (2 a) (3 hint))) (2 (2 of) (2 (2 that) (1 (1 typical) (1 (2 kiddie-flick) (2 sentimentality)))))))))) (2 .)))))
10 | (3 (3 (1 (2 (2 Nothing) (1 (2 's) (1 (2 at) (1 (2 (2 stake) (2 ,)) (1 (1 (2 just) (3 (2 a) (2 (2 twisty) (2 double-cross)))) (1 (2 you) (1 (2 can) (2 (2 (2 smell) (2 (2 a) (2 mile))) (2 away))))))))) (2 --)) (3 (2 still) (4 (2 ,) (3 (2 (2 the) (2 (3 derivative) (3 (2 Nine) (2 Queens)))) (3 (2 is) (3 (2 lots) (3 (2 of) (4 fun)))))))) (2 .))
11 | 


--------------------------------------------------------------------------------