├── NeuralQA
├── AnswerRerank
│ ├── demo.py
│ ├── rerank.py
│ └── util.py
├── EntityLinking
│ ├── change_format.py
│ ├── demo.py
│ ├── entity_linking.py
│ └── util.py
├── MentionDetection
│ ├── crf
│ │ ├── convert.py
│ │ ├── eval.py
│ │ ├── output.py
│ │ └── stanford-ner
│ │ │ └── NERDemo.java
│ └── nn
│ │ ├── args.py
│ │ ├── demo.py
│ │ ├── model.py
│ │ ├── test.py
│ │ ├── tmp_data.txt
│ │ ├── train.py
│ │ └── util
│ │ ├── datasets.py
│ │ └── util.py
└── RelationDetection
│ ├── nn
│ ├── args.py
│ ├── datasets.py
│ ├── demo.py
│ ├── model.py
│ ├── preprocess.py
│ ├── test.py
│ └── train.py
│ └── siamese
│ ├── args.py
│ ├── datasets.py
│ ├── eval.py
│ ├── model.py
│ ├── train.py
│ └── util.py
├── README.md
├── SiameseNetwork
├── config.py
├── qa_cnn.py
├── qa_lstm.py
├── qa_nn.py
├── train_cnn.py
├── train_lstm.py
├── train_nn.py
└── util
│ ├── dataset.py
│ └── util.py
├── WebQA
├── data
│ └── indexes
│ │ └── relation_sub_2M.pkl
├── src
│ ├── args.py
│ ├── main.py
│ ├── model.py
│ └── simpleQA.py
├── static
│ ├── css
│ │ └── bootstrap.min.css
│ ├── images
│ │ ├── favicon.ico
│ │ └── help.png
│ └── js
│ │ ├── bootstrap.js
│ │ ├── bootstrap.min.js
│ │ └── jquery-3.1.1.min.js
├── templates
│ ├── coming_soon.html
│ ├── entity.html
│ ├── homepage.html
│ ├── mention.html
│ ├── relation.html
│ └── test.html
└── util
│ ├── datasets.py
│ └── utils.py
└── demo.gif
/NeuralQA/AnswerRerank/demo.py:
--------------------------------------------------------------------------------
1 | import os
2 | import pickle
3 | from argparse import ArgumentParser
4 | from collections import defaultdict
5 |
6 | import math
7 |
8 | from util import clean_uri, www2fb, rdf2fb
9 |
10 |
11 | # Load up reachability graph
12 |
13 | def load_index(filename):
14 | # print("Loading index map from {}".format(filename))
15 | with open(filename, 'rb') as handler:
16 | index = pickle.load(handler)
17 | return index
18 |
19 |
20 | # Load predicted MIDs and relations for each question in valid/test set
21 | def get_mids(filename, hits):
22 | # print("Entity Source : {}".format(filename))
23 | id2mids = defaultdict(list)
24 | fin = open(filename)
25 | for line in fin.readlines():
26 | items = line.strip().split(' %%%% ')
27 | lineid = items[0]
28 | cand_mids = items[1:][:hits]
29 | for mid_entry in cand_mids:
30 | # TODO:WHY MID_TYPE ONLY ONE! TYPE IS FROM CFO NAME EITHER TYPE.TOPIC.NAME OR COMMON.TOPIC.ALIAS
31 | mid, mid_name, mid_type, score = mid_entry.split('\t')
32 | id2mids[lineid].append((mid, mid_name, mid_type, float(score)))
33 | return id2mids
34 |
35 |
36 | def get_rels(filename, hits):
37 | # print("Relation Source : {}".format(filename))
38 | id2rels = defaultdict(list)
39 | fin = open(filename)
40 | for line in fin.readlines():
41 | items = line.strip().split(' %%%% ')
42 | lineid = items[0].strip()
43 | rel = www2fb(items[1].strip())
44 | label = items[2].strip()
45 | score = items[3].strip()
46 | if len(id2rels[lineid]) < hits:
47 | id2rels[lineid].append((rel, label, float(score)))
48 | return id2rels
49 |
50 |
51 | def get_questions(filename):
52 | # print("getting questions ...")
53 | id2questions = {}
54 | id2goldmids = {}
55 | fin = open(filename)
56 | for line in fin.readlines():
57 | items = line.strip().split('\t')
58 | lineid = items[0].strip()
59 | mid = items[1].strip()
60 | question = items[5].strip()
61 | rel = items[3].strip()
62 | id2questions[lineid] = (question, rel)
63 | id2goldmids[lineid] = mid
64 | return id2questions, id2goldmids
65 |
66 |
67 | def get_mid2wiki(filename):
68 | # print("Loading Wiki")
69 | mid2wiki = defaultdict(bool)
70 | fin = open(filename)
71 | for line in fin.readlines():
72 | items = line.strip().split('\t')
73 | sub = rdf2fb(clean_uri(items[0]))
74 | mid2wiki[sub] = True
75 | return mid2wiki
76 |
77 |
78 | def evidence_integration(index_reach, index_degrees, mid2wiki, is_heuristics, input_entity, input_relation):
79 | id2answers = list()
80 | mids = input_entity.split("\n")
81 | rels = input_relation.split("\n")
82 | # print(rels)
83 |
84 | if is_heuristics:
85 | for item in mids:
86 | _, mid, mid_name, mid_type, mid_score = item.strip().split("\t")
87 | for item2 in rels:
88 | rel, rel_log_score = item2.strip().split("\t")
89 | # if this (mid, rel) exists in FB
90 | if rel in index_reach[mid]:
91 | rel_score = math.exp(float(rel_log_score))
92 | comb_score = (float(mid_score) ** 0.6) * (rel_score ** 0.1)
93 | id2answers.append((mid, rel, mid_name, mid_type, mid_score, rel_score, comb_score,
94 | int(mid2wiki[mid]), int(index_degrees[mid][0])))
95 | # I cannot use retrieved here because I use contain different name_type
96 | # if mid ==truth_mid and rel == truth_rel:
97 | # retrieved += 1
98 | id2answers.sort(key=lambda t: (t[6], t[3], t[7], t[8]), reverse=True)
99 | else:
100 | id2answers = [(mids[0][0], rels[0][0])]
101 |
102 | # write to file
103 | # TODO:CHANGED FOR SWITCH IS_HEURISTICS
104 | if is_heuristics:
105 | for answer in id2answers:
106 | mid, rel, mid_name, mid_type, mid_score, rel_score, comb_score, _, _ = answer
107 | print("{}\t{}\t{}\t{}\t{}".format(mid, rel, mid_name, mid_score, rel_score, comb_score))
108 | else:
109 | for answer in id2answers:
110 | mid, rel = answer
111 | print("{}\t{}".format(mid, rel))
112 | return id2answers
113 |
114 |
115 | if __name__ == "__main__":
116 | parser = ArgumentParser(description='Perform evidence integration')
117 | parser.add_argument('--ent_type', type=str, required=True, help="options are [crf|lstm|gru]")
118 | parser.add_argument('--rel_type', type=str, required=True, help="options are [lr|cnn|lstm|gru]")
119 | parser.add_argument('--index_reachpath', type=str, default="indexes/reachability_2M.pkl",
120 | help='path to the pickle for the reachability index')
121 | parser.add_argument('--index_degreespath', type=str, default="indexes/degrees_2M.pkl",
122 | help='path to the pickle for the index with the degree counts')
123 | parser.add_argument('--data_path', type=str, default="data/processed_simplequestions_dataset/test.txt")
124 | parser.add_argument('--ent_path', type=str, default="entity_linking/results/lstm/test-h100.txt",
125 | help='path to the entity linking results')
126 | parser.add_argument('--rel_path', type=str, default="relation_prediction/nn/results/cnn/test.txt",
127 | help='path to the relation prediction results')
128 | parser.add_argument('--wiki_path', type=str, default="data/fb2w.nt")
129 | parser.add_argument('--hits_ent', type=int, default=50,
130 | help='the hits here has to be <= the hits in entity linking')
131 | parser.add_argument('--hits_rel', type=int, default=5,
132 | help='the hits here has to be <= the hits in relation prediction retrieval')
133 | parser.add_argument('--no_heuristics', action='store_false', help='do not use heuristics', dest='heuristics')
134 | parser.add_argument('--output_dir', type=str, default="./results")
135 |
136 | # added for demo
137 | parser.add_argument('--input_ent_path', type=str, default='el_result.txt')
138 | parser.add_argument('--input_rel_path', type=str, default='rp_result.txt')
139 | args = parser.parse_args()
140 | # print(args)
141 |
142 | ent_type = args.ent_type.lower()
143 | rel_type = args.rel_type.lower()
144 | output_dir = os.path.join(args.output_dir, "{}-{}".format(ent_type, rel_type))
145 | os.makedirs(output_dir, exist_ok=True)
146 |
147 | index_reach = load_index(args.index_reachpath)
148 | # print(index_reach)
149 | index_degrees = load_index(args.index_degreespath)
150 | mid2wiki = get_mid2wiki(args.wiki_path)
151 |
152 | candidate_entity = open(args.input_ent_path).read().strip()
153 | candidate_relation = open(args.input_rel_path).read().strip()
154 | test_answers = evidence_integration(index_reach, index_degrees, mid2wiki, args.heuristics, candidate_entity,
155 | candidate_relation)
156 |
--------------------------------------------------------------------------------
/NeuralQA/AnswerRerank/rerank.py:
--------------------------------------------------------------------------------
1 | import os
2 | from argparse import ArgumentParser
3 | from collections import defaultdict
4 |
5 | import math
6 | from util import get_mid2wiki, get_rels, get_mids, get_questions, www2fb, load_index
7 |
8 |
9 | def answer_rerank(data_path, ent_path, rel_path, output_dir, index_reach, index_degrees, mid2wiki, is_heuristics,
10 | ent_hits, rel_hits):
11 | id2questions, id2goldmids = get_questions(data_path)
12 | id2mids = get_mids(ent_path, ent_hits)
13 | id2rels = get_rels(rel_path, rel_hits)
14 | file_base_name = os.path.basename(data_path)
15 | fout = open(os.path.join(output_dir, file_base_name), 'w')
16 |
17 | id2answers = defaultdict(list)
18 | found, notfound_both, notfound_mid, notfound_rel = 0, 0, 0, 0
19 | retrieved, retrieved_top1, retrieved_top2, retrieved_top3 = 0, 0, 0, 0
20 | lineids_found1 = []
21 | lineids_found2 = []
22 | lineids_found3 = []
23 |
24 | # for every lineid
25 | for line_id in id2goldmids:
26 | if line_id not in id2mids and line_id not in id2rels:
27 | notfound_both += 1
28 | continue
29 | elif line_id not in id2mids:
30 | notfound_mid += 1
31 | continue
32 | elif line_id not in id2rels:
33 | notfound_rel += 1
34 | continue
35 | found += 1
36 | question, truth_rel = id2questions[line_id]
37 | truth_rel = www2fb(truth_rel)
38 | truth_mid = id2goldmids[line_id]
39 | mids = id2mids[line_id]
40 | rels = id2rels[line_id]
41 |
42 | if is_heuristics:
43 | for (mid, mid_name, mid_type, mid_score) in mids:
44 | for (rel, rel_label, rel_log_score) in rels:
45 | # if this (mid, rel) exists in FB
46 | if rel in index_reach[mid]:
47 | rel_score = math.exp(float(rel_log_score))
48 | comb_score = (float(mid_score) ** 0.6) * (rel_score ** 0.1)
49 | id2answers[line_id].append((mid, rel, mid_name, mid_type, mid_score, rel_score, comb_score,
50 | int(mid2wiki[mid]), int(index_degrees[mid][0])))
51 | # I cannot use retrieved here because I use contain different name_type
52 | # if mid ==truth_mid and rel == truth_rel:
53 | # retrieved += 1
54 | id2answers[line_id].sort(key=lambda t: (t[6], t[3], t[7], t[8]), reverse=True)
55 | else:
56 | id2answers[line_id] = [(mids[0][0], rels[0][0])]
57 |
58 | # write to file
59 | fout.write("{}".format(line_id))
60 | if is_heuristics:
61 | for answer in id2answers[line_id]:
62 | mid, rel, mid_name, mid_type, mid_score, rel_score, comb_score, _, _ = answer
63 | fout.write(" %%%% {}\t{}\t{}\t{}\t{}".format(mid, rel, mid_name, mid_score, rel_score, comb_score))
64 | else:
65 | for answer in id2answers[line_id]:
66 | mid, rel = answer
67 | fout.write(" %%%% {}\t{}".format(mid, rel))
68 | fout.write('\n')
69 |
70 | if is_heuristics:
71 | if len(id2answers[line_id]) >= 1 and id2answers[line_id][0][1] == truth_rel: # id2answers[line_id][0][0] == truth_mid and
72 | retrieved_top1 += 1
73 | retrieved_top2 += 1
74 | retrieved_top3 += 1
75 | lineids_found1.append(line_id)
76 | elif len(id2answers[line_id]) >= 2 and id2answers[line_id][1][0] == truth_mid \
77 | and id2answers[line_id][1][1] == truth_rel:
78 | retrieved_top2 += 1
79 | retrieved_top3 += 1
80 | lineids_found2.append(line_id)
81 | elif len(id2answers[line_id]) >= 3 and id2answers[line_id][2][0] == truth_mid \
82 | and id2answers[line_id][2][1] == truth_rel:
83 | retrieved_top3 += 1
84 | lineids_found3.append(line_id)
85 | else:
86 | if len(id2answers[line_id]) >= 1 and id2answers[line_id][0][0] == truth_mid \
87 | and id2answers[line_id][0][1] == truth_rel:
88 | retrieved_top1 += 1
89 | retrieved_top2 += 1
90 | retrieved_top3 += 1
91 | lineids_found1.append(line_id)
92 | print()
93 | print("found: {}".format(found / len(id2goldmids) * 100.0))
94 | print("retrieved at top 1: {}".format(retrieved_top1 / len(id2goldmids) * 100.0))
95 | print("retrieved at top 2: {}".format(retrieved_top2 / len(id2goldmids) * 100.0))
96 | print("retrieved at top 3: {}".format(retrieved_top3 / len(id2goldmids) * 100.0))
97 | # print("retrieved at inf: {}".format(retrieved / len(id2goldmids) * 100.0))
98 | fout.close()
99 | return id2answers
100 |
101 |
102 | if __name__ == "__main__":
103 | parser = ArgumentParser(description='Perform evidence integration')
104 | parser.add_argument('--ent_type', type=str, required=True, help="options are [crf|lstm|gru]")
105 | parser.add_argument('--rel_type', type=str, required=True, help="options are [lr|cnn|lstm|gru]")
106 | parser.add_argument('--index_reachpath', type=str, default="../indexes/reachability_2M.pkl",
107 | help='path to the pickle for the reachability index')
108 | parser.add_argument('--index_degreespath', type=str, default="../indexes/degrees_2M.pkl",
109 | help='path to the pickle for the index with the degree counts')
110 | parser.add_argument('--data_path', type=str, default="../data/processed_simplequestions_dataset/test.txt")
111 | parser.add_argument('--ent_path', type=str, default="../entity_linking/results/crf/test-h100.txt",
112 | help='path to the entity linking results')
113 | parser.add_argument('--rel_path', type=str, default="../relation_prediction/nn/results/cnn/test.txt",
114 | help='path to the relation prediction results')
115 | parser.add_argument('--wiki_path', type=str, default="../data/fb2w.nt")
116 | parser.add_argument('--hits_ent', type=int, default=50,
117 | help='the hits here has to be <= the hits in entity linking')
118 | parser.add_argument('--hits_rel', type=int, default=5,
119 | help='the hits here has to be <= the hits in relation prediction retrieval')
120 | parser.add_argument('--no_heuristics', action='store_false', help='do not use heuristics', dest='heuristics')
121 | parser.add_argument('--output_dir', type=str, default="./results")
122 | args = parser.parse_args()
123 | print(args)
124 |
125 | ent_type = args.ent_type.lower()
126 | rel_type = args.rel_type.lower()
127 | # assert (ent_type == "crf" or ent_type == "lstm" or ent_type == "gru")
128 | # assert (rel_type == "lr" or rel_type == "cnn" or rel_type == "lstm" or rel_type == "gru")
129 | output_dir = os.path.join(args.output_dir, "{}-{}".format(ent_type, rel_type))
130 | os.makedirs(output_dir, exist_ok=True)
131 |
132 | index_reach = load_index(args.index_reachpath)
133 | index_degrees = load_index(args.index_degreespath)
134 | mid2wiki = get_mid2wiki(args.wiki_path)
135 |
136 | test_answers = answer_rerank(args.data_path, args.ent_path, args.rel_path, output_dir, index_reach,
137 | index_degrees, mid2wiki, args.heuristics, args.hits_ent, args.hits_rel)
138 |
--------------------------------------------------------------------------------
/NeuralQA/AnswerRerank/util.py:
--------------------------------------------------------------------------------
1 | import unicodedata
2 | from nltk.tokenize.treebank import TreebankWordTokenizer
3 |
4 | tokenizer = TreebankWordTokenizer()
5 |
6 |
7 | # Load up reachability graph
8 |
9 | def load_index(filename):
10 | print("Loading index map from {}".format(filename))
11 | with open(filename, 'rb') as handler:
12 | index = pickle.load(handler)
13 | return index
14 |
15 |
16 | # Load predicted MIDs and relations for each question in valid/test set
17 | def get_mids(filename, hits):
18 | print("Entity Source : {}".format(filename))
19 | id2mids = defaultdict(list)
20 | fin = open(filename)
21 | for line in fin.readlines():
22 | items = line.strip().split(' %%%% ')
23 | lineid = items[0]
24 | cand_mids = items[1:][:hits]
25 | for mid_entry in cand_mids:
26 | # TODO:WHY MID_TYPE ONLY ONE! TYPE IS FROM CFO NAME EITHER TYPE.TOPIC.NAME OR COMMON.TOPIC.ALIAS
27 | mid, mid_name, mid_type, score = mid_entry.split('\t')
28 | id2mids[lineid].append((mid, mid_name, mid_type, float(score)))
29 | return id2mids
30 |
31 |
32 | def get_rels(filename, hits):
33 | print("Relation Source : {}".format(filename))
34 | id2rels = defaultdict(list)
35 | fin = open(filename)
36 | for line in fin.readlines():
37 | items = line.strip().split(' %%%% ')
38 | lineid = items[0].strip()
39 | rel = www2fb(items[1].strip())
40 | label = items[2].strip()
41 | score = items[3].strip()
42 | if len(id2rels[lineid]) < hits:
43 | id2rels[lineid].append((rel, label, float(score)))
44 | return id2rels
45 |
46 |
47 | def get_questions(filename):
48 | print("getting questions ...")
49 | id2questions = {}
50 | id2goldmids = {}
51 | fin = open(filename)
52 | for line in fin.readlines():
53 | items = line.strip().split('\t')
54 | lineid = items[0].strip()
55 | mid = items[1].strip()
56 | question = items[5].strip()
57 | rel = items[3].strip()
58 | id2questions[lineid] = (question, rel)
59 | id2goldmids[lineid] = mid
60 | return id2questions, id2goldmids
61 |
62 |
63 | def get_mid2wiki(filename):
64 | print("Loading Wiki")
65 | mid2wiki = defaultdict(bool)
66 | fin = open(filename)
67 | for line in fin.readlines():
68 | items = line.strip().split('\t')
69 | sub = rdf2fb(clean_uri(items[0]))
70 | mid2wiki[sub] = True
71 | return mid2wiki
72 |
73 |
74 | def processed_text(text):
75 | text = text.replace('\\\\', '')
76 | # stripped = strip_accents(text.lower())
77 | stripped = text.lower()
78 | toks = tokenizer.tokenize(stripped)
79 | return " ".join(toks)
80 |
81 |
82 | def strip_accents(text):
83 | return ''.join(c for c in unicodedata.normalize('NFKD', text) if unicodedata.category(c) != 'Mn')
84 |
85 |
86 | def www2fb(in_str):
87 | if in_str.startswith("www.freebase.com"):
88 | in_str = 'fb:%s' % (in_str.split('www.freebase.com/')[-1].replace('/', '.'))
89 | if in_str == 'fb:m.07s9rl0':
90 | in_str = 'fb:m.02822'
91 | if in_str == 'fb:m.0bb56b6':
92 | in_str = 'fb:m.0dn0r'
93 | # Manual Correction
94 | if in_str == 'fb:m.01g81dw':
95 | in_str = 'fb:m.01g_bfh'
96 | if in_str == 'fb:m.0y7q89y':
97 | in_str = 'fb:m.0wrt1c5'
98 | if in_str == 'fb:m.0b0w7':
99 | in_str = 'fb:m.0fq0s89'
100 | if in_str == 'fb:m.09rmm6y':
101 | in_str = 'fb:m.03cnrcc'
102 | if in_str == 'fb:m.0crsn60':
103 | in_str = 'fb:m.02pnlqy'
104 | if in_str == 'fb:m.04t1f8y':
105 | in_str = 'fb:m.04t1fjr'
106 | if in_str == 'fb:m.027z990':
107 | in_str = 'fb:m.0ghdhcb'
108 | if in_str == 'fb:m.02xhc2v':
109 | in_str = 'fb:m.084sq'
110 | if in_str == 'fb:m.02z8b2h':
111 | in_str = 'fb:m.033vn1'
112 | if in_str == 'fb:m.0w43mcj':
113 | in_str = 'fb:m.0m0qffc'
114 | if in_str == 'fb:m.07rqy':
115 | in_str = 'fb:m.0py_0'
116 | if in_str == 'fb:m.0y9s5rm':
117 | in_str = 'fb:m.0ybxl2g'
118 | if in_str == 'fb:m.037ltr7':
119 | in_str = 'fb:m.0qjx99s'
120 | return in_str
121 |
122 |
123 | def clean_uri(uri):
124 | if uri.startswith("<") and uri.endswith(">"):
125 | return clean_uri(uri[1:-1])
126 | elif uri.startswith("\"") and uri.endswith("\""):
127 | return clean_uri(uri[1:-1])
128 | return uri
129 |
130 |
131 | def rdf2fb(in_str):
132 | if in_str.startswith('http://rdf.freebase.com/ns/'):
133 | return 'fb:%s' % (in_str.split('http://rdf.freebase.com/ns/')[-1])
134 |
--------------------------------------------------------------------------------
/NeuralQA/EntityLinking/change_format.py:
--------------------------------------------------------------------------------
1 | import argparse
2 |
3 | import FileUtil
4 |
5 | parser = argparse.ArgumentParser()
6 | parser.add_argument("--filename")
7 | args = parser.parse_args()
8 | filename = args.filename
9 |
10 | context = FileUtil.readFile(filename)
11 | output = []
12 | for i, c in enumerate(context):
13 | if i % 3 == 0:
14 | output.append("test-{} %%%% {}".format(int(i / 3 + 1), context[i + 1]))
15 | FileUtil.writeFile(output, filename + ".query")
16 | print("All done!")
17 |
--------------------------------------------------------------------------------
/NeuralQA/EntityLinking/demo.py:
--------------------------------------------------------------------------------
1 | import os
2 | import pickle
3 | from argparse import ArgumentParser
4 | from collections import defaultdict
5 |
6 | from fuzzywuzzy import fuzz
7 | from nltk.corpus import stopwords
8 | from tqdm import tqdm
9 |
10 | from util import www2fb
11 |
12 | inverted_index = defaultdict(list)
13 | stopword = set(stopwords.words('english'))
14 |
15 |
16 | def get_ngram(text):
17 | # ngram = set()
18 | ngram = []
19 | tokens = text.split()
20 | for i in range(len(tokens) + 1):
21 | for j in range(i):
22 | if i - j <= 3: # todo
23 | # ngram.add(" ".join(tokens[j:i]))
24 | temp = " ".join(tokens[j:i])
25 | if temp not in ngram:
26 | ngram.append(temp)
27 | # ngram = list(ngram)
28 | ngram = sorted(ngram, key=lambda x: len(x.split()), reverse=True)
29 | return ngram
30 |
31 |
32 | def get_stat_inverted_index(filename):
33 | """
34 | Get the number of entry and max length of the entry (How many mid in an entry)
35 | """
36 | with open(filename, "rb") as handler:
37 | global inverted_index
38 | inverted_index = pickle.load(handler)
39 | inverted_index = defaultdict(str, inverted_index)
40 | # print("Total type of text: {}".format(len(inverted_index)))
41 | max_len = 0
42 | _entry = ""
43 | for entry, value in inverted_index.items():
44 | if len(value) > max_len:
45 | max_len = len(value)
46 | _entry = entry
47 | # print("Max Length of entry is {}, text is {}".format(max_len, _entry))
48 |
49 |
50 | def entity_linking(pred_mention, top_num):
51 | C = []
52 | C_scored = []
53 | tokens = get_ngram(pred_mention)
54 |
55 | if len(tokens) > 0:
56 | maxlen = len(tokens[0].split())
57 | for item in tokens:
58 | if len(item.split()) < maxlen and len(C) == 0:
59 | maxlen = len(item.split())
60 | if len(item.split()) < maxlen and len(C) > 0:
61 | break
62 | if item in stopword:
63 | continue
64 | C.extend(inverted_index[item])
65 | # if len(C) > 0:
66 | # break
67 | for mid_text_type in sorted(set(C)):
68 | score = fuzz.ratio(mid_text_type[1], pred_mention) / 100.0
69 | # C_counts format : ((mid, text, type), score_based_on_fuzz)
70 | C_scored.append((mid_text_type, score))
71 |
72 | C_scored.sort(key=lambda t: t[1], reverse=True)
73 | cand_mids = C_scored[:top_num]
74 | for mid_text_type, score in cand_mids:
75 | print("{}\t{}\t{}\t{}\t{}".format(pred_mention, mid_text_type[0], mid_text_type[1], mid_text_type[2], score))
76 |
77 | if __name__ == "__main__":
78 | parser = ArgumentParser(description='Perform entity linking')
79 | parser.add_argument('--model_type', type=str, required=True, help="options are [crf|lstm|gru]")
80 | parser.add_argument('--index_ent', type=str, default="indexes/entity_2M.pkl",
81 | help='path to the pickle for the inverted entity index')
82 | parser.add_argument('--data_dir', type=str, default="data/processed_simplequestions_dataset")
83 | parser.add_argument('--query_dir', type=str, default="entity_detection/crf/query_text")
84 | parser.add_argument('--hits', type=int, default=10)
85 | parser.add_argument('--output_dir', type=str, default="./results")
86 |
87 | # added for demo
88 | # parser.add_argument('--input_mention', type=str, default='yao ming')
89 | parser.add_argument('--input_path', type=str, default='')
90 | args = parser.parse_args()
91 | # print(args)
92 |
93 | input_mentions = open(args.input_path).read().strip()
94 | # print("input_manetion:", input_mention)
95 |
96 | get_stat_inverted_index(args.index_ent)
97 | for mention in input_mentions.split():
98 | entity_linking(mention, args.hits)
99 |
--------------------------------------------------------------------------------
/NeuralQA/EntityLinking/entity_linking.py:
--------------------------------------------------------------------------------
1 | import os
2 | from argparse import ArgumentParser
3 | from collections import defaultdict
4 |
5 | from fuzzywuzzy import fuzz
6 | from nltk.corpus import stopwords
7 | from tqdm import tqdm
8 | from util import www2fb, get_ngram, get_stat_inverted_index
9 |
10 | stopword = set(stopwords.words('english'))
11 | inverted_index = defaultdict(list)
12 |
13 |
14 | def entity_linking(predicted_file, gold_file, hits, output):
15 | predicted = open(predicted_file)
16 | gold = open(gold_file)
17 | fout = open(output, 'w')
18 | total, top1, top3, top5, top10, top20, top50, top100 = 0, 0, 0, 0, 0, 0, 0, 0
19 | for idx, (line, gold_id) in tqdm(enumerate(zip(predicted.readlines(), gold.readlines()))):
20 | total += 1
21 | line = line.strip().split(" %%%% ")
22 | gold_id = gold_id.strip().split('\t')[1]
23 | cand_entity, cand_score = [], []
24 | line_id = line[0]
25 | if len(line) == 2:
26 | tokens = get_ngram(line[1])
27 | else:
28 | tokens = []
29 |
30 | if len(tokens) > 0:
31 | maxlen = len(tokens[0].split()) # 1, 2, 3
32 | # print(maxlen)
33 | for item in tokens: # todo
34 | if len(item.split()) < maxlen and len(cand_entity) == 0:
35 | maxlen = len(item.split())
36 | if len(item.split()) < maxlen and len(cand_entity) > 0:
37 | break
38 | if item in stopword:
39 | continue
40 | cand_entity.extend(inverted_index[item])
41 | print(item)
42 | print(inverted_index[item]) # all name/alias contain 'item'(string)
43 | # if len(cand_entity) > 0:
44 | # break
45 | print(cand_entity)
46 | for mid_text_type in sorted(set(cand_entity)):
47 | score = fuzz.ratio(mid_text_type[1], line[1]) / 100.0
48 | cand_score.append((mid_text_type, score))
49 |
50 | cand_score.sort(key=lambda t: t[1], reverse=True)
51 | cand_mids = cand_score[:hits]
52 | fout.write("{}".format(line_id))
53 | for mid_text_type, score in cand_mids:
54 | fout.write(" %%%% {}\t{}\t{}\t{}".format(mid_text_type[0], mid_text_type[1], mid_text_type[2], score))
55 | fout.write('\n')
56 | gold_id = www2fb(gold_id)
57 | mids_list = [x[0][0] for x in cand_mids]
58 | if gold_id in mids_list[:1]:
59 | top1 += 1
60 | if gold_id in mids_list[:3]:
61 | top3 += 1
62 | if gold_id in mids_list[:5]:
63 | top5 += 1
64 | if gold_id in mids_list[:10]:
65 | top10 += 1
66 | if gold_id in mids_list[:20]:
67 | top20 += 1
68 | if gold_id in mids_list[:50]:
69 | top50 += 1
70 | if gold_id in mids_list[:100]:
71 | top100 += 1
72 |
73 | print("total: {}".format(total))
74 | print("Top1 Entity Linking Accuracy: {}".format(top1 / total))
75 | print("Top3 Entity Linking Accuracy: {}".format(top3 / total))
76 | print("Top5 Entity Linking Accuracy: {}".format(top5 / total))
77 | print("Top10 Entity Linking Accuracy: {}".format(top10 / total))
78 | print("Top20 Entity Linking Accuracy: {}".format(top20 / total))
79 | print("Top50 Entity Linking Accuracy: {}".format(top50 / total))
80 | print("Top100 Entity Linking Accuracy: {}".format(top100 / total))
81 |
82 |
83 | if __name__ == "__main__":
84 | # print(get_ngram("Which team have LeBron played basketball ?"))
85 | parser = ArgumentParser(description='Perform entity linking')
86 | parser.add_argument('--model_type', type=str, required=True, help="options are [crf|lstm|gru]")
87 | parser.add_argument('--index_ent', type=str, default="../indexes/entity_2M.pkl",
88 | help='path to the pickle for the inverted entity index')
89 | parser.add_argument('--data_dir', type=str, default="../data/processed_simplequestions_dataset")
90 | parser.add_argument('--query_dir', type=str, default="../entity_detection/nn/query_text")
91 | parser.add_argument('--hits', type=int, default=100)
92 | parser.add_argument('--output_dir', type=str, default="./results")
93 | args = parser.parse_args()
94 | print(args)
95 |
96 | model_type = args.model_type.lower()
97 | # assert(model_type == "crf" or model_type == "lstm" or model_type == "gru")
98 | output_dir = os.path.join(args.output_dir, model_type)
99 | os.makedirs(output_dir, exist_ok=True)
100 |
101 | get_stat_inverted_index(args.index_ent)
102 | print("valid result:")
103 | entity_linking(
104 | os.path.join(args.query_dir, "query.valid"),
105 | os.path.join(args.data_dir, "valid.txt"),
106 | args.hits,
107 | os.path.join(output_dir, "valid-h{}.txt".format(args.hits)))
108 |
109 | print("test result:")
110 | entity_linking(
111 | os.path.join(args.query_dir, "query.test"),
112 | os.path.join(args.data_dir, "test.txt"),
113 | args.hits,
114 | os.path.join(output_dir, "test-h{}.txt".format(args.hits)))
115 |
--------------------------------------------------------------------------------
/NeuralQA/EntityLinking/util.py:
--------------------------------------------------------------------------------
1 | import unicodedata
2 | from nltk.tokenize.treebank import TreebankWordTokenizer
3 |
4 | tokenizer = TreebankWordTokenizer()
5 |
6 |
7 | def readFile(filename):
8 | context = open(filename).readlines()
9 | return [c.strip() for c in context]
10 |
11 |
12 | def writeFile(context, filename, append=False):
13 | if not append:
14 | with open(filename, 'w+') as fout:
15 | for co in context:
16 | fout.write(co + "\n")
17 | else:
18 | with open(filename, 'a+') as fout:
19 | for co in context:
20 | fout.write(co + "\n")
21 |
22 |
23 | def list2str(l, split=" "):
24 | a = ""
25 | for li in l:
26 | a += (str(li) + split)
27 | a = a[:-len(split)]
28 | return a
29 |
30 |
31 | def get_ngram(text):
32 | # ngram = set()
33 | ngram = []
34 | tokens = text.split()
35 | for i in range(len(tokens) + 1):
36 | for j in range(i):
37 | if i - j <= 3: # 3 ?
38 | # ngram.add(" ".join(tokens[j:i]))
39 | temp = " ".join(tokens[j:i])
40 | if temp not in ngram:
41 | ngram.append(temp)
42 | # ngram = list(ngram)
43 | ngram = sorted(ngram, key=lambda x: len(x.split()), reverse=True)
44 | return ngram
45 |
46 |
47 | def get_stat_inverted_index(filename):
48 | """
49 | Get the number of entry and max length of the entry (How many mid in an entry)
50 | """
51 | with open(filename, "rb") as handler:
52 | global inverted_index
53 | inverted_index = pickle.load(handler)
54 | inverted_index = defaultdict(str, inverted_index)
55 | print("Total type of text: {}".format(len(inverted_index)))
56 | max_len = 0
57 | _entry = ""
58 | for entry, value in inverted_index.items():
59 | if len(value) > max_len:
60 | max_len = len(value)
61 | _entry = entry
62 | print("Max Length of entry is {}, text is {}".format(max_len, _entry))
63 |
64 |
65 | def processed_text(text):
66 | text = text.replace('\\\\', '')
67 | # stripped = strip_accents(text.lower())
68 | stripped = text.lower()
69 | toks = tokenizer.tokenize(stripped)
70 | return " ".join(toks)
71 |
72 |
73 | def strip_accents(text):
74 | return ''.join(c for c in unicodedata.normalize('NFKD', text) if unicodedata.category(c) != 'Mn')
75 |
76 |
77 | def www2fb(in_str):
78 | if in_str.startswith("www.freebase.com"):
79 | in_str = 'fb:%s' % (in_str.split('www.freebase.com/')[-1].replace('/', '.'))
80 | if in_str == 'fb:m.07s9rl0':
81 | in_str = 'fb:m.02822'
82 | if in_str == 'fb:m.0bb56b6':
83 | in_str = 'fb:m.0dn0r'
84 | # Manual Correction
85 | if in_str == 'fb:m.01g81dw':
86 | in_str = 'fb:m.01g_bfh'
87 | if in_str == 'fb:m.0y7q89y':
88 | in_str = 'fb:m.0wrt1c5'
89 | if in_str == 'fb:m.0b0w7':
90 | in_str = 'fb:m.0fq0s89'
91 | if in_str == 'fb:m.09rmm6y':
92 | in_str = 'fb:m.03cnrcc'
93 | if in_str == 'fb:m.0crsn60':
94 | in_str = 'fb:m.02pnlqy'
95 | if in_str == 'fb:m.04t1f8y':
96 | in_str = 'fb:m.04t1fjr'
97 | if in_str == 'fb:m.027z990':
98 | in_str = 'fb:m.0ghdhcb'
99 | if in_str == 'fb:m.02xhc2v':
100 | in_str = 'fb:m.084sq'
101 | if in_str == 'fb:m.02z8b2h':
102 | in_str = 'fb:m.033vn1'
103 | if in_str == 'fb:m.0w43mcj':
104 | in_str = 'fb:m.0m0qffc'
105 | if in_str == 'fb:m.07rqy':
106 | in_str = 'fb:m.0py_0'
107 | if in_str == 'fb:m.0y9s5rm':
108 | in_str = 'fb:m.0ybxl2g'
109 | if in_str == 'fb:m.037ltr7':
110 | in_str = 'fb:m.0qjx99s'
111 | return in_str
112 |
113 |
114 | def clean_uri(uri):
115 | if uri.startswith("<") and uri.endswith(">"):
116 | return clean_uri(uri[1:-1])
117 | elif uri.startswith("\"") and uri.endswith("\""):
118 | return clean_uri(uri[1:-1])
119 | return uri
120 |
121 |
122 | def rdf2fb(in_str):
123 | if in_str.startswith('http://rdf.freebase.com/ns/'):
124 | return 'fb:%s' % (in_str.split('http://rdf.freebase.com/ns/')[-1])
125 |
--------------------------------------------------------------------------------
/NeuralQA/MentionDetection/crf/convert.py:
--------------------------------------------------------------------------------
1 | from argparse import ArgumentParser
2 |
3 |
4 | def convert(filename, output):
5 | fin = open(filename, 'r')
6 | fout = open(output, 'w')
7 | for line in fin.readlines():
8 | items = line.strip().split('\t')
9 | sent, label = items[5], items[6]
10 | for word, tag in zip(sent.strip().split(), label.strip().split()):
11 | fout.write("{}\t{}\n".format(word, tag))
12 | fout.write("\n")
13 | fout.close()
14 |
15 |
16 | if __name__ == '__main__':
17 | parser = ArgumentParser(description='Convert dataset to stanford format for training')
18 | parser.add_argument('--data_dir', type=str, default="../../../data/processed_simplequestions_dataset/train.txt")
19 | parser.add_argument('--save_path', type=str, default="data/stanford.train")
20 | args = parser.parse_args()
21 | convert(args.data_dir, args.save_path)
22 |
--------------------------------------------------------------------------------
/NeuralQA/MentionDetection/crf/eval.py:
--------------------------------------------------------------------------------
1 | import sys
2 |
3 |
4 | def get_span(label):
5 | start, end = 0, 0
6 | flag = False
7 | span = []
8 | for k, l in enumerate(label):
9 | if l == 'I' and not flag:
10 | start = k
11 | flag = True
12 | if l != 'I' and flag:
13 | flag = False
14 | en = k
15 | span.append((start, en))
16 | start, end = 0, 0
17 | if start != 0 and end == 0:
18 | end = len(label) + 1 # bug fixed: geoff
19 | span.append((start, end))
20 | return span
21 |
22 |
23 | def evaluation(filename):
24 | fin = open(filename, 'r')
25 | pred = []
26 | gold = []
27 | right = 0
28 | predicted = 0
29 | total_en = 0
30 | for line in fin.readlines():
31 | if line == '\n':
32 | gold_span = get_span(gold)
33 | pred_span = get_span(pred)
34 | total_en += len(gold_span)
35 | predicted += len(pred_span)
36 | for item in pred_span:
37 | if item in gold_span:
38 | right += 1
39 | gold = []
40 | pred = []
41 | else:
42 | word, gold_label, pred_label = line.strip().split()
43 | gold.append(gold_label)
44 | pred.append(pred_label)
45 |
46 | if gold != [] or pred != []:
47 | gold_span = get_span(gold)
48 | pred_span = get_span(pred)
49 | total_en += len(gold_span)
50 | predicted += len(pred_span)
51 | for item in pred_span:
52 | if item in gold_span:
53 | right += 1
54 |
55 | if predicted == 0:
56 | precision = 0
57 | else:
58 | precision = right / predicted
59 | if total_en == 0:
60 | recall = 0
61 | else:
62 | recall = right / total_en
63 | if precision + recall == 0:
64 | f1 = 0
65 | else:
66 | f1 = 2 * precision * recall / (precision + recall)
67 | print("Precision", precision, "Recall", recall, "F1", f1, "right", right, "predicted", predicted, "total", total_en)
68 |
69 |
70 | if __name__ == '__main__':
71 | if len(sys.argv) != 2:
72 | print("Need to specify the file")
73 | filename = sys.argv[1]
74 | evaluation(filename)
75 |
--------------------------------------------------------------------------------
/NeuralQA/MentionDetection/crf/output.py:
--------------------------------------------------------------------------------
1 | from argparse import ArgumentParser
2 |
3 |
4 | def convert(fileName, idFile, outputFile):
5 | fin = open(fileName)
6 | fid = open(idFile)
7 | fout = open(outputFile, "w")
8 | word_list = []
9 | pred_query = []
10 | line_id = []
11 | for line in fid.readlines():
12 | line_id.append(line.strip())
13 | index = 0
14 | for line in fin.readlines():
15 | if line == '\n':
16 | if len(pred_query) == 0:
17 | pred_query = word_list
18 | fout.write("{} %%%% {}\n".format(line_id[index], " ".join(pred_query)))
19 | index += 1
20 | pred_query = []
21 | word_list = []
22 | else:
23 | word, gold_label, pred_label = line.strip().split()
24 | word_list.append(word)
25 | if pred_label == 'I':
26 | pred_query.append(word)
27 | if (index != len(line_id)):
28 | print("Length Error")
29 |
30 |
31 | if __name__ == "__main__":
32 | parser = ArgumentParser(description='Convert result to query text')
33 | parser.add_argument('--data_dir', type=str, default="stanford-ner/data/stanford.predicted.valid")
34 | parser.add_argument('--valid_line', type=str,
35 | default="../../data/processed_simplequestions_dataset/lineids_valid.txt")
36 | parser.add_argument('--results_path', type=str, default="query_text/query.valid")
37 | args = parser.parse_args()
38 | convert(args.data_dir, args.valid_line, args.results_path)
39 |
--------------------------------------------------------------------------------
/NeuralQA/MentionDetection/crf/stanford-ner/NERDemo.java:
--------------------------------------------------------------------------------
1 | import edu.stanford.nlp.ie.AbstractSequenceClassifier;
2 | import edu.stanford.nlp.ie.crf.*;
3 | import edu.stanford.nlp.io.IOUtils;
4 | import edu.stanford.nlp.ling.CoreLabel;
5 | import edu.stanford.nlp.ling.CoreAnnotations;
6 | import edu.stanford.nlp.sequences.DocumentReaderAndWriter;
7 | import edu.stanford.nlp.util.Triple;
8 |
9 | import java.util.List;
10 |
11 |
12 | /** This is a demo of calling CRFClassifier programmatically.
13 | *
14 | * Usage: {@code java -mx400m -cp "*" NERDemo [serializedClassifier [fileName]] }
15 | *
16 | * If arguments aren't specified, they default to
17 | * classifiers/english.all.3class.distsim.crf.ser.gz and some hardcoded sample text.
18 | * If run with arguments, it shows some of the ways to get k-best labelings and
19 | * probabilities out with CRFClassifier. If run without arguments, it shows some of
20 | * the alternative output formats that you can get.
21 | *
22 | * To use CRFClassifier from the command line:
23 | *
24 | * {@code java -mx400m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier [classifier] -textFile [file] }
25 | *
26 | * Or if the file is already tokenized and one word per line, perhaps in
27 | * a tab-separated value format with extra columns for part-of-speech tag,
28 | * etc., use the version below (note the 's' instead of the 'x'):
29 | *
30 | * {@code java -mx400m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier [classifier] -testFile [file] }
31 | *
32 | *
33 | * @author Jenny Finkel
34 | * @author Christopher Manning
35 | */
36 |
37 | public class NERDemo {
38 |
39 | public static void main(String[] args) throws Exception {
40 |
41 | String serializedClassifier = "classifiers/english.all.3class.distsim.crf.ser.gz";
42 |
43 | if (args.length > 0) {
44 | serializedClassifier = args[0];
45 | }
46 |
47 | AbstractSequenceClassifier classifier = CRFClassifier.getClassifier(serializedClassifier);
48 |
49 | /* For either a file to annotate or for the hardcoded text example, this
50 | demo file shows several ways to process the input, for teaching purposes.
51 | */
52 |
53 | if (args.length > 1) {
54 |
55 | /* For the file, it shows (1) how to run NER on a String, (2) how
56 | to get the entities in the String with character offsets, and
57 | (3) how to run NER on a whole file (without loading it into a String).
58 | */
59 |
60 | String fileContents = IOUtils.slurpFile(args[1]);
61 | List> out = classifier.classify(fileContents);
62 | for (List sentence : out) {
63 | for (CoreLabel word : sentence) {
64 | System.out.print(word.word() + '/' + word.get(CoreAnnotations.AnswerAnnotation.class) + ' ');
65 | }
66 | System.out.println();
67 | }
68 |
69 | System.out.println("---");
70 | out = classifier.classifyFile(args[1]);
71 | for (List sentence : out) {
72 | for (CoreLabel word : sentence) {
73 | System.out.print(word.word() + '/' + word.get(CoreAnnotations.AnswerAnnotation.class) + ' ');
74 | }
75 | System.out.println();
76 | }
77 |
78 | System.out.println("---");
79 | List> list = classifier.classifyToCharacterOffsets(fileContents);
80 | for (Triple item : list) {
81 | System.out.println(item.first() + ": " + fileContents.substring(item.second(), item.third()));
82 | }
83 | System.out.println("---");
84 | System.out.println("Ten best entity labelings");
85 | DocumentReaderAndWriter readerAndWriter = classifier.makePlainTextReaderAndWriter();
86 | classifier.classifyAndWriteAnswersKBest(args[1], 10, readerAndWriter);
87 |
88 | System.out.println("---");
89 | System.out.println("Per-token marginalized probabilities");
90 | classifier.printProbs(args[1], readerAndWriter);
91 |
92 | // -- This code prints out the first order (token pair) clique probabilities.
93 | // -- But that output is a bit overwhelming, so we leave it commented out by default.
94 | // System.out.println("---");
95 | // System.out.println("First Order Clique Probabilities");
96 | // ((CRFClassifier) classifier).printFirstOrderProbs(args[1], readerAndWriter);
97 |
98 | } else {
99 |
100 | /* For the hard-coded String, it shows how to run it on a single
101 | sentence, and how to do this and produce several formats, including
102 | slash tags and an inline XML output format. It also shows the full
103 | contents of the {@code CoreLabel}s that are constructed by the
104 | classifier. And it shows getting out the probabilities of different
105 | assignments and an n-best list of classifications with probabilities.
106 | */
107 |
108 | String[] example = {"Good afternoon Rajat Raina, how are you today?",
109 | "I go to school at Stanford University, which is located in California." };
110 | for (String str : example) {
111 | System.out.println(classifier.classifyToString(str));
112 | }
113 | System.out.println("---");
114 |
115 | for (String str : example) {
116 | // This one puts in spaces and newlines between tokens, so just print not println.
117 | System.out.print(classifier.classifyToString(str, "slashTags", false));
118 | }
119 | System.out.println("---");
120 |
121 | for (String str : example) {
122 | // This one is best for dealing with the output as a TSV (tab-separated column) file.
123 | // The first column gives entities, the second their classes, and the third the remaining text in a document
124 | System.out.print(classifier.classifyToString(str, "tabbedEntities", false));
125 | }
126 | System.out.println("---");
127 |
128 | for (String str : example) {
129 | System.out.println(classifier.classifyWithInlineXML(str));
130 | }
131 | System.out.println("---");
132 |
133 | for (String str : example) {
134 | System.out.println(classifier.classifyToString(str, "xml", true));
135 | }
136 | System.out.println("---");
137 |
138 | for (String str : example) {
139 | System.out.print(classifier.classifyToString(str, "tsv", false));
140 | }
141 | System.out.println("---");
142 |
143 | // This gets out entities with character offsets
144 | int j = 0;
145 | for (String str : example) {
146 | j++;
147 | List> triples = classifier.classifyToCharacterOffsets(str);
148 | for (Triple trip : triples) {
149 | System.out.printf("%s over character offsets [%d, %d) in sentence %d.%n",
150 | trip.first(), trip.second(), trip.third, j);
151 | }
152 | }
153 | System.out.println("---");
154 |
155 | // This prints out all the details of what is stored for each token
156 | int i=0;
157 | for (String str : example) {
158 | for (List lcl : classifier.classify(str)) {
159 | for (CoreLabel cl : lcl) {
160 | System.out.print(i++ + ": ");
161 | System.out.println(cl.toShorterString());
162 | }
163 | }
164 | }
165 |
166 | System.out.println("---");
167 |
168 | }
169 | }
170 |
171 | }
172 |
--------------------------------------------------------------------------------
/NeuralQA/MentionDetection/nn/args.py:
--------------------------------------------------------------------------------
1 | from argparse import ArgumentParser
2 |
3 | def get_args():
4 | parser = ArgumentParser(description="Joint Prediction")
5 | parser.add_argument('--mention_detection_mode', type=str, required=True, help='options are LSTM, GRU')
6 | parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda')
7 | parser.add_argument('--gpu', type=int, default=-1) # Use -1 for CPU
8 | parser.add_argument('--epochs', type=int, default=30)
9 | parser.add_argument('--batch_size', type=int, default=32)
10 | parser.add_argument('--dataset', type=str, default="EntityDetection")
11 | parser.add_argument('--lr', type=float, default=1e-4)
12 | parser.add_argument('--seed', type=int, default=3435)
13 | parser.add_argument('--dev_every', type=int, default=2000)
14 | parser.add_argument('--log_every', type=int, default=1000)
15 | parser.add_argument('--patience', type=int, default=10)
16 | parser.add_argument('--save_path', type=str, default='saved_checkpoints')
17 | parser.add_argument('--specify_prefix', type=str, default='id1')
18 | parser.add_argument('--words_dim', type=int, default=300)
19 | parser.add_argument('--num_layer', type=int, default=2)
20 | parser.add_argument('--rnn_fc_dropout', type=float, default=0.3)
21 | parser.add_argument('--input_size', type=int, default=300)
22 | parser.add_argument('--hidden_size', type=int, default=300)
23 | parser.add_argument('--rnn_dropout', type=float, default=0.3)
24 | parser.add_argument('--clip_gradient', type=float, default=0.6, help='gradient clipping')
25 | parser.add_argument('--vector_cache', type=str, default="../../data/sq_glove300d.pt")
26 | parser.add_argument('--weight_decay',type=float, default=0)
27 | parser.add_argument('--fix_embed', action='store_false', dest='train_embed')
28 | parser.add_argument('--hits', type=int, default=100)
29 | # added for testing
30 | parser.add_argument('--trained_model', type=str, default='')
31 | parser.add_argument('--data_dir', type=str, default='../../data/processed_simplequestions_dataset')
32 | parser.add_argument('--results_path', type=str, default='query_text')
33 | # added for demo
34 | parser.add_argument('--input', type=str, default='Which teams have James played basketball?')
35 | args = parser.parse_args()
36 | return args
37 |
--------------------------------------------------------------------------------
/NeuralQA/MentionDetection/nn/demo.py:
--------------------------------------------------------------------------------
1 | import os
2 | import random
3 |
4 | import numpy as np
5 | import torch
6 | from args import get_args
7 | from datasets import SimpleQuestionDataset
8 | from torchtext import data
9 |
10 | os.environ["CUDA_VISIBLE_DEVICES"] = "3"
11 |
12 | np.set_printoptions(threshold=np.nan)
13 | # Set default configuration in : args.py
14 | args = get_args()
15 |
16 | # Set random seed for reproducibility
17 | torch.manual_seed(args.seed)
18 | np.random.seed(args.seed)
19 | random.seed(args.seed)
20 |
21 | print(args)
22 | if not args.cuda:
23 | args.gpu = -1
24 | if torch.cuda.is_available() and args.cuda:
25 | # print("Note: You are using GPU for training")
26 | torch.cuda.set_device(args.gpu)
27 | torch.cuda.manual_seed(args.seed)
28 | if torch.cuda.is_available() and not args.cuda:
29 | print("Warning: You have Cuda but not use it. You are using CPU for training.")
30 |
31 | TEXT = data.Field(lower=True)
32 | ED = data.Field()
33 |
34 | train, dev, test = SimpleQuestionDataset.splits(TEXT, ED, path=args.data_dir) # text_field, label_field
35 | TEXT.build_vocab(train, dev, test)
36 | ED.build_vocab(train, dev, test)
37 |
38 | # load the model
39 | model = torch.load(args.trained_model, map_location=lambda storage, location: storage.cuda(args.gpu))
40 |
41 | # print(model)
42 |
43 | if args.dataset == 'EntityDetection':
44 | index2tag = np.array(ED.vocab.itos)
45 | # print(index2tag)
46 | else:
47 | print("Wrong Dataset")
48 | exit(1)
49 |
50 | index2word = np.array(TEXT.vocab.itos)
51 |
52 | results_path = os.path.join(args.results_path, args.entity_detection_mode.lower())
53 | if not os.path.exists(results_path):
54 | os.makedirs(results_path, exist_ok=True)
55 |
56 | sentence = "who wrote the film Brave heart" # args.input which genre of album is harder ... faster
57 | # sentence = args.input
58 | # print(sentence.split())
59 | # print(index2word[:10]) # '', ''
60 |
61 | sent_idx = list()
62 | for word in sentence.split():
63 | if word.lower() in index2word:
64 | sent_idx.append(index2word.tolist().index(word.lower()))
65 | else:
66 | sent_idx.append(0) # 1?
67 |
68 |
69 | # print(sent_idx)
70 |
71 |
72 | def predict(data_name="demo"):
73 | model.eval()
74 |
75 | data_batch = np.array(sent_idx).reshape(-1, 1)
76 | scores = model(data_batch)
77 | # print(scores)
78 |
79 | if args.dataset == 'EntityDetection':
80 | # print(torch.max(scores, 1)[1])
81 | index_tag = np.transpose(torch.max(scores, 1)[1].cpu().data.numpy())
82 | tag_array = index2tag[index_tag]
83 | index_question = np.transpose(data_batch).reshape(-1)
84 | question_array = index2word[index_question]
85 |
86 | mentions = list()
87 | mention = ""
88 | for question, label in zip(question_array, tag_array):
89 | # print("{}\t{}\t".format("".join(question), " ".join(label)))
90 | if label == 'I' and question != "" and question != "":
91 | mention += question + " "
92 | if label == 'O' and mention != "":
93 | mentions.append(mention.strip())
94 | mention = ""
95 | if mention != "":
96 | mentions.append(mention.strip())
97 | for mention in mentions:
98 | print(mention)
99 | else:
100 | print("Wrong Dataset")
101 | exit()
102 |
103 |
104 | # run the model on the demo set and write the output to a file
105 | predict(data_name="demo")
106 |
--------------------------------------------------------------------------------
/NeuralQA/MentionDetection/nn/model.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn.functional as F
3 | from torch import nn
4 | from torch.autograd import Variable
5 |
6 |
7 | class MentionDetection(nn.Module):
8 | def __init__(self, config):
9 | super(MentionDetection, self).__init__()
10 | self.config = config
11 | self.embed = nn.Embedding(config.words_num, config.words_dim)
12 | if config.train_embed == False:
13 | self.embed.weight.requires_grad = False
14 | if config.mention_detection_mode.upper() == 'LSTM':
15 | self.lstm = nn.LSTM(input_size=config.input_size,
16 | hidden_size=config.hidden_size,
17 | num_layers=config.num_layer,
18 | dropout=config.rnn_dropout,
19 | bidirectional=True)
20 | elif config.mention_detection_mode.upper() == 'GRU':
21 | self.gru = nn.GRU(input_size=config.input_size,
22 | hidden_size=config.hidden_size,
23 | num_layers=config.num_layer,
24 | dropout=config.rnn_dropout,
25 | bidirectional=True)
26 | self.dropout = nn.Dropout(p=config.rnn_fc_dropout)
27 | self.relu = nn.ReLU()
28 | self.hidden2tag = nn.Sequential(
29 | nn.Linear(config.hidden_size * 2, config.hidden_size * 2),
30 | nn.BatchNorm1d(config.hidden_size * 2),
31 | self.relu,
32 | self.dropout,
33 | nn.Linear(config.hidden_size * 2, config.label)
34 | )
35 |
36 | def forward(self, x):
37 | # x = (sequence length, batch_size, dimension of embedding)
38 | # text = x.text # geoff: demo
39 |
40 | if self.config.cuda:
41 | text = torch.LongTensor(x).cuda()
42 | else:
43 | text = torch.LongTensor(x)
44 | # print("########")
45 | # print(text)
46 |
47 | batch_size = text.size()[1]
48 | x = self.embed(text)
49 | # h0 / c0 = (layer*direction, batch_size, hidden_dim)
50 | if self.config.mention_detection_mode.upper() == 'LSTM':
51 | if self.config.cuda:
52 | h0 = Variable(torch.zeros(self.config.num_layer * 2, batch_size,
53 | self.config.hidden_size).cuda())
54 | c0 = Variable(torch.zeros(self.config.num_layer * 2, batch_size,
55 | self.config.hidden_size).cuda())
56 | else:
57 | h0 = Variable(torch.zeros(self.config.num_layer * 2, batch_size,
58 | self.config.hidden_size))
59 | c0 = Variable(torch.zeros(self.config.num_layer * 2, batch_size,
60 | self.config.hidden_size))
61 | # output = (sentence length, batch_size, hidden_size * num_direction)
62 | # ht = (layer*direction, batch, hidden_dim)
63 | # ct = (layer*direction, batch, hidden_dim)
64 | outputs, (ht, ct) = self.lstm(x, (h0, c0))
65 | elif self.config.mention_detection_mode.upper() == 'GRU':
66 | if self.config.cuda:
67 | h0 = Variable(torch.zeros(self.config.num_layer * 2, batch_size,
68 | self.config.hidden_size).cuda())
69 | else:
70 | h0 = Variable(torch.zeros(self.config.num_layer * 2, batch_size,
71 | self.config.hidden_size))
72 | # output = (sentence length, batch_size, hidden_size * num_direction)
73 | # ht = (layer*direction, batch, hidden_dim)
74 | outputs, ht = self.gru(x, h0)
75 | else:
76 | print("Wrong Mention Detection Mode")
77 | exit(1)
78 | tags = self.hidden2tag(outputs.view(-1, outputs.size(2)))
79 | scores = F.log_softmax(tags)
80 | return scores
81 |
--------------------------------------------------------------------------------
/NeuralQA/MentionDetection/nn/test.py:
--------------------------------------------------------------------------------
1 | import os
2 | import random
3 |
4 | import numpy as np
5 | import torch
6 | from torchtext import data
7 | from tqdm import tqdm
8 |
9 | from args import get_args
10 | from util import evaluation
11 | from datasets import SimpleQuestionDataset
12 |
13 | os.environ["CUDA_VISIBLE_DEVICES"] = "0"
14 |
15 | np.set_printoptions(threshold=np.nan)
16 | # Set default configuration in : args.py
17 | args = get_args()
18 |
19 | # Set random seed for reproducibility
20 | torch.manual_seed(args.seed)
21 | np.random.seed(args.seed)
22 | random.seed(args.seed)
23 |
24 | if not args.cuda:
25 | args.gpu = -1
26 | if torch.cuda.is_available() and args.cuda:
27 | print("Note: You are using GPU for training")
28 | torch.cuda.set_device(args.gpu)
29 | torch.cuda.manual_seed(args.seed)
30 | if torch.cuda.is_available() and not args.cuda:
31 | print("Warning: You have Cuda but not use it. You are using CPU for training.")
32 |
33 | TEXT = data.Field(lower=True)
34 | ED = data.Field()
35 |
36 | train, dev, test = SimpleQuestionDataset.splits(TEXT, ED, path=args.data_dir) # text_field, label_field
37 | TEXT.build_vocab(train, dev, test)
38 | ED.build_vocab(train, dev, test)
39 |
40 | train_iter = data.Iterator(train, batch_size=args.batch_size, device=args.gpu, train=True, repeat=False,
41 | sort=False, shuffle=True)
42 | dev_iter = data.Iterator(dev, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False,
43 | sort=False, shuffle=False)
44 | test_iter = data.Iterator(test, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False,
45 | sort=False, shuffle=False)
46 |
47 | # load the model
48 | if not torch.cuda.is_available():
49 | model = torch.load(args.trained_model)
50 | else:
51 | model = torch.load(args.trained_model, map_location=lambda storage, location: storage.cuda(args.gpu))
52 |
53 | print(model)
54 |
55 | if args.dataset == 'EntityDetection':
56 | index2tag = np.array(ED.vocab.itos)
57 | # print(index2tag)
58 | else:
59 | print("Wrong Dataset")
60 | exit(1)
61 |
62 | index2word = np.array(TEXT.vocab.itos)
63 |
64 | results_path = os.path.join(args.results_path, args.entity_detection_mode.lower())
65 | if not os.path.exists(results_path):
66 | os.makedirs(results_path, exist_ok=True)
67 |
68 |
69 | def convert(fileName, idFile, outputFile):
70 | fin = open(fileName)
71 | fid = open(idFile)
72 | fout = open(outputFile, "w")
73 | # holiday barbie doll by bob mackie # h8583 O O O O O O I I I O O O I I
74 | # valid-10837 %%%% holiday barbie doll %%%% # h8583
75 | for line, line_id in tqdm(zip(fin.readlines(), fid.readlines())):
76 | query_list = []
77 | query_text = []
78 | line = line.strip().split('\t')
79 | sent = line[0].strip().split()
80 | pred = line[1].strip().split()
81 | for token, label in zip(sent, pred):
82 | if label == 'I':
83 | query_text.append(token)
84 | if label == 'O':
85 | query_text = list(filter(lambda x: x != '', query_text))
86 | if len(query_text) != 0:
87 | query_list.append(" ".join(list(filter(lambda x: x != '', query_text))))
88 | query_text = []
89 | query_text = list(filter(lambda x: x != '', query_text))
90 | if len(query_text) != 0:
91 | query_list.append(" ".join(list(filter(lambda x: x != '', query_text))))
92 | if len(query_list) == 0:
93 | query_list.append(" ".join(list(filter(lambda x: x != '', sent))))
94 | fout.write(" %%%% ".join([line_id.strip()] + query_list) + "\n")
95 |
96 |
97 | def predict(dataset_iter=test_iter, dataset=test, data_name="test"):
98 | print("Dataset: {}".format(data_name))
99 | model.eval()
100 | dataset_iter.init_epoch()
101 |
102 | n_correct = 0
103 | fname = "{}.txt".format(data_name)
104 | temp_file = 'tmp' + fname
105 | results_file = open(temp_file, 'w')
106 |
107 | gold_list = []
108 | pred_list = []
109 |
110 | for data_batch_idx, data_batch in enumerate(dataset_iter):
111 | scores = model(data_batch)
112 | if args.dataset == 'EntityDetection':
113 | n_correct += ((torch.max(scores, 1)[1].view(data_batch.ed.size()).data == data_batch.ed.data).sum(dim=0) \
114 | == data_batch.ed.size()[0]).sum()
115 | index_tag = np.transpose(torch.max(scores, 1)[1].view(data_batch.ed.size()).cpu().data.numpy())
116 | tag_array = index2tag[index_tag]
117 | index_question = np.transpose(data_batch.text.cpu().data.numpy())
118 | question_array = index2word[index_question]
119 | gold_list.append(np.transpose(data_batch.ed.cpu().data.numpy()))
120 | gold_array = index2tag[np.transpose(data_batch.ed.cpu().data.numpy())]
121 | pred_list.append(index_tag)
122 | for question, label, gold in zip(question_array, tag_array, gold_array):
123 | results_file.write("{}\t{}\t{}\n".format(" ".join(question), " ".join(label), " ".join(gold)))
124 | # print("{}\t{}\t{}\n".format(" ".join(question), " ".join(label), " ".join(gold)))
125 | else:
126 | print("Wrong Dataset")
127 | exit()
128 |
129 | if args.dataset == 'EntityDetection':
130 | P, R, F = evaluation(gold_list, pred_list, index2tag, type=False)
131 | print("{} Precision: {:10.6f}% Recall: {:10.6f}% F1 Score: {:10.6f}%".format("Dev", 100. * P, 100. * R,
132 | 100. * F))
133 | else:
134 | print("Wrong dataset")
135 | exit()
136 | results_file.flush()
137 | results_file.close()
138 | convert(temp_file, os.path.join(args.data_dir, "lineids_{}.txt".format(data_name)),
139 | os.path.join(results_path, "query.{}".format(data_name)))
140 | os.remove(temp_file)
141 |
142 |
143 | # run the model on the dev set and write the output to a file
144 | predict(dataset_iter=dev_iter, dataset=dev, data_name="valid2")
145 |
146 | # run the model on the test set and write the output to a file
147 | # predict(dataset_iter=test_iter, dataset=test, data_name="test")
148 |
--------------------------------------------------------------------------------
/NeuralQA/MentionDetection/nn/tmp_data.txt:
--------------------------------------------------------------------------------
1 | who was the trump ocean club international hotel and tower named after O O O I I I I I I I O O O O O I I I I I I I O O
2 | where was sasha vujačić born O O I I O O O I I O
3 | what is a region that dead combo was released in O O O O O I I O O O O O O O O I I O O O
4 | what is a film directed by wiebke von carolsfeld ? O O O O O O I I I O O O O O O O I I I O
5 | what country was music for stock exchange released in O O O I I I I O O O O O I I I I O O
6 | where is adler school of professional psychology located ? O O I I I I I O O O O I I I I I O O
7 | where was john miltern born O O I I O O O I I O
8 | what city is vancouver millionaires from ? O O O I I O O O O O I I O O
9 | what was seymour parker gilbert 's profession ? O O I I I O O O O O I I I O O O
10 | what does ( 12385 ) 1994 uo orbit O O I I I I I O O O I I I I I O
11 | who is the singer of only women bleed O O O O O I I I O O O O O I I I
12 | in what french city did antoine de févin die O O O O O I I I O O O O O O I I I O
13 | who published rama O O I O O I
14 | who was an advisor for irving langmuir ? O O O O O I I O O O O O O I I O
15 | what is the language of the film bon voyage ? O O O O O O O I I O O O O O O O O I I O
16 | which country was the hunyadi family from O O O I I I O O O O O I I O
17 | what major cities does u.s. route 2 run through O O O O I I I O O O O O O I I I O O
18 | who was a child of mithibai jinnah O O O O O I I O O O O O I I
19 | whats a version of the single titled star O O O O O O O I O O O O O O O I
20 | what is a song by john rutter ? O O O O O I I O