166 |
167 |
168 |
169 |
155 |
156 |
163 |
164 |
165 |
170 |
171 |
49 |
50 |
101 |
102 |
103 |
104 |
105 |
111 |
112 |
113 |
199 |
200 |
201 |
202 |
203 |
208 |
209 |
210 |
211 |
212 |
213 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
--------------------------------------------------------------------------------
/fastHan/model/camr_finetune_dataloader.py:
--------------------------------------------------------------------------------
1 | import re
2 | import torch
3 | import pandas as pd
4 |
5 | from datasets import Dataset
6 | from .camr_to_tuples import CAMR
7 |
8 | MAX_LEN = 300
9 |
10 |
11 | def FastCAMR_Parsing_Loader(data_path, tokenizer):
12 | sid_list, sent_list, id_token_list, amr_list, convert_amr_list = var_free_camrs(
13 | data_path)
14 | input_ids, attention_mask, decoder_input_ids, labels = [], [], [], []
15 | discard_num = 0
16 | discard_index = []
17 | for idx, sid in enumerate(sid_list):
18 | tokenize_result = tokenizer(convert_amr_list[idx])
19 | if len(tokenize_result['input_ids']) > MAX_LEN:
20 | discard_num += 1
21 | discard_index.append(idx)
22 | continue
23 | sent_tokenize_result = tokenizer(' '.join(sent_list[idx]),
24 | max_length=MAX_LEN,
25 | padding='max_length',
26 | truncation=True)
27 | amr_tokenize_result = tokenizer(convert_amr_list[idx],
28 | max_length=MAX_LEN,
29 | padding='max_length',
30 | truncation=True)
31 | input_ids.append(sent_tokenize_result['input_ids'])
32 | attention_mask.append(sent_tokenize_result['attention_mask'])
33 | decode_ids = amr_tokenize_result['input_ids']
34 | decoder_input_ids.append(decode_ids[:-1])
35 | labels.append(decode_ids[1:])
36 | # print("******* {0} sentences are discard! *******".format(discard_num))
37 | for idx in reversed(discard_index):
38 | sid_list.pop(idx)
39 | sent_list.pop(idx)
40 | id_token_list.pop(idx)
41 | amr_list.pop(idx)
42 | convert_amr_list.pop(idx)
43 | amr_data = {
44 | "input_ids": input_ids,
45 | "attention_mask": attention_mask,
46 | "decoder_input_ids": decoder_input_ids,
47 | "labels": labels
48 | }
49 | amr_data = pd.DataFrame(amr_data)
50 | dataset = Dataset.from_pandas(amr_data, preserve_index=False)
51 | return dataset
52 |
53 |
54 | def data_collator(amr_data):
55 | first = amr_data[0]
56 | batch = {}
57 | for k, v in first.items():
58 | if isinstance(v, torch.Tensor):
59 | batch[k] = torch.stack([f[k] for f in amr_data])
60 | else:
61 | batch[k] = torch.tensor([f[k] for f in amr_data])
62 | return batch
63 |
64 |
65 | def var_free_camrs(input_file):
66 | with open(input_file, 'r', encoding='utf-8') as f:
67 | lines = f.readlines()
68 | sid_list, sent_list, id_token_list, amr_list = read_raw_camrs(lines)
69 | convert_amr_list = [
70 | delete_camr_variables(convert_camr_to_single_line(amr),
71 | id_token_list[idx])
72 | for idx, amr in enumerate(amr_list)
73 | ]
74 | return sid_list, sent_list, id_token_list, amr_list, convert_amr_list
75 |
76 |
77 | def read_raw_camrs(lines):
78 | sid_list, sent_list, id_token_list, amr_list = [], [], [], []
79 | # 迭代输入文件中的每个句子
80 | cur_sent, cur_amr = [], []
81 | id_token_dict = {}
82 | has_content = False
83 | for line in lines:
84 | line = line.strip()
85 | if '\ufeff' in line:
86 | line = line.replace('\ufeff', '')
87 | if '\u200b' in line:
88 | line = line.replace('\u200b', '')
89 | if line == "":
90 | if has_content: # end of current CAMR
91 | sent_list.append(cur_sent)
92 | id_token_list.append(id_token_dict)
93 | amr_list.append(cur_amr)
94 | cur_sent, cur_amr = [], []
95 | id_token_dict = {}
96 | has_content = False
97 | continue
98 | if line.strip().startswith("#"):
99 | if '::id' in line:
100 | sid = re.findall(r'# ::id export_amr\.(.*?)\s*::', line)[0]
101 | sid_list.append(sid)
102 | elif '::wid' in line:
103 | wid = line[len('# ::wid '):].strip().split(' ')
104 | for i in wid:
105 | token_id, token = i.split('_')
106 | if token != '':
107 | # key: id number, value: token (e.g. "1":"我")
108 | cur_sent.append(token)
109 | id_token_dict[int(token_id[1:])] = token
110 | else:
111 | continue
112 | else:
113 | has_content = True
114 | cur_amr.append(line)
115 | if has_content:
116 | sent_list.append(cur_sent)
117 | id_token_list.append(id_token_dict)
118 | amr_list.append(cur_amr)
119 | return sid_list, sent_list, id_token_list, amr_list
120 |
121 |
122 | def convert_camr_to_single_line(amr):
123 | return "".join([line.strip() for line in amr])
124 |
125 |
126 | def delete_camr_variables(amr, id_token_dict):
127 | result_amr, coref_dict = CAMR.parse_AMR_line(amr, id_token_dict)
128 | node_dict = dict(zip(result_amr.nodes, result_amr.node_values))
129 | var = 'x\d+(?:_\d+)*(?:_x\d+(?:_\d+)*)*'
130 | coref_vars = '(' + var + '\s*/\s*' + var + ')'
131 | coref_var_list = re.findall(coref_vars, amr)
132 | for coref_v in coref_var_list:
133 | var0 = coref_v.split('/')[0].strip()
134 | var1 = coref_v.split('/')[1].strip()
135 | var0 = node_dict[var0[1:]]
136 | var1 = node_dict[var1[1:]]
137 | amr = re.sub(coref_v, var0 + '^' + var1, amr)
138 | normal_var = var + '\s*/\s*'
139 | amr = re.sub(normal_var, '', amr)
140 | return amr
--------------------------------------------------------------------------------
/docs/build/html/_modules/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
114 |
115 |
197 |
198 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
153 |
196 |
154 |
172 |
194 |
195 |
47 |
48 |
99 |
100 |
101 |
102 |
103 |
109 |
110 |
111 |
194 |
195 |
196 |
197 |
198 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
--------------------------------------------------------------------------------
/docs/source/conf.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | #
3 | # Configuration file for the Sphinx documentation builder.
4 | #
5 | # This file does only contain a selection of the most common options. For a
6 | # full list see the documentation:
7 | # http://www.sphinx-doc.org/en/master/config
8 |
9 | # -- Path setup --------------------------------------------------------------
10 |
11 | # If extensions (or modules to document with autodoc) are in another directory,
12 | # add these directories to sys.path here. If the directory is relative to the
13 | # documentation root, use os.path.abspath to make it absolute, like shown here.
14 | #
15 | import os
16 | import sys
17 |
18 | sys.path.insert(0, os.path.abspath('../../'))
19 |
20 | # -- Project information -----------------------------------------------------
21 |
22 | project = 'fastHan'
23 | copyright = '2020, fastHan'
24 | author = 'fastHan'
25 |
26 | # The short X.Y version
27 | version = '1.4'
28 | # The full version, including alpha/beta/rc tags
29 | release = '1.4'
30 |
31 | # -- General configuration ---------------------------------------------------
32 |
33 | # If your documentation needs a minimal Sphinx version, state it here.
34 | #
35 | # needs_sphinx = '1.0'
36 |
37 | # Add any Sphinx extension module names here, as strings. They can be
38 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
39 | # ones.
40 | extensions = [
41 | 'sphinx.ext.autodoc',
42 | 'sphinx.ext.viewcode',
43 | 'sphinx.ext.autosummary',
44 | 'sphinx.ext.mathjax',
45 | 'sphinx.ext.todo'
46 | ]
47 |
48 | autodoc_default_options = {
49 | 'member-order': 'bysource',
50 | 'special-members': '__init__',
51 | 'undoc-members': False,
52 | }
53 |
54 | autoclass_content = "class"
55 |
56 | # Add any paths that contain templates here, relative to this directory.
57 | templates_path = ['_templates']
58 | # template_bridge
59 | # The suffix(es) of source filenames.
60 | # You can specify multiple suffix as a list of string:
61 | #
62 | # source_suffix = ['.rst', '.md']
63 | source_suffix = '.rst'
64 |
65 | # The master toctree document.
66 | master_doc = 'index'
67 |
68 | # The language for content autogenerated by Sphinx. Refer to documentation
69 | # for a list of supported languages.
70 | #
71 | # This is also used if you do content translation via gettext catalogs.
72 | # Usually you set "language" from the command line for these cases.
73 | language = "zh_CN"
74 |
75 | # List of patterns, relative to source directory, that match files and
76 | # directories to ignore when looking for source files.
77 | # This pattern also affects html_static_path and html_extra_path .
78 | exclude_patterns = ['modules.rst']
79 |
80 | # The name of the Pygments (syntax highlighting) style to use.
81 | pygments_style = 'sphinx'
82 |
83 | # -- Options for HTML output -------------------------------------------------
84 |
85 | # The theme to use for HTML and HTML Help pages. See the documentation for
86 | # a list of builtin themes.
87 | #
88 | html_theme = 'sphinx_rtd_theme'
89 |
90 | # Theme options are theme-specific and customize the look and feel of a theme
91 | # further. For a list of options available for each theme, see the
92 | # documentation.
93 | #
94 | html_theme_options = {
95 | 'collapse_navigation': False,
96 | 'titles_only': True
97 | }
98 |
99 | # Add any paths that contain custom static files (such as style sheets) here,
100 | # relative to this directory. They are copied after the builtin static files,
101 | # so a file named "default.css" will overwrite the builtin "default.css".
102 | html_static_path = ['_static']
103 |
104 | # Custom sidebar templates, must be a dictionary that maps document names
105 | # to template names.
106 | #
107 | # The default sidebars (for documents that don't match any pattern) are
108 | # defined by theme itself. Builtin themes are using these templates by
109 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
110 | # 'searchbox.html']``.
111 | #
112 | # html_sidebars = {}
113 |
114 |
115 | # -- Options for HTMLHelp output ---------------------------------------------
116 |
117 | # Output file base name for HTML help builder.
118 | htmlhelp_basename = 'fastHan'
119 |
120 | # -- Options for LaTeX output ------------------------------------------------
121 |
122 | latex_elements = {
123 | # The paper size ('letterpaper' or 'a4paper').
124 | #
125 | # 'papersize': 'letterpaper',
126 |
127 | # The font size ('10pt', '11pt' or '12pt').
128 | #
129 | # 'pointsize': '10pt',
130 |
131 | # Additional stuff for the LaTeX preamble.
132 | #
133 | # 'preamble': '',
134 |
135 | # Latex figure (float) alignment
136 | #
137 | # 'figure_align': 'htbp',
138 | }
139 |
140 | # Grouping the document tree into LaTeX files. List of tuples
141 | # (source start file, target name, title,
142 | # author, documentclass [howto, manual, or own class]).
143 | latex_documents = []
144 |
145 | # -- Options for manual page output ------------------------------------------
146 |
147 | # One entry per manual page. List of tuples
148 | # (source start file, name, description, authors, manual section).
149 | man_pages = [
150 | (master_doc, 'fastHan', 'fastHan Documentation',
151 | [author], 1)
152 | ]
153 |
154 | # -- Options for Texinfo output ----------------------------------------------
155 |
156 | # Grouping the document tree into Texinfo files. List of tuples
157 | # (source start file, target name, title, author,
158 | # dir menu entry, description, category)
159 | texinfo_documents = [
160 | (master_doc, 'fastHan', 'fastHan Documentation',
161 | author, 'fastHan', 'One line description of project.',
162 | 'Miscellaneous'),
163 | ]
164 |
165 |
166 | # -- Extension configuration -------------------------------------------------
167 | def maybe_skip_member(app, what, name, obj, skip, options):
168 | if name == "__init__":
169 | return False
170 | if name.startswith("_"):
171 | return True
172 | return False
173 |
174 |
175 | def setup(app):
176 | app.connect('autodoc-skip-member', maybe_skip_member)
177 |
--------------------------------------------------------------------------------
/docs/build/html/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
112 |
113 |
192 |
193 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
149 |
191 |
150 |
167 |
189 |
190 |
151 |
152 |
165 |
166 | 代码可用的所有模块
153 |- fastHan.FastModel 154 |
- fastHan.model.BertCharParser 155 |
- fastHan.model.CharParser 156 |
- fastHan.model.UserDict 157 |
- fastHan.model.bert 158 |
- fastHan.model.bert_encoder_theseus 159 |
- fastHan.model.model 160 |
- fastHan.model.old_fastNLP_bert 161 |
- fastHan.model.utils 162 |
48 |
49 |
100 |
101 |
102 |
103 |
104 |
110 |
111 |
112 |
221 |
222 |
223 |
224 |
225 |
230 |
231 |
232 |
233 |
234 |
235 |
236 |
237 |
--------------------------------------------------------------------------------
/docs/build/html/fastHan.model.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
113 |
114 |
219 |
220 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
154 |
187 |
216 |
217 |
218 |
47 |
48 |
99 |
100 |
101 |
102 |
103 |
109 |
110 |
111 |
206 |
207 |
208 |
209 |
210 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
--------------------------------------------------------------------------------
/fastHan/model/metrics.py:
--------------------------------------------------------------------------------
1 | from fastNLP.core.metrics import Metric
2 | import torch
3 |
4 | # modified from https://github.com/yhcc/JointCwsParser
5 | class SegAppCharParseF1Metric(Metric):
6 | #
7 | def __init__(self,pun_index,app_index=0):
8 | super().__init__()
9 | self.app_index = app_index
10 | self.pun_index = pun_index
11 |
12 | self.parse_head_tp = 0
13 | self.parse_label_tp = 0
14 | self.rec_tol = 0
15 | self.pre_tol = 0
16 |
17 | def get_word_pairs(self,head_preds,label_preds,seq_lens,pun_masks):
18 | # 去掉root
19 | head_preds = head_preds[:, 1:].tolist()
20 | label_preds = label_preds[:, 1:].tolist()
21 | seq_lens = (seq_lens - 1).tolist()
22 |
23 | head_dep_tuples=[]
24 | head_label_dep_tuples = []
25 |
26 | for b in range(len(head_preds)):
27 | seq_len = seq_lens[b]
28 | head_pred = head_preds[b][:seq_len]
29 | label_pred = label_preds[b][:seq_len]
30 |
31 | words = [] # 存放[word_start, word_end)，相对起始位置，不考虑root
32 | heads = []
33 | labels = []
34 | ranges = [] # 对应该char是第几个word，长度是seq_len+1
35 | word_idx = 0
36 | word_start_idx = 0
37 | for idx, (label, head) in enumerate(zip(label_pred, head_pred)):
38 | ranges.append(word_idx)
39 | if label == self.app_index:
40 | pass
41 | else:
42 | labels.append(label)
43 | heads.append(head)
44 | words.append((word_start_idx, idx+1))
45 | word_start_idx = idx+1
46 | word_idx += 1
47 |
48 | head_dep_tuple = [] # head在前面
49 | head_label_dep_tuple = []
50 | for idx, head in enumerate(heads):
51 | span = words[idx]
52 | if span[0]==span[1]-1 and pun_masks[b, span[0]]:
53 | continue # exclude punctuations
54 | if head == 0:
55 | head_dep_tuple.append((('root', words[idx])))
56 | head_label_dep_tuple.append(('root', labels[idx], words[idx]))
57 | else:
58 | head_word_idx = ranges[head-1]
59 | head_word_span = words[head_word_idx]
60 | head_dep_tuple.append(((head_word_span, words[idx])))
61 | head_label_dep_tuple.append((head_word_span, labels[idx], words[idx]))
62 | head_dep_tuples.append(head_dep_tuple)
63 | head_label_dep_tuples.append(head_label_dep_tuple)
64 |
65 | return head_dep_tuples,head_label_dep_tuples
66 |
67 |
68 | # def update(self, gold_word_pairs, gold_label_word_pairs, head_preds, label_preds, seq_len,
69 | # pun_masks):
70 | def update(self, labels,heads,head_preds, label_preds, seq_len):
71 | """
72 |
73 | max_len是不包含root的character的长度
74 | :param gold_word_pairs: List[List[((head_start, head_end), (dep_start, dep_end)), ...]], batch_size
75 | :param gold_label_word_pairs: List[List[((head_start, head_end), label, (dep_start, dep_end)), ...]], batch_size
76 | :param head_preds: batch_size x max_len
77 | :param label_preds: batch_size x max_len
78 | :param seq_lens:
79 | :return:
80 | """
81 | pun_masks=(labels==self.pun_index).long()
82 | pun_masks=pun_masks[:,1:]
83 |
84 | head_dep_tuples,head_label_dep_tuples=self.get_word_pairs(head_preds,label_preds,seq_len,pun_masks)
85 | gold_head_dep_tuples,gold_head_label_dep_tuples=self.get_word_pairs(heads,labels,seq_len,pun_masks)
86 |
87 | for b in range(seq_len.shape[0]):
88 | head_dep_tuple=head_dep_tuples[b]
89 | head_label_dep_tuple=head_label_dep_tuples[b]
90 | gold_head_dep_tuple=gold_head_dep_tuples[b]
91 | gold_head_label_dep_tuple=gold_head_label_dep_tuples[b]
92 |
93 |
94 | for head_dep, head_label_dep in zip(head_dep_tuple, head_label_dep_tuple):
95 | if head_dep in gold_head_dep_tuple:
96 | self.parse_head_tp += 1
97 | if head_label_dep in gold_head_label_dep_tuple:
98 | self.parse_label_tp += 1
99 | self.pre_tol += len(head_dep_tuple)
100 | self.rec_tol += len(gold_head_dep_tuple)
101 |
102 | def get_metric(self, reset=True):
103 | u_p = self.parse_head_tp / self.pre_tol
104 | u_r = self.parse_head_tp / self.rec_tol
105 | u_f = 2*u_p*u_r/(1e-6 + u_p + u_r)
106 | l_p = self.parse_label_tp / self.pre_tol
107 | l_r = self.parse_label_tp / self.rec_tol
108 | l_f = 2*l_p*l_r/(1e-6 + l_p + l_r)
109 |
110 | if reset:
111 | self.parse_head_tp = 0
112 | self.parse_label_tp = 0
113 | self.rec_tol = 0
114 | self.pre_tol = 0
115 |
116 | return {'u_f1': round(u_f, 4), 'u_p': round(u_p, 4), 'u_r/uas':round(u_r, 4),
117 | 'f': round(l_f, 4), 'l_p': round(l_p, 4), 'l_r/las': round(l_r, 4)}
118 |
119 |
120 | class CWSMetric(Metric):
121 | def __init__(self, app_index=0):
122 | super().__init__()
123 | self.app_index = app_index
124 | self.pre = 0
125 | self.rec = 0
126 | self.tp = 0
127 |
128 |
129 | def label_to_seg(self,labels,seq_lens):
130 | segs=torch.zeros_like(labels)[:,1:]
131 | masks=torch.zeros_like(labels)[:,1:]
132 |
133 | seq_lens=(seq_lens-1).tolist()
134 | # [:,1:]是为了剔除root结点
135 | for idx,label in enumerate(labels[:, 1:].tolist()):
136 | seq_len=seq_lens[idx]
137 | label=label[:seq_len]
138 | word_len = 0
139 |
140 | for i,l in enumerate(label):
141 | if l==self.app_index and i!=len(label)-1:
142 | word_len+=1
143 | else:
144 | segs[idx,i]=word_len
145 | masks[idx,i]=1
146 | word_len=0
147 | return segs,masks
148 |
149 |
150 | def update(self, labels, label_preds, seq_len):
151 | """
152 | :param label_preds: batch_size x max_len
153 | :param seq_len: batch_size
154 | :return:
155 | """
156 |
157 | seg_targets,seg_masks=self.label_to_seg(labels,seq_len)
158 | pred_segs,pred_masks=self.label_to_seg(label_preds,seq_len)
159 |
160 |
161 | right_mask = seg_targets.eq(pred_segs) # 对长度的预测一致
162 | self.rec += seg_masks.sum().item()
163 | self.pre += pred_masks.sum().item()
164 | # 且pred和target在同一个地方有值
165 | self.tp += (right_mask.__and__(pred_masks.bool().__and__(seg_masks.bool()))).sum().item()
166 |
167 | def get_metric(self, reset=True):
168 | res = {}
169 | res['rec'] = round(self.tp/(self.rec+1e-6), 4)
170 | res['pre'] = round(self.tp/(self.pre+1e-6), 4)
171 | res['f1'] = round(2*res['rec']*res['pre']/(res['pre'] + res['rec'] + 1e-6), 4)
172 |
173 | if reset:
174 | self.pre = 0
175 | self.rec = 0
176 | self.tp = 0
177 |
178 | return res
--------------------------------------------------------------------------------
/docs/build/html/fastHan.model.utils.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
112 |
113 |
204 |
205 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
153 |
203 |
154 |
179 |
201 |
202 |
155 |
156 |
177 |
178 |
157 |
174 |
175 |
176 | fastHan.model package¶
158 |
159 |
173 | Submodules¶
160 |
161 |
172 | -
162 |
- fastHan.model.BertCharParser module 163 |
- fastHan.model.CharParser module 164 |
- fastHan.model.UserDict module 165 |
- fastHan.model.bert module 166 |
- fastHan.model.bert_encoder_theseus module 167 |
- fastHan.model.model module 168 |
- fastHan.model.old_fastNLP_bert module 169 |
- fastHan.model.utils module 170 |
47 |
48 |
99 |
100 |
101 |
102 |
103 |
109 |
110 |
111 |
220 |
221 |
222 |
223 |
224 |
229 |
230 |
231 |
232 |
233 |
234 |
235 |
236 |
--------------------------------------------------------------------------------
/fastHan/model/dependency_parsing_model.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn.functional as F
3 | from fastNLP.models.torch.biaffine_parser import (ArcBiaffine, BiaffineParser,
4 | LabelBilinear)
5 | from fastNLP.modules.torch.dropout import TimestepDropout
6 | from torch import nn
7 | from transformers import PreTrainedModel
8 |
9 |
10 | # modified from https://github.com/yhcc/JointCwsParser/blob/master/models/BertParser.py
11 | class BertParser(BiaffineParser):
12 | def __init__(self,
13 | num_label,
14 | embed_size=768,
15 | arc_mlp_size=500,
16 | label_mlp_size=100,
17 | dropout=0.1,
18 | use_greedy_infer=False,
19 | app_index=0):
20 | super(BiaffineParser, self).__init__()
21 |
22 | self.embed_size = embed_size
23 | self.mlp = nn.Sequential(
24 | nn.Linear(self.embed_size, arc_mlp_size * 2 + label_mlp_size * 2),
25 | nn.LeakyReLU(0.1),
26 | TimestepDropout(p=dropout),
27 | )
28 | self.arc_mlp_size = arc_mlp_size
29 | self.label_mlp_size = label_mlp_size
30 | self.arc_predictor = ArcBiaffine(arc_mlp_size, bias=True)
31 | self.label_predictor = LabelBilinear(label_mlp_size,
32 | label_mlp_size,
33 | num_label,
34 | bias=True)
35 | self.use_greedy_infer = use_greedy_infer
36 | self.reset_parameters()
37 |
38 | self.app_index = app_index
39 | self.num_label = num_label
40 | if self.app_index != 0:
41 | raise ValueError("现在app_index必须等于0")
42 |
43 | self.dropout = nn.Dropout(dropout)
44 |
45 | def reset_parameters(self):
46 | for name, m in self.named_modules():
47 | if hasattr(m, 'reset_parameters') or hasattr(m, 'init_param'):
48 | pass
49 | else:
50 | for p in m.parameters():
51 | if len(p.size()) > 1:
52 | nn.init.xavier_normal_(p, gain=0.1)
53 | else:
54 | nn.init.uniform_(p, -0.1, 0.1)
55 |
56 | def forward(self, feats, mask, gold_heads=None, char_labels=None):
57 | # 此处的mask与attention_mask不同，需要去除sep与cls
58 |
59 | batch_size = feats.shape[0]
60 | max_len = feats.shape[1]
61 |
62 | feats = self.dropout(feats)
63 | feats = self.mlp(feats)
64 | arc_sz, label_sz = self.arc_mlp_size, self.label_mlp_size
65 | arc_dep, arc_head = feats[:, :, :arc_sz], feats[:, :,
66 | arc_sz:2 * arc_sz]
67 | label_dep, label_head = feats[:, :, 2 * arc_sz:2 * arc_sz +
68 | label_sz], feats[:, :,
69 | 2 * arc_sz + label_sz:]
70 |
71 | arc_pred = self.arc_predictor(arc_head, arc_dep) # [N, L, L]
72 |
73 | if self.training:
74 | assert gold_heads is not None
75 | head_pred = None
76 | heads = gold_heads
77 | else:
78 | heads = self.mst_decoder(arc_pred, mask)
79 | head_pred = heads
80 |
81 | # 将pad的-100替换为-1，以免heads作为矩阵索引的时候报错
82 | padded_heads = torch.clone(heads)
83 | padded_heads[padded_heads == -100] = -1
84 |
85 | batch_range = torch.arange(start=0,
86 | end=batch_size,
87 | dtype=torch.long,
88 | device=mask.device).unsqueeze(1)
89 | label_head = label_head[batch_range, padded_heads].contiguous()
90 | label_pred = self.label_predictor(label_head,
91 | label_dep) # [N, max_len, num_label]
92 | # 这里限制一下，只有当head为下一个时，才能预测app这个label
93 | arange_index = torch.arange(1, max_len+1, dtype=torch.long, device=mask.device).unsqueeze(0)\
94 | .repeat(batch_size, 1) # batch_size x max_len
95 |
96 | app_masks = heads.ne(
97 | arange_index) # batch_size x max_len, 为1的位置不可以预测app
98 | app_masks = app_masks.unsqueeze(2).repeat(1, 1, self.num_label)
99 | app_masks[:, :, 1:] = 0
100 | label_pred = label_pred.masked_fill(app_masks, float('-inf'))
101 |
102 | if self.training:
103 | arc_loss, label_loss = self.loss(arc_pred, label_pred, gold_heads,
104 | char_labels, mask)
105 | res_dict = {
106 | 'loss': arc_loss + label_loss,
107 | 'arc_loss': arc_loss,
108 | 'label_loss': label_loss
109 | }
110 | else:
111 | res_dict = {
112 | 'label_preds': label_pred.max(2)[1],
113 | 'head_preds': head_pred
114 | }
115 |
116 | return res_dict
117 |
118 | @staticmethod
119 | def loss(arc_pred, label_pred, arc_true, label_true, mask):
120 | """
121 | Compute loss.
122 | :param arc_pred: [batch_size, seq_len, seq_len]
123 | :param label_pred: [batch_size, seq_len, n_tags]
124 | :param arc_true: [batch_size, seq_len]
125 | :param label_true: [batch_size, seq_len]
126 | :param mask: [batch_size, seq_len]
127 | :return: loss value
128 | """
129 |
130 | batch_size, seq_len, _ = arc_pred.shape
131 | flip_mask = (mask == 0)
132 | # _arc_pred = arc_pred.clone()
133 | _arc_pred = arc_pred.masked_fill(flip_mask.unsqueeze(1), -float('inf'))
134 |
135 | arc_true.data[:, 0].fill_(-100)
136 | label_true.data[:, 0].fill_(-100)
137 |
138 | arc_nll = F.cross_entropy(_arc_pred.view(-1, seq_len),
139 | arc_true.view(-1),
140 | ignore_index=-100)
141 | label_nll = F.cross_entropy(label_pred.view(-1, label_pred.size(-1)),
142 | label_true.view(-1),
143 | ignore_index=-100)
144 |
145 | return arc_nll, label_nll
146 |
147 |
148 | class DependencyParsingModel(PreTrainedModel):
149 | def __init__(self, encoder, config, labels):
150 | super().__init__(config)
151 |
152 | label_num = len(labels)
153 |
154 | self.num_labels = label_num
155 |
156 | self.encoder = encoder
157 | self.parser = BertParser(num_label=self.num_labels,
158 | embed_size=config.hidden_size,
159 | app_index=labels.index('app'))
160 |
161 | def forward(self,
162 | input_ids=None,
163 | attention_mask=None,
164 | labels=None,
165 | heads=None):
166 | # 生成不考虑cls和sep的mask
167 | seq_len = attention_mask.sum(dim=-1) - 2
168 | broad_cast_seq_len = torch.arange(attention_mask.shape[1] - 2).expand(
169 | attention_mask.shape[0], -1).to(seq_len.device)
170 | mask = broad_cast_seq_len < seq_len.unsqueeze(1)
171 |
172 | outputs = self.encoder(input_ids, attention_mask=attention_mask)
173 |
174 | feats = outputs[0]
175 | feats = feats[:, 1:-1]
176 |
177 | return self.parser.forward(feats=feats,
178 | mask=mask,
179 | gold_heads=heads,
180 | char_labels=labels)
181 |
--------------------------------------------------------------------------------
/docs/build/html/py-modindex.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
112 |
113 |
218 |
219 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
153 |
217 |
154 |
193 |
215 |
216 |
155 |
156 |
191 |
192 |
157 |
188 |
189 |
190 | fastHan.model.utils module¶
158 |-
159 |
-
160 |
fastHan.model.utils.check_dataloader_paths(paths: Union[str, Dict[str, str]]) → Dict[str, str][源代码]¶
161 | 检查传入dataloader的文件的合法性。如果为合法路径，将返回至少包含’train’这个key的dict。类似于下面的结果 162 | {
163 |164 |
167 |‘train’: ‘/some/path/to/’, # 一定包含，建词表应该在这上面建立，剩下的其它文件应该只需要处理并index。 165 | ‘test’: ‘xxx’ # 可能有，也可能没有 166 | …} 168 | 如果paths为不合法的，将直接进行raise相应的错误
169 |170 |
180 |171 | 172 | 173 | 参数: paths – 路径. 可以为一个文件路径(则认为该文件就是train的文件); 可以为一个文件目录，将在该目录下寻找train(文件名 174 | 中包含train这个字段), test.txt, dev.txt; 可以为一个dict, 则key是用户自定义的某个文件的名称，value是这个文件的路径。 175 |返回: 177 |
50 |
51 |
102 |
103 |
104 |
105 |
106 |
112 |
113 |
114 |
255 |
256 |
257 |
258 |
259 |
264 |
265 |
266 |
267 |
268 |
269 |
270 |
271 |
--------------------------------------------------------------------------------
/docs/build/html/fastHan.model.model.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
115 |
116 |
253 |
254 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
152 |
252 |
153 |
228 |
250 |
251 |
154 |
155 |
156 |
163 |
223 |
224 |
225 |
226 |
227 | Python 模块索引
157 | 158 |
159 | f
160 |
161 |
162 | | 165 | f | ||
| 169 | |
170 | fastHan | 171 | |
| 174 | |
175 | fastHan.FastModel | 176 | |
| 179 | |
180 | fastHan.model | 181 | |
| 184 | |
185 | fastHan.model.bert | 186 | |
| 189 | |
190 | fastHan.model.bert_encoder_theseus | 191 | |
| 194 | |
195 | fastHan.model.BertCharParser | 196 | |
| 199 | |
200 | fastHan.model.CharParser | 201 | |
| 204 | |
205 | fastHan.model.model | 206 | |
| 209 | |
210 | fastHan.model.old_fastNLP_bert | 211 | |
| 214 | |
215 | fastHan.model.UserDict | 216 | |
| 219 | |
220 | fastHan.model.utils | 221 | |
47 |
48 |
99 |
100 |
101 |
102 |
103 |
109 |
110 |
111 |
213 |
214 |
215 |
216 |
217 |
222 |
223 |
224 |
225 |
226 |
227 |
228 |
229 |
--------------------------------------------------------------------------------
/fastHan/model/finetune_dataloader.py:
--------------------------------------------------------------------------------
1 | import itertools
2 | import numpy as np
3 |
4 | from fastNLP import DataSet
5 |
6 | MAX_LEN = 300
7 |
8 |
9 | def fastHan_CWS_Loader(lines, label_to_index, tokenizer):
10 | data = {
11 | 'input_ids': [],
12 | 'attention_mask': [],
13 | 'labels': [],
14 | 'seq_len': [],
15 | }
16 |
17 | for line in lines:
18 | line = line.strip()
19 | if len(line) <= 1:
20 | continue
21 |
22 | line = line.split(' ')
23 |
24 | words = []
25 | labels = []
26 | for word in line:
27 | if len(word) == 0:
28 | continue
29 | words = words + list(word)
30 | if len(word) == 1:
31 | labels.append('s')
32 | else:
33 | labels = labels + ['b'] + ['m'] * (len(word) - 2) + ['e']
34 |
35 | words = words[:MAX_LEN]
36 | labels = labels[:MAX_LEN]
37 | labels = [-100] + [label_to_index[x] for x in labels] + [-100]
38 |
39 | tokenize_result = tokenizer(words, is_split_into_words=True)
40 |
41 | if len(tokenize_result['input_ids']) != len(labels):
42 | continue
43 |
44 | data['input_ids'].append(tokenize_result['input_ids'])
45 | data['attention_mask'].append(tokenize_result['attention_mask'])
46 | data['labels'].append(labels)
47 | data['seq_len'].append(len(labels) - 2)
48 |
49 | return DataSet(data)
50 |
51 |
52 | def fastHan_POS_loader(lines, label_to_index, tokenizer):
53 | data = {
54 | 'input_ids': [],
55 | 'attention_mask': [],
56 | 'labels': [],
57 | 'seq_len': [],
58 | }
59 |
60 | words = []
61 | labels = []
62 |
63 | for line in lines:
64 | if line == '\n' and len(words) > 0:
65 | words = words[:MAX_LEN]
66 | labels = labels[:MAX_LEN]
67 | labels = [-100] + [label_to_index[x] for x in labels] + [-100]
68 |
69 | tokenize_result = tokenizer(words, is_split_into_words=True)
70 | if len(tokenize_result['input_ids']) == len(labels):
71 | data['input_ids'].append(tokenize_result['input_ids'])
72 | data['attention_mask'].append(
73 | tokenize_result['attention_mask'])
74 | data['labels'].append(labels)
75 | data['seq_len'].append(len(labels) - 2)
76 | words = []
77 | labels = []
78 | else:
79 | line = line.strip()
80 | line = line.split('\t')
81 | word = line[1]
82 | label = line[3].lower()
83 |
84 | words = words + list(word)
85 | if len(word) == 1:
86 | labels.append('s-' + label)
87 | else:
88 | labels = labels + [
89 | 'b-' + label
90 | ] + ['m-' + label] * (len(word) - 2) + ['e-' + label]
91 | return DataSet(data)
92 |
93 |
94 | # def fastHan_CWS_guwen_Loader():
95 | # pass
96 |
97 |
98 | def fastHan_POS_guwen_loader(lines, label_to_index, tokenizer):
99 | data = {'input_ids': [], 'attention_mask': [], 'labels': [], 'seq_len': []}
100 |
101 | for line in lines:
102 | line = line.strip()
103 | if len(line) <= 1:
104 | continue
105 |
106 | line = line.split(' ')
107 |
108 | words = []
109 | labels = []
110 | for word_label in line:
111 | if len(word_label) == 0:
112 | continue
113 |
114 | word, label = word_label.split('/')
115 | words = words + list(word)
116 | if len(word) == 1:
117 | labels.append('s-' + label)
118 | else:
119 | labels = labels + [
120 | 'b-' + label
121 | ] + ['m-' + label] * (len(word) - 2) + ['e-' + label]
122 |
123 | words = words[:MAX_LEN]
124 | labels = labels[:MAX_LEN]
125 | labels = [-100] + [label_to_index[x] for x in labels] + [-100]
126 |
127 | tokenize_result = tokenizer(words, is_split_into_words=True)
128 |
129 | if len(tokenize_result['input_ids']) != len(labels):
130 | continue
131 |
132 | data['input_ids'].append(tokenize_result['input_ids'])
133 | data['attention_mask'].append(tokenize_result['attention_mask'])
134 | data['labels'].append(labels)
135 | data['seq_len'].append(len(labels) - 2)
136 |
137 | return DataSet(data)
138 |
139 |
140 | def fastHan_NER_Loader(lines, label_to_index, tokenizer):
141 | data = {
142 | 'input_ids': [],
143 | 'attention_mask': [],
144 | 'labels': [],
145 | 'seq_len': [],
146 | }
147 |
148 | words = []
149 | labels = []
150 |
151 | for line in lines:
152 | if line == '\n' and len(words) > 0:
153 | words = words[:MAX_LEN]
154 | labels = labels[:MAX_LEN]
155 | labels = [-100] + [label_to_index[x] for x in labels] + [-100]
156 |
157 | tokenize_result = tokenizer(words, is_split_into_words=True)
158 | if len(tokenize_result['input_ids']) == len(labels):
159 | data['input_ids'].append(tokenize_result['input_ids'])
160 | data['attention_mask'].append(
161 | tokenize_result['attention_mask'])
162 | data['labels'].append(labels)
163 | data['seq_len'].append(len(labels) - 2)
164 | words = []
165 | labels = []
166 | else:
167 | line = line.strip()
168 | word, label = line.split(' ')
169 | label = label.lower()
170 |
171 | words.append(word)
172 | labels.append(label)
173 |
174 | return DataSet(data)
175 |
176 |
177 | def fastHan_Parsing_Loader(lines, label_to_index, tokenizer):
178 | data = {
179 | 'input_ids': [],
180 | 'attention_mask': [],
181 | 'heads': [],
182 | 'labels': [],
183 | 'seq_len': []
184 | }
185 |
186 | words = []
187 | heads = []
188 | labels = []
189 |
190 | skip_1 = 0
191 | skip_2 = 0
192 | for line in lines:
193 | if line == '\n' and len(words) > 0:
194 | char_words = list(itertools.chain(*words))
195 | if len(char_words) > 300:
196 | skip_1 += 1
197 | words = []
198 | heads = []
199 | labels = []
200 | continue
201 | tokenize_result = tokenizer(char_words, is_split_into_words=True)
202 | if len(tokenize_result['input_ids']) - 2 != len(char_words):
203 | skip_2 += 1
204 | words = []
205 | heads = []
206 | labels = []
207 | continue
208 |
209 | # 添加根结点
210 | tokenize_result['input_ids'].insert(1, 1)
211 | tokenize_result['attention_mask'].insert(1, 1)
212 |
213 | head_end_indexes = np.cumsum(list(map(len, words))).tolist() + [0]
214 | char_index = 1
215 |
216 | char_heads = []
217 | char_labels = []
218 |
219 | for word, head, label in zip(words, heads, labels):
220 | for _ in range(len(word) - 1):
221 | char_index += 1
222 | char_heads.append(char_index)
223 | char_labels.append('app')
224 | char_index += 1
225 | char_heads.append(head_end_indexes[head - 1])
226 | char_labels.append(label)
227 |
228 | # 根节点的label都是-100
229 | labels = [-100] + [label_to_index[x] for x in char_labels]
230 | char_heads = [-100] + char_heads
231 |
232 | data['input_ids'].append(tokenize_result['input_ids'])
233 | data['attention_mask'].append(tokenize_result['attention_mask'])
234 | data['labels'].append(labels)
235 | data['heads'].append(char_heads)
236 | data['seq_len'].append(len(labels))
237 |
238 | words = []
239 | heads = []
240 | labels = []
241 | else:
242 | line = line.strip()
243 | line = line.split('\t')
244 |
245 | word = line[1]
246 | head = line[6].lower()
247 | label = line[7].lower()
248 |
249 | words.append(word)
250 | labels.append(label)
251 | heads.append(int(head))
252 | return DataSet(data)
--------------------------------------------------------------------------------
/fastHan/model/baseModel.py:
--------------------------------------------------------------------------------
1 | # from multitask_model_normloss2
2 | # 使用 wm来管理
3 |
4 | import torch
5 | import torch.nn as nn
6 | import torch.nn.functional as F
7 | from fastNLP.modules.torch import MLP, ConditionalRandomField, allowed_transitions
8 | from torch.nn import CrossEntropyLoss
9 | from transformers import PreTrainedModel
10 |
11 | from .dependency_parsing_model import BertParser
12 |
13 |
14 | # modified from https://github.com/THUDM/P-tuning-v2
15 | class PrefixEncoder(torch.nn.Module):
16 | r'''
17 | The torch.nn model to encode the prefix
18 | Input shape: (batch-size, prefix-length)
19 | Output shape: (batch-size, prefix-length, 2*layers*hidden)
20 | '''
21 | def __init__(self,
22 | config,
23 | num_tokens,
24 | prefix_projection,
25 | pre_seq_len,
26 | prefix_hidden_size=500):
27 | super().__init__()
28 | self.prefix_projection = prefix_projection
29 | if self.prefix_projection:
30 | # Use a two-layer MLP to encode the prefix
31 | self.embedding = torch.nn.Embedding(num_tokens, config.hidden_size)
32 | self.trans = torch.nn.Sequential(
33 | torch.nn.Linear(config.hidden_size, prefix_hidden_size),
34 | torch.nn.Tanh(),
35 | torch.nn.Linear(
36 | prefix_hidden_size,
37 | config.num_hidden_layers * 2 * config.hidden_size))
38 | else:
39 | self.embedding = torch.nn.Embedding(
40 | num_tokens, config.num_hidden_layers * 2 * config.hidden_size)
41 |
42 | def forward(self, prefix: torch.Tensor):
43 | if self.prefix_projection:
44 | prefix_tokens = self.embedding(prefix)
45 | past_key_values = self.trans(prefix_tokens)
46 | else:
47 | past_key_values = self.embedding(prefix)
48 | return past_key_values
49 |
50 |
51 | class MultiTaskModel(PreTrainedModel):
52 | def __init__(self,
53 | encoder,
54 | task_label_map,
55 | all_tasks,
56 | ensembledWeightManager,
57 | prefix_projection=False,
58 | pre_seq_len=6,
59 | biaffine_task='Parsing-ctb9'):
60 | super().__init__(encoder.config)
61 |
62 | self.all_tasks = all_tasks
63 | self.task_label_map = task_label_map
64 | self.ensembledWeightManager = ensembledWeightManager
65 | # sequence_labeling
66 | self.biaffine_task = biaffine_task
67 | self.seq_label_classifier = nn.ModuleDict()
68 | self.crf = nn.ModuleDict()
69 |
70 | for task in task_label_map:
71 | if task == self.biaffine_task:
72 | continue
73 |
74 | label_num = len(task_label_map[task])
75 | labels = {i: task_label_map[task][i] for i in range(label_num)}
76 |
77 | self.seq_label_classifier[task] = MLP(
78 | [encoder.config.hidden_size, 512, label_num])
79 | self.crf[task] = ConditionalRandomField(
80 | num_tags=label_num,
81 | allowed_transitions=allowed_transitions(labels))
82 | self.crf[task].trans_m.data *= 0
83 |
84 | self.parser = BertParser(
85 | num_label=len(task_label_map[self.biaffine_task]),
86 | embed_size=encoder.config.hidden_size,
87 | app_index=task_label_map[self.biaffine_task].index('app'))
88 | self.dropout = nn.Dropout(encoder.config.hidden_dropout_prob)
89 | self.encoder = encoder
90 |
91 | # prefix tuning
92 | self.pre_seq_len = pre_seq_len
93 | self.build_prefix_map()
94 | self.prefix_encoder = PrefixEncoder(
95 | config=encoder.config,
96 | prefix_projection=prefix_projection,
97 | num_tokens=self.num_tokens,
98 | pre_seq_len=self.pre_seq_len)
99 |
100 | def build_prefix_map(self):
101 | length = self.pre_seq_len // 2
102 | idx = 0
103 | prefix_map = dict()
104 | macro_map = dict()
105 | for task in self.all_tasks:
106 | macro_task, _ = task.split('-')
107 | if macro_task not in macro_map:
108 | macro_map[macro_task] = [idx + i for i in range(length)]
109 | idx += length
110 |
111 | for task in self.all_tasks:
112 | macro_task, _ = task.split('-')
113 | prefix_map[task] = torch.LongTensor(
114 | macro_map[macro_task] + [idx + i for i in range(length)])
115 | idx += length
116 | self.prefix_map = prefix_map
117 | self.num_tokens = idx
118 |
119 | def get_prompt(self, task, batch_size):
120 | prefix_tokens = self.prefix_map[task]
121 | prefix_tokens = prefix_tokens.unsqueeze(0).expand(batch_size, -1).to(
122 | self.encoder.device)
123 | past_key_values = self.prefix_encoder(prefix_tokens)
124 |
125 | past_key_values = past_key_values.view(
126 | batch_size, self.pre_seq_len,
127 | self.encoder.config.num_hidden_layers * 2,
128 | self.encoder.config.num_attention_heads,
129 | self.encoder.config.hidden_size //
130 | self.encoder.config.num_attention_heads)
131 | past_key_values = self.dropout(past_key_values)
132 | past_key_values = past_key_values.permute([2, 0, 3, 1, 4]).split(2)
133 | return past_key_values
134 |
135 | def __get_ud_diff(self, feats, tag_seqs, user_dict_weight):
136 | diff = torch.max(feats, dim=2)[0] - torch.mean(feats, dim=2)
137 | diff = diff.unsqueeze(dim=-1)
138 | diff = diff.expand(-1, -1, tag_seqs.size()[-1])
139 | diff = tag_seqs * diff * user_dict_weight
140 | return diff
141 |
142 | def forward(
143 | self,
144 | input_ids=None,
145 | attention_mask=None,
146 | task=None,
147 | labels=None,
148 | heads=None,
149 | tag_seqs=None,
150 | user_dict_weight=0.05,
151 | ):
152 | r"""
153 | labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`):
154 | Labels for computing the token classification loss. Indices should be in ``[0, ..., config.num_labels -
155 | 1]``.
156 | """
157 |
158 | task = task.tolist()[0]
159 | task = self.all_tasks[task]
160 |
161 | batch_size = input_ids.shape[0]
162 | past_key_values = self.get_prompt(task=task, batch_size=batch_size)
163 | prefix_attention_mask = torch.ones(batch_size, self.pre_seq_len).to(
164 | self.encoder.device)
165 | prefix_attention_mask = torch.cat(
166 | (prefix_attention_mask, attention_mask), dim=1)
167 |
168 | outputs = self.encoder(input_ids,
169 | attention_mask=prefix_attention_mask,
170 | past_key_values=past_key_values)
171 |
172 | feats = outputs[0]
173 | feats = self.dropout(feats)
174 |
175 | # 生成不考虑cls和sep的mask
176 | #if task==self.biaffine_task:
177 | seq_len_diff = 2
178 | #else:
179 | # seq_len_diff=1
180 | seq_len = attention_mask.sum(dim=-1) - seq_len_diff
181 | broad_cast_seq_len = torch.arange(attention_mask.shape[1] -
182 | seq_len_diff).expand(
183 | attention_mask.shape[0],
184 | -1).to(seq_len.device)
185 | mask = broad_cast_seq_len < seq_len.unsqueeze(1)
186 |
187 | # dependency parsing
188 | # 需要去掉cls和sep的影响
189 | if task == self.biaffine_task:
190 | feats = feats[:, 1:-1]
191 | output = self.parser.forward(feats=feats,
192 | mask=mask,
193 | gold_heads=heads,
194 | char_labels=labels)
195 | # 其他序列标注任务
196 | else:
197 | logits = self.seq_label_classifier[task](feats)
198 | if self.training:
199 | loss_fct = CrossEntropyLoss()
200 | loss = loss_fct(
201 | logits.view(-1, len(self.task_label_map[task])),
202 | labels.view(-1))
203 | output = {
204 | 'loss': loss,
205 | 'logits': logits,
206 | }
207 | else:
208 | # 预测阶段，利用crf模块中集成的维特比解码来预测
209 | probs = logits[:, 1:-1]
210 | if tag_seqs is not None:
211 | diff = self.__get_ud_diff(probs, tag_seqs, user_dict_weight)
212 | probs = probs + diff
213 | paths, scores = self.crf[task].viterbi_decode(logits=probs,
214 | mask=mask)
215 | paths[mask == 0] = -100
216 | output = {
217 | 'pred': paths,
218 | 'logits': logits,
219 | }
220 |
221 | if self.training:
222 | if task == self.biaffine_task:
223 | loss_weight = output['label_loss']
224 | else:
225 | loss_weight = output['loss']
226 | self.ensembledWeightManager.update(task=task,
227 | loss=float(loss_weight))
228 | weight = self.ensembledWeightManager.get(task)
229 | output['loss'] = output['loss'] * weight
230 |
231 | return output
--------------------------------------------------------------------------------
/docs/build/html/searchindex.js:
--------------------------------------------------------------------------------
1 | Search.setIndex({docnames:["fastHan","fastHan.FastModel","fastHan.model","fastHan.model.BertCharParser","fastHan.model.CharParser","fastHan.model.UserDict","fastHan.model.bert","fastHan.model.bert_encoder_theseus","fastHan.model.model","fastHan.model.old_fastNLP_bert","fastHan.model.utils","index","user/example"],envversion:{"sphinx.domains.c":1,"sphinx.domains.changeset":1,"sphinx.domains.cpp":1,"sphinx.domains.javascript":1,"sphinx.domains.math":2,"sphinx.domains.python":1,"sphinx.domains.rst":1,"sphinx.domains.std":1,"sphinx.ext.todo":1,"sphinx.ext.viewcode":1,sphinx:55},filenames:["fastHan.rst","fastHan.FastModel.rst","fastHan.model.rst","fastHan.model.BertCharParser.rst","fastHan.model.CharParser.rst","fastHan.model.UserDict.rst","fastHan.model.bert.rst","fastHan.model.bert_encoder_theseus.rst","fastHan.model.model.rst","fastHan.model.old_fastNLP_bert.rst","fastHan.model.utils.rst","index.rst","user/example.rst"],objects:{"":{fastHan:[0,0,0,"-"]},"fastHan.FastHan":{__init__:[0,2,1,""],add_user_dict:[0,2,1,""],remove_user_dict:[0,2,1,""],set_cws_style:[0,2,1,""],set_device:[0,2,1,""],set_user_dict_weight:[0,2,1,""]},"fastHan.FastModel":{FastHan:[1,1,1,""],Sentence:[1,1,1,""],Token:[1,1,1,""]},"fastHan.FastModel.FastHan":{__init__:[1,2,1,""],add_user_dict:[1,2,1,""],remove_user_dict:[1,2,1,""],set_cws_style:[1,2,1,""],set_device:[1,2,1,""],set_user_dict_weight:[1,2,1,""]},"fastHan.FastModel.Sentence":{__init__:[1,2,1,""]},"fastHan.FastModel.Token":{__init__:[1,2,1,""]},"fastHan.Sentence":{__init__:[0,2,1,""]},"fastHan.Token":{__init__:[0,2,1,""]},"fastHan.model":{BertCharParser:[3,0,0,"-"],CharParser:[4,0,0,"-"],UserDict:[5,0,0,"-"],bert:[6,0,0,"-"],bert_encoder_theseus:[7,0,0,"-"],model:[8,0,0,"-"],old_fastNLP_bert:[9,0,0,"-"],utils:[10,0,0,"-"]},"fastHan.model.BertCharParser":{BertCharParser:[3,1,1,""],CharBiaffineParser:[3,1,1,""],drop_input_independent:[3,4,1,""]},"fastHan.model.BertCharParser.BertCharParser":{__init__:[3,2,1,""],forward:[3,2,1,""],predict:[3,2,1,""]},"fastHan.model.BertCharParser.CharBiaffineParser":{__init__:[3,2,1,""],forward:[3,2,1,""],loss:[3,3,1,""],reset_parameters:[3,2,1,""]},"fastHan.model.CharParser":{CharBiaffineParser:[4,1,1,""],CharParser:[4,1,1,""],drop_input_independent:[4,4,1,""]},"fastHan.model.CharParser.CharBiaffineParser":{__init__:[4,2,1,""],forward:[4,2,1,""],loss:[4,3,1,""],predict:[4,2,1,""],reset_parameters:[4,2,1,""]},"fastHan.model.CharParser.CharParser":{__init__:[4,2,1,""],forward:[4,2,1,""],predict:[4,2,1,""]},"fastHan.model.UserDict":{Trie:[5,1,1,""],UserDict:[5,1,1,""]},"fastHan.model.UserDict.Trie":{__init__:[5,2,1,""],insert:[5,2,1,""],search:[5,2,1,""],startsWith:[5,2,1,""]},"fastHan.model.UserDict.UserDict":{__init__:[5,2,1,""],load_file:[5,2,1,""],load_list:[5,2,1,""],process_sentence:[5,2,1,""]},"fastHan.model.bert":{BertEmbedding:[6,1,1,""],BertWordPieceEncoder:[6,1,1,""]},"fastHan.model.bert.BertEmbedding":{__init__:[6,2,1,""],drop_word:[6,2,1,""],forward:[6,2,1,""]},"fastHan.model.bert.BertWordPieceEncoder":{__init__:[6,2,1,""],drop_word:[6,2,1,""],embed_size:[6,5,1,""],embedding_dim:[6,5,1,""],forward:[6,2,1,""],index_datasets:[6,2,1,""],num_embedding:[6,5,1,""]},"fastHan.model.bert_encoder_theseus":{BertModel:[7,1,1,""]},"fastHan.model.bert_encoder_theseus.BertModel":{__init__:[7,2,1,""],forward:[7,2,1,""],from_pretrained:[7,6,1,""],init_bert_weights:[7,2,1,""]},"fastHan.model.model":{CharModel:[8,1,1,""]},"fastHan.model.model.CharModel":{__init__:[8,2,1,""],forward:[8,2,1,""],predict:[8,2,1,""]},"fastHan.model.old_fastNLP_bert":{BertModel:[9,1,1,""]},"fastHan.model.old_fastNLP_bert.BertModel":{__init__:[9,2,1,""],forward:[9,2,1,""],from_pretrained:[9,6,1,""],init_bert_weights:[9,2,1,""]},"fastHan.model.utils":{check_dataloader_paths:[10,4,1,""],get_tokenizer:[10,4,1,""]},fastHan:{FastHan:[0,1,1,""],FastModel:[1,0,0,"-"],Sentence:[0,1,1,""],Token:[0,1,1,""],model:[2,0,0,"-"]}},objnames:{"0":["py","module","Python \u6a21\u5757"],"1":["py","class","Python \u7c7b"],"2":["py","method","Python \u65b9\u6cd5"],"3":["py","staticmethod","Python \u9759\u6001\u65b9\u6cd5"],"4":["py","function","Python \u51fd\u6570"],"5":["py","attribute","Python \u5c5e\u6027"],"6":["py","classmethod","Python \u7c7b\u65b9\u6cd5"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:staticmethod","4":"py:function","5":"py:attribute","6":"py:classmethod"},terms:{"02":[7,9],"05":[0,1],"10":[0,1,6],"100":[3,4,8],"12":[6,7,9],"2304":6,"30522":[7,9],"3072":[7,9],"31":8,"400":4,"430":6,"500":[3,4,8],"510":6,"512":[6,7,9],"768":[6,7,8,9],"800":4,"char":[3,4,8],"class":[0,1,3,4,5,6,7,8,9],"default":[3,4],"float":[0,1,6,7,9,12],"for":[0,1,5,6],"if":[3,4,5],"import":6,"in":[5,6],"int":[0,1,7,9],"long":12,"return":[3,4,5],"static":[3,4],"true":[6,7,8,9,12],"var":[3,4],"void":5,"with":[5,12],__init__:[0,1,3,4,5,6,7,8,9],accur:[0,1,5],add_cls_sep:6,add_user_dict:[0,1],add_word_lst:6,an:12,and:[3,4,7,8,9],ani:5,answer_list:[0,1],app_index:4,appli:12,arc_mlp_siz:[3,4],arc_pr:[3,4],arc_tru:[3,4],as:[0,1],attent:[7,9],attention_mask:[7,9],attention_probs_dropout_prob:[7,9],auto_trunc:6,avg:6,base:[0,1,6,7,9],batch:12,batch_siz:[3,4,6],bert:[0,1,2,7,9],bert_encoder_theseu:[0,2],bertcharpars:[0,2],bertembed:6,bertmodel:[7,9],bertpool:6,bertwordpieceencod:6,beta:12,biaffine_pars:[3,4],biaffinepars:[3,4],bidirect:[7,9],bigram:4,bigram_vocab_s:4,bin:6,bmeso:8,bodi:12,bool:[5,6],both:[3,4,7,8,9],by:[3,4,7,8,9],cell:12,char_head:[3,4],char_label:[3,4],char_vocab_s:4,charbiaffinepars:[3,4],charmodel:[0,1,8],charpars:[0,2],check_dataloader_path:10,cityu:[0,1],classmethod:[7,9],cls:6,cnc:[0,1],column:12,com:[7,9],comput:[3,4],concat:6,config:[7,9],contain:12,contextual_embed:6,contextualembed:6,core:6,corpu:[0,1],cpu:[0,1],ctb:[0,1],cuda:[0,1],cws:[0,1],data:5,dataload:10,dataset:[6,12],delta:12,dep_head:8,dep_label:8,depend:[0,1],dev:[0,1,10],devic:[0,1],device_id:[0,1],dic:[0,1],dict:[3,4,10],doc:12,docutil:12,drop_input_independ:[3,4],drop_word:6,dropout:[3,4,6,7,8,9],dropout_emb:[3,4],element:12,emb:[3,4,6,8],emb_dim:4,embed:[3,4,6,7,9],embed_s:6,embedding_dim:[3,4,6,8],emphas:12,emphasi:12,en:6,encod:[3,4,5,6],encoding_typ:8,english:[7,9],exampl:6,f_:12,f_beta:12,fals:[3,4,5,6,8,12],fastmodel:0,fastnlp:[3,4,6,12],feat:3,ffn:[7,9],field_nam:6,file:12,first:[6,12],floattensor:6,forward:[3,4,6,7,8,9],frac:12,from:[6,7,9],from_pretrain:[7,9],gelu:[7,9],get_token:10,github:[7,9],given:5,gold_head:[3,4],good:6,gpu:[0,1],gradient:6,h_2:12,h_2o:12,head:[0,1,3,4],head_label:[0,1],head_pr:[3,4],header:12,help:[0,1,5],here:5,hidden_act:[7,9],hidden_dropout_prob:[7,9],hidden_s:[7,9],html:12,http:[7,9,12],huggingfac:[7,9],include_cls_sep:6,index:10,index_dataset:6,infer:[3,4],init_bert_weight:[7,9],initi:[0,1,3,4,5,7,8,9],initializer_rang:[7,9],inlin:12,input:[6,7,9,12],input_id:[7,9],insert:5,instanc:12,intermediate_s:[7,9],intern:[3,4,7,8,9],into:5,is:[3,4,5,6,12],json:6,kei:10,kwarg:[7,9],label_mlp_s:[3,4],label_pr:[3,4],label_tru:[3,4],label_vocab:8,larg:[0,1],last:6,layer:6,layer_num:[6,7],layers_cut:9,len:[6,12],line:12,lineno:12,list:[0,1,6],liter:12,load_fil:5,load_list:5,longtensor:6,loss:[3,4],lstm:[3,4],mai:12,markdown:12,mask:[3,4],math:12,max:6,max_len:[3,4,6],max_position_embed:[7,9],mlp:[3,4],model:0,model_dir_or_nam:[6,7,9],model_typ:[0,1],modul:[0,2],msr:[0,1],mst:[3,4],n_tag:[3,4],na:12,naoh:12,ner:[0,1],net:12,ngram:[3,4],ngram_per_cha:4,ngram_per_char:[3,4],nn:[3,4,6,7,8,9],none:[0,1,3,4,6,7,8,9],not:[3,4],num_attention_head:[7,9],num_embed:[3,4,6],num_hidden_lay:[7,9],num_label:[3,4],object:[0,1,5],of:[3,4],old_fastnlp_bert:[0,2],option:12,or:12,output:[6,12],output_all_encoded_lay:[7,9],pad:6,param:[0,1,3,4,12],pars:[0,1,3,4],parsing_arc_mlp_s:8,parsing_label_mlp_s:8,parsing_rnn_lay:8,parsing_use_greedy_inf:8,part:[3,4],path:[5,10],piec:6,pku:[0,1],pool_method:6,pooled_cl:6,pos:[0,1,8],pos_emb_dim:[3,4],pos_idx:8,pos_vocab_s:[3,4],pre:12,pre_bigram:4,pre_bigrams_emb:4,pre_char:[3,4],pre_chars_emb:4,pre_ngram:[3,4],pre_trigram:4,pre_trigrams_emb:4,predict:[3,4,8],prefix:5,pretrain:[7,9],print:12,process_sent:5,provid:[3,4],pytorch:[7,9],quickref:12,rais:10,rec:12,regular:6,remove_user_dict:[0,1],represent:[7,9],requires_grad:6,reset_paramet:[3,4],result:[3,4],revers:5,rnn:[3,4],rnn_hidden_s:[3,4],rnn_layer:[3,4],root:[0,1,3,4],row:12,rst:12,rtype:5,same:12,scriptmodul:[3,4,7,8,9],search:5,second:12,see:[0,1,5],segment:[7,9],self:[0,1,5,6],sentenc:[0,1,5,12],sep:6,seq_len:[3,4,8],seq_len_for_wordlist:8,set:[0,1],set_cws_styl:[0,1],set_devic:[0,1],set_user_dict_weight:[0,1],sf:12,share:[3,4,7,8,9],signatur:[0,1,5],size:6,sohu:12,some:10,sourceforg:12,span:12,speech:[3,4],split:6,stackembed:6,start:5,startswith:5,state:[3,4,7,8,9],str:[0,1,5,6,7,9,10],strong:12,structur:5,style:[0,1],sxu:[0,1],tabl:12,tag:6,tag_seq:8,target:[0,1,8,12],task_class:8,test:[0,1,10],text:12,that:5,the:[3,4,5,6,7,9,12],there:5,thi:12,third:12,to:10,to_index:6,token:[0,1,6],token_type_id:[6,7,9],torch:[0,1,3,4,6,7,8,9],train:10,transform:[3,4,7,9],trie:5,trigram:4,trigram_vocab_s:4,tupl:[3,4],txt:[6,10],type:[0,1,5],type_vocab_s:[7,9],udc:[0,1],uncas:[6,7,9],undocu:[7,9],union:10,unk:6,unknown_index:6,uparrow:12,use_averag:8,use_greedy_inf:[3,4],use_pos_embed:8,user:12,userdict:[0,2],utf:5,util:[0,2],valu:[3,4,6,10,12],vector_s:3,vocab:6,vocab_s:[7,9],vocabulari:6,weight:[0,1,7,9],whether:6,word:[0,1,5,6],word_dropout:6,word_embed:[3,4],word_len:8,word_list:5,word_piec:6,wtb:[0,1],xxx:10,your:5,zx:[0,1]},titles:["fastHan package","fastHan.FastModel module","fastHan.model package","fastHan.model.BertCharParser module","fastHan.model.CharParser module","fastHan.model.UserDict module","fastHan.model.bert module","fastHan.model.bert_encoder_theseus module","fastHan.model.model module","fastHan.model.old_fastNLP_bert module","fastHan.model.utils module","demo \u4e2d\u6587\u6587\u6863","\u5927\u6807\u9898"],titleterms:{api:11,bert:6,bert_encoder_theseu:7,bertcharpars:3,charpars:4,csv:12,demo:11,fasthan:[0,1,2,3,4,5,6,7,8,9,10],fastmodel:1,model:[2,3,4,5,6,7,8,9,10],modul:[1,3,4,5,6,7,8,9,10],old_fastnlp_bert:9,packag:[0,2],submodul:[0,2],subpackag:0,userdict:5,util:10}})
--------------------------------------------------------------------------------
/fastHan/model/camr_restore.py:
--------------------------------------------------------------------------------
1 | import re
2 |
3 |
4 | def restore_camr(line, id_token_list):
5 | # 预处理内容
6 | line = re.sub('（', '(', line)
7 | line = re.sub('）', ')', line)
8 | line = re.sub('：', ':', line)
9 | time_list = re.findall('\d+\s*\:\s*\d+', line)
10 | line = re.sub('"', '', line)
11 | for s in time_list:
12 | ss = re.sub(':', '：', s)
13 | line = re.sub(s, ss, line)
14 |
15 | idx = 0
16 | amr_list = []
17 |
18 | new_mark = len(id_token_list) + 2
19 | node_dict = {}
20 | node_name_list = []
21 |
22 | def convert_node_value(node_value, is_bracket1=False):
23 | nonlocal id_token_list, node_name_list
24 | if not node_value:
25 | return node_value
26 | if '^' not in node_value:
27 | node_name = search_mark(node_value.split('-')[0])
28 | if not is_bracket1:
29 | node_name_list.append(node_name)
30 | return node_name + '/' + node_value
31 | else:
32 | node_value1, node_value2 = node_value.split('^')[0:2]
33 | node_name1 = search_mark(node_value1.split('-')[0])
34 | if int(node_name1[1:]) > len(id_token_list):
35 | node_name1 = 'x1'
36 | node_name2 = search_mark(node_value2.split('-')[0])
37 | if not node_name2 in node_name_list:
38 | node_name2 = node_name_list[0]
39 | return node_name1 + '/' + node_name2
40 |
41 | def search_mark(node_value):
42 | nonlocal id_token_list, node_dict, new_mark
43 | node_name = node_dict.get(node_value, 0)
44 | if node_name > len(id_token_list):
45 | node_name = new_mark
46 | new_mark += 1
47 | node_dict[node_value] = node_name
48 | return 'x' + str(node_name)
49 |
50 | i = node_name + 1
51 | while i <= len(id_token_list):
52 | if id_token_list[i] == node_value:
53 | break
54 | i += 1
55 | if i > len(id_token_list):
56 | j = 1
57 | while j <= node_name:
58 | if id_token_list[j] == node_value:
59 | break
60 | j += 1
61 | if node_name >= 1 and j <= node_name:
62 | node_name = j
63 | else:
64 | node_name = i
65 |
66 | if node_name <= 0:
67 | node_name = new_mark
68 | new_mark += 1
69 |
70 | node_dict[node_value] = node_name
71 | return 'x' + str(node_name)
72 |
73 | # 递归处理部分
74 | def bracket1():
75 | nonlocal idx
76 | nonlocal amr_list
77 | cur_charseq = []
78 | cur_idx = idx
79 | has_content = False
80 | while True:
81 | if cur_idx >= len(line):
82 | break
83 | if line[cur_idx] == '(' and not has_content:
84 | has_content = True
85 | if has_content:
86 | if line[cur_idx] == ':':
87 | cur_idx = idx
88 | cur_charseq = []
89 | break
90 | if line[cur_idx] == '(' and cur_charseq:
91 | break
92 | cur_charseq.append(line[cur_idx])
93 | if cur_charseq[-1] == ')':
94 | has_content = False
95 | cur_idx += 1
96 | break
97 | cur_idx += 1
98 |
99 | idx = cur_idx
100 | if not cur_charseq:
101 | amr_list.append('(')
102 | amr_list.append(')')
103 | cur_node_value = ''
104 | elif cur_charseq[-1] != ')':
105 | cur_node_value = get_seq_value(cur_charseq[1:])
106 | amr_list.append('(')
107 | amr_list.append(convert_node_value(cur_node_value, True))
108 | amr_list.append(')')
109 | else:
110 | cur_node_value = get_seq_value(cur_charseq[1:-1])
111 | amr_list.append('(')
112 | amr_list.append(convert_node_value(cur_node_value, True))
113 | amr_list.append(')')
114 | pass
115 |
116 | def bracket2():
117 | nonlocal idx
118 | nonlocal amr_list
119 | nonlocal id_token_list
120 | cur_charseq = []
121 | cur_idx = idx
122 |
123 | # 寻找左括号
124 | while cur_idx < len(line):
125 | if line[cur_idx] == '(':
126 | break
127 | cur_idx += 1
128 | if cur_idx < len(line):
129 | amr_list.append('(')
130 | else:
131 | amr_list.append('(')
132 | amr_list.append(convert_node_value('-'))
133 | amr_list.append(')')
134 | idx = cur_idx
135 | return
136 |
137 | # 寻找变量名
138 | cur_idx += 1
139 | while cur_idx < len(line):
140 | if line[cur_idx] == ')' or line[cur_idx] == ':':
141 | break
142 | if line[cur_idx] == '(':
143 | break
144 | cur_charseq.append(line[cur_idx])
145 | cur_idx += 1
146 | cur_node_value = get_seq_value(cur_charseq)
147 | if not cur_node_value:
148 | cur_node_value = '-'
149 | amr_list.append(convert_node_value(cur_node_value))
150 | cur_charseq.clear()
151 |
152 | if cur_node_value == 'name':
153 | name_idx = len(amr_list) - 1
154 | n_value_list = []
155 | while cur_idx < len(line) and line[cur_idx] != ')':
156 | cur_charseq.append(line[cur_idx])
157 | cur_idx += 1
158 | cur_charseq.append(':')
159 | l = r = 0
160 | r_value = ''
161 | for i, ch in enumerate(cur_charseq):
162 | if ch == ':':
163 | l = i
164 | n_value = get_seq_value(cur_charseq[r:l])
165 | if not n_value:
166 | n_value = '-'
167 | if r_value:
168 | amr_list.append(' ')
169 | amr_list.append(r_value)
170 | amr_list.append(' ')
171 | n_value = convert_node_value(n_value, True)
172 | if n_value.split('/')[-1].startswith('x'):
173 | n_value = n_value.split(
174 | '/')[0] + '/' + id_token_list[int(
175 | n_value.split('/')[0][1:])]
176 | amr_list.append(n_value)
177 | n_value_list.append(n_value.split('/')[0])
178 | n_value = r_value = ""
179 | elif ch.isspace() and i > 0 and cur_charseq[i - 1].isdigit():
180 | r = i
181 | r_value = get_seq_value(cur_charseq[l:r])
182 | if not r_value:
183 | r_value = 'op1'
184 | r_value = ':' + r_value
185 | cur_idx += 1
186 | amr_list.append(')')
187 | idx = cur_idx
188 | if n_value_list:
189 | amr_list[name_idx] = '_'.join(n_value_list) + '/' + 'name'
190 | return
191 |
192 | # 寻找关系名
193 | def relation():
194 | nonlocal cur_idx, idx
195 | cur_relationseq = []
196 | relation_content = False
197 | while cur_idx < len(line):
198 | if line[cur_idx] == ')':
199 | return relation_content
200 | if line[cur_idx] == ':':
201 | break
202 | cur_idx += 1
203 |
204 | if cur_idx >= len(line):
205 | return relation_content
206 |
207 | while cur_idx < len(line):
208 | if line[cur_idx] == ':':
209 | relation_content = True
210 | cur_relationseq = []
211 | if relation_content:
212 | if line[cur_idx] == '(':
213 | cur_relation_value = get_seq_value(cur_relationseq)
214 | amr_list.append(' ')
215 | amr_list.append(':' + cur_relation_value)
216 | idx = cur_idx
217 | bracket1()
218 | bracket2()
219 | cur_idx = idx
220 | break
221 | elif line[cur_idx] == ')':
222 | cur_relation_value = get_seq_value(cur_relationseq)
223 | amr_list.append(' ')
224 | amr_list.append(':' + cur_relation_value)
225 | amr_list.append('()(')
226 | amr_list.append(convert_node_value('-'))
227 | amr_list.append(')')
228 | cur_idx += 1
229 | break
230 | cur_relationseq.append(line[cur_idx])
231 | cur_idx += 1
232 | idx = cur_idx
233 | return relation_content
234 |
235 | while True:
236 | if not relation():
237 | break
238 |
239 | while cur_idx < len(line):
240 | if line[cur_idx] == ')':
241 | break
242 | cur_idx += 1
243 | amr_list.append(')')
244 | cur_idx = cur_idx + 1
245 | idx = cur_idx
246 | return
247 |
248 | bracket2()
249 | return amr_list
250 |
251 |
252 | def get_seq_value(cur_charseq):
253 | for idx, ch in enumerate(cur_charseq):
254 | if ch.isspace():
255 | cur_charseq[idx] = ''
256 | s = ''.join(cur_charseq)
257 | s = re.sub(':', '', s)
258 | s = re.sub('\(', '', s)
259 | s = re.sub('\)', '', s)
260 | return s
261 |
262 |
263 | def convert_camr_to_lines(amr):
264 | amr_list = amr.split(':')
265 | for i, line in enumerate(amr_list):
266 | if i == 0:
267 | continue
268 | amr_list[i] = ':' + line
269 |
270 | num = 0
271 | for i, line in enumerate(amr_list):
272 | if '(' not in line:
273 | continue
274 | amr_list[i] = '\t' * num + line
275 | num = num + line.count('(') - line.count(')')
276 |
277 | for i, line in enumerate(amr_list):
278 | if i == 0:
279 | continue
280 | if '(' in line:
281 | amr_list[i] = '\n' + line
282 |
283 | return ''.join(amr_list) + '\n'
--------------------------------------------------------------------------------
/docs/build/html/fastHan.model.UserDict.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
112 |
113 |
211 |
212 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
153 |
210 |
154 |
186 |
208 |
209 |
155 |
156 |
184 |
185 |
157 |
181 |
182 |
183 | fastHan.model.model module¶
158 |-
159 |
-
160 | class
fastHan.model.model.CharModel(embed, label_vocab, pos_idx=31, Parsing_rnn_layers=3, Parsing_arc_mlp_size=500, Parsing_label_mlp_size=100, Parsing_use_greedy_infer=False, encoding_type='bmeso', embedding_dim=768, dropout=0.1, use_pos_embedding=True, use_average=True)[源代码]¶
161 | 基类：
162 |torch.nn.modules.module.Module-
163 |
-
164 |
__init__(embed, label_vocab, pos_idx=31, Parsing_rnn_layers=3, Parsing_arc_mlp_size=500, Parsing_label_mlp_size=100, Parsing_use_greedy_infer=False, encoding_type='bmeso', embedding_dim=768, dropout=0.1, use_pos_embedding=True, use_average=True)[源代码]¶
165 | Initializes internal Module state, shared by both nn.Module and ScriptModule.
166 |
-
164 |
47 |
48 |
99 |
100 |
101 |
102 |
103 |
109 |
110 |
111 |
254 |
255 |
256 |
257 |
258 |
263 |
264 |
265 |
266 |
267 |
268 |
269 |
270 |
--------------------------------------------------------------------------------
112 |
113 |
252 |
253 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
153 |
251 |
154 |
227 |
249 |
250 |
155 |
156 |
225 |
226 |
157 |
222 |
223 |
224 | fastHan.model.UserDict module¶
158 |-
159 |
-
160 | class
fastHan.model.UserDict.Trie[源代码]¶
161 | 基类：
162 | 167 | 168 |object-
169 |
-
170 |
insert(word)[源代码]¶
171 | Inserts a word into the trie. 172 | :type word: str 173 | :rtype: void
174 |
-
170 |