├── README.md ├── dict ├── label_dict ├── postag_dict ├── sub_tag ├── obj_tag ├── p_eng └── type_dic ├── .DS_Store ├── RC ├── .DS_Store ├── RC.sh └── models.py ├── bert ├── .DS_Store ├── bert_code │ ├── .DS_Store │ ├── requirements.txt │ ├── __init__.py │ ├── CONTRIBUTING.md │ ├── optimization_test.py │ ├── sample_text.txt │ ├── tokenization_test.py │ ├── optimization.py │ ├── modeling_test.py │ ├── LICENSE │ └── multilingual.md └── bert_model │ └── bert_config.json ├── data ├── .DS_Store ├── RC_result │ └── .DS_Store └── ori_data │ └── all_50_schemas ├── SO_label ├── .DS_Store ├── SO.sh ├── lstm_crf_layer.py ├── models_SO.py └── data_reader_for_SO.py ├── baseline_with_tf.estimator ├── RC │ ├── .DS_Store │ ├── train_helper.py │ └── models.py └── NER │ ├── .DS_Store │ ├── NER_train.sh │ ├── cut_sentence.py │ ├── train_helper.py │ ├── lstm_crf_layer.py │ ├── models.py │ ├── conlleval.py │ └── data_reader_for_NER.py ├── __init__.py ├── NER ├── NER.sh ├── cut_sentence.py ├── train_helper.py ├── lstm_crf_layer.py ├── models.py └── conlleval.py ├── P_classification ├── PC.sh └── data_reader_for_PC.py ├── LICENSE └── .gitignore /README.md: -------------------------------------------------------------------------------- 1 | # baidu_IE 2 | 2019语言与智能技术竞赛 Information Extraction 3 | -------------------------------------------------------------------------------- /dict/label_dict: -------------------------------------------------------------------------------- 1 | B-SUB 2 | I-SUB 3 | E-SUB 4 | B-OBJ 5 | I-OBJ 6 | E-OBJ 7 | O 8 | -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yuhaitao1994/LIC2019_Information_Extraction/HEAD/.DS_Store -------------------------------------------------------------------------------- /RC/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yuhaitao1994/LIC2019_Information_Extraction/HEAD/RC/.DS_Store -------------------------------------------------------------------------------- /bert/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yuhaitao1994/LIC2019_Information_Extraction/HEAD/bert/.DS_Store -------------------------------------------------------------------------------- /data/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yuhaitao1994/LIC2019_Information_Extraction/HEAD/data/.DS_Store -------------------------------------------------------------------------------- /SO_label/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yuhaitao1994/LIC2019_Information_Extraction/HEAD/SO_label/.DS_Store -------------------------------------------------------------------------------- /bert/bert_code/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yuhaitao1994/LIC2019_Information_Extraction/HEAD/bert/bert_code/.DS_Store -------------------------------------------------------------------------------- /data/RC_result/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yuhaitao1994/LIC2019_Information_Extraction/HEAD/data/RC_result/.DS_Store -------------------------------------------------------------------------------- /bert/bert_code/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow >= 1.11.0 # CPU Version of TensorFlow. 2 | # tensorflow-gpu >= 1.11.0 # GPU version of TensorFlow. 3 | -------------------------------------------------------------------------------- /baseline_with_tf.estimator/RC/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yuhaitao1994/LIC2019_Information_Extraction/HEAD/baseline_with_tf.estimator/RC/.DS_Store -------------------------------------------------------------------------------- /baseline_with_tf.estimator/NER/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yuhaitao1994/LIC2019_Information_Extraction/HEAD/baseline_with_tf.estimator/NER/.DS_Store -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 5 | @Time : 2019/1/30 19:09 6 | @Author : MaCan (ma_cancan@163.com) 7 | @File : __init__.py.py 8 | """ -------------------------------------------------------------------------------- /dict/postag_dict: -------------------------------------------------------------------------------- 1 | n 2 | f 3 | s 4 | t 5 | nr 6 | ns 7 | nt 8 | nw 9 | nz 10 | v 11 | vd 12 | vn 13 | a 14 | ad 15 | an 16 | d 17 | m 18 | q 19 | r 20 | p 21 | c 22 | u 23 | xc 24 | w 25 | -------------------------------------------------------------------------------- /dict/sub_tag: -------------------------------------------------------------------------------- 1 | nz 10944 2 | ns 3782 3 | nw 200889 4 | vn 221 5 | t 120 6 | w 4 7 | m 95 8 | nr 84455 9 | an 45 10 | p 1 11 | s 5 12 | d 45 13 | f 12 14 | v 904 15 | a 235 16 | vd 5 17 | n 2530 18 | ad 11 19 | nt 14611 20 | u 3 21 | r 24 22 | c 2 23 | -------------------------------------------------------------------------------- /dict/obj_tag: -------------------------------------------------------------------------------- 1 | nz 26339 2 | ns 17384 3 | nw 9549 4 | vn 181 5 | t 17184 6 | w 1 7 | m 5736 8 | xc 42 9 | nr 152705 10 | an 83 11 | p 1 12 | s 17 13 | d 52 14 | f 8 15 | v 1181 16 | a 549 17 | vd 31 18 | n 8436 19 | ad 37 20 | nt 29200 21 | q 16 22 | u 4 23 | r 20 24 | c 1 25 | -------------------------------------------------------------------------------- /baseline_with_tf.estimator/NER/NER_train.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | python3 bert_lstm_ner.py \ 4 | -device_map=0 \ 5 | -do_train=True \ 6 | -do_eval=True \ 7 | -do_predict=False \ 8 | -max_seq_length=128 \ 9 | -batch_size=32 \ 10 | -learning_rate=2e-5 \ 11 | -num_train_epochs=15 12 | -------------------------------------------------------------------------------- /NER/NER.sh: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | 3 | python3 ner_main.py \ 4 | -device_map=0 \ 5 | -do_train=True \ 6 | -do_eval=True \ 7 | -do_predict=True \ 8 | -max_seq_length=128 \ 9 | -batch_size=32 \ 10 | -learning_rate=2e-5 \ 11 | -num_train_epochs=15 \ 12 | -save_summary_steps=500 \ 13 | -save_checkpoints_steps=500 \ 14 | -filter_adam_var=False \ 15 | -experiment_name=0428 16 | -------------------------------------------------------------------------------- /SO_label/SO.sh: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | 3 | python3 ptrNet_SO_main.py \ 4 | -device_map=0 \ 5 | -do_train=True \ 6 | -do_eval=True \ 7 | -do_predict=True \ 8 | -max_seq_length=150 \ 9 | -batch_size=32 \ 10 | -learning_rate=2e-5 \ 11 | -num_train_epochs=2 \ 12 | -save_summary_steps=5 \ 13 | -save_checkpoints_steps=5 \ 14 | -filter_adam_var=False \ 15 | -experiment_name=ptr_SO 16 | -------------------------------------------------------------------------------- /P_classification/PC.sh: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | 3 | python3 pclassification_main.py \ 4 | -device_map=0 \ 5 | -do_train=True \ 6 | -do_eval=True \ 7 | -do_predict=True \ 8 | -max_seq_length=150 \ 9 | -batch_size=32 \ 10 | -learning_rate=2e-5 \ 11 | -num_train_epochs=2 \ 12 | -save_summary_steps=5 \ 13 | -save_checkpoints_steps=5 \ 14 | -filter_adam_var=False \ 15 | -experiment_name=pclass 16 | -------------------------------------------------------------------------------- /bert/bert_model/bert_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention_probs_dropout_prob": 0.1, 3 | "directionality": "bidi", 4 | "hidden_act": "gelu", 5 | "hidden_dropout_prob": 0.1, 6 | "hidden_size": 768, 7 | "initializer_range": 0.02, 8 | "intermediate_size": 3072, 9 | "max_position_embeddings": 512, 10 | "num_attention_heads": 12, 11 | "num_hidden_layers": 12, 12 | "pooler_fc_size": 768, 13 | "pooler_num_attention_heads": 12, 14 | "pooler_num_fc_layers": 3, 15 | "pooler_size_per_head": 128, 16 | "pooler_type": "first_token_transform", 17 | "type_vocab_size": 2, 18 | "vocab_size": 21128 19 | } 20 | -------------------------------------------------------------------------------- /bert/bert_code/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /dict/p_eng: -------------------------------------------------------------------------------- 1 | 改编自 RP 2 | 主角 LA 3 | 丈夫 HB 4 | 号 PN 5 | 民族 NAT 6 | 所属专辑 AL 7 | 创始人 INVEN 8 | 毕业院校 GRA 9 | 总部地点 HQ 10 | 专业代码 SP 11 | 主演 ACT 12 | 董事长 CM 13 | 海拔 AT 14 | 朝代 DY 15 | 导演 DIR 16 | 简称 ABBR 17 | 首都 CP 18 | 注册资本 RG 19 | 出生地 BP 20 | 人口数量 PA 21 | 占地面积 AS 22 | 所在城市 CITY 23 | 上映时间 RS 24 | 父亲 FAR 25 | 出版社 PRESS 26 | 官方语言 OL 27 | 主持人 HOST 28 | 身高 HEIG 29 | 妻子 WIFE 30 | 气候 CLI 31 | 目 BO 32 | 歌手 SING 33 | 修业年限 SD 34 | 作词 LYR 35 | 连载网站 WEB 36 | 祖籍 AP 37 | 面积 AREA 38 | 母亲 MOT 39 | 出品公司 PC 40 | 编剧 WC 41 | 字 CN 42 | 作曲 MB 43 | 邮政编码 PCODE 44 | 制片人 FP 45 | 成立日期 BD 46 | 嘉宾 GUEST 47 | 国籍 NA 48 | 出生日期 BDATE 49 | 作者 WRITER 50 | 主演 STAR 51 | 木有关系 NORELATION -------------------------------------------------------------------------------- /dict/type_dic: -------------------------------------------------------------------------------- 1 | 主演 影视作品 人物 2 | 目 生物 目 3 | 身高 人物 Number 4 | 出生日期 人物 Date 5 | 国籍 人物 国家 6 | 连载网站 网络小说 网站 7 | 作者 图书作品 人物 8 | 歌手 歌曲 人物 9 | 海拔 地点 Number 10 | 出生地 人物 地点 11 | 导演 影视作品 人物 12 | 气候 行政区 气候 13 | 朝代 历史人物 Text 14 | 妻子 人物 人物 15 | 民族 人物 Text 16 | 毕业院校 人物 学校 17 | 编剧 影视作品 人物 18 | 出品公司 影视作品 企业 19 | 父亲 人物 人物 20 | 出版社 书籍 出版社 21 | 作词 歌曲 人物 22 | 作曲 歌曲 人物 23 | 母亲 人物 人物 24 | 成立日期 企业 Date 25 | 字 历史人物 Text 26 | 丈夫 人物 人物 27 | 号 历史人物 Text 28 | 所属专辑 歌曲 音乐专辑 29 | 所在城市 景点 城市 30 | 总部地点 企业 地点 31 | 主持人 电视综艺 人物 32 | 上映时间 影视作品 Date 33 | 首都 国家 城市 34 | 创始人 企业 人物 35 | 祖籍 人物 地点 36 | 改编自 影视作品 作品 37 | 制片人 影视作品 人物 38 | 注册资本 企业 Number 39 | 人口数量 行政区 Number 40 | 面积 行政区 Number 41 | 主角 网络小说 人物 42 | 占地面积 机构 Number 43 | 嘉宾 电视综艺 人物 44 | 简称 机构 Text 45 | 董事长 企业 人物 46 | 官方语言 国家 语言 47 | 邮政编码 行政区 Text 48 | 专业代码 学科专业 Text 49 | 修业年限 学科专业 Number 50 | -------------------------------------------------------------------------------- /RC/RC.sh: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | 3 | python3 rc_main.py \ 4 | -device_map=1 \ 5 | -do_train=True \ 6 | -do_eval=True \ 7 | -do_predict=True \ 8 | -max_seq_length=150 \ 9 | -batch_size=32 \ 10 | -learning_rate=2e-5 \ 11 | -num_train_epochs=10 \ 12 | -save_summary_steps=500 \ 13 | -save_checkpoints_steps=500 \ 14 | -filter_adam_var=False \ 15 | -experiment_name=bert_rc 16 | 17 | # python3 bert_pcnn_rc.py \ 18 | # -device_map=0 \ 19 | # -do_train=True \ 20 | # -do_eval=True \ 21 | # -do_predict=True \ 22 | # -max_seq_length=150 \ 23 | # -batch_size=32 \ 24 | # -learning_rate=2e-5 \ 25 | # -num_train_epochs=10 \ 26 | # -save_summary_steps=500 \ 27 | # -save_checkpoints_steps=500 \ 28 | # -filter_adam_var=False \ 29 | # -experiment_name=pcnn 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 yuhaitao1994 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /bert/bert_code/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | BERT needs to maintain permanent compatibility with the pre-trained model files, 4 | so we do not plan to make any major changes to this library (other than what was 5 | promised in the README). However, we can accept small patches related to 6 | re-factoring and documentation. To submit contributes, there are just a few 7 | small guidelines you need to follow. 8 | 9 | ## Contributor License Agreement 10 | 11 | Contributions to this project must be accompanied by a Contributor License 12 | Agreement. You (or your employer) retain the copyright to your contribution; 13 | this simply gives us permission to use and redistribute your contributions as 14 | part of the project. Head over to to see 15 | your current agreements on file or to sign a new one. 16 | 17 | You generally only need to submit a CLA once, so if you've already submitted one 18 | (even if it was for a different project), you probably don't need to do it 19 | again. 20 | 21 | ## Code reviews 22 | 23 | All submissions, including submissions by project members, require review. We 24 | use GitHub pull requests for this purpose. Consult 25 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 26 | information on using pull requests. 27 | 28 | ## Community Guidelines 29 | 30 | This project follows 31 | [Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /bert/bert_code/optimization_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import optimization 20 | import tensorflow as tf 21 | 22 | 23 | class OptimizationTest(tf.test.TestCase): 24 | 25 | def test_adam(self): 26 | with self.test_session() as sess: 27 | w = tf.get_variable( 28 | "w", 29 | shape=[3], 30 | initializer=tf.constant_initializer([0.1, -0.2, -0.1])) 31 | x = tf.constant([0.4, 0.2, -0.5]) 32 | loss = tf.reduce_mean(tf.square(x - w)) 33 | tvars = tf.trainable_variables() 34 | grads = tf.gradients(loss, tvars) 35 | global_step = tf.train.get_or_create_global_step() 36 | optimizer = optimization.AdamWeightDecayOptimizer(learning_rate=0.2) 37 | train_op = optimizer.apply_gradients(zip(grads, tvars), global_step) 38 | init_op = tf.group(tf.global_variables_initializer(), 39 | tf.local_variables_initializer()) 40 | sess.run(init_op) 41 | for _ in range(100): 42 | sess.run(train_op) 43 | w_np = sess.run(w) 44 | self.assertAllClose(w_np.flat, [0.4, 0.2, -0.5], rtol=1e-2, atol=1e-2) 45 | 46 | 47 | if __name__ == "__main__": 48 | tf.test.main() 49 | -------------------------------------------------------------------------------- /data/ori_data/all_50_schemas: -------------------------------------------------------------------------------- 1 | {"object_type": "地点", "predicate": "祖籍", "subject_type": "人物"} 2 | {"object_type": "人物", "predicate": "父亲", "subject_type": "人物"} 3 | {"object_type": "地点", "predicate": "总部地点", "subject_type": "企业"} 4 | {"object_type": "地点", "predicate": "出生地", "subject_type": "人物"} 5 | {"object_type": "目", "predicate": "目", "subject_type": "生物"} 6 | {"object_type": "Number", "predicate": "面积", "subject_type": "行政区"} 7 | {"object_type": "Text", "predicate": "简称", "subject_type": "机构"} 8 | {"object_type": "Date", "predicate": "上映时间", "subject_type": "影视作品"} 9 | {"object_type": "人物", "predicate": "妻子", "subject_type": "人物"} 10 | {"object_type": "音乐专辑", "predicate": "所属专辑", "subject_type": "歌曲"} 11 | {"object_type": "Number", "predicate": "注册资本", "subject_type": "企业"} 12 | {"object_type": "城市", "predicate": "首都", "subject_type": "国家"} 13 | {"object_type": "人物", "predicate": "导演", "subject_type": "影视作品"} 14 | {"object_type": "Text", "predicate": "字", "subject_type": "历史人物"} 15 | {"object_type": "Number", "predicate": "身高", "subject_type": "人物"} 16 | {"object_type": "企业", "predicate": "出品公司", "subject_type": "影视作品"} 17 | {"object_type": "Number", "predicate": "修业年限", "subject_type": "学科专业"} 18 | {"object_type": "Date", "predicate": "出生日期", "subject_type": "人物"} 19 | {"object_type": "人物", "predicate": "制片人", "subject_type": "影视作品"} 20 | {"object_type": "人物", "predicate": "母亲", "subject_type": "人物"} 21 | {"object_type": "人物", "predicate": "编剧", "subject_type": "影视作品"} 22 | {"object_type": "国家", "predicate": "国籍", "subject_type": "人物"} 23 | {"object_type": "Number", "predicate": "海拔", "subject_type": "地点"} 24 | {"object_type": "网站", "predicate": "连载网站", "subject_type": "网络小说"} 25 | {"object_type": "人物", "predicate": "丈夫", "subject_type": "人物"} 26 | {"object_type": "Text", "predicate": "朝代", "subject_type": "历史人物"} 27 | {"object_type": "Text", "predicate": "民族", "subject_type": "人物"} 28 | {"object_type": "Text", "predicate": "号", "subject_type": "历史人物"} 29 | {"object_type": "出版社", "predicate": "出版社", "subject_type": "书籍"} 30 | {"object_type": "人物", "predicate": "主持人", "subject_type": "电视综艺"} 31 | {"object_type": "Text", "predicate": "专业代码", "subject_type": "学科专业"} 32 | {"object_type": "人物", "predicate": "歌手", "subject_type": "歌曲"} 33 | {"object_type": "人物", "predicate": "作词", "subject_type": "歌曲"} 34 | {"object_type": "人物", "predicate": "主角", "subject_type": "网络小说"} 35 | {"object_type": "人物", "predicate": "董事长", "subject_type": "企业"} 36 | {"object_type": "Date", "predicate": "成立日期", "subject_type": "机构"} 37 | {"object_type": "学校", "predicate": "毕业院校", "subject_type": "人物"} 38 | {"object_type": "Number", "predicate": "占地面积", "subject_type": "机构"} 39 | {"object_type": "语言", "predicate": "官方语言", "subject_type": "国家"} 40 | {"object_type": "Text", "predicate": "邮政编码", "subject_type": "行政区"} 41 | {"object_type": "Number", "predicate": "人口数量", "subject_type": "行政区"} 42 | {"object_type": "城市", "predicate": "所在城市", "subject_type": "景点"} 43 | {"object_type": "人物", "predicate": "作者", "subject_type": "图书作品"} 44 | {"object_type": "Date", "predicate": "成立日期", "subject_type": "企业"} 45 | {"object_type": "人物", "predicate": "作曲", "subject_type": "歌曲"} 46 | {"object_type": "气候", "predicate": "气候", "subject_type": "行政区"} 47 | {"object_type": "人物", "predicate": "嘉宾", "subject_type": "电视综艺"} 48 | {"object_type": "人物", "predicate": "主演", "subject_type": "影视作品"} 49 | {"object_type": "作品", "predicate": "改编自", "subject_type": "影视作品"} 50 | {"object_type": "人物", "predicate": "创始人", "subject_type": "企业"} 51 | -------------------------------------------------------------------------------- /NER/cut_sentence.py: -------------------------------------------------------------------------------- 1 | # encoding=utf-8 2 | 3 | """ 4 | 用于语料库的处理 5 | 1. 全部处理成小于max_seq_length的序列，这样可以避免解码出现不合法的数据或者在最后算结果的时候出现out of range 的错误。 6 | """ 7 | 8 | 9 | import os 10 | import codecs 11 | import argparse 12 | 13 | 14 | def load_file(file_path): 15 | if not os.path.exists(file_path): 16 | return None 17 | with codecs.open(file_path, 'r', encoding='utf-8') as fd: 18 | for line in fd: 19 | yield line 20 | 21 | 22 | def _cut(sentence): 23 | new_sentence = [] 24 | sen = [] 25 | for i in sentence: 26 | if i.split(' ')[0] in ['。', '！', '？'] and len(sen) != 0: 27 | sen.append(i) 28 | new_sentence.append(sen) 29 | sen = [] 30 | continue 31 | sen.append(i) 32 | if len(new_sentence) == 1: # 娄底那种一句话超过max_seq_length的且没有句号的，用,分割，再长的不考虑了。。。 33 | new_sentence = [] 34 | sen = [] 35 | for i in sentence: 36 | if i.split(' ')[0] in ['，'] and len(sen) != 0: 37 | sen.append(i) 38 | new_sentence.append(sen) 39 | sen = [] 40 | continue 41 | sen.append(i) 42 | return new_sentence 43 | 44 | 45 | def cut_sentence(file, max_seq_length): 46 | """ 47 | 句子截断 48 | :param file: 49 | :param max_seq_length: 50 | :return: 51 | """ 52 | context = [] 53 | sentence = [] 54 | cnt = 0 55 | index = 0 56 | for line in load_file(file): 57 | line = line.strip() 58 | if line == '' and len(sentence) != 0: 59 | # 判断这一句是否超过最大长度 60 | index += 1 61 | if len(sentence) > max_seq_length: 62 | sentence = sentence[0:max_seq_length] 63 | context.append(sentence) 64 | sentence = [] 65 | continue 66 | cnt += 1 67 | sentence.append(line) 68 | print('index:{}'.format(index)) 69 | print('sentence num:{}'.format(len(context))) 70 | print('token cnt:{}'.format(cnt)) 71 | return context 72 | 73 | 74 | def write_to_file(file, context): 75 | # 首先将源文件改名为新文件名，避免覆盖 76 | os.rename(file, '{}.bak'.format(file)) 77 | with codecs.open(file, 'w', encoding='utf-8') as fd: 78 | for sen in context: 79 | for token in sen: 80 | fd.write(token + '\n') 81 | fd.write('\n') 82 | 83 | 84 | if __name__ == '__main__': 85 | parser = argparse.ArgumentParser(description='data pre process') 86 | parser.add_argument('--train_data', type=str, 87 | default='../data/NER_data/train.txt') 88 | parser.add_argument('--dev_data', type=str, 89 | default='../data/NER_data/dev.txt') 90 | parser.add_argument('--test_data', type=str, 91 | default='../data/NER_data/test.txt') 92 | parser.add_argument('--max_seq_length', type=int, default=148) 93 | args = parser.parse_args() 94 | 95 | print('cut train data to max sequence length:{}'.format(args.max_seq_length)) 96 | context = cut_sentence(args.train_data, args.max_seq_length) 97 | write_to_file(args.train_data, context) 98 | 99 | print('cut dev data to max sequence length:{}'.format(args.max_seq_length)) 100 | context = cut_sentence(args.dev_data, args.max_seq_length) 101 | write_to_file(args.dev_data, context) 102 | 103 | print('cut test data to max sequence length:{}'.format(args.max_seq_length)) 104 | context = cut_sentence(args.test_data, args.max_seq_length) 105 | write_to_file(args.test_data, context) 106 | -------------------------------------------------------------------------------- /baseline_with_tf.estimator/NER/cut_sentence.py: -------------------------------------------------------------------------------- 1 | # encoding=utf-8 2 | 3 | """ 4 | 用于语料库的处理 5 | 1. 全部处理成小于max_seq_length的序列，这样可以避免解码出现不合法的数据或者在最后算结果的时候出现out of range 的错误。 6 | """ 7 | 8 | 9 | import os 10 | import codecs 11 | import argparse 12 | 13 | 14 | def load_file(file_path): 15 | if not os.path.exists(file_path): 16 | return None 17 | with codecs.open(file_path, 'r', encoding='utf-8') as fd: 18 | for line in fd: 19 | yield line 20 | 21 | 22 | def _cut(sentence): 23 | new_sentence = [] 24 | sen = [] 25 | for i in sentence: 26 | if i.split(' ')[0] in ['。', '！', '？'] and len(sen) != 0: 27 | sen.append(i) 28 | new_sentence.append(sen) 29 | sen = [] 30 | continue 31 | sen.append(i) 32 | if len(new_sentence) == 1: # 娄底那种一句话超过max_seq_length的且没有句号的，用,分割，再长的不考虑了。。。 33 | new_sentence = [] 34 | sen = [] 35 | for i in sentence: 36 | if i.split(' ')[0] in ['，'] and len(sen) != 0: 37 | sen.append(i) 38 | new_sentence.append(sen) 39 | sen = [] 40 | continue 41 | sen.append(i) 42 | return new_sentence 43 | 44 | 45 | def cut_sentence(file, max_seq_length): 46 | """ 47 | 句子截断 48 | :param file: 49 | :param max_seq_length: 50 | :return: 51 | """ 52 | context = [] 53 | sentence = [] 54 | cnt = 0 55 | index = 0 56 | for line in load_file(file): 57 | line = line.strip() 58 | if line == '' and len(sentence) != 0: 59 | # 判断这一句是否超过最大长度 60 | index += 1 61 | if len(sentence) > max_seq_length: 62 | sentence = sentence[0:max_seq_length] 63 | context.append(sentence) 64 | sentence = [] 65 | continue 66 | cnt += 1 67 | sentence.append(line) 68 | print('index:{}'.format(index)) 69 | print('sentence num:{}'.format(len(context))) 70 | print('token cnt:{}'.format(cnt)) 71 | return context 72 | 73 | 74 | def write_to_file(file, context): 75 | # 首先将源文件改名为新文件名，避免覆盖 76 | os.rename(file, '{}.bak'.format(file)) 77 | with codecs.open(file, 'w', encoding='utf-8') as fd: 78 | for sen in context: 79 | for token in sen: 80 | fd.write(token + '\n') 81 | fd.write('\n') 82 | 83 | 84 | if __name__ == '__main__': 85 | parser = argparse.ArgumentParser(description='data pre process') 86 | parser.add_argument('--train_data', type=str, 87 | default='../data/NER_data/train.txt') 88 | parser.add_argument('--dev_data', type=str, 89 | default='../data/NER_data/dev.txt') 90 | parser.add_argument('--test_data', type=str, 91 | default='../data/NER_data/test.txt') 92 | parser.add_argument('--max_seq_length', type=int, default=126) 93 | args = parser.parse_args() 94 | 95 | print('cut train data to max sequence length:{}'.format(args.max_seq_length)) 96 | context = cut_sentence(args.train_data, args.max_seq_length) 97 | write_to_file(args.train_data, context) 98 | 99 | print('cut dev data to max sequence length:{}'.format(args.max_seq_length)) 100 | context = cut_sentence(args.dev_data, args.max_seq_length) 101 | write_to_file(args.dev_data, context) 102 | 103 | print('cut test data to max sequence length:{}'.format(args.max_seq_length)) 104 | context = cut_sentence(args.test_data, args.max_seq_length) 105 | write_to_file(args.test_data, context) 106 | -------------------------------------------------------------------------------- /bert/bert_code/sample_text.txt: -------------------------------------------------------------------------------- 1 | This text is included to make sure Unicode is handled properly: 力加勝北区ᴵᴺᵀᵃছজটডণত 2 | Text should be one-sentence-per-line, with empty lines between documents. 3 | This sample text is public domain and was randomly selected from Project Guttenberg. 4 | 5 | The rain had only ceased with the gray streaks of morning at Blazing Star, and the settlement awoke to a moral sense of cleanliness, and the finding of forgotten knives, tin cups, and smaller camp utensils, where the heavy showers had washed away the debris and dust heaps before the cabin doors. 6 | Indeed, it was recorded in Blazing Star that a fortunate early riser had once picked up on the highway a solid chunk of gold quartz which the rain had freed from its incumbering soil, and washed into immediate and glittering popularity. 7 | Possibly this may have been the reason why early risers in that locality, during the rainy season, adopted a thoughtful habit of body, and seldom lifted their eyes to the rifted or india-ink washed skies above them. 8 | "Cass" Beard had risen early that morning, but not with a view to discovery. 9 | A leak in his cabin roof,--quite consistent with his careless, improvident habits,--had roused him at 4 A. M., with a flooded "bunk" and wet blankets. 10 | The chips from his wood pile refused to kindle a fire to dry his bed-clothes, and he had recourse to a more provident neighbor's to supply the deficiency. 11 | This was nearly opposite. 12 | Mr. Cassius crossed the highway, and stopped suddenly. 13 | Something glittered in the nearest red pool before him. 14 | Gold, surely! 15 | But, wonderful to relate, not an irregular, shapeless fragment of crude ore, fresh from Nature's crucible, but a bit of jeweler's handicraft in the form of a plain gold ring. 16 | Looking at it more attentively, he saw that it bore the inscription, "May to Cass." 17 | Like most of his fellow gold-seekers, Cass was superstitious. 18 | 19 | The fountain of classic wisdom, Hypatia herself. 20 | As the ancient sage--the name is unimportant to a monk--pumped water nightly that he might study by day, so I, the guardian of cloaks and parasols, at the sacred doors of her lecture-room, imbibe celestial knowledge. 21 | From my youth I felt in me a soul above the matter-entangled herd. 22 | She revealed to me the glorious fact, that I am a spark of Divinity itself. 23 | A fallen star, I am, sir!' continued he, pensively, stroking his lean stomach--'a fallen star!--fallen, if the dignity of philosophy will allow of the simile, among the hogs of the lower world--indeed, even into the hog-bucket itself. Well, after all, I will show you the way to the Archbishop's. 24 | There is a philosophic pleasure in opening one's treasures to the modest young. 25 | Perhaps you will assist me by carrying this basket of fruit?' And the little man jumped up, put his basket on Philammon's head, and trotted off up a neighbouring street. 26 | Philammon followed, half contemptuous, half wondering at what this philosophy might be, which could feed the self-conceit of anything so abject as his ragged little apish guide; 27 | but the novel roar and whirl of the street, the perpetual stream of busy faces, the line of curricles, palanquins, laden asses, camels, elephants, which met and passed him, and squeezed him up steps and into doorways, as they threaded their way through the great Moon-gate into the ample street beyond, drove everything from his mind but wondering curiosity, and a vague, helpless dread of that great living wilderness, more terrible than any dead wilderness of sand which he had left behind. 28 | Already he longed for the repose, the silence of the Laura--for faces which knew him and smiled upon him; but it was too late to turn back now. 29 | His guide held on for more than a mile up the great main street, crossed in the centre of the city, at right angles, by one equally magnificent, at each end of which, miles away, appeared, dim and distant over the heads of the living stream of passengers, the yellow sand-hills of the desert; 30 | while at the end of the vista in front of them gleamed the blue harbour, through a network of countless masts. 31 | At last they reached the quay at the opposite end of the street; 32 | and there burst on Philammon's astonished eyes a vast semicircle of blue sea, ringed with palaces and towers. 33 | He stopped involuntarily; and his little guide stopped also, and looked askance at the young monk, to watch the effect which that grand panorama should produce on him. 34 | -------------------------------------------------------------------------------- /NER/train_helper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import argparse 4 | import os 5 | 6 | __all__ = ['get_args_parser'] 7 | 8 | 9 | def get_args_parser(): 10 | parser = argparse.ArgumentParser() 11 | 12 | bert_path = '../bert/bert_model' 13 | root_path = '../' 14 | 15 | def str2bool(v): 16 | if v.lower() in ('yes', 'true', 't', 'y', '1'): 17 | return True 18 | elif v.lower() in ('no', 'false', 'f', 'n', '0'): 19 | return False 20 | else: 21 | raise argparse.ArgumentTypeError('Unsupported value encountered.') 22 | 23 | parser.add_argument('-experiment_name', type=str, default='1', 24 | help='name') 25 | parser.add_argument('-data_dir', type=str, default=os.path.join(root_path, 'data/NER_data'), 26 | help='train, dev and test data dir') 27 | parser.add_argument('-bert_config_file', type=str, 28 | default=os.path.join(bert_path, 'bert_config.json')) 29 | parser.add_argument('-output_dir', type=str, 30 | default=os.path.join(root_path, 'data'), help='model_dir') 31 | parser.add_argument('-init_checkpoint', type=str, default=os.path.join(bert_path, 'bert_model.ckpt'), 32 | help='Initial checkpoint (usually from a pre-trained BERT model).') 33 | parser.add_argument('-vocab_file', type=str, default=os.path.join(bert_path, 'vocab.txt'), 34 | help='') 35 | parser.add_argument('-max_seq_length', type=int, default=128, 36 | help='The maximum total input sequence length after WordPiece tokenization.') 37 | parser.add_argument('-do_train', type=str2bool, default=False, 38 | help='Whether to run training.') 39 | parser.add_argument('-do_eval', type=str2bool, default=False, 40 | help='Whether to run eval on the dev set.') 41 | parser.add_argument('-do_predict', type=str2bool, default=False, 42 | help='Whether to run the predict in inference mode on the test set.') 43 | parser.add_argument('-batch_size', type=int, default=32, 44 | help='Total batch size for training, eval and predict.') 45 | parser.add_argument('-learning_rate', type=float, default=2e-5, 46 | help='The initial learning rate for Adam.') 47 | parser.add_argument('-num_train_epochs', type=float, default=15, 48 | help='Total number of training epochs to perform.') 49 | parser.add_argument('-dropout_rate', type=float, default=0.8, 50 | help='Dropout rate') 51 | parser.add_argument('-clip', type=float, default=0.5, 52 | help='Gradient clip') 53 | parser.add_argument('-warmup_proportion', type=float, default=0.1, 54 | help='Proportion of training to perform linear learning rate warmup for ' 55 | 'E.g., 0.1 = 10% of training.') 56 | parser.add_argument('-lstm_size', type=int, default=128, 57 | help='size of lstm units.') 58 | parser.add_argument('-num_layers', type=int, default=1, 59 | help='number of rnn layers, default is 1.') 60 | parser.add_argument('-cell', type=str, default='lstm', 61 | help='which rnn cell used.') 62 | parser.add_argument('-save_checkpoints_steps', type=int, default=500, 63 | help='save_checkpoints_steps') 64 | parser.add_argument('-save_summary_steps', type=int, default=500, 65 | help='save_summary_steps.') 66 | parser.add_argument('-filter_adam_var', type=str2bool, default=True, 67 | help='after training do filter Adam params from model and save no Adam params model in file.') 68 | parser.add_argument('-do_lower_case', type=str2bool, default=True, 69 | help='Whether to lower case the input text.') 70 | parser.add_argument('-clean', type=str2bool, default=True) 71 | parser.add_argument('-device_map', type=str, default='1', 72 | help='witch device using to train') 73 | 74 | # add labels 75 | parser.add_argument('-label_list', type=str, default=None, 76 | help='User define labels， can be a file with one label one line or a string using \',\' split') 77 | 78 | parser.add_argument('-verbose', action='store_true', default=False, 79 | help='turn on tensorflow logging for debug') 80 | parser.add_argument('-ner', type=str, default='ner', 81 | help='which modle to train') 82 | 83 | return parser.parse_args() 84 | -------------------------------------------------------------------------------- /baseline_with_tf.estimator/RC/train_helper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import argparse 4 | import os 5 | 6 | __all__ = ['get_args_parser'] 7 | 8 | 9 | def get_args_parser(): 10 | from bert_rc import __version__ 11 | parser = argparse.ArgumentParser() 12 | 13 | bert_path = '../bert/bert_model' 14 | root_path = '../data/' 15 | 16 | group1 = parser.add_argument_group('File Paths', 17 | 'config the path, checkpoint and filename of a pretrained/fine-tuned BERT model') 18 | group1.add_argument('-data_dir', type=str, default=os.path.join(root_path, 'RC_data'), 19 | help='train, dev and test data dir') 20 | group1.add_argument('-bert_config_file', type=str, 21 | default=os.path.join(bert_path, 'bert_config.json')) 22 | group1.add_argument('-output_dir', type=str, default=os.path.join(root_path, 'RC_model'), 23 | help='directory of a pretrained BERT model') 24 | group1.add_argument('-init_checkpoint', type=str, default=os.path.join(bert_path, 'bert_model.ckpt'), 25 | help='Initial checkpoint (usually from a pre-trained BERT model).') 26 | group1.add_argument('-vocab_file', type=str, default=os.path.join(bert_path, 'vocab.txt'), 27 | help='') 28 | 29 | group2 = parser.add_argument_group( 30 | 'Model Config', 'config the model params') 31 | group2.add_argument('-max_seq_length', type=int, default=150, 32 | help='The maximum total input sequence length after WordPiece tokenization.') 33 | group2.add_argument('-do_train', type=bool, default=False, 34 | help='Whether to run training.') 35 | group2.add_argument('-do_eval', type=bool, default=False, 36 | help='Whether to run eval on the dev set.') 37 | group2.add_argument('-do_predict', type=bool, default=True, 38 | help='Whether to run the predict in inference mode on the test set.') 39 | group2.add_argument('-batch_size', type=int, default=32, 40 | help='Total batch size for training, eval and predict.') 41 | group2.add_argument('-learning_rate', type=float, default=2e-5, 42 | help='The initial learning rate for Adam.') 43 | group2.add_argument('-num_train_epochs', type=float, default=5, 44 | help='Total number of training epochs to perform.') 45 | group2.add_argument('-dropout_rate', type=float, default=0.5, 46 | help='Dropout rate') 47 | group2.add_argument('-clip', type=float, default=0.5, 48 | help='Gradient clip') 49 | group2.add_argument('-warmup_proportion', type=float, default=0.1, 50 | help='Proportion of training to perform linear learning rate warmup for ' 51 | 'E.g., 0.1 = 10% of training.') 52 | group2.add_argument('-lstm_size', type=int, default=128, 53 | help='size of lstm units.') 54 | group2.add_argument('-num_layers', type=int, default=1, 55 | help='number of rnn layers, default is 1.') 56 | group2.add_argument('-cell', type=str, default='lstm', 57 | help='which rnn cell used.') 58 | group2.add_argument('-save_checkpoints_steps', type=int, default=1000, 59 | help='save_checkpoints_steps') 60 | group2.add_argument('-save_summary_steps', type=int, default=1000, 61 | help='save_summary_steps.') 62 | group2.add_argument('-filter_adam_var', type=bool, default=False, 63 | help='after training do filter Adam params from model and save no Adam params model in file.') 64 | group2.add_argument('-do_lower_case', type=bool, default=True, 65 | help='Whether to lower case the input text.') 66 | group2.add_argument('-clean', type=bool, default=True) 67 | group2.add_argument('-device_map', type=str, default='1', 68 | help='witch device using to train') 69 | 70 | # add labels 71 | group2.add_argument('-label_list', type=str, default='../dict/p_eng', 72 | help='User define labels， can be a file with one label one line or a string using \',\' split') 73 | 74 | parser.add_argument('-verbose', action='store_true', default=False, 75 | help='turn on tensorflow logging for debug') 76 | parser.add_argument('-rc', type=str, default='RC', 77 | help='which modle to train') 78 | parser.add_argument('-version', action='version', 79 | version='%(prog)s ' + __version__) 80 | return parser.parse_args() 81 | -------------------------------------------------------------------------------- /baseline_with_tf.estimator/NER/train_helper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import argparse 4 | import os 5 | 6 | __all__ = ['get_args_parser'] 7 | 8 | 9 | def get_args_parser(): 10 | from bert_lstm_ner import __version__ 11 | parser = argparse.ArgumentParser() 12 | 13 | bert_path = '../bert/bert_model' 14 | root_path = '../' 15 | 16 | group1 = parser.add_argument_group('File Paths', 17 | 'config the path, checkpoint and filename of a pretrained/fine-tuned BERT model') 18 | group1.add_argument('-data_dir', type=str, default=os.path.join(root_path, 'data/NER_data'), 19 | help='train, dev and test data dir') 20 | group1.add_argument('-bert_config_file', type=str, 21 | default=os.path.join(bert_path, 'bert_config.json')) 22 | group1.add_argument('-output_dir', type=str, default=os.path.join(root_path, 'data/NER_model'), 23 | help='directory of a pretrained BERT model') 24 | group1.add_argument('-init_checkpoint', type=str, default=os.path.join(bert_path, 'bert_model.ckpt'), 25 | help='Initial checkpoint (usually from a pre-trained BERT model).') 26 | group1.add_argument('-vocab_file', type=str, default=os.path.join(bert_path, 'vocab.txt'), 27 | help='') 28 | 29 | group2 = parser.add_argument_group( 30 | 'Model Config', 'config the model params') 31 | group2.add_argument('-max_seq_length', type=int, default=128, 32 | help='The maximum total input sequence length after WordPiece tokenization.') 33 | group2.add_argument('-do_train', type=bool, default=False, 34 | help='Whether to run training.') 35 | group2.add_argument('-do_eval', type=bool, default=False, 36 | help='Whether to run eval on the dev set.') 37 | group2.add_argument('-do_predict', type=bool, default=False, 38 | help='Whether to run the predict in inference mode on the test set.') 39 | group2.add_argument('-batch_size', type=int, default=32, 40 | help='Total batch size for training, eval and predict.') 41 | group2.add_argument('-learning_rate', type=float, default=2e-5, 42 | help='The initial learning rate for Adam.') 43 | group2.add_argument('-num_train_epochs', type=float, default=15, 44 | help='Total number of training epochs to perform.') 45 | group2.add_argument('-dropout_rate', type=float, default=0.5, 46 | help='Dropout rate') 47 | group2.add_argument('-clip', type=float, default=0.5, 48 | help='Gradient clip') 49 | group2.add_argument('-warmup_proportion', type=float, default=0.1, 50 | help='Proportion of training to perform linear learning rate warmup for ' 51 | 'E.g., 0.1 = 10% of training.') 52 | group2.add_argument('-lstm_size', type=int, default=128, 53 | help='size of lstm units.') 54 | group2.add_argument('-num_layers', type=int, default=1, 55 | help='number of rnn layers, default is 1.') 56 | group2.add_argument('-cell', type=str, default='lstm', 57 | help='which rnn cell used.') 58 | group2.add_argument('-save_checkpoints_steps', type=int, default=500, 59 | help='save_checkpoints_steps') 60 | group2.add_argument('-save_summary_steps', type=int, default=500, 61 | help='save_summary_steps.') 62 | group2.add_argument('-filter_adam_var', type=bool, default=False, 63 | help='after training do filter Adam params from model and save no Adam params model in file.') 64 | group2.add_argument('-do_lower_case', type=bool, default=True, 65 | help='Whether to lower case the input text.') 66 | group2.add_argument('-clean', type=bool, default=True) 67 | group2.add_argument('-device_map', type=str, default='0', 68 | help='witch device using to train') 69 | 70 | # add labels 71 | group2.add_argument('-label_list', type=str, default=None, 72 | help='User define labels， can be a file with one label one line or a string using \',\' split') 73 | 74 | parser.add_argument('-verbose', action='store_true', default=False, 75 | help='turn on tensorflow logging for debug') 76 | parser.add_argument('-ner', type=str, default='ner', 77 | help='which modle to train') 78 | parser.add_argument('-version', action='version', 79 | version='%(prog)s ' + __version__) 80 | return parser.parse_args() 81 | -------------------------------------------------------------------------------- /bert/bert_code/tokenization_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import os 20 | import tempfile 21 | import tokenization 22 | import six 23 | import tensorflow as tf 24 | 25 | 26 | class TokenizationTest(tf.test.TestCase): 27 | 28 | def test_full_tokenizer(self): 29 | vocab_tokens = [ 30 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 31 | "##ing", "," 32 | ] 33 | with tempfile.NamedTemporaryFile(delete=False) as vocab_writer: 34 | if six.PY2: 35 | vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) 36 | else: 37 | vocab_writer.write("".join( 38 | [x + "\n" for x in vocab_tokens]).encode("utf-8")) 39 | 40 | vocab_file = vocab_writer.name 41 | 42 | tokenizer = tokenization.FullTokenizer(vocab_file) 43 | os.unlink(vocab_file) 44 | 45 | tokens = tokenizer.tokenize(u"UNwant\u00E9d,running") 46 | self.assertAllEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"]) 47 | 48 | self.assertAllEqual( 49 | tokenizer.convert_tokens_to_ids(tokens), [7, 4, 5, 10, 8, 9]) 50 | 51 | def test_chinese(self): 52 | tokenizer = tokenization.BasicTokenizer() 53 | 54 | self.assertAllEqual( 55 | tokenizer.tokenize(u"ah\u535A\u63A8zz"), 56 | [u"ah", u"\u535A", u"\u63A8", u"zz"]) 57 | 58 | def test_basic_tokenizer_lower(self): 59 | tokenizer = tokenization.BasicTokenizer(do_lower_case=True) 60 | 61 | self.assertAllEqual( 62 | tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), 63 | ["hello", "!", "how", "are", "you", "?"]) 64 | self.assertAllEqual(tokenizer.tokenize(u"H\u00E9llo"), ["hello"]) 65 | 66 | def test_basic_tokenizer_no_lower(self): 67 | tokenizer = tokenization.BasicTokenizer(do_lower_case=False) 68 | 69 | self.assertAllEqual( 70 | tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), 71 | ["HeLLo", "!", "how", "Are", "yoU", "?"]) 72 | 73 | def test_wordpiece_tokenizer(self): 74 | vocab_tokens = [ 75 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 76 | "##ing" 77 | ] 78 | 79 | vocab = {} 80 | for (i, token) in enumerate(vocab_tokens): 81 | vocab[token] = i 82 | tokenizer = tokenization.WordpieceTokenizer(vocab=vocab) 83 | 84 | self.assertAllEqual(tokenizer.tokenize(""), []) 85 | 86 | self.assertAllEqual( 87 | tokenizer.tokenize("unwanted running"), 88 | ["un", "##want", "##ed", "runn", "##ing"]) 89 | 90 | self.assertAllEqual( 91 | tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) 92 | 93 | def test_convert_tokens_to_ids(self): 94 | vocab_tokens = [ 95 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 96 | "##ing" 97 | ] 98 | 99 | vocab = {} 100 | for (i, token) in enumerate(vocab_tokens): 101 | vocab[token] = i 102 | 103 | self.assertAllEqual( 104 | tokenization.convert_tokens_to_ids( 105 | vocab, ["un", "##want", "##ed", "runn", "##ing"]), [7, 4, 5, 8, 9]) 106 | 107 | def test_is_whitespace(self): 108 | self.assertTrue(tokenization._is_whitespace(u" ")) 109 | self.assertTrue(tokenization._is_whitespace(u"\t")) 110 | self.assertTrue(tokenization._is_whitespace(u"\r")) 111 | self.assertTrue(tokenization._is_whitespace(u"\n")) 112 | self.assertTrue(tokenization._is_whitespace(u"\u00A0")) 113 | 114 | self.assertFalse(tokenization._is_whitespace(u"A")) 115 | self.assertFalse(tokenization._is_whitespace(u"-")) 116 | 117 | def test_is_control(self): 118 | self.assertTrue(tokenization._is_control(u"\u0005")) 119 | 120 | self.assertFalse(tokenization._is_control(u"A")) 121 | self.assertFalse(tokenization._is_control(u" ")) 122 | self.assertFalse(tokenization._is_control(u"\t")) 123 | self.assertFalse(tokenization._is_control(u"\r")) 124 | 125 | def test_is_punctuation(self): 126 | self.assertTrue(tokenization._is_punctuation(u"-")) 127 | self.assertTrue(tokenization._is_punctuation(u"$")) 128 | self.assertTrue(tokenization._is_punctuation(u"`")) 129 | self.assertTrue(tokenization._is_punctuation(u".")) 130 | 131 | self.assertFalse(tokenization._is_punctuation(u"A")) 132 | self.assertFalse(tokenization._is_punctuation(u" ")) 133 | 134 | 135 | if __name__ == "__main__": 136 | tf.test.main() 137 | -------------------------------------------------------------------------------- /bert/bert_code/optimization.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Functions and classes related to optimization (weight updates).""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import re 22 | import tensorflow as tf 23 | 24 | 25 | def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, use_tpu): 26 | """Creates an optimizer training op.""" 27 | global_step = tf.train.get_or_create_global_step() 28 | 29 | learning_rate = tf.constant(value=init_lr, shape=[], dtype=tf.float32) 30 | 31 | # Implements linear decay of the learning rate. 32 | learning_rate = tf.train.polynomial_decay( 33 | learning_rate, 34 | global_step, 35 | num_train_steps, 36 | end_learning_rate=0.0, 37 | power=1.0, 38 | cycle=False) 39 | 40 | # Implements linear warmup. I.e., if global_step < num_warmup_steps, the 41 | # learning rate will be `global_step/num_warmup_steps * init_lr`. 42 | if num_warmup_steps: 43 | global_steps_int = tf.cast(global_step, tf.int32) 44 | warmup_steps_int = tf.constant(num_warmup_steps, dtype=tf.int32) 45 | 46 | global_steps_float = tf.cast(global_steps_int, tf.float32) 47 | warmup_steps_float = tf.cast(warmup_steps_int, tf.float32) 48 | 49 | warmup_percent_done = global_steps_float / warmup_steps_float 50 | warmup_learning_rate = init_lr * warmup_percent_done 51 | 52 | is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32) 53 | learning_rate = ( 54 | (1.0 - is_warmup) * learning_rate + is_warmup * warmup_learning_rate) 55 | 56 | # It is recommended that you use this optimizer for fine tuning, since this 57 | # is how the model was trained (note that the Adam m/v variables are NOT 58 | # loaded from init_checkpoint.) 59 | optimizer = AdamWeightDecayOptimizer( 60 | learning_rate=learning_rate, 61 | weight_decay_rate=0.01, 62 | beta_1=0.9, 63 | beta_2=0.999, 64 | epsilon=1e-6, 65 | exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) 66 | 67 | if use_tpu: 68 | optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) 69 | 70 | tvars = tf.trainable_variables() 71 | grads = tf.gradients(loss, tvars) 72 | 73 | # This is how the model was pre-trained. 74 | (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0) 75 | 76 | train_op = optimizer.apply_gradients( 77 | zip(grads, tvars), global_step=global_step) 78 | 79 | # Normally the global step update is done inside of `apply_gradients`. 80 | # However, `AdamWeightDecayOptimizer` doesn't do this. But if you use 81 | # a different optimizer, you should probably take this line out. 82 | new_global_step = global_step + 1 83 | train_op = tf.group(train_op, [global_step.assign(new_global_step)]) 84 | return train_op 85 | 86 | 87 | class AdamWeightDecayOptimizer(tf.train.Optimizer): 88 | """A basic Adam optimizer that includes "correct" L2 weight decay.""" 89 | 90 | def __init__(self, 91 | learning_rate, 92 | weight_decay_rate=0.0, 93 | beta_1=0.9, 94 | beta_2=0.999, 95 | epsilon=1e-6, 96 | exclude_from_weight_decay=None, 97 | name="AdamWeightDecayOptimizer"): 98 | """Constructs a AdamWeightDecayOptimizer.""" 99 | super(AdamWeightDecayOptimizer, self).__init__(False, name) 100 | 101 | self.learning_rate = learning_rate 102 | self.weight_decay_rate = weight_decay_rate 103 | self.beta_1 = beta_1 104 | self.beta_2 = beta_2 105 | self.epsilon = epsilon 106 | self.exclude_from_weight_decay = exclude_from_weight_decay 107 | 108 | def apply_gradients(self, grads_and_vars, global_step=None, name=None): 109 | """See base class.""" 110 | assignments = [] 111 | for (grad, param) in grads_and_vars: 112 | if grad is None or param is None: 113 | continue 114 | 115 | param_name = self._get_variable_name(param.name) 116 | 117 | m = tf.get_variable( 118 | name=param_name + "/adam_m", 119 | shape=param.shape.as_list(), 120 | dtype=tf.float32, 121 | trainable=False, 122 | initializer=tf.zeros_initializer()) 123 | v = tf.get_variable( 124 | name=param_name + "/adam_v", 125 | shape=param.shape.as_list(), 126 | dtype=tf.float32, 127 | trainable=False, 128 | initializer=tf.zeros_initializer()) 129 | 130 | # Standard Adam update. 131 | next_m = ( 132 | tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad)) 133 | next_v = ( 134 | tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2, 135 | tf.square(grad))) 136 | 137 | update = next_m / (tf.sqrt(next_v) + self.epsilon) 138 | 139 | # Just adding the square of the weights to the loss function is *not* 140 | # the correct way of using L2 regularization/weight decay with Adam, 141 | # since that will interact with the m and v parameters in strange ways. 142 | # 143 | # Instead we want ot decay the weights in a manner that doesn't interact 144 | # with the m/v parameters. This is equivalent to adding the square 145 | # of the weights to the loss with plain (non-momentum) SGD. 146 | if self._do_use_weight_decay(param_name): 147 | update += self.weight_decay_rate * param 148 | 149 | update_with_lr = self.learning_rate * update 150 | 151 | next_param = param - update_with_lr 152 | 153 | assignments.extend( 154 | [param.assign(next_param), 155 | m.assign(next_m), 156 | v.assign(next_v)]) 157 | return tf.group(*assignments, name=name) 158 | 159 | def _do_use_weight_decay(self, param_name): 160 | """Whether to use L2 weight decay for `param_name`.""" 161 | if not self.weight_decay_rate: 162 | return False 163 | if self.exclude_from_weight_decay: 164 | for r in self.exclude_from_weight_decay: 165 | if re.search(r, param_name) is not None: 166 | return False 167 | return True 168 | 169 | def _get_variable_name(self, param_name): 170 | """Get the variable name from the tensor name.""" 171 | m = re.match("^(.*):\\d+$", param_name) 172 | if m is not None: 173 | param_name = m.group(1) 174 | return param_name 175 | -------------------------------------------------------------------------------- /baseline_with_tf.estimator/NER/lstm_crf_layer.py: -------------------------------------------------------------------------------- 1 | # encoding=utf-8 2 | 3 | """ 4 | bert-blstm-crf layer 5 | """ 6 | 7 | import tensorflow as tf 8 | from tensorflow.contrib import rnn 9 | from tensorflow.contrib import crf 10 | 11 | 12 | class BLSTM_CRF(object): 13 | def __init__(self, embedded_chars, hidden_unit, cell_type, num_layers, dropout_rate, 14 | initializers, num_labels, seq_length, labels, lengths, is_training): 15 | """ 16 | BLSTM-CRF 网络 17 | :param embedded_chars: Fine-tuning embedding input 18 | :param hidden_unit: LSTM的隐含单元个数 19 | :param cell_type: RNN类型（LSTM OR GRU DICNN will be add in feature） 20 | :param num_layers: RNN的层数 21 | :param droupout_rate: droupout rate 22 | :param initializers: variable init class 23 | :param num_labels: 标签数量 24 | :param seq_length: 序列最大长度 25 | :param labels: 真实标签 26 | :param lengths: [batch_size] 每个batch下序列的真实长度 27 | :param is_training: 是否是训练过程 28 | """ 29 | self.hidden_unit = hidden_unit 30 | self.dropout_rate = dropout_rate 31 | self.cell_type = cell_type 32 | self.num_layers = num_layers 33 | self.embedded_chars = embedded_chars 34 | self.initializers = initializers 35 | self.seq_length = seq_length 36 | self.num_labels = num_labels 37 | self.labels = labels 38 | self.lengths = lengths 39 | self.embedding_dims = embedded_chars.shape[-1].value 40 | self.is_training = is_training 41 | 42 | def add_blstm_crf_layer(self, crf_only): 43 | """ 44 | blstm-crf网络 45 | :return: 46 | """ 47 | if self.is_training: 48 | # lstm input dropout rate i set 0.9 will get best score 49 | self.embedded_chars = tf.nn.dropout(self.embedded_chars, self.dropout_rate) 50 | 51 | if crf_only: 52 | logits = self.project_crf_layer(self.embedded_chars) 53 | else: 54 | # blstm 55 | lstm_output = self.blstm_layer(self.embedded_chars) 56 | # project 57 | logits = self.project_bilstm_layer(lstm_output) 58 | # crf 59 | loss, trans = self.crf_layer(logits) 60 | # CRF decode, pred_ids 是一条最大概率的标注路径 61 | pred_ids, _ = crf.crf_decode(potentials=logits, transition_params=trans, sequence_length=self.lengths) 62 | return (loss, logits, trans, pred_ids) 63 | 64 | def _witch_cell(self): 65 | """ 66 | RNN 类型 67 | :return: 68 | """ 69 | cell_tmp = None 70 | if self.cell_type == 'lstm': 71 | cell_tmp = rnn.LSTMCell(self.hidden_unit) 72 | elif self.cell_type == 'gru': 73 | cell_tmp = rnn.GRUCell(self.hidden_unit) 74 | return cell_tmp 75 | 76 | def _bi_dir_rnn(self): 77 | """ 78 | 双向RNN 79 | :return: 80 | """ 81 | cell_fw = self._witch_cell() 82 | cell_bw = self._witch_cell() 83 | if self.dropout_rate is not None: 84 | cell_bw = rnn.DropoutWrapper(cell_bw, output_keep_prob=self.dropout_rate) 85 | cell_fw = rnn.DropoutWrapper(cell_fw, output_keep_prob=self.dropout_rate) 86 | return cell_fw, cell_bw 87 | 88 | def blstm_layer(self, embedding_chars): 89 | """ 90 | 91 | :return: 92 | """ 93 | with tf.variable_scope('rnn_layer'): 94 | cell_fw, cell_bw = self._bi_dir_rnn() 95 | if self.num_layers > 1: 96 | cell_fw = rnn.MultiRNNCell([cell_fw] * self.num_layers, state_is_tuple=True) 97 | cell_bw = rnn.MultiRNNCell([cell_bw] * self.num_layers, state_is_tuple=True) 98 | 99 | outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, embedding_chars, 100 | dtype=tf.float32) 101 | outputs = tf.concat(outputs, axis=2) 102 | return outputs 103 | 104 | def project_bilstm_layer(self, lstm_outputs, name=None): 105 | """ 106 | hidden layer between lstm layer and logits 107 | :param lstm_outputs: [batch_size, num_steps, emb_size] 108 | :return: [batch_size, num_steps, num_tags] 109 | """ 110 | with tf.variable_scope("project" if not name else name): 111 | with tf.variable_scope("hidden"): 112 | W = tf.get_variable("W", shape=[self.hidden_unit * 2, self.hidden_unit], 113 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 114 | 115 | b = tf.get_variable("b", shape=[self.hidden_unit], dtype=tf.float32, 116 | initializer=tf.zeros_initializer()) 117 | output = tf.reshape(lstm_outputs, shape=[-1, self.hidden_unit * 2]) 118 | hidden = tf.tanh(tf.nn.xw_plus_b(output, W, b)) 119 | 120 | # project to score of tags 121 | with tf.variable_scope("logits"): 122 | W = tf.get_variable("W", shape=[self.hidden_unit, self.num_labels], 123 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 124 | 125 | b = tf.get_variable("b", shape=[self.num_labels], dtype=tf.float32, 126 | initializer=tf.zeros_initializer()) 127 | 128 | pred = tf.nn.xw_plus_b(hidden, W, b) 129 | return tf.reshape(pred, [-1, self.seq_length, self.num_labels]) 130 | 131 | def project_crf_layer(self, embedding_chars, name=None): 132 | """ 133 | hidden layer between input layer and logits 134 | :param lstm_outputs: [batch_size, num_steps, emb_size] 135 | :return: [batch_size, num_steps, num_tags] 136 | """ 137 | with tf.variable_scope("project" if not name else name): 138 | with tf.variable_scope("logits"): 139 | W = tf.get_variable("W", shape=[self.embedding_dims, self.num_labels], 140 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 141 | 142 | b = tf.get_variable("b", shape=[self.num_labels], dtype=tf.float32, 143 | initializer=tf.zeros_initializer()) 144 | output = tf.reshape(self.embedded_chars, 145 | shape=[-1, self.embedding_dims]) # [batch_size, embedding_dims] 146 | pred = tf.tanh(tf.nn.xw_plus_b(output, W, b)) 147 | return tf.reshape(pred, [-1, self.seq_length, self.num_labels]) 148 | 149 | def crf_layer(self, logits): 150 | """ 151 | calculate crf loss 152 | :param project_logits: [1, num_steps, num_tags] 153 | :return: scalar loss 154 | """ 155 | with tf.variable_scope("crf_loss"): 156 | trans = tf.get_variable( 157 | "transitions", 158 | shape=[self.num_labels, self.num_labels], 159 | initializer=self.initializers.xavier_initializer()) 160 | if self.labels is None: 161 | return None, trans 162 | else: 163 | log_likelihood, trans = tf.contrib.crf.crf_log_likelihood( 164 | inputs=logits, 165 | tag_indices=self.labels, 166 | transition_params=trans, 167 | sequence_lengths=self.lengths) 168 | return tf.reduce_mean(-log_likelihood), trans 169 | -------------------------------------------------------------------------------- /NER/lstm_crf_layer.py: -------------------------------------------------------------------------------- 1 | # encoding=utf-8 2 | 3 | """ 4 | bert-blstm-crf layer 5 | """ 6 | 7 | import tensorflow as tf 8 | from tensorflow.contrib import rnn 9 | from tensorflow.contrib import crf 10 | 11 | 12 | class BLSTM_CRF(object): 13 | def __init__(self, embedded_chars, hidden_unit, cell_type, num_layers, dropout_rate, 14 | initializers, num_labels, seq_length, labels, lengths, is_training): 15 | """ 16 | BLSTM-CRF 网络 17 | :param embedded_chars: Fine-tuning embedding input 18 | :param hidden_unit: LSTM的隐含单元个数 19 | :param cell_type: RNN类型（LSTM OR GRU DICNN will be add in feature） 20 | :param num_layers: RNN的层数 21 | :param droupout_rate: droupout rate 22 | :param initializers: variable init class 23 | :param num_labels: 标签数量 24 | :param seq_length: 序列最大长度 25 | :param labels: 真实标签 26 | :param lengths: [batch_size] 每个batch下序列的真实长度 27 | :param is_training: 是否是训练过程 28 | """ 29 | self.hidden_unit = hidden_unit 30 | self.dropout_rate = dropout_rate 31 | self.cell_type = cell_type 32 | self.num_layers = num_layers 33 | self.embedded_chars = embedded_chars 34 | self.initializers = initializers 35 | self.seq_length = seq_length 36 | self.num_labels = num_labels 37 | self.labels = labels 38 | self.lengths = lengths 39 | self.embedding_dims = embedded_chars.shape[-1].value 40 | self.is_training = is_training 41 | 42 | def add_blstm_crf_layer(self, crf_only): 43 | """ 44 | blstm-crf网络 45 | :return: 46 | """ 47 | self.embedded_chars = tf.cond(self.is_training, lambda: tf.nn.dropout( 48 | self.embedded_chars, self.dropout_rate), lambda: self.embedded_chars) 49 | 50 | if crf_only: 51 | logits = self.project_crf_layer(self.embedded_chars) 52 | else: 53 | # blstm 54 | lstm_output = self.blstm_layer(self.embedded_chars) 55 | # project 56 | logits = self.project_bilstm_layer(lstm_output) 57 | # crf 58 | loss, trans = self.crf_layer(logits) 59 | # CRF decode, pred_ids 是一条最大概率的标注路径 60 | pred_ids, _ = crf.crf_decode( 61 | potentials=logits, transition_params=trans, sequence_length=self.lengths) 62 | return (loss, logits, trans, pred_ids) 63 | 64 | def _witch_cell(self): 65 | """ 66 | RNN 类型 67 | :return: 68 | """ 69 | cell_tmp = None 70 | if self.cell_type == 'lstm': 71 | cell_tmp = rnn.LSTMCell(self.hidden_unit) 72 | elif self.cell_type == 'gru': 73 | cell_tmp = rnn.GRUCell(self.hidden_unit) 74 | return cell_tmp 75 | 76 | def _bi_dir_rnn(self): 77 | """ 78 | 双向RNN 79 | :return: 80 | """ 81 | cell_fw = self._witch_cell() 82 | cell_bw = self._witch_cell() 83 | if self.dropout_rate is not None: 84 | cell_bw = rnn.DropoutWrapper( 85 | cell_bw, output_keep_prob=self.dropout_rate) 86 | cell_fw = rnn.DropoutWrapper( 87 | cell_fw, output_keep_prob=self.dropout_rate) 88 | return cell_fw, cell_bw 89 | 90 | def blstm_layer(self, embedding_chars): 91 | """ 92 | 93 | :return: 94 | """ 95 | with tf.variable_scope('rnn_layer'): 96 | cell_fw, cell_bw = self._bi_dir_rnn() 97 | if self.num_layers > 1: 98 | cell_fw = rnn.MultiRNNCell( 99 | [cell_fw] * self.num_layers, state_is_tuple=True) 100 | cell_bw = rnn.MultiRNNCell( 101 | [cell_bw] * self.num_layers, state_is_tuple=True) 102 | 103 | outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, embedding_chars, 104 | dtype=tf.float32) 105 | outputs = tf.concat(outputs, axis=2) 106 | return outputs 107 | 108 | def project_bilstm_layer(self, lstm_outputs, name=None): 109 | """ 110 | hidden layer between lstm layer and logits 111 | :param lstm_outputs: [batch_size, num_steps, emb_size] 112 | :return: [batch_size, num_steps, num_tags] 113 | """ 114 | with tf.variable_scope("project" if not name else name): 115 | with tf.variable_scope("hidden"): 116 | W = tf.get_variable("W", shape=[self.hidden_unit * 2, self.hidden_unit], 117 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 118 | 119 | b = tf.get_variable("b", shape=[self.hidden_unit], dtype=tf.float32, 120 | initializer=tf.zeros_initializer()) 121 | output = tf.reshape( 122 | lstm_outputs, shape=[-1, self.hidden_unit * 2]) 123 | hidden = tf.tanh(tf.nn.xw_plus_b(output, W, b)) 124 | 125 | # project to score of tags 126 | with tf.variable_scope("logits"): 127 | W = tf.get_variable("W", shape=[self.hidden_unit, self.num_labels], 128 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 129 | 130 | b = tf.get_variable("b", shape=[self.num_labels], dtype=tf.float32, 131 | initializer=tf.zeros_initializer()) 132 | 133 | pred = tf.nn.xw_plus_b(hidden, W, b) 134 | return tf.reshape(pred, [-1, self.seq_length, self.num_labels]) 135 | 136 | def project_crf_layer(self, embedding_chars, name=None): 137 | """ 138 | hidden layer between input layer and logits 139 | :param lstm_outputs: [batch_size, num_steps, emb_size] 140 | :return: [batch_size, num_steps, num_tags] 141 | """ 142 | with tf.variable_scope("project" if not name else name): 143 | with tf.variable_scope("logits"): 144 | W = tf.get_variable("W", shape=[self.embedding_dims, self.num_labels], 145 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 146 | 147 | b = tf.get_variable("b", shape=[self.num_labels], dtype=tf.float32, 148 | initializer=tf.zeros_initializer()) 149 | output = tf.reshape(self.embedded_chars, 150 | shape=[-1, self.embedding_dims]) # [batch_size, embedding_dims] 151 | pred = tf.tanh(tf.nn.xw_plus_b(output, W, b)) 152 | return tf.reshape(pred, [-1, self.seq_length, self.num_labels]) 153 | 154 | def crf_layer(self, logits): 155 | """ 156 | calculate crf loss 157 | :param project_logits: [1, num_steps, num_tags] 158 | :return: scalar loss 159 | """ 160 | with tf.variable_scope("crf_loss"): 161 | trans = tf.get_variable( 162 | "transitions", 163 | shape=[self.num_labels, self.num_labels], 164 | initializer=self.initializers.xavier_initializer()) 165 | if self.labels is None: 166 | return None, trans 167 | else: 168 | log_likelihood, trans = tf.contrib.crf.crf_log_likelihood( 169 | inputs=logits, 170 | tag_indices=self.labels, 171 | transition_params=trans, 172 | sequence_lengths=self.lengths) 173 | return tf.reduce_mean(-log_likelihood), trans 174 | -------------------------------------------------------------------------------- /SO_label/lstm_crf_layer.py: -------------------------------------------------------------------------------- 1 | # encoding=utf-8 2 | 3 | """ 4 | bert-blstm-crf layer 5 | """ 6 | 7 | import tensorflow as tf 8 | from tensorflow.contrib import rnn 9 | from tensorflow.contrib import crf 10 | 11 | 12 | class BLSTM_CRF(object): 13 | def __init__(self, embedded_chars, hidden_unit, cell_type, num_layers, dropout_rate, 14 | initializers, num_labels, seq_length, labels, lengths, is_training): 15 | """ 16 | BLSTM-CRF 网络 17 | :param embedded_chars: Fine-tuning embedding input 18 | :param hidden_unit: LSTM的隐含单元个数 19 | :param cell_type: RNN类型（LSTM OR GRU DICNN will be add in feature） 20 | :param num_layers: RNN的层数 21 | :param droupout_rate: droupout rate 22 | :param initializers: variable init class 23 | :param num_labels: 标签数量 24 | :param seq_length: 序列最大长度 25 | :param labels: 真实标签 26 | :param lengths: [batch_size] 每个batch下序列的真实长度 27 | :param is_training: 是否是训练过程 28 | """ 29 | self.hidden_unit = hidden_unit 30 | self.dropout_rate = dropout_rate 31 | self.cell_type = cell_type 32 | self.num_layers = num_layers 33 | self.embedded_chars = embedded_chars 34 | self.initializers = initializers 35 | self.seq_length = seq_length 36 | self.num_labels = num_labels 37 | self.labels = labels 38 | self.lengths = lengths 39 | self.embedding_dims = embedded_chars.shape[-1].value 40 | self.is_training = is_training 41 | 42 | def add_blstm_crf_layer(self, crf_only): 43 | """ 44 | blstm-crf网络 45 | :return: 46 | """ 47 | self.embedded_chars = tf.cond(self.is_training, lambda: tf.nn.dropout( 48 | self.embedded_chars, self.dropout_rate), lambda: self.embedded_chars) 49 | 50 | if crf_only: 51 | logits = self.project_crf_layer(self.embedded_chars) 52 | else: 53 | # blstm 54 | lstm_output = self.blstm_layer(self.embedded_chars) 55 | # project 56 | logits = self.project_bilstm_layer(lstm_output) 57 | # crf 58 | loss, trans = self.crf_layer(logits) 59 | # CRF decode, pred_ids 是一条最大概率的标注路径 60 | pred_ids, _ = crf.crf_decode( 61 | potentials=logits, transition_params=trans, sequence_length=self.lengths) 62 | return (loss, logits, trans, pred_ids) 63 | 64 | def _witch_cell(self): 65 | """ 66 | RNN 类型 67 | :return: 68 | """ 69 | cell_tmp = None 70 | if self.cell_type == 'lstm': 71 | cell_tmp = rnn.LSTMCell(self.hidden_unit) 72 | elif self.cell_type == 'gru': 73 | cell_tmp = rnn.GRUCell(self.hidden_unit) 74 | return cell_tmp 75 | 76 | def _bi_dir_rnn(self): 77 | """ 78 | 双向RNN 79 | :return: 80 | """ 81 | cell_fw = self._witch_cell() 82 | cell_bw = self._witch_cell() 83 | if self.dropout_rate is not None: 84 | cell_bw = rnn.DropoutWrapper( 85 | cell_bw, output_keep_prob=self.dropout_rate) 86 | cell_fw = rnn.DropoutWrapper( 87 | cell_fw, output_keep_prob=self.dropout_rate) 88 | return cell_fw, cell_bw 89 | 90 | def blstm_layer(self, embedding_chars): 91 | """ 92 | 93 | :return: 94 | """ 95 | with tf.variable_scope('rnn_layer'): 96 | cell_fw, cell_bw = self._bi_dir_rnn() 97 | if self.num_layers > 1: 98 | cell_fw = rnn.MultiRNNCell( 99 | [cell_fw] * self.num_layers, state_is_tuple=True) 100 | cell_bw = rnn.MultiRNNCell( 101 | [cell_bw] * self.num_layers, state_is_tuple=True) 102 | 103 | outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, embedding_chars, 104 | dtype=tf.float32) 105 | outputs = tf.concat(outputs, axis=2) 106 | return outputs 107 | 108 | def project_bilstm_layer(self, lstm_outputs, name=None): 109 | """ 110 | hidden layer between lstm layer and logits 111 | :param lstm_outputs: [batch_size, num_steps, emb_size] 112 | :return: [batch_size, num_steps, num_tags] 113 | """ 114 | with tf.variable_scope("project" if not name else name): 115 | with tf.variable_scope("hidden"): 116 | W = tf.get_variable("W", shape=[self.hidden_unit * 2, self.hidden_unit], 117 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 118 | 119 | b = tf.get_variable("b", shape=[self.hidden_unit], dtype=tf.float32, 120 | initializer=tf.zeros_initializer()) 121 | output = tf.reshape( 122 | lstm_outputs, shape=[-1, self.hidden_unit * 2]) 123 | hidden = tf.tanh(tf.nn.xw_plus_b(output, W, b)) 124 | 125 | # project to score of tags 126 | with tf.variable_scope("logits"): 127 | W = tf.get_variable("W", shape=[self.hidden_unit, self.num_labels], 128 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 129 | 130 | b = tf.get_variable("b", shape=[self.num_labels], dtype=tf.float32, 131 | initializer=tf.zeros_initializer()) 132 | 133 | pred = tf.nn.xw_plus_b(hidden, W, b) 134 | return tf.reshape(pred, [-1, self.seq_length, self.num_labels]) 135 | 136 | def project_crf_layer(self, embedding_chars, name=None): 137 | """ 138 | hidden layer between input layer and logits 139 | :param lstm_outputs: [batch_size, num_steps, emb_size] 140 | :return: [batch_size, num_steps, num_tags] 141 | """ 142 | with tf.variable_scope("project" if not name else name): 143 | with tf.variable_scope("logits"): 144 | W = tf.get_variable("W", shape=[self.embedding_dims, self.num_labels], 145 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 146 | 147 | b = tf.get_variable("b", shape=[self.num_labels], dtype=tf.float32, 148 | initializer=tf.zeros_initializer()) 149 | output = tf.reshape(self.embedded_chars, 150 | shape=[-1, self.embedding_dims]) # [batch_size, embedding_dims] 151 | pred = tf.tanh(tf.nn.xw_plus_b(output, W, b)) 152 | return tf.reshape(pred, [-1, self.seq_length, self.num_labels]) 153 | 154 | def crf_layer(self, logits): 155 | """ 156 | calculate crf loss 157 | :param project_logits: [1, num_steps, num_tags] 158 | :return: scalar loss 159 | """ 160 | with tf.variable_scope("crf_loss"): 161 | trans = tf.get_variable( 162 | "transitions", 163 | shape=[self.num_labels, self.num_labels], 164 | initializer=self.initializers.xavier_initializer()) 165 | if self.labels is None: 166 | return None, trans 167 | else: 168 | log_likelihood, trans = tf.contrib.crf.crf_log_likelihood( 169 | inputs=logits, 170 | tag_indices=self.labels, 171 | transition_params=trans, 172 | sequence_lengths=self.lengths) 173 | return tf.reduce_mean(-log_likelihood), trans 174 | -------------------------------------------------------------------------------- /NER/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 一些公共模型代码 5 | """ 6 | 7 | import sys 8 | from lstm_crf_layer import BLSTM_CRF 9 | from tensorflow.contrib.layers.python.layers import initializers 10 | sys.path.append("../") 11 | from bert.bert_code import modeling, optimization, tokenization 12 | 13 | __all__ = ['InputExample', 'InputFeatures', 'decode_labels', 'create_model', 'convert_id_str', 14 | 'convert_id_to_label', 'result_to_json', 'create_classification_model'] 15 | 16 | 17 | class Model(object): 18 | def __init__(self, *args, **kwargs): 19 | pass 20 | 21 | 22 | class InputExample(object): 23 | """A single training/test example for simple sequence classification.""" 24 | 25 | def __init__(self, guid=None, text=None, label=None): 26 | """Constructs a InputExample. 27 | Args: 28 | guid: Unique id for the example. 29 | text_a: string. The untokenized text of the first sequence. For single 30 | sequence tasks, only this sequence must be specified. 31 | label: (Optional) string. The label of the example. This should be 32 | specified for train and dev examples, but not for test examples. 33 | """ 34 | self.guid = guid 35 | self.text = text 36 | self.label = label 37 | 38 | 39 | class InputFeatures(object): 40 | """A single set of features of data.""" 41 | 42 | def __init__(self, input_ids, input_mask, segment_ids, label_ids, ): 43 | self.input_ids = input_ids 44 | self.input_mask = input_mask 45 | self.segment_ids = segment_ids 46 | self.label_ids = label_ids 47 | # self.label_mask = label_mask 48 | 49 | 50 | class DataProcessor(object): 51 | """Base class for data converters for sequence classification data sets.""" 52 | 53 | def get_train_examples(self, data_dir): 54 | """Gets a collection of `InputExample`s for the train set.""" 55 | raise NotImplementedError() 56 | 57 | def get_dev_examples(self, data_dir): 58 | """Gets a collection of `InputExample`s for the dev set.""" 59 | raise NotImplementedError() 60 | 61 | def get_labels(self): 62 | """Gets the list of labels for this data set.""" 63 | raise NotImplementedError() 64 | 65 | 66 | def create_model(bert_config, is_training, input_ids, input_mask, 67 | segment_ids, labels, num_labels, use_one_hot_embeddings, 68 | dropout_rate=1.0, lstm_size=1, cell='lstm', num_layers=1): 69 | """ 70 | 创建X模型 71 | :param bert_config: bert 配置 72 | :param is_training: 73 | :param input_ids: 数据的idx 表示 74 | :param input_mask: 75 | :param segment_ids: 76 | :param labels: 标签的idx 表示 77 | :param num_labels: 类别数量 78 | :param use_one_hot_embeddings: 79 | :return: 80 | """ 81 | # 使用数据加载BertModel,获取对应的字embedding 82 | import tensorflow as tf 83 | model = modeling.BertModel( 84 | config=bert_config, 85 | is_training=is_training, 86 | input_ids=input_ids, 87 | input_mask=input_mask, 88 | token_type_ids=segment_ids, 89 | use_one_hot_embeddings=use_one_hot_embeddings 90 | ) 91 | # 获取对应的embedding 输入数据[batch_size, seq_length, embedding_size] 92 | embedding = model.get_sequence_output() 93 | max_seq_length = embedding.shape[1].value 94 | # 算序列真实长度 95 | used = tf.sign(tf.abs(input_ids)) 96 | # [batch_size] 大小的向量，包含了当前batch中的序列长度 97 | lengths = tf.reduce_sum(used, reduction_indices=1) 98 | # 添加CRF output layer 99 | blstm_crf = BLSTM_CRF(embedded_chars=embedding, hidden_unit=lstm_size, cell_type=cell, num_layers=num_layers, 100 | dropout_rate=dropout_rate, initializers=initializers, num_labels=num_labels, 101 | seq_length=max_seq_length, labels=labels, lengths=lengths, is_training=is_training) 102 | rst = blstm_crf.add_blstm_crf_layer(crf_only=True) 103 | return rst 104 | 105 | 106 | def decode_labels(labels, batch_size): 107 | new_labels = [] 108 | for row in range(batch_size): 109 | label = [] 110 | for i in labels[row]: 111 | i = i.decode('utf-8') 112 | if i == '**PAD**': 113 | break 114 | if i in ['[CLS]', '[SEP]']: 115 | continue 116 | label.append(i) 117 | new_labels.append(label) 118 | return new_labels 119 | 120 | 121 | def convert_id_str(input_ids, batch_size): 122 | res = [] 123 | for row in range(batch_size): 124 | line = [] 125 | for i in input_ids[row]: 126 | i = i.decode('utf-8') 127 | if i == '**PAD**': 128 | break 129 | if i in ['[CLS]', '[SEP]']: 130 | continue 131 | 132 | line.append(i) 133 | res.append(line) 134 | return res 135 | 136 | 137 | def convert_id_to_label(pred_ids_result, idx2label, batch_size): 138 | """ 139 | 将id形式的结果转化为真实序列结果 140 | :param pred_ids_result: 141 | :param idx2label: 142 | :return: 143 | """ 144 | result = [] 145 | index_result = [] 146 | for row in range(batch_size): 147 | curr_seq = [] 148 | curr_idx = [] 149 | ids = pred_ids_result[row] 150 | for idx, id in enumerate(ids): 151 | if id == 0: 152 | break 153 | curr_label = idx2label[id] 154 | if curr_label in ['[CLS]', '[SEP]']: 155 | if id == 102 and (idx < len(ids) and ids[idx + 1] == 0): 156 | break 157 | continue 158 | # elif curr_label == '[SEP]': 159 | # break 160 | curr_seq.append(curr_label) 161 | curr_idx.append(id) 162 | result.append(curr_seq) 163 | index_result.append(curr_idx) 164 | return result, index_result 165 | 166 | 167 | def result_to_json(self, string, tags): 168 | """ 169 | 将模型标注序列和输入序列结合 转化为结果 170 | :param string: 输入序列 171 | :param tags: 标注结果 172 | :return: 173 | """ 174 | item = {"entities": []} 175 | entity_name = "" 176 | entity_start = 0 177 | idx = 0 178 | last_tag = '' 179 | 180 | for char, tag in zip(string, tags): 181 | if tag[0] == "S": 182 | self.append(char, idx, idx + 1, tag[2:]) 183 | item["entities"].append( 184 | {"word": char, "start": idx, "end": idx + 1, "type": tag[2:]}) 185 | elif tag[0] == "B": 186 | if entity_name != '': 187 | self.append(entity_name, entity_start, idx, last_tag[2:]) 188 | item["entities"].append( 189 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 190 | entity_name = "" 191 | entity_name += char 192 | entity_start = idx 193 | elif tag[0] == "I": 194 | entity_name += char 195 | elif tag[0] == "O": 196 | if entity_name != '': 197 | self.append(entity_name, entity_start, idx, last_tag[2:]) 198 | item["entities"].append( 199 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 200 | entity_name = "" 201 | else: 202 | entity_name = "" 203 | entity_start = idx 204 | idx += 1 205 | last_tag = tag 206 | if entity_name != '': 207 | self.append(entity_name, entity_start, idx, last_tag[2:]) 208 | item["entities"].append( 209 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 210 | return item 211 | -------------------------------------------------------------------------------- /SO_label/models_SO.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Model for SO-labeling. 5 | @author:yuhaitao 6 | """ 7 | 8 | import sys 9 | import tensorflow as tf 10 | from tensorflow.contrib.layers.python.layers import initializers 11 | sys.path.append("../") 12 | from bert.bert_code import modeling, optimization, tokenization 13 | 14 | 15 | class InputExample(object): 16 | """A single training/test example for simple sequence classification.""" 17 | 18 | def __init__(self, guid, text_a, text_b=None, label=None): 19 | """Constructs a InputExample. 20 | Args: 21 | guid: Unique id for the example. 22 | text_a: string. The untokenized text of the first sequence. For single 23 | sequence tasks, only this sequence must be specified. 24 | label: (Optional) string. The label of the example. This should be 25 | specified for train and dev examples, but not for test examples. 26 | """ 27 | self.guid = guid 28 | self.text_a = text_a 29 | self.text_b = text_b 30 | self.label = label 31 | 32 | 33 | class InputFeatures_ptr(object): 34 | """A single set of features of data.""" 35 | 36 | def __init__(self, input_ids, input_mask, segment_ids, label_id, sub_ptr, obj_ptr, ): 37 | self.input_ids = input_ids 38 | self.input_mask = input_mask 39 | self.segment_ids = segment_ids 40 | self.label_id = label_id 41 | self.sub_ptr = sub_ptr 42 | self.obj_ptr = obj_ptr 43 | 44 | 45 | def relation_embedding(relation, num_relations, dim): 46 | """ 47 | relation embedding 48 | """ 49 | with tf.variable_scope("relation_embedding"): 50 | relation_embedding = tf.get_variable('relation_matrix', shape=[num_relations, dim], 51 | dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True) 52 | relation_output = tf.nn.embedding_lookup( 53 | relation_embedding, relation) 54 | return relation_output 55 | 56 | 57 | def softmax_mask(val, mask): 58 | INF = 1e30 59 | return -INF * (1 - tf.cast(mask, tf.float32)) + val 60 | 61 | 62 | def dense(inputs, hidden, use_bias=True, scope="dense"): 63 | """ 64 | 全连接层 65 | """ 66 | with tf.variable_scope(scope): 67 | shape = tf.shape(inputs) 68 | dim = inputs.get_shape().as_list()[-1] 69 | out_shape = [shape[idx] for idx in range( 70 | len(inputs.get_shape().as_list()) - 1)] + [hidden] 71 | # 三维的inputs，reshape成二维 72 | flat_inputs = tf.reshape(inputs, [-1, dim]) 73 | W = tf.get_variable("W", [dim, hidden]) 74 | res = tf.matmul(flat_inputs, W) 75 | if use_bias: 76 | b = tf.get_variable( 77 | "b", [hidden], initializer=tf.constant_initializer(0.)) 78 | res = tf.nn.bias_add(res, b) 79 | # outshape就是input的最后一维变成hidden 80 | res = tf.reshape(res, out_shape) 81 | return res 82 | 83 | 84 | class ptr_net: 85 | def __init__(self, hidden, keep_prob=1.0, is_train=None, scope="ptr_net"): 86 | self.gru = tf.contrib.rnn.GRUCell(hidden) 87 | self.scope = scope 88 | self.keep_prob = keep_prob 89 | self.is_train = is_train 90 | 91 | def __call__(self, init, match, hidden, mask): 92 | with tf.variable_scope(self.scope): 93 | d_match = tf.cond(self.is_train, lambda: tf.nn.dropout( 94 | match, keep_prob=self.keep_prob), lambda: match) 95 | inp, logits1 = pointer(d_match, init, hidden, mask) 96 | d_inp = tf.cond(self.is_train, lambda: tf.nn.dropout( 97 | inp, keep_prob=self.keep_prob), lambda: inp) 98 | tf.get_variable_scope().reuse_variables() 99 | ptr_out, logits2 = pointer(d_match, d_inp, hidden, mask) 100 | return logits1, logits2, ptr_out 101 | 102 | 103 | def pointer(inputs, state, hidden, mask, scope="pointer"): 104 | with tf.variable_scope(scope): 105 | u = tf.concat([tf.tile(tf.expand_dims(state, axis=1), [ 106 | 1, tf.shape(inputs)[1], 1]), inputs], axis=2) 107 | s0 = tf.nn.tanh(dense(u, hidden, use_bias=False, scope="s0")) 108 | s = dense(s0, 1, use_bias=False, scope="s") 109 | s1 = softmax_mask(tf.squeeze(s, [2]), mask) 110 | a = tf.expand_dims(tf.nn.softmax(s1), axis=2) 111 | res = tf.reduce_sum(a * inputs, axis=1) 112 | res = dense(res, 128, use_bias=False, scope="res") 113 | return res, s1 114 | 115 | 116 | def create_model_ptr(bert_config, is_training, input_ids, input_mask, segment_ids, labels, sub_ptr, obj_ptr, num_labels): 117 | """ 118 | SO labeling, 基于ptr Net 119 | """ 120 | # 首先使用bert的输出作为embedding 121 | model = modeling.BertModel( 122 | config=bert_config, 123 | is_training=is_training, 124 | input_ids=input_ids, 125 | input_mask=input_mask, 126 | token_type_ids=segment_ids, 127 | ) 128 | bert_out = model.get_sequence_output() 129 | 130 | bert_out = tf.cond(is_training, lambda: tf.nn.dropout( 131 | bert_out, keep_prob=0.9), lambda: bert_out) 132 | 133 | # relation embedding 和 pointer network 的流程 134 | relation_init = relation_embedding(labels, num_labels, 128) 135 | 136 | # 两个pointer network，分别指向主客体，参数不重用，但是第一个的结果作为第二个的init 137 | sub_ptr_net = ptr_net(hidden=512, keep_prob=0.9, 138 | is_train=is_training, scope='sub_ptrNet') 139 | obj_ptr_net = ptr_net(hidden=512, keep_prob=0.9, 140 | is_train=is_training, scope='obj_ptrNet') 141 | 142 | sub_logits1, sub_logits2, sub_state = sub_ptr_net( 143 | relation_init, bert_out, 512, input_mask) 144 | obj_logits1, obj_logits2, _ = obj_ptr_net( 145 | sub_state, bert_out, 512, input_mask) 146 | 147 | with tf.variable_scope('loss'): 148 | sub_outer = tf.matmul(tf.expand_dims(tf.nn.softmax( 149 | sub_logits1), axis=2), tf.expand_dims(tf.nn.softmax(sub_logits2), axis=1)) 150 | sub_outer = tf.matrix_band_part(sub_outer, 0, 15) 151 | sub_h_preds = tf.argmax(tf.reduce_max(sub_outer, axis=2), axis=1) 152 | sub_t_preds = tf.argmax(tf.reduce_max(sub_outer, axis=1), axis=1) 153 | 154 | obj_outer = tf.matmul(tf.expand_dims(tf.nn.softmax( 155 | obj_logits1), axis=2), tf.expand_dims(tf.nn.softmax(obj_logits2), axis=1)) 156 | obj_outer = tf.matrix_band_part(obj_outer, 0, 15) 157 | obj_h_preds = tf.argmax(tf.reduce_max(obj_outer, axis=2), axis=1) 158 | obj_t_preds = tf.argmax(tf.reduce_max(obj_outer, axis=1), axis=1) 159 | 160 | loss_s_h = tf.nn.softmax_cross_entropy_with_logits_v2( 161 | logits=sub_logits1, labels=tf.stop_gradient(tf.one_hot(sub_ptr[:, :1], depth=150, dtype=tf.float32))) 162 | loss_s_t = tf.nn.softmax_cross_entropy_with_logits_v2( 163 | logits=sub_logits2, labels=tf.stop_gradient(tf.one_hot(sub_ptr[:, 1:], depth=150, dtype=tf.float32))) 164 | loss_o_h = tf.nn.softmax_cross_entropy_with_logits_v2( 165 | logits=obj_logits1, labels=tf.stop_gradient(tf.one_hot(obj_ptr[:, :1], depth=150, dtype=tf.float32))) 166 | loss_o_t = tf.nn.softmax_cross_entropy_with_logits_v2( 167 | logits=obj_logits2, labels=tf.stop_gradient(tf.one_hot(obj_ptr[:, 1:], depth=150, dtype=tf.float32))) 168 | 169 | loss = tf.reduce_mean(loss_s_h + loss_s_t + loss_o_h + loss_o_t) 170 | 171 | preds = tf.concat([tf.expand_dims(sub_h_preds, axis=1), tf.expand_dims(sub_t_preds, axis=1), tf.expand_dims( 172 | obj_h_preds, axis=1), tf.expand_dims(obj_t_preds, axis=1)], axis=-1, name='pred_ids') 173 | 174 | return (loss, preds) 175 | -------------------------------------------------------------------------------- /baseline_with_tf.estimator/RC/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 一些公共模型代码 5 | """ 6 | 7 | import sys 8 | import tensorflow as tf 9 | from tensorflow.contrib.layers.python.layers import initializers 10 | sys.path.append("../") 11 | from bert.bert_code import modeling, optimization, tokenization 12 | 13 | __all__ = ['InputExample', 'InputFeatures', 'decode_labels', 'create_model', 'convert_id_str', 14 | 'convert_id_to_label', 'result_to_json', 'create_classification_model'] 15 | 16 | 17 | class Model(object): 18 | def __init__(self, *args, **kwargs): 19 | pass 20 | 21 | 22 | class InputExample(object): 23 | """A single training/test example for simple sequence classification.""" 24 | 25 | def __init__(self, guid, text_a, text_b=None, label=None): 26 | """Constructs a InputExample. 27 | Args: 28 | guid: Unique id for the example. 29 | text_a: string. The untokenized text of the first sequence. For single 30 | sequence tasks, only this sequence must be specified. 31 | label: (Optional) string. The label of the example. This should be 32 | specified for train and dev examples, but not for test examples. 33 | """ 34 | self.guid = guid 35 | self.text_a = text_a 36 | self.text_b = text_b 37 | self.label = label 38 | 39 | 40 | class InputFeatures(object): 41 | """A single set of features of data.""" 42 | 43 | def __init__(self, input_ids, input_mask, segment_ids, label_id, ): 44 | self.input_ids = input_ids 45 | self.input_mask = input_mask 46 | self.segment_ids = segment_ids 47 | self.label_id = label_id 48 | 49 | 50 | class DataProcessor(object): 51 | """Base class for data converters for sequence classification data sets.""" 52 | 53 | def get_train_examples(self, data_dir): 54 | """Gets a collection of `InputExample`s for the train set.""" 55 | raise NotImplementedError() 56 | 57 | def get_dev_examples(self, data_dir): 58 | """Gets a collection of `InputExample`s for the dev set.""" 59 | raise NotImplementedError() 60 | 61 | def get_labels(self): 62 | """Gets the list of labels for this data set.""" 63 | raise NotImplementedError() 64 | 65 | 66 | def create_model(bert_config, is_training, input_ids, input_mask, segment_ids, labels, num_labels): 67 | """ 68 | 69 | :param bert_config: 70 | :param is_training: 71 | :param input_ids: 72 | :param input_mask: 73 | :param segment_ids: 74 | :param labels: 75 | :param num_labels: 76 | :param use_one_hot_embedding: 77 | :return: 78 | """ 79 | # 通过传入的训练数据，进行representation 80 | model = modeling.BertModel( 81 | config=bert_config, 82 | is_training=is_training, 83 | input_ids=input_ids, 84 | input_mask=input_mask, 85 | token_type_ids=segment_ids, 86 | ) 87 | 88 | # In the demo, we are doing a simple classification task on the entire 89 | # segment. 90 | # 91 | # If you want to use the token-level output, use model.get_sequence_output() 92 | # instead. 93 | output_layer = model.get_pooled_output() 94 | 95 | hidden_size = output_layer.shape[-1].value 96 | 97 | output_weights = tf.get_variable( 98 | "output_weights", [num_labels, hidden_size], 99 | initializer=tf.truncated_normal_initializer(stddev=0.02)) 100 | 101 | output_bias = tf.get_variable( 102 | "output_bias", [num_labels], initializer=tf.zeros_initializer()) 103 | 104 | with tf.variable_scope("loss"): 105 | if is_training: 106 | # I.e., 0.1 dropout 107 | output_layer = tf.nn.dropout(output_layer, keep_prob=0.9) 108 | 109 | logits = tf.matmul(output_layer, output_weights, transpose_b=True) 110 | logits = tf.nn.bias_add(logits, output_bias) 111 | probabilities = tf.nn.softmax(logits, axis=-1) 112 | log_probs = tf.nn.log_softmax(logits, axis=-1) 113 | 114 | one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32) 115 | 116 | per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) 117 | loss = tf.reduce_mean(per_example_loss) 118 | 119 | return (loss, per_example_loss, logits, probabilities) 120 | 121 | 122 | def decode_labels(labels, batch_size): 123 | new_labels = [] 124 | for row in range(batch_size): 125 | label = [] 126 | for i in labels[row]: 127 | i = i.decode('utf-8') 128 | if i == '**PAD**': 129 | break 130 | if i in ['[CLS]', '[SEP]']: 131 | continue 132 | label.append(i) 133 | new_labels.append(label) 134 | return new_labels 135 | 136 | 137 | def convert_id_str(input_ids, batch_size): 138 | res = [] 139 | for row in range(batch_size): 140 | line = [] 141 | for i in input_ids[row]: 142 | i = i.decode('utf-8') 143 | if i == '**PAD**': 144 | break 145 | if i in ['[CLS]', '[SEP]']: 146 | continue 147 | 148 | line.append(i) 149 | res.append(line) 150 | return res 151 | 152 | 153 | def convert_id_to_label(pred_ids_result, idx2label, batch_size): 154 | """ 155 | 将id形式的结果转化为真实序列结果 156 | :param pred_ids_result: 157 | :param idx2label: 158 | :return: 159 | """ 160 | result = [] 161 | index_result = [] 162 | for row in range(batch_size): 163 | curr_seq = [] 164 | curr_idx = [] 165 | ids = pred_ids_result[row] 166 | for idx, id in enumerate(ids): 167 | if id == 0: 168 | break 169 | curr_label = idx2label[id] 170 | if curr_label in ['[CLS]', '[SEP]']: 171 | if id == 102 and (idx < len(ids) and ids[idx + 1] == 0): 172 | break 173 | continue 174 | # elif curr_label == '[SEP]': 175 | # break 176 | curr_seq.append(curr_label) 177 | curr_idx.append(id) 178 | result.append(curr_seq) 179 | index_result.append(curr_idx) 180 | return result, index_result 181 | 182 | 183 | def result_to_json(self, string, tags): 184 | """ 185 | 将模型标注序列和输入序列结合 转化为结果 186 | :param string: 输入序列 187 | :param tags: 标注结果 188 | :return: 189 | """ 190 | item = {"entities": []} 191 | entity_name = "" 192 | entity_start = 0 193 | idx = 0 194 | last_tag = '' 195 | 196 | for char, tag in zip(string, tags): 197 | if tag[0] == "S": 198 | self.append(char, idx, idx + 1, tag[2:]) 199 | item["entities"].append( 200 | {"word": char, "start": idx, "end": idx + 1, "type": tag[2:]}) 201 | elif tag[0] == "B": 202 | if entity_name != '': 203 | self.append(entity_name, entity_start, idx, last_tag[2:]) 204 | item["entities"].append( 205 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 206 | entity_name = "" 207 | entity_name += char 208 | entity_start = idx 209 | elif tag[0] == "I": 210 | entity_name += char 211 | elif tag[0] == "O": 212 | if entity_name != '': 213 | self.append(entity_name, entity_start, idx, last_tag[2:]) 214 | item["entities"].append( 215 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 216 | entity_name = "" 217 | else: 218 | entity_name = "" 219 | entity_start = idx 220 | idx += 1 221 | last_tag = tag 222 | if entity_name != '': 223 | self.append(entity_name, entity_start, idx, last_tag[2:]) 224 | item["entities"].append( 225 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 226 | return item 227 | -------------------------------------------------------------------------------- /bert/bert_code/modeling_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import collections 20 | import json 21 | import random 22 | import re 23 | 24 | import modeling 25 | import six 26 | import tensorflow as tf 27 | 28 | 29 | class BertModelTest(tf.test.TestCase): 30 | 31 | class BertModelTester(object): 32 | 33 | def __init__(self, 34 | parent, 35 | batch_size=13, 36 | seq_length=7, 37 | is_training=True, 38 | use_input_mask=True, 39 | use_token_type_ids=True, 40 | vocab_size=99, 41 | hidden_size=32, 42 | num_hidden_layers=5, 43 | num_attention_heads=4, 44 | intermediate_size=37, 45 | hidden_act="gelu", 46 | hidden_dropout_prob=0.1, 47 | attention_probs_dropout_prob=0.1, 48 | max_position_embeddings=512, 49 | type_vocab_size=16, 50 | initializer_range=0.02, 51 | scope=None): 52 | self.parent = parent 53 | self.batch_size = batch_size 54 | self.seq_length = seq_length 55 | self.is_training = is_training 56 | self.use_input_mask = use_input_mask 57 | self.use_token_type_ids = use_token_type_ids 58 | self.vocab_size = vocab_size 59 | self.hidden_size = hidden_size 60 | self.num_hidden_layers = num_hidden_layers 61 | self.num_attention_heads = num_attention_heads 62 | self.intermediate_size = intermediate_size 63 | self.hidden_act = hidden_act 64 | self.hidden_dropout_prob = hidden_dropout_prob 65 | self.attention_probs_dropout_prob = attention_probs_dropout_prob 66 | self.max_position_embeddings = max_position_embeddings 67 | self.type_vocab_size = type_vocab_size 68 | self.initializer_range = initializer_range 69 | self.scope = scope 70 | 71 | def create_model(self): 72 | input_ids = BertModelTest.ids_tensor([self.batch_size, self.seq_length], 73 | self.vocab_size) 74 | 75 | input_mask = None 76 | if self.use_input_mask: 77 | input_mask = BertModelTest.ids_tensor( 78 | [self.batch_size, self.seq_length], vocab_size=2) 79 | 80 | token_type_ids = None 81 | if self.use_token_type_ids: 82 | token_type_ids = BertModelTest.ids_tensor( 83 | [self.batch_size, self.seq_length], self.type_vocab_size) 84 | 85 | config = modeling.BertConfig( 86 | vocab_size=self.vocab_size, 87 | hidden_size=self.hidden_size, 88 | num_hidden_layers=self.num_hidden_layers, 89 | num_attention_heads=self.num_attention_heads, 90 | intermediate_size=self.intermediate_size, 91 | hidden_act=self.hidden_act, 92 | hidden_dropout_prob=self.hidden_dropout_prob, 93 | attention_probs_dropout_prob=self.attention_probs_dropout_prob, 94 | max_position_embeddings=self.max_position_embeddings, 95 | type_vocab_size=self.type_vocab_size, 96 | initializer_range=self.initializer_range) 97 | 98 | model = modeling.BertModel( 99 | config=config, 100 | is_training=self.is_training, 101 | input_ids=input_ids, 102 | input_mask=input_mask, 103 | token_type_ids=token_type_ids, 104 | scope=self.scope) 105 | 106 | outputs = { 107 | "embedding_output": model.get_embedding_output(), 108 | "sequence_output": model.get_sequence_output(), 109 | "pooled_output": model.get_pooled_output(), 110 | "all_encoder_layers": model.get_all_encoder_layers(), 111 | } 112 | return outputs 113 | 114 | def check_output(self, result): 115 | self.parent.assertAllEqual( 116 | result["embedding_output"].shape, 117 | [self.batch_size, self.seq_length, self.hidden_size]) 118 | 119 | self.parent.assertAllEqual( 120 | result["sequence_output"].shape, 121 | [self.batch_size, self.seq_length, self.hidden_size]) 122 | 123 | self.parent.assertAllEqual(result["pooled_output"].shape, 124 | [self.batch_size, self.hidden_size]) 125 | 126 | def test_default(self): 127 | self.run_tester(BertModelTest.BertModelTester(self)) 128 | 129 | def test_config_to_json_string(self): 130 | config = modeling.BertConfig(vocab_size=99, hidden_size=37) 131 | obj = json.loads(config.to_json_string()) 132 | self.assertEqual(obj["vocab_size"], 99) 133 | self.assertEqual(obj["hidden_size"], 37) 134 | 135 | def run_tester(self, tester): 136 | with self.test_session() as sess: 137 | ops = tester.create_model() 138 | init_op = tf.group(tf.global_variables_initializer(), 139 | tf.local_variables_initializer()) 140 | sess.run(init_op) 141 | output_result = sess.run(ops) 142 | tester.check_output(output_result) 143 | 144 | self.assert_all_tensors_reachable(sess, [init_op, ops]) 145 | 146 | @classmethod 147 | def ids_tensor(cls, shape, vocab_size, rng=None, name=None): 148 | """Creates a random int32 tensor of the shape within the vocab size.""" 149 | if rng is None: 150 | rng = random.Random() 151 | 152 | total_dims = 1 153 | for dim in shape: 154 | total_dims *= dim 155 | 156 | values = [] 157 | for _ in range(total_dims): 158 | values.append(rng.randint(0, vocab_size - 1)) 159 | 160 | return tf.constant(value=values, dtype=tf.int32, shape=shape, name=name) 161 | 162 | def assert_all_tensors_reachable(self, sess, outputs): 163 | """Checks that all the tensors in the graph are reachable from outputs.""" 164 | graph = sess.graph 165 | 166 | ignore_strings = [ 167 | "^.*/assert_less_equal/.*$", 168 | "^.*/dilation_rate$", 169 | "^.*/Tensordot/concat$", 170 | "^.*/Tensordot/concat/axis$", 171 | "^testing/.*$", 172 | ] 173 | 174 | ignore_regexes = [re.compile(x) for x in ignore_strings] 175 | 176 | unreachable = self.get_unreachable_ops(graph, outputs) 177 | filtered_unreachable = [] 178 | for x in unreachable: 179 | do_ignore = False 180 | for r in ignore_regexes: 181 | m = r.match(x.name) 182 | if m is not None: 183 | do_ignore = True 184 | if do_ignore: 185 | continue 186 | filtered_unreachable.append(x) 187 | unreachable = filtered_unreachable 188 | 189 | self.assertEqual( 190 | len(unreachable), 0, "The following ops are unreachable: %s" % 191 | (" ".join([x.name for x in unreachable]))) 192 | 193 | @classmethod 194 | def get_unreachable_ops(cls, graph, outputs): 195 | """Finds all of the tensors in graph that are unreachable from outputs.""" 196 | outputs = cls.flatten_recursive(outputs) 197 | output_to_op = collections.defaultdict(list) 198 | op_to_all = collections.defaultdict(list) 199 | assign_out_to_in = collections.defaultdict(list) 200 | 201 | for op in graph.get_operations(): 202 | for x in op.inputs: 203 | op_to_all[op.name].append(x.name) 204 | for y in op.outputs: 205 | output_to_op[y.name].append(op.name) 206 | op_to_all[op.name].append(y.name) 207 | if str(op.type) == "Assign": 208 | for y in op.outputs: 209 | for x in op.inputs: 210 | assign_out_to_in[y.name].append(x.name) 211 | 212 | assign_groups = collections.defaultdict(list) 213 | for out_name in assign_out_to_in.keys(): 214 | name_group = assign_out_to_in[out_name] 215 | for n1 in name_group: 216 | assign_groups[n1].append(out_name) 217 | for n2 in name_group: 218 | if n1 != n2: 219 | assign_groups[n1].append(n2) 220 | 221 | seen_tensors = {} 222 | stack = [x.name for x in outputs] 223 | while stack: 224 | name = stack.pop() 225 | if name in seen_tensors: 226 | continue 227 | seen_tensors[name] = True 228 | 229 | if name in output_to_op: 230 | for op_name in output_to_op[name]: 231 | if op_name in op_to_all: 232 | for input_name in op_to_all[op_name]: 233 | if input_name not in stack: 234 | stack.append(input_name) 235 | 236 | expanded_names = [] 237 | if name in assign_groups: 238 | for assign_name in assign_groups[name]: 239 | expanded_names.append(assign_name) 240 | 241 | for expanded_name in expanded_names: 242 | if expanded_name not in stack: 243 | stack.append(expanded_name) 244 | 245 | unreachable_ops = [] 246 | for op in graph.get_operations(): 247 | is_unreachable = False 248 | all_names = [x.name for x in op.inputs] + [x.name for x in op.outputs] 249 | for name in all_names: 250 | if name not in seen_tensors: 251 | is_unreachable = True 252 | if is_unreachable: 253 | unreachable_ops.append(op) 254 | return unreachable_ops 255 | 256 | @classmethod 257 | def flatten_recursive(cls, item): 258 | """Flattens (potentially nested) a tuple/dictionary/list to a list.""" 259 | output = [] 260 | if isinstance(item, list): 261 | output.extend(item) 262 | elif isinstance(item, tuple): 263 | output.extend(list(item)) 264 | elif isinstance(item, dict): 265 | for (_, v) in six.iteritems(item): 266 | output.append(v) 267 | else: 268 | return [item] 269 | 270 | flat_output = [] 271 | for x in output: 272 | flat_output.extend(cls.flatten_recursive(x)) 273 | return flat_output 274 | 275 | 276 | if __name__ == "__main__": 277 | tf.test.main() 278 | -------------------------------------------------------------------------------- /baseline_with_tf.estimator/NER/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | 一些公共模型代码 5 | """ 6 | 7 | import sys 8 | from lstm_crf_layer import BLSTM_CRF 9 | from tensorflow.contrib.layers.python.layers import initializers 10 | sys.path.append("../") 11 | from bert.bert_code import modeling, optimization, tokenization 12 | 13 | __all__ = ['InputExample', 'InputFeatures', 'decode_labels', 'create_model', 'convert_id_str', 14 | 'convert_id_to_label', 'result_to_json', 'create_classification_model'] 15 | 16 | 17 | class Model(object): 18 | def __init__(self, *args, **kwargs): 19 | pass 20 | 21 | 22 | class InputExample(object): 23 | """A single training/test example for simple sequence classification.""" 24 | 25 | def __init__(self, guid=None, text=None, label=None): 26 | """Constructs a InputExample. 27 | Args: 28 | guid: Unique id for the example. 29 | text_a: string. The untokenized text of the first sequence. For single 30 | sequence tasks, only this sequence must be specified. 31 | label: (Optional) string. The label of the example. This should be 32 | specified for train and dev examples, but not for test examples. 33 | """ 34 | self.guid = guid 35 | self.text = text 36 | self.label = label 37 | 38 | 39 | class InputFeatures(object): 40 | """A single set of features of data.""" 41 | 42 | def __init__(self, input_ids, input_mask, segment_ids, label_ids, ): 43 | self.input_ids = input_ids 44 | self.input_mask = input_mask 45 | self.segment_ids = segment_ids 46 | self.label_ids = label_ids 47 | # self.label_mask = label_mask 48 | 49 | 50 | class DataProcessor(object): 51 | """Base class for data converters for sequence classification data sets.""" 52 | 53 | def get_train_examples(self, data_dir): 54 | """Gets a collection of `InputExample`s for the train set.""" 55 | raise NotImplementedError() 56 | 57 | def get_dev_examples(self, data_dir): 58 | """Gets a collection of `InputExample`s for the dev set.""" 59 | raise NotImplementedError() 60 | 61 | def get_labels(self): 62 | """Gets the list of labels for this data set.""" 63 | raise NotImplementedError() 64 | 65 | 66 | def create_model(bert_config, is_training, input_ids, input_mask, 67 | segment_ids, labels, num_labels, use_one_hot_embeddings, 68 | dropout_rate=1.0, lstm_size=1, cell='lstm', num_layers=1): 69 | """ 70 | 创建X模型 71 | :param bert_config: bert 配置 72 | :param is_training: 73 | :param input_ids: 数据的idx 表示 74 | :param input_mask: 75 | :param segment_ids: 76 | :param labels: 标签的idx 表示 77 | :param num_labels: 类别数量 78 | :param use_one_hot_embeddings: 79 | :return: 80 | """ 81 | # 使用数据加载BertModel,获取对应的字embedding 82 | import tensorflow as tf 83 | model = modeling.BertModel( 84 | config=bert_config, 85 | is_training=is_training, 86 | input_ids=input_ids, 87 | input_mask=input_mask, 88 | token_type_ids=segment_ids, 89 | use_one_hot_embeddings=use_one_hot_embeddings 90 | ) 91 | # 获取对应的embedding 输入数据[batch_size, seq_length, embedding_size] 92 | embedding = model.get_sequence_output() 93 | max_seq_length = embedding.shape[1].value 94 | # 算序列真实长度 95 | used = tf.sign(tf.abs(input_ids)) 96 | # [batch_size] 大小的向量，包含了当前batch中的序列长度 97 | lengths = tf.reduce_sum(used, reduction_indices=1) 98 | # 添加CRF output layer 99 | blstm_crf = BLSTM_CRF(embedded_chars=embedding, hidden_unit=lstm_size, cell_type=cell, num_layers=num_layers, 100 | dropout_rate=dropout_rate, initializers=initializers, num_labels=num_labels, 101 | seq_length=max_seq_length, labels=labels, lengths=lengths, is_training=is_training) 102 | rst = blstm_crf.add_blstm_crf_layer(crf_only=False) 103 | return rst 104 | 105 | 106 | def create_classification_model(bert_config, is_training, input_ids, input_mask, segment_ids, labels, num_labels): 107 | """ 108 | 109 | :param bert_config: 110 | :param is_training: 111 | :param input_ids: 112 | :param input_mask: 113 | :param segment_ids: 114 | :param labels: 115 | :param num_labels: 116 | :param use_one_hot_embedding: 117 | :return: 118 | """ 119 | import tensorflow as tf 120 | # 通过传入的训练数据，进行representation 121 | model = modeling.BertModel( 122 | config=bert_config, 123 | is_training=is_training, 124 | input_ids=input_ids, 125 | input_mask=input_mask, 126 | token_type_ids=segment_ids, 127 | ) 128 | 129 | embedding_layer = model.get_sequence_output() 130 | output_layer = model.get_pooled_output() 131 | hidden_size = output_layer.shape[-1].value 132 | 133 | # predict = CNN_Classification(embedding_chars=embedding_layer, 134 | # labels=labels, 135 | # num_tags=num_labels, 136 | # sequence_length=FLAGS.max_seq_length, 137 | # embedding_dims=embedding_layer.shape[-1].value, 138 | # vocab_size=0, 139 | # filter_sizes=[3, 4, 5], 140 | # num_filters=3, 141 | # dropout_keep_prob=FLAGS.dropout_keep_prob, 142 | # l2_reg_lambda=0.001) 143 | # loss, predictions, probabilities = predict.add_cnn_layer() 144 | 145 | output_weights = tf.get_variable( 146 | "output_weights", [num_labels, hidden_size], 147 | initializer=tf.truncated_normal_initializer(stddev=0.02)) 148 | 149 | output_bias = tf.get_variable( 150 | "output_bias", [num_labels], initializer=tf.zeros_initializer()) 151 | 152 | with tf.variable_scope("loss"): 153 | if is_training: 154 | # I.e., 0.1 dropout 155 | output_layer = tf.nn.dropout(output_layer, keep_prob=0.9) 156 | 157 | logits = tf.matmul(output_layer, output_weights, transpose_b=True) 158 | logits = tf.nn.bias_add(logits, output_bias) 159 | probabilities = tf.nn.softmax(logits, axis=-1) 160 | log_probs = tf.nn.log_softmax(logits, axis=-1) 161 | 162 | if labels is not None: 163 | one_hot_labels = tf.one_hot( 164 | labels, depth=num_labels, dtype=tf.float32) 165 | 166 | per_example_loss = - \ 167 | tf.reduce_sum(one_hot_labels * log_probs, axis=-1) 168 | loss = tf.reduce_mean(per_example_loss) 169 | else: 170 | loss, per_example_loss = None, None 171 | return (loss, per_example_loss, logits, probabilities) 172 | 173 | 174 | def decode_labels(labels, batch_size): 175 | new_labels = [] 176 | for row in range(batch_size): 177 | label = [] 178 | for i in labels[row]: 179 | i = i.decode('utf-8') 180 | if i == '**PAD**': 181 | break 182 | if i in ['[CLS]', '[SEP]']: 183 | continue 184 | label.append(i) 185 | new_labels.append(label) 186 | return new_labels 187 | 188 | 189 | def convert_id_str(input_ids, batch_size): 190 | res = [] 191 | for row in range(batch_size): 192 | line = [] 193 | for i in input_ids[row]: 194 | i = i.decode('utf-8') 195 | if i == '**PAD**': 196 | break 197 | if i in ['[CLS]', '[SEP]']: 198 | continue 199 | 200 | line.append(i) 201 | res.append(line) 202 | return res 203 | 204 | 205 | def convert_id_to_label(pred_ids_result, idx2label, batch_size): 206 | """ 207 | 将id形式的结果转化为真实序列结果 208 | :param pred_ids_result: 209 | :param idx2label: 210 | :return: 211 | """ 212 | result = [] 213 | index_result = [] 214 | for row in range(batch_size): 215 | curr_seq = [] 216 | curr_idx = [] 217 | ids = pred_ids_result[row] 218 | for idx, id in enumerate(ids): 219 | if id == 0: 220 | break 221 | curr_label = idx2label[id] 222 | if curr_label in ['[CLS]', '[SEP]']: 223 | if id == 102 and (idx < len(ids) and ids[idx + 1] == 0): 224 | break 225 | continue 226 | # elif curr_label == '[SEP]': 227 | # break 228 | curr_seq.append(curr_label) 229 | curr_idx.append(id) 230 | result.append(curr_seq) 231 | index_result.append(curr_idx) 232 | return result, index_result 233 | 234 | 235 | def result_to_json(self, string, tags): 236 | """ 237 | 将模型标注序列和输入序列结合 转化为结果 238 | :param string: 输入序列 239 | :param tags: 标注结果 240 | :return: 241 | """ 242 | item = {"entities": []} 243 | entity_name = "" 244 | entity_start = 0 245 | idx = 0 246 | last_tag = '' 247 | 248 | for char, tag in zip(string, tags): 249 | if tag[0] == "S": 250 | self.append(char, idx, idx + 1, tag[2:]) 251 | item["entities"].append( 252 | {"word": char, "start": idx, "end": idx + 1, "type": tag[2:]}) 253 | elif tag[0] == "B": 254 | if entity_name != '': 255 | self.append(entity_name, entity_start, idx, last_tag[2:]) 256 | item["entities"].append( 257 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 258 | entity_name = "" 259 | entity_name += char 260 | entity_start = idx 261 | elif tag[0] == "I": 262 | entity_name += char 263 | elif tag[0] == "O": 264 | if entity_name != '': 265 | self.append(entity_name, entity_start, idx, last_tag[2:]) 266 | item["entities"].append( 267 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 268 | entity_name = "" 269 | else: 270 | entity_name = "" 271 | entity_start = idx 272 | idx += 1 273 | last_tag = tag 274 | if entity_name != '': 275 | self.append(entity_name, entity_start, idx, last_tag[2:]) 276 | item["entities"].append( 277 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 278 | return item 279 | -------------------------------------------------------------------------------- /P_classification/data_reader_for_PC.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """ 3 | Data processing for P-classification. 4 | @author: yuhaitao 5 | """ 6 | import json 7 | import os 8 | import codecs 9 | import sys 10 | import re 11 | import pandas as pd 12 | import numpy as np 13 | from tqdm import tqdm 14 | 15 | 16 | class MyDataReader(object): 17 | """ 18 | class for my data reader 19 | """ 20 | 21 | def __init__(self, 22 | postag_dict_path, 23 | label_dict_path, 24 | train_data_list_path='', 25 | dev_data_list_path=''): 26 | self._postag_dict_path = postag_dict_path 27 | self._label_dict_path = label_dict_path 28 | self.train_data_list_path = train_data_list_path 29 | self.dev_data_list_path = dev_data_list_path 30 | self._p_map_eng_dict = {} 31 | # load dictionary 32 | self._dict_path_dict = {'postag_dict': self._postag_dict_path, 33 | 'label_dict': self._label_dict_path} 34 | # check if the file exists 35 | for input_dict in [postag_dict_path, 36 | label_dict_path, train_data_list_path, dev_data_list_path]: 37 | if not os.path.exists(input_dict): 38 | raise ValueError("%s not found." % (input_dict)) 39 | return 40 | 41 | self._feature_dict = {} 42 | self._feature_dict['postag_dict'] = \ 43 | self._load_dict_from_file(self._dict_path_dict['postag_dict']) 44 | self._feature_dict['label_dict'], self.label_eng_dict = \ 45 | self._load_label_dict(self._dict_path_dict['label_dict']) 46 | print(self.label_eng_dict) 47 | # 将之前所有的字典反向 48 | self._reverse_dict = {name: self._get_reverse_dict(name) for name in 49 | self._dict_path_dict.keys()} 50 | self._reverse_dict['eng_map_p_dict'] = self._reverse_p_eng( 51 | self._p_map_eng_dict) 52 | self._UNK_IDX = 0 53 | 54 | # 统计在所有训练数据中主体和客体所覆盖的postag 55 | # 主体subject，客体object 56 | # self.subject_tags, self.object_tags = self.count_tags( 57 | # self.train_data_list_path, self._postag_dict_path) 58 | 59 | def _load_label_dict(self, dict_name): 60 | """这个函数重写了""" 61 | label_dict = {} 62 | label_to_eng = {} 63 | pattern = re.compile(r'\s+') 64 | with codecs.open(dict_name, 'r', 'utf-8') as fr: 65 | for idx, line in enumerate(fr): 66 | p, p_eng = re.split(pattern, line.strip()) 67 | label_to_eng[p] = p_eng 68 | label_dict[p_eng] = idx 69 | self._p_map_eng_dict[p] = p_eng 70 | # if p != '木有关系': 71 | # label_to_eng['反_' + p] = 'RE_' + p_eng 72 | # label_dict['RE_' + p_eng] = idx + 51 73 | # self._p_map_eng_dict['反_' + p] = 'RE_' + p_eng 74 | return label_dict, label_to_eng 75 | 76 | def _load_dict_from_file(self, dict_name, bias=0): 77 | """ 78 | Load vocabulary from file. 79 | """ 80 | dict_result = {} 81 | with codecs.open(dict_name, 'r', 'utf-8') as f_dict: 82 | for idx, line in enumerate(f_dict): 83 | line = line.strip() 84 | dict_result[line] = idx + bias 85 | return dict_result 86 | 87 | def _cal_mark_single_slot(self, spo_list, sentence): 88 | """ 89 | Calculate the value of the label 90 | """ 91 | mark_list = [0] * len(self._feature_dict['label_dict']) 92 | for spo in spo_list: 93 | predicate = spo['predicate'] 94 | p_idx = self._feature_dict['label_dict'][self._p_map_eng_dict[predicate]] 95 | mark_list[p_idx] = 1 96 | return mark_list 97 | 98 | def _is_valid_input_data(self, input_data): 99 | """is the input data valid""" 100 | try: 101 | dic = json.loads(input_data) 102 | except: 103 | return False 104 | if "text" not in dic or "postag" not in dic or \ 105 | type(dic["postag"]) is not list: 106 | return False 107 | for item in dic['postag']: 108 | if "word" not in item or "pos" not in item: 109 | return False 110 | return True 111 | 112 | def _get_feed_iterator(self, line, need_input=False, need_label=True): 113 | """ 114 | 生成一条数据，应修改为对每一个line生成一个样本列表，其中除了正样本，每个line还生成一个负样本 115 | """ 116 | # verify that the input format of each line meets the format 117 | if not self._is_valid_input_data(line): 118 | print >> sys.stderr, 'Format is error' 119 | return None 120 | dic = json.loads(line) 121 | sentence = dic['text'] 122 | sentence = ''.join(s.strip() for s in sentence.split()) 123 | text_and_label = sentence 124 | if need_label: 125 | label = set() 126 | for spo in dic['spo_list']: 127 | label.add(self.label_eng_dict[spo['predicate']]) 128 | for l in label: 129 | text_and_label += ('\t' + l) 130 | else: 131 | text_and_label += ('\t' + 'RP') 132 | 133 | return text_and_label 134 | 135 | def path_reader(self, data_path, need_input=False, need_label=True): 136 | """Read data from data_path""" 137 | def reader(): 138 | """Generator""" 139 | for line in open(data_path.strip()): 140 | # 对文件每一行生成数据 141 | text_and_label = self._get_feed_iterator( 142 | line.strip(), need_input, need_label) 143 | if text_and_label is None: 144 | continue 145 | yield text_and_label 146 | 147 | return reader 148 | 149 | def count_tags(self, train_file, postag_file): 150 | """ 151 | 统计所有主客体覆盖到的postag类别 152 | """ 153 | subject_tags, object_tags = {}, {} 154 | 155 | # 如果文件存在 156 | if os.path.isfile('../dict/sub_tag') and os.path.isfile('../dict/obj_tag'): 157 | with open('../dict/sub_tag', 'r') as fs: 158 | for line in fs: 159 | key, value = line.strip().split('\t') 160 | subject_tags[key] = int(value) 161 | with open('../dict/obj_tag', 'r') as fo: 162 | for line in fo: 163 | key, value = line.strip().split('\t') 164 | object_tags[key] = int(value) 165 | return subject_tags, object_tags 166 | 167 | # 如果文件不存在 168 | with open(postag_file, 'r') as f: 169 | for line in f: 170 | tag = line.strip() 171 | subject_tags[tag], object_tags[tag] = 0, 0 172 | print("开始统计主客体的postag类别...") 173 | with open(train_file, 'r') as f: 174 | for line in tqdm(f): 175 | dic = json.loads(line.strip()) 176 | for spo in dic['spo_list']: 177 | for postag in dic['postag']: 178 | if postag['word'] == spo['subject']: 179 | subject_tags[postag['pos']] += 1 180 | break 181 | for postag in dic['postag']: 182 | if postag['word'] == spo['object']: 183 | object_tags[postag['pos']] += 1 184 | break 185 | s = list(subject_tags.keys()) 186 | o = list(object_tags.keys()) 187 | for key in s: 188 | if subject_tags[key] == 0: 189 | del subject_tags[key] 190 | for key in o: 191 | if object_tags[key] == 0: 192 | del object_tags[key] 193 | with open('../dict/sub_tag', 'w') as fs: 194 | for key, value in subject_tags.items(): 195 | fs.write(key + '\t' + str(value) + '\n') 196 | with open('../dict/obj_tag', 'w') as fo: 197 | for key, value in object_tags.items(): 198 | fo.write(key + '\t' + str(value) + '\n') 199 | 200 | return subject_tags, object_tags 201 | 202 | def get_train_reader(self, need_input=False, need_label=True): 203 | """Data reader during training""" 204 | return self.path_reader(self.train_data_list_path, need_input, need_label) 205 | 206 | def get_dev_reader(self, need_input=True, need_label=True): 207 | """Data reader during dev""" 208 | return self.path_reader(self.dev_data_list_path, need_input, need_label) 209 | 210 | def get_test_reader(self, test_file_path='', need_input=True, need_label=False): 211 | """Data reader during predict""" 212 | return self.path_reader(test_file_path, need_input, need_label) 213 | 214 | def get_dict(self, dict_name): 215 | """Return dict""" 216 | if dict_name not in self._feature_dict: 217 | raise ValueError("dict name %s not found." % (dict_name)) 218 | return self._feature_dict[dict_name] 219 | 220 | def get_all_dict_name(self): 221 | """Get name of all dict""" 222 | return self._feature_dict.keys() 223 | 224 | def get_dict_size(self, dict_name): 225 | """Return dict length""" 226 | if dict_name not in self._feature_dict: 227 | raise ValueError("dict name %s not found." % (dict_name)) 228 | return len(self._feature_dict[dict_name]) 229 | 230 | def _get_reverse_dict(self, dict_name): 231 | dict_reverse = {} 232 | for key, value in self._feature_dict[dict_name].items(): 233 | dict_reverse[value] = key 234 | return dict_reverse 235 | 236 | def _reverse_p_eng(self, dic): 237 | dict_reverse = {} 238 | for key, value in dic.items(): 239 | dict_reverse[value] = key 240 | return dict_reverse 241 | 242 | def get_label_output(self, label_idx): 243 | """Output final label, used during predict and test""" 244 | dict_name = 'label_dict' 245 | if len(self._reverse_dict[dict_name]) == 0: 246 | self._get_reverse_dict(dict_name) 247 | p_eng = self._reverse_dict[dict_name][label_idx] 248 | return self._reverse_dict['eng_map_p_dict'][p_eng] 249 | 250 | 251 | if __name__ == '__main__': 252 | # initialize data generator 253 | data_generator = MyDataReader( 254 | postag_dict_path='../dict/postag_dict', 255 | label_dict_path='../dict/p_eng', 256 | train_data_list_path='../data/ori_data/train_data.json', 257 | dev_data_list_path='../data/ori_data/dev_data.json') 258 | 259 | # prepare data reader 260 | train = data_generator.get_train_reader() 261 | with open("../data/PC_data/train.txt", 'w') as f: 262 | for text_and_label in tqdm(train()): 263 | f.write(text_and_label + '\n') 264 | 265 | dev = data_generator.get_dev_reader() 266 | index = 0 267 | with open("../data/PC_data/dev.txt", 'w') as f: 268 | for text_and_label in tqdm(dev()): 269 | f.write(text_and_label + '\n') 270 | 271 | test = data_generator.get_test_reader( 272 | test_file_path='../data/ori_data/test1_data_postag.json') 273 | with open("../data/PC_data/test.txt", 'w') as f: 274 | for text_and_label in tqdm(test()): 275 | f.write(text_and_label + '\n') 276 | -------------------------------------------------------------------------------- /baseline_with_tf.estimator/NER/conlleval.py: -------------------------------------------------------------------------------- 1 | # Python version of the evaluation script from CoNLL'00- 2 | # Originates from: https://github.com/spyysalo/conlleval.py 3 | 4 | 5 | # Intentional differences: 6 | # - accept any space as delimiter by default 7 | # - optional file argument (default STDIN) 8 | # - option to set boundary (-b argument) 9 | # - LaTeX output (-l argument) not supported 10 | # - raw tags (-r argument) not supported 11 | 12 | # add function :evaluate(predicted_label, ori_label): which will not read from file 13 | 14 | import sys 15 | import re 16 | import codecs 17 | from collections import defaultdict, namedtuple 18 | 19 | ANY_SPACE = '' 20 | 21 | 22 | class FormatError(Exception): 23 | pass 24 | 25 | Metrics = namedtuple('Metrics', 'tp fp fn prec rec fscore') 26 | 27 | 28 | class EvalCounts(object): 29 | def __init__(self): 30 | self.correct_chunk = 0 # number of correctly identified chunks 31 | self.correct_tags = 0 # number of correct chunk tags 32 | self.found_correct = 0 # number of chunks in corpus 33 | self.found_guessed = 0 # number of identified chunks 34 | self.token_counter = 0 # token counter (ignores sentence breaks) 35 | 36 | # counts by type 37 | self.t_correct_chunk = defaultdict(int) 38 | self.t_found_correct = defaultdict(int) 39 | self.t_found_guessed = defaultdict(int) 40 | 41 | 42 | def parse_args(argv): 43 | import argparse 44 | parser = argparse.ArgumentParser( 45 | description='evaluate tagging results using CoNLL criteria', 46 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 47 | ) 48 | arg = parser.add_argument 49 | arg('-b', '--boundary', metavar='STR', default='-X-', 50 | help='sentence boundary') 51 | arg('-d', '--delimiter', metavar='CHAR', default=ANY_SPACE, 52 | help='character delimiting items in input') 53 | arg('-o', '--otag', metavar='CHAR', default='O', 54 | help='alternative outside tag') 55 | arg('file', nargs='?', default=None) 56 | return parser.parse_args(argv) 57 | 58 | 59 | def parse_tag(t): 60 | m = re.match(r'^([^-]*)-(.*)$', t) 61 | return m.groups() if m else (t, '') 62 | 63 | 64 | def evaluate(iterable, options=None): 65 | if options is None: 66 | options = parse_args([]) # use defaults 67 | 68 | counts = EvalCounts() 69 | num_features = None # number of features per line 70 | in_correct = False # currently processed chunks is correct until now 71 | last_correct = 'O' # previous chunk tag in corpus 72 | last_correct_type = '' # type of previously identified chunk tag 73 | last_guessed = 'O' # previously identified chunk tag 74 | last_guessed_type = '' # type of previous chunk tag in corpus 75 | 76 | for line in iterable: 77 | line = line.rstrip('\r\n') 78 | 79 | if options.delimiter == ANY_SPACE: 80 | features = line.split() 81 | else: 82 | features = line.split(options.delimiter) 83 | 84 | if num_features is None: 85 | num_features = len(features) 86 | elif num_features != len(features) and len(features) != 0: 87 | raise FormatError('unexpected number of features: %d (%d)' % 88 | (len(features), num_features)) 89 | 90 | if len(features) == 0 or features[0] == options.boundary: 91 | features = [options.boundary, 'O', 'O'] 92 | if len(features) < 3: 93 | raise FormatError('unexpected number of features in line %s' % line) 94 | 95 | guessed, guessed_type = parse_tag(features.pop()) 96 | correct, correct_type = parse_tag(features.pop()) 97 | first_item = features.pop(0) 98 | 99 | if first_item == options.boundary: 100 | guessed = 'O' 101 | 102 | end_correct = end_of_chunk(last_correct, correct, 103 | last_correct_type, correct_type) 104 | end_guessed = end_of_chunk(last_guessed, guessed, 105 | last_guessed_type, guessed_type) 106 | start_correct = start_of_chunk(last_correct, correct, 107 | last_correct_type, correct_type) 108 | start_guessed = start_of_chunk(last_guessed, guessed, 109 | last_guessed_type, guessed_type) 110 | 111 | if in_correct: 112 | if (end_correct and end_guessed and 113 | last_guessed_type == last_correct_type): 114 | in_correct = False 115 | counts.correct_chunk += 1 116 | counts.t_correct_chunk[last_correct_type] += 1 117 | elif (end_correct != end_guessed or guessed_type != correct_type): 118 | in_correct = False 119 | 120 | if start_correct and start_guessed and guessed_type == correct_type: 121 | in_correct = True 122 | 123 | if start_correct: 124 | counts.found_correct += 1 125 | counts.t_found_correct[correct_type] += 1 126 | if start_guessed: 127 | counts.found_guessed += 1 128 | counts.t_found_guessed[guessed_type] += 1 129 | if first_item != options.boundary: 130 | if correct == guessed and guessed_type == correct_type: 131 | counts.correct_tags += 1 132 | counts.token_counter += 1 133 | 134 | last_guessed = guessed 135 | last_correct = correct 136 | last_guessed_type = guessed_type 137 | last_correct_type = correct_type 138 | 139 | if in_correct: 140 | counts.correct_chunk += 1 141 | counts.t_correct_chunk[last_correct_type] += 1 142 | 143 | return counts 144 | 145 | 146 | 147 | def uniq(iterable): 148 | seen = set() 149 | return [i for i in iterable if not (i in seen or seen.add(i))] 150 | 151 | 152 | def calculate_metrics(correct, guessed, total): 153 | tp, fp, fn = correct, guessed-correct, total-correct 154 | p = 0 if tp + fp == 0 else 1.*tp / (tp + fp) 155 | r = 0 if tp + fn == 0 else 1.*tp / (tp + fn) 156 | f = 0 if p + r == 0 else 2 * p * r / (p + r) 157 | return Metrics(tp, fp, fn, p, r, f) 158 | 159 | 160 | def metrics(counts): 161 | c = counts 162 | overall = calculate_metrics( 163 | c.correct_chunk, c.found_guessed, c.found_correct 164 | ) 165 | by_type = {} 166 | for t in uniq(list(c.t_found_correct) + list(c.t_found_guessed)): 167 | by_type[t] = calculate_metrics( 168 | c.t_correct_chunk[t], c.t_found_guessed[t], c.t_found_correct[t] 169 | ) 170 | return overall, by_type 171 | 172 | 173 | def report(counts, out=None): 174 | if out is None: 175 | out = sys.stdout 176 | 177 | overall, by_type = metrics(counts) 178 | 179 | c = counts 180 | out.write('processed %d tokens with %d phrases; ' % 181 | (c.token_counter, c.found_correct)) 182 | out.write('found: %d phrases; correct: %d.\n' % 183 | (c.found_guessed, c.correct_chunk)) 184 | 185 | if c.token_counter > 0: 186 | out.write('accuracy: %6.2f%%; ' % 187 | (100.*c.correct_tags/c.token_counter)) 188 | out.write('precision: %6.2f%%; ' % (100.*overall.prec)) 189 | out.write('recall: %6.2f%%; ' % (100.*overall.rec)) 190 | out.write('FB1: %6.2f\n' % (100.*overall.fscore)) 191 | 192 | for i, m in sorted(by_type.items()): 193 | out.write('%17s: ' % i) 194 | out.write('precision: %6.2f%%; ' % (100.*m.prec)) 195 | out.write('recall: %6.2f%%; ' % (100.*m.rec)) 196 | out.write('FB1: %6.2f %d\n' % (100.*m.fscore, c.t_found_guessed[i])) 197 | 198 | 199 | def report_notprint(counts, out=None): 200 | if out is None: 201 | out = sys.stdout 202 | 203 | overall, by_type = metrics(counts) 204 | 205 | c = counts 206 | final_report = [] 207 | line = [] 208 | line.append('processed %d tokens with %d phrases; ' % 209 | (c.token_counter, c.found_correct)) 210 | line.append('found: %d phrases; correct: %d.\n' % 211 | (c.found_guessed, c.correct_chunk)) 212 | final_report.append("".join(line)) 213 | 214 | if c.token_counter > 0: 215 | line = [] 216 | line.append('accuracy: %6.2f%%; ' % 217 | (100.*c.correct_tags/c.token_counter)) 218 | line.append('precision: %6.2f%%; ' % (100.*overall.prec)) 219 | line.append('recall: %6.2f%%; ' % (100.*overall.rec)) 220 | line.append('FB1: %6.2f\n' % (100.*overall.fscore)) 221 | final_report.append("".join(line)) 222 | 223 | for i, m in sorted(by_type.items()): 224 | line = [] 225 | line.append('%17s: ' % i) 226 | line.append('precision: %6.2f%%; ' % (100.*m.prec)) 227 | line.append('recall: %6.2f%%; ' % (100.*m.rec)) 228 | line.append('FB1: %6.2f %d\n' % (100.*m.fscore, c.t_found_guessed[i])) 229 | final_report.append("".join(line)) 230 | return final_report 231 | 232 | 233 | def end_of_chunk(prev_tag, tag, prev_type, type_): 234 | # check if a chunk ended between the previous and current word 235 | # arguments: previous and current chunk tags, previous and current types 236 | chunk_end = False 237 | 238 | if prev_tag == 'E': chunk_end = True 239 | if prev_tag == 'S': chunk_end = True 240 | 241 | if prev_tag == 'B' and tag == 'B': chunk_end = True 242 | if prev_tag == 'B' and tag == 'S': chunk_end = True 243 | if prev_tag == 'B' and tag == 'O': chunk_end = True 244 | if prev_tag == 'I' and tag == 'B': chunk_end = True 245 | if prev_tag == 'I' and tag == 'S': chunk_end = True 246 | if prev_tag == 'I' and tag == 'O': chunk_end = True 247 | 248 | if prev_tag != 'O' and prev_tag != '.' and prev_type != type_: 249 | chunk_end = True 250 | 251 | # these chunks are assumed to have length 1 252 | if prev_tag == ']': chunk_end = True 253 | if prev_tag == '[': chunk_end = True 254 | 255 | return chunk_end 256 | 257 | 258 | def start_of_chunk(prev_tag, tag, prev_type, type_): 259 | # check if a chunk started between the previous and current word 260 | # arguments: previous and current chunk tags, previous and current types 261 | chunk_start = False 262 | 263 | if tag == 'B': chunk_start = True 264 | if tag == 'S': chunk_start = True 265 | 266 | if prev_tag == 'E' and tag == 'E': chunk_start = True 267 | if prev_tag == 'E' and tag == 'I': chunk_start = True 268 | if prev_tag == 'S' and tag == 'E': chunk_start = True 269 | if prev_tag == 'S' and tag == 'I': chunk_start = True 270 | if prev_tag == 'O' and tag == 'E': chunk_start = True 271 | if prev_tag == 'O' and tag == 'I': chunk_start = True 272 | 273 | if tag != 'O' and tag != '.' and prev_type != type_: 274 | chunk_start = True 275 | 276 | # these chunks are assumed to have length 1 277 | if tag == '[': chunk_start = True 278 | if tag == ']': chunk_start = True 279 | 280 | return chunk_start 281 | 282 | 283 | def return_report(input_file): 284 | with codecs.open(input_file, "r", "utf8") as f: 285 | counts = evaluate(f) 286 | return report_notprint(counts) 287 | 288 | 289 | def main(argv): 290 | args = parse_args(argv[1:]) 291 | 292 | if args.file is None: 293 | counts = evaluate(sys.stdin, args) 294 | else: 295 | with open(args.file) as f: 296 | counts = evaluate(f, args) 297 | report(counts) 298 | 299 | if __name__ == '__main__': 300 | sys.exit(main(sys.argv)) -------------------------------------------------------------------------------- /SO_label/data_reader_for_SO.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """ 3 | Data processing for SO-labeling. 4 | @author: yuhaitao 5 | """ 6 | import json 7 | import os 8 | import codecs 9 | import sys 10 | import re 11 | import random 12 | import pandas as pd 13 | import numpy as np 14 | from tqdm import tqdm 15 | 16 | 17 | class MyDataReader(object): 18 | """ 19 | class for my data reader 20 | """ 21 | 22 | def __init__(self, 23 | postag_dict_path, 24 | label_dict_path, 25 | train_data_list_path='', 26 | dev_data_list_path='', 27 | train_pc_file='', 28 | dev_pc_file=''): 29 | self._postag_dict_path = postag_dict_path 30 | self._label_dict_path = label_dict_path 31 | self.train_data_list_path = train_data_list_path 32 | self.dev_data_list_path = dev_data_list_path 33 | self.train_pc_file = train_pc_file 34 | self.dev_pc_file = dev_pc_file 35 | self._p_map_eng_dict = {} 36 | 37 | # 统计各种类别数据数量的词典 38 | self.label_num_dic = {} 39 | # load dictionary 40 | self._dict_path_dict = {'postag_dict': self._postag_dict_path, 41 | 'label_dict': self._label_dict_path} 42 | # check if the file exists 43 | for input_dict in [postag_dict_path, 44 | label_dict_path, train_data_list_path, dev_data_list_path]: 45 | if not os.path.exists(input_dict): 46 | raise ValueError("%s not found." % (input_dict)) 47 | return 48 | 49 | self._feature_dict = {} 50 | self._feature_dict['postag_dict'] = \ 51 | self._load_dict_from_file(self._dict_path_dict['postag_dict']) 52 | self._feature_dict['label_dict'], self.label_eng_dict = \ 53 | self._load_label_dict(self._dict_path_dict['label_dict']) 54 | print(self.label_eng_dict) 55 | # 将之前所有的字典反向 56 | self._reverse_dict = {name: self._get_reverse_dict(name) for name in 57 | self._dict_path_dict.keys()} 58 | self._reverse_dict['eng_map_p_dict'] = self._reverse_p_eng( 59 | self._p_map_eng_dict) 60 | self._UNK_IDX = 0 61 | 62 | # 统计在所有训练数据中主体和客体所覆盖的postag 63 | # 主体subject，客体object 64 | # self.subject_tags, self.object_tags = self.count_tags( 65 | # self.train_data_list_path, self._postag_dict_path) 66 | 67 | def _load_label_dict(self, dict_name): 68 | """这个函数重写了""" 69 | label_dict = {} 70 | label_to_eng = {} 71 | pattern = re.compile(r'\s+') 72 | with codecs.open(dict_name, 'r', 'utf-8') as fr: 73 | for idx, line in enumerate(fr): 74 | p, p_eng = re.split(pattern, line.strip()) 75 | label_to_eng[p] = p_eng 76 | label_dict[p_eng] = idx 77 | self._p_map_eng_dict[p] = p_eng 78 | self.label_num_dic[p] = 0 79 | # if p != '木有关系': 80 | # label_to_eng['反_' + p] = 'RE_' + p_eng 81 | # label_dict['RE_' + p_eng] = idx + 51 82 | # self._p_map_eng_dict['反_' + p] = 'RE_' + p_eng 83 | return label_dict, label_to_eng 84 | 85 | def _load_dict_from_file(self, dict_name, bias=0): 86 | """ 87 | Load vocabulary from file. 88 | """ 89 | dict_result = {} 90 | with codecs.open(dict_name, 'r', 'utf-8') as f_dict: 91 | for idx, line in enumerate(f_dict): 92 | line = line.strip() 93 | dict_result[line] = idx + bias 94 | return dict_result 95 | 96 | def _cal_mark_single_slot(self, spo_list, sentence): 97 | """ 98 | Calculate the value of the label 99 | """ 100 | mark_list = [0] * len(self._feature_dict['label_dict']) 101 | for spo in spo_list: 102 | predicate = spo['predicate'] 103 | p_idx = self._feature_dict['label_dict'][self._p_map_eng_dict[predicate]] 104 | mark_list[p_idx] = 1 105 | return mark_list 106 | 107 | def _is_valid_input_data(self, input_data): 108 | """is the input data valid""" 109 | try: 110 | dic = json.loads(input_data) 111 | except: 112 | return False 113 | if "text" not in dic or "postag" not in dic or \ 114 | type(dic["postag"]) is not list: 115 | return False 116 | for item in dic['postag']: 117 | if "word" not in item or "pos" not in item: 118 | return False 119 | return True 120 | 121 | def _get_feed_iterator(self, line, label_dict, eng_dict, pc_line=None, mode=None): 122 | """ 123 | 生成RC数据 124 | """ 125 | # verify that the input format of each line meets the format 126 | if not self._is_valid_input_data(line): 127 | print(line) 128 | print(sys.stderr, 'Format is error') 129 | raise ValueError 130 | dic = json.loads(line) 131 | # 注意sentence的长度被截断过 132 | sentence_ori = ''.join(s.strip() for s in dic['text'].split()) 133 | 134 | # 一个样本： text 主体 客体 类别 135 | sample_list = [] 136 | if mode == 'train': 137 | for spo in dic['spo_list']: 138 | sample = sentence_ori 139 | sample += ('\t' + label_dict[spo['predicate']]) 140 | sample += ('\t' + ''.join(s.strip() 141 | for s in spo['subject'].split())) 142 | sample += ('\t' + ''.join(s.strip() 143 | for s in spo['object'].split())) 144 | sample_list.append(sample) 145 | else: 146 | item = pc_line.strip().split('\t') 147 | sentence = item[0] 148 | if sentence != sentence_ori[0:len(sentence)]: 149 | print(sentence, sentence_ori) 150 | raise ValueError 151 | if len(item) == 1: 152 | return sample_list 153 | # 验证集 154 | if mode == 'dev': 155 | for label_eng in item[1:]: 156 | for spo in dic['spo_list']: 157 | if label_dict[spo['predicate']] == label_eng: 158 | sample = sentence_ori 159 | sample += ('\t' + label_eng) 160 | sample += ('\t' + ''.join(s.strip() 161 | for s in spo['subject'].split())) 162 | sample += ('\t' + ''.join(s.strip() 163 | for s in spo['object'].split())) 164 | sample_list.append(sample) 165 | break 166 | # 测试集 167 | elif mode == 'test': 168 | for label_eng in item[1:]: 169 | sample = sentence_ori 170 | sample += ('\t' + label_eng) 171 | sample += ('\t' + '主体') 172 | sample += ('\t' + '客体') 173 | sample_list.append(sample) 174 | 175 | return sample_list 176 | 177 | def path_reader(self, data_path, pc_path, mode): 178 | """Read data from data_path""" 179 | self._feature_dict['data_keylist'] = [] 180 | 181 | def reader(): 182 | """Generator""" 183 | if mode == "train": 184 | f = open(data_path.strip()) 185 | for line in f: 186 | sample_list = self._get_feed_iterator(line.strip( 187 | ), label_dict=self.label_eng_dict, eng_dict=self._reverse_dict['eng_map_p_dict'], pc_line=None, mode=mode) 188 | if sample_list is None: 189 | continue 190 | yield sample_list 191 | else: 192 | f_pc = open(pc_path, 'r') 193 | f = open(data_path.strip()) 194 | for line in f: 195 | pc_line = f_pc.readline() 196 | # 对文件每一行生成数据 197 | sample_list = self._get_feed_iterator(line.strip( 198 | ), label_dict=self.label_eng_dict, eng_dict=self._reverse_dict['eng_map_p_dict'], pc_line=pc_line.strip(), mode=mode) 199 | if sample_list is None: 200 | continue 201 | yield sample_list 202 | 203 | return reader 204 | 205 | def get_train_reader(self, mode='train'): 206 | """Data reader during training""" 207 | return self.path_reader(self.train_data_list_path, self.train_pc_file, mode) 208 | 209 | def get_dev_reader(self, mode='dev'): 210 | """Data reader during dev""" 211 | return self.path_reader(self.dev_data_list_path, self.dev_pc_file, mode) 212 | 213 | def get_test_reader(self, test_file_path='', test_pc_file='', mode='test'): 214 | """Data reader during predict""" 215 | return self.path_reader(test_file_path, test_pc_file, mode) 216 | 217 | def get_dict(self, dict_name): 218 | """Return dict""" 219 | if dict_name not in self._feature_dict: 220 | raise ValueError("dict name %s not found." % (dict_name)) 221 | return self._feature_dict[dict_name] 222 | 223 | def get_all_dict_name(self): 224 | """Get name of all dict""" 225 | return self._feature_dict.keys() 226 | 227 | def get_dict_size(self, dict_name): 228 | """Return dict length""" 229 | if dict_name not in self._feature_dict: 230 | raise ValueError("dict name %s not found." % (dict_name)) 231 | return len(self._feature_dict[dict_name]) 232 | 233 | def _get_reverse_dict(self, dict_name): 234 | dict_reverse = {} 235 | for key, value in self._feature_dict[dict_name].items(): 236 | dict_reverse[value] = key 237 | return dict_reverse 238 | 239 | def _reverse_p_eng(self, dic): 240 | dict_reverse = {} 241 | for key, value in dic.items(): 242 | dict_reverse[value] = key 243 | return dict_reverse 244 | 245 | def get_label_output(self, label_idx): 246 | """Output final label, used during predict and test""" 247 | dict_name = 'label_dict' 248 | if len(self._reverse_dict[dict_name]) == 0: 249 | self._get_reverse_dict(dict_name) 250 | p_eng = self._reverse_dict[dict_name][label_idx] 251 | return self._reverse_dict['eng_map_p_dict'][p_eng] 252 | 253 | 254 | if __name__ == '__main__': 255 | # initialize data generator 256 | data_generator = MyDataReader( 257 | postag_dict_path='../dict/postag_dict', 258 | label_dict_path='../dict/p_eng', 259 | train_data_list_path='../data/ori_data/train_data.json', 260 | dev_data_list_path='../data/ori_data/dev_data.json', 261 | train_pc_file='', 262 | dev_pc_file='../data/ori_data/PC_dev.txt') 263 | 264 | # prepare data reader 265 | train = data_generator.get_train_reader() 266 | with open("../data/SO_data/train.txt", 'w') as f: 267 | for sample_list in tqdm(train()): 268 | for sample in sample_list: 269 | f.write(sample + '\n') 270 | 271 | dev = data_generator.get_dev_reader() 272 | with open("../data/SO_data/dev.txt", 'w') as f: 273 | for sample_list in tqdm(dev()): 274 | for sample in sample_list: 275 | f.write(sample + '\n') 276 | 277 | test = data_generator.get_test_reader( 278 | test_file_path='../data/ori_data/test1_data_postag.json', test_pc_file='../data/ori_data/PC_test.txt') 279 | with open("../data/SO_data/test.txt", 'w') as f: 280 | for sample_list in tqdm(test()): 281 | for sample in sample_list: 282 | f.write(sample + '\n') 283 | -------------------------------------------------------------------------------- /NER/conlleval.py: -------------------------------------------------------------------------------- 1 | # Python version of the evaluation script from CoNLL'00- 2 | # Originates from: https://github.com/spyysalo/conlleval.py 3 | 4 | 5 | # Intentional differences: 6 | # - accept any space as delimiter by default 7 | # - optional file argument (default STDIN) 8 | # - option to set boundary (-b argument) 9 | # - LaTeX output (-l argument) not supported 10 | # - raw tags (-r argument) not supported 11 | 12 | # add function :evaluate(predicted_label, ori_label): which will not read from file 13 | 14 | import sys 15 | import re 16 | import codecs 17 | from collections import defaultdict, namedtuple 18 | 19 | ANY_SPACE = '' 20 | 21 | 22 | class FormatError(Exception): 23 | pass 24 | 25 | 26 | Metrics = namedtuple('Metrics', 'tp fp fn prec rec fscore') 27 | 28 | 29 | class EvalCounts(object): 30 | def __init__(self): 31 | self.correct_chunk = 0 # number of correctly identified chunks 32 | self.correct_tags = 0 # number of correct chunk tags 33 | self.found_correct = 0 # number of chunks in corpus 34 | self.found_guessed = 0 # number of identified chunks 35 | self.token_counter = 0 # token counter (ignores sentence breaks) 36 | 37 | # counts by type 38 | self.t_correct_chunk = defaultdict(int) 39 | self.t_found_correct = defaultdict(int) 40 | self.t_found_guessed = defaultdict(int) 41 | 42 | 43 | def parse_args(argv): 44 | import argparse 45 | parser = argparse.ArgumentParser( 46 | description='evaluate tagging results using CoNLL criteria', 47 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 48 | ) 49 | arg = parser.add_argument 50 | arg('-b', '--boundary', metavar='STR', default='-X-', 51 | help='sentence boundary') 52 | arg('-d', '--delimiter', metavar='CHAR', default=ANY_SPACE, 53 | help='character delimiting items in input') 54 | arg('-o', '--otag', metavar='CHAR', default='O', 55 | help='alternative outside tag') 56 | arg('file', nargs='?', default=None) 57 | return parser.parse_args(argv) 58 | 59 | 60 | def parse_tag(t): 61 | m = re.match(r'^([^-]*)-(.*)$', t) 62 | return m.groups() if m else (t, '') 63 | 64 | 65 | def evaluate(iterable, options=None): 66 | if options is None: 67 | options = parse_args([]) # use defaults 68 | 69 | counts = EvalCounts() 70 | num_features = None # number of features per line 71 | in_correct = False # currently processed chunks is correct until now 72 | last_correct = 'O' # previous chunk tag in corpus 73 | last_correct_type = '' # type of previously identified chunk tag 74 | last_guessed = 'O' # previously identified chunk tag 75 | last_guessed_type = '' # type of previous chunk tag in corpus 76 | 77 | for line in iterable: 78 | line = line.rstrip('\r\n') 79 | 80 | if options.delimiter == ANY_SPACE: 81 | features = line.split() 82 | else: 83 | features = line.split(options.delimiter) 84 | 85 | if num_features is None: 86 | num_features = len(features) 87 | elif num_features != len(features) and len(features) != 0: 88 | raise FormatError('unexpected number of features: %d (%d)' % 89 | (len(features), num_features)) 90 | 91 | if len(features) == 0 or features[0] == options.boundary: 92 | features = [options.boundary, 'O', 'O'] 93 | if len(features) < 3: 94 | raise FormatError( 95 | 'unexpected number of features in line %s' % line) 96 | 97 | guessed, guessed_type = parse_tag(features.pop()) 98 | correct, correct_type = parse_tag(features.pop()) 99 | first_item = features.pop(0) 100 | 101 | if first_item == options.boundary: 102 | guessed = 'O' 103 | 104 | end_correct = end_of_chunk(last_correct, correct, 105 | last_correct_type, correct_type) 106 | end_guessed = end_of_chunk(last_guessed, guessed, 107 | last_guessed_type, guessed_type) 108 | start_correct = start_of_chunk(last_correct, correct, 109 | last_correct_type, correct_type) 110 | start_guessed = start_of_chunk(last_guessed, guessed, 111 | last_guessed_type, guessed_type) 112 | 113 | if in_correct: 114 | if (end_correct and end_guessed and 115 | last_guessed_type == last_correct_type): 116 | in_correct = False 117 | counts.correct_chunk += 1 118 | counts.t_correct_chunk[last_correct_type] += 1 119 | elif (end_correct != end_guessed or guessed_type != correct_type): 120 | in_correct = False 121 | 122 | if start_correct and start_guessed and guessed_type == correct_type: 123 | in_correct = True 124 | 125 | if start_correct: 126 | counts.found_correct += 1 127 | counts.t_found_correct[correct_type] += 1 128 | if start_guessed: 129 | counts.found_guessed += 1 130 | counts.t_found_guessed[guessed_type] += 1 131 | if first_item != options.boundary: 132 | if correct == guessed and guessed_type == correct_type: 133 | counts.correct_tags += 1 134 | counts.token_counter += 1 135 | 136 | last_guessed = guessed 137 | last_correct = correct 138 | last_guessed_type = guessed_type 139 | last_correct_type = correct_type 140 | 141 | if in_correct: 142 | counts.correct_chunk += 1 143 | counts.t_correct_chunk[last_correct_type] += 1 144 | 145 | return counts 146 | 147 | 148 | def uniq(iterable): 149 | seen = set() 150 | return [i for i in iterable if not (i in seen or seen.add(i))] 151 | 152 | 153 | def calculate_metrics(correct, guessed, total): 154 | tp, fp, fn = correct, guessed - correct, total - correct 155 | p = 0 if tp + fp == 0 else 1. * tp / (tp + fp) 156 | r = 0 if tp + fn == 0 else 1. * tp / (tp + fn) 157 | f = 0 if p + r == 0 else 2 * p * r / (p + r) 158 | return Metrics(tp, fp, fn, p, r, f) 159 | 160 | 161 | def metrics(counts): 162 | c = counts 163 | overall = calculate_metrics( 164 | c.correct_chunk, c.found_guessed, c.found_correct 165 | ) 166 | by_type = {} 167 | for t in uniq(list(c.t_found_correct) + list(c.t_found_guessed)): 168 | by_type[t] = calculate_metrics( 169 | c.t_correct_chunk[t], c.t_found_guessed[t], c.t_found_correct[t] 170 | ) 171 | return overall, by_type 172 | 173 | 174 | def report(counts, out=None): 175 | if out is None: 176 | out = sys.stdout 177 | 178 | overall, by_type = metrics(counts) 179 | 180 | c = counts 181 | out.write('processed %d tokens with %d phrases; ' % 182 | (c.token_counter, c.found_correct)) 183 | out.write('found: %d phrases; correct: %d.\n' % 184 | (c.found_guessed, c.correct_chunk)) 185 | 186 | if c.token_counter > 0: 187 | out.write('accuracy: %6.2f%%; ' % 188 | (100. * c.correct_tags / c.token_counter)) 189 | out.write('precision: %6.2f%%; ' % (100. * overall.prec)) 190 | out.write('recall: %6.2f%%; ' % (100. * overall.rec)) 191 | out.write('FB1: %6.2f\n' % (100. * overall.fscore)) 192 | 193 | for i, m in sorted(by_type.items()): 194 | out.write('%17s: ' % i) 195 | out.write('precision: %6.2f%%; ' % (100. * m.prec)) 196 | out.write('recall: %6.2f%%; ' % (100. * m.rec)) 197 | out.write('FB1: %6.2f %d\n' % (100. * m.fscore, c.t_found_guessed[i])) 198 | 199 | 200 | def report_notprint(counts, out=None): 201 | if out is None: 202 | out = sys.stdout 203 | 204 | overall, by_type = metrics(counts) 205 | 206 | c = counts 207 | final_report = [] 208 | line = [] 209 | line.append('processed %d tokens with %d phrases; ' % 210 | (c.token_counter, c.found_correct)) 211 | line.append('found: %d phrases; correct: %d.\n' % 212 | (c.found_guessed, c.correct_chunk)) 213 | final_report.append("".join(line)) 214 | 215 | if c.token_counter > 0: 216 | line = [] 217 | line.append('accuracy: %6.2f%%; ' % 218 | (100. * c.correct_tags / c.token_counter)) 219 | line.append('precision: %6.2f%%; ' % (100. * overall.prec)) 220 | line.append('recall: %6.2f%%; ' % (100. * overall.rec)) 221 | line.append('FB1: %6.2f\n' % (100. * overall.fscore)) 222 | final_report.append("".join(line)) 223 | 224 | for i, m in sorted(by_type.items()): 225 | line = [] 226 | line.append('%17s: ' % i) 227 | line.append('precision: %6.2f%%; ' % (100. * m.prec)) 228 | line.append('recall: %6.2f%%; ' % (100. * m.rec)) 229 | line.append('FB1: %6.2f %d\n' % 230 | (100. * m.fscore, c.t_found_guessed[i])) 231 | final_report.append("".join(line)) 232 | return final_report, overall 233 | 234 | 235 | def end_of_chunk(prev_tag, tag, prev_type, type_): 236 | # check if a chunk ended between the previous and current word 237 | # arguments: previous and current chunk tags, previous and current types 238 | chunk_end = False 239 | 240 | if prev_tag == 'E': 241 | chunk_end = True 242 | if prev_tag == 'S': 243 | chunk_end = True 244 | 245 | if prev_tag == 'B' and tag == 'B': 246 | chunk_end = True 247 | if prev_tag == 'B' and tag == 'S': 248 | chunk_end = True 249 | if prev_tag == 'B' and tag == 'O': 250 | chunk_end = True 251 | if prev_tag == 'I' and tag == 'B': 252 | chunk_end = True 253 | if prev_tag == 'I' and tag == 'S': 254 | chunk_end = True 255 | if prev_tag == 'I' and tag == 'O': 256 | chunk_end = True 257 | 258 | if prev_tag != 'O' and prev_tag != '.' and prev_type != type_: 259 | chunk_end = True 260 | 261 | # these chunks are assumed to have length 1 262 | if prev_tag == ']': 263 | chunk_end = True 264 | if prev_tag == '[': 265 | chunk_end = True 266 | 267 | return chunk_end 268 | 269 | 270 | def start_of_chunk(prev_tag, tag, prev_type, type_): 271 | # check if a chunk started between the previous and current word 272 | # arguments: previous and current chunk tags, previous and current types 273 | chunk_start = False 274 | 275 | if tag == 'B': 276 | chunk_start = True 277 | if tag == 'S': 278 | chunk_start = True 279 | 280 | if prev_tag == 'E' and tag == 'E': 281 | chunk_start = True 282 | if prev_tag == 'E' and tag == 'I': 283 | chunk_start = True 284 | if prev_tag == 'S' and tag == 'E': 285 | chunk_start = True 286 | if prev_tag == 'S' and tag == 'I': 287 | chunk_start = True 288 | if prev_tag == 'O' and tag == 'E': 289 | chunk_start = True 290 | if prev_tag == 'O' and tag == 'I': 291 | chunk_start = True 292 | 293 | if tag != 'O' and tag != '.' and prev_type != type_: 294 | chunk_start = True 295 | 296 | # these chunks are assumed to have length 1 297 | if tag == '[': 298 | chunk_start = True 299 | if tag == ']': 300 | chunk_start = True 301 | 302 | return chunk_start 303 | 304 | 305 | def return_report(input_file): 306 | with codecs.open(input_file, "r", "utf8") as f: 307 | counts = evaluate(f) 308 | return report_notprint(counts) 309 | 310 | 311 | def main(argv): 312 | args = parse_args(argv[1:]) 313 | 314 | if args.file is None: 315 | counts = evaluate(sys.stdin, args) 316 | else: 317 | with open(args.file) as f: 318 | counts = evaluate(f, args) 319 | report(counts) 320 | 321 | 322 | if __name__ == '__main__': 323 | sys.exit(main(sys.argv)) 324 | -------------------------------------------------------------------------------- /bert/bert_code/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /bert/bert_code/multilingual.md: -------------------------------------------------------------------------------- 1 | ## Models 2 | 3 | There are two multilingual models currently available. We do not plan to release 4 | more single-language models, but we may release `BERT-Large` versions of these 5 | two in the future: 6 | 7 | * **[`BERT-Base, Multilingual Cased (New, recommended)`](https://storage.googleapis.com/bert_models/2018_11_23/multi_cased_L-12_H-768_A-12.zip)**: 8 | 104 languages, 12-layer, 768-hidden, 12-heads, 110M parameters 9 | * **[`BERT-Base, Multilingual Uncased (Orig, not recommended)`](https://storage.googleapis.com/bert_models/2018_11_03/multilingual_L-12_H-768_A-12.zip)**: 10 | 102 languages, 12-layer, 768-hidden, 12-heads, 110M parameters 11 | * **[`BERT-Base, Chinese`](https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip)**: 12 | Chinese Simplified and Traditional, 12-layer, 768-hidden, 12-heads, 110M 13 | parameters 14 | 15 | **The `Multilingual Cased (New)` model also fixes normalization issues in many 16 | languages, so it is recommended in languages with non-Latin alphabets (and is 17 | often better for most languages with Latin alphabets). When using this model, 18 | make sure to pass `--do_lower_case=false` to `run_pretraining.py` and other 19 | scripts.** 20 | 21 | See the [list of languages](#list-of-languages) that the Multilingual model 22 | supports. The Multilingual model does include Chinese (and English), but if your 23 | fine-tuning data is Chinese-only, then the Chinese model will likely produce 24 | better results. 25 | 26 | ## Results 27 | 28 | To evaluate these systems, we use the 29 | [XNLI dataset](https://github.com/facebookresearch/XNLI) dataset, which is a 30 | version of [MultiNLI](https://www.nyu.edu/projects/bowman/multinli/) where the 31 | dev and test sets have been translated (by humans) into 15 languages. Note that 32 | the training set was *machine* translated (we used the translations provided by 33 | XNLI, not Google NMT). For clarity, we only report on 6 languages below: 34 | 35 | 36 | 37 | | System | English | Chinese | Spanish | German | Arabic | Urdu | 38 | | --------------------------------- | -------- | -------- | -------- | -------- | -------- | -------- | 39 | | XNLI Baseline - Translate Train | 73.7 | 67.0 | 68.8 | 66.5 | 65.8 | 56.6 | 40 | | XNLI Baseline - Translate Test | 73.7 | 68.3 | 70.7 | 68.7 | 66.8 | 59.3 | 41 | | BERT - Translate Train Cased | **81.9** | **76.6** | **77.8** | **75.9** | **70.7** | 61.6 | 42 | | BERT - Translate Train Uncased | 81.4 | 74.2 | 77.3 | 75.2 | 70.5 | 61.7 | 43 | | BERT - Translate Test Uncased | 81.4 | 70.1 | 74.9 | 74.4 | 70.4 | **62.1** | 44 | | BERT - Zero Shot Uncased | 81.4 | 63.8 | 74.3 | 70.5 | 62.1 | 58.3 | 45 | 46 | 47 | 48 | The first two rows are baselines from the XNLI paper and the last three rows are 49 | our results with BERT. 50 | 51 | **Translate Train** means that the MultiNLI training set was machine translated 52 | from English into the foreign language. So training and evaluation were both 53 | done in the foreign language. Unfortunately, training was done on 54 | machine-translated data, so it is impossible to quantify how much of the lower 55 | accuracy (compared to English) is due to the quality of the machine translation 56 | vs. the quality of the pre-trained model. 57 | 58 | **Translate Test** means that the XNLI test set was machine translated from the 59 | foreign language into English. So training and evaluation were both done on 60 | English. However, test evaluation was done on machine-translated English, so the 61 | accuracy depends on the quality of the machine translation system. 62 | 63 | **Zero Shot** means that the Multilingual BERT system was fine-tuned on English 64 | MultiNLI, and then evaluated on the foreign language XNLI test. In this case, 65 | machine translation was not involved at all in either the pre-training or 66 | fine-tuning. 67 | 68 | Note that the English result is worse than the 84.2 MultiNLI baseline because 69 | this training used Multilingual BERT rather than English-only BERT. This implies 70 | that for high-resource languages, the Multilingual model is somewhat worse than 71 | a single-language model. However, it is not feasible for us to train and 72 | maintain dozens of single-language model. Therefore, if your goal is to maximize 73 | performance with a language other than English or Chinese, you might find it 74 | beneficial to run pre-training for additional steps starting from our 75 | Multilingual model on data from your language of interest. 76 | 77 | Here is a comparison of training Chinese models with the Multilingual 78 | `BERT-Base` and Chinese-only `BERT-Base`: 79 | 80 | System | Chinese 81 | ----------------------- | ------- 82 | XNLI Baseline | 67.0 83 | BERT Multilingual Model | 74.2 84 | BERT Chinese-only Model | 77.2 85 | 86 | Similar to English, the single-language model does 3% better than the 87 | Multilingual model. 88 | 89 | ## Fine-tuning Example 90 | 91 | The multilingual model does **not** require any special consideration or API 92 | changes. We did update the implementation of `BasicTokenizer` in 93 | `tokenization.py` to support Chinese character tokenization, so please update if 94 | you forked it. However, we did not change the tokenization API. 95 | 96 | To test the new models, we did modify `run_classifier.py` to add support for the 97 | [XNLI dataset](https://github.com/facebookresearch/XNLI). This is a 15-language 98 | version of MultiNLI where the dev/test sets have been human-translated, and the 99 | training set has been machine-translated. 100 | 101 | To run the fine-tuning code, please download the 102 | [XNLI dev/test set](https://s3.amazonaws.com/xnli/XNLI-1.0.zip) and the 103 | [XNLI machine-translated training set](https://s3.amazonaws.com/xnli/XNLI-MT-1.0.zip) 104 | and then unpack both .zip files into some directory `$XNLI_DIR`. 105 | 106 | To run fine-tuning on XNLI. The language is hard-coded into `run_classifier.py` 107 | (Chinese by default), so please modify `XnliProcessor` if you want to run on 108 | another language. 109 | 110 | This is a large dataset, so this will training will take a few hours on a GPU 111 | (or about 30 minutes on a Cloud TPU). To run an experiment quickly for 112 | debugging, just set `num_train_epochs` to a small value like `0.1`. 113 | 114 | ```shell 115 | export BERT_BASE_DIR=/path/to/bert/chinese_L-12_H-768_A-12 # or multilingual_L-12_H-768_A-12 116 | export XNLI_DIR=/path/to/xnli 117 | 118 | python run_classifier.py \ 119 | --task_name=XNLI \ 120 | --do_train=true \ 121 | --do_eval=true \ 122 | --data_dir=$XNLI_DIR \ 123 | --vocab_file=$BERT_BASE_DIR/vocab.txt \ 124 | --bert_config_file=$BERT_BASE_DIR/bert_config.json \ 125 | --init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt \ 126 | --max_seq_length=128 \ 127 | --train_batch_size=32 \ 128 | --learning_rate=5e-5 \ 129 | --num_train_epochs=2.0 \ 130 | --output_dir=/tmp/xnli_output/ 131 | ``` 132 | 133 | With the Chinese-only model, the results should look something like this: 134 | 135 | ``` 136 | ***** Eval results ***** 137 | eval_accuracy = 0.774116 138 | eval_loss = 0.83554 139 | global_step = 24543 140 | loss = 0.74603 141 | ``` 142 | 143 | ## Details 144 | 145 | ### Data Source and Sampling 146 | 147 | The languages chosen were the 148 | [top 100 languages with the largest Wikipedias](https://meta.wikimedia.org/wiki/List_of_Wikipedias). 149 | The entire Wikipedia dump for each language (excluding user and talk pages) was 150 | taken as the training data for each language 151 | 152 | However, the size of the Wikipedia for a given language varies greatly, and 153 | therefore low-resource languages may be "under-represented" in terms of the 154 | neural network model (under the assumption that languages are "competing" for 155 | limited model capacity to some extent). 156 | 157 | However, the size of a Wikipedia also correlates with the number of speakers of 158 | a language, and we also don't want to overfit the model by performing thousands 159 | of epochs over a tiny Wikipedia for a particular language. 160 | 161 | To balance these two factors, we performed exponentially smoothed weighting of 162 | the data during pre-training data creation (and WordPiece vocab creation). In 163 | other words, let's say that the probability of a language is *P(L)*, e.g., 164 | *P(English) = 0.21* means that after concatenating all of the Wikipedias 165 | together, 21% of our data is English. We exponentiate each probability by some 166 | factor *S* and then re-normalize, and sample from that distribution. In our case 167 | we use *S=0.7*. So, high-resource languages like English will be under-sampled, 168 | and low-resource languages like Icelandic will be over-sampled. E.g., in the 169 | original distribution English would be sampled 1000x more than Icelandic, but 170 | after smoothing it's only sampled 100x more. 171 | 172 | ### Tokenization 173 | 174 | For tokenization, we use a 110k shared WordPiece vocabulary. The word counts are 175 | weighted the same way as the data, so low-resource languages are upweighted by 176 | some factor. We intentionally do *not* use any marker to denote the input 177 | language (so that zero-shot training can work). 178 | 179 | Because Chinese (and Japanese Kanji and Korean Hanja) does not have whitespace 180 | characters, we add spaces around every character in the 181 | [CJK Unicode range](https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_\(Unicode_block\)) 182 | before applying WordPiece. This means that Chinese is effectively 183 | character-tokenized. Note that the CJK Unicode block only includes 184 | Chinese-origin characters and does *not* include Hangul Korean or 185 | Katakana/Hiragana Japanese, which are tokenized with whitespace+WordPiece like 186 | all other languages. 187 | 188 | For all other languages, we apply the 189 | [same recipe as English](https://github.com/google-research/bert#tokenization): 190 | (a) lower casing+accent removal, (b) punctuation splitting, (c) whitespace 191 | tokenization. We understand that accent markers have substantial meaning in some 192 | languages, but felt that the benefits of reducing the effective vocabulary make 193 | up for this. Generally the strong contextual models of BERT should make up for 194 | any ambiguity introduced by stripping accent markers. 195 | 196 | ### List of Languages 197 | 198 | The multilingual model supports the following languages. These languages were 199 | chosen because they are the top 100 languages with the largest Wikipedias: 200 | 201 | * Afrikaans 202 | * Albanian 203 | * Arabic 204 | * Aragonese 205 | * Armenian 206 | * Asturian 207 | * Azerbaijani 208 | * Bashkir 209 | * Basque 210 | * Bavarian 211 | * Belarusian 212 | * Bengali 213 | * Bishnupriya Manipuri 214 | * Bosnian 215 | * Breton 216 | * Bulgarian 217 | * Burmese 218 | * Catalan 219 | * Cebuano 220 | * Chechen 221 | * Chinese (Simplified) 222 | * Chinese (Traditional) 223 | * Chuvash 224 | * Croatian 225 | * Czech 226 | * Danish 227 | * Dutch 228 | * English 229 | * Estonian 230 | * Finnish 231 | * French 232 | * Galician 233 | * Georgian 234 | * German 235 | * Greek 236 | * Gujarati 237 | * Haitian 238 | * Hebrew 239 | * Hindi 240 | * Hungarian 241 | * Icelandic 242 | * Ido 243 | * Indonesian 244 | * Irish 245 | * Italian 246 | * Japanese 247 | * Javanese 248 | * Kannada 249 | * Kazakh 250 | * Kirghiz 251 | * Korean 252 | * Latin 253 | * Latvian 254 | * Lithuanian 255 | * Lombard 256 | * Low Saxon 257 | * Luxembourgish 258 | * Macedonian 259 | * Malagasy 260 | * Malay 261 | * Malayalam 262 | * Marathi 263 | * Minangkabau 264 | * Nepali 265 | * Newar 266 | * Norwegian (Bokmal) 267 | * Norwegian (Nynorsk) 268 | * Occitan 269 | * Persian (Farsi) 270 | * Piedmontese 271 | * Polish 272 | * Portuguese 273 | * Punjabi 274 | * Romanian 275 | * Russian 276 | * Scots 277 | * Serbian 278 | * Serbo-Croatian 279 | * Sicilian 280 | * Slovak 281 | * Slovenian 282 | * South Azerbaijani 283 | * Spanish 284 | * Sundanese 285 | * Swahili 286 | * Swedish 287 | * Tagalog 288 | * Tajik 289 | * Tamil 290 | * Tatar 291 | * Telugu 292 | * Turkish 293 | * Ukrainian 294 | * Urdu 295 | * Uzbek 296 | * Vietnamese 297 | * Volapük 298 | * Waray-Waray 299 | * Welsh 300 | * West Frisian 301 | * Western Punjabi 302 | * Yoruba 303 | 304 | The **Multilingual Cased (New)** release contains additionally **Thai** and 305 | **Mongolian**, which were not included in the original release. 306 | -------------------------------------------------------------------------------- /baseline_with_tf.estimator/NER/data_reader_for_NER.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """ 3 | 数据读取预处理，定义一个data reader类，生成CSV文件，供bert的data processor读取 4 | """ 5 | import json 6 | import os 7 | import codecs 8 | import sys 9 | import pandas as pd 10 | import numpy as np 11 | from tqdm import tqdm 12 | 13 | 14 | class MyDataReader(object): 15 | """ 16 | class for my data reader 17 | """ 18 | 19 | def __init__(self, 20 | postag_dict_path, 21 | label_dict_path, 22 | train_data_list_path='', 23 | dev_data_list_path=''): 24 | self._postag_dict_path = postag_dict_path 25 | self._label_dict_path = label_dict_path 26 | self.train_data_list_path = train_data_list_path 27 | self.dev_data_list_path = dev_data_list_path 28 | self._p_map_eng_dict = {} 29 | # load dictionary 30 | self._dict_path_dict = {'postag_dict': self._postag_dict_path, 31 | 'label_dict': self._label_dict_path} 32 | # check if the file exists 33 | for input_dict in [postag_dict_path, 34 | label_dict_path, train_data_list_path, dev_data_list_path]: 35 | if not os.path.exists(input_dict): 36 | raise ValueError("%s not found." % (input_dict)) 37 | return 38 | 39 | # self._feature_dict = {} 40 | # self._feature_dict['postag_dict'] = \ 41 | # self._load_dict_from_file(self._dict_path_dict['postag_dict']) 42 | # self._feature_dict['label_dict'] = \ 43 | # self._load_label_dict(self._dict_path_dict['label_dict']) 44 | # # 将之前所有的字典反向 45 | # self._reverse_dict = {name: self._get_reverse_dict(name) for name in 46 | # self._dict_path_dict.keys()} 47 | # self._reverse_dict['eng_map_p_dict'] = self._reverse_p_eng( 48 | # self._p_map_eng_dict) 49 | # self._UNK_IDX = 0 50 | 51 | # 统计在所有训练数据中主体和客体所覆盖的postag 52 | # 主体subject，客体object 53 | # self.subject_tags, self.object_tags = self.count_tags( 54 | # self.train_data_list_path, self._postag_dict_path) 55 | 56 | def _load_label_dict(self, dict_name): 57 | """load label dict from file""" 58 | label_dict = {} 59 | with codecs.open(dict_name, 'r', 'utf-8') as fr: 60 | for idx, line in enumerate(fr): 61 | p, p_eng = line.strip().split('\t') 62 | label_dict[p_eng] = idx 63 | self._p_map_eng_dict[p] = p_eng 64 | return label_dict 65 | 66 | def _load_dict_from_file(self, dict_name, bias=0): 67 | """ 68 | Load vocabulary from file. 69 | """ 70 | dict_result = {} 71 | with codecs.open(dict_name, 'r', 'utf-8') as f_dict: 72 | for idx, line in enumerate(f_dict): 73 | line = line.strip() 74 | dict_result[line] = idx + bias 75 | return dict_result 76 | 77 | def _cal_mark_single_slot(self, spo_list, sentence): 78 | """ 79 | Calculate the value of the label 80 | """ 81 | mark_list = [0] * len(self._feature_dict['label_dict']) 82 | for spo in spo_list: 83 | predicate = spo['predicate'] 84 | p_idx = self._feature_dict['label_dict'][self._p_map_eng_dict[predicate]] 85 | mark_list[p_idx] = 1 86 | return mark_list 87 | 88 | def _is_valid_input_data(self, input_data): 89 | """is the input data valid""" 90 | try: 91 | dic = json.loads(input_data) 92 | except: 93 | return False 94 | if "text" not in dic or "postag" not in dic or \ 95 | type(dic["postag"]) is not list: 96 | return False 97 | for item in dic['postag']: 98 | if "word" not in item or "pos" not in item: 99 | return False 100 | return True 101 | 102 | def _get_feed_iterator(self, line, need_input=False, need_label=True): 103 | """ 104 | 生成一条数据，应修改为对每一个line生成一个样本列表，其中除了正样本，每个line还生成一个负样本 105 | """ 106 | # verify that the input format of each line meets the format 107 | if not self._is_valid_input_data(line): 108 | print >> sys.stderr, 'Format is error' 109 | return None 110 | dic = json.loads(line) 111 | sentence = dic['text'] 112 | 113 | token_list = [] 114 | label_list = [] 115 | sentence = ''.join(s.strip() for s in sentence.split()) 116 | for s in sentence: 117 | token_list.append(s) 118 | if not need_label: 119 | label_list = ['X'] * len(token_list) 120 | return token_list, label_list 121 | else: 122 | label_list = ['O'] * len(token_list) 123 | entity_set = set() 124 | for spo in dic['spo_list']: 125 | entity_set.add(spo['subject']) 126 | entity_set.add(spo['object']) 127 | for entity in entity_set: 128 | try: 129 | index = sentence.index(entity) 130 | for i in range(len(entity)): 131 | if i == 0: 132 | label_list[index + i] = 'B' 133 | elif i == len(entity) - 1: 134 | label_list[index + i] = 'E' 135 | else: 136 | label_list[index + i] = 'I' 137 | except: 138 | continue 139 | return token_list, label_list 140 | 141 | def path_reader(self, data_path, need_input=False, need_label=True): 142 | """Read data from data_path""" 143 | def reader(): 144 | """Generator""" 145 | if os.path.isdir(data_path): 146 | input_files = os.listdir(data_path) 147 | for data_file in input_files: 148 | data_file_path = os.path.join(data_path, data_file) 149 | for line in open(data_file_path.strip()): 150 | token_list, label_list = self._get_feed_iterator( 151 | line.strip(), need_input, need_label) 152 | if token_list is None: 153 | continue 154 | yield token_list, label_list 155 | elif os.path.isfile(data_path): 156 | for line in open(data_path.strip()): 157 | # 对文件每一行生成数据 158 | token_list, label_list = self._get_feed_iterator( 159 | line.strip(), need_input, need_label) 160 | if token_list is None: 161 | continue 162 | yield token_list, label_list 163 | 164 | return reader 165 | 166 | def count_tags(self, train_file, postag_file): 167 | """ 168 | 统计所有主客体覆盖到的postag类别 169 | """ 170 | subject_tags, object_tags = {}, {} 171 | 172 | # 如果文件存在 173 | if os.path.isfile('../dict/sub_tag') and os.path.isfile('../dict/obj_tag'): 174 | with open('../dict/sub_tag', 'r') as fs: 175 | for line in fs: 176 | key, value = line.strip().split('\t') 177 | subject_tags[key] = int(value) 178 | with open('../dict/obj_tag', 'r') as fo: 179 | for line in fo: 180 | key, value = line.strip().split('\t') 181 | object_tags[key] = int(value) 182 | return subject_tags, object_tags 183 | 184 | # 如果文件不存在 185 | with open(postag_file, 'r') as f: 186 | for line in f: 187 | tag = line.strip() 188 | subject_tags[tag], object_tags[tag] = 0, 0 189 | print("开始统计主客体的postag类别...") 190 | with open(train_file, 'r') as f: 191 | for line in tqdm(f): 192 | dic = json.loads(line.strip()) 193 | for spo in dic['spo_list']: 194 | for postag in dic['postag']: 195 | if postag['word'] == spo['subject']: 196 | subject_tags[postag['pos']] += 1 197 | break 198 | for postag in dic['postag']: 199 | if postag['word'] == spo['object']: 200 | object_tags[postag['pos']] += 1 201 | break 202 | s = list(subject_tags.keys()) 203 | o = list(object_tags.keys()) 204 | for key in s: 205 | if subject_tags[key] == 0: 206 | del subject_tags[key] 207 | for key in o: 208 | if object_tags[key] == 0: 209 | del object_tags[key] 210 | with open('../dict/sub_tag', 'w') as fs: 211 | for key, value in subject_tags.items(): 212 | fs.write(key + '\t' + str(value) + '\n') 213 | with open('../dict/obj_tag', 'w') as fo: 214 | for key, value in object_tags.items(): 215 | fo.write(key + '\t' + str(value) + '\n') 216 | 217 | return subject_tags, object_tags 218 | 219 | def get_train_reader(self, need_input=False, need_label=True): 220 | """Data reader during training""" 221 | return self.path_reader(self.train_data_list_path, need_input, need_label) 222 | 223 | def get_dev_reader(self, need_input=True, need_label=True): 224 | """Data reader during dev""" 225 | return self.path_reader(self.dev_data_list_path, need_input, need_label) 226 | 227 | def get_test_reader(self, test_file_path='', need_input=True, need_label=False): 228 | """Data reader during predict""" 229 | return self.path_reader(test_file_path, need_input, need_label) 230 | 231 | def get_dict(self, dict_name): 232 | """Return dict""" 233 | if dict_name not in self._feature_dict: 234 | raise ValueError("dict name %s not found." % (dict_name)) 235 | return self._feature_dict[dict_name] 236 | 237 | def get_all_dict_name(self): 238 | """Get name of all dict""" 239 | return self._feature_dict.keys() 240 | 241 | def get_dict_size(self, dict_name): 242 | """Return dict length""" 243 | if dict_name not in self._feature_dict: 244 | raise ValueError("dict name %s not found." % (dict_name)) 245 | return len(self._feature_dict[dict_name]) 246 | 247 | def _get_reverse_dict(self, dict_name): 248 | dict_reverse = {} 249 | for key, value in self._feature_dict[dict_name].items(): 250 | dict_reverse[value] = key 251 | return dict_reverse 252 | 253 | def _reverse_p_eng(self, dic): 254 | dict_reverse = {} 255 | for key, value in dic.items(): 256 | dict_reverse[value] = key 257 | return dict_reverse 258 | 259 | def get_label_output(self, label_idx): 260 | """Output final label, used during predict and test""" 261 | dict_name = 'label_dict' 262 | if len(self._reverse_dict[dict_name]) == 0: 263 | self._get_reverse_dict(dict_name) 264 | p_eng = self._reverse_dict[dict_name][label_idx] 265 | return self._reverse_dict['eng_map_p_dict'][p_eng] 266 | 267 | 268 | if __name__ == '__main__': 269 | # initialize data generator 270 | data_generator = MyDataReader( 271 | postag_dict_path='../dict/postag_dict', 272 | label_dict_path='../dict/p_eng', 273 | train_data_list_path='../data/ori_data/train_data.json', 274 | dev_data_list_path='../data/ori_data/dev_data.json') 275 | 276 | # prepare data reader 277 | train = data_generator.get_train_reader() 278 | with open("../data/NER_data/train.txt", 'w') as f: 279 | for token_list, label_list in tqdm(train()): 280 | for i in range(len(token_list)): 281 | if token_list[i] == '' and label_list[i] == '': 282 | raise ValueError 283 | f.write(str(token_list[i]) + ' ' + str(label_list[i]) + '\n') 284 | f.write('\n') 285 | 286 | dev = data_generator.get_dev_reader() 287 | index = 0 288 | with open("../data/NER_data/dev.txt", 'w') as f: 289 | for token_list, label_list in tqdm(dev()): 290 | index += 1 291 | for i in range(len(token_list)): 292 | f.write(str(token_list[i]) + ' ' + str(label_list[i]) + '\n') 293 | f.write('\n') 294 | print('index:{}'.format(index)) 295 | 296 | test = data_generator.get_test_reader( 297 | test_file_path='../data/ori_data/test1_data_postag.json') 298 | with open("../data/NER_data/test.txt", 'w') as f: 299 | for token_list, label_list in tqdm(test()): 300 | for i in range(len(token_list)): 301 | f.write(str(token_list[i]) + ' ' + str(label_list[i]) + '\n') 302 | f.write('\n') 303 | -------------------------------------------------------------------------------- /RC/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Model structures for Relation Classification. 5 | @author: yuhaitao 6 | """ 7 | 8 | import sys 9 | import tensorflow as tf 10 | from tensorflow.contrib.layers.python.layers import initializers 11 | sys.path.append("../") 12 | from bert.bert_code import modeling, optimization, tokenization 13 | 14 | __all__ = ['InputExample', 'InputFeatures', 'decode_labels', 'create_model', 'convert_id_str', 15 | 'convert_id_to_label', 'result_to_json', 'create_classification_model'] 16 | 17 | 18 | class Model(object): 19 | def __init__(self, *args, **kwargs): 20 | pass 21 | 22 | 23 | class InputExample(object): 24 | """A single training/test example for simple sequence classification.""" 25 | 26 | def __init__(self, guid, text_a, text_b=None, label=None): 27 | """Constructs a InputExample. 28 | Args: 29 | guid: Unique id for the example. 30 | text_a: string. The untokenized text of the first sequence. For single 31 | sequence tasks, only this sequence must be specified. 32 | label: (Optional) string. The label of the example. This should be 33 | specified for train and dev examples, but not for test examples. 34 | """ 35 | self.guid = guid 36 | self.text_a = text_a 37 | self.text_b = text_b 38 | self.label = label 39 | 40 | 41 | class InputFeatures(object): 42 | """A single set of features of data.""" 43 | 44 | def __init__(self, input_ids, input_mask, segment_ids, label_id, ): 45 | self.input_ids = input_ids 46 | self.input_mask = input_mask 47 | self.segment_ids = segment_ids 48 | self.label_id = label_id 49 | 50 | 51 | class DataProcessor(object): 52 | """Base class for data converters for sequence classification data sets.""" 53 | 54 | def get_train_examples(self, data_dir): 55 | """Gets a collection of `InputExample`s for the train set.""" 56 | raise NotImplementedError() 57 | 58 | def get_dev_examples(self, data_dir): 59 | """Gets a collection of `InputExample`s for the dev set.""" 60 | raise NotImplementedError() 61 | 62 | def get_labels(self): 63 | """Gets the list of labels for this data set.""" 64 | raise NotImplementedError() 65 | 66 | 67 | def create_model(bert_config, is_training, input_ids, input_mask, segment_ids, labels, num_labels): 68 | """ 69 | 70 | :param bert_config: 71 | :param is_training: 72 | :param input_ids: 73 | :param input_mask: 74 | :param segment_ids: 75 | :param labels: 76 | :param num_labels: 77 | :param use_one_hot_embedding: 78 | :return: 79 | """ 80 | # 通过传入的训练数据，进行representation 81 | model = modeling.BertModel( 82 | config=bert_config, 83 | is_training=is_training, 84 | input_ids=input_ids, 85 | input_mask=input_mask, 86 | token_type_ids=segment_ids, 87 | ) 88 | 89 | # In the demo, we are doing a simple classification task on the entire 90 | # segment. 91 | # 92 | # If you want to use the token-level output, use model.get_sequence_output() 93 | # instead. 94 | output_layer = model.get_pooled_output() 95 | 96 | hidden_size = output_layer.shape[-1].value 97 | 98 | output_weights = tf.get_variable( 99 | "output_weights", [num_labels, hidden_size], 100 | initializer=tf.truncated_normal_initializer(stddev=0.02)) 101 | 102 | output_bias = tf.get_variable( 103 | "output_bias", [num_labels], initializer=tf.zeros_initializer()) 104 | 105 | with tf.variable_scope("loss"): 106 | if is_training: 107 | # I.e., 0.1 dropout 108 | output_layer = tf.nn.dropout(output_layer, keep_prob=0.9) 109 | 110 | logits = tf.matmul(output_layer, output_weights, transpose_b=True) 111 | logits = tf.nn.bias_add(logits, output_bias) 112 | probabilities = tf.nn.softmax(logits, axis=-1) 113 | log_probs = tf.nn.log_softmax(logits, axis=-1) 114 | 115 | one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32) 116 | 117 | per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) 118 | loss = tf.reduce_mean(per_example_loss) 119 | 120 | return (loss, per_example_loss, logits, probabilities) 121 | 122 | 123 | def create_model_PCNN(bert_config, is_training, input_ids, input_mask, segment_ids, labels, num_labels, positions, pcnn_mask): 124 | """ 125 | 使用bert与pcnn结合进行关系分类 126 | """ 127 | # PCNN所需要的组件 128 | def word_position_embedding(bert_out, positions, position_dim=10): 129 | """ 130 | bert 输出与 position embedding合并 131 | pos1:主体的位置 132 | pos2:客体的位置 133 | """ 134 | pos1_head, pos1_tail, pos2_head, pos2_tail = \ 135 | positions[:, :, 0], positions[:, :, 1], \ 136 | positions[:, :, 2], positions[:, :, 3] 137 | with tf.variable_scope('position_embedding'): 138 | max_len = bert_out.shape[1].value 139 | pos_tot = max_len * 2 # 因为position可以为正负，所以要乘2 140 | pos1_head_embedding = tf.get_variable('pos_1_head', shape=[pos_tot, position_dim], 141 | dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True) 142 | pos1_tail_embedding = tf.get_variable('pos_1_tail', shape=[pos_tot, position_dim], 143 | dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True) 144 | pos2_head_embedding = tf.get_variable('pos_2_head', shape=[pos_tot, position_dim], 145 | dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True) 146 | pos2_tail_embedding = tf.get_variable('pos_2_tail', shape=[pos_tot, position_dim], 147 | dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(), trainable=True) 148 | input_pos1_head = tf.nn.embedding_lookup( 149 | pos1_head_embedding, pos1_head) 150 | input_pos1_tail = tf.nn.embedding_lookup( 151 | pos1_tail_embedding, pos1_tail) 152 | input_pos2_head = tf.nn.embedding_lookup( 153 | pos2_head_embedding, pos2_head) 154 | input_pos2_tail = tf.nn.embedding_lookup( 155 | pos2_tail_embedding, pos2_tail) 156 | position_embedding = tf.concat( 157 | [input_pos1_head, input_pos1_tail, input_pos2_head, input_pos2_tail], -1) 158 | return tf.concat([bert_out, position_embedding], -1) 159 | 160 | def __cnn_cell__(x, hidden_size, kernel_size, stride_size=1): 161 | x = tf.layers.conv1d(inputs=x, 162 | filters=hidden_size, 163 | kernel_size=kernel_size, 164 | strides=stride_size, 165 | padding='same', 166 | kernel_initializer=tf.contrib.layers.xavier_initializer()) 167 | return x 168 | 169 | def __piecewise_pooling__(x, mask): 170 | mask_embedding = tf.constant( 171 | [[0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=tf.float32, name='mask_embedding') 172 | # mask表示三个通道，分别表示p1前，p1和p2之间，p2后，全0的表示填充部分 173 | mask = tf.nn.embedding_lookup(mask_embedding, mask) 174 | hidden_size = x.shape[-1].value 175 | x = tf.reduce_max(tf.expand_dims(mask * 100, 2) + 176 | tf.expand_dims(x, 3), axis=1) - 100 177 | return tf.reshape(x, [-1, hidden_size * 3]) 178 | 179 | def pcnn(x, mask, is_training, hidden_size=256, kernel_size=3, stride_size=1, activation=tf.nn.relu, keep_prob=0.9): 180 | with tf.variable_scope("pcnn"): 181 | max_length = x.shape[1] 182 | x = __cnn_cell__(x, hidden_size, kernel_size, stride_size) 183 | x = __piecewise_pooling__(x, mask) 184 | x = activation(x) 185 | return x 186 | 187 | # 首先使用bert的输出作为embedding 188 | model = modeling.BertModel( 189 | config=bert_config, 190 | is_training=is_training, 191 | input_ids=input_ids, 192 | input_mask=input_mask, 193 | token_type_ids=segment_ids, 194 | ) 195 | bert_out = model.get_sequence_output() 196 | 197 | # PCNN 流程 198 | pcnn_input = word_position_embedding(bert_out, positions) 199 | pcnn_output = pcnn(pcnn_input, pcnn_mask, is_training) 200 | 201 | # 输出 202 | hidden_size = pcnn_output.shape[-1].value 203 | output_weights = tf.get_variable( 204 | "output_weights", [num_labels, hidden_size], 205 | initializer=tf.truncated_normal_initializer(stddev=0.02)) 206 | output_bias = tf.get_variable( 207 | "output_bias", [num_labels], initializer=tf.zeros_initializer()) 208 | 209 | with tf.variable_scope("loss"): 210 | if is_training: 211 | # I.e., 0.1 dropout 212 | pcnn_output = tf.nn.dropout(pcnn_output, keep_prob=0.9) 213 | 214 | logits = tf.matmul(pcnn_output, output_weights, transpose_b=True) 215 | logits = tf.nn.bias_add(logits, output_bias) 216 | probabilities = tf.nn.softmax(logits, axis=-1) 217 | log_probs = tf.nn.log_softmax(logits, axis=-1) 218 | one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32) 219 | per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) 220 | loss = tf.reduce_mean(per_example_loss) 221 | 222 | return (loss, per_example_loss, logits, probabilities) 223 | 224 | 225 | def decode_labels(labels, batch_size): 226 | new_labels = [] 227 | for row in range(batch_size): 228 | label = [] 229 | for i in labels[row]: 230 | i = i.decode('utf-8') 231 | if i == '**PAD**': 232 | break 233 | if i in ['[CLS]', '[SEP]']: 234 | continue 235 | label.append(i) 236 | new_labels.append(label) 237 | return new_labels 238 | 239 | 240 | def convert_id_str(input_ids, batch_size): 241 | res = [] 242 | for row in range(batch_size): 243 | line = [] 244 | for i in input_ids[row]: 245 | i = i.decode('utf-8') 246 | if i == '**PAD**': 247 | break 248 | if i in ['[CLS]', '[SEP]']: 249 | continue 250 | 251 | line.append(i) 252 | res.append(line) 253 | return res 254 | 255 | 256 | def convert_id_to_label(pred_ids_result, idx2label, batch_size): 257 | """ 258 | 将id形式的结果转化为真实序列结果 259 | :param pred_ids_result: 260 | :param idx2label: 261 | :return: 262 | """ 263 | result = [] 264 | index_result = [] 265 | for row in range(batch_size): 266 | curr_seq = [] 267 | curr_idx = [] 268 | ids = pred_ids_result[row] 269 | for idx, id in enumerate(ids): 270 | if id == 0: 271 | break 272 | curr_label = idx2label[id] 273 | if curr_label in ['[CLS]', '[SEP]']: 274 | if id == 102 and (idx < len(ids) and ids[idx + 1] == 0): 275 | break 276 | continue 277 | # elif curr_label == '[SEP]': 278 | # break 279 | curr_seq.append(curr_label) 280 | curr_idx.append(id) 281 | result.append(curr_seq) 282 | index_result.append(curr_idx) 283 | return result, index_result 284 | 285 | 286 | def result_to_json(self, string, tags): 287 | """ 288 | 将模型标注序列和输入序列结合 转化为结果 289 | :param string: 输入序列 290 | :param tags: 标注结果 291 | :return: 292 | """ 293 | item = {"entities": []} 294 | entity_name = "" 295 | entity_start = 0 296 | idx = 0 297 | last_tag = '' 298 | 299 | for char, tag in zip(string, tags): 300 | if tag[0] == "S": 301 | self.append(char, idx, idx + 1, tag[2:]) 302 | item["entities"].append( 303 | {"word": char, "start": idx, "end": idx + 1, "type": tag[2:]}) 304 | elif tag[0] == "B": 305 | if entity_name != '': 306 | self.append(entity_name, entity_start, idx, last_tag[2:]) 307 | item["entities"].append( 308 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 309 | entity_name = "" 310 | entity_name += char 311 | entity_start = idx 312 | elif tag[0] == "I": 313 | entity_name += char 314 | elif tag[0] == "O": 315 | if entity_name != '': 316 | self.append(entity_name, entity_start, idx, last_tag[2:]) 317 | item["entities"].append( 318 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 319 | entity_name = "" 320 | else: 321 | entity_name = "" 322 | entity_start = idx 323 | idx += 1 324 | last_tag = tag 325 | if entity_name != '': 326 | self.append(entity_name, entity_start, idx, last_tag[2:]) 327 | item["entities"].append( 328 | {"word": entity_name, "start": entity_start, "end": idx, "type": last_tag[2:]}) 329 | return item 330 | --------------------------------------------------------------------------------