├── README.md ├── algorithm └── kg_qa │ ├── KG │ ├── KgAnswer.py │ ├── KgEval.py │ ├── KgPredict.py │ ├── __init__.py │ └── es.py │ ├── NER │ ├── DataMaking.py │ ├── EntityExtract.py │ ├── Eval.py │ ├── Predict.py │ ├── TrainAndValid.py │ └── __init__.py │ ├── NER_BERT_LSTM_CRF │ ├── EntityExtract.py │ ├── Eval.py │ ├── Predict.py │ ├── TrainAndValid.py │ ├── __init__.py │ └── lstm_crf_layer.py │ ├── SIM │ ├── DataMaking.py │ ├── Eval.py │ ├── Predict.py │ ├── TrainAndValid.py │ └── __init__.py │ ├── __init__.py │ ├── config.py │ └── 不同预训练模型的实验与评估.docx ├── bert ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── __init__.py ├── create_pretraining_data.py ├── extract_features.py ├── modeling.py ├── modeling_test.py ├── multilingual.md ├── optimization.py ├── optimization_test.py ├── predicting_movie_reviews_with_bert_on_tf_hub.ipynb ├── requirements.txt ├── run_classifier.py ├── run_classifier_with_tfhub.py ├── run_pretraining.py ├── run_squad.py ├── sample_text.txt ├── tokenization.py └── tokenization_test.py ├── config.py ├── pretraining_model └── readme.txt ├── raw_data └── kgClue │ ├── kgClue.yaml │ └── knowledge │ └── README.md ├── requirements.txt └── utils ├── DrawTrain.py ├── EvalReport.py ├── IdAndLabel.py ├── ListAndList.py └── __init__.py /README.md: -------------------------------------------------------------------------------- 1 | [toc] 2 | 3 | # KgClue_Bench 4 | 5 | 尽最大能力解耦代码，为NLP新手提供(BERT)学习平台 6 | 7 | ## 目录结构 8 | 9 | ├─algorithm # 算法
10 | │ └─kg_qa # 算法开发示例
11 | │ │ config.py
12 | │ ├─KG 每个模块对应一个package
13 | │ │ │ es.py # 将知识库导入es的脚本
14 | │ │ │ KgAnswer.py # 回答问题类
15 | │ │ │ KgEval.py# 回答问题的准确度评估方法
16 | │ │ │ KgPredict.py# 针对test.json文件生成预测结果，手动压缩之后可以提交到官网进行评估
17 | │ ├─NER
18 | │ │ │ DataMaking.py# NER训练数据集的制作脚本
19 | │ │ │ EntityExtract.py# 将序列标注标签转化为实体
20 | │ │ │ Eval.py# 评估代码（输出f1）
21 | │ │ │ Predict.py# 预测类
22 | │ │ │ TrainAndValid.py# 训练代码
23 | ├─bert 谷歌官方Bert代码存放
24 | │ │ .gitignore
25 | ├─pretraining_model # 存放bert的预训练模型
26 | │ ├─chinese_rbt3_L-3_H-768_A-12 #存放示例
27 | ├─raw_data # 数据集推荐添加方式,直接解压
28 | │ ├─kgClue # kg_qa项目中适配的数据集
29 | │ │ │ xxx.json
30 | │ │ └─knowledge # 知识库
31 | │ │ Knowledge.txt
32 | └─utils
33 | 34 | 35 | ## 算法排行 36 | 37 | ### **kg_qa任务** 以kgClue为训练数据集，旨在回答知识库中的问题 38 | 39 | > #### 不同算法结构性能比较(以chinese_rbtl3_L-3_H-1024_A-16为预训练模型) 40 | > 这里的评估是以问题回答准确度作为标准 41 | > 42 | >Model | F1 | EM | 43 | >:----:| :----: |:----: | 44 | >bert-crf | 70.7 | 70.7 | 45 | >bert-lstm-crf | 63.9 | 63.6 | 46 | 47 | #### 不同预训练模型性能比较(不代表每个模型的最佳性能) 48 | NER (bert+crf) seq_lan=32 epoch=5 49 | 50 | | pretraining_model | batch | micro-f1| macro-f1| f1(##WordPiece) |f1(B-NP/I-NP)| 51 | | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | 52 | | chinese_rbt3_L-3_H-768_A-12 | 40 | 93.1| 88.0 | 61.0 | 79.0 | 53 | | chinese_rbt4_L-4_H-768_A-12 | 40 | 92.0 | 87.0 | 62.0 | 75.0 | 54 | | chinese_rbt6_L-6_H-768_A-12 | 40 | 93.0 | 88.0 | 61.0 | 77.0 | 55 | | chinese_rbtl3_L-3_H-1024_A-16 | 40 | 93.0 | 89.0 | 66.0 | 77.0 | 56 | | chinese_wwm_ext_L-12_H-768_A-12 | 40 | 93.0 | 88.0 | 63.0 | 77.0 | 57 | 58 | SIM (bert) seq_lan=64 epoch=5 59 | 60 | | pretraining_model | batch | accuracy| precision| recall |macro-f1| 61 | | ----------- | ----------- | ----------- | ----------- | ----------- | ----------- | 62 | | chinese_rbt3_L-3_H-768_A-12 | 40 | 86.0| 44.3 | 2.0 | 49.0 | 63 | | chinese_rbt4_L-4_H-768_A-12 | 40 | 93.5 | 78.3 | 73.1 | 85.9 | 64 | | chinese_rbt6_L-6_H-768_A-12 | 40 | 93.8 | 79.2 | 74.9 | 86.7 | 65 | | chinese_rbtl3_L-3_H-1024_A-16 | 40 |96.5 |86.4| 89.1| 92.9 | 66 | | chinese_wwm_ext_L-12_H-768_A-12 | 40 | 95.5| 82.1| 86.6 | 90.9 | 67 | 68 | ## 使用示例 69 | 70 | ### 以 **kg_qa** 算法为例 71 | 72 | > 该项目下有三个文件夹KG\NER\SIM 73 | 74 | #### NER 75 | 76 | 1. python DataMaking.py **注意**: 1. 文件路径 2.脚本work路径,应该以整个KgCLUEBench为项目根目录运行 77 | 2. python TrainAndValid.py **注意** :训练之前设置好kg_qa目录下的config配置,其他注意点同上 78 | 3. python Precit.py 验证是否正常运行 79 | 4. python Eval.py 得出模型的评估结果,可以在训练时间段Eval模型,查看训练效果 80 | 5. python EntityExtract.py 将序列标注结果(Predict结果)转化为句子中的实体 81 | 82 | #### SIM 同理 83 | 84 | 1. python DataMaking.py **注意**: 1. 文件路径 2.脚本work路径,应该以整个KgCLUEBench为项目根目录运行 85 | 2. python TrainAndValid.py **注意** :训练之前设置好kg_qa目录下的config配置,其他注意点同上 86 | 3. python Precit.py 验证是否正常运行 87 | 4. python Eval.py 得出模型的评估结果,可以在训练时间段Eval模型,查看训练效果 88 | 89 | #### KG 90 | 1. es.py是将知识库（这里是Knowledge.txt）导入es系统的脚本文件，只需要执行一次 91 | 2. KgAnswer.py是回答问题的类，只需要输入一个句子，即可给出结果 92 | 3. KgEval是评估问题回答能力的代码，修改文件路径即可使用 93 | 4. KgPredict是回答test.json的代码，运行完成可以生成kgclue_predict.txt，用户压缩成zip文件之后可以直接提交至clue官网。 94 | 95 | ## algorithm 贡献方法 96 | 97 | > 在此目录下直接命名一个新的python包包含init和config文件 98 | > 不同算法可能有多个stage，不同stage建议使用独立的python包，多个stage共享一个config 99 | 100 | ## UPDATE 101 | 102 | ******* 2021-12-3,新项目开荒 103 | ******* 2021-12-12,完整流程测试通过 104 | 105 | ## 有问题联系1194370384@qq.com 106 | 107 | -------------------------------------------------------------------------------- /algorithm/kg_qa/KG/KgAnswer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/28 20:25 4 | # @Author : 刘鑫 5 | # @FileName: KgAnswer.py 6 | # @Software: PyCharm 7 | import json 8 | import os 9 | 10 | from elasticsearch import Elasticsearch 11 | 12 | from algorithm.kg_qa.NER.EntityExtract import EntityExtract 13 | from algorithm.kg_qa.SIM.Predict import Predict as SimPredict 14 | from algorithm.kg_qa.config import NerConfig, SimConfig 15 | 16 | 17 | class KgAnswer(object): 18 | def __init__(self, NER_MODEL_PATH, SIM_MODEL_PATH, es_host, es_port): 19 | # init the model we need to use 20 | self.ee = EntityExtract(NER_MODEL_PATH) 21 | self.sim = SimPredict(SIM_MODEL_PATH) 22 | self.es = Elasticsearch([":".join((es_host, es_port))]) 23 | 24 | def answer(self, sentence): 25 | 26 | entitys = "".join(self.ee.extract(sentence)) 27 | # to_do 需要添加适配，多个实体的情况 28 | # print(entitys) 29 | body = { 30 | "query": { 31 | "term": { 32 | "entity.keyword": entitys 33 | } 34 | } 35 | } 36 | 37 | es_results = self.es.search(index="kbqa-data", doc_type="kbList", body=body, size=1000) 38 | 39 | attribute_list, answer_list = list(), list() 40 | for i in range(len(es_results['hits']['hits'])): 41 | relation = es_results['hits']['hits'][i]['_source']['relation'] 42 | value = es_results['hits']['hits'][i]['_source']['value'] 43 | attribute_list.append(relation) 44 | answer_list.append(value) 45 | best_answer = "" 46 | best_attribute = "" 47 | probs_init = 0 48 | for attribute, answer in zip(attribute_list, answer_list): 49 | if attribute: 50 | isAttribute, probs = self.sim.predict_one(sentence, attribute, TEST_MODE=True) 51 | # if isAttribute: 52 | # # print("问题：%s，属性：%s,回答：%s" % (sentence, attribute, answer)) 53 | if probs[0][1] > probs_init: 54 | best_answer = answer 55 | best_attribute = attribute 56 | probs_init = probs[0][1] 57 | return best_answer, best_attribute, entitys 58 | 59 | 60 | if __name__ == '__main__': 61 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" 62 | NER_MODEL_PATH = NerConfig.model_out 63 | SIM_MODEL_PATH = SimConfig.model_out 64 | es_host = "127.0.0.1" 65 | es_port = "9200" 66 | kg = KgAnswer(NER_MODEL_PATH, SIM_MODEL_PATH, es_host, es_port) 67 | # print(kg.answer("NBA姚明学校在哪个地方啊？")) 68 | # print(kg.answer("NBA姚明学校所属地区是？")) 69 | # print(kg.answer("任宪韶的毕业院校是？")) 70 | # sentence = "巫山县疾病预防控制中心的机构职能是什么?" 71 | # print(kg.answer(sentence)) 72 | # kg_out_f = open("./kg_out.txt", "w", encoding='utf-8') 73 | with open(r"C:\Users\11943\Documents\GitHub\KgClue_Bench\raw_data\kgClue\test_public.json", 'r', 74 | encoding='utf-8') as f: 75 | count_number = 0 76 | true_answer = 0 77 | while True: 78 | 79 | line = f.readline() 80 | if line: 81 | count_number += 1 82 | line = json.loads(line) 83 | sentence = line["question"] 84 | answer = line["answer"].split("|||")[2].strip() 85 | p_answers = kg.answer(sentence) 86 | # kg_out_f.write(sentence + "\t" + answer + "\t" + "predict: \t" + p_answers + "\n") 87 | # break 88 | else: 89 | break 90 | # kg_out_f.close() 91 | -------------------------------------------------------------------------------- /algorithm/kg_qa/KG/KgEval.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/28 21:00 4 | # @Author : 刘鑫 5 | # @FileName: KgEval.py 6 | # @Software: PyCharm 7 | import json 8 | import os 9 | 10 | from KgAnswer import KgAnswer 11 | from algorithm.kg_qa.config import NerConfig, SimConfig 12 | 13 | def normalize_text(s): 14 | """Removing articles and punctuation, and standardizing whitespace are all typical text processing steps.""" 15 | import string, re 16 | 17 | def remove_articles(text): 18 | regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) 19 | return re.sub(regex, " ", text) 20 | 21 | def white_space_fix(text): 22 | return " ".join(text.split()) 23 | 24 | def remove_punc(text): 25 | exclude = set(string.punctuation) 26 | return "".join(ch for ch in text if ch not in exclude) 27 | 28 | def lower(text): 29 | return text.lower() 30 | 31 | return white_space_fix(remove_articles(remove_punc(lower(s)))) 32 | 33 | def compute_exact_match(prediction, truth): 34 | return int(normalize_text(prediction) == normalize_text(truth)) 35 | 36 | 37 | def compute_f1(prediction, truth): 38 | pred_tokens = normalize_text(prediction).split() 39 | truth_tokens = normalize_text(truth).split() 40 | 41 | # if either the prediction or the truth is no-answer then f1 = 1 if they agree, 0 otherwise 42 | if len(pred_tokens) == 0 or len(truth_tokens) == 0: 43 | return int(pred_tokens == truth_tokens) 44 | 45 | common_tokens = set(pred_tokens) & set(truth_tokens) 46 | 47 | # if there are no common tokens then f1 = 0 48 | if len(common_tokens) == 0: 49 | return 0 50 | 51 | prec = len(common_tokens) / len(pred_tokens) 52 | rec = len(common_tokens) / len(truth_tokens) 53 | 54 | return 2 * (prec * rec) / (prec + rec) 55 | 56 | 57 | if __name__ == '__main__': 58 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 59 | NER_MODEL_PATH = NerConfig.model_out 60 | SIM_MODEL_PATH = SimConfig.model_out 61 | es_host = "127.0.0.1" 62 | es_port = "9200" 63 | kg = KgAnswer(NER_MODEL_PATH, SIM_MODEL_PATH, es_host, es_port) 64 | em_score = 0 65 | f1_score = 0 66 | with open(r"C:\Users\11943\Documents\GitHub\KgCLUEbench\raw_data\kgClue\test_public.json", 'r', encoding='utf-8') as f: 67 | count_number = 0 68 | while True: 69 | 70 | line = f.readline() 71 | if line: 72 | count_number += 1 73 | line = json.loads(line) 74 | sentence = line["question"] 75 | answer = line["answer"].split("|||")[2].strip() 76 | p_answers,_,_ = kg.answer(sentence) 77 | # print(answer,p_answers) 78 | 79 | em_score += compute_exact_match(p_answers, answer) 80 | f1_score += compute_f1(p_answers, answer) 81 | 82 | else: 83 | break 84 | 85 | em = em_score/count_number 86 | f1 = f1_score/count_number 87 | print(em) 88 | print(f1) -------------------------------------------------------------------------------- /algorithm/kg_qa/KG/KgPredict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/12/10 13:11 4 | # @Author : 刘鑫 5 | # @FileName: KgPredict.py 6 | # @Software: PyCharm 7 | 8 | import json 9 | import os 10 | 11 | from KgAnswer import KgAnswer 12 | from algorithm.kg_qa.config import NerConfig, SimConfig 13 | 14 | if __name__ == '__main__': 15 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" 16 | NER_MODEL_PATH = NerConfig.model_out 17 | SIM_MODEL_PATH = SimConfig.model_out 18 | es_host = "127.0.0.1" 19 | es_port = "9200" 20 | kg = KgAnswer(NER_MODEL_PATH, SIM_MODEL_PATH, es_host, es_port) 21 | out_f = open("./kgclue_predict.txt", "w", encoding='utf-8') 22 | with open(r"C:\Users\11943\Documents\GitHub\KgCLUEbench\raw_data\kgClue\test.json", 'r', encoding='utf-8') as f: 23 | count_number = 0 24 | while True: 25 | 26 | line = f.readline() 27 | if line: 28 | 29 | line = json.loads(line) 30 | sentence = line["question"] 31 | best_answer, best_attribute, entitys = kg.answer(sentence) 32 | tmp = dict() 33 | tmp["id"] = count_number 34 | tmp["answer"] = str(entitys) + " ||| " + str(best_attribute) + " ||| " + str(best_answer) 35 | x = json.dumps(tmp, ensure_ascii=False) 36 | out_f.write(x+"\n") 37 | # print(x) 38 | count_number += 1 39 | # break 40 | # {"id": 0, "answer": "刘质平 ||| 师从 ||| 李叔同"} 41 | 42 | else: 43 | break 44 | out_f.close() 45 | -------------------------------------------------------------------------------- /algorithm/kg_qa/KG/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/12/1 12:08 4 | # @Author : 刘鑫 5 | # @FileName: __init__.py.py 6 | # @Software: PyCharm 7 | -------------------------------------------------------------------------------- /algorithm/kg_qa/KG/es.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | ''' 3 | ------------------------------------------------- 4 | Description : deploy es 5 | Author : nijinxin 6 | Date : 2021-11-24 7 | ------------------------------------------------- 8 | 9 | ''' 10 | 11 | from elasticsearch import Elasticsearch 12 | from elasticsearch.helpers import bulk 13 | 14 | def deployES(input_file, host, port): 15 | """ put triples into es 16 | input: input_file, host, port 17 | output: None 18 | """ 19 | es_dir = ":".join((host, port)) 20 | print("es_dir:",es_dir) 21 | es = Elasticsearch([es_dir]) 22 | es.indices.create(index='kbqa-data',ignore=[400, 404]) 23 | f = open(input_file, 'r', encoding='utf-8') 24 | 25 | id_num = 0 26 | lines = [] 27 | while True: 28 | line = f.readline() 29 | id_num += 1 30 | if line == "": 31 | break 32 | line = line.split("\t") 33 | if len(line[0].split("（")) == 2 and line[0][-1] == "）": 34 | entity = line[0].split("（")[0] 35 | ambiguity = line[0].split("（")[1].strip("）") 36 | 37 | else: 38 | entity = line[0] 39 | ambiguity = "None" 40 | 41 | action = { 42 | "_index": "kbqa-data", 43 | "_type": "kbList", 44 | "_id": id_num, #_id 也可以默认生成，不赋值 45 | "_source": { 46 | "entity": entity, 47 | "relation": line[1], 48 | "value": line[2].strip(), 49 | "ambiguity": ambiguity}} 50 | lines.append(action) 51 | if id_num % 5000 == 0: 52 | 53 | bulk(es, lines, index="kbqa-data", raise_on_error=True) 54 | lines = [] 55 | if id_num % 50000 == 0: 56 | print(id_num, " triples has been deployed") 57 | f.close() 58 | 59 | 60 | if __name__ == "__main__": 61 | input_file = "../knowledge/Knowledge.txt" 62 | host = "127.0.0.1" 63 | port = "9200" 64 | deployES(input_file, host, port) -------------------------------------------------------------------------------- /algorithm/kg_qa/NER/DataMaking.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/23 14:33 4 | # @Author : 刘鑫 5 | # @FileName: ner_data_making.py 6 | # @Software: PyCharm 7 | import collections 8 | import json 9 | import os 10 | import tensorflow as tf 11 | 12 | from bert import tokenization 13 | from algorithm.kg_qa.config import Properties,NerConfig as config 14 | from utils.ListAndList import _index_q_list_in_k_list 15 | 16 | 17 | class DataMaking(object): 18 | def __init__(self, do_lower_case=True, max_seq_length=128): 19 | self.task_name = "NER" 20 | # 初始化 bert_token 工具 21 | self.bert_tokenizer = tokenization.FullTokenizer(vocab_file=Properties.vocab_file, do_lower_case=do_lower_case) 22 | self.max_seq_length = max_seq_length 23 | 24 | def makdir(self, OUTPUT_DIR, DIR_NAME): 25 | if not os.path.exists(os.path.join(os.path.dirname(__file__), OUTPUT_DIR)): 26 | os.makedirs(os.path.join(os.path.dirname(__file__), OUTPUT_DIR)) 27 | if not os.path.exists(os.path.join(os.path.dirname(__file__), OUTPUT_DIR, DIR_NAME)): 28 | os.makedirs(os.path.join(os.path.dirname(__file__), OUTPUT_DIR, DIR_NAME)) 29 | 30 | def text2label(self, text, entity): 31 | text_token = self.bert_tokenizer.tokenize(text) 32 | entity_token = self.bert_tokenizer.tokenize(entity) 33 | entity_lenth = len(entity_token) 34 | idx_start = _index_q_list_in_k_list(entity_token, text_token) 35 | labeling_list = ["O"] * len(text_token) # 先用全O覆盖 36 | if idx_start is None: 37 | tokener_error_flag = True 38 | # self.bert_tokener_error_log_f.write(subject_object + " @@ " + text + "\n") 39 | else: 40 | labeling_list[idx_start] = "B-NP" 41 | if entity_lenth == 2: 42 | labeling_list[idx_start + 1] = "I-NP" 43 | elif entity_lenth >= 3: 44 | labeling_list[idx_start + 1: idx_start + entity_lenth] = ["I-NP"] * (entity_lenth - 1) 45 | 46 | for idx, token in enumerate(text_token): 47 | if token.startswith("##"): 48 | labeling_list[idx] = "[##WordPiece]" 49 | 50 | return labeling_list 51 | 52 | def convert_single_example(self, text, labeling_list=[], TEST_MODE=False): 53 | 54 | token_label_list = config.label_list 55 | 56 | token_label_map = {} 57 | for (i, label) in enumerate(token_label_list): 58 | token_label_map[label] = i 59 | 60 | text_token = self.bert_tokenizer.tokenize(text) 61 | 62 | # Account for [CLS] and [SEP] with "- 2" 63 | if len(text_token) > self.max_seq_length - 2: 64 | text_token = text_token[0:(self.max_seq_length - 2)] 65 | 66 | tokens = [] 67 | token_label_ids = [] 68 | segment_ids = [] 69 | # 添加起始位置 70 | tokens.append("[CLS]") 71 | segment_ids.append(0) 72 | token_label_ids.append(token_label_map["[CLS]"]) 73 | 74 | if TEST_MODE: 75 | labeling_list = ["O"] * len(text_token) 76 | 77 | for token, label in zip(text_token, labeling_list): 78 | tokens.append(token) 79 | segment_ids.append(0) 80 | token_label_ids.append(token_label_map[label]) 81 | 82 | tokens.append("[SEP]") 83 | segment_ids.append(0) 84 | token_label_ids.append(token_label_map["[SEP]"]) # 第一句话结束 85 | 86 | input_ids = self.bert_tokenizer.convert_tokens_to_ids(tokens) 87 | 88 | input_mask = [1] * len(input_ids) 89 | 90 | # Zero-pad up to the sequence length. 91 | while len(input_ids) < self.max_seq_length: 92 | input_ids.append(0) 93 | input_mask.append(0) 94 | segment_ids.append(0) 95 | token_label_ids.append(0) 96 | tokens.append("[Padding]") 97 | 98 | assert len(input_ids) == self.max_seq_length 99 | assert len(input_mask) == self.max_seq_length 100 | assert len(segment_ids) == self.max_seq_length 101 | assert len(token_label_ids) == self.max_seq_length 102 | 103 | if TEST_MODE: 104 | feature = (input_ids, input_mask, segment_ids) 105 | else: 106 | feature = config.InputFeatures( 107 | input_ids=input_ids, 108 | input_mask=input_mask, 109 | segment_ids=segment_ids, 110 | label_ids=token_label_ids, 111 | is_real_example=True) 112 | return feature 113 | 114 | def input2output(self, INPUT_DATA_PATHS, OUTPUT_DIR): 115 | for data_path in INPUT_DATA_PATHS: 116 | file_name = data_path.split('/')[-1] 117 | file_type = file_name.split('.')[0] 118 | self.makdir(OUTPUT_DIR, file_type) # 生成文件名对应的文件夹 119 | OUT_PATH = os.path.join(os.path.dirname(__file__), OUTPUT_DIR, file_type) 120 | # 创建需要写入的文件 121 | text_f = open(os.path.join(OUT_PATH, "text.txt"), "w", encoding='utf-8') 122 | token_in_f = open(os.path.join(OUT_PATH, "token_in.txt"), "w", encoding='utf-8') 123 | token_label_f = open(os.path.join(OUT_PATH, "token_label.txt"), "w", encoding='utf-8') 124 | tf_writer = tf.python_io.TFRecordWriter(os.path.join(OUT_PATH, file_type + ".tf_record")) 125 | with open(data_path, "r", encoding='utf-8') as f: 126 | count_numbers = 0 127 | while True: 128 | line = f.readline() 129 | if line: 130 | line = json.loads(line) 131 | text = line["question"] 132 | answer = line["answer"] 133 | entity = answer.split("|||")[0].split("（")[0] 134 | token_label = self.text2label(text, entity) 135 | 136 | # 写入txt文件 137 | text_f.write(text + "\n") 138 | token_in_f.write(" ".join(self.bert_tokenizer.tokenize(text)) + "\n") 139 | token_label_f.write(" ".join(token_label) + "\n") 140 | 141 | # 写入tf_record 142 | feature = self.convert_single_example(text, token_label) 143 | 144 | def create_int_feature(values): 145 | f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) 146 | return f 147 | 148 | features = collections.OrderedDict() 149 | features["input_ids"] = create_int_feature(feature.input_ids) 150 | features["input_mask"] = create_int_feature(feature.input_mask) 151 | features["segment_ids"] = create_int_feature(feature.segment_ids) 152 | features["label_ids"] = create_int_feature(feature.label_ids) 153 | features["is_real_example"] = create_int_feature([int(feature.is_real_example)]) 154 | 155 | tf_example = tf.train.Example(features=tf.train.Features(feature=features)) 156 | tf_writer.write(tf_example.SerializeToString()) 157 | count_numbers += 1 158 | if count_numbers % 10000 == 0: 159 | print("Writing example %d " % (count_numbers)) 160 | # break 161 | else: 162 | break 163 | print("all numbers", count_numbers) 164 | text_f.close() 165 | token_in_f.close() 166 | token_label_f.close() 167 | tf_writer.close() 168 | 169 | 170 | if __name__ == '__main__': 171 | INPUT_DATA_PATHS = ["raw_data/kgClue/train.json", "raw_data/kgClue/eval.json", "raw_data/kgClue/test_public.json"] 172 | OUTPUT_DIR = "./data" 173 | 174 | data_make = DataMaking(do_lower_case=True, max_seq_length=config.max_seq_length) 175 | data_make.input2output(INPUT_DATA_PATHS, OUTPUT_DIR) 176 | -------------------------------------------------------------------------------- /algorithm/kg_qa/NER/EntityExtract.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/27 14:56 4 | # @Author : 刘鑫 5 | # @FileName: entity_extract.py 6 | # @Software: PyCharm 7 | import json 8 | import os 9 | 10 | from bert import tokenization 11 | from algorithm.kg_qa.config import Properties, NerConfig as config 12 | from algorithm.kg_qa.NER.Predict import Predict 13 | 14 | 15 | def isEntityInText(text, entity): 16 | text_list = list(text) 17 | entity_list = list(entity) 18 | M = len(text_list) 19 | N = len(entity_list) 20 | 21 | i = 0 22 | while i <= M - N: 23 | j = 0 24 | space_nums = 0 25 | match = [] 26 | while j < N: 27 | tt = text_list[i + j].lower() 28 | ee = entity_list[j].lower() 29 | if tt == ee: 30 | j += 1 31 | match.append(ee) 32 | continue 33 | elif j > 0 and tt == " ": 34 | i += 1 35 | space_nums += 1 36 | else: 37 | j += 1 38 | break 39 | if j == N and entity_list == match: 40 | return i - space_nums, space_nums 41 | i += 1 42 | return -1, -1 43 | 44 | 45 | class EntityExtract(object): 46 | def __init__(self, MODEL_PATH): 47 | self.predict = Predict(MODEL_PATH) 48 | self.tokenizer_ = tokenization.FullTokenizer(vocab_file=Properties.vocab_file) 49 | 50 | def extract(self, sentence): 51 | ''' 52 | 将预测的token_label对应到句子中的字，抽取出中文实体，可能存在问题：空格缺失、大小写不一致，不过这里采用的策略是原样输出。在计算f1和em时这两点问题可以忽略 53 | :param sentence: 54 | :return: entitys 55 | ''' 56 | 57 | def _merge_WordPiece_and_single_word(entity_sort_list): 58 | entity_sort_tuple_list = [] 59 | for a_entity_list in entity_sort_list: 60 | entity_content = "" 61 | entity_type = None 62 | for idx, entity_part in enumerate(a_entity_list): 63 | if idx == 0: 64 | entity_type = entity_part 65 | if entity_type[:2] not in ["B-", "I-"]: 66 | break 67 | else: 68 | if entity_part.startswith("##"): 69 | entity_content += entity_part.replace("##", "") 70 | else: 71 | entity_content += entity_part 72 | if entity_content != "": 73 | entity_sort_tuple_list.append((entity_type[2:], entity_content)) 74 | return entity_sort_tuple_list 75 | 76 | ner_out = self.predict.predict_one(sentence, TEST_MODE=True) 77 | 78 | def preprocessing_model_token_lable(predicate_token_label_list, token_in_list_lenth): 79 | if predicate_token_label_list[0] == "[CLS]": 80 | predicate_token_label_list = predicate_token_label_list[1:] # y_predict.remove('[CLS]') 81 | if len(predicate_token_label_list) > token_in_list_lenth: # 只取输入序列长度即可 82 | predicate_token_label_list = predicate_token_label_list[:token_in_list_lenth] 83 | return predicate_token_label_list 84 | 85 | predicate_token_label_list = preprocessing_model_token_lable(ner_out, len(ner_out)) 86 | 87 | entity_sort_list = [] 88 | entity_part_list = [] 89 | 90 | token_in_not_UNK = self.tokenizer_.tokenize_not_UNK(sentence) 91 | for idx, token_label in enumerate(predicate_token_label_list): 92 | if token_label == "O": 93 | if len(entity_part_list) > 0: 94 | entity_sort_list.append(entity_part_list) 95 | entity_part_list = [] 96 | if token_label.startswith("B-"): 97 | if len(entity_part_list) > 0: # 适用于 B- B- *****的情况 98 | entity_sort_list.append(entity_part_list) 99 | entity_part_list = [] 100 | entity_part_list.append(token_label) 101 | entity_part_list.append(token_in_not_UNK[idx]) 102 | if idx == len(predicate_token_label_list) - 1: 103 | entity_sort_list.append(entity_part_list) 104 | if token_label.startswith("I-") or token_label == "[##WordPiece]": 105 | if len(entity_part_list) > 0: 106 | entity_part_list.append(token_in_not_UNK[idx]) 107 | if idx == len(predicate_token_label_list) - 1: 108 | entity_sort_list.append(entity_part_list) 109 | if token_label == "[SEP]": 110 | break 111 | 112 | entity_sort_tuple_list = _merge_WordPiece_and_single_word(entity_sort_list) 113 | entitys = [] 114 | for entity in entity_sort_tuple_list: 115 | if entity[0] == "NP": 116 | start, space_nums = isEntityInText(sentence, entity[1]) 117 | if start != -1: 118 | end = len(entity[1]) + space_nums + start 119 | entitys.append(sentence[start:end]) 120 | return entitys 121 | 122 | 123 | if __name__ == '__main__': 124 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" 125 | MODEL_PATH = config.model_out 126 | ee = EntityExtract(MODEL_PATH) 127 | # ff = open("./out.txt", 'w', encoding='utf-8') 128 | true_count=0 129 | with open(r"C:\Users\11943\Documents\GitHub\KgClue_Bench\raw_data\kgClue\test_public.json", "r", 130 | encoding='utf-8') as f: 131 | while True: 132 | line = f.readline() 133 | if line: 134 | line = json.loads(line) 135 | text = line["question"] 136 | answer = line["answer"] 137 | entity = answer.split("|||")[0].split("（")[0] 138 | p_entity = ee.extract(text) 139 | print(entity,"".join(p_entity)) 140 | if entity == "".join(p_entity): 141 | true_count+=1 142 | # ff.write(text + "\t" + entity + "\t" + "".join(p_entity) + "\n") 143 | else: 144 | break 145 | # ff.close() 146 | print(true_count/2000) 147 | -------------------------------------------------------------------------------- /algorithm/kg_qa/NER/Eval.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/20 10:54 4 | # @Author : 刘鑫 5 | # @FileName: seq_eval.py 6 | # @Software: PyCharm 7 | import json 8 | import os 9 | 10 | from algorithm.kg_qa.NER.Predict import Predict 11 | from algorithm.kg_qa.NER.DataMaking import DataMaking 12 | from algorithm.kg_qa.config import NerConfig as config 13 | from utils.EvalReport import report 14 | 15 | 16 | class Eval(object): 17 | def __init__(self, MODEL_PATH): 18 | self.data_make = DataMaking(do_lower_case=True, max_seq_length=config.max_seq_length) 19 | self.predict = Predict(MODEL_PATH) 20 | self.seq_id2label = self.predict.seq_id2label 21 | 22 | def id2label_f(self, id_list): 23 | predictions = [] 24 | for id in id_list: 25 | predictions.append(self.seq_id2label[id]) 26 | return predictions 27 | 28 | def do_eval(self, data_files=["../raw_data/test.json"]): 29 | 30 | for data_file in data_files: 31 | y_true = [] 32 | y_pred = [] 33 | with open(data_file, 'r', encoding='utf-8') as f: 34 | while True: 35 | line = f.readline() 36 | if line: 37 | line = json.loads(line) 38 | text = line["question"] 39 | answer = line["answer"] 40 | entity = answer.split("|||")[0].split("（")[0] 41 | 42 | token_label = self.data_make.text2label(text, entity) 43 | feature = self.data_make.convert_single_example(text, token_label) 44 | label = self.id2label_f(feature.label_ids) 45 | predict_label = self.predict.predict_one(text, TEST_MODE=True) 46 | 47 | y_true.append(label) 48 | y_pred.append(predict_label) 49 | 50 | else: 51 | break 52 | report(y_true, y_pred) 53 | 54 | 55 | if __name__ == '__main__': 56 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 57 | MODEL_PATH = config.model_out 58 | ner_eval = Eval(MODEL_PATH) 59 | ner_eval.do_eval(data_files=["raw_data/kgClue/test_public.json"]) 60 | -------------------------------------------------------------------------------- /algorithm/kg_qa/NER/Predict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/16 10:42 4 | # @Author : 刘鑫 5 | # @FileName: seq_predict.py 6 | # @Software: PyCharm 7 | 8 | import os 9 | import tensorflow as tf 10 | 11 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 12 | 13 | from bert import tokenization 14 | from algorithm.kg_qa.NER.DataMaking import DataMaking 15 | from utils.IdAndLabel import id2label 16 | from algorithm.kg_qa.config import Properties, NerConfig as config 17 | # 预测类 18 | class Predict(object): 19 | 20 | def __init__(self, MODEL_PATH): 21 | self.model_path = MODEL_PATH 22 | 23 | self.seq_id2label = id2label(config.label_list) 24 | 25 | # 准备token 26 | self.tokenizer_ = tokenization.FullTokenizer(vocab_file=Properties.vocab_file) 27 | self.data_making = DataMaking(do_lower_case=True, max_seq_length=config.max_seq_length) 28 | self.sess = self.load_model() 29 | 30 | self.input_ids = self.sess.graph.get_tensor_by_name("input_ids:0") 31 | self.input_mask = self.sess.graph.get_tensor_by_name("input_mask:0") 32 | self.segment_ids = self.sess.graph.get_tensor_by_name("segment_ids:0") 33 | self.keep_prob = self.sess.graph.get_tensor_by_name("keep_prob:0") 34 | # 预测的结果 35 | self.p = self.sess.graph.get_tensor_by_name("loss/ReverseSequence_1:0") 36 | # x = [n.name for n in tf.get_default_graph().as_graph_def().node] 37 | # print(x) 38 | 39 | def load_model(self): 40 | try: 41 | checkpoint = tf.train.get_checkpoint_state(self.model_path) 42 | input_checkpoint = checkpoint.model_checkpoint_path 43 | print("[INFO] input_checkpoint:", input_checkpoint) 44 | except Exception as e: 45 | input_checkpoint = self.model_path 46 | print("[INFO] Model folder", self.model_path, repr(e)) 47 | 48 | # We clear devices to allow TensorFlow to control on which device it will load operations 49 | clear_devices = True 50 | tf.reset_default_graph() 51 | # We import the meta graph and retrieve a Saver 52 | saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices) 53 | 54 | # We start a session and restore the graph weights 55 | sess_ = tf.Session() 56 | saver.restore(sess_, input_checkpoint) 57 | 58 | return sess_ 59 | 60 | def predict_one(self, PREDICT_TXT, TEST_MODE=False): 61 | 62 | feature = self.data_making.convert_single_example(PREDICT_TXT, TEST_MODE=TEST_MODE) 63 | 64 | def id2label_f(id_list): 65 | predictions = [] 66 | if TEST_MODE: 67 | for id in id_list: 68 | predictions.append(self.seq_id2label[id]) 69 | else: 70 | for id in id_list: 71 | predictions.append(self.seq_id2label[id]) 72 | if id == 4: 73 | break 74 | return predictions 75 | 76 | feed = {self.input_ids: [feature[0]], 77 | self.input_mask: [feature[1]], 78 | self.segment_ids: [feature[2]], 79 | self.keep_prob: 1.0} 80 | 81 | probs = self.sess.run(self.p, feed) 82 | 83 | out = id2label_f(probs[0]) 84 | 85 | if TEST_MODE: 86 | return out 87 | else: 88 | return out[1:-1] 89 | 90 | 91 | if __name__ == '__main__': 92 | MODEL_PATH = config.model_out 93 | PREDICT_TXT = "5·13土耳其索玛矿难的坐标是什么？" 94 | 95 | ner = Predict(MODEL_PATH) 96 | print("test predict: ", ner.predict_one(PREDICT_TXT, TEST_MODE=True)) 97 | -------------------------------------------------------------------------------- /algorithm/kg_qa/NER/TrainAndValid.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/12 11:13 4 | # @Author : 刘鑫 5 | # @FileName: sequnce_labeling_train.py 6 | # @Software: PyCharm 7 | 8 | # 假定已经带着正确的分类label去训练实体识别 9 | import tensorflow as tf 10 | import os, math 11 | 12 | from bert import modeling 13 | from bert import optimization 14 | from algorithm.kg_qa.config import Properties, NerConfig as config 15 | from utils.IdAndLabel import id2label 16 | from utils.EvalReport import report 17 | 18 | 19 | def load_bert_config(path): 20 | """ 21 | bert 模型配置文件 22 | """ 23 | return modeling.BertConfig.from_json_file(path) 24 | 25 | 26 | def create_model(bert_config, is_training, input_ids, input_mask, segment_ids, labels, keep_prob, num_labels, 27 | use_one_hot_embeddings): 28 | """Creates a classification model.""" 29 | model = modeling.BertModel( 30 | config=bert_config, 31 | is_training=is_training, 32 | input_ids=input_ids, 33 | input_mask=input_mask, 34 | token_type_ids=segment_ids, 35 | use_one_hot_embeddings=use_one_hot_embeddings, 36 | scope='bert' 37 | ) 38 | output_layer = model.get_sequence_output() 39 | hidden_size = output_layer.shape[-1].value 40 | seq_length = output_layer.shape[-2].value 41 | 42 | output_weight = tf.get_variable( 43 | "output_weights", [num_labels, hidden_size], 44 | initializer=tf.truncated_normal_initializer(stddev=0.02) 45 | ) 46 | output_bias = tf.get_variable( 47 | "output_bias", [num_labels], initializer=tf.zeros_initializer() 48 | ) 49 | with tf.variable_scope("loss"): 50 | if is_training: 51 | output_layer = tf.nn.dropout(output_layer, keep_prob=0.9) 52 | 53 | output_layer = tf.reshape(output_layer, [-1, hidden_size]) 54 | logits = tf.matmul(output_layer, output_weight, transpose_b=True) 55 | logits = tf.reshape(logits, [-1, seq_length, num_labels]) 56 | 57 | logits = tf.nn.bias_add(logits, output_bias) 58 | logits = tf.reshape(logits, shape=(-1, seq_length, num_labels)) 59 | 60 | input_m = tf.count_nonzero(input_mask, -1) 61 | 62 | log_likelihood, transition_matrix = tf.contrib.crf.crf_log_likelihood(logits, labels, input_m) 63 | loss = tf.reduce_mean(-log_likelihood) 64 | # inference 65 | viterbi_sequence, viterbi_score = tf.contrib.crf.crf_decode(logits, transition_matrix, input_m) 66 | # 不计算 padding 的 acc 67 | equals = tf.reduce_sum( 68 | tf.cast(tf.equal(tf.cast(viterbi_sequence, tf.int64), labels), tf.float32) * tf.cast(input_mask, 69 | tf.float32)) 70 | acc = equals / tf.cast(tf.reduce_sum(input_mask), tf.float32) 71 | return (loss, acc, logits, viterbi_sequence) 72 | 73 | 74 | def get_input_data(input_file, seq_length, batch_size, is_training=True): 75 | def parser(record): 76 | name_to_features = { 77 | "input_ids": tf.FixedLenFeature([seq_length], tf.int64), 78 | "input_mask": tf.FixedLenFeature([seq_length], tf.int64), 79 | "segment_ids": tf.FixedLenFeature([seq_length], tf.int64), 80 | "label_ids": tf.FixedLenFeature([seq_length], tf.int64), 81 | } 82 | 83 | example = tf.parse_single_example(record, features=name_to_features) 84 | input_ids = example["input_ids"] 85 | input_mask = example["input_mask"] 86 | segment_ids = example["segment_ids"] 87 | labels = example["label_ids"] 88 | return input_ids, input_mask, segment_ids, labels 89 | 90 | dataset = tf.data.TFRecordDataset(input_file) 91 | # 数据类别集中，需要较大的buffer_size，才能有效打乱，或者再数据处理的过程中进行打乱 92 | if is_training: 93 | dataset = dataset.map(parser).batch(batch_size).shuffle(buffer_size=2000) 94 | else: 95 | dataset = dataset.map(parser).batch(batch_size) 96 | iterator = dataset.make_one_shot_iterator() 97 | input_ids, input_mask, segment_ids, labels = iterator.get_next() 98 | return input_ids, input_mask, segment_ids, labels 99 | 100 | 101 | def id2label_f(id_list): 102 | seq_id2label = id2label(config.label_list) 103 | predictions = [] 104 | for id in id_list: 105 | predictions.append(seq_id2label[id]) 106 | return predictions 107 | 108 | 109 | def pre2out(predicts): 110 | ''' 111 | 可以考虑移动到utils里面 112 | :param predicts: 113 | :return: 114 | ''' 115 | outs = [] 116 | for tmp in predicts: 117 | tmp_out = id2label_f(tmp.tolist()) 118 | outs.append(tmp_out) 119 | return outs 120 | 121 | 122 | def main(): 123 | print("print start load the params...") 124 | 125 | tf.gfile.MakeDirs(config.model_out) 126 | 127 | # 配置超参数 128 | train_examples_len = config.train_examples_len 129 | valid_examples_len = config.valid_examples_len 130 | learning_rate = config.learning_rate 131 | eval_per_step = config.eval_per_step 132 | num_labels = config.num_labels 133 | 134 | num_train_steps = math.ceil(train_examples_len / config.train_batch_size) 135 | num_valid_steps = math.ceil(valid_examples_len / config.valid_batch_size) 136 | num_warmup_steps = math.ceil(num_train_steps * config.num_train_epochs * config.warmup_proportion) 137 | print("num_train_steps:{}, num_valid_steps:{}, num_warmup_steps:{}".format(num_train_steps, num_valid_steps, 138 | num_warmup_steps)) 139 | 140 | use_one_hot_embeddings = False 141 | is_training = True 142 | seq_len = config.max_seq_length 143 | 144 | init_checkpoint = Properties.init_checkpoint 145 | print("print start compile the bert model...") 146 | 147 | # 定义输入输出 148 | input_ids = tf.placeholder(tf.int64, shape=[None, seq_len], name='input_ids') 149 | input_mask = tf.placeholder(tf.int64, shape=[None, seq_len], name='input_mask') 150 | segment_ids = tf.placeholder(tf.int64, shape=[None, seq_len], name='segment_ids') 151 | token_labels = tf.placeholder(tf.int64, shape=[None, seq_len], name='token_labels') 152 | keep_prob = tf.placeholder(tf.float32, name='keep_prob') # , name='is_training' 153 | bert_config_ = load_bert_config(Properties.bert_config) 154 | (total_loss, acc, logits, probabilities) = create_model(bert_config_, is_training, input_ids, 155 | input_mask, segment_ids, token_labels, 156 | keep_prob, 157 | num_labels, use_one_hot_embeddings) 158 | train_op = optimization.create_optimizer(total_loss, learning_rate, num_train_steps * config.num_train_epochs, 159 | num_warmup_steps, False) 160 | 161 | print("print start train the bert model...") 162 | 163 | batch_size = config.train_batch_size 164 | valid_batch_size = config.valid_batch_size 165 | 166 | init_global = tf.global_variables_initializer() 167 | 168 | saver = tf.train.Saver([v for v in tf.global_variables() if 'adam_v' not in v.name and 'adam_m' not in v.name], 169 | max_to_keep=3) # 保存最后top3模型 170 | 171 | with tf.Session() as sess: 172 | 173 | print("start load the pre train model") 174 | 175 | if init_checkpoint: 176 | tvars = tf.trainable_variables() 177 | print("trainable_variables", len(tvars)) 178 | (assignment_map, initialized_variable_names) = modeling.get_assignment_map_from_checkpoint(tvars, 179 | init_checkpoint) 180 | print("initialized_variable_names:", len(initialized_variable_names)) 181 | 182 | saver_ = tf.train.Saver([v for v in tvars if v.name in initialized_variable_names]) 183 | saver_.restore(sess, init_checkpoint) 184 | sess.run(init_global) 185 | else: 186 | sess.run(tf.global_variables_initializer()) 187 | print("********* train start *********") 188 | 189 | def train_step(ids, mask, segment, y, step, train_out_f): 190 | feed = {input_ids: ids, 191 | input_mask: mask, 192 | segment_ids: segment, 193 | token_labels: y, 194 | keep_prob: 0.9} 195 | _, out_loss, acc_, p_ = sess.run([train_op, total_loss, acc, probabilities], feed_dict=feed) 196 | print("step :{},loss :{}, acc :{}".format(step, out_loss, acc_)) 197 | train_out_f.write("step :{}, loss :{}, acc :{} \n".format(step, out_loss, acc_)) 198 | return out_loss, p_, y 199 | 200 | def valid_step(ids, mask, segment, y): 201 | # 验证训练效果 202 | feed = {input_ids: ids, 203 | input_mask: mask, 204 | segment_ids: segment, 205 | token_labels: y, 206 | keep_prob: 1.0 207 | } 208 | out_loss, acc_, p_ = sess.run([total_loss, acc, probabilities], feed_dict=feed) 209 | print("loss :{}, acc :{}".format(out_loss, acc_)) 210 | return out_loss, p_, y 211 | 212 | min_total_loss_dev = 999999 213 | step = 0 214 | if not os.path.exists(config.training_log): 215 | os.makedirs(config.training_log) 216 | for epoch in range(config.num_train_epochs): 217 | _ = "{:*^100s}".format(("epoch-" + str(epoch)).center(20)) 218 | print(_) 219 | # 读取训练数据 220 | total_loss_train = 0 221 | 222 | input_ids2, input_mask2, segment_ids2, labels2 = get_input_data(config.train_data, seq_len, batch_size) 223 | 224 | train_out_f = open(os.path.join(config.training_log, "epoch-" + str(epoch) + ".txt"), 'w', encoding='utf-8') 225 | 226 | for i in range(num_train_steps): 227 | step += 1 228 | ids_train, mask_train, segment_train, y_train = sess.run( 229 | [input_ids2, input_mask2, segment_ids2, labels2]) 230 | out_loss, pre, y = train_step(ids_train, mask_train, segment_train, y_train, step, train_out_f) 231 | total_loss_train += out_loss 232 | 233 | if step % eval_per_step == 0 and step >= config.eval_start_step: 234 | total_loss_dev = 0 235 | dev_input_ids2, dev_input_mask2, dev_segment_ids2, dev_labels2 = get_input_data(config.valid_data, 236 | seq_len, 237 | valid_batch_size, 238 | False) 239 | 240 | for j in range(num_valid_steps): # 一个 epoch 的轮数 241 | ids_dev, mask_dev, segment_dev, y_dev = sess.run( 242 | [dev_input_ids2, dev_input_mask2, dev_segment_ids2, dev_labels2]) 243 | out_loss, pre, y = valid_step(ids_dev, mask_dev, segment_dev, y_dev) 244 | total_loss_dev += out_loss 245 | # report a batch data in valid_data 246 | report(pre2out(y), pre2out(pre)) 247 | print("total_loss_dev:{}".format(total_loss_dev)) 248 | # print(classification_report(total_true_dev, total_pre_dev, digits=4)) 249 | 250 | if total_loss_dev < min_total_loss_dev: 251 | print("save model:\t%f\t>%f" % (min_total_loss_dev, total_loss_dev)) 252 | min_total_loss_dev = total_loss_dev 253 | saver.save(sess, config.model_out + 'bert.ckpt', global_step=step) 254 | elif step < config.eval_start_step and step % config.auto_save == 0: 255 | saver.save(sess, config.model_out + 'bert.ckpt', global_step=step) 256 | train_out_f.close() 257 | _ = "{:*^100s}".format(("epoch-" + str(epoch) + " report:").center(20)) 258 | print("total_loss_train:{}".format(total_loss_train)) 259 | # print(classification_report(total_true_train, total_pre_train, digits=4)) 260 | 261 | 262 | if __name__ == "__main__": 263 | print("********* ner start *********") 264 | tf.logging.set_verbosity(tf.logging.INFO) 265 | main() 266 | -------------------------------------------------------------------------------- /algorithm/kg_qa/NER/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/29 22:25 4 | # @Author : 刘鑫 5 | # @FileName: __init__.py.py 6 | # @Software: PyCharm 7 | -------------------------------------------------------------------------------- /algorithm/kg_qa/NER_BERT_LSTM_CRF/EntityExtract.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/27 14:56 4 | # @Author : 刘鑫 5 | # @FileName: entity_extract.py 6 | # @Software: PyCharm 7 | import json 8 | import os 9 | 10 | from bert import tokenization 11 | from algorithm.kg_qa.config import Properties, LstmCRFConfig as config 12 | from algorithm.kg_qa.NER_BERT_LSTM_CRF.Predict import Predict 13 | 14 | 15 | def isEntityInText(text, entity): 16 | text_list = list(text) 17 | entity_list = list(entity) 18 | M = len(text_list) 19 | N = len(entity_list) 20 | 21 | i = 0 22 | while i <= M - N: 23 | j = 0 24 | space_nums = 0 25 | match = [] 26 | while j < N: 27 | tt = text_list[i + j].lower() 28 | ee = entity_list[j].lower() 29 | if tt == ee: 30 | j += 1 31 | match.append(ee) 32 | continue 33 | elif j > 0 and tt == " ": 34 | i += 1 35 | space_nums += 1 36 | else: 37 | j += 1 38 | break 39 | if j == N and entity_list == match: 40 | return i - space_nums, space_nums 41 | i += 1 42 | return -1, -1 43 | 44 | 45 | class EntityExtract(object): 46 | def __init__(self, MODEL_PATH): 47 | self.predict = Predict(MODEL_PATH) 48 | self.tokenizer_ = tokenization.FullTokenizer(vocab_file=Properties.vocab_file) 49 | 50 | def extract(self, sentence): 51 | ''' 52 | 将预测的token_label对应到句子中的字，抽取出中文实体，可能存在问题：空格缺失、大小写不一致，不过这里采用的策略是原样输出。在计算f1和em时这两点问题可以忽略 53 | :param sentence: 54 | :return: entitys 55 | ''' 56 | 57 | def _merge_WordPiece_and_single_word(entity_sort_list): 58 | entity_sort_tuple_list = [] 59 | for a_entity_list in entity_sort_list: 60 | entity_content = "" 61 | entity_type = None 62 | for idx, entity_part in enumerate(a_entity_list): 63 | if idx == 0: 64 | entity_type = entity_part 65 | if entity_type[:2] not in ["B-", "I-"]: 66 | break 67 | else: 68 | if entity_part.startswith("##"): 69 | entity_content += entity_part.replace("##", "") 70 | else: 71 | entity_content += entity_part 72 | if entity_content != "": 73 | entity_sort_tuple_list.append((entity_type[2:], entity_content)) 74 | return entity_sort_tuple_list 75 | 76 | ner_out = self.predict.predict_one(sentence, TEST_MODE=True) 77 | 78 | def preprocessing_model_token_lable(predicate_token_label_list, token_in_list_lenth): 79 | if predicate_token_label_list[0] == "[CLS]": 80 | predicate_token_label_list = predicate_token_label_list[1:] # y_predict.remove('[CLS]') 81 | if len(predicate_token_label_list) > token_in_list_lenth: # 只取输入序列长度即可 82 | predicate_token_label_list = predicate_token_label_list[:token_in_list_lenth] 83 | return predicate_token_label_list 84 | 85 | predicate_token_label_list = preprocessing_model_token_lable(ner_out, len(ner_out)) 86 | 87 | entity_sort_list = [] 88 | entity_part_list = [] 89 | 90 | token_in_not_UNK = self.tokenizer_.tokenize_not_UNK(sentence) 91 | for idx, token_label in enumerate(predicate_token_label_list): 92 | if token_label == "O": 93 | if len(entity_part_list) > 0: 94 | entity_sort_list.append(entity_part_list) 95 | entity_part_list = [] 96 | if token_label.startswith("B-"): 97 | if len(entity_part_list) > 0: # 适用于 B- B- *****的情况 98 | entity_sort_list.append(entity_part_list) 99 | entity_part_list = [] 100 | entity_part_list.append(token_label) 101 | entity_part_list.append(token_in_not_UNK[idx]) 102 | if idx == len(predicate_token_label_list) - 1: 103 | entity_sort_list.append(entity_part_list) 104 | if token_label.startswith("I-") or token_label == "[##WordPiece]": 105 | if len(entity_part_list) > 0: 106 | entity_part_list.append(token_in_not_UNK[idx]) 107 | if idx == len(predicate_token_label_list) - 1: 108 | entity_sort_list.append(entity_part_list) 109 | if token_label == "[SEP]": 110 | break 111 | 112 | entity_sort_tuple_list = _merge_WordPiece_and_single_word(entity_sort_list) 113 | entitys = [] 114 | for entity in entity_sort_tuple_list: 115 | if entity[0] == "NP": 116 | start, space_nums = isEntityInText(sentence, entity[1]) 117 | if start != -1: 118 | end = len(entity[1]) + space_nums + start 119 | entitys.append(sentence[start:end]) 120 | return entitys 121 | 122 | 123 | if __name__ == '__main__': 124 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" 125 | MODEL_PATH = config.model_out 126 | ee = EntityExtract(MODEL_PATH) 127 | ff = open("./out.txt", 'w', encoding='utf-8') 128 | true_count=0 129 | with open(r"C:\Users\11943\Documents\GitHub\KgClue_Bench\raw_data\kgClue\test_public.json", "r", 130 | encoding='utf-8') as f: 131 | while True: 132 | line = f.readline() 133 | if line: 134 | line = json.loads(line) 135 | text = line["question"] 136 | answer = line["answer"] 137 | entity = answer.split("|||")[0].split("（")[0] 138 | p_entity = ee.extract(text) 139 | print(entity,"".join(p_entity)) 140 | if entity == "".join(p_entity): 141 | true_count+=1 142 | ff.write(text + "\t" + entity + "\t" + "".join(p_entity) + "\n") 143 | else: 144 | break 145 | ff.close() 146 | print(true_count/2000) 147 | -------------------------------------------------------------------------------- /algorithm/kg_qa/NER_BERT_LSTM_CRF/Eval.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/20 10:54 4 | # @Author : 刘鑫 5 | # @FileName: seq_eval.py 6 | # @Software: PyCharm 7 | import json 8 | import os 9 | 10 | from algorithm.kg_qa.NER_BERT_LSTM_CRF.Predict import Predict 11 | from algorithm.kg_qa.NER.DataMaking import DataMaking 12 | from algorithm.kg_qa.config import LstmCRFConfig as config 13 | from utils.EvalReport import report 14 | 15 | 16 | class Eval(object): 17 | def __init__(self, MODEL_PATH): 18 | self.data_make = DataMaking(do_lower_case=True, max_seq_length=config.max_seq_length) 19 | self.predict = Predict(MODEL_PATH) 20 | self.seq_id2label = self.predict.seq_id2label 21 | 22 | def id2label_f(self, id_list): 23 | predictions = [] 24 | for id in id_list: 25 | predictions.append(self.seq_id2label[id]) 26 | return predictions 27 | 28 | def do_eval(self, data_files=["../raw_data/test.json"]): 29 | 30 | for data_file in data_files: 31 | y_true = [] 32 | y_pred = [] 33 | with open(data_file, 'r', encoding='utf-8') as f: 34 | while True: 35 | line = f.readline() 36 | if line: 37 | line = json.loads(line) 38 | text = line["question"] 39 | answer = line["answer"] 40 | entity = answer.split("|||")[0].split("（")[0] 41 | 42 | token_label = self.data_make.text2label(text, entity) 43 | feature = self.data_make.convert_single_example(text, token_label) 44 | label = self.id2label_f(feature.label_ids) 45 | predict_label = self.predict.predict_one(text, TEST_MODE=True) 46 | 47 | y_true.append(label) 48 | y_pred.append(predict_label) 49 | 50 | else: 51 | break 52 | report(y_true, y_pred) 53 | 54 | 55 | if __name__ == '__main__': 56 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 57 | MODEL_PATH = config.model_out 58 | ner_eval = Eval(MODEL_PATH) 59 | ner_eval.do_eval(data_files=["raw_data/kgClue/test_public.json"]) 60 | -------------------------------------------------------------------------------- /algorithm/kg_qa/NER_BERT_LSTM_CRF/Predict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/16 10:42 4 | # @Author : 刘鑫 5 | # @FileName: seq_predict.py 6 | # @Software: PyCharm 7 | 8 | import os 9 | import sys 10 | 11 | import tensorflow as tf 12 | 13 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 14 | 15 | from bert import tokenization 16 | from algorithm.kg_qa.NER.DataMaking import DataMaking 17 | from utils.IdAndLabel import id2label 18 | from algorithm.kg_qa.config import Properties, LstmCRFConfig as config 19 | 20 | 21 | # 预测类 22 | class Predict(object): 23 | 24 | def __init__(self, MODEL_PATH): 25 | self.model_path = MODEL_PATH 26 | 27 | self.seq_id2label = id2label(config.label_list) 28 | 29 | # 准备token 30 | self.tokenizer_ = tokenization.FullTokenizer(vocab_file=Properties.vocab_file) 31 | self.data_making = DataMaking(do_lower_case=True, max_seq_length=config.max_seq_length) 32 | self.sess = self.load_model() 33 | 34 | self.input_ids = self.sess.graph.get_tensor_by_name("input_ids:0") 35 | self.input_mask = self.sess.graph.get_tensor_by_name("input_mask:0") 36 | self.segment_ids = self.sess.graph.get_tensor_by_name("segment_ids:0") 37 | self.keep_prob = self.sess.graph.get_tensor_by_name("keep_prob:0") 38 | # 预测的结果 39 | self.p = self.sess.graph.get_tensor_by_name("ReverseSequence_1:0") 40 | 41 | # x =[n.name for n in tf.get_default_graph().as_graph_def().node] 42 | # print(x) 43 | 44 | def load_model(self): 45 | try: 46 | checkpoint = tf.train.get_checkpoint_state(self.model_path) 47 | input_checkpoint = checkpoint.model_checkpoint_path 48 | print("[INFO] input_checkpoint:", input_checkpoint) 49 | except Exception as e: 50 | input_checkpoint = self.model_path 51 | print("[INFO] Model folder", self.model_path, repr(e)) 52 | 53 | # We clear devices to allow TensorFlow to control on which device it will load operations 54 | clear_devices = True 55 | tf.reset_default_graph() 56 | # We import the meta graph and retrieve a Saver 57 | saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices) 58 | 59 | # We start a session and restore the graph weights 60 | sess_ = tf.Session() 61 | saver.restore(sess_, input_checkpoint) 62 | 63 | return sess_ 64 | 65 | def predict_one(self, PREDICT_TXT, TEST_MODE=False): 66 | 67 | feature = self.data_making.convert_single_example(PREDICT_TXT, TEST_MODE=TEST_MODE) 68 | 69 | def id2label_f(id_list): 70 | predictions = [] 71 | if TEST_MODE: 72 | for id in id_list: 73 | predictions.append(self.seq_id2label[id]) 74 | else: 75 | for id in id_list: 76 | predictions.append(self.seq_id2label[id]) 77 | if id == 4: 78 | break 79 | return predictions 80 | 81 | feed = {self.input_ids: [feature[0]], 82 | self.input_mask: [feature[1]], 83 | self.segment_ids: [feature[2]], 84 | self.keep_prob: 1.0} 85 | 86 | probs = self.sess.run(self.p, feed) 87 | 88 | out = id2label_f(probs[0]) 89 | 90 | if TEST_MODE: 91 | return out 92 | else: 93 | return out[1:-1] 94 | 95 | 96 | if __name__ == '__main__': 97 | MODEL_PATH = config.model_out 98 | PREDICT_TXT = "5·13土耳其索玛矿难的坐标是什么？" 99 | 100 | ner = Predict(MODEL_PATH) 101 | print("test predict: ", ner.predict_one(PREDICT_TXT, TEST_MODE=True)) 102 | -------------------------------------------------------------------------------- /algorithm/kg_qa/NER_BERT_LSTM_CRF/TrainAndValid.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/12 11:13 4 | # @Author : 刘鑫 5 | # @FileName: sequnce_labeling_train.py 6 | # @Software: PyCharm 7 | 8 | # 假定已经带着正确的分类label去训练实体识别 9 | import sys 10 | 11 | import tensorflow as tf 12 | import os, math 13 | from seqeval.metrics import accuracy_score, f1_score 14 | from bert import modeling 15 | from bert import optimization 16 | from tensorflow.contrib.layers.python.layers import initializers 17 | from algorithm.kg_qa.config import Properties, LstmCRFConfig as config 18 | from algorithm.kg_qa.NER_BERT_LSTM_CRF.lstm_crf_layer import BLSTM_CRF 19 | from utils.IdAndLabel import id2label 20 | from utils.EvalReport import report 21 | 22 | 23 | def load_bert_config(path): 24 | """ 25 | bert 模型配置文件 26 | """ 27 | return modeling.BertConfig.from_json_file(path) 28 | 29 | 30 | def create_model(bert_config, is_training, input_ids, input_mask, segment_ids, labels, keep_prob, num_labels, 31 | use_one_hot_embeddings): 32 | """Creates a classification model.""" 33 | model = modeling.BertModel( 34 | config=bert_config, 35 | is_training=is_training, 36 | input_ids=input_ids, 37 | input_mask=input_mask, 38 | token_type_ids=segment_ids, 39 | use_one_hot_embeddings=use_one_hot_embeddings, 40 | scope='bert' 41 | ) 42 | embedding = model.get_sequence_output() 43 | max_seq_length = embedding.shape[1].value 44 | 45 | used = tf.sign(tf.abs(input_ids)) 46 | lengths = tf.reduce_sum(used, reduction_indices=1) # [batch_size] 大小的向量，包含了当前batch中的序列长度 47 | 48 | blstm_crf = BLSTM_CRF(embedded_chars=embedding, hidden_unit=config.lstm_size, cell_type=config.cell, 49 | num_layers=config.num_layers, 50 | dropout_rate=config.droupout_rate, initializers=initializers, num_labels=num_labels, 51 | seq_length=max_seq_length, labels=labels, lengths=lengths, is_training=is_training) 52 | (total_loss, logits, trans, pred_ids) = blstm_crf.add_blstm_crf_layer(crf_only=False) 53 | 54 | return (total_loss, logits, trans, pred_ids) 55 | 56 | 57 | def get_input_data(input_file, seq_length, batch_size, is_training=True): 58 | def parser(record): 59 | name_to_features = { 60 | "input_ids": tf.FixedLenFeature([seq_length], tf.int64), 61 | "input_mask": tf.FixedLenFeature([seq_length], tf.int64), 62 | "segment_ids": tf.FixedLenFeature([seq_length], tf.int64), 63 | "label_ids": tf.FixedLenFeature([seq_length], tf.int64), 64 | } 65 | 66 | example = tf.parse_single_example(record, features=name_to_features) 67 | input_ids = example["input_ids"] 68 | input_mask = example["input_mask"] 69 | segment_ids = example["segment_ids"] 70 | labels = example["label_ids"] 71 | return input_ids, input_mask, segment_ids, labels 72 | 73 | dataset = tf.data.TFRecordDataset(input_file) 74 | # 数据类别集中，需要较大的buffer_size，才能有效打乱，或者再数据处理的过程中进行打乱 75 | if is_training: 76 | dataset = dataset.map(parser).batch(batch_size).shuffle(buffer_size=2000) 77 | else: 78 | dataset = dataset.map(parser).batch(batch_size) 79 | iterator = dataset.make_one_shot_iterator() 80 | input_ids, input_mask, segment_ids, labels = iterator.get_next() 81 | return input_ids, input_mask, segment_ids, labels 82 | 83 | 84 | def id2label_f(id_list): 85 | seq_id2label = id2label(config.label_list) 86 | predictions = [] 87 | for id in id_list: 88 | predictions.append(seq_id2label[id]) 89 | return predictions 90 | 91 | 92 | def pre2out(predicts): 93 | ''' 94 | 可以考虑移动到utils里面 95 | :param predicts: 96 | :return: 97 | ''' 98 | outs = [] 99 | for tmp in predicts: 100 | tmp_out = id2label_f(tmp.tolist()) 101 | outs.append(tmp_out) 102 | return outs 103 | 104 | 105 | def main(): 106 | print("print start load the params...") 107 | 108 | tf.gfile.MakeDirs(config.model_out) 109 | 110 | # 配置超参数 111 | train_examples_len = config.train_examples_len 112 | valid_examples_len = config.valid_examples_len 113 | learning_rate = config.learning_rate 114 | eval_per_step = config.eval_per_step 115 | num_labels = config.num_labels 116 | 117 | num_train_steps = math.ceil(train_examples_len / config.train_batch_size) 118 | num_valid_steps = math.ceil(valid_examples_len / config.valid_batch_size) 119 | num_warmup_steps = math.ceil(num_train_steps * config.num_train_epochs * config.warmup_proportion) 120 | print("num_train_steps:{}, num_valid_steps:{}, num_warmup_steps:{}".format(num_train_steps, num_valid_steps, 121 | num_warmup_steps)) 122 | 123 | use_one_hot_embeddings = False 124 | is_training = True 125 | seq_len = config.max_seq_length 126 | 127 | init_checkpoint = Properties.init_checkpoint 128 | print("print start compile the bert model...") 129 | 130 | # 定义输入输出 131 | input_ids = tf.placeholder(tf.int64, shape=[None, seq_len], name='input_ids') 132 | input_mask = tf.placeholder(tf.int64, shape=[None, seq_len], name='input_mask') 133 | segment_ids = tf.placeholder(tf.int64, shape=[None, seq_len], name='segment_ids') 134 | token_labels = tf.placeholder(tf.int64, shape=[None, seq_len], name='token_labels') 135 | keep_prob = tf.placeholder(tf.float32, name='keep_prob') # , name='is_training' 136 | bert_config_ = load_bert_config(Properties.bert_config) 137 | # total_loss, logits, trans, pred_ids 138 | (total_loss, logits, trans, pred_ids) = create_model(bert_config_, is_training, input_ids, 139 | input_mask, segment_ids, token_labels, 140 | keep_prob, 141 | num_labels, use_one_hot_embeddings) 142 | 143 | train_op = optimization.create_optimizer(total_loss, learning_rate, num_train_steps * config.num_train_epochs, 144 | num_warmup_steps, False) 145 | 146 | print("print start train the bert model...") 147 | 148 | batch_size = config.train_batch_size 149 | valid_batch_size = config.valid_batch_size 150 | 151 | init_global = tf.global_variables_initializer() 152 | 153 | saver = tf.train.Saver([v for v in tf.global_variables() if 'adam_v' not in v.name and 'adam_m' not in v.name], 154 | max_to_keep=3) # 保存最后top3模型 155 | 156 | with tf.Session() as sess: 157 | 158 | print("start load the pre train model") 159 | 160 | if init_checkpoint: 161 | tvars = tf.trainable_variables() 162 | print("trainable_variables", len(tvars)) 163 | (assignment_map, initialized_variable_names) = modeling.get_assignment_map_from_checkpoint(tvars, 164 | init_checkpoint) 165 | print("initialized_variable_names:", len(initialized_variable_names)) 166 | 167 | saver_ = tf.train.Saver([v for v in tvars if v.name in initialized_variable_names]) 168 | saver_.restore(sess, init_checkpoint) 169 | sess.run(init_global) 170 | else: 171 | sess.run(tf.global_variables_initializer()) 172 | print("********* train start *********") 173 | 174 | def train_step(ids, mask, segment, y, step, train_out_f): 175 | feed = {input_ids: ids, 176 | input_mask: mask, 177 | segment_ids: segment, 178 | token_labels: y, 179 | keep_prob: 0.9} 180 | _, out_loss, pred_ids_ = sess.run([train_op, total_loss, pred_ids], feed_dict=feed) 181 | acc = accuracy_score(pre2out(y), pre2out(pred_ids_)) 182 | f1 = f1_score(pre2out(y), pre2out(pred_ids_)) 183 | print("step :{},loss :{}, acc :{}, f1 :{}".format(step, out_loss, acc, f1)) 184 | train_out_f.write("step :{}, loss :{}, acc :{} , f1 :{} \n".format(step, out_loss, acc, f1)) 185 | return out_loss, pred_ids_, y 186 | 187 | def valid_step(ids, mask, segment, y): 188 | # 验证训练效果 189 | feed = {input_ids: ids, 190 | input_mask: mask, 191 | segment_ids: segment, 192 | token_labels: y, 193 | keep_prob: 1.0 194 | } 195 | 196 | out_loss, p_ = sess.run([total_loss, pred_ids], feed_dict=feed) 197 | # f1 = f1_score(pre2out(y), pre2out(p_)) 198 | # acc = accuracy_score(pre2out(y), pre2out(p_)) 199 | # print("loss :{}, acc :{}, f1:{}".format(out_loss, acc, f1)) 没有必要一个批次算一次 200 | return out_loss, p_, y 201 | 202 | min_total_loss_dev = 999999 203 | step = 0 204 | if not os.path.exists(config.training_log): 205 | os.makedirs(config.training_log) 206 | for epoch in range(config.num_train_epochs): 207 | _ = "{:*^100s}".format(("epoch-" + str(epoch)).center(20)) 208 | print(_) 209 | # 读取训练数据 210 | total_loss_train = 0 211 | 212 | input_ids2, input_mask2, segment_ids2, labels2 = get_input_data(config.train_data, seq_len, batch_size) 213 | train_out_f = open(os.path.join(config.training_log, "epoch-" + str(epoch) + ".txt"), 'w', encoding='utf-8') 214 | 215 | for i in range(num_train_steps): 216 | step += 1 217 | ids_train, mask_train, segment_train, y_train = sess.run( 218 | [input_ids2, input_mask2, segment_ids2, labels2]) 219 | out_loss, pre, y = train_step(ids_train, mask_train, segment_train, y_train, step, train_out_f) 220 | total_loss_train += out_loss 221 | 222 | if step % eval_per_step == 0 and step >= config.eval_start_step: 223 | total_loss_dev = 0 224 | dev_input_ids2, dev_input_mask2, dev_segment_ids2, dev_labels2 = get_input_data(config.valid_data, 225 | seq_len, 226 | valid_batch_size, 227 | False) 228 | pre_list = [] 229 | y_list = [] 230 | for j in range(num_valid_steps): # 一个 epoch 的轮数 231 | ids_dev, mask_dev, segment_dev, y_dev = sess.run( 232 | [dev_input_ids2, dev_input_mask2, dev_segment_ids2, dev_labels2]) 233 | out_loss, pre, y = valid_step(ids_dev, mask_dev, segment_dev, y_dev) 234 | total_loss_dev += out_loss 235 | pre_list.extend(pre) 236 | y_list.extend(y) 237 | print("total_loss_dev:{}".format(total_loss_dev)) 238 | report(pre2out(y_list), pre2out(pre_list)) 239 | 240 | if total_loss_dev < min_total_loss_dev: 241 | print("save model:\t%f\t>%f" % (min_total_loss_dev, total_loss_dev)) 242 | min_total_loss_dev = total_loss_dev 243 | saver.save(sess, config.model_out + 'bert.ckpt', global_step=step) 244 | elif step < config.eval_start_step and step % config.auto_save == 0: 245 | saver.save(sess, config.model_out + 'bert.ckpt', global_step=step) 246 | train_out_f.close() 247 | _ = "{:*^100s}".format(("epoch-" + str(epoch) + " report:").center(20)) 248 | print("total_loss_train:{}".format(total_loss_train)) 249 | # print(classification_report(total_true_train, total_pre_train, digits=4)) 250 | 251 | 252 | if __name__ == "__main__": 253 | print("********* ner start *********") 254 | tf.logging.set_verbosity(tf.logging.INFO) 255 | main() 256 | -------------------------------------------------------------------------------- /algorithm/kg_qa/NER_BERT_LSTM_CRF/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/29 22:25 4 | # @Author : 刘鑫 5 | # @FileName: __init__.py.py 6 | # @Software: PyCharm 7 | -------------------------------------------------------------------------------- /algorithm/kg_qa/NER_BERT_LSTM_CRF/lstm_crf_layer.py: -------------------------------------------------------------------------------- 1 | # encoding=utf-8 2 | 3 | """ 4 | bert-blstm-crf layer 5 | @Author:Macan 6 | """ 7 | 8 | import tensorflow as tf 9 | from tensorflow.contrib import rnn 10 | from tensorflow.contrib import crf 11 | 12 | 13 | class BLSTM_CRF(object): 14 | def __init__(self, embedded_chars, hidden_unit, cell_type, num_layers, dropout_rate, 15 | initializers, num_labels, seq_length, labels, lengths, is_training): 16 | """ 17 | BLSTM-CRF 网络 18 | :param embedded_chars: Fine-tuning embedding input 19 | :param hidden_unit: LSTM的隐含单元个数 20 | :param cell_type: RNN类型（LSTM OR GRU DICNN will be add in feature） 21 | :param num_layers: RNN的层数 22 | :param droupout_rate: droupout rate 23 | :param initializers: variable init class 24 | :param num_labels: 标签数量 25 | :param seq_length: 序列最大长度 26 | :param labels: 真实标签 27 | :param lengths: [batch_size] 每个batch下序列的真实长度 28 | :param is_training: 是否是训练过程 29 | """ 30 | self.hidden_unit = hidden_unit 31 | self.dropout_rate = dropout_rate 32 | self.cell_type = cell_type 33 | self.num_layers = num_layers 34 | self.embedded_chars = embedded_chars 35 | self.initializers = initializers 36 | self.seq_length = seq_length 37 | self.num_labels = num_labels 38 | self.labels = labels 39 | self.lengths = lengths 40 | self.embedding_dims = embedded_chars.shape[-1].value 41 | self.is_training = is_training 42 | 43 | def add_blstm_crf_layer(self, crf_only): 44 | """ 45 | blstm-crf网络 46 | :return: 47 | """ 48 | if self.is_training: 49 | # lstm input dropout rate i set 0.9 will get best score 50 | self.embedded_chars = tf.nn.dropout(self.embedded_chars, self.dropout_rate) 51 | 52 | if crf_only: 53 | logits = self.project_crf_layer(self.embedded_chars) 54 | else: 55 | # blstm 56 | lstm_output = self.blstm_layer(self.embedded_chars) 57 | # project 58 | logits = self.project_bilstm_layer(lstm_output) 59 | # crf 60 | loss, trans = self.crf_layer(logits) 61 | # CRF decode, pred_ids 是一条最大概率的标注路径 62 | pred_ids, _ = crf.crf_decode(potentials=logits, transition_params=trans, sequence_length=self.lengths) 63 | 64 | return ((loss, logits, trans, pred_ids)) 65 | 66 | def _witch_cell(self): 67 | """ 68 | RNN 类型 69 | :return: 70 | """ 71 | cell_tmp = None 72 | if self.cell_type == 'lstm': 73 | cell_tmp = rnn.BasicLSTMCell(self.hidden_unit) 74 | elif self.cell_type == 'gru': 75 | cell_tmp = rnn.GRUCell(self.hidden_unit) 76 | # 是否需要进行dropout 77 | if self.dropout_rate is not None: 78 | cell_tmp = rnn.DropoutWrapper(cell_tmp, output_keep_prob=self.dropout_rate) 79 | return cell_tmp 80 | 81 | def _bi_dir_rnn(self): 82 | """ 83 | 双向RNN 84 | :return: 85 | """ 86 | cell_fw = self._witch_cell() 87 | cell_bw = self._witch_cell() 88 | return cell_fw, cell_bw 89 | 90 | def blstm_layer(self, embedding_chars): 91 | """ 92 | 93 | :return: 94 | """ 95 | with tf.variable_scope('rnn_layer'): 96 | cell_fw, cell_bw = self._bi_dir_rnn() 97 | if self.num_layers > 1: 98 | cell_fw = rnn.MultiRNNCell([cell_fw] * self.num_layers, state_is_tuple=True) 99 | cell_bw = rnn.MultiRNNCell([cell_bw] * self.num_layers, state_is_tuple=True) 100 | 101 | outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, embedding_chars, 102 | dtype=tf.float32) 103 | outputs = tf.concat(outputs, axis=2) 104 | return outputs 105 | 106 | def project_bilstm_layer(self, lstm_outputs, name=None): 107 | """ 108 | hidden layer between lstm layer and logits 109 | :param lstm_outputs: [batch_size, num_steps, emb_size] 110 | :return: [batch_size, num_steps, num_tags] 111 | """ 112 | with tf.variable_scope("project" if not name else name): 113 | with tf.variable_scope("hidden"): 114 | W = tf.get_variable("W", shape=[self.hidden_unit * 2, self.hidden_unit], 115 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 116 | 117 | b = tf.get_variable("b", shape=[self.hidden_unit], dtype=tf.float32, 118 | initializer=tf.zeros_initializer()) 119 | output = tf.reshape(lstm_outputs, shape=[-1, self.hidden_unit * 2]) 120 | hidden = tf.tanh(tf.nn.xw_plus_b(output, W, b)) 121 | 122 | # project to score of tags 123 | with tf.variable_scope("logits"): 124 | W = tf.get_variable("W", shape=[self.hidden_unit, self.num_labels], 125 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 126 | 127 | b = tf.get_variable("b", shape=[self.num_labels], dtype=tf.float32, 128 | initializer=tf.zeros_initializer()) 129 | 130 | pred = tf.nn.xw_plus_b(hidden, W, b) 131 | return tf.reshape(pred, [-1, self.seq_length, self.num_labels]) 132 | 133 | def project_crf_layer(self, embedding_chars, name=None): 134 | """ 135 | hidden layer between input layer and logits 136 | :param lstm_outputs: [batch_size, num_steps, emb_size] 137 | :return: [batch_size, num_steps, num_tags] 138 | """ 139 | with tf.variable_scope("project" if not name else name): 140 | with tf.variable_scope("logits"): 141 | W = tf.get_variable("W", shape=[self.embedding_dims, self.num_labels], 142 | dtype=tf.float32, initializer=self.initializers.xavier_initializer()) 143 | 144 | b = tf.get_variable("b", shape=[self.num_labels], dtype=tf.float32, 145 | initializer=tf.zeros_initializer()) 146 | output = tf.reshape(self.embedded_chars, 147 | shape=[-1, self.embedding_dims]) # [batch_size, embedding_dims] 148 | pred = tf.tanh(tf.nn.xw_plus_b(output, W, b)) 149 | return tf.reshape(pred, [-1, self.seq_length, self.num_labels]) 150 | 151 | def crf_layer(self, logits): 152 | """ 153 | calculate crf loss 154 | :param project_logits: [1, num_steps, num_tags] 155 | :return: scalar loss 156 | """ 157 | with tf.variable_scope("crf_loss"): 158 | trans = tf.get_variable( 159 | "transitions", 160 | shape=[self.num_labels, self.num_labels], 161 | initializer=self.initializers.xavier_initializer()) 162 | log_likelihood, trans = tf.contrib.crf.crf_log_likelihood( 163 | inputs=logits, 164 | tag_indices=self.labels, 165 | transition_params=trans, 166 | sequence_lengths=self.lengths) 167 | return tf.reduce_mean(-log_likelihood), trans -------------------------------------------------------------------------------- /algorithm/kg_qa/SIM/DataMaking.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/23 14:33 4 | # @Author : 刘鑫 5 | # @FileName: ner_data_making.py 6 | # @Software: PyCharm 7 | import collections 8 | import json 9 | import os 10 | import tensorflow as tf 11 | from elasticsearch import Elasticsearch 12 | import random 13 | 14 | random.seed(1024) 15 | 16 | from bert import tokenization 17 | from algorithm.kg_qa.config import Properties, SimConfig as config 18 | 19 | 20 | def getAttribute(Attributes, attribute): 21 | index = len(Attributes) - 2 22 | flag = False 23 | if attribute in Attributes: 24 | Attributes.remove(attribute) 25 | flag = True 26 | 27 | x = random.randint(0, index) 28 | out = Attributes[x] 29 | 30 | if flag: 31 | Attributes.append(attribute) 32 | 33 | return out 34 | 35 | 36 | class DataMaking(object): 37 | def __init__(self, fake_example_nums=5, es_host="127.0.0.1", es_port="9200", do_lower_case=True, 38 | max_seq_length=128): 39 | self.task_name = "SIM" 40 | self.bert_tokenizer = tokenization.FullTokenizer(vocab_file=Properties.vocab_file, 41 | do_lower_case=do_lower_case) # 初始化 bert_token 工具 42 | self.fake_example_nums = fake_example_nums 43 | self.max_seq_length = max_seq_length 44 | 45 | self.es = Elasticsearch([":".join((es_host, es_port))]) 46 | # self.Attributes = list(set(self.getAttribute())) # 更换数据集时重新运行打印出来之后放到config里面 47 | 48 | def makdir(self, OUTPUT_DIR, DIR_NAME): 49 | if not os.path.exists(os.path.join(os.path.dirname(__file__), OUTPUT_DIR)): 50 | os.makedirs(os.path.join(os.path.dirname(__file__), OUTPUT_DIR)) 51 | if not os.path.exists(os.path.join(os.path.dirname(__file__), OUTPUT_DIR, DIR_NAME)): 52 | os.makedirs(os.path.join(os.path.dirname(__file__), OUTPUT_DIR, DIR_NAME)) 53 | 54 | def convert_single_example(self, text, attribute, label=0, test_mode=False): 55 | 56 | def _truncate_seq_pair(tokens_a, tokens_b, max_length): 57 | """Truncates a sequence pair in place to the maximum length.""" 58 | while True: 59 | total_length = len(tokens_a) + len(tokens_b) 60 | if total_length <= max_length: 61 | break 62 | if len(tokens_a) > len(tokens_b): 63 | tokens_a.pop() 64 | else: 65 | tokens_b.pop() 66 | 67 | text_token = self.bert_tokenizer.tokenize(text) 68 | attribute_tokens = self.bert_tokenizer.tokenize(attribute) 69 | # print("att", len(attribute), attribute) 调试 70 | tokens_b = self.bert_tokenizer.convert_tokens_to_ids(attribute_tokens) * (len(text_token) // len(attribute)) 71 | if int(label) == 1: 72 | label_ids = [0, 1] 73 | else: 74 | label_ids = [1, 0] 75 | 76 | _truncate_seq_pair(text_token, tokens_b, self.max_seq_length - 3) # 很重要 77 | # 78 | tokens = [] 79 | segment_ids = [] 80 | # 添加起始位置 81 | tokens.append("[CLS]") 82 | segment_ids.append(0) 83 | 84 | for token in text_token: 85 | tokens.append(token) 86 | segment_ids.append(0) 87 | 88 | tokens.append("[SEP]") 89 | segment_ids.append(0) 90 | 91 | input_ids = self.bert_tokenizer.convert_tokens_to_ids(tokens) 92 | 93 | for token in tokens_b: 94 | input_ids.append(token) 95 | segment_ids.append(1) 96 | 97 | input_ids.append(self.bert_tokenizer.convert_tokens_to_ids(["[SEP]"])[0]) # 102 98 | segment_ids.append(1) 99 | input_mask = [1] * len(input_ids) 100 | # Zero-pad up to the sequence length. 101 | while len(input_ids) < self.max_seq_length: 102 | input_ids.append(0) 103 | input_mask.append(0) 104 | segment_ids.append(0) 105 | tokens.append("[Padding]") 106 | 107 | assert len(input_ids) == self.max_seq_length 108 | assert len(input_mask) == self.max_seq_length 109 | assert len(segment_ids) == self.max_seq_length 110 | 111 | if test_mode: 112 | feature = (input_ids, input_mask, segment_ids) 113 | else: 114 | feature = config.InputFeatures( 115 | input_ids=input_ids, 116 | input_mask=input_mask, 117 | segment_ids=segment_ids, 118 | label_ids=label_ids, 119 | is_real_example=True) 120 | return feature 121 | 122 | def write2txt(self, text, attribute, label, token_label_f, tf_writer): 123 | # 写入txt 124 | example = text + "\t" + str(attribute) + "\t" + str(label) 125 | token_label_f.write(example + "\n") 126 | 127 | # 写人tf_record 128 | feature = self.convert_single_example(text, attribute, label) 129 | 130 | def create_int_feature(values): 131 | f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) 132 | return f 133 | 134 | features = collections.OrderedDict() 135 | features["input_ids"] = create_int_feature(feature.input_ids) 136 | features["input_mask"] = create_int_feature(feature.input_mask) 137 | features["segment_ids"] = create_int_feature(feature.segment_ids) 138 | features["label_ids"] = create_int_feature(feature.label_ids) 139 | features["is_real_example"] = create_int_feature([int(feature.is_real_example)]) 140 | 141 | tf_example = tf.train.Example(features=tf.train.Features(feature=features)) 142 | tf_writer.write(tf_example.SerializeToString()) 143 | 144 | def input2output(self, INPUT_DATA_PATHS, OUTPUT_DIR, is_init_attribute=False): 145 | if is_init_attribute: 146 | Attributes = [] 147 | for data_path in INPUT_DATA_PATHS: 148 | with open(data_path, "r", encoding='utf-8') as f: 149 | for line in f.readlines(): 150 | line = json.loads(line) 151 | attribute = line["answer"].split("|||")[1].strip() 152 | Attributes.append(attribute) 153 | Attributes = list(set(Attributes)) 154 | else: 155 | Attributes = config.Attributes 156 | 157 | for data_path in INPUT_DATA_PATHS: 158 | file_name = data_path.split('/')[-1] 159 | file_type = file_name.split('.')[0] 160 | self.makdir(OUTPUT_DIR, file_type) # 生成文件名对应的文件夹 161 | OUT_PATH = os.path.join(os.path.dirname(__file__), OUTPUT_DIR, file_type) 162 | # 创建需要写入的文件 163 | token_label_f = open(os.path.join(OUT_PATH, "token_label.txt"), "w", encoding='utf-8') 164 | tf_writer = tf.python_io.TFRecordWriter(os.path.join(OUT_PATH, file_type + ".tf_record")) 165 | with open(data_path, "r", encoding='utf-8') as f: 166 | count_numbers = 0 167 | while True: 168 | line = f.readline() 169 | if line: 170 | line = json.loads(line) 171 | text = line["question"] 172 | answer = line["answer"] 173 | attribute = answer.split("|||")[1].strip() 174 | entity = answer.split("|||")[0].split("（")[0] 175 | self.write2txt(text, attribute, 1, token_label_f, tf_writer) 176 | 177 | # 使用es获取相关属性的负类 178 | body = { 179 | "query": { 180 | "term": { 181 | "entity.keyword": entity 182 | } 183 | } 184 | } 185 | es_results = self.es.search(index="kbqa-data", doc_type="kbList", body=body, 186 | size=self.fake_example_nums) 187 | 188 | attribute_list = list() 189 | for i in range(len(es_results['hits']['hits'])): 190 | relation = es_results['hits']['hits'][i]['_source']['relation'] 191 | attribute_list.append(relation) 192 | 193 | # 去重复 194 | attribute_list = list(set(attribute_list)) 195 | for fake_attribute in attribute_list: 196 | # 1代表true 197 | # 0代表true 198 | if len(fake_attribute) == 0: 199 | continue 200 | self.write2txt(text, fake_attribute, 0, token_label_f, tf_writer) 201 | 202 | # 随机获得不相关负类 203 | for i in range(self.fake_example_nums): 204 | # 1代表true 205 | # 0代表true 206 | fake_attribute = getAttribute(Attributes, attribute) 207 | if len(fake_attribute) == 0: 208 | continue 209 | self.write2txt(text, fake_attribute, 0, token_label_f, tf_writer) 210 | 211 | count_numbers += 1 212 | if count_numbers % 10000 == 0: 213 | print("Writing example %d " % (count_numbers)) 214 | else: 215 | break 216 | print("all numbers", count_numbers) 217 | token_label_f.close() 218 | tf_writer.close() 219 | 220 | 221 | if __name__ == '__main__': 222 | INPUT_DATA_PATHS = ["raw_data/kgClue/train.json", "raw_data/kgClue/eval.json", "raw_data/kgClue/test_public.json"] 223 | OUTPUT_DIR = "./data" 224 | fake_example_nums = 2 # 会影响sim模型数据集的个数需要及时更新config里面的值 225 | es_host = "127.0.0.1" 226 | es_port = "9200" 227 | sim_data_make = DataMaking(fake_example_nums, es_host, es_port, do_lower_case=True, 228 | max_seq_length=config.max_seq_length) 229 | is_init_attribute = False 230 | sim_data_make.input2output(INPUT_DATA_PATHS, OUTPUT_DIR, is_init_attribute=is_init_attribute) 231 | -------------------------------------------------------------------------------- /algorithm/kg_qa/SIM/Eval.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/20 10:54 4 | # @Author : 刘鑫 5 | # @FileName: Eval.py 6 | # @Software: PyCharm 7 | import json 8 | import os 9 | 10 | from sklearn.metrics import f1_score, accuracy_score,recall_score,precision_score 11 | 12 | from algorithm.kg_qa.config import Properties, SimConfig as config 13 | from algorithm.kg_qa.SIM.Predict import Predict 14 | 15 | 16 | # 评估说明：既然是关注序列标注模型的分类效果，文本分类结果应该给予正确的 17 | 18 | class Eval(object): 19 | def __init__(self, MODEL_PATH): 20 | self.sim = Predict(MODEL_PATH) 21 | 22 | def do_eval(self, data_files=["../raw_data/test.json"]): 23 | 24 | for data_file in data_files: 25 | y_true = [] 26 | y_pred = [] 27 | with open(data_file, 'r', encoding='utf-8') as f: 28 | while True: 29 | line = f.readline() 30 | if line: 31 | text = line.split("\t")[0] 32 | attribute = line.split("\t")[1] 33 | t_label = line.split("\t")[2] 34 | 35 | if int(t_label) == 1: 36 | t_label = True 37 | else: 38 | t_label = False 39 | 40 | predict_label ,_= self.sim.predict_one(text, attribute, TEST_MODE=True) 41 | 42 | y_true.append(t_label) 43 | y_pred.append(predict_label) 44 | 45 | else: 46 | break 47 | 48 | macro = f1_score(y_true, y_pred, average='macro', zero_division=1) 49 | accuracy = accuracy_score(y_true, y_pred) 50 | recall = recall_score(y_true, y_pred) 51 | precision = precision_score(y_true, y_pred) 52 | 53 | print(f'\t\t准确率为： {accuracy}') 54 | print(f'\t\tf1值为： {macro}') 55 | print(f'\t\trecall值为： {recall}') 56 | print(f'\t\tprecision值为： {precision}') 57 | 58 | 59 | if __name__ == '__main__': 60 | os.environ["CUDA_VISIBLE_DEVICES"] = "0" # 使用不存在的显卡预测即用cpu 61 | MODEL_PATH = config.model_out 62 | sim_eval = Eval(MODEL_PATH) 63 | sim_eval.do_eval(data_files=[r"C:\Users\11943\Documents\GitHub\KgCLUEbench\algorithm\kg_qa\SIM\data\test_public\token_label.txt"]) 64 | # y = [True,False,False,True,False] 65 | # p = [False,False,False,True,True] 66 | # macro =f1_score(y, p, average='macro', zero_division=1) 67 | # accuracy = accuracy_score(y, p) 68 | # recall = recall_score(y, p) 69 | # precision = precision_score(y, p) 70 | # print(macro) 71 | # print(accuracy) 72 | # print(recall) 73 | # print(precision) 74 | -------------------------------------------------------------------------------- /algorithm/kg_qa/SIM/Predict.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/16 10:42 4 | # @Author : 刘鑫 5 | # @FileName: Predict.py 6 | # @Software: PyCharm 7 | 8 | import os 9 | import tensorflow as tf 10 | 11 | os.environ["CUDA_VISIBLE_DEVICES"] = "1" 12 | 13 | from bert import tokenization 14 | from algorithm.kg_qa.SIM.DataMaking import DataMaking 15 | from algorithm.kg_qa.config import Properties, SimConfig as config 16 | 17 | 18 | # 属性相似度 19 | class Predict(object): 20 | 21 | def __init__(self, MODEL_PATH): 22 | self.model_path = MODEL_PATH 23 | 24 | # 准备token 25 | self.tokenizer_ = tokenization.FullTokenizer(vocab_file=Properties.vocab_file) 26 | self.data_making = DataMaking(do_lower_case=True, max_seq_length=config.max_seq_length) 27 | self.sess = self.load_model() 28 | 29 | self.input_ids = self.sess.graph.get_tensor_by_name("input_ids:0") 30 | self.input_mask = self.sess.graph.get_tensor_by_name("input_mask:0") 31 | self.segment_ids = self.sess.graph.get_tensor_by_name("segment_ids:0") 32 | self.keep_prob = self.sess.graph.get_tensor_by_name("keep_prob:0") 33 | # 预测的结果 34 | self.p1 = self.sess.graph.get_tensor_by_name("loss/Cast_1:0") 35 | self.p2 = self.sess.graph.get_tensor_by_name("loss/Sigmoid:0") 36 | 37 | # tensor_name_list = [tensor.name for tensor in tf.get_default_graph().as_graph_def().node] 38 | # print(tensor_name_list) 39 | 40 | def load_model(self): 41 | try: 42 | checkpoint = tf.train.get_checkpoint_state(self.model_path) 43 | input_checkpoint = checkpoint.model_checkpoint_path 44 | print("[INFO] input_checkpoint:", input_checkpoint) 45 | except Exception as e: 46 | input_checkpoint = self.model_path 47 | print("[INFO] Model folder", self.model_path, repr(e)) 48 | 49 | # We clear devices to allow TensorFlow to control on which device it will load operations 50 | clear_devices = True 51 | tf.reset_default_graph() 52 | # We import the meta graph and retrieve a Saver 53 | saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices) 54 | 55 | # We start a session and restore the graph weights 56 | sess_ = tf.Session() 57 | saver.restore(sess_, input_checkpoint) 58 | 59 | return sess_ 60 | 61 | def predict_one(self, PREDICT_TXT, attribute, TEST_MODE=False): 62 | feature = self.data_making.convert_single_example(PREDICT_TXT, attribute, test_mode=TEST_MODE) 63 | feed = {self.input_ids: [feature[0]], 64 | self.input_mask: [feature[1]], 65 | self.segment_ids: [feature[2]], 66 | self.keep_prob: 1.0} 67 | 68 | probs, p2_ = self.sess.run([self.p1, self.p2], feed) 69 | 70 | for tmp in probs: 71 | tmp = list(tmp) 72 | if tmp[1] == 1: 73 | out = True 74 | else: 75 | out = False 76 | return out, p2_ 77 | 78 | 79 | if __name__ == '__main__': 80 | MODEL_PATH = config.model_out 81 | 82 | PREDICT_TXT = "东瓯王发生的主要事件是什么？" 83 | 84 | attributes = ["东瓯王后", "中文名", "主要事件", "代表人物", "姓氏", "所属地", "中文名", "位置"] 85 | label = [False, False, True, False, False, False, False, False] 86 | sim = Predict(MODEL_PATH) 87 | outs = [] 88 | for attribute in attributes: 89 | out = sim.predict_one(PREDICT_TXT, attribute, TEST_MODE=True) 90 | outs.append(out) 91 | print(outs) 92 | -------------------------------------------------------------------------------- /algorithm/kg_qa/SIM/TrainAndValid.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/12 11:13 4 | # @Author : 刘鑫 5 | # @FileName: sequnce_labeling_train.py 6 | # @Software: PyCharm 7 | 8 | # 假定已经带着正确的分类label去训练实体识别 9 | import tensorflow as tf 10 | import os, math 11 | 12 | from bert import modeling 13 | from bert import optimization 14 | from algorithm.kg_qa.config import Properties, SimConfig as config 15 | from utils.EvalReport import report 16 | 17 | 18 | def load_bert_config(path): 19 | """ 20 | bert 模型配置文件 21 | """ 22 | return modeling.BertConfig.from_json_file(path) 23 | 24 | 25 | def create_model(bert_config, is_training, input_ids, input_mask, segment_ids, labels, keep_prob, num_labels, 26 | use_one_hot_embeddings): 27 | """Creates a classification model.""" 28 | model = modeling.BertModel( 29 | config=bert_config, 30 | is_training=is_training, 31 | input_ids=input_ids, 32 | input_mask=input_mask, 33 | token_type_ids=segment_ids, 34 | use_one_hot_embeddings=use_one_hot_embeddings, 35 | scope='bert' 36 | ) 37 | output_layer = model.get_pooled_output() 38 | 39 | hidden_size = output_layer.shape[-1].value 40 | 41 | 42 | output_weights = tf.get_variable( 43 | "output_weights", [num_labels, hidden_size], 44 | initializer=tf.truncated_normal_initializer(stddev=0.02)) 45 | 46 | output_bias = tf.get_variable( 47 | "output_bias", [num_labels], initializer=tf.zeros_initializer()) 48 | 49 | with tf.variable_scope("loss"): 50 | if is_training: 51 | # I.e., 0.1 dropout 52 | output_layer = tf.nn.dropout(output_layer, keep_prob=0.9) 53 | 54 | logits_wx = tf.matmul(output_layer, output_weights, transpose_b=True) 55 | logits = tf.nn.bias_add(logits_wx, output_bias) 56 | 57 | probabilities = tf.sigmoid(logits) 58 | label_ids = tf.cast(labels, tf.float32) 59 | per_example_loss = tf.reduce_sum( 60 | tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=label_ids), axis=-1) 61 | loss = tf.reduce_mean(per_example_loss) 62 | 63 | predict_ids = tf.cast(probabilities > 0.5, tf.int32) 64 | label_ids = tf.cast(label_ids, tf.int32) 65 | elements_equal = tf.cast(tf.equal(predict_ids, label_ids), tf.int32) 66 | row_predict_ids = tf.reduce_sum(elements_equal, -1) 67 | row_label_ids = tf.reduce_sum(tf.ones_like(label_ids), -1) 68 | _, accuracy = tf.metrics.accuracy(labels=row_label_ids, predictions=row_predict_ids) 69 | 70 | return (loss, accuracy, logits, probabilities) 71 | 72 | 73 | def get_input_data(input_file, seq_length, batch_size, is_training=True): 74 | def parser(record): 75 | name_to_features = { 76 | "input_ids": tf.FixedLenFeature([seq_length], tf.int64), 77 | "input_mask": tf.FixedLenFeature([seq_length], tf.int64), 78 | "segment_ids": tf.FixedLenFeature([seq_length], tf.int64), 79 | "label_ids": tf.FixedLenFeature([config.num_labels], tf.int64), 80 | } 81 | 82 | example = tf.parse_single_example(record, features=name_to_features) 83 | input_ids = example["input_ids"] 84 | input_mask = example["input_mask"] 85 | segment_ids = example["segment_ids"] 86 | labels = example["label_ids"] 87 | return input_ids, input_mask, segment_ids, labels 88 | 89 | dataset = tf.data.TFRecordDataset(input_file) 90 | # 数据类别集中，需要较大的buffer_size，才能有效打乱，或者再数据处理的过程中进行打乱 91 | if is_training: 92 | dataset = dataset.map(parser).batch(batch_size).shuffle(buffer_size=2000) 93 | else: 94 | dataset = dataset.map(parser).batch(batch_size) 95 | iterator = dataset.make_one_shot_iterator() 96 | input_ids, input_mask, segment_ids, labels = iterator.get_next() 97 | return input_ids, input_mask, segment_ids, labels 98 | 99 | def main(): 100 | print("print start load the params...") 101 | 102 | tf.gfile.MakeDirs(config.model_out) 103 | 104 | # 配置超参数 105 | train_examples_len = config.train_examples_len 106 | valid_examples_len = config.valid_examples_len 107 | learning_rate = config.learning_rate 108 | eval_per_step = config.eval_per_step 109 | num_labels = config.num_labels 110 | 111 | num_train_steps = math.ceil(train_examples_len / config.train_batch_size) 112 | num_valid_steps = math.ceil(valid_examples_len / config.valid_batch_size) 113 | num_warmup_steps = math.ceil(num_train_steps * config.num_train_epochs * config.warmup_proportion) 114 | print("num_train_steps:{}, num_valid_steps:{}, num_warmup_steps:{}".format(num_train_steps, num_valid_steps, 115 | num_warmup_steps)) 116 | 117 | use_one_hot_embeddings = False 118 | is_training = True 119 | seq_len = config.max_seq_length 120 | 121 | init_checkpoint = Properties.init_checkpoint 122 | print("print start compile the bert model...") 123 | 124 | # 定义输入输出 125 | input_ids = tf.placeholder(tf.int64, shape=[None, seq_len], name='input_ids') 126 | input_mask = tf.placeholder(tf.int64, shape=[None, seq_len], name='input_mask') 127 | segment_ids = tf.placeholder(tf.int64, shape=[None, seq_len], name='segment_ids') 128 | token_labels = tf.placeholder(tf.int64, shape=[None, num_labels], name='token_labels') 129 | keep_prob = tf.placeholder(tf.float32, name='keep_prob') # , name='is_training' 130 | bert_config_ = load_bert_config(Properties.bert_config) 131 | (total_loss, acc, logits, probabilities) = create_model(bert_config_, is_training, input_ids, 132 | input_mask, segment_ids, token_labels, 133 | keep_prob, 134 | num_labels, use_one_hot_embeddings) 135 | train_op = optimization.create_optimizer(total_loss, learning_rate, num_train_steps * config.num_train_epochs, 136 | num_warmup_steps, False) 137 | 138 | print("print start train the bert model...") 139 | 140 | batch_size = config.train_batch_size 141 | valid_batch_size = config.valid_batch_size 142 | 143 | init_global = tf.global_variables_initializer() 144 | 145 | saver = tf.train.Saver([v for v in tf.global_variables() if 'adam_v' not in v.name and 'adam_m' not in v.name], 146 | max_to_keep=3) # 保存最后top3模型 147 | 148 | with tf.Session() as sess: 149 | sess.run(tf.local_variables_initializer()) 150 | print("start load the pre train model") 151 | 152 | if init_checkpoint: 153 | tvars = tf.trainable_variables() 154 | print("trainable_variables", len(tvars)) 155 | (assignment_map, initialized_variable_names) = modeling.get_assignment_map_from_checkpoint(tvars, 156 | init_checkpoint) 157 | print("initialized_variable_names:", len(initialized_variable_names)) 158 | 159 | saver_ = tf.train.Saver([v for v in tvars if v.name in initialized_variable_names]) 160 | saver_.restore(sess, init_checkpoint) 161 | sess.run(init_global) 162 | else: 163 | sess.run(tf.global_variables_initializer()) 164 | print("********* train start *********") 165 | 166 | def train_step(ids, mask, segment, y, step, train_out_f): 167 | feed = {input_ids: ids, 168 | input_mask: mask, 169 | segment_ids: segment, 170 | token_labels: y, 171 | keep_prob: 0.9} 172 | _, out_loss, acc_, p_ = sess.run([train_op, total_loss, acc, probabilities], feed_dict=feed) 173 | print("step :{},loss :{}, acc :{}".format(step, out_loss, acc_)) 174 | train_out_f.write("step :{}, loss :{}, acc :{} \n".format(step, out_loss, acc_)) 175 | return out_loss, p_, y 176 | 177 | def valid_step(ids, mask, segment, y): 178 | # 验证训练效果 179 | feed = {input_ids: ids, 180 | input_mask: mask, 181 | segment_ids: segment, 182 | token_labels: y, 183 | keep_prob: 1.0 184 | } 185 | out_loss, acc_, p_ = sess.run([total_loss, acc, probabilities], feed_dict=feed) 186 | print("loss :{}, acc :{}".format(out_loss, acc_)) 187 | return out_loss, p_, y 188 | 189 | min_total_loss_dev = 999999 190 | step = 0 191 | if not os.path.exists(config.training_log): 192 | os.makedirs(config.training_log) 193 | for epoch in range(config.num_train_epochs): 194 | _ = "{:*^100s}".format(("epoch-" + str(epoch)).center(20)) 195 | print(_) 196 | # 读取训练数据 197 | total_loss_train = 0 198 | 199 | input_ids2, input_mask2, segment_ids2, labels2 = get_input_data(config.train_data, seq_len, batch_size) 200 | 201 | train_out_f = open(os.path.join(config.training_log, "epoch-" + str(epoch) + ".txt"), 'w', encoding='utf-8') 202 | 203 | for i in range(num_train_steps): 204 | step += 1 205 | ids_train, mask_train, segment_train, y_train = sess.run( 206 | [input_ids2, input_mask2, segment_ids2, labels2]) 207 | out_loss, pre, y = train_step(ids_train, mask_train, segment_train, y_train, step, train_out_f) 208 | total_loss_train += out_loss 209 | 210 | if step % eval_per_step == 0 and step >= config.eval_start_step: 211 | total_loss_dev = 0 212 | dev_input_ids2, dev_input_mask2, dev_segment_ids2, dev_labels2 = get_input_data(config.valid_data, 213 | seq_len, 214 | valid_batch_size, 215 | False) 216 | 217 | for j in range(num_valid_steps): # 一个 epoch 的轮数 218 | ids_dev, mask_dev, segment_dev, y_dev = sess.run( 219 | [dev_input_ids2, dev_input_mask2, dev_segment_ids2, dev_labels2]) 220 | out_loss, pre, y = valid_step(ids_dev, mask_dev, segment_dev, y_dev) 221 | total_loss_dev += out_loss 222 | # report a batch data in valid_data 223 | # report(pre2out(y), pre2out(pre)) 224 | print("total_loss_dev:{}".format(total_loss_dev)) 225 | # print(classification_report(total_true_dev, total_pre_dev, digits=4)) 226 | 227 | if total_loss_dev < min_total_loss_dev: 228 | print("save model:\t%f\t>%f" % (min_total_loss_dev, total_loss_dev)) 229 | min_total_loss_dev = total_loss_dev 230 | saver.save(sess, config.model_out + 'bert.ckpt', global_step=step) 231 | elif step < config.eval_start_step and step % config.auto_save == 0: 232 | saver.save(sess, config.model_out + 'bert.ckpt', global_step=step) 233 | train_out_f.close() 234 | _ = "{:*^100s}".format(("epoch-" + str(epoch) + " report:").center(20)) 235 | print("total_loss_train:{}".format(total_loss_train)) 236 | # print(classification_report(total_true_train, total_pre_train, digits=4)) 237 | 238 | 239 | if __name__ == "__main__": 240 | print("********* sim start *********") 241 | tf.logging.set_verbosity(tf.logging.INFO) 242 | main() 243 | -------------------------------------------------------------------------------- /algorithm/kg_qa/SIM/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/29 22:25 4 | # @Author : 刘鑫 5 | # @FileName: __init__.py.py 6 | # @Software: PyCharm 7 | -------------------------------------------------------------------------------- /algorithm/kg_qa/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/29 22:24 4 | # @Author : 刘鑫 5 | # @FileName: __init__.py.py 6 | # @Software: PyCharm 7 | -------------------------------------------------------------------------------- /algorithm/kg_qa/不同预训练模型的实验与评估.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLUEbenchmark/KgCLUEbench/bece79c8bd7cb338d707cf71c03a98963cac1a24/algorithm/kg_qa/不同预训练模型的实验与评估.docx -------------------------------------------------------------------------------- /bert/.gitignore: -------------------------------------------------------------------------------- 1 | # Initially taken from Github's Python gitignore file 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # Environments 92 | .env 93 | .venv 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | .dmypy.json 113 | dmypy.json 114 | 115 | # Pyre type checker 116 | .pyre/ 117 | -------------------------------------------------------------------------------- /bert/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | BERT needs to maintain permanent compatibility with the pre-trained model files, 4 | so we do not plan to make any major changes to this library (other than what was 5 | promised in the README). However, we can accept small patches related to 6 | re-factoring and documentation. To submit contributes, there are just a few 7 | small guidelines you need to follow. 8 | 9 | ## Contributor License Agreement 10 | 11 | Contributions to this project must be accompanied by a Contributor License 12 | Agreement. You (or your employer) retain the copyright to your contribution; 13 | this simply gives us permission to use and redistribute your contributions as 14 | part of the project. Head over to to see 15 | your current agreements on file or to sign a new one. 16 | 17 | You generally only need to submit a CLA once, so if you've already submitted one 18 | (even if it was for a different project), you probably don't need to do it 19 | again. 20 | 21 | ## Code reviews 22 | 23 | All submissions, including submissions by project members, require review. We 24 | use GitHub pull requests for this purpose. Consult 25 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 26 | information on using pull requests. 27 | 28 | ## Community Guidelines 29 | 30 | This project follows 31 | [Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). 32 | -------------------------------------------------------------------------------- /bert/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /bert/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /bert/extract_features.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Extract pre-computed feature vectors from BERT.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import codecs 22 | import collections 23 | import json 24 | import re 25 | 26 | import modeling 27 | import tokenization 28 | import tensorflow as tf 29 | 30 | flags = tf.flags 31 | 32 | FLAGS = flags.FLAGS 33 | 34 | flags.DEFINE_string("input_file", None, "") 35 | 36 | flags.DEFINE_string("output_file", None, "") 37 | 38 | flags.DEFINE_string("layers", "-1,-2,-3,-4", "") 39 | 40 | flags.DEFINE_string( 41 | "bert_config_file", None, 42 | "The config json file corresponding to the pre-trained BERT model. " 43 | "This specifies the model architecture.") 44 | 45 | flags.DEFINE_integer( 46 | "max_seq_length", 128, 47 | "The maximum total input sequence length after WordPiece tokenization. " 48 | "Sequences longer than this will be truncated, and sequences shorter " 49 | "than this will be padded.") 50 | 51 | flags.DEFINE_string( 52 | "init_checkpoint", None, 53 | "Initial checkpoint (usually from a pre-trained BERT model).") 54 | 55 | flags.DEFINE_string("vocab_file", None, 56 | "The vocabulary file that the BERT model was trained on.") 57 | 58 | flags.DEFINE_bool( 59 | "do_lower_case", True, 60 | "Whether to lower case the input text. Should be True for uncased " 61 | "models and False for cased models.") 62 | 63 | flags.DEFINE_integer("batch_size", 32, "Batch size for predictions.") 64 | 65 | flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.") 66 | 67 | flags.DEFINE_string("master", None, 68 | "If using a TPU, the address of the master.") 69 | 70 | flags.DEFINE_integer( 71 | "num_tpu_cores", 8, 72 | "Only used if `use_tpu` is True. Total number of TPU cores to use.") 73 | 74 | flags.DEFINE_bool( 75 | "use_one_hot_embeddings", False, 76 | "If True, tf.one_hot will be used for embedding lookups, otherwise " 77 | "tf.nn.embedding_lookup will be used. On TPUs, this should be True " 78 | "since it is much faster.") 79 | 80 | 81 | class InputExample(object): 82 | 83 | def __init__(self, unique_id, text_a, text_b): 84 | self.unique_id = unique_id 85 | self.text_a = text_a 86 | self.text_b = text_b 87 | 88 | 89 | class InputFeatures(object): 90 | """A single set of features of data.""" 91 | 92 | def __init__(self, unique_id, tokens, input_ids, input_mask, input_type_ids): 93 | self.unique_id = unique_id 94 | self.tokens = tokens 95 | self.input_ids = input_ids 96 | self.input_mask = input_mask 97 | self.input_type_ids = input_type_ids 98 | 99 | 100 | def input_fn_builder(features, seq_length): 101 | """Creates an `input_fn` closure to be passed to TPUEstimator.""" 102 | 103 | all_unique_ids = [] 104 | all_input_ids = [] 105 | all_input_mask = [] 106 | all_input_type_ids = [] 107 | 108 | for feature in features: 109 | all_unique_ids.append(feature.unique_id) 110 | all_input_ids.append(feature.input_ids) 111 | all_input_mask.append(feature.input_mask) 112 | all_input_type_ids.append(feature.input_type_ids) 113 | 114 | def input_fn(params): 115 | """The actual input function.""" 116 | batch_size = params["batch_size"] 117 | 118 | num_examples = len(features) 119 | 120 | # This is for demo purposes and does NOT scale to large data sets. We do 121 | # not use Dataset.from_generator() because that uses tf.py_func which is 122 | # not TPU compatible. The right way to load data is with TFRecordReader. 123 | d = tf.data.Dataset.from_tensor_slices({ 124 | "unique_ids": 125 | tf.constant(all_unique_ids, shape=[num_examples], dtype=tf.int32), 126 | "input_ids": 127 | tf.constant( 128 | all_input_ids, shape=[num_examples, seq_length], 129 | dtype=tf.int32), 130 | "input_mask": 131 | tf.constant( 132 | all_input_mask, 133 | shape=[num_examples, seq_length], 134 | dtype=tf.int32), 135 | "input_type_ids": 136 | tf.constant( 137 | all_input_type_ids, 138 | shape=[num_examples, seq_length], 139 | dtype=tf.int32), 140 | }) 141 | 142 | d = d.batch(batch_size=batch_size, drop_remainder=False) 143 | return d 144 | 145 | return input_fn 146 | 147 | 148 | def model_fn_builder(bert_config, init_checkpoint, layer_indexes, use_tpu, 149 | use_one_hot_embeddings): 150 | """Returns `model_fn` closure for TPUEstimator.""" 151 | 152 | def model_fn(features, labels, mode, params): # pylint: disable=unused-argument 153 | """The `model_fn` for TPUEstimator.""" 154 | 155 | unique_ids = features["unique_ids"] 156 | input_ids = features["input_ids"] 157 | input_mask = features["input_mask"] 158 | input_type_ids = features["input_type_ids"] 159 | 160 | model = modeling.BertModel( 161 | config=bert_config, 162 | is_training=False, 163 | input_ids=input_ids, 164 | input_mask=input_mask, 165 | token_type_ids=input_type_ids, 166 | use_one_hot_embeddings=use_one_hot_embeddings) 167 | 168 | if mode != tf.estimator.ModeKeys.PREDICT: 169 | raise ValueError("Only PREDICT modes are supported: %s" % (mode)) 170 | 171 | tvars = tf.trainable_variables() 172 | scaffold_fn = None 173 | (assignment_map, 174 | initialized_variable_names) = modeling.get_assignment_map_from_checkpoint( 175 | tvars, init_checkpoint) 176 | if use_tpu: 177 | 178 | def tpu_scaffold(): 179 | tf.train.init_from_checkpoint(init_checkpoint, assignment_map) 180 | return tf.train.Scaffold() 181 | 182 | scaffold_fn = tpu_scaffold 183 | else: 184 | tf.train.init_from_checkpoint(init_checkpoint, assignment_map) 185 | 186 | tf.logging.info("**** Trainable Variables ****") 187 | for var in tvars: 188 | init_string = "" 189 | if var.name in initialized_variable_names: 190 | init_string = ", *INIT_FROM_CKPT*" 191 | tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape, 192 | init_string) 193 | 194 | all_layers = model.get_all_encoder_layers() 195 | 196 | predictions = { 197 | "unique_id": unique_ids, 198 | } 199 | 200 | for (i, layer_index) in enumerate(layer_indexes): 201 | predictions["layer_output_%d" % i] = all_layers[layer_index] 202 | 203 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 204 | mode=mode, predictions=predictions, scaffold_fn=scaffold_fn) 205 | return output_spec 206 | 207 | return model_fn 208 | 209 | 210 | def convert_examples_to_features(examples, seq_length, tokenizer): 211 | """Loads a data file into a list of `InputBatch`s.""" 212 | 213 | features = [] 214 | for (ex_index, example) in enumerate(examples): 215 | tokens_a = tokenizer.tokenize(example.text_a) 216 | 217 | tokens_b = None 218 | if example.text_b: 219 | tokens_b = tokenizer.tokenize(example.text_b) 220 | 221 | if tokens_b: 222 | # Modifies `tokens_a` and `tokens_b` in place so that the total 223 | # length is less than the specified length. 224 | # Account for [CLS], [SEP], [SEP] with "- 3" 225 | _truncate_seq_pair(tokens_a, tokens_b, seq_length - 3) 226 | else: 227 | # Account for [CLS] and [SEP] with "- 2" 228 | if len(tokens_a) > seq_length - 2: 229 | tokens_a = tokens_a[0:(seq_length - 2)] 230 | 231 | # The convention in BERT is: 232 | # (a) For sequence pairs: 233 | # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] 234 | # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 235 | # (b) For single sequences: 236 | # tokens: [CLS] the dog is hairy . [SEP] 237 | # type_ids: 0 0 0 0 0 0 0 238 | # 239 | # Where "type_ids" are used to indicate whether this is the first 240 | # sequence or the second sequence. The embedding vectors for `type=0` and 241 | # `type=1` were learned during pre-training and are added to the wordpiece 242 | # embedding vector (and position vector). This is not *strictly* necessary 243 | # since the [SEP] token unambiguously separates the sequences, but it makes 244 | # it easier for the model to learn the concept of sequences. 245 | # 246 | # For classification tasks, the first vector (corresponding to [CLS]) is 247 | # used as as the "sentence vector". Note that this only makes sense because 248 | # the entire model is fine-tuned. 249 | tokens = [] 250 | input_type_ids = [] 251 | tokens.append("[CLS]") 252 | input_type_ids.append(0) 253 | for token in tokens_a: 254 | tokens.append(token) 255 | input_type_ids.append(0) 256 | tokens.append("[SEP]") 257 | input_type_ids.append(0) 258 | 259 | if tokens_b: 260 | for token in tokens_b: 261 | tokens.append(token) 262 | input_type_ids.append(1) 263 | tokens.append("[SEP]") 264 | input_type_ids.append(1) 265 | 266 | input_ids = tokenizer.convert_tokens_to_ids(tokens) 267 | 268 | # The mask has 1 for real tokens and 0 for padding tokens. Only real 269 | # tokens are attended to. 270 | input_mask = [1] * len(input_ids) 271 | 272 | # Zero-pad up to the sequence length. 273 | while len(input_ids) < seq_length: 274 | input_ids.append(0) 275 | input_mask.append(0) 276 | input_type_ids.append(0) 277 | 278 | assert len(input_ids) == seq_length 279 | assert len(input_mask) == seq_length 280 | assert len(input_type_ids) == seq_length 281 | 282 | if ex_index < 5: 283 | tf.logging.info("*** Example ***") 284 | tf.logging.info("unique_id: %s" % (example.unique_id)) 285 | tf.logging.info("tokens: %s" % " ".join( 286 | [tokenization.printable_text(x) for x in tokens])) 287 | tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) 288 | tf.logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) 289 | tf.logging.info( 290 | "input_type_ids: %s" % " ".join([str(x) for x in input_type_ids])) 291 | 292 | features.append( 293 | InputFeatures( 294 | unique_id=example.unique_id, 295 | tokens=tokens, 296 | input_ids=input_ids, 297 | input_mask=input_mask, 298 | input_type_ids=input_type_ids)) 299 | return features 300 | 301 | 302 | def _truncate_seq_pair(tokens_a, tokens_b, max_length): 303 | """Truncates a sequence pair in place to the maximum length.""" 304 | 305 | # This is a simple heuristic which will always truncate the longer sequence 306 | # one token at a time. This makes more sense than truncating an equal percent 307 | # of tokens from each, since if one sequence is very short then each token 308 | # that's truncated likely contains more information than a longer sequence. 309 | while True: 310 | total_length = len(tokens_a) + len(tokens_b) 311 | if total_length <= max_length: 312 | break 313 | if len(tokens_a) > len(tokens_b): 314 | tokens_a.pop() 315 | else: 316 | tokens_b.pop() 317 | 318 | 319 | def read_examples(input_file): 320 | """Read a list of `InputExample`s from an input file.""" 321 | examples = [] 322 | unique_id = 0 323 | with tf.gfile.GFile(input_file, "r") as reader: 324 | while True: 325 | line = tokenization.convert_to_unicode(reader.readline()) 326 | if not line: 327 | break 328 | line = line.strip() 329 | text_a = None 330 | text_b = None 331 | m = re.match(r"^(.*) \|\|\| (.*)$", line) 332 | if m is None: 333 | text_a = line 334 | else: 335 | text_a = m.group(1) 336 | text_b = m.group(2) 337 | examples.append( 338 | InputExample(unique_id=unique_id, text_a=text_a, text_b=text_b)) 339 | unique_id += 1 340 | return examples 341 | 342 | 343 | def main(_): 344 | tf.logging.set_verbosity(tf.logging.INFO) 345 | 346 | layer_indexes = [int(x) for x in FLAGS.layers.split(",")] 347 | 348 | bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) 349 | 350 | tokenizer = tokenization.FullTokenizer( 351 | vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) 352 | 353 | is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 354 | run_config = tf.contrib.tpu.RunConfig( 355 | master=FLAGS.master, 356 | tpu_config=tf.contrib.tpu.TPUConfig( 357 | num_shards=FLAGS.num_tpu_cores, 358 | per_host_input_for_training=is_per_host)) 359 | 360 | examples = read_examples(FLAGS.input_file) 361 | 362 | features = convert_examples_to_features( 363 | examples=examples, seq_length=FLAGS.max_seq_length, tokenizer=tokenizer) 364 | 365 | unique_id_to_feature = {} 366 | for feature in features: 367 | unique_id_to_feature[feature.unique_id] = feature 368 | 369 | model_fn = model_fn_builder( 370 | bert_config=bert_config, 371 | init_checkpoint=FLAGS.init_checkpoint, 372 | layer_indexes=layer_indexes, 373 | use_tpu=FLAGS.use_tpu, 374 | use_one_hot_embeddings=FLAGS.use_one_hot_embeddings) 375 | 376 | # If TPU is not available, this will fall back to normal Estimator on CPU 377 | # or GPU. 378 | estimator = tf.contrib.tpu.TPUEstimator( 379 | use_tpu=FLAGS.use_tpu, 380 | model_fn=model_fn, 381 | config=run_config, 382 | predict_batch_size=FLAGS.batch_size) 383 | 384 | input_fn = input_fn_builder( 385 | features=features, seq_length=FLAGS.max_seq_length) 386 | 387 | with codecs.getwriter("utf-8")(tf.gfile.Open(FLAGS.output_file, 388 | "w")) as writer: 389 | for result in estimator.predict(input_fn, yield_single_examples=True): 390 | unique_id = int(result["unique_id"]) 391 | feature = unique_id_to_feature[unique_id] 392 | output_json = collections.OrderedDict() 393 | output_json["linex_index"] = unique_id 394 | all_features = [] 395 | for (i, token) in enumerate(feature.tokens): 396 | all_layers = [] 397 | for (j, layer_index) in enumerate(layer_indexes): 398 | layer_output = result["layer_output_%d" % j] 399 | layers = collections.OrderedDict() 400 | layers["index"] = layer_index 401 | layers["values"] = [ 402 | round(float(x), 6) for x in layer_output[i:(i + 1)].flat 403 | ] 404 | all_layers.append(layers) 405 | features = collections.OrderedDict() 406 | features["token"] = token 407 | features["layers"] = all_layers 408 | all_features.append(features) 409 | output_json["features"] = all_features 410 | writer.write(json.dumps(output_json) + "\n") 411 | 412 | 413 | if __name__ == "__main__": 414 | flags.mark_flag_as_required("input_file") 415 | flags.mark_flag_as_required("vocab_file") 416 | flags.mark_flag_as_required("bert_config_file") 417 | flags.mark_flag_as_required("init_checkpoint") 418 | flags.mark_flag_as_required("output_file") 419 | tf.app.run() 420 | -------------------------------------------------------------------------------- /bert/modeling_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import collections 20 | import json 21 | import random 22 | import re 23 | 24 | import modeling 25 | import six 26 | import tensorflow as tf 27 | 28 | 29 | class BertModelTest(tf.test.TestCase): 30 | 31 | class BertModelTester(object): 32 | 33 | def __init__(self, 34 | parent, 35 | batch_size=13, 36 | seq_length=7, 37 | is_training=True, 38 | use_input_mask=True, 39 | use_token_type_ids=True, 40 | vocab_size=99, 41 | hidden_size=32, 42 | num_hidden_layers=5, 43 | num_attention_heads=4, 44 | intermediate_size=37, 45 | hidden_act="gelu", 46 | hidden_dropout_prob=0.1, 47 | attention_probs_dropout_prob=0.1, 48 | max_position_embeddings=512, 49 | type_vocab_size=16, 50 | initializer_range=0.02, 51 | scope=None): 52 | self.parent = parent 53 | self.batch_size = batch_size 54 | self.seq_length = seq_length 55 | self.is_training = is_training 56 | self.use_input_mask = use_input_mask 57 | self.use_token_type_ids = use_token_type_ids 58 | self.vocab_size = vocab_size 59 | self.hidden_size = hidden_size 60 | self.num_hidden_layers = num_hidden_layers 61 | self.num_attention_heads = num_attention_heads 62 | self.intermediate_size = intermediate_size 63 | self.hidden_act = hidden_act 64 | self.hidden_dropout_prob = hidden_dropout_prob 65 | self.attention_probs_dropout_prob = attention_probs_dropout_prob 66 | self.max_position_embeddings = max_position_embeddings 67 | self.type_vocab_size = type_vocab_size 68 | self.initializer_range = initializer_range 69 | self.scope = scope 70 | 71 | def create_model(self): 72 | input_ids = BertModelTest.ids_tensor([self.batch_size, self.seq_length], 73 | self.vocab_size) 74 | 75 | input_mask = None 76 | if self.use_input_mask: 77 | input_mask = BertModelTest.ids_tensor( 78 | [self.batch_size, self.seq_length], vocab_size=2) 79 | 80 | token_type_ids = None 81 | if self.use_token_type_ids: 82 | token_type_ids = BertModelTest.ids_tensor( 83 | [self.batch_size, self.seq_length], self.type_vocab_size) 84 | 85 | config = modeling.BertConfig( 86 | vocab_size=self.vocab_size, 87 | hidden_size=self.hidden_size, 88 | num_hidden_layers=self.num_hidden_layers, 89 | num_attention_heads=self.num_attention_heads, 90 | intermediate_size=self.intermediate_size, 91 | hidden_act=self.hidden_act, 92 | hidden_dropout_prob=self.hidden_dropout_prob, 93 | attention_probs_dropout_prob=self.attention_probs_dropout_prob, 94 | max_position_embeddings=self.max_position_embeddings, 95 | type_vocab_size=self.type_vocab_size, 96 | initializer_range=self.initializer_range) 97 | 98 | model = modeling.BertModel( 99 | config=config, 100 | is_training=self.is_training, 101 | input_ids=input_ids, 102 | input_mask=input_mask, 103 | token_type_ids=token_type_ids, 104 | scope=self.scope) 105 | 106 | outputs = { 107 | "embedding_output": model.get_embedding_output(), 108 | "sequence_output": model.get_sequence_output(), 109 | "pooled_output": model.get_pooled_output(), 110 | "all_encoder_layers": model.get_all_encoder_layers(), 111 | } 112 | return outputs 113 | 114 | def check_output(self, result): 115 | self.parent.assertAllEqual( 116 | result["embedding_output"].shape, 117 | [self.batch_size, self.seq_length, self.hidden_size]) 118 | 119 | self.parent.assertAllEqual( 120 | result["sequence_output"].shape, 121 | [self.batch_size, self.seq_length, self.hidden_size]) 122 | 123 | self.parent.assertAllEqual(result["pooled_output"].shape, 124 | [self.batch_size, self.hidden_size]) 125 | 126 | def test_default(self): 127 | self.run_tester(BertModelTest.BertModelTester(self)) 128 | 129 | def test_config_to_json_string(self): 130 | config = modeling.BertConfig(vocab_size=99, hidden_size=37) 131 | obj = json.loads(config.to_json_string()) 132 | self.assertEqual(obj["vocab_size"], 99) 133 | self.assertEqual(obj["hidden_size"], 37) 134 | 135 | def run_tester(self, tester): 136 | with self.test_session() as sess: 137 | ops = tester.create_model() 138 | init_op = tf.group(tf.global_variables_initializer(), 139 | tf.local_variables_initializer()) 140 | sess.run(init_op) 141 | output_result = sess.run(ops) 142 | tester.check_output(output_result) 143 | 144 | self.assert_all_tensors_reachable(sess, [init_op, ops]) 145 | 146 | @classmethod 147 | def ids_tensor(cls, shape, vocab_size, rng=None, name=None): 148 | """Creates a random int32 tensor of the shape within the vocab size.""" 149 | if rng is None: 150 | rng = random.Random() 151 | 152 | total_dims = 1 153 | for dim in shape: 154 | total_dims *= dim 155 | 156 | values = [] 157 | for _ in range(total_dims): 158 | values.append(rng.randint(0, vocab_size - 1)) 159 | 160 | return tf.constant(value=values, dtype=tf.int32, shape=shape, name=name) 161 | 162 | def assert_all_tensors_reachable(self, sess, outputs): 163 | """Checks that all the tensors in the graph are reachable from outputs.""" 164 | graph = sess.graph 165 | 166 | ignore_strings = [ 167 | "^.*/assert_less_equal/.*$", 168 | "^.*/dilation_rate$", 169 | "^.*/Tensordot/concat$", 170 | "^.*/Tensordot/concat/axis$", 171 | "^testing/.*$", 172 | ] 173 | 174 | ignore_regexes = [re.compile(x) for x in ignore_strings] 175 | 176 | unreachable = self.get_unreachable_ops(graph, outputs) 177 | filtered_unreachable = [] 178 | for x in unreachable: 179 | do_ignore = False 180 | for r in ignore_regexes: 181 | m = r.match(x.name) 182 | if m is not None: 183 | do_ignore = True 184 | if do_ignore: 185 | continue 186 | filtered_unreachable.append(x) 187 | unreachable = filtered_unreachable 188 | 189 | self.assertEqual( 190 | len(unreachable), 0, "The following ops are unreachable: %s" % 191 | (" ".join([x.name for x in unreachable]))) 192 | 193 | @classmethod 194 | def get_unreachable_ops(cls, graph, outputs): 195 | """Finds all of the tensors in graph that are unreachable from outputs.""" 196 | outputs = cls.flatten_recursive(outputs) 197 | output_to_op = collections.defaultdict(list) 198 | op_to_all = collections.defaultdict(list) 199 | assign_out_to_in = collections.defaultdict(list) 200 | 201 | for op in graph.get_operations(): 202 | for x in op.inputs: 203 | op_to_all[op.name].append(x.name) 204 | for y in op.outputs: 205 | output_to_op[y.name].append(op.name) 206 | op_to_all[op.name].append(y.name) 207 | if str(op.type) == "Assign": 208 | for y in op.outputs: 209 | for x in op.inputs: 210 | assign_out_to_in[y.name].append(x.name) 211 | 212 | assign_groups = collections.defaultdict(list) 213 | for out_name in assign_out_to_in.keys(): 214 | name_group = assign_out_to_in[out_name] 215 | for n1 in name_group: 216 | assign_groups[n1].append(out_name) 217 | for n2 in name_group: 218 | if n1 != n2: 219 | assign_groups[n1].append(n2) 220 | 221 | seen_tensors = {} 222 | stack = [x.name for x in outputs] 223 | while stack: 224 | name = stack.pop() 225 | if name in seen_tensors: 226 | continue 227 | seen_tensors[name] = True 228 | 229 | if name in output_to_op: 230 | for op_name in output_to_op[name]: 231 | if op_name in op_to_all: 232 | for input_name in op_to_all[op_name]: 233 | if input_name not in stack: 234 | stack.append(input_name) 235 | 236 | expanded_names = [] 237 | if name in assign_groups: 238 | for assign_name in assign_groups[name]: 239 | expanded_names.append(assign_name) 240 | 241 | for expanded_name in expanded_names: 242 | if expanded_name not in stack: 243 | stack.append(expanded_name) 244 | 245 | unreachable_ops = [] 246 | for op in graph.get_operations(): 247 | is_unreachable = False 248 | all_names = [x.name for x in op.inputs] + [x.name for x in op.outputs] 249 | for name in all_names: 250 | if name not in seen_tensors: 251 | is_unreachable = True 252 | if is_unreachable: 253 | unreachable_ops.append(op) 254 | return unreachable_ops 255 | 256 | @classmethod 257 | def flatten_recursive(cls, item): 258 | """Flattens (potentially nested) a tuple/dictionary/list to a list.""" 259 | output = [] 260 | if isinstance(item, list): 261 | output.extend(item) 262 | elif isinstance(item, tuple): 263 | output.extend(list(item)) 264 | elif isinstance(item, dict): 265 | for (_, v) in six.iteritems(item): 266 | output.append(v) 267 | else: 268 | return [item] 269 | 270 | flat_output = [] 271 | for x in output: 272 | flat_output.extend(cls.flatten_recursive(x)) 273 | return flat_output 274 | 275 | 276 | if __name__ == "__main__": 277 | tf.test.main() 278 | -------------------------------------------------------------------------------- /bert/multilingual.md: -------------------------------------------------------------------------------- 1 | ## Models 2 | 3 | There are two multilingual models currently available. We do not plan to release 4 | more single-language models, but we may release `BERT-Large` versions of these 5 | two in the future: 6 | 7 | * **[`BERT-Base, Multilingual Cased (New, recommended)`](https://storage.googleapis.com/bert_models/2018_11_23/multi_cased_L-12_H-768_A-12.zip)**: 8 | 104 languages, 12-layer, 768-hidden, 12-heads, 110M parameters 9 | * **[`BERT-Base, Multilingual Uncased (Orig, not recommended)`](https://storage.googleapis.com/bert_models/2018_11_03/multilingual_L-12_H-768_A-12.zip)**: 10 | 102 languages, 12-layer, 768-hidden, 12-heads, 110M parameters 11 | * **[`BERT-Base, Chinese`](https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip)**: 12 | Chinese Simplified and Traditional, 12-layer, 768-hidden, 12-heads, 110M 13 | parameters 14 | 15 | **The `Multilingual Cased (New)` model also fixes normalization issues in many 16 | languages, so it is recommended in languages with non-Latin alphabets (and is 17 | often better for most languages with Latin alphabets). When using this model, 18 | make sure to pass `--do_lower_case=false` to `run_pretraining.py` and other 19 | scripts.** 20 | 21 | See the [list of languages](#list-of-languages) that the Multilingual model 22 | supports. The Multilingual model does include Chinese (and English), but if your 23 | fine-tuning data is Chinese-only, then the Chinese model will likely produce 24 | better results. 25 | 26 | ## Results 27 | 28 | To evaluate these systems, we use the 29 | [XNLI dataset](https://github.com/facebookresearch/XNLI) dataset, which is a 30 | version of [MultiNLI](https://www.nyu.edu/projects/bowman/multinli/) where the 31 | dev and test sets have been translated (by humans) into 15 languages. Note that 32 | the training set was *machine* translated (we used the translations provided by 33 | XNLI, not Google NMT). For clarity, we only report on 6 languages below: 34 | 35 | 36 | 37 | | System | English | Chinese | Spanish | German | Arabic | Urdu | 38 | | --------------------------------- | -------- | -------- | -------- | -------- | -------- | -------- | 39 | | XNLI Baseline - Translate Train | 73.7 | 67.0 | 68.8 | 66.5 | 65.8 | 56.6 | 40 | | XNLI Baseline - Translate Test | 73.7 | 68.3 | 70.7 | 68.7 | 66.8 | 59.3 | 41 | | BERT - Translate Train Cased | **81.9** | **76.6** | **77.8** | **75.9** | **70.7** | 61.6 | 42 | | BERT - Translate Train Uncased | 81.4 | 74.2 | 77.3 | 75.2 | 70.5 | 61.7 | 43 | | BERT - Translate Test Uncased | 81.4 | 70.1 | 74.9 | 74.4 | 70.4 | **62.1** | 44 | | BERT - Zero Shot Uncased | 81.4 | 63.8 | 74.3 | 70.5 | 62.1 | 58.3 | 45 | 46 | 47 | 48 | The first two rows are baselines from the XNLI paper and the last three rows are 49 | our results with BERT. 50 | 51 | **Translate Train** means that the MultiNLI training set was machine translated 52 | from English into the foreign language. So training and evaluation were both 53 | done in the foreign language. Unfortunately, training was done on 54 | machine-translated data, so it is impossible to quantify how much of the lower 55 | accuracy (compared to English) is due to the quality of the machine translation 56 | vs. the quality of the pre-trained model. 57 | 58 | **Translate Test** means that the XNLI test set was machine translated from the 59 | foreign language into English. So training and evaluation were both done on 60 | English. However, test evaluation was done on machine-translated English, so the 61 | accuracy depends on the quality of the machine translation system. 62 | 63 | **Zero Shot** means that the Multilingual BERT system was fine-tuned on English 64 | MultiNLI, and then evaluated on the foreign language XNLI test. In this case, 65 | machine translation was not involved at all in either the pre-training or 66 | fine-tuning. 67 | 68 | Note that the English result is worse than the 84.2 MultiNLI baseline because 69 | this training used Multilingual BERT rather than English-only BERT. This implies 70 | that for high-resource languages, the Multilingual model is somewhat worse than 71 | a single-language model. However, it is not feasible for us to train and 72 | maintain dozens of single-language models. Therefore, if your goal is to maximize 73 | performance with a language other than English or Chinese, you might find it 74 | beneficial to run pre-training for additional steps starting from our 75 | Multilingual model on data from your language of interest. 76 | 77 | Here is a comparison of training Chinese models with the Multilingual 78 | `BERT-Base` and Chinese-only `BERT-Base`: 79 | 80 | System | Chinese 81 | ----------------------- | ------- 82 | XNLI Baseline | 67.0 83 | BERT Multilingual Model | 74.2 84 | BERT Chinese-only Model | 77.2 85 | 86 | Similar to English, the single-language model does 3% better than the 87 | Multilingual model. 88 | 89 | ## Fine-tuning Example 90 | 91 | The multilingual model does **not** require any special consideration or API 92 | changes. We did update the implementation of `BasicTokenizer` in 93 | `tokenization.py` to support Chinese character tokenization, so please update if 94 | you forked it. However, we did not change the tokenization API. 95 | 96 | To test the new models, we did modify `run_classifier.py` to add support for the 97 | [XNLI dataset](https://github.com/facebookresearch/XNLI). This is a 15-language 98 | version of MultiNLI where the dev/test sets have been human-translated, and the 99 | training set has been machine-translated. 100 | 101 | To run the fine-tuning code, please download the 102 | [XNLI dev/test set](https://www.nyu.edu/projects/bowman/xnli/XNLI-1.0.zip) and the 103 | [XNLI machine-translated training set](https://www.nyu.edu/projects/bowman/xnli/XNLI-MT-1.0.zip) 104 | and then unpack both .zip files into some directory `$XNLI_DIR`. 105 | 106 | To run fine-tuning on XNLI. The language is hard-coded into `run_classifier.py` 107 | (Chinese by default), so please modify `XnliProcessor` if you want to run on 108 | another language. 109 | 110 | This is a large dataset, so this will training will take a few hours on a GPU 111 | (or about 30 minutes on a Cloud TPU). To run an experiment quickly for 112 | debugging, just set `num_train_epochs` to a small value like `0.1`. 113 | 114 | ```shell 115 | export BERT_BASE_DIR=/path/to/bert/chinese_L-12_H-768_A-12 # or multilingual_L-12_H-768_A-12 116 | export XNLI_DIR=/path/to/xnli 117 | 118 | python run_classifier.py \ 119 | --task_name=XNLI \ 120 | --do_train=true \ 121 | --do_eval=true \ 122 | --data_dir=$XNLI_DIR \ 123 | --vocab_file=$BERT_BASE_DIR/vocab.txt \ 124 | --bert_config_file=$BERT_BASE_DIR/bert_config.json \ 125 | --init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt \ 126 | --max_seq_length=128 \ 127 | --train_batch_size=32 \ 128 | --learning_rate=5e-5 \ 129 | --num_train_epochs=2.0 \ 130 | --output_dir=/tmp/xnli_output/ 131 | ``` 132 | 133 | With the Chinese-only model, the results should look something like this: 134 | 135 | ``` 136 | ***** Eval results ***** 137 | eval_accuracy = 0.774116 138 | eval_loss = 0.83554 139 | global_step = 24543 140 | loss = 0.74603 141 | ``` 142 | 143 | ## Details 144 | 145 | ### Data Source and Sampling 146 | 147 | The languages chosen were the 148 | [top 100 languages with the largest Wikipedias](https://meta.wikimedia.org/wiki/List_of_Wikipedias). 149 | The entire Wikipedia dump for each language (excluding user and talk pages) was 150 | taken as the training data for each language 151 | 152 | However, the size of the Wikipedia for a given language varies greatly, and 153 | therefore low-resource languages may be "under-represented" in terms of the 154 | neural network model (under the assumption that languages are "competing" for 155 | limited model capacity to some extent). At the same time, we also don't want 156 | to overfit the model by performing thousands of epochs over a tiny Wikipedia 157 | for a particular language. 158 | 159 | To balance these two factors, we performed exponentially smoothed weighting of 160 | the data during pre-training data creation (and WordPiece vocab creation). In 161 | other words, let's say that the probability of a language is *P(L)*, e.g., 162 | *P(English) = 0.21* means that after concatenating all of the Wikipedias 163 | together, 21% of our data is English. We exponentiate each probability by some 164 | factor *S* and then re-normalize, and sample from that distribution. In our case 165 | we use *S=0.7*. So, high-resource languages like English will be under-sampled, 166 | and low-resource languages like Icelandic will be over-sampled. E.g., in the 167 | original distribution English would be sampled 1000x more than Icelandic, but 168 | after smoothing it's only sampled 100x more. 169 | 170 | ### Tokenization 171 | 172 | For tokenization, we use a 110k shared WordPiece vocabulary. The word counts are 173 | weighted the same way as the data, so low-resource languages are upweighted by 174 | some factor. We intentionally do *not* use any marker to denote the input 175 | language (so that zero-shot training can work). 176 | 177 | Because Chinese (and Japanese Kanji and Korean Hanja) does not have whitespace 178 | characters, we add spaces around every character in the 179 | [CJK Unicode range](https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_$Unicode_block$) 180 | before applying WordPiece. This means that Chinese is effectively 181 | character-tokenized. Note that the CJK Unicode block only includes 182 | Chinese-origin characters and does *not* include Hangul Korean or 183 | Katakana/Hiragana Japanese, which are tokenized with whitespace+WordPiece like 184 | all other languages. 185 | 186 | For all other languages, we apply the 187 | [same recipe as English](https://github.com/google-research/bert#tokenization): 188 | (a) lower casing+accent removal, (b) punctuation splitting, (c) whitespace 189 | tokenization. We understand that accent markers have substantial meaning in some 190 | languages, but felt that the benefits of reducing the effective vocabulary make 191 | up for this. Generally the strong contextual models of BERT should make up for 192 | any ambiguity introduced by stripping accent markers. 193 | 194 | ### List of Languages 195 | 196 | The multilingual model supports the following languages. These languages were 197 | chosen because they are the top 100 languages with the largest Wikipedias: 198 | 199 | * Afrikaans 200 | * Albanian 201 | * Arabic 202 | * Aragonese 203 | * Armenian 204 | * Asturian 205 | * Azerbaijani 206 | * Bashkir 207 | * Basque 208 | * Bavarian 209 | * Belarusian 210 | * Bengali 211 | * Bishnupriya Manipuri 212 | * Bosnian 213 | * Breton 214 | * Bulgarian 215 | * Burmese 216 | * Catalan 217 | * Cebuano 218 | * Chechen 219 | * Chinese (Simplified) 220 | * Chinese (Traditional) 221 | * Chuvash 222 | * Croatian 223 | * Czech 224 | * Danish 225 | * Dutch 226 | * English 227 | * Estonian 228 | * Finnish 229 | * French 230 | * Galician 231 | * Georgian 232 | * German 233 | * Greek 234 | * Gujarati 235 | * Haitian 236 | * Hebrew 237 | * Hindi 238 | * Hungarian 239 | * Icelandic 240 | * Ido 241 | * Indonesian 242 | * Irish 243 | * Italian 244 | * Japanese 245 | * Javanese 246 | * Kannada 247 | * Kazakh 248 | * Kirghiz 249 | * Korean 250 | * Latin 251 | * Latvian 252 | * Lithuanian 253 | * Lombard 254 | * Low Saxon 255 | * Luxembourgish 256 | * Macedonian 257 | * Malagasy 258 | * Malay 259 | * Malayalam 260 | * Marathi 261 | * Minangkabau 262 | * Nepali 263 | * Newar 264 | * Norwegian (Bokmal) 265 | * Norwegian (Nynorsk) 266 | * Occitan 267 | * Persian (Farsi) 268 | * Piedmontese 269 | * Polish 270 | * Portuguese 271 | * Punjabi 272 | * Romanian 273 | * Russian 274 | * Scots 275 | * Serbian 276 | * Serbo-Croatian 277 | * Sicilian 278 | * Slovak 279 | * Slovenian 280 | * South Azerbaijani 281 | * Spanish 282 | * Sundanese 283 | * Swahili 284 | * Swedish 285 | * Tagalog 286 | * Tajik 287 | * Tamil 288 | * Tatar 289 | * Telugu 290 | * Turkish 291 | * Ukrainian 292 | * Urdu 293 | * Uzbek 294 | * Vietnamese 295 | * Volapük 296 | * Waray-Waray 297 | * Welsh 298 | * West Frisian 299 | * Western Punjabi 300 | * Yoruba 301 | 302 | The **Multilingual Cased (New)** release contains additionally **Thai** and 303 | **Mongolian**, which were not included in the original release. 304 | -------------------------------------------------------------------------------- /bert/optimization.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Functions and classes related to optimization (weight updates).""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import re 22 | import tensorflow as tf 23 | 24 | 25 | def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, use_tpu): 26 | """Creates an optimizer training op.""" 27 | global_step = tf.train.get_or_create_global_step() 28 | 29 | learning_rate = tf.constant(value=init_lr, shape=[], dtype=tf.float32) 30 | 31 | # Implements linear decay of the learning rate. 32 | learning_rate = tf.train.polynomial_decay( 33 | learning_rate, 34 | global_step, 35 | num_train_steps, 36 | end_learning_rate=0.0, 37 | power=1.0, 38 | cycle=False) 39 | 40 | # Implements linear warmup. I.e., if global_step < num_warmup_steps, the 41 | # learning rate will be `global_step/num_warmup_steps * init_lr`. 42 | if num_warmup_steps: 43 | global_steps_int = tf.cast(global_step, tf.int32) 44 | warmup_steps_int = tf.constant(num_warmup_steps, dtype=tf.int32) 45 | 46 | global_steps_float = tf.cast(global_steps_int, tf.float32) 47 | warmup_steps_float = tf.cast(warmup_steps_int, tf.float32) 48 | 49 | warmup_percent_done = global_steps_float / warmup_steps_float 50 | warmup_learning_rate = init_lr * warmup_percent_done 51 | 52 | is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32) 53 | learning_rate = ( 54 | (1.0 - is_warmup) * learning_rate + is_warmup * warmup_learning_rate) 55 | 56 | # It is recommended that you use this optimizer for fine tuning, since this 57 | # is how the model was trained (note that the Adam m/v variables are NOT 58 | # loaded from init_checkpoint.) 59 | optimizer = AdamWeightDecayOptimizer( 60 | learning_rate=learning_rate, 61 | weight_decay_rate=0.01, 62 | beta_1=0.9, 63 | beta_2=0.999, 64 | epsilon=1e-6, 65 | exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) 66 | 67 | if use_tpu: 68 | optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) 69 | 70 | tvars = tf.trainable_variables() 71 | grads = tf.gradients(loss, tvars) 72 | 73 | # This is how the model was pre-trained. 74 | (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0) 75 | 76 | train_op = optimizer.apply_gradients( 77 | zip(grads, tvars), global_step=global_step) 78 | 79 | # Normally the global step update is done inside of `apply_gradients`. 80 | # However, `AdamWeightDecayOptimizer` doesn't do this. But if you use 81 | # a different optimizer, you should probably take this line out. 82 | new_global_step = global_step + 1 83 | train_op = tf.group(train_op, [global_step.assign(new_global_step)]) 84 | return train_op 85 | 86 | 87 | class AdamWeightDecayOptimizer(tf.train.Optimizer): 88 | """A basic Adam optimizer that includes "correct" L2 weight decay.""" 89 | 90 | def __init__(self, 91 | learning_rate, 92 | weight_decay_rate=0.0, 93 | beta_1=0.9, 94 | beta_2=0.999, 95 | epsilon=1e-6, 96 | exclude_from_weight_decay=None, 97 | name="AdamWeightDecayOptimizer"): 98 | """Constructs a AdamWeightDecayOptimizer.""" 99 | super(AdamWeightDecayOptimizer, self).__init__(False, name) 100 | 101 | self.learning_rate = learning_rate 102 | self.weight_decay_rate = weight_decay_rate 103 | self.beta_1 = beta_1 104 | self.beta_2 = beta_2 105 | self.epsilon = epsilon 106 | self.exclude_from_weight_decay = exclude_from_weight_decay 107 | 108 | def apply_gradients(self, grads_and_vars, global_step=None, name=None): 109 | """See base class.""" 110 | assignments = [] 111 | for (grad, param) in grads_and_vars: 112 | if grad is None or param is None: 113 | continue 114 | 115 | param_name = self._get_variable_name(param.name) 116 | 117 | m = tf.get_variable( 118 | name=param_name + "/adam_m", 119 | shape=param.shape.as_list(), 120 | dtype=tf.float32, 121 | trainable=False, 122 | initializer=tf.zeros_initializer()) 123 | v = tf.get_variable( 124 | name=param_name + "/adam_v", 125 | shape=param.shape.as_list(), 126 | dtype=tf.float32, 127 | trainable=False, 128 | initializer=tf.zeros_initializer()) 129 | 130 | # Standard Adam update. 131 | next_m = ( 132 | tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad)) 133 | next_v = ( 134 | tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2, 135 | tf.square(grad))) 136 | 137 | update = next_m / (tf.sqrt(next_v) + self.epsilon) 138 | 139 | # Just adding the square of the weights to the loss function is *not* 140 | # the correct way of using L2 regularization/weight decay with Adam, 141 | # since that will interact with the m and v parameters in strange ways. 142 | # 143 | # Instead we want ot decay the weights in a manner that doesn't interact 144 | # with the m/v parameters. This is equivalent to adding the square 145 | # of the weights to the loss with plain (non-momentum) SGD. 146 | if self._do_use_weight_decay(param_name): 147 | update += self.weight_decay_rate * param 148 | 149 | update_with_lr = self.learning_rate * update 150 | 151 | next_param = param - update_with_lr 152 | 153 | assignments.extend( 154 | [param.assign(next_param), 155 | m.assign(next_m), 156 | v.assign(next_v)]) 157 | return tf.group(*assignments, name=name) 158 | 159 | def _do_use_weight_decay(self, param_name): 160 | """Whether to use L2 weight decay for `param_name`.""" 161 | if not self.weight_decay_rate: 162 | return False 163 | if self.exclude_from_weight_decay: 164 | for r in self.exclude_from_weight_decay: 165 | if re.search(r, param_name) is not None: 166 | return False 167 | return True 168 | 169 | def _get_variable_name(self, param_name): 170 | """Get the variable name from the tensor name.""" 171 | m = re.match("^(.*):\\d+$", param_name) 172 | if m is not None: 173 | param_name = m.group(1) 174 | return param_name 175 | -------------------------------------------------------------------------------- /bert/optimization_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import optimization 20 | import tensorflow as tf 21 | 22 | 23 | class OptimizationTest(tf.test.TestCase): 24 | 25 | def test_adam(self): 26 | with self.test_session() as sess: 27 | w = tf.get_variable( 28 | "w", 29 | shape=[3], 30 | initializer=tf.constant_initializer([0.1, -0.2, -0.1])) 31 | x = tf.constant([0.4, 0.2, -0.5]) 32 | loss = tf.reduce_mean(tf.square(x - w)) 33 | tvars = tf.trainable_variables() 34 | grads = tf.gradients(loss, tvars) 35 | global_step = tf.train.get_or_create_global_step() 36 | optimizer = optimization.AdamWeightDecayOptimizer(learning_rate=0.2) 37 | train_op = optimizer.apply_gradients(zip(grads, tvars), global_step) 38 | init_op = tf.group(tf.global_variables_initializer(), 39 | tf.local_variables_initializer()) 40 | sess.run(init_op) 41 | for _ in range(100): 42 | sess.run(train_op) 43 | w_np = sess.run(w) 44 | self.assertAllClose(w_np.flat, [0.4, 0.2, -0.5], rtol=1e-2, atol=1e-2) 45 | 46 | 47 | if __name__ == "__main__": 48 | tf.test.main() 49 | -------------------------------------------------------------------------------- /bert/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow >= 1.11.0 # CPU Version of TensorFlow. 2 | # tensorflow-gpu >= 1.11.0 # GPU version of TensorFlow. 3 | -------------------------------------------------------------------------------- /bert/run_classifier_with_tfhub.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """BERT finetuning runner with TF-Hub.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import os 22 | import optimization 23 | import run_classifier 24 | import tokenization 25 | import tensorflow as tf 26 | import tensorflow_hub as hub 27 | 28 | flags = tf.flags 29 | 30 | FLAGS = flags.FLAGS 31 | 32 | flags.DEFINE_string( 33 | "bert_hub_module_handle", None, 34 | "Handle for the BERT TF-Hub module.") 35 | 36 | 37 | def create_model(is_training, input_ids, input_mask, segment_ids, labels, 38 | num_labels, bert_hub_module_handle): 39 | """Creates a classification model.""" 40 | tags = set() 41 | if is_training: 42 | tags.add("train") 43 | bert_module = hub.Module(bert_hub_module_handle, tags=tags, trainable=True) 44 | bert_inputs = dict( 45 | input_ids=input_ids, 46 | input_mask=input_mask, 47 | segment_ids=segment_ids) 48 | bert_outputs = bert_module( 49 | inputs=bert_inputs, 50 | signature="tokens", 51 | as_dict=True) 52 | 53 | # In the demo, we are doing a simple classification task on the entire 54 | # segment. 55 | # 56 | # If you want to use the token-level output, use 57 | # bert_outputs["sequence_output"] instead. 58 | output_layer = bert_outputs["pooled_output"] 59 | 60 | hidden_size = output_layer.shape[-1].value 61 | 62 | output_weights = tf.get_variable( 63 | "output_weights", [num_labels, hidden_size], 64 | initializer=tf.truncated_normal_initializer(stddev=0.02)) 65 | 66 | output_bias = tf.get_variable( 67 | "output_bias", [num_labels], initializer=tf.zeros_initializer()) 68 | 69 | with tf.variable_scope("loss"): 70 | if is_training: 71 | # I.e., 0.1 dropout 72 | output_layer = tf.nn.dropout(output_layer, keep_prob=0.9) 73 | 74 | logits = tf.matmul(output_layer, output_weights, transpose_b=True) 75 | logits = tf.nn.bias_add(logits, output_bias) 76 | probabilities = tf.nn.softmax(logits, axis=-1) 77 | log_probs = tf.nn.log_softmax(logits, axis=-1) 78 | 79 | one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32) 80 | 81 | per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) 82 | loss = tf.reduce_mean(per_example_loss) 83 | 84 | return (loss, per_example_loss, logits, probabilities) 85 | 86 | 87 | def model_fn_builder(num_labels, learning_rate, num_train_steps, 88 | num_warmup_steps, use_tpu, bert_hub_module_handle): 89 | """Returns `model_fn` closure for TPUEstimator.""" 90 | 91 | def model_fn(features, labels, mode, params): # pylint: disable=unused-argument 92 | """The `model_fn` for TPUEstimator.""" 93 | 94 | tf.logging.info("*** Features ***") 95 | for name in sorted(features.keys()): 96 | tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape)) 97 | 98 | input_ids = features["input_ids"] 99 | input_mask = features["input_mask"] 100 | segment_ids = features["segment_ids"] 101 | label_ids = features["label_ids"] 102 | 103 | is_training = (mode == tf.estimator.ModeKeys.TRAIN) 104 | 105 | (total_loss, per_example_loss, logits, probabilities) = create_model( 106 | is_training, input_ids, input_mask, segment_ids, label_ids, num_labels, 107 | bert_hub_module_handle) 108 | 109 | output_spec = None 110 | if mode == tf.estimator.ModeKeys.TRAIN: 111 | train_op = optimization.create_optimizer( 112 | total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu) 113 | 114 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 115 | mode=mode, 116 | loss=total_loss, 117 | train_op=train_op) 118 | elif mode == tf.estimator.ModeKeys.EVAL: 119 | 120 | def metric_fn(per_example_loss, label_ids, logits): 121 | predictions = tf.argmax(logits, axis=-1, output_type=tf.int32) 122 | accuracy = tf.metrics.accuracy(label_ids, predictions) 123 | loss = tf.metrics.mean(per_example_loss) 124 | return { 125 | "eval_accuracy": accuracy, 126 | "eval_loss": loss, 127 | } 128 | 129 | eval_metrics = (metric_fn, [per_example_loss, label_ids, logits]) 130 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 131 | mode=mode, 132 | loss=total_loss, 133 | eval_metrics=eval_metrics) 134 | elif mode == tf.estimator.ModeKeys.PREDICT: 135 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 136 | mode=mode, predictions={"probabilities": probabilities}) 137 | else: 138 | raise ValueError( 139 | "Only TRAIN, EVAL and PREDICT modes are supported: %s" % (mode)) 140 | 141 | return output_spec 142 | 143 | return model_fn 144 | 145 | 146 | def create_tokenizer_from_hub_module(bert_hub_module_handle): 147 | """Get the vocab file and casing info from the Hub module.""" 148 | with tf.Graph().as_default(): 149 | bert_module = hub.Module(bert_hub_module_handle) 150 | tokenization_info = bert_module(signature="tokenization_info", as_dict=True) 151 | with tf.Session() as sess: 152 | vocab_file, do_lower_case = sess.run([tokenization_info["vocab_file"], 153 | tokenization_info["do_lower_case"]]) 154 | return tokenization.FullTokenizer( 155 | vocab_file=vocab_file, do_lower_case=do_lower_case) 156 | 157 | 158 | def main(_): 159 | tf.logging.set_verbosity(tf.logging.INFO) 160 | 161 | processors = { 162 | "cola": run_classifier.ColaProcessor, 163 | "mnli": run_classifier.MnliProcessor, 164 | "mrpc": run_classifier.MrpcProcessor, 165 | } 166 | 167 | if not FLAGS.do_train and not FLAGS.do_eval: 168 | raise ValueError("At least one of `do_train` or `do_eval` must be True.") 169 | 170 | tf.gfile.MakeDirs(FLAGS.output_dir) 171 | 172 | task_name = FLAGS.task_name.lower() 173 | 174 | if task_name not in processors: 175 | raise ValueError("Task not found: %s" % (task_name)) 176 | 177 | processor = processors[task_name]() 178 | 179 | label_list = processor.get_labels() 180 | 181 | tokenizer = create_tokenizer_from_hub_module(FLAGS.bert_hub_module_handle) 182 | 183 | tpu_cluster_resolver = None 184 | if FLAGS.use_tpu and FLAGS.tpu_name: 185 | tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( 186 | FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) 187 | 188 | is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 189 | run_config = tf.contrib.tpu.RunConfig( 190 | cluster=tpu_cluster_resolver, 191 | master=FLAGS.master, 192 | model_dir=FLAGS.output_dir, 193 | save_checkpoints_steps=FLAGS.save_checkpoints_steps, 194 | tpu_config=tf.contrib.tpu.TPUConfig( 195 | iterations_per_loop=FLAGS.iterations_per_loop, 196 | num_shards=FLAGS.num_tpu_cores, 197 | per_host_input_for_training=is_per_host)) 198 | 199 | train_examples = None 200 | num_train_steps = None 201 | num_warmup_steps = None 202 | if FLAGS.do_train: 203 | train_examples = processor.get_train_examples(FLAGS.data_dir) 204 | num_train_steps = int( 205 | len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) 206 | num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) 207 | 208 | model_fn = model_fn_builder( 209 | num_labels=len(label_list), 210 | learning_rate=FLAGS.learning_rate, 211 | num_train_steps=num_train_steps, 212 | num_warmup_steps=num_warmup_steps, 213 | use_tpu=FLAGS.use_tpu, 214 | bert_hub_module_handle=FLAGS.bert_hub_module_handle) 215 | 216 | # If TPU is not available, this will fall back to normal Estimator on CPU 217 | # or GPU. 218 | estimator = tf.contrib.tpu.TPUEstimator( 219 | use_tpu=FLAGS.use_tpu, 220 | model_fn=model_fn, 221 | config=run_config, 222 | train_batch_size=FLAGS.train_batch_size, 223 | eval_batch_size=FLAGS.eval_batch_size, 224 | predict_batch_size=FLAGS.predict_batch_size) 225 | 226 | if FLAGS.do_train: 227 | train_features = run_classifier.convert_examples_to_features( 228 | train_examples, label_list, FLAGS.max_seq_length, tokenizer) 229 | tf.logging.info("***** Running training *****") 230 | tf.logging.info(" Num examples = %d", len(train_examples)) 231 | tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) 232 | tf.logging.info(" Num steps = %d", num_train_steps) 233 | train_input_fn = run_classifier.input_fn_builder( 234 | features=train_features, 235 | seq_length=FLAGS.max_seq_length, 236 | is_training=True, 237 | drop_remainder=True) 238 | estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) 239 | 240 | if FLAGS.do_eval: 241 | eval_examples = processor.get_dev_examples(FLAGS.data_dir) 242 | eval_features = run_classifier.convert_examples_to_features( 243 | eval_examples, label_list, FLAGS.max_seq_length, tokenizer) 244 | 245 | tf.logging.info("***** Running evaluation *****") 246 | tf.logging.info(" Num examples = %d", len(eval_examples)) 247 | tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size) 248 | 249 | # This tells the estimator to run through the entire set. 250 | eval_steps = None 251 | # However, if running eval on the TPU, you will need to specify the 252 | # number of steps. 253 | if FLAGS.use_tpu: 254 | # Eval will be slightly WRONG on the TPU because it will truncate 255 | # the last batch. 256 | eval_steps = int(len(eval_examples) / FLAGS.eval_batch_size) 257 | 258 | eval_drop_remainder = True if FLAGS.use_tpu else False 259 | eval_input_fn = run_classifier.input_fn_builder( 260 | features=eval_features, 261 | seq_length=FLAGS.max_seq_length, 262 | is_training=False, 263 | drop_remainder=eval_drop_remainder) 264 | 265 | result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) 266 | 267 | output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") 268 | with tf.gfile.GFile(output_eval_file, "w") as writer: 269 | tf.logging.info("***** Eval results *****") 270 | for key in sorted(result.keys()): 271 | tf.logging.info(" %s = %s", key, str(result[key])) 272 | writer.write("%s = %s\n" % (key, str(result[key]))) 273 | 274 | if FLAGS.do_predict: 275 | predict_examples = processor.get_test_examples(FLAGS.data_dir) 276 | if FLAGS.use_tpu: 277 | # Discard batch remainder if running on TPU 278 | n = len(predict_examples) 279 | predict_examples = predict_examples[:(n - n % FLAGS.predict_batch_size)] 280 | 281 | predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record") 282 | run_classifier.file_based_convert_examples_to_features( 283 | predict_examples, label_list, FLAGS.max_seq_length, tokenizer, 284 | predict_file) 285 | 286 | tf.logging.info("***** Running prediction*****") 287 | tf.logging.info(" Num examples = %d", len(predict_examples)) 288 | tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size) 289 | 290 | predict_input_fn = run_classifier.file_based_input_fn_builder( 291 | input_file=predict_file, 292 | seq_length=FLAGS.max_seq_length, 293 | is_training=False, 294 | drop_remainder=FLAGS.use_tpu) 295 | 296 | result = estimator.predict(input_fn=predict_input_fn) 297 | 298 | output_predict_file = os.path.join(FLAGS.output_dir, "test_results.tsv") 299 | with tf.gfile.GFile(output_predict_file, "w") as writer: 300 | tf.logging.info("***** Predict results *****") 301 | for prediction in result: 302 | probabilities = prediction["probabilities"] 303 | output_line = "\t".join( 304 | str(class_probability) 305 | for class_probability in probabilities) + "\n" 306 | writer.write(output_line) 307 | 308 | 309 | if __name__ == "__main__": 310 | flags.mark_flag_as_required("data_dir") 311 | flags.mark_flag_as_required("task_name") 312 | flags.mark_flag_as_required("bert_hub_module_handle") 313 | flags.mark_flag_as_required("output_dir") 314 | tf.app.run() 315 | -------------------------------------------------------------------------------- /bert/sample_text.txt: -------------------------------------------------------------------------------- 1 | This text is included to make sure Unicode is handled properly: 力加勝北区ᴵᴺᵀᵃছজটডণত 2 | Text should be one-sentence-per-line, with empty lines between documents. 3 | This sample text is public domain and was randomly selected from Project Guttenberg. 4 | 5 | The rain had only ceased with the gray streaks of morning at Blazing Star, and the settlement awoke to a moral sense of cleanliness, and the finding of forgotten knives, tin cups, and smaller camp utensils, where the heavy showers had washed away the debris and dust heaps before the cabin doors. 6 | Indeed, it was recorded in Blazing Star that a fortunate early riser had once picked up on the highway a solid chunk of gold quartz which the rain had freed from its incumbering soil, and washed into immediate and glittering popularity. 7 | Possibly this may have been the reason why early risers in that locality, during the rainy season, adopted a thoughtful habit of body, and seldom lifted their eyes to the rifted or india-ink washed skies above them. 8 | "Cass" Beard had risen early that morning, but not with a view to discovery. 9 | A leak in his cabin roof,--quite consistent with his careless, improvident habits,--had roused him at 4 A. M., with a flooded "bunk" and wet blankets. 10 | The chips from his wood pile refused to kindle a fire to dry his bed-clothes, and he had recourse to a more provident neighbor's to supply the deficiency. 11 | This was nearly opposite. 12 | Mr. Cassius crossed the highway, and stopped suddenly. 13 | Something glittered in the nearest red pool before him. 14 | Gold, surely! 15 | But, wonderful to relate, not an irregular, shapeless fragment of crude ore, fresh from Nature's crucible, but a bit of jeweler's handicraft in the form of a plain gold ring. 16 | Looking at it more attentively, he saw that it bore the inscription, "May to Cass." 17 | Like most of his fellow gold-seekers, Cass was superstitious. 18 | 19 | The fountain of classic wisdom, Hypatia herself. 20 | As the ancient sage--the name is unimportant to a monk--pumped water nightly that he might study by day, so I, the guardian of cloaks and parasols, at the sacred doors of her lecture-room, imbibe celestial knowledge. 21 | From my youth I felt in me a soul above the matter-entangled herd. 22 | She revealed to me the glorious fact, that I am a spark of Divinity itself. 23 | A fallen star, I am, sir!' continued he, pensively, stroking his lean stomach--'a fallen star!--fallen, if the dignity of philosophy will allow of the simile, among the hogs of the lower world--indeed, even into the hog-bucket itself. Well, after all, I will show you the way to the Archbishop's. 24 | There is a philosophic pleasure in opening one's treasures to the modest young. 25 | Perhaps you will assist me by carrying this basket of fruit?' And the little man jumped up, put his basket on Philammon's head, and trotted off up a neighbouring street. 26 | Philammon followed, half contemptuous, half wondering at what this philosophy might be, which could feed the self-conceit of anything so abject as his ragged little apish guide; 27 | but the novel roar and whirl of the street, the perpetual stream of busy faces, the line of curricles, palanquins, laden asses, camels, elephants, which met and passed him, and squeezed him up steps and into doorways, as they threaded their way through the great Moon-gate into the ample street beyond, drove everything from his mind but wondering curiosity, and a vague, helpless dread of that great living wilderness, more terrible than any dead wilderness of sand which he had left behind. 28 | Already he longed for the repose, the silence of the Laura--for faces which knew him and smiled upon him; but it was too late to turn back now. 29 | His guide held on for more than a mile up the great main street, crossed in the centre of the city, at right angles, by one equally magnificent, at each end of which, miles away, appeared, dim and distant over the heads of the living stream of passengers, the yellow sand-hills of the desert; 30 | while at the end of the vista in front of them gleamed the blue harbour, through a network of countless masts. 31 | At last they reached the quay at the opposite end of the street; 32 | and there burst on Philammon's astonished eyes a vast semicircle of blue sea, ringed with palaces and towers. 33 | He stopped involuntarily; and his little guide stopped also, and looked askance at the young monk, to watch the effect which that grand panorama should produce on him. 34 | -------------------------------------------------------------------------------- /bert/tokenization.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Tokenization classes.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import collections 22 | import re 23 | import unicodedata 24 | import six 25 | import tensorflow as tf 26 | 27 | 28 | def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): 29 | """Checks whether the casing config is consistent with the checkpoint name.""" 30 | 31 | # The casing has to be passed in by the user and there is no explicit check 32 | # as to whether it matches the checkpoint. The casing information probably 33 | # should have been stored in the bert_config.json file, but it's not, so 34 | # we have to heuristically detect it to validate. 35 | 36 | if not init_checkpoint: 37 | return 38 | 39 | m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) 40 | if m is None: 41 | return 42 | 43 | model_name = m.group(1) 44 | 45 | lower_models = [ 46 | "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", 47 | "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" 48 | ] 49 | 50 | cased_models = [ 51 | "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", 52 | "multi_cased_L-12_H-768_A-12" 53 | ] 54 | 55 | is_bad_config = False 56 | if model_name in lower_models and not do_lower_case: 57 | is_bad_config = True 58 | actual_flag = "False" 59 | case_name = "lowercased" 60 | opposite_flag = "True" 61 | 62 | if model_name in cased_models and do_lower_case: 63 | is_bad_config = True 64 | actual_flag = "True" 65 | case_name = "cased" 66 | opposite_flag = "False" 67 | 68 | if is_bad_config: 69 | raise ValueError( 70 | "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " 71 | "However, `%s` seems to be a %s model, so you " 72 | "should pass in `--do_lower_case=%s` so that the fine-tuning matches " 73 | "how the model was pre-training. If this error is wrong, please " 74 | "just comment out this check." % (actual_flag, init_checkpoint, 75 | model_name, case_name, opposite_flag)) 76 | 77 | 78 | def convert_to_unicode(text): 79 | """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" 80 | if six.PY3: 81 | if isinstance(text, str): 82 | return text 83 | elif isinstance(text, bytes): 84 | return text.decode("utf-8", "ignore") 85 | else: 86 | raise ValueError("Unsupported string type: %s" % (type(text))) 87 | elif six.PY2: 88 | if isinstance(text, str): 89 | return text.decode("utf-8", "ignore") 90 | elif isinstance(text, unicode): 91 | return text 92 | else: 93 | raise ValueError("Unsupported string type: %s" % (type(text))) 94 | else: 95 | raise ValueError("Not running on Python2 or Python 3?") 96 | 97 | 98 | def printable_text(text): 99 | """Returns text encoded in a way suitable for print or `tf.logging`.""" 100 | 101 | # These functions want `str` for both Python2 and Python3, but in one case 102 | # it's a Unicode string and in the other it's a byte string. 103 | if six.PY3: 104 | if isinstance(text, str): 105 | return text 106 | elif isinstance(text, bytes): 107 | return text.decode("utf-8", "ignore") 108 | else: 109 | raise ValueError("Unsupported string type: %s" % (type(text))) 110 | elif six.PY2: 111 | if isinstance(text, str): 112 | return text 113 | elif isinstance(text, unicode): 114 | return text.encode("utf-8") 115 | else: 116 | raise ValueError("Unsupported string type: %s" % (type(text))) 117 | else: 118 | raise ValueError("Not running on Python2 or Python 3?") 119 | 120 | 121 | def load_vocab(vocab_file): 122 | """Loads a vocabulary file into a dictionary.""" 123 | vocab = collections.OrderedDict() 124 | index = 0 125 | with tf.gfile.GFile(vocab_file, "r") as reader: 126 | while True: 127 | token = convert_to_unicode(reader.readline()) 128 | if not token: 129 | break 130 | token = token.strip() 131 | vocab[token] = index 132 | index += 1 133 | return vocab 134 | 135 | 136 | def convert_by_vocab(vocab, items): 137 | """Converts a sequence of [tokens|ids] using the vocab.""" 138 | output = [] 139 | for item in items: 140 | output.append(vocab[item]) 141 | return output 142 | 143 | 144 | def convert_tokens_to_ids(vocab, tokens): 145 | return convert_by_vocab(vocab, tokens) 146 | 147 | 148 | def convert_ids_to_tokens(inv_vocab, ids): 149 | return convert_by_vocab(inv_vocab, ids) 150 | 151 | 152 | def whitespace_tokenize(text): 153 | """Runs basic whitespace cleaning and splitting on a piece of text.""" 154 | text = text.strip() 155 | if not text: 156 | return [] 157 | tokens = text.split() 158 | return tokens 159 | 160 | 161 | class FullTokenizer(object): 162 | """Runs end-to-end tokenziation.""" 163 | 164 | def __init__(self, vocab_file, do_lower_case=True): 165 | self.vocab = load_vocab(vocab_file) 166 | self.inv_vocab = {v: k for k, v in self.vocab.items()} 167 | self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) 168 | self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) 169 | self.wordpiece_tokenizer_not_UNK = WordpieceTokenizer_not_UNK(vocab=self.vocab) 170 | 171 | def tokenize(self, text): 172 | split_tokens = [] 173 | for token in self.basic_tokenizer.tokenize(text): 174 | for sub_token in self.wordpiece_tokenizer.tokenize(token): 175 | split_tokens.append(sub_token) 176 | 177 | return split_tokens 178 | 179 | def tokenize_not_UNK(self, text): 180 | split_tokens = [] 181 | for token in self.basic_tokenizer.tokenize(text): 182 | for sub_token in self.wordpiece_tokenizer_not_UNK.tokenize(token): 183 | split_tokens.append(sub_token) 184 | 185 | return split_tokens 186 | 187 | def convert_tokens_to_ids(self, tokens): 188 | return convert_by_vocab(self.vocab, tokens) 189 | 190 | def convert_ids_to_tokens(self, ids): 191 | return convert_by_vocab(self.inv_vocab, ids) 192 | 193 | 194 | class BasicTokenizer(object): 195 | """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" 196 | 197 | def __init__(self, do_lower_case=True): 198 | """Constructs a BasicTokenizer. 199 | 200 | Args: 201 | do_lower_case: Whether to lower case the input. 202 | """ 203 | self.do_lower_case = do_lower_case 204 | 205 | def tokenize(self, text): 206 | """Tokenizes a piece of text.""" 207 | text = convert_to_unicode(text) 208 | text = self._clean_text(text) 209 | 210 | # This was added on November 1st, 2018 for the multilingual and Chinese 211 | # models. This is also applied to the English models now, but it doesn't 212 | # matter since the English models were not trained on any Chinese data 213 | # and generally don't have any Chinese data in them (there are Chinese 214 | # characters in the vocabulary because Wikipedia does have some Chinese 215 | # words in the English Wikipedia.). 216 | text = self._tokenize_chinese_chars(text) 217 | 218 | orig_tokens = whitespace_tokenize(text) 219 | split_tokens = [] 220 | for token in orig_tokens: 221 | if self.do_lower_case: 222 | token = token.lower() 223 | token = self._run_strip_accents(token) 224 | split_tokens.extend(self._run_split_on_punc(token)) 225 | 226 | output_tokens = whitespace_tokenize(" ".join(split_tokens)) 227 | return output_tokens 228 | 229 | def _run_strip_accents(self, text): 230 | """Strips accents from a piece of text.""" 231 | text = unicodedata.normalize("NFD", text) 232 | output = [] 233 | for char in text: 234 | cat = unicodedata.category(char) 235 | if cat == "Mn": 236 | continue 237 | output.append(char) 238 | return "".join(output) 239 | 240 | def _run_split_on_punc(self, text): 241 | """Splits punctuation on a piece of text.""" 242 | chars = list(text) 243 | i = 0 244 | start_new_word = True 245 | output = [] 246 | while i < len(chars): 247 | char = chars[i] 248 | if _is_punctuation(char): 249 | output.append([char]) 250 | start_new_word = True 251 | else: 252 | if start_new_word: 253 | output.append([]) 254 | start_new_word = False 255 | output[-1].append(char) 256 | i += 1 257 | 258 | return ["".join(x) for x in output] 259 | 260 | def _tokenize_chinese_chars(self, text): 261 | """Adds whitespace around any CJK character.""" 262 | output = [] 263 | for char in text: 264 | cp = ord(char) 265 | if self._is_chinese_char(cp): 266 | output.append(" ") 267 | output.append(char) 268 | output.append(" ") 269 | else: 270 | output.append(char) 271 | return "".join(output) 272 | 273 | def _is_chinese_char(self, cp): 274 | """Checks whether CP is the codepoint of a CJK character.""" 275 | # This defines a "chinese character" as anything in the CJK Unicode block: 276 | # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) 277 | # 278 | # Note that the CJK Unicode block is NOT all Japanese and Korean characters, 279 | # despite its name. The modern Korean Hangul alphabet is a different block, 280 | # as is Japanese Hiragana and Katakana. Those alphabets are used to write 281 | # space-separated words, so they are not treated specially and handled 282 | # like the all of the other languages. 283 | if ((cp >= 0x4E00 and cp <= 0x9FFF) or # 284 | (cp >= 0x3400 and cp <= 0x4DBF) or # 285 | (cp >= 0x20000 and cp <= 0x2A6DF) or # 286 | (cp >= 0x2A700 and cp <= 0x2B73F) or # 287 | (cp >= 0x2B740 and cp <= 0x2B81F) or # 288 | (cp >= 0x2B820 and cp <= 0x2CEAF) or 289 | (cp >= 0xF900 and cp <= 0xFAFF) or # 290 | (cp >= 0x2F800 and cp <= 0x2FA1F)): # 291 | return True 292 | 293 | return False 294 | 295 | def _clean_text(self, text): 296 | """Performs invalid character removal and whitespace cleanup on text.""" 297 | output = [] 298 | for char in text: 299 | cp = ord(char) 300 | if cp == 0 or cp == 0xfffd or _is_control(char): 301 | continue 302 | if _is_whitespace(char): 303 | output.append(" ") 304 | else: 305 | output.append(char) 306 | return "".join(output) 307 | 308 | 309 | class WordpieceTokenizer_not_UNK(object): 310 | """Runs WordPiece tokenziation.""" 311 | def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): 312 | self.vocab = vocab 313 | self.unk_token = unk_token 314 | self.max_input_chars_per_word = max_input_chars_per_word 315 | 316 | def tokenize(self, text): 317 | """Tokenizes a piece of text into its word pieces. 318 | 319 | This uses a greedy longest-match-first algorithm to perform tokenization 320 | using the given vocabulary. 321 | 322 | For example: 323 | input = "unaffable" 324 | output = ["un", "##aff", "##able"] 325 | 326 | Args: 327 | text: A single token or whitespace separated tokens. This should have 328 | already been passed through `BasicTokenizer. 329 | 330 | Returns: 331 | A list of wordpiece tokens. 332 | """ 333 | 334 | text = convert_to_unicode(text) 335 | 336 | output_tokens = [] 337 | for token in whitespace_tokenize(text): 338 | chars = list(token) 339 | if len(chars) > self.max_input_chars_per_word: 340 | output_tokens.append(self.unk_token) 341 | continue 342 | 343 | is_bad = False 344 | start = 0 345 | sub_tokens = [] 346 | while start < len(chars): 347 | end = len(chars) 348 | cur_substr = None 349 | while start < end: 350 | substr = "".join(chars[start:end]) 351 | if start > 0: 352 | substr = "##" + substr 353 | if substr in self.vocab: 354 | cur_substr = substr 355 | break 356 | end -= 1 357 | if cur_substr is None: 358 | is_bad = True 359 | break 360 | sub_tokens.append(cur_substr) 361 | start = end 362 | 363 | if is_bad: 364 | #output_tokens.append(self.unk_token) 365 | output_tokens.append(token) 366 | else: 367 | output_tokens.extend(sub_tokens) 368 | return output_tokens 369 | 370 | 371 | class WordpieceTokenizer(object): 372 | """Runs WordPiece tokenziation.""" 373 | 374 | def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): 375 | self.vocab = vocab 376 | self.unk_token = unk_token 377 | self.max_input_chars_per_word = max_input_chars_per_word 378 | 379 | def tokenize(self, text): 380 | """Tokenizes a piece of text into its word pieces. 381 | 382 | This uses a greedy longest-match-first algorithm to perform tokenization 383 | using the given vocabulary. 384 | 385 | For example: 386 | input = "unaffable" 387 | output = ["un", "##aff", "##able"] 388 | 389 | Args: 390 | text: A single token or whitespace separated tokens. This should have 391 | already been passed through `BasicTokenizer. 392 | 393 | Returns: 394 | A list of wordpiece tokens. 395 | """ 396 | 397 | text = convert_to_unicode(text) 398 | 399 | output_tokens = [] 400 | for token in whitespace_tokenize(text): 401 | chars = list(token) 402 | if len(chars) > self.max_input_chars_per_word: 403 | output_tokens.append(self.unk_token) 404 | continue 405 | 406 | is_bad = False 407 | start = 0 408 | sub_tokens = [] 409 | while start < len(chars): 410 | end = len(chars) 411 | cur_substr = None 412 | while start < end: 413 | substr = "".join(chars[start:end]) 414 | if start > 0: 415 | substr = "##" + substr 416 | if substr in self.vocab: 417 | cur_substr = substr 418 | break 419 | end -= 1 420 | if cur_substr is None: 421 | is_bad = True 422 | break 423 | sub_tokens.append(cur_substr) 424 | start = end 425 | 426 | if is_bad: 427 | output_tokens.append(self.unk_token) 428 | else: 429 | output_tokens.extend(sub_tokens) 430 | return output_tokens 431 | 432 | 433 | def _is_whitespace(char): 434 | """Checks whether `chars` is a whitespace character.""" 435 | # \t, \n, and \r are technically contorl characters but we treat them 436 | # as whitespace since they are generally considered as such. 437 | if char == " " or char == "\t" or char == "\n" or char == "\r": 438 | return True 439 | cat = unicodedata.category(char) 440 | if cat == "Zs": 441 | return True 442 | return False 443 | 444 | 445 | def _is_control(char): 446 | """Checks whether `chars` is a control character.""" 447 | # These are technically control characters but we count them as whitespace 448 | # characters. 449 | if char == "\t" or char == "\n" or char == "\r": 450 | return False 451 | cat = unicodedata.category(char) 452 | if cat.startswith("C"): 453 | return True 454 | return False 455 | 456 | 457 | def _is_punctuation(char): 458 | """Checks whether `chars` is a punctuation character.""" 459 | cp = ord(char) 460 | # We treat all non-letter/number ASCII as punctuation. 461 | # Characters such as "^", "$", and "`" are not in the Unicode 462 | # Punctuation class but we treat them as punctuation anyways, for 463 | # consistency. 464 | if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or 465 | (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): 466 | return True 467 | cat = unicodedata.category(char) 468 | if cat.startswith("P"): 469 | return True 470 | return False 471 | -------------------------------------------------------------------------------- /bert/tokenization_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import os 20 | import tempfile 21 | import tokenization 22 | import six 23 | import tensorflow as tf 24 | 25 | 26 | class TokenizationTest(tf.test.TestCase): 27 | 28 | def test_full_tokenizer(self): 29 | vocab_tokens = [ 30 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 31 | "##ing", "," 32 | ] 33 | with tempfile.NamedTemporaryFile(delete=False) as vocab_writer: 34 | if six.PY2: 35 | vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) 36 | else: 37 | vocab_writer.write("".join( 38 | [x + "\n" for x in vocab_tokens]).encode("utf-8")) 39 | 40 | vocab_file = vocab_writer.name 41 | 42 | tokenizer = tokenization.FullTokenizer(vocab_file) 43 | os.unlink(vocab_file) 44 | 45 | tokens = tokenizer.tokenize(u"UNwant\u00E9d,running") 46 | self.assertAllEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"]) 47 | 48 | self.assertAllEqual( 49 | tokenizer.convert_tokens_to_ids(tokens), [7, 4, 5, 10, 8, 9]) 50 | 51 | def test_chinese(self): 52 | tokenizer = tokenization.BasicTokenizer() 53 | 54 | self.assertAllEqual( 55 | tokenizer.tokenize(u"ah\u535A\u63A8zz"), 56 | [u"ah", u"\u535A", u"\u63A8", u"zz"]) 57 | 58 | def test_basic_tokenizer_lower(self): 59 | tokenizer = tokenization.BasicTokenizer(do_lower_case=True) 60 | 61 | self.assertAllEqual( 62 | tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), 63 | ["hello", "!", "how", "are", "you", "?"]) 64 | self.assertAllEqual(tokenizer.tokenize(u"H\u00E9llo"), ["hello"]) 65 | 66 | def test_basic_tokenizer_no_lower(self): 67 | tokenizer = tokenization.BasicTokenizer(do_lower_case=False) 68 | 69 | self.assertAllEqual( 70 | tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), 71 | ["HeLLo", "!", "how", "Are", "yoU", "?"]) 72 | 73 | def test_wordpiece_tokenizer(self): 74 | vocab_tokens = [ 75 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 76 | "##ing" 77 | ] 78 | 79 | vocab = {} 80 | for (i, token) in enumerate(vocab_tokens): 81 | vocab[token] = i 82 | tokenizer = tokenization.WordpieceTokenizer(vocab=vocab) 83 | 84 | self.assertAllEqual(tokenizer.tokenize(""), []) 85 | 86 | self.assertAllEqual( 87 | tokenizer.tokenize("unwanted running"), 88 | ["un", "##want", "##ed", "runn", "##ing"]) 89 | 90 | self.assertAllEqual( 91 | tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) 92 | 93 | def test_convert_tokens_to_ids(self): 94 | vocab_tokens = [ 95 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 96 | "##ing" 97 | ] 98 | 99 | vocab = {} 100 | for (i, token) in enumerate(vocab_tokens): 101 | vocab[token] = i 102 | 103 | self.assertAllEqual( 104 | tokenization.convert_tokens_to_ids( 105 | vocab, ["un", "##want", "##ed", "runn", "##ing"]), [7, 4, 5, 8, 9]) 106 | 107 | def test_is_whitespace(self): 108 | self.assertTrue(tokenization._is_whitespace(u" ")) 109 | self.assertTrue(tokenization._is_whitespace(u"\t")) 110 | self.assertTrue(tokenization._is_whitespace(u"\r")) 111 | self.assertTrue(tokenization._is_whitespace(u"\n")) 112 | self.assertTrue(tokenization._is_whitespace(u"\u00A0")) 113 | 114 | self.assertFalse(tokenization._is_whitespace(u"A")) 115 | self.assertFalse(tokenization._is_whitespace(u"-")) 116 | 117 | def test_is_control(self): 118 | self.assertTrue(tokenization._is_control(u"\u0005")) 119 | 120 | self.assertFalse(tokenization._is_control(u"A")) 121 | self.assertFalse(tokenization._is_control(u" ")) 122 | self.assertFalse(tokenization._is_control(u"\t")) 123 | self.assertFalse(tokenization._is_control(u"\r")) 124 | self.assertFalse(tokenization._is_control(u"\U0001F4A9")) 125 | 126 | def test_is_punctuation(self): 127 | self.assertTrue(tokenization._is_punctuation(u"-")) 128 | self.assertTrue(tokenization._is_punctuation(u"$")) 129 | self.assertTrue(tokenization._is_punctuation(u"`")) 130 | self.assertTrue(tokenization._is_punctuation(u".")) 131 | 132 | self.assertFalse(tokenization._is_punctuation(u"A")) 133 | self.assertFalse(tokenization._is_punctuation(u" ")) 134 | 135 | 136 | if __name__ == "__main__": 137 | tf.test.main() 138 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/30 16:02 4 | # @Author : 刘鑫 5 | # @FileName: config.py 6 | # @Software: PyCharm 7 | import os 8 | 9 | 10 | class config(object): 11 | PROJECT_DIR = os.path.dirname(__file__) 12 | -------------------------------------------------------------------------------- /pretraining_model/readme.txt: -------------------------------------------------------------------------------- 1 | # 这里存放预训练模型 2 | # 直接将下载好的预训练模型解压放在这里即可 3 | # 尽可能的保持文件夹下内容名称定义一直，减少代码修改 -------------------------------------------------------------------------------- /raw_data/kgClue/kgClue.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLUEbenchmark/KgCLUEbench/bece79c8bd7cb338d707cf71c03a98963cac1a24/raw_data/kgClue/kgClue.yaml -------------------------------------------------------------------------------- /raw_data/kgClue/knowledge/README.md: -------------------------------------------------------------------------------- 1 | 请把knowledge.txt放在这里 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow==1.15.4 2 | tensorflow-gpu==1.15.4 3 | requests==2.23.0 4 | python>=3.6 -------------------------------------------------------------------------------- /utils/DrawTrain.py: -------------------------------------------------------------------------------- 1 | # 读取out里面的epoch文件绘制训练过程 2 | # 通过选择实现: 每一个epoch画一张图或所有epoch都绘制在一张图内 3 | 4 | import os 5 | import shutil 6 | import matplotlib 7 | from decimal import Decimal 8 | matplotlib.use('TkAgg') 9 | from matplotlib import pyplot as plt 10 | 11 | 12 | def draw_one_pic(path, out_path): 13 | with open(path, 'r', encoding='utf-8') as f: 14 | steps = [] 15 | loss_ = [] 16 | acc_ = [] 17 | while True: 18 | line = f.readline() 19 | if line: 20 | tmp = line.split(",") 21 | step = tmp[0].split(":")[1] 22 | loss = Decimal(tmp[1].split(":")[1]).quantize(Decimal('0.00')) 23 | acc = Decimal(float(tmp[2].split(":")[1])*100.0).quantize(Decimal('0.00')) 24 | steps.append(step) 25 | loss_.append(loss) 26 | acc_.append(acc) 27 | else: 28 | break 29 | 30 | x_axis_data = steps 31 | y_axis_data = loss_ 32 | y_axis_data_ = acc_ 33 | # plt.figure(figsize=(19.2,10.8)) 34 | # plot中参数的含义分别是横轴值，纵轴值，线的形状，颜色，透明度,线的宽度和标签 35 | plt.plot(x_axis_data, y_axis_data, color='b', alpha=0.8, linewidth=1.5, label='loss') 36 | plt.plot(x_axis_data, y_axis_data_, color='r', alpha=0.8, linewidth=1.5, label='acc %') 37 | 38 | # 显示标签，如果不加这句，即使在plot中加了label='一些数字'的参数，最终还是不会显示标签 39 | 40 | plt.legend(loc="upper right") 41 | plt.xlabel('steps') 42 | 43 | from matplotlib.pyplot import MultipleLocator 44 | x_major_locator = MultipleLocator(3000) 45 | 46 | ax = plt.gca() 47 | # ax为两条坐标轴的实例 48 | ax.xaxis.set_major_locator(x_major_locator) 49 | 50 | 51 | plt.savefig(out_path) # 保存该图片 52 | 53 | 54 | def main(divided=False): 55 | filenames = os.listdir(train_out_dir) 56 | filenames = sorted(filenames) 57 | 58 | if divided: 59 | for idx, filename in enumerate(filenames): 60 | file_path = os.path.join(train_out_dir, filename) 61 | out_path = os.path.join(pic_out, './training_epoch_' + str(idx) + '.png') 62 | draw_one_pic(file_path, out_path) 63 | else: 64 | # 思路合并所有的文件 65 | all_in_f = open("./tmp.txt", 'w', encoding='utf-8') 66 | for idx, filename in enumerate(filenames): 67 | if filename.startswith("epoch"): 68 | x = open(os.path.join(train_out_dir, filename), "r", encoding='utf-8') # 打开列表中的文件,读取文件内容 69 | all_in_f.write(x.read()) # 写入新建的log文件中 70 | x.close() 71 | else: 72 | all_in_f.close() 73 | raise ValueError("the path is not pure！！！") 74 | all_in_f.close() 75 | draw_one_pic("./tmp.txt", os.path.join(pic_out, './training_out.png')) 76 | os.remove("./tmp.txt") 77 | 78 | 79 | if __name__ == '__main__': 80 | train_out_dir = r"C:\Users\11943\Documents\GitHub\KgClue_Bench\algorithm\kg_qa\SIM\log" 81 | pic_out = r"C:\Users\11943\Documents\GitHub\KgClue_Bench\algorithm\kg_qa\SIM\pic_out" 82 | if not os.path.exists(pic_out): 83 | os.makedirs(pic_out) 84 | else: 85 | # 清空文件夹 86 | shutil.rmtree(pic_out) 87 | os.makedirs(pic_out) 88 | 89 | divided = False 90 | main(divided) 91 | -------------------------------------------------------------------------------- /utils/EvalReport.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/30 22:03 4 | # @Author : 刘鑫 5 | # @FileName: EvalReport.py 6 | # @Software: PyCharm 7 | 8 | 9 | def report(y_true, y_pred): 10 | ''' 11 | 12 | :param y_true: [[]] 13 | :param y_pred: [[]] 14 | :return: 15 | ''' 16 | 17 | from seqeval.metrics import accuracy_score, precision_score, recall_score, f1_score 18 | from seqeval.metrics import classification_report 19 | 20 | acc_s = accuracy_score(y_true, y_pred) 21 | precision_s = precision_score(y_true, y_pred) 22 | recall_s = recall_score(y_true, y_pred) 23 | f1_s = f1_score(y_true, y_pred) 24 | report = classification_report(y_true, y_pred) 25 | 26 | print(f'\t\t准确率为： {acc_s}') 27 | print(f'\t\t查准率为： {precision_s}') 28 | print(f'\t\t召回率为： {recall_s}') 29 | print(f'\t\tf1值为： {f1_s}') 30 | print(report) 31 | -------------------------------------------------------------------------------- /utils/IdAndLabel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/30 17:45 4 | # @Author : 刘鑫 5 | # @FileName: IdAndLabel.py 6 | # @Software: PyCharm 7 | 8 | def id2label(label_list): 9 | out = dict() 10 | for idx, label in enumerate(label_list): 11 | out[idx] = label 12 | return out 13 | -------------------------------------------------------------------------------- /utils/ListAndList.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/29 22:54 4 | # @Author : 刘鑫 5 | # @FileName: ListAndList.py 6 | # @Software: PyCharm 7 | import random 8 | 9 | 10 | def _index_q_list_in_k_list(q_list, k_list): 11 | """Known q_list in k_list, find index(first time) of q_list in k_list""" 12 | q_list_length = len(q_list) 13 | k_list_length = len(k_list) 14 | for idx in range(k_list_length - q_list_length + 1): 15 | t = [q == k for q, k in zip(q_list, k_list[idx: idx + q_list_length])] 16 | if all(t): 17 | idx_start = idx 18 | return idx_start 19 | 20 | 21 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2021/11/29 22:20 4 | # @Author : 刘鑫 5 | # @FileName: __init__.py.py 6 | # @Software: PyCharm 7 | --------------------------------------------------------------------------------