├── requirements.txt ├── wiki ├── pictures │ ├── bert_embedding.png │ ├── bert_similarity.png │ ├── similarity_eval.png │ ├── similarity_eval_1.png │ ├── similarity_infer.png │ ├── similarity_train.png │ ├── similarity_bert_model.png │ ├── similarity_data_process.png │ └── similarity_eualtion_1.png └── diagram │ └── bert_similarity.drawio ├── .gitignore ├── shell └── extract_log.sh ├── __init__.py ├── start.sh ├── visilize.py ├── args.py ├── logs ├── loss.log └── train.log ├── README.md ├── optimization.py ├── tokenization.py ├── similarity.py └── modeling.py /requirements.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | numpy 3 | tensorflow==1.14.0 -------------------------------------------------------------------------------- /wiki/pictures/bert_embedding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Brokenwind/BertSimilarity/HEAD/wiki/pictures/bert_embedding.png -------------------------------------------------------------------------------- /wiki/pictures/bert_similarity.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Brokenwind/BertSimilarity/HEAD/wiki/pictures/bert_similarity.png -------------------------------------------------------------------------------- /wiki/pictures/similarity_eval.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Brokenwind/BertSimilarity/HEAD/wiki/pictures/similarity_eval.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /chinese_L-12_H-768_A-12 2 | tmp/ 3 | __pycache__/ 4 | .idea/ 5 | tmp.tgz 6 | *.zip 7 | model_ouput 8 | pretrained_model -------------------------------------------------------------------------------- /wiki/pictures/similarity_eval_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Brokenwind/BertSimilarity/HEAD/wiki/pictures/similarity_eval_1.png -------------------------------------------------------------------------------- /wiki/pictures/similarity_infer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Brokenwind/BertSimilarity/HEAD/wiki/pictures/similarity_infer.png -------------------------------------------------------------------------------- /wiki/pictures/similarity_train.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Brokenwind/BertSimilarity/HEAD/wiki/pictures/similarity_train.png -------------------------------------------------------------------------------- /shell/extract_log.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd ../logs 4 | cat train.log | grep "INFO:tensorflow:loss" | awk -F '[=, ]' '{print $4,$9}' > loss.log -------------------------------------------------------------------------------- /wiki/pictures/similarity_bert_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Brokenwind/BertSimilarity/HEAD/wiki/pictures/similarity_bert_model.png -------------------------------------------------------------------------------- /wiki/pictures/similarity_data_process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Brokenwind/BertSimilarity/HEAD/wiki/pictures/similarity_data_process.png -------------------------------------------------------------------------------- /wiki/pictures/similarity_eualtion_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Brokenwind/BertSimilarity/HEAD/wiki/pictures/similarity_eualtion_1.png -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | read -p "running mode: train/eval/infer): " mode 4 | 5 | # 加入环境变量 6 | CUR_PATH=`pwd` 7 | export PYTHONPATH=$CUR_PATH:$PYTHONPATH 8 | 9 | if [ "$mode" == "train" ];then 10 | python similarity.py --mode=$mode > logs/"similarity_"$mode.log 2>&1 & 11 | echo "check train log with command: tail -f logs/similarity_$mode.log" 12 | elif [ "$mode" == "eval" ];then 13 | python similarity.py --mode=$mode > logs/"similarity_"$mode.log 2>&1 & 14 | echo "check eval log with command: tail -f logs/similarity_$mode.log" 15 | elif [ "$mode" == "infer" ];then 16 | python similarity.py --mode=$mode 17 | else 18 | echo "parameter error, choose from train/eval/infer" 19 | exit 1 20 | fi 21 | -------------------------------------------------------------------------------- /visilize.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pandas as pd 4 | import seaborn as sns 5 | from matplotlib import pyplot as plt 6 | 7 | LOG_FILE = 'logs/loss.log' 8 | 9 | 10 | def load_data(input_path): 11 | ''' 12 | 加载处理后的日志数据 13 | :return: 14 | ''' 15 | data = pd.read_csv(input_path, delimiter=' ',names=['loss', 'step'], header=None) 16 | print(data.head()) 17 | return data 18 | 19 | 20 | def plot_loss(): 21 | ''' 22 | 画loss的折线图 23 | ''' 24 | data = load_data(LOG_FILE) 25 | ax = sns.pointplot(x="step", y='loss', data=data) 26 | ax.set_xticklabels(ax.get_xticklabels(), rotation=90, ha="right",fontsize=5) 27 | plt.show() 28 | 29 | 30 | if __name__ == '__main__': 31 | plot_loss() 32 | 33 | -------------------------------------------------------------------------------- /args.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tensorflow as tf 3 | 4 | tf.logging.set_verbosity(tf.logging.INFO) 5 | 6 | PROJECT_ROOT_PATH = os.path.dirname(__file__) 7 | 8 | data_path = os.path.join(PROJECT_ROOT_PATH, 'data') 9 | dataset_path = os.path.join(data_path, 'dataset') 10 | model_dir = os.path.join(data_path, 'pretrained_model/chinese_L-12_H-768_A-12/') 11 | output_dir = os.path.join(data_path, 'model_ouput') 12 | 13 | config_name = os.path.join(model_dir, 'bert_config.json') 14 | ckpt_name = os.path.join(model_dir, 'bert_model.ckpt') 15 | vocab_file = os.path.join(model_dir, 'vocab.txt') 16 | 17 | num_train_epochs = 10 18 | batch_size = 128 19 | learning_rate = 0.00005 20 | 21 | # gpu使用率 22 | gpu_memory_fraction = 0.8 23 | 24 | # 默认取倒数第二层的输出值作为句向量 25 | layer_indexes = [-2] 26 | 27 | # 序列的最大程度，单文本建议把该值调小 28 | max_seq_len = 32 29 | -------------------------------------------------------------------------------- /logs/loss.log: -------------------------------------------------------------------------------- 1 | 1.0218862 0 2 | 0.5186956 100 3 | 0.35847408 200 4 | 0.39972323 300 5 | 0.3482466 400 6 | 0.3434133 500 7 | 0.38796747 600 8 | 0.30979925 700 9 | 0.34772998 800 10 | 0.3403089 900 11 | 0.22857589 1000 12 | 0.27272177 1100 13 | 0.23621395 1200 14 | 0.29772466 1300 15 | 0.23366493 1400 16 | 0.27182245 1500 17 | 0.23535305 1600 18 | 0.3185692 1700 19 | 0.23333019 1800 20 | 0.30707008 1900 21 | 0.18656304 2000 22 | 0.18715279 2100 23 | 0.1443583 2200 24 | 0.12635505 2300 25 | 0.15235075 2400 26 | 0.16846885 2500 27 | 0.08956495 2600 28 | 0.08271434 2700 29 | 0.111416705 2800 30 | 0.15365322 2900 31 | 0.13628608 3000 32 | 0.11042709 3100 33 | 0.06393729 3200 34 | 0.053375434 3300 35 | 0.114489384 3400 36 | 0.086134925 3500 37 | 0.066648535 3600 38 | 0.10406309 3700 39 | 0.04419697 3800 40 | 0.12511328 3900 41 | 0.11059156 4000 42 | 0.043235414 4100 43 | 0.052045528 4200 44 | 0.05638282 4300 45 | 0.05289631 4400 46 | 0.049040448 4500 47 | 0.030390669 4600 48 | 0.012548172 4700 49 | 0.11258875 4800 50 | 0.08994587 4900 51 | 0.025694255 5000 52 | 0.020793306 5100 53 | 0.06512974 5200 54 | 0.043220975 5300 55 | 0.01693129 5400 56 | -------------------------------------------------------------------------------- /wiki/diagram/bert_similarity.drawio: -------------------------------------------------------------------------------- 1 | 7V1tc6M4Ev41VO1+sAshXj8am2SnLpuam8zdXu2XFAFsM8GIBTyO59efBAKDEAm2wXaceKYmpIUF6ufpVndLMAKcrl5uYzta/olcLxAk0X0R4EyQJEmRFPyDSLaFRNZzySL23VwGdoIH/5dHhSKVrn3XS2onpggFqR/VhQ4KQ89JazI7jtGmftocBfWrRvbCawgeHDtoSv/y3XSZS3VF3Mn/8PzFsrgyEGnLyi5OpoJkabtoUxFBS4DTGKE0P1q9TL2AaK/QS/69m5bW8sZiL0y7fEFF28ftwyL9d6LZIQDze3MLRxJF46cdrOmI6d2m20IFcz8IpihAcfYrNCczycJXMJM0Rs9epUXMPrhlma4CLAL4MEbr0PXILZCG5j3TYfz04tR7qYjoGG49tPLSeItPoa2yJOdfoYyCOtXvZgePoau5bFmBBkJ6ok0psSj73mkNH1DF8ZX4OFP1283/7rZr958vo+B2/S1+HEkcHaoBvqw5R3icVWWq/6xR0TBKMrZP8AlQjl52jfhoQX5O7x6KjvB95X3lLQ2M6mreLP3Ue4hsh7RusGXWMakDF6LQI3dUR9kSjalhEnngR3/Qfsnxf2knfUCpKDUoNVUdi5UPaAALCmOqAqsNhSs8HleRh6vViup+vQON17ugmE6QCMrsbNy5uZnmfqAXjqg1jqiwae5cVqhDsaKYmXo39+/omYB61Savipdt8gC0gptEdngsuFIF3Ly/awJXky8c3HaH3ge49+cHtwcQASi+s0Px3LgpA+H2YH29DswkTWIwU8ZG5aOfG0GDgyCjYZwfReRwHngvE5K5Yd14oUsPZ05gJ4nv1BVdDAioY7X2kfMv03xSGyskTzCKf+UmHKY0lac6liOMqp8SPeoiL8lRsz+v4ea5tZSyiVoFBoWDQiGLvcBO/Z/1RJQHDb3CV+Rn4QclhQLkMeuPGXwTtI4dj36vmjq+2ZVqMPeT2vHCSxtdZWwph35EisULuj4JNCiBsBJYAinGYQRqdqUaYicCYfTsbeW0iJyQvHbTzC1T17PjY95jv+zkRY2f7ByUnbrYGzubXQ3FTp1Nf07CTl596pOdg7ITgKIseSw5Gz0NxU1yIdaiTkJPXlb2Sc9h6QkhkzAcTE+2p8HoiS90Fnq2J5+t5b4sPRwleX5I0s8QxSubXCrjhjhslXjPq/OryOcrQ/ZZPWbLjeevHkvqh6STdBV0YgucF0An7UPSKbwKOjVKrRfAJ95+gOvnk6CYiRddy5ppoxx8fl7B9nW1I/dI7I8Y6AcxDZiTabGKNtgOifrqtyyKF7WiVsT+l5u73RjYGCbXlLspoM4JaBi8tZ79a/xMt/C0FX54/G4bWed5CNP69r2rk8Bmme7pCajIDvxFSOiMWeNhuUmM3HfsYEIbVr7rkstwXU/vu+QMWC8aKZwZQOZwVBrMU8i9wys6pfXuhFAxVFXTmjQYZ5+PRwQN1mcQAIoqytmocEBF5ZMK/VNBhcaZiXBALYQhgtSJCKqtzUlZsrFvJ7bDZI7zBYzOR2ODytQyFOncMwRvj8gnG07DBgA0JmCQzjxLyINt1mvd8rXn7mulp7r5O8pH2Xo6yT2UC8pH5aF2Cr5Fmq6980kjtfZ+DaRhq+aXVsSQ22vol0waeNWkadTGiau5KNa0hytHbkq+blzZ2jQ09IuaQhReSXPP7FTirjHQXp5K94ABCf1fduqj6nrWE/uFDxOE4pykRg39hDEomN3/Decb53518zi7jf6co19/9/Bgp8xd7Jp5YeL9Jv0uSGaC5unKfumKNWOxumxOLG45+yzP4hpSAy/uo7i6MRBgvIICo8EF1kLUfezlk+X2U9GD+KpO9GInFdXJSGvqhMNh7AWrRXxNG0hDF7/ZrpcFm1e4cSGrOHI9unkD/q6LOEyv7BPv/S3hwLWz+ct93tykL3fi85f/fP0x/TE6fo23xV3GKELr9DdxDH4/0FHOdcdzHB6Nys2dp3SU5TR2Ck/JxYo3tV2fH7gokzdEkU+Cfa2c7ajcId6/nd+PvsSreWTa/0qndw9q+g3+gp24k6FUvDFFYsJTSqzVy4K8NWY8D9DGWdpxOrbDEKVZQPwIKvFp4M2JkWdbhL3Y+umRncKFqebn41+NnoxV17X6DA4UZQywV1YA0ICuNYNSyCFN+SKYY0yXq37uCwgY/RPKP9BfUZwu0QKFdmDtpIyP251zh1BEdfvDS9MttVh7naI6iq6dLMvs83DbO9aooF4HS4c1rIqnC/be1y6C/Toe2OaGeheJOBb1MjONG7lqvDvPkLqcd3jSGnv4XmmULRbWnilXMcnOPOwOMAfz8cw43oHNXVPCY1KLcfxw8T0j9UjuyUOoDDeAKL3uITSOh2C9dm8OgrfV4gL886g/B13fqjSSL8s/d3ga9AP55/qmgxGQa1iV74A6wEHv2fPAHhocXy2SuhUOBUsWTF3QZ4KlCYZJjolkSv4SyUTQZcFShYkq6BKR6JZAYpo+S4zvyFsDEdbT4pEBGy4C8AL/PgqLfK4cP5135sonRQ6hCKf0DHjTyHAU6TCLV7JxWtFvXQSoloT3yrSLGOE9ZNhyoaOyHn1o9C/DN3oaejI5fiNDyyqUpQj6jTBRyMFkJuCYjBzcCKbRdd3xMtabXD/2HBpZYlgJuzINFFEV5/2o+y9IKfUQw+C8axSoHD6zlZ3+vEL7lsmIS4oV5qkfjjLfSVghRoTmGawibXtCaYpWrc25N84a/bBsrKifNAVpXDatQ99Brjd68l0/b/VWTxi3on2DYnf0FHv2c6fHxTrOczXrKJ/qIq9QzZ7qsleEQuFTEuVXaYo6bNXYmVA+b+JuyKyKrUjJjhXBtARDy2bYzMZyoY7lMDs2SCu2OioXyWn5sQFIU/OmDr5JJXu7GW/sewwUjxIW964KpkjiCDpogwzruMEdBkKHSgefMhWNvHaVluAn2sMf9uB6ZKbEUL5xurpiUDxUUXvN8WCup32b1LW6HvVw19OF7P2KSq9UuqSmPyrttTTW0lL3MtPTiw5wWqXHarqrV4f/DpwD8ySHwal5ndY5vPIegiPjVdbaLGLQLs4A34DknYawfUStTGFM1eQzR63S8XslO7JDsPTM58HMsjXBnGReEGTR2idfWvhS37Ekc54L64sv+Nfd/3iRJ8u7/zgEWv8H -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BertSimilarity 2 | 3 | 基于Google的BERT模型来进行语义相似度计算。代码基于tensorflow 1。 4 | 5 | ## 1. 基本原理 6 | 7 | 简单来说就是将需要计算的相似的两个句子先拼接在一起，然后通过Bert模型获取获取整体的编码信息，接着通过全连接层将维，输出相似和不相似的概率。 8 | 9 | 10 | ## 1.1 模型结构 11 | 12 | 模型结构如图所示： 13 | 14 | ![](wiki/pictures/bert_similarity.png) 15 | 16 | ### 1.1.1 数据预处理 17 | 18 | 本文使用Bert模型计算相似度前，首先要对输入数据进行预处理，例如当要处理的文本是： 19 | 20 | ```text 21 | 如何得知关闭借呗 想永久关闭借呗 22 | ``` 23 | 24 | 首先要将文本按token化，切分成字数组: 25 | 26 | ```json 27 | [如 何 得 知 关 闭 借 呗] 28 | 29 | [想 永 久 关 闭 借 呗] 30 | ``` 31 | 32 | 然后将两个切分后的句子，按照如下的方式拼接： 33 | 34 | ```json 35 | [CLS] 如 何 得 知 关 闭 借 呗 [SEP] 想 永 久 关 闭 借 呗 [SEP] 36 | ``` 37 | 38 | 拼接后句子中的[CLS]是一个特殊分隔符表示每个样本的开头，[SEP]是样本中每个句子的结束标记符。拼接后的样本长度是len(A)+len(B)+3，因为加上了[CLS]和[SEP]特殊标识符的，3指的是1个[CLS]和2个[SEP]。在模型中，会指定最大样本长度LEN，本次实验中指定的是30），也就处理后的样本长度和不能超过LEN，即 39 | 40 | ```python 41 | len(A) + len(B) + 3 <= LEN 42 | ``` 43 | 44 | 超出最大长度的部分会被截取。然后继续对输入数据进行处理，将内容数字化。将token数组转化为token的索引数组，所有的样本会被转化为固定长度（LEN）的索引数组。[CLS]的索引是101，[SEP]的索引是102，不够最大样本长度的部分补0。然后对输入进行mask操作，样本有效部分是1，无效部分是0，0表示在训练时不会被关注。再对两个句子进行分割，分割操作是通过一个数组来标记，属于第一个句子标记为0，属于第二个句子的标记为1，不足部分也填0。 45 | 46 | 处理后结果如下图所示： 47 | 48 | ![](wiki/pictures/similarity_data_process.png) 49 | 50 | input_ids，input_mask，segment_ids都是Bert预训练模型需要的输入。 51 | 52 | ### 1.1.2 Bert编码阶段 53 | 54 | **Bert预训练简介** 55 | 56 | Bert模型结构如图所示： 57 | 58 | ![](wiki/pictures/similarity_bert_model.png) 59 | 60 | Bert没有使用word2vec进行词嵌入，而是直接在原始语料库上训练，并且在训练过程中同时进行两个预测任务，分别是遮蔽语言模型MLM（masked language model）任务和根据上一个句子预测下一个句子的任务。Bert使用MLM来解决单向局限，其随机地从输入中遮蔽一些词块，然后通过上下文语境来预测被遮蔽的词块。从Bert的双向Transformer结构中我们可以发现MLM任务并不只是从左到右进行预测，而是它融合了遮蔽词块左右两边的句子语境，充分利用两个方向的注意力机制。同时训练数据不需要人工标注，而是在训练之前随机产生训练数据，减少了由人工标注所带来的误差，只需要通过合适的方法，对现有语料中的句子进行随机的遮掩即可得到可以用来训练的语料。很多自然语言处理任务比如问答系统和自然语言推断任务都需要对两个句子之间关系进行理解，而这种理解并不能通过语言模型直接获得。在预训练任务中加入预测下一个句子的训练可以达到理解句子关系的目的，具体做法是随机替换一些句子，然后利用上一句进行预测下一句的真伪。 61 | 62 | **利用Bert对句子对进行编码** 63 | 64 | 使用已经训练好的Bert模型对句子对进行编码时，因为Bert中双向Attention机制，两个句子会相互Attention，也就是通过训练会学到两个句子的相似程度。 65 | 66 | ### 1.1.3 微调阶段 67 | 68 | 将句子对通过Bert预训练模型计算之后，获取两个句子的的最终编码。并对其进行0.1的Dropout，主要是为了防止模型过拟合，Dropout后的结果后边接一层全连接层，全连接层的输出维度为2，然后通过softmax计算，得到相似和不相似的概率。 69 | 70 | ## 2. 使用方式 71 | 72 | ### 2.1 数据准备 73 | 74 | 在使用前需要先下载Google预训练的Bert模型。 75 | 76 | 下载地址 [https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip](https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip) 77 | 78 | 下载完后，解压，然后将chinese_L-12_H-768_A-12文件夹复制到 data/pretrained_model/目录中 79 | 80 | ### 2.2 运行说明 81 | 82 | 为了方便执行，已经编写了Shell脚本，在linux机器上直接运行start.sh，根据提示输入对应的mode(train/eval/infer)即可。 83 | 84 | **2.2.1 训练阶段** 85 | 86 | ```bash 87 | cd BertSimilarity/ 88 | sh start.sh 89 | # 然后根据提示输入 train 90 | ``` 91 | 92 | 执行效果: 93 | 94 | ![](wiki/pictures/similarity_train.png) 95 | 96 | 执行日志查看: 97 | 98 | ```bash 99 | tail -f logs/similarity_train.log 100 | ``` 101 | 102 | 如果没有机器进行训练，我这里已经训练好了一份参数文件: 103 | 104 | [点击此处获取参数文件](https://pan.baidu.com/s/19pR3PS8AVIPpKZAXPkHdSA)，提取码: fud8 105 | 106 | 解压之后，model_output复制到项目根目录的data目录下边，然后就可以进行评价和测试 107 | 108 | **2.2.2 评价阶段** 109 | 110 | ```bash 111 | cd BertSimilarity/ 112 | sh start.sh 113 | # 然后根据提示输入 eval 114 | ``` 115 | 116 | 执行效果: 117 | 118 | ![](wiki/pictures/similarity_eval.png) 119 | 120 | 执行日志查看: 121 | 122 | ```bash 123 | tail -f logs/similarity_eval.log 124 | ``` 125 | 126 | 评价结果： 127 | ![](wiki/pictures/similarity_eval_1.png) 128 | 129 | 130 | 131 | **2.2.3 测试阶段** 132 | 133 | ```bash 134 | cd BertSimilarity/ 135 | sh start.sh 136 | # 然后根据提示输入 infer，待程序启动好之后，就可以分别输入两个句子，测试其相似程度 137 | ``` 138 | 139 | 执行效果: 140 | 141 | ![](wiki/pictures/similarity_infer.png) 142 | -------------------------------------------------------------------------------- /optimization.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Functions and classes related to optimization (weight updates).""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import re 22 | import tensorflow as tf 23 | 24 | 25 | def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, use_tpu): 26 | """Creates an optimizer training op.""" 27 | global_step = tf.train.get_or_create_global_step() 28 | 29 | learning_rate = tf.constant(value=init_lr, shape=[], dtype=tf.float32) 30 | 31 | # Implements linear decay of the learning rate. 32 | learning_rate = tf.train.polynomial_decay( 33 | learning_rate, 34 | global_step, 35 | num_train_steps, 36 | end_learning_rate=0.0, 37 | power=1.0, 38 | cycle=False) 39 | 40 | # Implements linear warmup. I.e., if global_step < num_warmup_steps, the 41 | # learning rate will be `global_step/num_warmup_steps * init_lr`. 42 | if num_warmup_steps: 43 | global_steps_int = tf.cast(global_step, tf.int32) 44 | warmup_steps_int = tf.constant(num_warmup_steps, dtype=tf.int32) 45 | 46 | global_steps_float = tf.cast(global_steps_int, tf.float32) 47 | warmup_steps_float = tf.cast(warmup_steps_int, tf.float32) 48 | 49 | warmup_percent_done = global_steps_float / warmup_steps_float 50 | warmup_learning_rate = init_lr * warmup_percent_done 51 | 52 | is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32) 53 | learning_rate = ( 54 | (1.0 - is_warmup) * learning_rate + is_warmup * warmup_learning_rate) 55 | 56 | # It is recommended that you use this optimizer for fine tuning, since this 57 | # is how the model was trained (note that the Adam m/v variables are NOT 58 | # loaded from init_checkpoint.) 59 | optimizer = AdamWeightDecayOptimizer( 60 | learning_rate=learning_rate, 61 | weight_decay_rate=0.01, 62 | beta_1=0.9, 63 | beta_2=0.999, 64 | epsilon=1e-6, 65 | exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) 66 | 67 | if use_tpu: 68 | optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) 69 | 70 | tvars = tf.trainable_variables() 71 | grads = tf.gradients(loss, tvars) 72 | 73 | # This is how the model was pre-trained. 74 | (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0) 75 | 76 | train_op = optimizer.apply_gradients( 77 | zip(grads, tvars), global_step=global_step) 78 | 79 | new_global_step = global_step + 1 80 | train_op = tf.group(train_op, [global_step.assign(new_global_step)]) 81 | return train_op 82 | 83 | 84 | class AdamWeightDecayOptimizer(tf.train.Optimizer): 85 | """A basic Adam optimizer that includes "correct" L2 weight decay.""" 86 | 87 | def __init__(self, 88 | learning_rate, 89 | weight_decay_rate=0.0, 90 | beta_1=0.9, 91 | beta_2=0.999, 92 | epsilon=1e-6, 93 | exclude_from_weight_decay=None, 94 | name="AdamWeightDecayOptimizer"): 95 | """Constructs a AdamWeightDecayOptimizer.""" 96 | super(AdamWeightDecayOptimizer, self).__init__(False, name) 97 | 98 | self.learning_rate = learning_rate 99 | self.weight_decay_rate = weight_decay_rate 100 | self.beta_1 = beta_1 101 | self.beta_2 = beta_2 102 | self.epsilon = epsilon 103 | self.exclude_from_weight_decay = exclude_from_weight_decay 104 | 105 | def apply_gradients(self, grads_and_vars, global_step=None, name=None): 106 | """See base class.""" 107 | assignments = [] 108 | for (grad, param) in grads_and_vars: 109 | if grad is None or param is None: 110 | continue 111 | 112 | param_name = self._get_variable_name(param.name) 113 | 114 | m = tf.get_variable( 115 | name=param_name + "/adam_m", 116 | shape=param.shape.as_list(), 117 | dtype=tf.float32, 118 | trainable=False, 119 | initializer=tf.zeros_initializer()) 120 | v = tf.get_variable( 121 | name=param_name + "/adam_v", 122 | shape=param.shape.as_list(), 123 | dtype=tf.float32, 124 | trainable=False, 125 | initializer=tf.zeros_initializer()) 126 | 127 | # Standard Adam update. 128 | next_m = ( 129 | tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad)) 130 | next_v = ( 131 | tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2, 132 | tf.square(grad))) 133 | 134 | update = next_m / (tf.sqrt(next_v) + self.epsilon) 135 | 136 | # Just adding the square of the weights to the loss function is *not* 137 | # the correct way of using L2 regularization/weight decay with Adam, 138 | # since that will interact with the m and v parameters in strange ways. 139 | # 140 | # Instead we want ot decay the weights in a manner that doesn't interact 141 | # with the m/v parameters. This is equivalent to adding the square 142 | # of the weights to the loss with plain (non-momentum) SGD. 143 | if self._do_use_weight_decay(param_name): 144 | update += self.weight_decay_rate * param 145 | 146 | update_with_lr = self.learning_rate * update 147 | 148 | next_param = param - update_with_lr 149 | 150 | assignments.extend( 151 | [param.assign(next_param), 152 | m.assign(next_m), 153 | v.assign(next_v)]) 154 | return tf.group(*assignments, name=name) 155 | 156 | def _do_use_weight_decay(self, param_name): 157 | """Whether to use L2 weight decay for `param_name`.""" 158 | if not self.weight_decay_rate: 159 | return False 160 | if self.exclude_from_weight_decay: 161 | for r in self.exclude_from_weight_decay: 162 | if re.search(r, param_name) is not None: 163 | return False 164 | return True 165 | 166 | def _get_variable_name(self, param_name): 167 | """Get the variable name from the tensor name.""" 168 | m = re.match("^(.*):\\d+$", param_name) 169 | if m is not None: 170 | param_name = m.group(1) 171 | return param_name 172 | -------------------------------------------------------------------------------- /tokenization.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Tokenization classes.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import collections 22 | import unicodedata 23 | import six 24 | import tensorflow as tf 25 | 26 | 27 | def convert_to_unicode(text): 28 | """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" 29 | if six.PY3: 30 | if isinstance(text, str): 31 | return text 32 | elif isinstance(text, bytes): 33 | return text.decode("utf-8", "ignore") 34 | else: 35 | raise ValueError("Unsupported string type: %s" % (type(text))) 36 | elif six.PY2: 37 | if isinstance(text, str): 38 | return text.decode("utf-8", "ignore") 39 | elif isinstance(text, unicode): 40 | return text 41 | else: 42 | raise ValueError("Unsupported string type: %s" % (type(text))) 43 | else: 44 | raise ValueError("Not running on Python2 or Python 3?") 45 | 46 | 47 | def printable_text(text): 48 | """Returns text encoded in a way suitable for print or `tf.logging`.""" 49 | 50 | # These functions want `str` for both Python2 and Python3, but in one case 51 | # it's a Unicode string and in the other it's a byte string. 52 | if six.PY3: 53 | if isinstance(text, str): 54 | return text 55 | elif isinstance(text, bytes): 56 | return text.decode("utf-8", "ignore") 57 | else: 58 | raise ValueError("Unsupported string type: %s" % (type(text))) 59 | elif six.PY2: 60 | if isinstance(text, str): 61 | return text 62 | elif isinstance(text, unicode): 63 | return text.encode("utf-8") 64 | else: 65 | raise ValueError("Unsupported string type: %s" % (type(text))) 66 | else: 67 | raise ValueError("Not running on Python2 or Python 3?") 68 | 69 | 70 | def load_vocab(vocab_file): 71 | """Loads a vocabulary file into a dictionary.""" 72 | vocab = collections.OrderedDict() 73 | index = 0 74 | with tf.gfile.GFile(vocab_file, "r") as reader: 75 | while True: 76 | token = convert_to_unicode(reader.readline()) 77 | if not token: 78 | break 79 | token = token.strip() 80 | vocab[token] = index 81 | index += 1 82 | return vocab 83 | 84 | 85 | def convert_by_vocab(vocab, items): 86 | """Converts a sequence of [tokens|ids] using the vocab.""" 87 | output = [] 88 | for item in items: 89 | output.append(vocab[item]) 90 | return output 91 | 92 | 93 | def convert_tokens_to_ids(vocab, tokens): 94 | return convert_by_vocab(vocab, tokens) 95 | 96 | 97 | def convert_ids_to_tokens(inv_vocab, ids): 98 | return convert_by_vocab(inv_vocab, ids) 99 | 100 | 101 | def whitespace_tokenize(text): 102 | """Runs basic whitespace cleaning and splitting on a piece of text.""" 103 | text = text.strip() 104 | if not text: 105 | return [] 106 | tokens = text.split() 107 | return tokens 108 | 109 | 110 | class FullTokenizer(object): 111 | """Runs end-to-end tokenziation.""" 112 | 113 | def __init__(self, vocab_file, do_lower_case=True): 114 | self.vocab = load_vocab(vocab_file) 115 | self.inv_vocab = {v: k for k, v in self.vocab.items()} 116 | self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) 117 | self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) 118 | 119 | def tokenize(self, text): 120 | split_tokens = [] 121 | for token in self.basic_tokenizer.tokenize(text): 122 | for sub_token in self.wordpiece_tokenizer.tokenize(token): 123 | split_tokens.append(sub_token) 124 | 125 | return split_tokens 126 | 127 | def convert_tokens_to_ids(self, tokens): 128 | return convert_by_vocab(self.vocab, tokens) 129 | 130 | def convert_ids_to_tokens(self, ids): 131 | return convert_by_vocab(self.inv_vocab, ids) 132 | 133 | 134 | class BasicTokenizer(object): 135 | """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" 136 | 137 | def __init__(self, do_lower_case=True): 138 | """Constructs a BasicTokenizer. 139 | 140 | Args: 141 | do_lower_case: Whether to lower case the input. 142 | """ 143 | self.do_lower_case = do_lower_case 144 | 145 | def tokenize(self, text): 146 | """Tokenizes a piece of text.""" 147 | text = convert_to_unicode(text) 148 | text = self._clean_text(text) 149 | 150 | # This was added on November 1st, 2018 for the multilingual and Chinese 151 | # models. This is also applied to the English models now, but it doesn't 152 | # matter since the English models were not trained on any Chinese data 153 | # and generally don't have any Chinese data in them (there are Chinese 154 | # characters in the vocabulary because Wikipedia does have some Chinese 155 | # words in the English Wikipedia.). 156 | text = self._tokenize_chinese_chars(text) 157 | 158 | orig_tokens = whitespace_tokenize(text) 159 | split_tokens = [] 160 | for token in orig_tokens: 161 | if self.do_lower_case: 162 | token = token.lower() 163 | token = self._run_strip_accents(token) 164 | split_tokens.extend(self._run_split_on_punc(token)) 165 | 166 | output_tokens = whitespace_tokenize(" ".join(split_tokens)) 167 | return output_tokens 168 | 169 | def _run_strip_accents(self, text): 170 | """Strips accents from a piece of text.""" 171 | text = unicodedata.normalize("NFD", text) 172 | output = [] 173 | for char in text: 174 | cat = unicodedata.category(char) 175 | if cat == "Mn": 176 | continue 177 | output.append(char) 178 | return "".join(output) 179 | 180 | def _run_split_on_punc(self, text): 181 | """Splits punctuation on a piece of text.""" 182 | chars = list(text) 183 | i = 0 184 | start_new_word = True 185 | output = [] 186 | while i < len(chars): 187 | char = chars[i] 188 | if _is_punctuation(char): 189 | output.append([char]) 190 | start_new_word = True 191 | else: 192 | if start_new_word: 193 | output.append([]) 194 | start_new_word = False 195 | output[-1].append(char) 196 | i += 1 197 | 198 | return ["".join(x) for x in output] 199 | 200 | def _tokenize_chinese_chars(self, text): 201 | """Adds whitespace around any CJK character.""" 202 | output = [] 203 | for char in text: 204 | cp = ord(char) 205 | if self._is_chinese_char(cp): 206 | output.append(" ") 207 | output.append(char) 208 | output.append(" ") 209 | else: 210 | output.append(char) 211 | return "".join(output) 212 | 213 | def _is_chinese_char(self, cp): 214 | """Checks whether CP is the codepoint of a CJK character.""" 215 | # This defines a "chinese character" as anything in the CJK Unicode block: 216 | # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) 217 | # 218 | # Note that the CJK Unicode block is NOT all Japanese and Korean characters, 219 | # despite its name. The modern Korean Hangul alphabet is a different block, 220 | # as is Japanese Hiragana and Katakana. Those alphabets are used to write 221 | # space-separated words, so they are not treated specially and handled 222 | # like the all of the other languages. 223 | if ((cp >= 0x4E00 and cp <= 0x9FFF) or # 224 | (cp >= 0x3400 and cp <= 0x4DBF) or # 225 | (cp >= 0x20000 and cp <= 0x2A6DF) or # 226 | (cp >= 0x2A700 and cp <= 0x2B73F) or # 227 | (cp >= 0x2B740 and cp <= 0x2B81F) or # 228 | (cp >= 0x2B820 and cp <= 0x2CEAF) or 229 | (cp >= 0xF900 and cp <= 0xFAFF) or # 230 | (cp >= 0x2F800 and cp <= 0x2FA1F)): # 231 | return True 232 | 233 | return False 234 | 235 | def _clean_text(self, text): 236 | """Performs invalid character removal and whitespace cleanup on text.""" 237 | output = [] 238 | for char in text: 239 | cp = ord(char) 240 | if cp == 0 or cp == 0xfffd or _is_control(char): 241 | continue 242 | if _is_whitespace(char): 243 | output.append(" ") 244 | else: 245 | output.append(char) 246 | return "".join(output) 247 | 248 | 249 | class WordpieceTokenizer(object): 250 | """Runs WordPiece tokenziation.""" 251 | 252 | def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): 253 | self.vocab = vocab 254 | self.unk_token = unk_token 255 | self.max_input_chars_per_word = max_input_chars_per_word 256 | 257 | def tokenize(self, text): 258 | """Tokenizes a piece of text into its word pieces. 259 | 260 | This uses a greedy longest-match-first algorithm to perform tokenization 261 | using the given vocabulary. 262 | 263 | For example: 264 | input = "unaffable" 265 | output = ["un", "##aff", "##able"] 266 | 267 | Args: 268 | text: A single token or whitespace separated tokens. This should have 269 | already been passed through `BasicTokenizer. 270 | 271 | Returns: 272 | A list of wordpiece tokens. 273 | """ 274 | 275 | text = convert_to_unicode(text) 276 | 277 | output_tokens = [] 278 | for token in whitespace_tokenize(text): 279 | chars = list(token) 280 | if len(chars) > self.max_input_chars_per_word: 281 | output_tokens.append(self.unk_token) 282 | continue 283 | 284 | is_bad = False 285 | start = 0 286 | sub_tokens = [] 287 | while start < len(chars): 288 | end = len(chars) 289 | cur_substr = None 290 | while start < end: 291 | substr = "".join(chars[start:end]) 292 | if start > 0: 293 | substr = "##" + substr 294 | if substr in self.vocab: 295 | cur_substr = substr 296 | break 297 | end -= 1 298 | if cur_substr is None: 299 | is_bad = True 300 | break 301 | sub_tokens.append(cur_substr) 302 | start = end 303 | 304 | if is_bad: 305 | output_tokens.append(self.unk_token) 306 | else: 307 | output_tokens.extend(sub_tokens) 308 | return output_tokens 309 | 310 | 311 | def _is_whitespace(char): 312 | """Checks whether `chars` is a whitespace character.""" 313 | # \t, \n, and \r are technically contorl characters but we treat them 314 | # as whitespace since they are generally considered as such. 315 | if char == " " or char == "\t" or char == "\n" or char == "\r": 316 | return True 317 | cat = unicodedata.category(char) 318 | if cat == "Zs": 319 | return True 320 | return False 321 | 322 | 323 | def _is_control(char): 324 | """Checks whether `chars` is a control character.""" 325 | # These are technically control characters but we count them as whitespace 326 | # characters. 327 | if char == "\t" or char == "\n" or char == "\r": 328 | return False 329 | cat = unicodedata.category(char) 330 | if cat.startswith("C"): 331 | return True 332 | return False 333 | 334 | 335 | def _is_punctuation(char): 336 | """Checks whether `chars` is a punctuation character.""" 337 | cp = ord(char) 338 | # We treat all non-letter/number ASCII as punctuation. 339 | # Characters such as "^", "$", and "`" are not in the Unicode 340 | # Punctuation class but we treat them as punctuation anyways, for 341 | # consistency. 342 | if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or 343 | (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): 344 | return True 345 | cat = unicodedata.category(char) 346 | if cat.startswith("P"): 347 | return True 348 | return False 349 | -------------------------------------------------------------------------------- /similarity.py: -------------------------------------------------------------------------------- 1 | import os 2 | from queue import Queue 3 | from threading import Thread 4 | 5 | import pandas as pd 6 | import tensorflow as tf 7 | import collections 8 | import args 9 | import tokenization 10 | import modeling 11 | import optimization 12 | 13 | # os.environ['CUDA_VISIBLE_DEVICES'] = '1' 14 | 15 | flags = tf.flags 16 | 17 | FLAGS = flags.FLAGS 18 | 19 | flags.DEFINE_string( 20 | "mode", tf.estimator.ModeKeys.TRAIN, 21 | "run mode, train/eval/predict") 22 | 23 | 24 | class InputExample(object): 25 | """A single training/test example for simple sequence classification.""" 26 | 27 | def __init__(self, guid, text_a, text_b=None, label=None): 28 | """Constructs a InputExample. 29 | 30 | Args: 31 | guid: Unique id for the example. 32 | text_a: string. The untokenized text of the first sequence. For single 33 | sequence tasks, only this sequence must be specified. 34 | text_b: (Optional) string. The untokenized text of the second sequence. 35 | Only must be specified for sequence pair tasks. 36 | label: (Optional) string. The label of the example. This should be 37 | specified for train and dev examples, but not for test examples. 38 | """ 39 | self.guid = guid 40 | self.text_a = text_a 41 | self.text_b = text_b 42 | self.label = label 43 | 44 | 45 | class InputFeatures(object): 46 | """A single set of features of data.""" 47 | 48 | def __init__(self, input_ids, input_mask, segment_ids, label_id): 49 | self.input_ids = input_ids 50 | self.input_mask = input_mask 51 | self.segment_ids = segment_ids 52 | self.label_id = label_id 53 | 54 | 55 | class DataProcessor(object): 56 | """Base class for data converters for sequence classification data sets.""" 57 | 58 | def get_train_examples(self, data_dir): 59 | """Gets a collection of `InputExample`s for the train set.""" 60 | raise NotImplementedError() 61 | 62 | def get_dev_examples(self, data_dir): 63 | """Gets a collection of `InputExample`s for the dev set.""" 64 | raise NotImplementedError() 65 | 66 | def get_test_examples(self, data_dir): 67 | """Gets a collection of `InputExample`s for prediction.""" 68 | raise NotImplementedError() 69 | 70 | def get_labels(self): 71 | """Gets the list of labels for this data set.""" 72 | raise NotImplementedError() 73 | 74 | 75 | class SimProcessor(DataProcessor): 76 | def get_train_examples(self, data_dir): 77 | file_path = os.path.join(data_dir, 'train.csv') 78 | train_df = pd.read_csv(file_path, encoding='utf-8') 79 | train_data = [] 80 | for index, train in enumerate(train_df.values): 81 | guid = 'train-%d' % index 82 | text_a = tokenization.convert_to_unicode(str(train[0])) 83 | text_b = tokenization.convert_to_unicode(str(train[1])) 84 | label = str(train[2]) 85 | train_data.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) 86 | return train_data 87 | 88 | def get_dev_examples(self, data_dir): 89 | file_path = os.path.join(data_dir, 'dev.csv') 90 | dev_df = pd.read_csv(file_path, encoding='utf-8') 91 | dev_data = [] 92 | for index, dev in enumerate(dev_df.values): 93 | guid = 'test-%d' % index 94 | text_a = tokenization.convert_to_unicode(str(dev[0])) 95 | text_b = tokenization.convert_to_unicode(str(dev[1])) 96 | label = str(dev[2]) 97 | dev_data.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) 98 | return dev_data 99 | 100 | def get_test_examples(self, data_dir): 101 | file_path = os.path.join(data_dir, 'test.csv') 102 | test_df = pd.read_csv(file_path, encoding='utf-8') 103 | test_data = [] 104 | for index, test in enumerate(test_df.values): 105 | guid = 'test-%d' % index 106 | text_a = tokenization.convert_to_unicode(str(test[0])) 107 | text_b = tokenization.convert_to_unicode(str(test[1])) 108 | label = str(test[2]) 109 | test_data.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) 110 | return test_data 111 | 112 | def get_sentence_examples(self, questions): 113 | for index, data in enumerate(questions): 114 | guid = 'test-%d' % index 115 | text_a = tokenization.convert_to_unicode(str(data[0])) 116 | text_b = tokenization.convert_to_unicode(str(data[1])) 117 | label = str(0) 118 | yield InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label) 119 | 120 | def get_labels(self): 121 | return ['0', '1'] 122 | 123 | 124 | class BertSim: 125 | 126 | def __init__(self, batch_size=args.batch_size): 127 | self.mode = None 128 | self.max_seq_length = args.max_seq_len 129 | self.tokenizer = tokenization.FullTokenizer(vocab_file=args.vocab_file, do_lower_case=True) 130 | self.batch_size = batch_size 131 | self.estimator = None 132 | self.processor = SimProcessor() 133 | tf.logging.set_verbosity(tf.logging.INFO) 134 | 135 | def set_mode(self, mode): 136 | self.mode = mode 137 | self.estimator = self.get_estimator() 138 | if mode == tf.estimator.ModeKeys.PREDICT: 139 | self.input_queue = Queue(maxsize=1) 140 | self.output_queue = Queue(maxsize=1) 141 | self.predict_thread = Thread(target=self.predict_from_queue, daemon=True) 142 | self.predict_thread.start() 143 | 144 | def create_model(bert_config, is_training, input_ids, input_mask, segment_ids, 145 | labels, num_labels, use_one_hot_embeddings): 146 | """ 147 | 建立微调模型 148 | :param is_training: 训练还是测试 149 | :param input_ids: 输入句子中每个字的索引列表 150 | :param input_mask: 字的屏蔽列表 151 | :param segment_ids: 分段列表，第一个句子用0表示，第二个句子用1表示，[0,0,0...,1,1,] 152 | :param labels: 两个句子是否相似,0:不相似，1：相似 153 | :param num_labels: 多少个样本，多少个标签 154 | :param use_one_hot_embeddings: 155 | :return: 156 | """ 157 | model = modeling.BertModel( 158 | config=bert_config, 159 | is_training=is_training, 160 | input_ids=input_ids, 161 | input_mask=input_mask, 162 | token_type_ids=segment_ids, 163 | use_one_hot_embeddings=use_one_hot_embeddings) 164 | 165 | # If you want to use the token-level output, use model.get_sequence_output() 166 | output_layer = model.get_pooled_output() 167 | 168 | hidden_size = output_layer.shape[-1].value 169 | 170 | output_weights = tf.get_variable( 171 | "output_weights", [num_labels, hidden_size], 172 | initializer=tf.truncated_normal_initializer(stddev=0.02)) 173 | 174 | output_bias = tf.get_variable( 175 | "output_bias", [num_labels], initializer=tf.zeros_initializer()) 176 | 177 | with tf.variable_scope("loss"): 178 | if is_training: 179 | # I.e., 0.1 dropout 180 | output_layer = tf.nn.dropout(output_layer, keep_prob=0.9) 181 | # transpose_b=True,在乘积之前先将第二个矩阵转置 182 | logits = tf.matmul(output_layer, output_weights, transpose_b=True) 183 | logits = tf.nn.bias_add(logits, output_bias) 184 | probabilities = tf.nn.softmax(logits, axis=-1) 185 | # 使用softmax losss 作为损失函数 186 | log_probs = tf.nn.log_softmax(logits, axis=-1) 187 | 188 | one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32) 189 | 190 | per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) 191 | loss = tf.reduce_mean(per_example_loss) 192 | 193 | return (loss, per_example_loss, logits, probabilities) 194 | 195 | def model_fn_builder(self, bert_config, num_labels, init_checkpoint, learning_rate, 196 | num_train_steps, num_warmup_steps, 197 | use_one_hot_embeddings): 198 | """Returns `model_fn` closurimport_tfe for TPUEstimator.""" 199 | 200 | def model_fn(features, labels, mode, params): # pylint: disable=unused-argument 201 | from tensorflow.python.estimator.model_fn import EstimatorSpec 202 | 203 | tf.logging.info("*** Features ***") 204 | for name in sorted(features.keys()): 205 | tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape)) 206 | 207 | input_ids = features["input_ids"] 208 | input_mask = features["input_mask"] 209 | segment_ids = features["segment_ids"] 210 | label_ids = features["label_ids"] 211 | 212 | is_training = (mode == tf.estimator.ModeKeys.TRAIN) 213 | 214 | (total_loss, per_example_loss, logits, probabilities) = BertSim.create_model( 215 | bert_config, is_training, input_ids, input_mask, segment_ids, label_ids, 216 | num_labels, use_one_hot_embeddings) 217 | 218 | tvars = tf.trainable_variables() 219 | initialized_variable_names = {} 220 | 221 | if init_checkpoint: 222 | (assignment_map, initialized_variable_names) \ 223 | = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint) 224 | tf.train.init_from_checkpoint(init_checkpoint, assignment_map) 225 | 226 | tf.logging.info("**** Trainable Variables ****") 227 | for var in tvars: 228 | init_string = "" 229 | if var.name in initialized_variable_names: 230 | init_string = ", *INIT_FROM_CKPT*" 231 | tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape, 232 | init_string) 233 | 234 | if mode == tf.estimator.ModeKeys.TRAIN: 235 | 236 | train_op = optimization.create_optimizer( 237 | total_loss, learning_rate, num_train_steps, num_warmup_steps, False) 238 | 239 | output_spec = EstimatorSpec( 240 | mode=mode, 241 | loss=total_loss, 242 | train_op=train_op) 243 | elif mode == tf.estimator.ModeKeys.EVAL: 244 | 245 | def metric_fn(per_example_loss, label_ids, logits): 246 | predictions = tf.argmax(logits, axis=-1, output_type=tf.int32) 247 | accuracy = tf.metrics.accuracy(label_ids, predictions) 248 | auc = tf.metrics.auc(label_ids, predictions) 249 | loss = tf.metrics.mean(per_example_loss) 250 | return { 251 | "eval_accuracy": accuracy, 252 | "eval_auc": auc, 253 | "eval_loss": loss, 254 | } 255 | 256 | eval_metrics = metric_fn(per_example_loss, label_ids, logits) 257 | output_spec = EstimatorSpec( 258 | mode=mode, 259 | loss=total_loss, 260 | eval_metric_ops=eval_metrics) 261 | else: 262 | output_spec = EstimatorSpec(mode=mode, predictions=probabilities) 263 | 264 | return output_spec 265 | 266 | return model_fn 267 | 268 | def get_estimator(self): 269 | 270 | from tensorflow.python.estimator.estimator import Estimator 271 | from tensorflow.python.estimator.run_config import RunConfig 272 | 273 | bert_config = modeling.BertConfig.from_json_file(args.config_name) 274 | label_list = self.processor.get_labels() 275 | train_examples = self.processor.get_train_examples(args.dataset_path) 276 | num_train_steps = int( 277 | len(train_examples) / self.batch_size * args.num_train_epochs) 278 | num_warmup_steps = int(num_train_steps * 0.1) 279 | 280 | if self.mode == tf.estimator.ModeKeys.TRAIN: 281 | init_checkpoint = args.ckpt_name 282 | else: 283 | init_checkpoint = args.output_dir 284 | 285 | model_fn = self.model_fn_builder( 286 | bert_config=bert_config, 287 | num_labels=len(label_list), 288 | init_checkpoint=init_checkpoint, 289 | learning_rate=args.learning_rate, 290 | num_train_steps=num_train_steps, 291 | num_warmup_steps=num_warmup_steps, 292 | use_one_hot_embeddings=False) 293 | 294 | config = tf.ConfigProto() 295 | config.gpu_options.allow_growth = True 296 | config.gpu_options.per_process_gpu_memory_fraction = args.gpu_memory_fraction 297 | config.log_device_placement = False 298 | 299 | return Estimator(model_fn=model_fn, config=RunConfig(session_config=config), model_dir=args.output_dir, 300 | params={'batch_size': self.batch_size}) 301 | 302 | def predict_from_queue(self): 303 | for i in self.estimator.predict(input_fn=self.queue_predict_input_fn, yield_single_examples=False): 304 | self.output_queue.put(i) 305 | 306 | def queue_predict_input_fn(self): 307 | return (tf.data.Dataset.from_generator( 308 | self.generate_from_queue, 309 | output_types={ 310 | 'input_ids': tf.int32, 311 | 'input_mask': tf.int32, 312 | 'segment_ids': tf.int32, 313 | 'label_ids': tf.int32}, 314 | output_shapes={ 315 | 'input_ids': (None, self.max_seq_length), 316 | 'input_mask': (None, self.max_seq_length), 317 | 'segment_ids': (None, self.max_seq_length), 318 | 'label_ids': (1,)}).prefetch(10)) 319 | 320 | def convert_examples_to_features(self, examples, label_list, max_seq_length, tokenizer): 321 | """Convert a set of `InputExample`s to a list of `InputFeatures`.""" 322 | 323 | for (ex_index, example) in enumerate(examples): 324 | label_map = {} 325 | for (i, label) in enumerate(label_list): 326 | label_map[label] = i 327 | 328 | tokens_a = tokenizer.tokenize(example.text_a) 329 | tokens_b = None 330 | if example.text_b: 331 | tokens_b = tokenizer.tokenize(example.text_b) 332 | 333 | if tokens_b: 334 | # Modifies `tokens_a` and `tokens_b` in place so that the total 335 | # length is less than the specified length. 336 | # Account for [CLS], [SEP], [SEP] with "- 3" 337 | self._truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) 338 | else: 339 | # Account for [CLS] and [SEP] with "- 2" 340 | if len(tokens_a) > max_seq_length - 2: 341 | tokens_a = tokens_a[0:(max_seq_length - 2)] 342 | 343 | # The convention in BERT is: 344 | # (a) For sequence pairs: 345 | # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] 346 | # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 347 | # (b) For single sequences: 348 | # tokens: [CLS] the dog is hairy . [SEP] 349 | # type_ids: 0 0 0 0 0 0 0 350 | # 351 | # Where "type_ids" are used to indicate whether this is the first 352 | # sequence or the second sequence. The embedding vectors for `type=0` and 353 | # `type=1` were learned during pre-training and are added to the wordpiece 354 | # embedding vector (and position vector). This is not *strictly* necessary 355 | # since the [SEP] token unambiguously separates the sequences, but it makes 356 | # it easier for the model to learn the concept of sequences. 357 | # 358 | # For classification tasks, the first vector (corresponding to [CLS]) is 359 | # used as as the "sentence vector". Note that this only makes sense because 360 | # the entire model is fine-tuned. 361 | tokens = [] 362 | segment_ids = [] 363 | tokens.append("[CLS]") 364 | segment_ids.append(0) 365 | for token in tokens_a: 366 | tokens.append(token) 367 | segment_ids.append(0) 368 | tokens.append("[SEP]") 369 | segment_ids.append(0) 370 | 371 | if tokens_b: 372 | for token in tokens_b: 373 | tokens.append(token) 374 | segment_ids.append(1) 375 | tokens.append("[SEP]") 376 | segment_ids.append(1) 377 | 378 | input_ids = tokenizer.convert_tokens_to_ids(tokens) 379 | 380 | # The mask has 1 for real tokens and 0 for padding tokens. Only real 381 | # tokens are attended to. 382 | input_mask = [1] * len(input_ids) 383 | 384 | # Zero-pad up to the sequence length. 385 | while len(input_ids) < max_seq_length: 386 | input_ids.append(0) 387 | input_mask.append(0) 388 | segment_ids.append(0) 389 | 390 | assert len(input_ids) == max_seq_length 391 | assert len(input_mask) == max_seq_length 392 | assert len(segment_ids) == max_seq_length 393 | 394 | label_id = label_map[example.label] 395 | if ex_index < 5: 396 | tf.logging.info("*** Example ***") 397 | tf.logging.info("guid: %s" % (example.guid)) 398 | tf.logging.info("tokens: %s" % " ".join( 399 | [tokenization.printable_text(x) for x in tokens])) 400 | tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) 401 | tf.logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) 402 | tf.logging.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) 403 | tf.logging.info("label: %s (id = %d)" % (example.label, label_id)) 404 | 405 | feature = InputFeatures( 406 | input_ids=input_ids, 407 | input_mask=input_mask, 408 | segment_ids=segment_ids, 409 | label_id=label_id) 410 | 411 | yield feature 412 | 413 | def generate_from_queue(self): 414 | while True: 415 | predict_examples = self.processor.get_sentence_examples(self.input_queue.get()) 416 | features = list(self.convert_examples_to_features(predict_examples, self.processor.get_labels(), 417 | args.max_seq_len, self.tokenizer)) 418 | yield { 419 | 'input_ids': [f.input_ids for f in features], 420 | 'input_mask': [f.input_mask for f in features], 421 | 'segment_ids': [f.segment_ids for f in features], 422 | 'label_ids': [f.label_id for f in features] 423 | } 424 | 425 | def _truncate_seq_pair(self, tokens_a, tokens_b, max_length): 426 | """Truncates a sequence pair in place to the maximum length.""" 427 | 428 | # This is a simple heuristic which will always truncate the longer sequence 429 | # one token at a time. This makes more sense than truncating an equal percent 430 | # of tokens from each, since if one sequence is very short then each token 431 | # that's truncated likely contains more information than a longer sequence. 432 | while True: 433 | total_length = len(tokens_a) + len(tokens_b) 434 | if total_length <= max_length: 435 | break 436 | if len(tokens_a) > len(tokens_b): 437 | tokens_a.pop() 438 | else: 439 | tokens_b.pop() 440 | 441 | def convert_single_example(self, ex_index, example, label_list, max_seq_length, tokenizer): 442 | """Converts a single `InputExample` into a single `InputFeatures`.""" 443 | label_map = {} 444 | for (i, label) in enumerate(label_list): 445 | label_map[label] = i 446 | 447 | tokens_a = tokenizer.tokenize(example.text_a) 448 | tokens_b = None 449 | if example.text_b: 450 | tokens_b = tokenizer.tokenize(example.text_b) 451 | 452 | if tokens_b: 453 | # Modifies `tokens_a` and `tokens_b` in place so that the total 454 | # length is less than the specified length. 455 | # Account for [CLS], [SEP], [SEP] with "- 3" 456 | self._truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) 457 | else: 458 | # Account for [CLS] and [SEP] with "- 2" 459 | if len(tokens_a) > max_seq_length - 2: 460 | tokens_a = tokens_a[0:(max_seq_length - 2)] 461 | 462 | # The convention in BERT is: 463 | # (a) For sequence pairs: 464 | # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] 465 | # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 466 | # (b) For single sequences: 467 | # tokens: [CLS] the dog is hairy . [SEP] 468 | # type_ids: 0 0 0 0 0 0 0 469 | # 470 | # Where "type_ids" are used to indicate whether this is the first 471 | # sequence or the second sequence. The embedding vectors for `type=0` and 472 | # `type=1` were learned during pre-training and are added to the wordpiece 473 | # embedding vector (and position vector). This is not *strictly* necessary 474 | # since the [SEP] token unambiguously separates the sequences, but it makes 475 | # it easier for the model to learn the concept of sequences. 476 | # 477 | # For classification tasks, the first vector (corresponding to [CLS]) is 478 | # used as as the "sentence vector". Note that this only makes sense because 479 | # the entire model is fine-tuned. 480 | tokens = [] 481 | segment_ids = [] 482 | tokens.append("[CLS]") 483 | segment_ids.append(0) 484 | for token in tokens_a: 485 | tokens.append(token) 486 | segment_ids.append(0) 487 | tokens.append("[SEP]") 488 | segment_ids.append(0) 489 | 490 | if tokens_b: 491 | for token in tokens_b: 492 | tokens.append(token) 493 | segment_ids.append(1) 494 | tokens.append("[SEP]") 495 | segment_ids.append(1) 496 | 497 | input_ids = tokenizer.convert_tokens_to_ids(tokens) 498 | 499 | # The mask has 1 for real tokens and 0 for padding tokens. Only real 500 | # tokens are attended to. 501 | input_mask = [1] * len(input_ids) 502 | 503 | # Zero-pad up to the sequence length. 504 | while len(input_ids) < max_seq_length: 505 | input_ids.append(0) 506 | input_mask.append(0) 507 | segment_ids.append(0) 508 | 509 | assert len(input_ids) == max_seq_length 510 | assert len(input_mask) == max_seq_length 511 | assert len(segment_ids) == max_seq_length 512 | 513 | label_id = label_map[example.label] 514 | if ex_index < 5: 515 | tf.logging.info("*** Example ***") 516 | tf.logging.info("guid: %s" % (example.guid)) 517 | tf.logging.info("tokens: %s" % " ".join( 518 | [tokenization.printable_text(x) for x in tokens])) 519 | tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) 520 | tf.logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) 521 | tf.logging.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) 522 | tf.logging.info("label: %s (id = %d)" % (example.label, label_id)) 523 | 524 | feature = InputFeatures( 525 | input_ids=input_ids, 526 | input_mask=input_mask, 527 | segment_ids=segment_ids, 528 | label_id=label_id) 529 | return feature 530 | 531 | def file_based_convert_examples_to_features(self, examples, label_list, max_seq_length, tokenizer, output_file): 532 | """Convert a set of `InputExample`s to a TFRecord file.""" 533 | 534 | writer = tf.python_io.TFRecordWriter(output_file) 535 | 536 | for (ex_index, example) in enumerate(examples): 537 | if ex_index % 10000 == 0: 538 | tf.logging.info("Writing example %d of %d" % (ex_index, len(examples))) 539 | 540 | feature = self.convert_single_example(ex_index, example, label_list, 541 | max_seq_length, tokenizer) 542 | 543 | def create_int_feature(values): 544 | f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) 545 | return f 546 | 547 | features = collections.OrderedDict() 548 | features["input_ids"] = create_int_feature(feature.input_ids) 549 | features["input_mask"] = create_int_feature(feature.input_mask) 550 | features["segment_ids"] = create_int_feature(feature.segment_ids) 551 | features["label_ids"] = create_int_feature([feature.label_id]) 552 | 553 | tf_example = tf.train.Example(features=tf.train.Features(feature=features)) 554 | writer.write(tf_example.SerializeToString()) 555 | 556 | def file_based_input_fn_builder(self, input_file, seq_length, is_training, drop_remainder): 557 | """Creates an `input_fn` closure to be passed to TPUEstimator.""" 558 | 559 | name_to_features = { 560 | "input_ids": tf.FixedLenFeature([seq_length], tf.int64), 561 | "input_mask": tf.FixedLenFeature([seq_length], tf.int64), 562 | "segment_ids": tf.FixedLenFeature([seq_length], tf.int64), 563 | "label_ids": tf.FixedLenFeature([], tf.int64), 564 | } 565 | 566 | def _decode_record(record, name_to_features): 567 | """Decodes a record to a TensorFlow example.""" 568 | example = tf.parse_single_example(record, name_to_features) 569 | 570 | # tf.Example only supports tf.int64, but the TPU only supports tf.int32. 571 | # So cast all int64 to int32. 572 | for name in list(example.keys()): 573 | t = example[name] 574 | if t.dtype == tf.int64: 575 | t = tf.to_int32(t) 576 | example[name] = t 577 | 578 | return example 579 | 580 | def input_fn(params): 581 | """The actual input function.""" 582 | batch_size = params["batch_size"] 583 | 584 | # For training, we want a lot of parallel reading and shuffling. 585 | # For eval, we want no shuffling and parallel reading doesn't matter. 586 | d = tf.data.TFRecordDataset(input_file) 587 | if is_training: 588 | d = d.repeat() 589 | d = d.shuffle(buffer_size=100) 590 | 591 | d = d.apply( 592 | tf.contrib.data.map_and_batch( 593 | lambda record: _decode_record(record, name_to_features), 594 | batch_size=batch_size, 595 | drop_remainder=drop_remainder)) 596 | 597 | return d 598 | 599 | return input_fn 600 | 601 | def train(self): 602 | if self.mode is None: 603 | raise ValueError("Please set the 'mode' parameter") 604 | 605 | bert_config = modeling.BertConfig.from_json_file(args.config_name) 606 | 607 | if args.max_seq_len > bert_config.max_position_embeddings: 608 | raise ValueError( 609 | "Cannot use sequence length %d because the BERT model " 610 | "was only trained up to sequence length %d" % 611 | (args.max_seq_len, bert_config.max_position_embeddings)) 612 | 613 | tf.gfile.MakeDirs(args.output_dir) 614 | 615 | label_list = self.processor.get_labels() 616 | 617 | train_examples = self.processor.get_train_examples(args.dataset_path) 618 | num_train_steps = int(len(train_examples) / args.batch_size * args.num_train_epochs) 619 | 620 | estimator = self.get_estimator() 621 | 622 | train_file = os.path.join(args.output_dir, "train.tf_record") 623 | self.file_based_convert_examples_to_features(train_examples, label_list, args.max_seq_len, self.tokenizer, 624 | train_file) 625 | tf.logging.info("***** Running training *****") 626 | tf.logging.info(" Num examples = %d", len(train_examples)) 627 | tf.logging.info(" Batch size = %d", args.batch_size) 628 | tf.logging.info(" Num steps = %d", num_train_steps) 629 | train_input_fn = self.file_based_input_fn_builder(input_file=train_file, seq_length=args.max_seq_len, 630 | is_training=True, 631 | drop_remainder=True) 632 | 633 | # early_stopping = tf.contrib.estimator.stop_if_no_decrease_hook( 634 | # estimator, 635 | # metric_name='loss', 636 | # max_steps_without_decrease=10, 637 | # min_steps=num_train_steps) 638 | 639 | # estimator.train(input_fn=train_input_fn, hooks=[early_stopping]) 640 | estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) 641 | 642 | def eval(self): 643 | if self.mode is None: 644 | raise ValueError("Please set the 'mode' parameter") 645 | eval_examples = self.processor.get_dev_examples(args.dataset_path) 646 | eval_file = os.path.join(args.output_dir, "eval.tf_record") 647 | label_list = self.processor.get_labels() 648 | self.file_based_convert_examples_to_features( 649 | eval_examples, label_list, args.max_seq_len, self.tokenizer, eval_file) 650 | 651 | tf.logging.info("***** Running evaluation *****") 652 | tf.logging.info(" Num examples = %d", len(eval_examples)) 653 | tf.logging.info(" Batch size = %d", self.batch_size) 654 | 655 | eval_input_fn = self.file_based_input_fn_builder( 656 | input_file=eval_file, 657 | seq_length=args.max_seq_len, 658 | is_training=False, 659 | drop_remainder=False) 660 | 661 | estimator = self.get_estimator() 662 | result = estimator.evaluate(input_fn=eval_input_fn, steps=None) 663 | 664 | output_eval_file = os.path.join(args.output_dir, "eval_results.txt") 665 | with tf.gfile.GFile(output_eval_file, "w") as writer: 666 | tf.logging.info("***** Eval results *****") 667 | for key in sorted(result.keys()): 668 | tf.logging.info(" %s = %s", key, str(result[key])) 669 | writer.write("%s = %s\n" % (key, str(result[key]))) 670 | 671 | def predict(self, sentence1, sentence2): 672 | if self.mode is None: 673 | raise ValueError("Please set the 'mode' parameter") 674 | self.input_queue.put([(sentence1, sentence2)]) 675 | prediction = self.output_queue.get() 676 | return prediction 677 | 678 | 679 | def main(_): 680 | sim = BertSim() 681 | if FLAGS.mode == tf.estimator.ModeKeys.TRAIN: 682 | sim.set_mode(tf.estimator.ModeKeys.TRAIN) 683 | sim.train() 684 | elif FLAGS.mode == tf.estimator.ModeKeys.EVAL: 685 | sim.set_mode(tf.estimator.ModeKeys.EVAL) 686 | sim.eval() 687 | elif FLAGS.mode == tf.estimator.ModeKeys.PREDICT: 688 | sim.set_mode(tf.estimator.ModeKeys.PREDICT) 689 | while True: 690 | sentence1 = input('sentence1: ') 691 | sentence2 = input('sentence2: ') 692 | predict = sim.predict(sentence1, sentence2) 693 | print(f'similarity：{predict[0][1]}') 694 | 695 | 696 | if __name__ == '__main__': 697 | flags.mark_flag_as_required("mode") 698 | tf.app.run() 699 | -------------------------------------------------------------------------------- /modeling.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """The main BERT model and related functions.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import collections 22 | import copy 23 | import json 24 | import math 25 | import re 26 | import six 27 | import tensorflow as tf 28 | 29 | 30 | class BertConfig(object): 31 | """Configuration for `BertModel`.""" 32 | 33 | def __init__(self, 34 | vocab_size, 35 | hidden_size=768, 36 | num_hidden_layers=12, 37 | num_attention_heads=12, 38 | intermediate_size=3072, 39 | hidden_act="gelu", 40 | hidden_dropout_prob=0.1, 41 | attention_probs_dropout_prob=0.1, 42 | max_position_embeddings=512, 43 | type_vocab_size=16, 44 | initializer_range=0.02): 45 | """Constructs BertConfig. 46 | 47 | Args: 48 | vocab_size: Vocabulary size of `inputs_ids` in `BertModel`. 49 | hidden_size: Size of the encoder layers and the pooler layer. 50 | num_hidden_layers: Number of hidden layers in the Transformer encoder. 51 | num_attention_heads: Number of attention heads for each attention layer in 52 | the Transformer encoder. 53 | intermediate_size: The size of the "intermediate" (i.e., feed-forward) 54 | layer in the Transformer encoder. 55 | hidden_act: The non-linear activation function (function or string) in the 56 | encoder and pooler. 57 | hidden_dropout_prob: The dropout probability for all fully connected 58 | layers in the embeddings, encoder, and pooler. 59 | attention_probs_dropout_prob: The dropout ratio for the attention 60 | probabilities. 61 | max_position_embeddings: The maximum sequence length that this model might 62 | ever be used with. Typically set this to something large just in case 63 | (e.g., 512 or 1024 or 2048). 64 | type_vocab_size: The vocabulary size of the `token_type_ids` passed into 65 | `BertModel`. 66 | initializer_range: The stdev of the truncated_normal_initializer for 67 | initializing all weight matrices. 68 | """ 69 | self.vocab_size = vocab_size 70 | self.hidden_size = hidden_size 71 | self.num_hidden_layers = num_hidden_layers 72 | self.num_attention_heads = num_attention_heads 73 | self.hidden_act = hidden_act 74 | self.intermediate_size = intermediate_size 75 | self.hidden_dropout_prob = hidden_dropout_prob 76 | self.attention_probs_dropout_prob = attention_probs_dropout_prob 77 | self.max_position_embeddings = max_position_embeddings 78 | self.type_vocab_size = type_vocab_size 79 | self.initializer_range = initializer_range 80 | 81 | @classmethod 82 | def from_dict(cls, json_object): 83 | """Constructs a `BertConfig` from a Python dictionary of parameters.""" 84 | config = BertConfig(vocab_size=None) 85 | for (key, value) in six.iteritems(json_object): 86 | config.__dict__[key] = value 87 | return config 88 | 89 | @classmethod 90 | def from_json_file(cls, json_file): 91 | """Constructs a `BertConfig` from a json file of parameters.""" 92 | with tf.gfile.GFile(json_file, "r") as reader: 93 | text = reader.read() 94 | return cls.from_dict(json.loads(text)) 95 | 96 | def to_dict(self): 97 | """Serializes this instance to a Python dictionary.""" 98 | output = copy.deepcopy(self.__dict__) 99 | return output 100 | 101 | def to_json_string(self): 102 | """Serializes this instance to a JSON string.""" 103 | return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" 104 | 105 | 106 | class BertModel(object): 107 | """BERT model ("Bidirectional Embedding Representations from a Transformer"). 108 | 109 | Example usage: 110 | 111 | ```python 112 | # Already been converted into WordPiece token ids 113 | input_ids = tf.constant([[31, 51, 99], [15, 5, 0]]) 114 | input_mask = tf.constant([[1, 1, 1], [1, 1, 0]]) 115 | token_type_ids = tf.constant([[0, 0, 1], [0, 2, 0]]) 116 | 117 | config = modeling.BertConfig(vocab_size=32000, hidden_size=512, 118 | num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024) 119 | 120 | model = modeling.BertModel(config=config, is_training=True, 121 | input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids) 122 | 123 | label_embeddings = tf.get_variable(...) 124 | pooled_output = model.get_pooled_output() 125 | logits = tf.matmul(pooled_output, label_embeddings) 126 | ... 127 | ``` 128 | """ 129 | 130 | def __init__(self, 131 | config, 132 | is_training, 133 | input_ids, 134 | input_mask=None, 135 | token_type_ids=None, 136 | use_one_hot_embeddings=True, 137 | scope=None): 138 | """Constructor for BertModel. 139 | 140 | Args: 141 | config: `BertConfig` instance. 142 | is_training: bool. rue for training model, false for eval model. Controls 143 | whether dropout will be applied. 144 | input_ids: int32 Tensor of shape [batch_size, seq_length]. 145 | input_mask: (optional) int32 Tensor of shape [batch_size, seq_length]. 146 | token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length]. 147 | use_one_hot_embeddings: (optional) bool. Whether to use one-hot word 148 | embeddings or tf.embedding_lookup() for the word embeddings. On the TPU, 149 | it is must faster if this is True, on the CPU or GPU, it is faster if 150 | this is False. 151 | scope: (optional) variable scope. Defaults to "bert". 152 | 153 | Raises: 154 | ValueError: The config is invalid or one of the input tensor shapes 155 | is invalid. 156 | """ 157 | config = copy.deepcopy(config) 158 | if not is_training: 159 | config.hidden_dropout_prob = 0.0 160 | config.attention_probs_dropout_prob = 0.0 161 | 162 | input_shape = get_shape_list(input_ids, expected_rank=2) 163 | batch_size = input_shape[0] 164 | seq_length = input_shape[1] 165 | 166 | if input_mask is None: 167 | input_mask = tf.ones(shape=[batch_size, seq_length], dtype=tf.int32) 168 | 169 | if token_type_ids is None: 170 | token_type_ids = tf.zeros(shape=[batch_size, seq_length], dtype=tf.int32) 171 | 172 | with tf.variable_scope(scope, default_name="bert"): 173 | with tf.variable_scope("embeddings"): 174 | # Perform embedding lookup on the word ids. 175 | (self.embedding_output, self.embedding_table) = embedding_lookup( 176 | input_ids=input_ids, 177 | vocab_size=config.vocab_size, 178 | embedding_size=config.hidden_size, 179 | initializer_range=config.initializer_range, 180 | word_embedding_name="word_embeddings", 181 | use_one_hot_embeddings=use_one_hot_embeddings) 182 | 183 | # Add positional embeddings and token type embeddings, then layer 184 | # normalize and perform dropout. 185 | self.embedding_output = embedding_postprocessor( 186 | input_tensor=self.embedding_output, 187 | use_token_type=True, 188 | token_type_ids=token_type_ids, 189 | token_type_vocab_size=config.type_vocab_size, 190 | token_type_embedding_name="token_type_embeddings", 191 | use_position_embeddings=True, 192 | position_embedding_name="position_embeddings", 193 | initializer_range=config.initializer_range, 194 | max_position_embeddings=config.max_position_embeddings, 195 | dropout_prob=config.hidden_dropout_prob) 196 | 197 | with tf.variable_scope("encoder"): 198 | # This converts a 2D mask of shape [batch_size, seq_length] to a 3D 199 | # mask of shape [batch_size, seq_length, seq_length] which is used 200 | # for the attention scores. 201 | attention_mask = create_attention_mask_from_input_mask( 202 | input_ids, input_mask) 203 | 204 | # Run the stacked transformer. 205 | # `sequence_output` shape = [batch_size, seq_length, hidden_size]. 206 | self.all_encoder_layers = transformer_model( 207 | input_tensor=self.embedding_output, 208 | attention_mask=attention_mask, 209 | hidden_size=config.hidden_size, 210 | num_hidden_layers=config.num_hidden_layers, 211 | num_attention_heads=config.num_attention_heads, 212 | intermediate_size=config.intermediate_size, 213 | intermediate_act_fn=get_activation(config.hidden_act), 214 | hidden_dropout_prob=config.hidden_dropout_prob, 215 | attention_probs_dropout_prob=config.attention_probs_dropout_prob, 216 | initializer_range=config.initializer_range, 217 | do_return_all_layers=True) 218 | 219 | self.sequence_output = self.all_encoder_layers[-1] 220 | # The "pooler" converts the encoded sequence tensor of shape 221 | # [batch_size, seq_length, hidden_size] to a tensor of shape 222 | # [batch_size, hidden_size]. This is necessary for segment-level 223 | # (or segment-pair-level) classification tasks where we need a fixed 224 | # dimensional representation of the segment. 225 | with tf.variable_scope("pooler"): 226 | # We "pool" the model by simply taking the hidden state corresponding 227 | # to the first token. We assume that this has been pre-trained 228 | first_token_tensor = tf.squeeze(self.sequence_output[:, 0:1, :], axis=1) 229 | self.pooled_output = tf.layers.dense( 230 | first_token_tensor, 231 | config.hidden_size, 232 | activation=tf.tanh, 233 | kernel_initializer=create_initializer(config.initializer_range)) 234 | 235 | def get_pooled_output(self): 236 | return self.pooled_output 237 | 238 | def get_sequence_output(self): 239 | """Gets final hidden layer of encoder. 240 | 241 | Returns: 242 | float Tensor of shape [batch_size, seq_length, hidden_size] corresponding 243 | to the final hidden of the transformer encoder. 244 | """ 245 | return self.sequence_output 246 | 247 | def get_all_encoder_layers(self): 248 | return self.all_encoder_layers 249 | 250 | def get_embedding_output(self): 251 | """Gets output of the embedding lookup (i.e., input to the transformer). 252 | 253 | Returns: 254 | float Tensor of shape [batch_size, seq_length, hidden_size] corresponding 255 | to the output of the embedding layer, after summing the word 256 | embeddings with the positional embeddings and the token type embeddings, 257 | then performing layer normalization. This is the input to the transformer. 258 | """ 259 | return self.embedding_output 260 | 261 | def get_embedding_table(self): 262 | return self.embedding_table 263 | 264 | 265 | def gelu(input_tensor): 266 | """Gaussian Error Linear Unit. 267 | 268 | This is a smoother version of the RELU. 269 | Original paper: https://arxiv.org/abs/1606.08415 270 | 271 | Args: 272 | input_tensor: float Tensor to perform activation. 273 | 274 | Returns: 275 | `input_tensor` with the GELU activation applied. 276 | """ 277 | cdf = 0.5 * (1.0 + tf.erf(input_tensor / tf.sqrt(2.0))) 278 | return input_tensor * cdf 279 | 280 | 281 | def get_activation(activation_string): 282 | """Maps a string to a Python function, e.g., "relu" => `tf.nn.relu`. 283 | 284 | Args: 285 | activation_string: String name of the activation function. 286 | 287 | Returns: 288 | A Python function corresponding to the activation function. If 289 | `activation_string` is None, empty, or "linear", this will return None. 290 | If `activation_string` is not a string, it will return `activation_string`. 291 | 292 | Raises: 293 | ValueError: The `activation_string` does not correspond to a known 294 | activation. 295 | """ 296 | 297 | # We assume that anything that"s not a string is already an activation 298 | # function, so we just return it. 299 | if not isinstance(activation_string, six.string_types): 300 | return activation_string 301 | 302 | if not activation_string: 303 | return None 304 | 305 | act = activation_string.lower() 306 | if act == "linear": 307 | return None 308 | elif act == "relu": 309 | return tf.nn.relu 310 | elif act == "gelu": 311 | return gelu 312 | elif act == "tanh": 313 | return tf.tanh 314 | else: 315 | raise ValueError("Unsupported activation: %s" % act) 316 | 317 | 318 | def get_assignment_map_from_checkpoint(tvars, init_checkpoint): 319 | """Compute the union of the current variables and checkpoint variables.""" 320 | assignment_map = {} 321 | initialized_variable_names = {} 322 | 323 | name_to_variable = collections.OrderedDict() 324 | for var in tvars: 325 | name = var.name 326 | m = re.match("^(.*):\\d+$", name) 327 | if m is not None: 328 | name = m.group(1) 329 | name_to_variable[name] = var 330 | 331 | init_vars = tf.train.list_variables(init_checkpoint) 332 | 333 | assignment_map = collections.OrderedDict() 334 | for x in init_vars: 335 | (name, var) = (x[0], x[1]) 336 | if name not in name_to_variable: 337 | continue 338 | assignment_map[name] = name 339 | initialized_variable_names[name] = 1 340 | initialized_variable_names[name + ":0"] = 1 341 | 342 | return (assignment_map, initialized_variable_names) 343 | 344 | 345 | def dropout(input_tensor, dropout_prob): 346 | """Perform dropout. 347 | 348 | Args: 349 | input_tensor: float Tensor. 350 | dropout_prob: Python float. The probability of dropping out a value (NOT of 351 | *keeping* a dimension as in `tf.nn.dropout`). 352 | 353 | Returns: 354 | A version of `input_tensor` with dropout applied. 355 | """ 356 | if dropout_prob is None or dropout_prob == 0.0: 357 | return input_tensor 358 | 359 | output = tf.nn.dropout(input_tensor, 1.0 - dropout_prob) 360 | return output 361 | 362 | 363 | def layer_norm(input_tensor, name=None): 364 | """Run layer normalization on the last dimension of the tensor.""" 365 | return tf.contrib.layers.layer_norm( 366 | inputs=input_tensor, begin_norm_axis=-1, begin_params_axis=-1, scope=name) 367 | 368 | 369 | def layer_norm_and_dropout(input_tensor, dropout_prob, name=None): 370 | """Runs layer normalization followed by dropout.""" 371 | output_tensor = layer_norm(input_tensor, name) 372 | output_tensor = dropout(output_tensor, dropout_prob) 373 | return output_tensor 374 | 375 | 376 | def create_initializer(initializer_range=0.02): 377 | """Creates a `truncated_normal_initializer` with the given range.""" 378 | return tf.truncated_normal_initializer(stddev=initializer_range) 379 | 380 | 381 | def embedding_lookup(input_ids, 382 | vocab_size, 383 | embedding_size=128, 384 | initializer_range=0.02, 385 | word_embedding_name="word_embeddings", 386 | use_one_hot_embeddings=False): 387 | """Looks up words embeddings for id tensor. 388 | 389 | Args: 390 | input_ids: int32 Tensor of shape [batch_size, seq_length] containing word 391 | ids. 392 | vocab_size: int. Size of the embedding vocabulary. 393 | embedding_size: int. Width of the word embeddings. 394 | initializer_range: float. Embedding initialization range. 395 | word_embedding_name: string. Name of the embedding table. 396 | use_one_hot_embeddings: bool. If True, use one-hot method for word 397 | embeddings. If False, use `tf.nn.embedding_lookup()`. One hot is better 398 | for TPUs. 399 | 400 | Returns: 401 | float Tensor of shape [batch_size, seq_length, embedding_size]. 402 | """ 403 | # This function assumes that the input is of shape [batch_size, seq_length, 404 | # num_inputs]. 405 | # 406 | # If the input is a 2D tensor of shape [batch_size, seq_length], we 407 | # reshape to [batch_size, seq_length, 1]. 408 | if input_ids.shape.ndims == 2: 409 | input_ids = tf.expand_dims(input_ids, axis=[-1]) 410 | 411 | embedding_table = tf.get_variable( 412 | name=word_embedding_name, 413 | shape=[vocab_size, embedding_size], 414 | initializer=create_initializer(initializer_range)) 415 | 416 | if use_one_hot_embeddings: 417 | flat_input_ids = tf.reshape(input_ids, [-1]) 418 | one_hot_input_ids = tf.one_hot(flat_input_ids, depth=vocab_size) 419 | output = tf.matmul(one_hot_input_ids, embedding_table) 420 | else: 421 | output = tf.nn.embedding_lookup(embedding_table, input_ids) 422 | 423 | input_shape = get_shape_list(input_ids) 424 | 425 | output = tf.reshape(output, 426 | input_shape[0:-1] + [input_shape[-1] * embedding_size]) 427 | return (output, embedding_table) 428 | 429 | 430 | def embedding_postprocessor(input_tensor, 431 | use_token_type=False, 432 | token_type_ids=None, 433 | token_type_vocab_size=16, 434 | token_type_embedding_name="token_type_embeddings", 435 | use_position_embeddings=True, 436 | position_embedding_name="position_embeddings", 437 | initializer_range=0.02, 438 | max_position_embeddings=512, 439 | dropout_prob=0.1): 440 | """Performs various post-processing on a word embedding tensor. 441 | 442 | Args: 443 | input_tensor: float Tensor of shape [batch_size, seq_length, 444 | embedding_size]. 445 | use_token_type: bool. Whether to add embeddings for `token_type_ids`. 446 | token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length]. 447 | Must be specified if `use_token_type` is True. 448 | token_type_vocab_size: int. The vocabulary size of `token_type_ids`. 449 | token_type_embedding_name: string. The name of the embedding table variable 450 | for token type ids. 451 | use_position_embeddings: bool. Whether to add position embeddings for the 452 | position of each token in the sequence. 453 | position_embedding_name: string. The name of the embedding table variable 454 | for positional embeddings. 455 | initializer_range: float. Range of the weight initialization. 456 | max_position_embeddings: int. Maximum sequence length that might ever be 457 | used with this model. This can be longer than the sequence length of 458 | input_tensor, but cannot be shorter. 459 | dropout_prob: float. Dropout probability applied to the final output tensor. 460 | 461 | Returns: 462 | float tensor with same shape as `input_tensor`. 463 | 464 | Raises: 465 | ValueError: One of the tensor shapes or input values is invalid. 466 | """ 467 | input_shape = get_shape_list(input_tensor, expected_rank=3) 468 | batch_size = input_shape[0] 469 | seq_length = input_shape[1] 470 | width = input_shape[2] 471 | 472 | output = input_tensor 473 | 474 | if use_token_type: 475 | if token_type_ids is None: 476 | raise ValueError("`token_type_ids` must be specified if" 477 | "`use_token_type` is True.") 478 | token_type_table = tf.get_variable( 479 | name=token_type_embedding_name, 480 | shape=[token_type_vocab_size, width], 481 | initializer=create_initializer(initializer_range)) 482 | # This vocab will be small so we always do one-hot here, since it is always 483 | # faster for a small vocabulary. 484 | flat_token_type_ids = tf.reshape(token_type_ids, [-1]) 485 | one_hot_ids = tf.one_hot(flat_token_type_ids, depth=token_type_vocab_size) 486 | token_type_embeddings = tf.matmul(one_hot_ids, token_type_table) 487 | token_type_embeddings = tf.reshape(token_type_embeddings, 488 | [batch_size, seq_length, width]) 489 | output += token_type_embeddings 490 | 491 | if use_position_embeddings: 492 | assert_op = tf.assert_less_equal(seq_length, max_position_embeddings) 493 | with tf.control_dependencies([assert_op]): 494 | full_position_embeddings = tf.get_variable( 495 | name=position_embedding_name, 496 | shape=[max_position_embeddings, width], 497 | initializer=create_initializer(initializer_range)) 498 | # Since the position embedding table is a learned variable, we create it 499 | # using a (long) sequence length `max_position_embeddings`. The actual 500 | # sequence length might be shorter than this, for faster training of 501 | # tasks that do not have long sequences. 502 | # 503 | # So `full_position_embeddings` is effectively an embedding table 504 | # for position [0, 1, 2, ..., max_position_embeddings-1], and the current 505 | # sequence has positions [0, 1, 2, ... seq_length-1], so we can just 506 | # perform a slice. 507 | position_embeddings = tf.slice(full_position_embeddings, [0, 0], 508 | [seq_length, -1]) 509 | num_dims = len(output.shape.as_list()) 510 | 511 | # Only the last two dimensions are relevant (`seq_length` and `width`), so 512 | # we broadcast among the first dimensions, which is typically just 513 | # the batch size. 514 | position_broadcast_shape = [] 515 | for _ in range(num_dims - 2): 516 | position_broadcast_shape.append(1) 517 | position_broadcast_shape.extend([seq_length, width]) 518 | position_embeddings = tf.reshape(position_embeddings, 519 | position_broadcast_shape) 520 | output += position_embeddings 521 | 522 | output = layer_norm_and_dropout(output, dropout_prob) 523 | return output 524 | 525 | 526 | def create_attention_mask_from_input_mask(from_tensor, to_mask): 527 | """Create 3D attention mask from a 2D tensor mask. 528 | 529 | Args: 530 | from_tensor: 2D or 3D Tensor of shape [batch_size, from_seq_length, ...]. 531 | to_mask: int32 Tensor of shape [batch_size, to_seq_length]. 532 | 533 | Returns: 534 | float Tensor of shape [batch_size, from_seq_length, to_seq_length]. 535 | """ 536 | from_shape = get_shape_list(from_tensor, expected_rank=[2, 3]) 537 | batch_size = from_shape[0] 538 | from_seq_length = from_shape[1] 539 | 540 | to_shape = get_shape_list(to_mask, expected_rank=2) 541 | to_seq_length = to_shape[1] 542 | 543 | to_mask = tf.cast( 544 | tf.reshape(to_mask, [batch_size, 1, to_seq_length]), tf.float32) 545 | 546 | # We don't assume that `from_tensor` is a mask (although it could be). We 547 | # don't actually care if we attend *from* padding tokens (only *to* padding) 548 | # tokens so we create a tensor of all ones. 549 | # 550 | # `broadcast_ones` = [batch_size, from_seq_length, 1] 551 | broadcast_ones = tf.ones( 552 | shape=[batch_size, from_seq_length, 1], dtype=tf.float32) 553 | 554 | # Here we broadcast along two dimensions to create the mask. 555 | mask = broadcast_ones * to_mask 556 | 557 | return mask 558 | 559 | 560 | def attention_layer(from_tensor, 561 | to_tensor, 562 | attention_mask=None, 563 | num_attention_heads=1, 564 | size_per_head=512, 565 | query_act=None, 566 | key_act=None, 567 | value_act=None, 568 | attention_probs_dropout_prob=0.0, 569 | initializer_range=0.02, 570 | do_return_2d_tensor=False, 571 | batch_size=None, 572 | from_seq_length=None, 573 | to_seq_length=None): 574 | """Performs multi-headed attention from `from_tensor` to `to_tensor`. 575 | 576 | This is an implementation of multi-headed attention based on "Attention 577 | is all you Need". If `from_tensor` and `to_tensor` are the same, then 578 | this is self-attention. Each timestep in `from_tensor` attends to the 579 | corresponding sequence in `to_tensor`, and returns a fixed-with vector. 580 | 581 | This function first projects `from_tensor` into a "query" tensor and 582 | `to_tensor` into "key" and "value" tensors. These are (effectively) a list 583 | of tensors of length `num_attention_heads`, where each tensor is of shape 584 | [batch_size, seq_length, size_per_head]. 585 | 586 | Then, the query and key tensors are dot-producted and scaled. These are 587 | softmaxed to obtain attention probabilities. The value tensors are then 588 | interpolated by these probabilities, then concatenated back to a single 589 | tensor and returned. 590 | 591 | In practice, the multi-headed attention are done with transposes and 592 | reshapes rather than actual separate tensors. 593 | 594 | Args: 595 | from_tensor: float Tensor of shape [batch_size, from_seq_length, 596 | from_width]. 597 | to_tensor: float Tensor of shape [batch_size, to_seq_length, to_width]. 598 | attention_mask: (optional) int32 Tensor of shape [batch_size, 599 | from_seq_length, to_seq_length]. The values should be 1 or 0. The 600 | attention scores will effectively be set to -infinity for any positions in 601 | the mask that are 0, and will be unchanged for positions that are 1. 602 | num_attention_heads: int. Number of attention heads. 603 | size_per_head: int. Size of each attention head. 604 | query_act: (optional) Activation function for the query transform. 605 | key_act: (optional) Activation function for the key transform. 606 | value_act: (optional) Activation function for the value transform. 607 | attention_probs_dropout_prob: (optional) float. Dropout probability of the 608 | attention probabilities. 609 | initializer_range: float. Range of the weight initializer. 610 | do_return_2d_tensor: bool. If True, the output will be of shape [batch_size 611 | * from_seq_length, num_attention_heads * size_per_head]. If False, the 612 | output will be of shape [batch_size, from_seq_length, num_attention_heads 613 | * size_per_head]. 614 | batch_size: (Optional) int. If the input is 2D, this might be the batch size 615 | of the 3D version of the `from_tensor` and `to_tensor`. 616 | from_seq_length: (Optional) If the input is 2D, this might be the seq length 617 | of the 3D version of the `from_tensor`. 618 | to_seq_length: (Optional) If the input is 2D, this might be the seq length 619 | of the 3D version of the `to_tensor`. 620 | 621 | Returns: 622 | float Tensor of shape [batch_size, from_seq_length, 623 | num_attention_heads * size_per_head]. (If `do_return_2d_tensor` is 624 | true, this will be of shape [batch_size * from_seq_length, 625 | num_attention_heads * size_per_head]). 626 | 627 | Raises: 628 | ValueError: Any of the arguments or tensor shapes are invalid. 629 | """ 630 | 631 | def transpose_for_scores(input_tensor, batch_size, num_attention_heads, 632 | seq_length, width): 633 | output_tensor = tf.reshape( 634 | input_tensor, [batch_size, seq_length, num_attention_heads, width]) 635 | 636 | output_tensor = tf.transpose(output_tensor, [0, 2, 1, 3]) 637 | return output_tensor 638 | 639 | from_shape = get_shape_list(from_tensor, expected_rank=[2, 3]) 640 | to_shape = get_shape_list(to_tensor, expected_rank=[2, 3]) 641 | 642 | if len(from_shape) != len(to_shape): 643 | raise ValueError( 644 | "The rank of `from_tensor` must match the rank of `to_tensor`.") 645 | 646 | if len(from_shape) == 3: 647 | batch_size = from_shape[0] 648 | from_seq_length = from_shape[1] 649 | to_seq_length = to_shape[1] 650 | elif len(from_shape) == 2: 651 | if (batch_size is None or from_seq_length is None or to_seq_length is None): 652 | raise ValueError( 653 | "When passing in rank 2 tensors to attention_layer, the values " 654 | "for `batch_size`, `from_seq_length`, and `to_seq_length` " 655 | "must all be specified.") 656 | 657 | # Scalar dimensions referenced here: 658 | # B = batch size (number of sequences) 659 | # F = `from_tensor` sequence length 660 | # T = `to_tensor` sequence length 661 | # N = `num_attention_heads` 662 | # H = `size_per_head` 663 | 664 | from_tensor_2d = reshape_to_matrix(from_tensor) 665 | to_tensor_2d = reshape_to_matrix(to_tensor) 666 | 667 | # `query_layer` = [B*F, N*H] 668 | query_layer = tf.layers.dense( 669 | from_tensor_2d, 670 | num_attention_heads * size_per_head, 671 | activation=query_act, 672 | name="query", 673 | kernel_initializer=create_initializer(initializer_range)) 674 | 675 | # `key_layer` = [B*T, N*H] 676 | key_layer = tf.layers.dense( 677 | to_tensor_2d, 678 | num_attention_heads * size_per_head, 679 | activation=key_act, 680 | name="key", 681 | kernel_initializer=create_initializer(initializer_range)) 682 | 683 | # `value_layer` = [B*T, N*H] 684 | value_layer = tf.layers.dense( 685 | to_tensor_2d, 686 | num_attention_heads * size_per_head, 687 | activation=value_act, 688 | name="value", 689 | kernel_initializer=create_initializer(initializer_range)) 690 | 691 | # `query_layer` = [B, N, F, H] 692 | query_layer = transpose_for_scores(query_layer, batch_size, 693 | num_attention_heads, from_seq_length, 694 | size_per_head) 695 | 696 | # `key_layer` = [B, N, T, H] 697 | key_layer = transpose_for_scores(key_layer, batch_size, num_attention_heads, 698 | to_seq_length, size_per_head) 699 | 700 | # Take the dot product between "query" and "key" to get the raw 701 | # attention scores. 702 | # `attention_scores` = [B, N, F, T] 703 | attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) 704 | attention_scores = tf.multiply(attention_scores, 705 | 1.0 / math.sqrt(float(size_per_head))) 706 | 707 | if attention_mask is not None: 708 | # `attention_mask` = [B, 1, F, T] 709 | attention_mask = tf.expand_dims(attention_mask, axis=[1]) 710 | 711 | # Since attention_mask is 1.0 for positions we want to attend and 0.0 for 712 | # masked positions, this operation will create a tensor which is 0.0 for 713 | # positions we want to attend and -10000.0 for masked positions. 714 | adder = (1.0 - tf.cast(attention_mask, tf.float32)) * -10000.0 715 | 716 | # Since we are adding it to the raw scores before the softmax, this is 717 | # effectively the same as removing these entirely. 718 | attention_scores += adder 719 | 720 | # Normalize the attention scores to probabilities. 721 | # `attention_probs` = [B, N, F, T] 722 | attention_probs = tf.nn.softmax(attention_scores) 723 | 724 | # This is actually dropping out entire tokens to attend to, which might 725 | # seem a bit unusual, but is taken from the original Transformer paper. 726 | attention_probs = dropout(attention_probs, attention_probs_dropout_prob) 727 | 728 | # `value_layer` = [B, T, N, H] 729 | value_layer = tf.reshape( 730 | value_layer, 731 | [batch_size, to_seq_length, num_attention_heads, size_per_head]) 732 | 733 | # `value_layer` = [B, N, T, H] 734 | value_layer = tf.transpose(value_layer, [0, 2, 1, 3]) 735 | 736 | # `context_layer` = [B, N, F, H] 737 | context_layer = tf.matmul(attention_probs, value_layer) 738 | 739 | # `context_layer` = [B, F, N, H] 740 | context_layer = tf.transpose(context_layer, [0, 2, 1, 3]) 741 | 742 | if do_return_2d_tensor: 743 | # `context_layer` = [B*F, N*V] 744 | context_layer = tf.reshape( 745 | context_layer, 746 | [batch_size * from_seq_length, num_attention_heads * size_per_head]) 747 | else: 748 | # `context_layer` = [B, F, N*V] 749 | context_layer = tf.reshape( 750 | context_layer, 751 | [batch_size, from_seq_length, num_attention_heads * size_per_head]) 752 | 753 | return context_layer 754 | 755 | 756 | def transformer_model(input_tensor, 757 | attention_mask=None, 758 | hidden_size=768, 759 | num_hidden_layers=12, 760 | num_attention_heads=12, 761 | intermediate_size=3072, 762 | intermediate_act_fn=gelu, 763 | hidden_dropout_prob=0.1, 764 | attention_probs_dropout_prob=0.1, 765 | initializer_range=0.02, 766 | do_return_all_layers=False): 767 | """Multi-headed, multi-layer Transformer from "Attention is All You Need". 768 | 769 | This is almost an exact implementation of the original Transformer encoder. 770 | 771 | See the original paper: 772 | https://arxiv.org/abs/1706.03762 773 | 774 | Also see: 775 | https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/models/transformer.py 776 | 777 | Args: 778 | input_tensor: float Tensor of shape [batch_size, seq_length, hidden_size]. 779 | attention_mask: (optional) int32 Tensor of shape [batch_size, seq_length, 780 | seq_length], with 1 for positions that can be attended to and 0 in 781 | positions that should not be. 782 | hidden_size: int. Hidden size of the Transformer. 783 | num_hidden_layers: int. Number of layers (blocks) in the Transformer. 784 | num_attention_heads: int. Number of attention heads in the Transformer. 785 | intermediate_size: int. The size of the "intermediate" (a.k.a., feed 786 | forward) layer. 787 | intermediate_act_fn: function. The non-linear activation function to apply 788 | to the output of the intermediate/feed-forward layer. 789 | hidden_dropout_prob: float. Dropout probability for the hidden layers. 790 | attention_probs_dropout_prob: float. Dropout probability of the attention 791 | probabilities. 792 | initializer_range: float. Range of the initializer (stddev of truncated 793 | normal). 794 | do_return_all_layers: Whether to also return all layers or just the final 795 | layer. 796 | 797 | Returns: 798 | float Tensor of shape [batch_size, seq_length, hidden_size], the final 799 | hidden layer of the Transformer. 800 | 801 | Raises: 802 | ValueError: A Tensor shape or parameter is invalid. 803 | """ 804 | if hidden_size % num_attention_heads != 0: 805 | raise ValueError( 806 | "The hidden size (%d) is not a multiple of the number of attention " 807 | "heads (%d)" % (hidden_size, num_attention_heads)) 808 | 809 | attention_head_size = int(hidden_size / num_attention_heads) 810 | input_shape = get_shape_list(input_tensor, expected_rank=3) 811 | batch_size = input_shape[0] 812 | seq_length = input_shape[1] 813 | input_width = input_shape[2] 814 | 815 | # The Transformer performs sum residuals on all layers so the input needs 816 | # to be the same as the hidden size. 817 | if input_width != hidden_size: 818 | raise ValueError("The width of the input tensor (%d) != hidden size (%d)" % 819 | (input_width, hidden_size)) 820 | 821 | # We keep the representation as a 2D tensor to avoid re-shaping it back and 822 | # forth from a 3D tensor to a 2D tensor. Re-shapes are normally free on 823 | # the GPU/CPU but may not be free on the TPU, so we want to minimize them to 824 | # help the optimizer. 825 | prev_output = reshape_to_matrix(input_tensor) 826 | 827 | all_layer_outputs = [] 828 | for layer_idx in range(num_hidden_layers): 829 | with tf.variable_scope("layer_%d" % layer_idx): 830 | layer_input = prev_output 831 | 832 | with tf.variable_scope("attention"): 833 | attention_heads = [] 834 | with tf.variable_scope("self"): 835 | attention_head = attention_layer( 836 | from_tensor=layer_input, 837 | to_tensor=layer_input, 838 | attention_mask=attention_mask, 839 | num_attention_heads=num_attention_heads, 840 | size_per_head=attention_head_size, 841 | attention_probs_dropout_prob=attention_probs_dropout_prob, 842 | initializer_range=initializer_range, 843 | do_return_2d_tensor=True, 844 | batch_size=batch_size, 845 | from_seq_length=seq_length, 846 | to_seq_length=seq_length) 847 | attention_heads.append(attention_head) 848 | 849 | attention_output = None 850 | if len(attention_heads) == 1: 851 | attention_output = attention_heads[0] 852 | else: 853 | # In the case where we have other sequences, we just concatenate 854 | # them to the self-attention head before the projection. 855 | attention_output = tf.concat(attention_heads, axis=-1) 856 | 857 | # Run a linear projection of `hidden_size` then add a residual 858 | # with `layer_input`. 859 | with tf.variable_scope("output"): 860 | attention_output = tf.layers.dense( 861 | attention_output, 862 | hidden_size, 863 | kernel_initializer=create_initializer(initializer_range)) 864 | attention_output = dropout(attention_output, hidden_dropout_prob) 865 | attention_output = layer_norm(attention_output + layer_input) 866 | 867 | # The activation is only applied to the "intermediate" hidden layer. 868 | with tf.variable_scope("intermediate"): 869 | intermediate_output = tf.layers.dense( 870 | attention_output, 871 | intermediate_size, 872 | activation=intermediate_act_fn, 873 | kernel_initializer=create_initializer(initializer_range)) 874 | 875 | # Down-project back to `hidden_size` then add the residual. 876 | with tf.variable_scope("output"): 877 | layer_output = tf.layers.dense( 878 | intermediate_output, 879 | hidden_size, 880 | kernel_initializer=create_initializer(initializer_range)) 881 | layer_output = dropout(layer_output, hidden_dropout_prob) 882 | layer_output = layer_norm(layer_output + attention_output) 883 | prev_output = layer_output 884 | all_layer_outputs.append(layer_output) 885 | 886 | if do_return_all_layers: 887 | final_outputs = [] 888 | for layer_output in all_layer_outputs: 889 | final_output = reshape_from_matrix(layer_output, input_shape) 890 | final_outputs.append(final_output) 891 | return final_outputs 892 | else: 893 | final_output = reshape_from_matrix(prev_output, input_shape) 894 | return final_output 895 | 896 | 897 | def get_shape_list(tensor, expected_rank=None, name=None): 898 | """Returns a list of the shape of tensor, preferring static dimensions. 899 | 900 | Args: 901 | tensor: A tf.Tensor object to find the shape of. 902 | expected_rank: (optional) int. The expected rank of `tensor`. If this is 903 | specified and the `tensor` has a different rank, and exception will be 904 | thrown. 905 | name: Optional name of the tensor for the error message. 906 | 907 | Returns: 908 | A list of dimensions of the shape of tensor. All static dimensions will 909 | be returned as python integers, and dynamic dimensions will be returned 910 | as tf.Tensor scalars. 911 | """ 912 | if name is None: 913 | name = tensor.name 914 | 915 | if expected_rank is not None: 916 | assert_rank(tensor, expected_rank, name) 917 | 918 | shape = tensor.shape.as_list() 919 | 920 | non_static_indexes = [] 921 | for (index, dim) in enumerate(shape): 922 | if dim is None: 923 | non_static_indexes.append(index) 924 | 925 | if not non_static_indexes: 926 | return shape 927 | 928 | dyn_shape = tf.shape(tensor) 929 | for index in non_static_indexes: 930 | shape[index] = dyn_shape[index] 931 | return shape 932 | 933 | 934 | def reshape_to_matrix(input_tensor): 935 | """Reshapes a >= rank 2 tensor to a rank 2 tensor (i.e., a matrix).""" 936 | ndims = input_tensor.shape.ndims 937 | if ndims < 2: 938 | raise ValueError("Input tensor must have at least rank 2. Shape = %s" % 939 | (input_tensor.shape)) 940 | if ndims == 2: 941 | return input_tensor 942 | 943 | width = input_tensor.shape[-1] 944 | output_tensor = tf.reshape(input_tensor, [-1, width]) 945 | return output_tensor 946 | 947 | 948 | def reshape_from_matrix(output_tensor, orig_shape_list): 949 | """Reshapes a rank 2 tensor back to its original rank >= 2 tensor.""" 950 | if len(orig_shape_list) == 2: 951 | return output_tensor 952 | 953 | output_shape = get_shape_list(output_tensor) 954 | 955 | orig_dims = orig_shape_list[0:-1] 956 | width = output_shape[-1] 957 | 958 | return tf.reshape(output_tensor, orig_dims + [width]) 959 | 960 | 961 | def assert_rank(tensor, expected_rank, name=None): 962 | """Raises an exception if the tensor rank is not of the expected rank. 963 | 964 | Args: 965 | tensor: A tf.Tensor to check the rank of. 966 | expected_rank: Python integer or list of integers, expected rank. 967 | name: Optional name of the tensor for the error message. 968 | 969 | Raises: 970 | ValueError: If the expected shape doesn't match the actual shape. 971 | """ 972 | if name is None: 973 | name = tensor.name 974 | 975 | expected_rank_dict = {} 976 | if isinstance(expected_rank, six.integer_types): 977 | expected_rank_dict[expected_rank] = True 978 | else: 979 | for x in expected_rank: 980 | expected_rank_dict[x] = True 981 | 982 | actual_rank = tensor.shape.ndims 983 | if actual_rank not in expected_rank_dict: 984 | scope_name = tf.get_variable_scope().name 985 | raise ValueError( 986 | "For the tensor `%s` in scope `%s`, the actual rank " 987 | "`%d` (shape = %s) is not equal to the expected rank `%s`" % 988 | (name, scope_name, actual_rank, str(tensor.shape), str(expected_rank))) 989 | -------------------------------------------------------------------------------- /logs/train.log: -------------------------------------------------------------------------------- 1 | nohup: ignoring input 2 | /usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`. 3 | from ._conv import register_converters as _register_converters 4 | INFO:tensorflow:Using config: {'_num_ps_replicas': 0, '_is_chief': True, '_train_distribute': None, '_model_dir': '/home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/', '_session_config': gpu_options { 5 | per_process_gpu_memory_fraction: 0.8 6 | allow_growth: true 7 | } 8 | , '_eval_distribute': None, '_keep_checkpoint_max': 5, '_service': None, '_num_worker_replicas': 1, '_device_fn': None, '_keep_checkpoint_every_n_hours': 10000, '_global_id_in_cluster': 0, '_save_checkpoints_secs': 600, '_log_step_count_steps': 100, '_protocol': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_evaluation_master': '', '_task_type': 'worker', '_cluster_spec': , '_experimental_distribute': None, '_tf_random_seed': None, '_master': '', '_task_id': 0} 9 | INFO:tensorflow:Using config: {'_num_ps_replicas': 0, '_is_chief': True, '_train_distribute': None, '_model_dir': '/home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/', '_session_config': gpu_options { 10 | per_process_gpu_memory_fraction: 0.8 11 | allow_growth: true 12 | } 13 | , '_eval_distribute': None, '_keep_checkpoint_max': 5, '_service': None, '_num_worker_replicas': 1, '_device_fn': None, '_keep_checkpoint_every_n_hours': 10000, '_global_id_in_cluster': 0, '_save_checkpoints_secs': 600, '_log_step_count_steps': 100, '_protocol': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_evaluation_master': '', '_task_type': 'worker', '_cluster_spec': , '_experimental_distribute': None, '_tf_random_seed': None, '_master': '', '_task_id': 0} 14 | INFO:tensorflow:Writing example 0 of 70000 15 | INFO:tensorflow:*** Example *** 16 | INFO:tensorflow:guid: train-0 17 | INFO:tensorflow:tokens: [CLS] 怎 么 更 改 花 呗 手 机 号 码 [SEP] 我 的 花 呗 是 以 前 的 手 机 号 码 ， 怎 么 更 改 成 现 [SEP] 18 | INFO:tensorflow:input_ids: 101 2582 720 3291 3121 5709 1446 2797 3322 1384 4772 102 2769 4638 5709 1446 3221 809 1184 4638 2797 3322 1384 4772 8024 2582 720 3291 3121 2768 4385 102 19 | INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 20 | INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 21 | INFO:tensorflow:label: 1 (id = 1) 22 | INFO:tensorflow:*** Example *** 23 | INFO:tensorflow:guid: train-1 24 | INFO:tensorflow:tokens: [CLS] 也 开 不 了 花 呗 ， 就 这 样 了 ？ 完 事 了 [SEP] 真 的 嘛 ？ 就 是 花 呗 付 款 [SEP] 25 | INFO:tensorflow:input_ids: 101 738 2458 679 749 5709 1446 8024 2218 6821 3416 749 8043 2130 752 749 102 4696 4638 1658 8043 2218 3221 5709 1446 802 3621 102 0 0 0 0 26 | INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 27 | INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 28 | INFO:tensorflow:label: 0 (id = 0) 29 | INFO:tensorflow:*** Example *** 30 | INFO:tensorflow:guid: train-2 31 | INFO:tensorflow:tokens: [CLS] 花 呗 冻 结 以 后 还 能 开 通 吗 [SEP] 我 的 条 件 可 以 开 通 花 呗 借 款 吗 [SEP] 32 | INFO:tensorflow:input_ids: 101 5709 1446 1108 5310 809 1400 6820 5543 2458 6858 1408 102 2769 4638 3340 816 1377 809 2458 6858 5709 1446 955 3621 1408 102 0 0 0 0 0 33 | INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 34 | INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 35 | INFO:tensorflow:label: 0 (id = 0) 36 | INFO:tensorflow:*** Example *** 37 | INFO:tensorflow:guid: train-3 38 | INFO:tensorflow:tokens: [CLS] 如 何 得 知 关 闭 借 呗 [SEP] 想 永 久 关 闭 借 呗 [SEP] 39 | INFO:tensorflow:input_ids: 101 1963 862 2533 4761 1068 7308 955 1446 102 2682 3719 719 1068 7308 955 1446 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 40 | INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 41 | INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 42 | INFO:tensorflow:label: 0 (id = 0) 43 | INFO:tensorflow:*** Example *** 44 | INFO:tensorflow:guid: train-4 45 | INFO:tensorflow:tokens: [CLS] 花 呗 扫 码 付 钱 [SEP] 二 维 码 扫 描 可 以 用 花 呗 吗 [SEP] 46 | INFO:tensorflow:input_ids: 101 5709 1446 2812 4772 802 7178 102 753 5335 4772 2812 2989 1377 809 4500 5709 1446 1408 102 0 0 0 0 0 0 0 0 0 0 0 0 47 | INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 48 | INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 49 | INFO:tensorflow:label: 0 (id = 0) 50 | INFO:tensorflow:Writing example 10000 of 70000 51 | INFO:tensorflow:Writing example 20000 of 70000 52 | INFO:tensorflow:Writing example 30000 of 70000 53 | INFO:tensorflow:Writing example 40000 of 70000 54 | INFO:tensorflow:Writing example 50000 of 70000 55 | INFO:tensorflow:Writing example 60000 of 70000 56 | INFO:tensorflow:***** Running training ***** 57 | INFO:tensorflow: Num examples = 70000 58 | INFO:tensorflow: Batch size = 128 59 | INFO:tensorflow: Num steps = 5468 60 | WARNING:tensorflow:From similarity.py:581: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version. 61 | Instructions for updating: 62 | Use `tf.data.experimental.map_and_batch(...)`. 63 | INFO:tensorflow:Calling model_fn. 64 | INFO:tensorflow:*** Features *** 65 | INFO:tensorflow: name = input_ids, shape = (128, 32) 66 | INFO:tensorflow: name = input_mask, shape = (128, 32) 67 | INFO:tensorflow: name = label_ids, shape = (128,) 68 | INFO:tensorflow: name = segment_ids, shape = (128, 32) 69 | INFO:tensorflow:**** Trainable Variables **** 70 | INFO:tensorflow: name = bert/embeddings/word_embeddings:0, shape = (21128, 768), *INIT_FROM_CKPT* 71 | INFO:tensorflow: name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT* 72 | INFO:tensorflow: name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT* 73 | INFO:tensorflow: name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 74 | INFO:tensorflow: name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 75 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 76 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 77 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 78 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 79 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 80 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 81 | INFO:tensorflow: name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 82 | INFO:tensorflow: name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 83 | INFO:tensorflow: name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 84 | INFO:tensorflow: name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 85 | INFO:tensorflow: name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 86 | INFO:tensorflow: name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 87 | INFO:tensorflow: name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 88 | INFO:tensorflow: name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 89 | INFO:tensorflow: name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 90 | INFO:tensorflow: name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 91 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 92 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 93 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 94 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 95 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 96 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 97 | INFO:tensorflow: name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 98 | INFO:tensorflow: name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 99 | INFO:tensorflow: name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 100 | INFO:tensorflow: name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 101 | INFO:tensorflow: name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 102 | INFO:tensorflow: name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 103 | INFO:tensorflow: name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 104 | INFO:tensorflow: name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 105 | INFO:tensorflow: name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 106 | INFO:tensorflow: name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 107 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 108 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 109 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 110 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 111 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 112 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 113 | INFO:tensorflow: name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 114 | INFO:tensorflow: name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 115 | INFO:tensorflow: name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 116 | INFO:tensorflow: name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 117 | INFO:tensorflow: name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 118 | INFO:tensorflow: name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 119 | INFO:tensorflow: name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 120 | INFO:tensorflow: name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 121 | INFO:tensorflow: name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 122 | INFO:tensorflow: name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 123 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 124 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 125 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 126 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 127 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 128 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 129 | INFO:tensorflow: name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 130 | INFO:tensorflow: name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 131 | INFO:tensorflow: name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 132 | INFO:tensorflow: name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 133 | INFO:tensorflow: name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 134 | INFO:tensorflow: name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 135 | INFO:tensorflow: name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 136 | INFO:tensorflow: name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 137 | INFO:tensorflow: name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 138 | INFO:tensorflow: name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 139 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 140 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 141 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 142 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 143 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 144 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 145 | INFO:tensorflow: name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 146 | INFO:tensorflow: name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 147 | INFO:tensorflow: name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 148 | INFO:tensorflow: name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 149 | INFO:tensorflow: name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 150 | INFO:tensorflow: name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 151 | INFO:tensorflow: name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 152 | INFO:tensorflow: name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 153 | INFO:tensorflow: name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 154 | INFO:tensorflow: name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 155 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 156 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 157 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 158 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 159 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 160 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 161 | INFO:tensorflow: name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 162 | INFO:tensorflow: name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 163 | INFO:tensorflow: name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 164 | INFO:tensorflow: name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 165 | INFO:tensorflow: name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 166 | INFO:tensorflow: name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 167 | INFO:tensorflow: name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 168 | INFO:tensorflow: name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 169 | INFO:tensorflow: name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 170 | INFO:tensorflow: name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 171 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 172 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 173 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 174 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 175 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 176 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 177 | INFO:tensorflow: name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 178 | INFO:tensorflow: name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 179 | INFO:tensorflow: name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 180 | INFO:tensorflow: name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 181 | INFO:tensorflow: name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 182 | INFO:tensorflow: name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 183 | INFO:tensorflow: name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 184 | INFO:tensorflow: name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 185 | INFO:tensorflow: name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 186 | INFO:tensorflow: name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 187 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 188 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 189 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 190 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 191 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 192 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 193 | INFO:tensorflow: name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 194 | INFO:tensorflow: name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 195 | INFO:tensorflow: name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 196 | INFO:tensorflow: name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 197 | INFO:tensorflow: name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 198 | INFO:tensorflow: name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 199 | INFO:tensorflow: name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 200 | INFO:tensorflow: name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 201 | INFO:tensorflow: name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 202 | INFO:tensorflow: name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 203 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 204 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 205 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 206 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 207 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 208 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 209 | INFO:tensorflow: name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 210 | INFO:tensorflow: name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 211 | INFO:tensorflow: name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 212 | INFO:tensorflow: name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 213 | INFO:tensorflow: name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 214 | INFO:tensorflow: name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 215 | INFO:tensorflow: name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 216 | INFO:tensorflow: name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 217 | INFO:tensorflow: name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 218 | INFO:tensorflow: name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 219 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 220 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 221 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 222 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 223 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 224 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 225 | INFO:tensorflow: name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 226 | INFO:tensorflow: name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 227 | INFO:tensorflow: name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 228 | INFO:tensorflow: name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 229 | INFO:tensorflow: name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 230 | INFO:tensorflow: name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 231 | INFO:tensorflow: name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 232 | INFO:tensorflow: name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 233 | INFO:tensorflow: name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 234 | INFO:tensorflow: name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 235 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 236 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 237 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 238 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 239 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 240 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 241 | INFO:tensorflow: name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 242 | INFO:tensorflow: name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 243 | INFO:tensorflow: name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 244 | INFO:tensorflow: name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 245 | INFO:tensorflow: name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 246 | INFO:tensorflow: name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 247 | INFO:tensorflow: name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 248 | INFO:tensorflow: name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 249 | INFO:tensorflow: name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 250 | INFO:tensorflow: name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 251 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 252 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 253 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 254 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 255 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 256 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 257 | INFO:tensorflow: name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 258 | INFO:tensorflow: name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 259 | INFO:tensorflow: name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 260 | INFO:tensorflow: name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 261 | INFO:tensorflow: name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 262 | INFO:tensorflow: name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 263 | INFO:tensorflow: name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 264 | INFO:tensorflow: name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 265 | INFO:tensorflow: name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 266 | INFO:tensorflow: name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 267 | INFO:tensorflow: name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 268 | INFO:tensorflow: name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 269 | INFO:tensorflow: name = output_weights:0, shape = (2, 768) 270 | INFO:tensorflow: name = output_bias:0, shape = (2,) 271 | INFO:tensorflow:Done calling model_fn. 272 | INFO:tensorflow:Create CheckpointSaverHook. 273 | INFO:tensorflow:Graph was finalized. 274 | 2019-02-18 05:54:48.046169: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA 275 | 2019-02-18 05:54:48.144305: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:964] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 276 | 2019-02-18 05:54:48.144766: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 0 with properties: 277 | name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235 278 | pciBusID: 0000:00:04.0 279 | totalMemory: 11.17GiB freeMemory: 11.09GiB 280 | 2019-02-18 05:54:48.144806: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1511] Adding visible gpu devices: 0 281 | 2019-02-18 05:54:48.424528: I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] Device interconnect StreamExecutor with strength 1 edge matrix: 282 | 2019-02-18 05:54:48.424620: I tensorflow/core/common_runtime/gpu/gpu_device.cc:988] 0 283 | 2019-02-18 05:54:48.424633: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 0: N 284 | 2019-02-18 05:54:48.424956: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 9151 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7) 285 | INFO:tensorflow:Running local_init_op. 286 | INFO:tensorflow:Done running local_init_op. 287 | INFO:tensorflow:Saving checkpoints for 0 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 288 | INFO:tensorflow:loss = 1.0218862, step = 0 289 | INFO:tensorflow:global_step/sec: 0.615146 290 | INFO:tensorflow:loss = 0.5186956, step = 100 (162.563 sec) 291 | INFO:tensorflow:global_step/sec: 0.693925 292 | INFO:tensorflow:loss = 0.35847408, step = 200 (144.108 sec) 293 | INFO:tensorflow:global_step/sec: 0.692983 294 | INFO:tensorflow:loss = 0.39972323, step = 300 (144.304 sec) 295 | INFO:tensorflow:Saving checkpoints for 399 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 296 | INFO:tensorflow:global_step/sec: 0.681772 297 | INFO:tensorflow:loss = 0.3482466, step = 400 (146.677 sec) 298 | INFO:tensorflow:global_step/sec: 0.693429 299 | INFO:tensorflow:loss = 0.3434133, step = 500 (144.211 sec) 300 | INFO:tensorflow:global_step/sec: 0.693451 301 | INFO:tensorflow:loss = 0.38796747, step = 600 (144.206 sec) 302 | INFO:tensorflow:global_step/sec: 0.692414 303 | INFO:tensorflow:loss = 0.30979925, step = 700 (144.422 sec) 304 | INFO:tensorflow:global_step/sec: 0.692319 305 | INFO:tensorflow:loss = 0.34772998, step = 800 (144.442 sec) 306 | INFO:tensorflow:Saving checkpoints for 814 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 307 | INFO:tensorflow:global_step/sec: 0.680927 308 | INFO:tensorflow:loss = 0.3403089, step = 900 (146.859 sec) 309 | INFO:tensorflow:global_step/sec: 0.6925 310 | INFO:tensorflow:loss = 0.22857589, step = 1000 (144.404 sec) 311 | INFO:tensorflow:global_step/sec: 0.692403 312 | INFO:tensorflow:loss = 0.27272177, step = 1100 (144.425 sec) 313 | INFO:tensorflow:global_step/sec: 0.692064 314 | INFO:tensorflow:loss = 0.23621395, step = 1200 (144.495 sec) 315 | INFO:tensorflow:Saving checkpoints for 1228 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 316 | INFO:tensorflow:global_step/sec: 0.677783 317 | INFO:tensorflow:loss = 0.29772466, step = 1300 (147.540 sec) 318 | INFO:tensorflow:global_step/sec: 0.692689 319 | INFO:tensorflow:loss = 0.23366493, step = 1400 (144.365 sec) 320 | INFO:tensorflow:global_step/sec: 0.692638 321 | INFO:tensorflow:loss = 0.27182245, step = 1500 (144.376 sec) 322 | INFO:tensorflow:global_step/sec: 0.692322 323 | INFO:tensorflow:loss = 0.23535305, step = 1600 (144.441 sec) 324 | INFO:tensorflow:Saving checkpoints for 1642 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 325 | INFO:tensorflow:global_step/sec: 0.681351 326 | INFO:tensorflow:loss = 0.3185692, step = 1700 (146.767 sec) 327 | INFO:tensorflow:global_step/sec: 0.692397 328 | INFO:tensorflow:loss = 0.23333019, step = 1800 (144.426 sec) 329 | INFO:tensorflow:global_step/sec: 0.692683 330 | INFO:tensorflow:loss = 0.30707008, step = 1900 (144.366 sec) 331 | INFO:tensorflow:global_step/sec: 0.692181 332 | INFO:tensorflow:loss = 0.18656304, step = 2000 (144.471 sec) 333 | INFO:tensorflow:Saving checkpoints for 2056 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 334 | INFO:tensorflow:global_step/sec: 0.680676 335 | INFO:tensorflow:loss = 0.18715279, step = 2100 (146.913 sec) 336 | INFO:tensorflow:global_step/sec: 0.693218 337 | INFO:tensorflow:loss = 0.1443583, step = 2200 (144.255 sec) 338 | INFO:tensorflow:global_step/sec: 0.693213 339 | INFO:tensorflow:loss = 0.12635505, step = 2300 (144.256 sec) 340 | INFO:tensorflow:global_step/sec: 0.693339 341 | INFO:tensorflow:loss = 0.15235075, step = 2400 (144.230 sec) 342 | INFO:tensorflow:Saving checkpoints for 2471 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 343 | INFO:tensorflow:global_step/sec: 0.681444 344 | INFO:tensorflow:loss = 0.16846885, step = 2500 (146.747 sec) 345 | INFO:tensorflow:global_step/sec: 0.693407 346 | INFO:tensorflow:loss = 0.08956495, step = 2600 (144.216 sec) 347 | INFO:tensorflow:global_step/sec: 0.692113 348 | INFO:tensorflow:loss = 0.08271434, step = 2700 (144.485 sec) 349 | INFO:tensorflow:global_step/sec: 0.692404 350 | INFO:tensorflow:loss = 0.111416705, step = 2800 (144.424 sec) 351 | INFO:tensorflow:Saving checkpoints for 2885 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 352 | INFO:tensorflow:global_step/sec: 0.680204 353 | INFO:tensorflow:loss = 0.15365322, step = 2900 (147.015 sec) 354 | INFO:tensorflow:global_step/sec: 0.692206 355 | INFO:tensorflow:loss = 0.13628608, step = 3000 (144.466 sec) 356 | INFO:tensorflow:global_step/sec: 0.690335 357 | INFO:tensorflow:loss = 0.11042709, step = 3100 (144.857 sec) 358 | INFO:tensorflow:global_step/sec: 0.692432 359 | INFO:tensorflow:loss = 0.06393729, step = 3200 (144.418 sec) 360 | INFO:tensorflow:Saving checkpoints for 3299 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 361 | INFO:tensorflow:global_step/sec: 0.680716 362 | INFO:tensorflow:loss = 0.053375434, step = 3300 (146.904 sec) 363 | INFO:tensorflow:global_step/sec: 0.692502 364 | INFO:tensorflow:loss = 0.114489384, step = 3400 (144.404 sec) 365 | INFO:tensorflow:global_step/sec: 0.692378 366 | INFO:tensorflow:loss = 0.086134925, step = 3500 (144.430 sec) 367 | INFO:tensorflow:global_step/sec: 0.692302 368 | INFO:tensorflow:loss = 0.066648535, step = 3600 (144.446 sec) 369 | INFO:tensorflow:global_step/sec: 0.692379 370 | INFO:tensorflow:loss = 0.10406309, step = 3700 (144.430 sec) 371 | INFO:tensorflow:Saving checkpoints for 3713 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 372 | INFO:tensorflow:global_step/sec: 0.680629 373 | INFO:tensorflow:loss = 0.04419697, step = 3800 (146.923 sec) 374 | INFO:tensorflow:global_step/sec: 0.692205 375 | INFO:tensorflow:loss = 0.12511328, step = 3900 (144.466 sec) 376 | INFO:tensorflow:global_step/sec: 0.692382 377 | INFO:tensorflow:loss = 0.11059156, step = 4000 (144.429 sec) 378 | INFO:tensorflow:global_step/sec: 0.6922 379 | INFO:tensorflow:loss = 0.043235414, step = 4100 (144.467 sec) 380 | INFO:tensorflow:Saving checkpoints for 4127 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 381 | INFO:tensorflow:global_step/sec: 0.680117 382 | INFO:tensorflow:loss = 0.052045528, step = 4200 (147.034 sec) 383 | INFO:tensorflow:global_step/sec: 0.692229 384 | INFO:tensorflow:loss = 0.05638282, step = 4300 (144.461 sec) 385 | INFO:tensorflow:global_step/sec: 0.692258 386 | INFO:tensorflow:loss = 0.05289631, step = 4400 (144.455 sec) 387 | INFO:tensorflow:global_step/sec: 0.692385 388 | INFO:tensorflow:loss = 0.049040448, step = 4500 (144.428 sec) 389 | INFO:tensorflow:Saving checkpoints for 4541 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 390 | INFO:tensorflow:global_step/sec: 0.680062 391 | INFO:tensorflow:loss = 0.030390669, step = 4600 (147.045 sec) 392 | INFO:tensorflow:global_step/sec: 0.69229 393 | INFO:tensorflow:loss = 0.012548172, step = 4700 (144.448 sec) 394 | INFO:tensorflow:global_step/sec: 0.692149 395 | INFO:tensorflow:loss = 0.11258875, step = 4800 (144.478 sec) 396 | INFO:tensorflow:global_step/sec: 0.692354 397 | INFO:tensorflow:loss = 0.08994587, step = 4900 (144.435 sec) 398 | INFO:tensorflow:Saving checkpoints for 4955 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 399 | INFO:tensorflow:global_step/sec: 0.68025 400 | INFO:tensorflow:loss = 0.025694255, step = 5000 (147.005 sec) 401 | INFO:tensorflow:global_step/sec: 0.691745 402 | INFO:tensorflow:loss = 0.020793306, step = 5100 (144.562 sec) 403 | INFO:tensorflow:global_step/sec: 0.692547 404 | INFO:tensorflow:loss = 0.06512974, step = 5200 (144.394 sec) 405 | INFO:tensorflow:global_step/sec: 0.691786 406 | INFO:tensorflow:loss = 0.043220975, step = 5300 (144.554 sec) 407 | INFO:tensorflow:Saving checkpoints for 5369 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 408 | INFO:tensorflow:global_step/sec: 0.679874 409 | INFO:tensorflow:loss = 0.01693129, step = 5400 (147.086 sec) 410 | INFO:tensorflow:Saving checkpoints for 5468 into /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt. 411 | INFO:tensorflow:Loss for final step: 0.0025181356. 412 | INFO:tensorflow:Using config: {'_num_ps_replicas': 0, '_is_chief': True, '_train_distribute': None, '_model_dir': '/home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/', '_session_config': gpu_options { 413 | per_process_gpu_memory_fraction: 0.8 414 | allow_growth: true 415 | } 416 | , '_eval_distribute': None, '_keep_checkpoint_max': 5, '_service': None, '_num_worker_replicas': 1, '_device_fn': None, '_keep_checkpoint_every_n_hours': 10000, '_global_id_in_cluster': 0, '_save_checkpoints_secs': 600, '_log_step_count_steps': 100, '_protocol': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_evaluation_master': '', '_task_type': 'worker', '_cluster_spec': , '_experimental_distribute': None, '_tf_random_seed': None, '_master': '', '_task_id': 0} 417 | INFO:tensorflow:Writing example 0 of 32477 418 | INFO:tensorflow:*** Example *** 419 | INFO:tensorflow:guid: test-0 420 | INFO:tensorflow:tokens: [CLS] 我 用 花 呗 交 的 优 拜 押 金 [SEP] 我 * * * 月 * * * 用 花 呗 支 付 的 小 蓝 单 车 押 [SEP] 421 | INFO:tensorflow:input_ids: 101 2769 4500 5709 1446 769 4638 831 2876 2852 7032 102 2769 115 115 115 3299 115 115 115 4500 5709 1446 3118 802 4638 2207 5905 1296 6756 2852 102 422 | INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 423 | INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 424 | INFO:tensorflow:label: 0 (id = 0) 425 | INFO:tensorflow:*** Example *** 426 | INFO:tensorflow:guid: test-1 427 | INFO:tensorflow:tokens: [CLS] 花 呗 有 漏 洞 吗 [SEP] 花 呗 有 * * * 天 宽 限 期 吗 [SEP] 428 | INFO:tensorflow:input_ids: 101 5709 1446 3300 4026 3822 1408 102 5709 1446 3300 115 115 115 1921 2160 7361 3309 1408 102 0 0 0 0 0 0 0 0 0 0 0 0 429 | INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 430 | INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 431 | INFO:tensorflow:label: 0 (id = 0) 432 | INFO:tensorflow:*** Example *** 433 | INFO:tensorflow:guid: test-2 434 | INFO:tensorflow:tokens: [CLS] 花 呗 可 以 在 淘 宝 上 使 用 吗 [SEP] 淘 宝 充 红 包 的 金 额 能 和 花 呗 叠 加 使 用 吗 [SEP] 435 | INFO:tensorflow:input_ids: 101 5709 1446 1377 809 1762 3905 2140 677 886 4500 1408 102 3905 2140 1041 5273 1259 4638 7032 7583 5543 1469 5709 1446 1363 1217 886 4500 1408 102 0 436 | INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 437 | INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 438 | INFO:tensorflow:label: 0 (id = 0) 439 | INFO:tensorflow:*** Example *** 440 | INFO:tensorflow:guid: test-3 441 | INFO:tensorflow:tokens: [CLS] 花 呗 还 款 了 ， 还 显 示 要 还 款 [SEP] 我 的 花 呗 需 要 还 款 [SEP] 442 | INFO:tensorflow:input_ids: 101 5709 1446 6820 3621 749 8024 6820 3227 4850 6206 6820 3621 102 2769 4638 5709 1446 7444 6206 6820 3621 102 0 0 0 0 0 0 0 0 0 443 | INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 444 | INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 445 | INFO:tensorflow:label: 0 (id = 0) 446 | INFO:tensorflow:*** Example *** 447 | INFO:tensorflow:guid: test-4 448 | INFO:tensorflow:tokens: [CLS] 我 的 花 呗 为 什 么 不 让 用 [SEP] 我 的 支 付 宝 花 呗 不 能 用 [SEP] 449 | INFO:tensorflow:input_ids: 101 2769 4638 5709 1446 711 784 720 679 6375 4500 102 2769 4638 3118 802 2140 5709 1446 679 5543 4500 102 0 0 0 0 0 0 0 0 0 450 | INFO:tensorflow:input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 451 | INFO:tensorflow:segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 452 | INFO:tensorflow:label: 0 (id = 0) 453 | INFO:tensorflow:Writing example 10000 of 32477 454 | INFO:tensorflow:Writing example 20000 of 32477 455 | INFO:tensorflow:Writing example 30000 of 32477 456 | INFO:tensorflow:***** Running evaluation ***** 457 | INFO:tensorflow: Num examples = 32477 458 | INFO:tensorflow: Batch size = 128 459 | INFO:tensorflow:Using config: {'_num_ps_replicas': 0, '_is_chief': True, '_train_distribute': None, '_model_dir': '/home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/', '_session_config': gpu_options { 460 | per_process_gpu_memory_fraction: 0.8 461 | allow_growth: true 462 | } 463 | , '_eval_distribute': None, '_keep_checkpoint_max': 5, '_service': None, '_num_worker_replicas': 1, '_device_fn': None, '_keep_checkpoint_every_n_hours': 10000, '_global_id_in_cluster': 0, '_save_checkpoints_secs': 600, '_log_step_count_steps': 100, '_protocol': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_evaluation_master': '', '_task_type': 'worker', '_cluster_spec': , '_experimental_distribute': None, '_tf_random_seed': None, '_master': '', '_task_id': 0} 464 | INFO:tensorflow:Calling model_fn. 465 | INFO:tensorflow:*** Features *** 466 | INFO:tensorflow: name = input_ids, shape = (?, 32) 467 | INFO:tensorflow: name = input_mask, shape = (?, 32) 468 | INFO:tensorflow: name = label_ids, shape = (?,) 469 | INFO:tensorflow: name = segment_ids, shape = (?, 32) 470 | INFO:tensorflow:**** Trainable Variables **** 471 | INFO:tensorflow: name = bert/embeddings/word_embeddings:0, shape = (21128, 768), *INIT_FROM_CKPT* 472 | INFO:tensorflow: name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT* 473 | INFO:tensorflow: name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT* 474 | INFO:tensorflow: name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 475 | INFO:tensorflow: name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 476 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 477 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 478 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 479 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 480 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 481 | INFO:tensorflow: name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 482 | INFO:tensorflow: name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 483 | INFO:tensorflow: name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 484 | INFO:tensorflow: name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 485 | INFO:tensorflow: name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 486 | INFO:tensorflow: name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 487 | INFO:tensorflow: name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 488 | INFO:tensorflow: name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 489 | INFO:tensorflow: name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 490 | INFO:tensorflow: name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 491 | INFO:tensorflow: name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 492 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 493 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 494 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 495 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 496 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 497 | INFO:tensorflow: name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 498 | INFO:tensorflow: name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 499 | INFO:tensorflow: name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 500 | INFO:tensorflow: name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 501 | INFO:tensorflow: name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 502 | INFO:tensorflow: name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 503 | INFO:tensorflow: name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 504 | INFO:tensorflow: name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 505 | INFO:tensorflow: name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 506 | INFO:tensorflow: name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 507 | INFO:tensorflow: name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 508 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 509 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 510 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 511 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 512 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 513 | INFO:tensorflow: name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 514 | INFO:tensorflow: name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 515 | INFO:tensorflow: name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 516 | INFO:tensorflow: name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 517 | INFO:tensorflow: name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 518 | INFO:tensorflow: name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 519 | INFO:tensorflow: name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 520 | INFO:tensorflow: name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 521 | INFO:tensorflow: name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 522 | INFO:tensorflow: name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 523 | INFO:tensorflow: name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 524 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 525 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 526 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 527 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 528 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 529 | INFO:tensorflow: name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 530 | INFO:tensorflow: name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 531 | INFO:tensorflow: name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 532 | INFO:tensorflow: name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 533 | INFO:tensorflow: name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 534 | INFO:tensorflow: name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 535 | INFO:tensorflow: name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 536 | INFO:tensorflow: name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 537 | INFO:tensorflow: name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 538 | INFO:tensorflow: name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 539 | INFO:tensorflow: name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 540 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 541 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 542 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 543 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 544 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 545 | INFO:tensorflow: name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 546 | INFO:tensorflow: name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 547 | INFO:tensorflow: name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 548 | INFO:tensorflow: name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 549 | INFO:tensorflow: name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 550 | INFO:tensorflow: name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 551 | INFO:tensorflow: name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 552 | INFO:tensorflow: name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 553 | INFO:tensorflow: name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 554 | INFO:tensorflow: name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 555 | INFO:tensorflow: name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 556 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 557 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 558 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 559 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 560 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 561 | INFO:tensorflow: name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 562 | INFO:tensorflow: name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 563 | INFO:tensorflow: name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 564 | INFO:tensorflow: name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 565 | INFO:tensorflow: name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 566 | INFO:tensorflow: name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 567 | INFO:tensorflow: name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 568 | INFO:tensorflow: name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 569 | INFO:tensorflow: name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 570 | INFO:tensorflow: name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 571 | INFO:tensorflow: name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 572 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 573 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 574 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 575 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 576 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 577 | INFO:tensorflow: name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 578 | INFO:tensorflow: name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 579 | INFO:tensorflow: name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 580 | INFO:tensorflow: name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 581 | INFO:tensorflow: name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 582 | INFO:tensorflow: name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 583 | INFO:tensorflow: name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 584 | INFO:tensorflow: name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 585 | INFO:tensorflow: name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 586 | INFO:tensorflow: name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 587 | INFO:tensorflow: name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 588 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 589 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 590 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 591 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 592 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 593 | INFO:tensorflow: name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 594 | INFO:tensorflow: name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 595 | INFO:tensorflow: name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 596 | INFO:tensorflow: name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 597 | INFO:tensorflow: name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 598 | INFO:tensorflow: name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 599 | INFO:tensorflow: name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 600 | INFO:tensorflow: name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 601 | INFO:tensorflow: name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 602 | INFO:tensorflow: name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 603 | INFO:tensorflow: name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 604 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 605 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 606 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 607 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 608 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 609 | INFO:tensorflow: name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 610 | INFO:tensorflow: name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 611 | INFO:tensorflow: name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 612 | INFO:tensorflow: name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 613 | INFO:tensorflow: name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 614 | INFO:tensorflow: name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 615 | INFO:tensorflow: name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 616 | INFO:tensorflow: name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 617 | INFO:tensorflow: name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 618 | INFO:tensorflow: name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 619 | INFO:tensorflow: name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 620 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 621 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 622 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 623 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 624 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 625 | INFO:tensorflow: name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 626 | INFO:tensorflow: name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 627 | INFO:tensorflow: name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 628 | INFO:tensorflow: name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 629 | INFO:tensorflow: name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 630 | INFO:tensorflow: name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 631 | INFO:tensorflow: name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 632 | INFO:tensorflow: name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 633 | INFO:tensorflow: name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 634 | INFO:tensorflow: name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 635 | INFO:tensorflow: name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 636 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 637 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 638 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 639 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 640 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 641 | INFO:tensorflow: name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 642 | INFO:tensorflow: name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 643 | INFO:tensorflow: name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 644 | INFO:tensorflow: name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 645 | INFO:tensorflow: name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 646 | INFO:tensorflow: name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 647 | INFO:tensorflow: name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 648 | INFO:tensorflow: name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 649 | INFO:tensorflow: name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 650 | INFO:tensorflow: name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 651 | INFO:tensorflow: name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 652 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 653 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT* 654 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 655 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT* 656 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 657 | INFO:tensorflow: name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT* 658 | INFO:tensorflow: name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 659 | INFO:tensorflow: name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 660 | INFO:tensorflow: name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 661 | INFO:tensorflow: name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 662 | INFO:tensorflow: name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT* 663 | INFO:tensorflow: name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT* 664 | INFO:tensorflow: name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT* 665 | INFO:tensorflow: name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 666 | INFO:tensorflow: name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT* 667 | INFO:tensorflow: name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT* 668 | INFO:tensorflow: name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT* 669 | INFO:tensorflow: name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT* 670 | INFO:tensorflow: name = output_weights:0, shape = (2, 768), *INIT_FROM_CKPT* 671 | INFO:tensorflow: name = output_bias:0, shape = (2,), *INIT_FROM_CKPT* 672 | INFO:tensorflow:Done calling model_fn. 673 | INFO:tensorflow:Starting evaluation at 2019-02-18-08:07:59 674 | INFO:tensorflow:Graph was finalized. 675 | 2019-02-18 08:08:00.322544: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1511] Adding visible gpu devices: 0 676 | 2019-02-18 08:08:00.322638: I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] Device interconnect StreamExecutor with strength 1 edge matrix: 677 | 2019-02-18 08:08:00.322651: I tensorflow/core/common_runtime/gpu/gpu_device.cc:988] 0 678 | 2019-02-18 08:08:00.322658: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 0: N 679 | 2019-02-18 08:08:00.322833: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 9151 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7) 680 | INFO:tensorflow:Restoring parameters from /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt-5468 681 | INFO:tensorflow:Running local_init_op. 682 | INFO:tensorflow:Done running local_init_op. 683 | INFO:tensorflow:Finished evaluation at 2019-02-18-08:09:56 684 | INFO:tensorflow:Saving dict for global step 5468: eval_accuracy = 0.8500477, eval_auc = 0.68875444, eval_loss = 0.7216634, global_step = 5468, loss = 0.7218847 685 | INFO:tensorflow:Saving 'checkpoint_path' summary for global step 5468: /home/amei/Worspace/bert-utils/chinese_L-12_H-768_A-12/../tmp/result/model.ckpt-5468 686 | INFO:tensorflow:***** Eval results ***** 687 | INFO:tensorflow: eval_accuracy = 0.8500477 688 | INFO:tensorflow: eval_auc = 0.68875444 689 | INFO:tensorflow: eval_loss = 0.7216634 690 | INFO:tensorflow: global_step = 5468 691 | INFO:tensorflow: loss = 0.7218847 692 | --------------------------------------------------------------------------------