├── Keyword-Attentive_Deep_Semantic_Matching.pdf ├── keyword-bert ├── .gitignore ├── LICENSE ├── __init__.py ├── create_pretraining_data.py ├── data │ ├── convert_to_bert_keyword.py │ ├── dev.tsv │ ├── test.tsv │ └── train.tsv ├── extract_features.py ├── modeling.py ├── modeling_test.py ├── optimization.py ├── optimization_test.py ├── pre_tokenize.py ├── pre_trained │ ├── bert_config.json │ ├── bert_config_6_layer.json │ └── vocab.txt ├── readme.md ├── requirements.txt ├── run.sh ├── run_classifier.py ├── run_pretraining.py ├── run_squad.py ├── sample_text.txt ├── tokenization.py └── tokenization_test.py └── readme.md /Keyword-Attentive_Deep_Semantic_Matching.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataTerminatorX/Keyword-BERT/1a01d128ce84eb7c057aaf5ac6a06dea744a7770/Keyword-Attentive_Deep_Semantic_Matching.pdf -------------------------------------------------------------------------------- /keyword-bert/.gitignore: -------------------------------------------------------------------------------- 1 | # Initially taken from Github's Python gitignore file 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # Environments 92 | .env 93 | .venv 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | .dmypy.json 113 | dmypy.json 114 | 115 | # Pyre type checker 116 | .pyre/ 117 | -------------------------------------------------------------------------------- /keyword-bert/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /keyword-bert/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /keyword-bert/create_pretraining_data.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Create masked LM/next sentence masked_lm TF examples for BERT.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import collections 22 | import random 23 | import tokenization 24 | import tensorflow as tf 25 | 26 | flags = tf.flags 27 | 28 | FLAGS = flags.FLAGS 29 | 30 | flags.DEFINE_string("input_file", None, 31 | "Input raw text file (or comma-separated list of files).") 32 | 33 | flags.DEFINE_string( 34 | "output_file", None, 35 | "Output TF example file (or comma-separated list of files).") 36 | 37 | flags.DEFINE_string("vocab_file", None, 38 | "The vocabulary file that the BERT model was trained on.") 39 | 40 | flags.DEFINE_bool( 41 | "do_lower_case", True, 42 | "Whether to lower case the input text. Should be True for uncased " 43 | "models and False for cased models.") 44 | 45 | flags.DEFINE_bool( 46 | "do_whole_word_mask", False, 47 | "Whether to use whole word masking rather than per-WordPiece masking.") 48 | 49 | flags.DEFINE_integer("max_seq_length", 128, "Maximum sequence length.") 50 | 51 | flags.DEFINE_integer("max_predictions_per_seq", 20, 52 | "Maximum number of masked LM predictions per sequence.") 53 | 54 | flags.DEFINE_integer("random_seed", 12345, "Random seed for data generation.") 55 | 56 | flags.DEFINE_integer( 57 | "dupe_factor", 10, 58 | "Number of times to duplicate the input data (with different masks).") 59 | 60 | flags.DEFINE_float("masked_lm_prob", 0.15, "Masked LM probability.") 61 | 62 | flags.DEFINE_float( 63 | "short_seq_prob", 0.1, 64 | "Probability of creating sequences which are shorter than the " 65 | "maximum length.") 66 | 67 | 68 | class TrainingInstance(object): 69 | """A single training instance (sentence pair).""" 70 | 71 | def __init__(self, tokens, segment_ids, masked_lm_positions, masked_lm_labels, 72 | is_random_next): 73 | self.tokens = tokens 74 | self.segment_ids = segment_ids 75 | self.is_random_next = is_random_next 76 | self.masked_lm_positions = masked_lm_positions 77 | self.masked_lm_labels = masked_lm_labels 78 | 79 | def __str__(self): 80 | s = "" 81 | s += "tokens: %s\n" % (" ".join( 82 | [tokenization.printable_text(x) for x in self.tokens])) 83 | s += "segment_ids: %s\n" % (" ".join([str(x) for x in self.segment_ids])) 84 | s += "is_random_next: %s\n" % self.is_random_next 85 | s += "masked_lm_positions: %s\n" % (" ".join( 86 | [str(x) for x in self.masked_lm_positions])) 87 | s += "masked_lm_labels: %s\n" % (" ".join( 88 | [tokenization.printable_text(x) for x in self.masked_lm_labels])) 89 | s += "\n" 90 | return s 91 | 92 | def __repr__(self): 93 | return self.__str__() 94 | 95 | 96 | def write_instance_to_example_files(instances, tokenizer, max_seq_length, 97 | max_predictions_per_seq, output_files): 98 | """Create TF example files from `TrainingInstance`s.""" 99 | writers = [] 100 | for output_file in output_files: 101 | writers.append(tf.python_io.TFRecordWriter(output_file)) 102 | 103 | writer_index = 0 104 | 105 | total_written = 0 106 | for (inst_index, instance) in enumerate(instances): 107 | input_ids = tokenizer.convert_tokens_to_ids(instance.tokens) 108 | input_mask = [1] * len(input_ids) 109 | segment_ids = list(instance.segment_ids) 110 | assert len(input_ids) <= max_seq_length 111 | 112 | while len(input_ids) < max_seq_length: 113 | input_ids.append(0) 114 | input_mask.append(0) 115 | segment_ids.append(0) 116 | 117 | assert len(input_ids) == max_seq_length 118 | assert len(input_mask) == max_seq_length 119 | assert len(segment_ids) == max_seq_length 120 | 121 | masked_lm_positions = list(instance.masked_lm_positions) 122 | masked_lm_ids = tokenizer.convert_tokens_to_ids(instance.masked_lm_labels) 123 | masked_lm_weights = [1.0] * len(masked_lm_ids) 124 | 125 | while len(masked_lm_positions) < max_predictions_per_seq: 126 | masked_lm_positions.append(0) 127 | masked_lm_ids.append(0) 128 | masked_lm_weights.append(0.0) 129 | 130 | next_sentence_label = 1 if instance.is_random_next else 0 131 | 132 | features = collections.OrderedDict() 133 | features["input_ids"] = create_int_feature(input_ids) 134 | features["input_mask"] = create_int_feature(input_mask) 135 | features["segment_ids"] = create_int_feature(segment_ids) 136 | features["masked_lm_positions"] = create_int_feature(masked_lm_positions) 137 | features["masked_lm_ids"] = create_int_feature(masked_lm_ids) 138 | features["masked_lm_weights"] = create_float_feature(masked_lm_weights) 139 | features["next_sentence_labels"] = create_int_feature([next_sentence_label]) 140 | 141 | tf_example = tf.train.Example(features=tf.train.Features(feature=features)) 142 | 143 | writers[writer_index].write(tf_example.SerializeToString()) 144 | writer_index = (writer_index + 1) % len(writers) 145 | 146 | total_written += 1 147 | 148 | if inst_index < 20: 149 | tf.logging.info("*** Example ***") 150 | tf.logging.info("tokens: %s" % " ".join( 151 | [tokenization.printable_text(x) for x in instance.tokens])) 152 | 153 | for feature_name in features.keys(): 154 | feature = features[feature_name] 155 | values = [] 156 | if feature.int64_list.value: 157 | values = feature.int64_list.value 158 | elif feature.float_list.value: 159 | values = feature.float_list.value 160 | tf.logging.info( 161 | "%s: %s" % (feature_name, " ".join([str(x) for x in values]))) 162 | 163 | for writer in writers: 164 | writer.close() 165 | 166 | tf.logging.info("Wrote %d total instances", total_written) 167 | 168 | 169 | def create_int_feature(values): 170 | feature = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) 171 | return feature 172 | 173 | 174 | def create_float_feature(values): 175 | feature = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) 176 | return feature 177 | 178 | 179 | def create_training_instances(input_files, tokenizer, max_seq_length, 180 | dupe_factor, short_seq_prob, masked_lm_prob, 181 | max_predictions_per_seq, rng): 182 | """Create `TrainingInstance`s from raw text.""" 183 | all_documents = [[]] 184 | 185 | # Input file format: 186 | # (1) One sentence per line. These should ideally be actual sentences, not 187 | # entire paragraphs or arbitrary spans of text. (Because we use the 188 | # sentence boundaries for the "next sentence prediction" task). 189 | # (2) Blank lines between documents. Document boundaries are needed so 190 | # that the "next sentence prediction" task doesn't span between documents. 191 | for input_file in input_files: 192 | with tf.gfile.GFile(input_file, "r") as reader: 193 | while True: 194 | line = tokenization.convert_to_unicode(reader.readline()) 195 | if not line: 196 | break 197 | line = line.strip() 198 | 199 | # Empty lines are used as document delimiters 200 | if not line: 201 | all_documents.append([]) 202 | tokens = tokenizer.tokenize(line) 203 | if tokens: 204 | all_documents[-1].append(tokens) 205 | 206 | # Remove empty documents 207 | all_documents = [x for x in all_documents if x] 208 | rng.shuffle(all_documents) 209 | 210 | vocab_words = list(tokenizer.vocab.keys()) 211 | instances = [] 212 | for _ in range(dupe_factor): 213 | for document_index in range(len(all_documents)): 214 | instances.extend( 215 | create_instances_from_document( 216 | all_documents, document_index, max_seq_length, short_seq_prob, 217 | masked_lm_prob, max_predictions_per_seq, vocab_words, rng)) 218 | 219 | rng.shuffle(instances) 220 | return instances 221 | 222 | 223 | def create_instances_from_document( 224 | all_documents, document_index, max_seq_length, short_seq_prob, 225 | masked_lm_prob, max_predictions_per_seq, vocab_words, rng): 226 | """Creates `TrainingInstance`s for a single document.""" 227 | document = all_documents[document_index] 228 | 229 | # Account for [CLS], [SEP], [SEP] 230 | max_num_tokens = max_seq_length - 3 231 | 232 | # We *usually* want to fill up the entire sequence since we are padding 233 | # to `max_seq_length` anyways, so short sequences are generally wasted 234 | # computation. However, we *sometimes* 235 | # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter 236 | # sequences to minimize the mismatch between pre-training and fine-tuning. 237 | # The `target_seq_length` is just a rough target however, whereas 238 | # `max_seq_length` is a hard limit. 239 | target_seq_length = max_num_tokens 240 | if rng.random() < short_seq_prob: 241 | target_seq_length = rng.randint(2, max_num_tokens) 242 | 243 | # We DON'T just concatenate all of the tokens from a document into a long 244 | # sequence and choose an arbitrary split point because this would make the 245 | # next sentence prediction task too easy. Instead, we split the input into 246 | # segments "A" and "B" based on the actual "sentences" provided by the user 247 | # input. 248 | instances = [] 249 | current_chunk = [] 250 | current_length = 0 251 | i = 0 252 | while i < len(document): 253 | segment = document[i] 254 | current_chunk.append(segment) 255 | current_length += len(segment) 256 | if i == len(document) - 1 or current_length >= target_seq_length: 257 | if current_chunk: 258 | # `a_end` is how many segments from `current_chunk` go into the `A` 259 | # (first) sentence. 260 | a_end = 1 261 | if len(current_chunk) >= 2: 262 | a_end = rng.randint(1, len(current_chunk) - 1) 263 | 264 | tokens_a = [] 265 | for j in range(a_end): 266 | tokens_a.extend(current_chunk[j]) 267 | 268 | tokens_b = [] 269 | # Random next 270 | is_random_next = False 271 | if len(current_chunk) == 1 or rng.random() < 0.5: 272 | is_random_next = True 273 | target_b_length = target_seq_length - len(tokens_a) 274 | 275 | # This should rarely go for more than one iteration for large 276 | # corpora. However, just to be careful, we try to make sure that 277 | # the random document is not the same as the document 278 | # we're processing. 279 | for _ in range(10): 280 | random_document_index = rng.randint(0, len(all_documents) - 1) 281 | if random_document_index != document_index: 282 | break 283 | 284 | random_document = all_documents[random_document_index] 285 | random_start = rng.randint(0, len(random_document) - 1) 286 | for j in range(random_start, len(random_document)): 287 | tokens_b.extend(random_document[j]) 288 | if len(tokens_b) >= target_b_length: 289 | break 290 | # We didn't actually use these segments so we "put them back" so 291 | # they don't go to waste. 292 | num_unused_segments = len(current_chunk) - a_end 293 | i -= num_unused_segments 294 | # Actual next 295 | else: 296 | is_random_next = False 297 | for j in range(a_end, len(current_chunk)): 298 | tokens_b.extend(current_chunk[j]) 299 | truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng) 300 | 301 | assert len(tokens_a) >= 1 302 | assert len(tokens_b) >= 1 303 | 304 | tokens = [] 305 | segment_ids = [] 306 | tokens.append("[CLS]") 307 | segment_ids.append(0) 308 | for token in tokens_a: 309 | tokens.append(token) 310 | segment_ids.append(0) 311 | 312 | tokens.append("[SEP]") 313 | segment_ids.append(0) 314 | 315 | for token in tokens_b: 316 | tokens.append(token) 317 | segment_ids.append(1) 318 | tokens.append("[SEP]") 319 | segment_ids.append(1) 320 | 321 | (tokens, masked_lm_positions, 322 | masked_lm_labels) = create_masked_lm_predictions( 323 | tokens, masked_lm_prob, max_predictions_per_seq, vocab_words, rng) 324 | instance = TrainingInstance( 325 | tokens=tokens, 326 | segment_ids=segment_ids, 327 | is_random_next=is_random_next, 328 | masked_lm_positions=masked_lm_positions, 329 | masked_lm_labels=masked_lm_labels) 330 | instances.append(instance) 331 | current_chunk = [] 332 | current_length = 0 333 | i += 1 334 | 335 | return instances 336 | 337 | 338 | MaskedLmInstance = collections.namedtuple("MaskedLmInstance", 339 | ["index", "label"]) 340 | 341 | 342 | def create_masked_lm_predictions(tokens, masked_lm_prob, 343 | max_predictions_per_seq, vocab_words, rng): 344 | """Creates the predictions for the masked LM objective.""" 345 | 346 | cand_indexes = [] 347 | for (i, token) in enumerate(tokens): 348 | if token == "[CLS]" or token == "[SEP]": 349 | continue 350 | # Whole Word Masking means that if we mask all of the wordpieces 351 | # corresponding to an original word. When a word has been split into 352 | # WordPieces, the first token does not have any marker and any subsequence 353 | # tokens are prefixed with ##. So whenever we see the ## token, we 354 | # append it to the previous set of word indexes. 355 | # 356 | # Note that Whole Word Masking does *not* change the training code 357 | # at all -- we still predict each WordPiece independently, softmaxed 358 | # over the entire vocabulary. 359 | if (FLAGS.do_whole_word_mask and len(cand_indexes) >= 1 and 360 | token.startswith("##")): 361 | cand_indexes[-1].append(i) 362 | else: 363 | cand_indexes.append([i]) 364 | 365 | rng.shuffle(cand_indexes) 366 | 367 | output_tokens = list(tokens) 368 | 369 | num_to_predict = min(max_predictions_per_seq, 370 | max(1, int(round(len(tokens) * masked_lm_prob)))) 371 | 372 | masked_lms = [] 373 | covered_indexes = set() 374 | for index_set in cand_indexes: 375 | if len(masked_lms) >= num_to_predict: 376 | break 377 | # If adding a whole-word mask would exceed the maximum number of 378 | # predictions, then just skip this candidate. 379 | if len(masked_lms) + len(index_set) > num_to_predict: 380 | continue 381 | is_any_index_covered = False 382 | for index in index_set: 383 | if index in covered_indexes: 384 | is_any_index_covered = True 385 | break 386 | if is_any_index_covered: 387 | continue 388 | for index in index_set: 389 | covered_indexes.add(index) 390 | 391 | masked_token = None 392 | # 80% of the time, replace with [MASK] 393 | if rng.random() < 0.8: 394 | masked_token = "[MASK]" 395 | else: 396 | # 10% of the time, keep original 397 | if rng.random() < 0.5: 398 | masked_token = tokens[index] 399 | # 10% of the time, replace with random word 400 | else: 401 | masked_token = vocab_words[rng.randint(0, len(vocab_words) - 1)] 402 | 403 | output_tokens[index] = masked_token 404 | 405 | masked_lms.append(MaskedLmInstance(index=index, label=tokens[index])) 406 | assert len(masked_lms) <= num_to_predict 407 | masked_lms = sorted(masked_lms, key=lambda x: x.index) 408 | 409 | masked_lm_positions = [] 410 | masked_lm_labels = [] 411 | for p in masked_lms: 412 | masked_lm_positions.append(p.index) 413 | masked_lm_labels.append(p.label) 414 | 415 | return (output_tokens, masked_lm_positions, masked_lm_labels) 416 | 417 | 418 | def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng): 419 | """Truncates a pair of sequences to a maximum sequence length.""" 420 | while True: 421 | total_length = len(tokens_a) + len(tokens_b) 422 | if total_length <= max_num_tokens: 423 | break 424 | 425 | trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b 426 | assert len(trunc_tokens) >= 1 427 | 428 | # We want to sometimes truncate from the front and sometimes from the 429 | # back to add more randomness and avoid biases. 430 | if rng.random() < 0.5: 431 | del trunc_tokens[0] 432 | else: 433 | trunc_tokens.pop() 434 | 435 | 436 | def main(_): 437 | tf.logging.set_verbosity(tf.logging.INFO) 438 | 439 | tokenizer = tokenization.FullTokenizer( 440 | vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) 441 | 442 | input_files = [] 443 | for input_pattern in FLAGS.input_file.split(","): 444 | input_files.extend(tf.gfile.Glob(input_pattern)) 445 | 446 | tf.logging.info("*** Reading from input files ***") 447 | for input_file in input_files: 448 | tf.logging.info(" %s", input_file) 449 | 450 | rng = random.Random(FLAGS.random_seed) 451 | instances = create_training_instances( 452 | input_files, tokenizer, FLAGS.max_seq_length, FLAGS.dupe_factor, 453 | FLAGS.short_seq_prob, FLAGS.masked_lm_prob, FLAGS.max_predictions_per_seq, 454 | rng) 455 | 456 | output_files = FLAGS.output_file.split(",") 457 | tf.logging.info("*** Writing to output files ***") 458 | for output_file in output_files: 459 | tf.logging.info(" %s", output_file) 460 | 461 | write_instance_to_example_files(instances, tokenizer, FLAGS.max_seq_length, 462 | FLAGS.max_predictions_per_seq, output_files) 463 | 464 | 465 | if __name__ == "__main__": 466 | flags.mark_flag_as_required("input_file") 467 | flags.mark_flag_as_required("output_file") 468 | flags.mark_flag_as_required("vocab_file") 469 | tf.app.run() 470 | -------------------------------------------------------------------------------- /keyword-bert/data/convert_to_bert_keyword.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | ######################################################################### 4 | # File Name: convert_to_bert_keyword.py 5 | # Author: changyumiao 6 | # Mail: changyumiao@tencent.com 7 | # Created Time: Tue 30 Jul 2019 09:30:00 PM CST 8 | # Python Version: 3.6 9 | ######################################################################### 10 | 11 | """This script convert keywords in text to keyword mask 12 | Example 13 | --- 14 | Input line: 15 | 0 \t 中 国 niu## #iub# ###bi 极 了 \t 中国 niubi \t 厉 害 了 我 的 国 \t 厉害 国 16 | Output line: 17 | 0 \t 同上 \t 1 1 1 1 1 0 0 \t 同上 \t 1 1 0 0 0 1 18 | 19 | """ 20 | 21 | import re 22 | import sys 23 | 24 | 25 | def match_en(s, kw): 26 | kw_index = [] 27 | for idx,e in enumerate(s): 28 | e.replace('#', '') 29 | if e in kw: 30 | kw_index.append(idx) 31 | return kw_index 32 | 33 | 34 | def match_ch(s, kw): 35 | kw_index = [] 36 | cur_rs = [] 37 | p1 = 0 38 | p2 = 0 39 | while(p1+p2 <= len(s)): 40 | if p2 == len(kw): # match succed 41 | kw_index += cur_rs 42 | p2 = 0 43 | p1 += 1 44 | cur_rs = [] 45 | if (p1 + p2) < len(s) and s[p1+p2] == kw[p2]: # matching 46 | cur_rs.append(p1+p2) 47 | p2 += 1 48 | else: # match failed 49 | p2 = 0 50 | p1 += 1 51 | cur_rs = [] 52 | return kw_index 53 | 54 | 55 | def match(s, kws): 56 | kw_index = set() 57 | for kw in kws: 58 | if re.match(r'^[\u4e00-\u9fff]+$', kw): 59 | kw_index |= set(match_ch(s, kw)) 60 | elif re.match(r'^[a-zA-Z]+$', kw): 61 | kw_index |= set(match_ch(s, kw)) 62 | else: 63 | continue 64 | return kw_index 65 | 66 | 67 | def main(in_file, out_file, drop_no_kw=None): 68 | with open(in_file, 'r', encoding='utf8') as fi, \ 69 | open(out_file, 'w', encoding='utf8') as fo: 70 | for idx, line in enumerate(fi): 71 | print('>> processing line %d' % idx, end='\r') 72 | line = line.strip('\n').split('\t') 73 | if len(line) != 5: 74 | continue 75 | label = line[0] 76 | text_a, kw_a = line[1].split(), line[2].split() 77 | text_b, kw_b = line[3].split(), line[4].split() 78 | if drop_no_kw and (not (kw_a and kw_b)): # keep sentence pairs both contain keywords 79 | continue 80 | kw_a_index = match(text_a, kw_a) 81 | kw_b_index = match(text_b, kw_b) 82 | kw_a_mask = ' '.join(['1' if idx in kw_a_index else '0' 83 | for idx in range(len(text_a))]) 84 | kw_b_mask = ' '.join(['1' if idx in kw_b_index else '0' 85 | for idx in range(len(text_b))]) 86 | new_line = '\t'.join([label, line[1], kw_a_mask, 87 | line[3], kw_b_mask]) 88 | print(new_line, file=fo) 89 | if (idx+1) % 5000 == 0: 90 | fo.flush() 91 | 92 | 93 | def test_match_ch(s, kw): 94 | a = match_ch("ababcdaaabcab", "abc") 95 | print(a) 96 | b = match_ch("中 国".split(),"中国") 97 | print(b) 98 | 99 | 100 | if __name__ == "__main__": 101 | if len(sys.argv) == 3: 102 | main(sys.argv[1], sys.argv[2]) 103 | elif len(sys.argv) == 4: 104 | main(sys.argv[1], sys.argv[2], sys.argv[3]) 105 | else: 106 | raise ValueError("Invalid arg number!") 107 | -------------------------------------------------------------------------------- /keyword-bert/data/dev.tsv: -------------------------------------------------------------------------------- 1 | 1 做 自 己 的 主 人 0 0 0 0 0 0 怎 样 做 自 己 的 主 人 0 0 0 0 0 0 0 0 2 | 1 我 想 了 解 下 魏 刚 0 0 0 0 1 1 0 谁 是 魏 刚 0 0 0 0 3 | 0 当 今 世 界 高 科 技 发 展 的 五 大 趋 势 0 0 0 0 1 1 1 0 0 0 1 1 0 0 为 什 么 硅 成 为 高 科 技 的 代 表 0 0 0 0 0 0 1 1 1 0 0 0 4 | 0 眼 睛 里 面 进 东 西 怎 么 办 1 1 0 0 0 1 1 0 0 0 宝 宝 眼 睛 老 是 流 泪 怎 么 办 1 1 1 1 0 0 1 1 0 0 0 5 | 0 光 纤 通 信 0 0 0 0 卫 星 通 信 0 0 0 0 6 | 1 71 年 后 才 得 的 金 牌 0 1 1 0 0 0 0 0 谁 71 年 后 才 得 到 金 牌 0 0 0 1 1 0 0 0 0 7 | 1 暗 中 听 评 0 0 0 0 暗 中 听 评 的 故 事 0 0 0 0 0 0 0 8 | 1 现 代 奥 运 会 是 怎 样 诞 生 的 0 0 1 1 1 0 0 0 0 0 0 现 代 奥 运 会 诞 生 的 由 来 0 0 1 1 1 0 0 0 0 0 9 | 1 赤 潮 与 赤 潮 生 物 1 1 0 1 1 0 0 赤 潮 与 赤 潮 生 物 有 什 么 区 别 1 1 0 1 1 1 1 0 0 0 0 0 10 | 0 云 南 石 林 是 怎 样 形 成 的 1 1 1 1 0 0 0 0 0 0 尿 液 是 怎 样 形 成 的 1 1 0 0 0 0 0 0 11 | 0 你 知 道 什 么 是 苹 果 0 0 0 0 0 0 0 0 你 知 道 什 么 是 麻 黄 0 0 0 0 0 0 0 0 12 | 0 鲑 鱼 回 游 之 谜 1 1 0 0 0 0 海 豚 之 谜 1 1 0 0 13 | 1 新 发 现 的 海 洋 杀 手 0 0 0 0 1 1 0 0 新 发 现 的 海 洋 杀 手 是 什 么 0 0 0 0 1 1 0 0 0 0 0 14 | 1 为 什 么 有 些 人 会 口 吃 0 0 0 0 0 1 1 0 0 为 什 么 有 些 人 会 口 吃 0 0 0 0 0 1 1 0 0 15 | 1 胳 膊 和 大 腿 为 什 么 不 容 易 起 包 1 1 0 1 1 0 0 0 0 0 0 0 0 胳 膊 和 大 腿 不 容 易 起 包 的 原 因 是 什 么 1 1 0 1 1 0 0 0 0 0 0 1 1 0 0 0 16 | 1 人 生 ， 没 有 什 么 事 是 重 要 的 1 1 0 0 0 0 0 0 0 0 0 0 为 什 么 说 人 生 没 有 什 么 事 是 重 要 的 0 0 0 0 1 1 0 0 0 0 0 0 0 0 0 17 | 1 蛋 品 的 种 类 1 1 0 0 0 蛋 品 有 哪 些 种 类 1 1 0 0 0 0 0 18 | 0 外 星 人 ， 你 在 哪 里 1 1 1 0 0 0 0 0 为 什 么 说 会 有 外 星 人 0 0 0 0 0 0 0 0 0 19 | 1 礼 仪 知 识 ： 畲 族 礼 仪 1 1 0 0 0 1 1 0 0 介 绍 一 下 畲 族 礼 仪 0 0 0 0 1 1 0 0 20 | 0 我 国 的 铁 矿 资 源 0 0 0 1 1 0 0 我 国 的 水 能 资 源 0 0 0 1 1 0 0 21 | 1 华 沙 美 人 鱼 1 1 0 0 0 华 沙 美 人 鱼 是 什 么 1 1 1 1 1 0 0 0 22 | 1 为 什 么 会 有 龙 卷 风 0 0 0 0 0 0 0 0 龙 卷 风 怎 么 来 的 1 1 1 0 0 0 0 23 | 1 帮 我 介 绍 下 香 椿 0 0 0 0 0 0 0 什 么 是 香 椿 0 0 0 0 0 24 | 0 酱 菜 类 制 法 1 1 0 0 0 糖 醋 菜 类 制 法 1 1 1 1 0 0 25 | 1 浅 研 植 物 的 茎 与 光 照 的 关 系 0 0 1 1 0 0 0 1 1 0 0 0 浅 研 植 物 的 茎 与 光 照 有 什 么 关 系 0 0 1 1 0 0 0 1 1 0 0 0 0 0 26 | 1 审 美 感 知 方 面 1 1 0 0 0 0 介 绍 一 下 审 美 感 知 方 面 0 0 0 0 1 1 0 0 0 0 27 | 0 麒 麟 是 什 么 动 物 1 1 0 0 0 0 0 狗 是 什 么 动 物 0 0 0 0 0 0 28 | 1 黑 龙 江 名 称 的 来 历 1 1 1 1 1 0 0 0 黑 龙 江 名 称 有 什 么 故 事 1 1 1 1 1 0 0 0 0 0 29 | 0 中 国 古 代 肃 贪 机 制 考 0 0 1 1 1 1 1 1 0 中 国 古 代 官 制 1 1 1 1 0 0 30 | 1 [UNK] 鼠 0 0 [UNK] 鼠 是 什 么 0 0 0 0 0 31 | 0 邮 箱 的 诞 生 1 1 0 0 0 磺 胺 的 诞 生 1 1 0 0 0 32 | 1 人 工 降 雨 的 发 明 1 1 1 1 0 0 0 人 工 降 雨 是 谁 发 明 1 1 1 1 0 0 0 0 33 | 0 上 官 婉 儿 为 什 么 不 记 武 则 天 灭 族 之 仇 1 1 1 1 0 0 0 0 0 1 1 1 1 1 0 0 武 则 天 为 什 么 要 立 无 字 碑 1 1 1 0 0 0 0 0 0 0 0 34 | 1 为 什 么 说 蝗 虫 是 害 虫 0 0 0 0 1 1 0 0 0 蝗 虫 怎 么 是 害 虫 啊 1 1 0 0 0 0 0 0 35 | 0 珍 贵 的 罗 汉 钱 0 0 0 1 1 0 珍 贵 的 丹 顶 鹤 0 0 0 0 0 0 36 | 1 为 什 么 李 世 民 会 发 动 " 玄 武 门 之 变 " 0 0 0 1 1 1 0 1 1 0 1 1 1 1 1 0 李 世 民 会 发 动 " 玄 武 门 之 变 " 的 原 因 是 什 么 1 1 1 0 1 1 0 1 1 1 1 1 0 0 0 0 0 0 0 37 | 0 为 什 么 胃 不 会 消 化 自 己 0 0 0 0 0 0 1 1 0 0 胃 是 怎 么 消 化 食 物 的 0 0 0 0 1 1 1 1 0 38 | 1 台 风 的 移 动 有 规 律 吗 1 1 0 1 1 0 1 1 0 台 风 的 移 动 的 规 律 是 什 么 吗 1 1 0 1 1 0 1 1 0 0 0 0 39 | 1 玛 雅 文 明 之 谜 1 1 1 1 0 0 什 么 是 玛 雅 文 明 0 0 0 1 1 0 0 40 | 1 为 什 么 称 猕 猴 桃 为 " 水 果 之 王 " 0 0 0 0 1 1 1 0 0 1 1 0 0 0 猕 猴 桃 被 称 为 " 水 果 之 王 " 的 原 因 是 什 么 1 1 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 0 41 | 0 植 物 的 报 复 行 为 1 1 0 0 0 0 0 动 物 的 报 复 性 为 1 1 0 1 1 1 0 42 | 0 斗 毛 眼 蝶 1 1 0 0 趣 蝶 莲 1 1 0 43 | 1 近 视 眼 0 0 0 介 绍 一 下 近 视 眼 0 0 0 0 0 0 0 44 | 1 声 音 的 妙 用 1 1 0 0 0 声 音 有 什 么 妙 用 1 1 0 0 0 0 0 45 | 0 巧 设 洗 衣 店 0 0 0 0 0 巧 设 问 题 让 孩 子 异 想 天 开 0 1 1 0 0 1 1 0 0 0 0 46 | 0 " 五 彩 神 池 " 话 九 寨 0 1 1 1 1 0 0 0 0 五 彩 稻 1 1 0 47 | 1 太 阳 黑 子 为 什 么 会 影 响 无 线 电 通 讯 1 1 1 1 0 0 0 0 0 0 1 1 1 0 0 太 阳 黑 子 怎 么 影 响 无 线 电 通 讯 的 1 1 1 1 0 0 0 0 1 1 1 1 1 0 48 | 0 直 升 机 为 什 么 需 要 装 长 尾 巴 1 1 1 0 0 0 0 0 0 0 0 0 为 什 么 直 升 机 可 以 在 空 中 停 留 0 0 0 1 1 1 0 0 0 1 1 0 0 49 | 0 为 什 么 称 猕 猴 桃 为 " 水 果 之 王 " 0 0 0 0 1 1 1 0 0 1 1 0 0 0 为 什 么 称 钢 琴 为 乐 器 之 王 0 0 0 0 1 1 0 1 1 0 0 50 | 0 为 什 么 人 造 卫 星 能 上 天 0 0 0 1 1 1 1 0 0 0 飞 机 无 燃 料 也 能 上 天 1 1 0 1 1 0 0 0 0 51 | 1 珍 贵 的 罗 汉 钱 0 0 0 1 1 0 介 绍 一 下 珍 贵 的 罗 汉 钱 0 0 0 0 0 0 0 1 1 0 52 | 1 船 舶 电 话 1 1 0 0 船 舶 电 话 的 故 事 1 1 1 1 0 0 0 53 | 0 指 甲 为 什 么 会 不 停 地 生 长 1 1 0 0 0 0 0 0 0 0 0 蜂 蟹 为 什 么 会 不 停 地 吐 泡 泡 1 1 0 0 0 0 0 0 1 1 0 0 54 | 0 独 角 鲸 一 牙 多 用 1 1 0 0 0 0 0 手 机 一 机 多 用 1 1 0 0 0 0 55 | 1 为 什 么 发 射 火 箭 要 沿 着 地 球 自 转 方 向 0 0 0 0 0 1 1 0 0 0 1 1 1 1 0 0 发 射 火 箭 要 沿 着 地 球 自 转 方 向 是 什 么 原 因 0 0 1 1 0 0 0 1 1 1 1 1 1 0 0 0 0 0 56 | 0 农 业 的 " 三 色 革 命 " 1 1 0 0 1 1 1 1 0 三 色 面 团 0 0 0 0 57 | 0 为 什 么 一 群 鸭 子 走 路 总 喜 欢 排 成 队 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 鸭 子 走 路 为 什 么 一 摇 一 摆 1 1 1 1 0 0 0 0 0 0 0 58 | 1 当 代 的 原 始 部 落 1 1 0 0 0 0 0 当 代 的 原 始 部 落 有 哪 些 1 1 0 1 1 1 1 0 0 0 59 | 1 为 什 么 麋 鹿 又 叫 四 不 像 0 0 0 1 1 0 0 0 0 0 麋 鹿 为 什 么 又 叫 四 不 像 1 1 0 0 0 0 0 0 0 0 60 | 0 布 拉 马 0 0 0 阿 马 鲁 1 1 0 61 | 0 饼 干 的 由 来 1 1 0 0 0 天 子 的 由 来 1 1 0 0 0 62 | 0 为 什 么 说 瑞 雪 兆 丰 年 0 0 0 0 0 0 0 0 0 为 什 么 说 谢 谢 0 0 0 0 0 0 63 | 1 什 么 是 脂 肪 0 0 0 0 0 介 绍 一 下 脂 肪 0 0 0 0 0 0 64 | 1 统 一 团 结 的 象 征 — — 人 民 大 会 堂 1 1 0 0 0 0 0 0 0 0 0 0 0 0 为 什 么 统 一 团 结 的 象 征 是 人 民 大 会 堂 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 0 65 | 0 爱 乐 乐 团 的 排 练 风 波 1 1 1 1 0 1 1 0 0 无 理 数 的 风 波 1 1 1 0 0 0 66 | 1 太 阳 的 耀 斑 是 怎 么 回 事 1 1 0 1 1 0 0 0 0 0 太 阳 的 耀 斑 是 什 么 1 1 0 1 1 0 0 0 67 | 0 水 底 气 源 1 1 0 0 水 底 家 园 1 1 0 0 68 | 0 " 超 声 " 和 " 超 音 " 0 0 0 0 0 0 1 1 0 什 么 是 超 声 波 0 0 0 0 0 0 69 | 1 为 什 么 企 鹅 走 起 路 来 一 摇 一 摆 0 0 0 1 1 1 1 1 1 0 0 0 0 企 鹅 走 起 路 来 为 什 么 会 一 摇 一 摆 1 1 1 1 1 1 0 0 0 0 0 0 0 0 70 | 0 什 么 是 大 西 洋 0 0 0 0 0 0 什 么 是 太 平 洋 0 0 0 0 0 0 71 | 1 伟 大 的 遗 迹 神 龛 诗 — — 玛 哈 达 寺 0 0 0 0 0 1 1 0 0 0 0 1 1 0 为 什 么 说 玛 哈 达 寺 是 伟 大 的 遗 迹 神 龛 诗 0 0 0 0 0 1 1 0 0 0 0 0 1 1 1 1 0 72 | 1 指 挥 果 断 配 合 密 切 的 柘 皋 之 战 0 0 0 0 0 0 0 0 0 0 0 0 0 指 挥 果 断 配 合 密 切 的 柘 皋 之 战 是 怎 样 的 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 73 | 0 心 慈 不 怕 世 道 险 、 胆 大 不 怕 妖 魔 扰 1 1 0 0 1 1 0 0 0 0 0 0 1 1 0 不 怕 苦 的 植 物 1 1 1 0 0 0 74 | 1 照 相 技 术 0 0 0 0 照 相 技 术 怎 么 发 展 的 0 0 1 1 0 0 1 1 0 75 | 1 智 力 发 育 为 什 么 要 依 靠 营 养 合 理 1 1 1 1 0 0 0 0 0 0 1 1 0 0 为 什 么 智 力 发 育 要 依 靠 营 养 合 理 0 0 0 1 1 1 1 0 0 0 1 1 0 0 76 | 0 为 什 么 林 肯 政 府 能 赢 得 美 国 南 北 战 争 的 胜 利 0 0 0 1 1 0 0 0 0 0 0 0 1 1 1 1 0 0 0 为 什 么 华 盛 顿 能 领 导 美 国 人 民 取 得 独 立 战 争 的 胜 利 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 77 | 0 我 们 的 身 体 是 由 什 么 组 成 的 0 0 0 1 1 0 0 0 0 0 0 0 蛋 壳 是 由 什 么 成 分 组 成 的 1 1 0 0 0 0 1 1 0 0 0 78 | 1 段 攻 羌 之 战 1 1 1 0 0 段 攻 羌 之 战 是 怎 样 的 1 1 1 1 1 0 0 0 0 79 | 1 管 道 运 输 技 术 1 1 1 1 0 0 什 么 是 管 道 运 输 技 术 0 0 0 1 1 1 1 0 0 80 | 0 鸟 中 女 王 0 0 0 0 鸟 中 的 强 盗 0 0 0 0 0 81 | 0 石 器 的 制 造 1 1 0 0 0 制 造 声 响 0 0 0 0 82 | 0 国 家 直 接 经 营 1 1 0 0 0 0 股 份 经 营 1 1 0 0 83 | 1 为 什 么 人 造 卫 星 能 上 天 0 0 0 1 1 1 1 0 0 0 人 造 卫 星 能 上 天 的 原 因 是 什 么 1 1 1 1 0 1 1 0 1 1 0 0 0 84 | 0 从 月 亮 上 看 到 的 地 球 0 1 1 0 0 0 0 0 0 从 星 星 上 看 到 的 地 球 0 1 1 0 0 0 0 0 0 85 | 0 靖 康 之 变 是 怎 样 发 生 的 1 1 0 0 0 0 0 0 0 0 龋 齿 是 怎 样 发 生 的 1 1 0 0 0 0 0 0 86 | 0 袅 娜 多 姿 的 壁 画 1 1 1 1 0 0 0 玻 璃 镶 嵌 壁 画 1 1 0 0 0 0 87 | 0 为 什 么 一 滴 墨 水 在 水 中 扩 散 以 后 再 也 不 会 自 动 聚 集 起 来 0 0 0 0 0 1 1 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 不 见 一 滴 水 的 热 气 泉 0 0 1 1 1 0 1 1 0 88 | 1 开 水 壶 用 久 了 为 什 么 会 生 水 垢 1 1 1 0 0 0 0 0 0 0 0 0 0 开 水 壶 用 久 了 会 生 水 垢 是 为 什 么 1 1 1 0 0 0 1 1 1 1 0 0 0 0 89 | 1 为 什 么 清 晨 起 床 后 要 打 开 窗 户 0 0 0 1 1 0 0 0 0 0 0 0 0 清 晨 起 床 后 为 什 么 要 打 开 窗 户 1 1 0 0 0 0 0 0 0 0 0 0 0 90 | 0 扁 形 动 物 0 0 0 0 线 形 动 物 0 0 0 0 91 | 0 泪 水 的 妙 用 1 1 0 0 0 声 音 的 妙 用 1 1 0 0 0 92 | 0 月 球 住 宅 1 1 0 0 月 球 的 七 大 谜 1 1 0 0 0 0 93 | 0 餐 桌 上 的 游 戏 1 1 1 0 0 0 宝 宝 餐 桌 上 的 保 健 学 问 1 1 1 1 1 0 0 0 0 0 94 | 0 负 数 的 引 入 1 1 0 0 0 生 物 的 引 入 1 1 0 0 0 95 | 1 越 王 台 与 浣 纱 石 1 1 1 0 1 1 0 越 王 台 与 浣 纱 石 分 别 是 什 么 1 1 1 0 1 1 0 0 0 0 0 0 96 | 1 为 什 么 肥 肉 和 瘦 肉 都 不 能 少 0 0 0 0 0 0 1 1 0 0 0 0 肥 肉 和 瘦 肉 都 不 能 少 的 原 因 是 什 么 0 0 0 1 1 0 0 0 0 0 1 1 0 0 0 97 | 0 天 然 而 成 的 嫁 接 树 1 1 0 0 0 0 0 0 分 株 嫁 接 亦 开 花 0 0 0 0 0 0 0 98 | 1 你 知 道 什 么 是 杨 虎 城 0 0 0 0 0 0 0 0 0 杨 虎 城 是 什 么 1 1 1 0 0 0 99 | 1 黄 金 邮 票 1 1 0 0 黄 金 邮 票 是 怎 样 的 1 1 1 1 0 0 0 0 100 | 0 蝙 蝠 是 怎 样 捕 食 昆 虫 的 1 1 0 0 0 0 0 1 1 0 食 虫 植 物 是 怎 样 捕 食 昆 虫 的 1 1 1 1 0 0 0 0 0 0 0 0 101 | -------------------------------------------------------------------------------- /keyword-bert/data/test.tsv: -------------------------------------------------------------------------------- 1 | 0 酒 与 醋 的 发 明 0 0 0 0 0 0 农 业 与 文 字 的 发 明 1 1 0 1 1 0 0 0 2 | 1 为 什 么 狗 是 人 类 首 先 驯 化 成 功 的 动 物 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 0 狗 是 人 类 首 先 驯 化 成 功 的 动 物 吗 0 0 1 1 0 0 0 0 0 0 0 1 1 0 3 | 1 为 什 么 自 行 车 比 赛 选 手 要 走 之 字 形 路 线 0 0 0 1 1 1 0 0 0 0 0 0 1 1 1 0 0 为 什 么 自 行 车 比 赛 选 手 要 走 之 字 形 路 线 0 0 0 1 1 1 0 0 0 0 0 0 1 1 1 0 0 4 | 1 为 什 么 每 上 一 节 课 要 休 息 10 分 钟 0 0 0 0 0 1 1 1 0 0 0 0 0 0 上 完 一 节 课 就 休 息 十 分 钟 有 什 么 根 据 吗 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 5 | 1 全 息 照 像 1 1 0 0 全 息 照 像 是 什 么 1 1 0 0 0 0 0 6 | 1 合 成 染 料 的 发 明 1 1 1 1 0 0 0 合 成 染 料 是 谁 发 明 的 1 1 1 1 0 0 0 0 0 7 | 0 帮 我 介 绍 下 军 曹 鱼 0 0 0 0 1 1 0 0 帮 我 介 绍 下 巨 骨 舌 鱼 0 0 0 0 0 1 1 0 0 8 | 1 影 视 制 作 的 最 后 一 道 工 序 1 1 1 1 0 0 0 0 0 0 0 影 视 制 作 的 最 后 一 道 工 序 是 什 么 1 1 1 1 0 0 0 0 0 1 1 0 0 0 9 | 0 什 么 是 中 国 民 主 促 进 会 0 0 0 0 0 0 0 0 0 0 什 么 是 科 学 民 主 的 家 教 方 式 0 0 0 1 1 1 1 0 1 1 0 0 10 | 0 你 知 道 什 么 是 中 华 虎 凤 蝶 0 0 0 0 0 0 1 1 0 0 0 你 知 道 什 么 是 柑 橘 凤 蝶 0 0 0 0 0 0 1 1 0 0 11 | 1 什 么 是 矛 盾 0 0 0 0 0 矛 盾 是 什 么 1 1 0 0 0 12 | 0 寄 生 性 种 子 植 物 1 1 1 1 1 0 0 寄 生 性 天 敌 昆 虫 1 1 1 1 1 0 0 13 | 0 银 波 草 0 0 0 孔 雀 草 1 1 0 14 | 0 为 什 么 有 的 恐 龙 身 上 要 穿 盔 甲 0 0 0 0 0 1 1 1 1 0 0 0 0 枪 械 为 什 么 要 穿 黑 衣 服 1 1 0 0 0 0 0 0 0 0 15 | 0 奥 运 会 会 歌 歌 词 1 1 1 1 1 0 0 残 运 会 会 歌 歌 词 1 1 1 1 1 0 0 16 | 1 香 蕉 为 什 么 没 有 籽 1 1 0 0 0 0 0 0 香 蕉 没 有 籽 是 什 么 原 因 1 1 0 0 0 0 0 0 0 0 17 | 0 为 什 么 说 埃 及 是 尼 罗 河 的 赠 礼 0 0 0 0 1 1 0 1 1 1 0 0 0 为 什 么 尼 罗 河 被 称 为 埃 及 的 母 亲 0 0 0 1 1 1 0 0 0 1 1 0 0 0 18 | 1 相 互 争 雄 的 金 辽 战 争 0 0 1 1 0 1 1 0 0 相 互 争 雄 的 金 辽 战 争 是 什 么 0 0 1 1 0 1 1 1 1 0 0 0 19 | 1 黑 枕 绿 啄 木 鸟 1 1 1 0 0 0 黑 枕 绿 啄 木 鸟 是 什 么 1 1 1 1 1 1 0 0 0 20 | 0 旅 游 得 来 " 灵 芝 草 " 1 1 0 0 0 1 1 1 0 风 流 草 ： 奇 特 的 风 流 草 1 1 0 0 1 1 0 1 1 0 21 | 0 听 讲 演 的 启 发 0 0 0 0 0 0 莱 特 兄 弟 讲 演 1 1 1 1 0 0 22 | 0 植 物 的 报 复 行 为 1 1 0 0 0 0 0 动 物 的 报 复 性 为 1 1 0 1 1 1 0 23 | 1 帮 我 介 绍 下 法 国 斗 牛 犬 0 0 0 0 0 1 1 0 0 0 什 么 是 法 国 斗 牛 犬 0 0 0 1 1 0 0 0 24 | 1 什 么 是 菲 尔 兹 奖 0 0 0 0 0 0 0 菲 尔 兹 奖 是 什 么 1 1 1 1 0 0 0 25 | 0 " 太 " 字 的 来 历 （ 二 ） 0 0 0 0 0 0 0 0 0 0 隋 朝 隋 字 的 来 历 1 1 0 0 0 0 0 26 | 1 迎 夏 与 迎 春 怎 样 区 别 1 1 0 0 0 0 0 0 0 迎 夏 与 迎 春 有 什 么 不 同 1 1 0 0 0 0 0 0 0 0 27 | 1 为 什 么 催 泪 弹 会 使 人 流 泪 0 0 0 1 1 1 0 0 0 0 0 催 泪 弹 会 使 人 流 泪 的 原 理 是 什 么 1 1 1 0 0 0 1 1 0 1 1 0 0 0 28 | 1 为 什 么 狗 要 四 处 嗅 来 嗅 去 0 0 0 0 0 0 0 0 0 0 0 狗 要 四 处 嗅 来 嗅 去 的 原 因 是 什 么 0 0 0 0 0 0 0 0 0 1 1 0 0 0 29 | 1 精 耕 细 作 源 远 流 长 1 1 1 1 0 0 0 0 精 耕 细 作 源 远 流 长 的 原 因 是 什 么 1 1 1 1 0 0 0 0 0 1 1 0 0 0 30 | 0 卓 别 林 轶 事 1 1 1 0 0 科 学 家 张 民 觉 轶 事 1 1 1 1 1 0 0 0 31 | 0 地 球 年 龄 知 多 少 1 1 1 1 0 0 0 天 池 湖 泊 知 多 少 1 1 1 1 0 0 0 32 | 1 蠢 太 子 " 考 场 " 舞 弊 0 1 1 0 1 1 0 0 0 蠢 太 子 " 考 场 " 舞 弊 是 什 么 故 事 0 1 1 0 1 1 0 1 1 0 0 0 0 0 33 | 0 " 胎 生 " 的 红 树 0 1 1 0 0 0 0 胎 生 的 动 物 1 1 0 0 0 34 | 0 为 什 么 每 个 人 都 长 着 肚 脐 0 0 0 0 0 0 0 0 0 0 0 为 什 么 每 个 人 的 声 音 都 不 一 样 0 0 0 0 0 0 0 1 1 0 0 0 0 35 | 1 伟 大 的 遗 迹 神 龛 诗 — — 玛 哈 达 寺 0 0 0 0 0 1 1 0 0 0 0 1 1 0 为 什 么 说 玛 哈 达 寺 是 伟 大 的 遗 迹 神 龛 诗 0 0 0 0 0 1 1 0 0 0 0 0 1 1 1 1 0 36 | 0 海 洋 变 沙 丘 的 神 话 1 1 0 1 1 0 0 0 海 洋 与 沙 丘 的 传 说 1 1 0 1 1 0 0 0 37 | 1 电 子 污 染 1 1 0 0 电 子 污 染 是 什 么 1 1 1 1 0 0 0 38 | 1 音 乐 疗 法 0 0 0 0 音 乐 疗 法 是 什 么 1 1 1 1 0 0 0 39 | 0 新 型 太 阳 能 电 池 1 1 1 1 1 0 0 太 阳 能 飞 艇 1 1 1 0 0 40 | 1 油 煎 食 物 为 什 么 不 容 易 消 化 1 1 1 1 0 0 0 0 0 0 0 0 油 煎 食 物 不 容 易 消 化 是 什 么 原 因 1 1 1 1 0 0 0 1 1 0 0 0 0 0 41 | 1 光 绪 为 何 叫 慈 禧 " 亲 爸 爸 " 1 1 0 0 0 1 1 0 0 1 1 0 光 绪 叫 慈 禧 " 亲 爸 爸 " 是 为 啥 1 1 0 1 1 0 0 1 1 0 0 0 0 42 | 1 礼 仪 知 识 ： 满 族 礼 仪 1 1 0 0 0 1 1 0 0 介 绍 一 下 礼 仪 知 识 ： 满 族 礼 仪 0 0 0 0 1 1 0 0 0 1 1 0 0 43 | 1 ss ##43 ##3 之 谜 0 0 0 0 0 ss ##43 ##3 为 什 么 成 为 一 个 谜 0 0 0 0 0 0 0 0 0 0 0 44 | 0 餐 桌 上 的 游 戏 1 1 1 0 0 0 宝 宝 餐 桌 上 的 保 健 学 问 1 1 1 1 1 0 0 0 0 0 45 | 0 帮 我 介 绍 下 耶 路 撒 冷 0 0 0 0 0 0 0 0 0 帮 我 介 绍 下 红 花 0 0 0 0 0 0 0 46 | 0 把 握 戏 剧 冲 突 0 0 1 1 0 0 理 解 戏 剧 语 言 0 0 1 1 0 0 47 | 1 老 抽 酱 油 和 生 抽 酱 油 的 区 别 1 1 1 1 0 0 0 1 1 0 0 0 老 抽 酱 油 和 生 抽 酱 油 有 什 么 不 同 1 1 1 1 0 0 0 1 1 0 0 0 0 0 48 | 1 北 洋 军 阀 混 战 之 二 第 一 次 直 奉 战 争 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 解 释 一 下 北 洋 军 阀 混 战 之 二 第 一 次 直 奉 战 争 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 49 | 0 脑 袋 〃 坚 硬 的 穿 甲 弹 1 1 0 0 0 0 0 0 0 坚 硬 的 金 刚 石 0 0 0 0 0 0 50 | 0 动 物 挂 图 与 课 本 1 1 0 0 0 0 0 植 物 挂 图 与 课 本 1 1 0 0 0 0 0 51 | 1 为 什 么 海 象 的 皮 肤 会 变 红 0 0 0 1 1 0 1 1 0 0 0 海 象 的 皮 肤 会 变 红 的 原 因 是 什 么 1 1 0 1 1 0 0 0 0 1 1 0 0 0 52 | 0 烤 箱 用 玻 璃 盘 和 铅 晶 质 玻 璃 1 1 0 1 1 0 0 1 1 1 0 0 玻 璃 奶 瓶 消 毒 和 清 洗 1 1 1 1 1 1 0 0 0 53 | 0 为 什 么 金 星 上 的 温 度 特 别 高 0 0 0 1 1 0 0 1 1 0 0 0 为 什 么 城 市 里 的 温 度 比 近 郊 高 0 0 0 1 1 0 0 1 1 0 1 1 0 54 | 0 向 鲨 鱼 学 习 设 计 飞 机 0 1 1 0 0 1 1 0 0 计 算 机 设 计 飞 机 1 1 1 1 1 0 0 55 | 1 为 什 么 鲤 鱼 和 泥 鳅 长 胡 子 0 0 0 1 1 0 1 1 0 0 0 鲤 鱼 和 泥 鳅 长 胡 子 的 原 因 是 什 么 1 1 0 1 1 1 1 1 0 0 0 0 0 0 56 | 1 黄 巾 起 义 0 0 0 0 介 绍 下 黄 巾 起 义 0 0 0 0 0 0 0 57 | 0 纪 实 风 格 1 1 0 0 电 视 风 格 1 1 0 0 58 | 1 奇 特 的 二 月 1 1 0 0 0 为 什 么 二 月 那 么 奇 特 0 0 0 0 0 0 0 0 0 59 | 0 刘 秀 统 一 全 国 的 战 争 1 1 1 1 0 0 0 0 0 隋 灭 陈 统 一 战 争 1 1 0 0 0 0 0 60 | 1 我 想 了 解 下 张 自 忠 0 0 0 0 0 0 0 0 谁 是 张 自 忠 0 0 0 0 0 61 | 0 取 得 上 帝 的 信 任 0 0 1 1 0 0 0 取 得 孩 子 的 信 任 0 0 1 1 0 0 0 62 | 0 人 人 爱 吃 的 苹 果 1 1 1 0 0 0 0 你 爱 吃 苹 果 么 0 0 0 1 1 0 63 | 1 中 国 名 岛 4 . 大 笔 架 山 1 1 1 1 0 0 0 0 0 0 中 国 名 岛 大 笔 架 山 在 哪 1 1 1 1 0 1 1 1 0 0 64 | 0 乳 粉 的 生 产 1 1 0 0 0 冲 调 婴 幼 儿 配 方 乳 粉 要 用 温 水 0 0 1 1 1 0 0 1 1 0 0 0 0 65 | 1 印 早 了 的 " 冠 军 纪 念 票 " 0 0 0 0 0 1 1 1 1 0 0 印 早 了 的 " 冠 军 纪 念 票 " 怎 么 办 0 0 0 0 0 1 1 1 1 0 0 0 0 0 66 | 1 为 什 么 早 晨 醒 来 时 会 有 眼 屎 0 0 0 1 1 1 1 1 0 0 0 0 早 晨 醒 来 时 会 有 眼 屎 的 原 因 是 什 么 1 1 1 1 1 0 0 1 1 0 0 0 0 0 0 67 | 1 我 想 了 解 下 魏 刚 0 0 0 0 1 1 0 谁 是 魏 刚 0 0 0 0 68 | 1 段 攻 羌 之 战 1 1 1 0 0 段 攻 羌 之 战 是 怎 样 的 1 1 1 1 1 0 0 0 0 69 | 0 淘 金 者 的 牛 仔 裤 1 1 1 0 0 0 0 牛 仔 裤 的 发 明 1 1 1 0 0 0 70 | 0 有 土 斯 有 人 ， 万 物 土 中 生 0 1 1 0 0 0 1 1 0 0 0 万 物 生 长 靠 月 亮 1 1 1 1 0 0 0 71 | 0 鱼 真 的 不 睡 觉 吗 0 0 0 0 0 0 0 海 豚 真 的 不 睡 觉 吗 1 1 0 0 0 0 0 0 72 | 1 猴 子 背 香 蕉 1 1 0 0 0 猴 子 背 香 蕉 的 问 题 1 1 0 1 1 0 0 0 73 | 0 制 陶 技 术 的 进 步 1 1 1 1 0 0 0 农 业 技 术 的 进 步 1 1 1 1 0 0 0 74 | 0 排 脏 退 敌 1 1 0 0 毛 泽 东 挥 笔 退 敌 军 1 1 1 0 0 0 0 0 75 | 1 介 绍 一 下 螳 螂 0 0 0 0 0 0 螳 螂 是 怎 样 的 1 1 0 0 0 0 76 | 1 光 绪 为 什 么 叫 慈 禧 " 亲 爸 爸 " 1 1 0 0 0 0 1 1 0 0 1 1 0 光 绪 叫 慈 禧 " 亲 爸 爸 " 是 为 啥 1 1 0 1 1 0 0 1 1 0 0 0 0 77 | 1 复 杂 的 食 物 链 0 0 0 0 0 0 解 释 一 下 复 杂 的 食 物 链 0 0 0 0 0 0 0 0 0 0 78 | 1 " 超 声 " 和 " 超 音 " 0 0 0 0 0 0 1 1 0 " 超 声 " 和 " 超 音 " 有 什 么 区 别 0 0 0 0 0 0 1 1 0 0 0 0 0 0 79 | 1 为 什 么 星 星 有 不 同 的 颜 色 0 0 0 1 1 0 0 0 0 0 0 星 星 有 不 同 的 颜 色 是 为 什 么 1 1 0 0 0 0 1 1 0 0 0 0 80 | 0 我 国 最 长 河 流 0 0 0 0 0 0 我 国 最 短 的 河 流 0 0 1 1 0 0 0 81 | 0 举 世 称 奇 的 帝 王 陵 墓 0 0 1 1 0 0 1 1 0 举 世 称 奇 的 永 乐 大 钟 0 0 1 1 0 1 1 0 0 82 | 1 什 么 是 父 与 子 0 0 0 0 0 0 父 与 子 是 啥 1 1 1 0 0 83 | 0 鲸 和 其 他 哺 乳 动 物 0 0 0 0 1 1 1 0 生 蛋 的 哺 乳 动 物 1 1 0 1 1 1 0 84 | 0 豆 制 品 加 工 1 1 1 0 0 腌 腊 制 品 的 加 工 1 1 1 1 0 0 0 85 | 0 怎 样 养 好 火 鹤 花 0 0 0 0 1 1 0 怎 样 养 好 西 瓜 皮 椒 草 0 0 0 0 1 1 1 0 0 86 | 0 安 居 人 体 的 细 菌 0 0 1 1 0 0 0 植 物 的 细 菌 1 1 0 0 0 87 | 0 有 趣 的 舌 头 0 0 0 0 0 爱 因 斯 坦 的 舌 头 0 1 1 1 0 0 0 88 | 1 海 上 巨 无 霸 1 1 0 0 0 海 上 巨 无 霸 是 什 么 1 1 1 1 1 0 0 0 89 | 0 为 什 么 人 会 衰 老 0 0 0 1 1 0 0 为 什 么 人 会 秃 顶 0 0 0 1 1 0 0 90 | 1 帮 我 介 绍 下 枣 宜 会 战 0 0 0 0 0 0 0 0 0 什 么 是 枣 宜 会 战 0 0 0 0 0 0 0 91 | 1 茅 爷 爷 的 记 忆 力 0 1 1 0 0 0 0 茅 爷 爷 的 记 忆 力 好 吗 0 1 1 0 1 1 1 0 0 92 | 0 为 什 么 鲤 鱼 和 泥 鳅 长 胡 子 0 0 0 1 1 0 1 1 0 0 0 为 什 么 男 人 长 胡 子 ， 女 人 不 长 胡 子 0 0 0 0 0 1 1 1 0 1 1 0 0 0 0 93 | 1 介 绍 一 下 白 毛 女 0 0 0 0 0 0 0 白 毛 女 是 谁 1 1 1 0 0 94 | 0 狂 暴 的 太 阳 风 0 0 0 0 0 0 什 么 是 太 阳 风 0 0 0 0 0 0 95 | 0 青 竹 开 的 " 玩 笑 " 1 1 0 0 0 0 0 0 邮 票 商 的 玩 笑 1 1 0 0 0 0 96 | 1 指 甲 为 什 么 会 不 停 地 生 长 1 1 0 0 0 0 0 0 0 0 0 指 甲 不 停 地 生 长 的 原 因 是 什 么 1 1 0 0 0 1 1 0 1 1 0 0 0 97 | 1 为 什 么 有 的 飞 机 翼 尖 向 上 折 起 0 0 0 0 0 1 1 1 1 0 0 0 0 有 的 飞 机 翼 尖 向 上 折 起 的 原 因 是 什 么 0 0 1 1 1 1 0 0 0 0 0 1 1 0 0 0 98 | 1 鲑 鱼 回 游 之 谜 1 1 0 0 0 0 鲑 鱼 是 怎 样 回 游 1 1 0 0 0 0 0 99 | 1 控 制 你 的 言 语 0 0 0 0 0 0 为 什 么 要 控 制 言 语 0 0 0 0 0 0 0 0 100 | 0 为 什 么 人 睡 着 了 会 做 梦 0 0 0 0 0 0 0 0 0 0 为 什 么 动 物 会 做 梦 0 0 0 1 1 0 0 0 101 | -------------------------------------------------------------------------------- /keyword-bert/extract_features.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Extract pre-computed feature vectors from BERT.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import codecs 22 | import collections 23 | import json 24 | import re 25 | 26 | import modeling 27 | import tokenization 28 | import tensorflow as tf 29 | 30 | flags = tf.flags 31 | 32 | FLAGS = flags.FLAGS 33 | 34 | flags.DEFINE_string("input_file", None, "") 35 | 36 | flags.DEFINE_string("output_file", None, "") 37 | 38 | flags.DEFINE_string("layers", "-1,-2,-3,-4", "") 39 | 40 | flags.DEFINE_string( 41 | "bert_config_file", None, 42 | "The config json file corresponding to the pre-trained BERT model. " 43 | "This specifies the model architecture.") 44 | 45 | flags.DEFINE_integer( 46 | "max_seq_length", 128, 47 | "The maximum total input sequence length after WordPiece tokenization. " 48 | "Sequences longer than this will be truncated, and sequences shorter " 49 | "than this will be padded.") 50 | 51 | flags.DEFINE_string( 52 | "init_checkpoint", None, 53 | "Initial checkpoint (usually from a pre-trained BERT model).") 54 | 55 | flags.DEFINE_string("vocab_file", None, 56 | "The vocabulary file that the BERT model was trained on.") 57 | 58 | flags.DEFINE_bool( 59 | "do_lower_case", True, 60 | "Whether to lower case the input text. Should be True for uncased " 61 | "models and False for cased models.") 62 | 63 | flags.DEFINE_integer("batch_size", 32, "Batch size for predictions.") 64 | 65 | flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.") 66 | 67 | flags.DEFINE_string("master", None, 68 | "If using a TPU, the address of the master.") 69 | 70 | flags.DEFINE_integer( 71 | "num_tpu_cores", 8, 72 | "Only used if `use_tpu` is True. Total number of TPU cores to use.") 73 | 74 | flags.DEFINE_bool( 75 | "use_one_hot_embeddings", False, 76 | "If True, tf.one_hot will be used for embedding lookups, otherwise " 77 | "tf.nn.embedding_lookup will be used. On TPUs, this should be True " 78 | "since it is much faster.") 79 | 80 | 81 | class InputExample(object): 82 | 83 | def __init__(self, unique_id, text_a, text_b): 84 | self.unique_id = unique_id 85 | self.text_a = text_a 86 | self.text_b = text_b 87 | 88 | 89 | class InputFeatures(object): 90 | """A single set of features of data.""" 91 | 92 | def __init__(self, unique_id, tokens, input_ids, input_mask, input_type_ids): 93 | self.unique_id = unique_id 94 | self.tokens = tokens 95 | self.input_ids = input_ids 96 | self.input_mask = input_mask 97 | self.input_type_ids = input_type_ids 98 | 99 | 100 | def input_fn_builder(features, seq_length): 101 | """Creates an `input_fn` closure to be passed to TPUEstimator.""" 102 | 103 | all_unique_ids = [] 104 | all_input_ids = [] 105 | all_input_mask = [] 106 | all_input_type_ids = [] 107 | 108 | for feature in features: 109 | all_unique_ids.append(feature.unique_id) 110 | all_input_ids.append(feature.input_ids) 111 | all_input_mask.append(feature.input_mask) 112 | all_input_type_ids.append(feature.input_type_ids) 113 | 114 | def input_fn(params): 115 | """The actual input function.""" 116 | batch_size = params["batch_size"] 117 | 118 | num_examples = len(features) 119 | 120 | # This is for demo purposes and does NOT scale to large data sets. We do 121 | # not use Dataset.from_generator() because that uses tf.py_func which is 122 | # not TPU compatible. The right way to load data is with TFRecordReader. 123 | d = tf.data.Dataset.from_tensor_slices({ 124 | "unique_ids": 125 | tf.constant(all_unique_ids, shape=[num_examples], dtype=tf.int32), 126 | "input_ids": 127 | tf.constant( 128 | all_input_ids, shape=[num_examples, seq_length], 129 | dtype=tf.int32), 130 | "input_mask": 131 | tf.constant( 132 | all_input_mask, 133 | shape=[num_examples, seq_length], 134 | dtype=tf.int32), 135 | "input_type_ids": 136 | tf.constant( 137 | all_input_type_ids, 138 | shape=[num_examples, seq_length], 139 | dtype=tf.int32), 140 | }) 141 | 142 | d = d.batch(batch_size=batch_size, drop_remainder=False) 143 | return d 144 | 145 | return input_fn 146 | 147 | 148 | def model_fn_builder(bert_config, init_checkpoint, layer_indexes, use_tpu, 149 | use_one_hot_embeddings): 150 | """Returns `model_fn` closure for TPUEstimator.""" 151 | 152 | def model_fn(features, labels, mode, params): # pylint: disable=unused-argument 153 | """The `model_fn` for TPUEstimator.""" 154 | 155 | unique_ids = features["unique_ids"] 156 | input_ids = features["input_ids"] 157 | input_mask = features["input_mask"] 158 | input_type_ids = features["input_type_ids"] 159 | 160 | model = modeling.BertModel( 161 | config=bert_config, 162 | is_training=False, 163 | input_ids=input_ids, 164 | input_mask=input_mask, 165 | token_type_ids=input_type_ids, 166 | use_one_hot_embeddings=use_one_hot_embeddings) 167 | 168 | if mode != tf.estimator.ModeKeys.PREDICT: 169 | raise ValueError("Only PREDICT modes are supported: %s" % (mode)) 170 | 171 | tvars = tf.trainable_variables() 172 | scaffold_fn = None 173 | (assignment_map, 174 | initialized_variable_names) = modeling.get_assignment_map_from_checkpoint( 175 | tvars, init_checkpoint) 176 | if use_tpu: 177 | 178 | def tpu_scaffold(): 179 | tf.train.init_from_checkpoint(init_checkpoint, assignment_map) 180 | return tf.train.Scaffold() 181 | 182 | scaffold_fn = tpu_scaffold 183 | else: 184 | tf.train.init_from_checkpoint(init_checkpoint, assignment_map) 185 | 186 | tf.logging.info("**** Trainable Variables ****") 187 | for var in tvars: 188 | init_string = "" 189 | if var.name in initialized_variable_names: 190 | init_string = ", *INIT_FROM_CKPT*" 191 | tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape, 192 | init_string) 193 | 194 | all_layers = model.get_all_encoder_layers() 195 | 196 | predictions = { 197 | "unique_id": unique_ids, 198 | } 199 | 200 | for (i, layer_index) in enumerate(layer_indexes): 201 | predictions["layer_output_%d" % i] = all_layers[layer_index] 202 | 203 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 204 | mode=mode, predictions=predictions, scaffold_fn=scaffold_fn) 205 | return output_spec 206 | 207 | return model_fn 208 | 209 | 210 | def convert_examples_to_features(examples, seq_length, tokenizer): 211 | """Loads a data file into a list of `InputBatch`s.""" 212 | 213 | features = [] 214 | for (ex_index, example) in enumerate(examples): 215 | tokens_a = tokenizer.tokenize(example.text_a) 216 | 217 | tokens_b = None 218 | if example.text_b: 219 | tokens_b = tokenizer.tokenize(example.text_b) 220 | 221 | if tokens_b: 222 | # Modifies `tokens_a` and `tokens_b` in place so that the total 223 | # length is less than the specified length. 224 | # Account for [CLS], [SEP], [SEP] with "- 3" 225 | _truncate_seq_pair(tokens_a, tokens_b, seq_length - 3) 226 | else: 227 | # Account for [CLS] and [SEP] with "- 2" 228 | if len(tokens_a) > seq_length - 2: 229 | tokens_a = tokens_a[0:(seq_length - 2)] 230 | 231 | # The convention in BERT is: 232 | # (a) For sequence pairs: 233 | # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] 234 | # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 235 | # (b) For single sequences: 236 | # tokens: [CLS] the dog is hairy . [SEP] 237 | # type_ids: 0 0 0 0 0 0 0 238 | # 239 | # Where "type_ids" are used to indicate whether this is the first 240 | # sequence or the second sequence. The embedding vectors for `type=0` and 241 | # `type=1` were learned during pre-training and are added to the wordpiece 242 | # embedding vector (and position vector). This is not *strictly* necessary 243 | # since the [SEP] token unambiguously separates the sequences, but it makes 244 | # it easier for the model to learn the concept of sequences. 245 | # 246 | # For classification tasks, the first vector (corresponding to [CLS]) is 247 | # used as as the "sentence vector". Note that this only makes sense because 248 | # the entire model is fine-tuned. 249 | tokens = [] 250 | input_type_ids = [] 251 | tokens.append("[CLS]") 252 | input_type_ids.append(0) 253 | for token in tokens_a: 254 | tokens.append(token) 255 | input_type_ids.append(0) 256 | tokens.append("[SEP]") 257 | input_type_ids.append(0) 258 | 259 | if tokens_b: 260 | for token in tokens_b: 261 | tokens.append(token) 262 | input_type_ids.append(1) 263 | tokens.append("[SEP]") 264 | input_type_ids.append(1) 265 | 266 | input_ids = tokenizer.convert_tokens_to_ids(tokens) 267 | 268 | # The mask has 1 for real tokens and 0 for padding tokens. Only real 269 | # tokens are attended to. 270 | input_mask = [1] * len(input_ids) 271 | 272 | # Zero-pad up to the sequence length. 273 | while len(input_ids) < seq_length: 274 | input_ids.append(0) 275 | input_mask.append(0) 276 | input_type_ids.append(0) 277 | 278 | assert len(input_ids) == seq_length 279 | assert len(input_mask) == seq_length 280 | assert len(input_type_ids) == seq_length 281 | 282 | if ex_index < 5: 283 | tf.logging.info("*** Example ***") 284 | tf.logging.info("unique_id: %s" % (example.unique_id)) 285 | tf.logging.info("tokens: %s" % " ".join( 286 | [tokenization.printable_text(x) for x in tokens])) 287 | tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) 288 | tf.logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) 289 | tf.logging.info( 290 | "input_type_ids: %s" % " ".join([str(x) for x in input_type_ids])) 291 | 292 | features.append( 293 | InputFeatures( 294 | unique_id=example.unique_id, 295 | tokens=tokens, 296 | input_ids=input_ids, 297 | input_mask=input_mask, 298 | input_type_ids=input_type_ids)) 299 | return features 300 | 301 | 302 | def _truncate_seq_pair(tokens_a, tokens_b, max_length): 303 | """Truncates a sequence pair in place to the maximum length.""" 304 | 305 | # This is a simple heuristic which will always truncate the longer sequence 306 | # one token at a time. This makes more sense than truncating an equal percent 307 | # of tokens from each, since if one sequence is very short then each token 308 | # that's truncated likely contains more information than a longer sequence. 309 | while True: 310 | total_length = len(tokens_a) + len(tokens_b) 311 | if total_length <= max_length: 312 | break 313 | if len(tokens_a) > len(tokens_b): 314 | tokens_a.pop() 315 | else: 316 | tokens_b.pop() 317 | 318 | 319 | def read_examples(input_file): 320 | """Read a list of `InputExample`s from an input file.""" 321 | examples = [] 322 | unique_id = 0 323 | with tf.gfile.GFile(input_file, "r") as reader: 324 | while True: 325 | line = tokenization.convert_to_unicode(reader.readline()) 326 | if not line: 327 | break 328 | line = line.strip() 329 | text_a = None 330 | text_b = None 331 | m = re.match(r"^(.*) \|\|\| (.*)$", line) 332 | if m is None: 333 | text_a = line 334 | else: 335 | text_a = m.group(1) 336 | text_b = m.group(2) 337 | examples.append( 338 | InputExample(unique_id=unique_id, text_a=text_a, text_b=text_b)) 339 | unique_id += 1 340 | return examples 341 | 342 | 343 | def main(_): 344 | tf.logging.set_verbosity(tf.logging.INFO) 345 | 346 | layer_indexes = [int(x) for x in FLAGS.layers.split(",")] 347 | 348 | bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) 349 | 350 | tokenizer = tokenization.FullTokenizer( 351 | vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) 352 | 353 | is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 354 | run_config = tf.contrib.tpu.RunConfig( 355 | master=FLAGS.master, 356 | tpu_config=tf.contrib.tpu.TPUConfig( 357 | num_shards=FLAGS.num_tpu_cores, 358 | per_host_input_for_training=is_per_host)) 359 | 360 | examples = read_examples(FLAGS.input_file) 361 | 362 | features = convert_examples_to_features( 363 | examples=examples, seq_length=FLAGS.max_seq_length, tokenizer=tokenizer) 364 | 365 | unique_id_to_feature = {} 366 | for feature in features: 367 | unique_id_to_feature[feature.unique_id] = feature 368 | 369 | model_fn = model_fn_builder( 370 | bert_config=bert_config, 371 | init_checkpoint=FLAGS.init_checkpoint, 372 | layer_indexes=layer_indexes, 373 | use_tpu=FLAGS.use_tpu, 374 | use_one_hot_embeddings=FLAGS.use_one_hot_embeddings) 375 | 376 | # If TPU is not available, this will fall back to normal Estimator on CPU 377 | # or GPU. 378 | estimator = tf.contrib.tpu.TPUEstimator( 379 | use_tpu=FLAGS.use_tpu, 380 | model_fn=model_fn, 381 | config=run_config, 382 | predict_batch_size=FLAGS.batch_size) 383 | 384 | input_fn = input_fn_builder( 385 | features=features, seq_length=FLAGS.max_seq_length) 386 | 387 | with codecs.getwriter("utf-8")(tf.gfile.Open(FLAGS.output_file, 388 | "w")) as writer: 389 | for result in estimator.predict(input_fn, yield_single_examples=True): 390 | unique_id = int(result["unique_id"]) 391 | feature = unique_id_to_feature[unique_id] 392 | output_json = collections.OrderedDict() 393 | output_json["linex_index"] = unique_id 394 | all_features = [] 395 | for (i, token) in enumerate(feature.tokens): 396 | all_layers = [] 397 | for (j, layer_index) in enumerate(layer_indexes): 398 | layer_output = result["layer_output_%d" % j] 399 | layers = collections.OrderedDict() 400 | layers["index"] = layer_index 401 | layers["values"] = [ 402 | round(float(x), 6) for x in layer_output[i:(i + 1)].flat 403 | ] 404 | all_layers.append(layers) 405 | features = collections.OrderedDict() 406 | features["token"] = token 407 | features["layers"] = all_layers 408 | all_features.append(features) 409 | output_json["features"] = all_features 410 | writer.write(json.dumps(output_json) + "\n") 411 | 412 | 413 | if __name__ == "__main__": 414 | flags.mark_flag_as_required("input_file") 415 | flags.mark_flag_as_required("vocab_file") 416 | flags.mark_flag_as_required("bert_config_file") 417 | flags.mark_flag_as_required("init_checkpoint") 418 | flags.mark_flag_as_required("output_file") 419 | tf.app.run() 420 | -------------------------------------------------------------------------------- /keyword-bert/modeling_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import collections 20 | import json 21 | import random 22 | import re 23 | 24 | import modeling 25 | import six 26 | import tensorflow as tf 27 | 28 | 29 | class BertModelTest(tf.test.TestCase): 30 | 31 | class BertModelTester(object): 32 | 33 | def __init__(self, 34 | parent, 35 | batch_size=13, 36 | seq_length=7, 37 | is_training=True, 38 | use_input_mask=True, 39 | use_token_type_ids=True, 40 | vocab_size=99, 41 | hidden_size=32, 42 | num_hidden_layers=5, 43 | num_attention_heads=4, 44 | intermediate_size=37, 45 | hidden_act="gelu", 46 | hidden_dropout_prob=0.1, 47 | attention_probs_dropout_prob=0.1, 48 | max_position_embeddings=512, 49 | type_vocab_size=16, 50 | initializer_range=0.02, 51 | scope=None): 52 | self.parent = parent 53 | self.batch_size = batch_size 54 | self.seq_length = seq_length 55 | self.is_training = is_training 56 | self.use_input_mask = use_input_mask 57 | self.use_token_type_ids = use_token_type_ids 58 | self.vocab_size = vocab_size 59 | self.hidden_size = hidden_size 60 | self.num_hidden_layers = num_hidden_layers 61 | self.num_attention_heads = num_attention_heads 62 | self.intermediate_size = intermediate_size 63 | self.hidden_act = hidden_act 64 | self.hidden_dropout_prob = hidden_dropout_prob 65 | self.attention_probs_dropout_prob = attention_probs_dropout_prob 66 | self.max_position_embeddings = max_position_embeddings 67 | self.type_vocab_size = type_vocab_size 68 | self.initializer_range = initializer_range 69 | self.scope = scope 70 | 71 | def create_model(self): 72 | input_ids = BertModelTest.ids_tensor([self.batch_size, self.seq_length], 73 | self.vocab_size) 74 | 75 | input_mask = None 76 | if self.use_input_mask: 77 | input_mask = BertModelTest.ids_tensor( 78 | [self.batch_size, self.seq_length], vocab_size=2) 79 | 80 | token_type_ids = None 81 | if self.use_token_type_ids: 82 | token_type_ids = BertModelTest.ids_tensor( 83 | [self.batch_size, self.seq_length], self.type_vocab_size) 84 | 85 | config = modeling.BertConfig( 86 | vocab_size=self.vocab_size, 87 | hidden_size=self.hidden_size, 88 | num_hidden_layers=self.num_hidden_layers, 89 | num_attention_heads=self.num_attention_heads, 90 | intermediate_size=self.intermediate_size, 91 | hidden_act=self.hidden_act, 92 | hidden_dropout_prob=self.hidden_dropout_prob, 93 | attention_probs_dropout_prob=self.attention_probs_dropout_prob, 94 | max_position_embeddings=self.max_position_embeddings, 95 | type_vocab_size=self.type_vocab_size, 96 | initializer_range=self.initializer_range) 97 | 98 | model = modeling.BertModel( 99 | config=config, 100 | is_training=self.is_training, 101 | input_ids=input_ids, 102 | input_mask=input_mask, 103 | token_type_ids=token_type_ids, 104 | scope=self.scope) 105 | 106 | outputs = { 107 | "embedding_output": model.get_embedding_output(), 108 | "sequence_output": model.get_sequence_output(), 109 | "pooled_output": model.get_pooled_output(), 110 | "all_encoder_layers": model.get_all_encoder_layers(), 111 | } 112 | return outputs 113 | 114 | def check_output(self, result): 115 | self.parent.assertAllEqual( 116 | result["embedding_output"].shape, 117 | [self.batch_size, self.seq_length, self.hidden_size]) 118 | 119 | self.parent.assertAllEqual( 120 | result["sequence_output"].shape, 121 | [self.batch_size, self.seq_length, self.hidden_size]) 122 | 123 | self.parent.assertAllEqual(result["pooled_output"].shape, 124 | [self.batch_size, self.hidden_size]) 125 | 126 | def test_default(self): 127 | self.run_tester(BertModelTest.BertModelTester(self)) 128 | 129 | def test_config_to_json_string(self): 130 | config = modeling.BertConfig(vocab_size=99, hidden_size=37) 131 | obj = json.loads(config.to_json_string()) 132 | self.assertEqual(obj["vocab_size"], 99) 133 | self.assertEqual(obj["hidden_size"], 37) 134 | 135 | def run_tester(self, tester): 136 | with self.test_session() as sess: 137 | ops = tester.create_model() 138 | init_op = tf.group(tf.global_variables_initializer(), 139 | tf.local_variables_initializer()) 140 | sess.run(init_op) 141 | output_result = sess.run(ops) 142 | tester.check_output(output_result) 143 | 144 | self.assert_all_tensors_reachable(sess, [init_op, ops]) 145 | 146 | @classmethod 147 | def ids_tensor(cls, shape, vocab_size, rng=None, name=None): 148 | """Creates a random int32 tensor of the shape within the vocab size.""" 149 | if rng is None: 150 | rng = random.Random() 151 | 152 | total_dims = 1 153 | for dim in shape: 154 | total_dims *= dim 155 | 156 | values = [] 157 | for _ in range(total_dims): 158 | values.append(rng.randint(0, vocab_size - 1)) 159 | 160 | return tf.constant(value=values, dtype=tf.int32, shape=shape, name=name) 161 | 162 | def assert_all_tensors_reachable(self, sess, outputs): 163 | """Checks that all the tensors in the graph are reachable from outputs.""" 164 | graph = sess.graph 165 | 166 | ignore_strings = [ 167 | "^.*/assert_less_equal/.*$", 168 | "^.*/dilation_rate$", 169 | "^.*/Tensordot/concat$", 170 | "^.*/Tensordot/concat/axis$", 171 | "^testing/.*$", 172 | ] 173 | 174 | ignore_regexes = [re.compile(x) for x in ignore_strings] 175 | 176 | unreachable = self.get_unreachable_ops(graph, outputs) 177 | filtered_unreachable = [] 178 | for x in unreachable: 179 | do_ignore = False 180 | for r in ignore_regexes: 181 | m = r.match(x.name) 182 | if m is not None: 183 | do_ignore = True 184 | if do_ignore: 185 | continue 186 | filtered_unreachable.append(x) 187 | unreachable = filtered_unreachable 188 | 189 | self.assertEqual( 190 | len(unreachable), 0, "The following ops are unreachable: %s" % 191 | (" ".join([x.name for x in unreachable]))) 192 | 193 | @classmethod 194 | def get_unreachable_ops(cls, graph, outputs): 195 | """Finds all of the tensors in graph that are unreachable from outputs.""" 196 | outputs = cls.flatten_recursive(outputs) 197 | output_to_op = collections.defaultdict(list) 198 | op_to_all = collections.defaultdict(list) 199 | assign_out_to_in = collections.defaultdict(list) 200 | 201 | for op in graph.get_operations(): 202 | for x in op.inputs: 203 | op_to_all[op.name].append(x.name) 204 | for y in op.outputs: 205 | output_to_op[y.name].append(op.name) 206 | op_to_all[op.name].append(y.name) 207 | if str(op.type) == "Assign": 208 | for y in op.outputs: 209 | for x in op.inputs: 210 | assign_out_to_in[y.name].append(x.name) 211 | 212 | assign_groups = collections.defaultdict(list) 213 | for out_name in assign_out_to_in.keys(): 214 | name_group = assign_out_to_in[out_name] 215 | for n1 in name_group: 216 | assign_groups[n1].append(out_name) 217 | for n2 in name_group: 218 | if n1 != n2: 219 | assign_groups[n1].append(n2) 220 | 221 | seen_tensors = {} 222 | stack = [x.name for x in outputs] 223 | while stack: 224 | name = stack.pop() 225 | if name in seen_tensors: 226 | continue 227 | seen_tensors[name] = True 228 | 229 | if name in output_to_op: 230 | for op_name in output_to_op[name]: 231 | if op_name in op_to_all: 232 | for input_name in op_to_all[op_name]: 233 | if input_name not in stack: 234 | stack.append(input_name) 235 | 236 | expanded_names = [] 237 | if name in assign_groups: 238 | for assign_name in assign_groups[name]: 239 | expanded_names.append(assign_name) 240 | 241 | for expanded_name in expanded_names: 242 | if expanded_name not in stack: 243 | stack.append(expanded_name) 244 | 245 | unreachable_ops = [] 246 | for op in graph.get_operations(): 247 | is_unreachable = False 248 | all_names = [x.name for x in op.inputs] + [x.name for x in op.outputs] 249 | for name in all_names: 250 | if name not in seen_tensors: 251 | is_unreachable = True 252 | if is_unreachable: 253 | unreachable_ops.append(op) 254 | return unreachable_ops 255 | 256 | @classmethod 257 | def flatten_recursive(cls, item): 258 | """Flattens (potentially nested) a tuple/dictionary/list to a list.""" 259 | output = [] 260 | if isinstance(item, list): 261 | output.extend(item) 262 | elif isinstance(item, tuple): 263 | output.extend(list(item)) 264 | elif isinstance(item, dict): 265 | for (_, v) in six.iteritems(item): 266 | output.append(v) 267 | else: 268 | return [item] 269 | 270 | flat_output = [] 271 | for x in output: 272 | flat_output.extend(cls.flatten_recursive(x)) 273 | return flat_output 274 | 275 | 276 | if __name__ == "__main__": 277 | tf.test.main() 278 | -------------------------------------------------------------------------------- /keyword-bert/optimization.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Functions and classes related to optimization (weight updates).""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import re 22 | import tensorflow as tf 23 | 24 | 25 | def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, use_tpu): 26 | """Creates an optimizer training op.""" 27 | global_step = tf.train.get_or_create_global_step() 28 | 29 | learning_rate = tf.constant(value=init_lr, shape=[], dtype=tf.float32) 30 | 31 | # Implements linear decay of the learning rate. 32 | learning_rate = tf.train.polynomial_decay( 33 | learning_rate, 34 | global_step, 35 | num_train_steps, 36 | end_learning_rate=0.0, 37 | power=1.0, 38 | cycle=False) 39 | 40 | # Implements linear warmup. I.e., if global_step < num_warmup_steps, the 41 | # learning rate will be `global_step/num_warmup_steps * init_lr`. 42 | if num_warmup_steps: 43 | global_steps_int = tf.cast(global_step, tf.int32) 44 | warmup_steps_int = tf.constant(num_warmup_steps, dtype=tf.int32) 45 | 46 | global_steps_float = tf.cast(global_steps_int, tf.float32) 47 | warmup_steps_float = tf.cast(warmup_steps_int, tf.float32) 48 | 49 | warmup_percent_done = global_steps_float / warmup_steps_float 50 | warmup_learning_rate = init_lr * warmup_percent_done 51 | 52 | is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32) 53 | learning_rate = ( 54 | (1.0 - is_warmup) * learning_rate + is_warmup * warmup_learning_rate) 55 | 56 | # It is recommended that you use this optimizer for fine tuning, since this 57 | # is how the model was trained (note that the Adam m/v variables are NOT 58 | # loaded from init_checkpoint.) 59 | optimizer = AdamWeightDecayOptimizer( 60 | learning_rate=learning_rate, 61 | weight_decay_rate=0.01, 62 | beta_1=0.9, 63 | beta_2=0.999, 64 | epsilon=1e-6, 65 | exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) 66 | 67 | if use_tpu: 68 | optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) 69 | 70 | tvars = tf.trainable_variables() 71 | grads = tf.gradients(loss, tvars) 72 | 73 | # This is how the model was pre-trained. 74 | (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0) 75 | 76 | train_op = optimizer.apply_gradients( 77 | zip(grads, tvars), global_step=global_step) 78 | 79 | # Normally the global step update is done inside of `apply_gradients`. 80 | # However, `AdamWeightDecayOptimizer` doesn't do this. But if you use 81 | # a different optimizer, you should probably take this line out. 82 | new_global_step = global_step + 1 83 | train_op = tf.group(train_op, [global_step.assign(new_global_step)]) 84 | return train_op 85 | 86 | 87 | class AdamWeightDecayOptimizer(tf.train.Optimizer): 88 | """A basic Adam optimizer that includes "correct" L2 weight decay.""" 89 | 90 | def __init__(self, 91 | learning_rate, 92 | weight_decay_rate=0.0, 93 | beta_1=0.9, 94 | beta_2=0.999, 95 | epsilon=1e-6, 96 | exclude_from_weight_decay=None, 97 | name="AdamWeightDecayOptimizer"): 98 | """Constructs a AdamWeightDecayOptimizer.""" 99 | super(AdamWeightDecayOptimizer, self).__init__(False, name) 100 | 101 | self.learning_rate = learning_rate 102 | self.weight_decay_rate = weight_decay_rate 103 | self.beta_1 = beta_1 104 | self.beta_2 = beta_2 105 | self.epsilon = epsilon 106 | self.exclude_from_weight_decay = exclude_from_weight_decay 107 | 108 | def apply_gradients(self, grads_and_vars, global_step=None, name=None): 109 | """See base class.""" 110 | assignments = [] 111 | for (grad, param) in grads_and_vars: 112 | if grad is None or param is None: 113 | continue 114 | 115 | param_name = self._get_variable_name(param.name) 116 | 117 | m = tf.get_variable( 118 | name=param_name + "/adam_m", 119 | shape=param.shape.as_list(), 120 | dtype=tf.float32, 121 | trainable=False, 122 | initializer=tf.zeros_initializer()) 123 | v = tf.get_variable( 124 | name=param_name + "/adam_v", 125 | shape=param.shape.as_list(), 126 | dtype=tf.float32, 127 | trainable=False, 128 | initializer=tf.zeros_initializer()) 129 | 130 | # Standard Adam update. 131 | next_m = ( 132 | tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad)) 133 | next_v = ( 134 | tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2, 135 | tf.square(grad))) 136 | 137 | update = next_m / (tf.sqrt(next_v) + self.epsilon) 138 | 139 | # Just adding the square of the weights to the loss function is *not* 140 | # the correct way of using L2 regularization/weight decay with Adam, 141 | # since that will interact with the m and v parameters in strange ways. 142 | # 143 | # Instead we want ot decay the weights in a manner that doesn't interact 144 | # with the m/v parameters. This is equivalent to adding the square 145 | # of the weights to the loss with plain (non-momentum) SGD. 146 | if self._do_use_weight_decay(param_name): 147 | update += self.weight_decay_rate * param 148 | 149 | update_with_lr = self.learning_rate * update 150 | 151 | next_param = param - update_with_lr 152 | 153 | assignments.extend( 154 | [param.assign(next_param), 155 | m.assign(next_m), 156 | v.assign(next_v)]) 157 | return tf.group(*assignments, name=name) 158 | 159 | def _do_use_weight_decay(self, param_name): 160 | """Whether to use L2 weight decay for `param_name`.""" 161 | if not self.weight_decay_rate: 162 | return False 163 | if self.exclude_from_weight_decay: 164 | for r in self.exclude_from_weight_decay: 165 | if re.search(r, param_name) is not None: 166 | return False 167 | return True 168 | 169 | def _get_variable_name(self, param_name): 170 | """Get the variable name from the tensor name.""" 171 | m = re.match("^(.*):\\d+$", param_name) 172 | if m is not None: 173 | param_name = m.group(1) 174 | return param_name 175 | -------------------------------------------------------------------------------- /keyword-bert/optimization_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import optimization 20 | import tensorflow as tf 21 | 22 | 23 | class OptimizationTest(tf.test.TestCase): 24 | 25 | def test_adam(self): 26 | with self.test_session() as sess: 27 | w = tf.get_variable( 28 | "w", 29 | shape=[3], 30 | initializer=tf.constant_initializer([0.1, -0.2, -0.1])) 31 | x = tf.constant([0.4, 0.2, -0.5]) 32 | loss = tf.reduce_mean(tf.square(x - w)) 33 | tvars = tf.trainable_variables() 34 | grads = tf.gradients(loss, tvars) 35 | global_step = tf.train.get_or_create_global_step() 36 | optimizer = optimization.AdamWeightDecayOptimizer(learning_rate=0.2) 37 | train_op = optimizer.apply_gradients(zip(grads, tvars), global_step) 38 | init_op = tf.group(tf.global_variables_initializer(), 39 | tf.local_variables_initializer()) 40 | sess.run(init_op) 41 | for _ in range(100): 42 | sess.run(train_op) 43 | w_np = sess.run(w) 44 | self.assertAllClose(w_np.flat, [0.4, 0.2, -0.5], rtol=1e-2, atol=1e-2) 45 | 46 | 47 | if __name__ == "__main__": 48 | tf.test.main() 49 | -------------------------------------------------------------------------------- /keyword-bert/pre_tokenize.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | """ 3 | Wilson Tam: This module tokenize questions and writes to output 4 | """ 5 | 6 | import sys 7 | #reload(sys) 8 | #sys.setdefaultencoding('utf8') 9 | import json 10 | 11 | import tensorflow as tf 12 | import codecs 13 | import gzip 14 | import tokenization 15 | 16 | flags = tf.flags 17 | 18 | FLAGS = flags.FLAGS 19 | 20 | flags.DEFINE_string("input_file", None, "Plain Input file (or comma-separated list of files).") 21 | flags.DEFINE_string("output_file", None, "Output file.") 22 | flags.DEFINE_string("vocab_file", "pre_trained/vocab.txt", "Output file in Squad format.") 23 | flags.DEFINE_bool( 24 | "do_lower_case", True, 25 | "Whether to lower case the input text. Should be True for uncased " 26 | "models and False for cased models.") 27 | 28 | def main(_): 29 | tf.logging.set_verbosity(tf.logging.INFO) 30 | 31 | tokenizer = tokenization.FullTokenizer(FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) 32 | 33 | input_files = [] 34 | for input_pattern in FLAGS.input_file.split(","): 35 | input_files.extend(tf.gfile.Glob(input_pattern)) 36 | 37 | tf.logging.info("*** Reading from input files ***") 38 | for input_file in input_files: 39 | tf.logging.info(" %s", input_file) 40 | 41 | with codecs.open(FLAGS.output_file, "w", encoding="utf-8") as output_json: 42 | for input_file in input_files: 43 | if input_file.endswith("gz"): 44 | reader = gzip.open(input_file, 'rb') 45 | else: 46 | reader = tf.gfile.GFile(input_file, "r") 47 | 48 | jj = 0 49 | while True: 50 | line = tokenization.convert_to_unicode(reader.readline()) 51 | if not line: 52 | break 53 | line = line.strip() 54 | line=line.split('\t') 55 | label=line[0] 56 | q=line[1] 57 | a=line[2] 58 | q_=" ".join(tokenizer.tokenize(q)) 59 | a_=" ".join(tokenizer.tokenize(a)) 60 | output_json.write(label+'\t'+ q_ +'\t'+ a_+'\n') 61 | 62 | 63 | ''' 64 | #sample = json.loads(line) 65 | 66 | tf.logging.info("line {}".format(jj)) 67 | if line!='': 68 | question = " ".join(tokenizer.tokenize(line)) 69 | #output_json.write(question.encode('utf-8')) 70 | output_json.write(question) 71 | output_json.write("\t" + " ".join([str(x) for x in tokenizer.convert_tokens_to_ids(tokenizer.tokenize(line))])) 72 | #output_json.write(question) 73 | output_json.write("\n") 74 | ''' 75 | ''' 76 | if sample['question']: 77 | question = " ".join(tokenizer.tokenize(sample['question'])) 78 | #output_json.write(question.encode('utf-8')) 79 | output_json.write(question) 80 | output_json.write("\t" + " ".join([str(x) for x in tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sample['question']))])) 81 | #print(tokenization.printable_text(question)) 82 | #output_json.write(tokenization.printable_text(question)) 83 | #output_json.write(question) 84 | output_json.write("\n") 85 | ''' 86 | jj += 1 87 | 88 | if __name__ == '__main__': 89 | flags.mark_flag_as_required("input_file") 90 | flags.mark_flag_as_required("output_file") 91 | tf.app.run() 92 | -------------------------------------------------------------------------------- /keyword-bert/pre_trained/bert_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention_probs_dropout_prob": 0.1, 3 | "directionality": "bidi", 4 | "hidden_act": "gelu", 5 | "hidden_dropout_prob": 0.1, 6 | "hidden_size": 768, 7 | "initializer_range": 0.02, 8 | "intermediate_size": 3072, 9 | "max_position_embeddings": 512, 10 | "num_attention_heads": 12, 11 | "num_hidden_layers": 12, 12 | "pooler_fc_size": 768, 13 | "pooler_num_attention_heads": 12, 14 | "pooler_num_fc_layers": 3, 15 | "pooler_size_per_head": 128, 16 | "pooler_type": "first_token_transform", 17 | "type_vocab_size": 2, 18 | "vocab_size": 21128 19 | } 20 | -------------------------------------------------------------------------------- /keyword-bert/pre_trained/bert_config_6_layer.json: -------------------------------------------------------------------------------- 1 | { 2 | "attention_probs_dropout_prob": 0.1, 3 | "directionality": "bidi", 4 | "hidden_act": "gelu", 5 | "hidden_dropout_prob": 0.1, 6 | "hidden_size": 768, 7 | "initializer_range": 0.02, 8 | "intermediate_size": 3072, 9 | "max_position_embeddings": 512, 10 | "num_attention_heads": 12, 11 | "num_hidden_layers": 4, 12 | "pooler_fc_size": 768, 13 | "pooler_num_attention_heads": 12, 14 | "pooler_num_fc_layers": 3, 15 | "pooler_size_per_head": 128, 16 | "pooler_type": "first_token_transform", 17 | "type_vocab_size": 2, 18 | "vocab_size": 21128 19 | } 20 | -------------------------------------------------------------------------------- /keyword-bert/readme.md: -------------------------------------------------------------------------------- 1 | # experiments descriptions 2 | ``` 3 | network structure of Keyword-BERT: 4 | 5 | fusion layer & output layer balabala... 6 | --------------------- 7 | h(CLS)<= transformer | keyword attention transformer => h_keyword(AB) 8 | layer n | layer n 9 | --------------------- 10 | transformer layer n-1 11 | --------------------- 12 | ...... 13 | --------------------- 14 | transformer layer 1 15 | --------------------- 16 | embedding layer 17 | ``` 18 | 19 | # change list (compared with vanilla BERT) 20 | `model.py`: 21 | * Add a keyword attention layer. 22 | * Add a flexible layer-num configuration 23 | 24 | `run_classifier.py`: 25 | * Add keyword mask (parsing from training data). 26 | * Add a fusion layer. 27 | * Add hooks to print training loss and 28 | iteration steps. 29 | 30 | # data preparation 31 | Add `keyword mask` to vanilla BERT training data. (plz refer to `data/convert_to_bert_keyword.py` and `pre_tokenize.py`) 32 | 33 | For data privacy, we only publish a small sample of our real data. 34 | 35 | # training procedure 36 | 1. Prepare you own training and test data 37 | 2. Copy bert pre-trained Chinese large model to `pre_trained` 38 | 3. Modify model hyper-parameters in `pre_trained/bert_config.json`. 39 | Modify running parameter in `run.sh` (if needed). 40 | 4. Execute `run.sh` 41 | 42 | -------------------------------------------------------------------------------- /keyword-bert/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow >= 1.11.0 # CPU Version of TensorFlow. 2 | # tensorflow-gpu >= 1.11.0 # GPU version of TensorFlow. 3 | -------------------------------------------------------------------------------- /keyword-bert/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # -*- coding: utf-8 -*- 3 | 4 | PRE_TRAINED_DIR="pre_trained" 5 | DATA_DIR="data" 6 | OUTOUT_DIR="outputs" 7 | GPU_ID="4" 8 | 9 | time_stamp=$(date +"%G-%m-%d - %H-%M") 10 | echo "#########################################" 11 | echo ">>>>> Bert running log @$time_stamp <<<<<" 12 | echo "#########################################" 13 | 14 | python run_classifier.py \ 15 | --task_name=Baike \ 16 | --do_train=False \ 17 | --do_eval=False \ 18 | --do_predict=True \ 19 | --do_train_and_eval=False \ 20 | --save_checkpoints_steps=100 \ 21 | --keep_checkpoint_max=3 \ 22 | --num_train_epochs=50 \ 23 | --max_seq_length=128 \ 24 | --train_batch_size=1 \ 25 | --data_dir=$DATA_DIR \ 26 | --vocab_file=$PRE_TRAINED_DIR/vocab.txt \ 27 | --bert_config_file=$PRE_TRAINED_DIR/bert_config_6_layer.json \ 28 | --init_checkpoint=$PRE_TRAINED_DIR/bert_model.ckpt \ 29 | --learning_rate=2e-5 \ 30 | --output_dir=$OUTOUT_DIR \ 31 | --gpu_device_id=$GPU_ID 32 | 33 | -------------------------------------------------------------------------------- /keyword-bert/run_classifier.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """BERT finetuning runner.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import collections 22 | import csv 23 | import os 24 | import copy 25 | import modeling 26 | import optimization 27 | import tokenization 28 | import tensorflow as tf 29 | 30 | flags = tf.flags 31 | 32 | FLAGS = flags.FLAGS 33 | 34 | ## Required parameters 35 | flags.DEFINE_string( 36 | "data_dir", None, 37 | "The input data dir. Should contain the .tsv files (or other data files) " 38 | "for the task.") 39 | 40 | flags.DEFINE_string( 41 | "bert_config_file", None, 42 | "The config json file corresponding to the pre-trained BERT model. " 43 | "This specifies the model architecture.") 44 | 45 | flags.DEFINE_string("task_name", None, "The name of the task to train.") 46 | 47 | flags.DEFINE_string("vocab_file", None, 48 | "The vocabulary file that the BERT model was trained on.") 49 | 50 | flags.DEFINE_string( 51 | "output_dir", None, 52 | "The output directory where the model checkpoints will be written.") 53 | 54 | ## Other parameters 55 | 56 | flags.DEFINE_string( 57 | "init_checkpoint", None, 58 | "Initial checkpoint (usually from a pre-trained BERT model).") 59 | 60 | flags.DEFINE_bool( 61 | "do_lower_case", True, 62 | "Whether to lower case the input text. Should be True for uncased " 63 | "models and False for cased models.") 64 | 65 | flags.DEFINE_integer( 66 | "max_seq_length", 128, 67 | "The maximum total input sequence length after WordPiece tokenization. " 68 | "Sequences longer than this will be truncated, and sequences shorter " 69 | "than this will be padded.") 70 | 71 | flags.DEFINE_bool("do_train", False, "Whether to run training.") 72 | 73 | flags.DEFINE_bool("do_eval", False, "Whether to run eval on the dev set.") 74 | 75 | flags.DEFINE_bool( 76 | "do_predict", False, 77 | "Whether to run the model in inference mode on the test set.") 78 | 79 | flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.") 80 | 81 | flags.DEFINE_integer("eval_batch_size", 8, "Total batch size for eval.") 82 | 83 | flags.DEFINE_integer("predict_batch_size", 8, "Total batch size for predict.") 84 | 85 | flags.DEFINE_float("learning_rate", 5e-5, "The initial learning rate for Adam.") 86 | 87 | flags.DEFINE_float("num_train_epochs", 3.0, 88 | "Total number of training epochs to perform.") 89 | 90 | flags.DEFINE_float( 91 | "warmup_proportion", 0.1, 92 | "Proportion of training to perform linear learning rate warmup for. " 93 | "E.g., 0.1 = 10% of training.") 94 | 95 | flags.DEFINE_integer("save_checkpoints_steps", 2000, 96 | "How often to save the model checkpoint.") 97 | 98 | flags.DEFINE_integer("iterations_per_loop", 1000, 99 | "How many steps to make in each estimator call.") 100 | 101 | flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.") 102 | 103 | tf.flags.DEFINE_string( 104 | "tpu_name", None, 105 | "The Cloud TPU to use for training. This should be either the name " 106 | "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 " 107 | "url.") 108 | 109 | tf.flags.DEFINE_string( 110 | "tpu_zone", None, 111 | "[Optional] GCE zone where the Cloud TPU is located in. If not " 112 | "specified, we will attempt to automatically detect the GCE project from " 113 | "metadata.") 114 | 115 | tf.flags.DEFINE_string( 116 | "gcp_project", None, 117 | "[Optional] Project name for the Cloud TPU-enabled project. If not " 118 | "specified, we will attempt to automatically detect the GCE project from " 119 | "metadata.") 120 | 121 | tf.flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.") 122 | 123 | flags.DEFINE_integer( 124 | "num_tpu_cores", 8, 125 | "Only used if `use_tpu` is True. Total number of TPU cores to use.") 126 | 127 | flags.DEFINE_string( 128 | "gpu_device_id", "0", 129 | "device id of gpu.") 130 | 131 | flags.DEFINE_integer( 132 | "keep_checkpoint_max", 10, 133 | "keep_checkpoint_max") 134 | 135 | flags.DEFINE_bool("do_train_and_eval", False, "Whether to run eval when training") 136 | flags.DEFINE_string("experiment_name", "", "Name of different experiments") 137 | 138 | class InputExample(object): 139 | """A single training/test example for simple sequence classification.""" 140 | 141 | def __init__(self, guid, text_a, text_b=None, label=None, 142 | kw_mask_a_raw=None, kw_mask_b_raw=None): 143 | """Constructs a InputExample. 144 | 145 | Args: 146 | guid: Unique id for the example. 147 | text_a: string. The untokenized text of the first sequence. For single 148 | sequence tasks, only this sequence must be specified. 149 | text_b: (Optional) string. The untokenized text of the second sequence. 150 | Only must be specified for sequence pair tasks. 151 | kw_mask_a_raw: list, [0, 1, 1] means 2nd and 3rd token in text_a is keyword 152 | kw_mask_b_raw: same as kw_mask_a_raw except on text_b 153 | label: (Optional) string. The label of the example. This should be 154 | specified for train and dev examples, but not for test examples. 155 | """ 156 | self.guid = guid 157 | self.text_a = text_a 158 | self.text_b = text_b 159 | self.kw_mask_a_raw = kw_mask_a_raw 160 | self.kw_mask_b_raw = kw_mask_b_raw 161 | self.label = label 162 | 163 | 164 | class PaddingInputExample(object): 165 | """Fake example so the num input examples is a multiple of the batch size. 166 | 167 | When running eval/predict on the TPU, we need to pad the number of examples 168 | to be a multiple of the batch size, because the TPU requires a fixed batch 169 | size. The alternative is to drop the last batch, which is bad because it means 170 | the entire output data won't be generated. 171 | 172 | We use this class instead of `None` because treating `None` as padding 173 | battches could cause silent errors. 174 | """ 175 | 176 | 177 | class InputFeatures(object): 178 | """A single set of features of data.""" 179 | 180 | def __init__(self, 181 | input_ids, 182 | input_mask, 183 | segment_ids, 184 | label_id, 185 | is_real_example=True, 186 | kw_mask_a=None, 187 | kw_mask_b=None, 188 | real_mask_a=None, 189 | real_mask_b=None): 190 | self.input_ids = input_ids 191 | self.input_mask = input_mask 192 | self.segment_ids = segment_ids 193 | if kw_mask_a and kw_mask_b: 194 | self.kw_mask_a = kw_mask_a 195 | self.kw_mask_b = kw_mask_b 196 | if real_mask_a and real_mask_b: 197 | self.real_mask_a = real_mask_a 198 | self.real_mask_b = real_mask_b 199 | self.label_id = label_id 200 | self.is_real_example = is_real_example 201 | 202 | 203 | class DataProcessor(object): 204 | """Base class for data converters for sequence classification data sets.""" 205 | 206 | def get_train_examples(self, data_dir): 207 | """Gets a collection of `InputExample`s for the train set.""" 208 | raise NotImplementedError() 209 | 210 | def get_dev_examples(self, data_dir): 211 | """Gets a collection of `InputExample`s for the dev set.""" 212 | raise NotImplementedError() 213 | 214 | def get_test_examples(self, data_dir): 215 | """Gets a collection of `InputExample`s for prediction.""" 216 | raise NotImplementedError() 217 | 218 | def get_labels(self): 219 | """Gets the list of labels for this data set.""" 220 | raise NotImplementedError() 221 | 222 | @classmethod 223 | def _read_tsv(cls, input_file, quotechar=None): 224 | """Reads a tab separated value file.""" 225 | with tf.gfile.Open(input_file, "r") as f: 226 | # reader = csv.reader(f, delimiter="\t", quotechar=quotechar) 227 | lines = [] 228 | # for line in reader: 229 | for line in f: 230 | line = line.strip().split('\t') 231 | lines.append(line) 232 | return lines 233 | 234 | 235 | class BaikeProcessor(DataProcessor): 236 | """Process baike data with keyword (q-q/q-a pairs) 237 | 238 | File line format: 239 | `label \t text_a \t keyword_a_mask' \t text_b \t keyword_b_mask` 240 | example: `1 \t 我 爱 你 中 国 \t 0 1 0 1 1 \t 爱 我 中 华 \t 1 0 1 1` 241 | """ 242 | 243 | def get_train_examples(self, data_dir): 244 | """See base class.""" 245 | return self._create_examples( 246 | self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") 247 | 248 | def get_dev_examples(self, data_dir): 249 | """See base class.""" 250 | return self._create_examples( 251 | self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") 252 | 253 | def get_test_examples(self, data_dir): 254 | """See base class.""" 255 | return self._create_examples( 256 | self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") 257 | 258 | def get_labels(self): 259 | """See base class.""" 260 | return ["0", "1"] 261 | 262 | def _create_examples(self, lines, set_type): 263 | """Creates examples for the training and dev sets.""" 264 | examples = [] 265 | for (i, line) in enumerate(lines): 266 | # if i == 0: 267 | # continue 268 | guid = "%s-%s" % (set_type, i) 269 | text_a = tokenization.convert_to_unicode(line[1]) 270 | kw_mask_a_raw = list(map(int, line[2].split(' '))) 271 | text_b = tokenization.convert_to_unicode(line[3]) 272 | kw_mask_b_raw = list(map(int, line[4].split(' '))) 273 | if set_type == "test": 274 | label = "0" 275 | else: 276 | label = tokenization.convert_to_unicode(line[0]) 277 | examples.append( 278 | InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label, 279 | kw_mask_a_raw=kw_mask_a_raw, kw_mask_b_raw=kw_mask_b_raw)) 280 | return examples 281 | 282 | 283 | def convert_single_example(ex_index, example, label_list, max_seq_length, 284 | tokenizer): 285 | """Converts a single `InputExample` into a single `InputFeatures`. 286 | example contains text keywords 287 | """ 288 | 289 | if isinstance(example, PaddingInputExample): 290 | return InputFeatures( 291 | input_ids=[0] * max_seq_length, 292 | input_mask=[0] * max_seq_length, 293 | segment_ids=[0] * max_seq_length, 294 | label_id=0, 295 | kw_mask_a=[0] * max_seq_length, 296 | kw_mask_b=[0] * max_seq_length, 297 | real_mask_a=[0] * max_seq_length, 298 | real_mask_b=[0] * max_seq_length, 299 | is_real_example=False) 300 | 301 | label_map = {} 302 | for (i, label) in enumerate(label_list): 303 | label_map[label] = i 304 | 305 | # assume example.text_a is already tokenized by `pre_tokenize.py` 306 | tokens_a = example.text_a.split(' ') 307 | tokens_b = example.text_b.split(' ') 308 | kw_mask_a_raw = copy.deepcopy(example.kw_mask_a_raw) # `raw` means no padding 309 | kw_mask_b_raw = copy.deepcopy(example.kw_mask_b_raw) 310 | 311 | if tokens_b: 312 | # Modifies `tokens_a` and `tokens_b` in place so that the total 313 | # length is less than the specified length. 314 | # Account for [CLS], [SEP], [SEP] with "- 3" 315 | _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3, 316 | kw_mask_a_raw, kw_mask_b_raw) 317 | else: 318 | # Account for [CLS] and [SEP] with "- 2" 319 | if len(tokens_a) > max_seq_length - 2: 320 | tokens_a = tokens_a[0:(max_seq_length - 2)] 321 | 322 | # The convention in BERT is: 323 | # (a) For sequence pairs: 324 | # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] 325 | # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 326 | # kw_mask_a: 0 0 0 1 1 1 0 0 0 0 0 0 0 0 327 | # kw_mask_b: 0 0 0 0 0 0 0 0 0 0 0 1 0 0 328 | #real_mask_a:0 1 1 1 1 1 1 0 0 0 0 0 0 0 329 | #real_mask_b:0 0 0 0 0 0 0 0 1 1 1 1 1 0 330 | # (b) For single sequences: 331 | # tokens: [CLS] the dog is hairy . [SEP] 332 | # type_ids: 0 0 0 0 0 0 0 333 | # 334 | # Where "type_ids" are used to indicate whether this is the first 335 | # sequence or the second sequence. The embedding vectors for `type=0` and 336 | # `type=1` were learned during pre-training and are added to the wordpiece 337 | # embedding vector (and position vector). This is not *strictly* necessary 338 | # since the [SEP] token unambiguously separates the sequences, but it makes 339 | # it easier for the model to learn the concept of sequences. 340 | # 341 | # For classification tasks, the first vector (corresponding to [CLS]) is 342 | # used as the "sentence vector". Note that this only makes sense because 343 | # the entire model is fine-tuned. 344 | tokens = [] 345 | kw_mask_a = [] 346 | kw_mask_b = [] 347 | real_mask_a = [] 348 | real_mask_b = [] 349 | segment_ids = [] 350 | tokens.append("[CLS]") 351 | segment_ids.append(0) 352 | kw_mask_a.append(0) 353 | kw_mask_b.append(0) 354 | real_mask_a.append(0) 355 | real_mask_b.append(0) 356 | 357 | for idx,token in enumerate(tokens_a): 358 | tokens.append(token) 359 | segment_ids.append(0) 360 | kw_mask_a.append(kw_mask_a_raw[idx]) 361 | real_mask_a.append(1) 362 | kw_mask_b.append(0) 363 | real_mask_b.append(0) 364 | 365 | tokens.append("[SEP]") 366 | segment_ids.append(0) 367 | kw_mask_a.append(0) 368 | kw_mask_b.append(0) 369 | real_mask_a.append(0) 370 | real_mask_b.append(0) 371 | 372 | for idx,token in enumerate(tokens_b): 373 | tokens.append(token) 374 | segment_ids.append(1) 375 | kw_mask_a.append(0) 376 | real_mask_a.append(0) 377 | kw_mask_b.append(kw_mask_b_raw[idx]) 378 | real_mask_b.append(1) 379 | 380 | tokens.append("[SEP]") 381 | segment_ids.append(1) 382 | kw_mask_a.append(0) 383 | kw_mask_b.append(0) 384 | real_mask_a.append(0) 385 | real_mask_b.append(0) 386 | 387 | input_ids = tokenizer.convert_tokens_to_ids(tokens) 388 | 389 | # The mask has 1 for real tokens and 0 for padding tokens. Only real 390 | # tokens are attended to. 391 | input_mask = [1] * len(input_ids) 392 | 393 | # Zero-pad up to the sequence length. 394 | while len(input_ids) < max_seq_length: 395 | input_ids.append(0) 396 | input_mask.append(0) 397 | segment_ids.append(0) 398 | kw_mask_a.append(0) 399 | kw_mask_b.append(0) 400 | real_mask_a.append(0) 401 | real_mask_b.append(0) 402 | 403 | assert len(input_ids) == max_seq_length 404 | assert len(input_mask) == max_seq_length 405 | assert len(segment_ids) == max_seq_length 406 | assert len(kw_mask_a) == max_seq_length 407 | assert len(kw_mask_b) == max_seq_length 408 | assert len(real_mask_a) == max_seq_length 409 | assert len(real_mask_b) == max_seq_length 410 | 411 | label_id = label_map[example.label] 412 | if ex_index < 5: 413 | tf.logging.info("*** Example ***") 414 | tf.logging.info("guid: %s" % (example.guid)) 415 | tf.logging.info("tokens: %s" % " ".join( 416 | [tokenization.printable_text(x) for x in tokens])) 417 | tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) 418 | tf.logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) 419 | tf.logging.info("real_mask_a: %s" % " ".join([str(x) for x in real_mask_a])) 420 | tf.logging.info("kw_mask_a: %s" % " ".join([str(x) for x in kw_mask_a])) 421 | tf.logging.info("real_mask_b: %s" % " ".join([str(x) for x in real_mask_b])) 422 | tf.logging.info("kw_mask_b: %s" % " ".join([str(x) for x in kw_mask_b])) 423 | tf.logging.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) 424 | tf.logging.info("label: %s (id = %d)" % (example.label, label_id)) 425 | 426 | feature = InputFeatures( 427 | input_ids=input_ids, 428 | input_mask=input_mask, 429 | segment_ids=segment_ids, 430 | label_id=label_id, 431 | kw_mask_a=kw_mask_a, 432 | kw_mask_b=kw_mask_b, 433 | real_mask_a=real_mask_a, 434 | real_mask_b=real_mask_b, 435 | is_real_example=True) 436 | return feature 437 | 438 | 439 | def file_based_convert_examples_to_features( 440 | examples, label_list, max_seq_length, tokenizer, output_file): 441 | """Convert a set of `InputExample`s to a TFRecord file.""" 442 | 443 | writer = tf.python_io.TFRecordWriter(output_file) 444 | 445 | for (ex_index, example) in enumerate(examples): 446 | if ex_index % 10000 == 0: 447 | tf.logging.info("Writing example %d of %d" % (ex_index, len(examples))) 448 | 449 | feature = convert_single_example(ex_index, example, label_list, 450 | max_seq_length, tokenizer) 451 | 452 | def create_int_feature(values): 453 | f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) 454 | return f 455 | 456 | features = collections.OrderedDict() 457 | features["input_ids"] = create_int_feature(feature.input_ids) 458 | features["input_mask"] = create_int_feature(feature.input_mask) 459 | features["kw_mask_a"] = create_int_feature(feature.kw_mask_a) 460 | features["kw_mask_b"] = create_int_feature(feature.kw_mask_b) 461 | features["real_mask_a"] = create_int_feature(feature.real_mask_a) 462 | features["real_mask_b"] = create_int_feature(feature.real_mask_b) 463 | features["segment_ids"] = create_int_feature(feature.segment_ids) 464 | features["label_ids"] = create_int_feature([feature.label_id]) 465 | features["is_real_example"] = create_int_feature( 466 | [int(feature.is_real_example)]) 467 | 468 | tf_example = tf.train.Example(features=tf.train.Features(feature=features)) 469 | writer.write(tf_example.SerializeToString()) 470 | writer.close() 471 | 472 | 473 | def file_based_input_fn_builder(input_file, seq_length, is_training, 474 | drop_remainder): 475 | """Creates an `input_fn` closure to be passed to TPUEstimator.""" 476 | 477 | name_to_features = { 478 | "input_ids": tf.FixedLenFeature([seq_length], tf.int64), 479 | "input_mask": tf.FixedLenFeature([seq_length], tf.int64), 480 | "segment_ids": tf.FixedLenFeature([seq_length], tf.int64), 481 | "kw_mask_a": tf.FixedLenFeature([seq_length], tf.int64), 482 | "kw_mask_b": tf.FixedLenFeature([seq_length], tf.int64), 483 | "real_mask_a": tf.FixedLenFeature([seq_length], tf.int64), 484 | "real_mask_b": tf.FixedLenFeature([seq_length], tf.int64), 485 | "label_ids": tf.FixedLenFeature([], tf.int64), 486 | "is_real_example": tf.FixedLenFeature([], tf.int64), 487 | } 488 | 489 | def _decode_record(record, name_to_features): 490 | """Decodes a record to a TensorFlow example.""" 491 | example = tf.parse_single_example(record, name_to_features) 492 | 493 | # tf.Example only supports tf.int64, but the TPU only supports tf.int32. 494 | # So cast all int64 to int32. 495 | for name in list(example.keys()): 496 | t = example[name] 497 | if t.dtype == tf.int64: 498 | t = tf.to_int32(t) 499 | example[name] = t 500 | 501 | return example 502 | 503 | def input_fn(params): 504 | """The actual input function.""" 505 | batch_size = params["batch_size"] 506 | 507 | # For training, we want a lot of parallel reading and shuffling. 508 | # For eval, we want no shuffling and parallel reading doesn't matter. 509 | d = tf.data.TFRecordDataset(input_file) 510 | if is_training: 511 | d = d.repeat() 512 | d = d.shuffle(buffer_size=100) 513 | 514 | d = d.apply( 515 | tf.contrib.data.map_and_batch( 516 | lambda record: _decode_record(record, name_to_features), 517 | batch_size=batch_size, 518 | drop_remainder=drop_remainder)) 519 | 520 | return d 521 | 522 | return input_fn 523 | 524 | 525 | def _truncate_seq_pair(tokens_a, tokens_b, max_length, 526 | kw_mask_a=None, kw_mask_b=None): 527 | """Truncates a sequence pair in place to the maximum length.""" 528 | 529 | # This is a simple heuristic which will always truncate the longer sequence 530 | # one token at a time. This makes more sense than truncating an equal percent 531 | # of tokens from each, since if one sequence is very short then each token 532 | # that's truncated likely contains more information than a longer sequence. 533 | while True: 534 | total_length = len(tokens_a) + len(tokens_b) 535 | if total_length <= max_length: 536 | break 537 | if len(tokens_a) > len(tokens_b): 538 | tokens_a.pop() 539 | if kw_mask_a: 540 | kw_mask_a.pop() 541 | else: 542 | tokens_b.pop() 543 | if kw_mask_b: 544 | kw_mask_b.pop() 545 | 546 | 547 | def create_model(bert_config, is_training, input_ids, input_mask, segment_ids, 548 | labels, num_labels, use_one_hot_embeddings, 549 | kw_mask_a, kw_mask_b, real_mask_a, real_mask_b): 550 | """Creates a classification model.""" 551 | model = modeling.BertModel( 552 | config=bert_config, 553 | is_training=is_training, 554 | input_ids=input_ids, 555 | input_mask=input_mask, 556 | kw_mask_a=kw_mask_a, 557 | kw_mask_b=kw_mask_b, 558 | real_mask_a=real_mask_a, 559 | real_mask_b=real_mask_b, 560 | token_type_ids=segment_ids, 561 | use_one_hot_embeddings=use_one_hot_embeddings) 562 | 563 | # In the demo, we are doing a simple classification task on the entire 564 | # segment. 565 | # 566 | # If you want to use the token-level output, use model.get_sequence_output() 567 | # instead. 568 | cls_output_layer = model.get_pooled_output() 569 | # h(cls) + h(A,B) 570 | if FLAGS.experiment_name == "addhAhB": 571 | ab_output_layer = model.get_ab_pooled_output() 572 | output_layer = tf.concat([cls_output_layer, ab_output_layer], -1) 573 | # h(cls) 574 | elif FLAGS.experiment_name == "no_kw": 575 | output_layer = cls_output_layer 576 | # h(cls) + h_keyword(A,B) 577 | else: 578 | kw_output_layer = model.get_kw_pooled_output() 579 | output_layer = tf.concat([cls_output_layer, kw_output_layer], -1) 580 | 581 | hidden_size = output_layer.shape[-1].value 582 | 583 | output_weights = tf.get_variable( 584 | "output_weights", [num_labels, hidden_size], 585 | initializer=tf.truncated_normal_initializer(stddev=0.02)) 586 | 587 | output_bias = tf.get_variable( 588 | "output_bias", [num_labels], initializer=tf.zeros_initializer()) 589 | 590 | with tf.variable_scope("loss"): 591 | if is_training: 592 | # I.e., 0.1 dropout 593 | output_layer = tf.nn.dropout(output_layer, keep_prob=0.9) 594 | 595 | logits = tf.matmul(output_layer, output_weights, transpose_b=True) 596 | logits = tf.nn.bias_add(logits, output_bias) 597 | probabilities = tf.nn.softmax(logits, axis=-1) 598 | log_probs = tf.nn.log_softmax(logits, axis=-1) 599 | 600 | one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32) 601 | 602 | per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) 603 | loss = tf.reduce_mean(per_example_loss) 604 | 605 | return (loss, per_example_loss, logits, probabilities) 606 | 607 | 608 | def model_fn_builder(bert_config, num_labels, init_checkpoint, learning_rate, 609 | num_train_steps, num_warmup_steps, use_tpu, 610 | use_one_hot_embeddings): 611 | """Returns `model_fn` closure for TPUEstimator.""" 612 | 613 | def model_fn(features, labels, mode, params): # pylint: disable=unused-argument 614 | """The `model_fn` for TPUEstimator.""" 615 | 616 | tf.logging.info("*** Features ***") 617 | for name in sorted(features.keys()): 618 | tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape)) 619 | 620 | input_ids = features["input_ids"] 621 | input_mask = features["input_mask"] 622 | kw_mask_a = features["kw_mask_a"] 623 | kw_mask_b = features["kw_mask_b"] 624 | real_mask_a = features["real_mask_a"] 625 | real_mask_b = features["real_mask_b"] 626 | segment_ids = features["segment_ids"] 627 | label_ids = features["label_ids"] 628 | is_real_example = None 629 | if "is_real_example" in features: 630 | is_real_example = tf.cast(features["is_real_example"], dtype=tf.float32) 631 | else: 632 | is_real_example = tf.ones(tf.shape(label_ids), dtype=tf.float32) 633 | 634 | is_training = (mode == tf.estimator.ModeKeys.TRAIN) 635 | 636 | (total_loss, per_example_loss, logits, probabilities) = create_model( 637 | bert_config, is_training, input_ids, input_mask, segment_ids, label_ids, 638 | num_labels, use_one_hot_embeddings, 639 | kw_mask_a, kw_mask_b, real_mask_a, real_mask_b) 640 | 641 | tvars = tf.trainable_variables() 642 | initialized_variable_names = {} 643 | scaffold_fn = None 644 | if init_checkpoint: 645 | (assignment_map, initialized_variable_names 646 | ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint) 647 | if use_tpu: 648 | 649 | def tpu_scaffold(): 650 | tf.train.init_from_checkpoint(init_checkpoint, assignment_map) 651 | return tf.train.Scaffold() 652 | 653 | scaffold_fn = tpu_scaffold 654 | else: 655 | tf.train.init_from_checkpoint(init_checkpoint, assignment_map) 656 | 657 | tf.logging.info("**** Trainable Variables ****") 658 | for var in tvars: 659 | init_string = "" 660 | if var.name in initialized_variable_names: 661 | init_string = ", *INIT_FROM_CKPT*" 662 | tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape, 663 | init_string) 664 | 665 | output_spec = None 666 | if mode == tf.estimator.ModeKeys.TRAIN: 667 | 668 | train_op = optimization.create_optimizer( 669 | total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu) 670 | 671 | # to print training loss, code from 672 | # https://github.com/google-research/bert/issues/70#issuecomment-436533131 673 | logging_hook = tf.train.LoggingTensorHook({"loss": total_loss}, every_n_iter=200) 674 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 675 | mode=mode, 676 | loss=total_loss, 677 | train_op=train_op, 678 | training_hooks=[logging_hook], 679 | scaffold_fn=scaffold_fn) 680 | elif mode == tf.estimator.ModeKeys.EVAL: 681 | 682 | def metric_fn(per_example_loss, label_ids, logits, is_real_example): 683 | predictions = tf.argmax(logits, axis=-1, output_type=tf.int32) 684 | accuracy = tf.metrics.accuracy( 685 | labels=label_ids, predictions=predictions, weights=is_real_example) 686 | auc = tf.metrics.auc(labels=label_ids, predictions=predictions, 687 | weights=is_real_example) 688 | loss = tf.metrics.mean(values=per_example_loss, weights=is_real_example) 689 | return { 690 | "eval_accuracy": accuracy, 691 | "eval_loss": loss, 692 | } 693 | 694 | eval_metrics = (metric_fn, 695 | [per_example_loss, label_ids, logits, is_real_example]) 696 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 697 | mode=mode, 698 | loss=total_loss, 699 | eval_metrics=eval_metrics, 700 | scaffold_fn=scaffold_fn) 701 | else: 702 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 703 | mode=mode, 704 | predictions={"probabilities": probabilities}, 705 | scaffold_fn=scaffold_fn) 706 | return output_spec 707 | 708 | return model_fn 709 | 710 | 711 | def main(_): 712 | os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu_device_id 713 | tf.logging.set_verbosity(tf.logging.INFO) 714 | 715 | processors = { 716 | "baike": BaikeProcessor 717 | } 718 | 719 | tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case, 720 | FLAGS.init_checkpoint) 721 | 722 | if (not FLAGS.do_train and not FLAGS.do_eval and 723 | not FLAGS.do_predict and not FLAGS.do_train_and_eval): 724 | raise ValueError( 725 | "At least one of `do_train`, `do_eval` or `do_predict' must be True.") 726 | 727 | bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) 728 | 729 | if FLAGS.max_seq_length > bert_config.max_position_embeddings: 730 | raise ValueError( 731 | "Cannot use sequence length %d because the BERT model " 732 | "was only trained up to sequence length %d" % 733 | (FLAGS.max_seq_length, bert_config.max_position_embeddings)) 734 | 735 | tf.gfile.MakeDirs(FLAGS.output_dir) 736 | 737 | task_name = FLAGS.task_name.lower() 738 | 739 | if task_name not in processors: 740 | raise ValueError("Task not found: %s" % (task_name)) 741 | 742 | processor = processors[task_name]() 743 | 744 | label_list = processor.get_labels() 745 | 746 | tokenizer = tokenization.FullTokenizer( 747 | vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) 748 | 749 | tpu_cluster_resolver = None 750 | if FLAGS.use_tpu and FLAGS.tpu_name: 751 | tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( 752 | FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) 753 | 754 | is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 755 | run_config = tf.contrib.tpu.RunConfig( 756 | cluster=tpu_cluster_resolver, 757 | master=FLAGS.master, 758 | model_dir=FLAGS.output_dir, 759 | save_checkpoints_steps=FLAGS.save_checkpoints_steps, 760 | keep_checkpoint_max=FLAGS.keep_checkpoint_max, 761 | tpu_config=tf.contrib.tpu.TPUConfig( 762 | iterations_per_loop=FLAGS.iterations_per_loop, 763 | num_shards=FLAGS.num_tpu_cores, 764 | per_host_input_for_training=is_per_host)) 765 | 766 | train_examples = None 767 | num_train_steps = None 768 | num_warmup_steps = None 769 | if FLAGS.do_train or FLAGS.do_train_and_eval: 770 | train_examples = processor.get_train_examples(FLAGS.data_dir) 771 | num_train_steps = int( 772 | len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) 773 | num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) 774 | 775 | model_fn = model_fn_builder( 776 | bert_config=bert_config, 777 | num_labels=len(label_list), 778 | init_checkpoint=FLAGS.init_checkpoint, 779 | learning_rate=FLAGS.learning_rate, 780 | num_train_steps=num_train_steps, 781 | num_warmup_steps=num_warmup_steps, 782 | use_tpu=FLAGS.use_tpu, 783 | use_one_hot_embeddings=FLAGS.use_tpu) 784 | 785 | # If TPU is not available, this will fall back to normal Estimator on CPU 786 | # or GPU. 787 | estimator = tf.contrib.tpu.TPUEstimator( 788 | use_tpu=FLAGS.use_tpu, 789 | model_fn=model_fn, 790 | config=run_config, 791 | train_batch_size=FLAGS.train_batch_size, 792 | eval_batch_size=FLAGS.eval_batch_size, 793 | predict_batch_size=FLAGS.predict_batch_size) 794 | 795 | if FLAGS.do_train: 796 | train_file = os.path.join(FLAGS.output_dir, "train.tf_record") 797 | if not os.path.exists(train_file): 798 | file_based_convert_examples_to_features( 799 | train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file) 800 | tf.logging.info("***** Running training *****") 801 | tf.logging.info(" Num examples = %d", len(train_examples)) 802 | tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) 803 | tf.logging.info(" Num steps = %d", num_train_steps) 804 | train_input_fn = file_based_input_fn_builder( 805 | input_file=train_file, 806 | seq_length=FLAGS.max_seq_length, 807 | is_training=True, 808 | drop_remainder=True) 809 | estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) 810 | 811 | if FLAGS.do_eval: 812 | eval_examples = processor.get_dev_examples(FLAGS.data_dir) 813 | num_actual_eval_examples = len(eval_examples) 814 | if FLAGS.use_tpu: 815 | # TPU requires a fixed batch size for all batches, therefore the number 816 | # of examples must be a multiple of the batch size, or else examples 817 | # will get dropped. So we pad with fake examples which are ignored 818 | # later on. These do NOT count towards the metric (all tf.metrics 819 | # support a per-instance weight, and these get a weight of 0.0). 820 | while len(eval_examples) % FLAGS.eval_batch_size != 0: 821 | eval_examples.append(PaddingInputExample()) 822 | 823 | eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") 824 | if not os.path.exists(eval_file): 825 | file_based_convert_examples_to_features( 826 | eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file) 827 | tf.logging.info("***** Running evaluation *****") 828 | tf.logging.info(" Num examples = %d (%d actual, %d padding)", 829 | len(eval_examples), num_actual_eval_examples, 830 | len(eval_examples) - num_actual_eval_examples) 831 | tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size) 832 | 833 | # This tells the estimator to run through the entire set. 834 | eval_steps = None 835 | # However, if running eval on the TPU, you will need to specify the 836 | # number of steps. 837 | if FLAGS.use_tpu: 838 | assert len(eval_examples) % FLAGS.eval_batch_size == 0 839 | eval_steps = int(len(eval_examples) // FLAGS.eval_batch_size) 840 | 841 | eval_drop_remainder = True if FLAGS.use_tpu else False 842 | eval_input_fn = file_based_input_fn_builder( 843 | input_file=eval_file, 844 | seq_length=FLAGS.max_seq_length, 845 | is_training=False, 846 | drop_remainder=eval_drop_remainder) 847 | 848 | result = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps) 849 | 850 | output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") 851 | with tf.gfile.GFile(output_eval_file, "w") as writer: 852 | tf.logging.info("***** Eval results *****") 853 | for key in sorted(result.keys()): 854 | tf.logging.info(" %s = %s", key, str(result[key])) 855 | writer.write("%s = %s\n" % (key, str(result[key]))) 856 | 857 | # mcy 858 | if FLAGS.do_train_and_eval: 859 | train_file = os.path.join(FLAGS.output_dir, "train.tf_record") 860 | if not os.path.exists(train_file): 861 | file_based_convert_examples_to_features( 862 | train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file) 863 | tf.logging.info("***** Running train and eval*****") 864 | tf.logging.info(">>Train Paras<<") 865 | tf.logging.info(" Num examples = %d", len(train_examples)) 866 | tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) 867 | tf.logging.info(" Num steps total = %d", num_train_steps) 868 | 869 | eval_examples = processor.get_dev_examples(FLAGS.data_dir) 870 | num_actual_eval_examples = len(eval_examples) 871 | eval_file = os.path.join(FLAGS.output_dir, "eval.tf_record") 872 | if not os.path.exists(eval_file): 873 | file_based_convert_examples_to_features( 874 | eval_examples, label_list, FLAGS.max_seq_length, tokenizer, eval_file) 875 | tf.logging.info(">>Eval Paras<<") 876 | tf.logging.info(" Num examples = %d (%d actual, %d padding)", 877 | len(eval_examples), num_actual_eval_examples, 878 | len(eval_examples) - num_actual_eval_examples) 879 | tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size) 880 | 881 | train_input_fn = file_based_input_fn_builder( 882 | input_file=train_file, 883 | seq_length=FLAGS.max_seq_length, 884 | is_training=True, 885 | drop_remainder=True) 886 | train_spec = tf.estimator.TrainSpec( 887 | input_fn=train_input_fn, 888 | max_steps=num_train_steps) 889 | eval_steps = None # This tells the estimator to run through the entire set. 890 | eval_drop_remainder = True if FLAGS.use_tpu else False 891 | eval_input_fn = file_based_input_fn_builder( 892 | input_file=eval_file, 893 | seq_length=FLAGS.max_seq_length, 894 | is_training=False, 895 | drop_remainder=eval_drop_remainder) 896 | eval_spec = tf.estimator.EvalSpec( 897 | input_fn=eval_input_fn, 898 | steps=eval_steps, 899 | start_delay_secs=0, 900 | throttle_secs=5) 901 | 902 | # tf.estimator.train_and_evaluate will call estimator.eval many times, print eval metrics 903 | # and only return last eval result 904 | tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec) 905 | 906 | # result = tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec) 907 | # output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") 908 | # with tf.gfile.GFile(output_eval_file, "w") as writer: 909 | # # tf.logging.info("***** Final eval results *****") 910 | # for key in sorted(result.keys()): 911 | # # tf.logging.info(" %s = %s", key, str(result[key])) 912 | # writer.write("%s = %s\n" % (key, str(result[key]))) 913 | # mcy-END 914 | 915 | if FLAGS.do_predict: 916 | predict_examples = processor.get_test_examples(FLAGS.data_dir) 917 | num_actual_predict_examples = len(predict_examples) 918 | if FLAGS.use_tpu: 919 | # TPU requires a fixed batch size for all batches, therefore the number 920 | # of examples must be a multiple of the batch size, or else examples 921 | # will get dropped. So we pad with fake examples which are ignored 922 | # later on. 923 | while len(predict_examples) % FLAGS.predict_batch_size != 0: 924 | predict_examples.append(PaddingInputExample()) 925 | 926 | predict_file = os.path.join(FLAGS.output_dir, "predict.tf_record") 927 | file_based_convert_examples_to_features(predict_examples, label_list, 928 | FLAGS.max_seq_length, tokenizer, 929 | predict_file) 930 | 931 | tf.logging.info("***** Running prediction*****") 932 | tf.logging.info(" Num examples = %d (%d actual, %d padding)", 933 | len(predict_examples), num_actual_predict_examples, 934 | len(predict_examples) - num_actual_predict_examples) 935 | tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size) 936 | 937 | predict_drop_remainder = True if FLAGS.use_tpu else False 938 | predict_input_fn = file_based_input_fn_builder( 939 | input_file=predict_file, 940 | seq_length=FLAGS.max_seq_length, 941 | is_training=False, 942 | drop_remainder=predict_drop_remainder) 943 | 944 | result = estimator.predict(input_fn=predict_input_fn) 945 | 946 | output_predict_file = os.path.join(FLAGS.output_dir, "test_results.tsv") 947 | with tf.gfile.GFile(output_predict_file, "w") as writer: 948 | num_written_lines = 0 949 | tf.logging.info("***** Predict results *****") 950 | for (i, prediction) in enumerate(result): 951 | probabilities = prediction["probabilities"] 952 | if i >= num_actual_predict_examples: 953 | break 954 | output_line = "\t".join( 955 | str(class_probability) 956 | for class_probability in probabilities) + "\n" 957 | writer.write(output_line) 958 | num_written_lines += 1 959 | assert num_written_lines == num_actual_predict_examples 960 | 961 | 962 | if __name__ == "__main__": 963 | flags.mark_flag_as_required("data_dir") 964 | flags.mark_flag_as_required("task_name") 965 | flags.mark_flag_as_required("vocab_file") 966 | flags.mark_flag_as_required("bert_config_file") 967 | flags.mark_flag_as_required("output_dir") 968 | tf.app.run() 969 | -------------------------------------------------------------------------------- /keyword-bert/run_pretraining.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Run masked LM/next sentence masked_lm pre-training for BERT.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import os 22 | import modeling 23 | import optimization 24 | import tensorflow as tf 25 | 26 | flags = tf.flags 27 | 28 | FLAGS = flags.FLAGS 29 | 30 | ## Required parameters 31 | flags.DEFINE_string( 32 | "bert_config_file", None, 33 | "The config json file corresponding to the pre-trained BERT model. " 34 | "This specifies the model architecture.") 35 | 36 | flags.DEFINE_string( 37 | "input_file", None, 38 | "Input TF example files (can be a glob or comma separated).") 39 | 40 | flags.DEFINE_string( 41 | "output_dir", None, 42 | "The output directory where the model checkpoints will be written.") 43 | 44 | ## Other parameters 45 | flags.DEFINE_string( 46 | "init_checkpoint", None, 47 | "Initial checkpoint (usually from a pre-trained BERT model).") 48 | 49 | flags.DEFINE_integer( 50 | "max_seq_length", 128, 51 | "The maximum total input sequence length after WordPiece tokenization. " 52 | "Sequences longer than this will be truncated, and sequences shorter " 53 | "than this will be padded. Must match data generation.") 54 | 55 | flags.DEFINE_integer( 56 | "max_predictions_per_seq", 20, 57 | "Maximum number of masked LM predictions per sequence. " 58 | "Must match data generation.") 59 | 60 | flags.DEFINE_bool("do_train", False, "Whether to run training.") 61 | 62 | flags.DEFINE_bool("do_eval", False, "Whether to run eval on the dev set.") 63 | 64 | flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.") 65 | 66 | flags.DEFINE_integer("eval_batch_size", 8, "Total batch size for eval.") 67 | 68 | flags.DEFINE_float("learning_rate", 5e-5, "The initial learning rate for Adam.") 69 | 70 | flags.DEFINE_integer("num_train_steps", 100000, "Number of training steps.") 71 | 72 | flags.DEFINE_integer("num_warmup_steps", 10000, "Number of warmup steps.") 73 | 74 | flags.DEFINE_integer("save_checkpoints_steps", 1000, 75 | "How often to save the model checkpoint.") 76 | 77 | flags.DEFINE_integer("iterations_per_loop", 1000, 78 | "How many steps to make in each estimator call.") 79 | 80 | flags.DEFINE_integer("max_eval_steps", 100, "Maximum number of eval steps.") 81 | 82 | flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.") 83 | 84 | tf.flags.DEFINE_string( 85 | "tpu_name", None, 86 | "The Cloud TPU to use for training. This should be either the name " 87 | "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 " 88 | "url.") 89 | 90 | tf.flags.DEFINE_string( 91 | "tpu_zone", None, 92 | "[Optional] GCE zone where the Cloud TPU is located in. If not " 93 | "specified, we will attempt to automatically detect the GCE project from " 94 | "metadata.") 95 | 96 | tf.flags.DEFINE_string( 97 | "gcp_project", None, 98 | "[Optional] Project name for the Cloud TPU-enabled project. If not " 99 | "specified, we will attempt to automatically detect the GCE project from " 100 | "metadata.") 101 | 102 | tf.flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.") 103 | 104 | flags.DEFINE_integer( 105 | "num_tpu_cores", 8, 106 | "Only used if `use_tpu` is True. Total number of TPU cores to use.") 107 | 108 | 109 | def model_fn_builder(bert_config, init_checkpoint, learning_rate, 110 | num_train_steps, num_warmup_steps, use_tpu, 111 | use_one_hot_embeddings): 112 | """Returns `model_fn` closure for TPUEstimator.""" 113 | 114 | def model_fn(features, labels, mode, params): # pylint: disable=unused-argument 115 | """The `model_fn` for TPUEstimator.""" 116 | 117 | tf.logging.info("*** Features ***") 118 | for name in sorted(features.keys()): 119 | tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape)) 120 | 121 | input_ids = features["input_ids"] 122 | input_mask = features["input_mask"] 123 | segment_ids = features["segment_ids"] 124 | masked_lm_positions = features["masked_lm_positions"] 125 | masked_lm_ids = features["masked_lm_ids"] 126 | masked_lm_weights = features["masked_lm_weights"] 127 | next_sentence_labels = features["next_sentence_labels"] 128 | 129 | is_training = (mode == tf.estimator.ModeKeys.TRAIN) 130 | 131 | model = modeling.BertModel( 132 | config=bert_config, 133 | is_training=is_training, 134 | input_ids=input_ids, 135 | input_mask=input_mask, 136 | token_type_ids=segment_ids, 137 | use_one_hot_embeddings=use_one_hot_embeddings) 138 | 139 | (masked_lm_loss, 140 | masked_lm_example_loss, masked_lm_log_probs) = get_masked_lm_output( 141 | bert_config, model.get_sequence_output(), model.get_embedding_table(), 142 | masked_lm_positions, masked_lm_ids, masked_lm_weights) 143 | 144 | (next_sentence_loss, next_sentence_example_loss, 145 | next_sentence_log_probs) = get_next_sentence_output( 146 | bert_config, model.get_pooled_output(), next_sentence_labels) 147 | 148 | total_loss = masked_lm_loss + next_sentence_loss 149 | 150 | tvars = tf.trainable_variables() 151 | 152 | initialized_variable_names = {} 153 | scaffold_fn = None 154 | if init_checkpoint: 155 | (assignment_map, initialized_variable_names 156 | ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint) 157 | if use_tpu: 158 | 159 | def tpu_scaffold(): 160 | tf.train.init_from_checkpoint(init_checkpoint, assignment_map) 161 | return tf.train.Scaffold() 162 | 163 | scaffold_fn = tpu_scaffold 164 | else: 165 | tf.train.init_from_checkpoint(init_checkpoint, assignment_map) 166 | 167 | tf.logging.info("**** Trainable Variables ****") 168 | for var in tvars: 169 | init_string = "" 170 | if var.name in initialized_variable_names: 171 | init_string = ", *INIT_FROM_CKPT*" 172 | tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape, 173 | init_string) 174 | 175 | output_spec = None 176 | if mode == tf.estimator.ModeKeys.TRAIN: 177 | train_op = optimization.create_optimizer( 178 | total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu) 179 | 180 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 181 | mode=mode, 182 | loss=total_loss, 183 | train_op=train_op, 184 | scaffold_fn=scaffold_fn) 185 | elif mode == tf.estimator.ModeKeys.EVAL: 186 | 187 | def metric_fn(masked_lm_example_loss, masked_lm_log_probs, masked_lm_ids, 188 | masked_lm_weights, next_sentence_example_loss, 189 | next_sentence_log_probs, next_sentence_labels): 190 | """Computes the loss and accuracy of the model.""" 191 | masked_lm_log_probs = tf.reshape(masked_lm_log_probs, 192 | [-1, masked_lm_log_probs.shape[-1]]) 193 | masked_lm_predictions = tf.argmax( 194 | masked_lm_log_probs, axis=-1, output_type=tf.int32) 195 | masked_lm_example_loss = tf.reshape(masked_lm_example_loss, [-1]) 196 | masked_lm_ids = tf.reshape(masked_lm_ids, [-1]) 197 | masked_lm_weights = tf.reshape(masked_lm_weights, [-1]) 198 | masked_lm_accuracy = tf.metrics.accuracy( 199 | labels=masked_lm_ids, 200 | predictions=masked_lm_predictions, 201 | weights=masked_lm_weights) 202 | masked_lm_mean_loss = tf.metrics.mean( 203 | values=masked_lm_example_loss, weights=masked_lm_weights) 204 | 205 | next_sentence_log_probs = tf.reshape( 206 | next_sentence_log_probs, [-1, next_sentence_log_probs.shape[-1]]) 207 | next_sentence_predictions = tf.argmax( 208 | next_sentence_log_probs, axis=-1, output_type=tf.int32) 209 | next_sentence_labels = tf.reshape(next_sentence_labels, [-1]) 210 | next_sentence_accuracy = tf.metrics.accuracy( 211 | labels=next_sentence_labels, predictions=next_sentence_predictions) 212 | next_sentence_mean_loss = tf.metrics.mean( 213 | values=next_sentence_example_loss) 214 | 215 | return { 216 | "masked_lm_accuracy": masked_lm_accuracy, 217 | "masked_lm_loss": masked_lm_mean_loss, 218 | "next_sentence_accuracy": next_sentence_accuracy, 219 | "next_sentence_loss": next_sentence_mean_loss, 220 | } 221 | 222 | eval_metrics = (metric_fn, [ 223 | masked_lm_example_loss, masked_lm_log_probs, masked_lm_ids, 224 | masked_lm_weights, next_sentence_example_loss, 225 | next_sentence_log_probs, next_sentence_labels 226 | ]) 227 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 228 | mode=mode, 229 | loss=total_loss, 230 | eval_metrics=eval_metrics, 231 | scaffold_fn=scaffold_fn) 232 | else: 233 | raise ValueError("Only TRAIN and EVAL modes are supported: %s" % (mode)) 234 | 235 | return output_spec 236 | 237 | return model_fn 238 | 239 | 240 | def get_masked_lm_output(bert_config, input_tensor, output_weights, positions, 241 | label_ids, label_weights): 242 | """Get loss and log probs for the masked LM.""" 243 | input_tensor = gather_indexes(input_tensor, positions) 244 | 245 | with tf.variable_scope("cls/predictions"): 246 | # We apply one more non-linear transformation before the output layer. 247 | # This matrix is not used after pre-training. 248 | with tf.variable_scope("transform"): 249 | input_tensor = tf.layers.dense( 250 | input_tensor, 251 | units=bert_config.hidden_size, 252 | activation=modeling.get_activation(bert_config.hidden_act), 253 | kernel_initializer=modeling.create_initializer( 254 | bert_config.initializer_range)) 255 | input_tensor = modeling.layer_norm(input_tensor) 256 | 257 | # The output weights are the same as the input embeddings, but there is 258 | # an output-only bias for each token. 259 | output_bias = tf.get_variable( 260 | "output_bias", 261 | shape=[bert_config.vocab_size], 262 | initializer=tf.zeros_initializer()) 263 | logits = tf.matmul(input_tensor, output_weights, transpose_b=True) 264 | logits = tf.nn.bias_add(logits, output_bias) 265 | log_probs = tf.nn.log_softmax(logits, axis=-1) 266 | 267 | label_ids = tf.reshape(label_ids, [-1]) 268 | label_weights = tf.reshape(label_weights, [-1]) 269 | 270 | one_hot_labels = tf.one_hot( 271 | label_ids, depth=bert_config.vocab_size, dtype=tf.float32) 272 | 273 | # The `positions` tensor might be zero-padded (if the sequence is too 274 | # short to have the maximum number of predictions). The `label_weights` 275 | # tensor has a value of 1.0 for every real prediction and 0.0 for the 276 | # padding predictions. 277 | per_example_loss = -tf.reduce_sum(log_probs * one_hot_labels, axis=[-1]) 278 | numerator = tf.reduce_sum(label_weights * per_example_loss) 279 | denominator = tf.reduce_sum(label_weights) + 1e-5 280 | loss = numerator / denominator 281 | 282 | return (loss, per_example_loss, log_probs) 283 | 284 | 285 | def get_next_sentence_output(bert_config, input_tensor, labels): 286 | """Get loss and log probs for the next sentence prediction.""" 287 | 288 | # Simple binary classification. Note that 0 is "next sentence" and 1 is 289 | # "random sentence". This weight matrix is not used after pre-training. 290 | with tf.variable_scope("cls/seq_relationship"): 291 | output_weights = tf.get_variable( 292 | "output_weights", 293 | shape=[2, bert_config.hidden_size], 294 | initializer=modeling.create_initializer(bert_config.initializer_range)) 295 | output_bias = tf.get_variable( 296 | "output_bias", shape=[2], initializer=tf.zeros_initializer()) 297 | 298 | logits = tf.matmul(input_tensor, output_weights, transpose_b=True) 299 | logits = tf.nn.bias_add(logits, output_bias) 300 | log_probs = tf.nn.log_softmax(logits, axis=-1) 301 | labels = tf.reshape(labels, [-1]) 302 | one_hot_labels = tf.one_hot(labels, depth=2, dtype=tf.float32) 303 | per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) 304 | loss = tf.reduce_mean(per_example_loss) 305 | return (loss, per_example_loss, log_probs) 306 | 307 | 308 | def gather_indexes(sequence_tensor, positions): 309 | """Gathers the vectors at the specific positions over a minibatch.""" 310 | sequence_shape = modeling.get_shape_list(sequence_tensor, expected_rank=3) 311 | batch_size = sequence_shape[0] 312 | seq_length = sequence_shape[1] 313 | width = sequence_shape[2] 314 | 315 | flat_offsets = tf.reshape( 316 | tf.range(0, batch_size, dtype=tf.int32) * seq_length, [-1, 1]) 317 | flat_positions = tf.reshape(positions + flat_offsets, [-1]) 318 | flat_sequence_tensor = tf.reshape(sequence_tensor, 319 | [batch_size * seq_length, width]) 320 | output_tensor = tf.gather(flat_sequence_tensor, flat_positions) 321 | return output_tensor 322 | 323 | 324 | def input_fn_builder(input_files, 325 | max_seq_length, 326 | max_predictions_per_seq, 327 | is_training, 328 | num_cpu_threads=4): 329 | """Creates an `input_fn` closure to be passed to TPUEstimator.""" 330 | 331 | def input_fn(params): 332 | """The actual input function.""" 333 | batch_size = params["batch_size"] 334 | 335 | name_to_features = { 336 | "input_ids": 337 | tf.FixedLenFeature([max_seq_length], tf.int64), 338 | "input_mask": 339 | tf.FixedLenFeature([max_seq_length], tf.int64), 340 | "segment_ids": 341 | tf.FixedLenFeature([max_seq_length], tf.int64), 342 | "masked_lm_positions": 343 | tf.FixedLenFeature([max_predictions_per_seq], tf.int64), 344 | "masked_lm_ids": 345 | tf.FixedLenFeature([max_predictions_per_seq], tf.int64), 346 | "masked_lm_weights": 347 | tf.FixedLenFeature([max_predictions_per_seq], tf.float32), 348 | "next_sentence_labels": 349 | tf.FixedLenFeature([1], tf.int64), 350 | } 351 | 352 | # For training, we want a lot of parallel reading and shuffling. 353 | # For eval, we want no shuffling and parallel reading doesn't matter. 354 | if is_training: 355 | d = tf.data.Dataset.from_tensor_slices(tf.constant(input_files)) 356 | d = d.repeat() 357 | d = d.shuffle(buffer_size=len(input_files)) 358 | 359 | # `cycle_length` is the number of parallel files that get read. 360 | cycle_length = min(num_cpu_threads, len(input_files)) 361 | 362 | # `sloppy` mode means that the interleaving is not exact. This adds 363 | # even more randomness to the training pipeline. 364 | d = d.apply( 365 | tf.contrib.data.parallel_interleave( 366 | tf.data.TFRecordDataset, 367 | sloppy=is_training, 368 | cycle_length=cycle_length)) 369 | d = d.shuffle(buffer_size=100) 370 | else: 371 | d = tf.data.TFRecordDataset(input_files) 372 | # Since we evaluate for a fixed number of steps we don't want to encounter 373 | # out-of-range exceptions. 374 | d = d.repeat() 375 | 376 | # We must `drop_remainder` on training because the TPU requires fixed 377 | # size dimensions. For eval, we assume we are evaluating on the CPU or GPU 378 | # and we *don't* want to drop the remainder, otherwise we wont cover 379 | # every sample. 380 | d = d.apply( 381 | tf.contrib.data.map_and_batch( 382 | lambda record: _decode_record(record, name_to_features), 383 | batch_size=batch_size, 384 | num_parallel_batches=num_cpu_threads, 385 | drop_remainder=True)) 386 | return d 387 | 388 | return input_fn 389 | 390 | 391 | def _decode_record(record, name_to_features): 392 | """Decodes a record to a TensorFlow example.""" 393 | example = tf.parse_single_example(record, name_to_features) 394 | 395 | # tf.Example only supports tf.int64, but the TPU only supports tf.int32. 396 | # So cast all int64 to int32. 397 | for name in list(example.keys()): 398 | t = example[name] 399 | if t.dtype == tf.int64: 400 | t = tf.to_int32(t) 401 | example[name] = t 402 | 403 | return example 404 | 405 | 406 | def main(_): 407 | tf.logging.set_verbosity(tf.logging.INFO) 408 | 409 | if not FLAGS.do_train and not FLAGS.do_eval: 410 | raise ValueError("At least one of `do_train` or `do_eval` must be True.") 411 | 412 | bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) 413 | 414 | tf.gfile.MakeDirs(FLAGS.output_dir) 415 | 416 | input_files = [] 417 | for input_pattern in FLAGS.input_file.split(","): 418 | input_files.extend(tf.gfile.Glob(input_pattern)) 419 | 420 | tf.logging.info("*** Input Files ***") 421 | for input_file in input_files: 422 | tf.logging.info(" %s" % input_file) 423 | 424 | tpu_cluster_resolver = None 425 | if FLAGS.use_tpu and FLAGS.tpu_name: 426 | tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( 427 | FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) 428 | 429 | is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 430 | run_config = tf.contrib.tpu.RunConfig( 431 | cluster=tpu_cluster_resolver, 432 | master=FLAGS.master, 433 | model_dir=FLAGS.output_dir, 434 | save_checkpoints_steps=FLAGS.save_checkpoints_steps, 435 | tpu_config=tf.contrib.tpu.TPUConfig( 436 | iterations_per_loop=FLAGS.iterations_per_loop, 437 | num_shards=FLAGS.num_tpu_cores, 438 | per_host_input_for_training=is_per_host)) 439 | 440 | model_fn = model_fn_builder( 441 | bert_config=bert_config, 442 | init_checkpoint=FLAGS.init_checkpoint, 443 | learning_rate=FLAGS.learning_rate, 444 | num_train_steps=FLAGS.num_train_steps, 445 | num_warmup_steps=FLAGS.num_warmup_steps, 446 | use_tpu=FLAGS.use_tpu, 447 | use_one_hot_embeddings=FLAGS.use_tpu) 448 | 449 | # If TPU is not available, this will fall back to normal Estimator on CPU 450 | # or GPU. 451 | estimator = tf.contrib.tpu.TPUEstimator( 452 | use_tpu=FLAGS.use_tpu, 453 | model_fn=model_fn, 454 | config=run_config, 455 | train_batch_size=FLAGS.train_batch_size, 456 | eval_batch_size=FLAGS.eval_batch_size) 457 | 458 | if FLAGS.do_train: 459 | tf.logging.info("***** Running training *****") 460 | tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) 461 | train_input_fn = input_fn_builder( 462 | input_files=input_files, 463 | max_seq_length=FLAGS.max_seq_length, 464 | max_predictions_per_seq=FLAGS.max_predictions_per_seq, 465 | is_training=True) 466 | estimator.train(input_fn=train_input_fn, max_steps=FLAGS.num_train_steps) 467 | 468 | if FLAGS.do_eval: 469 | tf.logging.info("***** Running evaluation *****") 470 | tf.logging.info(" Batch size = %d", FLAGS.eval_batch_size) 471 | 472 | eval_input_fn = input_fn_builder( 473 | input_files=input_files, 474 | max_seq_length=FLAGS.max_seq_length, 475 | max_predictions_per_seq=FLAGS.max_predictions_per_seq, 476 | is_training=False) 477 | 478 | result = estimator.evaluate( 479 | input_fn=eval_input_fn, steps=FLAGS.max_eval_steps) 480 | 481 | output_eval_file = os.path.join(FLAGS.output_dir, "eval_results.txt") 482 | with tf.gfile.GFile(output_eval_file, "w") as writer: 483 | tf.logging.info("***** Eval results *****") 484 | for key in sorted(result.keys()): 485 | tf.logging.info(" %s = %s", key, str(result[key])) 486 | writer.write("%s = %s\n" % (key, str(result[key]))) 487 | 488 | 489 | if __name__ == "__main__": 490 | flags.mark_flag_as_required("input_file") 491 | flags.mark_flag_as_required("bert_config_file") 492 | flags.mark_flag_as_required("output_dir") 493 | tf.app.run() 494 | -------------------------------------------------------------------------------- /keyword-bert/run_squad.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Run BERT on SQuAD 1.1 and SQuAD 2.0.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import collections 22 | import json 23 | import math 24 | import os 25 | import random 26 | import modeling 27 | import optimization 28 | import tokenization 29 | import six 30 | import tensorflow as tf 31 | 32 | flags = tf.flags 33 | 34 | FLAGS = flags.FLAGS 35 | 36 | ## Required parameters 37 | flags.DEFINE_string( 38 | "bert_config_file", None, 39 | "The config json file corresponding to the pre-trained BERT model. " 40 | "This specifies the model architecture.") 41 | 42 | flags.DEFINE_string("vocab_file", None, 43 | "The vocabulary file that the BERT model was trained on.") 44 | 45 | flags.DEFINE_string( 46 | "output_dir", None, 47 | "The output directory where the model checkpoints will be written.") 48 | 49 | ## Other parameters 50 | flags.DEFINE_string("train_file", None, 51 | "SQuAD json for training. E.g., train-v1.1.json") 52 | 53 | flags.DEFINE_string( 54 | "predict_file", None, 55 | "SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json") 56 | 57 | flags.DEFINE_string( 58 | "init_checkpoint", None, 59 | "Initial checkpoint (usually from a pre-trained BERT model).") 60 | 61 | flags.DEFINE_bool( 62 | "do_lower_case", True, 63 | "Whether to lower case the input text. Should be True for uncased " 64 | "models and False for cased models.") 65 | 66 | flags.DEFINE_integer( 67 | "max_seq_length", 384, 68 | "The maximum total input sequence length after WordPiece tokenization. " 69 | "Sequences longer than this will be truncated, and sequences shorter " 70 | "than this will be padded.") 71 | 72 | flags.DEFINE_integer( 73 | "doc_stride", 128, 74 | "When splitting up a long document into chunks, how much stride to " 75 | "take between chunks.") 76 | 77 | flags.DEFINE_integer( 78 | "max_query_length", 64, 79 | "The maximum number of tokens for the question. Questions longer than " 80 | "this will be truncated to this length.") 81 | 82 | flags.DEFINE_bool("do_train", False, "Whether to run training.") 83 | 84 | flags.DEFINE_bool("do_predict", False, "Whether to run eval on the dev set.") 85 | 86 | flags.DEFINE_integer("train_batch_size", 32, "Total batch size for training.") 87 | 88 | flags.DEFINE_integer("predict_batch_size", 8, 89 | "Total batch size for predictions.") 90 | 91 | flags.DEFINE_float("learning_rate", 5e-5, "The initial learning rate for Adam.") 92 | 93 | flags.DEFINE_float("num_train_epochs", 3.0, 94 | "Total number of training epochs to perform.") 95 | 96 | flags.DEFINE_float( 97 | "warmup_proportion", 0.1, 98 | "Proportion of training to perform linear learning rate warmup for. " 99 | "E.g., 0.1 = 10% of training.") 100 | 101 | flags.DEFINE_integer("save_checkpoints_steps", 1000, 102 | "How often to save the model checkpoint.") 103 | 104 | flags.DEFINE_integer("iterations_per_loop", 1000, 105 | "How many steps to make in each estimator call.") 106 | 107 | flags.DEFINE_integer( 108 | "n_best_size", 20, 109 | "The total number of n-best predictions to generate in the " 110 | "nbest_predictions.json output file.") 111 | 112 | flags.DEFINE_integer( 113 | "max_answer_length", 30, 114 | "The maximum length of an answer that can be generated. This is needed " 115 | "because the start and end predictions are not conditioned on one another.") 116 | 117 | flags.DEFINE_bool("use_tpu", False, "Whether to use TPU or GPU/CPU.") 118 | 119 | tf.flags.DEFINE_string( 120 | "tpu_name", None, 121 | "The Cloud TPU to use for training. This should be either the name " 122 | "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 " 123 | "url.") 124 | 125 | tf.flags.DEFINE_string( 126 | "tpu_zone", None, 127 | "[Optional] GCE zone where the Cloud TPU is located in. If not " 128 | "specified, we will attempt to automatically detect the GCE project from " 129 | "metadata.") 130 | 131 | tf.flags.DEFINE_string( 132 | "gcp_project", None, 133 | "[Optional] Project name for the Cloud TPU-enabled project. If not " 134 | "specified, we will attempt to automatically detect the GCE project from " 135 | "metadata.") 136 | 137 | tf.flags.DEFINE_string("master", None, "[Optional] TensorFlow master URL.") 138 | 139 | flags.DEFINE_integer( 140 | "num_tpu_cores", 8, 141 | "Only used if `use_tpu` is True. Total number of TPU cores to use.") 142 | 143 | flags.DEFINE_bool( 144 | "verbose_logging", False, 145 | "If true, all of the warnings related to data processing will be printed. " 146 | "A number of warnings are expected for a normal SQuAD evaluation.") 147 | 148 | flags.DEFINE_bool( 149 | "version_2_with_negative", False, 150 | "If true, the SQuAD examples contain some that do not have an answer.") 151 | 152 | flags.DEFINE_float( 153 | "null_score_diff_threshold", 0.0, 154 | "If null_score - best_non_null is greater than the threshold predict null.") 155 | 156 | 157 | class SquadExample(object): 158 | """A single training/test example for simple sequence classification. 159 | 160 | For examples without an answer, the start and end position are -1. 161 | """ 162 | 163 | def __init__(self, 164 | qas_id, 165 | question_text, 166 | doc_tokens, 167 | orig_answer_text=None, 168 | start_position=None, 169 | end_position=None, 170 | is_impossible=False): 171 | self.qas_id = qas_id 172 | self.question_text = question_text 173 | self.doc_tokens = doc_tokens 174 | self.orig_answer_text = orig_answer_text 175 | self.start_position = start_position 176 | self.end_position = end_position 177 | self.is_impossible = is_impossible 178 | 179 | def __str__(self): 180 | return self.__repr__() 181 | 182 | def __repr__(self): 183 | s = "" 184 | s += "qas_id: %s" % (tokenization.printable_text(self.qas_id)) 185 | s += ", question_text: %s" % ( 186 | tokenization.printable_text(self.question_text)) 187 | s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens)) 188 | if self.start_position: 189 | s += ", start_position: %d" % (self.start_position) 190 | if self.start_position: 191 | s += ", end_position: %d" % (self.end_position) 192 | if self.start_position: 193 | s += ", is_impossible: %r" % (self.is_impossible) 194 | return s 195 | 196 | 197 | class InputFeatures(object): 198 | """A single set of features of data.""" 199 | 200 | def __init__(self, 201 | unique_id, 202 | example_index, 203 | doc_span_index, 204 | tokens, 205 | token_to_orig_map, 206 | token_is_max_context, 207 | input_ids, 208 | input_mask, 209 | segment_ids, 210 | start_position=None, 211 | end_position=None, 212 | is_impossible=None): 213 | self.unique_id = unique_id 214 | self.example_index = example_index 215 | self.doc_span_index = doc_span_index 216 | self.tokens = tokens 217 | self.token_to_orig_map = token_to_orig_map 218 | self.token_is_max_context = token_is_max_context 219 | self.input_ids = input_ids 220 | self.input_mask = input_mask 221 | self.segment_ids = segment_ids 222 | self.start_position = start_position 223 | self.end_position = end_position 224 | self.is_impossible = is_impossible 225 | 226 | 227 | def read_squad_examples(input_file, is_training): 228 | """Read a SQuAD json file into a list of SquadExample.""" 229 | with tf.gfile.Open(input_file, "r") as reader: 230 | input_data = json.load(reader)["data"] 231 | 232 | def is_whitespace(c): 233 | if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: 234 | return True 235 | return False 236 | 237 | examples = [] 238 | for entry in input_data: 239 | for paragraph in entry["paragraphs"]: 240 | paragraph_text = paragraph["context"] 241 | doc_tokens = [] 242 | char_to_word_offset = [] 243 | prev_is_whitespace = True 244 | for c in paragraph_text: 245 | if is_whitespace(c): 246 | prev_is_whitespace = True 247 | else: 248 | if prev_is_whitespace: 249 | doc_tokens.append(c) 250 | else: 251 | doc_tokens[-1] += c 252 | prev_is_whitespace = False 253 | char_to_word_offset.append(len(doc_tokens) - 1) 254 | 255 | for qa in paragraph["qas"]: 256 | qas_id = qa["id"] 257 | question_text = qa["question"] 258 | start_position = None 259 | end_position = None 260 | orig_answer_text = None 261 | is_impossible = False 262 | if is_training: 263 | 264 | if FLAGS.version_2_with_negative: 265 | is_impossible = qa["is_impossible"] 266 | if (len(qa["answers"]) != 1) and (not is_impossible): 267 | raise ValueError( 268 | "For training, each question should have exactly 1 answer.") 269 | if not is_impossible: 270 | answer = qa["answers"][0] 271 | orig_answer_text = answer["text"] 272 | answer_offset = answer["answer_start"] 273 | answer_length = len(orig_answer_text) 274 | start_position = char_to_word_offset[answer_offset] 275 | end_position = char_to_word_offset[answer_offset + answer_length - 276 | 1] 277 | # Only add answers where the text can be exactly recovered from the 278 | # document. If this CAN'T happen it's likely due to weird Unicode 279 | # stuff so we will just skip the example. 280 | # 281 | # Note that this means for training mode, every example is NOT 282 | # guaranteed to be preserved. 283 | actual_text = " ".join( 284 | doc_tokens[start_position:(end_position + 1)]) 285 | cleaned_answer_text = " ".join( 286 | tokenization.whitespace_tokenize(orig_answer_text)) 287 | if actual_text.find(cleaned_answer_text) == -1: 288 | tf.logging.warning("Could not find answer: '%s' vs. '%s'", 289 | actual_text, cleaned_answer_text) 290 | continue 291 | else: 292 | start_position = -1 293 | end_position = -1 294 | orig_answer_text = "" 295 | 296 | example = SquadExample( 297 | qas_id=qas_id, 298 | question_text=question_text, 299 | doc_tokens=doc_tokens, 300 | orig_answer_text=orig_answer_text, 301 | start_position=start_position, 302 | end_position=end_position, 303 | is_impossible=is_impossible) 304 | examples.append(example) 305 | 306 | return examples 307 | 308 | def read_baike_examples(input_file, is_training): 309 | """Read a baike txt file into a list of SquadExample""" 310 | with tf.gfile.Open(input_file, "r") as reader: 311 | for line in reader: 312 | 313 | def convert_examples_to_features(examples, tokenizer, max_seq_length, 314 | doc_stride, max_query_length, is_training, 315 | output_fn): 316 | """Loads a data file into a list of `InputBatch`s.""" 317 | 318 | unique_id = 1000000000 319 | 320 | for (example_index, example) in enumerate(examples): 321 | query_tokens = tokenizer.tokenize(example.question_text) 322 | 323 | if len(query_tokens) > max_query_length: 324 | query_tokens = query_tokens[0:max_query_length] 325 | 326 | tok_to_orig_index = [] 327 | orig_to_tok_index = [] 328 | all_doc_tokens = [] 329 | for (i, token) in enumerate(example.doc_tokens): 330 | orig_to_tok_index.append(len(all_doc_tokens)) 331 | sub_tokens = tokenizer.tokenize(token) 332 | for sub_token in sub_tokens: 333 | tok_to_orig_index.append(i) 334 | all_doc_tokens.append(sub_token) 335 | 336 | tok_start_position = None 337 | tok_end_position = None 338 | if is_training and example.is_impossible: 339 | tok_start_position = -1 340 | tok_end_position = -1 341 | if is_training and not example.is_impossible: 342 | tok_start_position = orig_to_tok_index[example.start_position] 343 | if example.end_position < len(example.doc_tokens) - 1: 344 | tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 345 | else: 346 | tok_end_position = len(all_doc_tokens) - 1 347 | (tok_start_position, tok_end_position) = _improve_answer_span( 348 | all_doc_tokens, tok_start_position, tok_end_position, tokenizer, 349 | example.orig_answer_text) 350 | 351 | # The -3 accounts for [CLS], [SEP] and [SEP] 352 | max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 353 | 354 | # We can have documents that are longer than the maximum sequence length. 355 | # To deal with this we do a sliding window approach, where we take chunks 356 | # of the up to our max length with a stride of `doc_stride`. 357 | _DocSpan = collections.namedtuple( # pylint: disable=invalid-name 358 | "DocSpan", ["start", "length"]) 359 | doc_spans = [] 360 | start_offset = 0 361 | while start_offset < len(all_doc_tokens): 362 | length = len(all_doc_tokens) - start_offset 363 | if length > max_tokens_for_doc: 364 | length = max_tokens_for_doc 365 | doc_spans.append(_DocSpan(start=start_offset, length=length)) 366 | if start_offset + length == len(all_doc_tokens): 367 | break 368 | start_offset += min(length, doc_stride) 369 | 370 | for (doc_span_index, doc_span) in enumerate(doc_spans): 371 | tokens = [] 372 | token_to_orig_map = {} 373 | token_is_max_context = {} 374 | segment_ids = [] 375 | tokens.append("[CLS]") 376 | segment_ids.append(0) 377 | for token in query_tokens: 378 | tokens.append(token) 379 | segment_ids.append(0) 380 | tokens.append("[SEP]") 381 | segment_ids.append(0) 382 | 383 | for i in range(doc_span.length): 384 | split_token_index = doc_span.start + i 385 | token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] 386 | 387 | is_max_context = _check_is_max_context(doc_spans, doc_span_index, 388 | split_token_index) 389 | token_is_max_context[len(tokens)] = is_max_context 390 | tokens.append(all_doc_tokens[split_token_index]) 391 | segment_ids.append(1) 392 | tokens.append("[SEP]") 393 | segment_ids.append(1) 394 | 395 | input_ids = tokenizer.convert_tokens_to_ids(tokens) 396 | 397 | # The mask has 1 for real tokens and 0 for padding tokens. Only real 398 | # tokens are attended to. 399 | input_mask = [1] * len(input_ids) 400 | 401 | # Zero-pad up to the sequence length. 402 | while len(input_ids) < max_seq_length: 403 | input_ids.append(0) 404 | input_mask.append(0) 405 | segment_ids.append(0) 406 | 407 | assert len(input_ids) == max_seq_length 408 | assert len(input_mask) == max_seq_length 409 | assert len(segment_ids) == max_seq_length 410 | 411 | start_position = None 412 | end_position = None 413 | if is_training and not example.is_impossible: 414 | # For training, if our document chunk does not contain an annotation 415 | # we throw it out, since there is nothing to predict. 416 | doc_start = doc_span.start 417 | doc_end = doc_span.start + doc_span.length - 1 418 | out_of_span = False 419 | if not (tok_start_position >= doc_start and 420 | tok_end_position <= doc_end): 421 | out_of_span = True 422 | if out_of_span: 423 | start_position = 0 424 | end_position = 0 425 | else: 426 | doc_offset = len(query_tokens) + 2 427 | start_position = tok_start_position - doc_start + doc_offset 428 | end_position = tok_end_position - doc_start + doc_offset 429 | 430 | if is_training and example.is_impossible: 431 | start_position = 0 432 | end_position = 0 433 | 434 | if example_index < 20: 435 | tf.logging.info("*** Example ***") 436 | tf.logging.info("unique_id: %s" % (unique_id)) 437 | tf.logging.info("example_index: %s" % (example_index)) 438 | tf.logging.info("doc_span_index: %s" % (doc_span_index)) 439 | tf.logging.info("tokens: %s" % " ".join( 440 | [tokenization.printable_text(x) for x in tokens])) 441 | tf.logging.info("token_to_orig_map: %s" % " ".join( 442 | ["%d:%d" % (x, y) for (x, y) in six.iteritems(token_to_orig_map)])) 443 | tf.logging.info("token_is_max_context: %s" % " ".join([ 444 | "%d:%s" % (x, y) for (x, y) in six.iteritems(token_is_max_context) 445 | ])) 446 | tf.logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) 447 | tf.logging.info( 448 | "input_mask: %s" % " ".join([str(x) for x in input_mask])) 449 | tf.logging.info( 450 | "segment_ids: %s" % " ".join([str(x) for x in segment_ids])) 451 | if is_training and example.is_impossible: 452 | tf.logging.info("impossible example") 453 | if is_training and not example.is_impossible: 454 | answer_text = " ".join(tokens[start_position:(end_position + 1)]) 455 | tf.logging.info("start_position: %d" % (start_position)) 456 | tf.logging.info("end_position: %d" % (end_position)) 457 | tf.logging.info( 458 | "answer: %s" % (tokenization.printable_text(answer_text))) 459 | 460 | feature = InputFeatures( 461 | unique_id=unique_id, 462 | example_index=example_index, 463 | doc_span_index=doc_span_index, 464 | tokens=tokens, 465 | token_to_orig_map=token_to_orig_map, 466 | token_is_max_context=token_is_max_context, 467 | input_ids=input_ids, 468 | input_mask=input_mask, 469 | segment_ids=segment_ids, 470 | start_position=start_position, 471 | end_position=end_position, 472 | is_impossible=example.is_impossible) 473 | 474 | # Run callback 475 | output_fn(feature) 476 | 477 | unique_id += 1 478 | 479 | 480 | def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, 481 | orig_answer_text): 482 | """Returns tokenized answer spans that better match the annotated answer.""" 483 | 484 | # The SQuAD annotations are character based. We first project them to 485 | # whitespace-tokenized words. But then after WordPiece tokenization, we can 486 | # often find a "better match". For example: 487 | # 488 | # Question: What year was John Smith born? 489 | # Context: The leader was John Smith (1895-1943). 490 | # Answer: 1895 491 | # 492 | # The original whitespace-tokenized answer will be "(1895-1943).". However 493 | # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match 494 | # the exact answer, 1895. 495 | # 496 | # However, this is not always possible. Consider the following: 497 | # 498 | # Question: What country is the top exporter of electornics? 499 | # Context: The Japanese electronics industry is the lagest in the world. 500 | # Answer: Japan 501 | # 502 | # In this case, the annotator chose "Japan" as a character sub-span of 503 | # the word "Japanese". Since our WordPiece tokenizer does not split 504 | # "Japanese", we just use "Japanese" as the annotation. This is fairly rare 505 | # in SQuAD, but does happen. 506 | tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) 507 | 508 | for new_start in range(input_start, input_end + 1): 509 | for new_end in range(input_end, new_start - 1, -1): 510 | text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) 511 | if text_span == tok_answer_text: 512 | return (new_start, new_end) 513 | 514 | return (input_start, input_end) 515 | 516 | 517 | def _check_is_max_context(doc_spans, cur_span_index, position): 518 | """Check if this is the 'max context' doc span for the token.""" 519 | 520 | # Because of the sliding window approach taken to scoring documents, a single 521 | # token can appear in multiple documents. E.g. 522 | # Doc: the man went to the store and bought a gallon of milk 523 | # Span A: the man went to the 524 | # Span B: to the store and bought 525 | # Span C: and bought a gallon of 526 | # ... 527 | # 528 | # Now the word 'bought' will have two scores from spans B and C. We only 529 | # want to consider the score with "maximum context", which we define as 530 | # the *minimum* of its left and right context (the *sum* of left and 531 | # right context will always be the same, of course). 532 | # 533 | # In the example the maximum context for 'bought' would be span C since 534 | # it has 1 left context and 3 right context, while span B has 4 left context 535 | # and 0 right context. 536 | best_score = None 537 | best_span_index = None 538 | for (span_index, doc_span) in enumerate(doc_spans): 539 | end = doc_span.start + doc_span.length - 1 540 | if position < doc_span.start: 541 | continue 542 | if position > end: 543 | continue 544 | num_left_context = position - doc_span.start 545 | num_right_context = end - position 546 | score = min(num_left_context, num_right_context) + 0.01 * doc_span.length 547 | if best_score is None or score > best_score: 548 | best_score = score 549 | best_span_index = span_index 550 | 551 | return cur_span_index == best_span_index 552 | 553 | 554 | def create_model(bert_config, is_training, input_ids, input_mask, segment_ids, 555 | use_one_hot_embeddings): 556 | """Creates a classification model.""" 557 | model = modeling.BertModel( 558 | config=bert_config, 559 | is_training=is_training, 560 | input_ids=input_ids, 561 | input_mask=input_mask, 562 | token_type_ids=segment_ids, 563 | use_one_hot_embeddings=use_one_hot_embeddings) 564 | 565 | final_hidden = model.get_sequence_output() 566 | 567 | final_hidden_shape = modeling.get_shape_list(final_hidden, expected_rank=3) 568 | batch_size = final_hidden_shape[0] 569 | seq_length = final_hidden_shape[1] 570 | hidden_size = final_hidden_shape[2] 571 | 572 | output_weights = tf.get_variable( 573 | "cls/squad/output_weights", [2, hidden_size], 574 | initializer=tf.truncated_normal_initializer(stddev=0.02)) 575 | 576 | output_bias = tf.get_variable( 577 | "cls/squad/output_bias", [2], initializer=tf.zeros_initializer()) 578 | 579 | final_hidden_matrix = tf.reshape(final_hidden, 580 | [batch_size * seq_length, hidden_size]) 581 | logits = tf.matmul(final_hidden_matrix, output_weights, transpose_b=True) 582 | logits = tf.nn.bias_add(logits, output_bias) 583 | 584 | logits = tf.reshape(logits, [batch_size, seq_length, 2]) 585 | logits = tf.transpose(logits, [2, 0, 1]) 586 | 587 | unstacked_logits = tf.unstack(logits, axis=0) 588 | 589 | (start_logits, end_logits) = (unstacked_logits[0], unstacked_logits[1]) 590 | 591 | return (start_logits, end_logits) 592 | 593 | 594 | def model_fn_builder(bert_config, init_checkpoint, learning_rate, 595 | num_train_steps, num_warmup_steps, use_tpu, 596 | use_one_hot_embeddings): 597 | """Returns `model_fn` closure for TPUEstimator.""" 598 | 599 | def model_fn(features, labels, mode, params): # pylint: disable=unused-argument 600 | """The `model_fn` for TPUEstimator.""" 601 | 602 | tf.logging.info("*** Features ***") 603 | for name in sorted(features.keys()): 604 | tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape)) 605 | 606 | unique_ids = features["unique_ids"] 607 | input_ids = features["input_ids"] 608 | input_mask = features["input_mask"] 609 | segment_ids = features["segment_ids"] 610 | 611 | is_training = (mode == tf.estimator.ModeKeys.TRAIN) 612 | 613 | (start_logits, end_logits) = create_model( 614 | bert_config=bert_config, 615 | is_training=is_training, 616 | input_ids=input_ids, 617 | input_mask=input_mask, 618 | segment_ids=segment_ids, 619 | use_one_hot_embeddings=use_one_hot_embeddings) 620 | 621 | tvars = tf.trainable_variables() 622 | 623 | initialized_variable_names = {} 624 | scaffold_fn = None 625 | if init_checkpoint: 626 | (assignment_map, initialized_variable_names 627 | ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint) 628 | if use_tpu: 629 | 630 | def tpu_scaffold(): 631 | tf.train.init_from_checkpoint(init_checkpoint, assignment_map) 632 | return tf.train.Scaffold() 633 | 634 | scaffold_fn = tpu_scaffold 635 | else: 636 | tf.train.init_from_checkpoint(init_checkpoint, assignment_map) 637 | 638 | tf.logging.info("**** Trainable Variables ****") 639 | for var in tvars: 640 | init_string = "" 641 | if var.name in initialized_variable_names: 642 | init_string = ", *INIT_FROM_CKPT*" 643 | tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape, 644 | init_string) 645 | 646 | output_spec = None 647 | if mode == tf.estimator.ModeKeys.TRAIN: 648 | seq_length = modeling.get_shape_list(input_ids)[1] 649 | 650 | def compute_loss(logits, positions): 651 | one_hot_positions = tf.one_hot( 652 | positions, depth=seq_length, dtype=tf.float32) 653 | log_probs = tf.nn.log_softmax(logits, axis=-1) 654 | loss = -tf.reduce_mean( 655 | tf.reduce_sum(one_hot_positions * log_probs, axis=-1)) 656 | return loss 657 | 658 | start_positions = features["start_positions"] 659 | end_positions = features["end_positions"] 660 | 661 | start_loss = compute_loss(start_logits, start_positions) 662 | end_loss = compute_loss(end_logits, end_positions) 663 | 664 | total_loss = (start_loss + end_loss) / 2.0 665 | 666 | train_op = optimization.create_optimizer( 667 | total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu) 668 | 669 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 670 | mode=mode, 671 | loss=total_loss, 672 | train_op=train_op, 673 | scaffold_fn=scaffold_fn) 674 | elif mode == tf.estimator.ModeKeys.PREDICT: 675 | predictions = { 676 | "unique_ids": unique_ids, 677 | "start_logits": start_logits, 678 | "end_logits": end_logits, 679 | } 680 | output_spec = tf.contrib.tpu.TPUEstimatorSpec( 681 | mode=mode, predictions=predictions, scaffold_fn=scaffold_fn) 682 | else: 683 | raise ValueError( 684 | "Only TRAIN and PREDICT modes are supported: %s" % (mode)) 685 | 686 | return output_spec 687 | 688 | return model_fn 689 | 690 | 691 | def input_fn_builder(input_file, seq_length, is_training, drop_remainder): 692 | """Creates an `input_fn` closure to be passed to TPUEstimator.""" 693 | 694 | name_to_features = { 695 | "unique_ids": tf.FixedLenFeature([], tf.int64), 696 | "input_ids": tf.FixedLenFeature([seq_length], tf.int64), 697 | "input_mask": tf.FixedLenFeature([seq_length], tf.int64), 698 | "segment_ids": tf.FixedLenFeature([seq_length], tf.int64), 699 | } 700 | 701 | if is_training: 702 | name_to_features["start_positions"] = tf.FixedLenFeature([], tf.int64) 703 | name_to_features["end_positions"] = tf.FixedLenFeature([], tf.int64) 704 | 705 | def _decode_record(record, name_to_features): 706 | """Decodes a record to a TensorFlow example.""" 707 | example = tf.parse_single_example(record, name_to_features) 708 | 709 | # tf.Example only supports tf.int64, but the TPU only supports tf.int32. 710 | # So cast all int64 to int32. 711 | for name in list(example.keys()): 712 | t = example[name] 713 | if t.dtype == tf.int64: 714 | t = tf.to_int32(t) 715 | example[name] = t 716 | 717 | return example 718 | 719 | def input_fn(params): 720 | """The actual input function.""" 721 | batch_size = params["batch_size"] 722 | 723 | # For training, we want a lot of parallel reading and shuffling. 724 | # For eval, we want no shuffling and parallel reading doesn't matter. 725 | d = tf.data.TFRecordDataset(input_file) 726 | if is_training: 727 | d = d.repeat() 728 | d = d.shuffle(buffer_size=100) 729 | 730 | d = d.apply( 731 | tf.contrib.data.map_and_batch( 732 | lambda record: _decode_record(record, name_to_features), 733 | batch_size=batch_size, 734 | drop_remainder=drop_remainder)) 735 | 736 | return d 737 | 738 | return input_fn 739 | 740 | 741 | RawResult = collections.namedtuple("RawResult", 742 | ["unique_id", "start_logits", "end_logits"]) 743 | 744 | 745 | def write_predictions(all_examples, all_features, all_results, n_best_size, 746 | max_answer_length, do_lower_case, output_prediction_file, 747 | output_nbest_file, output_null_log_odds_file): 748 | """Write final predictions to the json file and log-odds of null if needed.""" 749 | tf.logging.info("Writing predictions to: %s" % (output_prediction_file)) 750 | tf.logging.info("Writing nbest to: %s" % (output_nbest_file)) 751 | 752 | example_index_to_features = collections.defaultdict(list) 753 | for feature in all_features: 754 | example_index_to_features[feature.example_index].append(feature) 755 | 756 | unique_id_to_result = {} 757 | for result in all_results: 758 | unique_id_to_result[result.unique_id] = result 759 | 760 | _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name 761 | "PrelimPrediction", 762 | ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]) 763 | 764 | all_predictions = collections.OrderedDict() 765 | all_nbest_json = collections.OrderedDict() 766 | scores_diff_json = collections.OrderedDict() 767 | 768 | for (example_index, example) in enumerate(all_examples): 769 | features = example_index_to_features[example_index] 770 | 771 | prelim_predictions = [] 772 | # keep track of the minimum score of null start+end of position 0 773 | score_null = 1000000 # large and positive 774 | min_null_feature_index = 0 # the paragraph slice with min mull score 775 | null_start_logit = 0 # the start logit at the slice with min null score 776 | null_end_logit = 0 # the end logit at the slice with min null score 777 | for (feature_index, feature) in enumerate(features): 778 | result = unique_id_to_result[feature.unique_id] 779 | start_indexes = _get_best_indexes(result.start_logits, n_best_size) 780 | end_indexes = _get_best_indexes(result.end_logits, n_best_size) 781 | # if we could have irrelevant answers, get the min score of irrelevant 782 | if FLAGS.version_2_with_negative: 783 | feature_null_score = result.start_logits[0] + result.end_logits[0] 784 | if feature_null_score < score_null: 785 | score_null = feature_null_score 786 | min_null_feature_index = feature_index 787 | null_start_logit = result.start_logits[0] 788 | null_end_logit = result.end_logits[0] 789 | for start_index in start_indexes: 790 | for end_index in end_indexes: 791 | # We could hypothetically create invalid predictions, e.g., predict 792 | # that the start of the span is in the question. We throw out all 793 | # invalid predictions. 794 | if start_index >= len(feature.tokens): 795 | continue 796 | if end_index >= len(feature.tokens): 797 | continue 798 | if start_index not in feature.token_to_orig_map: 799 | continue 800 | if end_index not in feature.token_to_orig_map: 801 | continue 802 | if not feature.token_is_max_context.get(start_index, False): 803 | continue 804 | if end_index < start_index: 805 | continue 806 | length = end_index - start_index + 1 807 | if length > max_answer_length: 808 | continue 809 | prelim_predictions.append( 810 | _PrelimPrediction( 811 | feature_index=feature_index, 812 | start_index=start_index, 813 | end_index=end_index, 814 | start_logit=result.start_logits[start_index], 815 | end_logit=result.end_logits[end_index])) 816 | 817 | if FLAGS.version_2_with_negative: 818 | prelim_predictions.append( 819 | _PrelimPrediction( 820 | feature_index=min_null_feature_index, 821 | start_index=0, 822 | end_index=0, 823 | start_logit=null_start_logit, 824 | end_logit=null_end_logit)) 825 | prelim_predictions = sorted( 826 | prelim_predictions, 827 | key=lambda x: (x.start_logit + x.end_logit), 828 | reverse=True) 829 | 830 | _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name 831 | "NbestPrediction", ["text", "start_logit", "end_logit"]) 832 | 833 | seen_predictions = {} 834 | nbest = [] 835 | for pred in prelim_predictions: 836 | if len(nbest) >= n_best_size: 837 | break 838 | feature = features[pred.feature_index] 839 | if pred.start_index > 0: # this is a non-null prediction 840 | tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] 841 | orig_doc_start = feature.token_to_orig_map[pred.start_index] 842 | orig_doc_end = feature.token_to_orig_map[pred.end_index] 843 | orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] 844 | tok_text = " ".join(tok_tokens) 845 | 846 | # De-tokenize WordPieces that have been split off. 847 | tok_text = tok_text.replace(" ##", "") 848 | tok_text = tok_text.replace("##", "") 849 | 850 | # Clean whitespace 851 | tok_text = tok_text.strip() 852 | tok_text = " ".join(tok_text.split()) 853 | orig_text = " ".join(orig_tokens) 854 | 855 | final_text = get_final_text(tok_text, orig_text, do_lower_case) 856 | if final_text in seen_predictions: 857 | continue 858 | 859 | seen_predictions[final_text] = True 860 | else: 861 | final_text = "" 862 | seen_predictions[final_text] = True 863 | 864 | nbest.append( 865 | _NbestPrediction( 866 | text=final_text, 867 | start_logit=pred.start_logit, 868 | end_logit=pred.end_logit)) 869 | 870 | # if we didn't inlude the empty option in the n-best, inlcude it 871 | if FLAGS.version_2_with_negative: 872 | if "" not in seen_predictions: 873 | nbest.append( 874 | _NbestPrediction( 875 | text="", start_logit=null_start_logit, 876 | end_logit=null_end_logit)) 877 | # In very rare edge cases we could have no valid predictions. So we 878 | # just create a nonce prediction in this case to avoid failure. 879 | if not nbest: 880 | nbest.append( 881 | _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) 882 | 883 | assert len(nbest) >= 1 884 | 885 | total_scores = [] 886 | best_non_null_entry = None 887 | for entry in nbest: 888 | total_scores.append(entry.start_logit + entry.end_logit) 889 | if not best_non_null_entry: 890 | if entry.text: 891 | best_non_null_entry = entry 892 | 893 | probs = _compute_softmax(total_scores) 894 | 895 | nbest_json = [] 896 | for (i, entry) in enumerate(nbest): 897 | output = collections.OrderedDict() 898 | output["text"] = entry.text 899 | output["probability"] = probs[i] 900 | output["start_logit"] = entry.start_logit 901 | output["end_logit"] = entry.end_logit 902 | nbest_json.append(output) 903 | 904 | assert len(nbest_json) >= 1 905 | 906 | if not FLAGS.version_2_with_negative: 907 | all_predictions[example.qas_id] = nbest_json[0]["text"] 908 | else: 909 | # predict "" iff the null score - the score of best non-null > threshold 910 | score_diff = score_null - best_non_null_entry.start_logit - ( 911 | best_non_null_entry.end_logit) 912 | scores_diff_json[example.qas_id] = score_diff 913 | if score_diff > FLAGS.null_score_diff_threshold: 914 | all_predictions[example.qas_id] = "" 915 | else: 916 | all_predictions[example.qas_id] = best_non_null_entry.text 917 | 918 | all_nbest_json[example.qas_id] = nbest_json 919 | 920 | with tf.gfile.GFile(output_prediction_file, "w") as writer: 921 | writer.write(json.dumps(all_predictions, indent=4) + "\n") 922 | 923 | with tf.gfile.GFile(output_nbest_file, "w") as writer: 924 | writer.write(json.dumps(all_nbest_json, indent=4) + "\n") 925 | 926 | if FLAGS.version_2_with_negative: 927 | with tf.gfile.GFile(output_null_log_odds_file, "w") as writer: 928 | writer.write(json.dumps(scores_diff_json, indent=4) + "\n") 929 | 930 | 931 | def get_final_text(pred_text, orig_text, do_lower_case): 932 | """Project the tokenized prediction back to the original text.""" 933 | 934 | # When we created the data, we kept track of the alignment between original 935 | # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So 936 | # now `orig_text` contains the span of our original text corresponding to the 937 | # span that we predicted. 938 | # 939 | # However, `orig_text` may contain extra characters that we don't want in 940 | # our prediction. 941 | # 942 | # For example, let's say: 943 | # pred_text = steve smith 944 | # orig_text = Steve Smith's 945 | # 946 | # We don't want to return `orig_text` because it contains the extra "'s". 947 | # 948 | # We don't want to return `pred_text` because it's already been normalized 949 | # (the SQuAD eval script also does punctuation stripping/lower casing but 950 | # our tokenizer does additional normalization like stripping accent 951 | # characters). 952 | # 953 | # What we really want to return is "Steve Smith". 954 | # 955 | # Therefore, we have to apply a semi-complicated alignment heruistic between 956 | # `pred_text` and `orig_text` to get a character-to-charcter alignment. This 957 | # can fail in certain cases in which case we just return `orig_text`. 958 | 959 | def _strip_spaces(text): 960 | ns_chars = [] 961 | ns_to_s_map = collections.OrderedDict() 962 | for (i, c) in enumerate(text): 963 | if c == " ": 964 | continue 965 | ns_to_s_map[len(ns_chars)] = i 966 | ns_chars.append(c) 967 | ns_text = "".join(ns_chars) 968 | return (ns_text, ns_to_s_map) 969 | 970 | # We first tokenize `orig_text`, strip whitespace from the result 971 | # and `pred_text`, and check if they are the same length. If they are 972 | # NOT the same length, the heuristic has failed. If they are the same 973 | # length, we assume the characters are one-to-one aligned. 974 | tokenizer = tokenization.BasicTokenizer(do_lower_case=do_lower_case) 975 | 976 | tok_text = " ".join(tokenizer.tokenize(orig_text)) 977 | 978 | start_position = tok_text.find(pred_text) 979 | if start_position == -1: 980 | if FLAGS.verbose_logging: 981 | tf.logging.info( 982 | "Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) 983 | return orig_text 984 | end_position = start_position + len(pred_text) - 1 985 | 986 | (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) 987 | (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) 988 | 989 | if len(orig_ns_text) != len(tok_ns_text): 990 | if FLAGS.verbose_logging: 991 | tf.logging.info("Length not equal after stripping spaces: '%s' vs '%s'", 992 | orig_ns_text, tok_ns_text) 993 | return orig_text 994 | 995 | # We then project the characters in `pred_text` back to `orig_text` using 996 | # the character-to-character alignment. 997 | tok_s_to_ns_map = {} 998 | for (i, tok_index) in six.iteritems(tok_ns_to_s_map): 999 | tok_s_to_ns_map[tok_index] = i 1000 | 1001 | orig_start_position = None 1002 | if start_position in tok_s_to_ns_map: 1003 | ns_start_position = tok_s_to_ns_map[start_position] 1004 | if ns_start_position in orig_ns_to_s_map: 1005 | orig_start_position = orig_ns_to_s_map[ns_start_position] 1006 | 1007 | if orig_start_position is None: 1008 | if FLAGS.verbose_logging: 1009 | tf.logging.info("Couldn't map start position") 1010 | return orig_text 1011 | 1012 | orig_end_position = None 1013 | if end_position in tok_s_to_ns_map: 1014 | ns_end_position = tok_s_to_ns_map[end_position] 1015 | if ns_end_position in orig_ns_to_s_map: 1016 | orig_end_position = orig_ns_to_s_map[ns_end_position] 1017 | 1018 | if orig_end_position is None: 1019 | if FLAGS.verbose_logging: 1020 | tf.logging.info("Couldn't map end position") 1021 | return orig_text 1022 | 1023 | output_text = orig_text[orig_start_position:(orig_end_position + 1)] 1024 | return output_text 1025 | 1026 | 1027 | def _get_best_indexes(logits, n_best_size): 1028 | """Get the n-best logits from a list.""" 1029 | index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) 1030 | 1031 | best_indexes = [] 1032 | for i in range(len(index_and_score)): 1033 | if i >= n_best_size: 1034 | break 1035 | best_indexes.append(index_and_score[i][0]) 1036 | return best_indexes 1037 | 1038 | 1039 | def _compute_softmax(scores): 1040 | """Compute softmax probability over raw logits.""" 1041 | if not scores: 1042 | return [] 1043 | 1044 | max_score = None 1045 | for score in scores: 1046 | if max_score is None or score > max_score: 1047 | max_score = score 1048 | 1049 | exp_scores = [] 1050 | total_sum = 0.0 1051 | for score in scores: 1052 | x = math.exp(score - max_score) 1053 | exp_scores.append(x) 1054 | total_sum += x 1055 | 1056 | probs = [] 1057 | for score in exp_scores: 1058 | probs.append(score / total_sum) 1059 | return probs 1060 | 1061 | 1062 | class FeatureWriter(object): 1063 | """Writes InputFeature to TF example file.""" 1064 | 1065 | def __init__(self, filename, is_training): 1066 | self.filename = filename 1067 | self.is_training = is_training 1068 | self.num_features = 0 1069 | self._writer = tf.python_io.TFRecordWriter(filename) 1070 | 1071 | def process_feature(self, feature): 1072 | """Write a InputFeature to the TFRecordWriter as a tf.train.Example.""" 1073 | self.num_features += 1 1074 | 1075 | def create_int_feature(values): 1076 | feature = tf.train.Feature( 1077 | int64_list=tf.train.Int64List(value=list(values))) 1078 | return feature 1079 | 1080 | features = collections.OrderedDict() 1081 | features["unique_ids"] = create_int_feature([feature.unique_id]) 1082 | features["input_ids"] = create_int_feature(feature.input_ids) 1083 | features["input_mask"] = create_int_feature(feature.input_mask) 1084 | features["segment_ids"] = create_int_feature(feature.segment_ids) 1085 | 1086 | if self.is_training: 1087 | features["start_positions"] = create_int_feature([feature.start_position]) 1088 | features["end_positions"] = create_int_feature([feature.end_position]) 1089 | impossible = 0 1090 | if feature.is_impossible: 1091 | impossible = 1 1092 | features["is_impossible"] = create_int_feature([impossible]) 1093 | 1094 | tf_example = tf.train.Example(features=tf.train.Features(feature=features)) 1095 | self._writer.write(tf_example.SerializeToString()) 1096 | 1097 | def close(self): 1098 | self._writer.close() 1099 | 1100 | 1101 | def validate_flags_or_throw(bert_config): 1102 | """Validate the input FLAGS or throw an exception.""" 1103 | tokenization.validate_case_matches_checkpoint(FLAGS.do_lower_case, 1104 | FLAGS.init_checkpoint) 1105 | 1106 | if not FLAGS.do_train and not FLAGS.do_predict: 1107 | raise ValueError("At least one of `do_train` or `do_predict` must be True.") 1108 | 1109 | if FLAGS.do_train: 1110 | if not FLAGS.train_file: 1111 | raise ValueError( 1112 | "If `do_train` is True, then `train_file` must be specified.") 1113 | if FLAGS.do_predict: 1114 | if not FLAGS.predict_file: 1115 | raise ValueError( 1116 | "If `do_predict` is True, then `predict_file` must be specified.") 1117 | 1118 | if FLAGS.max_seq_length > bert_config.max_position_embeddings: 1119 | raise ValueError( 1120 | "Cannot use sequence length %d because the BERT model " 1121 | "was only trained up to sequence length %d" % 1122 | (FLAGS.max_seq_length, bert_config.max_position_embeddings)) 1123 | 1124 | if FLAGS.max_seq_length <= FLAGS.max_query_length + 3: 1125 | raise ValueError( 1126 | "The max_seq_length (%d) must be greater than max_query_length " 1127 | "(%d) + 3" % (FLAGS.max_seq_length, FLAGS.max_query_length)) 1128 | 1129 | 1130 | def main(_): 1131 | tf.logging.set_verbosity(tf.logging.INFO) 1132 | 1133 | bert_config = modeling.BertConfig.from_json_file(FLAGS.bert_config_file) 1134 | 1135 | validate_flags_or_throw(bert_config) 1136 | 1137 | tf.gfile.MakeDirs(FLAGS.output_dir) 1138 | 1139 | tokenizer = tokenization.FullTokenizer( 1140 | vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) 1141 | 1142 | tpu_cluster_resolver = None 1143 | if FLAGS.use_tpu and FLAGS.tpu_name: 1144 | tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver( 1145 | FLAGS.tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project) 1146 | 1147 | is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2 1148 | run_config = tf.contrib.tpu.RunConfig( 1149 | cluster=tpu_cluster_resolver, 1150 | master=FLAGS.master, 1151 | model_dir=FLAGS.output_dir, 1152 | save_checkpoints_steps=FLAGS.save_checkpoints_steps, 1153 | tpu_config=tf.contrib.tpu.TPUConfig( 1154 | iterations_per_loop=FLAGS.iterations_per_loop, 1155 | num_shards=FLAGS.num_tpu_cores, 1156 | per_host_input_for_training=is_per_host)) 1157 | 1158 | train_examples = None 1159 | num_train_steps = None 1160 | num_warmup_steps = None 1161 | if FLAGS.do_train: 1162 | # train_examples = read_squad_examples( 1163 | train_examples = read_baike_examples( 1164 | input_file=FLAGS.train_file, is_training=True) 1165 | num_train_steps = int( 1166 | len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) 1167 | num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) 1168 | 1169 | # Pre-shuffle the input to avoid having to make a very large shuffle 1170 | # buffer in in the `input_fn`. 1171 | rng = random.Random(12345) 1172 | rng.shuffle(train_examples) 1173 | 1174 | model_fn = model_fn_builder( 1175 | bert_config=bert_config, 1176 | init_checkpoint=FLAGS.init_checkpoint, 1177 | learning_rate=FLAGS.learning_rate, 1178 | num_train_steps=num_train_steps, 1179 | num_warmup_steps=num_warmup_steps, 1180 | use_tpu=FLAGS.use_tpu, 1181 | use_one_hot_embeddings=FLAGS.use_tpu) 1182 | 1183 | # If TPU is not available, this will fall back to normal Estimator on CPU 1184 | # or GPU. 1185 | estimator = tf.contrib.tpu.TPUEstimator( 1186 | use_tpu=FLAGS.use_tpu, 1187 | model_fn=model_fn, 1188 | config=run_config, 1189 | train_batch_size=FLAGS.train_batch_size, 1190 | predict_batch_size=FLAGS.predict_batch_size) 1191 | 1192 | if FLAGS.do_train: 1193 | # We write to a temporary file to avoid storing very large constant tensors 1194 | # in memory. 1195 | train_writer = FeatureWriter( 1196 | filename=os.path.join(FLAGS.output_dir, "train.tf_record"), 1197 | is_training=True) 1198 | convert_examples_to_features( 1199 | examples=train_examples, 1200 | tokenizer=tokenizer, 1201 | max_seq_length=FLAGS.max_seq_length, 1202 | doc_stride=FLAGS.doc_stride, 1203 | max_query_length=FLAGS.max_query_length, 1204 | is_training=True, 1205 | output_fn=train_writer.process_feature) 1206 | train_writer.close() 1207 | 1208 | tf.logging.info("***** Running training *****") 1209 | tf.logging.info(" Num orig examples = %d", len(train_examples)) 1210 | tf.logging.info(" Num split examples = %d", train_writer.num_features) 1211 | tf.logging.info(" Batch size = %d", FLAGS.train_batch_size) 1212 | tf.logging.info(" Num steps = %d", num_train_steps) 1213 | del train_examples 1214 | 1215 | train_input_fn = input_fn_builder( 1216 | input_file=train_writer.filename, 1217 | seq_length=FLAGS.max_seq_length, 1218 | is_training=True, 1219 | drop_remainder=True) 1220 | estimator.train(input_fn=train_input_fn, max_steps=num_train_steps) 1221 | 1222 | if FLAGS.do_predict: 1223 | # eval_examples = read_squad_examples( 1224 | eval_examples = read_baike_examples( 1225 | input_file=FLAGS.predict_file, is_training=False) 1226 | 1227 | eval_writer = FeatureWriter( 1228 | filename=os.path.join(FLAGS.output_dir, "eval.tf_record"), 1229 | is_training=False) 1230 | eval_features = [] 1231 | 1232 | def append_feature(feature): 1233 | eval_features.append(feature) 1234 | eval_writer.process_feature(feature) 1235 | 1236 | convert_examples_to_features( 1237 | examples=eval_examples, 1238 | tokenizer=tokenizer, 1239 | max_seq_length=FLAGS.max_seq_length, 1240 | doc_stride=FLAGS.doc_stride, 1241 | max_query_length=FLAGS.max_query_length, 1242 | is_training=False, 1243 | output_fn=append_feature) 1244 | eval_writer.close() 1245 | 1246 | tf.logging.info("***** Running predictions *****") 1247 | tf.logging.info(" Num orig examples = %d", len(eval_examples)) 1248 | tf.logging.info(" Num split examples = %d", len(eval_features)) 1249 | tf.logging.info(" Batch size = %d", FLAGS.predict_batch_size) 1250 | 1251 | all_results = [] 1252 | 1253 | predict_input_fn = input_fn_builder( 1254 | input_file=eval_writer.filename, 1255 | seq_length=FLAGS.max_seq_length, 1256 | is_training=False, 1257 | drop_remainder=False) 1258 | 1259 | # If running eval on the TPU, you will need to specify the number of 1260 | # steps. 1261 | all_results = [] 1262 | for result in estimator.predict( 1263 | predict_input_fn, yield_single_examples=True): 1264 | if len(all_results) % 1000 == 0: 1265 | tf.logging.info("Processing example: %d" % (len(all_results))) 1266 | unique_id = int(result["unique_ids"]) 1267 | start_logits = [float(x) for x in result["start_logits"].flat] 1268 | end_logits = [float(x) for x in result["end_logits"].flat] 1269 | all_results.append( 1270 | RawResult( 1271 | unique_id=unique_id, 1272 | start_logits=start_logits, 1273 | end_logits=end_logits)) 1274 | 1275 | output_prediction_file = os.path.join(FLAGS.output_dir, "predictions.json") 1276 | output_nbest_file = os.path.join(FLAGS.output_dir, "nbest_predictions.json") 1277 | output_null_log_odds_file = os.path.join(FLAGS.output_dir, "null_odds.json") 1278 | 1279 | write_predictions(eval_examples, eval_features, all_results, 1280 | FLAGS.n_best_size, FLAGS.max_answer_length, 1281 | FLAGS.do_lower_case, output_prediction_file, 1282 | output_nbest_file, output_null_log_odds_file) 1283 | 1284 | 1285 | if __name__ == "__main__": 1286 | flags.mark_flag_as_required("vocab_file") 1287 | flags.mark_flag_as_required("bert_config_file") 1288 | flags.mark_flag_as_required("output_dir") 1289 | tf.app.run() 1290 | -------------------------------------------------------------------------------- /keyword-bert/sample_text.txt: -------------------------------------------------------------------------------- 1 | This text is included to make sure Unicode is handled properly: 力加勝北区ᴵᴺᵀᵃছজটডণত 2 | Text should be one-sentence-per-line, with empty lines between documents. 3 | This sample text is public domain and was randomly selected from Project Guttenberg. 4 | 5 | The rain had only ceased with the gray streaks of morning at Blazing Star, and the settlement awoke to a moral sense of cleanliness, and the finding of forgotten knives, tin cups, and smaller camp utensils, where the heavy showers had washed away the debris and dust heaps before the cabin doors. 6 | Indeed, it was recorded in Blazing Star that a fortunate early riser had once picked up on the highway a solid chunk of gold quartz which the rain had freed from its incumbering soil, and washed into immediate and glittering popularity. 7 | Possibly this may have been the reason why early risers in that locality, during the rainy season, adopted a thoughtful habit of body, and seldom lifted their eyes to the rifted or india-ink washed skies above them. 8 | "Cass" Beard had risen early that morning, but not with a view to discovery. 9 | A leak in his cabin roof,--quite consistent with his careless, improvident habits,--had roused him at 4 A. M., with a flooded "bunk" and wet blankets. 10 | The chips from his wood pile refused to kindle a fire to dry his bed-clothes, and he had recourse to a more provident neighbor's to supply the deficiency. 11 | This was nearly opposite. 12 | Mr. Cassius crossed the highway, and stopped suddenly. 13 | Something glittered in the nearest red pool before him. 14 | Gold, surely! 15 | But, wonderful to relate, not an irregular, shapeless fragment of crude ore, fresh from Nature's crucible, but a bit of jeweler's handicraft in the form of a plain gold ring. 16 | Looking at it more attentively, he saw that it bore the inscription, "May to Cass." 17 | Like most of his fellow gold-seekers, Cass was superstitious. 18 | 19 | The fountain of classic wisdom, Hypatia herself. 20 | As the ancient sage--the name is unimportant to a monk--pumped water nightly that he might study by day, so I, the guardian of cloaks and parasols, at the sacred doors of her lecture-room, imbibe celestial knowledge. 21 | From my youth I felt in me a soul above the matter-entangled herd. 22 | She revealed to me the glorious fact, that I am a spark of Divinity itself. 23 | A fallen star, I am, sir!' continued he, pensively, stroking his lean stomach--'a fallen star!--fallen, if the dignity of philosophy will allow of the simile, among the hogs of the lower world--indeed, even into the hog-bucket itself. Well, after all, I will show you the way to the Archbishop's. 24 | There is a philosophic pleasure in opening one's treasures to the modest young. 25 | Perhaps you will assist me by carrying this basket of fruit?' And the little man jumped up, put his basket on Philammon's head, and trotted off up a neighbouring street. 26 | Philammon followed, half contemptuous, half wondering at what this philosophy might be, which could feed the self-conceit of anything so abject as his ragged little apish guide; 27 | but the novel roar and whirl of the street, the perpetual stream of busy faces, the line of curricles, palanquins, laden asses, camels, elephants, which met and passed him, and squeezed him up steps and into doorways, as they threaded their way through the great Moon-gate into the ample street beyond, drove everything from his mind but wondering curiosity, and a vague, helpless dread of that great living wilderness, more terrible than any dead wilderness of sand which he had left behind. 28 | Already he longed for the repose, the silence of the Laura--for faces which knew him and smiled upon him; but it was too late to turn back now. 29 | His guide held on for more than a mile up the great main street, crossed in the centre of the city, at right angles, by one equally magnificent, at each end of which, miles away, appeared, dim and distant over the heads of the living stream of passengers, the yellow sand-hills of the desert; 30 | while at the end of the vista in front of them gleamed the blue harbour, through a network of countless masts. 31 | At last they reached the quay at the opposite end of the street; 32 | and there burst on Philammon's astonished eyes a vast semicircle of blue sea, ringed with palaces and towers. 33 | He stopped involuntarily; and his little guide stopped also, and looked askance at the young monk, to watch the effect which that grand panorama should produce on him. 34 | -------------------------------------------------------------------------------- /keyword-bert/tokenization.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Tokenization classes.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import collections 22 | import re 23 | import unicodedata 24 | import six 25 | import tensorflow as tf 26 | 27 | 28 | def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): 29 | """Checks whether the casing config is consistent with the checkpoint name.""" 30 | 31 | # The casing has to be passed in by the user and there is no explicit check 32 | # as to whether it matches the checkpoint. The casing information probably 33 | # should have been stored in the bert_config.json file, but it's not, so 34 | # we have to heuristically detect it to validate. 35 | 36 | if not init_checkpoint: 37 | return 38 | 39 | m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) 40 | if m is None: 41 | return 42 | 43 | model_name = m.group(1) 44 | 45 | lower_models = [ 46 | "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", 47 | "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" 48 | ] 49 | 50 | cased_models = [ 51 | "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", 52 | "multi_cased_L-12_H-768_A-12" 53 | ] 54 | 55 | is_bad_config = False 56 | if model_name in lower_models and not do_lower_case: 57 | is_bad_config = True 58 | actual_flag = "False" 59 | case_name = "lowercased" 60 | opposite_flag = "True" 61 | 62 | if model_name in cased_models and do_lower_case: 63 | is_bad_config = True 64 | actual_flag = "True" 65 | case_name = "cased" 66 | opposite_flag = "False" 67 | 68 | if is_bad_config: 69 | raise ValueError( 70 | "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " 71 | "However, `%s` seems to be a %s model, so you " 72 | "should pass in `--do_lower_case=%s` so that the fine-tuning matches " 73 | "how the model was pre-training. If this error is wrong, please " 74 | "just comment out this check." % (actual_flag, init_checkpoint, 75 | model_name, case_name, opposite_flag)) 76 | 77 | 78 | def convert_to_unicode(text): 79 | """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" 80 | if six.PY3: 81 | if isinstance(text, str): 82 | return text 83 | elif isinstance(text, bytes): 84 | return text.decode("utf-8", "ignore") 85 | else: 86 | raise ValueError("Unsupported string type: %s" % (type(text))) 87 | elif six.PY2: 88 | if isinstance(text, str): 89 | return text.decode("utf-8", "ignore") 90 | elif isinstance(text, unicode): 91 | return text 92 | else: 93 | raise ValueError("Unsupported string type: %s" % (type(text))) 94 | else: 95 | raise ValueError("Not running on Python2 or Python 3?") 96 | 97 | 98 | def printable_text(text): 99 | """Returns text encoded in a way suitable for print or `tf.logging`.""" 100 | 101 | # These functions want `str` for both Python2 and Python3, but in one case 102 | # it's a Unicode string and in the other it's a byte string. 103 | if six.PY3: 104 | if isinstance(text, str): 105 | return text 106 | elif isinstance(text, bytes): 107 | return text.decode("utf-8", "ignore") 108 | else: 109 | raise ValueError("Unsupported string type: %s" % (type(text))) 110 | elif six.PY2: 111 | if isinstance(text, str): 112 | return text 113 | elif isinstance(text, unicode): 114 | return text.encode("utf-8") 115 | else: 116 | raise ValueError("Unsupported string type: %s" % (type(text))) 117 | else: 118 | raise ValueError("Not running on Python2 or Python 3?") 119 | 120 | 121 | def load_vocab(vocab_file): 122 | """Loads a vocabulary file into a dictionary.""" 123 | vocab = collections.OrderedDict() 124 | index = 0 125 | with tf.gfile.GFile(vocab_file, "r") as reader: 126 | while True: 127 | token = convert_to_unicode(reader.readline()) 128 | if not token: 129 | break 130 | token = token.strip() 131 | vocab[token] = index 132 | index += 1 133 | return vocab 134 | 135 | 136 | def convert_by_vocab(vocab, items): 137 | """Converts a sequence of [tokens|ids] using the vocab.""" 138 | output = [] 139 | for item in items: 140 | output.append(vocab[item]) 141 | return output 142 | 143 | 144 | def convert_tokens_to_ids(vocab, tokens): 145 | return convert_by_vocab(vocab, tokens) 146 | 147 | 148 | def convert_ids_to_tokens(inv_vocab, ids): 149 | return convert_by_vocab(inv_vocab, ids) 150 | 151 | 152 | def whitespace_tokenize(text): 153 | """Runs basic whitespace cleaning and splitting on a piece of text.""" 154 | text = text.strip() 155 | if not text: 156 | return [] 157 | tokens = text.split() 158 | return tokens 159 | 160 | 161 | class FullTokenizer(object): 162 | """Runs end-to-end tokenziation.""" 163 | 164 | def __init__(self, vocab_file, do_lower_case=True): 165 | self.vocab = load_vocab(vocab_file) 166 | self.inv_vocab = {v: k for k, v in self.vocab.items()} 167 | self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) 168 | self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) 169 | 170 | def tokenize(self, text): 171 | split_tokens = [] 172 | for token in self.basic_tokenizer.tokenize(text): 173 | for sub_token in self.wordpiece_tokenizer.tokenize(token): 174 | split_tokens.append(sub_token) 175 | 176 | return split_tokens 177 | 178 | def convert_tokens_to_ids(self, tokens): 179 | return convert_by_vocab(self.vocab, tokens) 180 | 181 | def convert_ids_to_tokens(self, ids): 182 | return convert_by_vocab(self.inv_vocab, ids) 183 | 184 | 185 | class BasicTokenizer(object): 186 | """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" 187 | 188 | def __init__(self, do_lower_case=True): 189 | """Constructs a BasicTokenizer. 190 | 191 | Args: 192 | do_lower_case: Whether to lower case the input. 193 | """ 194 | self.do_lower_case = do_lower_case 195 | 196 | def tokenize(self, text): 197 | """Tokenizes a piece of text.""" 198 | text = convert_to_unicode(text) 199 | text = self._clean_text(text) 200 | 201 | # This was added on November 1st, 2018 for the multilingual and Chinese 202 | # models. This is also applied to the English models now, but it doesn't 203 | # matter since the English models were not trained on any Chinese data 204 | # and generally don't have any Chinese data in them (there are Chinese 205 | # characters in the vocabulary because Wikipedia does have some Chinese 206 | # words in the English Wikipedia.). 207 | text = self._tokenize_chinese_chars(text) 208 | 209 | orig_tokens = whitespace_tokenize(text) 210 | split_tokens = [] 211 | for token in orig_tokens: 212 | if self.do_lower_case: 213 | token = token.lower() 214 | token = self._run_strip_accents(token) 215 | split_tokens.extend(self._run_split_on_punc(token)) 216 | 217 | output_tokens = whitespace_tokenize(" ".join(split_tokens)) 218 | return output_tokens 219 | 220 | def _run_strip_accents(self, text): 221 | """Strips accents from a piece of text.""" 222 | text = unicodedata.normalize("NFD", text) 223 | output = [] 224 | for char in text: 225 | cat = unicodedata.category(char) 226 | if cat == "Mn": 227 | continue 228 | output.append(char) 229 | return "".join(output) 230 | 231 | def _run_split_on_punc(self, text): 232 | """Splits punctuation on a piece of text.""" 233 | chars = list(text) 234 | i = 0 235 | start_new_word = True 236 | output = [] 237 | while i < len(chars): 238 | char = chars[i] 239 | if _is_punctuation(char): 240 | output.append([char]) 241 | start_new_word = True 242 | else: 243 | if start_new_word: 244 | output.append([]) 245 | start_new_word = False 246 | output[-1].append(char) 247 | i += 1 248 | 249 | return ["".join(x) for x in output] 250 | 251 | def _tokenize_chinese_chars(self, text): 252 | """Adds whitespace around any CJK character.""" 253 | output = [] 254 | for char in text: 255 | cp = ord(char) 256 | if self._is_chinese_char(cp): 257 | output.append(" ") 258 | output.append(char) 259 | output.append(" ") 260 | else: 261 | output.append(char) 262 | return "".join(output) 263 | 264 | def _is_chinese_char(self, cp): 265 | """Checks whether CP is the codepoint of a CJK character.""" 266 | # This defines a "chinese character" as anything in the CJK Unicode block: 267 | # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) 268 | # 269 | # Note that the CJK Unicode block is NOT all Japanese and Korean characters, 270 | # despite its name. The modern Korean Hangul alphabet is a different block, 271 | # as is Japanese Hiragana and Katakana. Those alphabets are used to write 272 | # space-separated words, so they are not treated specially and handled 273 | # like the all of the other languages. 274 | if ((cp >= 0x4E00 and cp <= 0x9FFF) or # 275 | (cp >= 0x3400 and cp <= 0x4DBF) or # 276 | (cp >= 0x20000 and cp <= 0x2A6DF) or # 277 | (cp >= 0x2A700 and cp <= 0x2B73F) or # 278 | (cp >= 0x2B740 and cp <= 0x2B81F) or # 279 | (cp >= 0x2B820 and cp <= 0x2CEAF) or 280 | (cp >= 0xF900 and cp <= 0xFAFF) or # 281 | (cp >= 0x2F800 and cp <= 0x2FA1F)): # 282 | return True 283 | 284 | return False 285 | 286 | def _clean_text(self, text): 287 | """Performs invalid character removal and whitespace cleanup on text.""" 288 | output = [] 289 | for char in text: 290 | cp = ord(char) 291 | if cp == 0 or cp == 0xfffd or _is_control(char): 292 | continue 293 | if _is_whitespace(char): 294 | output.append(" ") 295 | else: 296 | output.append(char) 297 | return "".join(output) 298 | 299 | 300 | class WordpieceTokenizer(object): 301 | """Runs WordPiece tokenziation.""" 302 | 303 | def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): 304 | self.vocab = vocab 305 | self.unk_token = unk_token 306 | self.max_input_chars_per_word = max_input_chars_per_word 307 | 308 | def tokenize(self, text): 309 | """Tokenizes a piece of text into its word pieces. 310 | 311 | This uses a greedy longest-match-first algorithm to perform tokenization 312 | using the given vocabulary. 313 | 314 | For example: 315 | input = "unaffable" 316 | output = ["un", "##aff", "##able"] 317 | 318 | Args: 319 | text: A single token or whitespace separated tokens. This should have 320 | already been passed through `BasicTokenizer. 321 | 322 | Returns: 323 | A list of wordpiece tokens. 324 | """ 325 | 326 | text = convert_to_unicode(text) 327 | 328 | output_tokens = [] 329 | for token in whitespace_tokenize(text): 330 | chars = list(token) 331 | if len(chars) > self.max_input_chars_per_word: 332 | output_tokens.append(self.unk_token) 333 | continue 334 | 335 | is_bad = False 336 | start = 0 337 | sub_tokens = [] 338 | while start < len(chars): 339 | end = len(chars) 340 | cur_substr = None 341 | while start < end: 342 | substr = "".join(chars[start:end]) 343 | if start > 0: 344 | substr = "##" + substr 345 | if substr in self.vocab: 346 | cur_substr = substr 347 | break 348 | end -= 1 349 | if cur_substr is None: 350 | is_bad = True 351 | break 352 | sub_tokens.append(cur_substr) 353 | start = end 354 | 355 | if is_bad: 356 | output_tokens.append(self.unk_token) 357 | else: 358 | output_tokens.extend(sub_tokens) 359 | return output_tokens 360 | 361 | 362 | def _is_whitespace(char): 363 | """Checks whether `chars` is a whitespace character.""" 364 | # \t, \n, and \r are technically contorl characters but we treat them 365 | # as whitespace since they are generally considered as such. 366 | if char == " " or char == "\t" or char == "\n" or char == "\r": 367 | return True 368 | cat = unicodedata.category(char) 369 | if cat == "Zs": 370 | return True 371 | return False 372 | 373 | 374 | def _is_control(char): 375 | """Checks whether `chars` is a control character.""" 376 | # These are technically control characters but we count them as whitespace 377 | # characters. 378 | if char == "\t" or char == "\n" or char == "\r": 379 | return False 380 | cat = unicodedata.category(char) 381 | if cat in ("Cc", "Cf"): 382 | return True 383 | return False 384 | 385 | 386 | def _is_punctuation(char): 387 | """Checks whether `chars` is a punctuation character.""" 388 | cp = ord(char) 389 | # We treat all non-letter/number ASCII as punctuation. 390 | # Characters such as "^", "$", and "`" are not in the Unicode 391 | # Punctuation class but we treat them as punctuation anyways, for 392 | # consistency. 393 | if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or 394 | (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): 395 | return True 396 | cat = unicodedata.category(char) 397 | if cat.startswith("P"): 398 | return True 399 | return False 400 | -------------------------------------------------------------------------------- /keyword-bert/tokenization_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import os 20 | import tempfile 21 | import tokenization 22 | import six 23 | import tensorflow as tf 24 | 25 | 26 | class TokenizationTest(tf.test.TestCase): 27 | 28 | def test_full_tokenizer(self): 29 | vocab_tokens = [ 30 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 31 | "##ing", "," 32 | ] 33 | with tempfile.NamedTemporaryFile(delete=False) as vocab_writer: 34 | if six.PY2: 35 | vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) 36 | else: 37 | vocab_writer.write("".join( 38 | [x + "\n" for x in vocab_tokens]).encode("utf-8")) 39 | 40 | vocab_file = vocab_writer.name 41 | 42 | tokenizer = tokenization.FullTokenizer(vocab_file) 43 | os.unlink(vocab_file) 44 | 45 | tokens = tokenizer.tokenize(u"UNwant\u00E9d,running") 46 | self.assertAllEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"]) 47 | 48 | self.assertAllEqual( 49 | tokenizer.convert_tokens_to_ids(tokens), [7, 4, 5, 10, 8, 9]) 50 | 51 | def test_chinese(self): 52 | tokenizer = tokenization.BasicTokenizer() 53 | 54 | self.assertAllEqual( 55 | tokenizer.tokenize(u"ah\u535A\u63A8zz"), 56 | [u"ah", u"\u535A", u"\u63A8", u"zz"]) 57 | 58 | def test_basic_tokenizer_lower(self): 59 | tokenizer = tokenization.BasicTokenizer(do_lower_case=True) 60 | 61 | self.assertAllEqual( 62 | tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), 63 | ["hello", "!", "how", "are", "you", "?"]) 64 | self.assertAllEqual(tokenizer.tokenize(u"H\u00E9llo"), ["hello"]) 65 | 66 | def test_basic_tokenizer_no_lower(self): 67 | tokenizer = tokenization.BasicTokenizer(do_lower_case=False) 68 | 69 | self.assertAllEqual( 70 | tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), 71 | ["HeLLo", "!", "how", "Are", "yoU", "?"]) 72 | 73 | def test_wordpiece_tokenizer(self): 74 | vocab_tokens = [ 75 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 76 | "##ing" 77 | ] 78 | 79 | vocab = {} 80 | for (i, token) in enumerate(vocab_tokens): 81 | vocab[token] = i 82 | tokenizer = tokenization.WordpieceTokenizer(vocab=vocab) 83 | 84 | self.assertAllEqual(tokenizer.tokenize(""), []) 85 | 86 | self.assertAllEqual( 87 | tokenizer.tokenize("unwanted running"), 88 | ["un", "##want", "##ed", "runn", "##ing"]) 89 | 90 | self.assertAllEqual( 91 | tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) 92 | 93 | def test_convert_tokens_to_ids(self): 94 | vocab_tokens = [ 95 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 96 | "##ing" 97 | ] 98 | 99 | vocab = {} 100 | for (i, token) in enumerate(vocab_tokens): 101 | vocab[token] = i 102 | 103 | self.assertAllEqual( 104 | tokenization.convert_tokens_to_ids( 105 | vocab, ["un", "##want", "##ed", "runn", "##ing"]), [7, 4, 5, 8, 9]) 106 | 107 | def test_is_whitespace(self): 108 | self.assertTrue(tokenization._is_whitespace(u" ")) 109 | self.assertTrue(tokenization._is_whitespace(u"\t")) 110 | self.assertTrue(tokenization._is_whitespace(u"\r")) 111 | self.assertTrue(tokenization._is_whitespace(u"\n")) 112 | self.assertTrue(tokenization._is_whitespace(u"\u00A0")) 113 | 114 | self.assertFalse(tokenization._is_whitespace(u"A")) 115 | self.assertFalse(tokenization._is_whitespace(u"-")) 116 | 117 | def test_is_control(self): 118 | self.assertTrue(tokenization._is_control(u"\u0005")) 119 | 120 | self.assertFalse(tokenization._is_control(u"A")) 121 | self.assertFalse(tokenization._is_control(u" ")) 122 | self.assertFalse(tokenization._is_control(u"\t")) 123 | self.assertFalse(tokenization._is_control(u"\r")) 124 | self.assertFalse(tokenization._is_control(u"\U0001F4A9")) 125 | 126 | def test_is_punctuation(self): 127 | self.assertTrue(tokenization._is_punctuation(u"-")) 128 | self.assertTrue(tokenization._is_punctuation(u"$")) 129 | self.assertTrue(tokenization._is_punctuation(u"`")) 130 | self.assertTrue(tokenization._is_punctuation(u".")) 131 | 132 | self.assertFalse(tokenization._is_punctuation(u"A")) 133 | self.assertFalse(tokenization._is_punctuation(u" ")) 134 | 135 | 136 | if __name__ == "__main__": 137 | tf.test.main() 138 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # project discription 2 | 3 | An implementation of the `Keyword-BERT` model mentioned 4 | in my paper [Keyword-Attentive Deep Semantic Matching](https://arxiv.org/abs/2003.11516) 5 | 6 | (**Plz cite this github repo if using this code or relevant ideas from this paper**) 7 | 8 | For detailed descriptions of the whole pipeline ,refer to my [Chinese blog](https://mp.weixin.qq.com/s/_QY2EhB-TiBcb5q0379McQ) (**Highly suggested**) 9 | 10 | For detailed code running guidance, refer to `keyword-bert/readme.md` 11 | --------------------------------------------------------------------------------