├── BERT ├── MY_DATASET │ ├── devData.csv │ └── trainData.csv └── bert │ ├── .gitignore │ ├── CONTRIBUTING.md │ ├── LICENSE │ ├── README.md │ ├── __init__.py │ ├── create_pretraining_data.py │ ├── extract_features.py │ ├── modeling.py │ ├── modeling_test.py │ ├── multilingual.md │ ├── optimization.py │ ├── optimization_test.py │ ├── requirements.txt │ ├── run_classifier.py │ ├── run_pretraining.py │ ├── run_squad.py │ ├── sample_text.txt │ ├── tokenization.py │ └── tokenization_test.py ├── BiLstm ├── get_train_data.py ├── mode_structure.py ├── mode_trainning.py ├── parameter_config.py └── predict.py ├── BiLstmAttention ├── get_train_data.py ├── mode_structure.py ├── mode_trainning.py ├── parameter_config.py └── predict.py ├── ELMo ├── data.py ├── get_train_data.py ├── mode_structure.py ├── mode_trainning.py ├── parameter_config.py └── predict.py ├── README.md ├── Transformer ├── get_train_data.py ├── mode_structure.py ├── mode_trainning.py ├── parameter_config.py └── predict.py ├── charCNN ├── data_test.py ├── get_train_data.py ├── mode_structure.py ├── mode_trainning.py ├── parameter_config.py └── predict.py ├── textCNN ├── get_train_data.py ├── mode_structure.py ├── mode_trainning.py ├── parameter_config.py └── predict.py └── word2vec ├── word2Vec.bin ├── word2vec.py └── 数据提取.py /BERT/bert/.gitignore: -------------------------------------------------------------------------------- 1 | # Initially taken from Github's Python gitignore file 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # Environments 92 | .env 93 | .venv 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | .dmypy.json 113 | dmypy.json 114 | 115 | # Pyre type checker 116 | .pyre/ 117 | -------------------------------------------------------------------------------- /BERT/bert/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | BERT needs to maintain permanent compatibility with the pre-trained model files, 4 | so we do not plan to make any major changes to this library (other than what was 5 | promised in the README). However, we can accept small patches related to 6 | re-factoring and documentation. To submit contributes, there are just a few 7 | small guidelines you need to follow. 8 | 9 | ## Contributor License Agreement 10 | 11 | Contributions to this project must be accompanied by a Contributor License 12 | Agreement. You (or your employer) retain the copyright to your contribution; 13 | this simply gives us permission to use and redistribute your contributions as 14 | part of the project. Head over to to see 15 | your current agreements on file or to sign a new one. 16 | 17 | You generally only need to submit a CLA once, so if you've already submitted one 18 | (even if it was for a different project), you probably don't need to do it 19 | again. 20 | 21 | ## Code reviews 22 | 23 | All submissions, including submissions by project members, require review. We 24 | use GitHub pull requests for this purpose. Consult 25 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 26 | information on using pull requests. 27 | 28 | ## Community Guidelines 29 | 30 | This project follows 31 | [Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). 32 | -------------------------------------------------------------------------------- /BERT/bert/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /BERT/bert/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /BERT/bert/modeling_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import collections 20 | import json 21 | import random 22 | import re 23 | 24 | import modeling 25 | import six 26 | import tensorflow as tf 27 | 28 | 29 | class BertModelTest(tf.test.TestCase): 30 | 31 | class BertModelTester(object): 32 | 33 | def __init__(self, 34 | parent, 35 | batch_size=13, 36 | seq_length=7, 37 | is_training=True, 38 | use_input_mask=True, 39 | use_token_type_ids=True, 40 | vocab_size=99, 41 | hidden_size=32, 42 | num_hidden_layers=5, 43 | num_attention_heads=4, 44 | intermediate_size=37, 45 | hidden_act="gelu", 46 | hidden_dropout_prob=0.1, 47 | attention_probs_dropout_prob=0.1, 48 | max_position_embeddings=512, 49 | type_vocab_size=16, 50 | initializer_range=0.02, 51 | scope=None): 52 | self.parent = parent 53 | self.batch_size = batch_size 54 | self.seq_length = seq_length 55 | self.is_training = is_training 56 | self.use_input_mask = use_input_mask 57 | self.use_token_type_ids = use_token_type_ids 58 | self.vocab_size = vocab_size 59 | self.hidden_size = hidden_size 60 | self.num_hidden_layers = num_hidden_layers 61 | self.num_attention_heads = num_attention_heads 62 | self.intermediate_size = intermediate_size 63 | self.hidden_act = hidden_act 64 | self.hidden_dropout_prob = hidden_dropout_prob 65 | self.attention_probs_dropout_prob = attention_probs_dropout_prob 66 | self.max_position_embeddings = max_position_embeddings 67 | self.type_vocab_size = type_vocab_size 68 | self.initializer_range = initializer_range 69 | self.scope = scope 70 | 71 | def create_model(self): 72 | input_ids = BertModelTest.ids_tensor([self.batch_size, self.seq_length], 73 | self.vocab_size) 74 | 75 | input_mask = None 76 | if self.use_input_mask: 77 | input_mask = BertModelTest.ids_tensor( 78 | [self.batch_size, self.seq_length], vocab_size=2) 79 | 80 | token_type_ids = None 81 | if self.use_token_type_ids: 82 | token_type_ids = BertModelTest.ids_tensor( 83 | [self.batch_size, self.seq_length], self.type_vocab_size) 84 | 85 | config = modeling.BertConfig( 86 | vocab_size=self.vocab_size, 87 | hidden_size=self.hidden_size, 88 | num_hidden_layers=self.num_hidden_layers, 89 | num_attention_heads=self.num_attention_heads, 90 | intermediate_size=self.intermediate_size, 91 | hidden_act=self.hidden_act, 92 | hidden_dropout_prob=self.hidden_dropout_prob, 93 | attention_probs_dropout_prob=self.attention_probs_dropout_prob, 94 | max_position_embeddings=self.max_position_embeddings, 95 | type_vocab_size=self.type_vocab_size, 96 | initializer_range=self.initializer_range) 97 | 98 | model = modeling.BertModel( 99 | config=config, 100 | is_training=self.is_training, 101 | input_ids=input_ids, 102 | input_mask=input_mask, 103 | token_type_ids=token_type_ids, 104 | scope=self.scope) 105 | 106 | outputs = { 107 | "embedding_output": model.get_embedding_output(), 108 | "sequence_output": model.get_sequence_output(), 109 | "pooled_output": model.get_pooled_output(), 110 | "all_encoder_layers": model.get_all_encoder_layers(), 111 | } 112 | return outputs 113 | 114 | def check_output(self, result): 115 | self.parent.assertAllEqual( 116 | result["embedding_output"].shape, 117 | [self.batch_size, self.seq_length, self.hidden_size]) 118 | 119 | self.parent.assertAllEqual( 120 | result["sequence_output"].shape, 121 | [self.batch_size, self.seq_length, self.hidden_size]) 122 | 123 | self.parent.assertAllEqual(result["pooled_output"].shape, 124 | [self.batch_size, self.hidden_size]) 125 | 126 | def test_default(self): 127 | self.run_tester(BertModelTest.BertModelTester(self)) 128 | 129 | def test_config_to_json_string(self): 130 | config = modeling.BertConfig(vocab_size=99, hidden_size=37) 131 | obj = json.loads(config.to_json_string()) 132 | self.assertEqual(obj["vocab_size"], 99) 133 | self.assertEqual(obj["hidden_size"], 37) 134 | 135 | def run_tester(self, tester): 136 | with self.test_session() as sess: 137 | ops = tester.create_model() 138 | init_op = tf.group(tf.global_variables_initializer(), 139 | tf.local_variables_initializer()) 140 | sess.run(init_op) 141 | output_result = sess.run(ops) 142 | tester.check_output(output_result) 143 | 144 | self.assert_all_tensors_reachable(sess, [init_op, ops]) 145 | 146 | @classmethod 147 | def ids_tensor(cls, shape, vocab_size, rng=None, name=None): 148 | """Creates a random int32 tensor of the shape within the vocab size.""" 149 | if rng is None: 150 | rng = random.Random() 151 | 152 | total_dims = 1 153 | for dim in shape: 154 | total_dims *= dim 155 | 156 | values = [] 157 | for _ in range(total_dims): 158 | values.append(rng.randint(0, vocab_size - 1)) 159 | 160 | return tf.constant(value=values, dtype=tf.int32, shape=shape, name=name) 161 | 162 | def assert_all_tensors_reachable(self, sess, outputs): 163 | """Checks that all the tensors in the graph are reachable from outputs.""" 164 | graph = sess.graph 165 | 166 | ignore_strings = [ 167 | "^.*/assert_less_equal/.*$", 168 | "^.*/dilation_rate$", 169 | "^.*/Tensordot/concat$", 170 | "^.*/Tensordot/concat/axis$", 171 | "^testing/.*$", 172 | ] 173 | 174 | ignore_regexes = [re.compile(x) for x in ignore_strings] 175 | 176 | unreachable = self.get_unreachable_ops(graph, outputs) 177 | filtered_unreachable = [] 178 | for x in unreachable: 179 | do_ignore = False 180 | for r in ignore_regexes: 181 | m = r.match(x.name) 182 | if m is not None: 183 | do_ignore = True 184 | if do_ignore: 185 | continue 186 | filtered_unreachable.append(x) 187 | unreachable = filtered_unreachable 188 | 189 | self.assertEqual( 190 | len(unreachable), 0, "The following ops are unreachable: %s" % 191 | (" ".join([x.name for x in unreachable]))) 192 | 193 | @classmethod 194 | def get_unreachable_ops(cls, graph, outputs): 195 | """Finds all of the tensors in graph that are unreachable from outputs.""" 196 | outputs = cls.flatten_recursive(outputs) 197 | output_to_op = collections.defaultdict(list) 198 | op_to_all = collections.defaultdict(list) 199 | assign_out_to_in = collections.defaultdict(list) 200 | 201 | for op in graph.get_operations(): 202 | for x in op.inputs: 203 | op_to_all[op.name].append(x.name) 204 | for y in op.outputs: 205 | output_to_op[y.name].append(op.name) 206 | op_to_all[op.name].append(y.name) 207 | if str(op.type) == "Assign": 208 | for y in op.outputs: 209 | for x in op.inputs: 210 | assign_out_to_in[y.name].append(x.name) 211 | 212 | assign_groups = collections.defaultdict(list) 213 | for out_name in assign_out_to_in.keys(): 214 | name_group = assign_out_to_in[out_name] 215 | for n1 in name_group: 216 | assign_groups[n1].append(out_name) 217 | for n2 in name_group: 218 | if n1 != n2: 219 | assign_groups[n1].append(n2) 220 | 221 | seen_tensors = {} 222 | stack = [x.name for x in outputs] 223 | while stack: 224 | name = stack.pop() 225 | if name in seen_tensors: 226 | continue 227 | seen_tensors[name] = True 228 | 229 | if name in output_to_op: 230 | for op_name in output_to_op[name]: 231 | if op_name in op_to_all: 232 | for input_name in op_to_all[op_name]: 233 | if input_name not in stack: 234 | stack.append(input_name) 235 | 236 | expanded_names = [] 237 | if name in assign_groups: 238 | for assign_name in assign_groups[name]: 239 | expanded_names.append(assign_name) 240 | 241 | for expanded_name in expanded_names: 242 | if expanded_name not in stack: 243 | stack.append(expanded_name) 244 | 245 | unreachable_ops = [] 246 | for op in graph.get_operations(): 247 | is_unreachable = False 248 | all_names = [x.name for x in op.inputs] + [x.name for x in op.outputs] 249 | for name in all_names: 250 | if name not in seen_tensors: 251 | is_unreachable = True 252 | if is_unreachable: 253 | unreachable_ops.append(op) 254 | return unreachable_ops 255 | 256 | @classmethod 257 | def flatten_recursive(cls, item): 258 | """Flattens (potentially nested) a tuple/dictionary/list to a list.""" 259 | output = [] 260 | if isinstance(item, list): 261 | output.extend(item) 262 | elif isinstance(item, tuple): 263 | output.extend(list(item)) 264 | elif isinstance(item, dict): 265 | for (_, v) in six.iteritems(item): 266 | output.append(v) 267 | else: 268 | return [item] 269 | 270 | flat_output = [] 271 | for x in output: 272 | flat_output.extend(cls.flatten_recursive(x)) 273 | return flat_output 274 | 275 | 276 | if __name__ == "__main__": 277 | tf.test.main() 278 | -------------------------------------------------------------------------------- /BERT/bert/multilingual.md: -------------------------------------------------------------------------------- 1 | ## Models 2 | 3 | There are two multilingual models currently available. We do not plan to release 4 | more single-language models, but we may release `BERT-Large` versions of these 5 | two in the future: 6 | 7 | * **[`BERT-Base, Multilingual Cased (New, recommended)`](https://storage.googleapis.com/bert_models/2018_11_23/multi_cased_L-12_H-768_A-12.zip)**: 8 | 104 languages, 12-layer, 768-hidden, 12-heads, 110M parameters 9 | * **[`BERT-Base, Multilingual Uncased (Orig, not recommended)`](https://storage.googleapis.com/bert_models/2018_11_03/multilingual_L-12_H-768_A-12.zip)**: 10 | 102 languages, 12-layer, 768-hidden, 12-heads, 110M parameters 11 | * **[`BERT-Base, Chinese`](https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip)**: 12 | Chinese Simplified and Traditional, 12-layer, 768-hidden, 12-heads, 110M 13 | parameters 14 | 15 | **The `Multilingual Cased (New)` model also fixes normalization issues in many 16 | languages, so it is recommended in languages with non-Latin alphabets (and is 17 | often better for most languages with Latin alphabets). When using this model, 18 | make sure to pass `--do_lower_case=false` to `run_pretraining.py` and other 19 | scripts.** 20 | 21 | See the [list of languages](#list-of-languages) that the Multilingual model 22 | supports. The Multilingual model does include Chinese (and English), but if your 23 | fine-tuning data is Chinese-only, then the Chinese model will likely produce 24 | better results. 25 | 26 | ## Results 27 | 28 | To evaluate these systems, we use the 29 | [XNLI dataset](https://github.com/facebookresearch/XNLI) dataset, which is a 30 | version of [MultiNLI](https://www.nyu.edu/projects/bowman/multinli/) where the 31 | dev and test sets have been translated (by humans) into 15 languages. Note that 32 | the training set was *machine* translated (we used the translations provided by 33 | XNLI, not Google NMT). For clarity, we only report on 6 languages below: 34 | 35 | 36 | 37 | | System | English | Chinese | Spanish | German | Arabic | Urdu | 38 | | --------------------------------- | -------- | -------- | -------- | -------- | -------- | -------- | 39 | | XNLI Baseline - Translate Train | 73.7 | 67.0 | 68.8 | 66.5 | 65.8 | 56.6 | 40 | | XNLI Baseline - Translate Test | 73.7 | 68.3 | 70.7 | 68.7 | 66.8 | 59.3 | 41 | | BERT - Translate Train Cased | **81.9** | **76.6** | **77.8** | **75.9** | **70.7** | 61.6 | 42 | | BERT - Translate Train Uncased | 81.4 | 74.2 | 77.3 | 75.2 | 70.5 | 61.7 | 43 | | BERT - Translate Test Uncased | 81.4 | 70.1 | 74.9 | 74.4 | 70.4 | **62.1** | 44 | | BERT - Zero Shot Uncased | 81.4 | 63.8 | 74.3 | 70.5 | 62.1 | 58.3 | 45 | 46 | 47 | 48 | The first two rows are baselines from the XNLI paper and the last three rows are 49 | our results with BERT. 50 | 51 | **Translate Train** means that the MultiNLI training set was machine translated 52 | from English into the foreign language. So training and evaluation were both 53 | done in the foreign language. Unfortunately, training was done on 54 | machine-translated data, so it is impossible to quantify how much of the lower 55 | accuracy (compared to English) is due to the quality of the machine translation 56 | vs. the quality of the pre-trained model. 57 | 58 | **Translate Test** means that the XNLI test set was machine translated from the 59 | foreign language into English. So training and evaluation were both done on 60 | English. However, test evaluation was done on machine-translated English, so the 61 | accuracy depends on the quality of the machine translation system. 62 | 63 | **Zero Shot** means that the Multilingual BERT system was fine-tuned on English 64 | MultiNLI, and then evaluated on the foreign language XNLI test. In this case, 65 | machine translation was not involved at all in either the pre-training or 66 | fine-tuning. 67 | 68 | Note that the English result is worse than the 84.2 MultiNLI baseline because 69 | this training used Multilingual BERT rather than English-only BERT. This implies 70 | that for high-resource languages, the Multilingual model is somewhat worse than 71 | a single-language model. However, it is not feasible for us to train and 72 | maintain dozens of single-language model. Therefore, if your goal is to maximize 73 | performance with a language other than English or Chinese, you might find it 74 | beneficial to run pre-training for additional steps starting from our 75 | Multilingual model on data from your language of interest. 76 | 77 | Here is a comparison of training Chinese models with the Multilingual 78 | `BERT-Base` and Chinese-only `BERT-Base`: 79 | 80 | System | Chinese 81 | ----------------------- | ------- 82 | XNLI Baseline | 67.0 83 | BERT Multilingual Model | 74.2 84 | BERT Chinese-only Model | 77.2 85 | 86 | Similar to English, the single-language model does 3% better than the 87 | Multilingual model. 88 | 89 | ## Fine-tuning Example 90 | 91 | The multilingual model does **not** require any special consideration or API 92 | changes. We did update the implementation of `BasicTokenizer` in 93 | `tokenization.py` to support Chinese character tokenization, so please update if 94 | you forked it. However, we did not change the tokenization API. 95 | 96 | To test the new models, we did modify `run_classifier.py` to add support for the 97 | [XNLI dataset](https://github.com/facebookresearch/XNLI). This is a 15-language 98 | version of MultiNLI where the dev/test sets have been human-translated, and the 99 | training set has been machine-translated. 100 | 101 | To run the fine-tuning code, please download the 102 | [XNLI dev/test set](https://s3.amazonaws.com/xnli/XNLI-1.0.zip) and the 103 | [XNLI machine-translated training set](https://s3.amazonaws.com/xnli/XNLI-MT-1.0.zip) 104 | and then unpack both .zip files into some directory `$XNLI_DIR`. 105 | 106 | To run fine-tuning on XNLI. The language is hard-coded into `run_classifier.py` 107 | (Chinese by default), so please modify `XnliProcessor` if you want to run on 108 | another language. 109 | 110 | This is a large dataset, so this will training will take a few hours on a GPU 111 | (or about 30 minutes on a Cloud TPU). To run an experiment quickly for 112 | debugging, just set `num_train_epochs` to a small value like `0.1`. 113 | 114 | ```shell 115 | export BERT_BASE_DIR=/path/to/bert/chinese_L-12_H-768_A-12 # or multilingual_L-12_H-768_A-12 116 | export XNLI_DIR=/path/to/xnli 117 | 118 | python run_classifier.py \ 119 | --task_name=XNLI \ 120 | --do_train=true \ 121 | --do_eval=true \ 122 | --data_dir=$XNLI_DIR \ 123 | --vocab_file=$BERT_BASE_DIR/vocab.txt \ 124 | --bert_config_file=$BERT_BASE_DIR/bert_config.json \ 125 | --init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt \ 126 | --max_seq_length=128 \ 127 | --train_batch_size=32 \ 128 | --learning_rate=5e-5 \ 129 | --num_train_epochs=2.0 \ 130 | --output_dir=/tmp/xnli_output/ 131 | ``` 132 | 133 | With the Chinese-only model, the results should look something like this: 134 | 135 | ``` 136 | ***** Eval results ***** 137 | eval_accuracy = 0.774116 138 | eval_loss = 0.83554 139 | global_step = 24543 140 | loss = 0.74603 141 | ``` 142 | 143 | ## Details 144 | 145 | ### Data Source and Sampling 146 | 147 | The languages chosen were the 148 | [top 100 languages with the largest Wikipedias](https://meta.wikimedia.org/wiki/List_of_Wikipedias). 149 | The entire Wikipedia dump for each language (excluding user and talk pages) was 150 | taken as the training data for each language 151 | 152 | However, the size of the Wikipedia for a given language varies greatly, and 153 | therefore low-resource languages may be "under-represented" in terms of the 154 | neural network model (under the assumption that languages are "competing" for 155 | limited model capacity to some extent). 156 | 157 | However, the size of a Wikipedia also correlates with the number of speakers of 158 | a language, and we also don't want to overfit the model by performing thousands 159 | of epochs over a tiny Wikipedia for a particular language. 160 | 161 | To balance these two factors, we performed exponentially smoothed weighting of 162 | the data during pre-training data creation (and WordPiece vocab creation). In 163 | other words, let's say that the probability of a language is *P(L)*, e.g., 164 | *P(English) = 0.21* means that after concatenating all of the Wikipedias 165 | together, 21% of our data is English. We exponentiate each probability by some 166 | factor *S* and then re-normalize, and sample from that distribution. In our case 167 | we use *S=0.7*. So, high-resource languages like English will be under-sampled, 168 | and low-resource languages like Icelandic will be over-sampled. E.g., in the 169 | original distribution English would be sampled 1000x more than Icelandic, but 170 | after smoothing it's only sampled 100x more. 171 | 172 | ### Tokenization 173 | 174 | For tokenization, we use a 110k shared WordPiece vocabulary. The word counts are 175 | weighted the same way as the data, so low-resource languages are upweighted by 176 | some factor. We intentionally do *not* use any marker to denote the input 177 | language (so that zero-shot training can work). 178 | 179 | Because Chinese (and Japanese Kanji and Korean Hanja) does not have whitespace 180 | characters, we add spaces around every character in the 181 | [CJK Unicode range](https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_\(Unicode_block\)) 182 | before applying WordPiece. This means that Chinese is effectively 183 | character-tokenized. Note that the CJK Unicode block only includes 184 | Chinese-origin characters and does *not* include Hangul Korean or 185 | Katakana/Hiragana Japanese, which are tokenized with whitespace+WordPiece like 186 | all other languages. 187 | 188 | For all other languages, we apply the 189 | [same recipe as English](https://github.com/google-research/bert#tokenization): 190 | (a) lower casing+accent removal, (b) punctuation splitting, (c) whitespace 191 | tokenization. We understand that accent markers have substantial meaning in some 192 | languages, but felt that the benefits of reducing the effective vocabulary make 193 | up for this. Generally the strong contextual models of BERT should make up for 194 | any ambiguity introduced by stripping accent markers. 195 | 196 | ### List of Languages 197 | 198 | The multilingual model supports the following languages. These languages were 199 | chosen because they are the top 100 languages with the largest Wikipedias: 200 | 201 | * Afrikaans 202 | * Albanian 203 | * Arabic 204 | * Aragonese 205 | * Armenian 206 | * Asturian 207 | * Azerbaijani 208 | * Bashkir 209 | * Basque 210 | * Bavarian 211 | * Belarusian 212 | * Bengali 213 | * Bishnupriya Manipuri 214 | * Bosnian 215 | * Breton 216 | * Bulgarian 217 | * Burmese 218 | * Catalan 219 | * Cebuano 220 | * Chechen 221 | * Chinese (Simplified) 222 | * Chinese (Traditional) 223 | * Chuvash 224 | * Croatian 225 | * Czech 226 | * Danish 227 | * Dutch 228 | * English 229 | * Estonian 230 | * Finnish 231 | * French 232 | * Galician 233 | * Georgian 234 | * German 235 | * Greek 236 | * Gujarati 237 | * Haitian 238 | * Hebrew 239 | * Hindi 240 | * Hungarian 241 | * Icelandic 242 | * Ido 243 | * Indonesian 244 | * Irish 245 | * Italian 246 | * Japanese 247 | * Javanese 248 | * Kannada 249 | * Kazakh 250 | * Kirghiz 251 | * Korean 252 | * Latin 253 | * Latvian 254 | * Lithuanian 255 | * Lombard 256 | * Low Saxon 257 | * Luxembourgish 258 | * Macedonian 259 | * Malagasy 260 | * Malay 261 | * Malayalam 262 | * Marathi 263 | * Minangkabau 264 | * Nepali 265 | * Newar 266 | * Norwegian (Bokmal) 267 | * Norwegian (Nynorsk) 268 | * Occitan 269 | * Persian (Farsi) 270 | * Piedmontese 271 | * Polish 272 | * Portuguese 273 | * Punjabi 274 | * Romanian 275 | * Russian 276 | * Scots 277 | * Serbian 278 | * Serbo-Croatian 279 | * Sicilian 280 | * Slovak 281 | * Slovenian 282 | * South Azerbaijani 283 | * Spanish 284 | * Sundanese 285 | * Swahili 286 | * Swedish 287 | * Tagalog 288 | * Tajik 289 | * Tamil 290 | * Tatar 291 | * Telugu 292 | * Turkish 293 | * Ukrainian 294 | * Urdu 295 | * Uzbek 296 | * Vietnamese 297 | * Volapük 298 | * Waray-Waray 299 | * Welsh 300 | * West Frisian 301 | * Western Punjabi 302 | * Yoruba 303 | 304 | The **Multilingual Cased (New)** release contains additionally **Thai** and 305 | **Mongolian**, which were not included in the original release. 306 | -------------------------------------------------------------------------------- /BERT/bert/optimization.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Functions and classes related to optimization (weight updates).""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import re 22 | import tensorflow as tf 23 | 24 | 25 | def create_optimizer(loss, init_lr, num_train_steps, num_warmup_steps, use_tpu): 26 | """Creates an optimizer training op.""" 27 | global_step = tf.train.get_or_create_global_step() 28 | 29 | learning_rate = tf.constant(value=init_lr, shape=[], dtype=tf.float32) 30 | 31 | # Implements linear decay of the learning rate. 32 | learning_rate = tf.train.polynomial_decay( 33 | learning_rate, 34 | global_step, 35 | num_train_steps, 36 | end_learning_rate=0.0, 37 | power=1.0, 38 | cycle=False) 39 | 40 | # Implements linear warmup. I.e., if global_step < num_warmup_steps, the 41 | # learning rate will be `global_step/num_warmup_steps * init_lr`. 42 | if num_warmup_steps: 43 | global_steps_int = tf.cast(global_step, tf.int32) 44 | warmup_steps_int = tf.constant(num_warmup_steps, dtype=tf.int32) 45 | 46 | global_steps_float = tf.cast(global_steps_int, tf.float32) 47 | warmup_steps_float = tf.cast(warmup_steps_int, tf.float32) 48 | 49 | warmup_percent_done = global_steps_float / warmup_steps_float 50 | warmup_learning_rate = init_lr * warmup_percent_done 51 | 52 | is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32) 53 | learning_rate = ( 54 | (1.0 - is_warmup) * learning_rate + is_warmup * warmup_learning_rate) 55 | 56 | # It is recommended that you use this optimizer for fine tuning, since this 57 | # is how the model was trained (note that the Adam m/v variables are NOT 58 | # loaded from init_checkpoint.) 59 | optimizer = AdamWeightDecayOptimizer( 60 | learning_rate=learning_rate, 61 | weight_decay_rate=0.01, 62 | beta_1=0.9, 63 | beta_2=0.999, 64 | epsilon=1e-6, 65 | exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) 66 | 67 | if use_tpu: 68 | optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer) 69 | 70 | tvars = tf.trainable_variables() 71 | grads = tf.gradients(loss, tvars) 72 | 73 | # This is how the model was pre-trained. 74 | (grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0) 75 | 76 | train_op = optimizer.apply_gradients( 77 | zip(grads, tvars), global_step=global_step) 78 | 79 | # Normally the global step update is done inside of `apply_gradients`. 80 | # However, `AdamWeightDecayOptimizer` doesn't do this. But if you use 81 | # a different optimizer, you should probably take this line out. 82 | new_global_step = global_step + 1 83 | train_op = tf.group(train_op, [global_step.assign(new_global_step)]) 84 | return train_op 85 | 86 | 87 | class AdamWeightDecayOptimizer(tf.train.Optimizer): 88 | """A basic Adam optimizer that includes "correct" L2 weight decay.""" 89 | 90 | def __init__(self, 91 | learning_rate, 92 | weight_decay_rate=0.0, 93 | beta_1=0.9, 94 | beta_2=0.999, 95 | epsilon=1e-6, 96 | exclude_from_weight_decay=None, 97 | name="AdamWeightDecayOptimizer"): 98 | """Constructs a AdamWeightDecayOptimizer.""" 99 | super(AdamWeightDecayOptimizer, self).__init__(False, name) 100 | 101 | self.learning_rate = learning_rate 102 | self.weight_decay_rate = weight_decay_rate 103 | self.beta_1 = beta_1 104 | self.beta_2 = beta_2 105 | self.epsilon = epsilon 106 | self.exclude_from_weight_decay = exclude_from_weight_decay 107 | 108 | def apply_gradients(self, grads_and_vars, global_step=None, name=None): 109 | """See base class.""" 110 | assignments = [] 111 | for (grad, param) in grads_and_vars: 112 | if grad is None or param is None: 113 | continue 114 | 115 | param_name = self._get_variable_name(param.name) 116 | 117 | m = tf.get_variable( 118 | name=param_name + "/adam_m", 119 | shape=param.shape.as_list(), 120 | dtype=tf.float32, 121 | trainable=False, 122 | initializer=tf.zeros_initializer()) 123 | v = tf.get_variable( 124 | name=param_name + "/adam_v", 125 | shape=param.shape.as_list(), 126 | dtype=tf.float32, 127 | trainable=False, 128 | initializer=tf.zeros_initializer()) 129 | 130 | # Standard Adam update. 131 | next_m = ( 132 | tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad)) 133 | next_v = ( 134 | tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2, 135 | tf.square(grad))) 136 | 137 | update = next_m / (tf.sqrt(next_v) + self.epsilon) 138 | 139 | # Just adding the square of the weights to the loss function is *not* 140 | # the correct way of using L2 regularization/weight decay with Adam, 141 | # since that will interact with the m and v parameters in strange ways. 142 | # 143 | # Instead we want ot decay the weights in a manner that doesn't interact 144 | # with the m/v parameters. This is equivalent to adding the square 145 | # of the weights to the loss with plain (non-momentum) SGD. 146 | if self._do_use_weight_decay(param_name): 147 | update += self.weight_decay_rate * param 148 | 149 | update_with_lr = self.learning_rate * update 150 | 151 | next_param = param - update_with_lr 152 | 153 | assignments.extend( 154 | [param.assign(next_param), 155 | m.assign(next_m), 156 | v.assign(next_v)]) 157 | return tf.group(*assignments, name=name) 158 | 159 | def _do_use_weight_decay(self, param_name): 160 | """Whether to use L2 weight decay for `param_name`.""" 161 | if not self.weight_decay_rate: 162 | return False 163 | if self.exclude_from_weight_decay: 164 | for r in self.exclude_from_weight_decay: 165 | if re.search(r, param_name) is not None: 166 | return False 167 | return True 168 | 169 | def _get_variable_name(self, param_name): 170 | """Get the variable name from the tensor name.""" 171 | m = re.match("^(.*):\\d+$", param_name) 172 | if m is not None: 173 | param_name = m.group(1) 174 | return param_name 175 | -------------------------------------------------------------------------------- /BERT/bert/optimization_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import optimization 20 | import tensorflow as tf 21 | 22 | 23 | class OptimizationTest(tf.test.TestCase): 24 | 25 | def test_adam(self): 26 | with self.test_session() as sess: 27 | w = tf.get_variable( 28 | "w", 29 | shape=[3], 30 | initializer=tf.constant_initializer([0.1, -0.2, -0.1])) 31 | x = tf.constant([0.4, 0.2, -0.5]) 32 | loss = tf.reduce_mean(tf.square(x - w)) 33 | tvars = tf.trainable_variables() 34 | grads = tf.gradients(loss, tvars) 35 | global_step = tf.train.get_or_create_global_step() 36 | optimizer = optimization.AdamWeightDecayOptimizer(learning_rate=0.2) 37 | train_op = optimizer.apply_gradients(zip(grads, tvars), global_step) 38 | init_op = tf.group(tf.global_variables_initializer(), 39 | tf.local_variables_initializer()) 40 | sess.run(init_op) 41 | for _ in range(100): 42 | sess.run(train_op) 43 | w_np = sess.run(w) 44 | self.assertAllClose(w_np.flat, [0.4, 0.2, -0.5], rtol=1e-2, atol=1e-2) 45 | 46 | 47 | if __name__ == "__main__": 48 | tf.test.main() 49 | -------------------------------------------------------------------------------- /BERT/bert/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow >= 1.11.0 # CPU Version of TensorFlow. 2 | # tensorflow-gpu >= 1.11.0 # GPU version of TensorFlow. 3 | -------------------------------------------------------------------------------- /BERT/bert/sample_text.txt: -------------------------------------------------------------------------------- 1 | This text is included to make sure Unicode is handled properly: 力加勝北区ᴵᴺᵀᵃছজটডণত 2 | Text should be one-sentence-per-line, with empty lines between documents. 3 | This sample text is public domain and was randomly selected from Project Guttenberg. 4 | 5 | The rain had only ceased with the gray streaks of morning at Blazing Star, and the settlement awoke to a moral sense of cleanliness, and the finding of forgotten knives, tin cups, and smaller camp utensils, where the heavy showers had washed away the debris and dust heaps before the cabin doors. 6 | Indeed, it was recorded in Blazing Star that a fortunate early riser had once picked up on the highway a solid chunk of gold quartz which the rain had freed from its incumbering soil, and washed into immediate and glittering popularity. 7 | Possibly this may have been the reason why early risers in that locality, during the rainy season, adopted a thoughtful habit of body, and seldom lifted their eyes to the rifted or india-ink washed skies above them. 8 | "Cass" Beard had risen early that morning, but not with a view to discovery. 9 | A leak in his cabin roof,--quite consistent with his careless, improvident habits,--had roused him at 4 A. M., with a flooded "bunk" and wet blankets. 10 | The chips from his wood pile refused to kindle a fire to dry his bed-clothes, and he had recourse to a more provident neighbor's to supply the deficiency. 11 | This was nearly opposite. 12 | Mr. Cassius crossed the highway, and stopped suddenly. 13 | Something glittered in the nearest red pool before him. 14 | Gold, surely! 15 | But, wonderful to relate, not an irregular, shapeless fragment of crude ore, fresh from Nature's crucible, but a bit of jeweler's handicraft in the form of a plain gold ring. 16 | Looking at it more attentively, he saw that it bore the inscription, "May to Cass." 17 | Like most of his fellow gold-seekers, Cass was superstitious. 18 | 19 | The fountain of classic wisdom, Hypatia herself. 20 | As the ancient sage--the name is unimportant to a monk--pumped water nightly that he might study by day, so I, the guardian of cloaks and parasols, at the sacred doors of her lecture-room, imbibe celestial knowledge. 21 | From my youth I felt in me a soul above the matter-entangled herd. 22 | She revealed to me the glorious fact, that I am a spark of Divinity itself. 23 | A fallen star, I am, sir!' continued he, pensively, stroking his lean stomach--'a fallen star!--fallen, if the dignity of philosophy will allow of the simile, among the hogs of the lower world--indeed, even into the hog-bucket itself. Well, after all, I will show you the way to the Archbishop's. 24 | There is a philosophic pleasure in opening one's treasures to the modest young. 25 | Perhaps you will assist me by carrying this basket of fruit?' And the little man jumped up, put his basket on Philammon's head, and trotted off up a neighbouring street. 26 | Philammon followed, half contemptuous, half wondering at what this philosophy might be, which could feed the self-conceit of anything so abject as his ragged little apish guide; 27 | but the novel roar and whirl of the street, the perpetual stream of busy faces, the line of curricles, palanquins, laden asses, camels, elephants, which met and passed him, and squeezed him up steps and into doorways, as they threaded their way through the great Moon-gate into the ample street beyond, drove everything from his mind but wondering curiosity, and a vague, helpless dread of that great living wilderness, more terrible than any dead wilderness of sand which he had left behind. 28 | Already he longed for the repose, the silence of the Laura--for faces which knew him and smiled upon him; but it was too late to turn back now. 29 | His guide held on for more than a mile up the great main street, crossed in the centre of the city, at right angles, by one equally magnificent, at each end of which, miles away, appeared, dim and distant over the heads of the living stream of passengers, the yellow sand-hills of the desert; 30 | while at the end of the vista in front of them gleamed the blue harbour, through a network of countless masts. 31 | At last they reached the quay at the opposite end of the street; 32 | and there burst on Philammon's astonished eyes a vast semicircle of blue sea, ringed with palaces and towers. 33 | He stopped involuntarily; and his little guide stopped also, and looked askance at the young monk, to watch the effect which that grand panorama should produce on him. 34 | -------------------------------------------------------------------------------- /BERT/bert/tokenization_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2018 The Google AI Language Team Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import division 17 | from __future__ import print_function 18 | 19 | import os 20 | import tempfile 21 | import six 22 | import tensorflow as tf 23 | import tokenization 24 | 25 | 26 | class TokenizationTest(tf.test.TestCase): 27 | 28 | def test_full_tokenizer(self): 29 | vocab_tokens = [ 30 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 31 | "##ing", "," 32 | ] 33 | with tempfile.NamedTemporaryFile(delete=False) as vocab_writer: 34 | if six.PY2: 35 | vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) 36 | else: 37 | vocab_writer.write("".join( 38 | [x + "\n" for x in vocab_tokens]).encode("utf-8")) 39 | 40 | vocab_file = vocab_writer.name 41 | 42 | tokenizer = tokenization.FullTokenizer(vocab_file) 43 | os.unlink(vocab_file) 44 | 45 | tokens = tokenizer.tokenize(u"UNwant\u00E9d,running") 46 | self.assertAllEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"]) 47 | 48 | self.assertAllEqual( 49 | tokenizer.convert_tokens_to_ids(tokens), [7, 4, 5, 10, 8, 9]) 50 | 51 | def test_chinese(self): 52 | tokenizer = tokenization.BasicTokenizer() 53 | 54 | self.assertAllEqual( 55 | tokenizer.tokenize(u"ah\u535A\u63A8zz"), 56 | [u"ah", u"\u535A", u"\u63A8", u"zz"]) 57 | 58 | def test_basic_tokenizer_lower(self): 59 | tokenizer = tokenization.BasicTokenizer(do_lower_case=True) 60 | 61 | self.assertAllEqual( 62 | tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), 63 | ["hello", "!", "how", "are", "you", "?"]) 64 | self.assertAllEqual(tokenizer.tokenize(u"H\u00E9llo"), ["hello"]) 65 | 66 | def test_basic_tokenizer_no_lower(self): 67 | tokenizer = tokenization.BasicTokenizer(do_lower_case=False) 68 | 69 | self.assertAllEqual( 70 | tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), 71 | ["HeLLo", "!", "how", "Are", "yoU", "?"]) 72 | 73 | def test_wordpiece_tokenizer(self): 74 | vocab_tokens = [ 75 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 76 | "##ing" 77 | ] 78 | 79 | vocab = {} 80 | for (i, token) in enumerate(vocab_tokens): 81 | vocab[token] = i 82 | tokenizer = tokenization.WordpieceTokenizer(vocab=vocab) 83 | 84 | self.assertAllEqual(tokenizer.tokenize(""), []) 85 | 86 | self.assertAllEqual( 87 | tokenizer.tokenize("unwanted running"), 88 | ["un", "##want", "##ed", "runn", "##ing"]) 89 | 90 | self.assertAllEqual( 91 | tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) 92 | 93 | def test_convert_tokens_to_ids(self): 94 | vocab_tokens = [ 95 | "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", 96 | "##ing" 97 | ] 98 | 99 | vocab = {} 100 | for (i, token) in enumerate(vocab_tokens): 101 | vocab[token] = i 102 | 103 | self.assertAllEqual( 104 | tokenization.convert_tokens_to_ids( 105 | vocab, ["un", "##want", "##ed", "runn", "##ing"]), [7, 4, 5, 8, 9]) 106 | 107 | def test_is_whitespace(self): 108 | self.assertTrue(tokenization._is_whitespace(u" ")) 109 | self.assertTrue(tokenization._is_whitespace(u"\t")) 110 | self.assertTrue(tokenization._is_whitespace(u"\r")) 111 | self.assertTrue(tokenization._is_whitespace(u"\n")) 112 | self.assertTrue(tokenization._is_whitespace(u"\u00A0")) 113 | 114 | self.assertFalse(tokenization._is_whitespace(u"A")) 115 | self.assertFalse(tokenization._is_whitespace(u"-")) 116 | 117 | def test_is_control(self): 118 | self.assertTrue(tokenization._is_control(u"\u0005")) 119 | 120 | self.assertFalse(tokenization._is_control(u"A")) 121 | self.assertFalse(tokenization._is_control(u" ")) 122 | self.assertFalse(tokenization._is_control(u"\t")) 123 | self.assertFalse(tokenization._is_control(u"\r")) 124 | 125 | def test_is_punctuation(self): 126 | self.assertTrue(tokenization._is_punctuation(u"-")) 127 | self.assertTrue(tokenization._is_punctuation(u"$")) 128 | self.assertTrue(tokenization._is_punctuation(u"`")) 129 | self.assertTrue(tokenization._is_punctuation(u".")) 130 | 131 | self.assertFalse(tokenization._is_punctuation(u"A")) 132 | self.assertFalse(tokenization._is_punctuation(u" ")) 133 | 134 | 135 | if __name__ == "__main__": 136 | tf.test.main() 137 | -------------------------------------------------------------------------------- /BiLstm/get_train_data.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import json 3 | from collections import Counter 4 | import gensim 5 | import pandas as pd 6 | import numpy as np 7 | import parameter_config 8 | 9 | # 2 数据预处理的类，生成训练集和测试集 10 | # 　　1）将数据加载进来，将句子分割成词表示，并去除低频词和停用词。 11 | # 　　2）将词映射成索引表示，构建词汇-索引映射表，并保存成json的数据格式， 12 | # 之后做inference时可以用到。（注意，有的词可能不在word2vec的预训练词向量中，这种词直接用UNK表示） 13 | # 　　3）从预训练的词向量模型中读取出词向量，作为初始化值输入到模型中。 14 | # 　　4）将数据集分割成训练集和测试集 15 | 16 | class Dataset(object): 17 | def __init__(self, config): 18 | self.config = config 19 | self._dataSource = config.dataSource 20 | self._stopWordSource = config.stopWordSource 21 | self._sequenceLength = config.sequenceLength # 每条输入的序列处理为定长 22 | self._embeddingSize = config.model.embeddingSize 23 | self._batchSize = config.batchSize 24 | self._rate = config.rate 25 | self._stopWordDict = {} 26 | self.trainReviews = [] 27 | self.trainLabels = [] 28 | self.evalReviews = [] 29 | self.evalLabels = [] 30 | self.wordEmbedding = None 31 | self.labelList = [] 32 | 33 | def _readData(self, filePath): 34 | """ 35 | 从csv文件中读取数据集 36 | """ 37 | df = pd.read_csv(filePath) 38 | if self.config.numClasses == 1: 39 | labels = df["sentiment"].tolist() 40 | elif self.config.numClasses > 1: 41 | labels = df["rate"].tolist() 42 | review = df["review"].tolist() 43 | reviews = [line.strip().split() for line in review] 44 | return reviews, labels 45 | 46 | def _labelToIndex(self, labels, label2idx): 47 | """ 48 | 将标签转换成索引表示 49 | """ 50 | labelIds = [label2idx[label] for label in labels] 51 | return labelIds 52 | 53 | def _wordToIndex(self, reviews, word2idx): 54 | """ 55 | 将词转换成索引 56 | """ 57 | reviewIds = [[word2idx.get(item, word2idx["UNK"]) for item in review] for review in reviews] 58 | return reviewIds 59 | 60 | def _genTrainEvalData(self, x, y, word2idx, rate): 61 | """ 62 | 生成训练集和验证集 63 | """ 64 | reviews = [] 65 | for review in x: 66 | if len(review) >= self._sequenceLength: 67 | reviews.append(review[:self._sequenceLength]) 68 | else: 69 | reviews.append(review + [word2idx["PAD"]] * (self._sequenceLength - len(review))) 70 | trainIndex = int(len(x) * rate) 71 | trainReviews = np.asarray(reviews[:trainIndex], dtype="int64") 72 | trainLabels = np.array(y[:trainIndex], dtype="float32") 73 | evalReviews = np.asarray(reviews[trainIndex:], dtype="int64") 74 | evalLabels = np.array(y[trainIndex:], dtype="float32") 75 | return trainReviews, trainLabels, evalReviews, evalLabels 76 | 77 | def _getWordEmbedding(self, words): 78 | """ 79 | 按照我们的数据集中的单词取出预训练好的word2vec中的词向量 80 | """ 81 | wordVec = gensim.models.KeyedVectors.load_word2vec_format("../word2vec/word2Vec.bin", binary=True) 82 | vocab = [] 83 | wordEmbedding = [] 84 | # 添加 "pad" 和 "UNK", 85 | vocab.append("PAD") 86 | vocab.append("UNK") 87 | wordEmbedding.append(np.zeros(self._embeddingSize)) 88 | wordEmbedding.append(np.random.randn(self._embeddingSize)) 89 | 90 | for word in words: 91 | try: 92 | vector = wordVec.wv[word] 93 | vocab.append(word) 94 | wordEmbedding.append(vector) 95 | except: 96 | print(word + "不存在于词向量中") 97 | 98 | return vocab, np.array(wordEmbedding) 99 | 100 | def _genVocabulary(self, reviews, labels): 101 | """ 102 | 生成词向量和词汇-索引映射字典，可以用全数据集 103 | """ 104 | allWords = [word for review in reviews for word in review] 105 | 106 | # 去掉停用词 107 | subWords = [word for word in allWords if word not in self.stopWordDict] 108 | wordCount = Counter(subWords) # 统计词频 109 | sortWordCount = sorted(wordCount.items(), key=lambda x: x[1], reverse=True) 110 | # 去除低频词 111 | words = [item[0] for item in sortWordCount if item[1] >= 5] 112 | 113 | vocab, wordEmbedding = self._getWordEmbedding(words) 114 | self.wordEmbedding = wordEmbedding 115 | word2idx = dict(zip(vocab, list(range(len(vocab))))) 116 | 117 | uniqueLabel = list(set(labels)) 118 | label2idx = dict(zip(uniqueLabel, list(range(len(uniqueLabel))))) 119 | self.labelList = list(range(len(uniqueLabel))) 120 | # 将词汇-索引映射表保存为json数据，之后做inference时直接加载来处理数据 121 | with open("../data/wordJson/word2idx.json", "w", encoding="utf-8") as f: 122 | json.dump(word2idx, f) 123 | 124 | with open("../data/wordJson/label2idx.json", "w", encoding="utf-8") as f: 125 | json.dump(label2idx, f) 126 | 127 | return word2idx, label2idx 128 | 129 | def _readStopWord(self, stopWordPath): 130 | """ 131 | 读取停用词 132 | """ 133 | 134 | with open(stopWordPath, "r") as f: 135 | stopWords = f.read() 136 | stopWordList = stopWords.splitlines() 137 | # 将停用词用列表的形式生成，之后查找停用词时会比较快 138 | self.stopWordDict = dict(zip(stopWordList, list(range(len(stopWordList))))) 139 | 140 | def dataGen(self): 141 | """ 142 | 初始化训练集和验证集 143 | """ 144 | # 初始化停用词 145 | self._readStopWord(self._stopWordSource) 146 | 147 | # 初始化数据集 148 | reviews, labels = self._readData(self._dataSource) 149 | 150 | # 初始化词汇-索引映射表和词向量矩阵 151 | word2idx, label2idx = self._genVocabulary(reviews, labels) 152 | 153 | # 将标签和句子数值化 154 | labelIds = self._labelToIndex(labels, label2idx) 155 | reviewIds = self._wordToIndex(reviews, word2idx) 156 | 157 | # 初始化训练集和测试集 158 | trainReviews, trainLabels, evalReviews, evalLabels = self._genTrainEvalData(reviewIds, labelIds, word2idx, 159 | self._rate) 160 | self.trainReviews = trainReviews 161 | self.trainLabels = trainLabels 162 | 163 | self.evalReviews = evalReviews 164 | self.evalLabels = evalLabels 165 | 166 | #获取前些模块的数据 167 | config =parameter_config.Config() 168 | data = Dataset(config) 169 | data.dataGen() -------------------------------------------------------------------------------- /BiLstm/mode_structure.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | 3 | import tensorflow as tf 4 | import parameter_config 5 | 6 | # 3 构建模型 Bi-LSTM模型 7 | class BiLSTM(object): 8 | """ 9 | Bi-LSTM 用于文本分类 10 | """ 11 | def __init__(self, config, wordEmbedding): 12 | 13 | # 定义模型的输入 14 | self.inputX = tf.placeholder(tf.int32, [None, config.sequenceLength], name="inputX") 15 | self.inputY = tf.placeholder(tf.int32, [None], name="inputY") 16 | 17 | self.dropoutKeepProb = tf.placeholder(tf.float32, name="dropoutKeepProb") 18 | 19 | # 定义l2损失 20 | l2Loss = tf.constant(0.0) 21 | 22 | # 词嵌入层 23 | with tf.name_scope("embedding"): 24 | # 利用预训练的词向量初始化词嵌入矩阵 25 | self.W = tf.Variable(tf.cast(wordEmbedding, dtype=tf.float32, name="word2vec"), name="W") 26 | # 利用词嵌入矩阵将输入的数据中的词转换成词向量，维度[batch_size, sequence_length, embedding_size] 27 | self.embeddedWords = tf.nn.embedding_lookup(self.W, self.inputX) 28 | 29 | # 定义两层双向LSTM的模型结构 30 | with tf.name_scope("Bi-LSTM"): 31 | for idx, hiddenSize in enumerate(config.model.hiddenSizes): 32 | with tf.name_scope("Bi-LSTM" + str(idx)): 33 | # 定义前向LSTM结构 34 | lstmFwCell = tf.nn.rnn_cell.DropoutWrapper( 35 | tf.nn.rnn_cell.LSTMCell(num_units=hiddenSize, state_is_tuple=True), 36 | output_keep_prob=self.dropoutKeepProb) 37 | # 定义反向LSTM结构 38 | lstmBwCell = tf.nn.rnn_cell.DropoutWrapper( 39 | tf.nn.rnn_cell.LSTMCell(num_units=hiddenSize, state_is_tuple=True), 40 | output_keep_prob=self.dropoutKeepProb) 41 | 42 | # 采用动态rnn，可以动态的输入序列的长度，若没有输入，则取序列的全长 43 | # outputs是一个元祖(output_fw, output_bw)，其中两个元素的维度都是[batch_size, max_time, hidden_size],fw和bw的hidden_size一样 44 | # self.current_state 是最终的状态，二元组(state_fw, state_bw)，state_fw=[batch_size, s]，s是一个元祖(h, c) 45 | outputs, self.current_state = tf.nn.bidirectional_dynamic_rnn(lstmFwCell, lstmBwCell, 46 | self.embeddedWords, dtype=tf.float32, 47 | scope="bi-lstm" + str(idx)) 48 | 49 | # 对outputs中的fw和bw的结果拼接 [batch_size, time_step, hidden_size * 2] 50 | self.embeddedWords = tf.concat(outputs, 2) 51 | 52 | # 去除最后时间步的输出作为全连接的输入 53 | finalOutput = self.embeddedWords[:, 0, :] 54 | 55 | outputSize = config.model.hiddenSizes[-1] * 2 # 因为是双向LSTM，最终的输出值是fw和bw的拼接，因此要乘以2 56 | output = tf.reshape(finalOutput, [-1, outputSize]) # reshape成全连接层的输入维度 57 | 58 | # 全连接层的输出 59 | with tf.name_scope("output"): 60 | outputW = tf.get_variable( 61 | "outputW", 62 | shape=[outputSize, config.numClasses], 63 | initializer=tf.contrib.layers.xavier_initializer()) 64 | 65 | outputB = tf.Variable(tf.constant(0.1, shape=[config.numClasses]), name="outputB") 66 | l2Loss += tf.nn.l2_loss(outputW) 67 | l2Loss += tf.nn.l2_loss(outputB) 68 | self.logits = tf.nn.xw_plus_b(output, outputW, outputB, name="logits") 69 | if config.numClasses == 1: 70 | self.predictions = tf.cast(tf.greater_equal(self.logits, 0.0), tf.float32, name="predictions") 71 | elif config.numClasses > 1: 72 | self.predictions = tf.argmax(self.logits, axis=-1, name="predictions") 73 | 74 | # 计算二元交叉熵损失 75 | with tf.name_scope("loss"): 76 | if config.numClasses == 1: 77 | losses = tf.nn.sigmoid_cross_entropy_with_logits(logits=self.logits, 78 | labels=tf.cast(tf.reshape(self.inputY, [-1, 1]), 79 | dtype=tf.float32)) 80 | elif config.numClasses > 1: 81 | losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.inputY) 82 | 83 | self.loss = tf.reduce_mean(losses) + config.model.l2RegLambda * l2Loss -------------------------------------------------------------------------------- /BiLstm/parameter_config.py: -------------------------------------------------------------------------------- 1 | # 配置参数 2 | class TrainingConfig(object): 3 | epoches = 10 4 | evaluateEvery = 100 5 | checkpointEvery = 100 6 | learningRate = 0.001 7 | 8 | class ModelConfig(object): 9 | embeddingSize = 200 10 | hiddenSizes = [256, 256] # 单层LSTM结构的神经元个数 11 | dropoutKeepProb = 0.5 12 | l2RegLambda = 0.0 13 | 14 | class Config(object): 15 | sequenceLength = 200 # 取了所有序列长度的均值 16 | batchSize = 128 17 | dataSource = "../data/preProcess/labeledTrain.csv" 18 | stopWordSource = "../data/english" 19 | numClasses = 1 # 二分类设置为1，多分类设置为类别的数目 20 | rate = 0.8 # 训练集的比例 21 | training = TrainingConfig() 22 | model = ModelConfig() 23 | 24 | # 实例化配置参数对象 25 | # config = Config() -------------------------------------------------------------------------------- /BiLstm/predict.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import os 3 | import csv 4 | import time 5 | import datetime 6 | import random 7 | import json 8 | from collections import Counter 9 | from math import sqrt 10 | import gensim 11 | import pandas as pd 12 | import numpy as np 13 | import tensorflow as tf 14 | from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score 15 | import parameter_config 16 | config =parameter_config.Config() 17 | 18 | #7预测代码 19 | x = "this movie is full of references like mad max ii the wild one and many others the ladybug´s face it´s a clear reference or tribute to peter lorre this movie is a masterpiece we´ll talk much more about in the future" 20 | 21 | # 注：下面两个词典要保证和当前加载的模型对应的词典是一致的 22 | with open("../data/wordJson/word2idx.json", "r", encoding="utf-8") as f: 23 | word2idx = json.load(f) 24 | 25 | with open("../data/wordJson/label2idx.json", "r", encoding="utf-8") as f: 26 | label2idx = json.load(f) 27 | idx2label = {value: key for key, value in label2idx.items()} 28 | 29 | xIds = [word2idx.get(item, word2idx["UNK"]) for item in x.split(" ")] 30 | if len(xIds) >= config.sequenceLength: 31 | xIds = xIds[:config.sequenceLength] 32 | else: 33 | xIds = xIds + [word2idx["PAD"]] * (config.sequenceLength - len(xIds)) 34 | 35 | graph = tf.Graph() 36 | with graph.as_default(): 37 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) 38 | session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, gpu_options=gpu_options) 39 | sess = tf.Session(config=session_conf) 40 | 41 | with sess.as_default(): 42 | checkpoint_file = tf.train.latest_checkpoint("../model/Bi-LSTM/model/") 43 | saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file)) 44 | saver.restore(sess, checkpoint_file) 45 | 46 | # 获得需要喂给模型的参数，输出的结果依赖的输入值 47 | inputX = graph.get_operation_by_name("inputX").outputs[0] 48 | dropoutKeepProb = graph.get_operation_by_name("dropoutKeepProb").outputs[0] 49 | 50 | # 获得输出的结果 51 | predictions = graph.get_tensor_by_name("output/predictions:0") 52 | 53 | pred = sess.run(predictions, feed_dict={inputX: [xIds], dropoutKeepProb: 1.0})[0] 54 | 55 | # print(pred) 56 | pred = [idx2label[item] for item in pred] 57 | print(pred) -------------------------------------------------------------------------------- /BiLstmAttention/get_train_data.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import json 3 | from collections import Counter 4 | import gensim 5 | import pandas as pd 6 | import numpy as np 7 | import parameter_config 8 | 9 | # 2 数据预处理的类，生成训练集和测试集 10 | class Dataset(object): 11 | def __init__(self, config): 12 | self.config = config 13 | self._dataSource = config.dataSource 14 | self._stopWordSource = config.stopWordSource 15 | self._sequenceLength = config.sequenceLength # 每条输入的序列处理为定长 16 | self._embeddingSize = config.model.embeddingSize 17 | self._batchSize = config.batchSize 18 | self._rate = config.rate 19 | self._stopWordDict = {} 20 | self.trainReviews = [] 21 | self.trainLabels = [] 22 | self.evalReviews = [] 23 | self.evalLabels = [] 24 | self.wordEmbedding = None 25 | self.labelList = [] 26 | def _readData(self, filePath): 27 | """ 28 | 从csv文件中读取数据集,就本次测试的文件做记录 29 | """ 30 | df = pd.read_csv(filePath) #读取文件，是三列的数据，第一列是review,第二列sentiment，第三列rate 31 | if self.config.numClasses == 1: 32 | labels = df["sentiment"].tolist() #读取sentiment列的数据， 显示输出01序列数组25000条 33 | elif self.config.numClasses > 1: 34 | labels = df["rate"].tolist() #因为numClasses控制，本次取样没有取超过二分类 该处没有输出 35 | review = df["review"].tolist() 36 | reviews = [line.strip().split() for line in review] #按空格语句切分 37 | return reviews, labels 38 | def _labelToIndex(self, labels, label2idx): 39 | """ 40 | 将标签转换成索引表示 41 | """ 42 | labelIds = [label2idx[label] for label in labels] #print(labels==labelIds) 结果显示为true,也就是两个一样 43 | return labelIds 44 | def _wordToIndex(self, reviews, word2idx): 45 | """将词转换成索引""" 46 | reviewIds = [[word2idx.get(item, word2idx["UNK"]) for item in review] for review in reviews] 47 | # print(max(max(reviewIds))) 48 | # print(reviewIds) 49 | return reviewIds #返回25000个无序的数组 50 | def _genTrainEvalData(self, x, y, word2idx, rate): 51 | """生成训练集和验证集 """ 52 | reviews = [] 53 | # print(self._sequenceLength) 54 | # print(len(x)) 55 | for review in x: #self._sequenceLength为200，表示长的切成200，短的补齐，x数据依旧是25000 56 | if len(review) >= self._sequenceLength: 57 | reviews.append(review[:self._sequenceLength]) 58 | else: 59 | reviews.append(review + [word2idx["PAD"]] * (self._sequenceLength - len(review))) 60 | # print(len(review + [word2idx["PAD"]] * (self._sequenceLength - len(review)))) 61 | #以下是按照rate比例切分训练和测试数据： 62 | trainIndex = int(len(x) * rate) 63 | trainReviews = np.asarray(reviews[:trainIndex], dtype="int64") 64 | trainLabels = np.array(y[:trainIndex], dtype="float32") 65 | evalReviews = np.asarray(reviews[trainIndex:], dtype="int64") 66 | evalLabels = np.array(y[trainIndex:], dtype="float32") 67 | return trainReviews, trainLabels, evalReviews, evalLabels 68 | 69 | def _getWordEmbedding(self, words): 70 | """按照我们的数据集中的单词取出预训练好的word2vec中的词向量 71 | 反馈词和对应的向量（200维度），另外前面增加PAD对用0的数组，UNK对应随机数组。 72 | """ 73 | wordVec = gensim.models.KeyedVectors.load_word2vec_format("../word2vec/word2Vec.bin", binary=True) 74 | vocab = [] 75 | wordEmbedding = [] 76 | # 添加 "pad" 和 "UNK", 77 | vocab.append("PAD") 78 | vocab.append("UNK") 79 | wordEmbedding.append(np.zeros(self._embeddingSize)) # _embeddingSize 本文定义的是200 80 | wordEmbedding.append(np.random.randn(self._embeddingSize)) 81 | # print(wordEmbedding) 82 | for word in words: 83 | try: 84 | vector = wordVec.wv[word] 85 | vocab.append(word) 86 | wordEmbedding.append(vector) 87 | except: 88 | print(word + "不存在于词向量中") 89 | # print(vocab[:3],wordEmbedding[:3]) 90 | return vocab, np.array(wordEmbedding) 91 | def _genVocabulary(self, reviews, labels): 92 | """生成词向量和词汇-索引映射字典，可以用全数据集""" 93 | allWords = [word for review in reviews for word in review] #单词数量5738236 reviews是25000个观点句子【】 94 | subWords = [word for word in allWords if word not in self.stopWordDict] # 去掉停用词 95 | wordCount = Counter(subWords) # 统计词频 96 | sortWordCount = sorted(wordCount.items(), key=lambda x: x[1], reverse=True) #返回键值对，并按照数量排序 97 | # print(len(sortWordCount)) #161330 98 | # print(sortWordCount[:4],sortWordCount[-4:]) # [('movie', 41104), ('film', 36981), ('one', 24966), ('like', 19490)] [('daeseleires', 1), ('nice310', 1), ('shortsightedness', 1), ('unfairness', 1)] 99 | words = [item[0] for item in sortWordCount if item[1] >= 5] # 去除低频词，低于5的 100 | vocab, wordEmbedding = self._getWordEmbedding(words) 101 | self.wordEmbedding = wordEmbedding 102 | word2idx = dict(zip(vocab, list(range(len(vocab))))) #生成类似这种{'I': 0, 'love': 1, 'yanzi': 2} 103 | uniqueLabel = list(set(labels)) #标签去重 最后就 0 1了 104 | label2idx = dict(zip(uniqueLabel, list(range(len(uniqueLabel))))) #本文就 {0: 0, 1: 1} 105 | self.labelList = list(range(len(uniqueLabel))) 106 | # 将词汇-索引映射表保存为json数据，之后做inference时直接加载来处理数据 107 | with open("../data/wordJson/word2idx.json", "w", encoding="utf-8") as f: 108 | json.dump(word2idx, f) 109 | with open("../data/wordJson/label2idx.json", "w", encoding="utf-8") as f: 110 | json.dump(label2idx, f) 111 | return word2idx, label2idx 112 | 113 | def _readStopWord(self, stopWordPath): 114 | """ 115 | 读取停用词 116 | """ 117 | with open(stopWordPath, "r") as f: 118 | stopWords = f.read() 119 | stopWordList = stopWords.splitlines() 120 | # 将停用词用列表的形式生成，之后查找停用词时会比较快 121 | self.stopWordDict = dict(zip(stopWordList, list(range(len(stopWordList))))) 122 | 123 | def dataGen(self): 124 | """ 125 | 初始化训练集和验证集 126 | """ 127 | # 初始化停用词 128 | self._readStopWord(self._stopWordSource) 129 | # 初始化数据集 130 | reviews, labels = self._readData(self._dataSource) 131 | # 初始化词汇-索引映射表和词向量矩阵 132 | word2idx, label2idx = self._genVocabulary(reviews, labels) 133 | # 将标签和句子数值化 134 | labelIds = self._labelToIndex(labels, label2idx) 135 | reviewIds = self._wordToIndex(reviews, word2idx) 136 | # 初始化训练集和测试集 137 | trainReviews, trainLabels, evalReviews, evalLabels = self._genTrainEvalData(reviewIds, labelIds, word2idx, 138 | self._rate) 139 | self.trainReviews = trainReviews 140 | self.trainLabels = trainLabels 141 | 142 | self.evalReviews = evalReviews 143 | self.evalLabels = evalLabels 144 | 145 | #获取前些模块的数据 146 | # config =parameter_config.Config() 147 | # data = Dataset(config) 148 | # data.dataGen() -------------------------------------------------------------------------------- /BiLstmAttention/mode_structure.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | 3 | import tensorflow as tf 4 | import parameter_config 5 | 6 | config = parameter_config.Config() 7 | # 构建模型 3 Bi-LSTM + Attention模型 8 | # 构建模型 9 | class BiLSTMAttention(object): 10 | def __init__(self, config, wordEmbedding): 11 | 12 | # 定义模型的输入 13 | self.inputX = tf.placeholder(tf.int32, [None, config.sequenceLength], name="inputX") 14 | self.inputY = tf.placeholder(tf.int32, [None], name="inputY") 15 | 16 | self.dropoutKeepProb = tf.placeholder(tf.float32, name="dropoutKeepProb") 17 | 18 | # 定义l2损失 19 | l2Loss = tf.constant(0.0) 20 | 21 | # 词嵌入层 22 | with tf.name_scope("embedding"): 23 | 24 | # 利用预训练的词向量初始化词嵌入矩阵 25 | self.W = tf.Variable(tf.cast(wordEmbedding, dtype=tf.float32, name="word2vec") ,name="W") 26 | # 利用词嵌入矩阵将输入的数据中的词转换成词向量，维度[batch_size, sequence_length, embedding_size] 27 | self.embeddedWords = tf.nn.embedding_lookup(self.W, self.inputX) 28 | 29 | # 定义两层双向LSTM的模型结构 30 | with tf.name_scope("Bi-LSTM"): 31 | for idx, hiddenSize in enumerate(config.model.hiddenSizes): 32 | with tf.name_scope("Bi-LSTM" + str(idx)): 33 | # 定义前向LSTM结构 34 | lstmFwCell = tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.LSTMCell(num_units=hiddenSize, state_is_tuple=True), 35 | output_keep_prob=self.dropoutKeepProb) 36 | # 定义反向LSTM结构 37 | lstmBwCell = tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.LSTMCell(num_units=hiddenSize, state_is_tuple=True), 38 | output_keep_prob=self.dropoutKeepProb) 39 | 40 | 41 | # 采用动态rnn，可以动态的输入序列的长度，若没有输入，则取序列的全长 42 | # outputs是一个元祖(output_fw, output_bw)，其中两个元素的维度都是[batch_size, max_time, hidden_size],fw和bw的hidden_size一样 43 | # self.current_state 是最终的状态，二元组(state_fw, state_bw)，state_fw=[batch_size, s]，s是一个元祖(h, c) 44 | outputs_, self.current_state = tf.nn.bidirectional_dynamic_rnn(lstmFwCell, lstmBwCell, 45 | self.embeddedWords, dtype=tf.float32, 46 | scope="bi-lstm" + str(idx)) 47 | 48 | # 对outputs中的fw和bw的结果拼接 [batch_size, time_step, hidden_size * 2], 传入到下一层Bi-LSTM中 49 | self.embeddedWords = tf.concat(outputs_, 2) 50 | 51 | # 将最后一层Bi-LSTM输出的结果分割成前向和后向的输出 52 | outputs = tf.split(self.embeddedWords, 2, -1) 53 | 54 | # 在Bi-LSTM+Attention的论文中，将前向和后向的输出相加 55 | with tf.name_scope("Attention"): 56 | H = outputs[0] + outputs[1] 57 | 58 | # 得到Attention的输出 59 | output = self.attention(H) 60 | outputSize = config.model.hiddenSizes[-1] 61 | 62 | # 全连接层的输出 63 | with tf.name_scope("output"): 64 | outputW = tf.get_variable( 65 | "outputW", 66 | shape=[outputSize, config.numClasses], 67 | initializer=tf.contrib.layers.xavier_initializer()) 68 | 69 | outputB= tf.Variable(tf.constant(0.1, shape=[config.numClasses]), name="outputB") 70 | l2Loss += tf.nn.l2_loss(outputW) 71 | l2Loss += tf.nn.l2_loss(outputB) 72 | self.logits = tf.nn.xw_plus_b(output, outputW, outputB, name="logits") 73 | 74 | if config.numClasses == 1: 75 | self.predictions = tf.cast(tf.greater_equal(self.logits, 0.0), tf.float32, name="predictions") 76 | elif config.numClasses > 1: 77 | self.predictions = tf.argmax(self.logits, axis=-1, name="predictions") 78 | 79 | # 计算二元交叉熵损失 80 | with tf.name_scope("loss"): 81 | 82 | if config.numClasses == 1: 83 | losses = tf.nn.sigmoid_cross_entropy_with_logits(logits=self.logits, labels=tf.cast(tf.reshape(self.inputY, [-1, 1]), 84 | dtype=tf.float32)) 85 | elif config.numClasses > 1: 86 | losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.inputY) 87 | 88 | self.loss = tf.reduce_mean(losses) + config.model.l2RegLambda * l2Loss 89 | 90 | def attention(self, H): 91 | """ 92 | 利用Attention机制得到句子的向量表示 93 | """ 94 | # 获得最后一层LSTM的神经元数量 95 | hiddenSize = config.model.hiddenSizes[-1] 96 | 97 | # 初始化一个权重向量，是可训练的参数 98 | W = tf.Variable(tf.random_normal([hiddenSize], stddev=0.1)) 99 | 100 | # 对Bi-LSTM的输出用激活函数做非线性转换 101 | M = tf.tanh(H) 102 | 103 | # 对W和M做矩阵运算，W=[batch_size, time_step, hidden_size]，计算前做维度转换成[batch_size * time_step, hidden_size] 104 | # newM = [batch_size, time_step, 1]，每一个时间步的输出由向量转换成一个数字 105 | newM = tf.matmul(tf.reshape(M, [-1, hiddenSize]), tf.reshape(W, [-1, 1])) 106 | 107 | # 对newM做维度转换成[batch_size, time_step] 108 | restoreM = tf.reshape(newM, [-1, config.sequenceLength]) 109 | 110 | # 用softmax做归一化处理[batch_size, time_step] 111 | self.alpha = tf.nn.softmax(restoreM) 112 | 113 | # 利用求得的alpha的值对H进行加权求和，用矩阵运算直接操作 114 | r = tf.matmul(tf.transpose(H, [0, 2, 1]), tf.reshape(self.alpha, [-1, config.sequenceLength, 1])) 115 | 116 | # 将三维压缩成二维sequeezeR=[batch_size, hidden_size] 117 | sequeezeR = tf.reshape(r, [-1, hiddenSize]) 118 | 119 | sentenceRepren = tf.tanh(sequeezeR) 120 | 121 | # 对Attention的输出可以做dropout处理 122 | output = tf.nn.dropout(sentenceRepren, self.dropoutKeepProb) 123 | 124 | return output -------------------------------------------------------------------------------- /BiLstmAttention/mode_trainning.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | 3 | import os 4 | import datetime 5 | import numpy as np 6 | import tensorflow as tf 7 | import parameter_config 8 | import get_train_data 9 | import mode_structure 10 | 11 | #获取前些模块的数据 12 | config =parameter_config.Config() 13 | data = get_train_data.Dataset(config) 14 | data.dataGen() 15 | 16 | #4生成batch数据集 17 | def nextBatch(x, y, batchSize): 18 | # 生成batch数据集，用生成器的方式输出 19 | perm = np.arange(len(x)) #返回[0 1 2 ... len(x)]的数组 20 | np.random.shuffle(perm) #乱序 21 | x = x[perm] 22 | y = y[perm] 23 | numBatches = len(x) // batchSize 24 | 25 | for i in range(numBatches): 26 | start = i * batchSize 27 | end = start + batchSize 28 | batchX = np.array(x[start: end], dtype="int64") 29 | batchY = np.array(y[start: end], dtype="float32") 30 | yield batchX, batchY 31 | 32 | # 5 定义计算metrics的函数 33 | """ 34 | 定义各类性能指标 35 | """ 36 | def mean(item: list) -> float: 37 | """ 38 | 计算列表中元素的平均值 39 | :param item: 列表对象 40 | :return: 41 | """ 42 | res = sum(item) / len(item) if len(item) > 0 else 0 43 | return res 44 | 45 | def accuracy(pred_y, true_y): 46 | """ 47 | 计算二类和多类的准确率 48 | :param pred_y: 预测结果 49 | :param true_y: 真实结果 50 | :return: 51 | """ 52 | if isinstance(pred_y[0], list): 53 | pred_y = [item[0] for item in pred_y] 54 | corr = 0 55 | for i in range(len(pred_y)): 56 | if pred_y[i] == true_y[i]: 57 | corr += 1 58 | acc = corr / len(pred_y) if len(pred_y) > 0 else 0 59 | return acc 60 | 61 | def binary_precision(pred_y, true_y, positive=1): 62 | """ 63 | 二类的精确率计算 64 | :param pred_y: 预测结果 65 | :param true_y: 真实结果 66 | :param positive: 正例的索引表示 67 | :return: 68 | """ 69 | corr = 0 70 | pred_corr = 0 71 | for i in range(len(pred_y)): 72 | if pred_y[i] == positive: 73 | pred_corr += 1 74 | if pred_y[i] == true_y[i]: 75 | corr += 1 76 | 77 | prec = corr / pred_corr if pred_corr > 0 else 0 78 | return prec 79 | 80 | def binary_recall(pred_y, true_y, positive=1): 81 | """ 82 | 二类的召回率 83 | :param pred_y: 预测结果 84 | :param true_y: 真实结果 85 | :param positive: 正例的索引表示 86 | :return: 87 | """ 88 | corr = 0 89 | true_corr = 0 90 | for i in range(len(pred_y)): 91 | if true_y[i] == positive: 92 | true_corr += 1 93 | if pred_y[i] == true_y[i]: 94 | corr += 1 95 | 96 | rec = corr / true_corr if true_corr > 0 else 0 97 | return rec 98 | 99 | def binary_f_beta(pred_y, true_y, beta=1.0, positive=1): 100 | """ 101 | 二类的f beta值 102 | :param pred_y: 预测结果 103 | :param true_y: 真实结果 104 | :param beta: beta值 105 | :param positive: 正例的索引表示 106 | :return: 107 | """ 108 | precision = binary_precision(pred_y, true_y, positive) 109 | recall = binary_recall(pred_y, true_y, positive) 110 | try: 111 | f_b = (1 + beta * beta) * precision * recall / (beta * beta * precision + recall) 112 | except: 113 | f_b = 0 114 | return f_b 115 | 116 | def multi_precision(pred_y, true_y, labels): 117 | """ 118 | 多类的精确率 119 | :param pred_y: 预测结果 120 | :param true_y: 真实结果 121 | :param labels: 标签列表 122 | :return: 123 | """ 124 | if isinstance(pred_y[0], list): 125 | pred_y = [item[0] for item in pred_y] 126 | 127 | precisions = [binary_precision(pred_y, true_y, label) for label in labels] 128 | prec = mean(precisions) 129 | return prec 130 | 131 | def multi_recall(pred_y, true_y, labels): 132 | """ 133 | 多类的召回率 134 | :param pred_y: 预测结果 135 | :param true_y: 真实结果 136 | :param labels: 标签列表 137 | :return: 138 | """ 139 | if isinstance(pred_y[0], list): 140 | pred_y = [item[0] for item in pred_y] 141 | 142 | recalls = [binary_recall(pred_y, true_y, label) for label in labels] 143 | rec = mean(recalls) 144 | return rec 145 | 146 | def multi_f_beta(pred_y, true_y, labels, beta=1.0): 147 | """ 148 | 多类的f beta值 149 | :param pred_y: 预测结果 150 | :param true_y: 真实结果 151 | :param labels: 标签列表 152 | :param beta: beta值 153 | :return: 154 | """ 155 | if isinstance(pred_y[0], list): 156 | pred_y = [item[0] for item in pred_y] 157 | 158 | f_betas = [binary_f_beta(pred_y, true_y, beta, label) for label in labels] 159 | f_beta = mean(f_betas) 160 | return f_beta 161 | 162 | def get_binary_metrics(pred_y, true_y, f_beta=1.0): 163 | """ 164 | 得到二分类的性能指标 165 | :param pred_y: 166 | :param true_y: 167 | :param f_beta: 168 | :return: 169 | """ 170 | acc = accuracy(pred_y, true_y) 171 | recall = binary_recall(pred_y, true_y) 172 | precision = binary_precision(pred_y, true_y) 173 | f_beta = binary_f_beta(pred_y, true_y, f_beta) 174 | return acc, recall, precision, f_beta 175 | 176 | def get_multi_metrics(pred_y, true_y, labels, f_beta=1.0): 177 | """ 178 | 得到多分类的性能指标 179 | :param pred_y: 180 | :param true_y: 181 | :param labels: 182 | :param f_beta: 183 | :return: 184 | """ 185 | acc = accuracy(pred_y, true_y) 186 | recall = multi_recall(pred_y, true_y, labels) 187 | precision = multi_precision(pred_y, true_y, labels) 188 | f_beta = multi_f_beta(pred_y, true_y, labels, f_beta) 189 | return acc, recall, precision, f_beta 190 | 191 | # 6 训练模型 192 | # 生成训练集和验证集 193 | trainReviews = data.trainReviews 194 | trainLabels = data.trainLabels 195 | evalReviews = data.evalReviews 196 | evalLabels = data.evalLabels 197 | 198 | wordEmbedding = data.wordEmbedding 199 | labelList = data.labelList 200 | 201 | # 定义计算图 202 | with tf.Graph().as_default(): 203 | 204 | session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) 205 | session_conf.gpu_options.allow_growth=True 206 | session_conf.gpu_options.per_process_gpu_memory_fraction = 0.9 # 配置gpu占用率 207 | 208 | sess = tf.Session(config=session_conf) 209 | 210 | # 定义会话 211 | with sess.as_default(): 212 | bilstmattention = mode_structure.BiLSTMAttention(config, wordEmbedding) 213 | globalStep = tf.Variable(0, name="globalStep", trainable=False) 214 | # 定义优化函数，传入学习速率参数 215 | optimizer = tf.train.AdamOptimizer(config.training.learningRate) 216 | # 计算梯度,得到梯度和变量 217 | gradsAndVars = optimizer.compute_gradients(bilstmattention.loss) 218 | # 将梯度应用到变量下，生成训练器 219 | trainOp = optimizer.apply_gradients(gradsAndVars, global_step=globalStep) 220 | 221 | # 用summary绘制tensorBoard 222 | gradSummaries = [] 223 | for g, v in gradsAndVars: 224 | if g is not None: 225 | tf.summary.histogram("{}/grad/hist".format(v.name), g) 226 | tf.summary.scalar("{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) 227 | 228 | outDir = os.path.abspath(os.path.join(os.path.curdir, "summarys")) 229 | print("Writing to {}\n".format(outDir)) 230 | 231 | lossSummary = tf.summary.scalar("loss", bilstmattention.loss) 232 | summaryOp = tf.summary.merge_all() 233 | 234 | trainSummaryDir = os.path.join(outDir, "train") 235 | trainSummaryWriter = tf.summary.FileWriter(trainSummaryDir, sess.graph) 236 | 237 | evalSummaryDir = os.path.join(outDir, "eval") 238 | evalSummaryWriter = tf.summary.FileWriter(evalSummaryDir, sess.graph) 239 | 240 | 241 | # 初始化所有变量 242 | saver = tf.train.Saver(tf.global_variables(), max_to_keep=5) 243 | 244 | # 保存模型的一种方式，保存为pb文件 245 | savedModelPath = "../model/bilstm-atten/savedModel" 246 | if os.path.exists(savedModelPath): 247 | os.rmdir(savedModelPath) 248 | builder = tf.saved_model.builder.SavedModelBuilder(savedModelPath) 249 | 250 | sess.run(tf.global_variables_initializer()) 251 | 252 | def trainStep(batchX, batchY): 253 | """ 254 | 训练函数 255 | """ 256 | feed_dict = { 257 | bilstmattention.inputX: batchX, 258 | bilstmattention.inputY: batchY, 259 | bilstmattention.dropoutKeepProb: config.model.dropoutKeepProb 260 | } 261 | _, summary, step, loss, predictions = sess.run( 262 | [trainOp, summaryOp, globalStep, bilstmattention.loss, bilstmattention.predictions], 263 | feed_dict) 264 | timeStr = datetime.datetime.now().isoformat() 265 | 266 | if config.numClasses == 1: 267 | acc, recall, prec, f_beta = get_binary_metrics(pred_y=predictions, true_y=batchY) 268 | 269 | elif config.numClasses > 1: 270 | acc, recall, prec, f_beta = get_multi_metrics(pred_y=predictions, true_y=batchY, 271 | labels=labelList) 272 | 273 | trainSummaryWriter.add_summary(summary, step) 274 | 275 | return loss, acc, prec, recall, f_beta 276 | 277 | def devStep(batchX, batchY): 278 | """ 279 | 验证函数 280 | """ 281 | feed_dict = { 282 | bilstmattention.inputX: batchX, 283 | bilstmattention.inputY: batchY, 284 | bilstmattention.dropoutKeepProb: 1.0 285 | } 286 | summary, step, loss, predictions = sess.run( 287 | [summaryOp, globalStep, bilstmattention.loss, bilstmattention.predictions], 288 | feed_dict) 289 | 290 | if config.numClasses == 1: 291 | 292 | acc, precision, recall, f_beta = get_binary_metrics(pred_y=predictions, true_y=batchY) 293 | elif config.numClasses > 1: 294 | acc, precision, recall, f_beta = get_multi_metrics(pred_y=predictions, true_y=batchY, labels=labelList) 295 | 296 | evalSummaryWriter.add_summary(summary, step) 297 | 298 | return loss, acc, precision, recall, f_beta 299 | 300 | for i in range(config.training.epoches): 301 | # 训练模型 302 | print("start training model") 303 | for batchTrain in nextBatch(trainReviews, trainLabels, config.batchSize): 304 | loss, acc, prec, recall, f_beta = trainStep(batchTrain[0], batchTrain[1]) 305 | 306 | currentStep = tf.train.global_step(sess, globalStep) 307 | print("train: step: {}, loss: {}, acc: {}, recall: {}, precision: {}, f_beta: {}".format( 308 | currentStep, loss, acc, recall, prec, f_beta)) 309 | if currentStep % config.training.evaluateEvery == 0: 310 | print("\nEvaluation:") 311 | 312 | losses = [] 313 | accs = [] 314 | f_betas = [] 315 | precisions = [] 316 | recalls = [] 317 | 318 | for batchEval in nextBatch(evalReviews, evalLabels, config.batchSize): 319 | loss, acc, precision, recall, f_beta = devStep(batchEval[0], batchEval[1]) 320 | losses.append(loss) 321 | accs.append(acc) 322 | f_betas.append(f_beta) 323 | precisions.append(precision) 324 | recalls.append(recall) 325 | 326 | time_str = datetime.datetime.now().isoformat() 327 | print("{}, step: {}, loss: {}, acc: {},precision: {}, recall: {}, f_beta: {}".format(time_str, currentStep, mean(losses), 328 | mean(accs), mean(precisions), 329 | mean(recalls), mean(f_betas))) 330 | 331 | if currentStep % config.training.checkpointEvery == 0: 332 | # 保存模型的另一种方法，保存checkpoint文件 333 | path = saver.save(sess, "../model/bilstm-atten/model/my-model", global_step=currentStep) 334 | print("Saved model checkpoint to {}\n".format(path)) 335 | 336 | inputs = {"inputX": tf.saved_model.utils.build_tensor_info(bilstmattention.inputX), 337 | "keepProb": tf.saved_model.utils.build_tensor_info(bilstmattention.dropoutKeepProb)} 338 | 339 | outputs = {"predictions": tf.saved_model.utils.build_tensor_info(bilstmattention.predictions)} 340 | 341 | prediction_signature = tf.saved_model.signature_def_utils.build_signature_def(inputs=inputs, outputs=outputs, 342 | method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME) 343 | legacy_init_op = tf.group(tf.tables_initializer(), name="legacy_init_op") 344 | builder.add_meta_graph_and_variables(sess, [tf.saved_model.tag_constants.SERVING], 345 | signature_def_map={"predict": prediction_signature}, legacy_init_op=legacy_init_op) 346 | 347 | builder.save() 348 | 349 | 350 | -------------------------------------------------------------------------------- /BiLstmAttention/parameter_config.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | #需要的所有导入包，存放留用，转换到jupyter后直接使用 3 | # 1 配置训练参数 4 | class TrainingConfig(object): 5 | 6 | epoches = 4 7 | evaluateEvery = 100 8 | checkpointEvery = 100 9 | learningRate = 0.001 10 | 11 | class ModelConfig(object): 12 | embeddingSize = 200 13 | hiddenSizes = [256, 128] # LSTM结构的神经元个数 14 | dropoutKeepProb = 0.5 15 | l2RegLambda = 0.0 16 | 17 | class Config(object): 18 | sequenceLength = 200 # 取了所有序列长度的均值 19 | batchSize = 128 20 | dataSource = "../data/preProcess/labeledTrain.csv" 21 | stopWordSource = "../data/english" 22 | numClasses = 1 # 二分类设置为1，多分类设置为类别的数目 23 | rate = 0.8 # 训练集的比例 24 | training = TrainingConfig() 25 | model = ModelConfig() 26 | 27 | # 实例化配置参数对象 28 | config = Config() -------------------------------------------------------------------------------- /BiLstmAttention/predict.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import os 3 | import csv 4 | import time 5 | import datetime 6 | import random 7 | import json 8 | from collections import Counter 9 | from math import sqrt 10 | import gensim 11 | import pandas as pd 12 | import numpy as np 13 | import tensorflow as tf 14 | from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score 15 | import parameter_config 16 | config =parameter_config.Config() 17 | 18 | #7预测代码 19 | x = "this movie is full of references like mad max ii the wild one and many others the ladybug´s face it´s a clear reference or tribute to peter lorre this movie is a masterpiece we´ll talk much more about in the future" 20 | # x = "his movie is the same as the third level movie. There's no place to look good" 21 | # x = "This film is not good" #最终反馈为0 22 | # x = "This film is bad" #最终反馈为0 23 | 24 | x = "this movie is full of references like mad max ii the wild one and many others the ladybug´s face it´s a clear reference or tribute to peter lorre this movie is a masterpiece we´ll talk much more about in the future" 25 | 26 | # 注：下面两个词典要保证和当前加载的模型对应的词典是一致的 27 | with open("../data/wordJson/word2idx.json", "r", encoding="utf-8") as f: 28 | word2idx = json.load(f) 29 | 30 | with open("../data/wordJson/label2idx.json", "r", encoding="utf-8") as f: 31 | label2idx = json.load(f) 32 | idx2label = {value: key for key, value in label2idx.items()} 33 | 34 | xIds = [word2idx.get(item, word2idx["UNK"]) for item in x.split(" ")] 35 | if len(xIds) >= config.sequenceLength: 36 | xIds = xIds[:config.sequenceLength] 37 | else: 38 | xIds = xIds + [word2idx["PAD"]] * (config.sequenceLength - len(xIds)) 39 | 40 | graph = tf.Graph() 41 | with graph.as_default(): 42 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) 43 | session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, gpu_options=gpu_options) 44 | sess = tf.Session(config=session_conf) 45 | 46 | with sess.as_default(): 47 | checkpoint_file = tf.train.latest_checkpoint("../model/bilstm-atten/model/") 48 | saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file)) 49 | saver.restore(sess, checkpoint_file) 50 | 51 | # 获得需要喂给模型的参数，输出的结果依赖的输入值 52 | inputX = graph.get_operation_by_name("inputX").outputs[0] 53 | dropoutKeepProb = graph.get_operation_by_name("dropoutKeepProb").outputs[0] 54 | 55 | # 获得输出的结果 56 | predictions = graph.get_tensor_by_name("output/predictions:0") 57 | 58 | pred = sess.run(predictions, feed_dict={inputX: [xIds], dropoutKeepProb: 1.0})[0] 59 | 60 | # print(pred) 61 | pred = [idx2label[item] for item in pred] 62 | print(pred) -------------------------------------------------------------------------------- /ELMo/get_train_data.py: -------------------------------------------------------------------------------- 1 | # _*_ coding:utf-8 _*_ 2 | # Author:yifan 3 | import json 4 | from collections import Counter 5 | import gensim 6 | import pandas as pd 7 | import numpy as np 8 | import parameter_config 9 | 10 | # 2 数据预处理的类，生成训练集和测试集 11 | 12 | # 数据预处理的类，生成训练集和测试集 13 | 14 | class Dataset(object): 15 | def __init__(self, config): 16 | self._dataSource = config.dataSource 17 | self._stopWordSource = config.stopWordSource 18 | self._optionFile = config.optionFile 19 | self._weightFile = config.weightFile 20 | self._vocabFile = config.vocabFile 21 | self._tokenEmbeddingFile = config.tokenEmbeddingFile 22 | 23 | self._sequenceLength = config.sequenceLength # 每条输入的序列处理为定长 24 | self._embeddingSize = config.model.embeddingSize 25 | self._batchSize = config.batchSize 26 | self._rate = config.rate 27 | 28 | self.trainReviews = [] 29 | self.trainLabels = [] 30 | 31 | self.evalReviews = [] 32 | self.evalLabels = [] 33 | 34 | def _readData(self, filePath): 35 | """ 36 | 从csv文件中读取数据集 37 | """ 38 | 39 | df = pd.read_csv(filePath) 40 | labels = df["sentiment"].tolist() 41 | review = df["review"].tolist() 42 | reviews = [line.strip().split() for line in review] 43 | 44 | return reviews, labels 45 | 46 | def _genVocabFile(self, reviews): 47 | """ 48 | 用我们的训练数据生成一个词汇文件，并加入三个特殊字符 49 | """ 50 | allWords = [word for review in reviews for word in review] 51 | wordCount = Counter(allWords) # 统计词频 52 | sortWordCount = sorted(wordCount.items(), key=lambda x: x[1], reverse=True) 53 | words = [item[0] for item in sortWordCount.items()] 54 | allTokens = ['', '', ''] + words 55 | with open(self._vocabFile, 'w',encoding='UTF-8') as fout: 56 | fout.write('\n'.join(allTokens)) 57 | 58 | def _fixedSeq(self, reviews): 59 | """ 60 | 将长度超过200的截断为200的长度 61 | """ 62 | return [review[:self._sequenceLength] for review in reviews] 63 | 64 | def _genElmoEmbedding(self): 65 | """ 66 | 调用ELMO源码中的dump_token_embeddings方法，基于字符的表示生成词的向量表示。并保存成hdf5文件， 67 | 文件中的"embedding"键对应的value就是 68 | 词汇表文件中各词汇的向量表示，这些词汇的向量表示之后会作为BiLM的初始化输入。 69 | """ 70 | dump_token_embeddings( 71 | self._vocabFile, self._optionFile, self._weightFile, self._tokenEmbeddingFile) 72 | 73 | def _genTrainEvalData(self, x, y, rate): 74 | """ 75 | 生成训练集和验证集 76 | """ 77 | y = [[item] for item in y] 78 | trainIndex = int(len(x) * rate) 79 | 80 | trainReviews = x[:trainIndex] 81 | trainLabels = y[:trainIndex] 82 | 83 | evalReviews = x[trainIndex:] 84 | evalLabels = y[trainIndex:] 85 | 86 | return trainReviews, trainLabels, evalReviews, evalLabels 87 | 88 | 89 | def dataGen(self): 90 | """ 91 | 初始化训练集和验证集 92 | """ 93 | # 初始化数据集 94 | reviews, labels = self._readData(self._dataSource) 95 | # self._genVocabFile(reviews) # 生成vocabFile 96 | # self._genElmoEmbedding() # 生成elmo_token_embedding 97 | reviews = self._fixedSeq(reviews) 98 | # 初始化训练集和测试集 99 | trainReviews, trainLabels, evalReviews, evalLabels = self._genTrainEvalData(reviews, labels, self._rate) 100 | self.trainReviews = trainReviews 101 | self.trainLabels = trainLabels 102 | 103 | self.evalReviews = evalReviews 104 | self.evalLabels = evalLabels 105 | 106 | # from data import TokenBatcher 107 | # #获取前些模块的数据 108 | # config =parameter_config.Config() 109 | # data = Dataset(config) 110 | # data.dataGen() 111 | # batcher = TokenBatcher(config.vocabFile) 112 | # print(batcher) -------------------------------------------------------------------------------- /ELMo/mode_structure.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | # _*_ coding:utf-8 _*_ 3 | import tensorflow as tf 4 | import parameter_config 5 | 6 | config = parameter_config.Config() 7 | # 构建模型 3 ELMo模型 8 | # 构建模型 9 | class ELMo(object): 10 | """""" 11 | def __init__(self, config): 12 | # 定义模型的输入 13 | self.inputX = tf.placeholder(tf.float32, [None, config.sequenceLength, config.model.embeddingSize], name="inputX") 14 | self.inputY = tf.placeholder(tf.float32, [None, 1], name="inputY") 15 | self.dropoutKeepProb = tf.placeholder(tf.float32, name="dropoutKeepProb") 16 | 17 | # 定义l2损失 18 | l2Loss = tf.constant(0.0) 19 | 20 | with tf.name_scope("embedding"): 21 | embeddingW = tf.get_variable( 22 | "embeddingW", 23 | shape=[config.model.embeddingSize, config.model.embeddingSize], 24 | initializer=tf.contrib.layers.xavier_initializer()) 25 | reshapeInputX = tf.reshape(self.inputX, shape=[-1, config.model.embeddingSize]) 26 | 27 | self.embeddedWords = tf.reshape(tf.matmul(reshapeInputX, embeddingW), shape=[-1, config.sequenceLength, config.model.embeddingSize]) 28 | self.embeddedWords = tf.nn.dropout(self.embeddedWords, self.dropoutKeepProb) 29 | 30 | # 定义两层双向LSTM的模型结构 31 | with tf.name_scope("Bi-LSTM"): 32 | for idx, hiddenSize in enumerate(config.model.hiddenSizes): 33 | with tf.name_scope("Bi-LSTM" + str(idx)): 34 | # 定义前向LSTM结构 35 | lstmFwCell = tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.LSTMCell(num_units=hiddenSize, state_is_tuple=True), 36 | output_keep_prob=self.dropoutKeepProb) 37 | # 定义反向LSTM结构 38 | lstmBwCell = tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.LSTMCell(num_units=hiddenSize, state_is_tuple=True), 39 | output_keep_prob=self.dropoutKeepProb) 40 | 41 | 42 | # 采用动态rnn，可以动态的输入序列的长度，若没有输入，则取序列的全长 43 | # outputs是一个元组(output_fw, output_bw)，其中两个元素的维度都是[batch_size, max_time, hidden_size],fw和bw的hidden_size一样 44 | # self.current_state 是最终的状态，二元组(state_fw, state_bw)，state_fw=[batch_size, s]，s是一个元祖(h, c) 45 | outputs_, self.current_state = tf.nn.bidirectional_dynamic_rnn(lstmFwCell, lstmBwCell, 46 | self.embeddedWords, dtype=tf.float32, 47 | scope="bi-lstm" + str(idx)) 48 | 49 | # 对outputs中的fw和bw的结果拼接 [batch_size, time_step, hidden_size * 2], 传入到下一层Bi-LSTM中 50 | self.embeddedWords = tf.concat(outputs_, 2) 51 | # 将最后一层Bi-LSTM输出的结果分割成前向和后向的输出 52 | outputs = tf.split(self.embeddedWords, 2, -1) 53 | 54 | # 在Bi-LSTM+Attention的论文中，将前向和后向的输出相加 55 | with tf.name_scope("Attention"): 56 | H = outputs[0] + outputs[1] 57 | # 得到Attention的输出 58 | output = self._attention(H) 59 | outputSize = config.model.hiddenSizes[-1] 60 | 61 | # 全连接层的输出 62 | with tf.name_scope("output"): 63 | outputW = tf.get_variable( 64 | "outputW", 65 | shape=[outputSize, 1], 66 | initializer=tf.contrib.layers.xavier_initializer()) 67 | 68 | outputB= tf.Variable(tf.constant(0.1, shape=[1]), name="outputB") 69 | l2Loss += tf.nn.l2_loss(outputW) 70 | l2Loss += tf.nn.l2_loss(outputB) 71 | self.predictions = tf.nn.xw_plus_b(output, outputW, outputB, name="predictions") 72 | self.binaryPreds = tf.cast(tf.greater_equal(self.predictions, 0.0), tf.float32, name="binaryPreds") 73 | 74 | # 计算二元交叉熵损失 75 | with tf.name_scope("loss"): 76 | losses = tf.nn.sigmoid_cross_entropy_with_logits(logits=self.predictions, labels=self.inputY) 77 | self.loss = tf.reduce_mean(losses) + config.model.l2RegLambda * l2Loss 78 | 79 | def _attention(self, H): 80 | """ 81 | 利用Attention机制得到句子的向量表示 82 | """ 83 | # 获得最后一层LSTM的神经元数量 84 | hiddenSize = config.model.hiddenSizes[-1] 85 | 86 | # 初始化一个权重向量，是可训练的参数 87 | W = tf.Variable(tf.random_normal([hiddenSize], stddev=0.1)) 88 | 89 | # 对Bi-LSTM的输出用激活函数做非线性转换 90 | M = tf.tanh(H) 91 | 92 | # 对W和M做矩阵运算，W=[batch_size, time_step, hidden_size]，计算前做维度转换成[batch_size * time_step, hidden_size] 93 | # newM = [batch_size, time_step, 1]，每一个时间步的输出由向量转换成一个数字 94 | newM = tf.matmul(tf.reshape(M, [-1, hiddenSize]), tf.reshape(W, [-1, 1])) 95 | 96 | # 对newM做维度转换成[batch_size, time_step] 97 | restoreM = tf.reshape(newM, [-1, config.sequenceLength]) 98 | 99 | # 用softmax做归一化处理[batch_size, time_step] 100 | self.alpha = tf.nn.softmax(restoreM) 101 | 102 | # 利用求得的alpha的值对H进行加权求和，用矩阵运算直接操作 103 | r = tf.matmul(tf.transpose(H, [0, 2, 1]), tf.reshape(self.alpha, [-1, config.sequenceLength, 1])) 104 | 105 | # 将三维压缩成二维sequeezeR=[batch_size, hidden_size] 106 | sequeezeR = tf.squeeze(r) 107 | 108 | sentenceRepren = tf.tanh(sequeezeR) 109 | 110 | # 对Attention的输出可以做dropout处理 111 | output = tf.nn.dropout(sentenceRepren, self.dropoutKeepProb) 112 | 113 | return output -------------------------------------------------------------------------------- /ELMo/mode_trainning.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | # _*_ coding:utf-8 _*_ 3 | import os 4 | import datetime 5 | import numpy as np 6 | import tensorflow as tf 7 | import parameter_config 8 | import get_train_data 9 | import mode_structure 10 | from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score 11 | from data import TokenBatcher #不能从bilm直接导入TokenBatcher，因为需要修改内部的open为with open(filename, encoding="utf8") as f: 12 | from bilm import BidirectionalLanguageModel, weight_layers, dump_token_embeddings, Batcher 13 | 14 | #获取前些模块的数据 15 | config =parameter_config.Config() 16 | data = get_train_data.Dataset(config) 17 | data.dataGen() 18 | 19 | #4生成batch数据集 20 | def nextBatch(x, y, batchSize): 21 | # 生成batch数据集，用生成器的方式输出 22 | # perm = np.arange(len(x)) #返回[0 1 2 ... len(x)]的数组 23 | # np.random.shuffle(perm) #乱序 24 | # # x = x[perm] 25 | # # y = y[perm] 26 | # x = np.array(x)[perm] 27 | # y = np.array(y)[perm] 28 | # print(x) 29 | # # np.random.shuffle(x) #不能用这种，会导致x和y不一致 30 | # # np.random.shuffle(y) 31 | 32 | midVal = list(zip(x, y)) 33 | np.random.shuffle(midVal) 34 | x, y = zip(*midVal) 35 | x = list(x) 36 | y = list(y) 37 | print(x) 38 | numBatches = len(x) // batchSize 39 | 40 | for i in range(numBatches): 41 | start = i * batchSize 42 | end = start + batchSize 43 | batchX = np.array(x[start: end]) 44 | batchY = np.array(y[start: end]) 45 | yield batchX, batchY 46 | 47 | # 5 定义计算metrics的函数 48 | """ 49 | 定义各类性能指标 50 | """ 51 | def mean(item): 52 | return sum(item) / len(item) 53 | def genMetrics(trueY, predY, binaryPredY): 54 | """ 55 | 生成acc和auc值 56 | """ 57 | auc = roc_auc_score(trueY, predY) 58 | accuracy = accuracy_score(trueY, binaryPredY) 59 | precision = precision_score(trueY, binaryPredY) 60 | recall = recall_score(trueY, binaryPredY) 61 | 62 | return round(accuracy, 4), round(auc, 4), round(precision, 4), round(recall, 4) 63 | 64 | # 6 训练模型 65 | # 生成训练集和验证集 66 | trainReviews = data.trainReviews 67 | trainLabels = data.trainLabels 68 | evalReviews = data.evalReviews 69 | evalLabels = data.evalLabels 70 | 71 | # 定义计算图 72 | 73 | with tf.Graph().as_default(): 74 | 75 | session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) 76 | session_conf.gpu_options.allow_growth=True 77 | session_conf.gpu_options.per_process_gpu_memory_fraction = 0.9 # 配置gpu占用率 78 | 79 | sess = tf.Session(config=session_conf) 80 | 81 | # 定义会话 82 | with sess.as_default(): 83 | elmoMode = mode_structure.ELMo(config) 84 | 85 | # 实例化BiLM对象，这个必须放置在全局下，不能在elmo函数中定义，否则会出现重复生成tensorflow节点。 86 | with tf.variable_scope("bilm", reuse=True): 87 | bilm = BidirectionalLanguageModel( 88 | config.optionFile, 89 | config.weightFile, 90 | use_character_inputs=False, 91 | embedding_weight_file=config.tokenEmbeddingFile 92 | ) 93 | inputData = tf.placeholder('int32', shape=(None, None)) 94 | 95 | # 调用bilm中的__call__方法生成op对象 96 | inputEmbeddingsOp = bilm(inputData) 97 | 98 | # 计算ELMo向量表示 99 | elmoInput = weight_layers('input', inputEmbeddingsOp, l2_coef=0.0) 100 | 101 | globalStep = tf.Variable(0, name="globalStep", trainable=False) 102 | # 定义优化函数，传入学习速率参数 103 | optimizer = tf.train.AdamOptimizer(config.training.learningRate) 104 | # 计算梯度,得到梯度和变量 105 | gradsAndVars = optimizer.compute_gradients(elmoMode.loss) 106 | # 将梯度应用到变量下，生成训练器 107 | trainOp = optimizer.apply_gradients(gradsAndVars, global_step=globalStep) 108 | 109 | # 用summary绘制tensorBoard 110 | gradSummaries = [] 111 | for g, v in gradsAndVars: 112 | if g is not None: 113 | tf.summary.histogram("{}/grad/hist".format(v.name), g) 114 | tf.summary.scalar("{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) 115 | 116 | outDir = os.path.abspath(os.path.join(os.path.curdir, "summarys")) 117 | print("Writing to {}\n".format(outDir)) 118 | 119 | lossSummary = tf.summary.scalar("loss", elmoMode.loss) 120 | summaryOp = tf.summary.merge_all() 121 | 122 | trainSummaryDir = os.path.join(outDir, "train") 123 | trainSummaryWriter = tf.summary.FileWriter(trainSummaryDir, sess.graph) 124 | 125 | evalSummaryDir = os.path.join(outDir, "eval") 126 | evalSummaryWriter = tf.summary.FileWriter(evalSummaryDir, sess.graph) 127 | 128 | 129 | # 初始化所有变量 130 | saver = tf.train.Saver(tf.global_variables(), max_to_keep=5) 131 | 132 | savedModelPath ="../model/ELMo/savedModel" 133 | if os.path.exists(savedModelPath): 134 | os.rmdir(savedModelPath) 135 | 136 | # 保存模型的一种方式，保存为pb文件 137 | builder = tf.saved_model.builder.SavedModelBuilder(savedModelPath) 138 | 139 | sess.run(tf.global_variables_initializer()) 140 | 141 | def elmo(reviews): 142 | """ 143 | 对每一个输入的batch都动态的生成词向量表示 144 | """ 145 | # tf.reset_default_graph() 146 | # TokenBatcher是生成词表示的batch类 147 | # print("________") 148 | batcher = TokenBatcher(config.vocabFile) 149 | # 生成batch数据 150 | inputDataIndex = batcher.batch_sentences(reviews) 151 | # 计算ELMo的向量表示 152 | elmoInputVec = sess.run( 153 | [elmoInput['weighted_op']], 154 | feed_dict={inputData: inputDataIndex} 155 | ) 156 | return elmoInputVec 157 | 158 | def trainStep(batchX, batchY): 159 | """ 160 | 训练函数 161 | """ 162 | 163 | feed_dict = { 164 | elmoMode.inputX: elmo(batchX)[0], # inputX直接用动态生成的ELMo向量表示代入 165 | elmoMode.inputY: np.array(batchY, dtype="float32"), 166 | elmoMode.dropoutKeepProb: config.model.dropoutKeepProb 167 | } 168 | _, summary, step, loss, predictions, binaryPreds = sess.run( 169 | [trainOp, summaryOp, globalStep, elmoMode.loss, elmoMode.predictions, elmoMode.binaryPreds], 170 | feed_dict) 171 | timeStr = datetime.datetime.now().isoformat() 172 | acc, auc, precision, recall = genMetrics(batchY, predictions, binaryPreds) 173 | print("{}, step: {}, loss: {}, acc: {}, auc: {}, precision: {}, recall: {}".format(timeStr, step, loss, acc, auc, precision, recall)) 174 | trainSummaryWriter.add_summary(summary, step) 175 | 176 | def devStep(batchX, batchY): 177 | """ 178 | 验证函数 179 | """ 180 | feed_dict = { 181 | elmoMode.inputX: elmo(batchX)[0], 182 | elmoMode.inputY: np.array(batchY, dtype="float32"), 183 | elmoMode.dropoutKeepProb: 1.0 184 | } 185 | summary, step, loss, predictions, binaryPreds = sess.run( 186 | [summaryOp, globalStep, elmoMode.loss, elmoMode.predictions, elmoMode.binaryPreds], 187 | feed_dict) 188 | 189 | acc, auc, precision, recall = genMetrics(batchY, predictions, binaryPreds) 190 | 191 | evalSummaryWriter.add_summary(summary, step) 192 | 193 | return loss, acc, auc, precision, recall 194 | 195 | for i in range(config.training.epoches): 196 | # 训练模型 197 | print("start training model") 198 | for batchTrain in nextBatch(trainReviews, trainLabels, config.batchSize): 199 | trainStep(batchTrain[0], batchTrain[1]) 200 | 201 | currentStep = tf.train.global_step(sess, globalStep) 202 | if currentStep % config.training.evaluateEvery == 0: 203 | print("\nEvaluation:") 204 | 205 | losses = [] 206 | accs = [] 207 | aucs = [] 208 | precisions = [] 209 | recalls = [] 210 | 211 | for batchEval in nextBatch(evalReviews, evalLabels, config.batchSize): 212 | loss, acc, auc, precision, recall = devStep(batchEval[0], batchEval[1]) 213 | losses.append(loss) 214 | accs.append(acc) 215 | aucs.append(auc) 216 | precisions.append(precision) 217 | recalls.append(recall) 218 | 219 | time_str = datetime.datetime.now().isoformat() 220 | print("{}, step: {}, loss: {}, acc: {}, auc: {}, precision: {}, recall: {}".format(time_str, currentStep, mean(losses), 221 | mean(accs), mean(aucs), mean(precisions), 222 | mean(recalls))) 223 | 224 | if currentStep % config.training.checkpointEvery == 0: 225 | # 保存模型的另一种方法，保存checkpoint文件 226 | path = saver.save(sess, "../model/ELMo/model/my-model", global_step=currentStep) 227 | print("Saved model checkpoint to {}\n".format(path)) 228 | 229 | inputs = {"inputX": tf.saved_model.utils.build_tensor_info(elmoMode.inputX), 230 | "keepProb": tf.saved_model.utils.build_tensor_info(elmoMode.dropoutKeepProb)} 231 | 232 | outputs = {"binaryPreds": tf.saved_model.utils.build_tensor_info(elmoMode.binaryPreds)} 233 | 234 | prediction_signature = tf.saved_model.signature_def_utils.build_signature_def(inputs=inputs, outputs=outputs, 235 | method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME) 236 | legacy_init_op = tf.group(tf.tables_initializer(), name="legacy_init_op") 237 | builder.add_meta_graph_and_variables(sess, [tf.saved_model.tag_constants.SERVING], 238 | signature_def_map={"predict": prediction_signature}, legacy_init_op=legacy_init_op) 239 | 240 | builder.save() 241 | 242 | -------------------------------------------------------------------------------- /ELMo/parameter_config.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | # _*_ coding:utf-8 _*_ 3 | #需要的所有导入包，存放留用，转换到jupyter后直接使用 4 | # 1 配置训练参数 5 | 6 | class TrainingConfig(object): 7 | epoches = 5 8 | evaluateEvery = 100 9 | checkpointEvery = 100 10 | learningRate = 0.001 11 | 12 | class ModelConfig(object): 13 | embeddingSize = 256 # 这个值是和ELMo模型的output Size 对应的值 14 | 15 | hiddenSizes = [128] # LSTM结构的神经元个数 16 | 17 | dropoutKeepProb = 0.5 18 | l2RegLambda = 0.0 19 | 20 | class Config(object): 21 | sequenceLength = 200 # 取了所有序列长度的均值 22 | batchSize = 128 23 | 24 | dataSource = "../data/preProcess/labeledTrain.csv" 25 | 26 | stopWordSource = "../data/english" 27 | 28 | optionFile = "../data/elmodata/elmo_options.json" 29 | weightFile = "../data/elmodata/elmo_weights.hdf5" 30 | vocabFile = "../data/elmodata/vocab.txt" 31 | tokenEmbeddingFile = '../data/elmodata/elmo_token_embeddings.hdf5' 32 | 33 | numClasses = 2 34 | 35 | rate = 0.8 # 训练集的比例 36 | 37 | training = TrainingConfig() 38 | 39 | model = ModelConfig() 40 | 41 | 42 | # 实例化配置参数对象 43 | # config = Config() -------------------------------------------------------------------------------- /ELMo/predict.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | #测试没泡通 3 | import tensorflow as tf 4 | from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score 5 | import parameter_config 6 | config =parameter_config.Config() 7 | from data import TokenBatcher #不能从bilm直接导入TokenBatcher，因为需要修改内部的open为with open(filename, encoding="utf8") as f: 8 | from bilm import BidirectionalLanguageModel, weight_layers, dump_token_embeddings, Batcher 9 | 10 | #7预测代码 11 | reviews = "this movie is full of references like mad max ii the wild one and many others the ladybug´s face it´s a clear reference or tribute to peter lorre this movie is a masterpiece we´ll talk much more about in the future" 12 | # x = "his movie is the same as the third level movie. There's no place to look good" 13 | # x = "This film is not good" #最终反馈为0 14 | # x = "This film is bad" #最终反馈为0 15 | 16 | # 注：下面两个词典要保证和当前加载的模型对应的词典是一致的 17 | 18 | x1 = [review[:200] for review in reviews] 19 | 20 | graph = tf.Graph() 21 | with graph.as_default(): 22 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) 23 | session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, gpu_options=gpu_options) 24 | sess = tf.Session(config=session_conf) 25 | 26 | with sess.as_default(): 27 | # 恢复模型 28 | checkpoint_file = tf.train.latest_checkpoint("../model/ELMo/model/") 29 | saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file)) 30 | saver.restore(sess, checkpoint_file) 31 | 32 | with tf.variable_scope("bilm", reuse=True): 33 | bilm = BidirectionalLanguageModel( 34 | config.optionFile, 35 | config.weightFile, 36 | use_character_inputs=False, 37 | embedding_weight_file=config.tokenEmbeddingFile 38 | ) 39 | inputData = tf.placeholder('int32', shape=(None, None)) 40 | # 调用bilm中的__call__方法生成op对象 41 | inputEmbeddingsOp = bilm(inputData) 42 | # 计算ELMo向量表示 43 | elmoInput = weight_layers('input', inputEmbeddingsOp, l2_coef=0.0) 44 | def elmo(reviews): 45 | """ 46 | 对每一个输入的batch都动态的生成词向量表示 47 | """ 48 | batcher = TokenBatcher(config.vocabFile) 49 | # 生成batch数据 50 | inputDataIndex = batcher.batch_sentences(reviews) 51 | # 计算ELMo的向量表示x 52 | elmoInputVec = sess.run( 53 | [elmoInput['weighted_op']], 54 | feed_dict={inputData: inputDataIndex} 55 | ) 56 | return elmoInputVec 57 | 58 | 59 | # 获得需要喂给模型的参数，输出的结果依赖的输入值 60 | inputX = graph.get_operation_by_name("inputX").outputs[0] 61 | dropoutKeepProb = graph.get_operation_by_name("dropoutKeepProb").outputs[0] 62 | 63 | # 获得输出的结果 64 | binaryPreds = graph.get_tensor_by_name("output/binaryPreds:0") # mode_structure中的定义 65 | pred = sess.run(binaryPreds, feed_dict={inputX:elmo(x1)[0], dropoutKeepProb: 1.0})[0] 66 | 67 | # print(pred) 68 | # pred = [idx2label[item] for item in pred] 69 | print(pred) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NLP_textClassifier 2 | 基于word2vec预训练词向量； textCNN 模型 ；charCNN 模型 ；Bi-LSTM模型；Bi-LSTM + Attention 模型 ；Transformer 模型 ；ELMo 预训练模型 ；BERT 预训练模型的文本分类项目 3 | 4 | 5 | 一、大纲概述 6 | 文本分类这个系列将会有8篇左右文章，从github直接下载代码，从百度云下载训练数据，在pycharm上导入即可使用，包括基于word2vec预训练的文本分类，与及基于近几年的预训练模型（ELMo，BERT等）的文本分类。总共有以下系列： 7 | word2vec预训练词向量 8 | textCNN 模型 9 | charCNN 模型 10 | Bi-LSTM 模型 11 | Bi-LSTM + Attention 模型 12 | Transformer 模型 13 | ELMo 预训练模型 14 | BERT 预训练模型 15 | 16 | 二、数据集合 17 | 数据集为IMDB 电影影评，总共有三个数据文件，在/data/rawData目录下，包括unlabeledTrainData.tsv，labeledTrainData.tsv，testData.tsv。在进行文本分类时需要有标签的数据（labeledTrainData），但是在训练word2vec词向量模型（无监督学习）时可以将无标签的数据一起用上。 18 | 训练数据地址：链接：https://pan.baidu.com/s/1-XEwx1ai8kkGsMagIFKX_g 提取码：rtz8 19 | 20 | 相关的介绍：https://www.cnblogs.com/yifanrensheng/category/1758378.html 21 | -------------------------------------------------------------------------------- /Transformer/get_train_data.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import json 3 | from collections import Counter 4 | import gensim 5 | import pandas as pd 6 | import numpy as np 7 | import parameter_config 8 | 9 | # 2 数据预处理的类，生成训练集和测试集 10 | class Dataset(object): 11 | def __init__(self, config): 12 | self.config = config 13 | self._dataSource = config.dataSource 14 | self._stopWordSource = config.stopWordSource 15 | self._sequenceLength = config.sequenceLength # 每条输入的序列处理为定长 16 | self._embeddingSize = config.model.embeddingSize 17 | self._batchSize = config.batchSize 18 | self._rate = config.rate 19 | self._stopWordDict = {} 20 | self.trainReviews = [] 21 | self.trainLabels = [] 22 | self.evalReviews = [] 23 | self.evalLabels = [] 24 | self.wordEmbedding = None 25 | self.labelList = [] 26 | def _readData(self, filePath): 27 | """ 28 | 从csv文件中读取数据集,就本次测试的文件做记录 29 | """ 30 | df = pd.read_csv(filePath) #读取文件，是三列的数据，第一列是review,第二列sentiment，第三列rate 31 | if self.config.numClasses == 1: 32 | labels = df["sentiment"].tolist() #读取sentiment列的数据， 显示输出01序列数组25000条 33 | elif self.config.numClasses > 1: 34 | labels = df["rate"].tolist() #因为numClasses控制，本次取样没有取超过二分类 该处没有输出 35 | review = df["review"].tolist() 36 | reviews = [line.strip().split() for line in review] #按空格语句切分 37 | return reviews, labels 38 | def _labelToIndex(self, labels, label2idx): 39 | """ 40 | 将标签转换成索引表示 41 | """ 42 | labelIds = [label2idx[label] for label in labels] #print(labels==labelIds) 结果显示为true,也就是两个一样 43 | return labelIds 44 | def _wordToIndex(self, reviews, word2idx): 45 | """将词转换成索引""" 46 | reviewIds = [[word2idx.get(item, word2idx["UNK"]) for item in review] for review in reviews] 47 | # print(max(max(reviewIds))) 48 | # print(reviewIds) 49 | return reviewIds #返回25000个无序的数组 50 | def _genTrainEvalData(self, x, y, word2idx, rate): 51 | """生成训练集和验证集 """ 52 | reviews = [] 53 | # print(self._sequenceLength) 54 | # print(len(x)) 55 | for review in x: #self._sequenceLength为200，表示长的切成200，短的补齐，x数据依旧是25000 56 | if len(review) >= self._sequenceLength: 57 | reviews.append(review[:self._sequenceLength]) 58 | else: 59 | reviews.append(review + [word2idx["PAD"]] * (self._sequenceLength - len(review))) 60 | # print(len(review + [word2idx["PAD"]] * (self._sequenceLength - len(review)))) 61 | #以下是按照rate比例切分训练和测试数据： 62 | trainIndex = int(len(x) * rate) 63 | trainReviews = np.asarray(reviews[:trainIndex], dtype="int64") 64 | trainLabels = np.array(y[:trainIndex], dtype="float32") 65 | evalReviews = np.asarray(reviews[trainIndex:], dtype="int64") 66 | evalLabels = np.array(y[trainIndex:], dtype="float32") 67 | return trainReviews, trainLabels, evalReviews, evalLabels 68 | 69 | def _getWordEmbedding(self, words): 70 | """按照我们的数据集中的单词取出预训练好的word2vec中的词向量 71 | 反馈词和对应的向量（200维度），另外前面增加PAD对用0的数组，UNK对应随机数组。 72 | """ 73 | wordVec = gensim.models.KeyedVectors.load_word2vec_format("../word2vec/word2Vec.bin", binary=True) 74 | vocab = [] 75 | wordEmbedding = [] 76 | # 添加 "pad" 和 "UNK", 77 | vocab.append("PAD") 78 | vocab.append("UNK") 79 | wordEmbedding.append(np.zeros(self._embeddingSize)) # _embeddingSize 本文定义的是200 80 | wordEmbedding.append(np.random.randn(self._embeddingSize)) 81 | # print(wordEmbedding) 82 | for word in words: 83 | try: 84 | vector = wordVec.wv[word] 85 | vocab.append(word) 86 | wordEmbedding.append(vector) 87 | except: 88 | print(word + "不存在于词向量中") 89 | # print(vocab[:3],wordEmbedding[:3]) 90 | return vocab, np.array(wordEmbedding) 91 | def _genVocabulary(self, reviews, labels): 92 | """生成词向量和词汇-索引映射字典，可以用全数据集""" 93 | allWords = [word for review in reviews for word in review] #单词数量5738236 reviews是25000个观点句子【】 94 | subWords = [word for word in allWords if word not in self.stopWordDict] # 去掉停用词 95 | wordCount = Counter(subWords) # 统计词频 96 | sortWordCount = sorted(wordCount.items(), key=lambda x: x[1], reverse=True) #返回键值对，并按照数量排序 97 | # print(len(sortWordCount)) #161330 98 | # print(sortWordCount[:4],sortWordCount[-4:]) # [('movie', 41104), ('film', 36981), ('one', 24966), ('like', 19490)] [('daeseleires', 1), ('nice310', 1), ('shortsightedness', 1), ('unfairness', 1)] 99 | words = [item[0] for item in sortWordCount if item[1] >= 5] # 去除低频词，低于5的 100 | vocab, wordEmbedding = self._getWordEmbedding(words) 101 | self.wordEmbedding = wordEmbedding 102 | word2idx = dict(zip(vocab, list(range(len(vocab))))) #生成类似这种{'I': 0, 'love': 1, 'yanzi': 2} 103 | uniqueLabel = list(set(labels)) #标签去重 最后就 0 1了 104 | label2idx = dict(zip(uniqueLabel, list(range(len(uniqueLabel))))) #本文就 {0: 0, 1: 1} 105 | self.labelList = list(range(len(uniqueLabel))) 106 | # 将词汇-索引映射表保存为json数据，之后做inference时直接加载来处理数据 107 | with open("../data/wordJson/word2idx.json", "w", encoding="utf-8") as f: 108 | json.dump(word2idx, f) 109 | with open("../data/wordJson/label2idx.json", "w", encoding="utf-8") as f: 110 | json.dump(label2idx, f) 111 | return word2idx, label2idx 112 | 113 | def _readStopWord(self, stopWordPath): 114 | """ 115 | 读取停用词 116 | """ 117 | with open(stopWordPath, "r") as f: 118 | stopWords = f.read() 119 | stopWordList = stopWords.splitlines() 120 | # 将停用词用列表的形式生成，之后查找停用词时会比较快 121 | self.stopWordDict = dict(zip(stopWordList, list(range(len(stopWordList))))) 122 | 123 | def dataGen(self): 124 | """ 125 | 初始化训练集和验证集 126 | """ 127 | # 初始化停用词 128 | self._readStopWord(self._stopWordSource) 129 | # 初始化数据集 130 | reviews, labels = self._readData(self._dataSource) 131 | # 初始化词汇-索引映射表和词向量矩阵 132 | word2idx, label2idx = self._genVocabulary(reviews, labels) 133 | # 将标签和句子数值化 134 | labelIds = self._labelToIndex(labels, label2idx) 135 | reviewIds = self._wordToIndex(reviews, word2idx) 136 | # 初始化训练集和测试集 137 | trainReviews, trainLabels, evalReviews, evalLabels = self._genTrainEvalData(reviewIds, labelIds, word2idx, 138 | self._rate) 139 | self.trainReviews = trainReviews 140 | self.trainLabels = trainLabels 141 | 142 | self.evalReviews = evalReviews 143 | self.evalLabels = evalLabels 144 | 145 | #获取前些模块的数据 146 | # config =parameter_config.Config() 147 | # data = Dataset(config) 148 | # data.dataGen() -------------------------------------------------------------------------------- /Transformer/mode_structure.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import numpy as np 3 | import tensorflow as tf 4 | import parameter_config 5 | 6 | # 构建模型 3 Transformer模型 7 | # 生成位置嵌入 8 | def fixedPositionEmbedding(batchSize, sequenceLen): 9 | embeddedPosition = [] 10 | for batch in range(batchSize): 11 | x = [] 12 | for step in range(sequenceLen): #类似one-hot方式的构造 13 | a = np.zeros(sequenceLen) 14 | a[step] = 1 15 | x.append(a) 16 | embeddedPosition.append(x) 17 | return np.array(embeddedPosition, dtype="float32") 18 | 19 | # 模型构建 20 | class Transformer(object): 21 | """ 22 | Transformer Encoder 用于文本分类 23 | """ 24 | def __init__(self, config, wordEmbedding): 25 | # 定义模型的输入 26 | self.inputX = tf.placeholder(tf.int32, [None, config.sequenceLength], name="inputX") 27 | self.inputY = tf.placeholder(tf.int32, [None], name="inputY") 28 | self.dropoutKeepProb = tf.placeholder(tf.float32, name="dropoutKeepProb") 29 | self.embeddedPosition = tf.placeholder(tf.float32, [None, config.sequenceLength, config.sequenceLength], name="embeddedPosition") 30 | self.config = config 31 | # 定义l2损失 32 | l2Loss = tf.constant(0.0) 33 | 34 | # 词嵌入层, 位置向量的定义方式有两种：一是直接用固定的one-hot的形式传入，然后和词向量拼接， 35 | # 在当前的数据集上表现效果更好。另一种 36 | # 就是按照论文中的方法实现，这样的效果反而更差，可能是增大了模型的复杂度，在小数据集上表现不佳。 37 | with tf.name_scope("embedding"): 38 | # 利用预训练的词向量初始化词嵌入矩阵 39 | self.W = tf.Variable(tf.cast(wordEmbedding, dtype=tf.float32, name="word2vec") ,name="W") 40 | # 利用词嵌入矩阵将输入的数据中的词转换成词向量，维度[batch_size, sequence_length, embedding_size] 41 | self.embedded = tf.nn.embedding_lookup(self.W, self.inputX) 42 | self.embeddedWords = tf.concat([self.embedded, self.embeddedPosition], -1) 43 | 44 | with tf.name_scope("transformer"): 45 | for i in range(config.model.numBlocks): #numBlocks==1，文本只设置了一个 46 | with tf.name_scope("transformer-{}".format(i + 1)): 47 | # 维度[batch_size, sequence_length, embedding_size] 48 | multiHeadAtt = self._multiheadAttention(rawKeys=self.inputX, queries=self.embeddedWords, 49 | keys=self.embeddedWords) 50 | # 维度[batch_size, sequence_length, embedding_size] 51 | self.embeddedWords = self._feedForward(multiHeadAtt, 52 | [config.model.filters, config.model.embeddingSize + config.sequenceLength]) 53 | 54 | outputs = tf.reshape(self.embeddedWords, [-1, config.sequenceLength * (config.model.embeddingSize + config.sequenceLength)]) 55 | outputSize = outputs.get_shape()[-1].value 56 | 57 | with tf.name_scope("dropout"): 58 | outputs = tf.nn.dropout(outputs, keep_prob=self.dropoutKeepProb) 59 | 60 | # 全连接层的输出 61 | with tf.name_scope("output"): 62 | outputW = tf.get_variable( 63 | "outputW", 64 | shape=[outputSize, config.numClasses], 65 | initializer=tf.contrib.layers.xavier_initializer()) 66 | 67 | outputB= tf.Variable(tf.constant(0.1, shape=[config.numClasses]), name="outputB") 68 | l2Loss += tf.nn.l2_loss(outputW) 69 | l2Loss += tf.nn.l2_loss(outputB) 70 | self.logits = tf.nn.xw_plus_b(outputs, outputW, outputB, name="logits") 71 | 72 | if config.numClasses == 1: 73 | self.predictions = tf.cast(tf.greater_equal(self.logits, 0.0), tf.float32, name="predictions") 74 | elif config.numClasses > 1: 75 | self.predictions = tf.argmax(self.logits, axis=-1, name="predictions") 76 | 77 | # 计算二元交叉熵损失 78 | with tf.name_scope("loss"): 79 | 80 | if config.numClasses == 1: 81 | losses = tf.nn.sigmoid_cross_entropy_with_logits(logits=self.logits, labels=tf.cast(tf.reshape(self.inputY, [-1, 1]), 82 | dtype=tf.float32)) 83 | elif config.numClasses > 1: 84 | losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.inputY) 85 | 86 | self.loss = tf.reduce_mean(losses) + config.model.l2RegLambda * l2Loss 87 | 88 | def _layerNormalization(self, inputs, scope="layerNorm"): 89 | # LayerNorm层和BN层有所不同 90 | epsilon = self.config.model.epsilon 91 | inputsShape = inputs.get_shape() # [batch_size, sequence_length, embedding_size] 92 | paramsShape = inputsShape[-1:] 93 | # LayerNorm是在最后的维度上计算输入的数据的均值和方差，BN层是考虑所有维度的 94 | # mean, variance的维度都是[batch_size, sequence_len, 1] 95 | mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True) 96 | beta = tf.Variable(tf.zeros(paramsShape)) 97 | gamma = tf.Variable(tf.ones(paramsShape)) 98 | normalized = (inputs - mean) / ((variance + epsilon) ** .5) 99 | outputs = gamma * normalized + beta 100 | 101 | return outputs 102 | 103 | def _multiheadAttention(self, rawKeys, queries, keys, numUnits=None, causality=False, scope="multiheadAttention"): 104 | # rawKeys 的作用是为了计算mask时用的，因为keys是加上了position embedding的，其中不存在padding为0的值 105 | numHeads = self.config.model.numHeads #头数，目前设置为8 106 | keepProp = self.config.model.keepProp #dropout数量 107 | 108 | if numUnits is None: #若是没传入值，直接去输入数据的最后一维，即embedding size.embeddingSize = 200 109 | numUnits = queries.get_shape().as_list()[-1] 110 | 111 | # tf.layers.dense可以做多维tensor数据的非线性映射，在计算self-Attention时，一定要对这三个值进行非线性映射， 112 | # 其实这一步就是论文中Multi-Head Attention中的对分割后的数据进行权重映射的步骤，我们在这里先映射后分割，原则上是一样的。 113 | # Q, K, V的维度都是[batch_size, sequence_length, embedding_size] 114 | Q = tf.layers.dense(queries, numUnits, activation=tf.nn.relu) 115 | K = tf.layers.dense(keys, numUnits, activation=tf.nn.relu) 116 | V = tf.layers.dense(keys, numUnits, activation=tf.nn.relu) 117 | 118 | # 将数据按最后一维分割成num_heads个, 然后按照第一维拼接 119 | # Q, K, V 的维度都是[batch_size * numHeads, sequence_length, embedding_size/numHeads] 120 | Q_ = tf.concat(tf.split(Q, numHeads, axis=-1), axis=0) 121 | K_ = tf.concat(tf.split(K, numHeads, axis=-1), axis=0) 122 | V_ = tf.concat(tf.split(V, numHeads, axis=-1), axis=0) 123 | 124 | # 计算keys和queries之间的点积，维度[batch_size * numHeads, queries_len, key_len], 后两维是queries和keys的序列长度 125 | similary = tf.matmul(Q_, tf.transpose(K_, [0, 2, 1])) 126 | 127 | # 对计算的点积进行缩放处理，除以向量长度的根号值 128 | scaledSimilary = similary / (K_.get_shape().as_list()[-1] ** 0.5) 129 | 130 | # 在我们输入的序列中会存在padding这个样的填充词，这种词应该对最终的结果是毫无帮助的，原则上说当padding都是输入0时， 131 | # 计算出来的权重应该也是0，但是在transformer中引入了位置向量，当和位置向量相加之后，其值就不为0了，因此在添加位置向量 132 | # 之前，我们需要将其mask为0。虽然在queries中也存在这样的填充词，但原则上模型的结果之和输入有关，而且在self-Attention中 133 | # queryies = keys，因此只要一方为0，计算出的权重就为0。 134 | # 具体关于key mask的介绍可以看看这里： https://github.com/Kyubyong/transformer/issues/3 135 | 136 | # 利用tf，tile进行张量扩张， 维度[batch_size * numHeads, keys_len] keys_len = keys 的序列长度 137 | keyMasks = tf.tile(rawKeys, [numHeads, 1]) 138 | 139 | # 增加一个维度，并进行扩张，得到维度[batch_size * numHeads, queries_len, keys_len] 140 | keyMasks = tf.tile(tf.expand_dims(keyMasks, 1), [1, tf.shape(queries)[1], 1]) 141 | 142 | # tf.ones_like生成元素全为1，维度和scaledSimilary相同的tensor, 然后得到负无穷大的值 143 | paddings = tf.ones_like(scaledSimilary) * (-2 ** (32 + 1)) 144 | 145 | # tf.where(condition, x, y),condition中的元素为bool值，其中对应的True用x中的元素替换，对应的False用y中的元素替换 146 | # 因此condition,x,y的维度是一样的。下面就是keyMasks中的值为0就用paddings中的值替换 147 | maskedSimilary = tf.where(tf.equal(keyMasks, 0), paddings, scaledSimilary) # 维度[batch_size * numHeads, queries_len, key_len] 148 | 149 | # 在计算当前的词时，只考虑上文，不考虑下文，出现在Transformer Decoder中。在文本分类时，可以只用Transformer Encoder。 150 | # Decoder是生成模型，主要用在语言生成中 151 | if causality: 152 | diagVals = tf.ones_like(maskedSimilary[0, :, :]) # [queries_len, keys_len] 153 | tril = tf.contrib.linalg.LinearOperatorTriL(diagVals).to_dense() # [queries_len, keys_len] 154 | masks = tf.tile(tf.expand_dims(tril, 0), [tf.shape(maskedSimilary)[0], 1, 1]) # [batch_size * numHeads, queries_len, keys_len] 155 | 156 | paddings = tf.ones_like(masks) * (-2 ** (32 + 1)) 157 | maskedSimilary = tf.where(tf.equal(masks, 0), paddings, maskedSimilary) # [batch_size * numHeads, queries_len, keys_len] 158 | 159 | # 通过softmax计算权重系数，维度 [batch_size * numHeads, queries_len, keys_len] 160 | weights = tf.nn.softmax(maskedSimilary) 161 | 162 | # 加权和得到输出值, 维度[batch_size * numHeads, sequence_length, embedding_size/numHeads] 163 | outputs = tf.matmul(weights, V_) 164 | 165 | # 将多头Attention计算的得到的输出重组成最初的维度[batch_size, sequence_length, embedding_size] 166 | outputs = tf.concat(tf.split(outputs, numHeads, axis=0), axis=2) 167 | 168 | outputs = tf.nn.dropout(outputs, keep_prob=keepProp) 169 | 170 | # 对每个subLayers建立残差连接，即H(x) = F(x) + x 171 | outputs += queries 172 | # normalization 层 173 | outputs = self._layerNormalization(outputs) 174 | return outputs 175 | 176 | def _feedForward(self, inputs, filters, scope="multiheadAttention"): 177 | # 在这里的前向传播采用卷积神经网络 178 | 179 | # 内层 180 | params = {"inputs": inputs, "filters": filters[0], "kernel_size": 1, 181 | "activation": tf.nn.relu, "use_bias": True} 182 | outputs = tf.layers.conv1d(**params) 183 | 184 | # 外层 185 | params = {"inputs": outputs, "filters": filters[1], "kernel_size": 1, 186 | "activation": None, "use_bias": True} 187 | 188 | # 这里用到了一维卷积，实际上卷积核尺寸还是二维的，只是只需要指定高度，宽度和embedding size的尺寸一致 189 | # 维度[batch_size, sequence_length, embedding_size] 190 | outputs = tf.layers.conv1d(**params) 191 | 192 | # 残差连接 193 | outputs += inputs 194 | 195 | # 归一化处理 196 | outputs = self._layerNormalization(outputs) 197 | 198 | return outputs 199 | 200 | def _positionEmbedding(self, scope="positionEmbedding"): 201 | # 生成可训练的位置向量 202 | batchSize = self.config.batchSize 203 | sequenceLen = self.config.sequenceLength 204 | embeddingSize = self.config.model.embeddingSize 205 | 206 | # 生成位置的索引，并扩张到batch中所有的样本上 207 | positionIndex = tf.tile(tf.expand_dims(tf.range(sequenceLen), 0), [batchSize, 1]) 208 | 209 | # 根据正弦和余弦函数来获得每个位置上的embedding的第一部分 210 | positionEmbedding = np.array([[pos / np.power(10000, (i-i%2) / embeddingSize) for i in range(embeddingSize)] 211 | for pos in range(sequenceLen)]) 212 | 213 | # 然后根据奇偶性分别用sin和cos函数来包装 214 | positionEmbedding[:, 0::2] = np.sin(positionEmbedding[:, 0::2]) 215 | positionEmbedding[:, 1::2] = np.cos(positionEmbedding[:, 1::2]) 216 | 217 | # 将positionEmbedding转换成tensor的格式 218 | positionEmbedding_ = tf.cast(positionEmbedding, dtype=tf.float32) 219 | 220 | # 得到三维的矩阵[batchSize, sequenceLen, embeddingSize] 221 | positionEmbedded = tf.nn.embedding_lookup(positionEmbedding_, positionIndex) 222 | 223 | return positionEmbedded -------------------------------------------------------------------------------- /Transformer/mode_trainning.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | 3 | import os 4 | import datetime 5 | import numpy as np 6 | import tensorflow as tf 7 | import parameter_config 8 | import get_train_data 9 | import mode_structure 10 | 11 | #获取前些模块的数据 12 | config =parameter_config.Config() 13 | data = get_train_data.Dataset(config) 14 | data.dataGen() 15 | 16 | #4生成batch数据集 17 | def nextBatch(x, y, batchSize): 18 | # 生成batch数据集，用生成器的方式输出 19 | perm = np.arange(len(x)) #返回[0 1 2 ... len(x)]的数组 20 | np.random.shuffle(perm) #乱序 21 | x = x[perm] 22 | y = y[perm] 23 | numBatches = len(x) // batchSize 24 | 25 | for i in range(numBatches): 26 | start = i * batchSize 27 | end = start + batchSize 28 | batchX = np.array(x[start: end], dtype="int64") 29 | batchY = np.array(y[start: end], dtype="float32") 30 | yield batchX, batchY 31 | 32 | # 5 定义计算metrics的函数 33 | """ 34 | 定义各类性能指标 35 | """ 36 | def mean(item: list) -> float: 37 | """ 38 | 计算列表中元素的平均值 39 | :param item: 列表对象 40 | :return: 41 | """ 42 | res = sum(item) / len(item) if len(item) > 0 else 0 43 | return res 44 | 45 | def accuracy(pred_y, true_y): 46 | """ 47 | 计算二类和多类的准确率 48 | :param pred_y: 预测结果 49 | :param true_y: 真实结果 50 | :return: 51 | """ 52 | if isinstance(pred_y[0], list): 53 | pred_y = [item[0] for item in pred_y] 54 | corr = 0 55 | for i in range(len(pred_y)): 56 | if pred_y[i] == true_y[i]: 57 | corr += 1 58 | acc = corr / len(pred_y) if len(pred_y) > 0 else 0 59 | return acc 60 | 61 | def binary_precision(pred_y, true_y, positive=1): 62 | """ 63 | 二类的精确率计算 64 | :param pred_y: 预测结果 65 | :param true_y: 真实结果 66 | :param positive: 正例的索引表示 67 | :return: 68 | """ 69 | corr = 0 70 | pred_corr = 0 71 | for i in range(len(pred_y)): 72 | if pred_y[i] == positive: 73 | pred_corr += 1 74 | if pred_y[i] == true_y[i]: 75 | corr += 1 76 | 77 | prec = corr / pred_corr if pred_corr > 0 else 0 78 | return prec 79 | 80 | def binary_recall(pred_y, true_y, positive=1): 81 | """ 82 | 二类的召回率 83 | :param pred_y: 预测结果 84 | :param true_y: 真实结果 85 | :param positive: 正例的索引表示 86 | :return: 87 | """ 88 | corr = 0 89 | true_corr = 0 90 | for i in range(len(pred_y)): 91 | if true_y[i] == positive: 92 | true_corr += 1 93 | if pred_y[i] == true_y[i]: 94 | corr += 1 95 | 96 | rec = corr / true_corr if true_corr > 0 else 0 97 | return rec 98 | 99 | def binary_f_beta(pred_y, true_y, beta=1.0, positive=1): 100 | """ 101 | 二类的f beta值 102 | :param pred_y: 预测结果 103 | :param true_y: 真实结果 104 | :param beta: beta值 105 | :param positive: 正例的索引表示 106 | :return: 107 | """ 108 | precision = binary_precision(pred_y, true_y, positive) 109 | recall = binary_recall(pred_y, true_y, positive) 110 | try: 111 | f_b = (1 + beta * beta) * precision * recall / (beta * beta * precision + recall) 112 | except: 113 | f_b = 0 114 | return f_b 115 | 116 | def multi_precision(pred_y, true_y, labels): 117 | """ 118 | 多类的精确率 119 | :param pred_y: 预测结果 120 | :param true_y: 真实结果 121 | :param labels: 标签列表 122 | :return: 123 | """ 124 | if isinstance(pred_y[0], list): 125 | pred_y = [item[0] for item in pred_y] 126 | 127 | precisions = [binary_precision(pred_y, true_y, label) for label in labels] 128 | prec = mean(precisions) 129 | return prec 130 | 131 | def multi_recall(pred_y, true_y, labels): 132 | """ 133 | 多类的召回率 134 | :param pred_y: 预测结果 135 | :param true_y: 真实结果 136 | :param labels: 标签列表 137 | :return: 138 | """ 139 | if isinstance(pred_y[0], list): 140 | pred_y = [item[0] for item in pred_y] 141 | 142 | recalls = [binary_recall(pred_y, true_y, label) for label in labels] 143 | rec = mean(recalls) 144 | return rec 145 | 146 | def multi_f_beta(pred_y, true_y, labels, beta=1.0): 147 | """ 148 | 多类的f beta值 149 | :param pred_y: 预测结果 150 | :param true_y: 真实结果 151 | :param labels: 标签列表 152 | :param beta: beta值 153 | :return: 154 | """ 155 | if isinstance(pred_y[0], list): 156 | pred_y = [item[0] for item in pred_y] 157 | 158 | f_betas = [binary_f_beta(pred_y, true_y, beta, label) for label in labels] 159 | f_beta = mean(f_betas) 160 | return f_beta 161 | 162 | def get_binary_metrics(pred_y, true_y, f_beta=1.0): 163 | """ 164 | 得到二分类的性能指标 165 | :param pred_y: 166 | :param true_y: 167 | :param f_beta: 168 | :return: 169 | """ 170 | acc = accuracy(pred_y, true_y) 171 | recall = binary_recall(pred_y, true_y) 172 | precision = binary_precision(pred_y, true_y) 173 | f_beta = binary_f_beta(pred_y, true_y, f_beta) 174 | return acc, recall, precision, f_beta 175 | 176 | def get_multi_metrics(pred_y, true_y, labels, f_beta=1.0): 177 | """ 178 | 得到多分类的性能指标 179 | :param pred_y: 180 | :param true_y: 181 | :param labels: 182 | :param f_beta: 183 | :return: 184 | """ 185 | acc = accuracy(pred_y, true_y) 186 | recall = multi_recall(pred_y, true_y, labels) 187 | precision = multi_precision(pred_y, true_y, labels) 188 | f_beta = multi_f_beta(pred_y, true_y, labels, f_beta) 189 | return acc, recall, precision, f_beta 190 | 191 | # 6 训练模型 192 | # 生成训练集和验证集 193 | trainReviews = data.trainReviews 194 | trainLabels = data.trainLabels 195 | evalReviews = data.evalReviews 196 | evalLabels = data.evalLabels 197 | 198 | wordEmbedding = data.wordEmbedding 199 | labelList = data.labelList 200 | embeddedPosition = mode_structure.fixedPositionEmbedding(config.batchSize, config.sequenceLength) #使用的是one-hot形式 201 | 202 | # 训练模型 203 | # 定义计算图 204 | with tf.Graph().as_default(): 205 | session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) 206 | session_conf.gpu_options.allow_growth=True 207 | session_conf.gpu_options.per_process_gpu_memory_fraction = 0.9 # 配置gpu占用率 208 | sess = tf.Session(config=session_conf) 209 | 210 | # 定义会话 211 | with sess.as_default(): 212 | transformer = mode_structure.Transformer(config, wordEmbedding) 213 | globalStep = tf.Variable(0, name="globalStep", trainable=False) 214 | # 定义优化函数，传入学习速率参数 215 | optimizer = tf.train.AdamOptimizer(config.training.learningRate) 216 | # 计算梯度,得到梯度和变量 217 | gradsAndVars = optimizer.compute_gradients(transformer.loss) 218 | # 将梯度应用到变量下，生成训练器 219 | trainOp = optimizer.apply_gradients(gradsAndVars, global_step=globalStep) 220 | 221 | # 用summary绘制tensorBoard 222 | gradSummaries = [] 223 | for g, v in gradsAndVars: 224 | if g is not None: 225 | tf.summary.histogram("{}/grad/hist".format(v.name), g) 226 | tf.summary.scalar("{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) 227 | 228 | outDir = os.path.abspath(os.path.join(os.path.curdir, "summarys")) 229 | print("Writing to {}\n".format(outDir)) 230 | 231 | lossSummary = tf.summary.scalar("loss", transformer.loss) 232 | summaryOp = tf.summary.merge_all() 233 | 234 | trainSummaryDir = os.path.join(outDir, "train") 235 | trainSummaryWriter = tf.summary.FileWriter(trainSummaryDir, sess.graph) 236 | evalSummaryDir = os.path.join(outDir, "eval") 237 | evalSummaryWriter = tf.summary.FileWriter(evalSummaryDir, sess.graph) 238 | 239 | 240 | # 初始化所有变量 241 | saver = tf.train.Saver(tf.global_variables(), max_to_keep=5) 242 | 243 | # 保存模型的一种方式，保存为pb文件 244 | savedModelPath = "../model/transformer/savedModel" 245 | if os.path.exists(savedModelPath): 246 | os.rmdir(savedModelPath) 247 | builder = tf.saved_model.builder.SavedModelBuilder(savedModelPath) 248 | 249 | sess.run(tf.global_variables_initializer()) 250 | 251 | def trainStep(batchX, batchY): 252 | """ 253 | 训练函数 254 | """ 255 | feed_dict = { 256 | transformer.inputX: batchX, 257 | transformer.inputY: batchY, 258 | transformer.dropoutKeepProb: config.model.dropoutKeepProb, 259 | transformer.embeddedPosition: embeddedPosition 260 | } 261 | _, summary, step, loss, predictions = sess.run( 262 | [trainOp, summaryOp, globalStep, transformer.loss, transformer.predictions], 263 | feed_dict) 264 | 265 | if config.numClasses == 1: 266 | acc, recall, prec, f_beta = get_binary_metrics(pred_y=predictions, true_y=batchY) 267 | elif config.numClasses > 1: 268 | acc, recall, prec, f_beta = get_multi_metrics(pred_y=predictions, true_y=batchY, 269 | labels=labelList) 270 | 271 | trainSummaryWriter.add_summary(summary, step) 272 | return loss, acc, prec, recall, f_beta 273 | 274 | def devStep(batchX, batchY): 275 | """ 276 | 验证函数 277 | """ 278 | feed_dict = { 279 | transformer.inputX: batchX, 280 | transformer.inputY: batchY, 281 | transformer.dropoutKeepProb: 1.0, 282 | transformer.embeddedPosition: embeddedPosition 283 | } 284 | summary, step, loss, predictions = sess.run( 285 | [summaryOp, globalStep, transformer.loss, transformer.predictions], 286 | feed_dict) 287 | 288 | if config.numClasses == 1: 289 | acc, recall, prec, f_beta = get_binary_metrics(pred_y=predictions, true_y=batchY) 290 | 291 | 292 | elif config.numClasses > 1: 293 | acc, recall, prec, f_beta = get_multi_metrics(pred_y=predictions, true_y=batchY, 294 | labels=labelList) 295 | 296 | trainSummaryWriter.add_summary(summary, step) 297 | 298 | return loss, acc, prec, recall, f_beta 299 | 300 | for i in range(config.training.epoches): 301 | # 训练模型 302 | print("start training model") 303 | for batchTrain in nextBatch(trainReviews, trainLabels, config.batchSize): 304 | loss, acc, prec, recall, f_beta = trainStep(batchTrain[0], batchTrain[1]) 305 | 306 | currentStep = tf.train.global_step(sess, globalStep) 307 | print("train: step: {}, loss: {}, acc: {}, recall: {}, precision: {}, f_beta: {}".format( 308 | currentStep, loss, acc, recall, prec, f_beta)) 309 | if currentStep % config.training.evaluateEvery == 0: 310 | print("\nEvaluation:") 311 | 312 | losses = [] 313 | accs = [] 314 | f_betas = [] 315 | precisions = [] 316 | recalls = [] 317 | 318 | for batchEval in nextBatch(evalReviews, evalLabels, config.batchSize): 319 | loss, acc, precision, recall, f_beta = devStep(batchEval[0], batchEval[1]) 320 | losses.append(loss) 321 | accs.append(acc) 322 | f_betas.append(f_beta) 323 | precisions.append(precision) 324 | recalls.append(recall) 325 | 326 | time_str = datetime.datetime.now().isoformat() 327 | print("{}, step: {}, loss: {}, acc: {},precision: {}, recall: {}, f_beta: {}".format(time_str, currentStep, mean(losses), 328 | mean(accs), mean(precisions), 329 | mean(recalls), mean(f_betas))) 330 | 331 | if currentStep % config.training.checkpointEvery == 0: 332 | # 保存模型的另一种方法，保存checkpoint文件 333 | path = saver.save(sess, "../model/Transformer/model/my-model", global_step=currentStep) 334 | print("Saved model checkpoint to {}\n".format(path)) 335 | 336 | inputs = {"inputX": tf.saved_model.utils.build_tensor_info(transformer.inputX), 337 | "keepProb": tf.saved_model.utils.build_tensor_info(transformer.dropoutKeepProb)} 338 | 339 | outputs = {"predictions": tf.saved_model.utils.build_tensor_info(transformer.predictions)} 340 | 341 | prediction_signature = tf.saved_model.signature_def_utils.build_signature_def(inputs=inputs, outputs=outputs, 342 | method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME) 343 | legacy_init_op = tf.group(tf.tables_initializer(), name="legacy_init_op") 344 | builder.add_meta_graph_and_variables(sess, [tf.saved_model.tag_constants.SERVING], 345 | signature_def_map={"predict": prediction_signature}, legacy_init_op=legacy_init_op) 346 | 347 | builder.save() 348 | 349 | -------------------------------------------------------------------------------- /Transformer/parameter_config.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | #需要的所有导入包，存放留用，转换到jupyter后直接使用 3 | # 1 配置训练参数 4 | class TrainingConfig(object): 5 | epoches = 4 6 | evaluateEvery = 100 7 | checkpointEvery = 100 8 | learningRate = 0.001 9 | 10 | class ModelConfig(object): 11 | embeddingSize = 200 12 | filters = 128 # 内层一维卷积核的数量，外层卷积核的数量应该等于embeddingSize，因为要确保每个layer后的输出维度和输入维度是一致的。 13 | numHeads = 8 # Attention 的头数 14 | numBlocks = 1 # 设置transformer block的数量 15 | epsilon = 1e-8 # LayerNorm 层中的最小除数 16 | keepProp = 0.9 # multi head attention 中的dropout 17 | dropoutKeepProb = 0.5 # 全连接层的dropout 18 | l2RegLambda = 0.0 19 | 20 | class Config(object): 21 | sequenceLength = 200 # 取了所有序列长度的均值 22 | batchSize = 128 23 | dataSource = "../data/preProcess/labeledTrain.csv" 24 | stopWordSource = "../data/english" 25 | numClasses = 1 # 二分类设置为1，多分类设置为类别的数目 26 | rate = 0.8 # 训练集的比例 27 | training = TrainingConfig() 28 | model = ModelConfig() 29 | 30 | # 实例化配置参数对象 31 | config = Config() -------------------------------------------------------------------------------- /Transformer/predict.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | 3 | import json 4 | import tensorflow as tf 5 | from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score 6 | import parameter_config 7 | config =parameter_config.Config() 8 | import mode_structure 9 | embeddedPositions = mode_structure.fixedPositionEmbedding(config.batchSize, config.sequenceLength)[0] #使用的是one-hot形式 10 | # print(type(embeddedPositions)) 11 | # print(embeddedPositions.shape) 12 | #7预测代码 13 | # x = "this movie is full of references like mad max ii the wild one and many others the ladybug´s face it´s a clear reference or tribute to peter lorre this movie is a masterpiece we´ll talk much more about in the future" 14 | x = "his movie is the same as the third level movie. There's no place to look good" 15 | # x = "This film is not good" #最终反馈为1 感觉不准 16 | # x = "This film is bad" #最终反馈为0 17 | 18 | # 注：下面两个词典要保证和当前加载的模型对应的词典是一致的 19 | with open("../data/wordJson/word2idx.json", "r", encoding="utf-8") as f: 20 | word2idx = json.load(f) 21 | with open("../data/wordJson/label2idx.json", "r", encoding="utf-8") as f: #label2idx.json内容{"0": 0, "1": 1} 22 | label2idx = json.load(f) 23 | idx2label = {value: key for key, value in label2idx.items()} 24 | 25 | #x 的处理，变成模型能识别的向量xIds 26 | xIds = [word2idx.get(item, word2idx["UNK"]) for item in x.split(" ")] #返回x对应的向量 27 | if len(xIds) >= config.sequenceLength: #xIds 句子单词个数是否超过了sequenceLength（200） 28 | xIds = xIds[:config.sequenceLength] 29 | print("ddd",xIds) 30 | else: 31 | xIds = xIds + [word2idx["PAD"]] * (config.sequenceLength - len(xIds)) 32 | print("xxx", xIds) 33 | 34 | graph = tf.Graph() 35 | with graph.as_default(): 36 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) 37 | session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, gpu_options=gpu_options) 38 | sess = tf.Session(config=session_conf) 39 | 40 | with sess.as_default(): 41 | # 恢复模型 42 | checkpoint_file = tf.train.latest_checkpoint("../model/transformer/model/") 43 | saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file)) 44 | saver.restore(sess, checkpoint_file) 45 | 46 | # 获得需要喂给模型的参数，输出的结果依赖的输入值 47 | inputX = graph.get_operation_by_name("inputX").outputs[0] 48 | dropoutKeepProb = graph.get_operation_by_name("dropoutKeepProb").outputs[0] 49 | embeddedPosition = graph.get_operation_by_name("embeddedPosition").outputs[0] 50 | # inputX = tf.placeholder(tf.int32, [None, config.sequenceLength], name="inputX") 51 | # dropoutKeepProb = tf.placeholder(tf.float32, name="dropoutKeepProb") 52 | # embeddedPosition = tf.placeholder(tf.float32, [None, config.sequenceLength, config.sequenceLength], 53 | # name="embeddedPosition") #这种方式不行 54 | 55 | # 获得输出的结果 56 | predictions = graph.get_tensor_by_name("output/predictions:0") 57 | pred = sess.run(predictions, feed_dict={inputX: [xIds], dropoutKeepProb: 1.0, embeddedPosition: [embeddedPositions]})[0] 58 | 59 | # print(pred) 60 | pred = [idx2label[item] for item in pred] 61 | print(pred) -------------------------------------------------------------------------------- /charCNN/data_test.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import os 3 | import csv 4 | import time 5 | import datetime 6 | import random 7 | import json 8 | from collections import Counter 9 | from math import sqrt 10 | import gensim 11 | import pandas as pd 12 | import numpy as np 13 | import tensorflow as tf 14 | from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score 15 | import parameter_config 16 | import get_train_data 17 | config =parameter_config.Config() 18 | data = get_train_data.Dataset(config) 19 | #7预测代码 20 | # x = "this movie is full of references like mad max ii the wild one and many others the ladybug´s face it´s a clear reference or tribute to peter lorre this movie is a masterpiece we´ll talk much more about in the future" 21 | x = "his" 22 | # with open("../data/charJson/charToIndex.json", "w", encoding="utf-8") as f: 23 | # json.dump(self._charToIndex, f) 24 | # with open("../data/charJson/indexToChar.json", "w", encoding="utf-8") as f: 25 | # json.dump(self._indexToChar, f) 26 | 27 | # chars = [char for char in data._alphabet] 28 | # vocab, charEmbedding = data._getCharEmbedding(chars) 29 | # 30 | # data._charToIndex = dict(zip(vocab, list(range(len(vocab))))) 31 | # data._indexToChar = dict(zip(list(range(len(vocab))), vocab)) 32 | # 33 | # # 将词汇-索引映射表保存为json数据，之后做inference时直接加载来处理数据 34 | # with open("../data/charJson/charToIndex.json", "w", encoding="utf-8") as f: 35 | # json.dump(data._charToIndex, f) 36 | # with open("../data/charJson/indexToChar.json", "w", encoding="utf-8") as f: 37 | # json.dump(data._indexToChar, f) 38 | 39 | # reviews = [] 40 | # for i in range(len(x)): 41 | # reviewVec = data._reviewProcess(x[i], config.sequenceLength, data._charToIndex) 42 | # reviews.append(reviewVec) 43 | 44 | 45 | # 初始化词汇-索引映射表和词向量矩阵 46 | y = list(x) 47 | data._genVocabulary(y) 48 | print(x) 49 | reviewVec = data._reviewProcess(y, config.sequenceLength, data._charToIndex) 50 | print(reviewVec) -------------------------------------------------------------------------------- /charCNN/get_train_data.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import json 3 | import pandas as pd 4 | import numpy as np 5 | import parameter_config 6 | # 2、 训练数据生成 7 | # 　　1） 加载数据，将所有的句子分割成字符表示 8 | # 　　2） 构建字符-索引映射表，并保存成json的数据格式，方便在inference阶段加载使用 9 | # 　　3）将字符转换成one-hot的嵌入形式，作为模型中embedding层的初始化值。 10 | # 　　4） 将数据集分割成训练集和验证集 11 | # 数据预处理的类，生成训练集和测试集 12 | class Dataset(object): 13 | def __init__(self, config): #config.的部分都是从parameter.config.py中带出 14 | self._dataSource = config.dataSource #路径 15 | self._sequenceLength = config.sequenceLength # 字符表示的序列长度 16 | self._rate = config.rate # 训练集的比例 17 | self._alphabet = config.alphabet 18 | self.trainReviews = [] 19 | self.trainLabels = [] 20 | self.evalReviews = [] 21 | self.evalLabels = [] 22 | self.charEmbedding = None 23 | self._charToIndex = {} 24 | self._indexToChar = {} 25 | 26 | def _readData(self, filePath): 27 | """ 28 | 从csv文件中读取数据集 29 | """ 30 | df = pd.read_csv(filePath) 31 | labels = df["sentiment"].tolist() 32 | review = df["review"].tolist() 33 | reviews = [[char for char in line if char != " "] for line in review] 34 | return reviews, labels 35 | 36 | def _reviewProcess(self, review, sequenceLength, charToIndex): 37 | """ 38 | 将数据集中的每条评论用index表示 39 | wordToIndex中“pad”对应的index为0 40 | """ 41 | reviewVec = np.zeros((sequenceLength)) 42 | sequenceLen = sequenceLength 43 | # 判断当前的序列是否小于定义的固定序列长度 44 | if len(review) < sequenceLength: 45 | sequenceLen = len(review) 46 | for i in range(sequenceLen): 47 | if review[i] in charToIndex: 48 | reviewVec[i] = charToIndex[review[i]] 49 | else: 50 | reviewVec[i] = charToIndex["UNK"] 51 | return reviewVec 52 | 53 | def _genTrainEvalData(self, x, y, rate): 54 | """ 55 | 生成训练集和验证集,最后生成的一行表示一个句子，包含单词数为sequenceLength = 1014。每个单词用index表示 56 | """ 57 | reviews = [] 58 | labels = [] 59 | # 遍历所有的文本，将文本中的词转换成index表示 60 | for i in range(len(x)): 61 | reviewVec = self._reviewProcess(x[i], self._sequenceLength, self._charToIndex) 62 | reviews.append(reviewVec) 63 | labels.append([y[i]]) 64 | trainIndex = int(len(x) * rate) 65 | trainReviews = np.asarray(reviews[:trainIndex], dtype="int64") 66 | trainLabels = np.array(labels[:trainIndex], dtype="float32") 67 | evalReviews = np.asarray(reviews[trainIndex:], dtype="int64") 68 | evalLabels = np.array(labels[trainIndex:], dtype="float32") 69 | return trainReviews, trainLabels, evalReviews, evalLabels 70 | 71 | def _getCharEmbedding(self, chars): 72 | """ 73 | 按照one的形式将字符映射成向量 74 | 字母pad表示【0，0，0...】,UNK是【1，0，0...】，a表示【0，1，0...】等等 75 | """ 76 | alphabet = ["UNK"] + [char for char in self._alphabet] 77 | vocab = ["pad"] + alphabet 78 | charEmbedding = [] 79 | charEmbedding.append(np.zeros(len(alphabet), dtype="float32")) 80 | 81 | for i, alpha in enumerate(alphabet): 82 | onehot = np.zeros(len(alphabet), dtype="float32") 83 | # 生成每个字符对应的向量 84 | onehot[i] = 1 85 | # 生成字符嵌入的向量矩阵 86 | charEmbedding.append(onehot) 87 | return vocab, np.array(charEmbedding) 88 | 89 | def _genVocabulary(self, reviews): 90 | """ 91 | 生成字符向量和字符-索引映射字典 92 | """ 93 | chars = [char for char in self._alphabet] 94 | vocab, charEmbedding = self._getCharEmbedding(chars) 95 | self.charEmbedding = charEmbedding 96 | 97 | self._charToIndex = dict(zip(vocab, list(range(len(vocab))))) 98 | self._indexToChar = dict(zip(list(range(len(vocab))), vocab)) 99 | 100 | # 将词汇-索引映射表保存为json数据，之后做inference时直接加载来处理数据 101 | with open("../data/charJson/charToIndex.json", "w", encoding="utf-8") as f: 102 | json.dump(self._charToIndex, f) 103 | with open("../data/charJson/indexToChar.json", "w", encoding="utf-8") as f: 104 | json.dump(self._indexToChar, f) 105 | 106 | def dataGen(self): 107 | """ 108 | 初始化训练集和验证集 109 | """ 110 | # 初始化数据集 111 | # reviews: [['"', 'w', 'i', 't', 'h', 'a', 'l', 'l', 't', 'h', 'i', 's', 's', 't', 'u', 'f', 'f 112 | #labels:[1, ... 113 | reviews, labels = self._readData(self._dataSource) 114 | # print(reviews[0]) #['"', 'w', 'i', 't', 'h', 'a', 'l', 'l', 't', 'h', 'i', 115 | # 初始化词汇-索引映射表和词向量矩阵 116 | self._genVocabulary(reviews) 117 | # print(reviews[0]) #['"', 'w', 'i', 't', 'h', 'a', 'l', 'l', 't', 'h', 'i', 's', 's', 't', 'u', 'f', 'f', 'g', 'o', 'i' 118 | # 初始化训练集和测试集 训练集20000，测试集5000 每个trainReviews 长度位1014 119 | trainReviews, trainLabels, evalReviews, evalLabels = self._genTrainEvalData(reviews, labels, self._rate) 120 | # print("++++++++++") 121 | # print(trainReviews[0]) #[46 24 10 ... 6 5 17] 122 | self.trainReviews = trainReviews 123 | self.trainLabels = trainLabels 124 | self.evalReviews = evalReviews 125 | self.evalLabels = evalLabels 126 | # print(trainReviews) 127 | # print("++++") 128 | # print(trainLabels) 129 | # print(len(trainReviews[0])) 130 | # print(len(trainReviews[2])) 131 | # print(len(evalLabels)) 132 | #test 133 | config =parameter_config.Config() 134 | data = Dataset(config) 135 | data.dataGen() -------------------------------------------------------------------------------- /charCNN/mode_structure.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | 3 | import tensorflow as tf 4 | import math 5 | import parameter_config 6 | 7 | # 构建模型 3 textCNN 模型 8 | # 定义char-CNN分类器 9 | class CharCNN(object): 10 | """ 11 | char-CNN用于文本分类 　 12 | 在charCNN 模型中我们引入了BN层，但是效果并不明显，甚至存在一些收敛问题，待之后去探讨。 13 | """ 14 | def __init__(self, config, charEmbedding): 15 | # placeholders for input, output and dropuot 16 | self.inputX = tf.placeholder(tf.int32, [None, config.sequenceLength], name="inputX") 17 | self.inputY = tf.placeholder(tf.float32, [None, 1], name="inputY") 18 | self.dropoutKeepProb = tf.placeholder(tf.float32, name="dropoutKeepProb") 19 | self.isTraining = tf.placeholder(tf.bool, name="isTraining") 20 | self.epsilon = config.model.epsilon 21 | self.decay = config.model.decay 22 | 23 | # 字符嵌入 24 | with tf.name_scope("embedding"): 25 | # 利用one-hot的字符向量作为初始化词嵌入矩阵 26 | self.W = tf.Variable(tf.cast(charEmbedding, dtype=tf.float32, name="charEmbedding"), name="W") 27 | # 获得字符嵌入 28 | self.embededChars = tf.nn.embedding_lookup(self.W, self.inputX) 29 | # 添加一个通道维度 30 | self.embededCharsExpand = tf.expand_dims(self.embededChars, -1) 31 | 32 | for i, cl in enumerate(config.model.convLayers): 33 | print("开始第" + str(i + 1) + "卷积层的处理") 34 | # 利用命名空间name_scope来实现变量名复用 35 | with tf.name_scope("convLayer-%s" % (i + 1)): 36 | # 获取字符的向量长度 37 | filterWidth = self.embededCharsExpand.get_shape()[2].value 38 | # filterShape = [height, width, in_channels, out_channels] 39 | filterShape = [cl[1], filterWidth, 1, cl[0]] 40 | stdv = 1 / math.sqrt(cl[0] * cl[1]) 41 | 42 | # 初始化w和b的值 43 | wConv = tf.Variable(tf.random_uniform(filterShape, minval=-stdv, maxval=stdv), 44 | dtype='float32', name='w') 45 | bConv = tf.Variable(tf.random_uniform(shape=[cl[0]], minval=-stdv, maxval=stdv), name='b') 46 | 47 | # w_conv = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.05), name="w") 48 | # b_conv = tf.Variable(tf.constant(0.1, shape=[cl[0]]), name="b") 49 | # 构建卷积层，可以直接将卷积核的初始化方法传入（w_conv） 50 | conv = tf.nn.conv2d(self.embededCharsExpand, wConv, strides=[1, 1, 1, 1], padding="VALID", name="conv") 51 | # 加上偏差 52 | hConv = tf.nn.bias_add(conv, bConv) 53 | # 可以直接加上relu函数，因为tf.nn.conv2d事实上是做了一个卷积运算，然后在这个运算结果上加上偏差，再导入到relu函数中 54 | hConv = tf.nn.relu(hConv) 55 | 56 | # with tf.name_scope("batchNormalization"): 57 | # hConvBN = self._batchNorm(hConv) 58 | 59 | if cl[-1] is not None: 60 | ksizeShape = [1, cl[2], 1, 1] 61 | hPool = tf.nn.max_pool(hConv, ksize=ksizeShape, strides=ksizeShape, padding="VALID", name="pool") 62 | else: 63 | hPool = hConv 64 | 65 | print(hPool.shape) 66 | 67 | # 对维度进行转换，转换成卷积层的输入维度 68 | self.embededCharsExpand = tf.transpose(hPool, [0, 1, 3, 2], name="transpose") 69 | print(self.embededCharsExpand) 70 | with tf.name_scope("reshape"): 71 | fcDim = self.embededCharsExpand.get_shape()[1].value * self.embededCharsExpand.get_shape()[2].value 72 | self.inputReshape = tf.reshape(self.embededCharsExpand, [-1, fcDim]) 73 | 74 | weights = [fcDim] + config.model.fcLayers 75 | 76 | for i, fl in enumerate(config.model.fcLayers): #fcLayers = [512] 77 | with tf.name_scope("fcLayer-%s" % (i + 1)): 78 | print("开始第" + str(i + 1) + "全连接层的处理") 79 | stdv = 1 / math.sqrt(weights[i]) 80 | # 定义全连接层的初始化方法，均匀分布初始化w和b的值 81 | wFc = tf.Variable(tf.random_uniform([weights[i], fl], minval=-stdv, maxval=stdv), dtype="float32", 82 | name="w") 83 | bFc = tf.Variable(tf.random_uniform(shape=[fl], minval=-stdv, maxval=stdv), dtype="float32", name="b") 84 | 85 | # w_fc = tf.Variable(tf.truncated_normal([weights[i], fl], stddev=0.05), name="W") 86 | # b_fc = tf.Variable(tf.constant(0.1, shape=[fl]), name="b") 87 | 88 | self.fcInput = tf.nn.relu(tf.matmul(self.inputReshape, wFc) + bFc) 89 | with tf.name_scope("dropOut"): 90 | self.fcInputDrop = tf.nn.dropout(self.fcInput, self.dropoutKeepProb) 91 | self.inputReshape = self.fcInputDrop 92 | 93 | with tf.name_scope("outputLayer"): 94 | stdv = 1 / math.sqrt(weights[-1]) 95 | # 定义隐层到输出层的权重系数和偏差的初始化方法 96 | # w_out = tf.Variable(tf.truncated_normal([fc_layers[-1], num_classes], stddev=0.1), name="W") 97 | # b_out = tf.Variable(tf.constant(0.1, shape=[num_classes]), name="b") 98 | 99 | wOut = tf.Variable(tf.random_uniform([config.model.fcLayers[-1], 1], minval=-stdv, maxval=stdv), 100 | dtype="float32", name="w") 101 | bOut = tf.Variable(tf.random_uniform(shape=[1], minval=-stdv, maxval=stdv), name="b") 102 | # tf.nn.xw_plus_b就是x和w的乘积加上b 103 | self.predictions = tf.nn.xw_plus_b(self.inputReshape, wOut, bOut, name="predictions") 104 | # 进行二元分类 105 | self.binaryPreds = tf.cast(tf.greater_equal(self.predictions, 0.0), tf.float32, name="binaryPreds") 106 | 107 | with tf.name_scope("loss"): 108 | # 定义损失函数，对预测值进行softmax，再求交叉熵。 109 | losses = tf.nn.sigmoid_cross_entropy_with_logits(logits=self.predictions, labels=self.inputY) 110 | self.loss = tf.reduce_mean(losses) 111 | 112 | def _batchNorm(self, x): 113 | # BN层代码实现 114 | gamma = tf.Variable(tf.ones([x.get_shape()[3].value])) 115 | beta = tf.Variable(tf.zeros([x.get_shape()[3].value])) 116 | self.popMean = tf.Variable(tf.zeros([x.get_shape()[3].value]), trainable=False, name="popMean") 117 | self.popVariance = tf.Variable(tf.ones([x.get_shape()[3].value]), trainable=False, name="popVariance") 118 | 119 | def batchNormTraining(): 120 | # 一定要使用正确的维度确保计算的是每个特征图上的平均值和方差而不是整个网络节点上的统计分布值 121 | batchMean, batchVariance = tf.nn.moments(x, [0, 1, 2], keep_dims=False) 122 | decay = 0.99 123 | trainMean = tf.assign(self.popMean, self.popMean * self.decay + batchMean * (1 - self.decay)) 124 | trainVariance = tf.assign(self.popVariance, 125 | self.popVariance * self.decay + batchVariance * (1 - self.decay)) 126 | 127 | with tf.control_dependencies([trainMean, trainVariance]): 128 | return tf.nn.batch_normalization(x, batchMean, batchVariance, beta, gamma, self.epsilon) 129 | 130 | def batchNormInference(): 131 | return tf.nn.batch_normalization(x, self.popMean, self.popVariance, beta, gamma, self.epsilon) 132 | batchNormalizedOutput = tf.cond(self.isTraining, batchNormTraining, batchNormInference) 133 | return tf.nn.relu(batchNormalizedOutput) -------------------------------------------------------------------------------- /charCNN/mode_trainning.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import os 3 | import datetime 4 | import warnings 5 | import numpy as np 6 | import tensorflow as tf 7 | from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score 8 | warnings.filterwarnings("ignore") 9 | import parameter_config 10 | import get_train_data 11 | import mode_structure 12 | 13 | #获取前些模块的数据 14 | config =parameter_config.Config() 15 | data = get_train_data.Dataset(config) 16 | data.dataGen() 17 | 18 | #4生成batch数据集 19 | def nextBatch(x, y, batchSize): 20 | # 生成batch数据集，用生成器的方式输出 21 | perm = np.arange(len(x)) 22 | np.random.shuffle(perm) 23 | x = x[perm] 24 | y = y[perm] 25 | # print("++++++++++++++") 26 | # print(x) 27 | numBatches = len(x) // batchSize 28 | 29 | for i in range(numBatches): 30 | start = i * batchSize 31 | end = start + batchSize 32 | batchX = np.array(x[start: end], dtype="int64") 33 | batchY = np.array(y[start: end], dtype="float32") 34 | yield batchX, batchY 35 | 36 | # 5 定义计算metrics的函数 37 | """ 38 | 定义各类性能指标 39 | """ 40 | def mean(item): 41 | return sum(item) / len(item) 42 | 43 | def genMetrics(trueY, predY, binaryPredY): 44 | """ 45 | 生成acc和auc值 46 | """ 47 | auc = roc_auc_score(trueY, predY) 48 | accuracy = accuracy_score(trueY, binaryPredY) 49 | precision = precision_score(trueY, binaryPredY, average='macro') 50 | recall = recall_score(trueY, binaryPredY, average='macro') 51 | return round(accuracy, 4), round(auc, 4), round(precision, 4), round(recall, 4) 52 | 53 | # 6 训练模型 54 | # 生成训练集和验证集 55 | trainReviews = data.trainReviews 56 | trainLabels = data.trainLabels 57 | evalReviews = data.evalReviews 58 | evalLabels = data.evalLabels 59 | charEmbedding = data.charEmbedding 60 | 61 | # 定义计算图 62 | with tf.Graph().as_default(): 63 | session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) 64 | session_conf.gpu_options.allow_growth = True 65 | session_conf.gpu_options.per_process_gpu_memory_fraction = 0.9 # 配置gpu占用率 66 | sess = tf.Session(config=session_conf) 67 | 68 | # 定义会话 69 | with sess.as_default(): 70 | cnn = mode_structure.CharCNN(config, charEmbedding) 71 | globalStep = tf.Variable(0, name="globalStep", trainable=False) 72 | # 定义优化函数，传入学习速率参数 73 | optimizer = tf.train.RMSPropOptimizer(config.training.learningRate) 74 | # 计算梯度,得到梯度和变量 75 | gradsAndVars = optimizer.compute_gradients(cnn.loss) 76 | # 将梯度应用到变量下，生成训练器 77 | trainOp = optimizer.apply_gradients(gradsAndVars, global_step=globalStep) 78 | 79 | # 用summary绘制tensorBoard 80 | gradSummaries = [] 81 | for g, v in gradsAndVars: 82 | if g is not None: 83 | tf.summary.histogram("{}/grad/hist".format(v.name), g) 84 | tf.summary.scalar("{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) 85 | outDir = os.path.abspath(os.path.join(os.path.curdir, "summarys")) 86 | print("Writing to {}\n".format(outDir)) 87 | lossSummary = tf.summary.scalar("trainLoss", cnn.loss) 88 | 89 | summaryOp = tf.summary.merge_all() 90 | 91 | trainSummaryDir = os.path.join(outDir, "train") 92 | trainSummaryWriter = tf.summary.FileWriter(trainSummaryDir, sess.graph) 93 | evalSummaryDir = os.path.join(outDir, "eval") 94 | evalSummaryWriter = tf.summary.FileWriter(evalSummaryDir, sess.graph) 95 | 96 | # 初始化所有变量 97 | saver = tf.train.Saver(tf.global_variables(), max_to_keep=5) 98 | 99 | # 保存模型的一种方式，保存为pb文件 100 | builder = tf.saved_model.builder.SavedModelBuilder("../model/charCNN/savedModel") 101 | sess.run(tf.global_variables_initializer()) 102 | 103 | def trainStep(batchX, batchY): 104 | """ 105 | 训练函数 106 | """ 107 | feed_dict = { 108 | cnn.inputX: batchX, 109 | cnn.inputY: batchY, 110 | cnn.dropoutKeepProb: config.model.dropoutKeepProb, 111 | cnn.isTraining: True 112 | } 113 | _, summary, step, loss, predictions, binaryPreds = sess.run( 114 | [trainOp, summaryOp, globalStep, cnn.loss, cnn.predictions, cnn.binaryPreds], 115 | feed_dict) 116 | timeStr = datetime.datetime.now().isoformat() 117 | acc, auc, precision, recall = genMetrics(batchY, predictions, binaryPreds) 118 | print("{}, step: {}, loss: {}, acc: {}, auc: {}, precision: {}, recall: {}".format(timeStr, step, loss, 119 | acc, auc, precision, 120 | recall)) 121 | trainSummaryWriter.add_summary(summary, step) 122 | 123 | def devStep(batchX, batchY): 124 | """ 125 | 验证函数 126 | """ 127 | feed_dict = { 128 | cnn.inputX: batchX, 129 | cnn.inputY: batchY, 130 | cnn.dropoutKeepProb: 1.0, 131 | cnn.isTraining: False 132 | } 133 | summary, step, loss, predictions, binaryPreds = sess.run( 134 | [summaryOp, globalStep, cnn.loss, cnn.predictions, cnn.binaryPreds], 135 | feed_dict) 136 | 137 | acc, auc, precision, recall = genMetrics(batchY, predictions, binaryPreds) 138 | 139 | evalSummaryWriter.add_summary(summary, step) 140 | 141 | return loss, acc, auc, precision, recall 142 | 143 | for i in range(config.training.epoches): 144 | # 训练模型 145 | print("start training model") 146 | for batchTrain in nextBatch(trainReviews, trainLabels, config.batchSize): 147 | trainStep(batchTrain[0], batchTrain[1]) 148 | 149 | currentStep = tf.train.global_step(sess, globalStep) 150 | if currentStep % config.training.evaluateEvery == 0: 151 | print("\nEvaluation:") 152 | 153 | losses = [] 154 | accs = [] 155 | aucs = [] 156 | precisions = [] 157 | recalls = [] 158 | 159 | for batchEval in nextBatch(evalReviews, evalLabels, config.batchSize): 160 | loss, acc, auc, precision, recall = devStep(batchEval[0], batchEval[1]) 161 | losses.append(loss) 162 | accs.append(acc) 163 | aucs.append(auc) 164 | precisions.append(precision) 165 | recalls.append(recall) 166 | 167 | time_str = datetime.datetime.now().isoformat() 168 | print("{}, step: {}, loss: {}, acc: {}, auc: {}, precision: {}, recall: {}".format(time_str, 169 | currentStep, 170 | mean(losses), 171 | mean(accs), 172 | mean(aucs), 173 | mean( 174 | precisions), 175 | mean( 176 | recalls))) 177 | 178 | if currentStep % config.training.checkpointEvery == 0: 179 | # 保存模型的另一种方法，保存checkpoint文件 180 | path = saver.save(sess, "../model/charCNN/model/my-model", global_step=currentStep) 181 | print("Saved model checkpoint to {}\n".format(path)) 182 | 183 | inputs = {"inputX": tf.saved_model.utils.build_tensor_info(cnn.inputX), 184 | "keepProb": tf.saved_model.utils.build_tensor_info(cnn.dropoutKeepProb)} 185 | 186 | outputs = {"binaryPreds": tf.saved_model.utils.build_tensor_info(cnn.binaryPreds)} 187 | 188 | prediction_signature = tf.saved_model.signature_def_utils.build_signature_def(inputs=inputs, 189 | outputs=outputs, 190 | method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME) 191 | legacy_init_op = tf.group(tf.tables_initializer(), name="legacy_init_op") 192 | builder.add_meta_graph_and_variables(sess, [tf.saved_model.tag_constants.SERVING], 193 | signature_def_map={"predict": prediction_signature}, 194 | legacy_init_op=legacy_init_op) 195 | 196 | builder.save() -------------------------------------------------------------------------------- /charCNN/parameter_config.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | # 1、参数配置 3 | class TrainingConfig(object): 4 | epoches = 6 5 | evaluateEvery = 100 6 | checkpointEvery = 100 7 | learningRate = 0.001 8 | 9 | class ModelConfig(object): 10 | # 该列表中子列表的三个元素分别:卷积核的数量，卷积核的高度，池化的尺寸 11 | convLayers = [[256, 7, 4], 12 | [256, 7, 4], 13 | [256, 3, 4]] 14 | fcLayers = [512] 15 | dropoutKeepProb = 0.5 16 | epsilon = 1e-3 # BN层中防止分母为0而加入的极小值 17 | decay = 0.999 # BN层中用来计算滑动平均的值 18 | 19 | class Config(object): 20 | # 我们使用论文中提出的69个字符来表征输入数据 21 | alphabet = "abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{}" 22 | # alphabet = "abcdefghijklmnopqrstuvwxyz0123456789" 23 | sequenceLength = 1014 # 字符表示的序列长度 24 | batchSize = 128 25 | rate = 0.8 # 训练集的比例 26 | dataSource = "../data/preProcess/labeledCharTrain.csv" 27 | training = TrainingConfig() 28 | model = ModelConfig() 29 | config = Config() -------------------------------------------------------------------------------- /charCNN/predict.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | 3 | import tensorflow as tf 4 | import parameter_config 5 | import get_train_data 6 | config =parameter_config.Config() 7 | data = get_train_data.Dataset(config) 8 | 9 | #7预测代码 10 | x = "this movie is full of references like mad max ii the wild one and many others the ladybug´s face it´s a clear reference or tribute to peter lorre this movie is a masterpiece we´ll talk much more about in the future" 11 | # x = "This film is not good" #最终反馈为1 12 | # x = "This film is bad" #最终反馈为0 13 | # x = "This film is good" #最终反馈为1 14 | 15 | # 根据前面get_train_data获取，变成可以用来训练的向量。 16 | y = list(x) 17 | data._genVocabulary(y) 18 | print(x) 19 | reviewVec = data._reviewProcess(y, config.sequenceLength, data._charToIndex) 20 | print(reviewVec) 21 | 22 | graph = tf.Graph() 23 | with graph.as_default(): 24 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) 25 | session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, gpu_options=gpu_options) 26 | sess = tf.Session(config=session_conf) 27 | 28 | with sess.as_default(): 29 | # 恢复模型 30 | checkpoint_file = tf.train.latest_checkpoint("../model/charCNN/model/") 31 | saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file)) 32 | saver.restore(sess, checkpoint_file) 33 | 34 | # 获得需要喂给模型的参数，输出的结果依赖的输入值 35 | inputX = graph.get_operation_by_name("inputX").outputs[0] 36 | dropoutKeepProb = graph.get_operation_by_name("dropoutKeepProb").outputs[0] 37 | 38 | # 获得输出的结果 39 | predictions = graph.get_tensor_by_name("outputLayer/binaryPreds:0") 40 | pred = sess.run(predictions, feed_dict={inputX: [reviewVec], dropoutKeepProb: 1.0,})[0] 41 | 42 | # pred = [idx2label[item] for item in pred] 43 | print(pred) -------------------------------------------------------------------------------- /textCNN/get_train_data.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import json 3 | from collections import Counter 4 | import gensim 5 | import pandas as pd 6 | import numpy as np 7 | import parameter_config 8 | 9 | # 2 数据预处理的类，生成训练集和测试集 10 | class Dataset(object): 11 | def __init__(self, config): 12 | self.config = config 13 | self._dataSource = config.dataSource 14 | self._stopWordSource = config.stopWordSource 15 | self._sequenceLength = config.sequenceLength # 每条输入的序列处理为定长 16 | self._embeddingSize = config.model.embeddingSize 17 | self._batchSize = config.batchSize 18 | self._rate = config.rate 19 | self._stopWordDict = {} 20 | self.trainReviews = [] 21 | self.trainLabels = [] 22 | self.evalReviews = [] 23 | self.evalLabels = [] 24 | self.wordEmbedding = None 25 | self.labelList = [] 26 | def _readData(self, filePath): 27 | """ 28 | 从csv文件中读取数据集,就本次测试的文件做记录 29 | """ 30 | df = pd.read_csv(filePath) #读取文件，是三列的数据，第一列是review,第二列sentiment，第三列rate 31 | if self.config.numClasses == 1: 32 | labels = df["sentiment"].tolist() #读取sentiment列的数据， 显示输出01序列数组25000条 33 | elif self.config.numClasses > 1: 34 | labels = df["rate"].tolist() #因为numClasses控制，本次取样没有取超过二分类 该处没有输出 35 | review = df["review"].tolist() 36 | reviews = [line.strip().split() for line in review] #按空格语句切分 37 | return reviews, labels 38 | def _labelToIndex(self, labels, label2idx): 39 | """ 40 | 将标签转换成索引表示 41 | """ 42 | labelIds = [label2idx[label] for label in labels] #print(labels==labelIds) 结果显示为true,也就是两个一样 43 | return labelIds 44 | def _wordToIndex(self, reviews, word2idx): 45 | """将词转换成索引""" 46 | reviewIds = [[word2idx.get(item, word2idx["UNK"]) for item in review] for review in reviews] 47 | # print(max(max(reviewIds))) 48 | # print(reviewIds) 49 | return reviewIds #返回25000个无序的数组 50 | def _genTrainEvalData(self, x, y, word2idx, rate): 51 | """生成训练集和验证集 """ 52 | reviews = [] 53 | # print(self._sequenceLength) 54 | # print(len(x)) 55 | for review in x: #self._sequenceLength为200，表示长的切成200，短的补齐，x数据依旧是25000 56 | if len(review) >= self._sequenceLength: 57 | reviews.append(review[:self._sequenceLength]) 58 | else: 59 | reviews.append(review + [word2idx["PAD"]] * (self._sequenceLength - len(review))) 60 | # print(len(review + [word2idx["PAD"]] * (self._sequenceLength - len(review)))) 61 | 62 | #以下是按照rate比例切分训练和测试数据： 63 | trainIndex = int(len(x) * rate) 64 | trainReviews = np.asarray(reviews[:trainIndex], dtype="int64") 65 | trainLabels = np.array(y[:trainIndex], dtype="float32") 66 | evalReviews = np.asarray(reviews[trainIndex:], dtype="int64") 67 | evalLabels = np.array(y[trainIndex:], dtype="float32") 68 | return trainReviews, trainLabels, evalReviews, evalLabels 69 | def _getWordEmbedding(self, words): 70 | """按照我们的数据集中的单词取出预训练好的word2vec中的词向量 71 | 反馈词和对应的向量（200维度），另外前面增加PAD对用0的数组，UNK对应随机数组。 72 | """ 73 | wordVec = gensim.models.KeyedVectors.load_word2vec_format("../word2vec/word2Vec.bin", binary=True) 74 | vocab = [] 75 | wordEmbedding = [] 76 | # 添加 "pad" 和 "UNK", 77 | vocab.append("PAD") 78 | vocab.append("UNK") 79 | wordEmbedding.append(np.zeros(self._embeddingSize)) # _embeddingSize 本文定义的是200 80 | wordEmbedding.append(np.random.randn(self._embeddingSize)) 81 | # print(wordEmbedding) 82 | for word in words: 83 | try: 84 | vector = wordVec.wv[word] 85 | vocab.append(word) 86 | wordEmbedding.append(vector) 87 | except: 88 | print(word + "不存在于词向量中") 89 | # print(vocab[:3],wordEmbedding[:3]) 90 | return vocab, np.array(wordEmbedding) 91 | def _genVocabulary(self, reviews, labels): 92 | """生成词向量和词汇-索引映射字典，可以用全数据集""" 93 | allWords = [word for review in reviews for word in review] #单词数量5738236 reviews是25000个观点句子【】 94 | subWords = [word for word in allWords if word not in self.stopWordDict] # 去掉停用词 95 | wordCount = Counter(subWords) # 统计词频 96 | sortWordCount = sorted(wordCount.items(), key=lambda x: x[1], reverse=True) #返回键值对，并按照数量排序 97 | # print(len(sortWordCount)) #161330 98 | # print(sortWordCount[:4],sortWordCount[-4:]) # [('movie', 41104), ('film', 36981), ('one', 24966), ('like', 19490)] [('daeseleires', 1), ('nice310', 1), ('shortsightedness', 1), ('unfairness', 1)] 99 | words = [item[0] for item in sortWordCount if item[1] >= 5] # 去除低频词，低于5的 100 | vocab, wordEmbedding = self._getWordEmbedding(words) 101 | self.wordEmbedding = wordEmbedding 102 | word2idx = dict(zip(vocab, list(range(len(vocab))))) #生成类似这种{'I': 0, 'love': 1, 'yanzi': 2} 103 | uniqueLabel = list(set(labels)) #标签去重 最后就 0 1了 104 | label2idx = dict(zip(uniqueLabel, list(range(len(uniqueLabel))))) #本文就 {0: 0, 1: 1} 105 | self.labelList = list(range(len(uniqueLabel))) 106 | # 将词汇-索引映射表保存为json数据，之后做inference时直接加载来处理数据 107 | with open("../data/wordJson/word2idx.json", "w", encoding="utf-8") as f: 108 | json.dump(word2idx, f) 109 | with open("../data/wordJson/label2idx.json", "w", encoding="utf-8") as f: 110 | json.dump(label2idx, f) 111 | return word2idx, label2idx 112 | def _readStopWord(self, stopWordPath): 113 | """ 114 | 读取停用词 115 | """ 116 | with open(stopWordPath, "r") as f: 117 | stopWords = f.read() 118 | stopWordList = stopWords.splitlines() 119 | # 将停用词用列表的形式生成，之后查找停用词时会比较快 120 | self.stopWordDict = dict(zip(stopWordList, list(range(len(stopWordList))))) 121 | def dataGen(self): 122 | """ 123 | 初始化训练集和验证集 124 | """ 125 | # 初始化停用词 126 | self._readStopWord(self._stopWordSource) 127 | # 初始化数据集 128 | reviews, labels = self._readData(self._dataSource) 129 | # 初始化词汇-索引映射表和词向量矩阵 130 | word2idx, label2idx = self._genVocabulary(reviews, labels) 131 | # 将标签和句子数值化 132 | labelIds = self._labelToIndex(labels, label2idx) 133 | reviewIds = self._wordToIndex(reviews, word2idx) 134 | # 初始化训练集和测试集 135 | trainReviews, trainLabels, evalReviews, evalLabels = self._genTrainEvalData(reviewIds, labelIds, word2idx, 136 | self._rate) 137 | self.trainReviews = trainReviews 138 | self.trainLabels = trainLabels 139 | 140 | self.evalReviews = evalReviews 141 | self.evalLabels = evalLabels 142 | 143 | #获取前些模块的数据 144 | # config =parameter_config.Config() 145 | # data = Dataset(config) 146 | # data.dataGen() -------------------------------------------------------------------------------- /textCNN/mode_structure.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | 3 | import tensorflow as tf 4 | import parameter_config 5 | 6 | # 构建模型 3 textCNN 模型 7 | class TextCNN(object): 8 | """ 9 | Text CNN 用于文本分类 10 | """ 11 | def __init__(self, config, wordEmbedding): 12 | # 定义模型的输入 13 | self.inputX = tf.placeholder(tf.int32, [None, config.sequenceLength], name="inputX") #占据【不定，200】的矩阵 14 | self.inputY = tf.placeholder(tf.int32, [None], name="inputY") 15 | self.dropoutKeepProb = tf.placeholder(tf.float32, name="dropoutKeepProb") 16 | # 定义l2损失 17 | l2Loss = tf.constant(0.0) 18 | 19 | # 词嵌入层 20 | #词对应的向量wordEmbedding（200维度），另外前面增加PAD对用0的数组，UNK对应随机数组。 21 | with tf.name_scope("embedding"): 22 | # 利用预训练的词向量初始化词嵌入矩阵 23 | self.W = tf.Variable(tf.cast(wordEmbedding, dtype=tf.float32, name="word2vec"), name="W") 24 | # 利用词嵌入矩阵将输入的数据中的词转换成词向量，维度[batch_size, sequence_length, embedding_size] 25 | self.embeddedWords = tf.nn.embedding_lookup(self.W, self.inputX) 26 | # 卷积的输入是四维[batch_size, width, height, channel]，因此需要增加维度，用tf.expand_dims来增大维度（通道数） 27 | self.embeddedWordsExpanded = tf.expand_dims(self.embeddedWords, -1) 28 | 29 | # 创建卷积和池化层 30 | pooledOutputs = [] 31 | # 有三种size的filter，3， 4， 5，textCNN是个多通道单层卷积的模型，可以看作三个单层的卷积模型的融合 32 | for i, filterSize in enumerate(config.model.filterSizes): 33 | with tf.name_scope("conv-maxpool-%s" % filterSize): 34 | # 卷积层，卷积核尺寸为 filterSize * embeddingSize，卷积核的个数为numFilters filterSizes = [2, 3, 4, 5] 35 | # 初始化权重矩阵和偏置 36 | filterShape = [filterSize, config.model.embeddingSize, 1, config.model.numFilters] 37 | W = tf.Variable(tf.truncated_normal(filterShape, stddev=0.1), name="W") 38 | b = tf.Variable(tf.constant(0.1, shape=[config.model.numFilters]), name="b") 39 | conv = tf.nn.conv2d( 40 | self.embeddedWordsExpanded, 41 | W, 42 | strides=[1, 1, 1, 1], 43 | padding="VALID", 44 | name="conv") 45 | # relu函数的非线性映射 46 | h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu") 47 | 48 | # 池化层，最大池化，池化是对卷积后的序列取一个最大值 49 | pooled = tf.nn.max_pool( 50 | h, 51 | ksize=[1, config.sequenceLength - filterSize + 1, 1, 1], 52 | # ksize shape: [batch, height, width, channels] 53 | strides=[1, 1, 1, 1], 54 | padding='VALID', 55 | name="pool") 56 | pooledOutputs.append(pooled) # 将三种size的filter的输出一起加入到列表中 57 | 58 | # 得到CNN网络的输出长度 59 | numFiltersTotal = config.model.numFilters * len(config.model.filterSizes) #128*filterSizes的个数 60 | 61 | # 池化后的维度不变，按照最后的维度channel来concat 62 | self.hPool = tf.concat(pooledOutputs, 3) 63 | 64 | # 摊平成二维的数据输入到全连接层 65 | self.hPoolFlat = tf.reshape(self.hPool, [-1, numFiltersTotal]) 66 | 67 | # dropout 68 | with tf.name_scope("dropout"): 69 | self.hDrop = tf.nn.dropout(self.hPoolFlat, self.dropoutKeepProb) 70 | 71 | # 全连接层的输出 72 | with tf.name_scope("output"): #predice.py的graph.get_tensor_by_name("output/predictions:0")用到调用 73 | outputW = tf.get_variable( 74 | "outputW", 75 | shape=[numFiltersTotal, config.numClasses], 76 | initializer=tf.contrib.layers.xavier_initializer()) 77 | outputB = tf.Variable(tf.constant(0.1, shape=[config.numClasses]), name="outputB") 78 | l2Loss += tf.nn.l2_loss(outputW) 79 | l2Loss += tf.nn.l2_loss(outputB) 80 | self.logits = tf.nn.xw_plus_b(self.hDrop, outputW, outputB, name="logits") 81 | if config.numClasses == 1: 82 | self.predictions = tf.cast(tf.greater_equal(self.logits, 0.0), tf.int32, name="predictions")#≥0.0则输出1 83 | elif config.numClasses > 1: 84 | self.predictions = tf.argmax(self.logits, axis=-1, name="predictions") 85 | print(self.predictions) 86 | 87 | # 计算二元交叉熵损失 88 | with tf.name_scope("loss"): 89 | if config.numClasses == 1: 90 | losses = tf.nn.sigmoid_cross_entropy_with_logits(logits=self.logits, #二分类用sigmoid 91 | labels=tf.cast(tf.reshape(self.inputY, [-1, 1]), 92 | dtype=tf.float32)) 93 | elif config.numClasses > 1: 94 | #多分类使用softmax 95 | losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.inputY) 96 | 97 | self.loss = tf.reduce_mean(losses) + config.model.l2RegLambda * l2Loss -------------------------------------------------------------------------------- /textCNN/mode_trainning.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | 3 | import os 4 | import datetime 5 | import numpy as np 6 | import tensorflow as tf 7 | import parameter_config 8 | import get_train_data 9 | import mode_structure 10 | 11 | #获取前些模块的数据 12 | config =parameter_config.Config() 13 | data = get_train_data.Dataset(config) 14 | data.dataGen() 15 | 16 | #4生成batch数据集 17 | def nextBatch(x, y, batchSize): 18 | # 生成batch数据集，用生成器的方式输出 19 | perm = np.arange(len(x)) #返回[0 1 2 ... len(x)]的数组 20 | np.random.shuffle(perm) #乱序 21 | x = x[perm] 22 | y = y[perm] 23 | numBatches = len(x) // batchSize 24 | 25 | for i in range(numBatches): 26 | start = i * batchSize 27 | end = start + batchSize 28 | batchX = np.array(x[start: end], dtype="int64") 29 | batchY = np.array(y[start: end], dtype="float32") 30 | yield batchX, batchY 31 | 32 | # 5 定义计算metrics的函数 33 | """ 34 | 定义各类性能指标 35 | """ 36 | def mean(item: list) -> float: 37 | """ 38 | 计算列表中元素的平均值 39 | :param item: 列表对象 40 | :return: 41 | """ 42 | res = sum(item) / len(item) if len(item) > 0 else 0 43 | return res 44 | 45 | def accuracy(pred_y, true_y): 46 | """ 47 | 计算二类和多类的准确率 48 | :param pred_y: 预测结果 49 | :param true_y: 真实结果 50 | :return: 51 | """ 52 | if isinstance(pred_y[0], list): 53 | pred_y = [item[0] for item in pred_y] 54 | corr = 0 55 | for i in range(len(pred_y)): 56 | if pred_y[i] == true_y[i]: 57 | corr += 1 58 | acc = corr / len(pred_y) if len(pred_y) > 0 else 0 59 | return acc 60 | 61 | def binary_precision(pred_y, true_y, positive=1): 62 | """ 63 | 二类的精确率计算 64 | :param pred_y: 预测结果 65 | :param true_y: 真实结果 66 | :param positive: 正例的索引表示 67 | :return: 68 | """ 69 | corr = 0 70 | pred_corr = 0 71 | for i in range(len(pred_y)): 72 | if pred_y[i] == positive: 73 | pred_corr += 1 74 | if pred_y[i] == true_y[i]: 75 | corr += 1 76 | 77 | prec = corr / pred_corr if pred_corr > 0 else 0 78 | return prec 79 | 80 | def binary_recall(pred_y, true_y, positive=1): 81 | """ 82 | 二类的召回率 83 | :param pred_y: 预测结果 84 | :param true_y: 真实结果 85 | :param positive: 正例的索引表示 86 | :return: 87 | """ 88 | corr = 0 89 | true_corr = 0 90 | for i in range(len(pred_y)): 91 | if true_y[i] == positive: 92 | true_corr += 1 93 | if pred_y[i] == true_y[i]: 94 | corr += 1 95 | 96 | rec = corr / true_corr if true_corr > 0 else 0 97 | return rec 98 | 99 | def binary_f_beta(pred_y, true_y, beta=1.0, positive=1): 100 | """ 101 | 二类的f beta值 102 | :param pred_y: 预测结果 103 | :param true_y: 真实结果 104 | :param beta: beta值 105 | :param positive: 正例的索引表示 106 | :return: 107 | """ 108 | precision = binary_precision(pred_y, true_y, positive) 109 | recall = binary_recall(pred_y, true_y, positive) 110 | try: 111 | f_b = (1 + beta * beta) * precision * recall / (beta * beta * precision + recall) 112 | except: 113 | f_b = 0 114 | return f_b 115 | 116 | def multi_precision(pred_y, true_y, labels): 117 | """ 118 | 多类的精确率 119 | :param pred_y: 预测结果 120 | :param true_y: 真实结果 121 | :param labels: 标签列表 122 | :return: 123 | """ 124 | if isinstance(pred_y[0], list): 125 | pred_y = [item[0] for item in pred_y] 126 | 127 | precisions = [binary_precision(pred_y, true_y, label) for label in labels] 128 | prec = mean(precisions) 129 | return prec 130 | 131 | def multi_recall(pred_y, true_y, labels): 132 | """ 133 | 多类的召回率 134 | :param pred_y: 预测结果 135 | :param true_y: 真实结果 136 | :param labels: 标签列表 137 | :return: 138 | """ 139 | if isinstance(pred_y[0], list): 140 | pred_y = [item[0] for item in pred_y] 141 | 142 | recalls = [binary_recall(pred_y, true_y, label) for label in labels] 143 | rec = mean(recalls) 144 | return rec 145 | 146 | def multi_f_beta(pred_y, true_y, labels, beta=1.0): 147 | """ 148 | 多类的f beta值 149 | :param pred_y: 预测结果 150 | :param true_y: 真实结果 151 | :param labels: 标签列表 152 | :param beta: beta值 153 | :return: 154 | """ 155 | if isinstance(pred_y[0], list): 156 | pred_y = [item[0] for item in pred_y] 157 | 158 | f_betas = [binary_f_beta(pred_y, true_y, beta, label) for label in labels] 159 | f_beta = mean(f_betas) 160 | return f_beta 161 | 162 | def get_binary_metrics(pred_y, true_y, f_beta=1.0): 163 | """ 164 | 得到二分类的性能指标 165 | :param pred_y: 166 | :param true_y: 167 | :param f_beta: 168 | :return: 169 | """ 170 | acc = accuracy(pred_y, true_y) 171 | recall = binary_recall(pred_y, true_y) 172 | precision = binary_precision(pred_y, true_y) 173 | f_beta = binary_f_beta(pred_y, true_y, f_beta) 174 | return acc, recall, precision, f_beta 175 | 176 | def get_multi_metrics(pred_y, true_y, labels, f_beta=1.0): 177 | """ 178 | 得到多分类的性能指标 179 | :param pred_y: 180 | :param true_y: 181 | :param labels: 182 | :param f_beta: 183 | :return: 184 | """ 185 | acc = accuracy(pred_y, true_y) 186 | recall = multi_recall(pred_y, true_y, labels) 187 | precision = multi_precision(pred_y, true_y, labels) 188 | f_beta = multi_f_beta(pred_y, true_y, labels, f_beta) 189 | return acc, recall, precision, f_beta 190 | 191 | # 6 训练模型 192 | # 生成训练集和验证集 193 | trainReviews = data.trainReviews 194 | trainLabels = data.trainLabels 195 | evalReviews = data.evalReviews 196 | evalLabels = data.evalLabels 197 | 198 | wordEmbedding = data.wordEmbedding 199 | labelList = data.labelList 200 | 201 | # 定义计算图 202 | with tf.Graph().as_default(): 203 | session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) 204 | session_conf.gpu_options.allow_growth = True 205 | session_conf.gpu_options.per_process_gpu_memory_fraction = 0.9 # 配置gpu占用率 206 | sess = tf.Session(config=session_conf) 207 | 208 | # 定义会话 209 | with sess.as_default(): 210 | # cnn = textCNN.TextCNN(config, wordEmbedding) 211 | cnn = mode_structure.TextCNN(config, wordEmbedding) #调用之前的模型结构 212 | globalStep = tf.Variable(0, name="globalStep", trainable=False) 213 | # 定义优化函数，传入学习速率参数 214 | optimizer = tf.train.AdamOptimizer(config.training.learningRate) 215 | # 计算梯度,得到梯度和变量 216 | gradsAndVars = optimizer.compute_gradients(cnn.loss) 217 | # 将梯度应用到变量下，生成训练器 218 | trainOp = optimizer.apply_gradients(gradsAndVars, global_step=globalStep) 219 | 220 | # 用summary绘制tensorBoard 221 | gradSummaries = [] 222 | for g, v in gradsAndVars: 223 | if g is not None: 224 | tf.summary.histogram("{}/grad/hist".format(v.name), g) 225 | tf.summary.scalar("{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g)) 226 | 227 | outDir = os.path.abspath(os.path.join(os.path.curdir, "summarys")) 228 | print("Writing to {}\n".format(outDir)) 229 | 230 | trainSummaryDir = os.path.join(outDir, "train") 231 | trainSummaryWriter = tf.summary.FileWriter(trainSummaryDir, sess.graph) 232 | 233 | evalSummaryDir = os.path.join(outDir, "eval") 234 | evalSummaryWriter = tf.summary.FileWriter(evalSummaryDir, sess.graph) 235 | 236 | lossSummary = tf.summary.scalar("loss", cnn.loss) 237 | summaryOp = tf.summary.merge_all() 238 | 239 | # 初始化所有变量 240 | saver = tf.train.Saver(tf.global_variables(), max_to_keep=5) #保存5个模型 241 | # 保存模型的一种方式，保存为pb文件 242 | savedModelPath = "../model/textCNN/savedModel" 243 | if os.path.exists(savedModelPath): 244 | os.rmdir(savedModelPath) 245 | builder = tf.saved_model.builder.SavedModelBuilder(savedModelPath) 246 | sess.run(tf.global_variables_initializer()) 247 | 248 | 249 | def trainStep(batchX, batchY): 250 | """ 251 | 训练函数 252 | """ 253 | feed_dict = { 254 | cnn.inputX: batchX, 255 | cnn.inputY: batchY, 256 | cnn.dropoutKeepProb: config.model.dropoutKeepProb 257 | } 258 | _, summary, step, loss, predictions = sess.run( 259 | [trainOp, summaryOp, globalStep, cnn.loss, cnn.predictions], 260 | feed_dict) 261 | timeStr = datetime.datetime.now().isoformat() 262 | 263 | if config.numClasses == 1: 264 | acc, recall, prec, f_beta = get_binary_metrics(pred_y=predictions, true_y=batchY) 265 | elif config.numClasses > 1: 266 | acc, recall, prec, f_beta = get_multi_metrics(pred_y=predictions, true_y=batchY, 267 | labels=labelList) 268 | trainSummaryWriter.add_summary(summary, step) 269 | return loss, acc, prec, recall, f_beta 270 | 271 | def devStep(batchX, batchY): 272 | """ 273 | 验证函数 274 | """ 275 | feed_dict = { 276 | cnn.inputX: batchX, 277 | cnn.inputY: batchY, 278 | cnn.dropoutKeepProb: 1.0 279 | } 280 | summary, step, loss, predictions = sess.run( 281 | [summaryOp, globalStep, cnn.loss, cnn.predictions], 282 | feed_dict) 283 | 284 | if config.numClasses == 1: 285 | 286 | acc, precision, recall, f_beta = get_binary_metrics(pred_y=predictions, true_y=batchY) 287 | elif config.numClasses > 1: 288 | acc, precision, recall, f_beta = get_multi_metrics(pred_y=predictions, true_y=batchY, labels=labelList) 289 | 290 | evalSummaryWriter.add_summary(summary, step) 291 | 292 | return loss, acc, precision, recall, f_beta 293 | 294 | for i in range(config.training.epoches): 295 | # 训练模型 296 | print("start training model") 297 | for batchTrain in nextBatch(trainReviews, trainLabels, config.batchSize): 298 | loss, acc, prec, recall, f_beta = trainStep(batchTrain[0], batchTrain[1]) 299 | 300 | currentStep = tf.train.global_step(sess, globalStep) 301 | print("train: step: {}, loss: {}, acc: {}, recall: {}, precision: {}, f_beta: {}".format( 302 | currentStep, loss, acc, recall, prec, f_beta)) 303 | if currentStep % config.training.evaluateEvery == 0: 304 | print("\nEvaluation:") 305 | losses = [] 306 | accs = [] 307 | f_betas = [] 308 | precisions = [] 309 | recalls = [] 310 | 311 | for batchEval in nextBatch(evalReviews, evalLabels, config.batchSize): 312 | loss, acc, precision, recall, f_beta = devStep(batchEval[0], batchEval[1]) 313 | losses.append(loss) 314 | accs.append(acc) 315 | f_betas.append(f_beta) 316 | precisions.append(precision) 317 | recalls.append(recall) 318 | 319 | time_str = datetime.datetime.now().isoformat() 320 | print("{}, step: {}, loss: {}, acc: {},precision: {}, recall: {}, f_beta: {}".format(time_str, 321 | currentStep, 322 | mean(losses), 323 | mean(accs), 324 | mean( 325 | precisions), 326 | mean(recalls), 327 | mean(f_betas))) 328 | 329 | if currentStep % config.training.checkpointEvery == 0: 330 | # 保存模型的另一种方法，保存checkpoint文件 331 | path = saver.save(sess, "../model/textCNN/model/my-model", global_step=currentStep) 332 | print("Saved model checkpoint to {}\n".format(path)) 333 | 334 | inputs = {"inputX": tf.saved_model.utils.build_tensor_info(cnn.inputX), 335 | "keepProb": tf.saved_model.utils.build_tensor_info(cnn.dropoutKeepProb)} 336 | 337 | outputs = {"predictions": tf.saved_model.utils.build_tensor_info(cnn.predictions)} 338 | 339 | prediction_signature = tf.saved_model.signature_def_utils.build_signature_def(inputs=inputs, outputs=outputs, 340 | method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME) 341 | legacy_init_op = tf.group(tf.tables_initializer(), name="legacy_init_op") 342 | builder.add_meta_graph_and_variables(sess, [tf.saved_model.tag_constants.SERVING], 343 | signature_def_map={"predict": prediction_signature}, 344 | legacy_init_op=legacy_init_op) 345 | 346 | builder.save() -------------------------------------------------------------------------------- /textCNN/parameter_config.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | #需要的所有导入包，存放留用，转换到jupyter后直接使用 3 | # 1 配置训练参数 4 | class TrainingConfig(object): 5 | epoches = 5 6 | evaluateEvery = 100 7 | checkpointEvery = 100 8 | learningRate = 0.001 9 | 10 | class ModelConfig(object): 11 | embeddingSize = 200 12 | numFilters = 128 13 | filterSizes = [2, 3, 4, 5] 14 | dropoutKeepProb = 0.5 15 | l2RegLambda = 0.0 16 | 17 | #上面两个都放到Config一个类中，为了后期调用方便 18 | class Config(object): 19 | sequenceLength = 200 # 取了所有序列长度的均值 20 | batchSize = 128 21 | dataSource = "../data/preProcess/labeledTrain.csv" 22 | stopWordSource = "../data/english" 23 | numClasses = 1 # 二分类设置为1，多分类设置为类别的数目 24 | rate = 0.8 # 训练集的比例 25 | training = TrainingConfig() 26 | model = ModelConfig() 27 | # 实例化配置参数对象 28 | # config = Config() -------------------------------------------------------------------------------- /textCNN/predict.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import os 3 | import csv 4 | import time 5 | import datetime 6 | import random 7 | import json 8 | from collections import Counter 9 | from math import sqrt 10 | import gensim 11 | import pandas as pd 12 | import numpy as np 13 | import tensorflow as tf 14 | from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score 15 | import parameter_config 16 | config =parameter_config.Config() 17 | 18 | #7预测代码 19 | x = "this movie is full of references like mad max ii the wild one and many others the ladybug´s face it´s a clear reference or tribute to peter lorre this movie is a masterpiece we´ll talk much more about in the future" 20 | # x = "his movie is the same as the third level movie. There's no place to look good" 21 | # x = "This film is not good" #最终反馈为0 22 | # x = "This film is bad" #最终反馈为0 23 | 24 | # 注：下面两个词典要保证和当前加载的模型对应的词典是一致的 25 | with open("../data/wordJson/word2idx.json", "r", encoding="utf-8") as f: 26 | word2idx = json.load(f) 27 | with open("../data/wordJson/label2idx.json", "r", encoding="utf-8") as f: #label2idx.json内容{"0": 0, "1": 1} 28 | label2idx = json.load(f) 29 | idx2label = {value: key for key, value in label2idx.items()} 30 | 31 | #x 的处理，变成模型能识别的向量xIds 32 | xIds = [word2idx.get(item, word2idx["UNK"]) for item in x.split(" ")] #返回x对应的向量 33 | if len(xIds) >= config.sequenceLength: #xIds 句子单词个数是否超过了sequenceLength（200） 34 | xIds = xIds[:config.sequenceLength] 35 | print("ddd",xIds) 36 | else: 37 | xIds = xIds + [word2idx["PAD"]] * (config.sequenceLength - len(xIds)) 38 | print("xxx", xIds) 39 | 40 | graph = tf.Graph() 41 | with graph.as_default(): 42 | gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) 43 | session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False, gpu_options=gpu_options) 44 | sess = tf.Session(config=session_conf) 45 | 46 | with sess.as_default(): 47 | # 恢复模型 48 | checkpoint_file = tf.train.latest_checkpoint("../model/textCNN/model/") 49 | saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file)) 50 | saver.restore(sess, checkpoint_file) 51 | 52 | # 获得需要喂给模型的参数，输出的结果依赖的输入值 53 | inputX = graph.get_operation_by_name("inputX").outputs[0] 54 | dropoutKeepProb = graph.get_operation_by_name("dropoutKeepProb").outputs[0] 55 | 56 | # 获得输出的结果 57 | predictions = graph.get_tensor_by_name("output/predictions:0") # mode_structure中的定义 58 | pred = sess.run(predictions, feed_dict={inputX: [xIds], dropoutKeepProb: 1.0})[0] 59 | 60 | # print(pred) 61 | pred = [idx2label[item] for item in pred] 62 | print(pred) -------------------------------------------------------------------------------- /word2vec/word2Vec.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yifanhunter/NLP_textClassifier/fc1d496e669e49fa0bf33f034aa00a38c345f5ea/word2vec/word2Vec.bin -------------------------------------------------------------------------------- /word2vec/word2vec.py: -------------------------------------------------------------------------------- 1 | # Author:yifan 2 | import logging 3 | import gensim 4 | from gensim.models import word2vec 5 | # 设置输出日志 6 | logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) 7 | 8 | # 直接用gemsim提供的API去读取txt文件，读取文件的API有LineSentence 和 Text8Corpus, PathLineSentences等。 9 | sentences = word2vec.LineSentence("../data/preProcess/wordEmbdiing.txt") 10 | # a = list(sentences) 11 | # print(len(a)) 12 | 13 | # 训练模型，词向量的长度设置为200， 迭代次数为8，采用skip-gram模型，模型保存为bin格式 14 | model = gensim.models.Word2Vec(sentences, size=200, sg=1, iter=8) 15 | model.wv.save_word2vec_format("./word2Vec" + ".bin", binary=True) 16 | 17 | # 加载bin格式的模型 18 | word2Vec = gensim.models.KeyedVectors.load_word2vec_format("word2Vec.bin",binary=True) -------------------------------------------------------------------------------- /word2vec/数据提取.py: -------------------------------------------------------------------------------- 1 | # 数据集为IMDB 电影影评，总共有三个数据文件，在/data/rawData目录下， 2 | # 包括unlabeledTrainData.tsv，labeledTrainData.tsv，testData.tsv。 3 | # 在进行文本分类时需要有标签的数据（labeledTrainData），但是在训练word2vec词向量模型（无监督学习）时可以将无标签的数据一起用上。 4 | import pandas as pd 5 | from bs4 import BeautifulSoup 6 | 7 | # IMDB 电影影评属于英文文本，本序列主要是文本分类的模型介绍，因此数据预处理比较简单， 8 | # 只去除了各种标点符号，HTML标签，小写化等。代码如下： 9 | 10 | with open("../data/rawData/unlabeledTrainData.tsv", "r", encoding='UTF-8') as f: 11 | unlabeledTrain = [line.strip().split("\t") for line in f.readlines() if len(line.strip().split("\t")) == 2] 12 | 13 | with open("../data/rawData/labeledTrainData.tsv", "r", encoding='UTF-8') as f: 14 | labeledTrain = [line.strip().split("\t") for line in f.readlines() if len(line.strip().split("\t")) == 3] 15 | 16 | unlabel = pd.DataFrame(unlabeledTrain[1:], columns=unlabeledTrain[0]) 17 | label = pd.DataFrame(labeledTrain[1:], columns=labeledTrain[0]) #多一列数据 sentiment （0/1） 18 | # print("```````````") 19 | # print(unlabeledTrain) 20 | # print("========") 21 | # print(unlabel) 22 | # print("________") 23 | # print(label) 24 | 25 | def cleanReview(subject): 26 | # 数据处理函数 27 | beau = BeautifulSoup(subject) 28 | newSubject = beau.get_text() 29 | newSubject = newSubject.replace("\\", "").replace("\'", "").replace('/', '').replace('"', '').replace(',', 30 | '').replace( 31 | '.', '').replace('?', '').replace('(', '').replace(')', '') 32 | newSubject = newSubject.strip().split(" ") 33 | newSubject = [word.lower() for word in newSubject] 34 | newSubject = " ".join(newSubject) 35 | return newSubject 36 | 37 | unlabel["review"] = unlabel["review"].apply(cleanReview) 38 | label["review"] = label["review"].apply(cleanReview) 39 | 40 | # 将有标签的数据和无标签的数据合并 41 | newDf = pd.concat([unlabel["review"], label["review"]], axis=0) 42 | # 保存成txt文件 43 | newDf.to_csv("../data/preProcess/wordEmbdiing.txt", index=False) --------------------------------------------------------------------------------