├── .gitignore
├── BasicTokenizer.java
├── Demo.java
├── FullTokenizer.java
├── LICENSE
├── Preprocess.java
├── README.md
└── WordpieceTokenizer.java


/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | *.egg-info/
 24 | .installed.cfg
 25 | *.egg
 26 | MANIFEST
 27 | 
 28 | # PyInstaller
 29 | #  Usually these files are written by a python script from a template
 30 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 31 | *.manifest
 32 | *.spec
 33 | 
 34 | # Installer logs
 35 | pip-log.txt
 36 | pip-delete-this-directory.txt
 37 | 
 38 | # Unit test / coverage reports
 39 | htmlcov/
 40 | .tox/
 41 | .coverage
 42 | .coverage.*
 43 | .cache
 44 | nosetests.xml
 45 | coverage.xml
 46 | *.cover
 47 | .hypothesis/
 48 | .pytest_cache/
 49 | 
 50 | # Translations
 51 | *.mo
 52 | *.pot
 53 | 
 54 | # Django stuff:
 55 | *.log
 56 | local_settings.py
 57 | db.sqlite3
 58 | 
 59 | # Flask stuff:
 60 | instance/
 61 | .webassets-cache
 62 | 
 63 | # Scrapy stuff:
 64 | .scrapy
 65 | 
 66 | # Sphinx documentation
 67 | docs/_build/
 68 | 
 69 | # PyBuilder
 70 | target/
 71 | 
 72 | # Jupyter Notebook
 73 | .ipynb_checkpoints
 74 | 
 75 | # pyenv
 76 | .python-version
 77 | 
 78 | # celery beat schedule file
 79 | celerybeat-schedule
 80 | 
 81 | # SageMath parsed files
 82 | *.sage.py
 83 | 
 84 | # Environments
 85 | .env
 86 | .venv
 87 | env/
 88 | venv/
 89 | ENV/
 90 | env.bak/
 91 | venv.bak/
 92 | 
 93 | # Spyder project settings
 94 | .spyderproject
 95 | .spyproject
 96 | 
 97 | # Rope project settings
 98 | .ropeproject
 99 | 
100 | # mkdocs documentation
101 | /site
102 | 
103 | # mypy
104 | .mypy_cache/
105 | 


--------------------------------------------------------------------------------
/BasicTokenizer.java:
--------------------------------------------------------------------------------
  1 | package bert;
  2 | 
  3 | import java.util.ArrayList;
  4 | import java.util.Arrays;
  5 | import java.util.List;
  6 | import java.util.regex.Matcher;
  7 | import java.util.regex.Pattern;
  8 | 
  9 | public class BasicTokenizer {
 10 | 
 11 |     public List<String> tokenize(String text) {
 12 | 
 13 |         String cleanText = cleanText(text);
 14 | 
 15 |         String chineseTokens = tokenizeChineseChars(cleanText);
 16 | 
 17 |         List<String> origTokens = whiteSpaceTokenize(chineseTokens);
 18 | 
 19 |         String str = "";
 20 |         for (String token : origTokens) {
 21 | 
 22 |             List<String> list = runSplitOnPunc(token);
 23 |             for (int i = 0; i < list.size(); i++) {
 24 |                 str += list.get(i) + " ";
 25 |             }
 26 |         }
 27 | 
 28 |         List<String> resTokens = whiteSpaceTokenize(str);
 29 | 
 30 | 
 31 |         return resTokens;
 32 |     }
 33 | 
 34 |     private List<String> runSplitOnPunc(String token) {
 35 |         List<List<Character>> result = new ArrayList<List<Character>>();
 36 | 
 37 |         int length = token.length();
 38 |         int i = 0;
 39 |         boolean startNewWord = true;
 40 |         while (i < length) {
 41 |             char c = token.charAt(i);
 42 |             if (isPunctuation(c)) {
 43 |                 List<Character> list = Arrays.asList(c);
 44 |                 result.add(list);
 45 |                 startNewWord = true;
 46 |             } else {
 47 |                 if (startNewWord) {
 48 |                     result.add(new ArrayList<Character>());
 49 |                 }
 50 |                 startNewWord = false;
 51 |                 result.get(result.size() - 1).add(c);
 52 |             }
 53 |             i += 1;
 54 |         }
 55 | 
 56 |         List<String> res = new ArrayList<String>();
 57 |         for (int j = 0; j < result.size(); j++) {
 58 |             String str = "";
 59 |             for (int k = 0; k < result.get(j).size(); k++) {
 60 |                 str += result.get(j).get(k);
 61 |             }
 62 |             res.add(str);
 63 |         }
 64 |         return res;
 65 |     }
 66 | 
 67 |     private boolean isPunctuation(char c) {
 68 |         if ((c >= 33 && c <= 47) || (c >= 58 && c <= 64) ||
 69 |                 (c >= 91 && c <= 96) || (c >= 123 && c <= 126)) {
 70 |             return true;
 71 |         }
 72 | 
 73 |         if (c == '“' || c == '”' || c == '、' ||
 74 |                 c == '《' || c == '》' || c == '。' ||
 75 |                 c == '；' || c == '【' || c == '】') {
 76 |             return true;
 77 |         }
 78 |         return false;
 79 |     }
 80 | 
 81 |     private List<String> whiteSpaceTokenize(String text) {
 82 |         List<String> result = new ArrayList<String>();
 83 | 
 84 |         text = text.trim();
 85 |         if (null == text) {
 86 |             return result;
 87 |         }
 88 |         String[] tokens = text.split(" ");
 89 |         result = Arrays.asList(tokens);
 90 | 
 91 |         return result;
 92 |     }
 93 | 
 94 |     private String tokenizeChineseChars(String cleanText) {
 95 |         StringBuffer outStrBuf = new StringBuffer();
 96 | 
 97 |         for (int i = 0; i < cleanText.length(); i++) {
 98 |             char c = cleanText.charAt(i);
 99 |             if (isChineseChar(c)) {
100 |                 outStrBuf.append(" ");
101 |                 outStrBuf.append(c);
102 |                 outStrBuf.append(" ");
103 |             } else {
104 |                 outStrBuf.append(c);
105 |             }
106 |         }
107 | 
108 |         return outStrBuf.toString();
109 |     }
110 | 
111 |     private boolean isChineseChar(char c) {
112 |         /*
113 |                 if ((cp >= 0x4E00 && cp <= 0x9FFF) ||
114 |                         (cp >= 0x3400 && cp <= 0x4DBF) ||
115 |                         (cp >= 0x20000 && cp <= 0x2A6DF) ||
116 |                         (cp >= 0x2A700 && cp <= 0x2B73F) ||
117 |                         (cp >= 0x2B740 && cp <= 0x2B81F) ||
118 |                         (cp >= 0x2B820 && cp <= 0x2CEAF) ||
119 |                         (cp >= 0xF900 && cp <= 0xFAFF) ||
120 |                         (cp >= 0x2F800 && cp <= 0x2FA1F)){
121 |                     return true;
122 |                 }
123 |         */
124 |         String s = String.valueOf(c);
125 |         String  regex = "[\u4e00-\u9fa5]";
126 |         Pattern p = Pattern.compile(regex);
127 | 
128 |         Matcher m = p.matcher(s);
129 |         return m.matches();
130 |     }
131 | 
132 |     private String cleanText(String text) {
133 |         StringBuffer outStrBuf = new StringBuffer("");
134 | 
135 |         for (int i = 0; i < text.length(); i++) {
136 |             char c = text.charAt(i);
137 |             //if(c == 0 || c == 0xfffd || isControl(c)){
138 |             //  continue;
139 |             //}
140 | 
141 |             if (isWhiteSpace(c)) {
142 |                 outStrBuf.append(" ");
143 |             } else {
144 |                 outStrBuf.append(c);
145 |             }
146 |         }
147 |         return outStrBuf.toString();
148 |     }
149 | 
150 |     private boolean isControl(char c) {
151 |         if (c == '\t' || c == '\n' || c == '\r') {
152 |             return false;
153 |         }
154 |         return true;
155 |     }
156 | 
157 |     private boolean isWhiteSpace(char c) {
158 |         if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
159 |             return true;
160 |         }
161 |         return false;
162 |     }
163 | }
164 | 


--------------------------------------------------------------------------------
/Demo.java:
--------------------------------------------------------------------------------
 1 | package bert;
 2 | 
 3 | import java.io.*;
 4 | import java.util.*;
 5 | 
 6 | public class Demo {
 7 | 
 8 |     public static void main(String[] args) {
 9 |         String vocab_path = "F:\\0_codes\\java\\vocab_bert-base-chinese.txt";
10 |         boolean has_chinese = true;
11 |         boolean do_lower = false;
12 |         int maxSeqLength = 64;
13 | 
14 |         FullTokenizer fullTokenizer = new FullTokenizer(vocab_path, has_chinese, do_lower);
15 | 
16 |         String query = "这是BERT tokenizer。 I'm working hard for programing GPU CPU";
17 |         // List<String> tokensQuery = fullTokenizer.tokenize(query);
18 |         // TokenIds e = fullTokenizer.getTokenIdsSingle(tokensQuery, maxSeqLength);
19 | 
20 |         TokenIds e = fullTokenizer.TokenlizeSingle(query, maxSeqLength);
21 |         System.out.println("print token ids of query: " + query);
22 |         System.out.println("input_ids " + e.getInputIds());
23 |         System.out.println("input_mask " + e.getInputMask());
24 |         System.out.println("segment_ids " + e.getSegmentIds());
25 | 
26 |         // demo of pairs
27 |         List<String> docs = new ArrayList<String>();
28 |         docs.add("求七公主漫画1-52全集给我发一下谢谢");
29 |         docs.add("您也可以留一下邮箱的，QQ邮箱也是可以的。？");
30 | 
31 |         List<TokenIds> tokenIds = fullTokenizer.TokenlizeMultiPairs(query, docs, maxSeqLength);
32 |         System.out.println("print token ids of pairs");
33 |         System.out.println("input_ids " + tokenIds.get(0).getInputIds());
34 |         System.out.println("input_mask " + tokenIds.get(0).getInputMask());
35 |         System.out.println("segment_ids " + tokenIds.get(0).getSegmentIds());
36 | 
37 |     }
38 | 
39 | }


--------------------------------------------------------------------------------
/FullTokenizer.java:
--------------------------------------------------------------------------------
  1 | package bert;
  2 | 
  3 | import java.util.ArrayList;
  4 | import java.util.List;
  5 | import java.util.Map;
  6 | 
  7 | public class FullTokenizer {
  8 |     private Map<String, Integer> vocab;
  9 |     private BasicTokenizer basicTokenizer;
 10 |     private WordpieceTokenizer wordpieceTokenizer;
 11 |     private Preprocess preprocesser;
 12 |     boolean has_chinese;
 13 |     boolean do_lower;
 14 | 
 15 |     public FullTokenizer(String filePath, boolean has_chinese, boolean do_lower) {
 16 |         this.has_chinese = has_chinese;
 17 |         this.do_lower = do_lower;
 18 |         this.preprocesser = new Preprocess();
 19 |         this.vocab = this.preprocesser.load(filePath);
 20 | 
 21 |         this.basicTokenizer = new BasicTokenizer();
 22 |         this.wordpieceTokenizer = new WordpieceTokenizer(vocab);
 23 |     }
 24 | 
 25 |     public List<String> tokenize(String text) {
 26 |         if (has_chinese) {
 27 |             text = preprocesser.full2HalfChange(text);
 28 |         }
 29 |         if (do_lower) {
 30 |             text = text.toLowerCase();
 31 |         }
 32 |         this.do_lower = do_lower;
 33 |         List<String> splitTopkens = new ArrayList<String>();
 34 | 
 35 |         for (String token : basicTokenizer.tokenize(text)) {
 36 |             for (String subToken : wordpieceTokenizer.tokenize(token)) {
 37 |                 splitTopkens.add(subToken);
 38 |             }
 39 |         }
 40 |         return splitTopkens;
 41 |     }
 42 | 
 43 |     public List<Integer> convertTokensToIds(List<String> tokens) {
 44 |         List<Integer> outputIds = new ArrayList<Integer>();
 45 |         for (String token : tokens) {
 46 |             outputIds.add(this.vocab.get(token));
 47 |         }
 48 |         return outputIds;
 49 |     }
 50 | 
 51 |     // 单句映射id
 52 |     public TokenIds getTokenIdsSingle(List<String> tokensQuery, int maxSeqLength) {
 53 |         while (true) {
 54 |             int totalLength = tokensQuery.size();
 55 |             if (totalLength <= maxSeqLength - 2) {
 56 |                 break;
 57 |             } else {
 58 |                 tokensQuery.remove(tokensQuery.size() - 1);
 59 |             }
 60 |         }
 61 | 
 62 |         List<String> tokens = new ArrayList<String>();
 63 |         List<Integer> segmentIds = new ArrayList<Integer>();
 64 |         tokens.add("[CLS]");
 65 |         segmentIds.add(0);
 66 |         for (String token : tokensQuery) {
 67 |             tokens.add(token);
 68 |             segmentIds.add(0);
 69 |         }
 70 |         tokens.add("[SEP]");
 71 |         segmentIds.add(0);
 72 | 
 73 |         List<Integer> inputIds = convertTokensToIds(tokens);
 74 |         List<Integer> inputMask = new ArrayList<Integer>();
 75 | 
 76 |         for (int i = 0; i < inputIds.size(); i++) {
 77 |             inputMask.add(1);
 78 |         }
 79 | 
 80 |         while (inputIds.size() < maxSeqLength) {
 81 |             inputIds.add(0);
 82 |             inputMask.add(0);
 83 |             segmentIds.add(0);
 84 |         }
 85 | 
 86 |         return new TokenIds(inputIds, inputMask, segmentIds);
 87 |     }
 88 | 
 89 |     // 句对映射id
 90 |     public TokenIds getTokenIdsPair(List<String> tokensQuery, List<String> tokensDoc, int maxSeqLength) {
 91 |         while (true) {
 92 |             int totalLength = tokensQuery.size() + tokensDoc.size();
 93 |             if (totalLength <= maxSeqLength - 3) {
 94 |                 break;
 95 |             }
 96 |             if (tokensQuery.size() > tokensDoc.size()) {
 97 |                 tokensQuery.remove(tokensQuery.size() - 1);
 98 |             } else {
 99 |                 tokensDoc.remove(tokensDoc.size() - 1);
100 |             }
101 |         }
102 | 
103 |         List<String> tokens = new ArrayList<String>();
104 |         List<Integer> segmentIds = new ArrayList<Integer>();
105 |         tokens.add("[CLS]");
106 |         segmentIds.add(0);
107 |         for (String token : tokensQuery) {
108 |             tokens.add(token);
109 |             segmentIds.add(0);
110 |         }
111 |         tokens.add("[SEP]");
112 |         segmentIds.add(0);
113 | 
114 |         for (String token : tokensDoc) {
115 |             tokens.add(token);
116 |             segmentIds.add(1);
117 |         }
118 |         tokens.add("[SEP]");
119 |         segmentIds.add(1);
120 | 
121 |         List<Integer> inputIds = convertTokensToIds(tokens);
122 |         List<Integer> inputMask = new ArrayList<Integer>();
123 | 
124 |         for (int i = 0; i < inputIds.size(); i++) {
125 |             inputMask.add(1);
126 |         }
127 | 
128 |         while (inputIds.size() < maxSeqLength) {
129 |             inputIds.add(0);
130 |             inputMask.add(0);
131 |             segmentIds.add(0);
132 |         }
133 |         return new TokenIds(inputIds, inputMask, segmentIds);
134 |     }
135 | 
136 |     public TokenIds TokenlizeSingle(String query, int maxSeqLength) {
137 |         List<String> tokensQuery = tokenize(query);
138 |         return  getTokenIdsSingle(tokensQuery, maxSeqLength);
139 |     }
140 | 
141 |     public List<TokenIds> TokenlizeMultiPairs(String query, List<String> docs, int maxSeqLength) {
142 |         List<String> tokensQuery = tokenize(query);
143 | 
144 |         List<TokenIds> tokenIds = new ArrayList<TokenIds>();
145 |         for (String doc : docs) {
146 |             List<String> tokensDoc = tokenize(doc);
147 |             TokenIds e = getTokenIdsPair(tokensQuery, tokensDoc, maxSeqLength);
148 |             tokenIds.add(e);
149 |         }
150 |         return tokenIds;
151 |     }
152 | }
153 | 
154 | class TokenIds {
155 |     private List<Integer> inputIds;
156 |     private List<Integer> inputMask;
157 |     private List<Integer> segmentIds;
158 | 
159 |     public TokenIds(List<Integer> inputIds, List<Integer> inputMask, List<Integer> segmentIds) {
160 |         this.inputIds = inputIds;
161 |         this.inputMask = inputMask;
162 |         this.segmentIds = segmentIds;
163 |     }
164 | 
165 |     public List<Integer> getInputIds() {
166 |         return inputIds;
167 |     }
168 | 
169 |     public List<Integer> getInputMask() {
170 |         return inputMask;
171 |     }
172 | 
173 |     public void setInputMask(List<Integer> inputMask) {
174 |         this.inputMask = inputMask;
175 |     }
176 | 
177 |     public List<Integer> getSegmentIds() {
178 |         return segmentIds;
179 |     }
180 | 
181 |     public void setSegmentIds(List<Integer> segmentIds) {
182 |         this.segmentIds = segmentIds;
183 |     }
184 | 
185 |     public void setInputIds(List<Integer> inputIds) {
186 |         this.inputIds = inputIds;
187 |     }
188 | }


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/Preprocess.java:
--------------------------------------------------------------------------------
 1 | package bert;
 2 | 
 3 | import java.io.*;
 4 | import java.util.*;
 5 | 
 6 | public class Preprocess {
 7 | 
 8 |     public Preprocess() {
 9 |     }
10 | 
11 |     public Map<String, Integer> load(String filePath) {
12 |         Map<String, Integer> map = new HashMap<String, Integer>();
13 | 
14 |         /* 读取数据 */
15 |         try {
16 |             BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(filePath)),
17 |                                                    "UTF-8"));
18 |             int index = 0;
19 |             String token = null;
20 |             while ((token = br.readLine()) != null) {
21 |                 map.put(token, index);
22 |                 index += 1;
23 |             }
24 |             br.close();
25 |         } catch (Exception e) {
26 |             System.err.println("read errors :" + e);
27 |         }
28 |         return map;
29 |     }
30 | 
31 |     // 全角转半角
32 |     public String full2HalfChange(String QJstr) {
33 | 
34 |         StringBuffer outStrBuf = new StringBuffer();
35 | 
36 |         String Tstr = "";
37 | 
38 |         byte[] b = null;
39 |         try {
40 |             for (int i = 0; i < QJstr.length(); i++) {
41 | 
42 |                 Tstr = QJstr.substring(i, i + 1);
43 | 
44 |                 if (Tstr.equals("　")) {
45 |                     outStrBuf.append(" ");
46 |                     continue;
47 |                 }
48 | 
49 |                 b = Tstr.getBytes("unicode");
50 | 
51 |                 // 得到 unicode 字节数据
52 | 
53 |                 if (b[2] == -1) {
54 | 
55 |                     // 表示全角？
56 | 
57 |                     b[3] = (byte) (b[3] + 32);
58 | 
59 |                     b[2] = 0;
60 | 
61 |                     outStrBuf.append(new String(b, "unicode"));
62 | 
63 |                 } else {
64 |                     outStrBuf.append(Tstr);
65 |                 }
66 |             }
67 |         } catch (Exception e) {
68 |             e.printStackTrace();
69 |             return QJstr;
70 |         }
71 |         return outStrBuf.toString();
72 |     }
73 | }
74 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | This is a java version of Chinese tokenization descried in BERT, including basic tokenization and wordpiece tokenization.
 2 | 
 3 | ## Motivation
 4 | 
 5 | In production, we usually deploy the BERT related model by tensorflow serving for high performance and flexibility. However, our application may not developed by python. Hence, we have to rewrite the tokenization module.
 6 | 
 7 | ## Usage
 8 | 
 9 | Just run `Demo.java`, you can get result. Now, it support single and pair sentence both.
10 | 
11 | Moreover, for Chinese natural language processing, we add **full turn to half angle** and **uppercase to lowercase** operation.
12 | 
13 | ## Reporting issues
14 | 
15 | Please let me know, if you encounter any problems.
16 | 
17 | 


--------------------------------------------------------------------------------
/WordpieceTokenizer.java:
--------------------------------------------------------------------------------
 1 | package bert;
 2 | 
 3 | import java.util.ArrayList;
 4 | import java.util.Arrays;
 5 | import java.util.List;
 6 | import java.util.Map;
 7 | 
 8 | public class WordpieceTokenizer {
 9 | 
10 |     private Map<String, Integer> vocab;
11 |     private String unkToken = "[UNK]";
12 |     private int maxInputCharsPerWord = 200;
13 | 
14 |     public WordpieceTokenizer(Map<String, Integer> vocab) {
15 |         this.vocab = vocab;
16 |     }
17 | 
18 |     /*
19 |         For example:
20 |         input = "unaffable"
21 |         output = ["un", "##aff", "##able"]
22 |     */
23 |     public List<String> tokenize(String text) {
24 | 
25 |         List<String> tokens = whiteSpaceTokenize(text);
26 | 
27 |         List<String> outputTokens = new ArrayList<String>();
28 |         for (String token : tokens) {
29 |             int length = token.length();
30 |             if (length > this.maxInputCharsPerWord) {
31 |                 outputTokens.add(this.unkToken);
32 |                 continue;
33 |             }
34 | 
35 |             boolean isBad = false;
36 |             int start = 0;
37 |             List<String> subTokens = new ArrayList<String>();
38 | 
39 |             while (start < length) {
40 |                 int end = length;
41 |                 String curSubStr = null;
42 |                 while (start < end) {
43 |                     String subStr = token.substring(start, end);
44 |                     if (start > 0) {
45 |                         subStr = "##" + subStr;
46 |                     }
47 |                     if (this.vocab.containsKey(subStr)) {
48 |                         curSubStr = subStr;
49 |                         break;
50 |                     }
51 |                     end -= 1;
52 |                 }
53 |                 if (null == curSubStr) {
54 |                     isBad = true;
55 |                     break;
56 |                 }
57 |                 subTokens.add(curSubStr);
58 |                 start = end;
59 |             }
60 | 
61 |             if (isBad) {
62 |                 outputTokens.add(this.unkToken);
63 |             } else {
64 |                 outputTokens.addAll(subTokens);
65 |             }
66 | 
67 |         }
68 |         return outputTokens;
69 |     }
70 | 
71 |     private List<String> whiteSpaceTokenize(String text) {
72 |         List<String> result = new ArrayList<String>();
73 | 
74 |         text = text.trim();
75 |         if (null == text) {
76 |             return result;
77 |         }
78 |         String[] tokens = text.split(" ");
79 |         result = Arrays.asList(tokens);
80 | 
81 |         return result;
82 |     }
83 | 
84 | }
85 | 


--------------------------------------------------------------------------------