├── .gitignore ├── LICENSE ├── README.md ├── g2pc ├── __init__.py ├── cedict.pkl ├── crf100.bin └── g2pc.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![image](https://img.shields.io/pypi/v/g2pc.svg)](https://pypi.org/project/g2pC/) 2 | [![image](https://img.shields.io/pypi/l/g2pc.svg)](https://pypi.org/project/g2pC/) 3 | [![image](https://img.shields.io/pypi/pyversions/g2pc.svg)](https://pypi.org/project/g2pc/) 4 | # g2pC: A Context-aware Grapheme-to-Phoneme for Chinese 5 | There are several open source libraries of Chinese grapheme-to-phoneme 6 | conversion such as [python-pinyin](https://github.com/mozillazg/python-pinyin) or [xpinyin](https://github.com/lxneng/xpinyin). 7 | However, none of them seem to disambiguate Chinese polyphonic words like "行" 8 | ("xíng" (go, walk) vs. "háng" (line)) or "了" ("le" (completed action marker) 9 | vs. "liǎo" (finish, achieve)). Instead, they pick up the most frequent pronunciation. 10 | Although that may be a simple and economic strategy, machine learning techniques can be of help. 11 | We use CRF to determine the pronunciation of polyphonic words. In addition to the target word itself and its part-of-speech, which are tagged by pkuseg, its neighboring words are also featurized. 12 | ## Requirements 13 | * python >= 3.6 14 | * pkuseg 15 | * sklearn_crfsuite 16 | ## Installation 17 | ``` 18 | pip install g2pc 19 | ``` 20 | ## Main Features 21 | * Disambiguate polyphonic Chinese characters/words and return the most likely pinyin in the 22 | context using CRF implemented with [sklearn_crfsuite](https://github.com/TeamHG-Memex/sklearn-crfsuite). 23 | * By associating segmentation results provided by [pkuseg](https://arxiv.org/abs/1906.11455) with an open-source dictionary [CC-CEDICT](https://cc-cedict.org/wiki/), 24 | display the following comprehensive information. 25 | * word 26 | * part-of-speech 27 | * pinyin 28 | * descriptive pinyin: where Chinese tone change rules are applied 29 | * English meaning 30 | * traditional equivalent 31 | ## Algorithm (illustrated with an example) 32 | e.g., Input: 我写了几行代码。 (I wrote a few lines of codes.) 33 | * STEP 1. Segment input string using [pkuseg](https://arxiv.org/abs/1906.11455). 34 | * -> [('我', 'r'), ('写', 'v'), ('了', 'u'), ('几', 'm'), ('行', 'q'), ('代码', 'n'), ('。', 'w')] 35 | * STEP 2. Look up the [CC-CEDICT](https://cc-cedict.org/wiki/). Each token, a tuple, consists of 36 | word, pos, pronunciation candidates, meaning candidates, traditional character candidates. 37 | * -> [('我', 'r', ['wo3'], ['/I/me/my/'], ['我']),
38 | ('写', 'v', ['xie3'], ['/to write/'], ['寫']),
39 | ('了', 'u', ['le5', 'liao3', 'liao4'], [dal particle ..], ['了', '了', '瞭']),
40 | ('几', 'm', ['ji3', 'ji1'], ['/how much/..'], ['幾', '几']),
41 | ('行', 'q', ['xing2', 'hang2'], ['/to walk/.."], ['行', '行']),
42 | ('代码', 'n', ['dai4 ma3'], ['/code/'], ['代碼']),
43 | ('。', 'w', ['。'], [''], ['。'])] 44 | * STEP 3. For polyphonic words, we disambiguate them, using our pre-trained CRF model. 45 | * -> [('我', 'r', 'wo3', '/I/me/my/', '我'),
46 | ('写', 'v', 'xie3', '/to write/', '寫'),
47 | ('了', 'u', 'le5', '/(modal particle ..', '了'),
48 | ('几', 'm', 'ji3', '/how much/..', '幾'),
49 | ('行', 'q', 'hang2', "/row/..", '行'),
50 | ('代码', 'n', 'dai4 ma3', '/code/', '代碼'),
51 | ('。', 'w', '。', '。', '', '。')] 52 | 53 | * STEP 4. Tone change rules are applied. 54 | * -> [('我', 'r', 'wo3', 'wo2', '/I/me/my/', '我'),
55 | ('写', 'v', 'xie3', 'xie3', '/to write/', '寫'),
56 | ('了', 'u', 'le5', 'le5', '/(modal particle ..', '了'),
57 | ('几', 'm', 'ji3', 'ji3', '/how much/..', '幾'),
58 | ('行', 'q', 'hang2', 'hang2, "/row/..", '行'),
59 | ('代码', 'n', 'dai4 ma3', 'dai4 ma3', '/code/', '代碼'),
60 | ('。', 'w', '。', '。', '', '。')] 61 | ## Usage 62 | ``` 63 | >>> from g2pc import G2pC 64 | >>> g2p = G2pC() 65 | >>> g2p("一心一意") 66 | # This returns a list of tuples, each of which consists of 67 | # word, pos, pinyin, (tone changed) descriptive pinyin, English meaning, and equivanlent traditional character. 68 | [[('一心一意', 69 | 'i', 70 | 'yi1 xin1 yi1 yi4', 71 | 'yi4 xin1 yi2 yi4', 72 | "/concentrating one's thoughts and efforts/single-minded/bent on/intently/", 73 | '一心一意')] 74 | ``` 75 | ## Respectful comparison with other libraries 76 | ``` 77 | >>> text1 = "我写了几行代码。" # pay attention to the 行, which should be read as 'hang2', not 'xing2' 78 | >>> text2 = "来不了" # pay attention to the 了, which should be read as 'liao3', not 'le' 79 | # python-pinyin 80 | >>> pip install pypinyin 81 | >>> from pypinyin import pinyin 82 | >>> pinyin(text1) 83 | [['wǒ'], ['xiě'], ['le'], ['jǐ'], ['xíng'], ['dài'], ['mǎ'], ['。']] 84 | >>> pinyin(text2) 85 | [['lái'], ['bù'], ['le']] 86 | # xpinyin 87 | >>> pip install xpinyin 88 | >>> from xpinyin import Pinyin 89 | >>> p = Pinyin() 90 | >>> p.get_pinyin(text1, tone_marks="numbers") 91 | 'wo3-xie3-le5-ji1-xing2-dai4-ma3-。' 92 | >>> p.get_pinyin(text2, tone_marks="numbers") 93 | 'lai2-bu4-le5' 94 | ``` 95 | 96 | * Accuracy on internal test set (13,191 syllables) 97 | 98 | |Model|# Correct | # Incorrect | Acc. (%) | 99 | |--|--|--|--| 100 | |g2pC (0.9.9.3)| 13,033 | 158 | 98.80 | 101 | |pypinyin (0.35.3)|12,975| 216| 98.36| 102 | |xpinyin (0.5.6)|12,838 |353| 97.32| 103 | 104 | Accuracy 105 | ## Changelog 106 | ### 0.9.9.3 July 10, 2019 107 | * Refined the tone change rules. 108 | 109 | ### 0.9.9.2 July 10, 2019 110 | * Refined the `cedict.pkl`. 111 | 112 | ### 0.9.9.1 July 9, 2019 113 | * Fixed a bug of failing to find Chinese characters for names. (See [this](https://github.com/Kyubyong/g2pC/issues/3)) 114 | 115 | ### 0.9.6. July 7, 2019 116 | * Fixed a bug of failing to converting words not found in the dictionary. 117 | * Rearragned the `cedict.pkl`. 118 | * Refined the CRF model. 119 | * Added tone change rules. (See [this](https://github.com/Kyubyong/g2pC/issues/1)) 120 | ### 0.9.4. July 4, 2019 121 | * Initial launch 122 | ## References 123 | If you use our software for research, please cite: 124 | ``` 125 | @misc{gp2C2019, 126 | author = {Park, Kyubyong}, 127 | title = {g2pC}, 128 | year = {2019}, 129 | publisher = {GitHub}, 130 | journal = {GitHub repository}, 131 | howpublished = {\url{https://github.com/Kyubyong/g2pC}} 132 | } 133 | ``` 134 | -------------------------------------------------------------------------------- /g2pc/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | r"""g2pC 3 | """ 4 | from __future__ import absolute_import 5 | 6 | from .g2pc import G2pC -------------------------------------------------------------------------------- /g2pc/cedict.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyubyong/g2pC/869f36f0b1c23a48e90f7191af575b71976ce5bb/g2pc/cedict.pkl -------------------------------------------------------------------------------- /g2pc/crf100.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kyubyong/g2pC/869f36f0b1c23a48e90f7191af575b71976ce5bb/g2pc/crf100.bin -------------------------------------------------------------------------------- /g2pc/g2pc.py: -------------------------------------------------------------------------------- 1 | #-*- coding:utf8 -*- 2 | ''' 3 | g2pC: A context-aware grapheme-to-phoneme module for Chinese 4 | https://github.com/kyubyong/g2pC 5 | kbpark.linguist@gmail.com 6 | ''' 7 | import pickle 8 | import re 9 | import os 10 | import pkuseg 11 | from itertools import chain 12 | 13 | 14 | def convert_hanzi_string_to_number(string): 15 | return "/".join(str(ord(char)) for char in string) 16 | 17 | 18 | def word2features(sent, i): 19 | word = convert_hanzi_string_to_number(sent[i][0]) 20 | postag = sent[i][1] 21 | 22 | features = { 23 | 'bias': 1.0, 24 | 'word':word, 25 | 'postag': postag, 26 | } 27 | if i == 0: 28 | features['BOS'] = True 29 | else: 30 | if i > 0: 31 | word1 = convert_hanzi_string_to_number(sent[i-1][0]) 32 | postag1 = sent[i-1][1] 33 | features.update({ 34 | '-1:word': word1, 35 | '-1:postag': postag1, 36 | }) 37 | if i > 1: 38 | word1 = convert_hanzi_string_to_number(sent[i-2][0]) 39 | postag1 = sent[i-2][1] 40 | features.update({ 41 | '-2:word': word1, 42 | '-2:postag': postag1, 43 | }) 44 | if i > 2: 45 | word1 = convert_hanzi_string_to_number(sent[i-3][0]) 46 | postag1 = sent[i-3][1] 47 | features.update({ 48 | '-3:word': word1, 49 | '-3:postag': postag1, 50 | }) 51 | 52 | if i == len(sent)-1: 53 | features['EOS'] = True 54 | else: 55 | if i < len(sent)-1: 56 | word1 = convert_hanzi_string_to_number(sent[i+1][0]) 57 | postag1 = sent[i+1][1] 58 | features.update({ 59 | '+1:word': word1, 60 | '+1:postag': postag1, 61 | }) 62 | if i < len(sent)-2: 63 | word1 = convert_hanzi_string_to_number(sent[i+2][0]) 64 | postag1 = sent[i+2][1] 65 | features.update({ 66 | '+2:word': word1, 67 | '+2:postag': postag1, 68 | }) 69 | if i < len(sent)-3: 70 | word1 = convert_hanzi_string_to_number(sent[i+3][0]) 71 | postag1 = sent[i+3][1] 72 | features.update({ 73 | '+3:word': word1, 74 | '+3:postag': postag1, 75 | }) 76 | 77 | return features 78 | 79 | 80 | def sent2features(sent): 81 | return [word2features(sent, i) for i in range(len(sent))] 82 | 83 | 84 | def _tone_change(hanzis, pinyins): 85 | '''https://en.wikipedia.org/wiki/Standard_Chinese_phonology#Tone_sandhi 86 | ''' 87 | def tone33_to_23(pinyin): 88 | return re.sub("3( [^ ]+?3)", r"2\1", pinyin) 89 | 90 | # STEP 1. word-level 91 | ## Third tone change 92 | pinyins = [tone33_to_23(pinyin) for pinyin in pinyins] 93 | 94 | ## when it comes at the end of a multi-syllable word 95 | ##(regardless of the first tone of the next word), 96 | ## 一 is pronounced with first tone. 97 | _pinyins = [] 98 | for hanzi, pinyin in zip(hanzis, pinyins): 99 | if len(hanzi)>1 and hanzi[0]=="一": 100 | pinyin = re.sub("yi1$", "YI1", pinyin) 101 | _pinyins.append(pinyin) 102 | 103 | # STEP 2. phrase-level 104 | ## remove boundaries 105 | hanzis = "".join(hanzis) 106 | pinyins = " ".join(_pinyins) 107 | 108 | ## Third tone change 109 | pinyins = tone33_to_23(pinyins) 110 | pinyins = pinyins.split() 111 | 112 | hanzis_prev = "^" + hanzis[:-1] 113 | pinyins_prev = ["^"] + pinyins[:-1] 114 | 115 | hanzis_next = hanzis[1:] + "$" 116 | pinyins_next = pinyins[1:] + ["$"] 117 | 118 | _pinyins = [] 119 | for h, h_prev, h_next, p, p_prev, p_next in zip(hanzis, hanzis_prev, hanzis_next, pinyins, pinyins_prev, pinyins_next): 120 | if h == "一" and p == "yi1": 121 | if h_prev in "第初图表卷": 122 | p = "YI1" 123 | elif h_prev == "头" and h_next == "回": 124 | p = "YI1" 125 | elif h_prev == "末" and h_next == "次": 126 | p = "YI1" 127 | elif h_next in "号楼更等级一二三四五陆七八九十廿卅卌百皕千万亿": 128 | p = "YI1" 129 | elif h_prev == h_next: # 4. A一A -> A yi5 A 130 | p = "yi5" 131 | elif p_next[-1] == "4": # 一 + A4 -> yi2 A4 132 | p = "yi2" 133 | elif p_next[-1] in "123": # 一 + A{1,2,3} -> yi4 A{1,2,3} 134 | p = "yi4" 135 | if h == "不" and p=="bu4": 136 | if p_next[-1] == "4": # 不 + A4 -> bu2 A4 137 | p = "bu2" 138 | elif h_prev == h_next: # A不A -> A bu5 A 139 | p = "bu5" 140 | p = p.replace("YI", "yi") 141 | _pinyins.append(p) 142 | 143 | return _pinyins 144 | 145 | 146 | def tone_change(results): 147 | hanzis = [result[0] for result in results] 148 | pinyins = [result[2] for result in results] 149 | 150 | pinyins = _tone_change(hanzis, pinyins) 151 | 152 | # align 153 | rule_applied = [] 154 | for result in results: 155 | n_syls = len(result[2].split()) 156 | _pinyin = " ".join(pinyins[:n_syls]) 157 | pinyins = pinyins[n_syls:] 158 | result = (result[0], result[1], result[2], _pinyin, result[3], result[4]) 159 | rule_applied.append(result) 160 | return rule_applied 161 | 162 | 163 | class G2pC(object): 164 | def __init__(self): 165 | ''' 166 | self.cedict looks like: 167 | {行: {pron: [hang2, xing2], 168 | meaning: [/row/line, /to walk/to go], 169 | trad: [行, 行]} 170 | ''' 171 | self.seg = pkuseg.pkuseg(postag=True) 172 | self.cedict = pickle.load(open(os.path.dirname(os.path.abspath(__file__)) + '/cedict.pkl', 'rb')) 173 | self.crf = pickle.load(open(os.path.dirname(os.path.abspath(__file__)) + '/crf100.bin', 'rb')) 174 | 175 | def __call__(self, string): 176 | # fragment into sentences 177 | sents = re.sub("([！？。])", r"\1[SEP]", string) 178 | sents = sents.split("[SEP]") 179 | 180 | _sents = [] 181 | for sent in sents: 182 | if len(sent)==0: continue 183 | 184 | # STEP 1 185 | tokens = self.seg.cut(sent) 186 | 187 | # STEP 2 188 | analyzed = [] 189 | for word, pos in tokens: 190 | if word in self.cedict: 191 | features = self.cedict[word] 192 | prons = features["pron"] 193 | meanings = features["meaning"] 194 | trads = features["trad"] 195 | analyzed.append((word, pos, prons, meanings, trads)) 196 | else: 197 | for char in word: 198 | if char in self.cedict: 199 | features = self.cedict[char] 200 | prons = features["pron"] 201 | meanings = features["meaning"] 202 | trads = features["trad"] 203 | else: 204 | prons = [char] 205 | meanings = [""] 206 | trads = [char] 207 | analyzed.append((char, pos, prons, meanings, trads)) 208 | _sents.append(analyzed) 209 | 210 | # print("STEP1", tokens) 211 | # print("STEP2", analyzed) 212 | # STEP 3 213 | features = [sent2features(_sent) for _sent in _sents] 214 | preds = self.crf.predict(features) 215 | 216 | # concatenate sentences 217 | tokens = chain.from_iterable(_sents) 218 | preds = chain.from_iterable(preds) 219 | 220 | # determine pinyin 221 | ret = [] 222 | for (word, pos, prons, meanings, trads), p in zip(tokens, preds): 223 | # print(word, pos, prons, p) 224 | p = p.replace("-", " ") 225 | if p in prons: 226 | pinyin = p 227 | else: 228 | pinyin = prons[0] 229 | ind = prons.index(pinyin) 230 | meaning = meanings[ind] 231 | trad = trads[ind] 232 | ret.append((word, pos, pinyin, meaning, trad)) 233 | 234 | # print("STEP3", ret) 235 | 236 | # STEP 4 237 | ret = tone_change(ret) 238 | return ret 239 | 240 | 241 | if __name__ == "__main__": 242 | strings = ["有一次", "第一次", "十一二岁来到戏校", "同年十一月", "一九八二年英文版", "欧洲统一步伐", "吉林省一号工程", "一是选拔优秀干部"] 243 | g2p = G2pC() 244 | for string in strings: 245 | results = g2p(string) 246 | change = [each[3] for each in results] 247 | print(string, "/", "|".join(change)) 248 | 249 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", mode="r", encoding="utf-8") as fh: 4 | long_description = fh.read() 5 | 6 | REQUIRED_PACKAGES = [ 7 | 'pkuseg>=0.0.22', 8 | 'sklearn_crfsuite>=0.3.6', 9 | ] 10 | 11 | setuptools.setup( 12 | name="g2pC", 13 | version="0.9.9.3", 14 | author="Kyubyong Park", 15 | author_email="kbpark.linguist@gmail.com", 16 | description="g2pC: A Context-aware g2p module for Chinese", 17 | install_requires=REQUIRED_PACKAGES, 18 | license='Apache License 2.0', 19 | long_description=long_description, 20 | long_description_content_type="text/markdown", 21 | url="https://github.com/Kyubyong/g2pC", 22 | packages=setuptools.find_packages(), 23 | package_data={'g2pc': ['g2pc/cedict.pkl', 'g2pc/crf100.bin']}, 24 | python_requires=">=3.6", 25 | include_package_data=True, 26 | classifiers=[ 27 | 'Development Status :: 5 - Production/Stable', 28 | 'Intended Audience :: Developers', 29 | 'Intended Audience :: Science/Research', 30 | "License :: OSI Approved :: Apache Software License", 31 | "Operating System :: OS Independent", 32 | "Programming Language :: Python :: 3", 33 | 'Programming Language :: Python :: 3.6', 34 | 'Programming Language :: Python :: 3.7', 35 | ], 36 | ) --------------------------------------------------------------------------------