': 12, ...}
144 | Object.keys(hmmParams.word_prob).forEach((word_code) => {
145 | const weight = hmmParams.word_prob[word_code];
146 | const word_id = word_dict.pinyin_word[word_code];
147 |
148 | pred_dict.word_chars[word_id] = {
149 | word_id_: word_id,
150 | weight_: weight
151 | };
152 | });
153 |
154 | // =======================================================
155 | await asyncForEach(
156 | [
157 | {
158 | table: 'phrase_word',
159 | prop: 'word_chars',
160 | primaryKeys: ['word_id_'],
161 | update: (data, row) => {
162 | data.weight_ += row.weight_;
163 | }
164 | },
165 | {
166 | table: 'phrase_trans_prob',
167 | prop: 'trans_prob',
168 | primaryKeys: ['word_id_', 'prev_word_id_'],
169 | update: (data, row) => {
170 | data.value_ += row.value_;
171 | }
172 | }
173 | ],
174 | async ({ table, prop, primaryKeys, update }) => {
175 | const data = pred_dict[prop];
176 |
177 | (await phraseDictDB.all(`select * from ${table}`)).forEach((row) => {
178 | const code = primaryKeys.map((k) => row[k]).join(':');
179 |
180 | if (data[code]) {
181 | data[code].__exist__ = row;
182 | update(data[code], row);
183 | }
184 | });
185 |
186 | await saveToDB(phraseDictDB, table, data, true, primaryKeys);
187 | }
188 | );
189 | }
190 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/generate/sqlite/phrase/hmm/trans/index.mjs:
--------------------------------------------------------------------------------
1 | /* 生成 HMM 计算参数 */
2 | import {
3 | fromRootPath,
4 | appendLineToFile,
5 | getAllFiles,
6 | readJSONFromFile,
7 | readFile
8 | } from '#utils/utils.mjs';
9 | import { openDB, closeDB } from '#utils/sqlite.mjs';
10 | import { readWordsFromDB } from '../utils.mjs';
11 | import * as trans from './trans.mjs';
12 |
13 | // 样本文件。可试用样本如下：
14 | // - [已分词] https://raw.githubusercontent.com/InsaneLife/ChineseNLPCorpus/master/NER/MSRA/train1.txt
15 | // - [已分词] https://raw.githubusercontent.com/InsaneLife/ChineseNLPCorpus/master/NER/renMinRiBao/renmin.txt
16 | let phraseSampleFiles = [];
17 | let appendExistData = false;
18 |
19 | const args = process.argv.slice(2);
20 | for (let i = 0; i < args.length; i++) {
21 | const arg = args[i];
22 | if (arg == '-f') {
23 | phraseSampleFiles.push(args[++i]);
24 | } else if (arg == '-a') {
25 | appendExistData = true;
26 | }
27 | }
28 |
29 | if (!phraseSampleFiles) {
30 | console.log(
31 | 'Usage: npm run generate:sqlite:phrase:hmm:trans -- [-a] -f /file1 -f file2 ...'
32 | );
33 | console.log();
34 |
35 | process.exit(1);
36 | }
37 |
38 | // SQLite 字典库
39 | const wordDictSQLiteFile = fromRootPath('data', 'pinyin-word-dict.sqlite');
40 | // HMM 参数目录
41 | const transParamsDir = fromRootPath('data', 'hmm_params');
42 |
43 | console.log();
44 | console.log(`创建计算参数${appendExistData ? '（累积更新）' : ''} ...`);
45 | let wordDictDB = await openDB(wordDictSQLiteFile, true);
46 |
47 | let words;
48 | try {
49 | words = await readWordsFromDB(wordDictDB);
50 | } catch (e) {
51 | throw e;
52 | } finally {
53 | await closeDB(wordDictDB);
54 | }
55 |
56 | let transParams = appendExistData
57 | ? {
58 | word_prob: readJSONFromFile(transParamsDir + `/word_prob.json`),
59 | trans_prob: readJSONFromFile(transParamsDir + `/trans_prob.json`)
60 | }
61 | : null;
62 |
63 | getAllFiles(phraseSampleFiles).forEach((file) => {
64 | console.log(` - 分析文件: ${file} ...`);
65 |
66 | const sampleText = readFile(file);
67 |
68 | transParams = trans.countParams(sampleText, words, transParams);
69 | });
70 |
71 | Object.keys(transParams).forEach((name) => {
72 | appendLineToFile(
73 | transParamsDir + `/${name}.json`,
74 | JSON.stringify(transParams[name]),
75 | true
76 | );
77 | });
78 |
79 | console.log();
80 | console.log('Done');
81 | console.log();
82 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/generate/sqlite/phrase/hmm/trans/trans.mjs:
--------------------------------------------------------------------------------
1 | import { getPinyinTone, getPinyin } from '#utils/utils.mjs';
2 | import { countTrans, countWords } from '../utils.mjs';
3 |
4 | /**
5 | * HMM 参数计算
6 | * trans_prob - 汉字间转移概率
7 | */
8 | export function countParams(sampleText, words, existParams) {
9 | const clauses = extractClauses(sampleText, words);
10 |
11 | existParams = existParams || { word_prob: {}, trans_prob: {} };
12 |
13 | return {
14 | // 字的出现次数
15 | word_prob: countWords(clauses, existParams.word_prob),
16 | // 当前字为 EOS 且其前序为 BOS 的转移次数即为 训练的句子总数，
17 | // 而各个包含 BOS 前序的字即为句首字，且其出现次数即为 BOS 的值
18 | trans_prob: countTrans(clauses, existParams.trans_prob)
19 | };
20 | }
21 |
22 | // extractClauses('迈向/v 充满/v 希望/n 的/u 新/a 世纪/n', {
23 | // 力: true,
24 | // 争: true,
25 | // 达: true,
26 | // 到: true,
27 | // 万: true,
28 | // 标: true,
29 | // 准: true,
30 | // 箱: true
31 | // });
32 | /** 拆分样本数据，按汉字短句返回 */
33 | export function extractClauses(sampleText, words) {
34 | const clauses = getClauses(sampleText, words);
35 |
36 | const result = [];
37 | clauses.forEach((clause) => {
38 | const phrase = clause.join('');
39 | const pinyins = clause.map(getPinyin).reduce((ret, p) => ret.concat(p), []);
40 |
41 | // console.log(` - 获取拼音 for ${phrase} ...`);
42 |
43 | // 直接按 字:拼音 进行统计，故，无需再计算 拼音-汉字发射概率
44 | const pinyinWords = correctPinyin(phrase, pinyins, words);
45 | // 忽略包含无效拼音的短语
46 | if (pinyinWords.includes(null)) {
47 | console.log(
48 | ` - 忽略包含无效拼音的短语 '${phrase}': `,
49 | pinyinWords
50 | .map((p, i) => (p ? null : `${phrase.charAt(i)}:${pinyins[i]}`))
51 | .filter((w) => !!w)
52 | .join()
53 | );
54 | return;
55 | }
56 |
57 | result.push(pinyinWords);
58 | });
59 |
60 | return result;
61 | }
62 |
63 | /** @return [['迈向', '充满', '的'], [...], ...] */
64 | export function getClauses(sampleText, words) {
65 | const clauses = [];
66 |
67 | let clause = [];
68 | const splittedPhrases = sampleText.split(/\/[a-z]+\s+/g);
69 | for (let phrase of splittedPhrases) {
70 | if (isValidPhrase(phrase, words)) {
71 | clause.push(phrase);
72 | } else {
73 | if (clause.length > 0) {
74 | clauses.push(clause);
75 | }
76 | clause = [];
77 | }
78 | }
79 |
80 | return clauses;
81 | }
82 |
83 | function isValidPhrase(phrase, words) {
84 | const excludes = ['丨', '丶', '氵'];
85 |
86 | for (let i = 0; i < phrase.length; i++) {
87 | const word = phrase.charAt(i);
88 |
89 | if (!words[word] || excludes.includes(word)) {
90 | return false;
91 | }
92 | }
93 | return true;
94 | }
95 |
96 | /** @return ['迈:mài', ...] */
97 | function correctPinyin(clause, pinyins, words) {
98 | return pinyins.map((pinyin, index) => {
99 | let word = clause.charAt(index);
100 | const wordCode = `${word}:${pinyin}`;
101 | const prevWord = clause.charAt(index - 1);
102 | const postWord = clause.charAt(index + 1);
103 | const postPinyin = pinyins[index + 1] || '';
104 |
105 | // 唯一拼音
106 | const uniques = {
107 | '上:shang': 'shàng',
108 | '生:sheng': 'shēng',
109 | '娘:niang': 'niáng',
110 | '袱:fu': 'fú',
111 | '伍:wu': 'wǔ',
112 | '事:shi': 'shì',
113 | '情:qing': 'qíng',
114 | '下:xia': 'xià',
115 | '同:tong': 'tóng',
116 | '个:ge': 'gè',
117 | '喇:lā': 'lǎ',
118 | '姑:gu': 'gū',
119 | '闹:nao': 'nào',
120 | '实:shi': 'shí',
121 | '人:ren': 'rén',
122 | '萄:tao': 'táo',
123 | '究:jiu': 'jiū',
124 | '太:tai': 'tài',
125 | '芦:lu': 'lú',
126 | '嚣:áo': 'xiāo',
127 | '篷:peng': 'péng',
128 | '哈:ha': 'hā',
129 | '亮:liang': 'liàng',
130 | '栏:lan': 'lán',
131 | '悉:xi': 'xī',
132 | '桃:tao': 'táo',
133 | '气:qi': 'qì',
134 | '户:hu': 'hù',
135 | '脯:pú': 'fǔ',
136 | '敞:chang': 'chǎng',
137 | '复:fu': 'fù',
138 | '务:wu': 'wù',
139 | '歌:ge': 'gē',
140 | '欢:huan': 'huān',
141 | '氛:fen': 'fēn',
142 | '宝:bao': 'bǎo',
143 | '蟆:ma': 'má',
144 | '利:li': 'lì',
145 | '成:cheng': 'chéng',
146 | '婆:po': 'pó',
147 | '甲:jia': 'jiǎ',
148 | '腐:fu': 'fǔ',
149 | '摸:mo': 'mō',
150 | '郗:chī': 'xī',
151 | '女:nü': 'nǚ',
152 | '才:cai': 'cái',
153 | '蛐:qu': 'qū',
154 | '呼:hu': 'hū',
155 | '话:hua': 'huà',
156 | '嫂:sao': 'sǎo',
157 | '辑:ji': 'jí',
158 | '算:suan': 'suàn',
159 | '烦:fan': 'fán',
160 | '屉:ti': 'tì',
161 | '方:fang': 'fāng',
162 | '叔:shu': 'shū',
163 | '应:ying': 'yìng',
164 | '戚:qi': 'qī',
165 | '麻:ma': 'má',
166 | '拉:la': 'lā',
167 | '司:si': 'sī',
168 | '瑰:gui': 'guī',
169 | '牌:pai': 'pái',
170 | '疾:ji': 'jí',
171 | '误:wu': 'wù',
172 | '叭:ba': 'bā',
173 | '付:fu': 'fù',
174 | '蠡:lí': 'lǐ',
175 | '镗:táng': 'tāng',
176 | '毛:mao': 'máo',
177 | '荡:dang': 'dàng',
178 | '拾:shi': 'shí',
179 | '系:xi': 'xì',
180 | '妇:fu': 'fù',
181 | '仗:zhang': 'zhàng',
182 | '面:mian': 'miàn',
183 | '甥:sheng': 'shēng',
184 | '快:kuai': 'kuài',
185 | '婿:xu': 'xù',
186 | '计:ji': 'jì',
187 | '明:ming': 'míng',
188 | '琶:pa': 'pá',
189 | '遛:liú': 'liù',
190 | '兄:xiong': 'xiōng',
191 | '搁:ge': 'gē',
192 | '们:men': 'mén',
193 | '友:you': 'yǒu',
194 | '生:sheng': 'shēng',
195 | '难:nan': 'nán',
196 | '分:fen': 'fēn',
197 | '识:shi': 'shí',
198 | '食:shi': 'shí',
199 | '下:xia': 'xià',
200 | '氛:fen': 'fēn',
201 | '得:de': 'dé',
202 | '星:xing': 'xīng',
203 | '笼:long': 'lóng',
204 | '爷:ye': 'yé',
205 | '奶:nai': 'nǎi',
206 | '爸:ba': 'bà',
207 | '妈:ma': 'mā',
208 | '儿:er': 'ér',
209 | '哥:ge': 'gē',
210 | '服:fu': 'fú',
211 | '睛:jing': 'jīng',
212 | '弟:di': 'dì',
213 | '妹:mei': 'mèi',
214 | '司:si': 'sī',
215 | '候:hou': 'hòu',
216 | '腾:teng': 'téng',
217 | '璃:li': 'lí',
218 | '息:xi': 'xī',
219 | '傅:fu': 'fù',
220 | '娃:wa': 'wá',
221 | '卖:mai': 'mài',
222 | '屈:qu': 'qū',
223 | '思:si': 'sī',
224 | '活:huo': 'huó',
225 | '量:liang': 'liáng',
226 | '伯:bo': 'bó',
227 | '丧:sang': 'sàng',
228 | '嗦:suo': 'suō',
229 | '当:dang': 'dāng',
230 | '咕:gu': 'gū',
231 | '巴:ba': 'bā',
232 | '粑:ba': 'bā',
233 | '矩:ju': 'jǔ',
234 | '发:fa': 'fà',
235 | '合:he': 'hé',
236 | '帚:zhou': 'zhǒu',
237 | '蛋:dan': 'dàn',
238 | '枉:wang': 'wǎng',
239 | '泡:pao': 'pào',
240 | '酬:chou': 'chóu',
241 | '股:gu': 'gǔ',
242 | '剔:ti': 'tī',
243 | '西:xi': 'xī',
244 | '糊:hu': 'hú',
245 | '元:yuan': 'yuán',
246 | '杠:gang': 'gàng',
247 | '乎:hu': 'hū',
248 | '猬:wei': 'wèi',
249 | '指:zhi': 'zhǐ',
250 | '撒:sa': 'sā',
251 | '瞧:qiao': 'qiáo',
252 | '磨:mo': 'mó',
253 | '坊:fang': 'fáng',
254 | '叨:dao': 'dāo',
255 | '蹭:ceng': 'cèng',
256 | '姐:jie': 'jiě',
257 | '狸:li': 'lí',
258 | '楼:lou': 'lóu',
259 | '膊:bo': 'bó',
260 | '堂:tang': 'táng',
261 | '涂:tu': 'tú',
262 | '负:fu': 'fù',
263 | '灵:ling': 'líng',
264 | '菇:gu': 'gū',
265 | '舅:jiu': 'jiù',
266 | '饼:bing': 'bǐng',
267 | '罕:han': 'hǎn',
268 | '药:yao': 'yào',
269 | '筝:zheng': 'zhēng',
270 | '框:kuang': 'kuàng',
271 | '转:zhuan': 'zhuàn',
272 | '壳:ke': 'ké',
273 | '忽:hu': 'hū',
274 | '荒:huang': 'huāng',
275 | '莉:li': 'lì',
276 | '悠:you': 'yōu',
277 | '士:shi': 'shì',
278 | '嚷:rang': 'rāng',
279 | '笆:ba': 'bā',
280 | '窿:long': 'lóng',
281 | '缝:feng': 'féng',
282 | '口:kou': 'kǒu',
283 | '末:mo': 'mò',
284 | '里:li': 'lǐ',
285 | '叽:ji': 'jī',
286 | '心:xin': 'xīn',
287 | '宗:zong': 'zōng',
288 | '姥:lao': 'lǎo',
289 | '喝:he': 'hē',
290 | '伙:huo': 'huǒ',
291 | '囊:nang': 'nāng',
292 | '物:wu': 'wù',
293 | '嗽:sou': 'sòu',
294 | '咙:long': 'lóng',
295 | '': '',
296 | // 占位用
297 | _: ''
298 | };
299 |
300 | // 在 四声字 前念 二声：不要、不错、不是、不再、不认识
301 | if (word == '不' && ['bu'].includes(pinyin)) {
302 | const tone = getPinyinTone(postPinyin);
303 | if (tone == 4) {
304 | pinyin = 'bú';
305 | } else if (tone != 0 || ['得'].includes(postWord)) {
306 | pinyin = 'bù';
307 | }
308 | }
309 | // 在 四声 前念 二声：一样，一下子、一座、一位、一次、一块儿
310 | // 在 一声、二声、三声字 前念 四声：大吃一惊、一般、一年、一门、一口、一起、一种
311 | else if (word == '一' && ['yi'].includes(pinyin)) {
312 | const tone = getPinyinTone(postPinyin);
313 | if (tone == 4) {
314 | pinyin = 'yí';
315 | } else if (tone != 0) {
316 | pinyin = 'yì';
317 | }
318 | } else if (word == '同' && ['胡'].includes(prevWord)) {
319 | pinyin = 'tòng';
320 | } else if (word == '蕃' && ['茄'].includes(postWord)) {
321 | word = '番';
322 | pinyin = 'fān';
323 | } else if (word == '蕃' && ['吐'].includes(prevWord)) {
324 | pinyin = 'bō';
325 | } else if (word == '朵' && ['耳'].includes(prevWord)) {
326 | pinyin = 'duo';
327 | return `${word}:${pinyin}`;
328 | } else if (
329 | word == '脯' &&
330 | (['胸'].includes(prevWord) || ['子'].includes(postWord))
331 | ) {
332 | pinyin = 'pú';
333 | } else if (
334 | word == '夫' &&
335 | ['丈', '工', '功', '姐', '大', '妹'].includes(prevWord)
336 | ) {
337 | pinyin = 'fū';
338 | } else if (word == '喇' && ['喇', '哗', '呼', '喀'].includes(prevWord)) {
339 | pinyin = 'lā';
340 | }
341 | // 姓氏：https://baike.baidu.com/item/%E5%96%87%E5%A7%93/9730899
342 | else if (
343 | word == '喇' &&
344 | (['哈', '半'].includes(prevWord) || ['进', '敏', '秉'].includes(postWord))
345 | ) {
346 | pinyin = 'lá';
347 | } else if (
348 | word == '大' &&
349 | ['士'].includes(prevWord) &&
350 | ['夫'].includes(postWord)
351 | ) {
352 | pinyin = 'dà';
353 | } else if (word == '大' && ['夫'].includes(postWord)) {
354 | pinyin = 'dài';
355 | } else if (
356 | word == '个' &&
357 | ['自'].includes(prevWord) &&
358 | ['儿'].includes(postWord)
359 | ) {
360 | pinyin = 'gě';
361 | }
362 | //
363 | else if (uniques[wordCode]) {
364 | pinyin = uniques[wordCode];
365 | }
366 |
367 | if (!words[word].includes(pinyin)) {
368 | return null;
369 | }
370 |
371 | if (
372 | words[word].length > 1 &&
373 | getPinyinTone(pinyin) == 0 &&
374 | ![
375 | '的',
376 | '不',
377 | '一',
378 | '着',
379 | '么',
380 | '了',
381 | '子',
382 | '啊',
383 | '呢',
384 | '吧',
385 | '宜',
386 | '吗',
387 | '家',
388 | '头',
389 | '呀',
390 | '卜',
391 | '和',
392 | '嘛',
393 | '地',
394 | '匙',
395 | '啦',
396 | '裳',
397 | '瘩',
398 | '喽'
399 | ].includes(word)
400 | ) {
401 | return null;
402 | }
403 |
404 | return `${word}:${pinyin}`;
405 | });
406 | }
407 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/generate/sqlite/phrase/hmm/trans_kewen/index.mjs:
--------------------------------------------------------------------------------
1 | /* 生成 HMM 计算参数 */
2 | import {
3 | fromRootPath,
4 | appendLineToFile,
5 | readLineFromFile,
6 | getAllFiles,
7 | readJSONFromFile,
8 | asyncForEach
9 | } from '#utils/utils.mjs';
10 | import { openDB, closeDB } from '#utils/sqlite.mjs';
11 | import { readWordsFromDB } from '../utils.mjs';
12 | import * as trans from './trans.mjs';
13 |
14 | // 训练课文数据
15 | let phraseSampleFiles = [];
16 | let appendExistData = false;
17 |
18 | const args = process.argv.slice(2);
19 | for (let i = 0; i < args.length; i++) {
20 | const arg = args[i];
21 | if (arg == '-f') {
22 | phraseSampleFiles.push(args[++i]);
23 | } else if (arg == '-a') {
24 | appendExistData = true;
25 | }
26 | }
27 |
28 | if (!phraseSampleFiles) {
29 | console.log(
30 | 'Usage: npm run generate:sqlite:phrase:hmm:trans_kewen -- [-a] -f /file1 -f file2 ...'
31 | );
32 | console.log();
33 |
34 | process.exit(1);
35 | }
36 |
37 | // SQLite 字典库
38 | const wordDictSQLiteFile = fromRootPath('data', 'pinyin-word-dict.sqlite');
39 | // HMM 参数目录
40 | const transParamsDir = fromRootPath('data', 'hmm_params/kewen');
41 |
42 | console.log();
43 | console.log(`创建计算参数${appendExistData ? '（累积更新）' : ''} ...`);
44 | let wordDictDB = await openDB(wordDictSQLiteFile, true);
45 |
46 | let words;
47 | try {
48 | words = await readWordsFromDB(wordDictDB);
49 | } catch (e) {
50 | throw e;
51 | } finally {
52 | await closeDB(wordDictDB);
53 | }
54 |
55 | let transParams = appendExistData
56 | ? {
57 | word_prob: readJSONFromFile(transParamsDir + `/word_prob.json`),
58 | trans_prob: readJSONFromFile(transParamsDir + `/trans_prob.json`)
59 | }
60 | : null;
61 |
62 | await asyncForEach(getAllFiles(phraseSampleFiles), async (file) => {
63 | console.log(` - 分析文件: ${file} ...`);
64 |
65 | await readLineFromFile(file, (line) => {
66 | if (!line || !line.trim()) {
67 | return;
68 | }
69 |
70 | const json = JSON.parse(line);
71 | transParams = trans.countParams(json, words, transParams);
72 | });
73 | });
74 |
75 | Object.keys(transParams).forEach((name) => {
76 | appendLineToFile(
77 | transParamsDir + `/${name}.json`,
78 | JSON.stringify(transParams[name]),
79 | true
80 | );
81 | });
82 |
83 | console.log();
84 | console.log('Done');
85 | console.log();
86 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/generate/sqlite/phrase/hmm/trans_kewen/trans.mjs:
--------------------------------------------------------------------------------
1 | import { extractPinyinChars } from '#utils/utils.mjs';
2 | import { countTrans, countWords } from '../utils.mjs';
3 |
4 | /**
5 | * HMM 参数计算
6 | * trans_prob - 汉字间转移概率
7 | */
8 | export function countParams(articles, words, existParams) {
9 | existParams = existParams || { word_prob: {}, trans_prob: {} };
10 |
11 | const symbols = existParams.symbols || {};
12 | const clauses = readClausesFromArticles(articles, words, symbols);
13 |
14 | return {
15 | // 字的出现次数
16 | word_prob: countWords(clauses, existParams.word_prob),
17 | // 当前字为 EOS 且其前序为 BOS 的转移次数即为 训练的句子总数，
18 | // 而各个包含 BOS 前序的字即为句首字，且其出现次数即为 BOS 的值
19 | trans_prob: countTrans(clauses, existParams.trans_prob),
20 | // 所用到的符号及其出现次数
21 | symbols
22 | };
23 | }
24 |
25 | /**
26 | * @param articles [{title: [...], subtitle: [...], pargraphs: [[...], ...]}, {...}, ...]
27 | */
28 | function readClausesFromArticles(articles, words, symbols) {
29 | let clauses = [];
30 |
31 | articles.forEach(({ title, subtitle, pargraphs }) => {
32 | const titleText = title.map((w) => w.zi).join('');
33 | const subtitleText = subtitle.map((w) => w.zi).join('');
34 |
35 | if (
36 | titleText.includes('生字表') ||
37 | titleText.includes('写字表') ||
38 | titleText.includes('识字表') ||
39 | titleText.includes('练习版') ||
40 | titleText.includes('唐诗') ||
41 | titleText.includes('诗词') ||
42 | titleText.includes('诗歌') ||
43 | titleText.includes('词四首') ||
44 | titleText.includes('课文版') ||
45 | titleText.includes('世说新语') ||
46 | titleText.includes('短诗') ||
47 | titleText.includes('庄子') ||
48 | titleText.includes('老子') ||
49 | titleText.includes('离骚') ||
50 | titleText.includes('一年级') ||
51 | titleText.includes('二年级') ||
52 | subtitleText.includes('一年级') ||
53 | subtitleText.includes('二年级')
54 | ) {
55 | console.log(` - 忽略文章: ${titleText}`);
56 | return;
57 | }
58 | console.log(` - 分析文章: ${titleText}`);
59 |
60 | [title, subtitle].concat(pargraphs).forEach((p) => {
61 | clauses = clauses.concat(readClausesFromPargraph(p, words, symbols));
62 | });
63 | });
64 |
65 | return clauses;
66 | }
67 |
68 | /**
69 | * @param pargraph [{zi: '字', py: 'zì'}, {zi: '，'}, {...}, ...]
70 | */
71 | function readClausesFromPargraph(pargraph, words, symbols) {
72 | const clauses = [];
73 | const addClause = (c) => {
74 | c.length > 0 && clauses.push(c);
75 | // c.length > 0 && console.error(c.join(','));
76 | };
77 |
78 | let clause = [];
79 | for (let i = 0; i < pargraph.length; i++) {
80 | const prev = pargraph[i - 1] || {};
81 | const curr = pargraph[i];
82 | const zi = getCorrectWord(curr);
83 | const py = getCorrectPinyin(curr, prev);
84 |
85 | if (py) {
86 | const spells = words[zi] || [];
87 |
88 | if (/\w+/.test(zi)) {
89 | console.error(` - 非汉字：${curr.zi}:${curr.py}`);
90 | } else if (spells.includes(py)) {
91 | clause.push(`${zi}:${py}`);
92 | } else {
93 | console.error(` - 不存在拼音字: ${curr.zi}:${curr.py}`);
94 | }
95 | } else {
96 | symbols[zi] ||= 0;
97 | symbols[zi] += 1;
98 |
99 | // 短语结束
100 | if (isClauseEnd(zi)) {
101 | if (clause.length > 0) {
102 | addClause(clause);
103 | }
104 | clause = [];
105 | }
106 | }
107 | }
108 | addClause(clause);
109 |
110 | return clauses;
111 | }
112 |
113 | function isClauseEnd(zi) {
114 | return ['，', '。', '；', '：', '？', '！', '∶', '…'].includes(zi);
115 | }
116 |
117 | function getCorrectWord({ zi, py }) {
118 | switch (zi) {
119 | case '轮':
120 | py == 'lūn' && (zi = '抡');
121 | break;
122 | case '纤':
123 | // https://www.cngwzj.com/pygushi/SongDai/72474/
124 | py == 'lián' && (zi = '廉');
125 | break;
126 | case '沉':
127 | // https://www.cngwzj.com/pygushi/SongDai/61484/
128 | py == 'shěn' && (zi = '沈');
129 | break;
130 | case '挡':
131 | // https://www.cngwzj.com/pygushi/SongDai/57152/
132 | // https://baike.baidu.com/item/%E5%BA%86%E5%AE%AB%E6%98%A5%C2%B7%E5%8F%8C%E6%A1%A8%E8%8E%BC%E6%B3%A2/9918314
133 | py == 'dāng' && (zi = '珰');
134 | break;
135 | }
136 | return zi;
137 | }
138 |
139 | function getCorrectPinyin({ zi, py }, prev) {
140 | switch (zi) {
141 | // <<<<<<<<<<<<<<<<<<<<<<
142 | case '看':
143 | prev.zi == zi && (py = 'kàn');
144 | break;
145 | // <<<<<<<<<<<<<<< 叠词：第二个字为轻声
146 | case '爸':
147 | case '妈':
148 | case '哥':
149 | case '弟':
150 | case '姐':
151 | case '妹':
152 | case '爷':
153 | case '奶':
154 | case '婶':
155 | case '叔':
156 | prev.zi == zi && (py = extractPinyinChars(py));
157 | break;
158 | // <<<<<<<<<<<<<<<<<<<<<<<
159 | case '儿':
160 | ['墩', '褂', '势', '猴', '点', '劲'].includes(prev.zi) && (py = 'ér');
161 | break;
162 | // >>>>>>>>>>>>>>>>>>>>>
163 | default:
164 | const replacements = {
165 | 其: 'qí',
166 | 实: 'shí',
167 | 他: 'tā',
168 | 朴: 'pǔ',
169 | 笼: 'lóng',
170 | 牛: 'niú',
171 | 妞: 'niū',
172 | 剔: 'tī',
173 | 菇: 'gū',
174 | 活: 'huó',
175 | 笛: 'dí',
176 | 杵: 'chǔ',
177 | 釭: 'gāng',
178 | 墩: 'dūn',
179 | 褂: 'guà',
180 | 势: 'shì',
181 | 猴: 'hóu',
182 | 点: 'diǎn',
183 | //
184 | '景:ijǐng': 'jǐng',
185 | '温:yùn': 'wēn',
186 | '篷:peng': 'péng',
187 | '蓬:peng': 'péng',
188 | '晨:chen': 'chén',
189 | '袋:dai': 'dài',
190 | '来:lai': 'lái',
191 | '枉:wang': 'wǎng',
192 | '蟆:ma': 'má',
193 | '铛:dang': 'dāng',
194 | '闷:men': 'mèn',
195 | '粱:liang': 'liáng',
196 | '里:li': 'lǐ',
197 | '角:gǔ': 'jiǎo',
198 | '那:nàr': 'nà',
199 | '时:shi': 'shí',
200 | '焚:fèn': 'fén',
201 | '亮:liang': 'liàng',
202 | '道:dao': 'dào',
203 | '家:gū': 'jiā',
204 | '司:si': 'sī',
205 | '上:shang': 'shàng',
206 | '是:shi': 'shì',
207 | '不:bu': 'bù',
208 | '芦:lu': 'lú',
209 | '莫:mo': 'mò',
210 | '夫:fu': 'fū',
211 | '么:mò': 'me',
212 | '少:shāo': 'shǎo',
213 | '搁:ge': 'gē',
214 | '地:di': 'dì',
215 | '呵:ā': 'a',
216 | '劲:jìnr': 'jìn',
217 | '碌:lū': 'lù',
218 | '碌:lu': 'lù'
219 | };
220 |
221 | for (let key of Object.keys(replacements)) {
222 | if ([zi, `${zi}:${py}`].includes(key)) {
223 | py = replacements[key];
224 | break;
225 | }
226 | }
227 | }
228 |
229 | return py;
230 | }
231 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/generate/sqlite/phrase/hmm/utils.mjs:
--------------------------------------------------------------------------------
1 | /**
2 | * 从字典库中读取字及其拼音
3 | *
4 | * @returns 结构为 {'字': ['zì'], ...}
5 | */
6 | export async function readWordsFromDB(wordDictDB) {
7 | const words = {};
8 | (await wordDictDB.all(`select word_, spell_ from pinyin_word`)).forEach(
9 | (row) => {
10 | const { word_, spell_ } = row;
11 |
12 | words[word_] ||= [];
13 | words[word_].push(spell_);
14 | }
15 | );
16 |
17 | return words;
18 | }
19 |
20 | /**
21 | * 计算汉字（状态）间转移概率：每个句子中汉字转移概率
22 | *
23 | * @param clauses 结构为 [['字:zì', ...], [...], ...]
24 | */
25 | export function countTrans(clauses, existTransProb) {
26 | const transProb = existTransProb || {};
27 |
28 | clauses.forEach((clause) => {
29 | for (let i = 0; i <= clause.length; i++) {
30 | const curr = i == clause.length ? 'EOS' : clause[i];
31 | const prev = i == 0 ? 'BOS' : clause[i - 1];
32 |
33 | const prob = (transProb[curr] = transProb[curr] || {});
34 |
35 | prob[prev] = (prob[prev] || 0) + 1;
36 | // 转移概率: math.log(前序字出现次数 / total)
37 | prob.__total__ = (prob.__total__ || 0) + 1;
38 | }
39 | });
40 |
41 | return transProb;
42 | }
43 |
44 | /**
45 | * 统计短语中的汉字数量
46 | *
47 | * @param clauses 结构为 [['字:zì', ...], [...], ...]
48 | */
49 | export function countWords(clauses, existWordProp) {
50 | const wordProb = existWordProp || {};
51 |
52 | clauses.forEach((clause) => {
53 | for (let i = 0; i < clause.length; i++) {
54 | const curr = clause[i];
55 |
56 | wordProb[curr] ||= 0;
57 | wordProb[curr] += 1;
58 | }
59 | });
60 |
61 | return wordProb;
62 | }
63 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/generate/sqlite/word/diff.mjs:
--------------------------------------------------------------------------------
1 | /* 对比不同版本的 SQLite 字典库的数据差异 */
2 | import { fromRootPath, asyncForEach } from '#utils/utils.mjs';
3 |
4 | import { openDB, closeDB } from '#utils/sqlite.mjs';
5 |
6 | const oldDictDataSQLiteFile = fromRootPath(
7 | 'data',
8 | 'pinyin-word-dict.v2.sqlite'
9 | );
10 | const dictDataSQLiteFile = fromRootPath('data', 'pinyin-word-dict.sqlite');
11 |
12 | let oldDb = await openDB(oldDictDataSQLiteFile);
13 | let newDb = await openDB(dictDataSQLiteFile);
14 |
15 | try {
16 | console.log();
17 | console.log('对比元数据的差异 ...');
18 | await diffMetaData(oldDb, newDb);
19 |
20 | console.log();
21 | console.log('对比字数据的差异 ...');
22 | await diffWordData(oldDb, newDb);
23 | } catch (e) {
24 | throw e;
25 | } finally {
26 | await closeDB(oldDb);
27 | await closeDB(newDb);
28 | }
29 |
30 | async function diffMetaData(oldDb, newDb) {
31 | await asyncForEach(
32 | [
33 | 'meta_pinyin',
34 | 'meta_pinyin_chars',
35 | 'meta_zhuyin',
36 | 'meta_zhuyin_chars',
37 | 'meta_word'
38 | ],
39 | async (table) => {
40 | const oldData = {};
41 | const newData = {};
42 |
43 | (await oldDb.all(`select * from ${table}`)).forEach((row) => {
44 | const value = row.value_;
45 | const id_ = row.id_;
46 |
47 | oldData[value] = { id_ };
48 | });
49 | (await newDb.all(`select * from ${table}`)).forEach((row) => {
50 | const value = row.value_;
51 | const id_ = row.id_;
52 |
53 | newData[value] = { id_ };
54 | });
55 |
56 | Object.keys(newData).forEach((value) => {
57 | if (!oldData[value]) {
58 | console.log(`- ${table} => 元数据 ${value} 为新增`);
59 | return;
60 | }
61 |
62 | const oldId = oldData[value].id_;
63 | const newId = newData[value].id_;
64 |
65 | if (oldId != newId) {
66 | console.log(
67 | `- ${table} => 元数据 ${value} 的 id 不同: ${oldId} -> ${newId}`
68 | );
69 | }
70 | });
71 |
72 | Object.keys(oldData).forEach((value) => {
73 | if (!newData[value]) {
74 | console.log(`- ${table} => 元数据 ${value} 已被删除`);
75 | return;
76 | }
77 | });
78 | }
79 | );
80 | }
81 |
82 | async function diffWordData(oldDb, newDb) {
83 | await asyncForEach(['pinyin_word' /*, 'zhuyin_word'*/], async (table) => {
84 | const oldData = {};
85 | const newData = {};
86 |
87 | (await oldDb.all(`select * from ${table}`)).forEach((row) => {
88 | const id = row.id_;
89 |
90 | oldData[id] = row;
91 | });
92 | (await newDb.all(`select * from ${table}`)).forEach((row) => {
93 | const id = row.id_;
94 |
95 | newData[id] = row;
96 | });
97 |
98 | const genCode = (row) => {
99 | return `${row.word_id_ || row.source_id_}:${
100 | row.spell_id_ || row.target_id_
101 | }:${row.spell_chars_id_ || row.target_chars_id_}`;
102 | };
103 |
104 | Object.keys(newData).forEach((id) => {
105 | const oldRow = oldData[id];
106 | const newRow = newData[id];
107 |
108 | if (!oldRow) {
109 | console.log(
110 | `- ${table} => 字数据 ${id}:${newRow.word_}:${newRow.spell_} 为新增`
111 | );
112 | return;
113 | }
114 |
115 | const oldCode = genCode(oldRow);
116 | const newCode = genCode(newRow);
117 |
118 | if (oldCode != newCode) {
119 | console.log(
120 | `- ${table} => 字数据 ${id}:${newRow.word_}:${newRow.spell_} 的组合不同: ${oldCode} -> ${newCode}`
121 | );
122 | }
123 | });
124 |
125 | Object.keys(oldData).forEach((id) => {
126 | const oldRow = oldData[id];
127 | if (!newData[id]) {
128 | console.log(
129 | `- ${table} => 字数据 ${id}:${oldRow.word_}:${oldRow.spell_} 已被删除`
130 | );
131 | return;
132 | }
133 | });
134 | });
135 | }
136 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/generate/sqlite/word/index.mjs:
--------------------------------------------------------------------------------
1 | /* SQLite 字典库 */
2 | import { fromRootPath, readLineFromFile } from '#utils/utils.mjs';
3 | import { patch } from './patch.mjs';
4 | import * as sqlite from './sqlite.mjs';
5 |
6 | // 收集数据
7 | const wordDataValidFile = fromRootPath('data', 'pinyin-dict.valid.txt');
8 | const emojiDataFile = fromRootPath('data', 'emojis.json');
9 | // 分析数据
10 | const pinyinCharsFile = fromRootPath('..', 'analyze/files/pinyin.txt');
11 | const pinyinCharLinksFile = fromRootPath('..', 'analyze/files/char-links.json');
12 | const pinyinCharTreeFile = fromRootPath('..', 'analyze/files/char-tree.json');
13 |
14 | // SQLite 字典库
15 | const wordDictDataSQLiteFile = fromRootPath('data', 'pinyin-word-dict.sqlite');
16 |
17 | console.log();
18 | console.log('读取已收集的有效字信息 ...');
19 | const wordMetas = [];
20 | await readLineFromFile(wordDataValidFile, (line) => {
21 | if (!line || !line.trim()) {
22 | return;
23 | }
24 |
25 | const metas = JSON.parse(line);
26 | metas.forEach((meta) => {
27 | wordMetas.push(meta);
28 |
29 | patch(meta);
30 | });
31 | });
32 | console.log('- 有效字信息总数：' + wordMetas.length);
33 | console.log();
34 |
35 | console.log();
36 | console.log('写入字信息到 SQLite ...');
37 | let db1 = await sqlite.open(wordDictDataSQLiteFile);
38 |
39 | try {
40 | await sqlite.saveSpells(db1, wordMetas);
41 | console.log('- 已保存字读音信息');
42 |
43 | await sqlite.saveWords(db1, wordMetas);
44 | console.log('- 已保存字信息');
45 |
46 | await sqlite.savePhrases(db1, wordMetas);
47 | console.log('- 已保存词组信息');
48 | } catch (e) {
49 | throw e;
50 | } finally {
51 | await sqlite.close(db1);
52 | }
53 |
54 | console.log();
55 |
56 | console.log();
57 | console.log('读取已收集的表情符号 ...');
58 | const groupEmojiMetas = {};
59 | await readLineFromFile(emojiDataFile, (line) => {
60 | if (!line || !line.trim()) {
61 | return;
62 | }
63 |
64 | const groups = JSON.parse(line);
65 | groups.forEach((group) => {
66 | let groupName = group.name.zh;
67 | switch (groupName) {
68 | case '表情与情感':
69 | groupName = '表情';
70 | break;
71 | case '人物与身体':
72 | groupName = '人物';
73 | break;
74 | case '动物与自然':
75 | groupName = '动植物';
76 | break;
77 | case '食物与饮料':
78 | groupName = '饮食';
79 | break;
80 | case '旅行与地理':
81 | groupName = '旅行';
82 | break;
83 | case '符号标志':
84 | groupName = '符号';
85 | break;
86 | }
87 |
88 | groupEmojiMetas[groupName] = group.emojis;
89 | });
90 | });
91 | console.log(
92 | '- 表情符号总数：' +
93 | Object.values(groupEmojiMetas).reduce(
94 | (acc, emojis) => acc + emojis.length,
95 | 0
96 | )
97 | );
98 | console.log();
99 |
100 | console.log();
101 | console.log('写入表情符号到 SQLite ...');
102 | let db2 = await sqlite.open(wordDictDataSQLiteFile);
103 | try {
104 | await sqlite.saveEmojis(db2, groupEmojiMetas);
105 | console.log('- 已保存表情符号数据');
106 | } catch (e) {
107 | throw e;
108 | } finally {
109 | await sqlite.close(db2);
110 | }
111 | console.log();
112 |
113 | console.log();
114 | console.log('通过 SQLite 生成分析数据 ...');
115 | let db3 = await sqlite.open(wordDictDataSQLiteFile);
116 | try {
117 | await sqlite.generatePinyinChars(db3, pinyinCharsFile);
118 | console.log('- 已保存拼音字母组合数据');
119 |
120 | await sqlite.generatePinyinCharLinks(db3, pinyinCharLinksFile);
121 | console.log('- 已保存拼音字母关联数据');
122 |
123 | await sqlite.generatePinyinCharTree(db3, pinyinCharTreeFile);
124 | console.log('- 已保存拼音字母后继数据');
125 | } catch (e) {
126 | throw e;
127 | } finally {
128 | await sqlite.close(db3);
129 | }
130 | console.log();
131 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/generate/sqlite/word/patch.mjs:
--------------------------------------------------------------------------------
1 | import { extractPinyinChars } from '#utils/utils.mjs';
2 |
3 | /** 修正输入数据 */
4 | export function patch(meta) {
5 | const deleted = [
6 | '虾:hā' // -> 虾:há
7 | ];
8 |
9 | const added = [
10 | // “一”和“不”变调有规律：https://www.chinanews.com.cn/hwjy/news/2010/04-15/2228742.shtml
11 | '不:bú',
12 | '一:yì',
13 | '一:yí',
14 | '子:zi',
15 | // 便宜：pián yi
16 | '宜:yi',
17 | '噷:hm',
18 | '吒:zhà',
19 | '虎:hu',
20 | '枸:gōu',
21 | '焘:tāo',
22 | '喇:lā',
23 | '喇:lá',
24 | '蕃:bō',
25 | '蕃:fān',
26 | '脯:pú',
27 | '蕻:hóng',
28 | '朵:duo',
29 | '鏜:táng',
30 | '咔:kā',
31 | '蹬:dèng',
32 | '爸:ba',
33 | '叔:shu',
34 | '喝:he',
35 | // 《定风波·自春来》 - 无那。恨薄情一去，音书无个
36 | // https://www.cngwzj.com/pygushi/SongDai/48900/
37 | '那:nuó',
38 | // 《桂枝香·金陵怀古》 - 谩嗟荣辱
39 | // https://www.cngwzj.com/pygushi/SongDai/49417/
40 | '谩:màn',
41 | // 《贺新郎·春情》 - 殢酒厌厌病
42 | // https://www.cngwzj.com/pygushi/SongDai/61645/
43 | '厌:yǎn',
44 | // 《贺新郎·春情》 - 断鸿难倩
45 | // https://www.cngwzj.com/pygushi/SongDai/61645/
46 | '倩:qìng',
47 | // 《八声甘州·记玉关踏雪事清游》 - 长河饮马
48 | // https://www.cngwzj.com/pygushi/SongDai/61043/
49 | '饮:yìn',
50 | // 王维《青溪》 - 趣途无百里
51 | // https://www.cngwzj.com/pygushi/TangDai/10982/
52 | '趣:qū',
53 | // 李白《关山月》 - 戍客望边色
54 | // https://www.cngwzj.com/pygushi/TangDai/12860/
55 | '色:yì',
56 | // 《听董大弹胡笳声兼寄语弄房给事》 - 四郊秋叶惊摵摵
57 | // https://www.cngwzj.com/pygushi/TangDai/11474/
58 | '摵:shè',
59 | // 白居易《琵琶行》 - 自言本是京城女，家在虾蟆陵下住
60 | // https://www.cngwzj.com/pygushi/TangDai/25273/
61 | '虾:há',
62 | // 李白《将进酒》
63 | // https://www.cngwzj.com/pygushi/TangDai/12843/
64 | '将:qiāng',
65 | // 《行经华阴》- 借问路傍名利客
66 | // https://www.cngwzj.com/pygushi/TangDai/11353/
67 | '傍:páng',
68 | // 王维《鹿柴》
69 | // https://www.cngwzj.com/pygushi/TangDai/11206/
70 | '柴:zhài',
71 | // 礼记《虽有嘉肴》- 学学半
72 | // https://www.cngwzj.com/pygushi/LiangHan/76970/
73 | '学:xiào',
74 | // 屈原《离骚》- 肇锡余以嘉名
75 | // https://www.cngwzj.com/pygushi/XianQin/87343/
76 | '锡:cì',
77 | // - 来吾道夫先路
78 | '道:dǎo',
79 | // 论语《不义而富且贵，于我如浮云》- 久要不忘平生之言
80 | // https://www.cngwzj.com/pygushi/XianQin/88550/
81 | '要:yuē',
82 | // 论语《己所不欲，勿施于人》- 举皋陶
83 | // https://www.cngwzj.com/pygushi/XianQin/88549/
84 | '陶:yáo',
85 | // - 乡也
86 | '乡:xiàng',
87 | // 论语《好仁不好学，其蔽也愚》- 陈亢问
88 | // https://www.cngwzj.com/pygushi/XianQin/88551/
89 | '亢:gāng',
90 | // 荀子《劝学》- 君子生非异也
91 | // https://www.cngwzj.com/pygushi/XianQin/86629/
92 | '生:xìng',
93 | // 司马迁《陈涉世家》- 发闾左適戍渔阳
94 | // https://www.cngwzj.com/pygushi/LiangHan/88083/
95 | '適:zhé',
96 | '夏:jiǎ',
97 | '苦:hù',
98 | // 列子《杞人忧天》- 舍然大喜
99 | // https://www.cngwzj.com/pygushi/KeWen/87901/
100 | '舍:shì'
101 | ];
102 |
103 | // 先增改，
104 | extraWords(added).forEach(({ value, pinyin, chars }) => {
105 | if (
106 | meta.value == value &&
107 | meta.pinyins.filter(({ value }) => value == pinyin).length == 0
108 | ) {
109 | meta.pinyins.push({ value: pinyin, chars });
110 | }
111 | });
112 | // 再删除，以避免自增 id 发生较大变动
113 | extraWords(deleted).forEach(({ value, pinyin }) => {
114 | if (meta.value == value) {
115 | meta.pinyins = meta.pinyins.filter((py) => py.value !== pinyin);
116 | }
117 | });
118 | }
119 |
120 | function extraWords(words) {
121 | return words
122 | .map((w) => w.split(':'))
123 | .map((s) => ({
124 | value: s[0],
125 | pinyin: s[1],
126 | chars: extractPinyinChars(s[1])
127 | }));
128 | }
129 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/generate/test.mjs:
--------------------------------------------------------------------------------
1 | import { hasGlyphFontForCodePoint } from '#utils/utils.mjs';
2 | import { fetchWordMetas } from '#utils/zdic.mjs';
3 |
4 | const unicodes = [
5 | 'U+20C43' /* 𠱃 */,
6 | 'U+20C53' /* 𠱓 */,
7 | 'U+20C65' /* 𠱥 */,
8 | 'U+20C8D' /* 𠲍 */,
9 | 'U+20C96' /* 𠲖 */,
10 | 'U+20C9C' /* 𠲜 */,
11 | 'U+20CB5' /* 𠲵 */,
12 | 'U+20CD0' /* 𠳐 */,
13 | 'U+20CED' /* 𠳭 */
14 | ];
15 | for (let i = 0; i < unicodes.length; i++) {
16 | const unicode = unicodes[i];
17 | const codePoint = parseInt(unicode.replaceAll(/^U\+/g, '0x'), 16);
18 | const char = String.fromCharCode(codePoint);
19 | const exist = hasGlyphFontForCodePoint(unicode);
20 |
21 | console.log(unicode + ' - ' + char + ': ' + exist);
22 | }
23 |
24 | // const words = ['㑵', '𥁞', '尽', '国', '𣴘'];
25 | // const wordMetas = await fetchWordMetas(words);
26 | // console.log(JSON.stringify(wordMetas));
27 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/utils/sqlite.mjs:
--------------------------------------------------------------------------------
1 | // https://www.sqlitetutorial.net/sqlite-nodejs/connect/
2 | // https://github.com/TryGhost/node-sqlite3/wiki/API
3 | import sqlite3 from 'sqlite3';
4 | // https://www.npmjs.com/package/sqlite
5 | import * as sqlite from 'sqlite';
6 |
7 | import { splitChars, extractPinyinChars, asyncForEach } from './utils.mjs';
8 |
9 | export async function openDB(file, readonly) {
10 | const db = await sqlite.open({
11 | filename: file,
12 | mode: readonly
13 | ? sqlite3.OPEN_READONLY
14 | : sqlite3.OPEN_CREATE | sqlite3.OPEN_READWRITE,
15 | driver: sqlite3.Database
16 | });
17 |
18 | // 提升批量写入性能: https://avi.im/blag/2021/fast-sqlite-inserts/
19 | await execSQL(
20 | db,
21 | `
22 | pragma journal_mode = off;
23 | pragma synchronous = 0;
24 | pragma cache_size = 1000000;
25 | pragma locking_mode = exclusive;
26 | pragma temp_store = memory;
27 | `
28 | );
29 |
30 | return db;
31 | }
32 |
33 | export async function attachDB(db, sources) {
34 | // 附加数据库（连接期内有效）: https://www.sqlite.org/lang_attach.html
35 | await execSQL(
36 | db,
37 | Object.keys(sources)
38 | .map((name) => `attach database '${sources[name]}' as ${name}`)
39 | .join(';')
40 | );
41 |
42 | return db;
43 | }
44 |
45 | export async function closeDB(db, skipClean) {
46 | try {
47 | if (db.config.mode != sqlite3.OPEN_READONLY && !skipClean) {
48 | // 数据库无用空间回收
49 | await execSQL(db, 'vacuum');
50 | }
51 |
52 | await db.close();
53 | } catch (e) {
54 | console.error(e);
55 | }
56 | }
57 |
58 | /** 新增或更新数据 */
59 | export async function saveToDB(
60 | db,
61 | table,
62 | dataMap,
63 | disableSorting,
64 | primaryKeys
65 | ) {
66 | const dataArray = mapToArray(dataMap, disableSorting);
67 | if (dataArray.length === 0) {
68 | return;
69 | }
70 |
71 | primaryKeys = primaryKeys || ['id_'];
72 | const hasOnlyIdKey = primaryKeys.length == 1 && primaryKeys[0] == 'id_';
73 |
74 | const columnsWithPrimaryKey = Object.keys(dataArray[0]).filter(
75 | (k) => !k.startsWith('__')
76 | );
77 | const columns = columnsWithPrimaryKey.filter((k) => !primaryKeys.includes(k));
78 |
79 | const insertWithIdSql = `insert into ${table} (${columnsWithPrimaryKey.join(
80 | ', '
81 | )}) values (${columnsWithPrimaryKey.map(() => '?').join(', ')})
82 | `;
83 | const insertWithIdStatement = await db.prepare(insertWithIdSql);
84 | const insertStatement = hasOnlyIdKey
85 | ? await db.prepare(
86 | `insert into ${table} (${columns.join(', ')}) values (${columns
87 | .map(() => '?')
88 | .join(', ')})
89 | `
90 | )
91 | : await db.prepare(insertWithIdSql);
92 | const updateStatement =
93 | columns.length > 0
94 | ? await db.prepare(
95 | `update ${table} set ${columns
96 | .map((c) => c + ' = ?')
97 | .join(', ')} where ${primaryKeys
98 | .map((key) => key + ' = ?')
99 | .join(' and ')}
100 | `
101 | )
102 | : // 所有的列都为主键，则不需要更新
103 | null;
104 |
105 | const getId = (d) => primaryKeys.map((k) => d[k]).join('');
106 | await asyncForEach(dataArray, async (data) => {
107 | if (getId(data)) {
108 | const needToUpdate =
109 | data.__exist__ &&
110 | columns.reduce((r, c) => r || data[c] !== data.__exist__[c], false);
111 |
112 | if (needToUpdate) {
113 | await updateStatement.run(
114 | ...columns.concat(primaryKeys).map((c) => data[c])
115 | );
116 | }
117 | // 新增包含 id 的数据
118 | else if (!data.__exist__) {
119 | await insertWithIdStatement.run(
120 | ...columnsWithPrimaryKey.map((c) => data[c])
121 | );
122 | }
123 | } else {
124 | const params = (hasOnlyIdKey ? columns : columnsWithPrimaryKey).map(
125 | (c) => data[c]
126 | );
127 | await insertStatement.run(...params);
128 | }
129 | });
130 |
131 | await insertStatement.finalize();
132 | await insertWithIdStatement.finalize();
133 | updateStatement && (await updateStatement.finalize());
134 | }
135 |
136 | /** 删除数据 */
137 | export async function removeFromDB(db, table, data, primaryKeys) {
138 | if (data.length === 0) {
139 | return;
140 | }
141 |
142 | primaryKeys = primaryKeys || ['id_'];
143 |
144 | const deleteStatement = await db.prepare(
145 | `delete from ${table} where ${primaryKeys
146 | .map((key) => key + ' = ?')
147 | .join(' and ')}
148 | `
149 | );
150 |
151 | await asyncForEach(data, async (d) => {
152 | const params = typeof d == 'object' ? primaryKeys.map((c) => d[c]) : [d];
153 | await deleteStatement.run(...params);
154 | });
155 |
156 | await deleteStatement.finalize();
157 | }
158 |
159 | export async function hasTable(db, table) {
160 | const result = await db.get(
161 | `select count(*) as total from sqlite_master where type='table' and name='${table}'`
162 | );
163 | return result.total == 1;
164 | }
165 |
166 | export async function execSQL(db, sqls) {
167 | await asyncForEach(sqls.split(/;/g), async (sql) => {
168 | await db.exec(sql);
169 | });
170 | }
171 |
172 | function mapToArray(obj, disableSorting) {
173 | if (disableSorting === true) {
174 | return Object.keys(obj).map((k) => obj[k]);
175 | }
176 |
177 | const charSpecials = {
178 | a: ['ā', 'á', 'ǎ', 'à'],
179 | o: ['ō', 'ó', 'ǒ', 'ò'],
180 | e: ['ē', 'é', 'ě', 'è', 'ê', 'ê̄', 'ế', 'ê̌', 'ề'],
181 | i: ['ī', 'í', 'ǐ', 'ì'],
182 | u: ['ū', 'ú', 'ǔ', 'ù'],
183 | ü: ['ǖ', 'ǘ', 'ǚ', 'ǜ'],
184 | n: ['ń', 'ň', 'ǹ'],
185 | m: ['m̄', 'ḿ', 'm̀']
186 | };
187 | const charWeights = { ˉ: 10001, ˊ: 10002, ˇ: 10003, ˋ: 10004 };
188 | for (let i = 97, j = 1; i <= 122; i++, j++) {
189 | const ch = String.fromCharCode(i);
190 | const weight = j * 15;
191 | charWeights[ch] = weight;
192 |
193 | const specials = charSpecials[ch];
194 | if (specials) {
195 | for (let k = 0; k < specials.length; k++) {
196 | const special = specials[k];
197 |
198 | charWeights[special] = weight + (k + 1);
199 | }
200 | }
201 | }
202 | const getCharCode = (ch) => {
203 | let sum = 0;
204 | for (let i = 0; i < ch.length; i++) {
205 | sum += ch.charCodeAt(i);
206 | }
207 | return sum;
208 | };
209 |
210 | // Note: 主要排序带音调的拼音（注音规则暂时不清楚，故不处理），其余的按字符顺序排序
211 | const keys = Object.keys(obj).sort((a, b) => {
212 | const a_without_special = extractPinyinChars(a).replaceAll(/[ˊˇˋˉ]$/g, '');
213 | const b_without_special = extractPinyinChars(b).replaceAll(/[ˊˇˋˉ]$/g, '');
214 |
215 | if (a_without_special === b_without_special) {
216 | const a_weight = splitChars(a)
217 | .map((ch) => charWeights[ch] || getCharCode(ch))
218 | .reduce((acc, w) => acc + w, 0);
219 | const b_weight = splitChars(b)
220 | .map((ch) => charWeights[ch] || getCharCode(ch))
221 | .reduce((acc, w) => acc + w, 0);
222 |
223 | return a_weight - b_weight;
224 | }
225 |
226 | return a_without_special > b_without_special
227 | ? 1
228 | : a_without_special < b_without_special
229 | ? -1
230 | : 0;
231 | });
232 |
233 | return keys.map((k) => obj[k]);
234 | }
235 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/utils/utils.mjs:
--------------------------------------------------------------------------------
1 | import * as fs from 'fs';
2 | import * as path from 'path';
3 | import * as crypto from 'crypto';
4 | import { fileURLToPath } from 'url';
5 | import * as events from 'events';
6 | import * as readline from 'readline';
7 |
8 | import * as fontkit from 'fontkit';
9 | import getSystemFonts from 'get-system-fonts';
10 | import GraphemeSplitter from 'grapheme-splitter';
11 |
12 | import { pinyin as parsePinyin, addDict } from 'pinyin-pro';
13 | // https://pinyin-pro.cn/use/addDict.html
14 | import CompleteDict from '@pinyin-pro/data/complete';
15 |
16 | addDict(CompleteDict);
17 |
18 | const systemFonts = await prepareSystemFonts();
19 | const graphemeSplitter = new GraphemeSplitter();
20 |
21 | // https://codingbeautydev.com/blog/javascript-dirname-is-not-defined-in-es-module-scope/
22 | const __filename = fileURLToPath(import.meta.url);
23 | const __dirname = path.dirname(__filename);
24 |
25 | export async function asyncForEach(array, cb) {
26 | for (const e of array) {
27 | await cb(e);
28 | }
29 | }
30 |
31 | export function fromRootPath(...paths) {
32 | return path.join(__dirname, '../..', ...paths);
33 | }
34 |
35 | export function sleep(ms) {
36 | return new Promise((resolve) => setTimeout(() => resolve(), ms));
37 | }
38 |
39 | export function fileSHA256(filepath) {
40 | // https://gist.github.com/GuillermoPena/9233069#gistcomment-3149231-permalink
41 | const file = fs.readFileSync(filepath);
42 | const hash = crypto.createHash('sha256');
43 | hash.update(file);
44 |
45 | return hash.digest('hex');
46 | }
47 |
48 | export function existFile(filepath) {
49 | return fs.existsSync(filepath);
50 | }
51 |
52 | export function copyFile(source, target, override) {
53 | if (existFile(target) && override !== true) {
54 | return;
55 | }
56 |
57 | fs.copyFileSync(source, target);
58 | }
59 |
60 | export function readJSONFromFile(filepath, defaultValue = {}) {
61 | if (!existFile(filepath)) {
62 | return defaultValue;
63 | }
64 |
65 | return JSON.parse(readFile(filepath));
66 | }
67 |
68 | export function readFile(filepath) {
69 | return fs.readFileSync(filepath, 'utf8');
70 | }
71 |
72 | export function readAllFiles(dir) {
73 | return getAllFiles(dir).map((file) => readFile(file));
74 | }
75 |
76 | export function getAllFiles(dir) {
77 | if (Array.isArray(dir)) {
78 | return dir.map(getAllFiles).reduce((acc, files) => acc.concat(files), []);
79 | }
80 |
81 | if (fs.lstatSync(dir).isFile()) {
82 | return [dir];
83 | }
84 |
85 | let files = [];
86 | fs.readdirSync(dir).forEach((file) => {
87 | const filepath = path.join(dir, file);
88 |
89 | if (fs.lstatSync(filepath).isDirectory()) {
90 | files = files.concat(getAllFiles(filepath));
91 | } else {
92 | files.push(filepath);
93 | }
94 | });
95 |
96 | return files;
97 | }
98 |
99 | export async function readLineFromFile(filepath, consumer) {
100 | if (!existFile(filepath)) {
101 | return [];
102 | }
103 |
104 | const rl = readline.createInterface({
105 | input: fs.createReadStream(filepath),
106 | crlfDelay: Infinity
107 | });
108 |
109 | const results = [];
110 | rl.on('line', (line) => {
111 | const result = consumer(line);
112 | if (typeof result !== 'undefined') {
113 | results.push(result);
114 | }
115 | });
116 |
117 | await events.once(rl, 'close');
118 |
119 | return results;
120 | }
121 |
122 | export function appendLineToFile(filepath, line, doEmpty) {
123 | const dirpath = path.dirname(filepath);
124 |
125 | if (!fs.existsSync(dirpath)) {
126 | fs.mkdirSync(dirpath);
127 | }
128 |
129 | if (!fs.existsSync(filepath) || doEmpty) {
130 | fs.writeFileSync(filepath, '');
131 | }
132 |
133 | let fd;
134 | try {
135 | fd = fs.openSync(filepath, 'a');
136 | fs.appendFileSync(fd, line + '\n', 'utf8');
137 | } finally {
138 | fd && fs.closeSync(fd);
139 | }
140 | }
141 |
142 | export function naiveHTMLNodeInnerText(node) {
143 | // https://github.com/jsdom/jsdom/issues/1245#issuecomment-1243809196
144 | // We need Node(DOM's Node) for the constants,
145 | // but Node doesn't exist in the nodejs global space,
146 | // and any Node instance references the constants
147 | // through the prototype chain
148 | const Node = node;
149 |
150 | return node && node.childNodes
151 | ? [...node.childNodes]
152 | .map((node) => {
153 | switch (node.nodeType) {
154 | case Node.TEXT_NODE:
155 | return node.textContent;
156 | case Node.ELEMENT_NODE:
157 | return naiveHTMLNodeInnerText(node);
158 | default:
159 | return '';
160 | }
161 | })
162 | .join(' ')
163 | : '';
164 | }
165 |
166 | async function prepareSystemFonts() {
167 | // https://www.npmjs.com/package/get-system-fonts
168 | const fontFiles = await getSystemFonts();
169 | const fonts = [];
170 |
171 | // https://github.com/foliojs/fontkit#fonthasglyphforcodepointcodepoint
172 | fontFiles.forEach((file) => {
173 | try {
174 | const font = fontkit.openSync(file);
175 | if (!font.hasGlyphForCodePoint) {
176 | return;
177 | }
178 |
179 | //console.info('Read font file: ' + file);
180 | fonts.push(font);
181 | } catch (e) {
182 | //console.warn('Failed to read font file: ' + file, e);
183 | }
184 | });
185 |
186 | return fonts;
187 | }
188 |
189 | /** 判断系统字体中是否存在指定编码的字形，若不存在，则表示该编码的字不可读 */
190 | export function hasGlyphFontForCodePoint(unicode) {
191 | const codePoint = parseInt('0x' + unicode.replaceAll(/^U\+/g, ''), 16);
192 |
193 | for (let i = 0; i < systemFonts.length; i++) {
194 | const font = systemFonts[i];
195 |
196 | if (font.hasGlyphForCodePoint(codePoint)) {
197 | return true;
198 | }
199 | }
200 | return false;
201 | }
202 |
203 | /** 部分中文和表情符号等占用字节数大于 2，比如: 𫫇，需单独处理 */
204 | export function splitChars(str) {
205 | // https://github.com/orling/grapheme-splitter
206 | return graphemeSplitter.splitGraphemes(str);
207 | }
208 |
209 | /** @return ['nǐ', 'hǎo', 'ma'] */
210 | export function getPinyin(str) {
211 | // https://pinyin-pro.cn/use/pinyin.html
212 | return parsePinyin(str, {
213 | // 输出为数组
214 | type: 'array',
215 | // 作为音调符号带在拼音字母上
216 | toneType: 'symbol',
217 | // 识别字符串开头的姓氏
218 | surname: 'head',
219 | // 是否对一和不应用智能变调
220 | // 不（bù）在去声字前面读阳平声，如“～会”“～是”，这属于变调读音
221 | // http://www.moe.gov.cn/jyb_hygq/hygq_zczx/moe_1346/moe_1364/tnull_42118.html
222 | // “一”和“不”变调有规律：https://www.chinanews.com.cn/hwjy/news/2010/04-15/2228742.shtml
223 | toneSandhi: true
224 | });
225 | }
226 |
227 | /** 修正拼音 */
228 | export function correctPinyin(str) {
229 | return str
230 | .replaceAll('ā', 'ā')
231 | .replaceAll('ă', 'ǎ')
232 | .replaceAll('à', 'à')
233 | .replaceAll('ɑ', 'a')
234 | .replaceAll('ō', 'ō')
235 | .replaceAll('ŏ', 'ǒ')
236 | .replaceAll('ī', 'ī')
237 | .replaceAll('ĭ', 'ǐ')
238 | .replaceAll('ŭ', 'ǔ')
239 | .replaceAll('ɡ', 'g')
240 | .replaceAll('ē', 'ē')
241 | .replaceAll(/[·]/g, '');
242 | }
243 |
244 | /** 修正注音 */
245 | export function correctZhuyin(str) {
246 | return str.replaceAll('π', 'ㄫ').replaceAll('˙', '');
247 | }
248 |
249 | /** 拼音去掉声调后的字母组合 */
250 | export function extractPinyinChars(pinyin) {
251 | if ('m̀' === pinyin || 'ḿ' === pinyin || 'm̄' === pinyin) {
252 | return 'm';
253 | } else if (
254 | 'ê̄' === pinyin ||
255 | 'ế' === pinyin ||
256 | 'ê̌' === pinyin ||
257 | 'ề' === pinyin
258 | ) {
259 | return 'e';
260 | }
261 |
262 | const chars = [];
263 |
264 | const splits = splitChars(pinyin);
265 | for (let i = 0; i < splits.length; i++) {
266 | const ch = splits[i];
267 | switch (ch) {
268 | case 'ā':
269 | case 'á':
270 | case 'ǎ':
271 | case 'à':
272 | chars.push('a');
273 | break;
274 | case 'ō':
275 | case 'ó':
276 | case 'ǒ':
277 | case 'ò':
278 | chars.push('o');
279 | break;
280 | case 'ē':
281 | case 'é':
282 | case 'ě':
283 | case 'è':
284 | case 'ê':
285 | chars.push('e');
286 | break;
287 | case 'ī':
288 | case 'í':
289 | case 'ǐ':
290 | case 'ì':
291 | chars.push('i');
292 | break;
293 | case 'ū':
294 | case 'ú':
295 | case 'ǔ':
296 | case 'ù':
297 | chars.push('u');
298 | break;
299 | case 'ǖ':
300 | case 'ǘ':
301 | case 'ǚ':
302 | case 'ǜ':
303 | chars.push('ü');
304 | break;
305 | case 'ń':
306 | case 'ň':
307 | case 'ǹ':
308 | chars.push('n');
309 | break;
310 | default:
311 | chars.push(ch);
312 | }
313 | }
314 |
315 | return chars.join('');
316 | }
317 |
318 | export function getPinyinTone(pinyin) {
319 | const tones = {
320 | ā: 1,
321 | á: 2,
322 | ǎ: 3,
323 | à: 4,
324 | //
325 | ō: 1,
326 | ó: 2,
327 | ǒ: 3,
328 | ò: 4,
329 | //
330 | ē: 1,
331 | é: 2,
332 | ě: 3,
333 | è: 4,
334 | ê: 0,
335 | ê̄: 1,
336 | ế: 2,
337 | ê̌: 3,
338 | ề: 4,
339 | //
340 | ī: 1,
341 | í: 2,
342 | ǐ: 3,
343 | ì: 4,
344 | //
345 | ū: 1,
346 | ú: 2,
347 | ǔ: 3,
348 | ù: 4,
349 | //
350 | ǖ: 1,
351 | ǘ: 2,
352 | ǚ: 3,
353 | ǜ: 4,
354 | //
355 | ń: 2,
356 | ň: 3,
357 | ǹ: 4,
358 | //
359 | m̄: 1,
360 | ḿ: 2,
361 | m̀: 4
362 | };
363 |
364 | for (let ch in tones) {
365 | if (pinyin.includes(ch)) {
366 | return tones[ch];
367 | }
368 | }
369 |
370 | return 0;
371 | }
372 |
373 | /** 注音去掉声调后的字符组合 */
374 | export function extractZhuyinChars(zhuyin) {
375 | return zhuyin.replaceAll(/[ˊˇˋˉ˙]/g, '');
376 | }
377 |
378 | /**
379 | * 计算两个笔画的相似度（Levenshtein Distance）：
380 | * - [Sort an array by the "Levenshtein Distance" with best performance in Javascript](https://stackoverflow.com/a/11958496)
381 | * - [字符串编辑距离之 Damerau–Levenshtein Distance](https://blog.csdn.net/asty9000/article/details/81570627)
382 | * - [字符串编辑距离之 Levenshtein Distance](https://blog.csdn.net/asty9000/article/details/81384650)
383 | * - [Damerau–Levenshtein distance](https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance)
384 | */
385 | export function calculateStrokeSimilarity(s, t) {
386 | const d = []; // 2d matrix
387 |
388 | // Step 1
389 | const n = s.length;
390 | const m = t.length;
391 |
392 | if (n == 0) return 0;
393 | if (m == 0) return 0;
394 |
395 | // Create an array of arrays in javascript (a descending loop is quicker)
396 | for (let i = n; i >= 0; i--) d[i] = [];
397 |
398 | // Step 2
399 | for (let i = n; i >= 0; i--) d[i][0] = i;
400 | for (let j = m; j >= 0; j--) d[0][j] = j;
401 |
402 | // Step 3
403 | for (let i = 1; i <= n; i++) {
404 | const s_i = s.charAt(i - 1);
405 |
406 | // Step 4
407 | for (let j = 1; j <= m; j++) {
408 | // Check the jagged ld total so far
409 | if (i == j && d[i][j] > 4) return n;
410 |
411 | const t_j = t.charAt(j - 1);
412 | const cost = s_i == t_j ? 0 : 1; // Step 5
413 |
414 | // Calculate the minimum
415 | let mi = d[i - 1][j] + 1;
416 | const b = d[i][j - 1] + 1;
417 | const c = d[i - 1][j - 1] + cost;
418 |
419 | if (b < mi) mi = b;
420 | if (c < mi) mi = c;
421 |
422 | d[i][j] = mi; // Step 6
423 |
424 | // Note: 不做转换变换
425 | // // Damerau transposition
426 | // if (i > 1 && j > 1 && s_i == t.charAt(j - 2) && s.charAt(i - 2) == t_j) {
427 | // d[i][j] = Math.min(d[i][j], d[i - 2][j - 2] + cost);
428 | // }
429 | }
430 | }
431 |
432 | // Step 7
433 | return 1 - d[n][m] / Math.max(n, m);
434 | }
435 |
--------------------------------------------------------------------------------
/tools/pinyin-dict/src/utils/zdic.mjs:
--------------------------------------------------------------------------------
1 | import got from 'got';
2 | import { JSDOM } from 'jsdom';
3 |
4 | import {
5 | sleep,
6 | splitChars,
7 | hasGlyphFontForCodePoint,
8 | naiveHTMLNodeInnerText
9 | } from './utils.mjs';
10 |
11 | // 根据 zdic.net 获取字的详细数据
12 | const baseUrl = 'https://www.zdic.net/hans/';
13 | const gotOptions = { timeout: { connect: 50000 } };
14 |
15 | export async function fetchWordMetas(words) {
16 | return await Promise.all(words.map(fetchWordMeta));
17 | }
18 |
19 | async function fetchWordMeta(word) {
20 | const srcUrl = baseUrl + word;
21 | // const html = await (await fetch(srcUrl)).text();
22 | const html = await got(srcUrl, gotOptions).text();
23 | const $dom = new JSDOM(html);
24 | const $doc = (($dom || {}).window || {}).document;
25 | if (!$doc) {
26 | return { value: word };
27 | }
28 |
29 | const wordMeta = {
30 | value: word,
31 | unicode: '',
32 | src_url: srcUrl,
33 | glyph_svg_url: '',
34 | glyph_gif_url: '',
35 | glyph_struct: '',
36 | glyph_font_exists: true,
37 | // 注音与拼音的区别和历史: https://sspai.com/post/75248
38 | pinyins: [],
39 | zhuyins: [],
40 | radical: '',
41 | stroke_order: '',
42 | total_stroke_count: 0,
43 | radical_stroke_count: 0,
44 | traditional: false,
45 | simple_words: [],
46 | variant_words: [],
47 | traditional_words: [],
48 | wubi_codes: [],
49 | cangjie_codes: [],
50 | zhengma_codes: [],
51 | sijiao_codes: [],
52 | phrases: []
53 | };
54 |
55 | // 字形图片和笔顺动画
56 | const $img = $doc.querySelector('.ziif .zipic img');
57 | if ($img) {
58 | const src = $img.getAttribute('src');
59 | const gif = $img.getAttribute('data-gif');
60 |
61 | src && (wordMeta.glyph_svg_url = 'https:' + src);
62 | gif && (wordMeta.glyph_gif_url = 'https:' + gif);
63 | }
64 |
65 | // 拼音
66 | const $pinyin = $doc.querySelectorAll('.ziif .dsk .z_py .z_d');
67 | $pinyin.forEach(($el) => {
68 | const value = naiveHTMLNodeInnerText($el).trim();
69 | const $audio = $el.querySelector('a[data-src-mp3]');
70 | const audio = ($audio && $audio.getAttribute('data-src-mp3')) || '';
71 |
72 | wordMeta.pinyins.push({
73 | value,
74 | audio_url: audio ? 'https:' + audio : ''
75 | });
76 | });
77 |
78 | // 注音，与拼音按顺序对应
79 | const $zhuyin = $doc.querySelectorAll('.ziif .dsk .z_zy .z_d');
80 | $zhuyin.forEach(($el) => {
81 | const value = naiveHTMLNodeInnerText($el).trim();
82 | const $audio = $el.querySelector('a[data-src-mp3]');
83 | const audio = ($audio && $audio.getAttribute('data-src-mp3')) || '';
84 |
85 | wordMeta.zhuyins.push({
86 | value,
87 | audio_url: audio ? 'https:' + audio : ''
88 | });
89 | });
90 |
91 | // 总笔画数
92 | const $totalStrokeCount = $doc.querySelector('.ziif .dsk .z_bs2 .z_ts3');
93 | $totalStrokeCount &&
94 | (wordMeta.total_stroke_count = parseInt(
95 | naiveHTMLNodeInnerText($totalStrokeCount.parentElement)
96 | .replaceAll(/^.+\s+/g, '')
97 | .trim()
98 | ));
99 |
100 | // 部首、部外笔画数
101 | const $radical = $doc.querySelectorAll('.ziif .dsk .z_bs2 .z_ts2');
102 | $radical.forEach(($el) => {
103 | const text = naiveHTMLNodeInnerText($el.parentElement);
104 | const value = text.replaceAll(/^.+\s+/g, '').trim();
105 |
106 | if (text.includes('部首')) {
107 | wordMeta.radical = value;
108 | } else if (text.includes('部外')) {
109 | wordMeta.radical_stroke_count = Math.max(
110 | 0,
111 | wordMeta.total_stroke_count - parseInt(value)
112 | );
113 | }
114 | });
115 |
116 | // 简繁字
117 | const $jianfan = $doc.querySelectorAll('.ziif .dsk .z_jfz > p > a');
118 | $jianfan.forEach(($el) => {
119 | if ($el.querySelector('img')) {
120 | return;
121 | }
122 |
123 | const parentText = naiveHTMLNodeInnerText($el.parentElement);
124 | const value = naiveHTMLNodeInnerText($el).trim();
125 |
126 | if (parentText.includes('繁体')) {
127 | wordMeta.traditional = false;
128 | wordMeta.traditional_words = value.split(/\s+/g);
129 | } else if (parentText.includes('简体')) {
130 | wordMeta.traditional = true;
131 | wordMeta.simple_words = value.split(/\s+/g);
132 | }
133 | });
134 |
135 | // 异体字
136 | const $variant = $doc.querySelectorAll('.ziif .dsk .z_ytz2 > a');
137 | $variant.forEach(($el) => {
138 | if ($el.querySelector('img')) {
139 | return;
140 | }
141 |
142 | const value = naiveHTMLNodeInnerText($el).trim();
143 | wordMeta.variant_words.push(value);
144 | });
145 |
146 | // 笔顺
147 | const $strokeOrder = $doc.querySelector('.ziif .dsk .z_bis2');
148 | $strokeOrder &&
149 | (wordMeta.stroke_order = naiveHTMLNodeInnerText($strokeOrder).trim());
150 |
151 | // 编码信息
152 | const codeTitles = [];
153 | const $codeTitle = $doc.querySelectorAll('.ziif .dsk .dsk_2_1 > p > span');
154 | $codeTitle.forEach(($el) => {
155 | const value = naiveHTMLNodeInnerText($el).trim();
156 |
157 | codeTitles.push(value);
158 | });
159 |
160 | const codes = [];
161 | $doc.querySelectorAll('.ziif .dsk .dsk_2_1').forEach(($el) => {
162 | const value = naiveHTMLNodeInnerText($el).trim();
163 |
164 | if (!codeTitles.includes(value)) {
165 | codes.push(value);
166 | }
167 | });
168 | for (let i = 0; i < codeTitles.length; i++) {
169 | const title = codeTitles[i];
170 | const value = codes[i];
171 |
172 | if (title === '统一码') {
173 | wordMeta.unicode = value.replaceAll(/^.+(U\+.+)\s*/g, '$1');
174 | } else if (title === '字形分析') {
175 | wordMeta.glyph_struct = value;
176 | } else if (title === '五笔') {
177 | wordMeta.wubi_codes = value.split(/\|/g);
178 | } else if (title === '仓颉') {
179 | wordMeta.cangjie_codes = value.split(/\|/g);
180 | } else if (title === '郑码') {
181 | wordMeta.zhengma_codes = value.split(/\|/g);
182 | } else if (title === '四角') {
183 | wordMeta.sijiao_codes = value.split(/\|/g);
184 | }
185 | }
186 |
187 | wordMeta.glyph_font_exists = hasGlyphFontForCodePoint(wordMeta.unicode);
188 |
189 | // 词组、短语
190 | const phrases = [];
191 | const $phrase = $doc.querySelectorAll('.crefe');
192 | $phrase.forEach((el) => {
193 | const text = naiveHTMLNodeInnerText(el).trim();
194 |
195 | phrases.push(text);
196 | });
197 |
198 | const batchSize = 10;
199 | for (let i = 0; i < phrases.length; i += batchSize) {
200 | const phraseMetas = await Promise.all(
201 | phrases.slice(i, i + batchSize).map(fetchPhraseMeta)
202 | );
203 |
204 | phraseMetas.forEach((phrase) => {
205 | wordMeta.phrases.push(...phrase);
206 | });
207 |
208 | await sleep(1500);
209 | }
210 |
211 | return wordMeta;
212 | }
213 |
214 | async function fetchPhraseMeta(phrase) {
215 | const srcUrl = baseUrl + phrase;
216 | // const html = await (await fetch(srcUrl)).text();
217 | const html = await got(srcUrl, gotOptions).text();
218 | const $dom = new JSDOM(html);
219 | const $doc = (($dom || {}).window || {}).document;
220 |
221 | // https://www.zdic.net/hans/不塞不流，不止不行
222 | const phrases = phrase.split(/[，,、；]/g);
223 | const phraseCharsArray = phrases
224 | .map((p) => splitChars(p))
225 | .filter((p) => p.length > 1);
226 | if (!$doc || phraseCharsArray.length === 0) {
227 | return phraseCharsArray.map((p) => ({
228 | value: p,
229 | pinyins: [],
230 | zhuyins: []
231 | }));
232 | }
233 |
234 | // 拼音及注音
235 | const pinyinsArray = [];
236 | const zhuyinsArray = [];
237 | const $duyin = $doc.querySelectorAll('.ciif p .z_ts2');
238 | $duyin.forEach(($el) => {
239 | const text = naiveHTMLNodeInnerText($el);
240 | const $dicpy = $el.parentElement.querySelectorAll('.dicpy');
241 |
242 | $dicpy.forEach(($e) => {
243 | naiveHTMLNodeInnerText($e)
244 | .split(/[，,；]/g)
245 | .forEach((val) => {
246 | const splits = val
247 | .replaceAll(/[·]/g, '')
248 | .replaceAll(/\s+([ˊˇˋˉ])/g, '$1')
249 | .split(/\s+/g);
250 |
251 | if (text === '拼音') {
252 | pinyinsArray.push({ value: splits });
253 | } else if (text === '注音') {
254 | zhuyinsArray.push({ value: splits });
255 | }
256 | });
257 | });
258 | });
259 |
260 | const metas = [];
261 | if (pinyinsArray.length === 0) {
262 | return metas;
263 | }
264 |
265 | phraseCharsArray.forEach((chars, i) => {
266 | metas.push({
267 | value: chars,
268 | src_url: srcUrl,
269 | pinyins: [pinyinsArray[i]],
270 | zhuyins: [zhuyinsArray[i]]
271 | });
272 | });
273 |
274 | return metas;
275 | }
276 |
--------------------------------------------------------------------------------
7 |
8 | > To download the latest version for Android,
9 | > please go to https://github.com/crazydan-studio/kuaizi-ime-android/releases/latest
10 |
11 | [
](https://f-droid.org/packages/org.crazydan.studio.app.ime.kuaizi)
14 |
15 | > **Warning**：筷字输入法(Kuaizi IME) is currently exclusively released
16 | > on [F-Droid](https://f-droid.org) and has not been published on other application platforms.
17 | > Please be cautious and aware of related risks. If you need to download and use it,
18 | > please use the link above.
19 |
20 | > If you find any errors or have improvement suggestions,
21 | > please visit [Issues](https://github.com/crazydan-studio/kuaizi-ime/issues)
22 |
23 | **筷字输入法** (Kuaizi IME) is a system based on the combinatorial features of Chinese Pinyin.
24 | It is an efficient and fast input method editor, suitable for electronic touchscreen devices.
25 |
26 | **筷字输入法** (Kuaizi IME) abandones the traditional method of tapping on virtual keyboard keys one by one,
27 | it instead offers a swipe input method for Pinyin, combined with the combinatorial features of Pinyin letters.
28 | The key layout is designed to make Pinyin input more accurate and convenient.
29 |
30 | **筷字输入法** (Kuaizi IME) provides editing support such as cursor movement
31 | and text selection for the target editor, thereby eliminating the frustration
32 | of being unable to accurately position inputs and text due to the small screen size of mobile devices.
33 | This further enhances text input and editing efficiency on small-screen devices.
34 |
35 | User-data generated by **筷字输入法** (Kuaizi IME) is stored locally.
36 | It does not connect to the internet and will not collect nor analyze user data or behavior habits.
37 | Furthermore, it does not support or provide predictive input nor fuzzy matching mechanisms.
38 | To some extent, this input method editor aims to enhance the users' ability to recognize
39 | and memorize Chinese characters and Pinyin, ensuring that the reliance on digital tools
40 | does not lead to the gradual forgetting and eventual abandonment of
41 | 「汉字」 (Hànzì, Chinese characters), a treasure of Chinese civilization.
42 |
43 | ## About the icon
44 |
45 | **筷字输入法** The icon is composed of the Chinese character 「字」 (zì, character)
46 | and the Chinese-style utensil 「筷子」 (Kuàizi, chopsticks).
47 | This combination closely links Chinese civilization with the unique culinary culture of its people,
48 | showcasing the long history of Chinese civilization.
49 | It also helps to spread traditional Chinese culture to the world,
50 | deepening global friends' understanding and appreciation of Chinese culture.
51 | At the same time, it allows the Chinese people to have a stronger confidence in their national culture.
52 |
53 | ## How to clone the repository
54 |
55 | ```bash
56 | git clone git@github.com:crazydan-studio/kuaizi-ime.git
57 |
58 | cd kuaizi-ime
59 | git submodule update --init android
60 | ```
61 |
62 | > For detailed build instructions, please refer to the README documentation of each module.
63 |
64 | ## Project structure
65 |
66 | - [android/](https://github.com/crazydan-studio/kuaizi-ime-android):
67 | The complete project code for the 筷字输入法 (Kuaizi IME) Android client.
68 | Note: Use the command `git submodule update --init android` to initialize this git submodule.
69 | - [docs/](./docs/): The documentation directory of this project.
70 | - [Cross-shaped input interaction design](./docs/design/cross-mode.md):
71 | Optimization and improvement plan for cross-shaped input.
72 | - [tools/analyze/](./tools/analyze/):
73 | Pinyin key layout online analysis tool, used to analyze the letter combination structure of Pinyin
74 | and validate the layout scheme of the regular hexagonal keys.
75 | - [tools/pinyin-dict/](./tools/pinyin-dict/):
76 | Pinyin dictionary data collection and correction program,
77 | which collects Chinese character data and generates an SQLite database.
78 | The character/dictionary database for the 筷字输入法 (Kuaizi IME) client is also generated by this tool.
79 | - [thirdparty/](./thirdparty/):
80 | Third-party character and word data used in this project. **Note:** Use the command
81 | `git submodule update --init thirdparty/OpenCC thirdparty/pinyin-data`
82 | to initialize the git-submodules within this directory.
83 |
84 | ## License
85 |
86 | [Apache 2.0](./LICENSE)
87 |
88 | ## Donations
89 |
90 | **Note**：Please add the remark `筷字输入法` when donating.
91 | For a complete list of donations, please check the [Donation list](./docs/donate/index.md)。
92 |
93 | | 支付宝 (Alipay) | 微信支付 (WeChat Pay) |
94 | | -- | -- |
95 | | 
| 
|
96 |
97 | ## Feature Highlights
98 |
99 | - Input Pinyin via a swiping motion, with a maximum of only two swipes required for a single input.
100 | - Built-in
101 | [8VIM](https://github.com/8VIM/8VIM)/[8pen](./docs/Increasing_the_Efficiency_of_Text_Input_in_the_8pen_Method.pdf)
102 | input mode, allowing continuous input of Pinyin, English, and numbers by drawing circles,
103 | providing a smooth input experience.
104 | - All candidate characters are displayed with their complete Pinyin,
105 | making it easy to identify the accurate pronunciation of each character.
106 | - Input data is stored locally; no collection or analysis of user data.
107 | - Utilizes Hidden Markov Models (HMM) and the Viterbi algorithm for Pinyin input prediction.
108 | - Supports only precise Pinyin input matching; does not support Pinyin predictive input or fuzzy matching mechanisms.
109 | - Provides editing functions such as cursor movement, text selection, copy, paste, and cut.
110 | Additionally, supports undoing inputs submitted to the target editor for easy corrections.
111 | - Supports left-handed and right-handed mode switching to accommodate different user habits.
112 | - Supports basic mathematical operations (addition, subtraction, multiplication, and division),
113 | allowing calculations while typing.
114 | - Supports input of punctuation, emojis, and other text symbols, offering rich forms of expression.
115 | - Supports direct input of paired punctuation marks (e.g., brackets, quotes)
116 | and automatically wraps selected content within them.
117 | - Automatically adds spaces between Latin characters and Chinese characters,
118 | as well as between operands and operators, to standardize input formatting.
119 |
120 | https://github.com/user-attachments/assets/c5a3c769-0e6d-42e1-bc2f-babe85607bfb
121 |
122 | > Note: The audio and subtitles were provided by [GitPodcast](https://www.gitpodcast.com/),
123 | > the final videos were generated and produced using [Veed](https://www.veed.io)
124 |
125 | 
126 |
127 | ## Feature demonstration
128 |
129 | > The latest version's key layout may differ from the demonstration videos.
130 | > Please refer to the latest version.
131 |
132 | ### Pinyin Swipe Input
133 |
134 | https://github.com/crazydan-studio/kuaizi-ime/assets/1321315/eb2f74f9-f64e-4d02-ad80-98e3ecb9d61b
135 |
136 | ### Arithmetic Input
137 |
138 | https://github.com/crazydan-studio/kuaizi-ime/assets/1321315/f461b564-0ac4-4257-82ad-11afcd3e1d6c
139 |
140 | ### Content Editing
141 |
142 | https://github.com/crazydan-studio/kuaizi-ime/assets/1321315/992a0a5e-7e1e-4b93-a1ac-c893d0e3ff2e
143 |
144 | ### Emoji Matching
145 |
146 | https://github.com/crazydan-studio/kuaizi-ime/assets/1321315/7fff6ddf-9e10-408f-b160-3b3b8e2ab215
147 |
148 | ### Paired Symbol Input
149 |
150 | https://github.com/crazydan-studio/kuaizi-ime/assets/1321315/97b0f99b-92e1-4b28-a5b6-d45150c4bada
151 |
152 | ### Pinyin Input Submission Options
153 |
154 | https://github.com/crazydan-studio/kuaizi-ime/assets/1321315/274d41dd-e858-4b71-a041-31df3dd24f7d
155 |
156 | ### 8VIM/8pen-style Input
157 |
158 | https://github.com/crazydan-studio/kuaizi-ime/assets/1321315/534fa61e-34dc-4e81-a7d1-5eb7cc3b291f
159 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | 筷字（Kuaizi）输入法
2 | =======================================
3 |
4 | 中文版 | [English](./README-en.md)
5 |
6 |