├── data
    ├── dict_pinyin.dat
    ├── user_dict_pinyin
    ├── valid_utf16.txt
    ├── user_dict_pinyin.dat
    └── rawdict_utf16_65105_freq.txt
├── main.cpp
├── googlelib
    ├── pinyinime_global.h
    ├── lib.pri
    ├── mystdlib.h
    ├── mystdlib.cpp
    ├── utf16reader.h
    ├── utf16char.h
    ├── lpicache.h
    ├── sync.h
    ├── lpicache.cpp
    ├── sync.cpp
    ├── ngram.h
    ├── utf16reader.cpp
    ├── spellingtable.h
    ├── dictlist.h
    ├── splparser.h
    ├── utf16char.cpp
    ├── dictdef.h
    ├── pinyinime.cpp
    ├── searchutility.h
    ├── dictbuilder.h
    ├── searchutility.cpp
    ├── pinyinime.h
    ├── spellingtable.cpp
    ├── dicttrie.h
    ├── spellingtrie.h
    ├── ngram.cpp
    ├── splparser.cpp
    ├── atomdictbase.h
    ├── dictlist.cpp
    ├── userdict.h
    └── matrixsearch.h
├── MyVirtualKeyboard.pro
├── customerqpushbutton.h
├── customerqpushbutton.cpp
├── virtualkeyboard.h
├── keyboard.h
├── README.md
└── LICENSE


/data/dict_pinyin.dat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aron566/google_pinyinim/HEAD/data/dict_pinyin.dat


--------------------------------------------------------------------------------
/data/user_dict_pinyin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aron566/google_pinyinim/HEAD/data/user_dict_pinyin


--------------------------------------------------------------------------------
/data/valid_utf16.txt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aron566/google_pinyinim/HEAD/data/valid_utf16.txt


--------------------------------------------------------------------------------
/data/user_dict_pinyin.dat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aron566/google_pinyinim/HEAD/data/user_dict_pinyin.dat


--------------------------------------------------------------------------------
/data/rawdict_utf16_65105_freq.txt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aron566/google_pinyinim/HEAD/data/rawdict_utf16_65105_freq.txt


--------------------------------------------------------------------------------
/main.cpp:
--------------------------------------------------------------------------------
 1 | #include "keyboard.h"
 2 | 
 3 | #include <QApplication>
 4 | 
 5 | int main(int argc, char *argv[])
 6 | {
 7 |     QApplication a(argc, argv);
 8 |     keyboard w;
 9 |     w.show();
10 |     return a.exec();
11 | }
12 | 


--------------------------------------------------------------------------------
/googlelib/pinyinime_global.h:
--------------------------------------------------------------------------------
 1 | #ifndef PINYINIME_GLOBAL_H
 2 | #define PINYINIME_GLOBAL_H
 3 | 
 4 | #include <QtCore/qglobal.h>
 5 | 
 6 | #if defined(PINYINIME_LIBRARY)
 7 | #  define PINYINIMESHARED_EXPORT Q_DECL_EXPORT
 8 | #else
 9 | #  define PINYINIMESHARED_EXPORT Q_DECL_IMPORT
10 | #endif
11 | 
12 | #endif // PINYINIME_GLOBAL_H
13 | 


--------------------------------------------------------------------------------
/googlelib/lib.pri:
--------------------------------------------------------------------------------
 1 | INCLUDEPATH += $$PWD
 2 | 
 3 | HEADERS += \
 4 |     $$PWD/atomdictbase.h \
 5 |     $$PWD/dictbuilder.h \
 6 |     $$PWD/dictdef.h \
 7 |     $$PWD/dictlist.h \
 8 |     $$PWD/dicttrie.h \
 9 |     $$PWD/lpicache.h \
10 |     $$PWD/matrixsearch.h \
11 |     $$PWD/mystdlib.h \
12 |     $$PWD/ngram.h \
13 |     $$PWD/pinyinime.h \
14 |     $$PWD/pinyinime_global.h \
15 |     $$PWD/searchutility.h \
16 |     $$PWD/spellingtable.h \
17 |     $$PWD/spellingtrie.h \
18 |     $$PWD/splparser.h \
19 |     $$PWD/sync.h \
20 |     $$PWD/userdict.h \
21 |     $$PWD/utf16char.h \
22 |     $$PWD/utf16reader.h
23 | 
24 | SOURCES += \
25 |     $$PWD/dictbuilder.cpp \
26 |     $$PWD/dictlist.cpp \
27 |     $$PWD/dicttrie.cpp \
28 |     $$PWD/lpicache.cpp \
29 |     $$PWD/matrixsearch.cpp \
30 |     $$PWD/mystdlib.cpp \
31 |     $$PWD/ngram.cpp \
32 |     $$PWD/pinyinime.cpp \
33 |     $$PWD/searchutility.cpp \
34 |     $$PWD/spellingtable.cpp \
35 |     $$PWD/spellingtrie.cpp \
36 |     $$PWD/splparser.cpp \
37 |     $$PWD/sync.cpp \
38 |     $$PWD/userdict.cpp \
39 |     $$PWD/utf16char.cpp \
40 |     $$PWD/utf16reader.cpp
41 | 


--------------------------------------------------------------------------------
/googlelib/mystdlib.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (C) 2009 The Android Open Source Project
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *      http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | 
17 | #ifndef PINYINIME_INCLUDE_MYSTDLIB_H__
18 | #define PINYINIME_INCLUDE_MYSTDLIB_H__
19 | 
20 | #include <stdlib.h>
21 | 
22 | namespace ime_pinyin {
23 | 
24 | void myqsort(void *p, size_t n, size_t es,
25 |              int (*cmp)(const void *, const void *));
26 | 
27 | void *mybsearch(const void *key, const void *base,
28 |                 size_t nmemb, size_t size,
29 |                 int (*compar)(const void *, const void *));
30 | }
31 | 
32 | #endif  // PINYINIME_INCLUDE_MYSTDLIB_H__
33 | 


--------------------------------------------------------------------------------
/googlelib/mystdlib.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (C) 2009 The Android Open Source Project
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *      http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | 
17 | #include <stdlib.h>
18 | 
19 | namespace ime_pinyin {
20 | 
21 | // For debug purpose. You can add a fixed version of qsort and bsearch functions
22 | // here so that the output will be totally the same under different platforms.
23 | 
24 | void myqsort(void *p, size_t n, size_t es,
25 |              int (*cmp)(const void *, const void *)) {
26 |   qsort(p,n, es, cmp);
27 | }
28 | 
29 | void *mybsearch(const void *k, const void *b,
30 |                 size_t n, size_t es,
31 |                 int (*cmp)(const void *, const void *)) {
32 |   return bsearch(k, b, n, es, cmp);
33 | }
34 | }  // namespace ime_pinyin
35 | 


--------------------------------------------------------------------------------
/MyVirtualKeyboard.pro:
--------------------------------------------------------------------------------
 1 | QT       += core gui
 2 | 
 3 | greaterThan(QT_MAJOR_VERSION, 4): QT += widgets
 4 | 
 5 | CONFIG += c++11
 6 | 
 7 | # 生成库文件
 8 | TARGET = virtualkeyboard
 9 | TEMPLATE = lib
10 | # 只生成静态库
11 | #CONFIG += staticlib
12 | # 导出到dll
13 | DEFINES += VIRTUALKEYBOARD_LIBRARY
14 | 
15 | # 谷歌拼音库的依赖
16 | #DEFINES += PINYINIME_LIBRARY
17 | include(googlelib/lib.pri)
18 | 
19 | # The following define makes your compiler emit warnings if you use
20 | # any Qt feature that has been marked deprecated (the exact warnings
21 | # depend on your compiler). Please consult the documentation of the
22 | # deprecated API in order to know how to port your code away from it.
23 | DEFINES += QT_DEPRECATED_WARNINGS
24 | 
25 | # You can also make your code fail to compile if it uses deprecated APIs.
26 | # In order to do so, uncomment the following line.
27 | # You can also select to disable deprecated APIs only up to a certain version of Qt.
28 | #DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000    # disables all the APIs deprecated before Qt 6.0.0
29 | 
30 | SOURCES += \
31 |     customerqpushbutton.cpp \
32 |     keyboard.cpp
33 | #    main.cpp #\单键盘页面测试使用，编译库时不可带入
34 | 
35 | HEADERS += \
36 |     customerqpushbutton.h \
37 |     keyboard.h \
38 |     virtualkeyboard.h \
39 |     ui_keyboard.h
40 | # 屏蔽，否则打开项目会覆盖相对位置信息
41 | #FORMS += \
42 | #    keyboard.ui
43 | 
44 | # Default rules for deployment.
45 | qnx: target.path = /tmp/$${TARGET}/bin
46 | else: unix:!android: target.path = /opt/$${TARGET}/bin
47 | !isEmpty(target.path): INSTALLS += target
48 | 


--------------------------------------------------------------------------------
/googlelib/utf16reader.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (C) 2009 The Android Open Source Project
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *      http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | 
17 | #ifndef PINYINIME_INCLUDE_UTF16READER_H__
18 | #define PINYINIME_INCLUDE_UTF16READER_H__
19 | 
20 | #include <stdio.h>
21 | #include "./utf16char.h"
22 | 
23 | namespace ime_pinyin {
24 | 
25 | class Utf16Reader {
26 |  private:
27 |   FILE *fp_;
28 |   char16 *buffer_;
29 |   size_t buffer_total_len_;
30 |   size_t buffer_next_pos_;
31 | 
32 |   // Always less than buffer_total_len_ - buffer_next_pos_
33 |   size_t buffer_valid_len_;
34 | 
35 |  public:
36 |   Utf16Reader();
37 |   ~Utf16Reader();
38 | 
39 |   // filename is the name of the file to open.
40 |   // buffer_len specifies how long buffer should be allocated to speed up the
41 |   // future reading
42 |   bool open(const char* filename, size_t buffer_len);
43 |   char16* readline(char16* read_buf, size_t max_len);
44 |   bool close();
45 | };
46 | }
47 | 
48 | #endif  // PINYINIME_INCLUDE_UTF16READER_H__
49 | 


--------------------------------------------------------------------------------
/customerqpushbutton.h:
--------------------------------------------------------------------------------
 1 | /**
 2 |  *  @file customerqpushbutton.h
 3 |  *
 4 |  *  @date 2020-08-16
 5 |  *
 6 |  *  @author aron566
 7 |  *
 8 |  *  @brief 自定按钮控件
 9 |  *
10 |  *  @version V1.0
11 |  */
12 | #ifndef CUSTOMERQPUSHBUTTON_H
13 | #define CUSTOMERQPUSHBUTTON_H
14 | /** Includes -----------------------------------------------------------------*/
15 | #include <QPushButton>
16 | /** Private includes ---------------------------------------------------------*/
17 | 
18 | /** Private defines ----------------------------------------------------------*/
19 | 
20 | /** Exported typedefines -----------------------------------------------------*/
21 | 
22 | /** Exported constants -------------------------------------------------------*/
23 | 
24 | /** Exported macros-----------------------------------------------------------*/
25 | /** Exported variables -------------------------------------------------------*/
26 | /** Exported functions prototypes --------------------------------------------*/
27 | /**
28 |  * @brief The customerqpushbutton class
29 |  */
30 | class customerqpushbutton : public QPushButton
31 | {
32 |     Q_OBJECT
33 | 
34 | public:
35 |     explicit customerqpushbutton(QWidget *parent = nullptr);
36 |     explicit customerqpushbutton(const QString &text, QWidget *parent = nullptr);
37 |     customerqpushbutton(const QIcon& icon, const QString &text, QWidget *parent = nullptr);
38 |     ~customerqpushbutton();
39 | signals:
40 |     void clicked(customerqpushbutton *pbtn);
41 | protected:
42 |     virtual void mouseReleaseEvent(QMouseEvent *ev);
43 | };
44 | 
45 | #endif // CUSTOMERQPUSHBUTTON_H
46 | /******************************** End of file *********************************/
47 | 


--------------------------------------------------------------------------------
/googlelib/utf16char.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (C) 2009 The Android Open Source Project
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *      http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | 
17 | #ifndef PINYINIME_INCLUDE_UTF16CHAR_H__
18 | #define PINYINIME_INCLUDE_UTF16CHAR_H__
19 | 
20 | #include <stdlib.h>
21 | 
22 | namespace ime_pinyin {
23 | 
24 | #ifdef __cplusplus
25 | extern "C" {
26 | #endif
27 | 
28 |   typedef unsigned short char16;
29 | 
30 |   // Get a token from utf16_str,
31 |   // Returned pointer is a '\0'-terminated utf16 string, or NULL
32 |   // *utf16_str_next returns the next part of the string for further tokenizing
33 |   char16* utf16_strtok(char16 *utf16_str, size_t *token_size,
34 |                        char16 **utf16_str_next);
35 | 
36 |   int utf16_atoi(const char16 *utf16_str);
37 | 
38 |   float utf16_atof(const char16 *utf16_str);
39 | 
40 |   size_t utf16_strlen(const char16 *utf16_str);
41 | 
42 |   int utf16_strcmp(const char16 *str1, const char16 *str2);
43 |   int utf16_strncmp(const char16 *str1, const char16 *str2, size_t size);
44 | 
45 |   char16* utf16_strcpy(char16 *dst, const char16 *src);
46 |   char16* utf16_strncpy(char16 *dst, const char16 *src, size_t size);
47 | 
48 | 
49 |   char* utf16_strcpy_tochar(char *dst, const char16 *src);
50 | 
51 | #ifdef __cplusplus
52 | }
53 | #endif
54 | }
55 | 
56 | #endif  // PINYINIME_INCLUDE_UTF16CHAR_H__
57 | 


--------------------------------------------------------------------------------
/customerqpushbutton.cpp:
--------------------------------------------------------------------------------
 1 | /**
 2 |   ******************************************************************************
 3 |   * @file    customerqpushbutton.cpp
 4 |   * @author  aron566
 5 |   * @version v1.0
 6 |   * @date    2020-08-16
 7 |   * @brief   自定义按钮控件.
 8 |   ******************************************************************************
 9 |   */
10 | /* Header includes -----------------------------------------------------------*/
11 | #include "customerqpushbutton.h"
12 | /* Macro definitions ---------------------------------------------------------*/
13 | /* Type definitions ----------------------------------------------------------*/
14 | /* Variable declarations -----------------------------------------------------*/
15 | /* Variable definitions ------------------------------------------------------*/
16 | /* Function declarations -----------------------------------------------------*/
17 | /* Function definitions ------------------------------------------------------*/
18 | /**
19 |  * @brief customerqpushbutton::customerqpushbutton
20 |  * @param parent
21 |  */
22 | customerqpushbutton::customerqpushbutton(QWidget *parent)
23 |     :QPushButton(parent)
24 | {
25 | 
26 | }
27 | 
28 | /**
29 |  * @brief customerqpushbutton::customerqpushbutton
30 |  * @param text
31 |  * @param parent
32 |  */
33 | customerqpushbutton::customerqpushbutton(const QString &text, QWidget *parent)
34 |     :QPushButton(text ,parent)
35 | {
36 | 
37 | }
38 | 
39 | /**
40 |  * @brief customerqpushbutton::customerqpushbutton
41 |  * @param icon
42 |  * @param text
43 |  * @param parent
44 |  */
45 | customerqpushbutton::customerqpushbutton(const QIcon& icon, const QString &text, QWidget *parent)
46 |     :QPushButton(icon ,text ,parent)
47 | {
48 | 
49 | }
50 | 
51 | /**
52 |  * @brief customerqpushbutton::~customerqpushbutton
53 |  */
54 | customerqpushbutton::~customerqpushbutton()
55 | {
56 | 
57 | }
58 | 
59 | /**
60 |  * @brief customerqlabel::mouseReleaseEvent
61 |  * @param ev
62 |  */
63 | void customerqpushbutton::mouseReleaseEvent(QMouseEvent *ev)
64 | {
65 |     Q_UNUSED(ev)
66 |     emit clicked(this);
67 | }
68 | /* ---------------------------- end of file ----------------------------------*/
69 | 


--------------------------------------------------------------------------------
/googlelib/lpicache.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (C) 2009 The Android Open Source Project
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *      http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | 
17 | #ifndef PINYINIME_ANDPY_INCLUDE_LPICACHE_H__
18 | #define PINYINIME_ANDPY_INCLUDE_LPICACHE_H__
19 | 
20 | #include <stdlib.h>
21 | #include "./searchutility.h"
22 | #include "./spellingtrie.h"
23 | 
24 | namespace ime_pinyin {
25 | 
26 | // Used to cache LmaPsbItem list for half spelling ids.
27 | class LpiCache {
28 |  private:
29 |   static LpiCache *instance_;
30 |   static const int kMaxLpiCachePerId = 15;
31 | 
32 |   LmaPsbItem *lpi_cache_;
33 |   uint16 *lpi_cache_len_;
34 | 
35 |  public:
36 |   LpiCache();
37 |   ~LpiCache();
38 | 
39 |   static LpiCache& get_instance();
40 | 
41 |   // Test if the LPI list of the given splid  has been cached.
42 |   // If splid is a full spelling id, it returns false, because we only cache
43 |   // list for half ids.
44 |   bool is_cached(uint16 splid);
45 | 
46 |   // Put LPI list to cahce. If the length of the list, lpi_num, is longer than
47 |   // the cache buffer. the list will be truncated, and function returns the
48 |   // maximum length of the cache buffer.
49 |   // Note: splid must be a half id, and lpi_items must be not NULL. The
50 |   // caller of this function should guarantee this.
51 |   size_t put_cache(uint16 splid, LmaPsbItem lpi_items[], size_t lpi_num);
52 | 
53 |   // Get the cached list for the given half id.
54 |   // Return the length of the cached buffer.
55 |   // Note: splid must be a half id, and lpi_items must be not NULL. The
56 |   // caller of this function should guarantee this.
57 |   size_t get_cache(uint16 splid, LmaPsbItem lpi_items[], size_t lpi_max);
58 | };
59 | 
60 | }  // namespace
61 | 
62 | #endif  // PINYINIME_ANDPY_INCLUDE_LPICACHE_H__
63 | 


--------------------------------------------------------------------------------
/googlelib/sync.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (C) 2009 The Android Open Source Project
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *      http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | 
17 | #ifndef PINYINIME_INCLUDE_SYNC_H__
18 | #define PINYINIME_INCLUDE_SYNC_H__
19 | 
20 | #define ___SYNC_ENABLED___
21 | 
22 | #ifdef ___SYNC_ENABLED___
23 | 
24 | #include "userdict.h"
25 | 
26 | namespace ime_pinyin {
27 | 
28 | // Class for user dictionary synchronization
29 | // This class is not thread safe
30 | // Normal invoking flow will be
31 | //   begin() ->
32 | //   put_lemmas() x N ->
33 | //   {
34 | //     get_lemmas() ->
35 | //     [ get_last_got_count() ] ->
36 | //     clear_last_got() ->
37 | //   } x N ->
38 | //   finish()
39 | class Sync {
40 |  public:
41 |   Sync();
42 |   ~Sync();
43 | 
44 |   static const int kUserDictMaxLemmaCount = 5000;
45 |   static const int kUserDictMaxLemmaSize = 200000;
46 |   static const int kUserDictRatio = 20;
47 | 
48 |   bool begin(const char * filename);
49 | 
50 |   // Merge lemmas downloaded from sync server into local dictionary
51 |   // lemmas, lemmas string encoded in UTF16LE
52 |   // len, length of lemmas string
53 |   // Return how many lemmas merged successfully
54 |   int put_lemmas(char16 * lemmas, int len);
55 | 
56 |   // Get local new user lemmas into UTF16LE string
57 |   // str, buffer ptr to store new user lemmas
58 |   // size, size of buffer
59 |   // Return length of returned buffer in measure of UTF16LE
60 |   int get_lemmas(char16 * str, int size);
61 | 
62 |   // Return lemmas count in last get_lemmas()
63 |   int get_last_got_count();
64 | 
65 |   // Return total lemmas count need get_lemmas()
66 |   int get_total_count();
67 | 
68 |   // Clear lemmas got by recent get_lemmas()
69 |   void clear_last_got();
70 | 
71 |   void finish();
72 | 
73 |   int get_capacity();
74 | 
75 |  private:
76 |   UserDict * userdict_;
77 |   char * dictfile_;
78 |   int last_count_;
79 | };
80 | 
81 | }
82 | 
83 | #endif
84 | 
85 | #endif  // PINYINIME_INCLUDE_SYNC_H__
86 | 


--------------------------------------------------------------------------------
/googlelib/lpicache.cpp:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (C) 2009 The Android Open Source Project
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *      http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | 
17 | #include <assert.h>
18 | #include "lpicache.h"
19 | 
20 | namespace ime_pinyin {
21 | 
22 | LpiCache* LpiCache::instance_ = NULL;
23 | 
24 | LpiCache::LpiCache() {
25 |   lpi_cache_ = new LmaPsbItem[kFullSplIdStart * kMaxLpiCachePerId];
26 |   lpi_cache_len_ = new uint16[kFullSplIdStart];
27 |   assert(NULL != lpi_cache_);
28 |   assert(NULL != lpi_cache_len_);
29 |   for (uint16 id = 0; id < kFullSplIdStart; id++)
30 |     lpi_cache_len_[id] = 0;
31 | }
32 | 
33 | LpiCache::~LpiCache() {
34 |   if (NULL != lpi_cache_)
35 |     delete [] lpi_cache_;
36 | 
37 |   if (NULL != lpi_cache_len_)
38 |     delete [] lpi_cache_len_;
39 | }
40 | 
41 | LpiCache& LpiCache::get_instance() {
42 |   if (NULL == instance_) {
43 |     instance_ = new LpiCache();
44 |     assert(NULL != instance_);
45 |   }
46 |   return *instance_;
47 | }
48 | 
49 | bool LpiCache::is_cached(uint16 splid) {
50 |   if (splid >= kFullSplIdStart)
51 |     return false;
52 |   return lpi_cache_len_[splid] != 0;
53 | }
54 | 
55 | size_t LpiCache::put_cache(uint16 splid, LmaPsbItem lpi_items[],
56 |                            size_t lpi_num) {
57 |   uint16 num = kMaxLpiCachePerId;
58 |   if (num > lpi_num)
59 |     num = static_cast<uint16>(lpi_num);
60 | 
61 |   LmaPsbItem *lpi_cache_this = lpi_cache_ + splid * kMaxLpiCachePerId;
62 |   for (uint16 pos = 0; pos < num; pos++)
63 |     lpi_cache_this[pos] = lpi_items[pos];
64 | 
65 |   lpi_cache_len_[splid] = num;
66 |   return num;
67 | }
68 | 
69 | size_t LpiCache::get_cache(uint16 splid, LmaPsbItem lpi_items[],
70 |                            size_t lpi_max) {
71 |   if (lpi_max > lpi_cache_len_[splid])
72 |     lpi_max = lpi_cache_len_[splid];
73 | 
74 |   LmaPsbItem *lpi_cache_this = lpi_cache_ + splid * kMaxLpiCachePerId;
75 |   for (uint16 pos = 0; pos < lpi_max; pos++) {
76 |     lpi_items[pos] = lpi_cache_this[pos];
77 |   }
78 |   return lpi_max;
79 | }
80 | 
81 | }  // namespace ime_pinyin
82 | 


--------------------------------------------------------------------------------
/googlelib/sync.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #include "sync.h"
 18 | #include <assert.h>
 19 | #include <string.h>
 20 | 
 21 | #ifdef ___SYNC_ENABLED___
 22 | 
 23 | namespace ime_pinyin {
 24 | 
 25 | Sync::Sync()
 26 |   : userdict_(NULL),
 27 |     dictfile_(NULL),
 28 |     last_count_(0) {
 29 | };
 30 | 
 31 | Sync::~Sync() {
 32 | }
 33 | 
 34 | 
 35 | bool Sync::begin(const char * filename) {
 36 |   if (userdict_) {
 37 |     finish();
 38 |   }
 39 | 
 40 |   if (!filename) {
 41 |     return false;
 42 |   }
 43 | 
 44 |   dictfile_ = strdup(filename);
 45 |   if (!dictfile_) {
 46 |     return false;
 47 |   }
 48 | 
 49 |   userdict_ = new UserDict();
 50 |   if (!userdict_) {
 51 |     free(dictfile_);
 52 |     dictfile_ = NULL;
 53 |     return false;
 54 |   }
 55 | 
 56 |   if (userdict_->load_dict((const char*)dictfile_, kUserDictIdStart,
 57 |                            kUserDictIdEnd) == false) {
 58 |     delete userdict_;
 59 |     userdict_ = NULL;
 60 |     free(dictfile_);
 61 |     dictfile_ = NULL;
 62 |     return false;
 63 |   }
 64 | 
 65 |   userdict_->set_limit(kUserDictMaxLemmaCount, kUserDictMaxLemmaSize, kUserDictRatio);
 66 | 
 67 |   return true;
 68 | }
 69 | 
 70 | int Sync::put_lemmas(char16 * lemmas, int len) {
 71 |   return userdict_->put_lemmas_no_sync_from_utf16le_string(lemmas, len);
 72 | }
 73 | 
 74 | int Sync::get_lemmas(char16 * str, int size) {
 75 |   return userdict_->get_sync_lemmas_in_utf16le_string_from_beginning(str, size, &last_count_);
 76 | }
 77 | 
 78 | int Sync::get_last_got_count() {
 79 |   return last_count_;
 80 | }
 81 | 
 82 | int Sync::get_total_count() {
 83 |   return userdict_->get_sync_count();
 84 | }
 85 | 
 86 | void Sync::clear_last_got() {
 87 |   if (last_count_ < 0) {
 88 |     return;
 89 |   }
 90 |   userdict_->clear_sync_lemmas(0, last_count_);
 91 |   last_count_ = 0;
 92 | }
 93 | 
 94 | void Sync::finish() {
 95 |   if (userdict_) {
 96 |     userdict_->close_dict();
 97 |     delete userdict_;
 98 |     userdict_ = NULL;
 99 |     free(dictfile_);
100 |     dictfile_ = NULL;
101 |     last_count_ = 0;
102 |   }
103 | }
104 | 
105 | int Sync::get_capacity() {
106 |   UserDict::UserDictStat stat;
107 |   userdict_->state(&stat);
108 |   return stat.limit_lemma_count - stat.lemma_count;
109 | }
110 | 
111 | }
112 | #endif
113 | 


--------------------------------------------------------------------------------
/googlelib/ngram.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (C) 2009 The Android Open Source Project
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *      http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | 
17 | #ifndef PINYINIME_INCLUDE_NGRAM_H__
18 | #define PINYINIME_INCLUDE_NGRAM_H__
19 | 
20 | #include <stdio.h>
21 | #include <stdlib.h>
22 | #include "./dictdef.h"
23 | #include <QFile>
24 | 
25 | namespace ime_pinyin {
26 | 
27 | typedef unsigned char CODEBOOK_TYPE;
28 | 
29 | static const size_t kCodeBookSize = 256;
30 | 
31 | class NGram {
32 |  public:
33 |   // The maximum score of a lemma item.
34 |   static const LmaScoreType kMaxScore = 0x3fff;
35 | 
36 |   // In order to reduce the storage size, the original log value is amplified by
37 |   // kScoreAmplifier, and we use LmaScoreType to store.
38 |   // After this process, an item with a lower score has a higher frequency.
39 |   static const int kLogValueAmplifier = -800;
40 | 
41 |   // System words' total frequency. It is not the real total frequency, instead,
42 |   // It is only used to adjust system lemmas' scores when the user dictionary's
43 |   // total frequency changes.
44 |   // In this version, frequencies of system lemmas are fixed. We are considering
45 |   // to make them changable in next version.
46 |   static const size_t kSysDictTotalFreq = 100000000;
47 | 
48 |  private:
49 | 
50 |   static NGram* instance_;
51 | 
52 |   bool initialized_;
53 |   uint32 idx_num_;
54 | 
55 |   size_t total_freq_none_sys_;
56 | 
57 |   // Score compensation for system dictionary lemmas.
58 |   // Because after user adds some user lemmas, the total frequency changes, and
59 |   // we use this value to normalize the score.
60 |   float sys_score_compensation_;
61 | 
62 | #ifdef ___BUILD_MODEL___
63 |   double *freq_codes_df_;
64 | #endif
65 |   LmaScoreType *freq_codes_;
66 |   CODEBOOK_TYPE *lma_freq_idx_;
67 | 
68 |  public:
69 |   NGram();
70 |   ~NGram();
71 | 
72 |   static NGram& get_instance();
73 | 
74 |   bool save_ngram(FILE *fp);
75 |   bool load_ngram(QFile *fp);
76 | 
77 |   // Set the total frequency of all none system dictionaries.
78 |   void set_total_freq_none_sys(size_t freq_none_sys);
79 | 
80 |   float get_uni_psb(LemmaIdType lma_id);
81 | 
82 |   // Convert a probability to score. Actually, the score will be limited to
83 |   // kMaxScore, but at runtime, we also need float expression to get accurate
84 |   // value of the score.
85 |   // After the conversion, a lower score indicates a higher probability of the
86 |   // item.
87 |   static float convert_psb_to_score(double psb);
88 | 
89 | #ifdef ___BUILD_MODEL___
90 |   // For constructing the unigram mode model.
91 |   bool build_unigram(LemmaEntry *lemma_arr, size_t num,
92 |                      LemmaIdType next_idx_unused);
93 | #endif
94 | };
95 | }
96 | 
97 | #endif  // PINYINIME_INCLUDE_NGRAM_H__
98 | 


--------------------------------------------------------------------------------
/googlelib/utf16reader.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #include "utf16reader.h"
 18 | 
 19 | namespace ime_pinyin {
 20 | 
 21 | #define MIN_BUF_LEN 128
 22 | #define MAX_BUF_LEN 65535
 23 | 
 24 | Utf16Reader::Utf16Reader() {
 25 |   fp_ = NULL;
 26 |   buffer_ = NULL;
 27 |   buffer_total_len_ = 0;
 28 |   buffer_next_pos_ = 0;
 29 |   buffer_valid_len_ = 0;
 30 | }
 31 | 
 32 | Utf16Reader::~Utf16Reader() {
 33 |   if (NULL != fp_)
 34 |     fclose(fp_);
 35 | 
 36 |   if (NULL != buffer_)
 37 |     delete [] buffer_;
 38 | }
 39 | 
 40 | 
 41 | bool Utf16Reader::open(const char* filename, size_t buffer_len) {
 42 |   if (filename == NULL)
 43 |     return false;
 44 | 
 45 |   if (buffer_len < MIN_BUF_LEN)
 46 |     buffer_len = MIN_BUF_LEN;
 47 |   else if (buffer_len > MAX_BUF_LEN)
 48 |     buffer_len = MAX_BUF_LEN;
 49 | 
 50 |   buffer_total_len_ = buffer_len;
 51 | 
 52 |   if (NULL != buffer_)
 53 |     delete [] buffer_;
 54 |   buffer_ = new char16[buffer_total_len_];
 55 |   if (NULL == buffer_)
 56 |     return false;
 57 | 
 58 |   if ((fp_ = fopen(filename, "rb")) == NULL)
 59 |     return false;
 60 | 
 61 |   // the UTF16 file header, skip
 62 |   char16 header;
 63 |   if (fread(&header, sizeof(header), 1, fp_) != 1 || header != 0xfeff) {
 64 |     fclose(fp_);
 65 |     fp_ = NULL;
 66 |     return false;
 67 |   }
 68 | 
 69 |   return true;
 70 | }
 71 | 
 72 | char16* Utf16Reader::readline(char16* read_buf, size_t max_len) {
 73 |   if (NULL == fp_ || NULL == read_buf || 0 == max_len)
 74 |     return NULL;
 75 | 
 76 |   size_t ret_len = 0;
 77 | 
 78 |   do {
 79 |     if (buffer_valid_len_ == 0) {
 80 |       buffer_next_pos_ = 0;
 81 |       buffer_valid_len_ = fread(buffer_, sizeof(char16),
 82 |                                 buffer_total_len_, fp_);
 83 |       if (buffer_valid_len_ == 0) {
 84 |         if (0 == ret_len)
 85 |           return NULL;
 86 |         read_buf[ret_len] = (char16)'\0';
 87 |         return read_buf;
 88 |       }
 89 |     }
 90 | 
 91 |     for (size_t i = 0; i < buffer_valid_len_; i++) {
 92 |       if (i == max_len - 1 ||
 93 |           buffer_[buffer_next_pos_ + i] == (char16)'\n') {
 94 |         if (ret_len + i > 0 && read_buf[ret_len + i - 1] == (char16)'\r') {
 95 |           read_buf[ret_len + i - 1] = (char16)'\0';
 96 |         } else {
 97 |           read_buf[ret_len + i] = (char16)'\0';
 98 |         }
 99 | 
100 |         i++;
101 |         buffer_next_pos_ += i;
102 |         buffer_valid_len_ -= i;
103 |         if (buffer_next_pos_ == buffer_total_len_) {
104 |           buffer_next_pos_ = 0;
105 |           buffer_valid_len_ = 0;
106 |         }
107 |         return read_buf;
108 |       } else {
109 |         read_buf[ret_len + i] = buffer_[buffer_next_pos_ + i];
110 |       }
111 |     }
112 | 
113 |     ret_len += buffer_valid_len_;
114 |     buffer_valid_len_ = 0;
115 |   } while (true);
116 | 
117 |   // Never reach here
118 |   return NULL;
119 | }
120 | 
121 | bool Utf16Reader::close() {
122 |   if (NULL != fp_)
123 |     fclose(fp_);
124 |   fp_ = NULL;
125 | 
126 |   if (NULL != buffer_)
127 |     delete [] buffer_;
128 |   buffer_ = NULL;
129 |   return true;
130 | }
131 | }  // namespace ime_pinyin
132 | 


--------------------------------------------------------------------------------
/virtualkeyboard.h:
--------------------------------------------------------------------------------
 1 | /**
 2 |  *  @file virtualkeyboard.h
 3 |  *
 4 |  *  @date 2020-08-14
 5 |  *
 6 |  *  @author aron566
 7 |  *
 8 |  *  @brief 调用软件键盘接口，及相关配置
 9 |  *
10 |  *  @version V1.0
11 |  */
12 | #ifndef VIRTUALKEYBOARD_H
13 | #define VIRTUALKEYBOARD_H
14 | /** Includes -----------------------------------------------------------------*/
15 | #include "keyboard.h"
16 | 
17 | /** Private includes ---------------------------------------------------------*/
18 | 
19 | /** Private defines ----------------------------------------------------------*/
20 | #define UI_KEYBOARDWINDOW_WIDTH     1280U
21 | #define UI_KEYBOARDWINDOW_HEIGHT    720U
22 | 
23 | #define SEARCH_RESULT_NUM_MAX       200U/**< 最大候选数目*/
24 | /*中文结果起始高度，可大不可小*/
25 | #define CHINESESEARCH_RESULT_START_Y  (UI_KEYBOARDWINDOW_HEIGHT*140/720)
26 | /*高度自动，无需修改*/
27 | #define CHINESESEARCH_RESULT_END_Y    (UI_KEYBOARDWINDOW_HEIGHT*330/720-CHINESESEARCH_RESULT_START_Y)
28 | /*===设定搜索页面文本显示属性===*/
29 | #define CHINESESEARCH_RECT_START_Y         (CHINESESEARCH_RESULT_START_Y)
30 | #define CHINESESEARCH_RECT_END_Y           (CHINESESEARCH_RESULT_END_Y)
31 | #define CHINESESEARCH_BLOCKSTYLE_MULTI_RECT_X_GAP     (5U)        /*搜索结果页面矩形水平间隙*/
32 | #define CHINESESEARCH_BLOCKSTYLE_MULTI_RECT_Y_GAP     (5U)        /*搜索结果页面矩形垂直间隙*/
33 | #define CHINESESEARCH_BLOCKSTYLE_RECT_H_NUM_MAX       (7U)        /*搜索结果页面最大允许每行矩形数目*/
34 | #define CHINESESEARCH_BLOCKSTYLE_RECT_V_NUM_MAX       (3U)        /*搜索结果页面最大允许矩形行数目*/
35 | #define CHINESESEARCH_BLOCKSTYLE_RECT_LEFT_INVALID_LEN     (5U)   /*搜索结果页面左边不可用长度*/
36 | #define CHINESESEARCH_BLOCKSTYLE_RECT_RIGHT_INVALID_LEN    (UI_KEYBOARDWINDOW_WIDTH-UI_KEYBOARDWINDOW_WIDTH*1190/1280U)   /*搜索结果页面右边不可用长度*/
37 | #define CHINESESEARCH_BLOCKSTYLE_RECT_UP_INVALID_LEN       (5U)   /*搜索结果页面上边不可用长度*/
38 | #define CHINESESEARCH_BLOCKSTYLE_RECT_DOWN_INVALID_LEN     (0U)   /*搜索结果页面下边不可用长度*/
39 | #define CHINESESEARCH_BLOCKSTYLE_RECT_ORIGIN_Y        (CHINESESEARCH_RESULT_START_Y+CHINESESEARCH_BLOCKSTYLE_RECT_UP_INVALID_LEN)   /*搜索结果页面矩形起始Y*/
40 | #define CHINESESEARCH_BLOCKSTYLE_RECT_ORIGIN_X        (CHINESESEARCH_BLOCKSTYLE_RECT_LEFT_INVALID_LEN)
41 | 
42 | /*通过计算每矩形允许宽度X*/
43 | #define CHINESESEARCH_BLOCKSTYLE_RECT_WIDTH       \
44 | ((UI_KEYBOARDWINDOW_WIDTH-CHINESESEARCH_BLOCKSTYLE_RECT_LEFT_INVALID_LEN-CHINESESEARCH_BLOCKSTYLE_RECT_RIGHT_INVALID_LEN-(CHINESESEARCH_BLOCKSTYLE_MULTI_RECT_X_GAP*(CHINESESEARCH_BLOCKSTYLE_RECT_H_NUM_MAX+1)))/CHINESESEARCH_BLOCKSTYLE_RECT_H_NUM_MAX)
45 | /*通过计算每矩形允许高度Y*/
46 | #define CHINESESEARCH_BLOCKSTYLE_RECT_END_Y       \
47 | ((CHINESESEARCH_RECT_END_Y-CHINESESEARCH_BLOCKSTYLE_RECT_DOWN_INVALID_LEN-CHINESESEARCH_BLOCKSTYLE_RECT_UP_INVALID_LEN-(CHINESESEARCH_BLOCKSTYLE_MULTI_RECT_Y_GAP*(CHINESESEARCH_BLOCKSTYLE_RECT_V_NUM_MAX+1)))/CHINESESEARCH_BLOCKSTYLE_RECT_V_NUM_MAX)
48 | 
49 | /*计算多个视频矩形X相对坐标起点,数目从0开始则间隙X需+1，高度x不变*/
50 | #define CHINESESEARCH_BLOCKSTYLE_MULTI_RECT_H_X(X)   (CHINESESEARCH_BLOCKSTYLE_RECT_ORIGIN_X+CHINESESEARCH_BLOCKSTYLE_MULTI_RECT_X_GAP*((X%CHINESESEARCH_BLOCKSTYLE_RECT_H_NUM_MAX)+1)+CHINESESEARCH_BLOCKSTYLE_RECT_WIDTH*(X%CHINESESEARCH_BLOCKSTYLE_RECT_H_NUM_MAX))
51 | /*计算多个视频矩形Y相对坐标起点,数目从0开始则间隙X需+1，高度x不变*/
52 | #define CHINESESEARCH_BLOCKSTYLE_MULTI_RECT_Y_X(X)   (CHINESESEARCH_BLOCKSTYLE_RECT_ORIGIN_Y+CHINESESEARCH_BLOCKSTYLE_MULTI_RECT_Y_GAP*((X/CHINESESEARCH_BLOCKSTYLE_RECT_H_NUM_MAX)+1)+CHINESESEARCH_BLOCKSTYLE_RECT_END_Y*(X/CHINESESEARCH_BLOCKSTYLE_RECT_H_NUM_MAX))
53 | /** Exported typedefines -----------------------------------------------------*/
54 | 
55 | /** Exported constants -------------------------------------------------------*/
56 | 
57 | /** Exported macros-----------------------------------------------------------*/
58 | 
59 | /** Exported variables -------------------------------------------------------*/
60 | /** Exported functions prototypes --------------------------------------------*/
61 | 
62 | #endif // VIRTUALKEYBOARD_H
63 | /******************************** End of file *********************************/
64 | 


--------------------------------------------------------------------------------
/googlelib/spellingtable.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #ifndef PINYINIME_INCLUDE_SPELLINGTABLE_H__
 18 | #define PINYINIME_INCLUDE_SPELLINGTABLE_H__
 19 | 
 20 | #include <stdlib.h>
 21 | #include "./dictdef.h"
 22 | 
 23 | namespace ime_pinyin {
 24 | 
 25 | #ifdef ___BUILD_MODEL___
 26 | 
 27 | const size_t kMaxSpellingSize = kMaxPinyinSize;
 28 | 
 29 | typedef struct {
 30 |   char str[kMaxSpellingSize + 1];
 31 |   double freq;
 32 | } RawSpelling, *PRawSpelling;
 33 | 
 34 | // This class is used to store the spelling strings
 35 | // The length of the input spelling string should be less or equal to the
 36 | // spelling_size_ (set by init_table). If the input string is too long,
 37 | // we only keep its first spelling_size_ chars.
 38 | class SpellingTable {
 39 |  private:
 40 |   static const size_t kNotSupportNum = 3;
 41 |   static const char kNotSupportList[kNotSupportNum][kMaxSpellingSize + 1];
 42 | 
 43 |   bool need_score_;
 44 | 
 45 |   size_t spelling_max_num_;
 46 | 
 47 |   RawSpelling *raw_spellings_;
 48 | 
 49 |   // Used to store spelling strings. If the spelling table needs to calculate
 50 |   // score, an extra char after each spelling string is the score.
 51 |   // An item with a lower score has a higher probability.
 52 |   char *spelling_buf_;
 53 |   size_t spelling_size_;
 54 | 
 55 |   double total_freq_;
 56 | 
 57 |   size_t spelling_num_;
 58 | 
 59 |   double score_amplifier_;
 60 | 
 61 |   unsigned char average_score_;
 62 | 
 63 |   // If frozen is true, put_spelling() and contain() are not allowed to call.
 64 |   bool frozen_;
 65 | 
 66 |   size_t get_hash_pos(const char* spelling_str);
 67 |   size_t hash_pos_next(size_t hash_pos);
 68 |   void free_resource();
 69 |  public:
 70 |   SpellingTable();
 71 |   ~SpellingTable();
 72 | 
 73 |   // pure_spl_size is the pure maximum spelling string size. For example,
 74 |   // "zhuang" is the longgest item in Pinyin, so pure_spl_size should be 6.
 75 |   // spl_max_num is the maximum number of spelling strings to store.
 76 |   // need_score is used to indicate whether the caller needs to calculate a
 77 |   // score for each spelling.
 78 |   bool init_table(size_t pure_spl_size, size_t spl_max_num, bool need_score);
 79 | 
 80 |   // Put a spelling string to the table.
 81 |   // It always returns false if called after arrange() withtout a new
 82 |   // init_table() operation.
 83 |   // freq is the spelling's occuring count.
 84 |   // If the spelling has been in the table, occuring count will accumulated.
 85 |   bool put_spelling(const char* spelling_str, double spl_count);
 86 | 
 87 |   // Test whether a spelling string is in the table.
 88 |   // It always returns false, when being called after arrange() withtout a new
 89 |   // init_table() operation.
 90 |   bool contain(const char* spelling_str);
 91 | 
 92 |   // Sort the spelling strings and put them from the begin of the buffer.
 93 |   // Return the pointer of the sorted spelling strings.
 94 |   // item_size and spl_num return the item size and number of spelling.
 95 |   // Because each spelling uses a '\0' as terminator, the returned item_size is
 96 |   // at least one char longer than the spl_size parameter specified by
 97 |   // init_table(). If the table is initialized to calculate score, item_size
 98 |   // will be increased by 1, and current_spl_str[item_size - 1] stores an
 99 |   // unsinged char score.
100 |   // An item with a lower score has a higher probability.
101 |   // Do not call put_spelling() and contains() after arrange().
102 |   const char* arrange(size_t *item_size, size_t *spl_num);
103 | 
104 |   float get_score_amplifier();
105 | 
106 |   unsigned char get_average_score();
107 | };
108 | #endif  // ___BUILD_MODEL___
109 | }
110 | 
111 | #endif  // PINYINIME_INCLUDE_SPELLINGTABLE_H__
112 | 


--------------------------------------------------------------------------------
/googlelib/dictlist.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #ifndef PINYINIME_INCLUDE_DICTLIST_H__
 18 | #define PINYINIME_INCLUDE_DICTLIST_H__
 19 | 
 20 | #include <stdlib.h>
 21 | #include <stdio.h>
 22 | #include "./dictdef.h"
 23 | #include "./searchutility.h"
 24 | #include "./spellingtrie.h"
 25 | #include "./utf16char.h"
 26 | 
 27 | namespace ime_pinyin {
 28 | 
 29 | class DictList {
 30 |  private:
 31 |   bool initialized_;
 32 | 
 33 |   const SpellingTrie *spl_trie_;
 34 | 
 35 |   // Number of SingCharItem. The first is blank, because id 0 is invalid.
 36 |   uint32 scis_num_;
 37 |   char16 *scis_hz_;
 38 |   SpellingId *scis_splid_;
 39 | 
 40 |   // The large memory block to store the word list.
 41 |   char16 *buf_;
 42 | 
 43 |   // Starting position of those words whose lengths are i+1, counted in
 44 |   // char16
 45 |   uint32 start_pos_[kMaxLemmaSize + 1];
 46 | 
 47 |   uint32 start_id_[kMaxLemmaSize + 1];
 48 | 
 49 |   int (*cmp_func_[kMaxLemmaSize])(const void *, const void *);
 50 | 
 51 |   bool alloc_resource(size_t buf_size, size_t scim_num);
 52 | 
 53 |   void free_resource();
 54 | 
 55 | #ifdef ___BUILD_MODEL___
 56 |   // Calculate the requsted memory, including the start_pos[] buffer.
 57 |   size_t calculate_size(const LemmaEntry *lemma_arr, size_t lemma_num);
 58 | 
 59 |   void fill_scis(const SingleCharItem *scis, size_t scis_num);
 60 | 
 61 |   // Copy the related content to the inner buffer
 62 |   // It should be called after calculate_size()
 63 |   void fill_list(const LemmaEntry *lemma_arr, size_t lemma_num);
 64 | 
 65 |   // Find the starting position for the buffer of those 2-character Chinese word
 66 |   // whose first character is the given Chinese character.
 67 |   char16* find_pos2_startedbyhz(char16 hz_char);
 68 | #endif
 69 | 
 70 |   // Find the starting position for the buffer of those words whose lengths are
 71 |   // word_len. The given parameter cmp_func decides how many characters from
 72 |   // beginning will be used to compare.
 73 |   char16* find_pos_startedbyhzs(const char16 last_hzs[],
 74 |                                 size_t word_Len,
 75 |                                 int (*cmp_func)(const void *, const void *));
 76 | 
 77 |  public:
 78 | 
 79 |   DictList();
 80 |   ~DictList();
 81 | 
 82 |   bool save_list(FILE *fp);
 83 |   bool load_list(QFile *fp);
 84 | 
 85 | #ifdef ___BUILD_MODEL___
 86 |   // Init the list from the LemmaEntry array.
 87 |   // lemma_arr should have been sorted by the hanzi_str, and have been given
 88 |   // ids from 1
 89 |   bool init_list(const SingleCharItem *scis, size_t scis_num,
 90 |                  const LemmaEntry *lemma_arr, size_t lemma_num);
 91 | #endif
 92 | 
 93 |   // Get the hanzi string for the given id
 94 |   uint16 get_lemma_str(LemmaIdType id_hz, char16 *str_buf, uint16 str_max);
 95 | 
 96 |   void convert_to_hanzis(char16 *str, uint16 str_len);
 97 | 
 98 |   void convert_to_scis_ids(char16 *str, uint16 str_len);
 99 | 
100 |   // last_hzs stores the last n Chinese characters history, its length should be
101 |   // less or equal than kMaxPredictSize.
102 |   // hzs_len specifies the length(<= kMaxPredictSize).
103 |   // predict_buf is used to store the result.
104 |   // buf_len specifies the buffer length.
105 |   // b4_used specifies how many items before predict_buf have been used.
106 |   // Returned value is the number of newly added items.
107 |   size_t predict(const char16 last_hzs[], uint16 hzs_len,
108 |                  NPredictItem *npre_items, size_t npre_max,
109 |                  size_t b4_used);
110 | 
111 |   // If half_splid is a valid half spelling id, return those full spelling
112 |   // ids which share this half id.
113 |   uint16 get_splids_for_hanzi(char16 hanzi, uint16 half_splid,
114 |                               uint16 *splids, uint16 max_splids);
115 | 
116 |   LemmaIdType get_lemma_id(const char16 *str, uint16 str_len);
117 | };
118 | }
119 | 
120 | #endif  // PINYINIME_INCLUDE_DICTLIST_H__
121 | 


--------------------------------------------------------------------------------
/googlelib/splparser.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (C) 2009 The Android Open Source Project
 3 |  *
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *      http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License.
15 |  */
16 | 
17 | #ifndef PINYINIME_INCLUDE_SPLPARSER_H__
18 | #define PINYINIME_INCLUDE_SPLPARSER_H__
19 | 
20 | #include "./dictdef.h"
21 | #include "./spellingtrie.h"
22 | 
23 | namespace ime_pinyin {
24 | 
25 | class SpellingParser {
26 |  protected:
27 |   const SpellingTrie *spl_trie_;
28 | 
29 |  public:
30 |   SpellingParser();
31 | 
32 |   // Given a string, parse it into a spelling id stream.
33 |   // If the whole string are sucessfully parsed, last_is_pre will be true;
34 |   // if the whole string is not fullly parsed, last_is_pre will return whether
35 |   // the last part of the string is a prefix of a full spelling string. For
36 |   // example, given string "zhengzhon", "zhon" is not a valid speling, but it is
37 |   // the prefix of "zhong".
38 |   //
39 |   // If splstr starts with a character not in ['a'-z'] (it is a split char),
40 |   // return 0.
41 |   // Split char can only appear in the middle of the string or at the end.
42 |   uint16 splstr_to_idxs(const char *splstr, uint16 str_len, uint16 splidx[],
43 |                         uint16 start_pos[], uint16 max_size, bool &last_is_pre);
44 | 
45 |   // Similar to splstr_to_idxs(), the only difference is that splstr_to_idxs()
46 |   // convert single-character Yunmus into half ids, while this function converts
47 |   // them into full ids.
48 |   uint16 splstr_to_idxs_f(const char *splstr, uint16 str_len, uint16 splidx[],
49 |           uint16 start_pos[], uint16 max_size, bool &last_is_pre);
50 | 
51 |   // Similar to splstr_to_idxs(), the only difference is that this function
52 |   // uses char16 instead of char8.
53 |   uint16 splstr16_to_idxs(const char16 *splstr, uint16 str_len, uint16 splidx[],
54 |                         uint16 start_pos[], uint16 max_size, bool &last_is_pre);
55 | 
56 |   // Similar to splstr_to_idxs_f(), the only difference is that this function
57 |   // uses char16 instead of char8.
58 |   uint16 splstr16_to_idxs_f(const char16 *splstr16, uint16 str_len,
59 |                             uint16 splidx[], uint16 start_pos[],
60 |                             uint16 max_size, bool &last_is_pre);
61 | 
62 |   // If the given string is a spelling, return the id, others, return 0.
63 |   // If the give string is a single char Yunmus like "A", and the char is
64 |   // enabled in ShouZiMu mode, the returned spelling id will be a half id.
65 |   // When the returned spelling id is a half id, *is_pre returns whether it
66 |   // is a prefix of a full spelling string.
67 |   uint16 get_splid_by_str(const char *splstr, uint16 str_len, bool *is_pre);
68 | 
69 |   // If the given string is a spelling, return the id, others, return 0.
70 |   // If the give string is a single char Yunmus like "a", no matter the char
71 |   // is enabled in ShouZiMu mode or not, the returned spelling id will be
72 |   // a full id.
73 |   // When the returned spelling id is a half id, *p_is_pre returns whether it
74 |   // is a prefix of a full spelling string.
75 |   uint16 get_splid_by_str_f(const char *splstr, uint16 str_len, bool *is_pre);
76 | 
77 |   // Splitter chars are not included.
78 |   bool is_valid_to_parse(char ch);
79 | 
80 |   // When auto-correction is not enabled, get_splid_by_str() will be called to
81 |   // return the single result. When auto-correction is enabled, this function
82 |   // will be called to get the results. Auto-correction is not ready.
83 |   // full_id_num returns number of full spelling ids.
84 |   // is_pre returns whether the given string is the prefix of a full spelling
85 |   // string.
86 |   // If splstr starts with a character not in [a-zA-Z] (it is a split char),
87 |   // return 0.
88 |   // Split char can only appear in the middle of the string or at the end.
89 |   // The caller should guarantee NULL != splstr && str_len > 0 && NULL != splidx
90 |   uint16 get_splids_parallel(const char *splstr, uint16 str_len,
91 |                              uint16 splidx[], uint16 max_size,
92 |                              uint16 &full_id_num, bool &is_pre);
93 | };
94 | }
95 | 
96 | #endif  // PINYINIME_INCLUDE_SPLPARSER_H__
97 | 


--------------------------------------------------------------------------------
/googlelib/utf16char.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #include <stdlib.h>
 18 | #include "utf16char.h"
 19 | 
 20 | namespace ime_pinyin {
 21 | 
 22 | #ifdef __cplusplus
 23 | extern "C" {
 24 | #endif
 25 | 
 26 |   char16* utf16_strtok(char16 *utf16_str, size_t *token_size,
 27 |                        char16 **utf16_str_next) {
 28 |     if (NULL == utf16_str || NULL == token_size || NULL == utf16_str_next) {
 29 |       return NULL;
 30 |     }
 31 | 
 32 |     // Skip the splitters
 33 |     size_t pos = 0;
 34 |     while ((char16)' ' == utf16_str[pos] || (char16)'\n' == utf16_str[pos]
 35 |            || (char16)'\t' == utf16_str[pos])
 36 |       pos++;
 37 | 
 38 |     utf16_str += pos;
 39 |     pos = 0;
 40 | 
 41 |     while ((char16)'\0' != utf16_str[pos] && (char16)' ' != utf16_str[pos]
 42 |            && (char16)'\n' != utf16_str[pos]
 43 |            && (char16)'\t' != utf16_str[pos]) {
 44 |       pos++;
 45 |     }
 46 | 
 47 |     char16 *ret_val = utf16_str;
 48 |     if ((char16)'\0' == utf16_str[pos]) {
 49 |       *utf16_str_next = NULL;
 50 |       if (0 == pos)
 51 |         return NULL;
 52 |     } else {
 53 |       *utf16_str_next = utf16_str + pos + 1;
 54 |     }
 55 | 
 56 |     utf16_str[pos] = (char16)'\0';
 57 |     *token_size = pos;
 58 | 
 59 |     return ret_val;
 60 |   }
 61 | 
 62 |   int utf16_atoi(const char16 *utf16_str) {
 63 |     if (NULL == utf16_str)
 64 |       return 0;
 65 | 
 66 |     int value = 0;
 67 |     int sign = 1;
 68 |     size_t pos = 0;
 69 | 
 70 |     if ((char16)'-' == utf16_str[pos]) {
 71 |       sign = -1;
 72 |       pos++;
 73 |     }
 74 | 
 75 |     while ((char16)'0' <=  utf16_str[pos] &&
 76 |            (char16)'9' >= utf16_str[pos]) {
 77 |       value = value * 10 + static_cast<int>(utf16_str[pos] - (char16)'0');
 78 |       pos++;
 79 |     }
 80 | 
 81 |     return value*sign;
 82 |   }
 83 | 
 84 |   float utf16_atof(const char16 *utf16_str) {
 85 |     // A temporary implemetation.
 86 |     char char8[256];
 87 |     if (utf16_strlen(utf16_str) >= 256) return 0;
 88 | 
 89 |     utf16_strcpy_tochar(char8, utf16_str);
 90 |     return atof(char8);
 91 |   }
 92 | 
 93 |   size_t utf16_strlen(const char16 *utf16_str) {
 94 |     if (NULL == utf16_str)
 95 |       return 0;
 96 | 
 97 |     size_t size = 0;
 98 |     while ((char16)'\0' != utf16_str[size])
 99 |       size++;
100 |     return size;
101 |   }
102 | 
103 |   int utf16_strcmp(const char16* str1, const char16* str2) {
104 |     size_t pos = 0;
105 |     while (str1[pos] == str2[pos] && (char16)'\0' != str1[pos])
106 |       pos++;
107 | 
108 |     return static_cast<int>(str1[pos]) - static_cast<int>(str2[pos]);
109 |   }
110 | 
111 |   int utf16_strncmp(const char16 *str1, const char16 *str2, size_t size) {
112 |     size_t pos = 0;
113 |     while (pos < size && str1[pos] == str2[pos] && (char16)'\0' != str1[pos])
114 |       pos++;
115 | 
116 |     if (pos == size)
117 |       return 0;
118 | 
119 |     return static_cast<int>(str1[pos]) - static_cast<int>(str2[pos]);
120 |   }
121 | 
122 |   // we do not consider overlapping
123 |   char16* utf16_strcpy(char16 *dst, const char16 *src) {
124 |     if (NULL == src || NULL == dst)
125 |       return NULL;
126 | 
127 |     char16* cp = dst;
128 | 
129 |     while ((char16)'\0' != *src) {
130 |       *cp = *src;
131 |       cp++;
132 |       src++;
133 |     }
134 | 
135 |     *cp = *src;
136 | 
137 |     return dst;
138 |   }
139 | 
140 |   char16* utf16_strncpy(char16 *dst, const char16 *src, size_t size) {
141 |     if (NULL == src || NULL == dst || 0 == size)
142 |       return NULL;
143 | 
144 |     if (src == dst)
145 |       return dst;
146 | 
147 |     char16* cp = dst;
148 | 
149 |     if (dst < src || (dst > src && dst >= src + size)) {
150 |       while (size-- && (*cp++ = *src++))
151 |         ;
152 |     } else {
153 |       cp += size - 1;
154 |       src += size - 1;
155 |       while (size-- && (*cp-- == *src--))
156 |         ;
157 |     }
158 |     return dst;
159 |   }
160 | 
161 |   // We do not handle complicated cases like overlapping, because in this
162 |   // codebase, it is not necessary.
163 |   char* utf16_strcpy_tochar(char *dst, const char16 *src) {
164 |     if (NULL == src || NULL == dst)
165 |       return NULL;
166 | 
167 |     char* cp = dst;
168 | 
169 |     while ((char16)'\0' != *src) {
170 |       *cp = static_cast<char>(*src);
171 |       cp++;
172 |       src++;
173 |     }
174 |     *cp = *src;
175 | 
176 |     return dst;
177 |   }
178 | 
179 | #ifdef __cplusplus
180 | }
181 | #endif
182 | }  // namespace ime_pinyin
183 | 


--------------------------------------------------------------------------------
/keyboard.h:
--------------------------------------------------------------------------------
  1 | /**
  2 |  *  @file keyboard.h
  3 |  *
  4 |  *  @date 2020-08-14
  5 |  *
  6 |  *  @author aron566
  7 |  *
  8 |  *  @brief 软键盘实现
  9 |  *
 10 |  *  @version V1.0
 11 |  */
 12 | #ifndef KEYBOARD_H
 13 | #define KEYBOARD_H
 14 | /** Includes -----------------------------------------------------------------*/
 15 | #include <QMainWindow>
 16 | #include <QPushButton>
 17 | #include <QMap>
 18 | #include <pinyinime.h>
 19 | #include "customerqpushbutton.h"
 20 | /** Private includes ---------------------------------------------------------*/
 21 | 
 22 | /** Private defines ----------------------------------------------------------*/
 23 | #if defined(VIRTUALKEYBOARD_LIBRARY)
 24 | #  define VIRTUALKEYBOARD_EXPORT Q_DECL_EXPORT
 25 | #else
 26 | #  define VIRTUALKEYBOARD_EXPORT Q_DECL_IMPORT
 27 | #endif
 28 | /** Exported typedefines -----------------------------------------------------*/
 29 | 
 30 | /** Exported constants -------------------------------------------------------*/
 31 | 
 32 | /** Exported macros-----------------------------------------------------------*/
 33 | /** Exported variables -------------------------------------------------------*/
 34 | /** Exported functions prototypes --------------------------------------------*/
 35 | 
 36 | QT_BEGIN_NAMESPACE
 37 | namespace Ui { class keyboard; }
 38 | QT_END_NAMESPACE
 39 | 
 40 | class VIRTUALKEYBOARD_EXPORT keyboard : public QMainWindow
 41 | {
 42 |     Q_OBJECT
 43 | public:
 44 |     explicit keyboard(QWidget *parent = nullptr ,const char* datapath = nullptr ,const char* userdatapt = nullptr);
 45 |     ~keyboard();
 46 |     void InitTabOrder(void);
 47 |     void Init_MAP_Key_value(void);
 48 | public:
 49 |     QMap<int,QPushButton *>keyboardmap;
 50 |     QMap<size_t,QPushButton *>Resultmap;
 51 |     bool keycapsmode = false;
 52 |     bool IsZhMode = false;
 53 | private:
 54 |     const char *dict_datapath = nullptr;
 55 |     const char *dict_userdatapath = nullptr;
 56 |     QStringList ResultStr;
 57 |     quint16 CurrentResultPageNUM = 1;
 58 |     quint16 TotalResultPageNUM = 1;
 59 | public:
 60 |     enum KEYBOARD_MODE
 61 |     {
 62 |         NUM_ONLY = 0,/**< 数字模式*/
 63 |         EN_ONLY,/**< 英文模式*/
 64 |         ANY,/**< 全功能*/
 65 |     };
 66 |     void showKeyboard(QString title = "键入xx的内容：" ,QString str = "2020");
 67 |     void set_editTips(QString title = "键入xx的内容：");
 68 |     void set_editBox(QString str = "2020");
 69 |     void set_keyboardmode(KEYBOARD_MODE mode);
 70 | private:
 71 |     void Google_PinyinInit(const char* datapath = nullptr ,const char* userdatapt = nullptr);
 72 |     void ZhResultWigdetInit();
 73 |     void set_capsLockmode(bool enable);
 74 |     void UpdateSymbolDisplay();
 75 |     void writeABC(int index);
 76 |     void AppendOnKeyInputWindow(QString &str,bool result = false);
 77 |     void ShowSearchResult(QStringList &result);
 78 |     void ShowNextPageResult();
 79 |     void ShowLastPageResult();
 80 |     void SetResultHidden();
 81 |     void SetEnableNUM(bool state);
 82 |     void SetEnableEN(bool state);
 83 | signals:
 84 |     void editisModifiedok(QString);
 85 | private slots:
 86 | 
 87 | 	void on_key_clear_clicked();
 88 | 	
 89 |     void on_key_caps_clicked();
 90 | 
 91 |     void on_key_del_clicked();
 92 | 
 93 |     void on_key_ok_clicked();
 94 | 	
 95 |     void on_key_1_clicked();
 96 | 
 97 |     void on_key_2_clicked();
 98 | 
 99 |     void on_key_3_clicked();
100 | 
101 |     void on_key_4_clicked();
102 | 
103 |     void on_key_5_clicked();
104 | 
105 |     void on_key_6_clicked();
106 | 
107 |     void on_key_7_clicked();
108 | 
109 |     void on_key_8_clicked();
110 | 
111 |     void on_key_9_clicked();
112 | 
113 |     void on_key_0_clicked();
114 | 
115 |     void on_key_q_clicked();
116 | 
117 |     void on_key_w_clicked();
118 | 
119 |     void on_key_e_clicked();
120 | 
121 |     void on_key_r_clicked();
122 | 
123 |     void on_key_t_clicked();
124 | 
125 |     void on_key_y_clicked();
126 | 
127 |     void on_key_u_clicked();
128 | 
129 |     void on_key_i_clicked();
130 | 
131 |     void on_key_o_clicked();
132 | 
133 |     void on_key_p_clicked();
134 | 
135 |     void on_key_a_clicked();
136 | 
137 |     void on_key_s_clicked();
138 | 
139 |     void on_key_d_clicked();
140 | 
141 |     void on_key_f_clicked();
142 | 
143 |     void on_key_g_clicked();
144 | 
145 |     void on_key_h_clicked();
146 | 
147 |     void on_key_j_clicked();
148 | 
149 |     void on_key_k_clicked();
150 | 
151 |     void on_key_l_clicked();
152 | 
153 |     void on_key_z_clicked();
154 | 
155 |     void on_key_x_clicked();
156 | 
157 |     void on_key_c_clicked();
158 | 
159 |     void on_key_v_clicked();
160 | 
161 |     void on_key_b_clicked();
162 | 
163 |     void on_key_n_clicked();
164 | 
165 |     void on_key_m_clicked();
166 | 
167 |     void on_key_language_clicked();
168 | 
169 |     void on_key_comma_clicked();
170 | 
171 |     void on_key_dash_clicked();
172 | 
173 |     void on_key_virgule_clicked();
174 | 
175 |     void on_key_colon_clicked();
176 | 
177 |     void on_key_at_clicked();
178 | 
179 |     void on_key_point_clicked();
180 | 
181 |     void on_key_pinyininput_textChanged(const QString &arg1);
182 | 
183 |     void on_NEXTpushButton_clicked();
184 | 
185 |     void on_LASTpushButton_clicked();
186 | 
187 |     void on_key_clear_clicked();
188 | 
189 |     void slotResult(customerqpushbutton *pbtn);
190 | 
191 | private:
192 |     Ui::keyboard *ui;
193 | };
194 | #endif // KEYBOARD_H
195 | 
196 | /******************************** End of file *********************************/
197 | 


--------------------------------------------------------------------------------
/googlelib/dictdef.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #ifndef PINYINIME_INCLUDE_DICTDEF_H__
 18 | #define PINYINIME_INCLUDE_DICTDEF_H__
 19 | 
 20 | #include <stdlib.h>
 21 | #include "./utf16char.h"
 22 | 
 23 | namespace ime_pinyin {
 24 | 
 25 | // Enable the following line when building the binary dictionary model.
 26 | // #define ___BUILD_MODEL___
 27 | 
 28 | typedef unsigned char      uint8;
 29 | typedef unsigned short     uint16;
 30 | typedef unsigned int       uint32;
 31 | 
 32 | typedef signed char        int8;
 33 | typedef short              int16;
 34 | typedef int                int32;
 35 | typedef long long          int64;
 36 | typedef unsigned long long uint64;
 37 | 
 38 | const bool kPrintDebug0 = false;
 39 | const bool kPrintDebug1 = false;
 40 | const bool kPrintDebug2 = false;
 41 | 
 42 | // The max length of a lemma.
 43 | const size_t kMaxLemmaSize = 8;
 44 | 
 45 | // The max length of a Pinyin (spelling).
 46 | const size_t kMaxPinyinSize = 6;
 47 | 
 48 | // The number of half spelling ids. For Chinese Pinyin, there 30 half ids.
 49 | // See SpellingTrie.h for details.
 50 | const size_t kHalfSpellingIdNum = 29;
 51 | 
 52 | // The maximum number of full spellings. For Chinese Pinyin, there are only
 53 | // about 410 spellings.
 54 | // If change this value is bigger(needs more bits), please also update
 55 | // other structures like SpellingNode, to make sure than a spelling id can be
 56 | // stored.
 57 | // -1 is because that 0 is never used.
 58 | const size_t kMaxSpellingNum = 512 - kHalfSpellingIdNum - 1;
 59 | const size_t kMaxSearchSteps = 40;
 60 | 
 61 | // One character predicts its following characters.
 62 | const size_t kMaxPredictSize = (kMaxLemmaSize - 1);
 63 | 
 64 | // LemmaIdType must always be size_t.
 65 | typedef size_t LemmaIdType;
 66 | const size_t kLemmaIdSize = 3;  // Actually, a Id occupies 3 bytes in storage.
 67 | const size_t kLemmaIdComposing = 0xffffff;
 68 | 
 69 | typedef uint16 LmaScoreType;
 70 | typedef uint16 KeyScoreType;
 71 | 
 72 | // Number of items with highest score are kept for prediction purpose.
 73 | const size_t kTopScoreLemmaNum = 10;
 74 | 
 75 | const size_t kMaxPredictNumByGt3 = 1;
 76 | const size_t kMaxPredictNumBy3 = 2;
 77 | const size_t kMaxPredictNumBy2 = 2;
 78 | 
 79 | // The last lemma id (included) for the system dictionary. The system
 80 | // dictionary's ids always start from 1.
 81 | const LemmaIdType kSysDictIdEnd = 500000;
 82 | 
 83 | // The first lemma id for the user dictionary.
 84 | const LemmaIdType kUserDictIdStart = 500001;
 85 | 
 86 | // The last lemma id (included) for the user dictionary.
 87 | const LemmaIdType kUserDictIdEnd = 600000;
 88 | 
 89 | typedef struct {
 90 |   uint16 half_splid:5;
 91 |   uint16 full_splid:11;
 92 | } SpellingId, *PSpellingId;
 93 | 
 94 | 
 95 | /**
 96 |  * We use different node types for different layers
 97 |  * Statistical data of the building result for a testing dictionary:
 98 |  *                              root,   level 0,   level 1,   level 2,   level 3
 99 |  * max son num of one node:     406        280         41          2          -
100 |  * max homo num of one node:      0         90         23          2          2
101 |  * total node num of a layer:     1        406      31766      13516        993
102 |  * total homo num of a layer:     9       5674      44609      12667        995
103 |  *
104 |  * The node number for root and level 0 won't be larger than 500
105 |  * According to the information above, two kinds of nodes can be used; one for
106 |  * root and level 0, the other for these layers deeper than 0.
107 |  *
108 |  * LE = less and equal,
109 |  * A node occupies 16 bytes. so, totallly less than 16 * 500 = 8K
110 |  */
111 | struct LmaNodeLE0 {
112 |   uint32 son_1st_off;
113 |   uint32 homo_idx_buf_off;
114 |   uint16 spl_idx;
115 |   uint16 num_of_son;
116 |   uint16 num_of_homo;
117 | };
118 | 
119 | /**
120 |  * GE = great and equal
121 |  * A node occupies 8 bytes.
122 |  */
123 | struct LmaNodeGE1 {
124 |   uint16 son_1st_off_l;        // Low bits of the son_1st_off
125 |   uint16 homo_idx_buf_off_l;   // Low bits of the homo_idx_buf_off_1
126 |   uint16 spl_idx;
127 |   unsigned char num_of_son;            // number of son nodes
128 |   unsigned char num_of_homo;           // number of homo words
129 |   unsigned char son_1st_off_h;         // high bits of the son_1st_off
130 |   unsigned char homo_idx_buf_off_h;    // high bits of the homo_idx_buf_off
131 | };
132 | 
133 | #ifdef ___BUILD_MODEL___
134 | struct SingleCharItem {
135 |   float freq;
136 |   char16 hz;
137 |   SpellingId splid;
138 | };
139 | 
140 | struct LemmaEntry {
141 |   LemmaIdType idx_by_py;
142 |   LemmaIdType idx_by_hz;
143 |   char16 hanzi_str[kMaxLemmaSize + 1];
144 | 
145 |   // The SingleCharItem id for each Hanzi.
146 |   uint16 hanzi_scis_ids[kMaxLemmaSize];
147 | 
148 |   uint16 spl_idx_arr[kMaxLemmaSize + 1];
149 |   char pinyin_str[kMaxLemmaSize][kMaxPinyinSize + 1];
150 |   unsigned char hz_str_len;
151 |   float freq;
152 | };
153 | #endif  // ___BUILD_MODEL___
154 | 
155 | }  //  namespace ime_pinyin
156 | 
157 | #endif  // PINYINIME_INCLUDE_DICTDEF_H__
158 | 


--------------------------------------------------------------------------------
/googlelib/pinyinime.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #include <stdlib.h>
 18 | #include "pinyinime.h"
 19 | #include "dicttrie.h"
 20 | #include "matrixsearch.h"
 21 | #include "spellingtrie.h"
 22 | 
 23 | #ifdef __cplusplus
 24 | extern "C" {
 25 | #endif
 26 | 
 27 |   using namespace ime_pinyin;
 28 | 
 29 |   // The maximum number of the prediction items.
 30 |   static const size_t kMaxPredictNum = 500;
 31 | 
 32 |   // Used to search Pinyin string and give the best candidate.
 33 |   MatrixSearch* matrix_search = NULL;
 34 | 
 35 |   char16 predict_buf[kMaxPredictNum][kMaxPredictSize + 1];
 36 | 
 37 |   bool im_open_decoder(const char *fn_sys_dict, const char *fn_usr_dict) {
 38 |     if (NULL != matrix_search)
 39 |       delete matrix_search;
 40 | 
 41 |     matrix_search = new MatrixSearch();
 42 |     if (NULL == matrix_search) {
 43 |       return false;
 44 |     }
 45 | 
 46 |     return matrix_search->init(fn_sys_dict, fn_usr_dict);
 47 |   }
 48 | 
 49 |   bool im_open_decoder_fd(int sys_fd, long start_offset, long length,
 50 |                           const char *fn_usr_dict) {
 51 |     if (NULL != matrix_search)
 52 |       delete matrix_search;
 53 | 
 54 |     matrix_search = new MatrixSearch();
 55 |     if (NULL == matrix_search)
 56 |       return false;
 57 | 
 58 |     return matrix_search->init_fd(sys_fd, start_offset, length, fn_usr_dict);
 59 |   }
 60 | 
 61 |   void im_close_decoder() {
 62 |     if (NULL != matrix_search) {
 63 |       matrix_search->close();
 64 |       delete matrix_search;
 65 |     }
 66 |     matrix_search = NULL;
 67 |   }
 68 | 
 69 |   void im_set_max_lens(size_t max_sps_len, size_t max_hzs_len) {
 70 |     if (NULL != matrix_search) {
 71 |       matrix_search->set_max_lens(max_sps_len, max_hzs_len);
 72 |     }
 73 |   }
 74 | 
 75 |   void im_flush_cache() {
 76 |     if (NULL != matrix_search)
 77 |       matrix_search->flush_cache();
 78 |   }
 79 | 
 80 |   // To be updated.
 81 |   size_t im_search(const char* pybuf, size_t pylen) {
 82 |     if (NULL == matrix_search)
 83 |       return 0;
 84 | 
 85 |     matrix_search->search(pybuf, pylen);
 86 |     return matrix_search->get_candidate_num();
 87 |   }
 88 | 
 89 |   size_t im_delsearch(size_t pos, bool is_pos_in_splid,
 90 |                       bool clear_fixed_this_step) {
 91 |     if (NULL == matrix_search)
 92 |       return 0;
 93 |     matrix_search->delsearch(pos, is_pos_in_splid, clear_fixed_this_step);
 94 |     return matrix_search->get_candidate_num();
 95 |   }
 96 | 
 97 |   void im_reset_search() {
 98 |     if (NULL == matrix_search)
 99 |       return;
100 | 
101 |     matrix_search->reset_search();
102 |   }
103 | 
104 |   // To be removed
105 |   size_t im_add_letter(char ch) {
106 |     return 0;
107 |   }
108 | 
109 |   const char* im_get_sps_str(size_t *decoded_len) {
110 |     if (NULL == matrix_search)
111 |       return NULL;
112 | 
113 |     return matrix_search->get_pystr(decoded_len);
114 |   }
115 | 
116 |   char16* im_get_candidate(size_t cand_id, char16* cand_str,
117 |                         size_t max_len) {
118 |     if (NULL == matrix_search)
119 |       return NULL;
120 | 
121 |     return matrix_search->get_candidate(cand_id, cand_str, max_len);
122 |   }
123 | 
124 |   size_t im_get_spl_start_pos(const uint16 *&spl_start) {
125 |     if (NULL == matrix_search)
126 |       return 0;
127 | 
128 |     return matrix_search->get_spl_start(spl_start);
129 |   }
130 | 
131 |   size_t im_choose(size_t choice_id) {
132 |     if (NULL == matrix_search)
133 |       return 0;
134 | 
135 |     return matrix_search->choose(choice_id);
136 |   }
137 | 
138 |   size_t im_cancel_last_choice() {
139 |     if (NULL == matrix_search)
140 |       return 0;
141 | 
142 |     return matrix_search->cancel_last_choice();
143 |   }
144 | 
145 |   size_t im_get_fixed_len() {
146 |     if (NULL == matrix_search)
147 |       return 0;
148 | 
149 |     return matrix_search->get_fixedlen();
150 |   }
151 | 
152 |   // To be removed
153 |   bool im_cancel_input() {
154 |     return true;
155 |   }
156 | 
157 | 
158 |   size_t im_get_predicts(const char16 *his_buf,
159 |                          char16 (*&pre_buf)[kMaxPredictSize + 1]) {
160 |     if (NULL == his_buf)
161 |       return 0;
162 | 
163 |     size_t fixed_len = utf16_strlen(his_buf);
164 |     const char16 *fixed_ptr = his_buf;
165 |     if (fixed_len > kMaxPredictSize) {
166 |       fixed_ptr += fixed_len - kMaxPredictSize;
167 |       fixed_len = kMaxPredictSize;
168 |     }
169 | 
170 |     pre_buf = predict_buf;
171 |     return matrix_search->get_predicts(his_buf, pre_buf, kMaxPredictNum);
172 |   }
173 | 
174 |   void im_enable_shm_as_szm(bool enable) {
175 |     SpellingTrie &spl_trie = SpellingTrie::get_instance();
176 |     spl_trie.szm_enable_shm(enable);
177 |   }
178 | 
179 |   void im_enable_ym_as_szm(bool enable) {
180 |     SpellingTrie &spl_trie = SpellingTrie::get_instance();
181 |     spl_trie.szm_enable_ym(enable);
182 |   }
183 | 
184 |   void im_init_user_dictionary(const char *fn_usr_dict) {
185 |     if (!matrix_search)
186 |         return;
187 |     matrix_search->flush_cache();
188 |     matrix_search->init_user_dictionary(fn_usr_dict);
189 |   }
190 | 
191 |   bool im_is_user_dictionary_enabled(void) {
192 |     return NULL != matrix_search ? matrix_search->is_user_dictionary_enabled() : false;
193 |   }
194 | 
195 | #ifdef __cplusplus
196 | }
197 | #endif
198 | 


--------------------------------------------------------------------------------
/googlelib/searchutility.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #ifndef PINYINIME_ANDPY_INCLUDE_SEARCHCOMMON_H__
 18 | #define PINYINIME_ANDPY_INCLUDE_SEARCHCOMMON_H__
 19 | 
 20 | #include <stdlib.h>
 21 | #include "./spellingtrie.h"
 22 | 
 23 | namespace ime_pinyin {
 24 | 
 25 | // Type used to identify the size of a pool, such as id pool, etc.
 26 | typedef uint16 PoolPosType;
 27 | 
 28 | // Type used to identify a parsing mile stone in an atom dictionary.
 29 | typedef uint16 MileStoneHandle;
 30 | 
 31 | // Type used to express a lemma and its probability score.
 32 | typedef struct {
 33 |   size_t id:(kLemmaIdSize * 8);
 34 |   size_t lma_len:4;
 35 |   uint16 psb;  // The score, the lower psb, the higher possibility.
 36 |   // For single character items, we may also need Hanzi.
 37 |   // For multiple characer items, ignore it.
 38 |   char16 hanzi;
 39 | } LmaPsbItem, *PLmaPsbItem;
 40 | 
 41 | // LmaPsbItem extended with string.
 42 | typedef struct {
 43 |   LmaPsbItem lpi;
 44 |   char16 str[kMaxLemmaSize + 1];
 45 | } LmaPsbStrItem, *PLmaPsbStrItem;
 46 | 
 47 | 
 48 | typedef struct {
 49 |   float psb;
 50 |   char16 pre_hzs[kMaxPredictSize];
 51 |   uint16 his_len;  // The length of the history used to do the prediction.
 52 | } NPredictItem, *PNPredictItem;
 53 | 
 54 | // Parameter structure used to extend in a dictionary. All dictionaries
 55 | // receives the same DictExtPara and a dictionary specific MileStoneHandle for
 56 | // extending.
 57 | //
 58 | // When the user inputs a new character, AtomDictBase::extend_dict() will be
 59 | // called at least once for each dictionary.
 60 | //
 61 | // For example, when the user inputs "wm", extend_dict() will be called twice,
 62 | // and the DictExtPara parameter are as follows respectively:
 63 | // 1. splids = {w, m}; splids_extended = 1; ext_len = 1; step_no = 1;
 64 | // splid_end_split = false; id_start = wa(the first id start with 'w');
 65 | // id_num = number of ids starting with 'w'.
 66 | // 2. splids = {m}; splids_extended = 0; ext_len = 1; step_no = 1;
 67 | // splid_end_split = false; id_start = wa; id_num = number of ids starting with
 68 | // 'w'.
 69 | //
 70 | // For string "women", one of the cases of the DictExtPara parameter is:
 71 | // splids = {wo, men}, splids_extended = 1, ext_len = 3 (length of "men"),
 72 | // step_no = 4; splid_end_split = false; id_start = men, id_num = 1.
 73 | //
 74 | typedef struct {
 75 |   // Spelling ids for extending, there are splids_extended + 1 ids in the
 76 |   // buffer.
 77 |   // For a normal lemma, there can only be kMaxLemmaSize spelling ids in max,
 78 |   // but for a composing phrase, there can kMaxSearchSteps spelling ids.
 79 |   uint16 splids[kMaxSearchSteps];
 80 | 
 81 |   // Number of ids that have been used before. splids[splids_extended] is the
 82 |   // newly added id for the current extension.
 83 |   uint16 splids_extended;
 84 | 
 85 |   // The step span of the extension. It is also the size of the string for
 86 |   // the newly added spelling id.
 87 |   uint16 ext_len;
 88 | 
 89 |   // The step number for the current extension. It is also the ending position
 90 |   // in the input Pinyin string for the substring of spelling ids in splids[].
 91 |   // For example, when the user inputs "women", step_no = 4.
 92 |   // This parameter may useful to manage the MileStoneHandle list for each
 93 |   // step. When the user deletes a character from the string, MileStoneHandle
 94 |   // objects for the the steps after that character should be reset; when the
 95 |   // user begins a new string, all MileStoneHandle objects should be reset.
 96 |   uint16 step_no;
 97 | 
 98 |   // Indicate whether the newly added spelling ends with a splitting character
 99 |   bool splid_end_split;
100 | 
101 |   // If the newly added id is a half id, id_start is the first id of the
102 |   // corresponding full ids; if the newly added id is a full id, id_start is
103 |   // that id.
104 |   uint16 id_start;
105 | 
106 |   // If the newly added id is a half id, id_num is the number of corresponding
107 |   // ids; if it is a full id, id_num == 1.
108 |   uint16 id_num;
109 | }DictExtPara, *PDictExtPara;
110 | 
111 | bool is_system_lemma(LemmaIdType lma_id);
112 | bool is_user_lemma(LemmaIdType lma_id);
113 | bool is_composing_lemma(LemmaIdType lma_id);
114 | 
115 | int cmp_lpi_with_psb(const void *p1, const void *p2);
116 | int cmp_lpi_with_unified_psb(const void *p1, const void *p2);
117 | int cmp_lpi_with_id(const void *p1, const void *p2);
118 | int cmp_lpi_with_hanzi(const void *p1, const void *p2);
119 | 
120 | int cmp_lpsi_with_str(const void *p1, const void *p2);
121 | 
122 | int cmp_hanzis_1(const void *p1, const void *p2);
123 | int cmp_hanzis_2(const void *p1, const void *p2);
124 | int cmp_hanzis_3(const void *p1, const void *p2);
125 | int cmp_hanzis_4(const void *p1, const void *p2);
126 | int cmp_hanzis_5(const void *p1, const void *p2);
127 | int cmp_hanzis_6(const void *p1, const void *p2);
128 | int cmp_hanzis_7(const void *p1, const void *p2);
129 | int cmp_hanzis_8(const void *p1, const void *p2);
130 | 
131 | int cmp_npre_by_score(const void *p1, const void *p2);
132 | int cmp_npre_by_hislen_score(const void *p1, const void *p2);
133 | int cmp_npre_by_hanzi_score(const void *p1, const void *p2);
134 | 
135 | 
136 | size_t remove_duplicate_npre(NPredictItem *npre_items, size_t npre_num);
137 | 
138 | size_t align_to_size_t(size_t size);
139 | 
140 | }  // namespace
141 | 
142 | #endif  // PINYINIME_ANDPY_INCLUDE_SEARCHCOMMON_H__
143 | 


--------------------------------------------------------------------------------
/googlelib/dictbuilder.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #ifndef PINYINIME_INCLUDE_DICTBUILDER_H__
 18 | #define PINYINIME_INCLUDE_DICTBUILDER_H__
 19 | 
 20 | #include <stdlib.h>
 21 | #include "./utf16char.h"
 22 | #include "./dictdef.h"
 23 | #include "./dictlist.h"
 24 | #include "./spellingtable.h"
 25 | #include "./spellingtrie.h"
 26 | #include "./splparser.h"
 27 | 
 28 | namespace ime_pinyin {
 29 | 
 30 | #ifdef ___BUILD_MODEL___
 31 | 
 32 | #define ___DO_STATISTICS___
 33 | 
 34 | class DictTrie;
 35 | 
 36 | class DictBuilder {
 37 |  private:
 38 |   // The raw lemma array buffer.
 39 |   LemmaEntry *lemma_arr_;
 40 |   size_t lemma_num_;
 41 | 
 42 |   // Used to store all possible single char items.
 43 |   // Two items may have the same Hanzi while their spelling ids are different.
 44 |   SingleCharItem *scis_;
 45 |   size_t scis_num_;
 46 | 
 47 |   // In the tree, root's level is -1.
 48 |   // Lemma nodes for root, and level 0
 49 |   LmaNodeLE0 *lma_nodes_le0_;
 50 | 
 51 |   // Lemma nodes for layers whose levels are deeper than 0
 52 |   LmaNodeGE1 *lma_nodes_ge1_;
 53 | 
 54 |   // Number of used lemma nodes
 55 |   size_t lma_nds_used_num_le0_;
 56 |   size_t lma_nds_used_num_ge1_;
 57 | 
 58 |   // Used to store homophonies' ids.
 59 |   LemmaIdType *homo_idx_buf_;
 60 |   // Number of homophonies each of which only contains one Chinese character.
 61 |   size_t homo_idx_num_eq1_;
 62 |   // Number of homophonies each of which contains more than one character.
 63 |   size_t homo_idx_num_gt1_;
 64 | 
 65 |   // The items with highest scores.
 66 |   LemmaEntry *top_lmas_;
 67 |   size_t top_lmas_num_;
 68 | 
 69 |   SpellingTable *spl_table_;
 70 |   SpellingParser *spl_parser_;
 71 | 
 72 | #ifdef ___DO_STATISTICS___
 73 |   size_t max_sonbuf_len_[kMaxLemmaSize];
 74 |   size_t max_homobuf_len_[kMaxLemmaSize];
 75 | 
 76 |   size_t total_son_num_[kMaxLemmaSize];
 77 |   size_t total_node_hasson_[kMaxLemmaSize];
 78 |   size_t total_sonbuf_num_[kMaxLemmaSize];
 79 |   size_t total_sonbuf_allnoson_[kMaxLemmaSize];
 80 |   size_t total_node_in_sonbuf_allnoson_[kMaxLemmaSize];
 81 |   size_t total_homo_num_[kMaxLemmaSize];
 82 | 
 83 |   size_t sonbufs_num1_;     // Number of son buffer with only 1 son
 84 |   size_t sonbufs_numgt1_;   // Number of son buffer with more 1 son;
 85 | 
 86 |   size_t total_lma_node_num_;
 87 | 
 88 |   void stat_init();
 89 |   void stat_print();
 90 | #endif
 91 | 
 92 |  public:
 93 | 
 94 |   DictBuilder();
 95 |   ~DictBuilder();
 96 | 
 97 |   // Build dictionary trie from the file fn_raw. File fn_validhzs provides
 98 |   // valid chars. If fn_validhzs is NULL, only chars in GB2312 will be
 99 |   // included.
100 |   bool build_dict(const char* fn_raw, const char* fn_validhzs,
101 |                   DictTrie *dict_trie);
102 | 
103 |  private:
104 |   // Fill in the buffer with id. The caller guarantees that the paramters are
105 |   // vaild.
106 |   void id_to_charbuf(unsigned char *buf, LemmaIdType id);
107 | 
108 |   // Update the offset of sons for a node.
109 |   void set_son_offset(LmaNodeGE1 *node, size_t offset);
110 | 
111 |   // Update the offset of homophonies' ids for a node.
112 |   void set_homo_id_buf_offset(LmaNodeGE1 *node, size_t offset);
113 | 
114 |   // Format a speling string.
115 |   void format_spelling_str(char *spl_str);
116 | 
117 |   // Sort the lemma_arr by the hanzi string, and give each of unique items
118 |   // a id. Why we need to sort the lemma list according to their Hanzi string
119 |   // is to find items started by a given prefix string to do prediction.
120 |   // Actually, the single char items are be in other order, for example,
121 |   // in spelling id order, etc.
122 |   // Return value is next un-allocated idx available.
123 |   LemmaIdType sort_lemmas_by_hz();
124 | 
125 |   // Build the SingleCharItem list, and fill the hanzi_scis_ids in the
126 |   // lemma buffer lemma_arr_.
127 |   // This function should be called after the lemma array is ready.
128 |   // Return the number of unique SingleCharItem elements.
129 |   size_t build_scis();
130 | 
131 |   // Construct a subtree using a subset of the spelling array (from
132 |   // item_star to item_end)
133 |   // parent is the parent node to update the necessary information
134 |   // parent can be a member of LmaNodeLE0 or LmaNodeGE1
135 |   bool construct_subset(void* parent, LemmaEntry* lemma_arr,
136 |                         size_t item_start, size_t item_end, size_t level);
137 | 
138 | 
139 |   // Read valid Chinese Hanzis from the given file.
140 |   // num is used to return number of chars.
141 |   // The return buffer is sorted and caller needs to free the returned buffer.
142 |   char16* read_valid_hanzis(const char *fn_validhzs, size_t *num);
143 | 
144 | 
145 |   // Read a raw dictionary. max_item is the maximum number of items. If there
146 |   // are more items in the ditionary, only the first max_item will be read.
147 |   // Returned value is the number of items successfully read from the file.
148 |   size_t read_raw_dict(const char* fn_raw, const char *fn_validhzs,
149 |                        size_t max_item);
150 | 
151 |   // Try to find if a character is in hzs buffer.
152 |   bool hz_in_hanzis_list(const char16 *hzs, size_t hzs_len, char16 hz);
153 | 
154 |   // Try to find if all characters in str are in hzs buffer.
155 |   bool str_in_hanzis_list(const char16 *hzs, size_t hzs_len,
156 |                           const char16 *str, size_t str_len);
157 | 
158 |   // Get these lemmas with toppest scores.
159 |   void get_top_lemmas();
160 | 
161 |   // Allocate resource to build dictionary.
162 |   // lma_num is the number of items to be loaded
163 |   bool alloc_resource(size_t lma_num);
164 | 
165 |   // Free resource.
166 |   void free_resource();
167 | };
168 | #endif  // ___BUILD_MODEL___
169 | }
170 | 
171 | #endif  // PINYINIME_INCLUDE_DICTBUILDER_H__
172 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # google_pinyinim
  2 | 
  3 | ![forks](https://img.shields.io/github/forks/aron566/google_pinyinim)
  4 | ![stars](https://img.shields.io/github/stars/aron566/google_pinyinim)
  5 | [![license](https://img.shields.io/github/license/aron566/google_pinyinim)](https://github.com/aron566/google_pinyinim/blob/master/LICENSE)
  6 | ![](https://img.shields.io/badge/platform-Linux|Windows|Mac|Embedded-orange.svg)
  7 | 
  8 | ```
  9 |                              _                       _                _         _            
 10 |                             | |                     (_)              (_)       (_)           
 11 |   __ _   ___    ___    __ _ | |  ___          _ __   _  _ __   _   _  _  _ __   _  _ __ ___  
 12 |  / _` | / _ \  / _ \  / _` || | / _ \        | '_ \ | || '_ \ | | | || || '_ \ | || '_ ` _ \ 
 13 | | (_| || (_) || (_) || (_| || ||  __/        | |_) || || | | || |_| || || | | || || | | | | |
 14 |  \__, | \___/  \___/  \__, ||_| \___|        | .__/ |_||_| |_| \__, ||_||_| |_||_||_| |_| |_|
 15 |   __/ |                __/ |          ______ | |                __/ |                        
 16 |  |___/                |___/          |______||_|               |___/                         
 17 | ```
 18 | 
 19 | 谷歌拼音输入法移植至QT
 20 | 
 21 | # 移植方法
 22 | 
 23 | ## 第一种直接带入源码编译
 24 | ### 这个无需多讲，直接将所有的工程包含到你所需的工程中去即可，可查看.pro文件涉及哪些文件。
 25 | ## 第二种链接方式
 26 | 1、clone工程到你的本地目录，打开项目
 27 | 2、动态库的生成：修改项目中.pro文件中TEMPLATE = **app**改为**lib**即可生成动态库文件
 28 | 
 29 | ```bash
 30 | # 生成库文件
 31 | TARGET = virtualkeyboard
 32 | TEMPLATE = lib
 33 | ```
 34 | 
 35 | 3、静态库的生成（不会更新之前已生成的动态库文件！）：
 36 | 
 37 | ```bash
 38 | # 生成库文件
 39 | TARGET = virtualkeyboard
 40 | TEMPLATE = lib
 41 | # 指定生成为静态库
 42 | CONFIG += staticlib
 43 | ```
 44 | 
 45 | 4、取出工程中所有.h文件及编译生成的库文件
 46 | 5、将.h文件加入到**你需要用到的工程中**，.pro文件中链接此动态库文件即可
 47 | 
 48 | 项目中`.pro`文件增加的内容如下（或者右键添加库->选择外部->选择对应库文件）：
 49 | 
 50 | ```bash
 51 | # 软键盘
 52 | include(keyboard/keyboard.pri)
 53 | 
 54 | # 软键盘linux
 55 | unix:!macx: LIBS += -L$$PWD/keyboard/
 56 | unix:!macx: LIBS += -lvirtualkeyboard
 57 | 
 58 | # 软键盘windows
 59 | win32: LIBS += -L$$PWD/keyboard/
 60 | win32: LIBS += -lvirtualkeyboard
 61 | ```
 62 | 
 63 | ```bash
 64 | # 链接静态库时，多添加一行定义，否者在.h的头文件中声明了导入（Q_DECL_IMPORT），则在windows平台编译会找动态库，此时找不到会报未定义的错误
 65 | DEFINES += VIRTUALKEYBOARD_LIBRARY
 66 | ```
 67 | 
 68 | 
 69 | 
 70 | 5.1、在你的工程中新建keyboard文件夹
 71 | 
 72 | 5.2、在keyboard文件夹中新建keyboard.pri文件
 73 | 
 74 | 填入如下内容：
 75 | 
 76 | ```bash
 77 | HEADERS += \
 78 |     $$PWD/customerqpushbutton.h \
 79 |     $$PWD/keyboard.h \
 80 |     $$PWD/lib/atomdictbase.h \
 81 |     $$PWD/lib/dictbuilder.h \
 82 |     $$PWD/lib/dictdef.h \
 83 |     $$PWD/lib/dictlist.h \
 84 |     $$PWD/lib/dicttrie.h \
 85 |     $$PWD/lib/lpicache.h \
 86 |     $$PWD/lib/matrixsearch.h \
 87 |     $$PWD/lib/mystdlib.h \
 88 |     $$PWD/lib/ngram.h \
 89 |     $$PWD/lib/pinyinime.h \
 90 |     $$PWD/lib/pinyinime_global.h \
 91 |     $$PWD/lib/searchutility.h \
 92 |     $$PWD/lib/spellingtable.h \
 93 |     $$PWD/lib/spellingtrie.h \
 94 |     $$PWD/lib/splparser.h \
 95 |     $$PWD/lib/sync.h \
 96 |     $$PWD/lib/userdict.h \
 97 |     $$PWD/lib/utf16char.h \
 98 |     $$PWD/lib/utf16reader.h \
 99 |     $$PWD/virtualkeyboard.h
100 | ```
101 | 
102 | 5.3、复制`customerqpushbutton.h` `keyboard.h` `virtualkeyboard.h`到`keyboard`文件夹
103 | 
104 | 5.4、复制编译的动态库文件或者静态库文件到`keyboard`文件夹
105 | 
106 | 5.5、在**keyboard**文件夹中新建**lib**文件夹
107 | 
108 | 将克隆的工程中`googlelib`文件夹中所有.h后缀的文件复制到此
109 | 
110 | 5.6、修改`keyboard.h` 中`#include <pinyinime.h>`改为 `#include <keyboard/lib/pinyinime.h>`
111 | 
112 | 5.7、复制`data`目录下的词典文件到**keyboard**文件夹内，（在其他目标板上运行，需将词典复制过去，实例化时指定加载路径信息）
113 | 
114 | 6、点击小锤子编译即可
115 | 
116 | 7、如果动态库则上传动态库文件到特定硬件上
117 | 
118 | ## 修改相关参数
119 | ### 修改输入法界面尺寸大小
120 | - 打开`virtualkeyboard.h`文件,文件中定义皆以**pixel**为单位
121 |   修改其中`UI_KEYBOARDWINDOW_WIDTH`与`UI_KEYBOARDWINDOW_HEIGHT`宏定义为多少pixel
122 |   修改显示中文结果的间隙大小：水平间隙`CHINESESEARCH_BLOCKSTYLE_MULTI_RECT_X_GAP`，垂直间隙`CHINESESEARCH_BLOCKSTYLE_MULTI_RECT_Y_GAP`
123 |   修改每页显示数目：列数目`CHINESESEARCH_BLOCKSTYLE_RECT_H_NUM_MAX`，行数目`CHINESESEARCH_BLOCKSTYLE_RECT_V_NUM_MAX`
124 | 
125 |   **其他修改直接看文件中说明**
126 | 
127 |   # 使用方式
128 | 
129 |   输入框使用单一`QLineEdit`控件便于管理
130 |   
131 |   ## 初始化部分
132 |   
133 |   1、实例化键盘
134 |   
135 |   ```cpp
136 |   keyboard *pKeyboard = new keyboard(this ,"谷歌词典文件路径" ,"用户词典文件路径");
137 |   ```
138 |   
139 |   2、连接键盘输入结束信号
140 |   
141 |   ```cpp
142 |   connect(pKeyboard ,&keyboard::editisModifiedok ,this ,&MainWindow::slotKeyboardReturn);
143 |   ```
144 |   
145 |   6、槽函数处理，参数str为输入的内容
146 |   
147 |   ```cpp
148 |   void MainWindow::slotKeyboardReturn(QString str)
149 |   {
150 |       /*do something...*/
151 |   }
152 |   ```
153 |   
154 |   ## 调用键盘
155 |   
156 |   ```cpp
157 |   /*设置显示键盘*/
158 |   void showKeyboard(QString title = "键入xx的内容：" ,QString str = "2020");
159 |   /*单独设置标题*/
160 |   void set_editTips(QString title = "键入xx的内容：");
161 |   /*单独设置输入框内容*/
162 |   void set_editBox(QString str = "2020");
163 |   /*设置键盘模式*/
164 |   void set_keyboardmode(KEYBOARD_MODE mode);
165 |   ```
166 | 
167 | **1、调整键盘模式**
168 | 
169 | ```c
170 | enum KEYBOARD_MODE
171 | {
172 |     NUM_ONLY = 0,/**< 数字模式,字母键将不可用*/
173 |     EN_ONLY,/**< 英文模式,数字键将不可用*/
174 |     ANY,/**< 全功能,默认模式*/
175 | };
176 | ```
177 | 
178 | eg：
179 | 
180 | ```cpp
181 | pKeyboard->set_keyboardmode(keyboard::NUM_ONLY);
182 | ```
183 | 
184 | ## 互动
185 | 
186 | 1、主线程的主界面部分管理
187 | 
188 | ```cpp
189 | private:
190 |     QWidget* pLastCallobj;/**< 保存着上次隐藏的页面，也可不用，但需要调用键盘的页面不隐藏*/
191 |     QLineEdit *pLastCallwidget;/**< 保存上次调用键盘的控件*/
192 | ```
193 | 
194 | 2、主线程建立槽，调用键盘显示
195 | 
196 | ```cpp
197 | /*连接子界面要求显示键盘的信号*/
198 | connect(parameterui, &parameter::show_keyboard, this, &MainWindow::slotprocessedit);
199 | ```
200 | 
201 | ```cpp
202 | void MainWindow::slotprocessedit(QWidget *pObject, QLineEdit *pwidget, QString title, QString edittext)
203 | {
204 |     pLastCallobj = pObject;
205 |     pLastCallwidget = pwidget;
206 | 	pKeyboard->showKeyboard(title, edittext);
207 | }
208 | ```
209 | 
210 | 3、主线程，连接键盘输入完成信号
211 | 
212 | ```cpp
213 | connect(pKeyboard, &keyboard::editisModifiedok, this, &MainWindow::slotKeyboardReturn);
214 | ```
215 | 
216 | ```cpp
217 | void MainWindow::slotKeyboardReturn(QString str)
218 | {
219 |     /*重新显示子级页面*/
220 |     pLastCallobj->show();
221 |     /*将键盘的字符串给编辑框*/
222 |     pLastCallwidget->setText(str);
223 | }
224 | ```
225 | 
226 | 4、主线程子级页面，输入控件被点击处理
227 | 
228 | QLineEdit控件有以下几个信号：
229 | 
230 | ```cpp
231 | // returnPressed：聚焦在控件上按下回车键时发出，通常用作不带触摸屏的环境
232 | // selectionChanged：聚焦到时发出一次信号
233 | // 连接信号：
234 | connect(ui->clientiplineEdit, &QLineEdit::selectionChanged, this, &parameter::on_clientip_Pressed);
235 | connect(ui->clientiplineEdit, &QLineEdit::returnPressed, this, &parameter::on_clientip_Pressed);
236 | /*需要注意的是，当控件代码由UI设计器自动生成时，信号与槽的建立应当在UI设计器中完成*/
237 | ```
238 | 
239 | ```cpp
240 | signals:
241 |     void show_keyboard(QWidget *, QLineEdit *, QString, QString);
242 | ```
243 | 
244 | ```cpp
245 | /*在parameter界面，clientip输入框被点击*/
246 | void parameter::on_clientip_Pressed()
247 | {
248 |     emit show_keyboard(this ,ui->clientip,"输入客户端IP" ,ui->clientip->text());
249 |     this->hide();
250 | }
251 | ```
252 | 
253 | 5、连接信号与槽
254 | 
255 | ```cpp
256 | connect(parameterui ,&parameter::show_keyboard ,this ,&MainWindow::slotprocessedit);/*parameterui为主界面的子级页面*/
257 | ```
258 | 
259 | 


--------------------------------------------------------------------------------
/googlelib/searchutility.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #include <assert.h>
 18 | #include "mystdlib.h"
 19 | #include "searchutility.h"
 20 | 
 21 | namespace ime_pinyin {
 22 | 
 23 | bool is_system_lemma(LemmaIdType lma_id) {
 24 |   return (0 < lma_id && lma_id <= kSysDictIdEnd);
 25 | }
 26 | 
 27 | bool is_user_lemma(LemmaIdType lma_id) {
 28 |   return (kUserDictIdStart <= lma_id && lma_id <= kUserDictIdEnd);
 29 | }
 30 | 
 31 | bool is_composing_lemma(LemmaIdType lma_id) {
 32 |   return (kLemmaIdComposing == lma_id);
 33 | }
 34 | 
 35 | int cmp_lpi_with_psb(const void *p1, const void *p2) {
 36 |   if ((static_cast<const LmaPsbItem*>(p1))->psb >
 37 |       (static_cast<const LmaPsbItem*>(p2))->psb)
 38 |     return 1;
 39 |   if ((static_cast<const LmaPsbItem*>(p1))->psb <
 40 |       (static_cast<const LmaPsbItem*>(p2))->psb)
 41 |     return -1;
 42 |   return 0;
 43 | }
 44 | 
 45 | int cmp_lpi_with_unified_psb(const void *p1, const void *p2) {
 46 |   const LmaPsbItem *item1 = static_cast<const LmaPsbItem*>(p1);
 47 |   const LmaPsbItem *item2 = static_cast<const LmaPsbItem*>(p2);
 48 | 
 49 |   // The real unified psb is psb1 / lma_len1 and psb2 * lma_len2
 50 |   // But we use psb1 * lma_len2 and psb2 * lma_len1 to get better
 51 |   // precision.
 52 |   size_t up1 = item1->psb * (item2->lma_len);
 53 |   size_t up2 = item2->psb * (item1->lma_len);
 54 |   if (up1 < up2) {
 55 |     return -1;
 56 |   }
 57 |   if (up1 > up2) {
 58 |     return 1;
 59 |   }
 60 |   return 0;
 61 | }
 62 | 
 63 | int cmp_lpi_with_id(const void *p1, const void *p2) {
 64 |   if ((static_cast<const LmaPsbItem*>(p1))->id <
 65 |       (static_cast<const LmaPsbItem*>(p2))->id)
 66 |     return -1;
 67 |   if ((static_cast<const LmaPsbItem*>(p1))->id >
 68 |       (static_cast<const LmaPsbItem*>(p2))->id)
 69 |     return 1;
 70 |   return 0;
 71 | }
 72 | 
 73 | int cmp_lpi_with_hanzi(const void *p1, const void *p2) {
 74 |   if ((static_cast<const LmaPsbItem*>(p1))->hanzi <
 75 |       (static_cast<const LmaPsbItem*>(p2))->hanzi)
 76 |     return -1;
 77 |   if ((static_cast<const LmaPsbItem*>(p1))->hanzi >
 78 |       (static_cast<const LmaPsbItem*>(p2))->hanzi)
 79 |     return 1;
 80 | 
 81 |   return 0;
 82 | }
 83 | 
 84 | int cmp_lpsi_with_str(const void *p1, const void *p2) {
 85 |   return utf16_strcmp((static_cast<const LmaPsbStrItem*>(p1))->str,
 86 |                       (static_cast<const LmaPsbStrItem*>(p2))->str);
 87 | }
 88 | 
 89 | 
 90 | int cmp_hanzis_1(const void *p1, const void *p2) {
 91 |   if (*static_cast<const char16*>(p1) <
 92 |       *static_cast<const char16*>(p2))
 93 |     return -1;
 94 | 
 95 |   if (*static_cast<const char16*>(p1) >
 96 |       *static_cast<const char16*>(p2))
 97 |     return 1;
 98 |   return 0;
 99 | }
100 | 
101 | int cmp_hanzis_2(const void *p1, const void *p2) {
102 |   return  utf16_strncmp(static_cast<const char16*>(p1),
103 |                         static_cast<const char16*>(p2), 2);
104 | }
105 | 
106 | int cmp_hanzis_3(const void *p1, const void *p2) {
107 |   return  utf16_strncmp(static_cast<const char16*>(p1),
108 |                         static_cast<const char16*>(p2), 3);
109 | }
110 | 
111 | int cmp_hanzis_4(const void *p1, const void *p2) {
112 |   return  utf16_strncmp(static_cast<const char16*>(p1),
113 |                         static_cast<const char16*>(p2), 4);
114 | }
115 | 
116 | int cmp_hanzis_5(const void *p1, const void *p2) {
117 |   return  utf16_strncmp(static_cast<const char16*>(p1),
118 |                         static_cast<const char16*>(p2), 5);
119 | }
120 | 
121 | int cmp_hanzis_6(const void *p1, const void *p2) {
122 |   return  utf16_strncmp(static_cast<const char16*>(p1),
123 |                         static_cast<const char16*>(p2), 6);
124 | }
125 | 
126 | int cmp_hanzis_7(const void *p1, const void *p2) {
127 |   return  utf16_strncmp(static_cast<const char16*>(p1),
128 |                         static_cast<const char16*>(p2), 7);
129 | }
130 | 
131 | int cmp_hanzis_8(const void *p1, const void *p2) {
132 |   return  utf16_strncmp(static_cast<const char16*>(p1),
133 |                         static_cast<const char16*>(p2), 8);
134 | }
135 | 
136 | int cmp_npre_by_score(const void *p1, const void *p2) {
137 |   if ((static_cast<const NPredictItem*>(p1))->psb >
138 |       (static_cast<const NPredictItem*>(p2))->psb)
139 |     return 1;
140 | 
141 |   if ((static_cast<const NPredictItem*>(p1))->psb <
142 |       (static_cast<const NPredictItem*>(p2))->psb)
143 |     return -1;
144 | 
145 |   return 0;
146 | }
147 | 
148 | int cmp_npre_by_hislen_score(const void *p1, const void *p2) {
149 |   if ((static_cast<const NPredictItem*>(p1))->his_len <
150 |       (static_cast<const NPredictItem*>(p2))->his_len)
151 |     return 1;
152 | 
153 |   if ((static_cast<const NPredictItem*>(p1))->his_len >
154 |       (static_cast<const NPredictItem*>(p2))->his_len)
155 |     return -1;
156 | 
157 |   if ((static_cast<const NPredictItem*>(p1))->psb >
158 |       (static_cast<const NPredictItem*>(p2))->psb)
159 |     return 1;
160 | 
161 |   if ((static_cast<const NPredictItem*>(p1))->psb <
162 |       (static_cast<const NPredictItem*>(p2))->psb)
163 |     return -1;
164 | 
165 |   return 0;
166 | }
167 | 
168 | int cmp_npre_by_hanzi_score(const void *p1, const void *p2) {
169 |   int ret_v = (utf16_strncmp((static_cast<const NPredictItem*>(p1))->pre_hzs,
170 |       (static_cast<const NPredictItem*>(p2))->pre_hzs, kMaxPredictSize));
171 |   if (0 != ret_v)
172 |     return ret_v;
173 | 
174 |   if ((static_cast<const NPredictItem*>(p1))->psb >
175 |       (static_cast<const NPredictItem*>(p2))->psb)
176 |     return 1;
177 | 
178 |   if ((static_cast<const NPredictItem*>(p1))->psb <
179 |       (static_cast<const NPredictItem*>(p2))->psb)
180 |     return -1;
181 | 
182 |   return 0;
183 | }
184 | 
185 | size_t remove_duplicate_npre(NPredictItem *npre_items, size_t npre_num) {
186 |   if (NULL == npre_items || 0 == npre_num)
187 |     return 0;
188 | 
189 |   myqsort(npre_items, npre_num, sizeof(NPredictItem), cmp_npre_by_hanzi_score);
190 | 
191 |   size_t remain_num = 1;  // The first one is reserved.
192 |   for (size_t pos = 1; pos < npre_num; pos++) {
193 |     if (utf16_strncmp(npre_items[pos].pre_hzs,
194 |                       npre_items[remain_num - 1].pre_hzs,
195 |                       kMaxPredictSize) != 0) {
196 |       if (remain_num != pos) {
197 |         npre_items[remain_num] = npre_items[pos];
198 |       }
199 |       remain_num++;
200 |     }
201 |   }
202 |   return remain_num;
203 | }
204 | 
205 | size_t align_to_size_t(size_t size) {
206 |   size_t s = sizeof(size_t);
207 |   return (size + s -1) / s * s;
208 | }
209 | 
210 | }  // namespace ime_pinyin
211 | 


--------------------------------------------------------------------------------
/googlelib/pinyinime.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #ifndef PINYINIME_INCLUDE_ANDPYIME_H__
 18 | #define PINYINIME_INCLUDE_ANDPYIME_H__
 19 | 
 20 | #include <stdlib.h>
 21 | #include "./dictdef.h"
 22 | 
 23 | #ifdef __cplusplus
 24 | extern "C" {
 25 | #endif
 26 | 
 27 |   namespace ime_pinyin {
 28 | 
 29 |   /**
 30 |    * Open the decoder engine via the system and user dictionary file names.
 31 |    *
 32 |    * @param fn_sys_dict The file name of the system dictionary.
 33 |    * @param fn_usr_dict The file name of the user dictionary.
 34 |    * @return true if open the decoder engine successfully.
 35 |    */
 36 |   bool im_open_decoder(const char *fn_sys_dict, const char *fn_usr_dict);
 37 | 
 38 |   /**
 39 |    * Open the decoder engine via the system dictionary FD and user dictionary
 40 |    * file name. Because on Android, the system dictionary is embedded in the
 41 |    * whole application apk file.
 42 |    *
 43 |    * @param sys_fd The file in which the system dictionary is embedded.
 44 |    * @param start_offset The starting position of the system dictionary in the
 45 |    * file sys_fd.
 46 |    * @param length The length of the system dictionary in the file sys_fd,
 47 |    * counted in byte.
 48 |    * @return true if succeed.
 49 |    */
 50 |   bool im_open_decoder_fd(int sys_fd, long start_offset, long length,
 51 |                           const char *fn_usr_dict);
 52 | 
 53 |   /**
 54 |    * Close the decoder engine.
 55 |    */
 56 |   void im_close_decoder();
 57 | 
 58 |   /**
 59 |    * Set maximum limitations for decoding. If this function is not called,
 60 |    * default values will be used. For example, due to screen size limitation,
 61 |    * the UI engine of the IME can only show a certain number of letters(input)
 62 |    * to decode, and a certain number of Chinese characters(output). If after
 63 |    * user adds a new letter, the input or the output string is longer than the
 64 |    * limitations, the engine will discard the recent letter.
 65 |    *
 66 |    * @param max_sps_len Maximum length of the spelling string(Pinyin string).
 67 |    * @max_hzs_len Maximum length of the decoded Chinese character string.
 68 |    */
 69 |   void im_set_max_lens(size_t max_sps_len, size_t max_hzs_len);
 70 | 
 71 |   /**
 72 |    * Flush cached data to persistent memory. Because at runtime, in order to
 73 |    * achieve best performance, some data is only store in memory.
 74 |    */
 75 |   void im_flush_cache();
 76 | 
 77 |   /**
 78 |    * Use a spelling string(Pinyin string) to search. The engine will try to do
 79 |    * an incremental search based on its previous search result, so if the new
 80 |    * string has the same prefix with the previous one stored in the decoder,
 81 |    * the decoder will only continue the search from the end of the prefix.
 82 |    * If the caller needs to do a brand new search, please call im_reset_search()
 83 |    * first. Calling im_search() is equivalent to calling im_add_letter() one by
 84 |    * one.
 85 |    *
 86 |    * @param sps_buf The spelling string buffer to decode.
 87 |    * @param sps_len The length of the spelling string buffer.
 88 |    * @return The number of candidates.
 89 |    */
 90 |   size_t im_search(const char* sps_buf, size_t sps_len);
 91 | 
 92 |   /**
 93 |    * Make a delete operation in the current search result, and make research if
 94 |    * necessary.
 95 |    *
 96 |    * @param pos The posistion of char in spelling string to delete, or the
 97 |    * position of spelling id in result string to delete.
 98 |    * @param is_pos_in_splid Indicate whether the pos parameter is the position
 99 |    * in the spelling string, or the position in the result spelling id string.
100 |    * @return The number of candidates.
101 |    */
102 |   size_t im_delsearch(size_t pos, bool is_pos_in_splid,
103 |                       bool clear_fixed_this_step);
104 | 
105 |   /**
106 |    * Reset the previous search result.
107 |    */
108 |   void im_reset_search();
109 | 
110 |   /**
111 |    * Add a Pinyin letter to the current spelling string kept by decoder. If the
112 |    * decoder fails in adding the letter, it will do nothing. im_get_sps_str()
113 |    * can be used to get the spelling string kept by decoder currently.
114 |    *
115 |    * @param ch The letter to add.
116 |    * @return The number of candidates.
117 |    */
118 |   size_t im_add_letter(char ch);
119 | 
120 |   /**
121 |    * Get the spelling string kept by the decoder.
122 |    *
123 |    * @param decoded_len Used to return how many characters in the spelling
124 |    * string is successfully parsed.
125 |    * @return The spelling string kept by the decoder.
126 |    */
127 |   const char *im_get_sps_str(size_t *decoded_len);
128 | 
129 |   /**
130 |    * Get a candidate(or choice) string.
131 |    *
132 |    * @param cand_id The id to get a candidate. Started from 0. Usually, id 0
133 |    * is a sentence-level candidate.
134 |    * @param cand_str The buffer to store the candidate.
135 |    * @param max_len The maximum length of the buffer.
136 |    * @return cand_str if succeeds, otherwise NULL.
137 |    */
138 |   char16* im_get_candidate(size_t cand_id, char16* cand_str,
139 |                            size_t max_len);
140 | 
141 |   /**
142 |    * Get the segmentation information(the starting positions) of the spelling
143 |    * string.
144 |    *
145 |    * @param spl_start Used to return the starting posistions.
146 |    * @return The number of spelling ids. If it is L, there will be L+1 valid
147 |    * elements in spl_start, and spl_start[L] is the posistion after the end of
148 |    * the last spelling id.
149 |    */
150 |   size_t im_get_spl_start_pos(const uint16 *&spl_start);
151 | 
152 |   /**
153 |    * Choose a candidate and make it fixed. If the candidate does not match
154 |    * the end of all spelling ids, new candidates will be provided from the
155 |    * first unfixed position. If the candidate matches the end of the all
156 |    * spelling ids, there will be only one new candidates, or the whole fixed
157 |    * sentence.
158 |    *
159 |    * @param cand_id The id of candidate to select and make it fixed.
160 |    * @return The number of candidates. If after the selection, the whole result
161 |    * string has been fixed, there will be only one candidate.
162 |    */
163 |   size_t im_choose(size_t cand_id);
164 | 
165 |   /**
166 |    * Cancel the last selection, or revert the last operation of im_choose().
167 |    *
168 |    * @return The number of candidates.
169 |    */
170 |   size_t im_cancel_last_choice();
171 | 
172 |   /**
173 |    * Get the number of fixed spelling ids, or Chinese characters.
174 |    *
175 |    * @return The number of fixed spelling ids, of Chinese characters.
176 |    */
177 |   size_t im_get_fixed_len();
178 | 
179 |   /**
180 |    * Cancel the input state and reset the search workspace.
181 |    */
182 |   bool im_cancel_input();
183 | 
184 |   /**
185 |    * Get prediction candiates based on the given fixed Chinese string as the
186 |    * history.
187 |    *
188 |    * @param his_buf The history buffer to do the prediction. It should be ended
189 |    * with '\0'.
190 |    * @param pre_buf Used to return prediction result list.
191 |    * @return The number of predicted result string.
192 |    */
193 |   size_t im_get_predicts(const char16 *his_buf,
194 |                          char16 (*&pre_buf)[kMaxPredictSize + 1]);
195 | 
196 |   /**
197 |    * Enable Shengmus in ShouZiMu mode.
198 |    */
199 |   void im_enable_shm_as_szm(bool enable);
200 | 
201 |   /**
202 |    * Enable Yunmus in ShouZiMu mode.
203 |    */
204 |   void im_enable_ym_as_szm(bool enable);
205 | 
206 |   /**
207 |    * Initializes or uninitializes the user dictionary.
208 |    *
209 |    * @param fn_usr_dict The file name of the user dictionary.
210 |    */
211 |   void im_init_user_dictionary(const char *fn_usr_dict);
212 | 
213 |   /**
214 |    * Returns the current status of user dictinary.
215 |    */
216 |   bool im_is_user_dictionary_enabled(void);
217 | }
218 | 
219 | #ifdef __cplusplus
220 | }
221 | #endif
222 | 
223 | #endif  // PINYINIME_INCLUDE_ANDPYIME_H__
224 | 


--------------------------------------------------------------------------------
/googlelib/spellingtable.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #include <assert.h>
 18 | #include <stdlib.h>
 19 | #include <stdio.h>
 20 | #include <string.h>
 21 | #include <math.h>
 22 | #include "spellingtable.h"
 23 | 
 24 | namespace ime_pinyin {
 25 | 
 26 | #ifdef ___BUILD_MODEL___
 27 | 
 28 | const char SpellingTable::
 29 |     kNotSupportList[kNotSupportNum][kMaxSpellingSize + 1] = {"HM", "HNG", "NG"};
 30 | 
 31 | // "" is the biggest, so that all empty strings will be moved to the end
 32 | // _eb mean empty is biggest
 33 | int compare_raw_spl_eb(const void* p1, const void* p2) {
 34 |   if ('\0' == (static_cast<const RawSpelling*>(p1))->str[0])
 35 |     return 1;
 36 | 
 37 |   if ('\0' == (static_cast<const RawSpelling*>(p2))->str[0])
 38 |     return -1;
 39 | 
 40 |   return strcmp((static_cast<const RawSpelling*>(p1))->str,
 41 |                 (static_cast<const RawSpelling*>(p2))->str);
 42 | }
 43 | 
 44 | size_t get_odd_next(size_t value) {
 45 |   size_t v_next = value;
 46 |   while (true) {
 47 |     size_t v_next_sqrt = (size_t)sqrt(v_next);
 48 | 
 49 |     bool is_odd = true;
 50 |     for (size_t v_dv = 2; v_dv < v_next_sqrt + 1; v_dv++) {
 51 |       if (v_next % v_dv == 0) {
 52 |         is_odd = false;
 53 |         break;
 54 |       }
 55 |     }
 56 | 
 57 |     if (is_odd)
 58 |       return v_next;
 59 | 
 60 |     v_next++;
 61 |   }
 62 | 
 63 |   // never reach here
 64 |   return 0;
 65 | }
 66 | 
 67 | SpellingTable::SpellingTable() {
 68 |   need_score_ = false;
 69 |   raw_spellings_ = NULL;
 70 |   spelling_buf_ = NULL;
 71 |   spelling_num_ = 0;
 72 |   total_freq_ = 0;
 73 |   frozen_ = true;
 74 | }
 75 | 
 76 | SpellingTable::~SpellingTable() {
 77 |   free_resource();
 78 | }
 79 | 
 80 | size_t SpellingTable::get_hash_pos(const char* spelling_str) {
 81 |   size_t hash_pos = 0;
 82 |   for (size_t pos = 0; pos < spelling_size_; pos++) {
 83 |     if ('\0' == spelling_str[pos])
 84 |       break;
 85 |     hash_pos += (size_t)spelling_str[pos];
 86 |   }
 87 | 
 88 |   hash_pos = hash_pos % spelling_max_num_;
 89 |   return hash_pos;
 90 | }
 91 | 
 92 | size_t SpellingTable::hash_pos_next(size_t hash_pos) {
 93 |   hash_pos += 123;
 94 |   hash_pos = hash_pos % spelling_max_num_;
 95 |   return hash_pos;
 96 | }
 97 | 
 98 | void SpellingTable::free_resource() {
 99 |   if (NULL != raw_spellings_)
100 |     delete [] raw_spellings_;
101 |   raw_spellings_ = NULL;
102 | 
103 |   if (NULL != spelling_buf_)
104 |     delete [] spelling_buf_;
105 |   spelling_buf_ = NULL;
106 | }
107 | 
108 | bool SpellingTable::init_table(size_t pure_spl_size, size_t spl_max_num,
109 |                                bool need_score) {
110 |   if (pure_spl_size == 0 || spl_max_num ==0)
111 |     return false;
112 | 
113 |   need_score_ = need_score;
114 | 
115 |   free_resource();
116 | 
117 |   spelling_size_ = pure_spl_size + 1;
118 |   if (need_score)
119 |     spelling_size_ += 1;
120 |   spelling_max_num_ = get_odd_next(spl_max_num);
121 |   spelling_num_ = 0;
122 | 
123 |   raw_spellings_ = new RawSpelling[spelling_max_num_];
124 |   spelling_buf_ = new char[spelling_max_num_ * (spelling_size_)];
125 |   if (NULL == raw_spellings_ || NULL == spelling_buf_) {
126 |     free_resource();
127 |     return false;
128 |   }
129 | 
130 |   memset(raw_spellings_, 0, spelling_max_num_ * sizeof(RawSpelling));
131 |   memset(spelling_buf_, 0, spelling_max_num_ * (spelling_size_));
132 |   frozen_ = false;
133 |   total_freq_ = 0;
134 |   return true;
135 | }
136 | 
137 | bool SpellingTable::put_spelling(const char* spelling_str, double freq) {
138 |   if (frozen_ || NULL == spelling_str)
139 |     return false;
140 | 
141 |   for (size_t pos = 0; pos < kNotSupportNum; pos++) {
142 |     if (strcmp(spelling_str, kNotSupportList[pos]) == 0) {
143 |       return false;
144 |     }
145 |   }
146 | 
147 |   total_freq_ += freq;
148 | 
149 |   size_t hash_pos = get_hash_pos(spelling_str);
150 | 
151 |   raw_spellings_[hash_pos].str[spelling_size_ - 1] = '\0';
152 | 
153 |   if (strncmp(raw_spellings_[hash_pos].str, spelling_str,
154 |               spelling_size_ - 1) == 0) {
155 |     raw_spellings_[hash_pos].freq += freq;
156 |     return true;
157 |   }
158 | 
159 |   size_t hash_pos_ori = hash_pos;
160 | 
161 |   while (true) {
162 |     if (strncmp(raw_spellings_[hash_pos].str,
163 |                 spelling_str, spelling_size_ - 1) == 0) {
164 |       raw_spellings_[hash_pos].freq += freq;
165 |       return true;
166 |     }
167 | 
168 |     if ('\0' == raw_spellings_[hash_pos].str[0]) {
169 |       raw_spellings_[hash_pos].freq += freq;
170 |       strncpy(raw_spellings_[hash_pos].str, spelling_str, spelling_size_ - 1);
171 |       raw_spellings_[hash_pos].str[spelling_size_ - 1] = '\0';
172 |       spelling_num_++;
173 |       return true;
174 |     }
175 | 
176 |     hash_pos = hash_pos_next(hash_pos);
177 |     if (hash_pos_ori == hash_pos)
178 |       return false;
179 |   }
180 | 
181 |   // never reach here
182 |   return false;
183 | }
184 | 
185 | bool SpellingTable::contain(const char* spelling_str) {
186 |   if (NULL == spelling_str || NULL == spelling_buf_ || frozen_)
187 |     return false;
188 | 
189 |   size_t hash_pos = get_hash_pos(spelling_str);
190 | 
191 |   if ('\0' == raw_spellings_[hash_pos].str[0])
192 |     return false;
193 | 
194 |   if (strncmp(raw_spellings_[hash_pos].str, spelling_str, spelling_size_ - 1)
195 |       == 0)
196 |     return true;
197 | 
198 |   size_t hash_pos_ori = hash_pos;
199 | 
200 |   while (true) {
201 |     hash_pos = hash_pos_next(hash_pos);
202 |     if (hash_pos_ori == hash_pos)
203 |       return false;
204 | 
205 |     if ('\0' == raw_spellings_[hash_pos].str[0])
206 |       return false;
207 | 
208 |     if (strncmp(raw_spellings_[hash_pos].str, spelling_str, spelling_size_ - 1)
209 |         == 0)
210 |       return true;
211 |   }
212 | 
213 |   // never reach here
214 |   return false;
215 | }
216 | 
217 | const char* SpellingTable::arrange(size_t *item_size, size_t *spl_num) {
218 |   if (NULL == raw_spellings_ || NULL == spelling_buf_ ||
219 |       NULL == item_size || NULL == spl_num)
220 |     return NULL;
221 | 
222 |   qsort(raw_spellings_, spelling_max_num_, sizeof(RawSpelling),
223 |         compare_raw_spl_eb);
224 | 
225 |   // After sorting, only the first spelling_num_ items are valid.
226 |   // Copy them to the destination buffer.
227 |   for (size_t pos = 0; pos < spelling_num_; pos++) {
228 |     strncpy(spelling_buf_ + pos * spelling_size_, raw_spellings_[pos].str,
229 |             spelling_size_);
230 |   }
231 | 
232 |   if (need_score_) {
233 |     if (kPrintDebug0)
234 |       printf("------------Spelling Possiblities--------------\n");
235 | 
236 |     double max_score = 0;
237 |     double min_score = 0;
238 | 
239 |     // After sorting, only the first spelling_num_ items are valid.
240 |     for (size_t pos = 0; pos < spelling_num_; pos++) {
241 |       raw_spellings_[pos].freq /= total_freq_;
242 |       if (need_score_) {
243 |         if (0 == pos) {
244 |           max_score = raw_spellings_[0].freq;
245 |           min_score = max_score;
246 |         } else {
247 |           if (raw_spellings_[pos].freq > max_score)
248 |             max_score = raw_spellings_[pos].freq;
249 |           if (raw_spellings_[pos].freq < min_score)
250 |             min_score = raw_spellings_[pos].freq;
251 |         }
252 |       }
253 |     }
254 | 
255 |     if (kPrintDebug0)
256 |       printf("-----max psb: %f, min psb: %f\n", max_score, min_score);
257 | 
258 |     max_score = log(max_score);
259 |     min_score = log(min_score);
260 | 
261 |     if (kPrintDebug0)
262 |       printf("-----max log value: %f, min log value: %f\n",
263 |              max_score, min_score);
264 | 
265 |     // The absolute value of min_score is bigger than that of max_score because
266 |     // both of them are negative after log function.
267 |     score_amplifier_ = 1.0 * 255 / min_score;
268 | 
269 |     double average_score = 0;
270 |     for (size_t pos = 0; pos < spelling_num_; pos++) {
271 |       double score = log(raw_spellings_[pos].freq) * score_amplifier_;
272 |       assert(score >= 0);
273 | 
274 |       average_score += score;
275 | 
276 |       // Because of calculation precision issue, score might be a little bigger
277 |       // than 255 after being amplified.
278 |       if (score > 255)
279 |         score = 255;
280 |       char *this_spl_buf = spelling_buf_ + pos * spelling_size_;
281 |       this_spl_buf[spelling_size_ - 1] =
282 |           static_cast<char>((unsigned char)score);
283 | 
284 |       if (kPrintDebug0) {
285 |         printf("---pos:%d, %s, psb:%d\n", pos, this_spl_buf,
286 |                (unsigned char)this_spl_buf[spelling_size_ -1]);
287 |       }
288 |     }
289 |     average_score /= spelling_num_;
290 |     assert(average_score <= 255);
291 |     average_score_ = static_cast<uint8>(average_score);
292 | 
293 |     if (kPrintDebug0)
294 |       printf("\n----Score Amplifier: %f, Average Score: %d\n", score_amplifier_,
295 |              average_score_);
296 |   }
297 | 
298 |   *item_size = spelling_size_;
299 |   *spl_num = spelling_num_;
300 |   frozen_ = true;
301 |   return spelling_buf_;
302 | }
303 | 
304 | float SpellingTable::get_score_amplifier() {
305 |   return static_cast<float>(score_amplifier_);
306 | }
307 | 
308 | unsigned char SpellingTable::get_average_score() {
309 |   return average_score_;
310 | }
311 | 
312 | #endif  // ___BUILD_MODEL___
313 | }  // namespace ime_pinyin
314 | 


--------------------------------------------------------------------------------
/googlelib/dicttrie.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #ifndef PINYINIME_INCLUDE_DICTTRIE_H__
 18 | #define PINYINIME_INCLUDE_DICTTRIE_H__
 19 | 
 20 | #include <stdlib.h>
 21 | #include "./atomdictbase.h"
 22 | #include "./dictdef.h"
 23 | #include "./dictlist.h"
 24 | #include "./searchutility.h"
 25 | #include <QFile>
 26 | 
 27 | namespace ime_pinyin {
 28 | 
 29 | class DictTrie : AtomDictBase {
 30 |  private:
 31 |   struct ParsingMark {
 32 |     size_t node_offset:24;
 33 |     size_t node_num:8;           // Number of nodes with this spelling id given
 34 |                                  // by spl_id. If spl_id is a Shengmu, for nodes
 35 |                                  // in the first layer of DictTrie, it equals to
 36 |                                  // SpellingTrie::shm2full_num(); but for those
 37 |                                  // nodes which are not in the first layer,
 38 |                                  // node_num < SpellingTrie::shm2full_num().
 39 |                                  // For a full spelling id, node_num = 1;
 40 |   };
 41 | 
 42 |   // Used to indicate an extended mile stone.
 43 |   // An extended mile stone is used to mark a partial match in the dictionary
 44 |   // trie to speed up further potential extending.
 45 |   // For example, when the user inputs "w", a mile stone is created to mark the
 46 |   // partial match status, so that when user inputs another char 'm', it will be
 47 |   // faster to extend search space based on this mile stone.
 48 |   //
 49 |   // For partial match status of "wm", there can be more than one sub mile
 50 |   // stone, for example, "wm" can be matched to "wanm", "wom", ..., etc, so
 51 |   // there may be more one parsing mark used to mark these partial matchings.
 52 |   // A mile stone records the starting position in the mark list and number of
 53 |   // marks.
 54 |   struct MileStone {
 55 |     uint16 mark_start;
 56 |     uint16 mark_num;
 57 |   };
 58 | 
 59 |   DictList* dict_list_;
 60 | 
 61 |   const SpellingTrie *spl_trie_;
 62 | 
 63 |   LmaNodeLE0* root_;        // Nodes for root and the first layer.
 64 |   LmaNodeGE1* nodes_ge1_;   // Nodes for other layers.
 65 | 
 66 |   // An quick index from spelling id to the LmaNodeLE0 node buffer, or
 67 |   // to the root_ buffer.
 68 |   // Index length:
 69 |   // SpellingTrie::get_instance().get_spelling_num() + 1. The last one is used
 70 |   // to get the end.
 71 |   // All Shengmu ids are not indexed because they will be converted into
 72 |   // corresponding full ids.
 73 |   // So, given an id splid, the son is:
 74 |   // root_[splid_le0_index_[splid - kFullSplIdStart]]
 75 |   uint16 *splid_le0_index_;
 76 | 
 77 |   uint32 lma_node_num_le0_;
 78 |   uint32 lma_node_num_ge1_;
 79 | 
 80 |   // The first part is for homophnies, and the last  top_lma_num_ items are
 81 |   // lemmas with highest scores.
 82 |   unsigned char *lma_idx_buf_;
 83 |   uint32 lma_idx_buf_len_;  // The total size of lma_idx_buf_ in byte.
 84 |   uint32 total_lma_num_;    // Total number of lemmas in this dictionary.
 85 |   uint32 top_lmas_num_;     // Number of lemma with highest scores.
 86 | 
 87 |   // Parsing mark list used to mark the detailed extended statuses.
 88 |   ParsingMark *parsing_marks_;
 89 |   // The position for next available mark.
 90 |   uint16 parsing_marks_pos_;
 91 | 
 92 |   // Mile stone list used to mark the extended status.
 93 |   MileStone *mile_stones_;
 94 |   // The position for the next available mile stone. We use positions (except 0)
 95 |   // as handles.
 96 |   MileStoneHandle mile_stones_pos_;
 97 | 
 98 |   // Get the offset of sons for a node.
 99 |   inline size_t get_son_offset(const LmaNodeGE1 *node);
100 | 
101 |   // Get the offset of homonious ids for a node.
102 |   inline size_t get_homo_idx_buf_offset(const LmaNodeGE1 *node);
103 | 
104 |   // Get the lemma id by the offset.
105 |   inline LemmaIdType get_lemma_id(size_t id_offset);
106 | 
107 |   void free_resource(bool free_dict_list);
108 | 
109 |   bool load_dict(QFile *fp);
110 | 
111 |   // Given a LmaNodeLE0 node, extract the lemmas specified by it, and fill
112 |   // them into the lpi_items buffer.
113 |   // This function is called by the search engine.
114 |   size_t fill_lpi_buffer(LmaPsbItem lpi_items[], size_t max_size,
115 |                          LmaNodeLE0 *node);
116 | 
117 |   // Given a LmaNodeGE1 node, extract the lemmas specified by it, and fill
118 |   // them into the lpi_items buffer.
119 |   // This function is called by inner functions extend_dict0(), extend_dict1()
120 |   // and extend_dict2().
121 |   size_t fill_lpi_buffer(LmaPsbItem lpi_items[], size_t max_size,
122 |                          size_t homo_buf_off, LmaNodeGE1 *node,
123 |                          uint16 lma_len);
124 | 
125 |   // Extend in the trie from level 0.
126 |   MileStoneHandle extend_dict0(MileStoneHandle from_handle,
127 |                                const DictExtPara *dep, LmaPsbItem *lpi_items,
128 |                                size_t lpi_max, size_t *lpi_num);
129 | 
130 |   // Extend in the trie from level 1.
131 |   MileStoneHandle extend_dict1(MileStoneHandle from_handle,
132 |                                const DictExtPara *dep, LmaPsbItem *lpi_items,
133 |                                size_t lpi_max, size_t *lpi_num);
134 | 
135 |   // Extend in the trie from level 2.
136 |   MileStoneHandle extend_dict2(MileStoneHandle from_handle,
137 |                                const DictExtPara *dep, LmaPsbItem *lpi_items,
138 |                                size_t lpi_max, size_t *lpi_num);
139 | 
140 |   // Try to extend the given spelling id buffer, and if the given id_lemma can
141 |   // be successfully gotten, return true;
142 |   // The given spelling ids are all valid full ids.
143 |   bool try_extend(const uint16 *splids, uint16 splid_num, LemmaIdType id_lemma);
144 | 
145 | #ifdef ___BUILD_MODEL___
146 |   bool save_dict(FILE *fp);
147 | #endif  // ___BUILD_MODEL___
148 | 
149 |   static const int kMaxMileStone = 100;
150 |   static const int kMaxParsingMark = 600;
151 |   static const MileStoneHandle kFirstValidMileStoneHandle = 1;
152 | 
153 |   friend class DictParser;
154 |   friend class DictBuilder;
155 | 
156 |  public:
157 | 
158 |   DictTrie();
159 |   ~DictTrie();
160 | 
161 | #ifdef ___BUILD_MODEL___
162 |   // Construct the tree from the file fn_raw.
163 |   // fn_validhzs provide the valid hanzi list. If fn_validhzs is
164 |   // NULL, only chars in GB2312 will be included.
165 |   bool build_dict(const char *fn_raw, const char *fn_validhzs);
166 | 
167 |   // Save the binary dictionary
168 |   // Actually, the SpellingTrie/DictList instance will be also saved.
169 |   bool save_dict(const char *filename);
170 | #endif  // ___BUILD_MODEL___
171 | 
172 |   void convert_to_hanzis(char16 *str, uint16 str_len);
173 | 
174 |   void convert_to_scis_ids(char16 *str, uint16 str_len);
175 | 
176 |   // Load a binary dictionary
177 |   // The SpellingTrie instance/DictList will be also loaded
178 |   bool load_dict(const char *filename, LemmaIdType start_id,
179 |                  LemmaIdType end_id);
180 |   bool load_dict_fd(int sys_fd, long start_offset, long length,
181 |                     LemmaIdType start_id, LemmaIdType end_id);
182 |   bool close_dict() {return true;}
183 |   size_t number_of_lemmas() {return 0;}
184 | 
185 |   void reset_milestones(uint16 from_step, MileStoneHandle from_handle);
186 | 
187 |   MileStoneHandle extend_dict(MileStoneHandle from_handle,
188 |                               const DictExtPara *dep,
189 |                               LmaPsbItem *lpi_items,
190 |                               size_t lpi_max, size_t *lpi_num);
191 | 
192 |   size_t get_lpis(const uint16 *splid_str, uint16 splid_str_len,
193 |                   LmaPsbItem *lpi_items, size_t lpi_max);
194 | 
195 |   uint16 get_lemma_str(LemmaIdType id_lemma, char16 *str_buf, uint16 str_max);
196 | 
197 |   uint16 get_lemma_splids(LemmaIdType id_lemma, uint16 *splids,
198 |                           uint16 splids_max, bool arg_valid);
199 | 
200 |   size_t predict(const char16 *last_hzs, uint16 hzs_len,
201 |                  NPredictItem *npre_items, size_t npre_max,
202 |                  size_t b4_used);
203 | 
204 |   LemmaIdType put_lemma(char16 /*lemma_str*/[], uint16 /*splids*/[],
205 |                         uint16 /*lemma_len*/, uint16 /*count*/) {return 0;}
206 | 
207 |   LemmaIdType update_lemma(LemmaIdType /*lemma_id*/, int16 /*delta_count*/,
208 |                            bool /*selected*/) {return 0;}
209 | 
210 |   LemmaIdType get_lemma_id(char16 /*lemma_str*/[], uint16 /*splids*/[],
211 |                            uint16 /*lemma_len*/) {return 0;}
212 | 
213 |   LmaScoreType get_lemma_score(LemmaIdType /*lemma_id*/) {return 0;}
214 | 
215 |   LmaScoreType get_lemma_score(char16 /*lemma_str*/[], uint16 /*splids*/[],
216 |                         uint16 /*lemma_len*/) {return 0;}
217 | 
218 |   bool remove_lemma(LemmaIdType /*lemma_id*/) {return false;}
219 | 
220 |   size_t get_total_lemma_count() {return 0;}
221 |   void set_total_lemma_count_of_others(size_t count);
222 | 
223 |   void flush_cache() {}
224 | 
225 |   LemmaIdType get_lemma_id(const char16 lemma_str[], uint16 lemma_len);
226 | 
227 |   // Fill the lemmas with highest scores to the prediction buffer.
228 |   // his_len is the history length to fill in the prediction buffer.
229 |   size_t predict_top_lmas(size_t his_len, NPredictItem *npre_items,
230 |                           size_t npre_max, size_t b4_used);
231 | };
232 | }
233 | 
234 | #endif  // PINYINIME_INCLUDE_DICTTRIE_H__
235 | 


--------------------------------------------------------------------------------
/googlelib/spellingtrie.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #ifndef PINYINIME_INCLUDE_SPELLINGTRIE_H__
 18 | #define PINYINIME_INCLUDE_SPELLINGTRIE_H__
 19 | 
 20 | #include <stdio.h>
 21 | #include <stdlib.h>
 22 | #include "./dictdef.h"
 23 | #include <QFile>
 24 | 
 25 | namespace ime_pinyin {
 26 | 
 27 | static const unsigned short kFullSplIdStart = kHalfSpellingIdNum + 1;
 28 | 
 29 | // Node used for the trie of spellings
 30 | struct SpellingNode {
 31 |   SpellingNode *first_son;
 32 |   // The spelling id for each node. If you need more bits to store
 33 |   // spelling id, please adjust this structure.
 34 |   uint16 spelling_idx:11;
 35 |   uint16  num_of_son:5;
 36 |   char char_this_node;
 37 |   unsigned char score;
 38 | };
 39 | 
 40 | class SpellingTrie {
 41 |  private:
 42 |   static const int kMaxYmNum = 64;
 43 |   static const size_t kValidSplCharNum = 26;
 44 | 
 45 |   static const uint16 kHalfIdShengmuMask = 0x01;
 46 |   static const uint16 kHalfIdYunmuMask = 0x02;
 47 |   static const uint16 kHalfIdSzmMask = 0x04;
 48 | 
 49 |   // Map from half spelling id to single char.
 50 |   // For half ids of Zh/Ch/Sh, map to z/c/s (low case) respectively.
 51 |   // For example, 1 to 'A', 2 to 'B', 3 to 'C', 4 to 'c', 5 to 'D', ...,
 52 |   // 28 to 'Z', 29 to 'z'.
 53 |   // [0] is not used to achieve better efficiency.
 54 |   static const char kHalfId2Sc_[kFullSplIdStart + 1];
 55 | 
 56 |   static unsigned char char_flags_[];
 57 |   static SpellingTrie* instance_;
 58 | 
 59 |   // The spelling table
 60 |   char *spelling_buf_;
 61 | 
 62 |   // The size of longest spelling string, includes '\0' and an extra char to
 63 |   // store score. For example, "zhuang" is the longgest item in Pinyin list,
 64 |   // so spelling_size_ is 8.
 65 |   // Structure: The string ended with '\0' + score char.
 66 |   // An item with a lower score has a higher probability.
 67 |   uint32 spelling_size_;
 68 | 
 69 |   // Number of full spelling ids.
 70 |   uint32 spelling_num_;
 71 | 
 72 |   float score_amplifier_;
 73 |   unsigned char average_score_;
 74 | 
 75 |   // The Yunmu id list for the spelling ids (for half ids of Shengmu,
 76 |   // the Yunmu id is 0).
 77 |   // The length of the list is spelling_num_ + kFullSplIdStart,
 78 |   // so that spl_ym_ids_[splid] is the Yunmu id of the splid.
 79 |   uint8 *spl_ym_ids_;
 80 | 
 81 |   // The Yunmu table.
 82 |   // Each Yunmu will be assigned with Yunmu id from 1.
 83 |   char *ym_buf_;
 84 |   size_t ym_size_;  // The size of longest Yunmu string, '\0'included.
 85 |   size_t ym_num_;
 86 | 
 87 |   // The spelling string just queried
 88 |   char *splstr_queried_;
 89 | 
 90 |   // The spelling string just queried
 91 |   char16 *splstr16_queried_;
 92 | 
 93 |   // The root node of the spelling tree
 94 |   SpellingNode* root_;
 95 | 
 96 |   // If a none qwerty key such as a fnction key like ENTER is given, this node
 97 |   // will be used to indicate that this is not a QWERTY node.
 98 |   SpellingNode* dumb_node_;
 99 | 
100 |   // If a splitter key is pressed, this node will be used to indicate that this
101 |   // is a splitter key.
102 |   SpellingNode* splitter_node_;
103 | 
104 |   // Used to get the first level sons.
105 |   SpellingNode* level1_sons_[kValidSplCharNum];
106 | 
107 |   // The full spl_id range for specific half id.
108 |   // h2f means half to full.
109 |   // A half id can be a ShouZiMu id (id to represent the first char of a full
110 |   // spelling, including Shengmu and Yunmu), or id of zh/ch/sh.
111 |   // [1..kFullSplIdStart-1] is the arrange of half id.
112 |   uint16 h2f_start_[kFullSplIdStart];
113 |   uint16 h2f_num_[kFullSplIdStart];
114 | 
115 |   // Map from full id to half id.
116 |   uint16 *f2h_;
117 | 
118 | #ifdef ___BUILD_MODEL___
119 |   // How many node used to build the trie.
120 |   size_t node_num_;
121 | #endif
122 | 
123 |   SpellingTrie();
124 | 
125 |   void free_son_trie(SpellingNode* node);
126 | 
127 |   // Construct a subtree using a subset of the spelling array (from
128 |   // item_star to item_end).
129 |   // Member spelliing_buf_ and spelling_size_ should be valid.
130 |   // parent is used to update its num_of_son and score.
131 |   SpellingNode* construct_spellings_subset(size_t item_start, size_t item_end,
132 |                                            size_t level, SpellingNode *parent);
133 |   bool build_f2h();
134 | 
135 |   // The caller should guarantee ch >= 'A' && ch <= 'Z'
136 |   bool is_shengmu_char(char ch) const;
137 | 
138 |   // The caller should guarantee ch >= 'A' && ch <= 'Z'
139 |   bool is_yunmu_char(char ch) const;
140 | 
141 | #ifdef ___BUILD_MODEL___
142 |   // Given a spelling string, return its Yunmu string.
143 |   // The caller guaratees spl_str is valid.
144 |   const char* get_ym_str(const char *spl_str);
145 | 
146 |   // Build the Yunmu list, and the mapping relation between the full ids and the
147 |   // Yunmu ids. This functin is called after the spelling trie is built.
148 |   bool build_ym_info();
149 | #endif
150 | 
151 |   friend class SpellingParser;
152 |   friend class SmartSplParser;
153 |   friend class SmartSplParser2;
154 | 
155 |  public:
156 |   ~SpellingTrie();
157 | 
158 |   inline static bool is_valid_spl_char(char ch) {
159 |     return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z');
160 |   }
161 | 
162 |   // The caller guarantees that the two chars are valid spelling chars.
163 |   inline static bool is_same_spl_char(char ch1, char ch2) {
164 |     return ch1 == ch2 || ch1 - ch2 == 'a' - 'A' || ch2 - ch1 == 'a' - 'A';
165 |   }
166 | 
167 |   // Construct the tree from the input pinyin array
168 |   // The given string list should have been sorted.
169 |   // score_amplifier is used to convert a possibility value into score.
170 |   // average_score is the average_score of all spellings. The dumb node is
171 |   // assigned with this score.
172 |   bool construct(const char* spelling_arr, size_t item_size, size_t item_num,
173 |                  float score_amplifier, unsigned char average_score);
174 | 
175 |   // Test if the given id is a valid spelling id.
176 |   // If function returns true, the given splid may be updated like this:
177 |   // When 'A' is not enabled in ShouZiMu mode, the parsing result for 'A' is
178 |   // first given as a half id 1, but because 'A' is a one-char Yunmu and
179 |   // it is a valid id, it needs to updated to its corresponding full id.
180 |   bool if_valid_id_update(uint16 *splid) const;
181 | 
182 |   // Test if the given id is a half id.
183 |   bool is_half_id(uint16 splid) const;
184 | 
185 |   bool is_full_id(uint16 splid) const;
186 | 
187 |   // Test if the given id is a one-char Yunmu id (obviously, it is also a half
188 |   // id), such as 'A', 'E' and 'O'.
189 |   bool is_half_id_yunmu(uint16 splid) const;
190 | 
191 |   // Test if this char is a ShouZiMu char. This ShouZiMu char may be not enabled.
192 |   // For Pinyin, only i/u/v is not a ShouZiMu char.
193 |   // The caller should guarantee that ch >= 'A' && ch <= 'Z'
194 |   bool is_szm_char(char ch) const;
195 | 
196 |   // Test If this char is enabled in ShouZiMu mode.
197 |   // The caller should guarantee that ch >= 'A' && ch <= 'Z'
198 |   bool szm_is_enabled(char ch) const;
199 | 
200 |   // Enable/disable Shengmus in ShouZiMu mode(using the first char of a spelling
201 |   // to input).
202 |   void szm_enable_shm(bool enable);
203 | 
204 |   // Enable/disable Yunmus in ShouZiMu mode.
205 |   void szm_enable_ym(bool enable);
206 | 
207 |   // Test if this char is enabled in ShouZiMu mode.
208 |   // The caller should guarantee ch >= 'A' && ch <= 'Z'
209 |   bool is_szm_enabled(char ch) const;
210 | 
211 |   // Return the number of full ids for the given half id.
212 |   uint16 half2full_num(uint16 half_id) const;
213 | 
214 |   // Return the number of full ids for the given half id, and fill spl_id_start
215 |   // to return the first full id.
216 |   uint16 half_to_full(uint16 half_id, uint16 *spl_id_start) const;
217 | 
218 |   // Return the corresponding half id for the given full id.
219 |   // Not frequently used, low efficient.
220 |   // Return 0 if fails.
221 |   uint16 full_to_half(uint16 full_id) const;
222 | 
223 |   // To test whether a half id is compatible with a full id.
224 |   // Generally, when half_id == full_to_half(full_id), return true.
225 |   // But for "Zh, Ch, Sh", if fussy mode is on, half id for 'Z' is compatible
226 |   // with a full id like "Zhe". (Fussy mode is not ready).
227 |   bool half_full_compatible(uint16 half_id, uint16 full_id) const;
228 | 
229 |   static const SpellingTrie* get_cpinstance();
230 | 
231 |   static SpellingTrie& get_instance();
232 | 
233 |   // Save to the file stream
234 |   bool save_spl_trie(FILE *fp);
235 | 
236 |   // Load from the file stream
237 |   bool load_spl_trie(QFile *fp);
238 | 
239 |   // Get the number of spellings
240 |   size_t get_spelling_num();
241 | 
242 |   // Return the Yunmu id for the given Yunmu string.
243 |   // If the string is not valid, return 0;
244 |   uint8 get_ym_id(const char* ym_str);
245 | 
246 |   // Get the readonly Pinyin string for a given spelling id
247 |   const char* get_spelling_str(uint16 splid);
248 | 
249 |   // Get the readonly Pinyin string for a given spelling id
250 |   const char16* get_spelling_str16(uint16 splid);
251 | 
252 |   // Get Pinyin string for a given spelling id. Return the length of the
253 |   // string, and fill-in '\0' at the end.
254 |   size_t get_spelling_str16(uint16 splid, char16 *splstr16,
255 |                             size_t splstr16_len);
256 | };
257 | }
258 | 
259 | #endif  // PINYINIME_INCLUDE_SPELLINGTRIE_H__
260 | 


--------------------------------------------------------------------------------
/googlelib/ngram.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #include <assert.h>
 18 | #include <math.h>
 19 | #include <stdio.h>
 20 | #include <string.h>
 21 | #include <time.h>
 22 | #include "mystdlib.h"
 23 | #include "ngram.h"
 24 | 
 25 | namespace ime_pinyin {
 26 | 
 27 | #define ADD_COUNT 0.3
 28 | 
 29 | int comp_double(const void *p1, const void *p2) {
 30 |   if (*static_cast<const double*>(p1) < *static_cast<const double*>(p2))
 31 |     return -1;
 32 |   if (*static_cast<const double*>(p1) > *static_cast<const double*>(p2))
 33 |     return 1;
 34 |   return 0;
 35 | }
 36 | 
 37 | inline double distance(double freq, double code) {
 38 |   // return fabs(freq - code);
 39 |   return freq * fabs(log(freq) - log(code));
 40 | }
 41 | 
 42 | // Find the index of the code value which is nearest to the given freq
 43 | int qsearch_nearest(double code_book[], double freq, int start, int end) {
 44 |   if (start == end)
 45 |     return start;
 46 | 
 47 |   if (start + 1 == end) {
 48 |     if (distance(freq, code_book[end]) > distance(freq, code_book[start]))
 49 |       return start;
 50 |     return end;
 51 |   }
 52 | 
 53 |   int mid = (start + end) / 2;
 54 | 
 55 |   if (code_book[mid] > freq)
 56 |     return qsearch_nearest(code_book, freq, start, mid);
 57 |   else
 58 |     return qsearch_nearest(code_book, freq, mid, end);
 59 | }
 60 | 
 61 | size_t update_code_idx(double freqs[], size_t num, double code_book[],
 62 |                        CODEBOOK_TYPE *code_idx) {
 63 |   size_t changed = 0;
 64 |   for (size_t pos = 0; pos < num; pos++) {
 65 |     CODEBOOK_TYPE idx;
 66 |     idx = qsearch_nearest(code_book, freqs[pos], 0, kCodeBookSize - 1);
 67 |     if (idx != code_idx[pos])
 68 |       changed++;
 69 |     code_idx[pos] = idx;
 70 |   }
 71 |   return changed;
 72 | }
 73 | 
 74 | double recalculate_kernel(double freqs[], size_t num, double code_book[],
 75 |                           CODEBOOK_TYPE *code_idx) {
 76 |   double ret = 0;
 77 | 
 78 |   size_t *item_num =  new size_t[kCodeBookSize];
 79 |   assert(item_num);
 80 |   memset(item_num, 0, sizeof(size_t) * kCodeBookSize);
 81 | 
 82 |   double *cb_new = new double[kCodeBookSize];
 83 |   assert(cb_new);
 84 |   memset(cb_new, 0, sizeof(double) * kCodeBookSize);
 85 | 
 86 |   for (size_t pos = 0; pos < num; pos++) {
 87 |     ret += distance(freqs[pos], code_book[code_idx[pos]]);
 88 | 
 89 |     cb_new[code_idx[pos]] += freqs[pos];
 90 |     item_num[code_idx[pos]] += 1;
 91 |   }
 92 | 
 93 |   for (size_t code = 0; code < kCodeBookSize; code++) {
 94 |     assert(item_num[code] > 0);
 95 |     code_book[code] = cb_new[code] / item_num[code];
 96 |   }
 97 | 
 98 |   delete [] item_num;
 99 |   delete [] cb_new;
100 | 
101 |   return ret;
102 | }
103 | 
104 | void iterate_codes(double freqs[], size_t num, double code_book[],
105 |                    CODEBOOK_TYPE *code_idx) {
106 |   size_t iter_num = 0;
107 |   double delta_last = 0;
108 |   do {
109 |     size_t changed = update_code_idx(freqs, num, code_book, code_idx);
110 | 
111 |     double delta = recalculate_kernel(freqs, num, code_book, code_idx);
112 | 
113 |     if (kPrintDebug0) {
114 |       printf("---Unigram codebook iteration: %d : %d, %.9f\n",
115 |              iter_num, changed, delta);
116 |     }
117 |     iter_num++;
118 | 
119 |     if (iter_num > 1 &&
120 |         (delta == 0 || fabs(delta_last - delta)/fabs(delta) < 0.000000001))
121 |       break;
122 |     delta_last = delta;
123 |   } while (true);
124 | }
125 | 
126 | 
127 | NGram* NGram::instance_ = NULL;
128 | 
129 | NGram::NGram() {
130 |   initialized_ = false;
131 |   idx_num_ = 0;
132 |   lma_freq_idx_ = NULL;
133 |   sys_score_compensation_ = 0;
134 | 
135 | #ifdef ___BUILD_MODEL___
136 |   freq_codes_df_ = NULL;
137 | #endif
138 |   freq_codes_ = NULL;
139 | }
140 | 
141 | NGram::~NGram() {
142 |   if (NULL != lma_freq_idx_)
143 |     free(lma_freq_idx_);
144 | 
145 | #ifdef ___BUILD_MODEL___
146 |   if (NULL != freq_codes_df_)
147 |     free(freq_codes_df_);
148 | #endif
149 | 
150 |   if (NULL != freq_codes_)
151 |     free(freq_codes_);
152 | }
153 | 
154 | NGram& NGram::get_instance() {
155 |   if (NULL == instance_)
156 |     instance_ = new NGram();
157 |   return *instance_;
158 | }
159 | 
160 | bool NGram::save_ngram(FILE *fp) {
161 |   if (!initialized_ || NULL == fp)
162 |     return false;
163 | 
164 |   if (0 == idx_num_ || NULL == freq_codes_ ||  NULL == lma_freq_idx_)
165 |     return false;
166 | 
167 |   if (fwrite(&idx_num_, sizeof(uint32), 1, fp) != 1)
168 |     return false;
169 | 
170 |   if (fwrite(freq_codes_, sizeof(LmaScoreType), kCodeBookSize, fp) !=
171 |       kCodeBookSize)
172 |     return false;
173 | 
174 |   if (fwrite(lma_freq_idx_, sizeof(CODEBOOK_TYPE), idx_num_, fp) != idx_num_)
175 |     return false;
176 | 
177 |   return true;
178 | }
179 | 
180 | bool NGram::load_ngram(QFile *fp) {
181 |   if (NULL == fp)
182 |     return false;
183 | 
184 |   initialized_ = false;
185 | 
186 |   if (fp->read((char *)&idx_num_, sizeof(uint32)) != sizeof(uint32) )
187 |     return false;
188 | 
189 |   if (NULL != lma_freq_idx_)
190 |     free(lma_freq_idx_);
191 | 
192 |   if (NULL != freq_codes_)
193 |     free(freq_codes_);
194 | 
195 |   lma_freq_idx_ = static_cast<CODEBOOK_TYPE*>
196 |                   (malloc(idx_num_ * sizeof(CODEBOOK_TYPE)));
197 |   freq_codes_ = static_cast<LmaScoreType*>
198 |       (malloc(kCodeBookSize * sizeof(LmaScoreType)));
199 | 
200 |   if (NULL == lma_freq_idx_ || NULL == freq_codes_)
201 |     return false;
202 | 
203 |   if (fp->read((char *)freq_codes_, sizeof(LmaScoreType) * kCodeBookSize) !=
204 |       sizeof(LmaScoreType) * kCodeBookSize)
205 |     return false;
206 | 
207 |   if (fp->read((char *)lma_freq_idx_, sizeof(CODEBOOK_TYPE) * idx_num_) != sizeof(CODEBOOK_TYPE) * idx_num_)
208 |     return false;
209 | 
210 |   initialized_ = true;
211 | 
212 |   total_freq_none_sys_ = 0;
213 |   return true;
214 | }
215 | 
216 | void NGram::set_total_freq_none_sys(size_t freq_none_sys) {
217 |   total_freq_none_sys_ = freq_none_sys;
218 |   if (0 == total_freq_none_sys_) {
219 |     sys_score_compensation_ = 0;
220 |   } else {
221 |     double factor = static_cast<double>(kSysDictTotalFreq) / (
222 |         kSysDictTotalFreq + total_freq_none_sys_);
223 |     sys_score_compensation_ = static_cast<float>(
224 |         log(factor) * kLogValueAmplifier);
225 |   }
226 | }
227 | 
228 | // The caller makes sure this oject is initialized.
229 | float NGram::get_uni_psb(LemmaIdType lma_id) {
230 |   return  static_cast<float>(freq_codes_[lma_freq_idx_[lma_id]]) +
231 |       sys_score_compensation_;
232 | }
233 | 
234 | float NGram::convert_psb_to_score(double psb) {
235 |   float score = static_cast<float>(
236 |       log(psb) * static_cast<double>(kLogValueAmplifier));
237 |   if (score > static_cast<float>(kMaxScore)) {
238 |     score = static_cast<float>(kMaxScore);
239 |   }
240 |   return score;
241 | }
242 | 
243 | #ifdef ___BUILD_MODEL___
244 | bool NGram::build_unigram(LemmaEntry *lemma_arr, size_t lemma_num,
245 |                           LemmaIdType next_idx_unused) {
246 |   if (NULL == lemma_arr || 0 == lemma_num || next_idx_unused <= 1)
247 |     return false;
248 | 
249 |   double total_freq = 0;
250 |   double *freqs = new double[next_idx_unused];
251 |   if (NULL == freqs)
252 |     return false;
253 | 
254 |   freqs[0] = ADD_COUNT;
255 |   total_freq += freqs[0];
256 |   LemmaIdType idx_now = 0;
257 |   for (size_t pos = 0; pos < lemma_num; pos++) {
258 |     if (lemma_arr[pos].idx_by_hz == idx_now)
259 |       continue;
260 |     idx_now++;
261 | 
262 |     assert(lemma_arr[pos].idx_by_hz == idx_now);
263 | 
264 |     freqs[idx_now] = lemma_arr[pos].freq;
265 |     if (freqs[idx_now] <= 0)
266 |       freqs[idx_now] = 0.3;
267 | 
268 |     total_freq += freqs[idx_now];
269 |   }
270 | 
271 |   double max_freq = 0;
272 |   idx_num_ = idx_now + 1;
273 |   assert(idx_now + 1 == next_idx_unused);
274 | 
275 |   for (size_t pos = 0; pos < idx_num_; pos++) {
276 |     freqs[pos] = freqs[pos] / total_freq;
277 |     assert(freqs[pos] > 0);
278 |     if (freqs[pos] > max_freq)
279 |       max_freq = freqs[pos];
280 |   }
281 | 
282 |   // calculate the code book
283 |   if (NULL == freq_codes_df_)
284 |     freq_codes_df_ = new double[kCodeBookSize];
285 |   assert(freq_codes_df_);
286 |   memset(freq_codes_df_, 0, sizeof(double) * kCodeBookSize);
287 | 
288 |   if (NULL == freq_codes_)
289 |     freq_codes_ = new LmaScoreType[kCodeBookSize];
290 |   assert(freq_codes_);
291 |   memset(freq_codes_, 0, sizeof(LmaScoreType) * kCodeBookSize);
292 | 
293 |   size_t freq_pos = 0;
294 |   for (size_t code_pos = 0; code_pos < kCodeBookSize; code_pos++) {
295 |     bool found = true;
296 | 
297 |     while (found) {
298 |       found = false;
299 |       double cand = freqs[freq_pos];
300 |       for (size_t i = 0; i < code_pos; i++)
301 |         if (freq_codes_df_[i] == cand) {
302 |           found = true;
303 |           break;
304 |         }
305 |       if (found)
306 |         freq_pos++;
307 |     }
308 | 
309 |     freq_codes_df_[code_pos] = freqs[freq_pos];
310 |     freq_pos++;
311 |   }
312 | 
313 |   myqsort(freq_codes_df_, kCodeBookSize, sizeof(double), comp_double);
314 | 
315 |   if (NULL == lma_freq_idx_)
316 |     lma_freq_idx_ = new CODEBOOK_TYPE[idx_num_];
317 |   assert(lma_freq_idx_);
318 | 
319 |   iterate_codes(freqs, idx_num_, freq_codes_df_, lma_freq_idx_);
320 | 
321 |   delete [] freqs;
322 | 
323 |   if (kPrintDebug0) {
324 |     printf("\n------Language Model Unigram Codebook------\n");
325 |   }
326 | 
327 |   for (size_t code_pos = 0; code_pos < kCodeBookSize; code_pos++) {
328 |     double log_score = log(freq_codes_df_[code_pos]);
329 |     float final_score = convert_psb_to_score(freq_codes_df_[code_pos]);
330 |     if (kPrintDebug0) {
331 |       printf("code:%d, probability:%.9f, log score:%.3f, final score: %.3f\n",
332 |              code_pos, freq_codes_df_[code_pos], log_score, final_score);
333 |     }
334 |     freq_codes_[code_pos] = static_cast<LmaScoreType>(final_score);
335 |   }
336 | 
337 |   initialized_ = true;
338 |   return true;
339 | }
340 | #endif
341 | 
342 | }  // namespace ime_pinyin
343 | 


--------------------------------------------------------------------------------
/googlelib/splparser.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #include <assert.h>
 18 | #include "splparser.h"
 19 | 
 20 | namespace ime_pinyin {
 21 | 
 22 | SpellingParser::SpellingParser() {
 23 |   spl_trie_ = SpellingTrie::get_cpinstance();
 24 | }
 25 | 
 26 | bool SpellingParser::is_valid_to_parse(char ch) {
 27 |   return SpellingTrie::is_valid_spl_char(ch);
 28 | }
 29 | 
 30 | uint16 SpellingParser::splstr_to_idxs(const char *splstr, uint16 str_len,
 31 |                                       uint16 spl_idx[], uint16 start_pos[],
 32 |                                       uint16 max_size, bool &last_is_pre) {
 33 |   if (NULL == splstr || 0 == max_size || 0 == str_len)
 34 |     return 0;
 35 | 
 36 |   if (!SpellingTrie::is_valid_spl_char(splstr[0]))
 37 |     return 0;
 38 | 
 39 |   last_is_pre = false;
 40 | 
 41 |   const SpellingNode *node_this = spl_trie_->root_;
 42 | 
 43 |   uint16 str_pos = 0;
 44 |   uint16 idx_num = 0;
 45 |   if (NULL != start_pos)
 46 |     start_pos[0] = 0;
 47 |   bool last_is_splitter = false;
 48 | 
 49 |   while (str_pos < str_len) {
 50 |     char char_this = splstr[str_pos];
 51 |     // all characters outside of [a, z] are considered as splitters
 52 |     if (!SpellingTrie::is_valid_spl_char(char_this)) {
 53 |       // test if the current node is endable
 54 |       uint16 id_this = node_this->spelling_idx;
 55 |       if (spl_trie_->if_valid_id_update(&id_this)) {
 56 |         spl_idx[idx_num] = id_this;
 57 | 
 58 |         idx_num++;
 59 |         str_pos++;
 60 |         if (NULL != start_pos)
 61 |           start_pos[idx_num] = str_pos;
 62 |         if (idx_num >= max_size)
 63 |           return idx_num;
 64 | 
 65 |         node_this = spl_trie_->root_;
 66 |         last_is_splitter = true;
 67 |         continue;
 68 |       } else {
 69 |         if (last_is_splitter) {
 70 |           str_pos++;
 71 |           if (NULL != start_pos)
 72 |             start_pos[idx_num] = str_pos;
 73 |           continue;
 74 |         } else {
 75 |           return idx_num;
 76 |         }
 77 |       }
 78 |     }
 79 | 
 80 |     last_is_splitter = false;
 81 | 
 82 |     SpellingNode *found_son = NULL;
 83 | 
 84 |     if (0 == str_pos) {
 85 |       if (char_this >= 'a')
 86 |         found_son = spl_trie_->level1_sons_[char_this - 'a'];
 87 |       else
 88 |         found_son = spl_trie_->level1_sons_[char_this - 'A'];
 89 |     } else {
 90 |       SpellingNode *first_son = node_this->first_son;
 91 |       // Because for Zh/Ch/Sh nodes, they are the last in the buffer and
 92 |       // frequently used, so we scan from the end.
 93 |       for (int i = 0; i < node_this->num_of_son; i++) {
 94 |         SpellingNode *this_son = first_son + i;
 95 |         if (SpellingTrie::is_same_spl_char(
 96 |             this_son->char_this_node, char_this)) {
 97 |           found_son = this_son;
 98 |           break;
 99 |         }
100 |       }
101 |     }
102 | 
103 |     // found, just move the current node pointer to the the son
104 |     if (NULL != found_son) {
105 |       node_this = found_son;
106 |     } else {
107 |       // not found, test if it is endable
108 |       uint16 id_this = node_this->spelling_idx;
109 |       if (spl_trie_->if_valid_id_update(&id_this)) {
110 |         // endable, remember the index
111 |         spl_idx[idx_num] = id_this;
112 | 
113 |         idx_num++;
114 |         if (NULL != start_pos)
115 |           start_pos[idx_num] = str_pos;
116 |         if (idx_num >= max_size)
117 |           return idx_num;
118 |         node_this = spl_trie_->root_;
119 |         continue;
120 |       } else {
121 |         return idx_num;
122 |       }
123 |     }
124 | 
125 |     str_pos++;
126 |   }
127 | 
128 |   uint16 id_this = node_this->spelling_idx;
129 |   if (spl_trie_->if_valid_id_update(&id_this)) {
130 |     // endable, remember the index
131 |     spl_idx[idx_num] = id_this;
132 | 
133 |     idx_num++;
134 |     if (NULL != start_pos)
135 |       start_pos[idx_num] = str_pos;
136 |   }
137 | 
138 |   last_is_pre = !last_is_splitter;
139 | 
140 |   return idx_num;
141 | }
142 | 
143 | uint16 SpellingParser::splstr_to_idxs_f(const char *splstr, uint16 str_len,
144 |                                         uint16 spl_idx[], uint16 start_pos[],
145 |                                         uint16 max_size, bool &last_is_pre) {
146 |   uint16 idx_num = splstr_to_idxs(splstr, str_len, spl_idx, start_pos,
147 |                                   max_size, last_is_pre);
148 |   for (uint16 pos = 0; pos < idx_num; pos++) {
149 |     if (spl_trie_->is_half_id_yunmu(spl_idx[pos])) {
150 |       spl_trie_->half_to_full(spl_idx[pos], spl_idx + pos);
151 |       if (pos == idx_num - 1) {
152 |         last_is_pre = false;
153 |       }
154 |     }
155 |   }
156 |   return idx_num;
157 | }
158 | 
159 | uint16 SpellingParser::splstr16_to_idxs(const char16 *splstr, uint16 str_len,
160 |                                         uint16 spl_idx[], uint16 start_pos[],
161 |                                         uint16 max_size, bool &last_is_pre) {
162 |   if (NULL == splstr || 0 == max_size || 0 == str_len)
163 |     return 0;
164 | 
165 |   if (!SpellingTrie::is_valid_spl_char(splstr[0]))
166 |     return 0;
167 | 
168 |   last_is_pre = false;
169 | 
170 |   const SpellingNode *node_this = spl_trie_->root_;
171 | 
172 |   uint16 str_pos = 0;
173 |   uint16 idx_num = 0;
174 |   if (NULL != start_pos)
175 |     start_pos[0] = 0;
176 |   bool last_is_splitter = false;
177 | 
178 |   while (str_pos < str_len) {
179 |     char16 char_this = splstr[str_pos];
180 |     // all characters outside of [a, z] are considered as splitters
181 |     if (!SpellingTrie::is_valid_spl_char(char_this)) {
182 |       // test if the current node is endable
183 |       uint16 id_this = node_this->spelling_idx;
184 |       if (spl_trie_->if_valid_id_update(&id_this)) {
185 |         spl_idx[idx_num] = id_this;
186 | 
187 |         idx_num++;
188 |         str_pos++;
189 |         if (NULL != start_pos)
190 |           start_pos[idx_num] = str_pos;
191 |         if (idx_num >= max_size)
192 |           return idx_num;
193 | 
194 |         node_this = spl_trie_->root_;
195 |         last_is_splitter = true;
196 |         continue;
197 |       } else {
198 |         if (last_is_splitter) {
199 |           str_pos++;
200 |           if (NULL != start_pos)
201 |             start_pos[idx_num] = str_pos;
202 |           continue;
203 |         } else {
204 |           return idx_num;
205 |         }
206 |       }
207 |     }
208 | 
209 |     last_is_splitter = false;
210 | 
211 |     SpellingNode *found_son = NULL;
212 | 
213 |     if (0 == str_pos) {
214 |       if (char_this >= 'a')
215 |         found_son = spl_trie_->level1_sons_[char_this - 'a'];
216 |       else
217 |         found_son = spl_trie_->level1_sons_[char_this - 'A'];
218 |     } else {
219 |       SpellingNode *first_son = node_this->first_son;
220 |       // Because for Zh/Ch/Sh nodes, they are the last in the buffer and
221 |       // frequently used, so we scan from the end.
222 |       for (int i = 0; i < node_this->num_of_son; i++) {
223 |         SpellingNode *this_son = first_son + i;
224 |         if (SpellingTrie::is_same_spl_char(
225 |             this_son->char_this_node, char_this)) {
226 |           found_son = this_son;
227 |           break;
228 |         }
229 |       }
230 |     }
231 | 
232 |     // found, just move the current node pointer to the the son
233 |     if (NULL != found_son) {
234 |       node_this = found_son;
235 |     } else {
236 |       // not found, test if it is endable
237 |       uint16 id_this = node_this->spelling_idx;
238 |       if (spl_trie_->if_valid_id_update(&id_this)) {
239 |         // endable, remember the index
240 |         spl_idx[idx_num] = id_this;
241 | 
242 |         idx_num++;
243 |         if (NULL != start_pos)
244 |           start_pos[idx_num] = str_pos;
245 |         if (idx_num >= max_size)
246 |           return idx_num;
247 |         node_this = spl_trie_->root_;
248 |         continue;
249 |       } else {
250 |         return idx_num;
251 |       }
252 |     }
253 | 
254 |     str_pos++;
255 |   }
256 | 
257 |   uint16 id_this = node_this->spelling_idx;
258 |   if (spl_trie_->if_valid_id_update(&id_this)) {
259 |     // endable, remember the index
260 |     spl_idx[idx_num] = id_this;
261 | 
262 |     idx_num++;
263 |     if (NULL != start_pos)
264 |       start_pos[idx_num] = str_pos;
265 |   }
266 | 
267 |   last_is_pre = !last_is_splitter;
268 | 
269 |   return idx_num;
270 | }
271 | 
272 | uint16 SpellingParser::splstr16_to_idxs_f(const char16 *splstr, uint16 str_len,
273 |                                           uint16 spl_idx[], uint16 start_pos[],
274 |                                           uint16 max_size, bool &last_is_pre) {
275 |   uint16 idx_num = splstr16_to_idxs(splstr, str_len, spl_idx, start_pos,
276 |                                     max_size, last_is_pre);
277 |   for (uint16 pos = 0; pos < idx_num; pos++) {
278 |     if (spl_trie_->is_half_id_yunmu(spl_idx[pos])) {
279 |       spl_trie_->half_to_full(spl_idx[pos], spl_idx + pos);
280 |       if (pos == idx_num - 1) {
281 |         last_is_pre = false;
282 |       }
283 |     }
284 |   }
285 |   return idx_num;
286 | }
287 | 
288 | uint16 SpellingParser::get_splid_by_str(const char *splstr, uint16 str_len,
289 |                                         bool *is_pre) {
290 |   if (NULL == is_pre)
291 |     return 0;
292 | 
293 |   uint16 spl_idx[2];
294 |   uint16 start_pos[3];
295 | 
296 |   if (splstr_to_idxs(splstr, str_len, spl_idx, start_pos, 2, *is_pre) != 1)
297 |     return 0;
298 | 
299 |   if (start_pos[1] != str_len)
300 |     return 0;
301 |   return spl_idx[0];
302 | }
303 | 
304 | uint16 SpellingParser::get_splid_by_str_f(const char *splstr, uint16 str_len,
305 |                                           bool *is_pre) {
306 |   if (NULL == is_pre)
307 |     return 0;
308 | 
309 |   uint16 spl_idx[2];
310 |   uint16 start_pos[3];
311 | 
312 |   if (splstr_to_idxs(splstr, str_len, spl_idx, start_pos, 2, *is_pre) != 1)
313 |     return 0;
314 | 
315 |   if (start_pos[1] != str_len)
316 |     return 0;
317 |   if (spl_trie_->is_half_id_yunmu(spl_idx[0])) {
318 |     spl_trie_->half_to_full(spl_idx[0], spl_idx);
319 |     *is_pre = false;
320 |   }
321 | 
322 |   return spl_idx[0];
323 | }
324 | 
325 | uint16 SpellingParser::get_splids_parallel(const char *splstr, uint16 str_len,
326 |     uint16 splidx[], uint16 max_size,
327 |     uint16 &full_id_num, bool &is_pre) {
328 |   if (max_size <= 0 || !is_valid_to_parse(splstr[0]))
329 |     return 0;
330 | 
331 |   splidx[0] = get_splid_by_str(splstr, str_len, &is_pre);
332 |   full_id_num = 0;
333 |   if (0 != splidx[0]) {
334 |     if (splidx[0] >= kFullSplIdStart)
335 |       full_id_num = 1;
336 |     return 1;
337 |   }
338 |   return 0;
339 | }
340 | 
341 | }  // namespace ime_pinyin
342 | 


--------------------------------------------------------------------------------
/googlelib/atomdictbase.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | /**
 18 |  * This class defines AtomDictBase class which is the base class for all atom
 19 |  * dictionaries. Atom dictionaries are managed by the decoder class
 20 |  * MatrixSearch.
 21 |  *
 22 |  * When the user appends a new character to the Pinyin string, all enabled atom
 23 |  * dictionaries' extend_dict() will be called at least once to get candidates
 24 |  * ended in this step (the information of starting step is also given in the
 25 |  * parameter). Usually, when extend_dict() is called, a MileStoneHandle object
 26 |  * returned by a previous calling for a earlier step is given to speed up the
 27 |  * look-up process, and a new MileStoneHandle object will be returned if
 28 |  * the extension is successful.
 29 |  *
 30 |  * A returned MileStoneHandle object should keep alive until Function
 31 |  * reset_milestones() is called and this object is noticed to be reset.
 32 |  *
 33 |  * Usually, the atom dictionary can use step information to manage its
 34 |  * MileStoneHandle objects, or it can make the objects in ascendant order to
 35 |  * make the reset easier.
 36 |  *
 37 |  * When the decoder loads the dictionary, it will give a starting lemma id for
 38 |  * this atom dictionary to map a inner id to a global id. Global ids should be
 39 |  * used when an atom dictionary talks to any component outside.
 40 |  */
 41 | #ifndef PINYINIME_INCLUDE_ATOMDICTBASE_H__
 42 | #define PINYINIME_INCLUDE_ATOMDICTBASE_H__
 43 | 
 44 | #include <stdlib.h>
 45 | #include "./dictdef.h"
 46 | #include "./searchutility.h"
 47 | 
 48 | namespace ime_pinyin {
 49 | class AtomDictBase {
 50 |  public:
 51 |   virtual ~AtomDictBase() {}
 52 | 
 53 |   /**
 54 |    * Load an atom dictionary from a file.
 55 |    *
 56 |    * @param file_name The file name to load dictionary.
 57 |    * @param start_id The starting id used for this atom dictionary.
 58 |    * @param end_id The end id (included) which can be used for this atom
 59 |    * dictionary. User dictionary will always use the last id space, so it can
 60 |    * ignore this paramter. All other atom dictionaries should check this
 61 |    * parameter.
 62 |    * @return True if succeed.
 63 |    */
 64 |   virtual bool load_dict(const char *file_name, LemmaIdType start_id,
 65 |                          LemmaIdType end_id) = 0;
 66 | 
 67 |   /**
 68 |    * Close this atom dictionary.
 69 |    *
 70 |    * @return True if succeed.
 71 |    */
 72 |   virtual bool close_dict() = 0;
 73 | 
 74 |   /**
 75 |    * Get the total number of lemmas in this atom dictionary.
 76 |    *
 77 |    * @return The total number of lemmas.
 78 |    */
 79 |   virtual size_t number_of_lemmas() = 0;
 80 | 
 81 |   /**
 82 |    * This function is called by the decoder when user deletes a character from
 83 |    * the input string, or begins a new input string.
 84 |    *
 85 |    * Different atom dictionaries may implement this function in different way.
 86 |    * an atom dictionary can use one of these two parameters (or both) to reset
 87 |    * its corresponding MileStoneHandle objects according its detailed
 88 |    * implementation.
 89 |    *
 90 |    * For example, if an atom dictionary uses step information to manage its
 91 |    * MileStoneHandle objects, parameter from_step can be used to identify which
 92 |    * objects should be reset; otherwise, if another atom dictionary does not
 93 |    * use the detailed step information, it only uses ascendant handles
 94 |    * (according to step. For the same step, earlier call, smaller handle), it
 95 |    * can easily reset those MileStoneHandle which are larger than from_handle.
 96 |    *
 97 |    * The decoder always reset the decoding state by step. So when it begins
 98 |    * resetting, it will call reset_milestones() of its atom dictionaries with
 99 |    * the step information, and the MileStoneHandle objects returned by the
100 |    * earliest calling of extend_dict() for that step.
101 |    *
102 |    * If an atom dictionary does not implement incremental search, this function
103 |    * can be totally ignored.
104 |    *
105 |    * @param from_step From which step(included) the MileStoneHandle
106 |    * objects should be reset.
107 |    * @param from_handle The ealiest MileStoneHandle object for step from_step
108 |    */
109 |   virtual void reset_milestones(uint16 from_step,
110 |                                 MileStoneHandle from_handle) = 0;
111 | 
112 |   /**
113 |    * Used to extend in this dictionary. The handle returned should keep valid
114 |    * until reset_milestones() is called.
115 |    *
116 |    * @param from_handle Its previous returned extended handle without the new
117 |    * spelling id, it can be used to speed up the extending.
118 |    * @param dep The paramter used for extending.
119 |    * @param lpi_items Used to fill in the lemmas matched.
120 |    * @param lpi_max The length of the buffer
121 |    * @param lpi_num Used to return the newly added items.
122 |    * @return The new mile stone for this extending. 0 if fail.
123 |    */
124 |   virtual MileStoneHandle extend_dict(MileStoneHandle from_handle,
125 |                                       const DictExtPara *dep,
126 |                                       LmaPsbItem *lpi_items,
127 |                                       size_t lpi_max, size_t *lpi_num) = 0;
128 | 
129 |   /**
130 |    * Get lemma items with scores according to a spelling id stream.
131 |    * This atom dictionary does not need to sort the returned items.
132 |    *
133 |    * @param splid_str The spelling id stream buffer.
134 |    * @param splid_str_len The length of the spelling id stream buffer.
135 |    * @param lpi_items Used to return matched lemma items with scores.
136 |    * @param lpi_max The maximum size of the buffer to return result.
137 |    * @return The number of matched items which have been filled in to lpi_items.
138 |    */
139 |   virtual size_t get_lpis(const uint16 *splid_str, uint16 splid_str_len,
140 |                           LmaPsbItem *lpi_items, size_t lpi_max) = 0;
141 | 
142 |   /**
143 |    * Get a lemma string (The Chinese string) by the given lemma id.
144 |    *
145 |    * @param id_lemma The lemma id to get the string.
146 |    * @param str_buf The buffer to return the Chinese string.
147 |    * @param str_max The maximum size of the buffer.
148 |    * @return The length of the string, 0 if fail.
149 |    */
150 |   virtual uint16 get_lemma_str(LemmaIdType id_lemma, char16 *str_buf,
151 |                                uint16 str_max) = 0;
152 | 
153 |   /**
154 |    * Get the full spelling ids for the given lemma id.
155 |    * If the given buffer is too short, return 0.
156 |    *
157 |    * @param splids Used to return the spelling ids.
158 |    * @param splids_max The maximum buffer length of splids.
159 |    * @param arg_valid Used to indicate if the incoming parameters have been
160 |    * initialized are valid. If it is true, the splids and splids_max are valid
161 |    * and there may be half ids in splids to be updated to full ids. In this
162 |    * case, splids_max is the number of valid ids in splids.
163 |    * @return The number of ids in the buffer.
164 |    */
165 |   virtual uint16 get_lemma_splids(LemmaIdType id_lemma, uint16 *splids,
166 |                                   uint16 splids_max, bool arg_valid) = 0;
167 | 
168 |   /**
169 |    * Function used for prediction.
170 |    * No need to sort the newly added items.
171 |    *
172 |    * @param last_hzs The last n Chinese chracters(called Hanzi), its length
173 |    * should be less than or equal to kMaxPredictSize.
174 |    * @param hzs_len specifies the length(<= kMaxPredictSize) of the history.
175 |    * @param npre_items Used used to return the result.
176 |    * @param npre_max The length of the buffer to return result
177 |    * @param b4_used Number of prediction result (from npre_items[-b4_used])
178 |    * from other atom dictionaries. A atom ditionary can just ignore it.
179 |    * @return The number of prediction result from this atom dictionary.
180 |    */
181 |   virtual size_t predict(const char16 last_hzs[], uint16 hzs_len,
182 |                          NPredictItem *npre_items, size_t npre_max,
183 |                          size_t b4_used) = 0;
184 | 
185 |   /**
186 |    * Add a lemma to the dictionary. If the dictionary allows to add new
187 |    * items and this item does not exist, add it.
188 |    *
189 |    * @param lemma_str The Chinese string of the lemma.
190 |    * @param splids The spelling ids of the lemma.
191 |    * @param lemma_len The length of the Chinese lemma.
192 |    * @param count The frequency count for this lemma.
193 |    */
194 |   virtual LemmaIdType put_lemma(char16 lemma_str[], uint16 splids[],
195 |                                 uint16 lemma_len, uint16 count) = 0;
196 | 
197 |   /**
198 |    * Update a lemma's occuring count.
199 |    *
200 |    * @param lemma_id The lemma id to update.
201 |    * @param delta_count The frequnecy count to ajust.
202 |    * @param selected Indicate whether this lemma is selected by user and
203 |    * submitted to target edit box.
204 |    * @return The id if succeed, 0 if fail.
205 |    */
206 |   virtual LemmaIdType update_lemma(LemmaIdType lemma_id, int16 delta_count,
207 |                                    bool selected) = 0;
208 | 
209 |   /**
210 |    * Get the lemma id for the given lemma.
211 |    *
212 |    * @param lemma_str The Chinese string of the lemma.
213 |    * @param splids The spelling ids of the lemma.
214 |    * @param lemma_len The length of the lemma.
215 |    * @return The matched lemma id, or 0 if fail.
216 |    */
217 |   virtual LemmaIdType get_lemma_id(char16 lemma_str[], uint16 splids[],
218 |                                    uint16 lemma_len) = 0;
219 | 
220 |   /**
221 |    * Get the lemma score.
222 |    *
223 |    * @param lemma_id The lemma id to get score.
224 |    * @return The score of the lemma, or 0 if fail.
225 |    */
226 |   virtual LmaScoreType get_lemma_score(LemmaIdType lemma_id) = 0;
227 | 
228 |   /**
229 |    * Get the lemma score.
230 |    *
231 |    * @param lemma_str The Chinese string of the lemma.
232 |    * @param splids The spelling ids of the lemma.
233 |    * @param lemma_len The length of the lemma.
234 |    * @return The score of the lamm, or 0 if fail.
235 |    */
236 |   virtual LmaScoreType get_lemma_score(char16 lemma_str[], uint16 splids[],
237 |                                 uint16 lemma_len) = 0;
238 | 
239 |   /**
240 |    * If the dictionary allowed, remove a lemma from it.
241 |    *
242 |    * @param lemma_id The id of the lemma to remove.
243 |    * @return True if succeed.
244 |    */
245 |   virtual bool remove_lemma(LemmaIdType lemma_id) = 0;
246 | 
247 |   /**
248 |    * Get the total occuring count of this atom dictionary.
249 |    *
250 |    * @return The total occuring count of this atom dictionary.
251 |    */
252 |   virtual size_t get_total_lemma_count() = 0;
253 | 
254 |   /**
255 |    * Set the total occuring count of other atom dictionaries.
256 |    *
257 |    * @param count The total occuring count of other atom dictionaies.
258 |    */
259 |   virtual void set_total_lemma_count_of_others(size_t count) = 0;
260 | 
261 |   /**
262 |    * Notify this atom dictionary to flush the cached data to persistent storage
263 |    * if necessary.
264 |    */
265 |   virtual void flush_cache() = 0;
266 | };
267 | }
268 | 
269 | #endif  // PINYINIME_INCLUDE_ATOMDICTBASE_H__
270 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/googlelib/dictlist.cpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #include <assert.h>
 18 | #include <stdlib.h>
 19 | #include <string.h>
 20 | #include "dictlist.h"
 21 | #include "mystdlib.h"
 22 | #include "ngram.h"
 23 | #include "searchutility.h"
 24 | 
 25 | namespace ime_pinyin {
 26 | 
 27 | DictList::DictList() {
 28 |   initialized_ = false;
 29 |   scis_num_ = 0;
 30 |   scis_hz_ = NULL;
 31 |   scis_splid_ = NULL;
 32 |   buf_ = NULL;
 33 |   spl_trie_ = SpellingTrie::get_cpinstance();
 34 | 
 35 |   assert(kMaxLemmaSize == 8);
 36 |   cmp_func_[0] = cmp_hanzis_1;
 37 |   cmp_func_[1] = cmp_hanzis_2;
 38 |   cmp_func_[2] = cmp_hanzis_3;
 39 |   cmp_func_[3] = cmp_hanzis_4;
 40 |   cmp_func_[4] = cmp_hanzis_5;
 41 |   cmp_func_[5] = cmp_hanzis_6;
 42 |   cmp_func_[6] = cmp_hanzis_7;
 43 |   cmp_func_[7] = cmp_hanzis_8;
 44 | }
 45 | 
 46 | DictList::~DictList() {
 47 |   free_resource();
 48 | }
 49 | 
 50 | bool DictList::alloc_resource(size_t buf_size, size_t scis_num) {
 51 |   // Allocate memory
 52 |   buf_ = static_cast<char16*>(malloc(buf_size * sizeof(char16)));
 53 |   if (NULL == buf_)
 54 |     return false;
 55 | 
 56 |   scis_num_ = scis_num;
 57 | 
 58 |   scis_hz_ = static_cast<char16*>(malloc(scis_num_ * sizeof(char16)));
 59 |   if (NULL == scis_hz_)
 60 |     return false;
 61 | 
 62 |   scis_splid_ = static_cast<SpellingId*>
 63 |       (malloc(scis_num_ * sizeof(SpellingId)));
 64 | 
 65 |   if (NULL == scis_splid_)
 66 |     return false;
 67 | 
 68 |   return true;
 69 | }
 70 | 
 71 | void DictList::free_resource() {
 72 |   if (NULL != buf_)
 73 |     free(buf_);
 74 |   buf_ = NULL;
 75 | 
 76 |   if (NULL != scis_hz_)
 77 |     free(scis_hz_);
 78 |   scis_hz_ = NULL;
 79 | 
 80 |   if (NULL != scis_splid_)
 81 |     free(scis_splid_);
 82 |   scis_splid_ = NULL;
 83 | }
 84 | 
 85 | #ifdef ___BUILD_MODEL___
 86 | bool DictList::init_list(const SingleCharItem *scis, size_t scis_num,
 87 |                          const LemmaEntry *lemma_arr, size_t lemma_num) {
 88 |   if (NULL == scis || 0 == scis_num || NULL == lemma_arr || 0 == lemma_num)
 89 |     return false;
 90 | 
 91 |   initialized_ = false;
 92 | 
 93 |   if (NULL != buf_)
 94 |     free(buf_);
 95 | 
 96 |   // calculate the size
 97 |   size_t buf_size = calculate_size(lemma_arr, lemma_num);
 98 |   if (0 == buf_size)
 99 |     return false;
100 | 
101 |   if (!alloc_resource(buf_size, scis_num))
102 |     return false;
103 | 
104 |   fill_scis(scis, scis_num);
105 | 
106 |   // Copy the related content from the array to inner buffer
107 |   fill_list(lemma_arr, lemma_num);
108 | 
109 |   initialized_ = true;
110 |   return true;
111 | }
112 | 
113 | size_t DictList::calculate_size(const LemmaEntry* lemma_arr, size_t lemma_num) {
114 |   size_t last_hz_len = 0;
115 |   size_t list_size = 0;
116 |   size_t id_num = 0;
117 | 
118 |   for (size_t i = 0; i < lemma_num; i++) {
119 |     if (0 == i) {
120 |       last_hz_len = lemma_arr[i].hz_str_len;
121 | 
122 |       assert(last_hz_len > 0);
123 |       assert(lemma_arr[0].idx_by_hz == 1);
124 | 
125 |       id_num++;
126 |       start_pos_[0] = 0;
127 |       start_id_[0] = id_num;
128 | 
129 |       last_hz_len = 1;
130 |       list_size += last_hz_len;
131 |     } else {
132 |       size_t current_hz_len = lemma_arr[i].hz_str_len;
133 | 
134 |       assert(current_hz_len >= last_hz_len);
135 | 
136 |       if (current_hz_len == last_hz_len) {
137 |           list_size += current_hz_len;
138 |           id_num++;
139 |       } else {
140 |         for (size_t len = last_hz_len; len < current_hz_len - 1; len++) {
141 |           start_pos_[len] = start_pos_[len - 1];
142 |           start_id_[len] = start_id_[len - 1];
143 |         }
144 | 
145 |         start_pos_[current_hz_len - 1] = list_size;
146 | 
147 |         id_num++;
148 |         start_id_[current_hz_len - 1] = id_num;
149 | 
150 |         last_hz_len = current_hz_len;
151 |         list_size += current_hz_len;
152 |       }
153 |     }
154 |   }
155 | 
156 |   for (size_t i = last_hz_len; i <= kMaxLemmaSize; i++) {
157 |     if (0 == i) {
158 |       start_pos_[0] = 0;
159 |       start_id_[0] = 1;
160 |     } else {
161 |       start_pos_[i] = list_size;
162 |       start_id_[i] = id_num;
163 |     }
164 |   }
165 | 
166 |   return start_pos_[kMaxLemmaSize];
167 | }
168 | 
169 | void DictList::fill_scis(const SingleCharItem *scis, size_t scis_num) {
170 |   assert(scis_num_ == scis_num);
171 | 
172 |   for (size_t pos = 0; pos < scis_num_; pos++) {
173 |     scis_hz_[pos] = scis[pos].hz;
174 |     scis_splid_[pos] = scis[pos].splid;
175 |   }
176 | }
177 | 
178 | void DictList::fill_list(const LemmaEntry* lemma_arr, size_t lemma_num) {
179 |   size_t current_pos = 0;
180 | 
181 |   utf16_strncpy(buf_, lemma_arr[0].hanzi_str,
182 |                 lemma_arr[0].hz_str_len);
183 | 
184 |   current_pos = lemma_arr[0].hz_str_len;
185 | 
186 |   size_t id_num = 1;
187 | 
188 |   for (size_t i = 1; i < lemma_num; i++) {
189 |     utf16_strncpy(buf_ + current_pos, lemma_arr[i].hanzi_str,
190 |                   lemma_arr[i].hz_str_len);
191 | 
192 |     id_num++;
193 |     current_pos += lemma_arr[i].hz_str_len;
194 |   }
195 | 
196 |   assert(current_pos == start_pos_[kMaxLemmaSize]);
197 |   assert(id_num == start_id_[kMaxLemmaSize]);
198 | }
199 | 
200 | char16* DictList::find_pos2_startedbyhz(char16 hz_char) {
201 |   char16 *found_2w = static_cast<char16*>
202 |                      (mybsearch(&hz_char, buf_ + start_pos_[1],
203 |                                 (start_pos_[2] - start_pos_[1]) / 2,
204 |                                 sizeof(char16) * 2, cmp_hanzis_1));
205 |   if (NULL == found_2w)
206 |     return NULL;
207 | 
208 |   while (found_2w > buf_ + start_pos_[1] && *found_2w == *(found_2w - 1))
209 |     found_2w -= 2;
210 | 
211 |   return found_2w;
212 | }
213 | #endif  // ___BUILD_MODEL___
214 | 
215 | char16* DictList::find_pos_startedbyhzs(const char16 last_hzs[],
216 |     size_t word_len, int (*cmp_func)(const void *, const void *)) {
217 |   char16 *found_w = static_cast<char16*>
218 |                     (mybsearch(last_hzs, buf_ + start_pos_[word_len - 1],
219 |                                (start_pos_[word_len] - start_pos_[word_len - 1])
220 |                                / word_len,
221 |                                sizeof(char16) * word_len, cmp_func));
222 | 
223 |   if (NULL == found_w)
224 |     return NULL;
225 | 
226 |   while (found_w > buf_ + start_pos_[word_len -1] &&
227 |          cmp_func(found_w, found_w - word_len) == 0)
228 |     found_w -= word_len;
229 | 
230 |   return found_w;
231 | }
232 | 
233 | size_t DictList::predict(const char16 last_hzs[], uint16 hzs_len,
234 |                          NPredictItem *npre_items, size_t npre_max,
235 |                          size_t b4_used) {
236 |   assert(hzs_len <= kMaxPredictSize && hzs_len > 0);
237 | 
238 |   // 1. Prepare work
239 |   int (*cmp_func)(const void *, const void *) = cmp_func_[hzs_len - 1];
240 | 
241 |   NGram& ngram = NGram::get_instance();
242 | 
243 |   size_t item_num = 0;
244 | 
245 |   // 2. Do prediction
246 |   for (uint16 pre_len = 1; pre_len <= kMaxPredictSize + 1 - hzs_len;
247 |        pre_len++) {
248 |     uint16 word_len = hzs_len + pre_len;
249 |     char16 *w_buf = find_pos_startedbyhzs(last_hzs, word_len, cmp_func);
250 |     if (NULL == w_buf)
251 |       continue;
252 |     while (w_buf < buf_ + start_pos_[word_len] &&
253 |            cmp_func(w_buf, last_hzs) == 0 &&
254 |            item_num < npre_max) {
255 |       memset(npre_items + item_num, 0, sizeof(NPredictItem));
256 |       utf16_strncpy(npre_items[item_num].pre_hzs, w_buf + hzs_len, pre_len);
257 |       npre_items[item_num].psb =
258 |         ngram.get_uni_psb((size_t)(w_buf - buf_ - start_pos_[word_len - 1])
259 |                           / word_len + start_id_[word_len - 1]);
260 |       npre_items[item_num].his_len = hzs_len;
261 |       item_num++;
262 |       w_buf += word_len;
263 |     }
264 |   }
265 | 
266 |   size_t new_num = 0;
267 |   for (size_t i = 0; i < item_num; i++) {
268 |     // Try to find it in the existing items
269 |     size_t e_pos;
270 |     for (e_pos = 1; e_pos <= b4_used; e_pos++) {
271 |       if (utf16_strncmp((*(npre_items - e_pos)).pre_hzs, npre_items[i].pre_hzs,
272 |                         kMaxPredictSize) == 0)
273 |         break;
274 |     }
275 |     if (e_pos <= b4_used)
276 |       continue;
277 | 
278 |     // If not found, append it to the buffer
279 |     npre_items[new_num] = npre_items[i];
280 |     new_num++;
281 |   }
282 | 
283 |   return new_num;
284 | }
285 | 
286 | uint16 DictList::get_lemma_str(LemmaIdType id_lemma, char16 *str_buf,
287 |                                uint16 str_max) {
288 |   if (!initialized_ || id_lemma >= start_id_[kMaxLemmaSize] || NULL == str_buf
289 |       || str_max <= 1)
290 |     return 0;
291 | 
292 |   // Find the range
293 |   for (uint16 i = 0; i < kMaxLemmaSize; i++) {
294 |     if (i + 1 > str_max - 1)
295 |       return 0;
296 |     if (start_id_[i] <= id_lemma && start_id_[i + 1] > id_lemma) {
297 |       size_t id_span = id_lemma - start_id_[i];
298 | 
299 |       uint16 *buf = buf_ + start_pos_[i] + id_span * (i + 1);
300 |       for (uint16 len = 0; len <= i; len++) {
301 |         str_buf[len] = buf[len];
302 |       }
303 |       str_buf[i+1] = (char16)'\0';
304 |       return i + 1;
305 |     }
306 |   }
307 |   return 0;
308 | }
309 | 
310 | uint16 DictList::get_splids_for_hanzi(char16 hanzi, uint16 half_splid,
311 |                                       uint16 *splids, uint16 max_splids) {
312 |   char16 *hz_found = static_cast<char16*>
313 |       (mybsearch(&hanzi, scis_hz_, scis_num_, sizeof(char16), cmp_hanzis_1));
314 |   assert(NULL != hz_found && hanzi == *hz_found);
315 | 
316 |   // Move to the first one.
317 |   while (hz_found > scis_hz_ && hanzi == *(hz_found - 1))
318 |     hz_found--;
319 | 
320 |   // First try to found if strict comparison result is not zero.
321 |   char16 *hz_f = hz_found;
322 |   bool strict = false;
323 |   while (hz_f < scis_hz_ + scis_num_ && hanzi == *hz_f) {
324 |     uint16 pos = hz_f - scis_hz_;
325 |     if (0 == half_splid || scis_splid_[pos].half_splid == half_splid) {
326 |       strict = true;
327 |     }
328 |     hz_f++;
329 |   }
330 | 
331 |   uint16 found_num = 0;
332 |   while (hz_found < scis_hz_ + scis_num_ && hanzi == *hz_found) {
333 |     uint16 pos = hz_found - scis_hz_;
334 |     if (0 == half_splid ||
335 |         (strict && scis_splid_[pos].half_splid == half_splid) ||
336 |         (!strict && spl_trie_->half_full_compatible(half_splid,
337 |         scis_splid_[pos].full_splid))) {
338 |       assert(found_num + 1 < max_splids);
339 |       splids[found_num] = scis_splid_[pos].full_splid;
340 |       found_num++;
341 |     }
342 |     hz_found++;
343 |   }
344 | 
345 |   return found_num;
346 | }
347 | 
348 | LemmaIdType DictList::get_lemma_id(const char16 *str, uint16 str_len) {
349 |   if (NULL == str || str_len > kMaxLemmaSize)
350 |     return 0;
351 | 
352 |   char16 *found = find_pos_startedbyhzs(str, str_len, cmp_func_[str_len - 1]);
353 |   if (NULL == found)
354 |     return 0;
355 | 
356 |   assert(found > buf_);
357 |   assert(static_cast<size_t>(found - buf_) >= start_pos_[str_len - 1]);
358 |   return static_cast<LemmaIdType>
359 |       (start_id_[str_len - 1] +
360 |        (found - buf_ - start_pos_[str_len - 1]) / str_len);
361 | }
362 | 
363 | void DictList::convert_to_hanzis(char16 *str, uint16 str_len) {
364 |   assert(NULL != str);
365 | 
366 |   for (uint16 str_pos = 0; str_pos < str_len; str_pos++) {
367 |     str[str_pos] = scis_hz_[str[str_pos]];
368 |   }
369 | }
370 | 
371 | void DictList::convert_to_scis_ids(char16 *str, uint16 str_len) {
372 |   assert(NULL != str);
373 | 
374 |   for (uint16 str_pos = 0; str_pos < str_len; str_pos++) {
375 |     str[str_pos] = 0x100;
376 |   }
377 | }
378 | 
379 | bool DictList::save_list(FILE *fp) {
380 |   if (!initialized_ || NULL == fp)
381 |     return false;
382 | 
383 |   if (NULL == buf_ || 0 == start_pos_[kMaxLemmaSize] ||
384 |       NULL == scis_hz_ || NULL == scis_splid_ || 0 == scis_num_)
385 |     return false;
386 | 
387 |   if (fwrite(&scis_num_, sizeof(uint32), 1, fp) != 1)
388 |     return false;
389 | 
390 |   if (fwrite(start_pos_, sizeof(uint32), kMaxLemmaSize + 1, fp) !=
391 |       kMaxLemmaSize + 1)
392 |     return false;
393 | 
394 |   if (fwrite(start_id_, sizeof(uint32), kMaxLemmaSize + 1, fp) !=
395 |       kMaxLemmaSize + 1)
396 |     return false;
397 | 
398 |   if (fwrite(scis_hz_, sizeof(char16), scis_num_, fp) != scis_num_)
399 |     return false;
400 | 
401 |   if (fwrite(scis_splid_, sizeof(SpellingId), scis_num_, fp) != scis_num_)
402 |     return false;
403 | 
404 |   if (fwrite(buf_, sizeof(char16), start_pos_[kMaxLemmaSize], fp) !=
405 |       start_pos_[kMaxLemmaSize])
406 |     return false;
407 | 
408 |   return true;
409 | }
410 | 
411 | bool DictList::load_list(QFile *fp) {
412 |   if (NULL == fp)
413 |     return false;
414 | 
415 |   initialized_ = false;
416 | 
417 |   if (fp->read((char *)&scis_num_, sizeof(uint32)) != sizeof(uint32))
418 |     return false;
419 | 
420 |   if (fp->read((char *)start_pos_, sizeof(uint32) * (kMaxLemmaSize + 1)) !=
421 |       sizeof(uint32) * (kMaxLemmaSize + 1))
422 |     return false;
423 | 
424 |   if (fp->read((char *)start_id_, sizeof(uint32) * (kMaxLemmaSize + 1)) !=
425 |       sizeof(uint32) * (kMaxLemmaSize + 1))
426 |     return false;
427 | 
428 |   free_resource();
429 | 
430 |   if (!alloc_resource(start_pos_[kMaxLemmaSize], scis_num_))
431 |     return false;
432 | 
433 |   if (fp->read((char *)scis_hz_, sizeof(char16) * scis_num_) != sizeof(char16) * scis_num_)
434 |     return false;
435 | 
436 |   if (fp->read((char *)scis_splid_, sizeof(SpellingId) * scis_num_) != sizeof(SpellingId) * scis_num_)
437 |     return false;
438 | 
439 |   if (fp->read((char *)buf_, sizeof(char16) * start_pos_[kMaxLemmaSize]) !=
440 |       sizeof(char16) * start_pos_[kMaxLemmaSize])
441 |     return false;
442 | 
443 |   initialized_ = true;
444 |   return true;
445 | }
446 | }  // namespace ime_pinyin
447 | 


--------------------------------------------------------------------------------
/googlelib/userdict.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #ifndef PINYINIME_INCLUDE_USERDICT_H__
 18 | #define PINYINIME_INCLUDE_USERDICT_H__
 19 | 
 20 | #define ___CACHE_ENABLED___
 21 | #define ___SYNC_ENABLED___
 22 | #define ___PREDICT_ENABLED___
 23 | 
 24 | // Debug performance for operations
 25 | // #define ___DEBUG_PERF___
 26 | 
 27 | #ifdef _WIN32
 28 | #include <time.h>
 29 | #include <winsock.h> // timeval
 30 | #else
 31 | #include <pthread.h>
 32 | #include <sys/time.h>
 33 | #endif
 34 | #include "atomdictbase.h"
 35 | 
 36 | namespace ime_pinyin {
 37 | 
 38 | class UserDict : public AtomDictBase {
 39 |  public:
 40 |   UserDict();
 41 |   ~UserDict();
 42 | 
 43 |   bool load_dict(const char *file_name, LemmaIdType start_id,
 44 |                  LemmaIdType end_id);
 45 | 
 46 |   bool close_dict();
 47 | 
 48 |   size_t number_of_lemmas();
 49 | 
 50 |   void reset_milestones(uint16 from_step, MileStoneHandle from_handle);
 51 | 
 52 |   MileStoneHandle extend_dict(MileStoneHandle from_handle,
 53 |                               const DictExtPara *dep, LmaPsbItem *lpi_items,
 54 |                               size_t lpi_max, size_t *lpi_num);
 55 | 
 56 |   size_t get_lpis(const uint16 *splid_str, uint16 splid_str_len,
 57 |                   LmaPsbItem *lpi_items, size_t lpi_max);
 58 | 
 59 |   uint16 get_lemma_str(LemmaIdType id_lemma, char16* str_buf,
 60 |                        uint16 str_max);
 61 | 
 62 |   uint16 get_lemma_splids(LemmaIdType id_lemma, uint16 *splids,
 63 |                           uint16 splids_max, bool arg_valid);
 64 | 
 65 |   size_t predict(const char16 last_hzs[], uint16 hzs_len,
 66 |                  NPredictItem *npre_items, size_t npre_max,
 67 |                  size_t b4_used);
 68 | 
 69 |   // Full spelling ids are required
 70 |   LemmaIdType put_lemma(char16 lemma_str[], uint16 splids[],
 71 |                         uint16 lemma_len, uint16 count);
 72 | 
 73 |   LemmaIdType update_lemma(LemmaIdType lemma_id, int16 delta_count,
 74 |                            bool selected);
 75 | 
 76 |   LemmaIdType get_lemma_id(char16 lemma_str[], uint16 splids[],
 77 |                            uint16 lemma_len);
 78 | 
 79 |   LmaScoreType get_lemma_score(LemmaIdType lemma_id);
 80 | 
 81 |   LmaScoreType get_lemma_score(char16 lemma_str[], uint16 splids[],
 82 |                         uint16 lemma_len);
 83 | 
 84 |   bool remove_lemma(LemmaIdType lemma_id);
 85 | 
 86 |   size_t get_total_lemma_count();
 87 |   void set_total_lemma_count_of_others(size_t count);
 88 | 
 89 |   void flush_cache();
 90 | 
 91 |   void set_limit(uint32 max_lemma_count, uint32 max_lemma_size,
 92 |                  uint32 reclaim_ratio);
 93 | 
 94 |   void reclaim();
 95 | 
 96 |   void defragment();
 97 | 
 98 | #ifdef ___SYNC_ENABLED___
 99 |   void clear_sync_lemmas(unsigned int start, unsigned int end);
100 | 
101 |   int get_sync_count();
102 | 
103 |   LemmaIdType put_lemma_no_sync(char16 lemma_str[], uint16 splids[],
104 |                         uint16 lemma_len, uint16 count, uint64 lmt);
105 |    /**
106 |     * Add lemmas encoded in UTF-16LE into dictionary without adding sync flag.
107 |     *
108 |     * @param lemmas in format of 'wo men,WM,0.32;da jia,DJ,0.12'
109 |     * @param len length of lemmas string in UTF-16LE
110 |     * @return newly added lemma count
111 |     */
112 |   int put_lemmas_no_sync_from_utf16le_string(char16 * lemmas, int len);
113 | 
114 |   /**
115 |    * Get lemmas need sync to a UTF-16LE string of above format.
116 |    * Note: input buffer (str) must not be too small. If str is too small to
117 |    *       contain single one lemma, there might be a dead loop.
118 |    *
119 |    * @param str buffer to write lemmas
120 |    * @param size buffer size in UTF-16LE
121 |    * @param count output value of lemma returned
122 |    * @return UTF-16LE string length
123 |    */
124 |   int get_sync_lemmas_in_utf16le_string_from_beginning(
125 |       char16 * str, int size, int * count);
126 | 
127 | #endif
128 | 
129 |   struct UserDictStat {
130 |     uint32 version;
131 |     const char * file_name;
132 |     struct timeval load_time;
133 |     struct timeval last_update;
134 |     uint32 disk_size;
135 |     uint32 lemma_count;
136 |     uint32 lemma_size;
137 |     uint32 delete_count;
138 |     uint32 delete_size;
139 | #ifdef ___SYNC_ENABLED___
140 |     uint32 sync_count;
141 | #endif
142 |     uint32 reclaim_ratio;
143 |     uint32 limit_lemma_count;
144 |     uint32 limit_lemma_size;
145 |   };
146 | 
147 |   bool state(UserDictStat * stat);
148 | 
149 |  private:
150 |   uint32 total_other_nfreq_;
151 |   struct timeval load_time_;
152 |   LemmaIdType start_id_;
153 |   uint32 version_;
154 |   uint8 * lemmas_;
155 | 
156 |   // In-Memory-Only flag for each lemma
157 |   static const uint8 kUserDictLemmaFlagRemove = 1;
158 |   // Inuse lemmas' offset
159 |   uint32 * offsets_;
160 |   // Highest bit in offset tells whether corresponding lemma is removed
161 |   static const uint32 kUserDictOffsetFlagRemove = (1 << 31);
162 |   // Maximum possible for the offset
163 |   static const uint32 kUserDictOffsetMask = ~(kUserDictOffsetFlagRemove);
164 |   // Bit width for last modified time, from 1 to 16
165 |   static const uint32 kUserDictLMTBitWidth = 16;
166 |   // Granularity for last modified time in second
167 |   static const uint32 kUserDictLMTGranularity = 60 * 60 * 24 * 7;
168 |   // Maximum frequency count
169 |   static const uint16 kUserDictMaxFrequency = 0xFFFF;
170 | 
171 | #define COARSE_UTC(year, month, day, hour, minute, second) \
172 |   ( \
173 |     (year - 1970) * 365 * 24 * 60 * 60 + \
174 |     (month - 1) * 30 * 24 * 60 * 60 + \
175 |     (day - 1) * 24 * 60 * 60 + \
176 |     (hour - 0) * 60 * 60 + \
177 |     (minute - 0) * 60 + \
178 |     (second - 0) \
179 |   )
180 |   static const uint64 kUserDictLMTSince = COARSE_UTC(2009, 1, 1, 0, 0, 0);
181 | 
182 |   // Correspond to offsets_
183 |   uint32 * scores_;
184 |   // Following two fields are only valid in memory
185 |   uint32 * ids_;
186 | #ifdef ___PREDICT_ENABLED___
187 |   uint32 * predicts_;
188 | #endif
189 | #ifdef ___SYNC_ENABLED___
190 |   uint32 * syncs_;
191 |   size_t sync_count_size_;
192 | #endif
193 |   uint32 * offsets_by_id_;
194 | 
195 |   size_t lemma_count_left_;
196 |   size_t lemma_size_left_;
197 | 
198 |   const char * dict_file_;
199 | 
200 |   // Be sure size is 4xN
201 |   struct UserDictInfo {
202 |     // When limitation reached, how much percentage will be reclaimed (1 ~ 100)
203 |     uint32 reclaim_ratio;
204 |     // maximum lemma count, 0 means no limitation
205 |     uint32 limit_lemma_count;
206 |     // Maximum lemma size, it's different from
207 |     // whole disk file size or in-mem dict size
208 |     // 0 means no limitation
209 |     uint32 limit_lemma_size;
210 |     // Total lemma count including deleted and inuse
211 |     // Also indicate offsets_ size
212 |     uint32 lemma_count;
213 |     // Total size of lemmas including used and freed
214 |     uint32 lemma_size;
215 |     // Freed lemma count
216 |     uint32 free_count;
217 |     // Freed lemma size in byte
218 |     uint32 free_size;
219 | #ifdef ___SYNC_ENABLED___
220 |     uint32 sync_count;
221 | #endif
222 |     int32 total_nfreq;
223 |   } dict_info_;
224 | 
225 |   static const uint32 kUserDictVersion = 0x0ABCDEF0;
226 | 
227 |   static const uint32 kUserDictPreAlloc = 32;
228 |   static const uint32 kUserDictAverageNchar = 8;
229 | 
230 |   enum UserDictState {
231 |     // Keep in order
232 |     USER_DICT_NONE = 0,
233 |     USER_DICT_SYNC,
234 | #ifdef ___SYNC_ENABLED___
235 |     USER_DICT_SYNC_DIRTY,
236 | #endif
237 |     USER_DICT_SCORE_DIRTY,
238 |     USER_DICT_OFFSET_DIRTY,
239 |     USER_DICT_LEMMA_DIRTY,
240 | 
241 |     USER_DICT_DEFRAGMENTED,
242 |   } state_;
243 | 
244 |   struct UserDictSearchable {
245 |     uint16 splids_len;
246 |     uint16 splid_start[kMaxLemmaSize];
247 |     uint16 splid_count[kMaxLemmaSize];
248 |     // Compact inital letters for both FuzzyCompareSpellId and cache system
249 |     uint32 signature[kMaxLemmaSize / 4];
250 |   };
251 | 
252 | #ifdef ___CACHE_ENABLED___
253 |   enum UserDictCacheType {
254 |     USER_DICT_CACHE,
255 |     USER_DICT_MISS_CACHE,
256 |   };
257 | 
258 |   static const int kUserDictCacheSize = 4;
259 |   static const int kUserDictMissCacheSize = kMaxLemmaSize - 1;
260 | 
261 |   struct UserDictMissCache {
262 |     uint32 signatures[kUserDictMissCacheSize][kMaxLemmaSize / 4];
263 |     uint16 head, tail;
264 |   } miss_caches_[kMaxLemmaSize];
265 | 
266 |   struct UserDictCache {
267 |     uint32 signatures[kUserDictCacheSize][kMaxLemmaSize / 4];
268 |     uint32 offsets[kUserDictCacheSize];
269 |     uint32 lengths[kUserDictCacheSize];
270 |     // Ring buffer
271 |     uint16 head, tail;
272 |   } caches_[kMaxLemmaSize];
273 | 
274 |   void cache_init();
275 | 
276 |   void cache_push(UserDictCacheType type,
277 |                  UserDictSearchable *searchable,
278 |                  uint32 offset, uint32 length);
279 | 
280 |   bool cache_hit(UserDictSearchable *searchable,
281 |                  uint32 *offset, uint32 *length);
282 | 
283 |   bool load_cache(UserDictSearchable *searchable,
284 |                   uint32 *offset, uint32 *length);
285 | 
286 |   void save_cache(UserDictSearchable *searchable,
287 |                   uint32 offset, uint32 length);
288 | 
289 |   void reset_cache();
290 | 
291 |   bool load_miss_cache(UserDictSearchable *searchable);
292 | 
293 |   void save_miss_cache(UserDictSearchable *searchable);
294 | 
295 |   void reset_miss_cache();
296 | #endif
297 | 
298 |   LmaScoreType translate_score(int f);
299 | 
300 |   int extract_score_freq(int raw_score);
301 | 
302 |   uint64 extract_score_lmt(int raw_score);
303 | 
304 |   inline int build_score(uint64 lmt, int freq);
305 | 
306 |   inline int64 utf16le_atoll(uint16 *s, int len);
307 | 
308 |   inline int utf16le_lltoa(int64 v, uint16 *s, int size);
309 | 
310 |   LemmaIdType _put_lemma(char16 lemma_str[], uint16 splids[],
311 |                         uint16 lemma_len, uint16 count, uint64 lmt);
312 | 
313 |   size_t _get_lpis(const uint16 *splid_str, uint16 splid_str_len,
314 |                    LmaPsbItem *lpi_items, size_t lpi_max, bool * need_extend);
315 | 
316 |   int _get_lemma_score(char16 lemma_str[], uint16 splids[], uint16 lemma_len);
317 | 
318 |   int _get_lemma_score(LemmaIdType lemma_id);
319 | 
320 |   int is_fuzzy_prefix_spell_id(const uint16 * id1, uint16 len1,
321 |                                const UserDictSearchable *searchable);
322 | 
323 |   bool is_prefix_spell_id(const uint16 * fullids,
324 |                           uint16 fulllen, const UserDictSearchable *searchable);
325 | 
326 |   uint32 get_dict_file_size(UserDictInfo * info);
327 | 
328 |   bool reset(const char *file);
329 | 
330 |   bool validate(const char *file);
331 | 
332 |   bool load(const char *file, LemmaIdType start_id);
333 | 
334 |   bool is_valid_state();
335 | 
336 |   bool is_valid_lemma_id(LemmaIdType id);
337 | 
338 |   LemmaIdType get_max_lemma_id();
339 | 
340 |   void set_lemma_flag(uint32 offset, uint8 flag);
341 | 
342 |   char get_lemma_flag(uint32 offset);
343 | 
344 |   char get_lemma_nchar(uint32 offset);
345 | 
346 |   uint16 * get_lemma_spell_ids(uint32 offset);
347 | 
348 |   uint16 * get_lemma_word(uint32 offset);
349 | 
350 |   // Prepare searchable to fasten locate process
351 |   void prepare_locate(UserDictSearchable *searchable,
352 |                       const uint16 * splids, uint16 len);
353 | 
354 |   // Compare initial letters only
355 |   int32 fuzzy_compare_spell_id(const uint16 * id1, uint16 len1,
356 |                                const UserDictSearchable *searchable);
357 | 
358 |   // Compare exactly two spell ids
359 |   // First argument must be a full id spell id
360 |   bool equal_spell_id(const uint16 * fullids,
361 |                       uint16 fulllen, const UserDictSearchable *searchable);
362 | 
363 |   // Find first item by initial letters
364 |   int32 locate_first_in_offsets(const UserDictSearchable *searchable);
365 | 
366 |   LemmaIdType append_a_lemma(char16 lemma_str[], uint16 splids[],
367 |                            uint16 lemma_len, uint16 count, uint64 lmt);
368 | 
369 |   // Check if a lemma is in dictionary
370 |   int32 locate_in_offsets(char16 lemma_str[],
371 |                           uint16 splid_str[], uint16 lemma_len);
372 | 
373 |   bool remove_lemma_by_offset_index(int offset_index);
374 | #ifdef ___PREDICT_ENABLED___
375 |   uint32 locate_where_to_insert_in_predicts(const uint16 * words,
376 |                                             int lemma_len);
377 | 
378 |   int32 locate_first_in_predicts(const uint16 * words, int lemma_len);
379 | 
380 |   void remove_lemma_from_predict_list(uint32 offset);
381 | #endif
382 | #ifdef ___SYNC_ENABLED___
383 |   void queue_lemma_for_sync(LemmaIdType id);
384 | 
385 |   void remove_lemma_from_sync_list(uint32 offset);
386 | 
387 |   void write_back_sync(int fd);
388 | #endif
389 |   void write_back_score(int fd);
390 |   void write_back_offset(int fd);
391 |   void write_back_lemma(int fd);
392 |   void write_back_all(int fd);
393 |   void write_back();
394 | 
395 |   struct UserDictScoreOffsetPair {
396 |     int score;
397 |     uint32 offset_index;
398 |   };
399 | 
400 |   inline void swap(UserDictScoreOffsetPair * sop, int i, int j);
401 | 
402 |   void shift_down(UserDictScoreOffsetPair * sop, int i, int n);
403 | 
404 |   // On-disk format for each lemma
405 |   // +-------------+
406 |   // | Version (4) |
407 |   // +-------------+
408 |   // +-----------+-----------+--------------------+-------------------+
409 |   // | Spare (1) | Nchar (1) | Splids (2 x Nchar) | Lemma (2 x Nchar) |
410 |   // +-----------+-----------+--------------------+-------------------+
411 |   // ...
412 |   // +-----------------------+     +-------------+      <---Offset of offset
413 |   // | Offset1 by_splids (4) | ... | OffsetN (4) |
414 |   // +-----------------------+     +-------------+
415 | #ifdef ___PREDICT_ENABLED___
416 |   // +----------------------+     +-------------+
417 |   // | Offset1 by_lemma (4) | ... | OffsetN (4) |
418 |   // +----------------------+     +-------------+
419 | #endif
420 |   // +------------+     +------------+
421 |   // | Score1 (4) | ... | ScoreN (4) |
422 |   // +------------+     +------------+
423 | #ifdef ___SYNC_ENABLED___
424 |   // +-------------+     +-------------+
425 |   // | NewAdd1 (4) | ... | NewAddN (4) |
426 |   // +-------------+     +-------------+
427 | #endif
428 |   // +----------------+
429 |   // | Dict Info (4x) |
430 |   // +----------------+
431 | };
432 | }
433 | 
434 | #endif
435 | 


--------------------------------------------------------------------------------
/googlelib/matrixsearch.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (C) 2009 The Android Open Source Project
  3 |  *
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *      http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License.
 15 |  */
 16 | 
 17 | #ifndef PINYINIME_ANDPY_INCLUDE_MATRIXSEARCH_H__
 18 | #define PINYINIME_ANDPY_INCLUDE_MATRIXSEARCH_H__
 19 | 
 20 | #include <stdlib.h>
 21 | #include "./atomdictbase.h"
 22 | #include "./dicttrie.h"
 23 | #include "./searchutility.h"
 24 | #include "./spellingtrie.h"
 25 | #include "./splparser.h"
 26 | 
 27 | namespace ime_pinyin {
 28 | 
 29 | static const size_t kMaxRowNum = kMaxSearchSteps;
 30 | 
 31 | typedef struct {
 32 |   // MileStoneHandle objects for the system and user dictionaries.
 33 |   MileStoneHandle dict_handles[2];
 34 |   // From which DMI node. -1 means it's from root.
 35 |   PoolPosType dmi_fr;
 36 |   // The spelling id for the Pinyin string from the previous DMI to this node.
 37 |   // If it is a half id like Shengmu, the node pointed by dict_node is the first
 38 |   // node with this Shengmu,
 39 |   uint16 spl_id;
 40 |   // What's the level of the dict node. Level of root is 0, but root is never
 41 |   // recorded by dict_node.
 42 |   unsigned char dict_level:7;
 43 |   // If this node is for composing phrase, this bit is 1.
 44 |   unsigned char c_phrase:1;
 45 |   // Whether the spl_id is parsed with a split character at the end.
 46 |   unsigned char splid_end_split:1;
 47 |   // What's the length of the spelling string for this match, for the whole
 48 |   // word.
 49 |   unsigned char splstr_len:7;
 50 |   // Used to indicate whether all spelling ids from the root are full spelling
 51 |   // ids. This information is useful for keymapping mode(not finished). Because
 52 |   // in this mode, there is no clear boundaries, we prefer those results which
 53 |   // have full spelling ids.
 54 |   unsigned char all_full_id:1;
 55 | } DictMatchInfo, *PDictMatchInfo;
 56 | 
 57 | typedef struct MatrixNode {
 58 |   LemmaIdType id;
 59 |   float score;
 60 |   MatrixNode *from;
 61 |   // From which DMI node. Used to trace the spelling segmentation.
 62 |   PoolPosType dmi_fr;
 63 |   uint16 step;
 64 | } MatrixNode, *PMatrixNode;
 65 | 
 66 | typedef struct {
 67 |   // The MatrixNode position in the matrix pool
 68 |   PoolPosType mtrx_nd_pos;
 69 |   // The DictMatchInfo position in the DictMatchInfo pool.
 70 |   PoolPosType dmi_pos;
 71 |   uint16 mtrx_nd_num;
 72 |   uint16 dmi_num:15;
 73 |   // Used to indicate whether there are dmi nodes in this step with full
 74 |   // spelling id. This information is used to decide whether a substring of a
 75 |   // valid Pinyin should be extended.
 76 |   //
 77 |   // Example1: shoudao
 78 |   // When the last char 'o' is added, the parser will find "dao" is a valid
 79 |   // Pinyin, and because all dmi nodes at location 'd' (including those for
 80 |   // "shoud", and those for "d") have Shengmu id only, so it is not necessary
 81 |   // to extend "ao", otherwise the result may be "shoud ao", that is not
 82 |   // reasonable.
 83 |   //
 84 |   // Example2: hengao
 85 |   // When the last 'o' is added, the parser finds "gao" is a valid Pinyin.
 86 |   // Because some dmi nodes at 'g' has Shengmu ids (hen'g and g), but some dmi
 87 |   // nodes at 'g' has full ids ('heng'), so it is necessary to extend "ao", thus
 88 |   // "heng ao" can also be the result.
 89 |   //
 90 |   // Similarly, "ganga" is expanded to "gang a".
 91 |   //
 92 |   // For Pinyin string "xian", because "xian" is a valid Pinyin, because all dmi
 93 |   // nodes at 'x' only have Shengmu ids, the parser will not try "x ian" (and it
 94 |   // is not valid either). If the parser uses break in the loop, the result
 95 |   // always be "xian"; but if the parser uses continue in the loop, "xi an" will
 96 |   // also be tried. This behaviour can be set via the function
 97 |   // set_xi_an_switch().
 98 |   uint16 dmi_has_full_id:1;
 99 |   // Points to a MatrixNode of the current step to indicate which choice the
100 |   // user selects.
101 |   MatrixNode *mtrx_nd_fixed;
102 | } MatrixRow, *PMatrixRow;
103 | 
104 | // When user inputs and selects candidates, the fixed lemma ids are stored in
105 | // lma_id_ of class MatrixSearch, and fixed_lmas_ is used to indicate how many
106 | // lemmas from the beginning are fixed. If user deletes Pinyin characters one
107 | // by one from the end, these fixed lemmas can be unlocked one by one when
108 | // necessary. Whenever user deletes a Chinese character and its spelling string
109 | // in these fixed lemmas, all fixed lemmas will be merged together into a unit
110 | // named ComposingPhrase with a lemma id kLemmaIdComposing, and this composing
111 | // phrase will be the first lemma in the sentence. Because it contains some
112 | // modified lemmas (by deleting a character), these merged lemmas are called
113 | // sub lemmas (sublma), and each of them are represented individually, so that
114 | // when user deletes Pinyin characters from the end, these sub lemmas can also
115 | // be unlocked one by one.
116 | typedef struct {
117 |   uint16 spl_ids[kMaxRowNum];
118 |   uint16 spl_start[kMaxRowNum];
119 |   char16 chn_str[kMaxRowNum];       // Chinese string.
120 |   uint16 sublma_start[kMaxRowNum];  // Counted in Chinese characters.
121 |   size_t sublma_num;
122 |   uint16 length;                    // Counted in Chinese characters.
123 | } ComposingPhrase, *TComposingPhrase;
124 | 
125 | class MatrixSearch {
126 |  private:
127 |   // If it is true, prediction list by string whose length is greater than 1
128 |   // will be limited to a reasonable number.
129 |   static const bool kPredictLimitGt1 = false;
130 | 
131 |   // If it is true, the engine will prefer long history based prediction,
132 |   // for example, when user inputs "BeiJing", we prefer "DaXue", etc., which are
133 |   // based on the two-character history.
134 |   static const bool kPreferLongHistoryPredict = true;
135 | 
136 |   // If it is true, prediction will only be based on user dictionary. this flag
137 |   // is for debug purpose.
138 |   static const bool kOnlyUserDictPredict = false;
139 | 
140 |   // The maximum buffer to store LmaPsbItems.
141 |   static const size_t kMaxLmaPsbItems = 1450;
142 | 
143 |   // How many rows for each step.
144 |   static const size_t kMaxNodeARow = 5;
145 | 
146 |   // The maximum length of the sentence candidates counted in chinese
147 |   // characters
148 |   static const size_t kMaxSentenceLength = 16;
149 | 
150 |   // The size of the matrix node pool.
151 |   static const size_t kMtrxNdPoolSize = 200;
152 | 
153 |   // The size of the DMI node pool.
154 |   static const size_t kDmiPoolSize = 800;
155 | 
156 |   // Used to indicate whether this object has been initialized.
157 |   bool inited_;
158 | 
159 |   // Spelling trie.
160 |   const SpellingTrie *spl_trie_;
161 | 
162 |   // Used to indicate this switcher status: when "xian" is parseed, should
163 |   // "xi an" also be extended. Default is false.
164 |   // These cases include: xia, xian, xiang, zhuan, jiang..., etc. The string
165 |   // should be valid for a FULL spelling, or a combination of two spellings,
166 |   // first of which is a FULL id too. So even it is true, "da" will never be
167 |   // split into "d a", because "d" is not a full spelling id.
168 |   bool xi_an_enabled_;
169 | 
170 |   // System dictionary.
171 |   DictTrie* dict_trie_;
172 | 
173 |   // User dictionary.
174 |   AtomDictBase* user_dict_;
175 | 
176 |   // Spelling parser.
177 |   SpellingParser* spl_parser_;
178 | 
179 |   // The maximum allowed length of spelling string (such as a Pinyin string).
180 |   size_t max_sps_len_;
181 | 
182 |   // The maximum allowed length of a result Chinese string.
183 |   size_t max_hzs_len_;
184 | 
185 |   // Pinyin string. Max length: kMaxRowNum - 1
186 |   char pys_[kMaxRowNum];
187 | 
188 |   // The length of the string that has been decoded successfully.
189 |   size_t pys_decoded_len_;
190 | 
191 |   // Shared buffer for multiple purposes.
192 |   size_t *share_buf_;
193 | 
194 |   MatrixNode *mtrx_nd_pool_;
195 |   PoolPosType mtrx_nd_pool_used_;    // How many nodes used in the pool
196 |   DictMatchInfo *dmi_pool_;
197 |   PoolPosType dmi_pool_used_;        // How many items used in the pool
198 | 
199 |   MatrixRow *matrix_;                // The first row is for starting
200 | 
201 |   DictExtPara *dep_;                 // Parameter used to extend DMI nodes.
202 | 
203 |   NPredictItem *npre_items_;         // Used to do prediction
204 |   size_t npre_items_len_;
205 | 
206 |   // The starting positions and lemma ids for the full sentence candidate.
207 |   size_t lma_id_num_;
208 |   uint16 lma_start_[kMaxRowNum];     // Counted in spelling ids.
209 |   LemmaIdType lma_id_[kMaxRowNum];
210 |   size_t fixed_lmas_;
211 | 
212 |   // If fixed_lmas_ is bigger than i,  Element i is used to indicate whether
213 |   // the i'th lemma id in lma_id_ is the first candidate for that step.
214 |   // If all candidates are the first one for that step, the whole string can be
215 |   // decoded by the engine automatically, so no need to add it to user
216 |   // dictionary. (We are considering to add it to user dictionary in the
217 |   // future).
218 |   uint8 fixed_lmas_no1_[kMaxRowNum];
219 | 
220 |   // Composing phrase
221 |   ComposingPhrase c_phrase_;
222 | 
223 |   // If dmi_c_phrase_ is true, the decoder will try to match the
224 |   // composing phrase (And definitely it will match successfully). If it
225 |   // is false, the decoder will try to match lemmas items in dictionaries.
226 |   bool dmi_c_phrase_;
227 | 
228 |   // The starting positions and spelling ids for the first full sentence
229 |   // candidate.
230 |   size_t spl_id_num_;                // Number of splling ids
231 |   uint16 spl_start_[kMaxRowNum];     // Starting positions
232 |   uint16 spl_id_[kMaxRowNum];        // Spelling ids
233 |   // Used to remember the last fixed position, counted in Hanzi.
234 |   size_t fixed_hzs_;
235 | 
236 |   // Lemma Items with possibility score, two purposes:
237 |   // 1. In Viterbi decoding, this buffer is used to get all possible candidates
238 |   // for current step;
239 |   // 2. When the search is done, this buffer is used to get candiates from the
240 |   // first un-fixed step and show them to the user.
241 |   LmaPsbItem lpi_items_[kMaxLmaPsbItems];
242 |   size_t lpi_total_;
243 | 
244 |   // Assign the pointers with NULL. The caller makes sure that all pointers are
245 |   // not valid before calling it. This function only will be called in the
246 |   // construction function and free_resource().
247 |   void reset_pointers_to_null();
248 | 
249 |   bool alloc_resource();
250 | 
251 |   void free_resource();
252 | 
253 |   // Reset the search space totally.
254 |   bool reset_search0();
255 | 
256 |   // Reset the search space from ch_pos step. For example, if the original
257 |   // input Pinyin is "an", reset_search(1) will reset the search space to the
258 |   // result of "a". If the given position is out of range, return false.
259 |   // if clear_fixed_this_step is true, and the ch_pos step is a fixed step,
260 |   // clear its fixed status. if clear_dmi_his_step is true, clear the DMI nodes.
261 |   // If clear_mtrx_this_sTep is true, clear the mtrx nodes of this step.
262 |   // The DMI nodes will be kept.
263 |   //
264 |   // Note: this function should not destroy content of pys_.
265 |   bool reset_search(size_t ch_pos, bool clear_fixed_this_step,
266 |                     bool clear_dmi_this_step, bool clear_mtrx_this_step);
267 | 
268 |   // Delete a part of the content in pys_.
269 |   void del_in_pys(size_t start, size_t len);
270 | 
271 |   // Delete a spelling id and its corresponding Chinese character, and merge
272 |   // the fixed lemmas into the composing phrase.
273 |   // del_spl_pos indicates which spelling id needs to be delete.
274 |   // This function will update the lemma and spelling segmentation information.
275 |   // The caller guarantees that fixed_lmas_ > 0 and del_spl_pos is within
276 |   // the fixed lemmas.
277 |   void merge_fixed_lmas(size_t del_spl_pos);
278 | 
279 |   // Get spelling start posistions and ids. The result will be stored in
280 |   // spl_id_num_, spl_start_[], spl_id_[].
281 |   // fixed_hzs_ will be also assigned.
282 |   void get_spl_start_id();
283 | 
284 |   // Get all lemma ids with match the given spelling id stream(shorter than the
285 |   // maximum length of a word).
286 |   // If pfullsent is not NULL, means the full sentence candidate may be the
287 |   // same with the coming lemma string, if so, remove that lemma.
288 |   // The result is sorted in descendant order by the frequency score.
289 |   size_t get_lpis(const uint16* splid_str, size_t splid_str_len,
290 |                   LmaPsbItem* lma_buf, size_t max_lma_buf,
291 |                   const char16 *pfullsent, bool sort_by_psb);
292 | 
293 |   uint16 get_lemma_str(LemmaIdType id_lemma, char16 *str_buf, uint16 str_max);
294 | 
295 |   uint16 get_lemma_splids(LemmaIdType id_lemma, uint16 *splids,
296 |                           uint16 splids_max, bool arg_valid);
297 | 
298 | 
299 |   // Extend a DMI node with a spelling id. ext_len is the length of the rows
300 |   // to extend, actually, it is the size of the spelling string of splid.
301 |   // return value can be 1 or 0.
302 |   // 1 means a new DMI is filled in (dmi_pool_used_ is the next blank DMI in
303 |   // the pool).
304 |   // 0 means either the dmi node can not be extended with splid, or the splid
305 |   // is a Shengmu id, which is only used to get lpi_items, or the result node
306 |   // in DictTrie has no son, it is not nccessary to keep the new DMI.
307 |   //
308 |   // This function modifies the content of lpi_items_ and lpi_total_.
309 |   // lpi_items_ is used to get the LmaPsbItem list, lpi_total_ returns the size.
310 |   // The function's returned value has no relation with the value of lpi_num.
311 |   //
312 |   // If dmi == NULL, this function will extend the root node of DictTrie
313 |   //
314 |   // This function will not change dmi_nd_pool_used_. Please change it after
315 |   // calling this function if necessary.
316 |   //
317 |   // The caller should guarantees that NULL != dep.
318 |   size_t extend_dmi(DictExtPara *dep, DictMatchInfo *dmi_s);
319 | 
320 |   // Extend dmi for the composing phrase.
321 |   size_t extend_dmi_c(DictExtPara *dep, DictMatchInfo *dmi_s);
322 | 
323 |   // Extend a MatrixNode with the give LmaPsbItem list.
324 |   // res_row is the destination row number.
325 |   // This function does not change mtrx_nd_pool_used_. Please change it after
326 |   // calling this function if necessary.
327 |   // return 0 always.
328 |   size_t extend_mtrx_nd(MatrixNode *mtrx_nd, LmaPsbItem lpi_items[],
329 |                         size_t lpi_num, PoolPosType dmi_fr, size_t res_row);
330 | 
331 | 
332 |   // Try to find a dmi node at step_to position, and the found dmi node should
333 |   // match the given spelling id strings.
334 |   PoolPosType match_dmi(size_t step_to, uint16 spl_ids[], uint16 spl_id_num);
335 | 
336 |   bool add_char(char ch);
337 |   bool prepare_add_char(char ch);
338 | 
339 |   // Called after prepare_add_char, so the input char has been saved.
340 |   bool add_char_qwerty();
341 | 
342 |   // Prepare candidates from the last fixed hanzi position.
343 |   void prepare_candidates();
344 | 
345 |   // Is the character in step pos a splitter character?
346 |   // The caller guarantees that the position is valid.
347 |   bool is_split_at(uint16 pos);
348 | 
349 |   void fill_dmi(DictMatchInfo *dmi, MileStoneHandle *handles,
350 |                 PoolPosType dmi_fr,
351 |                 uint16 spl_id, uint16 node_num, unsigned char dict_level,
352 |                 bool splid_end_split, unsigned char splstr_len,
353 |                 unsigned char all_full_id);
354 | 
355 |   size_t inner_predict(const char16 fixed_scis_ids[], uint16 scis_num,
356 |                        char16 predict_buf[][kMaxPredictSize + 1],
357 |                        size_t buf_len);
358 | 
359 |   // Add the first candidate to the user dictionary.
360 |   bool try_add_cand0_to_userdict();
361 | 
362 |   // Add a user lemma to the user dictionary. This lemma is a subset of
363 |   // candidate 0. lma_from is from which lemma in lma_ids_, lma_num is the
364 |   // number of lemmas to be combined together as a new lemma. The caller
365 |   // gurantees that the combined new lemma's length is less or equal to
366 |   // kMaxLemmaSize.
367 |   bool add_lma_to_userdict(uint16 lma_from, uint16 lma_num, float score);
368 | 
369 |   // Update dictionary frequencies.
370 |   void update_dict_freq();
371 | 
372 |   void debug_print_dmi(PoolPosType dmi_pos, uint16 nest_level);
373 | 
374 |  public:
375 |   MatrixSearch();
376 |   ~MatrixSearch();
377 | 
378 |   bool init(const char *fn_sys_dict, const char *fn_usr_dict);
379 | 
380 |   bool init_fd(int sys_fd, long start_offset, long length,
381 |                const char *fn_usr_dict);
382 | 
383 |   void init_user_dictionary(const char *fn_usr_dict);
384 | 
385 |   bool is_user_dictionary_enabled() const;
386 | 
387 |   void set_max_lens(size_t max_sps_len, size_t max_hzs_len);
388 | 
389 |   void close();
390 | 
391 |   void flush_cache();
392 | 
393 |   void set_xi_an_switch(bool xi_an_enabled);
394 | 
395 |   bool get_xi_an_switch();
396 | 
397 |   // Reset the search space. Equivalent to reset_search(0).
398 |   // If inited, always return true;
399 |   bool reset_search();
400 | 
401 |   // Search a Pinyin string.
402 |   // Return value is the position successfully parsed.
403 |   size_t search(const char *py, size_t py_len);
404 | 
405 |   // Used to delete something in the Pinyin string kept by the engine, and do
406 |   // a re-search.
407 |   // Return value is the new length of Pinyin string kept by the engine which
408 |   // is parsed successfully.
409 |   // If is_pos_in_splid is false, pos is used to indicate that pos-th Pinyin
410 |   // character needs to be deleted. If is_pos_in_splid is true, all Pinyin
411 |   // characters for pos-th spelling id needs to be deleted.
412 |   // If the deleted character(s) is just after a fixed lemma or sub lemma in
413 |   // composing phrase, clear_fixed_this_step indicates whether we needs to
414 |   // unlock the last fixed lemma or sub lemma.
415 |   // If is_pos_in_splid is false, and pos-th character is in the range for the
416 |   // fixed lemmas or composing string, this function will do nothing and just
417 |   // return the result of the previous search.
418 |   size_t delsearch(size_t pos, bool is_pos_in_splid,
419 |                    bool clear_fixed_this_step);
420 | 
421 |   // Get the number of candiates, called after search().
422 |   size_t get_candidate_num();
423 | 
424 |   // Get the Pinyin string stored by the engine.
425 |   // *decoded_len returns the length of the successfully decoded string.
426 |   const char* get_pystr(size_t *decoded_len);
427 | 
428 |   // Get the spelling boundaries for the first sentence candidate.
429 |   // Number of spellings will be returned. The number of valid elements in
430 |   // spl_start is one more than the return value because the last one is used
431 |   // to indicate the beginning of the next un-input speling.
432 |   // For a Pinyin "women", the returned value is 2, spl_start is [0, 2, 5] .
433 |   size_t get_spl_start(const uint16 *&spl_start);
434 | 
435 |   // Get one candiate string. If full sentence candidate is available, it will
436 |   // be the first one.
437 |   char16* get_candidate(size_t cand_id, char16 *cand_str, size_t max_len);
438 | 
439 |   // Get the first candiate, which is a "full sentence".
440 |   // retstr_len is not NULL, it will be used to return the string length.
441 |   // If only_unfixed is true, only unfixed part will be fetched.
442 |   char16* get_candidate0(char16* cand_str, size_t max_len,
443 |                          uint16 *retstr_len, bool only_unfixed);
444 | 
445 |   // Choose a candidate. The decoder will do a search after the fixed position.
446 |   size_t choose(size_t cand_id);
447 | 
448 |   // Cancel the last choosing operation, and return the new number of choices.
449 |   size_t cancel_last_choice();
450 | 
451 |   // Get the length of fixed Hanzis.
452 |   size_t get_fixedlen();
453 | 
454 |   size_t get_predicts(const char16 fixed_buf[],
455 |                       char16 predict_buf[][kMaxPredictSize + 1],
456 |                       size_t buf_len);
457 | };
458 | }
459 | 
460 | #endif  // PINYINIME_ANDPY_INCLUDE_MATRIXSEARCH_H__
461 | 


--------------------------------------------------------------------------------