├── .github ├── CONTRIBUTING.md ├── issue_template.md ├── pull_request_template.md └── workflows │ └── go.yml ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── README_ja.md ├── README_zh.md ├── circle.yml ├── consts └── dict_file.go ├── crf └── crf.go ├── dag.go ├── data ├── README.md ├── dict │ ├── README.md │ ├── en │ │ └── dict.txt │ ├── jp │ │ ├── README.md │ │ └── dict.txt │ └── zh │ │ ├── idf.txt │ │ ├── s_1.txt │ │ ├── stop_tokens.txt │ │ ├── stop_word.txt │ │ ├── t_1.txt │ │ ├── tf_idf.txt │ │ └── tf_idf_origin.txt └── main.go ├── dict_1.16.go ├── dict_1.16_test.go ├── dict_embed.go ├── dict_no_embed.go ├── dict_util.go ├── dict_util_test.go ├── dictionary.go ├── examples ├── dict │ ├── embed │ │ ├── main.go │ │ └── test_dict3.txt │ └── main.go ├── en │ ├── main.go │ └── testdata │ │ ├── test_en.txt │ │ ├── test_en2.txt │ │ └── test_en_dict3.txt ├── hmm │ └── main.go ├── jp │ └── main.go └── main.go ├── go.mod ├── go.sum ├── gonn ├── cnn │ └── cnn.go └── rnn │ └── rnn.go ├── gse.go ├── gse_bm_test.go ├── gse_test.go ├── hmm ├── extracker │ ├── tag_extracker.go │ └── textrank.go ├── hmm_seg.go ├── hmm_seg_test.go ├── pos │ ├── char_state_tab.go │ ├── dict.go │ ├── pos_seg.go │ ├── pos_seg_test.go │ ├── prob_emit.go │ ├── prob_start.go │ ├── prob_trans.go │ └── viterbi.go ├── prob_emit.go ├── prob_trans.go ├── relevance │ ├── bm25.go │ ├── bm25_test.go │ ├── idf.go │ ├── idf_test.go │ ├── relevance.go │ ├── tfidf.go │ └── tfidf_test.go ├── segment │ └── segment.go ├── stopwords │ ├── stop_words.go │ └── stop_words.txt ├── util │ └── util.go └── viterbi.go ├── seg_utils.go ├── segmenter.go ├── segmenter_test.go ├── stop.go ├── testdata ├── CHANGELOG.md ├── stop.txt ├── test_en.txt ├── test_en2.txt ├── test_en_dict3.txt └── zh │ ├── bailuyuan_zh.txt │ ├── test_dict.txt │ ├── test_dict1.txt │ └── test_zh_dict2.txt ├── tf ├── nlp │ └── nlp.go └── tf.go ├── token.go ├── token_test.go ├── tools ├── benchmark │ ├── benchmark.go │ └── goroutines │ │ └── goroutines.go └── server │ ├── server.go │ └── static │ ├── index.html │ └── jquery.min.js ├── trim.go └── types └── dict_file.go /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Ego 2 | 3 | -------------------------------------------------------------------------------- /.github/issue_template.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/.github/issue_template.md -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/.github/pull_request_template.md -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/.github/workflows/go.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/.gitignore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/.travis.yml -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/README.md -------------------------------------------------------------------------------- /README_ja.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/README_ja.md -------------------------------------------------------------------------------- /README_zh.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/README_zh.md -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/circle.yml -------------------------------------------------------------------------------- /consts/dict_file.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/consts/dict_file.go -------------------------------------------------------------------------------- /crf/crf.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/crf/crf.go -------------------------------------------------------------------------------- /dag.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/dag.go -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- 1 | # data 2 | 3 | Gse segment templates and dictionaries -------------------------------------------------------------------------------- /data/dict/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/data/dict/README.md -------------------------------------------------------------------------------- /data/dict/en/dict.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/dict/jp/README.md: -------------------------------------------------------------------------------- 1 | dict.txt 通过内部工具生成, Copyright 2017 ego authors. 商用和拷贝请注明来源和版权 -------------------------------------------------------------------------------- /data/dict/jp/dict.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/data/dict/jp/dict.txt -------------------------------------------------------------------------------- /data/dict/zh/idf.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/data/dict/zh/idf.txt -------------------------------------------------------------------------------- /data/dict/zh/s_1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/data/dict/zh/s_1.txt -------------------------------------------------------------------------------- /data/dict/zh/stop_tokens.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/data/dict/zh/stop_tokens.txt -------------------------------------------------------------------------------- /data/dict/zh/stop_word.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/data/dict/zh/stop_word.txt -------------------------------------------------------------------------------- /data/dict/zh/t_1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/data/dict/zh/t_1.txt -------------------------------------------------------------------------------- /data/dict/zh/tf_idf.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/data/dict/zh/tf_idf.txt -------------------------------------------------------------------------------- /data/dict/zh/tf_idf_origin.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/data/dict/zh/tf_idf_origin.txt -------------------------------------------------------------------------------- /data/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/data/main.go -------------------------------------------------------------------------------- /dict_1.16.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/dict_1.16.go -------------------------------------------------------------------------------- /dict_1.16_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/dict_1.16_test.go -------------------------------------------------------------------------------- /dict_embed.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/dict_embed.go -------------------------------------------------------------------------------- /dict_no_embed.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/dict_no_embed.go -------------------------------------------------------------------------------- /dict_util.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/dict_util.go -------------------------------------------------------------------------------- /dict_util_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/dict_util_test.go -------------------------------------------------------------------------------- /dictionary.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/dictionary.go -------------------------------------------------------------------------------- /examples/dict/embed/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/examples/dict/embed/main.go -------------------------------------------------------------------------------- /examples/dict/embed/test_dict3.txt: -------------------------------------------------------------------------------- 1 | hi 32 2 | hello 20 3 | world 20 n 4 | -------------------------------------------------------------------------------- /examples/dict/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/examples/dict/main.go -------------------------------------------------------------------------------- /examples/en/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/examples/en/main.go -------------------------------------------------------------------------------- /examples/en/testdata/test_en.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/examples/en/testdata/test_en.txt -------------------------------------------------------------------------------- /examples/en/testdata/test_en2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/examples/en/testdata/test_en2.txt -------------------------------------------------------------------------------- /examples/en/testdata/test_en_dict3.txt: -------------------------------------------------------------------------------- 1 | hi 32 2 | hello 20 3 | world 20 n 4 | -------------------------------------------------------------------------------- /examples/hmm/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/examples/hmm/main.go -------------------------------------------------------------------------------- /examples/jp/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/examples/jp/main.go -------------------------------------------------------------------------------- /examples/main.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/examples/main.go -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/go.mod -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/go.sum -------------------------------------------------------------------------------- /gonn/cnn/cnn.go: -------------------------------------------------------------------------------- 1 | package cnn 2 | -------------------------------------------------------------------------------- /gonn/rnn/rnn.go: -------------------------------------------------------------------------------- 1 | package rnn 2 | -------------------------------------------------------------------------------- /gse.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/gse.go -------------------------------------------------------------------------------- /gse_bm_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/gse_bm_test.go -------------------------------------------------------------------------------- /gse_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/gse_test.go -------------------------------------------------------------------------------- /hmm/extracker/tag_extracker.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/extracker/tag_extracker.go -------------------------------------------------------------------------------- /hmm/extracker/textrank.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/extracker/textrank.go -------------------------------------------------------------------------------- /hmm/hmm_seg.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/hmm_seg.go -------------------------------------------------------------------------------- /hmm/hmm_seg_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/hmm_seg_test.go -------------------------------------------------------------------------------- /hmm/pos/char_state_tab.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/pos/char_state_tab.go -------------------------------------------------------------------------------- /hmm/pos/dict.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/pos/dict.go -------------------------------------------------------------------------------- /hmm/pos/pos_seg.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/pos/pos_seg.go -------------------------------------------------------------------------------- /hmm/pos/pos_seg_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/pos/pos_seg_test.go -------------------------------------------------------------------------------- /hmm/pos/prob_emit.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/pos/prob_emit.go -------------------------------------------------------------------------------- /hmm/pos/prob_start.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/pos/prob_start.go -------------------------------------------------------------------------------- /hmm/pos/prob_trans.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/pos/prob_trans.go -------------------------------------------------------------------------------- /hmm/pos/viterbi.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/pos/viterbi.go -------------------------------------------------------------------------------- /hmm/prob_emit.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/prob_emit.go -------------------------------------------------------------------------------- /hmm/prob_trans.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/prob_trans.go -------------------------------------------------------------------------------- /hmm/relevance/bm25.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/relevance/bm25.go -------------------------------------------------------------------------------- /hmm/relevance/bm25_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/relevance/bm25_test.go -------------------------------------------------------------------------------- /hmm/relevance/idf.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/relevance/idf.go -------------------------------------------------------------------------------- /hmm/relevance/idf_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/relevance/idf_test.go -------------------------------------------------------------------------------- /hmm/relevance/relevance.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/relevance/relevance.go -------------------------------------------------------------------------------- /hmm/relevance/tfidf.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/relevance/tfidf.go -------------------------------------------------------------------------------- /hmm/relevance/tfidf_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/relevance/tfidf_test.go -------------------------------------------------------------------------------- /hmm/segment/segment.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/segment/segment.go -------------------------------------------------------------------------------- /hmm/stopwords/stop_words.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/stopwords/stop_words.go -------------------------------------------------------------------------------- /hmm/stopwords/stop_words.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/stopwords/stop_words.txt -------------------------------------------------------------------------------- /hmm/util/util.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/util/util.go -------------------------------------------------------------------------------- /hmm/viterbi.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/hmm/viterbi.go -------------------------------------------------------------------------------- /seg_utils.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/seg_utils.go -------------------------------------------------------------------------------- /segmenter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/segmenter.go -------------------------------------------------------------------------------- /segmenter_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/segmenter_test.go -------------------------------------------------------------------------------- /stop.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/stop.go -------------------------------------------------------------------------------- /testdata/CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/testdata/CHANGELOG.md -------------------------------------------------------------------------------- /testdata/stop.txt: -------------------------------------------------------------------------------- 1 | leave 2 | 离开 -------------------------------------------------------------------------------- /testdata/test_en.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/testdata/test_en.txt -------------------------------------------------------------------------------- /testdata/test_en2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/testdata/test_en2.txt -------------------------------------------------------------------------------- /testdata/test_en_dict3.txt: -------------------------------------------------------------------------------- 1 | hi 32 2 | hello 20 3 | world 20 n 4 | -------------------------------------------------------------------------------- /testdata/zh/bailuyuan_zh.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/testdata/zh/bailuyuan_zh.txt -------------------------------------------------------------------------------- /testdata/zh/test_dict.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/testdata/zh/test_dict.txt -------------------------------------------------------------------------------- /testdata/zh/test_dict1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/testdata/zh/test_dict1.txt -------------------------------------------------------------------------------- /testdata/zh/test_zh_dict2.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/testdata/zh/test_zh_dict2.txt -------------------------------------------------------------------------------- /tf/nlp/nlp.go: -------------------------------------------------------------------------------- 1 | package nlp 2 | -------------------------------------------------------------------------------- /tf/tf.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/tf/tf.go -------------------------------------------------------------------------------- /token.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/token.go -------------------------------------------------------------------------------- /token_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/token_test.go -------------------------------------------------------------------------------- /tools/benchmark/benchmark.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/tools/benchmark/benchmark.go -------------------------------------------------------------------------------- /tools/benchmark/goroutines/goroutines.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/tools/benchmark/goroutines/goroutines.go -------------------------------------------------------------------------------- /tools/server/server.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/tools/server/server.go -------------------------------------------------------------------------------- /tools/server/static/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/tools/server/static/index.html -------------------------------------------------------------------------------- /tools/server/static/jquery.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/tools/server/static/jquery.min.js -------------------------------------------------------------------------------- /trim.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/trim.go -------------------------------------------------------------------------------- /types/dict_file.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-ego/gse/HEAD/types/dict_file.go --------------------------------------------------------------------------------