├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── bidirectional_maximum_matching.go ├── bidirectional_maximum_matching_test.go ├── bidirectional_minimum_matching.go ├── bidirectional_minimum_matching_test.go ├── bigram_dict.go ├── common_test.go ├── data └── zh │ ├── README.md │ ├── bigram.txt │ ├── dict.txt │ └── stop_tokens.txt ├── dict.go ├── maximum_matching.go ├── maximum_matching_test.go ├── minimum_matching.go ├── minimum_matching_test.go ├── num_letter_wordfilter.go ├── reverse_maximum_matching.go ├── reverse_maximum_matching_test.go ├── reverse_minimum_matching.go ├── reverse_minimum_matching_test.go ├── stop_tokens.go ├── stop_tokens_test.go ├── tokenizer.go ├── utils.go ├── utils_test.go └── wordfilter.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/.travis.yml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/README.md -------------------------------------------------------------------------------- /bidirectional_maximum_matching.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/bidirectional_maximum_matching.go -------------------------------------------------------------------------------- /bidirectional_maximum_matching_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/bidirectional_maximum_matching_test.go -------------------------------------------------------------------------------- /bidirectional_minimum_matching.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/bidirectional_minimum_matching.go -------------------------------------------------------------------------------- /bidirectional_minimum_matching_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/bidirectional_minimum_matching_test.go -------------------------------------------------------------------------------- /bigram_dict.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/bigram_dict.go -------------------------------------------------------------------------------- /common_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/common_test.go -------------------------------------------------------------------------------- /data/zh/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/data/zh/README.md -------------------------------------------------------------------------------- /data/zh/bigram.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/data/zh/bigram.txt -------------------------------------------------------------------------------- /data/zh/dict.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/data/zh/dict.txt -------------------------------------------------------------------------------- /data/zh/stop_tokens.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/data/zh/stop_tokens.txt -------------------------------------------------------------------------------- /dict.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/dict.go -------------------------------------------------------------------------------- /maximum_matching.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/maximum_matching.go -------------------------------------------------------------------------------- /maximum_matching_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/maximum_matching_test.go -------------------------------------------------------------------------------- /minimum_matching.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/minimum_matching.go -------------------------------------------------------------------------------- /minimum_matching_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/minimum_matching_test.go -------------------------------------------------------------------------------- /num_letter_wordfilter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/num_letter_wordfilter.go -------------------------------------------------------------------------------- /reverse_maximum_matching.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/reverse_maximum_matching.go -------------------------------------------------------------------------------- /reverse_maximum_matching_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/reverse_maximum_matching_test.go -------------------------------------------------------------------------------- /reverse_minimum_matching.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/reverse_minimum_matching.go -------------------------------------------------------------------------------- /reverse_minimum_matching_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/reverse_minimum_matching_test.go -------------------------------------------------------------------------------- /stop_tokens.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/stop_tokens.go -------------------------------------------------------------------------------- /stop_tokens_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/stop_tokens_test.go -------------------------------------------------------------------------------- /tokenizer.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/tokenizer.go -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/utils.go -------------------------------------------------------------------------------- /utils_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/utils_test.go -------------------------------------------------------------------------------- /wordfilter.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xujiajun/gotokenizer/HEAD/wordfilter.go --------------------------------------------------------------------------------