├── .gitignore ├── LICENSE ├── README.md ├── README_zh-hans.md ├── benchmark_test.go ├── bpe.go ├── core_bpe.go ├── doc └── test_result.md ├── encoding.go ├── go.mod ├── go.sum ├── load.go ├── regex_test.go ├── test ├── benchmark.py ├── benchmark_test.go ├── test.txt ├── token_num.go └── token_num.py ├── tiktoken.go └── tiktoken_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/README.md -------------------------------------------------------------------------------- /README_zh-hans.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/README_zh-hans.md -------------------------------------------------------------------------------- /benchmark_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/benchmark_test.go -------------------------------------------------------------------------------- /bpe.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/bpe.go -------------------------------------------------------------------------------- /core_bpe.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/core_bpe.go -------------------------------------------------------------------------------- /doc/test_result.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/doc/test_result.md -------------------------------------------------------------------------------- /encoding.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/encoding.go -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/go.mod -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/go.sum -------------------------------------------------------------------------------- /load.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/load.go -------------------------------------------------------------------------------- /regex_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/regex_test.go -------------------------------------------------------------------------------- /test/benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/test/benchmark.py -------------------------------------------------------------------------------- /test/benchmark_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/test/benchmark_test.go -------------------------------------------------------------------------------- /test/test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/test/test.txt -------------------------------------------------------------------------------- /test/token_num.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/test/token_num.go -------------------------------------------------------------------------------- /test/token_num.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/test/token_num.py -------------------------------------------------------------------------------- /tiktoken.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/tiktoken.go -------------------------------------------------------------------------------- /tiktoken_test.go: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linux-do/tiktoken-go/HEAD/tiktoken_test.go --------------------------------------------------------------------------------