├── .editorconfig ├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .gitmodules ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Cargo.toml ├── Justfile ├── LICENSE ├── README.md ├── SECURITY.md ├── renovate.json ├── rustfmt.toml ├── scripts ├── download_assets.sh ├── get_current_version.sh └── update_vendored_code.sh └── tiktoken-rs ├── Cargo.toml ├── README.md ├── assets ├── cl100k_base.tiktoken ├── encoder.json ├── o200k_base.tiktoken ├── p50k_base.tiktoken ├── r50k_base.tiktoken └── vocab.bpe ├── benches ├── basic.rs ├── init.rs └── locking.rs ├── examples ├── count_chat_tokens.rs ├── count_tokens.rs ├── example_text.txt └── num_tokens_memory.rs ├── src ├── api.rs ├── lib.rs ├── model.rs ├── patched_tiktoken.rs ├── singleton.rs ├── tiktoken_ext │ ├── mod.rs │ └── openai_public.rs ├── tokenizer.rs └── vendor_tiktoken.rs └── tests ├── model.rs └── tiktoken.rs /.editorconfig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/.editorconfig -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/.github/workflows/ci.yml -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/.github/workflows/release.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/.gitmodules -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/Cargo.toml -------------------------------------------------------------------------------- /Justfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/Justfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ./tiktoken-rs/README.md -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/SECURITY.md -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/renovate.json -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # Defaults are great! 2 | -------------------------------------------------------------------------------- /scripts/download_assets.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/scripts/download_assets.sh -------------------------------------------------------------------------------- /scripts/get_current_version.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/scripts/get_current_version.sh -------------------------------------------------------------------------------- /scripts/update_vendored_code.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/scripts/update_vendored_code.sh -------------------------------------------------------------------------------- /tiktoken-rs/Cargo.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/Cargo.toml -------------------------------------------------------------------------------- /tiktoken-rs/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/README.md -------------------------------------------------------------------------------- /tiktoken-rs/assets/cl100k_base.tiktoken: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/assets/cl100k_base.tiktoken -------------------------------------------------------------------------------- /tiktoken-rs/assets/encoder.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/assets/encoder.json -------------------------------------------------------------------------------- /tiktoken-rs/assets/o200k_base.tiktoken: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/assets/o200k_base.tiktoken -------------------------------------------------------------------------------- /tiktoken-rs/assets/p50k_base.tiktoken: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/assets/p50k_base.tiktoken -------------------------------------------------------------------------------- /tiktoken-rs/assets/r50k_base.tiktoken: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/assets/r50k_base.tiktoken -------------------------------------------------------------------------------- /tiktoken-rs/assets/vocab.bpe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/assets/vocab.bpe -------------------------------------------------------------------------------- /tiktoken-rs/benches/basic.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/benches/basic.rs -------------------------------------------------------------------------------- /tiktoken-rs/benches/init.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/benches/init.rs -------------------------------------------------------------------------------- /tiktoken-rs/benches/locking.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/benches/locking.rs -------------------------------------------------------------------------------- /tiktoken-rs/examples/count_chat_tokens.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/examples/count_chat_tokens.rs -------------------------------------------------------------------------------- /tiktoken-rs/examples/count_tokens.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/examples/count_tokens.rs -------------------------------------------------------------------------------- /tiktoken-rs/examples/example_text.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/examples/example_text.txt -------------------------------------------------------------------------------- /tiktoken-rs/examples/num_tokens_memory.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/examples/num_tokens_memory.rs -------------------------------------------------------------------------------- /tiktoken-rs/src/api.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/src/api.rs -------------------------------------------------------------------------------- /tiktoken-rs/src/lib.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/src/lib.rs -------------------------------------------------------------------------------- /tiktoken-rs/src/model.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/src/model.rs -------------------------------------------------------------------------------- /tiktoken-rs/src/patched_tiktoken.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/src/patched_tiktoken.rs -------------------------------------------------------------------------------- /tiktoken-rs/src/singleton.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/src/singleton.rs -------------------------------------------------------------------------------- /tiktoken-rs/src/tiktoken_ext/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod openai_public; 2 | -------------------------------------------------------------------------------- /tiktoken-rs/src/tiktoken_ext/openai_public.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/src/tiktoken_ext/openai_public.rs -------------------------------------------------------------------------------- /tiktoken-rs/src/tokenizer.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/src/tokenizer.rs -------------------------------------------------------------------------------- /tiktoken-rs/src/vendor_tiktoken.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/src/vendor_tiktoken.rs -------------------------------------------------------------------------------- /tiktoken-rs/tests/model.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/tests/model.rs -------------------------------------------------------------------------------- /tiktoken-rs/tests/tiktoken.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zurawiki/tiktoken-rs/HEAD/tiktoken-rs/tests/tiktoken.rs --------------------------------------------------------------------------------