├── .github └── ISSUE_TEMPLATE │ ├── 🎇custom-issue-template.md │ ├── 🐛bug-report.md │ └── 💡feature-request.md ├── .gitignore ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── assets └── playground.jpg ├── examples └── demo.html ├── package.json ├── packages ├── aya_expanse │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── baichuan2 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── chatglm3 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── claude │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── claude1 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── command_r_plus │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── deepseek_v3 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── gemini │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── gemma │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── gemma2 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── gemma3 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── gpt2 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── gpt3 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── gpt35turbo │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── gpt35turbo16k │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── gpt4 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── gpt4o │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── gptoss │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── internlm2 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── llama2 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── llama3 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── llama3_1 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── llama3_2 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── minicpm_v4_5 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── mistral_nemo │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── qwen1_5 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── qwen2_5 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── qwen3 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── tests │ ├── README.md │ ├── babel.config.js │ ├── benchmarks │ │ ├── baseline.ts │ │ └── main.ts │ ├── jest.config.js │ ├── package.json │ ├── test-inputs │ │ └── t5-small │ │ │ ├── tokenizer.json │ │ │ └── tokenizer_config.json │ ├── tests │ │ ├── tokenizers.test.ts │ │ └── utils.ts │ └── tsconfig.json ├── text_davinci002 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── text_davinci003 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── text_embedding_ada002 │ ├── package.json │ ├── readme.md │ ├── src │ │ ├── data.ts │ │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── tokenizers │ ├── package.json │ ├── pnpm-lock.yaml │ ├── readme.md │ ├── src │ │ ├── TokenizerLoader.ts │ │ ├── main.ts │ │ ├── prepatch.ts │ │ ├── tokenizers │ │ │ ├── models │ │ │ │ └── whisper │ │ │ │ │ └── common_whisper.js │ │ │ ├── readme.md │ │ │ ├── tokenizers.js │ │ │ └── utils │ │ │ │ ├── constants.js │ │ │ │ ├── core.js │ │ │ │ ├── data-structures.js │ │ │ │ ├── generic.js │ │ │ │ ├── hub.js │ │ │ │ ├── maths.js │ │ │ │ └── tensor.js │ │ └── types.ts │ ├── tsconfig.json │ └── tsup.config.ts └── yi │ ├── package.json │ ├── readme.md │ ├── src │ ├── data.ts │ └── main.ts │ ├── tsconfig.json │ └── tsup.config.ts ├── pnpm-lock.yaml ├── pnpm-workspace.yaml └── scripts ├── dl_hf_models.mjs ├── export_model.py ├── pkg_gen.mjs ├── pkg_template ├── package.json ├── readme.md ├── src │ ├── data.ts │ └── main.ts ├── tsconfig.json └── tsup.config.ts ├── pkgs.config.json └── smart_publish.js /.github/ISSUE_TEMPLATE/🎇custom-issue-template.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/.github/ISSUE_TEMPLATE/🎇custom-issue-template.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/🐛bug-report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/.github/ISSUE_TEMPLATE/🐛bug-report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/💡feature-request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/.github/ISSUE_TEMPLATE/💡feature-request.md -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/.gitignore -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/.vscode/settings.json -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/README.md -------------------------------------------------------------------------------- /assets/playground.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/assets/playground.jpg -------------------------------------------------------------------------------- /examples/demo.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/examples/demo.html -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/package.json -------------------------------------------------------------------------------- /packages/aya_expanse/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/aya_expanse/package.json -------------------------------------------------------------------------------- /packages/aya_expanse/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/aya_expanse/readme.md -------------------------------------------------------------------------------- /packages/aya_expanse/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/aya_expanse/src/data.ts -------------------------------------------------------------------------------- /packages/aya_expanse/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/aya_expanse/src/main.ts -------------------------------------------------------------------------------- /packages/aya_expanse/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/aya_expanse/tsconfig.json -------------------------------------------------------------------------------- /packages/aya_expanse/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/aya_expanse/tsup.config.ts -------------------------------------------------------------------------------- /packages/baichuan2/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/baichuan2/package.json -------------------------------------------------------------------------------- /packages/baichuan2/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/baichuan2/readme.md -------------------------------------------------------------------------------- /packages/baichuan2/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/baichuan2/src/data.ts -------------------------------------------------------------------------------- /packages/baichuan2/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/baichuan2/src/main.ts -------------------------------------------------------------------------------- /packages/baichuan2/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/baichuan2/tsconfig.json -------------------------------------------------------------------------------- /packages/baichuan2/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/baichuan2/tsup.config.ts -------------------------------------------------------------------------------- /packages/chatglm3/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/chatglm3/package.json -------------------------------------------------------------------------------- /packages/chatglm3/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/chatglm3/readme.md -------------------------------------------------------------------------------- /packages/chatglm3/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/chatglm3/src/data.ts -------------------------------------------------------------------------------- /packages/chatglm3/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/chatglm3/src/main.ts -------------------------------------------------------------------------------- /packages/chatglm3/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/chatglm3/tsconfig.json -------------------------------------------------------------------------------- /packages/chatglm3/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/chatglm3/tsup.config.ts -------------------------------------------------------------------------------- /packages/claude/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude/package.json -------------------------------------------------------------------------------- /packages/claude/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude/readme.md -------------------------------------------------------------------------------- /packages/claude/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude/src/data.ts -------------------------------------------------------------------------------- /packages/claude/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude/src/main.ts -------------------------------------------------------------------------------- /packages/claude/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude/tsconfig.json -------------------------------------------------------------------------------- /packages/claude/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude/tsup.config.ts -------------------------------------------------------------------------------- /packages/claude1/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude1/package.json -------------------------------------------------------------------------------- /packages/claude1/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude1/readme.md -------------------------------------------------------------------------------- /packages/claude1/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude1/src/data.ts -------------------------------------------------------------------------------- /packages/claude1/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude1/src/main.ts -------------------------------------------------------------------------------- /packages/claude1/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude1/tsconfig.json -------------------------------------------------------------------------------- /packages/claude1/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/claude1/tsup.config.ts -------------------------------------------------------------------------------- /packages/command_r_plus/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/command_r_plus/package.json -------------------------------------------------------------------------------- /packages/command_r_plus/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/command_r_plus/readme.md -------------------------------------------------------------------------------- /packages/command_r_plus/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/command_r_plus/src/data.ts -------------------------------------------------------------------------------- /packages/command_r_plus/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/command_r_plus/src/main.ts -------------------------------------------------------------------------------- /packages/command_r_plus/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/command_r_plus/tsconfig.json -------------------------------------------------------------------------------- /packages/command_r_plus/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/command_r_plus/tsup.config.ts -------------------------------------------------------------------------------- /packages/deepseek_v3/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/deepseek_v3/package.json -------------------------------------------------------------------------------- /packages/deepseek_v3/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/deepseek_v3/readme.md -------------------------------------------------------------------------------- /packages/deepseek_v3/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/deepseek_v3/src/data.ts -------------------------------------------------------------------------------- /packages/deepseek_v3/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/deepseek_v3/src/main.ts -------------------------------------------------------------------------------- /packages/deepseek_v3/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/deepseek_v3/tsconfig.json -------------------------------------------------------------------------------- /packages/deepseek_v3/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/deepseek_v3/tsup.config.ts -------------------------------------------------------------------------------- /packages/gemini/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemini/package.json -------------------------------------------------------------------------------- /packages/gemini/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemini/readme.md -------------------------------------------------------------------------------- /packages/gemini/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemini/src/data.ts -------------------------------------------------------------------------------- /packages/gemini/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemini/src/main.ts -------------------------------------------------------------------------------- /packages/gemini/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemini/tsconfig.json -------------------------------------------------------------------------------- /packages/gemini/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemini/tsup.config.ts -------------------------------------------------------------------------------- /packages/gemma/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma/package.json -------------------------------------------------------------------------------- /packages/gemma/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma/readme.md -------------------------------------------------------------------------------- /packages/gemma/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma/src/data.ts -------------------------------------------------------------------------------- /packages/gemma/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma/src/main.ts -------------------------------------------------------------------------------- /packages/gemma/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma/tsconfig.json -------------------------------------------------------------------------------- /packages/gemma/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma/tsup.config.ts -------------------------------------------------------------------------------- /packages/gemma2/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma2/package.json -------------------------------------------------------------------------------- /packages/gemma2/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma2/readme.md -------------------------------------------------------------------------------- /packages/gemma2/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma2/src/data.ts -------------------------------------------------------------------------------- /packages/gemma2/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma2/src/main.ts -------------------------------------------------------------------------------- /packages/gemma2/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma2/tsconfig.json -------------------------------------------------------------------------------- /packages/gemma2/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma2/tsup.config.ts -------------------------------------------------------------------------------- /packages/gemma3/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma3/package.json -------------------------------------------------------------------------------- /packages/gemma3/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma3/readme.md -------------------------------------------------------------------------------- /packages/gemma3/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma3/src/data.ts -------------------------------------------------------------------------------- /packages/gemma3/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma3/src/main.ts -------------------------------------------------------------------------------- /packages/gemma3/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma3/tsconfig.json -------------------------------------------------------------------------------- /packages/gemma3/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gemma3/tsup.config.ts -------------------------------------------------------------------------------- /packages/gpt2/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt2/package.json -------------------------------------------------------------------------------- /packages/gpt2/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt2/readme.md -------------------------------------------------------------------------------- /packages/gpt2/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt2/src/data.ts -------------------------------------------------------------------------------- /packages/gpt2/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt2/src/main.ts -------------------------------------------------------------------------------- /packages/gpt2/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt2/tsconfig.json -------------------------------------------------------------------------------- /packages/gpt2/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt2/tsup.config.ts -------------------------------------------------------------------------------- /packages/gpt3/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt3/package.json -------------------------------------------------------------------------------- /packages/gpt3/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt3/readme.md -------------------------------------------------------------------------------- /packages/gpt3/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt3/src/data.ts -------------------------------------------------------------------------------- /packages/gpt3/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt3/src/main.ts -------------------------------------------------------------------------------- /packages/gpt3/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt3/tsconfig.json -------------------------------------------------------------------------------- /packages/gpt3/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt3/tsup.config.ts -------------------------------------------------------------------------------- /packages/gpt35turbo/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo/package.json -------------------------------------------------------------------------------- /packages/gpt35turbo/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo/readme.md -------------------------------------------------------------------------------- /packages/gpt35turbo/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo/src/data.ts -------------------------------------------------------------------------------- /packages/gpt35turbo/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo/src/main.ts -------------------------------------------------------------------------------- /packages/gpt35turbo/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo/tsconfig.json -------------------------------------------------------------------------------- /packages/gpt35turbo/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo/tsup.config.ts -------------------------------------------------------------------------------- /packages/gpt35turbo16k/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo16k/package.json -------------------------------------------------------------------------------- /packages/gpt35turbo16k/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo16k/readme.md -------------------------------------------------------------------------------- /packages/gpt35turbo16k/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo16k/src/data.ts -------------------------------------------------------------------------------- /packages/gpt35turbo16k/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo16k/src/main.ts -------------------------------------------------------------------------------- /packages/gpt35turbo16k/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo16k/tsconfig.json -------------------------------------------------------------------------------- /packages/gpt35turbo16k/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt35turbo16k/tsup.config.ts -------------------------------------------------------------------------------- /packages/gpt4/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4/package.json -------------------------------------------------------------------------------- /packages/gpt4/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4/readme.md -------------------------------------------------------------------------------- /packages/gpt4/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4/src/data.ts -------------------------------------------------------------------------------- /packages/gpt4/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4/src/main.ts -------------------------------------------------------------------------------- /packages/gpt4/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4/tsconfig.json -------------------------------------------------------------------------------- /packages/gpt4/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4/tsup.config.ts -------------------------------------------------------------------------------- /packages/gpt4o/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4o/package.json -------------------------------------------------------------------------------- /packages/gpt4o/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4o/readme.md -------------------------------------------------------------------------------- /packages/gpt4o/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4o/src/data.ts -------------------------------------------------------------------------------- /packages/gpt4o/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4o/src/main.ts -------------------------------------------------------------------------------- /packages/gpt4o/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4o/tsconfig.json -------------------------------------------------------------------------------- /packages/gpt4o/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gpt4o/tsup.config.ts -------------------------------------------------------------------------------- /packages/gptoss/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gptoss/package.json -------------------------------------------------------------------------------- /packages/gptoss/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gptoss/readme.md -------------------------------------------------------------------------------- /packages/gptoss/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gptoss/src/data.ts -------------------------------------------------------------------------------- /packages/gptoss/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gptoss/src/main.ts -------------------------------------------------------------------------------- /packages/gptoss/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gptoss/tsconfig.json -------------------------------------------------------------------------------- /packages/gptoss/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/gptoss/tsup.config.ts -------------------------------------------------------------------------------- /packages/internlm2/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/internlm2/package.json -------------------------------------------------------------------------------- /packages/internlm2/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/internlm2/readme.md -------------------------------------------------------------------------------- /packages/internlm2/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/internlm2/src/data.ts -------------------------------------------------------------------------------- /packages/internlm2/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/internlm2/src/main.ts -------------------------------------------------------------------------------- /packages/internlm2/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/internlm2/tsconfig.json -------------------------------------------------------------------------------- /packages/internlm2/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/internlm2/tsup.config.ts -------------------------------------------------------------------------------- /packages/llama2/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama2/package.json -------------------------------------------------------------------------------- /packages/llama2/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama2/readme.md -------------------------------------------------------------------------------- /packages/llama2/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama2/src/data.ts -------------------------------------------------------------------------------- /packages/llama2/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama2/src/main.ts -------------------------------------------------------------------------------- /packages/llama2/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama2/tsconfig.json -------------------------------------------------------------------------------- /packages/llama2/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama2/tsup.config.ts -------------------------------------------------------------------------------- /packages/llama3/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3/package.json -------------------------------------------------------------------------------- /packages/llama3/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3/readme.md -------------------------------------------------------------------------------- /packages/llama3/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3/src/data.ts -------------------------------------------------------------------------------- /packages/llama3/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3/src/main.ts -------------------------------------------------------------------------------- /packages/llama3/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3/tsconfig.json -------------------------------------------------------------------------------- /packages/llama3/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3/tsup.config.ts -------------------------------------------------------------------------------- /packages/llama3_1/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_1/package.json -------------------------------------------------------------------------------- /packages/llama3_1/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_1/readme.md -------------------------------------------------------------------------------- /packages/llama3_1/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_1/src/data.ts -------------------------------------------------------------------------------- /packages/llama3_1/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_1/src/main.ts -------------------------------------------------------------------------------- /packages/llama3_1/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_1/tsconfig.json -------------------------------------------------------------------------------- /packages/llama3_1/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_1/tsup.config.ts -------------------------------------------------------------------------------- /packages/llama3_2/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_2/package.json -------------------------------------------------------------------------------- /packages/llama3_2/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_2/readme.md -------------------------------------------------------------------------------- /packages/llama3_2/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_2/src/data.ts -------------------------------------------------------------------------------- /packages/llama3_2/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_2/src/main.ts -------------------------------------------------------------------------------- /packages/llama3_2/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_2/tsconfig.json -------------------------------------------------------------------------------- /packages/llama3_2/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/llama3_2/tsup.config.ts -------------------------------------------------------------------------------- /packages/minicpm_v4_5/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/minicpm_v4_5/package.json -------------------------------------------------------------------------------- /packages/minicpm_v4_5/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/minicpm_v4_5/readme.md -------------------------------------------------------------------------------- /packages/minicpm_v4_5/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/minicpm_v4_5/src/data.ts -------------------------------------------------------------------------------- /packages/minicpm_v4_5/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/minicpm_v4_5/src/main.ts -------------------------------------------------------------------------------- /packages/minicpm_v4_5/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/minicpm_v4_5/tsconfig.json -------------------------------------------------------------------------------- /packages/minicpm_v4_5/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/minicpm_v4_5/tsup.config.ts -------------------------------------------------------------------------------- /packages/mistral_nemo/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/mistral_nemo/package.json -------------------------------------------------------------------------------- /packages/mistral_nemo/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/mistral_nemo/readme.md -------------------------------------------------------------------------------- /packages/mistral_nemo/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/mistral_nemo/src/data.ts -------------------------------------------------------------------------------- /packages/mistral_nemo/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/mistral_nemo/src/main.ts -------------------------------------------------------------------------------- /packages/mistral_nemo/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/mistral_nemo/tsconfig.json -------------------------------------------------------------------------------- /packages/mistral_nemo/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/mistral_nemo/tsup.config.ts -------------------------------------------------------------------------------- /packages/qwen1_5/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen1_5/package.json -------------------------------------------------------------------------------- /packages/qwen1_5/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen1_5/readme.md -------------------------------------------------------------------------------- /packages/qwen1_5/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen1_5/src/data.ts -------------------------------------------------------------------------------- /packages/qwen1_5/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen1_5/src/main.ts -------------------------------------------------------------------------------- /packages/qwen1_5/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen1_5/tsconfig.json -------------------------------------------------------------------------------- /packages/qwen1_5/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen1_5/tsup.config.ts -------------------------------------------------------------------------------- /packages/qwen2_5/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen2_5/package.json -------------------------------------------------------------------------------- /packages/qwen2_5/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen2_5/readme.md -------------------------------------------------------------------------------- /packages/qwen2_5/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen2_5/src/data.ts -------------------------------------------------------------------------------- /packages/qwen2_5/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen2_5/src/main.ts -------------------------------------------------------------------------------- /packages/qwen2_5/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen2_5/tsconfig.json -------------------------------------------------------------------------------- /packages/qwen2_5/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen2_5/tsup.config.ts -------------------------------------------------------------------------------- /packages/qwen3/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen3/package.json -------------------------------------------------------------------------------- /packages/qwen3/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen3/readme.md -------------------------------------------------------------------------------- /packages/qwen3/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen3/src/data.ts -------------------------------------------------------------------------------- /packages/qwen3/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen3/src/main.ts -------------------------------------------------------------------------------- /packages/qwen3/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen3/tsconfig.json -------------------------------------------------------------------------------- /packages/qwen3/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/qwen3/tsup.config.ts -------------------------------------------------------------------------------- /packages/tests/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tests/README.md -------------------------------------------------------------------------------- /packages/tests/babel.config.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tests/babel.config.js -------------------------------------------------------------------------------- /packages/tests/benchmarks/baseline.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tests/benchmarks/baseline.ts -------------------------------------------------------------------------------- /packages/tests/benchmarks/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tests/benchmarks/main.ts -------------------------------------------------------------------------------- /packages/tests/jest.config.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tests/jest.config.js -------------------------------------------------------------------------------- /packages/tests/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tests/package.json -------------------------------------------------------------------------------- /packages/tests/test-inputs/t5-small/tokenizer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tests/test-inputs/t5-small/tokenizer.json -------------------------------------------------------------------------------- /packages/tests/test-inputs/t5-small/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tests/test-inputs/t5-small/tokenizer_config.json -------------------------------------------------------------------------------- /packages/tests/tests/tokenizers.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tests/tests/tokenizers.test.ts -------------------------------------------------------------------------------- /packages/tests/tests/utils.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tests/tests/utils.ts -------------------------------------------------------------------------------- /packages/tests/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tests/tsconfig.json -------------------------------------------------------------------------------- /packages/text_davinci002/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci002/package.json -------------------------------------------------------------------------------- /packages/text_davinci002/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci002/readme.md -------------------------------------------------------------------------------- /packages/text_davinci002/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci002/src/data.ts -------------------------------------------------------------------------------- /packages/text_davinci002/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci002/src/main.ts -------------------------------------------------------------------------------- /packages/text_davinci002/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci002/tsconfig.json -------------------------------------------------------------------------------- /packages/text_davinci002/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci002/tsup.config.ts -------------------------------------------------------------------------------- /packages/text_davinci003/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci003/package.json -------------------------------------------------------------------------------- /packages/text_davinci003/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci003/readme.md -------------------------------------------------------------------------------- /packages/text_davinci003/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci003/src/data.ts -------------------------------------------------------------------------------- /packages/text_davinci003/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci003/src/main.ts -------------------------------------------------------------------------------- /packages/text_davinci003/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci003/tsconfig.json -------------------------------------------------------------------------------- /packages/text_davinci003/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_davinci003/tsup.config.ts -------------------------------------------------------------------------------- /packages/text_embedding_ada002/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_embedding_ada002/package.json -------------------------------------------------------------------------------- /packages/text_embedding_ada002/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_embedding_ada002/readme.md -------------------------------------------------------------------------------- /packages/text_embedding_ada002/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_embedding_ada002/src/data.ts -------------------------------------------------------------------------------- /packages/text_embedding_ada002/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_embedding_ada002/src/main.ts -------------------------------------------------------------------------------- /packages/text_embedding_ada002/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_embedding_ada002/tsconfig.json -------------------------------------------------------------------------------- /packages/text_embedding_ada002/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/text_embedding_ada002/tsup.config.ts -------------------------------------------------------------------------------- /packages/tokenizers/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/package.json -------------------------------------------------------------------------------- /packages/tokenizers/pnpm-lock.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/pnpm-lock.yaml -------------------------------------------------------------------------------- /packages/tokenizers/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/readme.md -------------------------------------------------------------------------------- /packages/tokenizers/src/TokenizerLoader.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/TokenizerLoader.ts -------------------------------------------------------------------------------- /packages/tokenizers/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/main.ts -------------------------------------------------------------------------------- /packages/tokenizers/src/prepatch.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/prepatch.ts -------------------------------------------------------------------------------- /packages/tokenizers/src/tokenizers/models/whisper/common_whisper.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/tokenizers/models/whisper/common_whisper.js -------------------------------------------------------------------------------- /packages/tokenizers/src/tokenizers/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/tokenizers/readme.md -------------------------------------------------------------------------------- /packages/tokenizers/src/tokenizers/tokenizers.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/tokenizers/tokenizers.js -------------------------------------------------------------------------------- /packages/tokenizers/src/tokenizers/utils/constants.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/tokenizers/utils/constants.js -------------------------------------------------------------------------------- /packages/tokenizers/src/tokenizers/utils/core.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/tokenizers/utils/core.js -------------------------------------------------------------------------------- /packages/tokenizers/src/tokenizers/utils/data-structures.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/tokenizers/utils/data-structures.js -------------------------------------------------------------------------------- /packages/tokenizers/src/tokenizers/utils/generic.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/tokenizers/utils/generic.js -------------------------------------------------------------------------------- /packages/tokenizers/src/tokenizers/utils/hub.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/tokenizers/utils/hub.js -------------------------------------------------------------------------------- /packages/tokenizers/src/tokenizers/utils/maths.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/tokenizers/utils/maths.js -------------------------------------------------------------------------------- /packages/tokenizers/src/tokenizers/utils/tensor.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/tokenizers/utils/tensor.js -------------------------------------------------------------------------------- /packages/tokenizers/src/types.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/src/types.ts -------------------------------------------------------------------------------- /packages/tokenizers/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/tsconfig.json -------------------------------------------------------------------------------- /packages/tokenizers/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/tokenizers/tsup.config.ts -------------------------------------------------------------------------------- /packages/yi/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/yi/package.json -------------------------------------------------------------------------------- /packages/yi/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/yi/readme.md -------------------------------------------------------------------------------- /packages/yi/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/yi/src/data.ts -------------------------------------------------------------------------------- /packages/yi/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/yi/src/main.ts -------------------------------------------------------------------------------- /packages/yi/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/yi/tsconfig.json -------------------------------------------------------------------------------- /packages/yi/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/packages/yi/tsup.config.ts -------------------------------------------------------------------------------- /pnpm-lock.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/pnpm-lock.yaml -------------------------------------------------------------------------------- /pnpm-workspace.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/pnpm-workspace.yaml -------------------------------------------------------------------------------- /scripts/dl_hf_models.mjs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/scripts/dl_hf_models.mjs -------------------------------------------------------------------------------- /scripts/export_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/scripts/export_model.py -------------------------------------------------------------------------------- /scripts/pkg_gen.mjs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/scripts/pkg_gen.mjs -------------------------------------------------------------------------------- /scripts/pkg_template/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/scripts/pkg_template/package.json -------------------------------------------------------------------------------- /scripts/pkg_template/readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/scripts/pkg_template/readme.md -------------------------------------------------------------------------------- /scripts/pkg_template/src/data.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/scripts/pkg_template/src/data.ts -------------------------------------------------------------------------------- /scripts/pkg_template/src/main.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/scripts/pkg_template/src/main.ts -------------------------------------------------------------------------------- /scripts/pkg_template/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/scripts/pkg_template/tsconfig.json -------------------------------------------------------------------------------- /scripts/pkg_template/tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/scripts/pkg_template/tsup.config.ts -------------------------------------------------------------------------------- /scripts/pkgs.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/scripts/pkgs.config.json -------------------------------------------------------------------------------- /scripts/smart_publish.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lenML/tokenizers/HEAD/scripts/smart_publish.js --------------------------------------------------------------------------------