├── .github ├── FUNDING.yml └── workflows │ ├── main.yml │ └── size.yml ├── .gitignore ├── LICENSE ├── README.md ├── package.json ├── src ├── array-keyed-map.d.ts ├── bpe-regex.ts ├── bpe-vocab.ts ├── encodings.ts ├── index-browser.ts ├── index.ts ├── text-decoder.ts ├── text-encoder.ts └── tokenizer.ts ├── test └── tokenizer.test.ts ├── tsconfig.json ├── tsdx.config.js └── yarn.lock /.github/FUNDING.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/.github/FUNDING.yml -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/.github/workflows/main.yml -------------------------------------------------------------------------------- /.github/workflows/size.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/.github/workflows/size.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/README.md -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/package.json -------------------------------------------------------------------------------- /src/array-keyed-map.d.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/src/array-keyed-map.d.ts -------------------------------------------------------------------------------- /src/bpe-regex.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/src/bpe-regex.ts -------------------------------------------------------------------------------- /src/bpe-vocab.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/src/bpe-vocab.ts -------------------------------------------------------------------------------- /src/encodings.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/src/encodings.ts -------------------------------------------------------------------------------- /src/index-browser.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/src/index-browser.ts -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/src/index.ts -------------------------------------------------------------------------------- /src/text-decoder.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/src/text-decoder.ts -------------------------------------------------------------------------------- /src/text-encoder.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/src/text-encoder.ts -------------------------------------------------------------------------------- /src/tokenizer.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/src/tokenizer.ts -------------------------------------------------------------------------------- /test/tokenizer.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/test/tokenizer.test.ts -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/tsconfig.json -------------------------------------------------------------------------------- /tsdx.config.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/tsdx.config.js -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/botisan-ai/gpt3-tokenizer/HEAD/yarn.lock --------------------------------------------------------------------------------