├── .changeset └── config.json ├── .github └── workflows │ ├── build_js.yml │ └── publish.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE ├── MANIFEST.in ├── README.md ├── core ├── Cargo.toml └── src │ └── lib.rs ├── js ├── CHANGELOG.md ├── README.md ├── examples │ ├── dynamic.ts │ └── simple.ts ├── index.d.ts ├── index.js ├── lite.d.ts ├── lite.js ├── package.json ├── src │ ├── core.ts │ ├── index.ts │ ├── lite.ts │ └── utils.ts ├── test │ └── compatibility.test.ts └── tsconfig.json ├── package.json ├── perf.svg ├── pyproject.toml ├── python ├── Cargo.toml └── src │ └── lib.rs ├── scripts ├── benchmark.py ├── ranks.ts ├── redact.py └── tsconfig.json ├── setup.py ├── static ├── .gitignore ├── package.json └── src │ └── _redirects ├── tests ├── __init__.py ├── test_encoding.py ├── test_helpers.py ├── test_misc.py ├── test_offsets.py └── test_simple_public.py ├── tiktoken ├── __init__.py ├── _educational.py ├── core.py ├── load.py ├── model.py ├── model_to_encoding.json ├── py.typed ├── registry.json └── registry.py ├── turbo.json ├── wasm ├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── alias │ ├── CHANGELOG.md │ └── package.json ├── package.json ├── scripts │ ├── alias.ts │ └── postprocess.ts ├── src │ ├── init.ts │ ├── lib.rs │ └── load.ts ├── test │ ├── init_error.test.ts │ └── test_simple_public.test.ts ├── tsconfig.json └── yarn.lock └── yarn.lock /.changeset/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/.changeset/config.json -------------------------------------------------------------------------------- /.github/workflows/build_js.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/.github/workflows/build_js.yml -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/.github/workflows/publish.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/.gitignore -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/Cargo.toml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/README.md -------------------------------------------------------------------------------- /core/Cargo.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/core/Cargo.toml -------------------------------------------------------------------------------- /core/src/lib.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/core/src/lib.rs -------------------------------------------------------------------------------- /js/CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/js/CHANGELOG.md -------------------------------------------------------------------------------- /js/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/js/README.md -------------------------------------------------------------------------------- /js/examples/dynamic.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/js/examples/dynamic.ts -------------------------------------------------------------------------------- /js/examples/simple.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/js/examples/simple.ts -------------------------------------------------------------------------------- /js/index.d.ts: -------------------------------------------------------------------------------- 1 | export * from "./dist/index.js"; 2 | -------------------------------------------------------------------------------- /js/index.js: -------------------------------------------------------------------------------- 1 | module.exports = require("./dist/index.cjs"); 2 | -------------------------------------------------------------------------------- /js/lite.d.ts: -------------------------------------------------------------------------------- 1 | export * from "./dist/lite.js"; 2 | -------------------------------------------------------------------------------- /js/lite.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./dist/lite.cjs'); -------------------------------------------------------------------------------- /js/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/js/package.json -------------------------------------------------------------------------------- /js/src/core.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/js/src/core.ts -------------------------------------------------------------------------------- /js/src/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/js/src/index.ts -------------------------------------------------------------------------------- /js/src/lite.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/js/src/lite.ts -------------------------------------------------------------------------------- /js/src/utils.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/js/src/utils.ts -------------------------------------------------------------------------------- /js/test/compatibility.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/js/test/compatibility.test.ts -------------------------------------------------------------------------------- /js/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/js/tsconfig.json -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/package.json -------------------------------------------------------------------------------- /perf.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/perf.svg -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/pyproject.toml -------------------------------------------------------------------------------- /python/Cargo.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/python/Cargo.toml -------------------------------------------------------------------------------- /python/src/lib.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/python/src/lib.rs -------------------------------------------------------------------------------- /scripts/benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/scripts/benchmark.py -------------------------------------------------------------------------------- /scripts/ranks.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/scripts/ranks.ts -------------------------------------------------------------------------------- /scripts/redact.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/scripts/redact.py -------------------------------------------------------------------------------- /scripts/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/scripts/tsconfig.json -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/setup.py -------------------------------------------------------------------------------- /static/.gitignore: -------------------------------------------------------------------------------- 1 | src/js 2 | src/wasm -------------------------------------------------------------------------------- /static/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/static/package.json -------------------------------------------------------------------------------- /static/src/_redirects: -------------------------------------------------------------------------------- 1 | / https://github.com/dqbd/tiktoken 301 -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_encoding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tests/test_encoding.py -------------------------------------------------------------------------------- /tests/test_helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tests/test_helpers.py -------------------------------------------------------------------------------- /tests/test_misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tests/test_misc.py -------------------------------------------------------------------------------- /tests/test_offsets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tests/test_offsets.py -------------------------------------------------------------------------------- /tests/test_simple_public.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tests/test_simple_public.py -------------------------------------------------------------------------------- /tiktoken/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tiktoken/__init__.py -------------------------------------------------------------------------------- /tiktoken/_educational.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tiktoken/_educational.py -------------------------------------------------------------------------------- /tiktoken/core.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tiktoken/core.py -------------------------------------------------------------------------------- /tiktoken/load.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tiktoken/load.py -------------------------------------------------------------------------------- /tiktoken/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tiktoken/model.py -------------------------------------------------------------------------------- /tiktoken/model_to_encoding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tiktoken/model_to_encoding.json -------------------------------------------------------------------------------- /tiktoken/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tiktoken/registry.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tiktoken/registry.json -------------------------------------------------------------------------------- /tiktoken/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/tiktoken/registry.py -------------------------------------------------------------------------------- /turbo.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/turbo.json -------------------------------------------------------------------------------- /wasm/.gitignore: -------------------------------------------------------------------------------- 1 | # WASM 2 | ranks/ 3 | node_modules -------------------------------------------------------------------------------- /wasm/CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/CHANGELOG.md -------------------------------------------------------------------------------- /wasm/Cargo.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/Cargo.toml -------------------------------------------------------------------------------- /wasm/alias/CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/alias/CHANGELOG.md -------------------------------------------------------------------------------- /wasm/alias/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/alias/package.json -------------------------------------------------------------------------------- /wasm/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/package.json -------------------------------------------------------------------------------- /wasm/scripts/alias.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/scripts/alias.ts -------------------------------------------------------------------------------- /wasm/scripts/postprocess.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/scripts/postprocess.ts -------------------------------------------------------------------------------- /wasm/src/init.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/src/init.ts -------------------------------------------------------------------------------- /wasm/src/lib.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/src/lib.rs -------------------------------------------------------------------------------- /wasm/src/load.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/src/load.ts -------------------------------------------------------------------------------- /wasm/test/init_error.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/test/init_error.test.ts -------------------------------------------------------------------------------- /wasm/test/test_simple_public.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/test/test_simple_public.test.ts -------------------------------------------------------------------------------- /wasm/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/tsconfig.json -------------------------------------------------------------------------------- /wasm/yarn.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/wasm/yarn.lock -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dqbd/tiktoken/HEAD/yarn.lock --------------------------------------------------------------------------------