├── CHANGELOG.md ├── LICENSE.md ├── README.md ├── composer.json ├── config └── tokenizer-x.php ├── phpunit.xml.bak └── src ├── OpenAI ├── characters.json ├── encoder.json └── vocab.bpe ├── TokenCalculator.php ├── TokenizerX.php └── TokenizerXServiceProvider.php /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rajentrivedi/tokenizer-x/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rajentrivedi/tokenizer-x/HEAD/LICENSE.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rajentrivedi/tokenizer-x/HEAD/README.md -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rajentrivedi/tokenizer-x/HEAD/composer.json -------------------------------------------------------------------------------- /config/tokenizer-x.php: -------------------------------------------------------------------------------- 1 |