├── .github ├── ISSUE_TEMPLATE │ └── good_first_issue.yml ├── actions │ ├── find_wheel │ │ ├── action.yml │ │ ├── dist │ │ │ └── index.js │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── src │ │ │ └── index.js │ │ └── tests │ │ │ └── index.test.js │ └── install_poetry │ │ └── action.yml ├── dependabot.yml ├── dependency_review.yml ├── labeler.yml └── workflows │ ├── coverity.yml │ ├── labeler.yml │ ├── linux.yml │ ├── mac.yml │ ├── manylinux_2_28.yml │ ├── sdl.yml │ └── windows.yml ├── .gitignore ├── CMakeLists.txt ├── Jenkinsfile ├── LICENSE ├── README.md ├── SECURITY.md ├── benchmark ├── .gitignore ├── README.md └── benchmark.py ├── cmake ├── external │ └── icu.cmake ├── modules │ └── FindICU.cmake ├── platforms.cmake ├── templates │ ├── __version__.py.in │ └── vs_version.rc.in ├── version.cmake └── vs_version.cmake ├── js ├── .gitignore ├── .npmignore ├── README.md ├── openvino-tokenizers.d.ts ├── openvino-tokenizers.js ├── package-lock.json ├── package.json ├── scripts │ └── download-runtime.js └── tests │ └── openvino-tokenizers.test.js ├── poetry.lock ├── pyproject.toml ├── python └── openvino_tokenizers │ ├── __init__.py │ ├── __version__.py │ ├── build_tokenizer.py │ ├── cli.py │ ├── constants.py │ ├── convert_tokenizer.py │ ├── hf_parser.py │ ├── tiktoken_parser.py │ ├── tokenizer_pipeline.py │ ├── tokenizer_transformations.py │ └── utils.py ├── src ├── CMakeLists.txt ├── bpe_tokenizer.cpp ├── bpe_tokenizer.hpp ├── byte_fallback.cpp ├── byte_fallback.hpp ├── bytes_to_chars.cpp ├── bytes_to_chars.hpp ├── case_fold.cpp ├── case_fold.hpp ├── chars_to_bytes.cpp ├── chars_to_bytes.hpp ├── charsmap_normalization.cpp ├── charsmap_normalization.hpp ├── combine_segments.cpp ├── combine_segments.hpp ├── equal_str.cpp ├── equal_str.hpp ├── fuze.cpp ├── fuze.hpp ├── normalize_unicode.cpp ├── normalize_unicode.hpp ├── ov_extension.cpp ├── ragged_tensor_pack.cpp ├── ragged_tensor_pack.hpp ├── ragged_to_dense.cpp ├── ragged_to_dense.hpp ├── ragged_to_ragged.cpp ├── ragged_to_ragged.hpp ├── ragged_to_sparse.cpp ├── ragged_to_sparse.hpp ├── regex_normalization.cpp ├── regex_normalization.hpp ├── regex_split.cpp ├── regex_split.hpp ├── sentence_piece.cpp ├── sentence_piece.hpp ├── special_tokens_split.cpp ├── special_tokens_split.hpp ├── string_tensor_pack.hpp ├── string_tensor_unpack.cpp ├── string_tensor_unpack.hpp ├── string_to_hash_bucket.cpp ├── string_to_hash_bucket.hpp ├── tensorflow_translators.cpp ├── tensorflow_translators.hpp ├── tokenizer.hpp ├── tokenizers_factory.cpp ├── tokenizers_factory.hpp ├── trie_tokenizer.cpp ├── trie_tokenizer.hpp ├── truncate.cpp ├── truncate.hpp ├── unigram_tokenizer.cpp ├── unigram_tokenizer.hpp ├── utf8_validate.cpp ├── utf8_validate.hpp ├── utils.cpp ├── utils.hpp ├── vocab_decoder.cpp ├── vocab_decoder.hpp ├── vocab_encoder.cpp ├── vocab_encoder.hpp ├── wordpiece_tokenizer.cpp └── wordpiece_tokenizer.hpp ├── tests ├── __init__.py ├── conftest.py ├── layer_tests.py ├── pass_rates.json ├── stats.json ├── tokenizer_differential_fuzzing.py ├── tokenizers_test.py └── utils.py └── third-party-programs.txt /.github/ISSUE_TEMPLATE/good_first_issue.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/ISSUE_TEMPLATE/good_first_issue.yml -------------------------------------------------------------------------------- /.github/actions/find_wheel/action.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/actions/find_wheel/action.yml -------------------------------------------------------------------------------- /.github/actions/find_wheel/dist/index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/actions/find_wheel/dist/index.js -------------------------------------------------------------------------------- /.github/actions/find_wheel/package-lock.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/actions/find_wheel/package-lock.json -------------------------------------------------------------------------------- /.github/actions/find_wheel/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/actions/find_wheel/package.json -------------------------------------------------------------------------------- /.github/actions/find_wheel/src/index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/actions/find_wheel/src/index.js -------------------------------------------------------------------------------- /.github/actions/find_wheel/tests/index.test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/actions/find_wheel/tests/index.test.js -------------------------------------------------------------------------------- /.github/actions/install_poetry/action.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/actions/install_poetry/action.yml -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/dependency_review.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/dependency_review.yml -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/labeler.yml -------------------------------------------------------------------------------- /.github/workflows/coverity.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/workflows/coverity.yml -------------------------------------------------------------------------------- /.github/workflows/labeler.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/workflows/labeler.yml -------------------------------------------------------------------------------- /.github/workflows/linux.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/workflows/linux.yml -------------------------------------------------------------------------------- /.github/workflows/mac.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/workflows/mac.yml -------------------------------------------------------------------------------- /.github/workflows/manylinux_2_28.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/workflows/manylinux_2_28.yml -------------------------------------------------------------------------------- /.github/workflows/sdl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/workflows/sdl.yml -------------------------------------------------------------------------------- /.github/workflows/windows.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.github/workflows/windows.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/.gitignore -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/CMakeLists.txt -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/Jenkinsfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/README.md -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/SECURITY.md -------------------------------------------------------------------------------- /benchmark/.gitignore: -------------------------------------------------------------------------------- 1 | venv* 2 | latency_* 3 | ShareGPT_* -------------------------------------------------------------------------------- /benchmark/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/benchmark/README.md -------------------------------------------------------------------------------- /benchmark/benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/benchmark/benchmark.py -------------------------------------------------------------------------------- /cmake/external/icu.cmake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/cmake/external/icu.cmake -------------------------------------------------------------------------------- /cmake/modules/FindICU.cmake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/cmake/modules/FindICU.cmake -------------------------------------------------------------------------------- /cmake/platforms.cmake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/cmake/platforms.cmake -------------------------------------------------------------------------------- /cmake/templates/__version__.py.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/cmake/templates/__version__.py.in -------------------------------------------------------------------------------- /cmake/templates/vs_version.rc.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/cmake/templates/vs_version.rc.in -------------------------------------------------------------------------------- /cmake/version.cmake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/cmake/version.cmake -------------------------------------------------------------------------------- /cmake/vs_version.cmake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/cmake/vs_version.cmake -------------------------------------------------------------------------------- /js/.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | node_modules 3 | -------------------------------------------------------------------------------- /js/.npmignore: -------------------------------------------------------------------------------- 1 | bin 2 | tests 3 | thirdparty 4 | 5 | *.tgz 6 | -------------------------------------------------------------------------------- /js/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/js/README.md -------------------------------------------------------------------------------- /js/openvino-tokenizers.d.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/js/openvino-tokenizers.d.ts -------------------------------------------------------------------------------- /js/openvino-tokenizers.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/js/openvino-tokenizers.js -------------------------------------------------------------------------------- /js/package-lock.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/js/package-lock.json -------------------------------------------------------------------------------- /js/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/js/package.json -------------------------------------------------------------------------------- /js/scripts/download-runtime.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/js/scripts/download-runtime.js -------------------------------------------------------------------------------- /js/tests/openvino-tokenizers.test.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/js/tests/openvino-tokenizers.test.js -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/poetry.lock -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/pyproject.toml -------------------------------------------------------------------------------- /python/openvino_tokenizers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/python/openvino_tokenizers/__init__.py -------------------------------------------------------------------------------- /python/openvino_tokenizers/__version__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/python/openvino_tokenizers/__version__.py -------------------------------------------------------------------------------- /python/openvino_tokenizers/build_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/python/openvino_tokenizers/build_tokenizer.py -------------------------------------------------------------------------------- /python/openvino_tokenizers/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/python/openvino_tokenizers/cli.py -------------------------------------------------------------------------------- /python/openvino_tokenizers/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/python/openvino_tokenizers/constants.py -------------------------------------------------------------------------------- /python/openvino_tokenizers/convert_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/python/openvino_tokenizers/convert_tokenizer.py -------------------------------------------------------------------------------- /python/openvino_tokenizers/hf_parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/python/openvino_tokenizers/hf_parser.py -------------------------------------------------------------------------------- /python/openvino_tokenizers/tiktoken_parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/python/openvino_tokenizers/tiktoken_parser.py -------------------------------------------------------------------------------- /python/openvino_tokenizers/tokenizer_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/python/openvino_tokenizers/tokenizer_pipeline.py -------------------------------------------------------------------------------- /python/openvino_tokenizers/tokenizer_transformations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/python/openvino_tokenizers/tokenizer_transformations.py -------------------------------------------------------------------------------- /python/openvino_tokenizers/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/python/openvino_tokenizers/utils.py -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/CMakeLists.txt -------------------------------------------------------------------------------- /src/bpe_tokenizer.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/bpe_tokenizer.cpp -------------------------------------------------------------------------------- /src/bpe_tokenizer.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/bpe_tokenizer.hpp -------------------------------------------------------------------------------- /src/byte_fallback.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/byte_fallback.cpp -------------------------------------------------------------------------------- /src/byte_fallback.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/byte_fallback.hpp -------------------------------------------------------------------------------- /src/bytes_to_chars.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/bytes_to_chars.cpp -------------------------------------------------------------------------------- /src/bytes_to_chars.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/bytes_to_chars.hpp -------------------------------------------------------------------------------- /src/case_fold.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/case_fold.cpp -------------------------------------------------------------------------------- /src/case_fold.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/case_fold.hpp -------------------------------------------------------------------------------- /src/chars_to_bytes.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/chars_to_bytes.cpp -------------------------------------------------------------------------------- /src/chars_to_bytes.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/chars_to_bytes.hpp -------------------------------------------------------------------------------- /src/charsmap_normalization.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/charsmap_normalization.cpp -------------------------------------------------------------------------------- /src/charsmap_normalization.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/charsmap_normalization.hpp -------------------------------------------------------------------------------- /src/combine_segments.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/combine_segments.cpp -------------------------------------------------------------------------------- /src/combine_segments.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/combine_segments.hpp -------------------------------------------------------------------------------- /src/equal_str.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/equal_str.cpp -------------------------------------------------------------------------------- /src/equal_str.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/equal_str.hpp -------------------------------------------------------------------------------- /src/fuze.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/fuze.cpp -------------------------------------------------------------------------------- /src/fuze.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/fuze.hpp -------------------------------------------------------------------------------- /src/normalize_unicode.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/normalize_unicode.cpp -------------------------------------------------------------------------------- /src/normalize_unicode.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/normalize_unicode.hpp -------------------------------------------------------------------------------- /src/ov_extension.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/ov_extension.cpp -------------------------------------------------------------------------------- /src/ragged_tensor_pack.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/ragged_tensor_pack.cpp -------------------------------------------------------------------------------- /src/ragged_tensor_pack.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/ragged_tensor_pack.hpp -------------------------------------------------------------------------------- /src/ragged_to_dense.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/ragged_to_dense.cpp -------------------------------------------------------------------------------- /src/ragged_to_dense.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/ragged_to_dense.hpp -------------------------------------------------------------------------------- /src/ragged_to_ragged.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/ragged_to_ragged.cpp -------------------------------------------------------------------------------- /src/ragged_to_ragged.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/ragged_to_ragged.hpp -------------------------------------------------------------------------------- /src/ragged_to_sparse.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/ragged_to_sparse.cpp -------------------------------------------------------------------------------- /src/ragged_to_sparse.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/ragged_to_sparse.hpp -------------------------------------------------------------------------------- /src/regex_normalization.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/regex_normalization.cpp -------------------------------------------------------------------------------- /src/regex_normalization.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/regex_normalization.hpp -------------------------------------------------------------------------------- /src/regex_split.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/regex_split.cpp -------------------------------------------------------------------------------- /src/regex_split.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/regex_split.hpp -------------------------------------------------------------------------------- /src/sentence_piece.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/sentence_piece.cpp -------------------------------------------------------------------------------- /src/sentence_piece.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/sentence_piece.hpp -------------------------------------------------------------------------------- /src/special_tokens_split.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/special_tokens_split.cpp -------------------------------------------------------------------------------- /src/special_tokens_split.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/special_tokens_split.hpp -------------------------------------------------------------------------------- /src/string_tensor_pack.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/string_tensor_pack.hpp -------------------------------------------------------------------------------- /src/string_tensor_unpack.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/string_tensor_unpack.cpp -------------------------------------------------------------------------------- /src/string_tensor_unpack.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/string_tensor_unpack.hpp -------------------------------------------------------------------------------- /src/string_to_hash_bucket.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/string_to_hash_bucket.cpp -------------------------------------------------------------------------------- /src/string_to_hash_bucket.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/string_to_hash_bucket.hpp -------------------------------------------------------------------------------- /src/tensorflow_translators.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/tensorflow_translators.cpp -------------------------------------------------------------------------------- /src/tensorflow_translators.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/tensorflow_translators.hpp -------------------------------------------------------------------------------- /src/tokenizer.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/tokenizer.hpp -------------------------------------------------------------------------------- /src/tokenizers_factory.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/tokenizers_factory.cpp -------------------------------------------------------------------------------- /src/tokenizers_factory.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/tokenizers_factory.hpp -------------------------------------------------------------------------------- /src/trie_tokenizer.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/trie_tokenizer.cpp -------------------------------------------------------------------------------- /src/trie_tokenizer.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/trie_tokenizer.hpp -------------------------------------------------------------------------------- /src/truncate.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/truncate.cpp -------------------------------------------------------------------------------- /src/truncate.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/truncate.hpp -------------------------------------------------------------------------------- /src/unigram_tokenizer.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/unigram_tokenizer.cpp -------------------------------------------------------------------------------- /src/unigram_tokenizer.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/unigram_tokenizer.hpp -------------------------------------------------------------------------------- /src/utf8_validate.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/utf8_validate.cpp -------------------------------------------------------------------------------- /src/utf8_validate.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/utf8_validate.hpp -------------------------------------------------------------------------------- /src/utils.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/utils.cpp -------------------------------------------------------------------------------- /src/utils.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/utils.hpp -------------------------------------------------------------------------------- /src/vocab_decoder.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/vocab_decoder.cpp -------------------------------------------------------------------------------- /src/vocab_decoder.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/vocab_decoder.hpp -------------------------------------------------------------------------------- /src/vocab_encoder.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/vocab_encoder.cpp -------------------------------------------------------------------------------- /src/vocab_encoder.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/vocab_encoder.hpp -------------------------------------------------------------------------------- /src/wordpiece_tokenizer.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/wordpiece_tokenizer.cpp -------------------------------------------------------------------------------- /src/wordpiece_tokenizer.hpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/src/wordpiece_tokenizer.hpp -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/layer_tests.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/tests/layer_tests.py -------------------------------------------------------------------------------- /tests/pass_rates.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/tests/pass_rates.json -------------------------------------------------------------------------------- /tests/stats.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/tests/stats.json -------------------------------------------------------------------------------- /tests/tokenizer_differential_fuzzing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/tests/tokenizer_differential_fuzzing.py -------------------------------------------------------------------------------- /tests/tokenizers_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/tests/tokenizers_test.py -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/tests/utils.py -------------------------------------------------------------------------------- /third-party-programs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openvinotoolkit/openvino_tokenizers/HEAD/third-party-programs.txt --------------------------------------------------------------------------------