├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── dependabot.yml ├── pull_request_template.md └── workflows │ ├── cifuzz.yml │ ├── cmake.yml │ ├── cross_build.yml │ ├── requirements │ ├── base.in │ ├── base.txt │ ├── cibuildwheel.in │ └── cibuildwheel.txt │ └── wheel.yml ├── .gitignore ├── CMakeLists.txt ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── VERSION.txt ├── cmake └── ios.toolchain.cmake ├── config.h.in ├── data ├── Scripts.txt ├── botchan.txt ├── extract_headers.pl ├── gen_spec_parser.pl ├── gen_unicode_scripts_code.pl ├── ids_denorm.tsv ├── ids_norm.tsv ├── nfc.tsv ├── nfc_cf.tsv ├── nfd.tsv ├── nfd_cf.tsv ├── nfkc.bin ├── nfkc.tsv ├── nfkc_cf.bin ├── nfkc_cf.tsv ├── nfkd.tsv ├── nfkd_cf.tsv ├── nmt_nfkc.bin ├── nmt_nfkc.tsv ├── nmt_nfkc_cf.bin ├── nmt_nfkc_cf.tsv └── wagahaiwa_nekodearu.txt ├── doc ├── api.md ├── experiments.md ├── normalization.md ├── options.md └── special_symbols.md ├── python ├── .gitignore ├── MANIFEST.in ├── README.md ├── add_new_vocab.ipynb ├── build_bundled.sh ├── build_sdist.sh ├── pyproject.toml ├── sentencepiece_python_module_example.ipynb ├── setup.cfg ├── setup.py ├── src │ └── sentencepiece │ │ ├── __init__.py │ │ ├── _version.py │ │ ├── sentencepiece.i │ │ ├── sentencepiece_model_pb2.py │ │ ├── sentencepiece_pb2.py │ │ └── sentencepiece_wrap.cxx └── test │ ├── __init__.py │ ├── botchan.txt │ ├── sentencepiece_test.py │ ├── test_ja_model.model │ └── test_model.model ├── sentencepiece.pc.in ├── src ├── CMakeLists.txt ├── bpe_model.cc ├── bpe_model.h ├── bpe_model_test.cc ├── bpe_model_trainer.cc ├── bpe_model_trainer.h ├── bpe_model_trainer_test.cc ├── builder.cc ├── builder.h ├── builder_test.cc ├── builtin_pb │ ├── sentencepiece.pb.cc │ ├── sentencepiece.pb.h │ ├── sentencepiece_model.pb.cc │ └── sentencepiece_model.pb.h ├── char_model.cc ├── char_model.h ├── char_model_test.cc ├── char_model_trainer.cc ├── char_model_trainer.h ├── char_model_trainer_test.cc ├── common.h ├── compile_charsmap_main.cc ├── error.cc ├── filesystem.cc ├── filesystem.h ├── filesystem_test.cc ├── freelist.h ├── freelist_test.cc ├── init.cc ├── init.h ├── init_test.cc ├── model_factory.cc ├── model_factory.h ├── model_factory_test.cc ├── model_interface.cc ├── model_interface.h ├── model_interface_test.cc ├── normalization_rule.h ├── normalizer.cc ├── normalizer.h ├── normalizer_test.cc ├── pretokenizer_for_training.cc ├── pretokenizer_for_training.h ├── pretokenizer_for_training_test.cc ├── sentencepiece.proto ├── sentencepiece_model.proto ├── sentencepiece_processor.cc ├── sentencepiece_processor.h ├── sentencepiece_processor_test.cc ├── sentencepiece_trainer.cc ├── sentencepiece_trainer.h ├── sentencepiece_trainer_test.cc ├── spec_parser.h ├── spm_decode_main.cc ├── spm_encode_main.cc ├── spm_export_vocab_main.cc ├── spm_normalize_main.cc ├── spm_train_main.cc ├── test_main.cc ├── testharness.cc ├── testharness.h ├── trainer_factory.cc ├── trainer_factory.h ├── trainer_factory_test.cc ├── trainer_interface.cc ├── trainer_interface.h ├── trainer_interface_test.cc ├── unicode_script.cc ├── unicode_script.h ├── unicode_script_map.h ├── unicode_script_test.cc ├── unigram_model.cc ├── unigram_model.h ├── unigram_model_test.cc ├── unigram_model_trainer.cc ├── unigram_model_trainer.h ├── unigram_model_trainer_test.cc ├── util.cc ├── util.h ├── util_test.cc ├── word_model.cc ├── word_model.h ├── word_model_test.cc ├── word_model_trainer.cc ├── word_model_trainer.h └── word_model_trainer_test.cc └── third_party ├── CMakeLists.txt ├── absl ├── LICENSE ├── container │ ├── btree_set.h │ ├── flat_hash_map.h │ └── flat_hash_set.h ├── flags │ ├── flag.cc │ ├── flag.h │ ├── parse.h │ ├── usage.h │ └── usage_config.h ├── log │ ├── check.h │ ├── globals.h │ ├── log.cc │ └── log.h └── strings │ ├── ascii.h │ ├── match.h │ ├── numbers.h │ ├── str_cat.h │ ├── str_format.h │ ├── str_join.h │ ├── str_replace.h │ ├── str_split.h │ ├── string_view.h │ └── strip.h ├── darts_clone ├── LICENSE └── darts.h ├── esaxx ├── LICENSE ├── esa.hxx └── sais.hxx └── protobuf-lite ├── LICENSE ├── arena.cc ├── arenastring.cc ├── bytestream.cc ├── coded_stream.cc ├── common.cc ├── extension_set.cc ├── generated_enum_util.cc ├── generated_message_table_driven_lite.cc ├── generated_message_util.cc ├── google └── protobuf │ ├── any.h │ ├── arena.h │ ├── arena_impl.h │ ├── arenastring.h │ ├── descriptor.h │ ├── extension_set.h │ ├── extension_set_inl.h │ ├── generated_enum_reflection.h │ ├── generated_enum_util.h │ ├── generated_message_table_driven.h │ ├── generated_message_table_driven_lite.h │ ├── generated_message_util.h │ ├── has_bits.h │ ├── implicit_weak_message.h │ ├── io │ ├── coded_stream.h │ ├── io_win32.h │ ├── zero_copy_stream.h │ ├── zero_copy_stream_impl.h │ └── zero_copy_stream_impl_lite.h │ ├── map.h │ ├── map_entry_lite.h │ ├── map_field_lite.h │ ├── map_type_handler.h │ ├── message_lite.h │ ├── metadata_lite.h │ ├── parse_context.h │ ├── port.h │ ├── port_def.inc │ ├── port_undef.inc │ ├── repeated_field.h │ ├── stubs │ ├── bytestream.h │ ├── callback.h │ ├── casts.h │ ├── common.h │ ├── hash.h │ ├── int128.h │ ├── logging.h │ ├── macros.h │ ├── map_util.h │ ├── mutex.h │ ├── once.h │ ├── platform_macros.h │ ├── port.h │ ├── status.h │ ├── statusor.h │ ├── stl_util.h │ ├── stringpiece.h │ ├── stringprintf.h │ ├── strutil.h │ └── time.h │ ├── unknown_field_set.h │ └── wire_format_lite.h ├── implicit_weak_message.cc ├── int128.cc ├── io_win32.cc ├── message_lite.cc ├── parse_context.cc ├── repeated_field.cc ├── status.cc ├── statusor.cc ├── stringpiece.cc ├── stringprintf.cc ├── structurally_valid.cc ├── strutil.cc ├── time.cc ├── wire_format_lite.cc ├── zero_copy_stream.cc ├── zero_copy_stream_impl.cc └── zero_copy_stream_impl_lite.cc /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/.github/ISSUE_TEMPLATE/bug_report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/.github/ISSUE_TEMPLATE/feature_request.md -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/.github/pull_request_template.md -------------------------------------------------------------------------------- /.github/workflows/cifuzz.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/.github/workflows/cifuzz.yml -------------------------------------------------------------------------------- /.github/workflows/cmake.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/.github/workflows/cmake.yml -------------------------------------------------------------------------------- /.github/workflows/cross_build.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/.github/workflows/cross_build.yml -------------------------------------------------------------------------------- /.github/workflows/requirements/base.in: -------------------------------------------------------------------------------- 1 | pip 2 | setuptools 3 | wheel 4 | twine 5 | pytest 6 | -------------------------------------------------------------------------------- /.github/workflows/requirements/base.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/.github/workflows/requirements/base.txt -------------------------------------------------------------------------------- /.github/workflows/requirements/cibuildwheel.in: -------------------------------------------------------------------------------- 1 | -c base.txt 2 | cibuildwheel 3 | -------------------------------------------------------------------------------- /.github/workflows/requirements/cibuildwheel.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/.github/workflows/requirements/cibuildwheel.txt -------------------------------------------------------------------------------- /.github/workflows/wheel.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/.github/workflows/wheel.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/.gitignore -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/CMakeLists.txt -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/README.md -------------------------------------------------------------------------------- /VERSION.txt: -------------------------------------------------------------------------------- 1 | 0.2.2 2 | -------------------------------------------------------------------------------- /cmake/ios.toolchain.cmake: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/cmake/ios.toolchain.cmake -------------------------------------------------------------------------------- /config.h.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/config.h.in -------------------------------------------------------------------------------- /data/Scripts.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/Scripts.txt -------------------------------------------------------------------------------- /data/botchan.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/botchan.txt -------------------------------------------------------------------------------- /data/extract_headers.pl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/extract_headers.pl -------------------------------------------------------------------------------- /data/gen_spec_parser.pl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/gen_spec_parser.pl -------------------------------------------------------------------------------- /data/gen_unicode_scripts_code.pl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/gen_unicode_scripts_code.pl -------------------------------------------------------------------------------- /data/ids_denorm.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/ids_denorm.tsv -------------------------------------------------------------------------------- /data/ids_norm.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/ids_norm.tsv -------------------------------------------------------------------------------- /data/nfc.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nfc.tsv -------------------------------------------------------------------------------- /data/nfc_cf.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nfc_cf.tsv -------------------------------------------------------------------------------- /data/nfd.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nfd.tsv -------------------------------------------------------------------------------- /data/nfd_cf.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nfd_cf.tsv -------------------------------------------------------------------------------- /data/nfkc.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nfkc.bin -------------------------------------------------------------------------------- /data/nfkc.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nfkc.tsv -------------------------------------------------------------------------------- /data/nfkc_cf.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nfkc_cf.bin -------------------------------------------------------------------------------- /data/nfkc_cf.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nfkc_cf.tsv -------------------------------------------------------------------------------- /data/nfkd.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nfkd.tsv -------------------------------------------------------------------------------- /data/nfkd_cf.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nfkd_cf.tsv -------------------------------------------------------------------------------- /data/nmt_nfkc.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nmt_nfkc.bin -------------------------------------------------------------------------------- /data/nmt_nfkc.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nmt_nfkc.tsv -------------------------------------------------------------------------------- /data/nmt_nfkc_cf.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nmt_nfkc_cf.bin -------------------------------------------------------------------------------- /data/nmt_nfkc_cf.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/nmt_nfkc_cf.tsv -------------------------------------------------------------------------------- /data/wagahaiwa_nekodearu.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/data/wagahaiwa_nekodearu.txt -------------------------------------------------------------------------------- /doc/api.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/doc/api.md -------------------------------------------------------------------------------- /doc/experiments.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/doc/experiments.md -------------------------------------------------------------------------------- /doc/normalization.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/doc/normalization.md -------------------------------------------------------------------------------- /doc/options.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/doc/options.md -------------------------------------------------------------------------------- /doc/special_symbols.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/doc/special_symbols.md -------------------------------------------------------------------------------- /python/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/.gitignore -------------------------------------------------------------------------------- /python/MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/MANIFEST.in -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/README.md -------------------------------------------------------------------------------- /python/add_new_vocab.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/add_new_vocab.ipynb -------------------------------------------------------------------------------- /python/build_bundled.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/build_bundled.sh -------------------------------------------------------------------------------- /python/build_sdist.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/build_sdist.sh -------------------------------------------------------------------------------- /python/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/pyproject.toml -------------------------------------------------------------------------------- /python/sentencepiece_python_module_example.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/sentencepiece_python_module_example.ipynb -------------------------------------------------------------------------------- /python/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description_file = README.md 3 | -------------------------------------------------------------------------------- /python/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/setup.py -------------------------------------------------------------------------------- /python/src/sentencepiece/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/src/sentencepiece/__init__.py -------------------------------------------------------------------------------- /python/src/sentencepiece/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.2.2' 2 | -------------------------------------------------------------------------------- /python/src/sentencepiece/sentencepiece.i: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/src/sentencepiece/sentencepiece.i -------------------------------------------------------------------------------- /python/src/sentencepiece/sentencepiece_model_pb2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/src/sentencepiece/sentencepiece_model_pb2.py -------------------------------------------------------------------------------- /python/src/sentencepiece/sentencepiece_pb2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/src/sentencepiece/sentencepiece_pb2.py -------------------------------------------------------------------------------- /python/src/sentencepiece/sentencepiece_wrap.cxx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/src/sentencepiece/sentencepiece_wrap.cxx -------------------------------------------------------------------------------- /python/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/test/botchan.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/test/botchan.txt -------------------------------------------------------------------------------- /python/test/sentencepiece_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/test/sentencepiece_test.py -------------------------------------------------------------------------------- /python/test/test_ja_model.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/test/test_ja_model.model -------------------------------------------------------------------------------- /python/test/test_model.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/python/test/test_model.model -------------------------------------------------------------------------------- /sentencepiece.pc.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/sentencepiece.pc.in -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/CMakeLists.txt -------------------------------------------------------------------------------- /src/bpe_model.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/bpe_model.cc -------------------------------------------------------------------------------- /src/bpe_model.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/bpe_model.h -------------------------------------------------------------------------------- /src/bpe_model_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/bpe_model_test.cc -------------------------------------------------------------------------------- /src/bpe_model_trainer.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/bpe_model_trainer.cc -------------------------------------------------------------------------------- /src/bpe_model_trainer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/bpe_model_trainer.h -------------------------------------------------------------------------------- /src/bpe_model_trainer_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/bpe_model_trainer_test.cc -------------------------------------------------------------------------------- /src/builder.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/builder.cc -------------------------------------------------------------------------------- /src/builder.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/builder.h -------------------------------------------------------------------------------- /src/builder_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/builder_test.cc -------------------------------------------------------------------------------- /src/builtin_pb/sentencepiece.pb.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/builtin_pb/sentencepiece.pb.cc -------------------------------------------------------------------------------- /src/builtin_pb/sentencepiece.pb.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/builtin_pb/sentencepiece.pb.h -------------------------------------------------------------------------------- /src/builtin_pb/sentencepiece_model.pb.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/builtin_pb/sentencepiece_model.pb.cc -------------------------------------------------------------------------------- /src/builtin_pb/sentencepiece_model.pb.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/builtin_pb/sentencepiece_model.pb.h -------------------------------------------------------------------------------- /src/char_model.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/char_model.cc -------------------------------------------------------------------------------- /src/char_model.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/char_model.h -------------------------------------------------------------------------------- /src/char_model_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/char_model_test.cc -------------------------------------------------------------------------------- /src/char_model_trainer.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/char_model_trainer.cc -------------------------------------------------------------------------------- /src/char_model_trainer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/char_model_trainer.h -------------------------------------------------------------------------------- /src/char_model_trainer_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/char_model_trainer_test.cc -------------------------------------------------------------------------------- /src/common.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/common.h -------------------------------------------------------------------------------- /src/compile_charsmap_main.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/compile_charsmap_main.cc -------------------------------------------------------------------------------- /src/error.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/error.cc -------------------------------------------------------------------------------- /src/filesystem.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/filesystem.cc -------------------------------------------------------------------------------- /src/filesystem.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/filesystem.h -------------------------------------------------------------------------------- /src/filesystem_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/filesystem_test.cc -------------------------------------------------------------------------------- /src/freelist.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/freelist.h -------------------------------------------------------------------------------- /src/freelist_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/freelist_test.cc -------------------------------------------------------------------------------- /src/init.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/init.cc -------------------------------------------------------------------------------- /src/init.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/init.h -------------------------------------------------------------------------------- /src/init_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/init_test.cc -------------------------------------------------------------------------------- /src/model_factory.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/model_factory.cc -------------------------------------------------------------------------------- /src/model_factory.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/model_factory.h -------------------------------------------------------------------------------- /src/model_factory_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/model_factory_test.cc -------------------------------------------------------------------------------- /src/model_interface.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/model_interface.cc -------------------------------------------------------------------------------- /src/model_interface.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/model_interface.h -------------------------------------------------------------------------------- /src/model_interface_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/model_interface_test.cc -------------------------------------------------------------------------------- /src/normalization_rule.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/normalization_rule.h -------------------------------------------------------------------------------- /src/normalizer.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/normalizer.cc -------------------------------------------------------------------------------- /src/normalizer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/normalizer.h -------------------------------------------------------------------------------- /src/normalizer_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/normalizer_test.cc -------------------------------------------------------------------------------- /src/pretokenizer_for_training.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/pretokenizer_for_training.cc -------------------------------------------------------------------------------- /src/pretokenizer_for_training.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/pretokenizer_for_training.h -------------------------------------------------------------------------------- /src/pretokenizer_for_training_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/pretokenizer_for_training_test.cc -------------------------------------------------------------------------------- /src/sentencepiece.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/sentencepiece.proto -------------------------------------------------------------------------------- /src/sentencepiece_model.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/sentencepiece_model.proto -------------------------------------------------------------------------------- /src/sentencepiece_processor.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/sentencepiece_processor.cc -------------------------------------------------------------------------------- /src/sentencepiece_processor.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/sentencepiece_processor.h -------------------------------------------------------------------------------- /src/sentencepiece_processor_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/sentencepiece_processor_test.cc -------------------------------------------------------------------------------- /src/sentencepiece_trainer.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/sentencepiece_trainer.cc -------------------------------------------------------------------------------- /src/sentencepiece_trainer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/sentencepiece_trainer.h -------------------------------------------------------------------------------- /src/sentencepiece_trainer_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/sentencepiece_trainer_test.cc -------------------------------------------------------------------------------- /src/spec_parser.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/spec_parser.h -------------------------------------------------------------------------------- /src/spm_decode_main.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/spm_decode_main.cc -------------------------------------------------------------------------------- /src/spm_encode_main.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/spm_encode_main.cc -------------------------------------------------------------------------------- /src/spm_export_vocab_main.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/spm_export_vocab_main.cc -------------------------------------------------------------------------------- /src/spm_normalize_main.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/spm_normalize_main.cc -------------------------------------------------------------------------------- /src/spm_train_main.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/spm_train_main.cc -------------------------------------------------------------------------------- /src/test_main.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/test_main.cc -------------------------------------------------------------------------------- /src/testharness.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/testharness.cc -------------------------------------------------------------------------------- /src/testharness.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/testharness.h -------------------------------------------------------------------------------- /src/trainer_factory.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/trainer_factory.cc -------------------------------------------------------------------------------- /src/trainer_factory.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/trainer_factory.h -------------------------------------------------------------------------------- /src/trainer_factory_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/trainer_factory_test.cc -------------------------------------------------------------------------------- /src/trainer_interface.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/trainer_interface.cc -------------------------------------------------------------------------------- /src/trainer_interface.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/trainer_interface.h -------------------------------------------------------------------------------- /src/trainer_interface_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/trainer_interface_test.cc -------------------------------------------------------------------------------- /src/unicode_script.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/unicode_script.cc -------------------------------------------------------------------------------- /src/unicode_script.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/unicode_script.h -------------------------------------------------------------------------------- /src/unicode_script_map.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/unicode_script_map.h -------------------------------------------------------------------------------- /src/unicode_script_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/unicode_script_test.cc -------------------------------------------------------------------------------- /src/unigram_model.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/unigram_model.cc -------------------------------------------------------------------------------- /src/unigram_model.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/unigram_model.h -------------------------------------------------------------------------------- /src/unigram_model_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/unigram_model_test.cc -------------------------------------------------------------------------------- /src/unigram_model_trainer.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/unigram_model_trainer.cc -------------------------------------------------------------------------------- /src/unigram_model_trainer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/unigram_model_trainer.h -------------------------------------------------------------------------------- /src/unigram_model_trainer_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/unigram_model_trainer_test.cc -------------------------------------------------------------------------------- /src/util.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/util.cc -------------------------------------------------------------------------------- /src/util.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/util.h -------------------------------------------------------------------------------- /src/util_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/util_test.cc -------------------------------------------------------------------------------- /src/word_model.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/word_model.cc -------------------------------------------------------------------------------- /src/word_model.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/word_model.h -------------------------------------------------------------------------------- /src/word_model_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/word_model_test.cc -------------------------------------------------------------------------------- /src/word_model_trainer.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/word_model_trainer.cc -------------------------------------------------------------------------------- /src/word_model_trainer.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/word_model_trainer.h -------------------------------------------------------------------------------- /src/word_model_trainer_test.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/src/word_model_trainer_test.cc -------------------------------------------------------------------------------- /third_party/CMakeLists.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/CMakeLists.txt -------------------------------------------------------------------------------- /third_party/absl/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/LICENSE -------------------------------------------------------------------------------- /third_party/absl/container/btree_set.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/container/btree_set.h -------------------------------------------------------------------------------- /third_party/absl/container/flat_hash_map.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/container/flat_hash_map.h -------------------------------------------------------------------------------- /third_party/absl/container/flat_hash_set.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/container/flat_hash_set.h -------------------------------------------------------------------------------- /third_party/absl/flags/flag.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/flags/flag.cc -------------------------------------------------------------------------------- /third_party/absl/flags/flag.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/flags/flag.h -------------------------------------------------------------------------------- /third_party/absl/flags/parse.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/flags/parse.h -------------------------------------------------------------------------------- /third_party/absl/flags/usage.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/flags/usage.h -------------------------------------------------------------------------------- /third_party/absl/flags/usage_config.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/flags/usage_config.h -------------------------------------------------------------------------------- /third_party/absl/log/check.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/log/check.h -------------------------------------------------------------------------------- /third_party/absl/log/globals.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/log/globals.h -------------------------------------------------------------------------------- /third_party/absl/log/log.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/log/log.cc -------------------------------------------------------------------------------- /third_party/absl/log/log.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/log/log.h -------------------------------------------------------------------------------- /third_party/absl/strings/ascii.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/strings/ascii.h -------------------------------------------------------------------------------- /third_party/absl/strings/match.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/strings/match.h -------------------------------------------------------------------------------- /third_party/absl/strings/numbers.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/strings/numbers.h -------------------------------------------------------------------------------- /third_party/absl/strings/str_cat.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/strings/str_cat.h -------------------------------------------------------------------------------- /third_party/absl/strings/str_format.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/strings/str_format.h -------------------------------------------------------------------------------- /third_party/absl/strings/str_join.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/strings/str_join.h -------------------------------------------------------------------------------- /third_party/absl/strings/str_replace.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/strings/str_replace.h -------------------------------------------------------------------------------- /third_party/absl/strings/str_split.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/strings/str_split.h -------------------------------------------------------------------------------- /third_party/absl/strings/string_view.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/strings/string_view.h -------------------------------------------------------------------------------- /third_party/absl/strings/strip.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/absl/strings/strip.h -------------------------------------------------------------------------------- /third_party/darts_clone/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/darts_clone/LICENSE -------------------------------------------------------------------------------- /third_party/darts_clone/darts.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/darts_clone/darts.h -------------------------------------------------------------------------------- /third_party/esaxx/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/esaxx/LICENSE -------------------------------------------------------------------------------- /third_party/esaxx/esa.hxx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/esaxx/esa.hxx -------------------------------------------------------------------------------- /third_party/esaxx/sais.hxx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/esaxx/sais.hxx -------------------------------------------------------------------------------- /third_party/protobuf-lite/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/LICENSE -------------------------------------------------------------------------------- /third_party/protobuf-lite/arena.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/arena.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/arenastring.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/arenastring.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/bytestream.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/bytestream.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/coded_stream.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/coded_stream.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/common.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/common.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/extension_set.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/extension_set.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/generated_enum_util.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/generated_enum_util.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/generated_message_table_driven_lite.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/generated_message_table_driven_lite.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/generated_message_util.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/generated_message_util.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/any.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/any.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/arena.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/arena.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/arena_impl.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/arena_impl.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/arenastring.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/arenastring.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/descriptor.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/descriptor.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/extension_set.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/extension_set.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/extension_set_inl.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/extension_set_inl.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/generated_enum_reflection.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/generated_enum_reflection.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/generated_enum_util.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/generated_enum_util.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/generated_message_table_driven.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/generated_message_table_driven.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/generated_message_table_driven_lite.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/generated_message_table_driven_lite.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/generated_message_util.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/generated_message_util.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/has_bits.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/has_bits.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/implicit_weak_message.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/implicit_weak_message.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/io/coded_stream.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/io/coded_stream.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/io/io_win32.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/io/io_win32.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/io/zero_copy_stream.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl_lite.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl_lite.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/map.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/map.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/map_entry_lite.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/map_entry_lite.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/map_field_lite.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/map_field_lite.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/map_type_handler.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/map_type_handler.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/message_lite.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/message_lite.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/metadata_lite.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/metadata_lite.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/parse_context.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/parse_context.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/port.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/port.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/port_def.inc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/port_def.inc -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/port_undef.inc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/port_undef.inc -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/repeated_field.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/repeated_field.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/bytestream.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/bytestream.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/callback.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/callback.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/casts.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/casts.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/common.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/common.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/hash.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/hash.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/int128.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/int128.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/logging.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/logging.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/macros.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/macros.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/map_util.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/map_util.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/mutex.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/mutex.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/once.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/once.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/platform_macros.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/platform_macros.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/port.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/port.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/status.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/status.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/statusor.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/statusor.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/stl_util.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/stl_util.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/stringpiece.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/stringpiece.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/stringprintf.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/stringprintf.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/strutil.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/strutil.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/stubs/time.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/stubs/time.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/unknown_field_set.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/unknown_field_set.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/google/protobuf/wire_format_lite.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/google/protobuf/wire_format_lite.h -------------------------------------------------------------------------------- /third_party/protobuf-lite/implicit_weak_message.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/implicit_weak_message.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/int128.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/int128.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/io_win32.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/io_win32.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/message_lite.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/message_lite.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/parse_context.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/parse_context.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/repeated_field.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/repeated_field.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/status.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/status.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/statusor.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/statusor.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/stringpiece.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/stringpiece.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/stringprintf.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/stringprintf.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/structurally_valid.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/structurally_valid.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/strutil.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/strutil.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/time.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/time.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/wire_format_lite.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/wire_format_lite.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/zero_copy_stream.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/zero_copy_stream.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/zero_copy_stream_impl.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/zero_copy_stream_impl.cc -------------------------------------------------------------------------------- /third_party/protobuf-lite/zero_copy_stream_impl_lite.cc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sentencepiece/HEAD/third_party/protobuf-lite/zero_copy_stream_impl_lite.cc --------------------------------------------------------------------------------