├── .gitignore ├── README.md ├── data ├── 2020-07-29_covid_news_sents.txt └── very_small_alphabet_corpus.txt ├── examples ├── README.md ├── bert_config.json └── run_konlpy_bert.py ├── huggingface_konlpy ├── __init__.py ├── about.py ├── tokenizers_konlpy │ ├── __init__.py │ ├── pretokenizers.py │ └── tokenizers.py ├── transformers_konlpy │ ├── __init__.py │ └── pretrained_tokenizers.py └── utils.py ├── requirements.txt ├── setup.py └── tutorials ├── 00_huggingface_tokenizers_usage.ipynb ├── 01_huggingface_konlpy_usage.ipynb └── tokenizers └── BertStyleMecab ├── notag-vocab.txt ├── notag-vocab_3000.txt └── usetag-vocab.txt /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/README.md -------------------------------------------------------------------------------- /data/2020-07-29_covid_news_sents.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/data/2020-07-29_covid_news_sents.txt -------------------------------------------------------------------------------- /data/very_small_alphabet_corpus.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/data/very_small_alphabet_corpus.txt -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/examples/README.md -------------------------------------------------------------------------------- /examples/bert_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/examples/bert_config.json -------------------------------------------------------------------------------- /examples/run_konlpy_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/examples/run_konlpy_bert.py -------------------------------------------------------------------------------- /huggingface_konlpy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/huggingface_konlpy/__init__.py -------------------------------------------------------------------------------- /huggingface_konlpy/about.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/huggingface_konlpy/about.py -------------------------------------------------------------------------------- /huggingface_konlpy/tokenizers_konlpy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/huggingface_konlpy/tokenizers_konlpy/__init__.py -------------------------------------------------------------------------------- /huggingface_konlpy/tokenizers_konlpy/pretokenizers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/huggingface_konlpy/tokenizers_konlpy/pretokenizers.py -------------------------------------------------------------------------------- /huggingface_konlpy/tokenizers_konlpy/tokenizers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/huggingface_konlpy/tokenizers_konlpy/tokenizers.py -------------------------------------------------------------------------------- /huggingface_konlpy/transformers_konlpy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/huggingface_konlpy/transformers_konlpy/__init__.py -------------------------------------------------------------------------------- /huggingface_konlpy/transformers_konlpy/pretrained_tokenizers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/huggingface_konlpy/transformers_konlpy/pretrained_tokenizers.py -------------------------------------------------------------------------------- /huggingface_konlpy/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/huggingface_konlpy/utils.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/requirements.txt -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/setup.py -------------------------------------------------------------------------------- /tutorials/00_huggingface_tokenizers_usage.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/tutorials/00_huggingface_tokenizers_usage.ipynb -------------------------------------------------------------------------------- /tutorials/01_huggingface_konlpy_usage.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/tutorials/01_huggingface_konlpy_usage.ipynb -------------------------------------------------------------------------------- /tutorials/tokenizers/BertStyleMecab/notag-vocab.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/tutorials/tokenizers/BertStyleMecab/notag-vocab.txt -------------------------------------------------------------------------------- /tutorials/tokenizers/BertStyleMecab/notag-vocab_3000.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/tutorials/tokenizers/BertStyleMecab/notag-vocab_3000.txt -------------------------------------------------------------------------------- /tutorials/tokenizers/BertStyleMecab/usetag-vocab.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lovit/huggingface_konlpy/HEAD/tutorials/tokenizers/BertStyleMecab/usetag-vocab.txt --------------------------------------------------------------------------------