├── .gitignore ├── LICENSE ├── README.md ├── ext ├── data │ ├── bert_data │ │ └── .gitignore │ ├── json_data │ │ └── .gitignore │ └── raw │ │ └── .gitignore ├── logs │ └── .gitignore ├── models │ └── .gitignore └── results │ └── .gitignore ├── main.py ├── requirements.txt ├── requirements_prepro.txt ├── src ├── distributed.py ├── make_data.py ├── make_submission.py ├── models │ ├── __init__.py │ ├── adam.py │ ├── data_loader.py │ ├── decoder.py │ ├── encoder.py │ ├── loss.py │ ├── model_builder.py │ ├── neural.py │ ├── optimizers.py │ ├── reporter.py │ ├── reporter_ext.py │ ├── trainer.py │ └── trainer_ext.py ├── others │ ├── __init__.py │ ├── install_mecab.py │ ├── logging.py │ ├── pyrouge.py │ ├── rouge_metric.py │ ├── test_rouge_score.py │ └── utils.py ├── prepro │ ├── __init__.py │ ├── data_builder.py │ ├── smart_common_words.txt │ ├── tokenization_kobert.py │ └── utils.py ├── preprocess.py ├── train.py ├── train_extractive.py └── translate │ ├── __init__.py │ ├── beam.py │ └── penalties.py ├── temp ├── src │ ├── cal_rouge.py │ ├── models │ │ └── predictor.py │ ├── post_stats.py │ ├── prepo │ │ └── tokenization.py │ └── train_abstractive.py └── urls │ ├── cnn_mapping_test.txt │ ├── cnn_mapping_train.txt │ ├── cnn_mapping_valid.txt │ ├── mapping_test.txt │ ├── mapping_train.txt │ └── mapping_valid.txt └── tutorials ├── .ipynb_checkpoints ├── EDA_hangil-checkpoint.ipynb └── PreSumm_preprocessing-checkpoint.ipynb ├── EDA.ipynb ├── PreSumm_preprocessing.ipynb ├── PreSumm_train.ipynb ├── images └── BERTSUM_structure.PNG └── used_tokens_ids_kobert.txt /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | __pycache__ 3 | tutorials/.ipynb_checkpoints/ 4 | # temp/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/README.md -------------------------------------------------------------------------------- /ext/data/bert_data/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /ext/data/json_data/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /ext/data/raw/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /ext/logs/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /ext/models/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /ext/results/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/main.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/requirements.txt -------------------------------------------------------------------------------- /requirements_prepro.txt: -------------------------------------------------------------------------------- 1 | #Cython 2 | kss 3 | bs4 4 | pandas 5 | 6 | # test 7 | # py-rouge 8 | nltk 9 | konlpy -------------------------------------------------------------------------------- /src/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/distributed.py -------------------------------------------------------------------------------- /src/make_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/make_data.py -------------------------------------------------------------------------------- /src/make_submission.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/make_submission.py -------------------------------------------------------------------------------- /src/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/models/adam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/adam.py -------------------------------------------------------------------------------- /src/models/data_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/data_loader.py -------------------------------------------------------------------------------- /src/models/decoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/decoder.py -------------------------------------------------------------------------------- /src/models/encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/encoder.py -------------------------------------------------------------------------------- /src/models/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/loss.py -------------------------------------------------------------------------------- /src/models/model_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/model_builder.py -------------------------------------------------------------------------------- /src/models/neural.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/neural.py -------------------------------------------------------------------------------- /src/models/optimizers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/optimizers.py -------------------------------------------------------------------------------- /src/models/reporter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/reporter.py -------------------------------------------------------------------------------- /src/models/reporter_ext.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/reporter_ext.py -------------------------------------------------------------------------------- /src/models/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/trainer.py -------------------------------------------------------------------------------- /src/models/trainer_ext.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/models/trainer_ext.py -------------------------------------------------------------------------------- /src/others/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/others/install_mecab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/others/install_mecab.py -------------------------------------------------------------------------------- /src/others/logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/others/logging.py -------------------------------------------------------------------------------- /src/others/pyrouge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/others/pyrouge.py -------------------------------------------------------------------------------- /src/others/rouge_metric.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/others/rouge_metric.py -------------------------------------------------------------------------------- /src/others/test_rouge_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/others/test_rouge_score.py -------------------------------------------------------------------------------- /src/others/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/others/utils.py -------------------------------------------------------------------------------- /src/prepro/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/prepro/data_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/prepro/data_builder.py -------------------------------------------------------------------------------- /src/prepro/smart_common_words.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/prepro/smart_common_words.txt -------------------------------------------------------------------------------- /src/prepro/tokenization_kobert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/prepro/tokenization_kobert.py -------------------------------------------------------------------------------- /src/prepro/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/prepro/utils.py -------------------------------------------------------------------------------- /src/preprocess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/preprocess.py -------------------------------------------------------------------------------- /src/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/train.py -------------------------------------------------------------------------------- /src/train_extractive.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/train_extractive.py -------------------------------------------------------------------------------- /src/translate/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/translate/beam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/translate/beam.py -------------------------------------------------------------------------------- /src/translate/penalties.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/src/translate/penalties.py -------------------------------------------------------------------------------- /temp/src/cal_rouge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/temp/src/cal_rouge.py -------------------------------------------------------------------------------- /temp/src/models/predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/temp/src/models/predictor.py -------------------------------------------------------------------------------- /temp/src/post_stats.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/temp/src/post_stats.py -------------------------------------------------------------------------------- /temp/src/prepo/tokenization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/temp/src/prepo/tokenization.py -------------------------------------------------------------------------------- /temp/src/train_abstractive.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/temp/src/train_abstractive.py -------------------------------------------------------------------------------- /temp/urls/cnn_mapping_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/temp/urls/cnn_mapping_test.txt -------------------------------------------------------------------------------- /temp/urls/cnn_mapping_train.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/temp/urls/cnn_mapping_train.txt -------------------------------------------------------------------------------- /temp/urls/cnn_mapping_valid.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/temp/urls/cnn_mapping_valid.txt -------------------------------------------------------------------------------- /temp/urls/mapping_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/temp/urls/mapping_test.txt -------------------------------------------------------------------------------- /temp/urls/mapping_train.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/temp/urls/mapping_train.txt -------------------------------------------------------------------------------- /temp/urls/mapping_valid.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/temp/urls/mapping_valid.txt -------------------------------------------------------------------------------- /tutorials/.ipynb_checkpoints/EDA_hangil-checkpoint.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/tutorials/.ipynb_checkpoints/EDA_hangil-checkpoint.ipynb -------------------------------------------------------------------------------- /tutorials/.ipynb_checkpoints/PreSumm_preprocessing-checkpoint.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/tutorials/.ipynb_checkpoints/PreSumm_preprocessing-checkpoint.ipynb -------------------------------------------------------------------------------- /tutorials/EDA.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/tutorials/EDA.ipynb -------------------------------------------------------------------------------- /tutorials/PreSumm_preprocessing.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/tutorials/PreSumm_preprocessing.ipynb -------------------------------------------------------------------------------- /tutorials/PreSumm_train.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/tutorials/PreSumm_train.ipynb -------------------------------------------------------------------------------- /tutorials/images/BERTSUM_structure.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/tutorials/images/BERTSUM_structure.PNG -------------------------------------------------------------------------------- /tutorials/used_tokens_ids_kobert.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uoneway/KoBertSum/HEAD/tutorials/used_tokens_ids_kobert.txt --------------------------------------------------------------------------------