├── .circleci └── config.yml ├── .flake8 ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── examples ├── README.md ├── __init__.py ├── entity_disambiguation │ ├── README.md │ ├── dataloader.py │ ├── dataset.py │ ├── deepspeed_config │ │ ├── stage1.json │ │ └── stage2.json │ ├── evaluate.py │ ├── model.py │ ├── scripts │ │ ├── convert_checkpoint.py │ │ ├── create_candidate_data.py │ │ ├── create_redirect_data.py │ │ └── create_title_data.py │ └── train.py ├── entity_typing │ ├── README.md │ ├── __init__.py │ ├── configs │ │ ├── lib │ │ │ ├── base.libsonnet │ │ │ ├── transformers_model.jsonnet │ │ │ ├── transformers_model_luke.jsonnet │ │ │ └── transformers_model_luke_with_entity_aware_attention.jsonnet │ │ ├── transformers.jsonnet │ │ ├── transformers_luke.jsonnet │ │ └── transformers_luke_with_entity_aware_attention.jsonnet │ ├── evaluate_transformers_checkpoint.py │ ├── model.py │ ├── modules │ │ ├── __init__.py │ │ ├── entity_based.py │ │ ├── feature_extractor.py │ │ └── token_based.py │ ├── reader.py │ └── tests │ │ ├── __init__.py │ │ ├── fixtures │ │ └── test.json │ │ └── test_dataset_reader.py ├── legacy │ ├── README.md │ ├── __init__.py │ ├── cli.py │ ├── entity_span_qa │ │ ├── __init__.py │ │ ├── main.py │ │ ├── model.py │ │ ├── record_eval.py │ │ └── utils.py │ ├── entity_typing │ │ ├── __init__.py │ │ ├── main.py │ │ ├── model.py │ │ └── utils.py │ ├── ner │ │ ├── __init__.py │ │ ├── main.py │ │ ├── model.py │ │ └── utils.py │ ├── reading_comprehension │ │ ├── __init__.py │ │ ├── main.py │ │ ├── model.py │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── dataset.py │ │ │ ├── feature.py │ │ │ ├── result_writer.py │ │ │ ├── squad_eval.py │ │ │ └── wiki_link_db.py │ ├── relation_classification │ │ ├── __init__.py │ │ ├── main.py │ │ ├── model.py │ │ └── utils.py │ └── utils │ │ ├── __init__.py │ │ ├── experiment_logger.py │ │ ├── mention_db.py │ │ └── trainer.py ├── ner │ ├── README.md │ ├── __init__.py │ ├── configs │ │ ├── lib │ │ │ ├── base.libsonnet │ │ │ ├── transformers_model.jsonnet │ │ │ ├── transformers_model_luke.jsonnet │ │ │ └── transformers_model_luke_with_entity_aware_attention.jsonnet │ │ ├── transformers.jsonnet │ │ ├── transformers_luke.jsonnet │ │ └── transformers_luke_with_entity_aware_attention.jsonnet │ ├── convert_allennlp_to_huggingface_model.py │ ├── evaluate_transformers_checkpoint.py │ ├── metrics │ │ ├── __init__.py │ │ └── span_to_label_f1.py │ ├── model.py │ ├── modules │ │ ├── __init__.py │ │ ├── entity_based.py │ │ ├── feature_extractor.py │ │ ├── token_and_entity.py │ │ └── token_based.py │ ├── reader.py │ └── utils │ │ └── convert_io_to_bio_format.py ├── reading_comprehension │ ├── README.md │ ├── __init__.py │ ├── configs_squad │ │ ├── lib │ │ │ ├── base.libsonnet │ │ │ ├── dataset_reader.jsonnet │ │ │ ├── dataset_reader_with_entity.jsonnet │ │ │ ├── transformers_model.jsonnet │ │ │ ├── transformers_model_luke.jsonnet │ │ │ └── transformers_model_luke_with_entity_aware_attention.jsonnet │ │ ├── transformers.jsonnet │ │ ├── transformers_luke.jsonnet │ │ └── transformers_luke_with_entity_aware_attention.jsonnet │ ├── evaluate_qa.py │ ├── mention_candidates │ │ └── squad │ │ │ └── train-dev-v1.1.json │ ├── metrics │ │ ├── __init__.py │ │ ├── qa_metric.py │ │ └── squad │ │ │ ├── __init__.py │ │ │ ├── answer_string.py │ │ │ ├── mlqa_extensions │ │ │ ├── __init__.py │ │ │ ├── mlqa_answer_string.py │ │ │ └── mlqa_languages.py │ │ │ ├── squad_evaluate.py │ │ │ └── squad_metric.py │ ├── model.py │ └── readers │ │ ├── __init__.py │ │ └── squad_reader.py ├── relation_classification │ ├── README.md │ ├── __init__.py │ ├── configs │ │ ├── lib │ │ │ ├── base.libsonnet │ │ │ ├── transformers_model.jsonnet │ │ │ ├── transformers_model_luke.jsonnet │ │ │ └── transformers_model_luke_with_entity_aware_attention.jsonnet │ │ ├── transformers.jsonnet │ │ ├── transformers_luke.jsonnet │ │ └── transformers_luke_with_entity_aware_attention.jsonnet │ ├── convert_allennlp_to_huggingface_model.py │ ├── evaluate_transformers_checkpoint.py │ ├── luke_for_entity_pair_classification.py │ ├── metrics │ │ └── multiway_f1.py │ ├── model.py │ ├── modules │ │ ├── __init__.py │ │ ├── entity_based.py │ │ ├── feature_extractor.py │ │ ├── token_based.py │ │ └── token_entity_based.py │ └── reader.py └── utils │ ├── __init__.py │ ├── embedders │ ├── __init__.py │ ├── scalar_mix_transoformer_embedder.py │ └── transformers_luke_embedder.py │ ├── learning_rate_schedulers │ ├── __init__.py │ └── linear_with_warmup.py │ ├── span_utils.py │ ├── util.py │ └── wiki_entity_linker │ ├── __init__.py │ ├── entity_db.py │ ├── make_mention_candidates_from_squad.py │ ├── mention_candidate_generator.py │ ├── mention_candidate_translator.py │ ├── mention_db.py │ ├── translate_mention_candidates.py │ ├── wiki_entity_linker.py │ └── wiki_link_db.py ├── luke ├── __init__.py ├── cli.py ├── model.py ├── pretraining │ ├── __init__.py │ ├── batch_generator.py │ ├── dataset.py │ ├── metrics.py │ ├── model.py │ ├── tokenization.py │ └── train.py └── utils │ ├── __init__.py │ ├── convert_luke_to_huggingface_model.py │ ├── entity_vocab.py │ ├── interwiki_db.py │ ├── model_utils.py │ ├── resources │ ├── en-sent.bin │ ├── opennlp-maxent-3.0.3.jar │ └── opennlp-tools-1.5.3.jar │ ├── sentence_splitter.py │ └── util.py ├── notebooks ├── huggingface_conll_2003.ipynb ├── huggingface_open_entity.ipynb └── huggingface_tacred.ipynb ├── poetry.lock ├── poetry_coling2020.lock ├── poetry_emnlp2020.lock ├── pretraining.md ├── pretraining_config ├── luke_base_stage1.json ├── luke_base_stage2.json ├── luke_large_stage1.json ├── luke_large_stage2.json ├── mluke_base_stage1.json ├── mluke_base_stage2.json ├── mluke_large_stage1.json └── mluke_large_stage2.json ├── pyproject.toml ├── pytest.ini ├── requirements.txt ├── resources └── luke_logo.png └── tests ├── __init__.py ├── fixtures ├── dummy_entity_vocab.tsv ├── en_ja_multilingual_vocab_test.jsonl ├── enwiki_20181220_entvocab_100.tsv └── wikidata_20180423_sitelinks10.json ├── pretraining ├── __init__.py ├── dummy_dump_db.py └── test_dataset.py ├── test_model.py ├── test_tokenization.py └── utils ├── __init__.py ├── test_entity_vocab.py ├── test_interwiki_db.py └── test_sentence_splitter.py /.circleci/config.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/.circleci/config.yml -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/.flake8 -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/README.md -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/README.md -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/entity_disambiguation/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/README.md -------------------------------------------------------------------------------- /examples/entity_disambiguation/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/dataloader.py -------------------------------------------------------------------------------- /examples/entity_disambiguation/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/dataset.py -------------------------------------------------------------------------------- /examples/entity_disambiguation/deepspeed_config/stage1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/deepspeed_config/stage1.json -------------------------------------------------------------------------------- /examples/entity_disambiguation/deepspeed_config/stage2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/deepspeed_config/stage2.json -------------------------------------------------------------------------------- /examples/entity_disambiguation/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/evaluate.py -------------------------------------------------------------------------------- /examples/entity_disambiguation/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/model.py -------------------------------------------------------------------------------- /examples/entity_disambiguation/scripts/convert_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/scripts/convert_checkpoint.py -------------------------------------------------------------------------------- /examples/entity_disambiguation/scripts/create_candidate_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/scripts/create_candidate_data.py -------------------------------------------------------------------------------- /examples/entity_disambiguation/scripts/create_redirect_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/scripts/create_redirect_data.py -------------------------------------------------------------------------------- /examples/entity_disambiguation/scripts/create_title_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/scripts/create_title_data.py -------------------------------------------------------------------------------- /examples/entity_disambiguation/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_disambiguation/train.py -------------------------------------------------------------------------------- /examples/entity_typing/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/README.md -------------------------------------------------------------------------------- /examples/entity_typing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/entity_typing/configs/lib/base.libsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/configs/lib/base.libsonnet -------------------------------------------------------------------------------- /examples/entity_typing/configs/lib/transformers_model.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/configs/lib/transformers_model.jsonnet -------------------------------------------------------------------------------- /examples/entity_typing/configs/lib/transformers_model_luke.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/configs/lib/transformers_model_luke.jsonnet -------------------------------------------------------------------------------- /examples/entity_typing/configs/lib/transformers_model_luke_with_entity_aware_attention.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/configs/lib/transformers_model_luke_with_entity_aware_attention.jsonnet -------------------------------------------------------------------------------- /examples/entity_typing/configs/transformers.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/configs/transformers.jsonnet -------------------------------------------------------------------------------- /examples/entity_typing/configs/transformers_luke.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/configs/transformers_luke.jsonnet -------------------------------------------------------------------------------- /examples/entity_typing/configs/transformers_luke_with_entity_aware_attention.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/configs/transformers_luke_with_entity_aware_attention.jsonnet -------------------------------------------------------------------------------- /examples/entity_typing/evaluate_transformers_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/evaluate_transformers_checkpoint.py -------------------------------------------------------------------------------- /examples/entity_typing/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/model.py -------------------------------------------------------------------------------- /examples/entity_typing/modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/entity_typing/modules/entity_based.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/modules/entity_based.py -------------------------------------------------------------------------------- /examples/entity_typing/modules/feature_extractor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/modules/feature_extractor.py -------------------------------------------------------------------------------- /examples/entity_typing/modules/token_based.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/modules/token_based.py -------------------------------------------------------------------------------- /examples/entity_typing/reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/reader.py -------------------------------------------------------------------------------- /examples/entity_typing/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/entity_typing/tests/fixtures/test.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/tests/fixtures/test.json -------------------------------------------------------------------------------- /examples/entity_typing/tests/test_dataset_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/entity_typing/tests/test_dataset_reader.py -------------------------------------------------------------------------------- /examples/legacy/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/README.md -------------------------------------------------------------------------------- /examples/legacy/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/legacy/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/cli.py -------------------------------------------------------------------------------- /examples/legacy/entity_span_qa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/legacy/entity_span_qa/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/entity_span_qa/main.py -------------------------------------------------------------------------------- /examples/legacy/entity_span_qa/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/entity_span_qa/model.py -------------------------------------------------------------------------------- /examples/legacy/entity_span_qa/record_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/entity_span_qa/record_eval.py -------------------------------------------------------------------------------- /examples/legacy/entity_span_qa/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/entity_span_qa/utils.py -------------------------------------------------------------------------------- /examples/legacy/entity_typing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/legacy/entity_typing/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/entity_typing/main.py -------------------------------------------------------------------------------- /examples/legacy/entity_typing/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/entity_typing/model.py -------------------------------------------------------------------------------- /examples/legacy/entity_typing/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/entity_typing/utils.py -------------------------------------------------------------------------------- /examples/legacy/ner/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/legacy/ner/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/ner/main.py -------------------------------------------------------------------------------- /examples/legacy/ner/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/ner/model.py -------------------------------------------------------------------------------- /examples/legacy/ner/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/ner/utils.py -------------------------------------------------------------------------------- /examples/legacy/reading_comprehension/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/legacy/reading_comprehension/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/reading_comprehension/main.py -------------------------------------------------------------------------------- /examples/legacy/reading_comprehension/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/reading_comprehension/model.py -------------------------------------------------------------------------------- /examples/legacy/reading_comprehension/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/legacy/reading_comprehension/utils/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/reading_comprehension/utils/dataset.py -------------------------------------------------------------------------------- /examples/legacy/reading_comprehension/utils/feature.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/reading_comprehension/utils/feature.py -------------------------------------------------------------------------------- /examples/legacy/reading_comprehension/utils/result_writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/reading_comprehension/utils/result_writer.py -------------------------------------------------------------------------------- /examples/legacy/reading_comprehension/utils/squad_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/reading_comprehension/utils/squad_eval.py -------------------------------------------------------------------------------- /examples/legacy/reading_comprehension/utils/wiki_link_db.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/reading_comprehension/utils/wiki_link_db.py -------------------------------------------------------------------------------- /examples/legacy/relation_classification/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/legacy/relation_classification/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/relation_classification/main.py -------------------------------------------------------------------------------- /examples/legacy/relation_classification/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/relation_classification/model.py -------------------------------------------------------------------------------- /examples/legacy/relation_classification/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/relation_classification/utils.py -------------------------------------------------------------------------------- /examples/legacy/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/utils/__init__.py -------------------------------------------------------------------------------- /examples/legacy/utils/experiment_logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/utils/experiment_logger.py -------------------------------------------------------------------------------- /examples/legacy/utils/mention_db.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/utils/mention_db.py -------------------------------------------------------------------------------- /examples/legacy/utils/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/legacy/utils/trainer.py -------------------------------------------------------------------------------- /examples/ner/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/README.md -------------------------------------------------------------------------------- /examples/ner/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/ner/configs/lib/base.libsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/configs/lib/base.libsonnet -------------------------------------------------------------------------------- /examples/ner/configs/lib/transformers_model.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/configs/lib/transformers_model.jsonnet -------------------------------------------------------------------------------- /examples/ner/configs/lib/transformers_model_luke.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/configs/lib/transformers_model_luke.jsonnet -------------------------------------------------------------------------------- /examples/ner/configs/lib/transformers_model_luke_with_entity_aware_attention.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/configs/lib/transformers_model_luke_with_entity_aware_attention.jsonnet -------------------------------------------------------------------------------- /examples/ner/configs/transformers.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/configs/transformers.jsonnet -------------------------------------------------------------------------------- /examples/ner/configs/transformers_luke.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/configs/transformers_luke.jsonnet -------------------------------------------------------------------------------- /examples/ner/configs/transformers_luke_with_entity_aware_attention.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/configs/transformers_luke_with_entity_aware_attention.jsonnet -------------------------------------------------------------------------------- /examples/ner/convert_allennlp_to_huggingface_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/convert_allennlp_to_huggingface_model.py -------------------------------------------------------------------------------- /examples/ner/evaluate_transformers_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/evaluate_transformers_checkpoint.py -------------------------------------------------------------------------------- /examples/ner/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/ner/metrics/span_to_label_f1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/metrics/span_to_label_f1.py -------------------------------------------------------------------------------- /examples/ner/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/model.py -------------------------------------------------------------------------------- /examples/ner/modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/ner/modules/entity_based.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/modules/entity_based.py -------------------------------------------------------------------------------- /examples/ner/modules/feature_extractor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/modules/feature_extractor.py -------------------------------------------------------------------------------- /examples/ner/modules/token_and_entity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/modules/token_and_entity.py -------------------------------------------------------------------------------- /examples/ner/modules/token_based.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/modules/token_based.py -------------------------------------------------------------------------------- /examples/ner/reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/reader.py -------------------------------------------------------------------------------- /examples/ner/utils/convert_io_to_bio_format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/ner/utils/convert_io_to_bio_format.py -------------------------------------------------------------------------------- /examples/reading_comprehension/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/README.md -------------------------------------------------------------------------------- /examples/reading_comprehension/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/reading_comprehension/configs_squad/lib/base.libsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/configs_squad/lib/base.libsonnet -------------------------------------------------------------------------------- /examples/reading_comprehension/configs_squad/lib/dataset_reader.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/configs_squad/lib/dataset_reader.jsonnet -------------------------------------------------------------------------------- /examples/reading_comprehension/configs_squad/lib/dataset_reader_with_entity.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/configs_squad/lib/dataset_reader_with_entity.jsonnet -------------------------------------------------------------------------------- /examples/reading_comprehension/configs_squad/lib/transformers_model.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/configs_squad/lib/transformers_model.jsonnet -------------------------------------------------------------------------------- /examples/reading_comprehension/configs_squad/lib/transformers_model_luke.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/configs_squad/lib/transformers_model_luke.jsonnet -------------------------------------------------------------------------------- /examples/reading_comprehension/configs_squad/lib/transformers_model_luke_with_entity_aware_attention.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/configs_squad/lib/transformers_model_luke_with_entity_aware_attention.jsonnet -------------------------------------------------------------------------------- /examples/reading_comprehension/configs_squad/transformers.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/configs_squad/transformers.jsonnet -------------------------------------------------------------------------------- /examples/reading_comprehension/configs_squad/transformers_luke.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/configs_squad/transformers_luke.jsonnet -------------------------------------------------------------------------------- /examples/reading_comprehension/configs_squad/transformers_luke_with_entity_aware_attention.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/configs_squad/transformers_luke_with_entity_aware_attention.jsonnet -------------------------------------------------------------------------------- /examples/reading_comprehension/evaluate_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/evaluate_qa.py -------------------------------------------------------------------------------- /examples/reading_comprehension/mention_candidates/squad/train-dev-v1.1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/mention_candidates/squad/train-dev-v1.1.json -------------------------------------------------------------------------------- /examples/reading_comprehension/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/metrics/__init__.py -------------------------------------------------------------------------------- /examples/reading_comprehension/metrics/qa_metric.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/metrics/qa_metric.py -------------------------------------------------------------------------------- /examples/reading_comprehension/metrics/squad/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/reading_comprehension/metrics/squad/answer_string.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/metrics/squad/answer_string.py -------------------------------------------------------------------------------- /examples/reading_comprehension/metrics/squad/mlqa_extensions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/metrics/squad/mlqa_extensions/__init__.py -------------------------------------------------------------------------------- /examples/reading_comprehension/metrics/squad/mlqa_extensions/mlqa_answer_string.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/metrics/squad/mlqa_extensions/mlqa_answer_string.py -------------------------------------------------------------------------------- /examples/reading_comprehension/metrics/squad/mlqa_extensions/mlqa_languages.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/metrics/squad/mlqa_extensions/mlqa_languages.py -------------------------------------------------------------------------------- /examples/reading_comprehension/metrics/squad/squad_evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/metrics/squad/squad_evaluate.py -------------------------------------------------------------------------------- /examples/reading_comprehension/metrics/squad/squad_metric.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/metrics/squad/squad_metric.py -------------------------------------------------------------------------------- /examples/reading_comprehension/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/model.py -------------------------------------------------------------------------------- /examples/reading_comprehension/readers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/reading_comprehension/readers/squad_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/reading_comprehension/readers/squad_reader.py -------------------------------------------------------------------------------- /examples/relation_classification/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/README.md -------------------------------------------------------------------------------- /examples/relation_classification/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/relation_classification/configs/lib/base.libsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/configs/lib/base.libsonnet -------------------------------------------------------------------------------- /examples/relation_classification/configs/lib/transformers_model.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/configs/lib/transformers_model.jsonnet -------------------------------------------------------------------------------- /examples/relation_classification/configs/lib/transformers_model_luke.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/configs/lib/transformers_model_luke.jsonnet -------------------------------------------------------------------------------- /examples/relation_classification/configs/lib/transformers_model_luke_with_entity_aware_attention.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/configs/lib/transformers_model_luke_with_entity_aware_attention.jsonnet -------------------------------------------------------------------------------- /examples/relation_classification/configs/transformers.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/configs/transformers.jsonnet -------------------------------------------------------------------------------- /examples/relation_classification/configs/transformers_luke.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/configs/transformers_luke.jsonnet -------------------------------------------------------------------------------- /examples/relation_classification/configs/transformers_luke_with_entity_aware_attention.jsonnet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/configs/transformers_luke_with_entity_aware_attention.jsonnet -------------------------------------------------------------------------------- /examples/relation_classification/convert_allennlp_to_huggingface_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/convert_allennlp_to_huggingface_model.py -------------------------------------------------------------------------------- /examples/relation_classification/evaluate_transformers_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/evaluate_transformers_checkpoint.py -------------------------------------------------------------------------------- /examples/relation_classification/luke_for_entity_pair_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/luke_for_entity_pair_classification.py -------------------------------------------------------------------------------- /examples/relation_classification/metrics/multiway_f1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/metrics/multiway_f1.py -------------------------------------------------------------------------------- /examples/relation_classification/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/model.py -------------------------------------------------------------------------------- /examples/relation_classification/modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/relation_classification/modules/entity_based.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/modules/entity_based.py -------------------------------------------------------------------------------- /examples/relation_classification/modules/feature_extractor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/modules/feature_extractor.py -------------------------------------------------------------------------------- /examples/relation_classification/modules/token_based.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/modules/token_based.py -------------------------------------------------------------------------------- /examples/relation_classification/modules/token_entity_based.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/modules/token_entity_based.py -------------------------------------------------------------------------------- /examples/relation_classification/reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/relation_classification/reader.py -------------------------------------------------------------------------------- /examples/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/__init__.py -------------------------------------------------------------------------------- /examples/utils/embedders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/utils/embedders/scalar_mix_transoformer_embedder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/embedders/scalar_mix_transoformer_embedder.py -------------------------------------------------------------------------------- /examples/utils/embedders/transformers_luke_embedder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/embedders/transformers_luke_embedder.py -------------------------------------------------------------------------------- /examples/utils/learning_rate_schedulers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/utils/learning_rate_schedulers/linear_with_warmup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/learning_rate_schedulers/linear_with_warmup.py -------------------------------------------------------------------------------- /examples/utils/span_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/span_utils.py -------------------------------------------------------------------------------- /examples/utils/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/util.py -------------------------------------------------------------------------------- /examples/utils/wiki_entity_linker/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/wiki_entity_linker/__init__.py -------------------------------------------------------------------------------- /examples/utils/wiki_entity_linker/entity_db.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/wiki_entity_linker/entity_db.py -------------------------------------------------------------------------------- /examples/utils/wiki_entity_linker/make_mention_candidates_from_squad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/wiki_entity_linker/make_mention_candidates_from_squad.py -------------------------------------------------------------------------------- /examples/utils/wiki_entity_linker/mention_candidate_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/wiki_entity_linker/mention_candidate_generator.py -------------------------------------------------------------------------------- /examples/utils/wiki_entity_linker/mention_candidate_translator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/wiki_entity_linker/mention_candidate_translator.py -------------------------------------------------------------------------------- /examples/utils/wiki_entity_linker/mention_db.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/wiki_entity_linker/mention_db.py -------------------------------------------------------------------------------- /examples/utils/wiki_entity_linker/translate_mention_candidates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/wiki_entity_linker/translate_mention_candidates.py -------------------------------------------------------------------------------- /examples/utils/wiki_entity_linker/wiki_entity_linker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/wiki_entity_linker/wiki_entity_linker.py -------------------------------------------------------------------------------- /examples/utils/wiki_entity_linker/wiki_link_db.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/examples/utils/wiki_entity_linker/wiki_link_db.py -------------------------------------------------------------------------------- /luke/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/__init__.py -------------------------------------------------------------------------------- /luke/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/cli.py -------------------------------------------------------------------------------- /luke/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/model.py -------------------------------------------------------------------------------- /luke/pretraining/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /luke/pretraining/batch_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/pretraining/batch_generator.py -------------------------------------------------------------------------------- /luke/pretraining/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/pretraining/dataset.py -------------------------------------------------------------------------------- /luke/pretraining/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/pretraining/metrics.py -------------------------------------------------------------------------------- /luke/pretraining/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/pretraining/model.py -------------------------------------------------------------------------------- /luke/pretraining/tokenization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/pretraining/tokenization.py -------------------------------------------------------------------------------- /luke/pretraining/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/pretraining/train.py -------------------------------------------------------------------------------- /luke/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /luke/utils/convert_luke_to_huggingface_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/utils/convert_luke_to_huggingface_model.py -------------------------------------------------------------------------------- /luke/utils/entity_vocab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/utils/entity_vocab.py -------------------------------------------------------------------------------- /luke/utils/interwiki_db.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/utils/interwiki_db.py -------------------------------------------------------------------------------- /luke/utils/model_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/utils/model_utils.py -------------------------------------------------------------------------------- /luke/utils/resources/en-sent.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/utils/resources/en-sent.bin -------------------------------------------------------------------------------- /luke/utils/resources/opennlp-maxent-3.0.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/utils/resources/opennlp-maxent-3.0.3.jar -------------------------------------------------------------------------------- /luke/utils/resources/opennlp-tools-1.5.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/utils/resources/opennlp-tools-1.5.3.jar -------------------------------------------------------------------------------- /luke/utils/sentence_splitter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/utils/sentence_splitter.py -------------------------------------------------------------------------------- /luke/utils/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/luke/utils/util.py -------------------------------------------------------------------------------- /notebooks/huggingface_conll_2003.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/notebooks/huggingface_conll_2003.ipynb -------------------------------------------------------------------------------- /notebooks/huggingface_open_entity.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/notebooks/huggingface_open_entity.ipynb -------------------------------------------------------------------------------- /notebooks/huggingface_tacred.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/notebooks/huggingface_tacred.ipynb -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/poetry.lock -------------------------------------------------------------------------------- /poetry_coling2020.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/poetry_coling2020.lock -------------------------------------------------------------------------------- /poetry_emnlp2020.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/poetry_emnlp2020.lock -------------------------------------------------------------------------------- /pretraining.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/pretraining.md -------------------------------------------------------------------------------- /pretraining_config/luke_base_stage1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/pretraining_config/luke_base_stage1.json -------------------------------------------------------------------------------- /pretraining_config/luke_base_stage2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/pretraining_config/luke_base_stage2.json -------------------------------------------------------------------------------- /pretraining_config/luke_large_stage1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/pretraining_config/luke_large_stage1.json -------------------------------------------------------------------------------- /pretraining_config/luke_large_stage2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/pretraining_config/luke_large_stage2.json -------------------------------------------------------------------------------- /pretraining_config/mluke_base_stage1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/pretraining_config/mluke_base_stage1.json -------------------------------------------------------------------------------- /pretraining_config/mluke_base_stage2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/pretraining_config/mluke_base_stage2.json -------------------------------------------------------------------------------- /pretraining_config/mluke_large_stage1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/pretraining_config/mluke_large_stage1.json -------------------------------------------------------------------------------- /pretraining_config/mluke_large_stage2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/pretraining_config/mluke_large_stage2.json -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/pyproject.toml -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/pytest.ini -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/requirements.txt -------------------------------------------------------------------------------- /resources/luke_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/resources/luke_logo.png -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/fixtures/dummy_entity_vocab.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/tests/fixtures/dummy_entity_vocab.tsv -------------------------------------------------------------------------------- /tests/fixtures/en_ja_multilingual_vocab_test.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/tests/fixtures/en_ja_multilingual_vocab_test.jsonl -------------------------------------------------------------------------------- /tests/fixtures/enwiki_20181220_entvocab_100.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/tests/fixtures/enwiki_20181220_entvocab_100.tsv -------------------------------------------------------------------------------- /tests/fixtures/wikidata_20180423_sitelinks10.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/tests/fixtures/wikidata_20180423_sitelinks10.json -------------------------------------------------------------------------------- /tests/pretraining/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/pretraining/dummy_dump_db.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/tests/pretraining/dummy_dump_db.py -------------------------------------------------------------------------------- /tests/pretraining/test_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/tests/pretraining/test_dataset.py -------------------------------------------------------------------------------- /tests/test_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/tests/test_model.py -------------------------------------------------------------------------------- /tests/test_tokenization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/tests/test_tokenization.py -------------------------------------------------------------------------------- /tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/utils/test_entity_vocab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/tests/utils/test_entity_vocab.py -------------------------------------------------------------------------------- /tests/utils/test_interwiki_db.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/tests/utils/test_interwiki_db.py -------------------------------------------------------------------------------- /tests/utils/test_sentence_splitter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/studio-ousia/luke/HEAD/tests/utils/test_sentence_splitter.py --------------------------------------------------------------------------------