├── LICENSE ├── README.md ├── __pycache__ └── conlleval.cpython-37.pyc ├── algorithms ├── __init__.py ├── dict_merge.py └── maximum_spanning_tree.py ├── config ├── de_crf_ner.yaml ├── de_maxent_ner.yaml ├── en_crf_ner.yaml ├── en_maxent_ner.yaml ├── es_crf_ner.yaml ├── es_maxent_ner.yaml ├── eu_crf_ner.yaml ├── fa_crf_ner.yaml ├── he_crf_ner.yaml ├── multi-bert_10epoch_32batch_0.00005lr_10000lrrate_multilingual_nocrf_fast_relearn_sentbatch_sentloss_finetune_conlllang_nodev_panx_ner9.yaml ├── multi_bert_origflair_300epoch_2000batch_0.1lr_256hidden_de_monolingual_crf_sentloss_10patience_baseline_nodev_ner0.yaml ├── multi_bert_origflair_300epoch_2000batch_0.1lr_256hidden_en_monolingual_crf_sentloss_10patience_baseline_nodev_ner0.yaml ├── multi_bert_origflair_300epoch_2000batch_0.1lr_256hidden_es_monolingual_crf_sentloss_10patience_baseline_nodev_ner1.yaml ├── multi_bert_origflair_300epoch_2000batch_0.1lr_256hidden_nl_monolingual_crf_sentloss_10patience_baseline_nodev_ner1.yaml ├── nl_crf_ner.yaml ├── nl_maxent_ner.yaml └── ta_crf_ner.yaml ├── extract_features.py ├── flair ├── CRF.py ├── __init__.py ├── __init__.pyc ├── __pycache__ │ ├── __init__.cpython-36.pyc │ ├── __init__.cpython-37.pyc │ ├── __init__.cpython-38.pyc │ ├── config_parser.cpython-36.pyc │ ├── config_parser.cpython-37.pyc │ ├── corpus_mapping.cpython-36.pyc │ ├── corpus_mapping.cpython-37.pyc │ ├── custom_data_loader.cpython-36.pyc │ ├── custom_data_loader.cpython-37.pyc │ ├── data.cpython-36.pyc │ ├── data.cpython-37.pyc │ ├── datasets.cpython-36.pyc │ ├── datasets.cpython-37.pyc │ ├── embeddings.cpython-36.pyc │ ├── embeddings.cpython-37.pyc │ ├── file_utils.cpython-36.pyc │ ├── file_utils.cpython-37.pyc │ ├── image_encoder.cpython-36.pyc │ ├── image_encoder.cpython-37.pyc │ ├── linear_functions.cpython-36.pyc │ ├── linear_functions.cpython-37.pyc │ ├── list_data.cpython-36.pyc │ ├── list_data.cpython-37.pyc │ ├── loss_functions.cpython-37.pyc │ ├── ner_dp_utils.cpython-36.pyc │ ├── ner_dp_utils.cpython-37.pyc │ ├── nn.cpython-36.pyc │ ├── nn.cpython-37.pyc │ ├── optim.cpython-36.pyc │ ├── optim.cpython-37.pyc │ ├── training_utils.cpython-36.pyc │ ├── training_utils.cpython-37.pyc │ ├── variational_inference.cpython-36.pyc │ └── variational_inference.cpython-37.pyc ├── algorithms │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── dict_merge.cpython-36.pyc │ │ ├── dict_merge.cpython-37.pyc │ │ └── maximum_spanning_tree.cpython-36.pyc │ ├── dict_merge.py │ └── maximum_spanning_tree.py ├── config_parser.py ├── corpus_mapping.py ├── custom_data_loader.py ├── data.py ├── data_fetcher.py ├── datasets.py ├── embeddings.py ├── file_utils.py ├── hyperparameter │ ├── __init__.py │ ├── param_selection.py │ └── parameter.py ├── linear_functions.py ├── list_data.py ├── loss_functions.py ├── models │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── biaffine_attention.cpython-36.pyc │ │ ├── biaffine_attention.cpython-37.pyc │ │ ├── biaffine_dp.cpython-36.pyc │ │ ├── biaffine_dp.cpython-37.pyc │ │ ├── controller.cpython-36.pyc │ │ ├── controller.cpython-37.pyc │ │ ├── dependency_model.cpython-36.pyc │ │ ├── dependency_model.cpython-37.pyc │ │ ├── dependency_model_1.cpython-36.pyc │ │ ├── dependency_model_1.cpython-37.pyc │ │ ├── ensemble_model.cpython-36.pyc │ │ ├── ensemble_model.cpython-37.pyc │ │ ├── language_model.cpython-36.pyc │ │ ├── language_model.cpython-37.pyc │ │ ├── mst_decoder.cpython-36.pyc │ │ ├── mst_decoder.cpython-37.pyc │ │ ├── sequence_tagger_model.cpython-36.pyc │ │ ├── sequence_tagger_model.cpython-37.pyc │ │ ├── text_classification_model.cpython-36.pyc │ │ └── text_classification_model.cpython-37.pyc │ ├── biaffine_attention.py │ ├── biaffine_dp.py │ ├── controller.py │ ├── dependency_model.py │ ├── dependency_model_1.py │ ├── ensemble_model.py │ ├── finetune_sequence_tagger.py │ ├── language_model.py │ ├── mst_decoder.py │ ├── sequence_tagger_model.py │ ├── text_classification_model.py │ └── text_regression_model.py ├── ner_dp_utils.py ├── nn.py ├── optim.py ├── parser │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── model.cpython-36.pyc │ │ └── model.cpython-37.pyc │ ├── cmds │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-36.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── cmd.cpython-36.pyc │ │ │ ├── cmd.cpython-37.pyc │ │ │ ├── evaluate.cpython-36.pyc │ │ │ ├── evaluate.cpython-37.pyc │ │ │ ├── predict.cpython-37.pyc │ │ │ └── train.cpython-37.pyc │ │ ├── cmd.py │ │ ├── evaluate.py │ │ ├── predict.py │ │ └── train.py │ ├── config.py │ ├── model.py │ ├── modules │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-36.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── bert.cpython-36.pyc │ │ │ ├── bert.cpython-37.pyc │ │ │ ├── biaffine.cpython-36.pyc │ │ │ ├── biaffine.cpython-37.pyc │ │ │ ├── bilstm.cpython-36.pyc │ │ │ ├── bilstm.cpython-37.pyc │ │ │ ├── char_lstm.cpython-36.pyc │ │ │ ├── char_lstm.cpython-37.pyc │ │ │ ├── dropout.cpython-36.pyc │ │ │ ├── dropout.cpython-37.pyc │ │ │ ├── mlp.cpython-36.pyc │ │ │ ├── mlp.cpython-37.pyc │ │ │ ├── scalar_mix.cpython-36.pyc │ │ │ ├── scalar_mix.cpython-37.pyc │ │ │ ├── trilinear_attention.cpython-36.pyc │ │ │ └── trilinear_attention.cpython-37.pyc │ │ ├── bert.py │ │ ├── biaffine.py │ │ ├── bilstm.py │ │ ├── char_lstm.py │ │ ├── dropout.py │ │ ├── mlp.py │ │ ├── scalar_mix.py │ │ └── trilinear_attention.py │ └── utils │ │ ├── __init__.py │ │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── alg.cpython-36.pyc │ │ ├── alg.cpython-37.pyc │ │ ├── common.cpython-36.pyc │ │ ├── common.cpython-37.pyc │ │ ├── corpus.cpython-36.pyc │ │ ├── corpus.cpython-37.pyc │ │ ├── data.cpython-36.pyc │ │ ├── data.cpython-37.pyc │ │ ├── embedding.cpython-36.pyc │ │ ├── embedding.cpython-37.pyc │ │ ├── field.cpython-36.pyc │ │ ├── field.cpython-37.pyc │ │ ├── fn.cpython-36.pyc │ │ ├── fn.cpython-37.pyc │ │ ├── metric.cpython-36.pyc │ │ ├── metric.cpython-37.pyc │ │ ├── vocab.cpython-36.pyc │ │ └── vocab.cpython-37.pyc │ │ ├── alg.py │ │ ├── common.py │ │ ├── corpus.py │ │ ├── data.py │ │ ├── embedding.py │ │ ├── field.py │ │ ├── fn.py │ │ ├── metric.py │ │ └── vocab.py ├── samplers.py ├── trainers │ ├── .finetune_trainer.py.swp │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── distillation_trainer.cpython-36.pyc │ │ ├── distillation_trainer.cpython-37.pyc │ │ ├── finetune_trainer.cpython-36.pyc │ │ ├── finetune_trainer.cpython-37.pyc │ │ ├── reinforcement_trainer.cpython-36.pyc │ │ ├── reinforcement_trainer.cpython-37.pyc │ │ ├── swaf_trainer.cpython-36.pyc │ │ ├── swaf_trainer.cpython-37.pyc │ │ ├── trainer.cpython-36.pyc │ │ └── trainer.cpython-37.pyc │ ├── distillation_trainer.py │ ├── distillation_trainer_old.py │ ├── finetune_trainer.py │ ├── language_model_trainer.py │ ├── old_kd_trainer.py │ ├── reinforcement_trainer.py │ ├── swaf_trainer.py │ └── trainer.py ├── training_utils.py ├── utils │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── archival.cpython-36.pyc │ │ ├── checks.cpython-36.pyc │ │ ├── checks.cpython-37.pyc │ │ ├── environment.cpython-36.pyc │ │ ├── exception_hook.cpython-36.pyc │ │ ├── exception_hook.cpython-37.pyc │ │ ├── file.cpython-36.pyc │ │ ├── from_params.cpython-36.pyc │ │ ├── from_params.cpython-37.pyc │ │ ├── logging.cpython-36.pyc │ │ ├── logging.cpython-37.pyc │ │ ├── nn.cpython-36.pyc │ │ ├── params.cpython-36.pyc │ │ ├── params.cpython-37.pyc │ │ ├── registrable.cpython-36.pyc │ │ ├── string.cpython-36.pyc │ │ ├── time.cpython-36.pyc │ │ └── tqdm.cpython-36.pyc │ ├── archival.py │ ├── checks.py │ ├── environment.py │ ├── exception_hook.py │ ├── extract_tokens_from_amr.py │ ├── file.py │ ├── from_params.py │ ├── logging.py │ ├── nn.py │ ├── params.py │ ├── registrable.py │ ├── string.py │ ├── time.py │ └── tqdm.py ├── variational_inference.py └── visual │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-36.pyc │ ├── __init__.cpython-37.pyc │ ├── activations.cpython-36.pyc │ ├── activations.cpython-37.pyc │ ├── manifold.cpython-36.pyc │ ├── manifold.cpython-37.pyc │ ├── training_curves.cpython-36.pyc │ └── training_curves.cpython-37.pyc │ ├── activations.py │ ├── html_templates.py │ ├── manifold.py │ ├── ner_html.py │ └── training_curves.py ├── requirements.txt ├── resources ├── docs │ └── named_entity_recognition.md └── taggers │ ├── ast_tags.pkl │ ├── ast_tags_new.pkl │ ├── dependency_projective_tags.pkl │ ├── dm_tags.pkl │ ├── enhancedud_tags.pkl │ ├── ner_tags.pkl │ ├── np_tags.pkl │ ├── pos_tags.pkl │ └── ud_dependency_tags.pkl ├── script ├── __pycache__ │ ├── evaluate.cpython-37.pyc │ └── semdep_eval.cpython-37.pyc ├── evaluate.py └── semdep_eval.py ├── tests ├── conftest.py ├── resources │ ├── corpora │ │ └── lorem_ipsum │ │ │ ├── test.txt │ │ │ ├── train │ │ │ ├── split_1.txt │ │ │ └── split_2.txt │ │ │ └── valid.txt │ ├── tasks │ │ ├── ag_news │ │ │ ├── README.md │ │ │ ├── dev.txt │ │ │ ├── test.txt │ │ │ └── train.txt │ │ ├── fashion │ │ │ ├── eng.testa │ │ │ ├── eng.testb │ │ │ └── eng.train │ │ ├── fashion_nodev │ │ │ ├── test.tsv │ │ │ └── train.tsv │ │ ├── germeval │ │ │ ├── NER-de-dev.tsv │ │ │ ├── NER-de-test.tsv │ │ │ └── NER-de-train.tsv │ │ ├── imdb │ │ │ ├── README.md │ │ │ ├── dev.txt │ │ │ ├── test.txt │ │ │ └── train.txt │ │ ├── multi_class │ │ │ ├── dev.txt │ │ │ ├── test.txt │ │ │ └── train.txt │ │ ├── regression │ │ │ ├── README.md │ │ │ ├── dev.txt │ │ │ ├── test.txt │ │ │ └── train.txt │ │ └── ud_english │ │ │ ├── en_ewt-ud-dev.conllu │ │ │ ├── en_ewt-ud-test.conllu │ │ │ └── en_ewt-ud-train.conllu │ └── visual │ │ ├── loss.tsv │ │ ├── snippet.txt │ │ └── weights.txt ├── test_data.py ├── test_data_fetchers.py ├── test_embeddings.py ├── test_hyperparameter.py ├── test_language_model.py ├── test_model_integration.py ├── test_text_classifier.py ├── test_text_regressor.py ├── test_transformer_embeddings.py ├── test_utils.py └── test_visual.py ├── tools ├── conlleval ├── convertResultTagScheme.py └── split.py ├── train.py └── utils ├── LICENSE ├── README.md ├── __init__.py ├── __pycache__ └── conlleval.cpython-37.pyc ├── algorithms ├── __init__.py ├── dict_merge.py └── maximum_spanning_tree.py ├── archival.py ├── checks.py ├── config ├── bert-de-03-ner-finetune.yaml ├── bert-en-ner-finetune.yaml ├── conll_03_de_model.yaml ├── conll_03_english.yaml ├── dm_parsing_model.yaml ├── doc_ner_best.yaml ├── doc_ner_de_03.yaml ├── doc_ner_de_03_best.yaml ├── doc_ner_de_06.yaml ├── doc_ner_de_06_best.yaml ├── doc_ner_en.yaml ├── doc_ner_es.yaml ├── doc_ner_es_best.yaml ├── doc_ner_nl.yaml ├── doc_ner_nl_best.yaml ├── en-bert-extract.yaml ├── en-bert-finetune-ner.yaml ├── en-bert-finetune-ptb.yaml ├── mbert-de-03-ner-finetune.yaml ├── mbert-en-ner-finetune.yaml ├── pas_parsing_model.yaml ├── psd_parsing_model.yaml ├── ptb_parsing_model.yaml ├── roberta-doc-en-ner-finetune.yaml ├── tweebank_finetune_pos.yaml ├── xlmr-doc-de-03-ner-finetune.yaml ├── xlmr-doc-de-06-ner-finetune.yaml ├── xlmr-doc-en-ner-finetune.yaml ├── xlmr-doc-es-ner-finetune.yaml ├── xlmr-doc-nl-ner-finetune.yaml └── xlnet-doc-en-ner-finetune.yaml ├── environment.py ├── exception_hook.py ├── extract_features.py ├── extract_tokens_from_amr.py ├── file.py ├── flair ├── CRF.py ├── __init__.py ├── __init__.pyc ├── __pycache__ │ ├── __init__.cpython-36.pyc │ ├── __init__.cpython-37.pyc │ ├── config_parser.cpython-36.pyc │ ├── config_parser.cpython-37.pyc │ ├── corpus_mapping.cpython-36.pyc │ ├── corpus_mapping.cpython-37.pyc │ ├── custom_data_loader.cpython-36.pyc │ ├── custom_data_loader.cpython-37.pyc │ ├── data.cpython-36.pyc │ ├── data.cpython-37.pyc │ ├── datasets.cpython-36.pyc │ ├── datasets.cpython-37.pyc │ ├── embeddings.cpython-36.pyc │ ├── embeddings.cpython-37.pyc │ ├── file_utils.cpython-36.pyc │ ├── file_utils.cpython-37.pyc │ ├── linear_functions.cpython-36.pyc │ ├── linear_functions.cpython-37.pyc │ ├── list_data.cpython-36.pyc │ ├── list_data.cpython-37.pyc │ ├── ner_dp_utils.cpython-36.pyc │ ├── ner_dp_utils.cpython-37.pyc │ ├── nn.cpython-36.pyc │ ├── nn.cpython-37.pyc │ ├── optim.cpython-36.pyc │ ├── optim.cpython-37.pyc │ ├── training_utils.cpython-36.pyc │ ├── training_utils.cpython-37.pyc │ ├── variational_inference.cpython-36.pyc │ └── variational_inference.cpython-37.pyc ├── algorithms │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── dict_merge.cpython-36.pyc │ │ ├── dict_merge.cpython-37.pyc │ │ └── maximum_spanning_tree.cpython-36.pyc │ ├── dict_merge.py │ └── maximum_spanning_tree.py ├── config_parser.py ├── corpus_mapping.py ├── custom_data_loader.py ├── data.py ├── data_fetcher.py ├── datasets.py ├── embeddings.py ├── file_utils.py ├── hyperparameter │ ├── __init__.py │ ├── param_selection.py │ └── parameter.py ├── linear_functions.py ├── list_data.py ├── models │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── biaffine_attention.cpython-36.pyc │ │ ├── biaffine_attention.cpython-37.pyc │ │ ├── biaffine_dp.cpython-36.pyc │ │ ├── biaffine_dp.cpython-37.pyc │ │ ├── controller.cpython-36.pyc │ │ ├── controller.cpython-37.pyc │ │ ├── dependency_model.cpython-36.pyc │ │ ├── dependency_model.cpython-37.pyc │ │ ├── dependency_model_1.cpython-36.pyc │ │ ├── dependency_model_1.cpython-37.pyc │ │ ├── ensemble_model.cpython-36.pyc │ │ ├── ensemble_model.cpython-37.pyc │ │ ├── language_model.cpython-36.pyc │ │ ├── language_model.cpython-37.pyc │ │ ├── mst_decoder.cpython-36.pyc │ │ ├── mst_decoder.cpython-37.pyc │ │ ├── sequence_tagger_model.cpython-36.pyc │ │ ├── sequence_tagger_model.cpython-37.pyc │ │ ├── text_classification_model.cpython-36.pyc │ │ └── text_classification_model.cpython-37.pyc │ ├── biaffine_attention.py │ ├── controller.py │ ├── dependency_model.py │ ├── ensemble_model.py │ ├── finetune_sequence_tagger.py │ ├── language_model.py │ ├── mst_decoder.py │ ├── sequence_tagger_model.py │ ├── text_classification_model.py │ └── text_regression_model.py ├── ner_dp_utils.py ├── nn.py ├── optim.py ├── parser │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── model.cpython-36.pyc │ │ └── model.cpython-37.pyc │ ├── cmds │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-36.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── cmd.cpython-36.pyc │ │ │ ├── cmd.cpython-37.pyc │ │ │ ├── evaluate.cpython-36.pyc │ │ │ ├── evaluate.cpython-37.pyc │ │ │ ├── predict.cpython-37.pyc │ │ │ └── train.cpython-37.pyc │ │ ├── cmd.py │ │ ├── evaluate.py │ │ ├── predict.py │ │ └── train.py │ ├── config.py │ ├── model.py │ ├── modules │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-36.pyc │ │ │ ├── __init__.cpython-37.pyc │ │ │ ├── bert.cpython-36.pyc │ │ │ ├── bert.cpython-37.pyc │ │ │ ├── biaffine.cpython-36.pyc │ │ │ ├── biaffine.cpython-37.pyc │ │ │ ├── bilstm.cpython-36.pyc │ │ │ ├── bilstm.cpython-37.pyc │ │ │ ├── char_lstm.cpython-36.pyc │ │ │ ├── char_lstm.cpython-37.pyc │ │ │ ├── dropout.cpython-36.pyc │ │ │ ├── dropout.cpython-37.pyc │ │ │ ├── mlp.cpython-36.pyc │ │ │ ├── mlp.cpython-37.pyc │ │ │ ├── scalar_mix.cpython-36.pyc │ │ │ ├── scalar_mix.cpython-37.pyc │ │ │ ├── trilinear_attention.cpython-36.pyc │ │ │ └── trilinear_attention.cpython-37.pyc │ │ ├── bert.py │ │ ├── biaffine.py │ │ ├── bilstm.py │ │ ├── char_lstm.py │ │ ├── dropout.py │ │ ├── mlp.py │ │ ├── scalar_mix.py │ │ └── trilinear_attention.py │ └── utils │ │ ├── __init__.py │ │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── alg.cpython-36.pyc │ │ ├── alg.cpython-37.pyc │ │ ├── common.cpython-36.pyc │ │ ├── common.cpython-37.pyc │ │ ├── corpus.cpython-36.pyc │ │ ├── corpus.cpython-37.pyc │ │ ├── data.cpython-36.pyc │ │ ├── data.cpython-37.pyc │ │ ├── embedding.cpython-36.pyc │ │ ├── embedding.cpython-37.pyc │ │ ├── field.cpython-36.pyc │ │ ├── field.cpython-37.pyc │ │ ├── fn.cpython-36.pyc │ │ ├── fn.cpython-37.pyc │ │ ├── metric.cpython-36.pyc │ │ ├── metric.cpython-37.pyc │ │ ├── vocab.cpython-36.pyc │ │ └── vocab.cpython-37.pyc │ │ ├── alg.py │ │ ├── common.py │ │ ├── corpus.py │ │ ├── data.py │ │ ├── embedding.py │ │ ├── field.py │ │ ├── fn.py │ │ ├── metric.py │ │ └── vocab.py ├── samplers.py ├── trainers │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── distillation_trainer.cpython-36.pyc │ │ ├── distillation_trainer.cpython-37.pyc │ │ ├── finetune_trainer.cpython-36.pyc │ │ ├── finetune_trainer.cpython-37.pyc │ │ ├── reinforcement_trainer.cpython-36.pyc │ │ ├── reinforcement_trainer.cpython-37.pyc │ │ ├── swaf_trainer.cpython-36.pyc │ │ ├── swaf_trainer.cpython-37.pyc │ │ ├── trainer.cpython-36.pyc │ │ └── trainer.cpython-37.pyc │ ├── distillation_trainer.py │ ├── finetune_trainer.py │ ├── language_model_trainer.py │ ├── reinforcement_trainer.py │ └── trainer.py ├── training_utils.py ├── utils │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-36.pyc │ │ ├── __init__.cpython-37.pyc │ │ ├── archival.cpython-36.pyc │ │ ├── checks.cpython-36.pyc │ │ ├── checks.cpython-37.pyc │ │ ├── environment.cpython-36.pyc │ │ ├── exception_hook.cpython-36.pyc │ │ ├── exception_hook.cpython-37.pyc │ │ ├── file.cpython-36.pyc │ │ ├── from_params.cpython-36.pyc │ │ ├── from_params.cpython-37.pyc │ │ ├── logging.cpython-36.pyc │ │ ├── logging.cpython-37.pyc │ │ ├── nn.cpython-36.pyc │ │ ├── params.cpython-36.pyc │ │ ├── params.cpython-37.pyc │ │ ├── registrable.cpython-36.pyc │ │ ├── string.cpython-36.pyc │ │ ├── time.cpython-36.pyc │ │ └── tqdm.cpython-36.pyc │ ├── archival.py │ ├── checks.py │ ├── environment.py │ ├── exception_hook.py │ ├── extract_tokens_from_amr.py │ ├── file.py │ ├── from_params.py │ ├── logging.py │ ├── nn.py │ ├── params.py │ ├── registrable.py │ ├── string.py │ ├── time.py │ └── tqdm.py ├── variational_inference.py └── visual │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-36.pyc │ ├── __init__.cpython-37.pyc │ ├── activations.cpython-36.pyc │ ├── activations.cpython-37.pyc │ ├── manifold.cpython-36.pyc │ ├── manifold.cpython-37.pyc │ ├── training_curves.cpython-36.pyc │ └── training_curves.cpython-37.pyc │ ├── activations.py │ ├── html_templates.py │ ├── manifold.py │ ├── ner_html.py │ └── training_curves.py ├── from_params.py ├── logging.py ├── nn.py ├── params.py ├── registrable.py ├── requirements.txt ├── resources ├── docs │ └── named_entity_recognition.md └── taggers │ ├── ast_tags.pkl │ ├── ast_tags_new.pkl │ ├── dependency_projective_tags.pkl │ ├── dm_tags.pkl │ ├── enhancedud_tags.pkl │ ├── ner_tags.pkl │ ├── np_tags.pkl │ ├── pos_tags.pkl │ └── ud_dependency_tags.pkl ├── script ├── __pycache__ │ ├── evaluate.cpython-37.pyc │ └── semdep_eval.cpython-37.pyc ├── evaluate.py └── semdep_eval.py ├── string.py ├── tests ├── conftest.py ├── resources │ ├── corpora │ │ └── lorem_ipsum │ │ │ ├── test.txt │ │ │ ├── train │ │ │ ├── split_1.txt │ │ │ └── split_2.txt │ │ │ └── valid.txt │ ├── tasks │ │ ├── ag_news │ │ │ ├── README.md │ │ │ ├── dev.txt │ │ │ ├── test.txt │ │ │ └── train.txt │ │ ├── fashion │ │ │ ├── eng.testa │ │ │ ├── eng.testb │ │ │ └── eng.train │ │ ├── fashion_nodev │ │ │ ├── test.tsv │ │ │ └── train.tsv │ │ ├── germeval │ │ │ ├── NER-de-dev.tsv │ │ │ ├── NER-de-test.tsv │ │ │ └── NER-de-train.tsv │ │ ├── imdb │ │ │ ├── README.md │ │ │ ├── dev.txt │ │ │ ├── test.txt │ │ │ └── train.txt │ │ ├── multi_class │ │ │ ├── dev.txt │ │ │ ├── test.txt │ │ │ └── train.txt │ │ ├── regression │ │ │ ├── README.md │ │ │ ├── dev.txt │ │ │ ├── test.txt │ │ │ └── train.txt │ │ └── ud_english │ │ │ ├── en_ewt-ud-dev.conllu │ │ │ ├── en_ewt-ud-test.conllu │ │ │ └── en_ewt-ud-train.conllu │ └── visual │ │ ├── loss.tsv │ │ ├── snippet.txt │ │ └── weights.txt ├── test_data.py ├── test_data_fetchers.py ├── test_embeddings.py ├── test_hyperparameter.py ├── test_language_model.py ├── test_model_integration.py ├── test_text_classifier.py ├── test_text_regressor.py ├── test_transformer_embeddings.py ├── test_utils.py └── test_visual.py ├── time.py ├── tools ├── conlleval ├── convertResultTagScheme.py └── split.py ├── tqdm.py └── train.py /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Flair is licensed under the following MIT License (MIT) Copyright © 2018 Zalando SE, https://tech.zalando.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /__pycache__/conlleval.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/__pycache__/conlleval.cpython-37.pyc -------------------------------------------------------------------------------- /algorithms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/algorithms/__init__.py -------------------------------------------------------------------------------- /algorithms/dict_merge.py: -------------------------------------------------------------------------------- 1 | # Recursive dictionary merge 2 | # Copyright (C) 2016 Paul Durivage 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program. If not, see . 16 | 17 | # Found here https://gist.github.com/angstwad/bf22d1822c38a92ec0a9 18 | # Using jpopelka's modified solution. 19 | 20 | import collections 21 | 22 | def dict_merge(dct, merge_dct): 23 | """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of 24 | updating only top-level keys, dict_merge recurses down into dicts nested 25 | to an arbitrary depth, updating keys. The ``merge_dct`` is merged into 26 | ``dct``. 27 | 28 | :param dct: dict onto which the merge is executed 29 | :param merge_dct: dct merged into dct 30 | :return: None 31 | """ 32 | for k, v in merge_dct.items(): 33 | if isinstance(dct.get(k), dict) and isinstance(v, collections.Mapping): 34 | dict_merge(dct[k], v) 35 | else: 36 | dct[k] = v 37 | -------------------------------------------------------------------------------- /config/de_maxent_ner.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | distill_mode: true 3 | optimizer: SGD 4 | sentence_level_batch: true 5 | anneal_factor: 0.5 6 | embeddings: 7 | FastCharacterEmbeddings: 8 | char_cnn: true 9 | char_embedding_dim: 25 10 | hidden_size_char: 25 11 | FastWordEmbeddings-0: 12 | embeddings: de 13 | freeze: true 14 | interpolation: 0.5 15 | is_teacher_list: true 16 | model: 17 | FastSequenceTagger: 18 | crf_attention: false 19 | distill_crf: false 20 | distill_posterior: true 21 | dropout: 0.0 22 | hidden_size: 256 23 | relearn_embeddings: true 24 | sentence_loss: true 25 | temperature: 3.0 26 | use_crf: false 27 | model_name: word_char_charcnn_0.5anneal_300epoch_32batch_0.1lr_256hidden_de_monolingual_nocrf_sentloss_distill_posterior_3temperature_10patience_fast_sentbatch_relearn_fasttext_freeze_nodev_ner35 28 | ner: 29 | Corpus: CONLL_03_GERMAN 30 | tag_dictionary: resources/taggers/ner_tags.pkl 31 | teachers: 32 | ? config_gen/multi_bert_origflair_300epoch_2000batch_0.1lr_256hidden_de_monolingual_crf_sentloss_10patience_baseline_nodev_ner0.yaml 33 | : CONLL_03_GERMAN 34 | target_dir: resources/taggers/ 35 | targets: ner 36 | teacher_annealing: true 37 | train: 38 | learning_rate: 0.1 39 | max_epochs: 300 40 | mini_batch_size: 32 41 | monitor_test: false 42 | patience: 10 43 | save_final_model: false 44 | train_with_dev: false 45 | true_reshuffle: false 46 | trainer: ModelFinetuner 47 | -------------------------------------------------------------------------------- /config/en_maxent_ner.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | distill_mode: true 3 | optimizer: SGD 4 | sentence_level_batch: true 5 | anneal_factor: 0.5 6 | embeddings: 7 | FastCharacterEmbeddings: 8 | char_cnn: true 9 | char_embedding_dim: 25 10 | hidden_size_char: 25 11 | FastWordEmbeddings-0: 12 | embeddings: en 13 | freeze: true 14 | interpolation: 0.5 15 | is_teacher_list: true 16 | model: 17 | FastSequenceTagger: 18 | crf_attention: false 19 | distill_crf: false 20 | distill_posterior: true 21 | dropout: 0.0 22 | hidden_size: 256 23 | relearn_embeddings: true 24 | sentence_loss: true 25 | temperature: 5.0 26 | use_crf: false 27 | model_name: word_char_charcnn_0.5anneal_300epoch_32batch_0.1lr_256hidden_en_monolingual_nocrf_sentloss_distill_posterior_5temperature_10patience_fast_sentbatch_relearn_fasttext_freeze_nodev_ner35 28 | ner: 29 | Corpus: CONLL_03 30 | professors: 31 | config/single-de-ner.yaml: CONLL_03_GERMAN 32 | config/single-en-ner.yaml: CONLL_03 33 | config/single-es-ner.yaml: CONLL_03_SPANISH 34 | config/single-nl-ner.yaml: CONLL_03_DUTCH 35 | tag_dictionary: resources/taggers/ner_tags.pkl 36 | teachers: 37 | ? config_gen/multi_bert_origflair_300epoch_2000batch_0.1lr_256hidden_en_monolingual_crf_sentloss_10patience_baseline_nodev_ner0.yaml 38 | : CONLL_03 39 | target_dir: resources/taggers/ 40 | targets: ner 41 | teacher_annealing: true 42 | train: 43 | learning_rate: 0.1 44 | max_epochs: 300 45 | mini_batch_size: 32 46 | monitor_test: false 47 | patience: 10 48 | save_final_model: false 49 | train_with_dev: false 50 | true_reshuffle: false 51 | trainer: ModelFinetuner 52 | -------------------------------------------------------------------------------- /config/es_maxent_ner.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | distill_mode: true 3 | optimizer: SGD 4 | sentence_level_batch: true 5 | anneal_factor: 0.5 6 | embeddings: 7 | FastCharacterEmbeddings: 8 | char_cnn: true 9 | char_embedding_dim: 25 10 | hidden_size_char: 25 11 | FastWordEmbeddings-0: 12 | embeddings: es 13 | freeze: true 14 | interpolation: 0.5 15 | is_teacher_list: true 16 | model: 17 | FastSequenceTagger: 18 | crf_attention: false 19 | distill_crf: false 20 | distill_posterior: true 21 | dropout: 0.0 22 | hidden_size: 256 23 | relearn_embeddings: true 24 | sentence_loss: true 25 | temperature: 7.0 26 | use_crf: false 27 | model_name: word_char_charcnn_0.5anneal_300epoch_32batch_0.1lr_256hidden_es_monolingual_nocrf_sentloss_distill_posterior_7temperature_10patience_fast_sentbatch_relearn_fasttext_freeze_nodev_ner35 28 | ner: 29 | Corpus: CONLL_03_SPANISH 30 | professors: 31 | config/single-de-ner.yaml: CONLL_03_GERMAN 32 | config/single-en-ner.yaml: CONLL_03 33 | config/single-es-ner.yaml: CONLL_03_SPANISH 34 | config/single-nl-ner.yaml: CONLL_03_DUTCH 35 | tag_dictionary: resources/taggers/ner_tags.pkl 36 | teachers: 37 | ? config_gen/multi_bert_origflair_300epoch_2000batch_0.1lr_256hidden_es_monolingual_crf_sentloss_10patience_baseline_nodev_ner1.yaml 38 | : CONLL_03_SPANISH 39 | target_dir: resources/taggers/ 40 | targets: ner 41 | teacher_annealing: true 42 | train: 43 | learning_rate: 0.1 44 | max_epochs: 300 45 | mini_batch_size: 32 46 | monitor_test: false 47 | patience: 10 48 | save_final_model: false 49 | train_with_dev: false 50 | true_reshuffle: false 51 | trainer: ModelFinetuner 52 | -------------------------------------------------------------------------------- /config/fa_crf_ner.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | distill_mode: true 3 | down_sample_amount: 30000 4 | optimizer: SGD 5 | sentence_level_batch: true 6 | anneal_factor: 0.5 7 | embeddings: 8 | FastCharacterEmbeddings: 9 | char_cnn: true 10 | char_embedding_dim: 25 11 | hidden_size_char: 25 12 | FastWordEmbeddings-0: 13 | embeddings: fa 14 | freeze: true 15 | interpolation: 0.5 16 | is_teacher_list: true 17 | model: 18 | FastSequenceTagger: 19 | crf_attention: false 20 | distill_crf: false 21 | distill_posterior: true 22 | dropout: 0.0 23 | hidden_size: 256 24 | relearn_embeddings: true 25 | sentence_loss: true 26 | temperature: 1.0 27 | use_crf: true 28 | model_name: word_char_charcnn_0.5anneal_300epoch_32batch_0.1lr_256hidden_fa_monolingual_crf_sentloss_distill_posterior_softmaxteacher_1temperature_10patience_fast_sentbatch_relearn_fasttext_freeze_mberttunedkd_30000downsample_nodev_panxpred_newteacher_panx_ner39 29 | ner: 30 | Corpus: PANXPRED-FA 31 | tag_dictionary: resources/taggers/ner_tags.pkl 32 | teachers: 33 | ? config_gen/multi-bert_10epoch_32batch_0.00005lr_10000lrrate_multilingual_nocrf_fast_relearn_sentbatch_sentloss_finetune_conlllang_nodev_panx_ner9.yaml 34 | : PANXPRED-FA 35 | target_dir: resources/taggers/ 36 | targets: ner 37 | teacher_annealing: true 38 | train: 39 | learning_rate: 0.1 40 | max_epochs: 300 41 | mini_batch_size: 32 42 | monitor_test: false 43 | patience: 10 44 | save_final_model: false 45 | train_with_dev: false 46 | true_reshuffle: false 47 | trainer: ModelFinetuner 48 | -------------------------------------------------------------------------------- /config/he_crf_ner.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | distill_mode: true 3 | down_sample_amount: 30000 4 | optimizer: SGD 5 | sentence_level_batch: true 6 | anneal_factor: 0.5 7 | embeddings: 8 | FastCharacterEmbeddings: 9 | char_cnn: true 10 | char_embedding_dim: 25 11 | hidden_size_char: 25 12 | FastWordEmbeddings-0: 13 | embeddings: he 14 | freeze: true 15 | interpolation: 0.5 16 | is_teacher_list: true 17 | model: 18 | FastSequenceTagger: 19 | crf_attention: false 20 | distill_crf: false 21 | distill_posterior: true 22 | dropout: 0.0 23 | hidden_size: 256 24 | relearn_embeddings: true 25 | sentence_loss: true 26 | temperature: 1.0 27 | use_crf: true 28 | model_name: word_char_charcnn_0.5anneal_300epoch_32batch_0.1lr_256hidden_he_monolingual_crf_sentloss_distill_posterior_softmaxteacher_1temperature_10patience_fast_sentbatch_relearn_fasttext_freeze_mberttunedkd_30000downsample_nodev_panxpred_newteacher_panx_ner39 29 | ner: 30 | Corpus: PANXPRED-HE 31 | tag_dictionary: resources/taggers/ner_tags.pkl 32 | teachers: 33 | ? config_gen/multi-bert_10epoch_32batch_0.00005lr_10000lrrate_multilingual_nocrf_fast_relearn_sentbatch_sentloss_finetune_conlllang_nodev_panx_ner9.yaml 34 | : PANXPRED-HE 35 | target_dir: resources/taggers/ 36 | targets: ner 37 | teacher_annealing: true 38 | train: 39 | learning_rate: 0.1 40 | max_epochs: 300 41 | mini_batch_size: 32 42 | monitor_test: false 43 | patience: 10 44 | save_final_model: false 45 | train_with_dev: false 46 | true_reshuffle: false 47 | trainer: ModelFinetuner 48 | -------------------------------------------------------------------------------- /config/nl_crf_ner.yaml: -------------------------------------------------------------------------------- 1 | MFVI: 2 | hexa_rank: 150 3 | hexa_std: 1 4 | iterations: 3 5 | normalize_weight: true 6 | quad_rank: 150 7 | quad_std: 1 8 | tag_dim: 150 9 | use_hexalinear: false 10 | use_quadrilinear: false 11 | use_second_order: false 12 | use_third_order: false 13 | window_size: 1 14 | ModelFinetuner: 15 | distill_mode: true 16 | optimizer: SGD 17 | sentence_level_batch: true 18 | anneal_factor: 0.5 19 | embeddings: 20 | FastCharacterEmbeddings: 21 | char_cnn: true 22 | char_embedding_dim: 25 23 | hidden_size_char: 25 24 | FastWordEmbeddings-0: 25 | embeddings: nl 26 | freeze: true 27 | interpolation: 0.5 28 | is_teacher_list: true 29 | model: 30 | FastSequenceTagger: 31 | crf_attention: false 32 | distill_crf: false 33 | distill_exact: true 34 | dropout: 0.0 35 | hidden_size: 256 36 | relearn_embeddings: true 37 | sentence_loss: true 38 | temperature: 2.0 39 | use_crf: true 40 | model_name: word_char_charcnn_0.5anneal_300epoch_32batch_0.1lr_256hidden_nl_monolingual_crf_sentloss_distill_exact_2temperature_10patience_fast_sentbatch_relearn_fasttext_freeze_nodev_ner1 41 | ner: 42 | Corpus: CONLL_03_DUTCH 43 | tag_dictionary: resources/taggers/ner_tags.pkl 44 | teachers: 45 | ? config_gen/multi_bert_origflair_300epoch_2000batch_0.1lr_256hidden_nl_monolingual_crf_sentloss_10patience_baseline_nodev_ner1.yaml 46 | : CONLL_03_DUTCH 47 | target_dir: resources/taggers/ 48 | targets: ner 49 | teacher_annealing: true 50 | train: 51 | learning_rate: 0.1 52 | max_epochs: 300 53 | mini_batch_size: 32 54 | monitor_test: false 55 | patience: 10 56 | save_final_model: false 57 | train_with_dev: false 58 | true_reshuffle: false 59 | trainer: ModelFinetuner 60 | -------------------------------------------------------------------------------- /config/nl_maxent_ner.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | distill_mode: true 3 | optimizer: SGD 4 | sentence_level_batch: true 5 | anneal_factor: 0.5 6 | embeddings: 7 | FastCharacterEmbeddings: 8 | char_cnn: true 9 | char_embedding_dim: 25 10 | hidden_size_char: 25 11 | FastWordEmbeddings-0: 12 | embeddings: nl 13 | freeze: true 14 | interpolation: 0.5 15 | is_teacher_list: true 16 | model: 17 | FastSequenceTagger: 18 | crf_attention: false 19 | distill_crf: false 20 | distill_posterior: true 21 | dropout: 0.0 22 | hidden_size: 256 23 | relearn_embeddings: true 24 | sentence_loss: true 25 | temperature: 3.0 26 | use_crf: false 27 | model_name: word_char_charcnn_0.5anneal_300epoch_32batch_0.1lr_256hidden_nl_monolingual_nocrf_sentloss_distill_posterior_3temperature_10patience_fast_sentbatch_relearn_fasttext_freeze_nodev_ner35 28 | ner: 29 | Corpus: CONLL_03_DUTCH 30 | professors: 31 | config/single-de-ner.yaml: CONLL_03_GERMAN 32 | config/single-en-ner.yaml: CONLL_03 33 | config/single-es-ner.yaml: CONLL_03_SPANISH 34 | config/single-nl-ner.yaml: CONLL_03_DUTCH 35 | tag_dictionary: resources/taggers/ner_tags.pkl 36 | teachers: 37 | ? config_gen/multi_bert_origflair_300epoch_2000batch_0.1lr_256hidden_nl_monolingual_crf_sentloss_10patience_baseline_nodev_ner1.yaml 38 | : CONLL_03_DUTCH 39 | target_dir: resources/taggers/ 40 | targets: ner 41 | teacher_annealing: true 42 | train: 43 | learning_rate: 0.1 44 | max_epochs: 300 45 | mini_batch_size: 32 46 | monitor_test: false 47 | patience: 10 48 | save_final_model: false 49 | train_with_dev: false 50 | true_reshuffle: false 51 | trainer: ModelFinetuner 52 | -------------------------------------------------------------------------------- /config/ta_crf_ner.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | distill_mode: true 3 | down_sample_amount: 30000 4 | optimizer: SGD 5 | sentence_level_batch: true 6 | anneal_factor: 0.5 7 | atis: 8 | Corpus: ATIS-EN:ATIS-TR:ATIS-HI 9 | chunk: 10 | Corpus: CONLL_03:CONLL_03_GERMAN 11 | embeddings: 12 | FastCharacterEmbeddings: 13 | char_cnn: true 14 | char_embedding_dim: 25 15 | hidden_size_char: 25 16 | FastWordEmbeddings-0: 17 | embeddings: ta 18 | freeze: true 19 | interpolation: 0.5 20 | is_teacher_list: true 21 | model: 22 | FastSequenceTagger: 23 | crf_attention: false 24 | distill_crf: false 25 | distill_posterior: true 26 | dropout: 0.0 27 | hidden_size: 256 28 | relearn_embeddings: true 29 | sentence_loss: true 30 | temperature: 1.0 31 | use_crf: true 32 | model_name: word_char_charcnn_0.5anneal_300epoch_32batch_0.1lr_256hidden_ta_monolingual_crf_sentloss_distill_posterior_softmaxteacher_1temperature_10patience_fast_sentbatch_relearn_fasttext_freeze_mberttunedkd_30000downsample_nodev_panxpred_newteacher_panx_ner39 33 | ner: 34 | Corpus: PANXPRED-TA 35 | tag_dictionary: resources/taggers/ner_tags.pkl 36 | teachers: 37 | ? config_gen/multi-bert_10epoch_32batch_0.00005lr_10000lrrate_multilingual_nocrf_fast_relearn_sentbatch_sentloss_finetune_conlllang_nodev_panx_ner9.yaml 38 | : PANXPRED-TA 39 | target_dir: resources/taggers/ 40 | targets: ner 41 | teacher_annealing: true 42 | train: 43 | learning_rate: 0.1 44 | max_epochs: 300 45 | mini_batch_size: 32 46 | monitor_test: false 47 | patience: 10 48 | save_final_model: false 49 | train_with_dev: false 50 | true_reshuffle: false 51 | trainer: ModelFinetuner 52 | -------------------------------------------------------------------------------- /flair/__init__.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | 4 | # global variable: cache_root 5 | cache_root = os.path.expanduser(os.path.join("~", ".flair")) 6 | 7 | # global variable: device 8 | device = None 9 | if torch.cuda.is_available(): 10 | device = torch.device("cuda:0") 11 | else: 12 | device = torch.device("cpu") 13 | 14 | from . import data 15 | from . import models 16 | from . import visual 17 | from . import trainers 18 | from . import nn 19 | 20 | import logging.config 21 | 22 | __version__ = "0.4.3" 23 | 24 | logging.config.dictConfig( 25 | { 26 | "version": 1, 27 | "disable_existing_loggers": False, 28 | "formatters": {"standard": {"format": "%(asctime)-15s %(message)s"}}, 29 | "handlers": { 30 | "console": { 31 | "level": "INFO", 32 | "class": "logging.StreamHandler", 33 | "formatter": "standard", 34 | "stream": "ext://sys.stdout", 35 | } 36 | }, 37 | "loggers": { 38 | "flair": {"handlers": ["console"], "level": "INFO", "propagate": False} 39 | }, 40 | "root": {"handlers": ["console"], "level": "WARNING"}, 41 | } 42 | ) 43 | 44 | logger = logging.getLogger("flair") 45 | -------------------------------------------------------------------------------- /flair/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__init__.pyc -------------------------------------------------------------------------------- /flair/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /flair/__pycache__/config_parser.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/config_parser.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/config_parser.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/config_parser.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/corpus_mapping.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/corpus_mapping.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/corpus_mapping.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/corpus_mapping.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/custom_data_loader.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/custom_data_loader.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/custom_data_loader.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/custom_data_loader.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/data.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/data.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/data.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/data.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/datasets.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/datasets.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/datasets.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/datasets.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/embeddings.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/embeddings.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/embeddings.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/embeddings.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/file_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/file_utils.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/file_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/file_utils.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/image_encoder.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/image_encoder.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/image_encoder.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/image_encoder.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/linear_functions.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/linear_functions.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/linear_functions.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/linear_functions.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/list_data.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/list_data.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/list_data.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/list_data.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/loss_functions.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/loss_functions.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/ner_dp_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/ner_dp_utils.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/ner_dp_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/ner_dp_utils.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/nn.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/nn.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/nn.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/nn.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/optim.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/optim.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/optim.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/optim.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/training_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/training_utils.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/training_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/training_utils.cpython-37.pyc -------------------------------------------------------------------------------- /flair/__pycache__/variational_inference.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/variational_inference.cpython-36.pyc -------------------------------------------------------------------------------- /flair/__pycache__/variational_inference.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/__pycache__/variational_inference.cpython-37.pyc -------------------------------------------------------------------------------- /flair/algorithms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/algorithms/__init__.py -------------------------------------------------------------------------------- /flair/algorithms/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/algorithms/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /flair/algorithms/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/algorithms/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /flair/algorithms/__pycache__/dict_merge.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/algorithms/__pycache__/dict_merge.cpython-36.pyc -------------------------------------------------------------------------------- /flair/algorithms/__pycache__/dict_merge.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/algorithms/__pycache__/dict_merge.cpython-37.pyc -------------------------------------------------------------------------------- /flair/algorithms/__pycache__/maximum_spanning_tree.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/algorithms/__pycache__/maximum_spanning_tree.cpython-36.pyc -------------------------------------------------------------------------------- /flair/algorithms/dict_merge.py: -------------------------------------------------------------------------------- 1 | # Recursive dictionary merge 2 | # Copyright (C) 2016 Paul Durivage 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program. If not, see . 16 | 17 | # Found here https://gist.github.com/angstwad/bf22d1822c38a92ec0a9 18 | # Using jpopelka's modified solution. 19 | 20 | import collections 21 | 22 | def dict_merge(dct, merge_dct): 23 | """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of 24 | updating only top-level keys, dict_merge recurses down into dicts nested 25 | to an arbitrary depth, updating keys. The ``merge_dct`` is merged into 26 | ``dct``. 27 | 28 | :param dct: dict onto which the merge is executed 29 | :param merge_dct: dct merged into dct 30 | :return: None 31 | """ 32 | for k, v in merge_dct.items(): 33 | if isinstance(dct.get(k), dict) and isinstance(v, collections.Mapping): 34 | dict_merge(dct[k], v) 35 | else: 36 | dct[k] = v 37 | -------------------------------------------------------------------------------- /flair/hyperparameter/__init__.py: -------------------------------------------------------------------------------- 1 | from .parameter import ( 2 | Parameter, 3 | SEQUENCE_TAGGER_PARAMETERS, 4 | TRAINING_PARAMETERS, 5 | DOCUMENT_EMBEDDING_PARAMETERS, 6 | ) 7 | from .param_selection import ( 8 | SequenceTaggerParamSelector, 9 | TextClassifierParamSelector, 10 | SearchSpace, 11 | ) 12 | -------------------------------------------------------------------------------- /flair/list_data.py: -------------------------------------------------------------------------------- 1 | from .data import * 2 | class ListCorpus(Corpus): 3 | def __init__( 4 | self, 5 | train: List[FlairDataset], 6 | dev: List[FlairDataset], 7 | test: List[FlairDataset], 8 | name: str = "listcorpus", 9 | targets: list = [], 10 | ): 11 | # In this Corpus, we set train list to be our target to train, we keep self._train the same as the Class Corpus as the counting and preprocessing is needed 12 | self.train_list: List[FlairDataset] = train 13 | self.dev_list: List[FlairDataset] = dev 14 | self.test_list: List[FlairDataset] = test 15 | self._train: FlairDataset = ConcatDataset([data for data in train]) 16 | self._dev: FlairDataset = ConcatDataset([data for data in dev]) 17 | self._test: FlairDataset = ConcatDataset([data for data in test]) 18 | self.name: str = name 19 | self.targets = targets -------------------------------------------------------------------------------- /flair/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .sequence_tagger_model import SequenceTagger, FastSequenceTagger 2 | from .dependency_model import SemanticDependencyParser 3 | from .language_model import LanguageModel 4 | from .text_classification_model import TextClassifier 5 | from .controller import EmbedController 6 | from .dependency_model_1 import SemanticDependencyParser1 7 | from .ensemble_model import EnsembleModel -------------------------------------------------------------------------------- /flair/models/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/biaffine_attention.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/biaffine_attention.cpython-36.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/biaffine_attention.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/biaffine_attention.cpython-37.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/biaffine_dp.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/biaffine_dp.cpython-36.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/biaffine_dp.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/biaffine_dp.cpython-37.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/controller.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/controller.cpython-36.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/controller.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/controller.cpython-37.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/dependency_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/dependency_model.cpython-36.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/dependency_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/dependency_model.cpython-37.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/dependency_model_1.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/dependency_model_1.cpython-36.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/dependency_model_1.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/dependency_model_1.cpython-37.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/ensemble_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/ensemble_model.cpython-36.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/ensemble_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/ensemble_model.cpython-37.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/language_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/language_model.cpython-36.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/language_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/language_model.cpython-37.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/mst_decoder.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/mst_decoder.cpython-36.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/mst_decoder.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/mst_decoder.cpython-37.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/sequence_tagger_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/sequence_tagger_model.cpython-36.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/sequence_tagger_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/sequence_tagger_model.cpython-37.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/text_classification_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/text_classification_model.cpython-36.pyc -------------------------------------------------------------------------------- /flair/models/__pycache__/text_classification_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/models/__pycache__/text_classification_model.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .model import Model 4 | 5 | __all__ = ['Model'] 6 | -------------------------------------------------------------------------------- /flair/parser/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/__pycache__/model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/__pycache__/model.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/__pycache__/model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/__pycache__/model.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/cmds/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .evaluate import Evaluate 4 | from .predict import Predict 5 | from .train import Train 6 | 7 | __all__ = ['Evaluate', 'Predict', 'Train'] 8 | -------------------------------------------------------------------------------- /flair/parser/cmds/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/cmds/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/cmds/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/cmds/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/cmds/__pycache__/cmd.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/cmds/__pycache__/cmd.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/cmds/__pycache__/cmd.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/cmds/__pycache__/cmd.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/cmds/__pycache__/evaluate.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/cmds/__pycache__/evaluate.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/cmds/__pycache__/evaluate.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/cmds/__pycache__/evaluate.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/cmds/__pycache__/predict.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/cmds/__pycache__/predict.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/cmds/__pycache__/train.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/cmds/__pycache__/train.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from ast import literal_eval 4 | from configparser import ConfigParser 5 | from argparse import Namespace 6 | 7 | 8 | class Config(ConfigParser): 9 | 10 | def __init__(self, path): 11 | super(Config, self).__init__() 12 | 13 | self.read(path) 14 | self.namespace = Namespace() 15 | self.update(dict((name, literal_eval(value)) 16 | for section in self.sections() 17 | for name, value in self.items(section))) 18 | 19 | def __repr__(self): 20 | s = line = "-" * 15 + "-+-" + "-" * 25 + "\n" 21 | s += f"{'Param':15} | {'Value':^25}\n" + line 22 | for name, value in vars(self.namespace).items(): 23 | s += f"{name:15} | {str(value):^25}\n" 24 | s += line 25 | 26 | return s 27 | 28 | def __getattr__(self, attr): 29 | return getattr(self.namespace, attr) 30 | 31 | def __getstate__(self): 32 | return vars(self) 33 | 34 | def __setstate__(self, state): 35 | self.__dict__.update(state) 36 | 37 | def update(self, kwargs): 38 | for name, value in kwargs.items(): 39 | setattr(self.namespace, name, value) 40 | 41 | return self 42 | -------------------------------------------------------------------------------- /flair/parser/modules/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from . import dropout 4 | from .bert import BertEmbedding 5 | from .biaffine import Biaffine 6 | from .trilinear_attention import TrilinearScorer 7 | from .bilstm import BiLSTM 8 | from .char_lstm import CHAR_LSTM 9 | from .mlp import MLP 10 | 11 | __all__ = ['CHAR_LSTM', 'MLP', 'BertEmbedding', 12 | 'Biaffine', 'BiLSTM', 'dropout', 'TrilinearScorer'] 13 | -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/bert.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/bert.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/bert.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/bert.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/biaffine.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/biaffine.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/biaffine.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/biaffine.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/bilstm.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/bilstm.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/bilstm.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/bilstm.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/char_lstm.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/char_lstm.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/char_lstm.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/char_lstm.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/dropout.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/dropout.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/dropout.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/dropout.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/mlp.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/mlp.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/mlp.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/mlp.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/scalar_mix.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/scalar_mix.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/scalar_mix.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/scalar_mix.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/trilinear_attention.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/trilinear_attention.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/modules/__pycache__/trilinear_attention.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/modules/__pycache__/trilinear_attention.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/modules/char_lstm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import torch 4 | import torch.nn as nn 5 | from torch.nn.utils.rnn import pack_padded_sequence 6 | 7 | 8 | class CHAR_LSTM(nn.Module): 9 | 10 | def __init__(self, n_chars, n_embed, n_out): 11 | super(CHAR_LSTM, self).__init__() 12 | 13 | # the embedding layer 14 | self.embed = nn.Embedding(num_embeddings=n_chars, 15 | embedding_dim=n_embed) 16 | # the lstm layer 17 | self.lstm = nn.LSTM(input_size=n_embed, 18 | hidden_size=n_out//2, 19 | batch_first=True, 20 | bidirectional=True) 21 | 22 | def forward(self, x): 23 | mask = x.gt(0) 24 | lens = mask.sum(dim=1) 25 | 26 | x = pack_padded_sequence(self.embed(x), lens, True, False) 27 | x, (hidden, _) = self.lstm(x) 28 | hidden = torch.cat(torch.unbind(hidden), dim=-1) 29 | 30 | return hidden 31 | -------------------------------------------------------------------------------- /flair/parser/modules/dropout.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import torch 4 | import torch.nn as nn 5 | import pdb 6 | 7 | class SharedDropout(nn.Module): 8 | 9 | def __init__(self, p=0.5, batch_first=True): 10 | super(SharedDropout, self).__init__() 11 | 12 | self.p = p 13 | self.batch_first = batch_first 14 | 15 | def extra_repr(self): 16 | s = f"p={self.p}" 17 | if self.batch_first: 18 | s += f", batch_first={self.batch_first}" 19 | 20 | return s 21 | 22 | def forward(self, x): 23 | if self.training: 24 | if self.batch_first: 25 | mask = self.get_mask(x[:, 0], self.p) 26 | else: 27 | mask = self.get_mask(x[0], self.p) 28 | x *= mask.unsqueeze(1) if self.batch_first else mask 29 | 30 | return x 31 | 32 | @staticmethod 33 | def get_mask(x, p): 34 | mask = x.new_empty(x.shape).bernoulli_(1 - p) 35 | mask = mask / (1 - p) 36 | 37 | return mask 38 | 39 | 40 | class IndependentDropout(nn.Module): 41 | 42 | def __init__(self, p=0.5): 43 | super(IndependentDropout, self).__init__() 44 | 45 | self.p = p 46 | 47 | def extra_repr(self): 48 | return f"p={self.p}" 49 | 50 | def forward(self, *items): 51 | if self.training: 52 | masks = [x.new_empty(x.shape[:2]).bernoulli_(1 - self.p) 53 | for x in items] 54 | total = sum(masks) 55 | scale = len(items) / total.max(torch.ones_like(total)) 56 | masks = [mask * scale for mask in masks] 57 | items = [item * mask.unsqueeze(dim=-1) 58 | for item, mask in zip(items, masks)] 59 | 60 | return items 61 | -------------------------------------------------------------------------------- /flair/parser/modules/mlp.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from flair.parser.modules.dropout import SharedDropout 4 | 5 | import torch.nn as nn 6 | 7 | 8 | class MLP(nn.Module): 9 | 10 | def __init__(self, n_in, n_hidden, dropout=0, identity = False): 11 | super(MLP, self).__init__() 12 | 13 | self.linear = nn.Linear(n_in, n_hidden) 14 | self.identity = identity 15 | if not self.identity: 16 | self.activation = nn.LeakyReLU(negative_slope=0.1) 17 | self.dropout = SharedDropout(p=dropout) 18 | 19 | self.reset_parameters() 20 | 21 | def reset_parameters(self): 22 | nn.init.orthogonal_(self.linear.weight) 23 | nn.init.zeros_(self.linear.bias) 24 | 25 | def forward(self, x): 26 | x = self.linear(x) 27 | if not self.identity: 28 | x = self.activation(x) 29 | x = self.dropout(x) 30 | 31 | return x 32 | -------------------------------------------------------------------------------- /flair/parser/modules/scalar_mix.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | 7 | class ScalarMix(nn.Module): 8 | 9 | def __init__(self, n_layers, dropout=0): 10 | super(ScalarMix, self).__init__() 11 | 12 | self.n_layers = n_layers 13 | self.dropout = dropout 14 | 15 | self.weights = nn.Parameter(torch.zeros(n_layers)) 16 | self.gamma = nn.Parameter(torch.tensor([1.0])) 17 | self.dropout = nn.Dropout(dropout) 18 | 19 | def extra_repr(self): 20 | s = f"n_layers={self.n_layers}" 21 | if self.dropout.p > 0: 22 | s += f", dropout={self.dropout.p}" 23 | 24 | return s 25 | 26 | def forward(self, tensors): 27 | normed_weights = self.dropout(self.weights.softmax(-1)) 28 | weighted_sum = sum(w * h for w, h in zip(normed_weights, tensors)) 29 | 30 | return self.gamma * weighted_sum 31 | -------------------------------------------------------------------------------- /flair/parser/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from . import corpus, data, field, fn, metric 4 | from .embedding import Embedding 5 | from .vocab import Vocab 6 | 7 | __all__ = ['Corpus', 'Embedding', 'Vocab', 8 | 'corpus', 'data', 'field', 'fn', 'metric'] 9 | -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/alg.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/alg.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/alg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/alg.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/common.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/common.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/common.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/common.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/corpus.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/corpus.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/corpus.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/corpus.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/data.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/data.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/data.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/data.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/embedding.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/embedding.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/embedding.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/embedding.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/field.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/field.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/field.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/field.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/fn.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/fn.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/fn.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/fn.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/metric.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/metric.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/metric.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/metric.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/vocab.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/vocab.cpython-36.pyc -------------------------------------------------------------------------------- /flair/parser/utils/__pycache__/vocab.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/parser/utils/__pycache__/vocab.cpython-37.pyc -------------------------------------------------------------------------------- /flair/parser/utils/common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | pad = '' 4 | unk = '' 5 | bos = '' 6 | eos = '' 7 | -------------------------------------------------------------------------------- /flair/parser/utils/embedding.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import torch 4 | 5 | 6 | class Embedding(object): 7 | 8 | def __init__(self, tokens, vectors, unk=None): 9 | super(Embedding, self).__init__() 10 | 11 | self.tokens = tokens 12 | self.vectors = torch.tensor(vectors) 13 | self.pretrained = {w: v for w, v in zip(tokens, vectors)} 14 | self.unk = unk 15 | 16 | def __len__(self): 17 | return len(self.tokens) 18 | 19 | def __contains__(self, token): 20 | return token in self.pretrained 21 | 22 | @property 23 | def dim(self): 24 | return self.vectors.size(1) 25 | 26 | @property 27 | def unk_index(self): 28 | if self.unk is not None: 29 | return self.tokens.index(self.unk) 30 | else: 31 | raise AttributeError 32 | 33 | @classmethod 34 | def load(cls, path, unk=None): 35 | with open(path, 'r') as f: 36 | lines = [line for line in f] 37 | splits = [line.split() for line in lines] 38 | tokens, vectors = zip(*[(s[0], list(map(float, s[1:]))) 39 | for s in splits]) 40 | 41 | return cls(tokens, vectors, unk=unk) 42 | -------------------------------------------------------------------------------- /flair/parser/utils/vocab.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from collections.abc import Iterable 4 | from flair.parser.utils.common import unk 5 | 6 | 7 | class Vocab(object): 8 | 9 | def __init__(self, counter, min_freq=1, specials=[]): 10 | self.itos = specials 11 | self.stoi = {token: i for i, token in enumerate(self.itos)} 12 | 13 | self.extend([token for token, freq in counter.items() 14 | if freq >= min_freq]) 15 | self.unk_index = self.stoi.get(unk, 0) 16 | self.n_init = len(self) 17 | 18 | def __len__(self): 19 | return len(self.itos) 20 | 21 | def __getitem__(self, key): 22 | return self.stoi[key] 23 | 24 | def __contains__(self, token): 25 | return token in self.stoi 26 | 27 | def token2id(self, sequence): 28 | return [self.stoi.get(token, self.unk_index) for token in sequence] 29 | 30 | def id2token(self, ids): 31 | if isinstance(ids, Iterable): 32 | return [self.itos[i] for i in ids] 33 | else: 34 | return self.itos[ids] 35 | 36 | def extend(self, tokens): 37 | self.itos.extend(sorted(set(tokens).difference(self.stoi))) 38 | self.stoi = {token: i for i, token in enumerate(self.itos)} 39 | -------------------------------------------------------------------------------- /flair/trainers/.finetune_trainer.py.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/.finetune_trainer.py.swp -------------------------------------------------------------------------------- /flair/trainers/__init__.py: -------------------------------------------------------------------------------- 1 | from .trainer import ModelTrainer 2 | from .distillation_trainer import ModelDistiller 3 | from .finetune_trainer import ModelFinetuner 4 | from .reinforcement_trainer import ReinforcementTrainer 5 | from .swaf_trainer import SWAFTrainer -------------------------------------------------------------------------------- /flair/trainers/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /flair/trainers/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /flair/trainers/__pycache__/distillation_trainer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/distillation_trainer.cpython-36.pyc -------------------------------------------------------------------------------- /flair/trainers/__pycache__/distillation_trainer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/distillation_trainer.cpython-37.pyc -------------------------------------------------------------------------------- /flair/trainers/__pycache__/finetune_trainer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/finetune_trainer.cpython-36.pyc -------------------------------------------------------------------------------- /flair/trainers/__pycache__/finetune_trainer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/finetune_trainer.cpython-37.pyc -------------------------------------------------------------------------------- /flair/trainers/__pycache__/reinforcement_trainer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/reinforcement_trainer.cpython-36.pyc -------------------------------------------------------------------------------- /flair/trainers/__pycache__/reinforcement_trainer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/reinforcement_trainer.cpython-37.pyc -------------------------------------------------------------------------------- /flair/trainers/__pycache__/swaf_trainer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/swaf_trainer.cpython-36.pyc -------------------------------------------------------------------------------- /flair/trainers/__pycache__/swaf_trainer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/swaf_trainer.cpython-37.pyc -------------------------------------------------------------------------------- /flair/trainers/__pycache__/trainer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/trainer.cpython-36.pyc -------------------------------------------------------------------------------- /flair/trainers/__pycache__/trainer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/trainers/__pycache__/trainer.cpython-37.pyc -------------------------------------------------------------------------------- /flair/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .exception_hook import ExceptionHook 2 | from typing import Iterator, List, TypeVar, Iterable, Dict 3 | import random 4 | from itertools import zip_longest, islice 5 | 6 | A = TypeVar('A') 7 | def lazy_groups_of(iterator: Iterator[A], group_size: int) -> Iterator[List[A]]: 8 | """ 9 | Takes an iterator and batches the invididual instances into lists of the 10 | specified size. The last list may be smaller if there are instances left over. 11 | """ 12 | return iter(lambda: list(islice(iterator, 0, group_size)), []) 13 | 14 | def ensure_list(iterable: Iterable[A]) -> List[A]: 15 | """ 16 | An Iterable may be a list or a generator. 17 | This ensures we get a list without making an unnecessary copy. 18 | """ 19 | if isinstance(iterable, list): 20 | return iterable 21 | else: 22 | return list(iterable) 23 | 24 | def is_lazy(iterable: Iterable[A]) -> bool: 25 | """ 26 | Checks if the given iterable is lazy, 27 | which here just means it's not a list. 28 | """ 29 | return not isinstance(iterable, list) 30 | 31 | def add_noise_to_dict_values(dictionary: Dict[A, float], noise_param: float) -> Dict[A, float]: 32 | """ 33 | Returns a new dictionary with noise added to every key in ``dictionary``. The noise is 34 | uniformly distributed within ``noise_param`` percent of the value for every value in the 35 | dictionary. 36 | """ 37 | new_dict = {} 38 | for key, value in dictionary.items(): 39 | noise_value = value * noise_param 40 | noise = random.uniform(-noise_value, noise_value) 41 | new_dict[key] = value + noise 42 | return new_dict 43 | -------------------------------------------------------------------------------- /flair/utils/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/archival.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/archival.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/checks.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/checks.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/checks.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/checks.cpython-37.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/environment.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/environment.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/exception_hook.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/exception_hook.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/exception_hook.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/exception_hook.cpython-37.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/file.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/file.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/from_params.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/from_params.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/from_params.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/from_params.cpython-37.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/logging.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/logging.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/logging.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/logging.cpython-37.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/nn.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/nn.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/params.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/params.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/params.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/params.cpython-37.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/registrable.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/registrable.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/string.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/string.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/time.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/time.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/__pycache__/tqdm.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/utils/__pycache__/tqdm.cpython-36.pyc -------------------------------------------------------------------------------- /flair/utils/checks.py: -------------------------------------------------------------------------------- 1 | """ 2 | Adopted from AllenNLP: 3 | https://github.com/allenai/allennlp/tree/v0.6.1/allennlp/common 4 | 5 | Functions and exceptions for checking that 6 | AllenNLP and its models are configured correctly. 7 | """ 8 | 9 | from torch import cuda 10 | 11 | from flair.utils import logging 12 | 13 | logger = logging.init_logger() # pylint: disable=invalid-name 14 | 15 | 16 | class ConfigurationError(Exception): 17 | """ 18 | The exception raised by any AllenNLP object when it's misconfigured 19 | (e.g. missing properties, invalid properties, unknown properties). 20 | """ 21 | 22 | def __init__(self, message): 23 | super(ConfigurationError, self).__init__() 24 | self.message = message 25 | 26 | def __str__(self): 27 | return repr(self.message) 28 | 29 | 30 | def log_pytorch_version_info(): 31 | import torch 32 | logger.info("Pytorch version: %s", torch.__version__) 33 | 34 | 35 | def check_dimensions_match(dimension_1: int, 36 | dimension_2: int, 37 | dim_1_name: str, 38 | dim_2_name: str) -> None: 39 | if dimension_1 != dimension_2: 40 | raise ConfigurationError(f"{dim_1_name} must match {dim_2_name}, but got {dimension_1} " 41 | f"and {dimension_2} instead") 42 | 43 | 44 | def check_for_gpu(device_id: int): 45 | if device_id is not None and device_id >= cuda.device_count(): 46 | raise ConfigurationError("Experiment specified a GPU but none is available;" 47 | " if you want to run on CPU use the override" 48 | " 'trainer.cuda_device=-1' in the json config file.") 49 | -------------------------------------------------------------------------------- /flair/utils/exception_hook.py: -------------------------------------------------------------------------------- 1 | class ExceptionHook: 2 | instance = None 3 | def __call__(self, *args, **kwargs): 4 | if self.instance is None: 5 | from IPython.core import ultratb 6 | self.instance = ultratb.FormattedTB(mode="Plain", color_scheme="Linux", call_pdb=1) 7 | return self.instance(*args, **kwargs) 8 | 9 | -------------------------------------------------------------------------------- /flair/utils/extract_tokens_from_amr.py: -------------------------------------------------------------------------------- 1 | from stog.data.dataset_readers import AbstractMeaningRepresentationDatasetReader 2 | import sys 3 | from stog.utils import logging 4 | 5 | logger = logging.init_logger() 6 | def extract_amr_token(file_path): 7 | dataset_reader = AbstractMeaningRepresentationDatasetReader() 8 | for instance in dataset_reader.read(file_path): 9 | amr_tokens = instance.fields["amr_tokens"]["decoder_tokens"] 10 | yield " ".join(amr_tokens) 11 | 12 | 13 | if __name__ == "__main__": 14 | if len(sys.argv) < 2: 15 | print("""Usage: 16 | python {} [amr_file] 17 | 18 | The output will in stdout. 19 | """) 20 | for filename in sys.argv[1:]: 21 | for line in extract_amr_token(filename): 22 | sys.stdout.write(line + "\n") 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /flair/utils/time.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | 4 | def time_to_str(timestamp: int) -> str: 5 | """ 6 | Convert seconds past Epoch to human readable string. 7 | """ 8 | datetimestamp = datetime.datetime.fromtimestamp(timestamp) 9 | return '{:04d}-{:02d}-{:02d}-{:02d}-{:02d}-{:02d}'.format( 10 | datetimestamp.year, datetimestamp.month, datetimestamp.day, 11 | datetimestamp.hour, datetimestamp.minute, datetimestamp.second 12 | ) 13 | 14 | 15 | def str_to_time(time_str: str) -> datetime.datetime: 16 | """ 17 | Convert human readable string to datetime.datetime. 18 | """ 19 | pieces = [int(piece) for piece in time_str.split('-')] 20 | return datetime.datetime(*pieces) 21 | -------------------------------------------------------------------------------- /flair/visual/__init__.py: -------------------------------------------------------------------------------- 1 | from .manifold import Visualizer 2 | from .activations import Highlighter 3 | -------------------------------------------------------------------------------- /flair/visual/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/visual/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /flair/visual/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/visual/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /flair/visual/__pycache__/activations.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/visual/__pycache__/activations.cpython-36.pyc -------------------------------------------------------------------------------- /flair/visual/__pycache__/activations.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/visual/__pycache__/activations.cpython-37.pyc -------------------------------------------------------------------------------- /flair/visual/__pycache__/manifold.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/visual/__pycache__/manifold.cpython-36.pyc -------------------------------------------------------------------------------- /flair/visual/__pycache__/manifold.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/visual/__pycache__/manifold.cpython-37.pyc -------------------------------------------------------------------------------- /flair/visual/__pycache__/training_curves.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/visual/__pycache__/training_curves.cpython-36.pyc -------------------------------------------------------------------------------- /flair/visual/__pycache__/training_curves.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/flair/visual/__pycache__/training_curves.cpython-37.pyc -------------------------------------------------------------------------------- /flair/visual/html_templates.py: -------------------------------------------------------------------------------- 1 | TAGGED_ENTITY = ''' 2 | 3 | {entity} 4 | {label} 5 | 6 | ''' 7 | 8 | HTML_PAGE = """ 9 | 10 | 11 | 12 | Flair 13 | 14 | 15 | {text} 16 | 17 | """ 18 | -------------------------------------------------------------------------------- /resources/taggers/ast_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/resources/taggers/ast_tags.pkl -------------------------------------------------------------------------------- /resources/taggers/ast_tags_new.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/resources/taggers/ast_tags_new.pkl -------------------------------------------------------------------------------- /resources/taggers/dependency_projective_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/resources/taggers/dependency_projective_tags.pkl -------------------------------------------------------------------------------- /resources/taggers/dm_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/resources/taggers/dm_tags.pkl -------------------------------------------------------------------------------- /resources/taggers/enhancedud_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/resources/taggers/enhancedud_tags.pkl -------------------------------------------------------------------------------- /resources/taggers/ner_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/resources/taggers/ner_tags.pkl -------------------------------------------------------------------------------- /resources/taggers/np_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/resources/taggers/np_tags.pkl -------------------------------------------------------------------------------- /resources/taggers/pos_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/resources/taggers/pos_tags.pkl -------------------------------------------------------------------------------- /resources/taggers/ud_dependency_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/resources/taggers/ud_dependency_tags.pkl -------------------------------------------------------------------------------- /script/__pycache__/evaluate.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/script/__pycache__/evaluate.cpython-37.pyc -------------------------------------------------------------------------------- /script/__pycache__/semdep_eval.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/script/__pycache__/semdep_eval.cpython-37.pyc -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pathlib import Path 3 | 4 | 5 | @pytest.fixture(scope="module") 6 | def resources_path(): 7 | return Path(__file__).parent / "resources" 8 | 9 | 10 | @pytest.fixture(scope="module") 11 | def tasks_base_path(resources_path): 12 | return resources_path / "tasks" 13 | 14 | 15 | @pytest.fixture(scope="module") 16 | def results_base_path(resources_path): 17 | return resources_path / "results" 18 | 19 | 20 | def pytest_addoption(parser): 21 | parser.addoption( 22 | "--runslow", action="store_true", default=False, help="run slow tests" 23 | ) 24 | parser.addoption( 25 | "--runintegration", 26 | action="store_true", 27 | default=False, 28 | help="run integration tests", 29 | ) 30 | 31 | 32 | def pytest_collection_modifyitems(config, items): 33 | if config.getoption("--runslow") and config.getoption("--runintegration"): 34 | return 35 | 36 | if not config.getoption("--runslow"): 37 | skip_slow = pytest.mark.skip(reason="need --runslow option to run") 38 | for item in items: 39 | if "slow" in item.keywords: 40 | item.add_marker(skip_slow) 41 | 42 | if not config.getoption("--runintegration"): 43 | skip_integration = pytest.mark.skip( 44 | reason="need --runintegration option to run" 45 | ) 46 | for item in items: 47 | if "integration" in item.keywords: 48 | item.add_marker(skip_integration) 49 | -------------------------------------------------------------------------------- /tests/resources/corpora/lorem_ipsum/test.txt: -------------------------------------------------------------------------------- 1 | Adipiscing commodo elit at imperdiet. Consequat interdum varius sit amet mattis vulputate enim nulla. Nulla aliquet porttitor lacus luctus accumsan tortor. Curabitur gravida arcu ac tortor. Adipiscing elit pellentesque habitant morbi. Sed viverra tellus in hac habitasse platea dictumst. Turpis cursus in hac habitasse. Pharetra vel turpis nunc eget. Enim facilisis gravida neque convallis a cras semper auctor neque. Interdum posuere lorem ipsum dolor sit amet consectetur adipiscing elit. 2 | 3 | Mauris sit amet massa vitae tortor condimentum lacinia. Neque gravida in fermentum et sollicitudin. Blandit volutpat maecenas volutpat blandit aliquam. Gravida neque convallis a cras semper auctor neque vitae. Viverra aliquet eget sit amet tellus cras adipiscing enim eu. Risus sed vulputate odio ut enim blandit volutpat maecenas. Amet tellus cras adipiscing enim eu. Viverra tellus in hac habitasse platea dictumst vestibulum rhoncus est. Magna etiam tempor orci eu lobortis elementum. Leo vel fringilla est ullamcorper eget. Nisl nisi scelerisque eu ultrices. Eros donec ac odio tempor orci dapibus ultrices in. Nisl nisi scelerisque eu ultrices vitae auctor eu augue. Hac habitasse platea dictumst vestibulum rhoncus est pellentesque elit. Habitasse platea dictumst vestibulum rhoncus est pellentesque elit. In ornare quam viverra orci sagittis. Morbi quis commodo odio aenean. Nam at lectus urna duis convallis convallis tellus id interdum. -------------------------------------------------------------------------------- /tests/resources/corpora/lorem_ipsum/valid.txt: -------------------------------------------------------------------------------- 1 | Nulla at volutpat diam ut venenatis tellus in metus vulputate. Porttitor leo a diam sollicitudin tempor. Tincidunt vitae semper quis lectus nulla at volutpat diam. Ornare aenean euismod elementum nisi quis eleifend quam adipiscing. Tortor pretium viverra suspendisse potenti. Arcu risus quis varius quam quisque id. Non sodales neque sodales ut etiam sit amet nisl. Porttitor lacus luctus accumsan tortor posuere ac ut consequat. Diam sit amet nisl suscipit. Ut sem nulla pharetra diam sit amet nisl suscipit adipiscing. Varius quam quisque id diam. Elementum tempus egestas sed sed risus pretium quam vulputate. Eu ultrices vitae auctor eu augue ut lectus. Tincidunt id aliquet risus feugiat in ante metus dictum at. Mauris cursus mattis molestie a iaculis at erat pellentesque. Leo urna molestie at elementum eu. 2 | 3 | Posuere morbi leo urna molestie. Tincidunt nunc pulvinar sapien et. Mattis molestie a iaculis at erat pellentesque. Arcu cursus euismod quis viverra nibh cras pulvinar mattis nunc. Phasellus vestibulum lorem sed risus ultricies tristique nulla aliquet enim. Aenean et tortor at risus viverra. Ut placerat orci nulla pellentesque dignissim. Est lorem ipsum dolor sit amet. Eros donec ac odio tempor. Elementum integer enim neque volutpat ac tincidunt vitae. -------------------------------------------------------------------------------- /tests/resources/tasks/ag_news/README.md: -------------------------------------------------------------------------------- 1 | ## AG_NEWS 2 | 3 | Data is taken from [here](https://www.di.unipi.it/~gulli/AG_corpus_of_news_articles.html). 4 | 5 | The dataset contains a collection of news articles grouped into different categories. 6 | We took a small random sample and converted it to the expected format of our data fetcher: 7 | ``` 8 | __label__ 9 | ``` 10 | 11 | #### Publications Using the Dataset 12 | 13 | * G. M. Del Corso, A. Gulli, and F. Romani. Ranking a stream of news. In Proceedings of 14th International World Wide Web Conference, pages 97–106, Chiba, Japan, 2005. 14 | * A. Gulli. The anatomy of a news search engine. In Proceedings of 14th International World Wide Web Conference, pages 880–881, Chiba, Japan, 2005. -------------------------------------------------------------------------------- /tests/resources/tasks/fashion/eng.testa: -------------------------------------------------------------------------------- 1 | Most _ _ O 2 | wedding _ _ B-Occasion 3 | dresses _ _ B-NominalProduct 4 | , _ _ O 5 | for _ _ O 6 | example _ _ O 7 | , _ _ O 8 | are _ _ O 9 | simply _ _ O 10 | too _ _ O 11 | enormous _ _ O 12 | and _ _ O 13 | terrifyingly _ _ O 14 | loaded _ _ O 15 | with _ _ O 16 | sentimental _ _ O 17 | value _ _ O 18 | for _ _ O 19 | DIY _ _ B-ProductDesign 20 | dyeing _ _ I-ProductDesign 21 | . _ _ O -------------------------------------------------------------------------------- /tests/resources/tasks/fashion/eng.testb: -------------------------------------------------------------------------------- 1 | For _ _ O 2 | my _ _ O 3 | Nurse _ _ B-NamedOrganizationBrand 4 | Ratched _ _ I-NamedOrganizationBrand 5 | dress _ _ B-NominalProduct 6 | , _ _ O 7 | I _ _ O 8 | had _ _ O 9 | brought _ _ O 10 | two _ _ O 11 | dyeing _ _ O 12 | options _ _ O 13 | — _ _ O 14 | one _ _ O 15 | more _ _ O 16 | ambitious _ _ O 17 | than _ _ O 18 | the _ _ O 19 | other _ _ O 20 | . _ _ O -------------------------------------------------------------------------------- /tests/resources/tasks/fashion_nodev/test.tsv: -------------------------------------------------------------------------------- 1 | Most _ _ O 2 | wedding _ _ B-Occasion 3 | dresses _ _ B-NominalProduct 4 | , _ _ O 5 | for _ _ O 6 | example _ _ O 7 | , _ _ O 8 | are _ _ O 9 | simply _ _ O 10 | too _ _ O 11 | enormous _ _ O 12 | and _ _ O 13 | terrifyingly _ _ O 14 | loaded _ _ O 15 | with _ _ O 16 | sentimental _ _ O 17 | value _ _ O 18 | for _ _ O 19 | DIY _ _ B-ProductDesign 20 | dyeing _ _ I-ProductDesign 21 | . _ _ O -------------------------------------------------------------------------------- /tests/resources/tasks/germeval/NER-de-dev.tsv: -------------------------------------------------------------------------------- 1 | # http://de.wikipedia.org/wiki/Toyota_Crown [2009-08-13] 2 | 1 1980 O O 3 | 2 kam O O 4 | 3 der O O 5 | 4 Crown B-OTH O 6 | 5 als O O 7 | 6 Versuch O O 8 | 7 von O O 9 | 8 Toyota B-ORG O 10 | 9 , O O 11 | 10 sich O O 12 | 11 in O O 13 | 12 der O O 14 | 13 Oberen O O 15 | 14 Mittelklasse O O 16 | 15 zu O O 17 | 16 etablieren O O 18 | 17 , O O 19 | 18 auch O O 20 | 19 nach O O 21 | 20 Deutschland B-LOC O 22 | 21 . O O 23 | -------------------------------------------------------------------------------- /tests/resources/tasks/germeval/NER-de-test.tsv: -------------------------------------------------------------------------------- 1 | # http://de.wikipedia.org/wiki/Schönburg_(Rhein) [2009-10-23] 2 | 1 1951 O O 3 | 2 bis O O 4 | 3 1953 O O 5 | 4 wurde O O 6 | 5 der O O 7 | 6 nördliche O O 8 | 7 Teil O O 9 | 8 als O O 10 | 9 Jugendburg O O 11 | 10 des O O 12 | 11 Kolpingwerkes B-OTH O 13 | 12 gebaut O O 14 | 13 . O O -------------------------------------------------------------------------------- /tests/resources/tasks/germeval/NER-de-train.tsv: -------------------------------------------------------------------------------- 1 | # n-tv.de vom 26.02.2005 [2005-02-26] 2 | 1 Schartau B-PER O 3 | 2 sagte O O 4 | 3 dem O O 5 | 4 " O O 6 | 5 Tagesspiegel B-ORG O 7 | 6 " O O 8 | 7 vom O O 9 | 8 Freitag O O 10 | 9 , O O 11 | 10 Fischer B-PER O 12 | 11 sei O O 13 | 12 " O O 14 | 13 in O O 15 | 14 einer O O 16 | 15 Weise O O 17 | 16 aufgetreten O O 18 | 17 , O O 19 | 18 die O O 20 | 19 alles O O 21 | 20 andere O O 22 | 21 als O O 23 | 22 überzeugend O O 24 | 23 war O O 25 | 24 " O O 26 | 25 . O O 27 | 28 | # welt.de vom 29.10.2005 [2005-10-29] 29 | 1 Firmengründer O O 30 | 2 Wolf B-PER O 31 | 3 Peter I-PER O 32 | 4 Bree I-PER O 33 | 5 arbeitete O O 34 | 6 Anfang O O 35 | 7 der O O 36 | 8 siebziger O O 37 | 9 Jahre O O 38 | 10 als O O 39 | 11 Möbelvertreter O O 40 | 12 , O O 41 | 13 als O O 42 | 14 er O O 43 | 15 einen O O 44 | 16 fliegenden O O 45 | 17 Händler O O 46 | 18 aus O O 47 | 19 dem O O 48 | 20 Libanon B-LOC O 49 | 21 traf O O 50 | 22 . O O -------------------------------------------------------------------------------- /tests/resources/tasks/imdb/README.md: -------------------------------------------------------------------------------- 1 | ## IMDB 2 | 3 | Data is taken from [here](http://ai.stanford.edu/~amaas/data/sentiment/). 4 | 5 | The dataset contains data for a binary sentiment classification. 6 | We took a small random sample and converted it to the expected format of our data fetcher: 7 | ``` 8 | __label__ 9 | ``` 10 | 11 | #### Publications Using the Dataset 12 | 13 | * Andrew L. Maas, Raymond E. Daly, Peter T. Pham, Dan Huang, Andrew Y. Ng, and Christopher Potts. (2011). Learning Word Vectors for Sentiment Analysis. The 49th Annual Meeting of the Association for Computational Linguistics (ACL 2011). -------------------------------------------------------------------------------- /tests/resources/tasks/multi_class/dev.txt: -------------------------------------------------------------------------------- 1 | __label__apple apple 2 | __label__tv tv 3 | __label__guitar guitar 4 | __label__apple __label__tv apple tv -------------------------------------------------------------------------------- /tests/resources/tasks/multi_class/test.txt: -------------------------------------------------------------------------------- 1 | __label__guitar guitar 2 | __label__apple apple 3 | __label__tv tv 4 | __label__apple __label__tv apple tv 5 | __label__apple __label__guitar apple tv -------------------------------------------------------------------------------- /tests/resources/tasks/multi_class/train.txt: -------------------------------------------------------------------------------- 1 | __label__tv tv 2 | __label__apple __label__tv apple tv 3 | __label__apple apple 4 | __label__tv tv 5 | __label__apple __label__tv apple tv 6 | __label__guitar guitar 7 | __label__guitar guitar -------------------------------------------------------------------------------- /tests/resources/tasks/regression/README.md: -------------------------------------------------------------------------------- 1 | ## REGRESSION 2 | 3 | Data is taken from [here](http://saifmohammad.com/WebPages/EmotionIntensity-SharedTask.html). 4 | 5 | The dataset contains a collection of tweets with joy intensity value. 6 | We took the joy dataset and converted it to the expected format of our data fetcher: 7 | ``` 8 | __label__ 9 | ``` 10 | 11 | #### Publication About the Dataset 12 | 13 | * Emotion Intensities in Tweets. Saif M. Mohammad and Felipe Bravo-Marquez. In Proceedings of the sixth joint conference on lexical and computational semantics (*Sem), August 2017, Vancouver, Canada. 14 | * WASSA-2017 Shared Task on Emotion Intensity. Saif M. Mohammad and Felipe Bravo-Marquez. In Proceedings of the EMNLP 2017 Workshop on Computational Approaches to Subjectivity, Sentiment, and Social Media (WASSA), September 2017, Copenhagen, Denmark. -------------------------------------------------------------------------------- /tests/resources/visual/snippet.txt: -------------------------------------------------------------------------------- 1 | The U.S. Centers for Disease Control and Prevention initially advised school systems to close if outbreaks occurred , then reversed itself , saying the apparent mildness of the virus meant most schools and day care centers should stay open , even if they had confirmed cases of swine flu . 2 | When Ms. Winfrey invited Suzanne Somers to share her controversial views about bio-identical hormone treatment on her syndicated show in 2009 , it won Ms. Winfrey a rare dollop of unflattering press , including a Newsweek cover story titled " Crazy Talk : Oprah , Wacky Cures & You . " 3 | Elk calling -- a skill that hunters perfected long ago to lure game with the promise of a little romance -- is now its own sport . 4 | Don 't ! 5 | Fish , ranked 98th in the world , fired 22 aces en route to a 6-3 , 6-7 ( 5 / 7 ) , 7-6 ( 7 / 4 ) win over seventh-seeded Argentinian David Nalbandian . 6 | Why does everything have to become such a big issue ? 7 | AMMAN ( Reuters ) - King Abdullah of Jordan will meet U.S. President Barack Obama in Washington on April 21 to lobby on behalf of Arab states for a stronger U.S. role in Middle East peacemaking , palace officials said on Sunday . 8 | To help keep traffic flowing the Congestion Charge will remain in operation through-out the strike and TfL will be suspending road works on major London roads wherever possible . 9 | If no candidate wins an absolute majority , there will be a runoff between the top two contenders , most likely in mid-October . 10 | Authorities previously served search warrants at Murray 's Las Vegas home and his businesses in Las Vegas and Houston . -------------------------------------------------------------------------------- /tools/split.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import random 3 | 4 | # percent = 0.5 5 | file_path = 'conll_03_vi.train_bioes' 6 | percent = 0.8 7 | file_path_out = 'conll_03_vietnamese.train_bioes'.format(percent) 8 | file_out = open(file_path_out, 'w', encoding='utf-8') 9 | with open(file_path, 'r', encoding='utf-8') as file: 10 | lines = file.readlines() 11 | all_samples = [] 12 | sample = [] 13 | for line in lines: 14 | if line == '\n': 15 | all_samples.append(sample) 16 | sample = [] 17 | continue 18 | sample.append(line) 19 | # file_out.write('\n') 20 | 21 | random.shuffle(all_samples) 22 | train_sents = int(percent * len(all_samples)) 23 | dev_sents = int(0.9 * len(all_samples)) 24 | test_sents = len(all_samples) - dev_sents 25 | train_samples = all_samples[:train_sents] 26 | print(f'train: 0-{train_sents}, dev: {train_sents}-{dev_sents}, test: {train_sents}-{len(all_samples)}') 27 | for sample in train_samples: 28 | for line in sample: 29 | file_out.write(line) 30 | file_out.write('\n') 31 | 32 | dev_file = 'conll_03_vietnamese.testa_bioes' 33 | dev_file = open(dev_file, 'w', encoding='utf-8') 34 | for sample in all_samples[train_sents: dev_sents]: 35 | for line in sample: 36 | dev_file.write(line) 37 | dev_file.write('\n') 38 | 39 | test_file = 'conll_03_vietnamese.testb_bioes' 40 | test_file = open(test_file, 'w', encoding='utf-8') 41 | for sample in all_samples[dev_sents:]: 42 | for line in sample: 43 | test_file.write(line) 44 | test_file.write('\n') 45 | 46 | 47 | file_out.close() 48 | test_file.close() 49 | dev_file.close() -------------------------------------------------------------------------------- /utils/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Flair is licensed under the following MIT License (MIT) Copyright © 2018 Zalando SE, https://tech.zalando.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .exception_hook import ExceptionHook 2 | from typing import Iterator, List, TypeVar, Iterable, Dict 3 | import random 4 | from itertools import zip_longest, islice 5 | 6 | A = TypeVar('A') 7 | def lazy_groups_of(iterator: Iterator[A], group_size: int) -> Iterator[List[A]]: 8 | """ 9 | Takes an iterator and batches the invididual instances into lists of the 10 | specified size. The last list may be smaller if there are instances left over. 11 | """ 12 | return iter(lambda: list(islice(iterator, 0, group_size)), []) 13 | 14 | def ensure_list(iterable: Iterable[A]) -> List[A]: 15 | """ 16 | An Iterable may be a list or a generator. 17 | This ensures we get a list without making an unnecessary copy. 18 | """ 19 | if isinstance(iterable, list): 20 | return iterable 21 | else: 22 | return list(iterable) 23 | 24 | def is_lazy(iterable: Iterable[A]) -> bool: 25 | """ 26 | Checks if the given iterable is lazy, 27 | which here just means it's not a list. 28 | """ 29 | return not isinstance(iterable, list) 30 | 31 | def add_noise_to_dict_values(dictionary: Dict[A, float], noise_param: float) -> Dict[A, float]: 32 | """ 33 | Returns a new dictionary with noise added to every key in ``dictionary``. The noise is 34 | uniformly distributed within ``noise_param`` percent of the value for every value in the 35 | dictionary. 36 | """ 37 | new_dict = {} 38 | for key, value in dictionary.items(): 39 | noise_value = value * noise_param 40 | noise = random.uniform(-noise_value, noise_value) 41 | new_dict[key] = value + noise 42 | return new_dict 43 | -------------------------------------------------------------------------------- /utils/__pycache__/conlleval.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/__pycache__/conlleval.cpython-37.pyc -------------------------------------------------------------------------------- /utils/algorithms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/algorithms/__init__.py -------------------------------------------------------------------------------- /utils/algorithms/dict_merge.py: -------------------------------------------------------------------------------- 1 | # Recursive dictionary merge 2 | # Copyright (C) 2016 Paul Durivage 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program. If not, see . 16 | 17 | # Found here https://gist.github.com/angstwad/bf22d1822c38a92ec0a9 18 | # Using jpopelka's modified solution. 19 | 20 | import collections 21 | 22 | def dict_merge(dct, merge_dct): 23 | """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of 24 | updating only top-level keys, dict_merge recurses down into dicts nested 25 | to an arbitrary depth, updating keys. The ``merge_dct`` is merged into 26 | ``dct``. 27 | 28 | :param dct: dict onto which the merge is executed 29 | :param merge_dct: dct merged into dct 30 | :return: None 31 | """ 32 | for k, v in merge_dct.items(): 33 | if isinstance(dct.get(k), dict) and isinstance(v, collections.Mapping): 34 | dict_merge(dct[k], v) 35 | else: 36 | dct[k] = v 37 | -------------------------------------------------------------------------------- /utils/checks.py: -------------------------------------------------------------------------------- 1 | """ 2 | Adopted from AllenNLP: 3 | https://github.com/allenai/allennlp/tree/v0.6.1/allennlp/common 4 | 5 | Functions and exceptions for checking that 6 | AllenNLP and its models are configured correctly. 7 | """ 8 | 9 | from torch import cuda 10 | 11 | from utils import logging 12 | 13 | logger = logging.init_logger() # pylint: disable=invalid-name 14 | 15 | 16 | class ConfigurationError(Exception): 17 | """ 18 | The exception raised by any AllenNLP object when it's misconfigured 19 | (e.g. missing properties, invalid properties, unknown properties). 20 | """ 21 | 22 | def __init__(self, message): 23 | super(ConfigurationError, self).__init__() 24 | self.message = message 25 | 26 | def __str__(self): 27 | return repr(self.message) 28 | 29 | 30 | def log_pytorch_version_info(): 31 | import torch 32 | logger.info("Pytorch version: %s", torch.__version__) 33 | 34 | 35 | def check_dimensions_match(dimension_1: int, 36 | dimension_2: int, 37 | dim_1_name: str, 38 | dim_2_name: str) -> None: 39 | if dimension_1 != dimension_2: 40 | raise ConfigurationError(f"{dim_1_name} must match {dim_2_name}, but got {dimension_1} " 41 | f"and {dimension_2} instead") 42 | 43 | 44 | def check_for_gpu(device_id: int): 45 | if device_id is not None and device_id >= cuda.device_count(): 46 | raise ConfigurationError("Experiment specified a GPU but none is available;" 47 | " if you want to run on CPU use the override" 48 | " 'trainer.cuda_device=-1' in the json config file.") 49 | -------------------------------------------------------------------------------- /utils/config/roberta-doc-en-ner-finetune.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | assign_doc_id: true 3 | distill_mode: false 4 | sentence_level_batch: true 5 | train_with_doc: true 6 | anneal_factor: 2 7 | embeddings: 8 | TransformerWordEmbeddings-0: 9 | fine_tune: true 10 | layers: '-1' 11 | model: roberta-large 12 | pooling_operation: first 13 | v2_doc: true 14 | interpolation: 0.5 15 | is_teacher_list: true 16 | model: 17 | FastSequenceTagger: 18 | crf_attention: false 19 | dropout: 0.0 20 | hidden_size: 256 21 | locked_dropout: 0.0 22 | sentence_loss: true 23 | use_cnn: false 24 | use_crf: false 25 | use_rnn: false 26 | word_dropout: 0.1 27 | model_name: en-xlmr-first-docv2_10epoch_1batch_4accumulate_0.000005lr_10000lrrate_eng_monolingual_nocrf_fast_norelearn_sentbatch_sentloss_finetune_nodev_saving_ner5 28 | ner: 29 | Corpus: CONLL_03_ENGLISH 30 | tag_dictionary: resources/taggers/ner_tags.pkl 31 | target_dir: resources/taggers/ 32 | targets: ner 33 | teacher_annealing: false 34 | train: 35 | embeddings_storage_mode: none 36 | fine_tune_mode: true 37 | gradient_accumulation_steps: 4 38 | learning_rate: 5.0e-06 39 | lr_rate: 10000 40 | max_epochs: 10 41 | mini_batch_size: 1 42 | monitor_test: false 43 | save_final_model: false 44 | save_finetuned_embedding: true 45 | train_with_dev: false 46 | true_reshuffle: false 47 | use_warmup: false 48 | trainer: ModelFinetuner 49 | -------------------------------------------------------------------------------- /utils/config/xlmr-doc-de-03-ner-finetune.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | assign_doc_id: true 3 | distill_mode: false 4 | sentence_level_batch: true 5 | train_with_doc: true 6 | anneal_factor: 2 7 | ast: 8 | Corpus: SEMEVAL16-TR:SEMEVAL16-ES:SEMEVAL16-NL:SEMEVAL16-EN:SEMEVAL16-RU 9 | atis: 10 | Corpus: ATIS-EN:ATIS-TR:ATIS-HI 11 | chunk: 12 | Corpus: CONLL_03:CONLL_03_GERMAN 13 | embeddings: 14 | TransformerWordEmbeddings-0: 15 | fine_tune: true 16 | layers: '-1' 17 | model: xlm-roberta-large 18 | pooling_operation: first 19 | v2_doc: true 20 | interpolation: 0.5 21 | is_teacher_list: true 22 | model: 23 | FastSequenceTagger: 24 | crf_attention: false 25 | dropout: 0.0 26 | hidden_size: 256 27 | locked_dropout: 0.0 28 | sentence_loss: true 29 | use_cnn: false 30 | use_crf: false 31 | use_rnn: false 32 | word_dropout: 0.1 33 | model_name: xlmr-first-docv2_10epoch_1batch_4accumulate_0.000005lr_10000lrrate_de_monolingual_nocrf_fast_norelearn_sentbatch_sentloss_finetune_nodev_saving_newner3 34 | ner: 35 | Corpus: CONLL_03_GERMAN_NEW 36 | tag_dictionary: resources/taggers/ner_tags.pkl 37 | target_dir: resources/taggers/ 38 | targets: ner 39 | teacher_annealing: false 40 | train: 41 | embeddings_storage_mode: none 42 | fine_tune_mode: true 43 | gradient_accumulation_steps: 4 44 | learning_rate: 5.0e-06 45 | lr_rate: 10000 46 | max_epochs: 10 47 | mini_batch_size: 1 48 | monitor_test: false 49 | save_final_model: false 50 | save_finetuned_embedding: true 51 | train_with_dev: false 52 | true_reshuffle: false 53 | use_warmup: false 54 | trainer: ModelFinetuner 55 | -------------------------------------------------------------------------------- /utils/config/xlmr-doc-de-06-ner-finetune.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | assign_doc_id: true 3 | distill_mode: false 4 | sentence_level_batch: true 5 | train_with_doc: true 6 | anneal_factor: 2 7 | ast: 8 | Corpus: SEMEVAL16-TR:SEMEVAL16-ES:SEMEVAL16-NL:SEMEVAL16-EN:SEMEVAL16-RU 9 | atis: 10 | Corpus: ATIS-EN:ATIS-TR:ATIS-HI 11 | chunk: 12 | Corpus: CONLL_03:CONLL_03_GERMAN 13 | embeddings: 14 | TransformerWordEmbeddings-0: 15 | fine_tune: true 16 | layers: '-1' 17 | model: xlm-roberta-large 18 | pooling_operation: first 19 | v2_doc: true 20 | interpolation: 0.5 21 | is_teacher_list: true 22 | model: 23 | FastSequenceTagger: 24 | crf_attention: false 25 | dropout: 0.0 26 | hidden_size: 256 27 | locked_dropout: 0.0 28 | sentence_loss: true 29 | use_cnn: false 30 | use_crf: false 31 | use_rnn: false 32 | word_dropout: 0.1 33 | model_name: xlmr-first-docv2_10epoch_1batch_4accumulate_0.000005lr_10000lrrate_de_monolingual_nocrf_fast_norelearn_sentbatch_sentloss_finetune_nodev_saving_newner3 34 | ner: 35 | Corpus: CONLL_03_GERMAN_NEW 36 | tag_dictionary: resources/taggers/ner_tags.pkl 37 | target_dir: resources/taggers/ 38 | targets: ner 39 | teacher_annealing: false 40 | train: 41 | embeddings_storage_mode: none 42 | fine_tune_mode: true 43 | gradient_accumulation_steps: 4 44 | learning_rate: 5.0e-06 45 | lr_rate: 10000 46 | max_epochs: 10 47 | mini_batch_size: 1 48 | monitor_test: false 49 | save_final_model: false 50 | save_finetuned_embedding: true 51 | train_with_dev: false 52 | true_reshuffle: false 53 | use_warmup: false 54 | trainer: ModelFinetuner 55 | -------------------------------------------------------------------------------- /utils/config/xlmr-doc-en-ner-finetune.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | assign_doc_id: true 3 | distill_mode: false 4 | sentence_level_batch: true 5 | train_with_doc: true 6 | anneal_factor: 2 7 | ast: 8 | Corpus: SEMEVAL16-TR:SEMEVAL16-ES:SEMEVAL16-NL:SEMEVAL16-EN:SEMEVAL16-RU 9 | atis: 10 | Corpus: ATIS-EN:ATIS-TR:ATIS-HI 11 | chunk: 12 | Corpus: CONLL_03:CONLL_03_GERMAN 13 | embeddings: 14 | TransformerWordEmbeddings-0: 15 | fine_tune: true 16 | layers: '-1' 17 | model: xlm-roberta-large 18 | pooling_operation: first 19 | v2_doc: true 20 | interpolation: 0.5 21 | is_teacher_list: true 22 | model: 23 | FastSequenceTagger: 24 | crf_attention: false 25 | dropout: 0.0 26 | hidden_size: 256 27 | locked_dropout: 0.0 28 | sentence_loss: true 29 | use_cnn: false 30 | use_crf: false 31 | use_rnn: false 32 | word_dropout: 0.1 33 | model_name: xlmr-first-docv2_10epoch_1batch_4accumulate_0.000005lr_10000lrrate_eng_monolingual_nocrf_fast_norelearn_sentbatch_sentloss_finetune_nodev_saving_ner3 34 | ner: 35 | Corpus: CONLL_03_ENGLISH 36 | tag_dictionary: resources/taggers/ner_tags.pkl 37 | target_dir: resources/taggers/ 38 | targets: ner 39 | teacher_annealing: false 40 | train: 41 | embeddings_storage_mode: none 42 | fine_tune_mode: true 43 | gradient_accumulation_steps: 4 44 | learning_rate: 5.0e-06 45 | lr_rate: 10000 46 | max_epochs: 10 47 | mini_batch_size: 1 48 | monitor_test: false 49 | save_final_model: false 50 | save_finetuned_embedding: true 51 | train_with_dev: false 52 | true_reshuffle: false 53 | use_warmup: false 54 | trainer: ModelFinetuner 55 | -------------------------------------------------------------------------------- /utils/config/xlmr-doc-es-ner-finetune.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | assign_doc_id: true 3 | distill_mode: false 4 | sentence_level_batch: true 5 | train_with_doc: true 6 | anneal_factor: 2 7 | ast: 8 | Corpus: SEMEVAL16-TR:SEMEVAL16-ES:SEMEVAL16-NL:SEMEVAL16-EN:SEMEVAL16-RU 9 | atis: 10 | Corpus: ATIS-EN:ATIS-TR:ATIS-HI 11 | chunk: 12 | Corpus: CONLL_03:CONLL_03_GERMAN 13 | embeddings: 14 | TransformerWordEmbeddings-0: 15 | fine_tune: true 16 | layers: '-1' 17 | model: xlm-roberta-large 18 | pooling_operation: first 19 | v2_doc: true 20 | interpolation: 0.5 21 | is_teacher_list: true 22 | model: 23 | FastSequenceTagger: 24 | crf_attention: false 25 | dropout: 0.0 26 | hidden_size: 256 27 | locked_dropout: 0.0 28 | sentence_loss: true 29 | use_cnn: false 30 | use_crf: false 31 | use_rnn: false 32 | word_dropout: 0.1 33 | model_name: xlmr-first-docv2_10epoch_1batch_4accumulate_0.000005lr_10000lrrate_es_monolingual_nocrf_fast_norelearn_sentbatch_sentloss_finetune_nodev_saving_newner3 34 | ner: 35 | Corpus: CONLL_03_SPANISH_NEW 36 | tag_dictionary: resources/taggers/ner_tags.pkl 37 | target_dir: resources/taggers/ 38 | targets: ner 39 | teacher_annealing: false 40 | train: 41 | embeddings_storage_mode: none 42 | fine_tune_mode: true 43 | gradient_accumulation_steps: 4 44 | learning_rate: 5.0e-06 45 | lr_rate: 10000 46 | max_epochs: 10 47 | mini_batch_size: 1 48 | monitor_test: false 49 | save_final_model: false 50 | save_finetuned_embedding: true 51 | train_with_dev: false 52 | true_reshuffle: false 53 | use_warmup: false 54 | trainer: ModelFinetuner 55 | -------------------------------------------------------------------------------- /utils/config/xlmr-doc-nl-ner-finetune.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | assign_doc_id: true 3 | distill_mode: false 4 | sentence_level_batch: true 5 | train_with_doc: true 6 | anneal_factor: 2 7 | ast: 8 | Corpus: SEMEVAL16-TR:SEMEVAL16-ES:SEMEVAL16-NL:SEMEVAL16-EN:SEMEVAL16-RU 9 | atis: 10 | Corpus: ATIS-EN:ATIS-TR:ATIS-HI 11 | chunk: 12 | Corpus: CONLL_03:CONLL_03_GERMAN 13 | embeddings: 14 | TransformerWordEmbeddings-0: 15 | fine_tune: true 16 | layers: '-1' 17 | model: xlm-roberta-large 18 | pooling_operation: first 19 | v2_doc: true 20 | interpolation: 0.5 21 | is_teacher_list: true 22 | model: 23 | FastSequenceTagger: 24 | crf_attention: false 25 | dropout: 0.0 26 | hidden_size: 256 27 | locked_dropout: 0.0 28 | sentence_loss: true 29 | use_cnn: false 30 | use_crf: false 31 | use_rnn: false 32 | word_dropout: 0.1 33 | model_name: xlmr-first-docv2_10epoch_1batch_4accumulate_0.000005lr_10000lrrate_nl_monolingual_nocrf_fast_norelearn_sentbatch_sentloss_finetune_nodev_saving_newner4 34 | ner: 35 | Corpus: CONLL_03_DUTCH_NEW 36 | tag_dictionary: resources/taggers/ner_tags.pkl 37 | target_dir: resources/taggers/ 38 | targets: ner 39 | teacher_annealing: false 40 | train: 41 | embeddings_storage_mode: none 42 | fine_tune_mode: true 43 | gradient_accumulation_steps: 4 44 | learning_rate: 5.0e-06 45 | lr_rate: 10000 46 | max_epochs: 10 47 | mini_batch_size: 1 48 | monitor_test: false 49 | save_final_model: false 50 | save_finetuned_embedding: true 51 | train_with_dev: false 52 | true_reshuffle: false 53 | use_warmup: false 54 | trainer: ModelFinetuner 55 | -------------------------------------------------------------------------------- /utils/config/xlnet-doc-en-ner-finetune.yaml: -------------------------------------------------------------------------------- 1 | ModelFinetuner: 2 | assign_doc_id: true 3 | distill_mode: false 4 | sentence_level_batch: true 5 | train_with_doc: true 6 | anneal_factor: 2 7 | ast: 8 | Corpus: SEMEVAL16-TR:SEMEVAL16-ES:SEMEVAL16-NL:SEMEVAL16-EN:SEMEVAL16-RU 9 | atis: 10 | Corpus: ATIS-EN:ATIS-TR:ATIS-HI 11 | chunk: 12 | Corpus: CONLL_03:CONLL_03_GERMAN 13 | embeddings: 14 | TransformerWordEmbeddings-0: 15 | fine_tune: true 16 | layers: '-1' 17 | model: xlnet-large-cased 18 | pooling_operation: first 19 | v2_doc: true 20 | interpolation: 0.5 21 | is_teacher_list: true 22 | model: 23 | FastSequenceTagger: 24 | crf_attention: false 25 | dropout: 0.0 26 | hidden_size: 256 27 | locked_dropout: 0.0 28 | sentence_loss: true 29 | use_cnn: false 30 | use_crf: false 31 | use_rnn: false 32 | word_dropout: 0.1 33 | model_name: xlnet-first-docv2_10epoch_1batch_4accumulate_0.000005lr_10000lrrate_eng_monolingual_nocrf_fast_norelearn_sentbatch_sentloss_finetune_nodev_saving_ner4 34 | ner: 35 | Corpus: CONLL_03_ENGLISH 36 | tag_dictionary: resources/taggers/ner_tags.pkl 37 | target_dir: resources/taggers/ 38 | targets: ner 39 | teacher_annealing: false 40 | train: 41 | embeddings_storage_mode: none 42 | fine_tune_mode: true 43 | gradient_accumulation_steps: 4 44 | learning_rate: 5.0e-06 45 | lr_rate: 10000 46 | max_epochs: 10 47 | mini_batch_size: 1 48 | monitor_test: false 49 | save_final_model: false 50 | save_finetuned_embedding: true 51 | train_with_dev: false 52 | true_reshuffle: false 53 | use_warmup: false 54 | trainer: ModelFinetuner 55 | -------------------------------------------------------------------------------- /utils/exception_hook.py: -------------------------------------------------------------------------------- 1 | class ExceptionHook: 2 | instance = None 3 | def __call__(self, *args, **kwargs): 4 | if self.instance is None: 5 | from IPython.core import ultratb 6 | self.instance = ultratb.FormattedTB(mode="Plain", color_scheme="Linux", call_pdb=1) 7 | return self.instance(*args, **kwargs) 8 | 9 | -------------------------------------------------------------------------------- /utils/extract_tokens_from_amr.py: -------------------------------------------------------------------------------- 1 | from stog.data.dataset_readers import AbstractMeaningRepresentationDatasetReader 2 | import sys 3 | from stog.utils import logging 4 | 5 | logger = logging.init_logger() 6 | def extract_amr_token(file_path): 7 | dataset_reader = AbstractMeaningRepresentationDatasetReader() 8 | for instance in dataset_reader.read(file_path): 9 | amr_tokens = instance.fields["amr_tokens"]["decoder_tokens"] 10 | yield " ".join(amr_tokens) 11 | 12 | 13 | if __name__ == "__main__": 14 | if len(sys.argv) < 2: 15 | print("""Usage: 16 | python {} [amr_file] 17 | 18 | The output will in stdout. 19 | """) 20 | for filename in sys.argv[1:]: 21 | for line in extract_amr_token(filename): 22 | sys.stdout.write(line + "\n") 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /utils/flair/__init__.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import os 3 | 4 | # global variable: cache_root 5 | cache_root = os.path.expanduser(os.path.join("~", ".flair")) 6 | 7 | # global variable: device 8 | device = None 9 | if torch.cuda.is_available(): 10 | device = torch.device("cuda:0") 11 | else: 12 | device = torch.device("cpu") 13 | 14 | from . import data 15 | from . import models 16 | from . import visual 17 | from . import trainers 18 | from . import nn 19 | 20 | import logging.config 21 | 22 | __version__ = "0.4.3" 23 | 24 | logging.config.dictConfig( 25 | { 26 | "version": 1, 27 | "disable_existing_loggers": False, 28 | "formatters": {"standard": {"format": "%(asctime)-15s %(message)s"}}, 29 | "handlers": { 30 | "console": { 31 | "level": "INFO", 32 | "class": "logging.StreamHandler", 33 | "formatter": "standard", 34 | "stream": "ext://sys.stdout", 35 | } 36 | }, 37 | "loggers": { 38 | "flair": {"handlers": ["console"], "level": "INFO", "propagate": False} 39 | }, 40 | "root": {"handlers": ["console"], "level": "WARNING"}, 41 | } 42 | ) 43 | 44 | logger = logging.getLogger("flair") 45 | -------------------------------------------------------------------------------- /utils/flair/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__init__.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/config_parser.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/config_parser.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/config_parser.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/config_parser.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/corpus_mapping.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/corpus_mapping.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/corpus_mapping.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/corpus_mapping.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/custom_data_loader.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/custom_data_loader.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/custom_data_loader.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/custom_data_loader.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/data.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/data.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/data.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/data.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/datasets.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/datasets.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/datasets.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/datasets.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/embeddings.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/embeddings.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/embeddings.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/embeddings.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/file_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/file_utils.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/file_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/file_utils.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/linear_functions.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/linear_functions.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/linear_functions.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/linear_functions.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/list_data.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/list_data.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/list_data.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/list_data.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/ner_dp_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/ner_dp_utils.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/ner_dp_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/ner_dp_utils.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/nn.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/nn.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/nn.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/nn.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/optim.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/optim.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/optim.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/optim.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/training_utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/training_utils.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/training_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/training_utils.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/variational_inference.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/variational_inference.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/__pycache__/variational_inference.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/__pycache__/variational_inference.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/algorithms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/algorithms/__init__.py -------------------------------------------------------------------------------- /utils/flair/algorithms/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/algorithms/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/algorithms/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/algorithms/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/algorithms/__pycache__/dict_merge.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/algorithms/__pycache__/dict_merge.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/algorithms/__pycache__/dict_merge.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/algorithms/__pycache__/dict_merge.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/algorithms/__pycache__/maximum_spanning_tree.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/algorithms/__pycache__/maximum_spanning_tree.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/algorithms/dict_merge.py: -------------------------------------------------------------------------------- 1 | # Recursive dictionary merge 2 | # Copyright (C) 2016 Paul Durivage 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU General Public License 15 | # along with this program. If not, see . 16 | 17 | # Found here https://gist.github.com/angstwad/bf22d1822c38a92ec0a9 18 | # Using jpopelka's modified solution. 19 | 20 | import collections 21 | 22 | def dict_merge(dct, merge_dct): 23 | """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of 24 | updating only top-level keys, dict_merge recurses down into dicts nested 25 | to an arbitrary depth, updating keys. The ``merge_dct`` is merged into 26 | ``dct``. 27 | 28 | :param dct: dict onto which the merge is executed 29 | :param merge_dct: dct merged into dct 30 | :return: None 31 | """ 32 | for k, v in merge_dct.items(): 33 | if isinstance(dct.get(k), dict) and isinstance(v, collections.Mapping): 34 | dict_merge(dct[k], v) 35 | else: 36 | dct[k] = v 37 | -------------------------------------------------------------------------------- /utils/flair/hyperparameter/__init__.py: -------------------------------------------------------------------------------- 1 | from .parameter import ( 2 | Parameter, 3 | SEQUENCE_TAGGER_PARAMETERS, 4 | TRAINING_PARAMETERS, 5 | DOCUMENT_EMBEDDING_PARAMETERS, 6 | ) 7 | from .param_selection import ( 8 | SequenceTaggerParamSelector, 9 | TextClassifierParamSelector, 10 | SearchSpace, 11 | ) 12 | -------------------------------------------------------------------------------- /utils/flair/list_data.py: -------------------------------------------------------------------------------- 1 | from .data import * 2 | class ListCorpus(Corpus): 3 | def __init__( 4 | self, 5 | train: List[FlairDataset], 6 | dev: List[FlairDataset], 7 | test: List[FlairDataset], 8 | name: str = "listcorpus", 9 | targets: list = [], 10 | ): 11 | # In this Corpus, we set train list to be our target to train, we keep self._train the same as the Class Corpus as the counting and preprocessing is needed 12 | self.train_list: List[FlairDataset] = train 13 | self.dev_list: List[FlairDataset] = dev 14 | self.test_list: List[FlairDataset] = test 15 | self._train: FlairDataset = ConcatDataset([data for data in train]) 16 | self._dev: FlairDataset = ConcatDataset([data for data in dev]) 17 | self._test: FlairDataset = ConcatDataset([data for data in test]) 18 | self.name: str = name 19 | self.targets = targets -------------------------------------------------------------------------------- /utils/flair/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .sequence_tagger_model import SequenceTagger, FastSequenceTagger 2 | from .dependency_model import SemanticDependencyParser 3 | from .language_model import LanguageModel 4 | from .text_classification_model import TextClassifier 5 | from .controller import EmbedController 6 | from .ensemble_model import EnsembleModel -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/biaffine_attention.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/biaffine_attention.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/biaffine_attention.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/biaffine_attention.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/biaffine_dp.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/biaffine_dp.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/biaffine_dp.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/biaffine_dp.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/controller.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/controller.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/controller.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/controller.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/dependency_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/dependency_model.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/dependency_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/dependency_model.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/dependency_model_1.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/dependency_model_1.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/dependency_model_1.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/dependency_model_1.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/ensemble_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/ensemble_model.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/ensemble_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/ensemble_model.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/language_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/language_model.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/language_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/language_model.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/mst_decoder.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/mst_decoder.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/mst_decoder.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/mst_decoder.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/sequence_tagger_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/sequence_tagger_model.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/sequence_tagger_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/sequence_tagger_model.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/text_classification_model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/text_classification_model.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/models/__pycache__/text_classification_model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/models/__pycache__/text_classification_model.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .model import Model 4 | 5 | __all__ = ['Model'] 6 | -------------------------------------------------------------------------------- /utils/flair/parser/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/__pycache__/model.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/__pycache__/model.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/__pycache__/model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/__pycache__/model.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/cmds/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .evaluate import Evaluate 4 | from .predict import Predict 5 | from .train import Train 6 | 7 | __all__ = ['Evaluate', 'Predict', 'Train'] 8 | -------------------------------------------------------------------------------- /utils/flair/parser/cmds/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/cmds/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/cmds/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/cmds/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/cmds/__pycache__/cmd.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/cmds/__pycache__/cmd.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/cmds/__pycache__/cmd.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/cmds/__pycache__/cmd.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/cmds/__pycache__/evaluate.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/cmds/__pycache__/evaluate.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/cmds/__pycache__/evaluate.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/cmds/__pycache__/evaluate.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/cmds/__pycache__/predict.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/cmds/__pycache__/predict.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/cmds/__pycache__/train.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/cmds/__pycache__/train.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/config.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from ast import literal_eval 4 | from configparser import ConfigParser 5 | from argparse import Namespace 6 | 7 | 8 | class Config(ConfigParser): 9 | 10 | def __init__(self, path): 11 | super(Config, self).__init__() 12 | 13 | self.read(path) 14 | self.namespace = Namespace() 15 | self.update(dict((name, literal_eval(value)) 16 | for section in self.sections() 17 | for name, value in self.items(section))) 18 | 19 | def __repr__(self): 20 | s = line = "-" * 15 + "-+-" + "-" * 25 + "\n" 21 | s += f"{'Param':15} | {'Value':^25}\n" + line 22 | for name, value in vars(self.namespace).items(): 23 | s += f"{name:15} | {str(value):^25}\n" 24 | s += line 25 | 26 | return s 27 | 28 | def __getattr__(self, attr): 29 | return getattr(self.namespace, attr) 30 | 31 | def __getstate__(self): 32 | return vars(self) 33 | 34 | def __setstate__(self, state): 35 | self.__dict__.update(state) 36 | 37 | def update(self, kwargs): 38 | for name, value in kwargs.items(): 39 | setattr(self.namespace, name, value) 40 | 41 | return self 42 | -------------------------------------------------------------------------------- /utils/flair/parser/modules/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from . import dropout 4 | from .bert import BertEmbedding 5 | from .biaffine import Biaffine 6 | from .trilinear_attention import TrilinearScorer 7 | from .bilstm import BiLSTM 8 | from .char_lstm import CHAR_LSTM 9 | from .mlp import MLP 10 | 11 | __all__ = ['CHAR_LSTM', 'MLP', 'BertEmbedding', 12 | 'Biaffine', 'BiLSTM', 'dropout', 'TrilinearScorer'] 13 | -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/bert.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/bert.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/bert.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/bert.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/biaffine.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/biaffine.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/biaffine.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/biaffine.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/bilstm.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/bilstm.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/bilstm.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/bilstm.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/char_lstm.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/char_lstm.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/char_lstm.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/char_lstm.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/dropout.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/dropout.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/dropout.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/dropout.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/mlp.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/mlp.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/mlp.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/mlp.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/scalar_mix.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/scalar_mix.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/scalar_mix.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/scalar_mix.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/trilinear_attention.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/trilinear_attention.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/__pycache__/trilinear_attention.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/modules/__pycache__/trilinear_attention.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/modules/char_lstm.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import torch 4 | import torch.nn as nn 5 | from torch.nn.utils.rnn import pack_padded_sequence 6 | 7 | 8 | class CHAR_LSTM(nn.Module): 9 | 10 | def __init__(self, n_chars, n_embed, n_out): 11 | super(CHAR_LSTM, self).__init__() 12 | 13 | # the embedding layer 14 | self.embed = nn.Embedding(num_embeddings=n_chars, 15 | embedding_dim=n_embed) 16 | # the lstm layer 17 | self.lstm = nn.LSTM(input_size=n_embed, 18 | hidden_size=n_out//2, 19 | batch_first=True, 20 | bidirectional=True) 21 | 22 | def forward(self, x): 23 | mask = x.gt(0) 24 | lens = mask.sum(dim=1) 25 | 26 | x = pack_padded_sequence(self.embed(x), lens, True, False) 27 | x, (hidden, _) = self.lstm(x) 28 | hidden = torch.cat(torch.unbind(hidden), dim=-1) 29 | 30 | return hidden 31 | -------------------------------------------------------------------------------- /utils/flair/parser/modules/dropout.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import torch 4 | import torch.nn as nn 5 | import pdb 6 | 7 | class SharedDropout(nn.Module): 8 | 9 | def __init__(self, p=0.5, batch_first=True): 10 | super(SharedDropout, self).__init__() 11 | 12 | self.p = p 13 | self.batch_first = batch_first 14 | 15 | def extra_repr(self): 16 | s = f"p={self.p}" 17 | if self.batch_first: 18 | s += f", batch_first={self.batch_first}" 19 | 20 | return s 21 | 22 | def forward(self, x): 23 | if self.training: 24 | if self.batch_first: 25 | mask = self.get_mask(x[:, 0], self.p) 26 | else: 27 | mask = self.get_mask(x[0], self.p) 28 | x *= mask.unsqueeze(1) if self.batch_first else mask 29 | 30 | return x 31 | 32 | @staticmethod 33 | def get_mask(x, p): 34 | mask = x.new_empty(x.shape).bernoulli_(1 - p) 35 | mask = mask / (1 - p) 36 | 37 | return mask 38 | 39 | 40 | class IndependentDropout(nn.Module): 41 | 42 | def __init__(self, p=0.5): 43 | super(IndependentDropout, self).__init__() 44 | 45 | self.p = p 46 | 47 | def extra_repr(self): 48 | return f"p={self.p}" 49 | 50 | def forward(self, *items): 51 | if self.training: 52 | masks = [x.new_empty(x.shape[:2]).bernoulli_(1 - self.p) 53 | for x in items] 54 | total = sum(masks) 55 | scale = len(items) / total.max(torch.ones_like(total)) 56 | masks = [mask * scale for mask in masks] 57 | items = [item * mask.unsqueeze(dim=-1) 58 | for item, mask in zip(items, masks)] 59 | 60 | return items 61 | -------------------------------------------------------------------------------- /utils/flair/parser/modules/mlp.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from flair.parser.modules.dropout import SharedDropout 4 | 5 | import torch.nn as nn 6 | 7 | 8 | class MLP(nn.Module): 9 | 10 | def __init__(self, n_in, n_hidden, dropout=0, identity = False): 11 | super(MLP, self).__init__() 12 | 13 | self.linear = nn.Linear(n_in, n_hidden) 14 | self.identity = identity 15 | if not self.identity: 16 | self.activation = nn.LeakyReLU(negative_slope=0.1) 17 | self.dropout = SharedDropout(p=dropout) 18 | 19 | self.reset_parameters() 20 | 21 | def reset_parameters(self): 22 | nn.init.orthogonal_(self.linear.weight) 23 | nn.init.zeros_(self.linear.bias) 24 | 25 | def forward(self, x): 26 | x = self.linear(x) 27 | if not self.identity: 28 | x = self.activation(x) 29 | x = self.dropout(x) 30 | 31 | return x 32 | -------------------------------------------------------------------------------- /utils/flair/parser/modules/scalar_mix.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | 7 | class ScalarMix(nn.Module): 8 | 9 | def __init__(self, n_layers, dropout=0): 10 | super(ScalarMix, self).__init__() 11 | 12 | self.n_layers = n_layers 13 | self.dropout = dropout 14 | 15 | self.weights = nn.Parameter(torch.zeros(n_layers)) 16 | self.gamma = nn.Parameter(torch.tensor([1.0])) 17 | self.dropout = nn.Dropout(dropout) 18 | 19 | def extra_repr(self): 20 | s = f"n_layers={self.n_layers}" 21 | if self.dropout.p > 0: 22 | s += f", dropout={self.dropout.p}" 23 | 24 | return s 25 | 26 | def forward(self, tensors): 27 | normed_weights = self.dropout(self.weights.softmax(-1)) 28 | weighted_sum = sum(w * h for w, h in zip(normed_weights, tensors)) 29 | 30 | return self.gamma * weighted_sum 31 | -------------------------------------------------------------------------------- /utils/flair/parser/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from . import corpus, data, field, fn, metric 4 | from .embedding import Embedding 5 | from .vocab import Vocab 6 | 7 | __all__ = ['Corpus', 'Embedding', 'Vocab', 8 | 'corpus', 'data', 'field', 'fn', 'metric'] 9 | -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/alg.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/alg.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/alg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/alg.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/common.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/common.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/common.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/common.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/corpus.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/corpus.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/corpus.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/corpus.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/data.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/data.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/data.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/data.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/embedding.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/embedding.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/embedding.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/embedding.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/field.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/field.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/field.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/field.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/fn.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/fn.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/fn.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/fn.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/metric.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/metric.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/metric.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/metric.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/vocab.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/vocab.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/__pycache__/vocab.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/parser/utils/__pycache__/vocab.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/parser/utils/common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | pad = '' 4 | unk = '' 5 | bos = '' 6 | eos = '' 7 | -------------------------------------------------------------------------------- /utils/flair/parser/utils/embedding.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import torch 4 | 5 | 6 | class Embedding(object): 7 | 8 | def __init__(self, tokens, vectors, unk=None): 9 | super(Embedding, self).__init__() 10 | 11 | self.tokens = tokens 12 | self.vectors = torch.tensor(vectors) 13 | self.pretrained = {w: v for w, v in zip(tokens, vectors)} 14 | self.unk = unk 15 | 16 | def __len__(self): 17 | return len(self.tokens) 18 | 19 | def __contains__(self, token): 20 | return token in self.pretrained 21 | 22 | @property 23 | def dim(self): 24 | return self.vectors.size(1) 25 | 26 | @property 27 | def unk_index(self): 28 | if self.unk is not None: 29 | return self.tokens.index(self.unk) 30 | else: 31 | raise AttributeError 32 | 33 | @classmethod 34 | def load(cls, path, unk=None): 35 | with open(path, 'r') as f: 36 | lines = [line for line in f] 37 | splits = [line.split() for line in lines] 38 | tokens, vectors = zip(*[(s[0], list(map(float, s[1:]))) 39 | for s in splits]) 40 | 41 | return cls(tokens, vectors, unk=unk) 42 | -------------------------------------------------------------------------------- /utils/flair/parser/utils/vocab.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from collections.abc import Iterable 4 | from flair.parser.utils.common import unk 5 | 6 | 7 | class Vocab(object): 8 | 9 | def __init__(self, counter, min_freq=1, specials=[]): 10 | self.itos = specials 11 | self.stoi = {token: i for i, token in enumerate(self.itos)} 12 | 13 | self.extend([token for token, freq in counter.items() 14 | if freq >= min_freq]) 15 | self.unk_index = self.stoi.get(unk, 0) 16 | self.n_init = len(self) 17 | 18 | def __len__(self): 19 | return len(self.itos) 20 | 21 | def __getitem__(self, key): 22 | return self.stoi[key] 23 | 24 | def __contains__(self, token): 25 | return token in self.stoi 26 | 27 | def token2id(self, sequence): 28 | return [self.stoi.get(token, self.unk_index) for token in sequence] 29 | 30 | def id2token(self, ids): 31 | if isinstance(ids, Iterable): 32 | return [self.itos[i] for i in ids] 33 | else: 34 | return self.itos[ids] 35 | 36 | def extend(self, tokens): 37 | self.itos.extend(sorted(set(tokens).difference(self.stoi))) 38 | self.stoi = {token: i for i, token in enumerate(self.itos)} 39 | -------------------------------------------------------------------------------- /utils/flair/trainers/__init__.py: -------------------------------------------------------------------------------- 1 | from .trainer import ModelTrainer 2 | from .distillation_trainer import ModelDistiller 3 | from .finetune_trainer import ModelFinetuner 4 | from .reinforcement_trainer import ReinforcementTrainer -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/distillation_trainer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/distillation_trainer.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/distillation_trainer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/distillation_trainer.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/finetune_trainer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/finetune_trainer.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/finetune_trainer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/finetune_trainer.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/reinforcement_trainer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/reinforcement_trainer.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/reinforcement_trainer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/reinforcement_trainer.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/swaf_trainer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/swaf_trainer.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/swaf_trainer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/swaf_trainer.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/trainer.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/trainer.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/trainers/__pycache__/trainer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/trainers/__pycache__/trainer.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .exception_hook import ExceptionHook 2 | from typing import Iterator, List, TypeVar, Iterable, Dict 3 | import random 4 | from itertools import zip_longest, islice 5 | 6 | A = TypeVar('A') 7 | def lazy_groups_of(iterator: Iterator[A], group_size: int) -> Iterator[List[A]]: 8 | """ 9 | Takes an iterator and batches the invididual instances into lists of the 10 | specified size. The last list may be smaller if there are instances left over. 11 | """ 12 | return iter(lambda: list(islice(iterator, 0, group_size)), []) 13 | 14 | def ensure_list(iterable: Iterable[A]) -> List[A]: 15 | """ 16 | An Iterable may be a list or a generator. 17 | This ensures we get a list without making an unnecessary copy. 18 | """ 19 | if isinstance(iterable, list): 20 | return iterable 21 | else: 22 | return list(iterable) 23 | 24 | def is_lazy(iterable: Iterable[A]) -> bool: 25 | """ 26 | Checks if the given iterable is lazy, 27 | which here just means it's not a list. 28 | """ 29 | return not isinstance(iterable, list) 30 | 31 | def add_noise_to_dict_values(dictionary: Dict[A, float], noise_param: float) -> Dict[A, float]: 32 | """ 33 | Returns a new dictionary with noise added to every key in ``dictionary``. The noise is 34 | uniformly distributed within ``noise_param`` percent of the value for every value in the 35 | dictionary. 36 | """ 37 | new_dict = {} 38 | for key, value in dictionary.items(): 39 | noise_value = value * noise_param 40 | noise = random.uniform(-noise_value, noise_value) 41 | new_dict[key] = value + noise 42 | return new_dict 43 | -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/archival.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/archival.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/checks.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/checks.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/checks.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/checks.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/environment.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/environment.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/exception_hook.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/exception_hook.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/exception_hook.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/exception_hook.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/file.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/file.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/from_params.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/from_params.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/from_params.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/from_params.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/logging.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/logging.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/logging.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/logging.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/nn.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/nn.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/params.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/params.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/params.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/params.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/registrable.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/registrable.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/string.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/string.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/time.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/time.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/__pycache__/tqdm.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/utils/__pycache__/tqdm.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/utils/checks.py: -------------------------------------------------------------------------------- 1 | """ 2 | Adopted from AllenNLP: 3 | https://github.com/allenai/allennlp/tree/v0.6.1/allennlp/common 4 | 5 | Functions and exceptions for checking that 6 | AllenNLP and its models are configured correctly. 7 | """ 8 | 9 | from torch import cuda 10 | 11 | from flair.utils import logging 12 | 13 | logger = logging.init_logger() # pylint: disable=invalid-name 14 | 15 | 16 | class ConfigurationError(Exception): 17 | """ 18 | The exception raised by any AllenNLP object when it's misconfigured 19 | (e.g. missing properties, invalid properties, unknown properties). 20 | """ 21 | 22 | def __init__(self, message): 23 | super(ConfigurationError, self).__init__() 24 | self.message = message 25 | 26 | def __str__(self): 27 | return repr(self.message) 28 | 29 | 30 | def log_pytorch_version_info(): 31 | import torch 32 | logger.info("Pytorch version: %s", torch.__version__) 33 | 34 | 35 | def check_dimensions_match(dimension_1: int, 36 | dimension_2: int, 37 | dim_1_name: str, 38 | dim_2_name: str) -> None: 39 | if dimension_1 != dimension_2: 40 | raise ConfigurationError(f"{dim_1_name} must match {dim_2_name}, but got {dimension_1} " 41 | f"and {dimension_2} instead") 42 | 43 | 44 | def check_for_gpu(device_id: int): 45 | if device_id is not None and device_id >= cuda.device_count(): 46 | raise ConfigurationError("Experiment specified a GPU but none is available;" 47 | " if you want to run on CPU use the override" 48 | " 'trainer.cuda_device=-1' in the json config file.") 49 | -------------------------------------------------------------------------------- /utils/flair/utils/exception_hook.py: -------------------------------------------------------------------------------- 1 | class ExceptionHook: 2 | instance = None 3 | def __call__(self, *args, **kwargs): 4 | if self.instance is None: 5 | from IPython.core import ultratb 6 | self.instance = ultratb.FormattedTB(mode="Plain", color_scheme="Linux", call_pdb=1) 7 | return self.instance(*args, **kwargs) 8 | 9 | -------------------------------------------------------------------------------- /utils/flair/utils/extract_tokens_from_amr.py: -------------------------------------------------------------------------------- 1 | from stog.data.dataset_readers import AbstractMeaningRepresentationDatasetReader 2 | import sys 3 | from stog.utils import logging 4 | 5 | logger = logging.init_logger() 6 | def extract_amr_token(file_path): 7 | dataset_reader = AbstractMeaningRepresentationDatasetReader() 8 | for instance in dataset_reader.read(file_path): 9 | amr_tokens = instance.fields["amr_tokens"]["decoder_tokens"] 10 | yield " ".join(amr_tokens) 11 | 12 | 13 | if __name__ == "__main__": 14 | if len(sys.argv) < 2: 15 | print("""Usage: 16 | python {} [amr_file] 17 | 18 | The output will in stdout. 19 | """) 20 | for filename in sys.argv[1:]: 21 | for line in extract_amr_token(filename): 22 | sys.stdout.write(line + "\n") 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /utils/flair/utils/time.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | 4 | def time_to_str(timestamp: int) -> str: 5 | """ 6 | Convert seconds past Epoch to human readable string. 7 | """ 8 | datetimestamp = datetime.datetime.fromtimestamp(timestamp) 9 | return '{:04d}-{:02d}-{:02d}-{:02d}-{:02d}-{:02d}'.format( 10 | datetimestamp.year, datetimestamp.month, datetimestamp.day, 11 | datetimestamp.hour, datetimestamp.minute, datetimestamp.second 12 | ) 13 | 14 | 15 | def str_to_time(time_str: str) -> datetime.datetime: 16 | """ 17 | Convert human readable string to datetime.datetime. 18 | """ 19 | pieces = [int(piece) for piece in time_str.split('-')] 20 | return datetime.datetime(*pieces) 21 | -------------------------------------------------------------------------------- /utils/flair/visual/__init__.py: -------------------------------------------------------------------------------- 1 | from .manifold import Visualizer 2 | from .activations import Highlighter 3 | -------------------------------------------------------------------------------- /utils/flair/visual/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/visual/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/visual/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/visual/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/visual/__pycache__/activations.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/visual/__pycache__/activations.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/visual/__pycache__/activations.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/visual/__pycache__/activations.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/visual/__pycache__/manifold.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/visual/__pycache__/manifold.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/visual/__pycache__/manifold.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/visual/__pycache__/manifold.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/visual/__pycache__/training_curves.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/visual/__pycache__/training_curves.cpython-36.pyc -------------------------------------------------------------------------------- /utils/flair/visual/__pycache__/training_curves.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/flair/visual/__pycache__/training_curves.cpython-37.pyc -------------------------------------------------------------------------------- /utils/flair/visual/html_templates.py: -------------------------------------------------------------------------------- 1 | TAGGED_ENTITY = ''' 2 | 3 | {entity} 4 | {label} 5 | 6 | ''' 7 | 8 | HTML_PAGE = """ 9 | 10 | 11 | 12 | Flair 13 | 14 | 15 | {text} 16 | 17 | """ 18 | -------------------------------------------------------------------------------- /utils/resources/taggers/ast_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/resources/taggers/ast_tags.pkl -------------------------------------------------------------------------------- /utils/resources/taggers/ast_tags_new.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/resources/taggers/ast_tags_new.pkl -------------------------------------------------------------------------------- /utils/resources/taggers/dependency_projective_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/resources/taggers/dependency_projective_tags.pkl -------------------------------------------------------------------------------- /utils/resources/taggers/dm_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/resources/taggers/dm_tags.pkl -------------------------------------------------------------------------------- /utils/resources/taggers/enhancedud_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/resources/taggers/enhancedud_tags.pkl -------------------------------------------------------------------------------- /utils/resources/taggers/ner_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/resources/taggers/ner_tags.pkl -------------------------------------------------------------------------------- /utils/resources/taggers/np_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/resources/taggers/np_tags.pkl -------------------------------------------------------------------------------- /utils/resources/taggers/pos_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/resources/taggers/pos_tags.pkl -------------------------------------------------------------------------------- /utils/resources/taggers/ud_dependency_tags.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/resources/taggers/ud_dependency_tags.pkl -------------------------------------------------------------------------------- /utils/script/__pycache__/evaluate.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/script/__pycache__/evaluate.cpython-37.pyc -------------------------------------------------------------------------------- /utils/script/__pycache__/semdep_eval.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Alibaba-NLP/StructuralKD/1aa5471562c4f36a1f229accd1ed30f4188f9558/utils/script/__pycache__/semdep_eval.cpython-37.pyc -------------------------------------------------------------------------------- /utils/tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pathlib import Path 3 | 4 | 5 | @pytest.fixture(scope="module") 6 | def resources_path(): 7 | return Path(__file__).parent / "resources" 8 | 9 | 10 | @pytest.fixture(scope="module") 11 | def tasks_base_path(resources_path): 12 | return resources_path / "tasks" 13 | 14 | 15 | @pytest.fixture(scope="module") 16 | def results_base_path(resources_path): 17 | return resources_path / "results" 18 | 19 | 20 | def pytest_addoption(parser): 21 | parser.addoption( 22 | "--runslow", action="store_true", default=False, help="run slow tests" 23 | ) 24 | parser.addoption( 25 | "--runintegration", 26 | action="store_true", 27 | default=False, 28 | help="run integration tests", 29 | ) 30 | 31 | 32 | def pytest_collection_modifyitems(config, items): 33 | if config.getoption("--runslow") and config.getoption("--runintegration"): 34 | return 35 | 36 | if not config.getoption("--runslow"): 37 | skip_slow = pytest.mark.skip(reason="need --runslow option to run") 38 | for item in items: 39 | if "slow" in item.keywords: 40 | item.add_marker(skip_slow) 41 | 42 | if not config.getoption("--runintegration"): 43 | skip_integration = pytest.mark.skip( 44 | reason="need --runintegration option to run" 45 | ) 46 | for item in items: 47 | if "integration" in item.keywords: 48 | item.add_marker(skip_integration) 49 | -------------------------------------------------------------------------------- /utils/tests/resources/corpora/lorem_ipsum/test.txt: -------------------------------------------------------------------------------- 1 | Adipiscing commodo elit at imperdiet. Consequat interdum varius sit amet mattis vulputate enim nulla. Nulla aliquet porttitor lacus luctus accumsan tortor. Curabitur gravida arcu ac tortor. Adipiscing elit pellentesque habitant morbi. Sed viverra tellus in hac habitasse platea dictumst. Turpis cursus in hac habitasse. Pharetra vel turpis nunc eget. Enim facilisis gravida neque convallis a cras semper auctor neque. Interdum posuere lorem ipsum dolor sit amet consectetur adipiscing elit. 2 | 3 | Mauris sit amet massa vitae tortor condimentum lacinia. Neque gravida in fermentum et sollicitudin. Blandit volutpat maecenas volutpat blandit aliquam. Gravida neque convallis a cras semper auctor neque vitae. Viverra aliquet eget sit amet tellus cras adipiscing enim eu. Risus sed vulputate odio ut enim blandit volutpat maecenas. Amet tellus cras adipiscing enim eu. Viverra tellus in hac habitasse platea dictumst vestibulum rhoncus est. Magna etiam tempor orci eu lobortis elementum. Leo vel fringilla est ullamcorper eget. Nisl nisi scelerisque eu ultrices. Eros donec ac odio tempor orci dapibus ultrices in. Nisl nisi scelerisque eu ultrices vitae auctor eu augue. Hac habitasse platea dictumst vestibulum rhoncus est pellentesque elit. Habitasse platea dictumst vestibulum rhoncus est pellentesque elit. In ornare quam viverra orci sagittis. Morbi quis commodo odio aenean. Nam at lectus urna duis convallis convallis tellus id interdum. -------------------------------------------------------------------------------- /utils/tests/resources/corpora/lorem_ipsum/valid.txt: -------------------------------------------------------------------------------- 1 | Nulla at volutpat diam ut venenatis tellus in metus vulputate. Porttitor leo a diam sollicitudin tempor. Tincidunt vitae semper quis lectus nulla at volutpat diam. Ornare aenean euismod elementum nisi quis eleifend quam adipiscing. Tortor pretium viverra suspendisse potenti. Arcu risus quis varius quam quisque id. Non sodales neque sodales ut etiam sit amet nisl. Porttitor lacus luctus accumsan tortor posuere ac ut consequat. Diam sit amet nisl suscipit. Ut sem nulla pharetra diam sit amet nisl suscipit adipiscing. Varius quam quisque id diam. Elementum tempus egestas sed sed risus pretium quam vulputate. Eu ultrices vitae auctor eu augue ut lectus. Tincidunt id aliquet risus feugiat in ante metus dictum at. Mauris cursus mattis molestie a iaculis at erat pellentesque. Leo urna molestie at elementum eu. 2 | 3 | Posuere morbi leo urna molestie. Tincidunt nunc pulvinar sapien et. Mattis molestie a iaculis at erat pellentesque. Arcu cursus euismod quis viverra nibh cras pulvinar mattis nunc. Phasellus vestibulum lorem sed risus ultricies tristique nulla aliquet enim. Aenean et tortor at risus viverra. Ut placerat orci nulla pellentesque dignissim. Est lorem ipsum dolor sit amet. Eros donec ac odio tempor. Elementum integer enim neque volutpat ac tincidunt vitae. -------------------------------------------------------------------------------- /utils/tests/resources/tasks/ag_news/README.md: -------------------------------------------------------------------------------- 1 | ## AG_NEWS 2 | 3 | Data is taken from [here](https://www.di.unipi.it/~gulli/AG_corpus_of_news_articles.html). 4 | 5 | The dataset contains a collection of news articles grouped into different categories. 6 | We took a small random sample and converted it to the expected format of our data fetcher: 7 | ``` 8 | __label__ 9 | ``` 10 | 11 | #### Publications Using the Dataset 12 | 13 | * G. M. Del Corso, A. Gulli, and F. Romani. Ranking a stream of news. In Proceedings of 14th International World Wide Web Conference, pages 97–106, Chiba, Japan, 2005. 14 | * A. Gulli. The anatomy of a news search engine. In Proceedings of 14th International World Wide Web Conference, pages 880–881, Chiba, Japan, 2005. -------------------------------------------------------------------------------- /utils/tests/resources/tasks/fashion/eng.testa: -------------------------------------------------------------------------------- 1 | Most _ _ O 2 | wedding _ _ B-Occasion 3 | dresses _ _ B-NominalProduct 4 | , _ _ O 5 | for _ _ O 6 | example _ _ O 7 | , _ _ O 8 | are _ _ O 9 | simply _ _ O 10 | too _ _ O 11 | enormous _ _ O 12 | and _ _ O 13 | terrifyingly _ _ O 14 | loaded _ _ O 15 | with _ _ O 16 | sentimental _ _ O 17 | value _ _ O 18 | for _ _ O 19 | DIY _ _ B-ProductDesign 20 | dyeing _ _ I-ProductDesign 21 | . _ _ O -------------------------------------------------------------------------------- /utils/tests/resources/tasks/fashion/eng.testb: -------------------------------------------------------------------------------- 1 | For _ _ O 2 | my _ _ O 3 | Nurse _ _ B-NamedOrganizationBrand 4 | Ratched _ _ I-NamedOrganizationBrand 5 | dress _ _ B-NominalProduct 6 | , _ _ O 7 | I _ _ O 8 | had _ _ O 9 | brought _ _ O 10 | two _ _ O 11 | dyeing _ _ O 12 | options _ _ O 13 | — _ _ O 14 | one _ _ O 15 | more _ _ O 16 | ambitious _ _ O 17 | than _ _ O 18 | the _ _ O 19 | other _ _ O 20 | . _ _ O -------------------------------------------------------------------------------- /utils/tests/resources/tasks/fashion_nodev/test.tsv: -------------------------------------------------------------------------------- 1 | Most _ _ O 2 | wedding _ _ B-Occasion 3 | dresses _ _ B-NominalProduct 4 | , _ _ O 5 | for _ _ O 6 | example _ _ O 7 | , _ _ O 8 | are _ _ O 9 | simply _ _ O 10 | too _ _ O 11 | enormous _ _ O 12 | and _ _ O 13 | terrifyingly _ _ O 14 | loaded _ _ O 15 | with _ _ O 16 | sentimental _ _ O 17 | value _ _ O 18 | for _ _ O 19 | DIY _ _ B-ProductDesign 20 | dyeing _ _ I-ProductDesign 21 | . _ _ O -------------------------------------------------------------------------------- /utils/tests/resources/tasks/germeval/NER-de-dev.tsv: -------------------------------------------------------------------------------- 1 | # http://de.wikipedia.org/wiki/Toyota_Crown [2009-08-13] 2 | 1 1980 O O 3 | 2 kam O O 4 | 3 der O O 5 | 4 Crown B-OTH O 6 | 5 als O O 7 | 6 Versuch O O 8 | 7 von O O 9 | 8 Toyota B-ORG O 10 | 9 , O O 11 | 10 sich O O 12 | 11 in O O 13 | 12 der O O 14 | 13 Oberen O O 15 | 14 Mittelklasse O O 16 | 15 zu O O 17 | 16 etablieren O O 18 | 17 , O O 19 | 18 auch O O 20 | 19 nach O O 21 | 20 Deutschland B-LOC O 22 | 21 . O O 23 | -------------------------------------------------------------------------------- /utils/tests/resources/tasks/germeval/NER-de-test.tsv: -------------------------------------------------------------------------------- 1 | # http://de.wikipedia.org/wiki/Schönburg_(Rhein) [2009-10-23] 2 | 1 1951 O O 3 | 2 bis O O 4 | 3 1953 O O 5 | 4 wurde O O 6 | 5 der O O 7 | 6 nördliche O O 8 | 7 Teil O O 9 | 8 als O O 10 | 9 Jugendburg O O 11 | 10 des O O 12 | 11 Kolpingwerkes B-OTH O 13 | 12 gebaut O O 14 | 13 . O O -------------------------------------------------------------------------------- /utils/tests/resources/tasks/germeval/NER-de-train.tsv: -------------------------------------------------------------------------------- 1 | # n-tv.de vom 26.02.2005 [2005-02-26] 2 | 1 Schartau B-PER O 3 | 2 sagte O O 4 | 3 dem O O 5 | 4 " O O 6 | 5 Tagesspiegel B-ORG O 7 | 6 " O O 8 | 7 vom O O 9 | 8 Freitag O O 10 | 9 , O O 11 | 10 Fischer B-PER O 12 | 11 sei O O 13 | 12 " O O 14 | 13 in O O 15 | 14 einer O O 16 | 15 Weise O O 17 | 16 aufgetreten O O 18 | 17 , O O 19 | 18 die O O 20 | 19 alles O O 21 | 20 andere O O 22 | 21 als O O 23 | 22 überzeugend O O 24 | 23 war O O 25 | 24 " O O 26 | 25 . O O 27 | 28 | # welt.de vom 29.10.2005 [2005-10-29] 29 | 1 Firmengründer O O 30 | 2 Wolf B-PER O 31 | 3 Peter I-PER O 32 | 4 Bree I-PER O 33 | 5 arbeitete O O 34 | 6 Anfang O O 35 | 7 der O O 36 | 8 siebziger O O 37 | 9 Jahre O O 38 | 10 als O O 39 | 11 Möbelvertreter O O 40 | 12 , O O 41 | 13 als O O 42 | 14 er O O 43 | 15 einen O O 44 | 16 fliegenden O O 45 | 17 Händler O O 46 | 18 aus O O 47 | 19 dem O O 48 | 20 Libanon B-LOC O 49 | 21 traf O O 50 | 22 . O O -------------------------------------------------------------------------------- /utils/tests/resources/tasks/imdb/README.md: -------------------------------------------------------------------------------- 1 | ## IMDB 2 | 3 | Data is taken from [here](http://ai.stanford.edu/~amaas/data/sentiment/). 4 | 5 | The dataset contains data for a binary sentiment classification. 6 | We took a small random sample and converted it to the expected format of our data fetcher: 7 | ``` 8 | __label__ 9 | ``` 10 | 11 | #### Publications Using the Dataset 12 | 13 | * Andrew L. Maas, Raymond E. Daly, Peter T. Pham, Dan Huang, Andrew Y. Ng, and Christopher Potts. (2011). Learning Word Vectors for Sentiment Analysis. The 49th Annual Meeting of the Association for Computational Linguistics (ACL 2011). -------------------------------------------------------------------------------- /utils/tests/resources/tasks/multi_class/dev.txt: -------------------------------------------------------------------------------- 1 | __label__apple apple 2 | __label__tv tv 3 | __label__guitar guitar 4 | __label__apple __label__tv apple tv -------------------------------------------------------------------------------- /utils/tests/resources/tasks/multi_class/test.txt: -------------------------------------------------------------------------------- 1 | __label__guitar guitar 2 | __label__apple apple 3 | __label__tv tv 4 | __label__apple __label__tv apple tv 5 | __label__apple __label__guitar apple tv -------------------------------------------------------------------------------- /utils/tests/resources/tasks/multi_class/train.txt: -------------------------------------------------------------------------------- 1 | __label__tv tv 2 | __label__apple __label__tv apple tv 3 | __label__apple apple 4 | __label__tv tv 5 | __label__apple __label__tv apple tv 6 | __label__guitar guitar 7 | __label__guitar guitar -------------------------------------------------------------------------------- /utils/tests/resources/tasks/regression/README.md: -------------------------------------------------------------------------------- 1 | ## REGRESSION 2 | 3 | Data is taken from [here](http://saifmohammad.com/WebPages/EmotionIntensity-SharedTask.html). 4 | 5 | The dataset contains a collection of tweets with joy intensity value. 6 | We took the joy dataset and converted it to the expected format of our data fetcher: 7 | ``` 8 | __label__ 9 | ``` 10 | 11 | #### Publication About the Dataset 12 | 13 | * Emotion Intensities in Tweets. Saif M. Mohammad and Felipe Bravo-Marquez. In Proceedings of the sixth joint conference on lexical and computational semantics (*Sem), August 2017, Vancouver, Canada. 14 | * WASSA-2017 Shared Task on Emotion Intensity. Saif M. Mohammad and Felipe Bravo-Marquez. In Proceedings of the EMNLP 2017 Workshop on Computational Approaches to Subjectivity, Sentiment, and Social Media (WASSA), September 2017, Copenhagen, Denmark. -------------------------------------------------------------------------------- /utils/tests/resources/visual/snippet.txt: -------------------------------------------------------------------------------- 1 | The U.S. Centers for Disease Control and Prevention initially advised school systems to close if outbreaks occurred , then reversed itself , saying the apparent mildness of the virus meant most schools and day care centers should stay open , even if they had confirmed cases of swine flu . 2 | When Ms. Winfrey invited Suzanne Somers to share her controversial views about bio-identical hormone treatment on her syndicated show in 2009 , it won Ms. Winfrey a rare dollop of unflattering press , including a Newsweek cover story titled " Crazy Talk : Oprah , Wacky Cures & You . " 3 | Elk calling -- a skill that hunters perfected long ago to lure game with the promise of a little romance -- is now its own sport . 4 | Don 't ! 5 | Fish , ranked 98th in the world , fired 22 aces en route to a 6-3 , 6-7 ( 5 / 7 ) , 7-6 ( 7 / 4 ) win over seventh-seeded Argentinian David Nalbandian . 6 | Why does everything have to become such a big issue ? 7 | AMMAN ( Reuters ) - King Abdullah of Jordan will meet U.S. President Barack Obama in Washington on April 21 to lobby on behalf of Arab states for a stronger U.S. role in Middle East peacemaking , palace officials said on Sunday . 8 | To help keep traffic flowing the Congestion Charge will remain in operation through-out the strike and TfL will be suspending road works on major London roads wherever possible . 9 | If no candidate wins an absolute majority , there will be a runoff between the top two contenders , most likely in mid-October . 10 | Authorities previously served search warrants at Murray 's Las Vegas home and his businesses in Las Vegas and Houston . -------------------------------------------------------------------------------- /utils/time.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | 4 | def time_to_str(timestamp: int) -> str: 5 | """ 6 | Convert seconds past Epoch to human readable string. 7 | """ 8 | datetimestamp = datetime.datetime.fromtimestamp(timestamp) 9 | return '{:04d}-{:02d}-{:02d}-{:02d}-{:02d}-{:02d}'.format( 10 | datetimestamp.year, datetimestamp.month, datetimestamp.day, 11 | datetimestamp.hour, datetimestamp.minute, datetimestamp.second 12 | ) 13 | 14 | 15 | def str_to_time(time_str: str) -> datetime.datetime: 16 | """ 17 | Convert human readable string to datetime.datetime. 18 | """ 19 | pieces = [int(piece) for piece in time_str.split('-')] 20 | return datetime.datetime(*pieces) 21 | -------------------------------------------------------------------------------- /utils/tools/split.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import random 3 | 4 | # percent = 0.5 5 | file_path = 'conll_03_vi.train_bioes' 6 | percent = 0.8 7 | file_path_out = 'conll_03_vietnamese.train_bioes'.format(percent) 8 | file_out = open(file_path_out, 'w', encoding='utf-8') 9 | with open(file_path, 'r', encoding='utf-8') as file: 10 | lines = file.readlines() 11 | all_samples = [] 12 | sample = [] 13 | for line in lines: 14 | if line == '\n': 15 | all_samples.append(sample) 16 | sample = [] 17 | continue 18 | sample.append(line) 19 | # file_out.write('\n') 20 | 21 | random.shuffle(all_samples) 22 | train_sents = int(percent * len(all_samples)) 23 | dev_sents = int(0.9 * len(all_samples)) 24 | test_sents = len(all_samples) - dev_sents 25 | train_samples = all_samples[:train_sents] 26 | print(f'train: 0-{train_sents}, dev: {train_sents}-{dev_sents}, test: {train_sents}-{len(all_samples)}') 27 | for sample in train_samples: 28 | for line in sample: 29 | file_out.write(line) 30 | file_out.write('\n') 31 | 32 | dev_file = 'conll_03_vietnamese.testa_bioes' 33 | dev_file = open(dev_file, 'w', encoding='utf-8') 34 | for sample in all_samples[train_sents: dev_sents]: 35 | for line in sample: 36 | dev_file.write(line) 37 | dev_file.write('\n') 38 | 39 | test_file = 'conll_03_vietnamese.testb_bioes' 40 | test_file = open(test_file, 'w', encoding='utf-8') 41 | for sample in all_samples[dev_sents:]: 42 | for line in sample: 43 | test_file.write(line) 44 | test_file.write('\n') 45 | 46 | 47 | file_out.close() 48 | test_file.close() 49 | dev_file.close() -------------------------------------------------------------------------------- /utils/tqdm.py: -------------------------------------------------------------------------------- 1 | """ 2 | :class:`~allennlp.common.tqdm.Tqdm` wraps tqdm so we can add configurable 3 | global defaults for certain tqdm parameters. 4 | 5 | Adopted from AllenNLP: 6 | https://github.com/allenai/allennlp/blob/v0.6.1/allennlp/common/tqdm.py 7 | """ 8 | 9 | from tqdm import tqdm as _tqdm 10 | # This is neccesary to stop tqdm from hanging 11 | # when exceptions are raised inside iterators. 12 | # It should have been fixed in 4.2.1, but it still 13 | # occurs. 14 | # TODO(Mark): Remove this once tqdm cleans up after itself properly. 15 | # https://github.com/tqdm/tqdm/issues/469 16 | _tqdm.monitor_interval = 0 17 | 18 | class Tqdm: 19 | # These defaults are the same as the argument defaults in tqdm. 20 | default_mininterval: float = 0.1 21 | 22 | @staticmethod 23 | def set_default_mininterval(value: float) -> None: 24 | Tqdm.default_mininterval = value 25 | 26 | @staticmethod 27 | def set_slower_interval(use_slower_interval: bool) -> None: 28 | """ 29 | If ``use_slower_interval`` is ``True``, we will dramatically slow down ``tqdm's`` default 30 | output rate. ``tqdm's`` default output rate is great for interactively watching progress, 31 | but it is not great for log files. You might want to set this if you are primarily going 32 | to be looking at output through log files, not the terminal. 33 | """ 34 | if use_slower_interval: 35 | Tqdm.default_mininterval = 10.0 36 | else: 37 | Tqdm.default_mininterval = 0.1 38 | 39 | @staticmethod 40 | def tqdm(*args, **kwargs): 41 | new_kwargs = { 42 | 'mininterval': Tqdm.default_mininterval, 43 | **kwargs 44 | } 45 | 46 | return _tqdm(*args, **new_kwargs) 47 | --------------------------------------------------------------------------------