├── .gitignore ├── LICENSE ├── README.md ├── datasets ├── __init__.py ├── collate_functions.py ├── data_sampling.py ├── mrc_ner_dataset.py ├── mrpc_dataset.py ├── mrpc_processor.py ├── squad_dataset.py ├── tnews_dataset.py └── truncate_dataset.py ├── loss ├── __init__.py ├── dice_loss.py └── focal_loss.py ├── metrics ├── __init__.py ├── classification_acc_f1.py ├── functional │ ├── __init__.py │ ├── cls_acc_f1.py │ ├── ner_span_f1.py │ └── squad │ │ ├── README.md │ │ ├── __init__.py │ │ ├── eval.sh │ │ ├── evaluate_v1.py │ │ ├── evaluate_v2.py │ │ └── postprocess_predication.py ├── mrc_ner_span_f1.py └── squad_em_f1.py ├── models ├── __init__.py ├── bert_classification.py ├── bert_qa.py ├── bert_query_ner.py ├── classifier.py └── model_config.py ├── requirements.txt ├── scripts ├── download_ckpt.sh ├── glue_mrpc │ ├── bert_base_ce.sh │ ├── bert_base_dice.sh │ ├── bert_base_focal.sh │ ├── bert_large_ce.sh │ ├── bert_large_dice.sh │ ├── bert_large_focal.sh │ └── eval.sh ├── mrc_squad1 │ ├── bert_base_ce.sh │ ├── bert_base_dice.sh │ ├── bert_base_focal.sh │ ├── bert_large_ce.sh │ ├── bert_large_dice.sh │ ├── bert_large_focal.sh │ ├── eval_pred_file.sh │ └── eval_saved_ckpt.sh ├── ner_enconll03 │ ├── bert_dice.sh │ └── bert_focal.sh ├── ner_enontonotes5 │ ├── bert_dice.sh │ └── bert_focal.sh ├── ner_zhmsra │ ├── bert_dice.sh │ └── bert_focal.sh ├── ner_zhonto4 │ ├── bert_dice.sh │ └── bert_focal.sh ├── prepare_ckpt.sh ├── prepare_mrpc_data.sh └── textcl_tnews │ ├── bert_dice.sh │ └── bert_focal.sh ├── tasks ├── glue │ ├── download_glue_data.py │ ├── evaluate_models.py │ ├── evaluate_predictions.py │ ├── mrpc_dev_ids.tsv │ ├── process_mrpc.py │ └── train.py ├── mrc_ner │ ├── data_preprocess │ │ ├── file_utils.py │ │ ├── label_utils.py │ │ └── query_map.py │ ├── eval.sh │ ├── evaluate.py │ ├── generate_mrc_dataset.py │ ├── train.py │ └── train.sh ├── pos │ └── train.py ├── squad │ ├── evaluate_models.py │ ├── evaluate_predictions.py │ └── train.py └── tnews │ ├── train.py │ └── train.sh ├── tests ├── count_length_autotokenizer.py └── count_length_glue.py └── utils ├── __init__.py ├── get_parser.py └── random_seed.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/README.md -------------------------------------------------------------------------------- /datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /datasets/collate_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/datasets/collate_functions.py -------------------------------------------------------------------------------- /datasets/data_sampling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/datasets/data_sampling.py -------------------------------------------------------------------------------- /datasets/mrc_ner_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/datasets/mrc_ner_dataset.py -------------------------------------------------------------------------------- /datasets/mrpc_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/datasets/mrpc_dataset.py -------------------------------------------------------------------------------- /datasets/mrpc_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/datasets/mrpc_processor.py -------------------------------------------------------------------------------- /datasets/squad_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/datasets/squad_dataset.py -------------------------------------------------------------------------------- /datasets/tnews_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/datasets/tnews_dataset.py -------------------------------------------------------------------------------- /datasets/truncate_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/datasets/truncate_dataset.py -------------------------------------------------------------------------------- /loss/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/loss/__init__.py -------------------------------------------------------------------------------- /loss/dice_loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/loss/dice_loss.py -------------------------------------------------------------------------------- /loss/focal_loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/loss/focal_loss.py -------------------------------------------------------------------------------- /metrics/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics/classification_acc_f1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/metrics/classification_acc_f1.py -------------------------------------------------------------------------------- /metrics/functional/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics/functional/cls_acc_f1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/metrics/functional/cls_acc_f1.py -------------------------------------------------------------------------------- /metrics/functional/ner_span_f1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/metrics/functional/ner_span_f1.py -------------------------------------------------------------------------------- /metrics/functional/squad/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/metrics/functional/squad/README.md -------------------------------------------------------------------------------- /metrics/functional/squad/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metrics/functional/squad/eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/metrics/functional/squad/eval.sh -------------------------------------------------------------------------------- /metrics/functional/squad/evaluate_v1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/metrics/functional/squad/evaluate_v1.py -------------------------------------------------------------------------------- /metrics/functional/squad/evaluate_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/metrics/functional/squad/evaluate_v2.py -------------------------------------------------------------------------------- /metrics/functional/squad/postprocess_predication.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/metrics/functional/squad/postprocess_predication.py -------------------------------------------------------------------------------- /metrics/mrc_ner_span_f1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/metrics/mrc_ner_span_f1.py -------------------------------------------------------------------------------- /metrics/squad_em_f1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/metrics/squad_em_f1.py -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/bert_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/models/bert_classification.py -------------------------------------------------------------------------------- /models/bert_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/models/bert_qa.py -------------------------------------------------------------------------------- /models/bert_query_ner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/models/bert_query_ner.py -------------------------------------------------------------------------------- /models/classifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/models/classifier.py -------------------------------------------------------------------------------- /models/model_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/models/model_config.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/download_ckpt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/download_ckpt.sh -------------------------------------------------------------------------------- /scripts/glue_mrpc/bert_base_ce.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/glue_mrpc/bert_base_ce.sh -------------------------------------------------------------------------------- /scripts/glue_mrpc/bert_base_dice.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/glue_mrpc/bert_base_dice.sh -------------------------------------------------------------------------------- /scripts/glue_mrpc/bert_base_focal.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/glue_mrpc/bert_base_focal.sh -------------------------------------------------------------------------------- /scripts/glue_mrpc/bert_large_ce.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/glue_mrpc/bert_large_ce.sh -------------------------------------------------------------------------------- /scripts/glue_mrpc/bert_large_dice.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/glue_mrpc/bert_large_dice.sh -------------------------------------------------------------------------------- /scripts/glue_mrpc/bert_large_focal.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/glue_mrpc/bert_large_focal.sh -------------------------------------------------------------------------------- /scripts/glue_mrpc/eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/glue_mrpc/eval.sh -------------------------------------------------------------------------------- /scripts/mrc_squad1/bert_base_ce.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/mrc_squad1/bert_base_ce.sh -------------------------------------------------------------------------------- /scripts/mrc_squad1/bert_base_dice.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/mrc_squad1/bert_base_dice.sh -------------------------------------------------------------------------------- /scripts/mrc_squad1/bert_base_focal.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/mrc_squad1/bert_base_focal.sh -------------------------------------------------------------------------------- /scripts/mrc_squad1/bert_large_ce.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/mrc_squad1/bert_large_ce.sh -------------------------------------------------------------------------------- /scripts/mrc_squad1/bert_large_dice.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/mrc_squad1/bert_large_dice.sh -------------------------------------------------------------------------------- /scripts/mrc_squad1/bert_large_focal.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/mrc_squad1/bert_large_focal.sh -------------------------------------------------------------------------------- /scripts/mrc_squad1/eval_pred_file.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/mrc_squad1/eval_pred_file.sh -------------------------------------------------------------------------------- /scripts/mrc_squad1/eval_saved_ckpt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/mrc_squad1/eval_saved_ckpt.sh -------------------------------------------------------------------------------- /scripts/ner_enconll03/bert_dice.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/ner_enconll03/bert_dice.sh -------------------------------------------------------------------------------- /scripts/ner_enconll03/bert_focal.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/ner_enconll03/bert_focal.sh -------------------------------------------------------------------------------- /scripts/ner_enontonotes5/bert_dice.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/ner_enontonotes5/bert_dice.sh -------------------------------------------------------------------------------- /scripts/ner_enontonotes5/bert_focal.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/ner_enontonotes5/bert_focal.sh -------------------------------------------------------------------------------- /scripts/ner_zhmsra/bert_dice.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/ner_zhmsra/bert_dice.sh -------------------------------------------------------------------------------- /scripts/ner_zhmsra/bert_focal.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/ner_zhmsra/bert_focal.sh -------------------------------------------------------------------------------- /scripts/ner_zhonto4/bert_dice.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/ner_zhonto4/bert_dice.sh -------------------------------------------------------------------------------- /scripts/ner_zhonto4/bert_focal.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/ner_zhonto4/bert_focal.sh -------------------------------------------------------------------------------- /scripts/prepare_ckpt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/prepare_ckpt.sh -------------------------------------------------------------------------------- /scripts/prepare_mrpc_data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/prepare_mrpc_data.sh -------------------------------------------------------------------------------- /scripts/textcl_tnews/bert_dice.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/textcl_tnews/bert_dice.sh -------------------------------------------------------------------------------- /scripts/textcl_tnews/bert_focal.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/scripts/textcl_tnews/bert_focal.sh -------------------------------------------------------------------------------- /tasks/glue/download_glue_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/glue/download_glue_data.py -------------------------------------------------------------------------------- /tasks/glue/evaluate_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/glue/evaluate_models.py -------------------------------------------------------------------------------- /tasks/glue/evaluate_predictions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/glue/evaluate_predictions.py -------------------------------------------------------------------------------- /tasks/glue/mrpc_dev_ids.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/glue/mrpc_dev_ids.tsv -------------------------------------------------------------------------------- /tasks/glue/process_mrpc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/glue/process_mrpc.py -------------------------------------------------------------------------------- /tasks/glue/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/glue/train.py -------------------------------------------------------------------------------- /tasks/mrc_ner/data_preprocess/file_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/mrc_ner/data_preprocess/file_utils.py -------------------------------------------------------------------------------- /tasks/mrc_ner/data_preprocess/label_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/mrc_ner/data_preprocess/label_utils.py -------------------------------------------------------------------------------- /tasks/mrc_ner/data_preprocess/query_map.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/mrc_ner/data_preprocess/query_map.py -------------------------------------------------------------------------------- /tasks/mrc_ner/eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/mrc_ner/eval.sh -------------------------------------------------------------------------------- /tasks/mrc_ner/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/mrc_ner/evaluate.py -------------------------------------------------------------------------------- /tasks/mrc_ner/generate_mrc_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/mrc_ner/generate_mrc_dataset.py -------------------------------------------------------------------------------- /tasks/mrc_ner/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/mrc_ner/train.py -------------------------------------------------------------------------------- /tasks/mrc_ner/train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/mrc_ner/train.sh -------------------------------------------------------------------------------- /tasks/pos/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/pos/train.py -------------------------------------------------------------------------------- /tasks/squad/evaluate_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/squad/evaluate_models.py -------------------------------------------------------------------------------- /tasks/squad/evaluate_predictions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/squad/evaluate_predictions.py -------------------------------------------------------------------------------- /tasks/squad/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/squad/train.py -------------------------------------------------------------------------------- /tasks/tnews/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/tnews/train.py -------------------------------------------------------------------------------- /tasks/tnews/train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tasks/tnews/train.sh -------------------------------------------------------------------------------- /tests/count_length_autotokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tests/count_length_autotokenizer.py -------------------------------------------------------------------------------- /tests/count_length_glue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/tests/count_length_glue.py -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/get_parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/utils/get_parser.py -------------------------------------------------------------------------------- /utils/random_seed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShannonAI/dice_loss_for_NLP/HEAD/utils/random_seed.py --------------------------------------------------------------------------------