├── .gitignore ├── README.md ├── heatmap ├── __init__.py ├── heatmap_utils.py └── sequence_scorer.py ├── parse_nmt.py ├── preprocess_nstack2seq_merge.py ├── runs ├── get_last_checkpoint.py ├── infer_model.sh ├── run_nstack_nmt.sh ├── train_classification.py ├── train_fairseq.sh └── treetfm_train.py ├── scripts ├── __init__.py ├── average_checkpoints.py ├── build_sym_alignment.py ├── compound_split_bleu.sh ├── convert_dictionary.lua ├── convert_imdb.py ├── convert_model.lua ├── read_binarized.py └── sacrebleu_pregen.sh └── src ├── __init__.py ├── binarization.py ├── bpe ├── __init__.py └── bpe_utils.py ├── criterions ├── __init__.py ├── classification_cross_entropy.py └── masked_lm_loss.py ├── data ├── __init__.py ├── dptree2seq_dataset.py ├── dptree2seq_sep_dataset.py ├── dptree_dictionary.py ├── dptree_index_dataset.py ├── dptree_mono_class_dataset.py ├── dptree_sep_mono_class_dataset.py ├── monolingual_classification_dataset.py ├── nstack2seq_dataset.py ├── nstack_merge_monoclass_dataset.py ├── nstack_mono_class_dataset.py ├── task_utils.py └── transforms │ ├── __init__.py │ ├── gpt2_bpe.py │ ├── moses_tokenizer.py │ ├── nltk_tokenizer.py │ ├── sentencepiece_bpe.py │ ├── space_tokenizer.py │ └── subword_nmt_bpe.py ├── dptree ├── __init__.py ├── nstack_process.py ├── tree_builder.py └── tree_process.py ├── dptree2seq_generator.py ├── dptree_tokenizer.py ├── models ├── __init__.py ├── dptree2seg_transformer.py ├── nstack_archs.py └── nstack_transformer.py ├── modules ├── __init__.py ├── default_dy_conv.py ├── default_multihead_attention.py ├── dptree_individual_multihead_attention.py ├── dptree_multihead_attention.py ├── dptree_onseq_multihead_attention.py ├── dptree_sep_multihead_attention.py ├── dptree_transformer_layer.py ├── embeddings.py ├── nstack_merge_tree_attention.py ├── nstack_transformer_layers.py ├── nstack_tree_attention.py └── nstack_tree_attention_eff.py ├── nstack2seq_generator.py ├── nstack_tokenizer.py ├── optim ├── __init__.py └── lr_scheduler │ ├── __init__.py │ └── flex_inv_sqrt_schedule.py ├── tasks ├── __init__.py ├── dptree2seq_sep_translation.py ├── dptree2seq_translation.py ├── dptree_classification.py ├── dptree_sep_classification.py ├── fairseq_classification.py ├── nstack2seq_translation.py ├── nstack_classification.py └── nstack_from_dptree_classification.py ├── trainers └── __init__.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/README.md -------------------------------------------------------------------------------- /heatmap/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/heatmap/__init__.py -------------------------------------------------------------------------------- /heatmap/heatmap_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/heatmap/heatmap_utils.py -------------------------------------------------------------------------------- /heatmap/sequence_scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/heatmap/sequence_scorer.py -------------------------------------------------------------------------------- /parse_nmt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/parse_nmt.py -------------------------------------------------------------------------------- /preprocess_nstack2seq_merge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/preprocess_nstack2seq_merge.py -------------------------------------------------------------------------------- /runs/get_last_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/runs/get_last_checkpoint.py -------------------------------------------------------------------------------- /runs/infer_model.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/runs/infer_model.sh -------------------------------------------------------------------------------- /runs/run_nstack_nmt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/runs/run_nstack_nmt.sh -------------------------------------------------------------------------------- /runs/train_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/runs/train_classification.py -------------------------------------------------------------------------------- /runs/train_fairseq.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/runs/train_fairseq.sh -------------------------------------------------------------------------------- /runs/treetfm_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/runs/treetfm_train.py -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/average_checkpoints.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/scripts/average_checkpoints.py -------------------------------------------------------------------------------- /scripts/build_sym_alignment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/scripts/build_sym_alignment.py -------------------------------------------------------------------------------- /scripts/compound_split_bleu.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/scripts/compound_split_bleu.sh -------------------------------------------------------------------------------- /scripts/convert_dictionary.lua: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/scripts/convert_dictionary.lua -------------------------------------------------------------------------------- /scripts/convert_imdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/scripts/convert_imdb.py -------------------------------------------------------------------------------- /scripts/convert_model.lua: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/scripts/convert_model.lua -------------------------------------------------------------------------------- /scripts/read_binarized.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/scripts/read_binarized.py -------------------------------------------------------------------------------- /scripts/sacrebleu_pregen.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/scripts/sacrebleu_pregen.sh -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/__init__.py -------------------------------------------------------------------------------- /src/binarization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/binarization.py -------------------------------------------------------------------------------- /src/bpe/__init__.py: -------------------------------------------------------------------------------- 1 | from . import bpe_utils -------------------------------------------------------------------------------- /src/bpe/bpe_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/bpe/bpe_utils.py -------------------------------------------------------------------------------- /src/criterions/__init__.py: -------------------------------------------------------------------------------- 1 | from . import classification_cross_entropy 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /src/criterions/classification_cross_entropy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/criterions/classification_cross_entropy.py -------------------------------------------------------------------------------- /src/criterions/masked_lm_loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/criterions/masked_lm_loss.py -------------------------------------------------------------------------------- /src/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/__init__.py -------------------------------------------------------------------------------- /src/data/dptree2seq_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/dptree2seq_dataset.py -------------------------------------------------------------------------------- /src/data/dptree2seq_sep_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/dptree2seq_sep_dataset.py -------------------------------------------------------------------------------- /src/data/dptree_dictionary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/dptree_dictionary.py -------------------------------------------------------------------------------- /src/data/dptree_index_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/dptree_index_dataset.py -------------------------------------------------------------------------------- /src/data/dptree_mono_class_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/dptree_mono_class_dataset.py -------------------------------------------------------------------------------- /src/data/dptree_sep_mono_class_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/dptree_sep_mono_class_dataset.py -------------------------------------------------------------------------------- /src/data/monolingual_classification_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/monolingual_classification_dataset.py -------------------------------------------------------------------------------- /src/data/nstack2seq_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/nstack2seq_dataset.py -------------------------------------------------------------------------------- /src/data/nstack_merge_monoclass_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/nstack_merge_monoclass_dataset.py -------------------------------------------------------------------------------- /src/data/nstack_mono_class_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/nstack_mono_class_dataset.py -------------------------------------------------------------------------------- /src/data/task_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/task_utils.py -------------------------------------------------------------------------------- /src/data/transforms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/transforms/__init__.py -------------------------------------------------------------------------------- /src/data/transforms/gpt2_bpe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/transforms/gpt2_bpe.py -------------------------------------------------------------------------------- /src/data/transforms/moses_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/transforms/moses_tokenizer.py -------------------------------------------------------------------------------- /src/data/transforms/nltk_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/transforms/nltk_tokenizer.py -------------------------------------------------------------------------------- /src/data/transforms/sentencepiece_bpe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/transforms/sentencepiece_bpe.py -------------------------------------------------------------------------------- /src/data/transforms/space_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/transforms/space_tokenizer.py -------------------------------------------------------------------------------- /src/data/transforms/subword_nmt_bpe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/data/transforms/subword_nmt_bpe.py -------------------------------------------------------------------------------- /src/dptree/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/dptree/__init__.py -------------------------------------------------------------------------------- /src/dptree/nstack_process.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/dptree/nstack_process.py -------------------------------------------------------------------------------- /src/dptree/tree_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/dptree/tree_builder.py -------------------------------------------------------------------------------- /src/dptree/tree_process.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/dptree/tree_process.py -------------------------------------------------------------------------------- /src/dptree2seq_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/dptree2seq_generator.py -------------------------------------------------------------------------------- /src/dptree_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/dptree_tokenizer.py -------------------------------------------------------------------------------- /src/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/models/__init__.py -------------------------------------------------------------------------------- /src/models/dptree2seg_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/models/dptree2seg_transformer.py -------------------------------------------------------------------------------- /src/models/nstack_archs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/models/nstack_archs.py -------------------------------------------------------------------------------- /src/models/nstack_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/models/nstack_transformer.py -------------------------------------------------------------------------------- /src/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/__init__.py -------------------------------------------------------------------------------- /src/modules/default_dy_conv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/default_dy_conv.py -------------------------------------------------------------------------------- /src/modules/default_multihead_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/default_multihead_attention.py -------------------------------------------------------------------------------- /src/modules/dptree_individual_multihead_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/dptree_individual_multihead_attention.py -------------------------------------------------------------------------------- /src/modules/dptree_multihead_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/dptree_multihead_attention.py -------------------------------------------------------------------------------- /src/modules/dptree_onseq_multihead_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/dptree_onseq_multihead_attention.py -------------------------------------------------------------------------------- /src/modules/dptree_sep_multihead_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/dptree_sep_multihead_attention.py -------------------------------------------------------------------------------- /src/modules/dptree_transformer_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/dptree_transformer_layer.py -------------------------------------------------------------------------------- /src/modules/embeddings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/embeddings.py -------------------------------------------------------------------------------- /src/modules/nstack_merge_tree_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/nstack_merge_tree_attention.py -------------------------------------------------------------------------------- /src/modules/nstack_transformer_layers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/nstack_transformer_layers.py -------------------------------------------------------------------------------- /src/modules/nstack_tree_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/nstack_tree_attention.py -------------------------------------------------------------------------------- /src/modules/nstack_tree_attention_eff.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/modules/nstack_tree_attention_eff.py -------------------------------------------------------------------------------- /src/nstack2seq_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/nstack2seq_generator.py -------------------------------------------------------------------------------- /src/nstack_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/nstack_tokenizer.py -------------------------------------------------------------------------------- /src/optim/__init__.py: -------------------------------------------------------------------------------- 1 | from . import lr_scheduler -------------------------------------------------------------------------------- /src/optim/lr_scheduler/__init__.py: -------------------------------------------------------------------------------- 1 | from . import flex_inv_sqrt_schedule 2 | 3 | 4 | -------------------------------------------------------------------------------- /src/optim/lr_scheduler/flex_inv_sqrt_schedule.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/optim/lr_scheduler/flex_inv_sqrt_schedule.py -------------------------------------------------------------------------------- /src/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/tasks/__init__.py -------------------------------------------------------------------------------- /src/tasks/dptree2seq_sep_translation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/tasks/dptree2seq_sep_translation.py -------------------------------------------------------------------------------- /src/tasks/dptree2seq_translation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/tasks/dptree2seq_translation.py -------------------------------------------------------------------------------- /src/tasks/dptree_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/tasks/dptree_classification.py -------------------------------------------------------------------------------- /src/tasks/dptree_sep_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/tasks/dptree_sep_classification.py -------------------------------------------------------------------------------- /src/tasks/fairseq_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/tasks/fairseq_classification.py -------------------------------------------------------------------------------- /src/tasks/nstack2seq_translation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/tasks/nstack2seq_translation.py -------------------------------------------------------------------------------- /src/tasks/nstack_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/tasks/nstack_classification.py -------------------------------------------------------------------------------- /src/tasks/nstack_from_dptree_classification.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/tasks/nstack_from_dptree_classification.py -------------------------------------------------------------------------------- /src/trainers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nxphi47/tree_transformer/HEAD/src/utils.py --------------------------------------------------------------------------------