├── .gitignore ├── LICENSE ├── README.md ├── data └── snips │ ├── test │ ├── label │ ├── seq.in │ └── seq.out │ ├── train │ ├── label │ ├── seq.in │ └── seq.out │ └── valid │ ├── label │ ├── seq.in │ └── seq.out ├── examples ├── README.md ├── convert_to_tflite.py ├── evaluate_bert_crf_nlu.py ├── evaluate_tflite_transformer_nlu.py ├── evaluate_transformer_nlu.py ├── incremental_training.py ├── predict_auto_nlu.py ├── predict_tflite_nlu.py ├── train_bert_crf_nlu.py ├── train_bert_nlu.py ├── train_transformer_nlu.py └── train_transformer_nlu_with_layer_pruning.py ├── img ├── Layer_dropping_Strategies.PNG └── joint_bert.PNG ├── notebooks ├── tflite_compress_nlu_model.ipynb ├── using_bert_crf_nlu.ipynb ├── using_bert_nlu.ipynb ├── using_compressed_nlu_model.ipynb ├── using_trans_nlu.ipynb └── using_trans_nlu_with_pruning.ipynb ├── scripts ├── README.md ├── evaluate.py ├── nlu_basic_api.py ├── train_joint_bert.py └── train_joint_trans.py ├── setup.py └── src └── dialognlu ├── __init__.py ├── auto_nlu.py ├── compression ├── __init__.py ├── commons.py └── layer_pruning.py ├── layers ├── __init__.py └── crf_layer.py ├── models ├── __init__.py ├── base_joint_trans.py ├── callbacks.py ├── joint_bert.py ├── joint_bert_crf.py ├── joint_trans_albert.py ├── joint_trans_bert.py ├── joint_trans_distilbert.py ├── joint_trans_roberta.py ├── joint_trans_xlnet.py ├── model_pool.py ├── nlu_model.py └── trans_auto_model.py ├── nlu_components.py ├── readers ├── __init__.py ├── dataset.py └── goo_format_reader.py ├── utils ├── __init__.py ├── data_utils.py ├── io_utils.py └── tf_utils.py └── vectorizers ├── __init__.py ├── albert_tokenization.py ├── bert_vectorizer.py ├── tags_vectorizer.py ├── tokenization.py └── trans_vectorizer.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/README.md -------------------------------------------------------------------------------- /data/snips/test/label: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/data/snips/test/label -------------------------------------------------------------------------------- /data/snips/test/seq.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/data/snips/test/seq.in -------------------------------------------------------------------------------- /data/snips/test/seq.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/data/snips/test/seq.out -------------------------------------------------------------------------------- /data/snips/train/label: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/data/snips/train/label -------------------------------------------------------------------------------- /data/snips/train/seq.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/data/snips/train/seq.in -------------------------------------------------------------------------------- /data/snips/train/seq.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/data/snips/train/seq.out -------------------------------------------------------------------------------- /data/snips/valid/label: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/data/snips/valid/label -------------------------------------------------------------------------------- /data/snips/valid/seq.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/data/snips/valid/seq.in -------------------------------------------------------------------------------- /data/snips/valid/seq.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/data/snips/valid/seq.out -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/README.md -------------------------------------------------------------------------------- /examples/convert_to_tflite.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/convert_to_tflite.py -------------------------------------------------------------------------------- /examples/evaluate_bert_crf_nlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/evaluate_bert_crf_nlu.py -------------------------------------------------------------------------------- /examples/evaluate_tflite_transformer_nlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/evaluate_tflite_transformer_nlu.py -------------------------------------------------------------------------------- /examples/evaluate_transformer_nlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/evaluate_transformer_nlu.py -------------------------------------------------------------------------------- /examples/incremental_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/incremental_training.py -------------------------------------------------------------------------------- /examples/predict_auto_nlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/predict_auto_nlu.py -------------------------------------------------------------------------------- /examples/predict_tflite_nlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/predict_tflite_nlu.py -------------------------------------------------------------------------------- /examples/train_bert_crf_nlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/train_bert_crf_nlu.py -------------------------------------------------------------------------------- /examples/train_bert_nlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/train_bert_nlu.py -------------------------------------------------------------------------------- /examples/train_transformer_nlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/train_transformer_nlu.py -------------------------------------------------------------------------------- /examples/train_transformer_nlu_with_layer_pruning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/examples/train_transformer_nlu_with_layer_pruning.py -------------------------------------------------------------------------------- /img/Layer_dropping_Strategies.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/img/Layer_dropping_Strategies.PNG -------------------------------------------------------------------------------- /img/joint_bert.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/img/joint_bert.PNG -------------------------------------------------------------------------------- /notebooks/tflite_compress_nlu_model.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/notebooks/tflite_compress_nlu_model.ipynb -------------------------------------------------------------------------------- /notebooks/using_bert_crf_nlu.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/notebooks/using_bert_crf_nlu.ipynb -------------------------------------------------------------------------------- /notebooks/using_bert_nlu.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/notebooks/using_bert_nlu.ipynb -------------------------------------------------------------------------------- /notebooks/using_compressed_nlu_model.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/notebooks/using_compressed_nlu_model.ipynb -------------------------------------------------------------------------------- /notebooks/using_trans_nlu.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/notebooks/using_trans_nlu.ipynb -------------------------------------------------------------------------------- /notebooks/using_trans_nlu_with_pruning.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/notebooks/using_trans_nlu_with_pruning.ipynb -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/scripts/README.md -------------------------------------------------------------------------------- /scripts/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/scripts/evaluate.py -------------------------------------------------------------------------------- /scripts/nlu_basic_api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/scripts/nlu_basic_api.py -------------------------------------------------------------------------------- /scripts/train_joint_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/scripts/train_joint_bert.py -------------------------------------------------------------------------------- /scripts/train_joint_trans.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/scripts/train_joint_trans.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/setup.py -------------------------------------------------------------------------------- /src/dialognlu/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/__init__.py -------------------------------------------------------------------------------- /src/dialognlu/auto_nlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/auto_nlu.py -------------------------------------------------------------------------------- /src/dialognlu/compression/__init__.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------------------------------- /src/dialognlu/compression/commons.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/compression/commons.py -------------------------------------------------------------------------------- /src/dialognlu/compression/layer_pruning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/compression/layer_pruning.py -------------------------------------------------------------------------------- /src/dialognlu/layers/__init__.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------------------------------- /src/dialognlu/layers/crf_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/layers/crf_layer.py -------------------------------------------------------------------------------- /src/dialognlu/models/__init__.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------------------------------- /src/dialognlu/models/base_joint_trans.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/base_joint_trans.py -------------------------------------------------------------------------------- /src/dialognlu/models/callbacks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/callbacks.py -------------------------------------------------------------------------------- /src/dialognlu/models/joint_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/joint_bert.py -------------------------------------------------------------------------------- /src/dialognlu/models/joint_bert_crf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/joint_bert_crf.py -------------------------------------------------------------------------------- /src/dialognlu/models/joint_trans_albert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/joint_trans_albert.py -------------------------------------------------------------------------------- /src/dialognlu/models/joint_trans_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/joint_trans_bert.py -------------------------------------------------------------------------------- /src/dialognlu/models/joint_trans_distilbert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/joint_trans_distilbert.py -------------------------------------------------------------------------------- /src/dialognlu/models/joint_trans_roberta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/joint_trans_roberta.py -------------------------------------------------------------------------------- /src/dialognlu/models/joint_trans_xlnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/joint_trans_xlnet.py -------------------------------------------------------------------------------- /src/dialognlu/models/model_pool.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/model_pool.py -------------------------------------------------------------------------------- /src/dialognlu/models/nlu_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/nlu_model.py -------------------------------------------------------------------------------- /src/dialognlu/models/trans_auto_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/models/trans_auto_model.py -------------------------------------------------------------------------------- /src/dialognlu/nlu_components.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/nlu_components.py -------------------------------------------------------------------------------- /src/dialognlu/readers/__init__.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------------------------------- /src/dialognlu/readers/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/readers/dataset.py -------------------------------------------------------------------------------- /src/dialognlu/readers/goo_format_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/readers/goo_format_reader.py -------------------------------------------------------------------------------- /src/dialognlu/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------------------------------- /src/dialognlu/utils/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/utils/data_utils.py -------------------------------------------------------------------------------- /src/dialognlu/utils/io_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/utils/io_utils.py -------------------------------------------------------------------------------- /src/dialognlu/utils/tf_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/utils/tf_utils.py -------------------------------------------------------------------------------- /src/dialognlu/vectorizers/__init__.py: -------------------------------------------------------------------------------- 1 | # -------------------------------------------------------------------------------- /src/dialognlu/vectorizers/albert_tokenization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/vectorizers/albert_tokenization.py -------------------------------------------------------------------------------- /src/dialognlu/vectorizers/bert_vectorizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/vectorizers/bert_vectorizer.py -------------------------------------------------------------------------------- /src/dialognlu/vectorizers/tags_vectorizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/vectorizers/tags_vectorizer.py -------------------------------------------------------------------------------- /src/dialognlu/vectorizers/tokenization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/vectorizers/tokenization.py -------------------------------------------------------------------------------- /src/dialognlu/vectorizers/trans_vectorizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MahmoudWahdan/dialog-nlu/HEAD/src/dialognlu/vectorizers/trans_vectorizer.py --------------------------------------------------------------------------------