├── .gitignore ├── README.md ├── mxnet ├── bert-large │ ├── gluon-nlp │ │ ├── .coveragerc │ │ ├── .github │ │ │ ├── ISSUE_TEMPLATE │ │ │ │ ├── bug_report.md │ │ │ │ └── feature_request.md │ │ │ └── PULL_REQUEST_TEMPLATE.md │ │ ├── .gitignore │ │ ├── .gitmodules │ │ ├── .pylintrc │ │ ├── CODEOWNERS │ │ ├── CODE_OF_CONDUCT.md │ │ ├── CONTRIBUTING.md │ │ ├── LICENSE │ │ ├── MANIFEST.in │ │ ├── Makefile │ │ ├── README.rst │ │ ├── ci │ │ │ ├── batch │ │ │ │ ├── docker │ │ │ │ │ ├── Dockerfile │ │ │ │ │ └── gluon_nlp_job.sh │ │ │ │ ├── submit-job.py │ │ │ │ └── wait-job.py │ │ │ ├── codecov.sh │ │ │ ├── jenkins │ │ │ │ ├── Jenkinsfile_py3-master_cpu_unittest │ │ │ │ ├── Jenkinsfile_py3-master_gpu_doc │ │ │ │ ├── Jenkinsfile_py3-master_gpu_integration │ │ │ │ ├── Jenkinsfile_py3-master_gpu_unittest │ │ │ │ ├── Jenkinsfile_py3_cpu_unittest │ │ │ │ ├── Jenkinsfile_py3_gpu_integration │ │ │ │ ├── Jenkinsfile_py3_gpu_unittest │ │ │ │ ├── build_steps.groovy │ │ │ │ └── utils.groovy │ │ │ ├── prepare_clean_env.sh │ │ │ ├── rat │ │ │ │ └── rat-excludes │ │ │ └── upload_doc.sh │ │ ├── codecov.yml │ │ ├── conftest.py │ │ ├── contributor.rst │ │ ├── docs │ │ │ ├── .gitignore │ │ │ ├── .nojekyll │ │ │ ├── 404.rst │ │ │ ├── Doxyfile │ │ │ ├── Makefile │ │ │ ├── README.txt │ │ │ ├── _static │ │ │ │ ├── 404.jpg │ │ │ │ ├── gluon-logo.svg │ │ │ │ ├── gluon.ico │ │ │ │ ├── gluon_black.png │ │ │ │ └── google_analytics.js │ │ │ ├── api │ │ │ │ ├── index.rst │ │ │ │ ├── modules │ │ │ │ │ ├── data.batchify.rst │ │ │ │ │ ├── data.rst │ │ │ │ │ ├── embedding.rst │ │ │ │ │ ├── index.rst │ │ │ │ │ ├── initializer.rst │ │ │ │ │ ├── loss.rst │ │ │ │ │ ├── model.rst │ │ │ │ │ ├── model.train.rst │ │ │ │ │ ├── optimizer.rst │ │ │ │ │ ├── utils.rst │ │ │ │ │ └── vocab.rst │ │ │ │ └── notes │ │ │ │ │ ├── data_api.rst │ │ │ │ │ ├── images │ │ │ │ │ ├── fixed_bucket_strategy_ratio0.0.png │ │ │ │ │ ├── fixed_bucket_strategy_ratio0.7.png │ │ │ │ │ ├── no_bucket_strategy.png │ │ │ │ │ └── sorted_bucket_strategy.png │ │ │ │ │ ├── index.rst │ │ │ │ │ └── vocab_emb.rst │ │ │ ├── community │ │ │ │ ├── contribute.rst │ │ │ │ ├── index.rst │ │ │ │ └── release.rst │ │ │ ├── conf.py │ │ │ ├── examples │ │ │ │ ├── index.rst │ │ │ │ ├── language_model │ │ │ │ │ ├── cache_model.png │ │ │ │ │ ├── language_model.md │ │ │ │ │ └── language_model_intro.png │ │ │ │ ├── machine_translation │ │ │ │ │ ├── dataprocessor.py │ │ │ │ │ ├── gnmt.md │ │ │ │ │ ├── hyperparameters.py │ │ │ │ │ ├── nmt │ │ │ │ │ ├── transformer.md │ │ │ │ │ ├── transformer.png │ │ │ │ │ └── utils.py │ │ │ │ ├── sentence_embedding │ │ │ │ │ ├── Bi-LSTM-Rep.png │ │ │ │ │ ├── attention-nlp.png │ │ │ │ │ ├── bert │ │ │ │ │ ├── bert-embed.png │ │ │ │ │ ├── bert-sentence-pair.png │ │ │ │ │ ├── bert.md │ │ │ │ │ ├── bert.png │ │ │ │ │ ├── dev.tsv │ │ │ │ │ ├── elmo_sentence_representation.md │ │ │ │ │ ├── self_attentive_sentence_embedding.md │ │ │ │ │ └── sentences.json │ │ │ │ ├── sentiment_analysis │ │ │ │ │ ├── samodel-v3.png │ │ │ │ │ └── sentiment_analysis.md │ │ │ │ ├── sequence_sampling │ │ │ │ │ └── sequence_sampling.md │ │ │ │ └── word_embedding │ │ │ │ │ ├── data.py │ │ │ │ │ ├── model.py │ │ │ │ │ ├── utils.py │ │ │ │ │ ├── word_embedding.md │ │ │ │ │ └── word_embedding_training.md │ │ │ ├── genindex.rst │ │ │ ├── index.rst │ │ │ ├── install.rst │ │ │ ├── md2ipynb.py │ │ │ ├── model_zoo │ │ │ └── model_zoo.rst │ │ ├── examples │ │ ├── pytest.ini │ │ ├── scripts │ │ │ ├── __init__.py │ │ │ ├── bert │ │ │ │ ├── __init__.py │ │ │ │ ├── bert_qa_evaluate.py │ │ │ │ ├── cmd.sh │ │ │ │ ├── conversion_tools │ │ │ │ │ ├── compare_gluon_ernie.py │ │ │ │ │ ├── compare_tf_gluon_model.py │ │ │ │ │ ├── convert_fairseq_model.py │ │ │ │ │ ├── convert_paddle_to_gluon.py │ │ │ │ │ ├── convert_pytorch_model.py │ │ │ │ │ ├── convert_tf_model.py │ │ │ │ │ ├── ernie_top_layer_emb.npy │ │ │ │ │ ├── infer_pytorch_gluon_parameter_name_mapping.py │ │ │ │ │ ├── input.txt │ │ │ │ │ ├── input_cn.txt │ │ │ │ │ └── utils.py │ │ │ │ ├── create_pretraining_data.py │ │ │ │ ├── data │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── baidu_ernie_data.py │ │ │ │ │ ├── classification.py │ │ │ │ │ ├── dataloader.py │ │ │ │ │ ├── embedding.py │ │ │ │ │ ├── ner.py │ │ │ │ │ ├── pretrain.py │ │ │ │ │ ├── qa.py │ │ │ │ │ └── transform.py │ │ │ │ ├── embedding.py │ │ │ │ ├── export.py │ │ │ │ ├── finetune_classifier.py │ │ │ │ ├── finetune_icsl.py │ │ │ │ ├── finetune_ner.py │ │ │ │ ├── finetune_squad.py │ │ │ │ ├── fp16_utils.py │ │ │ │ ├── hvd.sh │ │ │ │ ├── index.rst │ │ │ │ ├── kvstore.sh │ │ │ │ ├── model │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── ner.py │ │ │ │ │ └── qa.py │ │ │ │ ├── mul-hvd.sh │ │ │ │ ├── ner_utils.py │ │ │ │ ├── ompi_bind_DGX1.sh │ │ │ │ ├── predict_ner.py │ │ │ │ ├── pretraining_utils.py │ │ │ │ ├── run_bps.sh │ │ │ │ ├── run_pretraining.py │ │ │ │ ├── sample_text.txt │ │ │ │ └── utils.py │ │ │ ├── index.rst │ │ │ ├── language_model │ │ │ │ ├── __init__.py │ │ │ │ ├── cache_language_model.py │ │ │ │ ├── conversion_utils │ │ │ │ │ ├── compare_pytorch_gluon_model.py │ │ │ │ │ ├── convert_transformer_xl.py │ │ │ │ │ └── utils.py │ │ │ │ ├── index.rst │ │ │ │ ├── large_word_language_model.py │ │ │ │ ├── sampler.py │ │ │ │ ├── transformer │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── attention_cell.py │ │ │ │ │ ├── embedding.py │ │ │ │ │ ├── model.py │ │ │ │ │ ├── softmax.py │ │ │ │ │ └── transformer.py │ │ │ │ ├── transformer_xl.py │ │ │ │ └── word_language_model.py │ │ │ ├── machine_translation │ │ │ │ ├── __init__.py │ │ │ │ ├── _constants.py │ │ │ │ ├── bleu.py │ │ │ │ ├── dataprocessor.py │ │ │ │ ├── dataset.py │ │ │ │ ├── gnmt.py │ │ │ │ ├── hyperparameters.py │ │ │ │ ├── index.rst │ │ │ │ ├── inference_transformer.py │ │ │ │ ├── train_gnmt.py │ │ │ │ ├── train_transformer.py │ │ │ │ ├── translation.py │ │ │ │ └── utils.py │ │ │ ├── natural_language_inference │ │ │ │ ├── dataset.py │ │ │ │ ├── decomposable_attention.py │ │ │ │ ├── esim.py │ │ │ │ ├── index.rst │ │ │ │ ├── main.py │ │ │ │ ├── preprocess.py │ │ │ │ └── utils.py │ │ │ ├── parsing │ │ │ │ ├── __init__.py │ │ │ │ ├── common │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── config.py │ │ │ │ │ ├── data.py │ │ │ │ │ ├── exponential_scheduler.py │ │ │ │ │ ├── k_means.py │ │ │ │ │ ├── savable.py │ │ │ │ │ ├── tarjan.py │ │ │ │ │ └── utils.py │ │ │ │ ├── index.rst │ │ │ │ └── parser │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── biaffine_parser.py │ │ │ │ │ ├── dep_parser.py │ │ │ │ │ └── evaluate │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── evaluate.py │ │ │ ├── question_answering │ │ │ │ ├── __init__.py │ │ │ │ ├── data_pipeline.py │ │ │ │ └── utils.py │ │ │ ├── sentiment_analysis │ │ │ │ ├── __init__.py │ │ │ │ ├── finetune_lm.py │ │ │ │ ├── index.rst │ │ │ │ ├── process_data.py │ │ │ │ ├── sentiment_analysis_cnn.py │ │ │ │ └── text_cnn.py │ │ │ ├── tests │ │ │ │ ├── __init__.py │ │ │ │ ├── conftest.py │ │ │ │ ├── multi-bleu-detok.perl │ │ │ │ ├── multi-bleu.perl │ │ │ │ ├── test_bert_dataset_transform.py │ │ │ │ ├── test_bert_embedding.py │ │ │ │ ├── test_bleu.py │ │ │ │ ├── test_dataprocessor.py │ │ │ │ ├── test_encoder_decoder.py │ │ │ │ ├── test_models.py │ │ │ │ ├── test_question_answering.py │ │ │ │ ├── test_references.txt │ │ │ │ ├── test_sampler.py │ │ │ │ ├── test_sanity.py │ │ │ │ ├── test_scripts.py │ │ │ │ ├── test_transformer_xl.py │ │ │ │ ├── test_translations.txt │ │ │ │ └── word_embeddings │ │ │ │ │ └── glove │ │ │ │ │ ├── cooccurrences.npz │ │ │ │ │ └── vocab.txt │ │ │ ├── text_classification │ │ │ │ ├── fasttext_word_ngram.py │ │ │ │ └── index.rst │ │ │ ├── text_generation │ │ │ │ ├── __init__.py │ │ │ │ ├── index.rst │ │ │ │ ├── model │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── gpt.py │ │ │ │ └── sequence_sampling.py │ │ │ └── word_embeddings │ │ │ │ ├── data.py │ │ │ │ ├── evaluate_pretrained.py │ │ │ │ ├── evaluation.py │ │ │ │ ├── executors.py │ │ │ │ ├── extract_vocab.py │ │ │ │ ├── index.rst │ │ │ │ ├── model.py │ │ │ │ ├── run_all.sh │ │ │ │ ├── tools │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── cooccur.cc │ │ │ │ ├── utils.h │ │ │ │ └── vocab_count.cc │ │ │ │ ├── train_glove.py │ │ │ │ ├── train_sg_cbow.py │ │ │ │ └── utils.py │ │ ├── setup.py │ │ ├── src │ │ │ └── gluonnlp │ │ │ │ ├── __init__.py │ │ │ │ ├── _constants.py │ │ │ │ ├── base.py │ │ │ │ ├── data │ │ │ │ ├── __init__.py │ │ │ │ ├── batchify │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── batchify.py │ │ │ │ │ ├── embedding.py │ │ │ │ │ └── language_model.py │ │ │ │ ├── candidate_sampler.py │ │ │ │ ├── conll.py │ │ │ │ ├── corpora │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── google_billion_word.py │ │ │ │ │ ├── large_text_compression_benchmark.py │ │ │ │ │ └── wikitext.py │ │ │ │ ├── dataloader.py │ │ │ │ ├── dataset.py │ │ │ │ ├── glue.py │ │ │ │ ├── intent_slot.py │ │ │ │ ├── question_answering.py │ │ │ │ ├── registry.py │ │ │ │ ├── sampler.py │ │ │ │ ├── sentiment.py │ │ │ │ ├── stream.py │ │ │ │ ├── transforms.py │ │ │ │ ├── translation.py │ │ │ │ ├── utils.py │ │ │ │ └── word_embedding_evaluation.py │ │ │ │ ├── embedding │ │ │ │ ├── __init__.py │ │ │ │ ├── evaluation.py │ │ │ │ └── token_embedding.py │ │ │ │ ├── initializer │ │ │ │ ├── __init__.py │ │ │ │ └── initializer.py │ │ │ │ ├── loss │ │ │ │ ├── __init__.py │ │ │ │ ├── activation_regularizer.py │ │ │ │ ├── label_smoothing.py │ │ │ │ └── loss.py │ │ │ │ ├── metric │ │ │ │ ├── __init__.py │ │ │ │ └── masked_accuracy.py │ │ │ │ ├── model │ │ │ │ ├── __init__.py │ │ │ │ ├── attention_cell.py │ │ │ │ ├── bert.py │ │ │ │ ├── bilm_encoder.py │ │ │ │ ├── block.py │ │ │ │ ├── convolutional_encoder.py │ │ │ │ ├── elmo.py │ │ │ │ ├── highway.py │ │ │ │ ├── language_model.py │ │ │ │ ├── lstmpcellwithclip.py │ │ │ │ ├── parameter.py │ │ │ │ ├── sampled_block.py │ │ │ │ ├── seq2seq_encoder_decoder.py │ │ │ │ ├── sequence_sampler.py │ │ │ │ ├── train │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── cache.py │ │ │ │ │ ├── embedding.py │ │ │ │ │ └── language_model.py │ │ │ │ ├── transformer.py │ │ │ │ ├── translation.py │ │ │ │ └── utils.py │ │ │ │ ├── optimizer │ │ │ │ ├── __init__.py │ │ │ │ ├── bert_adam.py │ │ │ │ └── lamb.py │ │ │ │ ├── utils │ │ │ │ ├── __init__.py │ │ │ │ ├── files.py │ │ │ │ ├── parallel.py │ │ │ │ ├── parameter.py │ │ │ │ └── version.py │ │ │ │ └── vocab │ │ │ │ ├── __init__.py │ │ │ │ ├── bert.py │ │ │ │ ├── elmo.py │ │ │ │ ├── subwords.py │ │ │ │ └── vocab.py │ │ ├── tests │ │ │ └── unittest │ │ │ │ ├── batchify │ │ │ │ ├── test_batchify.py │ │ │ │ ├── test_batchify_embedding.py │ │ │ │ └── test_batchify_language_model.py │ │ │ │ ├── conftest.py │ │ │ │ ├── corpora │ │ │ │ ├── test_gbw.py │ │ │ │ ├── test_large_text_compression_benchmark.py │ │ │ │ └── test_wikitext.py │ │ │ │ ├── test_attention_cell.py │ │ │ │ ├── test_bertvocab.py │ │ │ │ ├── test_bilm_encoder.py │ │ │ │ ├── test_candidate_sampler.py │ │ │ │ ├── test_convolutional_encoder.py │ │ │ │ ├── test_datasets.py │ │ │ │ ├── test_elmo.py │ │ │ │ ├── test_highway.py │ │ │ │ ├── test_initializer.py │ │ │ │ ├── test_lamb.py │ │ │ │ ├── test_loss.py │ │ │ │ ├── test_lstmpcellwithclip.py │ │ │ │ ├── test_metrics.py │ │ │ │ ├── test_models.py │ │ │ │ ├── test_optimizer.py │ │ │ │ ├── test_pytest.py │ │ │ │ ├── test_sampled_logits.py │ │ │ │ ├── test_sampler.py │ │ │ │ ├── test_sanity.py │ │ │ │ ├── test_sequence_sampler.py │ │ │ │ ├── test_stream.py │ │ │ │ ├── test_token_embedding.py │ │ │ │ ├── test_transforms.py │ │ │ │ ├── test_utils.py │ │ │ │ ├── test_vocab_embed.py │ │ │ │ └── train │ │ │ │ ├── test_dataloader.py │ │ │ │ ├── test_embedding.py │ │ │ │ └── test_embedding │ │ │ │ ├── lorem_ipsum.bin │ │ │ │ ├── lorem_ipsum.vec │ │ │ │ ├── lorem_ipsum_w2v.bin │ │ │ │ └── lorem_ipsum_w2v.vec │ │ └── tools │ │ │ ├── diagnose.py │ │ │ └── plot_bucketing_strategies.py │ └── run.sh └── vgg-16 │ ├── common │ ├── __init__.py │ ├── data.py │ ├── data_byteps.py │ ├── find_mxnet.py │ ├── fit.py │ ├── fit_byteps.py │ ├── modelzoo.py │ └── util.py │ ├── data │ ├── caltech256.sh │ └── imagenet1k-val.sh │ ├── run.sh │ ├── symbols │ ├── README.md │ ├── __init__.py │ ├── alexnet.py │ ├── googlenet.py │ ├── inception-bn.py │ ├── inception-resnet-v2.py │ ├── inception-v3.py │ ├── inception-v4.py │ ├── lenet.py │ ├── mlp.py │ ├── mobilenet.py │ ├── mobilenetv2.py │ ├── resnet-v1.py │ ├── resnet.py │ ├── resnetv1.py │ ├── resnext.py │ └── vgg.py │ └── train_imagenet.py ├── osdi20ae.pdf ├── pytorch ├── gpt-2 │ ├── README.md │ ├── gpt-2 │ │ ├── .circleci │ │ │ ├── config.yml │ │ │ └── deploy.sh │ │ ├── .coveragerc │ │ ├── .github │ │ │ ├── ISSUE_TEMPLATE │ │ │ │ ├── ---new-benchmark.md │ │ │ │ ├── --new-model-addition.md │ │ │ │ ├── bug-report.md │ │ │ │ ├── feature-request.md │ │ │ │ ├── migration.md │ │ │ │ └── question-help.md │ │ │ └── stale.yml │ │ ├── .gitignore │ │ ├── CONTRIBUTING.md │ │ ├── LICENSE │ │ ├── MANIFEST.in │ │ ├── Makefile │ │ ├── README.md │ │ ├── deploy_multi_version_doc.sh │ │ ├── docker │ │ │ └── Dockerfile │ │ ├── docs │ │ │ ├── Makefile │ │ │ ├── README.md │ │ │ └── source │ │ │ │ ├── _static │ │ │ │ ├── css │ │ │ │ │ ├── Calibre-Light.ttf │ │ │ │ │ ├── Calibre-Medium.otf │ │ │ │ │ ├── Calibre-Regular.otf │ │ │ │ │ ├── Calibre-Thin.otf │ │ │ │ │ ├── code-snippets.css │ │ │ │ │ └── huggingface.css │ │ │ │ └── js │ │ │ │ │ ├── custom.js │ │ │ │ │ └── huggingface_logo.svg │ │ │ │ ├── benchmarks.md │ │ │ │ ├── bertology.rst │ │ │ │ ├── conf.py │ │ │ │ ├── converting_tensorflow_models.rst │ │ │ │ ├── examples.md │ │ │ │ ├── glossary.rst │ │ │ │ ├── imgs │ │ │ │ ├── transformers_logo_name.png │ │ │ │ ├── warmup_constant_schedule.png │ │ │ │ ├── warmup_cosine_hard_restarts_schedule.png │ │ │ │ ├── warmup_cosine_schedule.png │ │ │ │ ├── warmup_cosine_warm_restarts_schedule.png │ │ │ │ └── warmup_linear_schedule.png │ │ │ │ ├── index.rst │ │ │ │ ├── installation.md │ │ │ │ ├── main_classes │ │ │ │ ├── configuration.rst │ │ │ │ ├── model.rst │ │ │ │ ├── optimizer_schedules.rst │ │ │ │ ├── processors.rst │ │ │ │ └── tokenizer.rst │ │ │ │ ├── migration.md │ │ │ │ ├── model_doc │ │ │ │ ├── albert.rst │ │ │ │ ├── auto.rst │ │ │ │ ├── bert.rst │ │ │ │ ├── camembert.rst │ │ │ │ ├── ctrl.rst │ │ │ │ ├── distilbert.rst │ │ │ │ ├── flaubert.rst │ │ │ │ ├── gpt.rst │ │ │ │ ├── gpt2.rst │ │ │ │ ├── roberta.rst │ │ │ │ ├── transformerxl.rst │ │ │ │ ├── xlm.rst │ │ │ │ ├── xlmroberta.rst │ │ │ │ └── xlnet.rst │ │ │ │ ├── model_sharing.md │ │ │ │ ├── multilingual.rst │ │ │ │ ├── notebooks.rst │ │ │ │ ├── pretrained_models.rst │ │ │ │ ├── quickstart.md │ │ │ │ ├── serialization.rst │ │ │ │ └── torchscript.rst │ │ ├── examples │ │ │ ├── README.md │ │ │ ├── benchmarks.py │ │ │ ├── contrib │ │ │ │ ├── README.md │ │ │ │ ├── run_camembert.py │ │ │ │ ├── run_openai_gpt.py │ │ │ │ ├── run_swag.py │ │ │ │ └── run_transfo_xl.py │ │ │ ├── distillation │ │ │ │ ├── README.md │ │ │ │ ├── distiller.py │ │ │ │ ├── grouped_batch_sampler.py │ │ │ │ ├── lm_seqs_dataset.py │ │ │ │ ├── requirements.txt │ │ │ │ ├── run_squad_w_distillation.py │ │ │ │ ├── scripts │ │ │ │ │ ├── binarized_data.py │ │ │ │ │ ├── extract.py │ │ │ │ │ ├── extract_distilbert.py │ │ │ │ │ └── token_counts.py │ │ │ │ ├── train.py │ │ │ │ ├── training_configs │ │ │ │ │ ├── distilbert-base-multilingual-cased.json │ │ │ │ │ ├── distilbert-base-uncased.json │ │ │ │ │ ├── distilgpt2.json │ │ │ │ │ └── distilroberta-base.json │ │ │ │ └── utils.py │ │ │ ├── hans │ │ │ │ ├── hans_processors.py │ │ │ │ ├── test_hans.py │ │ │ │ └── utils_hans.py │ │ │ ├── mm-imdb │ │ │ │ ├── run_mmimdb.py │ │ │ │ └── utils_mmimdb.py │ │ │ ├── pplm │ │ │ │ ├── README.md │ │ │ │ ├── imgs │ │ │ │ │ ├── headfigure.png │ │ │ │ │ └── wooly.png │ │ │ │ ├── pplm_classification_head.py │ │ │ │ ├── run_pplm.py │ │ │ │ └── run_pplm_discrim_train.py │ │ │ ├── requirements.txt │ │ │ ├── run_bertology.py │ │ │ ├── run_generation.py │ │ │ ├── run_glue.py │ │ │ ├── run_lm_finetuning.py │ │ │ ├── run_lm_finetuning_bps.py │ │ │ ├── run_multiple_choice.py │ │ │ ├── run_ner.py │ │ │ ├── run_squad.py │ │ │ ├── run_tf_glue.py │ │ │ ├── run_tf_ner.py │ │ │ ├── run_xnli.py │ │ │ ├── summarization │ │ │ │ ├── README.md │ │ │ │ ├── configuration_bertabs.py │ │ │ │ ├── convert_bertabs_original_pytorch_checkpoint.py │ │ │ │ ├── modeling_bertabs.py │ │ │ │ ├── requirements.txt │ │ │ │ ├── run_summarization.py │ │ │ │ ├── test_utils_summarization.py │ │ │ │ └── utils_summarization.py │ │ │ ├── test_examples.py │ │ │ ├── tests_samples │ │ │ │ ├── .gitignore │ │ │ │ ├── MRPC │ │ │ │ │ ├── dev.tsv │ │ │ │ │ └── train.tsv │ │ │ │ └── SQUAD │ │ │ │ │ ├── dev-v2.0.json │ │ │ │ │ └── train-v2.0.json │ │ │ ├── utils_multiple_choice.py │ │ │ └── utils_ner.py │ │ ├── hubconf.py │ │ ├── notebooks │ │ │ ├── Comparing-PT-and-TF-models.ipynb │ │ │ ├── Comparing-TF-and-PT-models-MLM-NSP.ipynb │ │ │ ├── Comparing-TF-and-PT-models-SQuAD.ipynb │ │ │ └── Comparing-TF-and-PT-models.ipynb │ │ ├── setup.cfg │ │ ├── setup.py │ │ ├── src │ │ │ └── transformers │ │ │ │ ├── __init__.py │ │ │ │ ├── commands │ │ │ │ ├── __init__.py │ │ │ │ ├── convert.py │ │ │ │ ├── download.py │ │ │ │ ├── run.py │ │ │ │ ├── serving.py │ │ │ │ ├── train.py │ │ │ │ └── user.py │ │ │ │ ├── configuration_albert.py │ │ │ │ ├── configuration_auto.py │ │ │ │ ├── configuration_bert.py │ │ │ │ ├── configuration_camembert.py │ │ │ │ ├── configuration_ctrl.py │ │ │ │ ├── configuration_distilbert.py │ │ │ │ ├── configuration_flaubert.py │ │ │ │ ├── configuration_gpt2.py │ │ │ │ ├── configuration_mmbt.py │ │ │ │ ├── configuration_openai.py │ │ │ │ ├── configuration_roberta.py │ │ │ │ ├── configuration_t5.py │ │ │ │ ├── configuration_transfo_xl.py │ │ │ │ ├── configuration_utils.py │ │ │ │ ├── configuration_xlm.py │ │ │ │ ├── configuration_xlm_roberta.py │ │ │ │ ├── configuration_xlnet.py │ │ │ │ ├── convert_albert_original_tf_checkpoint_to_pytorch.py │ │ │ │ ├── convert_bert_original_tf_checkpoint_to_pytorch.py │ │ │ │ ├── convert_bert_pytorch_checkpoint_to_original_tf.py │ │ │ │ ├── convert_gpt2_original_tf_checkpoint_to_pytorch.py │ │ │ │ ├── convert_openai_original_tf_checkpoint_to_pytorch.py │ │ │ │ ├── convert_pytorch_checkpoint_to_tf2.py │ │ │ │ ├── convert_roberta_original_pytorch_checkpoint_to_pytorch.py │ │ │ │ ├── convert_t5_original_tf_checkpoint_to_pytorch.py │ │ │ │ ├── convert_transfo_xl_original_tf_checkpoint_to_pytorch.py │ │ │ │ ├── convert_xlm_original_pytorch_checkpoint_to_pytorch.py │ │ │ │ ├── convert_xlnet_original_tf_checkpoint_to_pytorch.py │ │ │ │ ├── data │ │ │ │ ├── __init__.py │ │ │ │ ├── metrics │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── squad_metrics.py │ │ │ │ └── processors │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── glue.py │ │ │ │ │ ├── squad.py │ │ │ │ │ ├── utils.py │ │ │ │ │ └── xnli.py │ │ │ │ ├── file_utils.py │ │ │ │ ├── hf_api.py │ │ │ │ ├── modelcard.py │ │ │ │ ├── modeling_albert.py │ │ │ │ ├── modeling_auto.py │ │ │ │ ├── modeling_bert.py │ │ │ │ ├── modeling_camembert.py │ │ │ │ ├── modeling_ctrl.py │ │ │ │ ├── modeling_distilbert.py │ │ │ │ ├── modeling_encoder_decoder.py │ │ │ │ ├── modeling_flaubert.py │ │ │ │ ├── modeling_gpt2.py │ │ │ │ ├── modeling_mmbt.py │ │ │ │ ├── modeling_openai.py │ │ │ │ ├── modeling_roberta.py │ │ │ │ ├── modeling_t5.py │ │ │ │ ├── modeling_tf_albert.py │ │ │ │ ├── modeling_tf_auto.py │ │ │ │ ├── modeling_tf_bert.py │ │ │ │ ├── modeling_tf_camembert.py │ │ │ │ ├── modeling_tf_ctrl.py │ │ │ │ ├── modeling_tf_distilbert.py │ │ │ │ ├── modeling_tf_gpt2.py │ │ │ │ ├── modeling_tf_openai.py │ │ │ │ ├── modeling_tf_pytorch_utils.py │ │ │ │ ├── modeling_tf_roberta.py │ │ │ │ ├── modeling_tf_t5.py │ │ │ │ ├── modeling_tf_transfo_xl.py │ │ │ │ ├── modeling_tf_transfo_xl_utilities.py │ │ │ │ ├── modeling_tf_utils.py │ │ │ │ ├── modeling_tf_xlm.py │ │ │ │ ├── modeling_tf_xlm_roberta.py │ │ │ │ ├── modeling_tf_xlnet.py │ │ │ │ ├── modeling_transfo_xl.py │ │ │ │ ├── modeling_transfo_xl_utilities.py │ │ │ │ ├── modeling_utils.py │ │ │ │ ├── modeling_xlm.py │ │ │ │ ├── modeling_xlm_roberta.py │ │ │ │ ├── modeling_xlnet.py │ │ │ │ ├── optimization.py │ │ │ │ ├── optimization_tf.py │ │ │ │ ├── pipelines.py │ │ │ │ ├── tokenization_albert.py │ │ │ │ ├── tokenization_auto.py │ │ │ │ ├── tokenization_bert.py │ │ │ │ ├── tokenization_bert_japanese.py │ │ │ │ ├── tokenization_camembert.py │ │ │ │ ├── tokenization_ctrl.py │ │ │ │ ├── tokenization_distilbert.py │ │ │ │ ├── tokenization_flaubert.py │ │ │ │ ├── tokenization_gpt2.py │ │ │ │ ├── tokenization_openai.py │ │ │ │ ├── tokenization_roberta.py │ │ │ │ ├── tokenization_t5.py │ │ │ │ ├── tokenization_transfo_xl.py │ │ │ │ ├── tokenization_utils.py │ │ │ │ ├── tokenization_xlm.py │ │ │ │ ├── tokenization_xlm_roberta.py │ │ │ │ └── tokenization_xlnet.py │ │ ├── templates │ │ │ ├── adding_a_new_example_script │ │ │ │ ├── README.md │ │ │ │ ├── run_xxx.py │ │ │ │ └── utils_xxx.py │ │ │ └── adding_a_new_model │ │ │ │ ├── README.md │ │ │ │ ├── configuration_xxx.py │ │ │ │ ├── convert_xxx_original_tf_checkpoint_to_pytorch.py │ │ │ │ ├── modeling_tf_xxx.py │ │ │ │ ├── modeling_xxx.py │ │ │ │ ├── tests │ │ │ │ ├── test_modeling_tf_xxx.py │ │ │ │ ├── test_modeling_xxx.py │ │ │ │ └── test_tokenization_xxx.py │ │ │ │ └── tokenization_xxx.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── fixtures │ │ │ │ ├── dummy-config.json │ │ │ │ ├── empty.txt │ │ │ │ ├── input.txt │ │ │ │ ├── sample_text.txt │ │ │ │ └── test_sentencepiece.model │ │ │ ├── test_configuration_auto.py │ │ │ ├── test_configuration_common.py │ │ │ ├── test_doc_samples.py │ │ │ ├── test_hf_api.py │ │ │ ├── test_model_card.py │ │ │ ├── test_modeling_albert.py │ │ │ ├── test_modeling_auto.py │ │ │ ├── test_modeling_bert.py │ │ │ ├── test_modeling_common.py │ │ │ ├── test_modeling_ctrl.py │ │ │ ├── test_modeling_distilbert.py │ │ │ ├── test_modeling_encoder_decoder.py │ │ │ ├── test_modeling_gpt2.py │ │ │ ├── test_modeling_openai.py │ │ │ ├── test_modeling_roberta.py │ │ │ ├── test_modeling_t5.py │ │ │ ├── test_modeling_tf_albert.py │ │ │ ├── test_modeling_tf_auto.py │ │ │ ├── test_modeling_tf_bert.py │ │ │ ├── test_modeling_tf_common.py │ │ │ ├── test_modeling_tf_ctrl.py │ │ │ ├── test_modeling_tf_distilbert.py │ │ │ ├── test_modeling_tf_gpt2.py │ │ │ ├── test_modeling_tf_openai_gpt.py │ │ │ ├── test_modeling_tf_roberta.py │ │ │ ├── test_modeling_tf_t5.py │ │ │ ├── test_modeling_tf_transfo_xl.py │ │ │ ├── test_modeling_tf_xlm.py │ │ │ ├── test_modeling_tf_xlnet.py │ │ │ ├── test_modeling_transfo_xl.py │ │ │ ├── test_modeling_xlm.py │ │ │ ├── test_modeling_xlnet.py │ │ │ ├── test_optimization.py │ │ │ ├── test_optimization_tf.py │ │ │ ├── test_pipelines.py │ │ │ ├── test_tokenization_albert.py │ │ │ ├── test_tokenization_auto.py │ │ │ ├── test_tokenization_bert.py │ │ │ ├── test_tokenization_bert_japanese.py │ │ │ ├── test_tokenization_common.py │ │ │ ├── test_tokenization_ctrl.py │ │ │ ├── test_tokenization_distilbert.py │ │ │ ├── test_tokenization_gpt2.py │ │ │ ├── test_tokenization_openai.py │ │ │ ├── test_tokenization_roberta.py │ │ │ ├── test_tokenization_t5.py │ │ │ ├── test_tokenization_transfo_xl.py │ │ │ ├── test_tokenization_utils.py │ │ │ ├── test_tokenization_xlm.py │ │ │ ├── test_tokenization_xlnet.py │ │ │ └── utils.py │ │ ├── transformers-cli │ │ ├── utils │ │ │ ├── download_glue_data.py │ │ │ └── link_tester.py │ │ └── valohai.yaml │ └── run.sh └── ugatit │ ├── README.md │ ├── run.sh │ └── ugatit │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── UGATIT.py │ ├── assets │ ├── ablation.png │ ├── discriminator.png │ ├── generator.png │ ├── kid.png │ ├── teaser.png │ └── user_study.png │ ├── dataset.py │ ├── main.py │ ├── networks.py │ ├── requirements.txt │ └── utils.py └── tensorflow ├── resnet-50 ├── benchmark.py └── run.sh └── transformer ├── README.md ├── official ├── .gitignore ├── Dockerfile.cpu ├── Dockerfile.gpu ├── LICENSE ├── README-TPU.md ├── README.md ├── __init__.py ├── benchmark │ ├── __init__.py │ ├── benchmark_uploader.py │ ├── benchmark_uploader_main.py │ ├── benchmark_uploader_test.py │ ├── bert_benchmark.py │ ├── bert_benchmark_utils.py │ ├── bert_squad_benchmark.py │ ├── datastore │ │ └── schema │ │ │ ├── benchmark_metric.json │ │ │ ├── benchmark_run.json │ │ │ └── benchmark_run_status.json │ ├── keras_benchmark.py │ ├── keras_cifar_benchmark.py │ ├── keras_imagenet_benchmark.py │ ├── models │ │ ├── __init__.py │ │ └── trivial_model.py │ ├── ncf_keras_benchmark.py │ ├── resnet_ctl_imagenet_benchmark.py │ ├── retinanet_benchmark.py │ ├── squad_evaluate_v1_1.py │ ├── transformer_benchmark.py │ └── xlnet_benchmark.py ├── modeling │ ├── __init__.py │ ├── activations │ │ ├── __init__.py │ │ ├── gelu.py │ │ ├── gelu_test.py │ │ ├── swish.py │ │ └── swish_test.py │ ├── hyperparams │ │ ├── __init__.py │ │ ├── params_dict.py │ │ └── params_dict_test.py │ ├── model_training_utils.py │ ├── model_training_utils_test.py │ ├── tf_utils.py │ └── training │ │ ├── __init__.py │ │ └── distributed_executor.py ├── nlp │ ├── README.md │ ├── __init__.py │ ├── bert │ │ ├── README.md │ │ ├── __init__.py │ │ ├── bert_cloud_tpu.md │ │ ├── classifier_data_lib.py │ │ ├── common_flags.py │ │ ├── create_finetuning_data.py │ │ ├── create_pretraining_data.py │ │ ├── export_tfhub.py │ │ ├── export_tfhub_test.py │ │ ├── input_pipeline.py │ │ ├── model_saving_utils.py │ │ ├── run_classifier.py │ │ ├── run_pretraining.py │ │ ├── run_squad.py │ │ ├── squad_lib.py │ │ ├── squad_lib_sp.py │ │ ├── tf1_checkpoint_converter_lib.py │ │ ├── tf2_albert_encoder_checkpoint_converter.py │ │ ├── tf2_encoder_checkpoint_converter.py │ │ ├── tokenization.py │ │ └── tokenization_test.py │ ├── bert_modeling.py │ ├── bert_models.py │ ├── modeling │ │ ├── __init__.py │ │ ├── layers │ │ │ ├── __init__.py │ │ │ ├── attention.py │ │ │ ├── attention_test.py │ │ │ ├── dense_einsum.py │ │ │ ├── dense_einsum_test.py │ │ │ ├── masked_softmax.py │ │ │ ├── masked_softmax_test.py │ │ │ ├── on_device_embedding.py │ │ │ ├── on_device_embedding_test.py │ │ │ ├── position_embedding.py │ │ │ ├── position_embedding_test.py │ │ │ ├── self_attention_mask.py │ │ │ ├── transformer.py │ │ │ ├── transformer_scaffold.py │ │ │ ├── transformer_scaffold_test.py │ │ │ └── transformer_test.py │ │ ├── losses │ │ │ ├── __init__.py │ │ │ ├── weighted_sparse_categorical_crossentropy.py │ │ │ └── weighted_sparse_categorical_crossentropy_test.py │ │ └── networks │ │ │ ├── __init__.py │ │ │ ├── albert_transformer_encoder.py │ │ │ ├── albert_transformer_encoder_test.py │ │ │ ├── bert_classifier.py │ │ │ ├── bert_classifier_test.py │ │ │ ├── bert_pretrainer.py │ │ │ ├── bert_pretrainer_test.py │ │ │ ├── bert_span_labeler.py │ │ │ ├── bert_span_labeler_test.py │ │ │ ├── classification.py │ │ │ ├── classification_test.py │ │ │ ├── encoder_scaffold.py │ │ │ ├── encoder_scaffold_test.py │ │ │ ├── masked_lm.py │ │ │ ├── masked_lm_test.py │ │ │ ├── span_labeling.py │ │ │ ├── span_labeling_test.py │ │ │ ├── transformer_encoder.py │ │ │ └── transformer_encoder_test.py │ ├── optimization.py │ ├── transformer │ │ ├── __init__.py │ │ ├── beam_search_v1.py │ │ ├── beam_search_v1_test.py │ │ ├── model_params.py │ │ ├── model_utils.py │ │ └── model_utils_test.py │ └── xlnet │ │ ├── README.md │ │ ├── __init__.py │ │ ├── classifier_utils.py │ │ ├── common_flags.py │ │ ├── data_utils.py │ │ ├── optimization.py │ │ ├── preprocess_classification_data.py │ │ ├── preprocess_pretrain_data.py │ │ ├── preprocess_squad_data.py │ │ ├── preprocess_utils.py │ │ ├── run_classifier.py │ │ ├── run_pretrain.py │ │ ├── run_squad.py │ │ ├── squad_utils.py │ │ ├── training_utils.py │ │ ├── xlnet_config.py │ │ ├── xlnet_modeling.py │ │ └── xlnet_modeling_test.py ├── pip_package │ └── setup.py ├── r1 │ ├── __init__.py │ ├── boosted_trees │ │ ├── README.md │ │ ├── __init__.py │ │ ├── data_download.py │ │ ├── train_higgs.py │ │ ├── train_higgs_test.csv │ │ └── train_higgs_test.py │ ├── mnist │ │ ├── README.md │ │ ├── __init__.py │ │ ├── dataset.py │ │ ├── example3.png │ │ ├── example5.png │ │ ├── examples.npy │ │ ├── mnist.py │ │ ├── mnist_eager.py │ │ ├── mnist_eager_test.py │ │ ├── mnist_test.py │ │ └── mnist_tpu.py │ ├── resnet │ │ ├── README.md │ │ ├── __init__.py │ │ ├── cifar10_download_and_extract.py │ │ ├── cifar10_main.py │ │ ├── cifar10_test.py │ │ ├── estimator_benchmark.py │ │ ├── imagenet_main.py │ │ ├── imagenet_preprocessing.py │ │ ├── imagenet_test.py │ │ ├── layer_test.py │ │ ├── resnet_model.py │ │ └── resnet_run_loop.py │ ├── transformer │ │ ├── README.md │ │ ├── __init__.py │ │ ├── attention_layer.py │ │ ├── dataset.py │ │ ├── embedding_layer.py │ │ ├── ffn_layer.py │ │ ├── schedule.py │ │ ├── schedule_test.py │ │ ├── transformer.py │ │ ├── transformer_main.py │ │ └── translate.py │ ├── utils │ │ ├── __init__.py │ │ ├── data │ │ │ ├── __init__.py │ │ │ ├── file_io.py │ │ │ └── file_io_test.py │ │ ├── export.py │ │ ├── export_test.py │ │ ├── tpu.py │ │ └── tpu_test.py │ └── wide_deep │ │ ├── README.md │ │ ├── __init__.py │ │ ├── census_dataset.py │ │ ├── census_main.py │ │ ├── census_test.csv │ │ ├── census_test.py │ │ ├── movielens_dataset.py │ │ ├── movielens_main.py │ │ ├── movielens_test.py │ │ └── wide_deep_run_loop.py ├── recommendation │ ├── README.md │ ├── __init__.py │ ├── constants.py │ ├── create_ncf_data.py │ ├── data_pipeline.py │ ├── data_preprocessing.py │ ├── data_test.py │ ├── movielens.py │ ├── ncf_common.py │ ├── ncf_estimator_main.py │ ├── ncf_input_pipeline.py │ ├── ncf_keras_main.py │ ├── ncf_test.py │ ├── neumf_model.py │ ├── popen_helper.py │ ├── run.sh │ ├── run_tpu.sh │ └── stat_utils.py ├── requirements.txt ├── staging │ ├── __init__.py │ └── shakespeare │ │ ├── README.md │ │ ├── __init__.py │ │ ├── shakespeare_benchmark.py │ │ └── shakespeare_main.py ├── transformer │ ├── README.md │ ├── __init__.py │ ├── compute_bleu.py │ ├── compute_bleu_test.py │ ├── data_download.py │ ├── utils │ │ ├── __init__.py │ │ ├── metrics.py │ │ ├── tokenizer.py │ │ └── tokenizer_test.py │ └── v2 │ │ ├── __init__.py │ │ ├── attention_layer.py │ │ ├── beam_search.py │ │ ├── data_pipeline.py │ │ ├── embedding_layer.py │ │ ├── ffn_layer.py │ │ ├── metrics.py │ │ ├── misc.py │ │ ├── optimizer.py │ │ ├── transformer.py │ │ ├── transformer_layers_test.py │ │ ├── transformer_main.py │ │ ├── transformer_main_test.py │ │ ├── transformer_test.py │ │ └── translate.py ├── utils │ ├── __init__.py │ ├── flags │ │ ├── README.md │ │ ├── __init__.py │ │ ├── _base.py │ │ ├── _benchmark.py │ │ ├── _conventions.py │ │ ├── _device.py │ │ ├── _distribution.py │ │ ├── _misc.py │ │ ├── _performance.py │ │ ├── core.py │ │ ├── flags_test.py │ │ └── guidelines.md │ ├── hyperparams_flags.py │ ├── logs │ │ ├── __init__.py │ │ ├── cloud_lib.py │ │ ├── cloud_lib_test.py │ │ ├── guidelines.md │ │ ├── hooks.py │ │ ├── hooks_helper.py │ │ ├── hooks_helper_test.py │ │ ├── hooks_test.py │ │ ├── logger.py │ │ ├── logger_test.py │ │ ├── metric_hook.py │ │ ├── metric_hook_test.py │ │ └── mlperf_helper.py │ ├── misc │ │ ├── __init__.py │ │ ├── callstack_sampler.py │ │ ├── distribution_utils.py │ │ ├── distribution_utils_test.py │ │ ├── keras_utils.py │ │ ├── model_helpers.py │ │ ├── model_helpers_test.py │ │ └── tpu_lib.py │ └── testing │ │ ├── __init__.py │ │ ├── benchmark_wrappers.py │ │ ├── integration.py │ │ ├── mock_lib.py │ │ ├── perfzero_benchmark.py │ │ ├── pylint.rcfile │ │ ├── reference_data.py │ │ ├── reference_data │ │ ├── reference_data_test │ │ │ ├── dense │ │ │ │ ├── expected_graph │ │ │ │ ├── model.ckpt.data-00000-of-00001 │ │ │ │ ├── model.ckpt.index │ │ │ │ ├── results.json │ │ │ │ └── tf_version.json │ │ │ └── uniform_random │ │ │ │ ├── expected_graph │ │ │ │ ├── model.ckpt.data-00000-of-00001 │ │ │ │ ├── model.ckpt.index │ │ │ │ ├── results.json │ │ │ │ └── tf_version.json │ │ └── resnet │ │ │ ├── batch-size-32_bottleneck_projection_version-1_width-8_channels-4 │ │ │ ├── expected_graph │ │ │ ├── model.ckpt.data-00000-of-00001 │ │ │ ├── model.ckpt.index │ │ │ ├── results.json │ │ │ └── tf_version.json │ │ │ ├── batch-size-32_bottleneck_projection_version-2_width-8_channels-4 │ │ │ ├── expected_graph │ │ │ ├── model.ckpt.data-00000-of-00001 │ │ │ ├── model.ckpt.index │ │ │ ├── results.json │ │ │ └── tf_version.json │ │ │ ├── batch-size-32_bottleneck_version-1_width-8_channels-4 │ │ │ ├── expected_graph │ │ │ ├── model.ckpt.data-00000-of-00001 │ │ │ ├── model.ckpt.index │ │ │ ├── results.json │ │ │ └── tf_version.json │ │ │ ├── batch-size-32_bottleneck_version-2_width-8_channels-4 │ │ │ ├── expected_graph │ │ │ ├── model.ckpt.data-00000-of-00001 │ │ │ ├── model.ckpt.index │ │ │ ├── results.json │ │ │ └── tf_version.json │ │ │ ├── batch-size-32_building_projection_version-1_width-8_channels-4 │ │ │ ├── expected_graph │ │ │ ├── model.ckpt.data-00000-of-00001 │ │ │ ├── model.ckpt.index │ │ │ ├── results.json │ │ │ └── tf_version.json │ │ │ ├── batch-size-32_building_projection_version-2_width-8_channels-4 │ │ │ ├── expected_graph │ │ │ ├── model.ckpt.data-00000-of-00001 │ │ │ ├── model.ckpt.index │ │ │ ├── results.json │ │ │ └── tf_version.json │ │ │ ├── batch-size-32_building_version-1_width-8_channels-4 │ │ │ ├── expected_graph │ │ │ ├── model.ckpt.data-00000-of-00001 │ │ │ ├── model.ckpt.index │ │ │ ├── results.json │ │ │ └── tf_version.json │ │ │ ├── batch-size-32_building_version-2_width-8_channels-4 │ │ │ ├── expected_graph │ │ │ ├── model.ckpt.data-00000-of-00001 │ │ │ ├── model.ckpt.index │ │ │ ├── results.json │ │ │ └── tf_version.json │ │ │ └── batch_norm │ │ │ ├── expected_graph │ │ │ ├── model.ckpt.data-00000-of-00001 │ │ │ ├── model.ckpt.index │ │ │ ├── results.json │ │ │ └── tf_version.json │ │ ├── reference_data_test.py │ │ └── scripts │ │ ├── builds_common.sh │ │ ├── ci_sanity.sh │ │ └── presubmit.sh └── vision │ ├── __init__.py │ ├── detection │ ├── README.md │ ├── __init__.py │ ├── configs │ │ ├── __init__.py │ │ ├── base_config.py │ │ ├── factory.py │ │ ├── maskrcnn_config.py │ │ └── retinanet_config.py │ ├── dataloader │ │ ├── __init__.py │ │ ├── anchor.py │ │ ├── factory.py │ │ ├── input_reader.py │ │ ├── maskrcnn_parser.py │ │ ├── mode_keys.py │ │ ├── retinanet_parser.py │ │ ├── shapemask_parser.py │ │ └── tf_example_decoder.py │ ├── evaluation │ │ ├── __init__.py │ │ ├── coco_evaluator.py │ │ ├── coco_utils.py │ │ └── factory.py │ ├── executor │ │ ├── __init__.py │ │ └── detection_executor.py │ ├── main.py │ ├── modeling │ │ ├── __init__.py │ │ ├── architecture │ │ │ ├── __init__.py │ │ │ ├── factory.py │ │ │ ├── fpn.py │ │ │ ├── heads.py │ │ │ ├── identity.py │ │ │ ├── nn_ops.py │ │ │ └── resnet.py │ │ ├── base_model.py │ │ ├── checkpoint_utils.py │ │ ├── factory.py │ │ ├── learning_rates.py │ │ ├── losses.py │ │ └── retinanet_model.py │ ├── ops │ │ ├── nms.py │ │ ├── postprocess_ops.py │ │ ├── roi_ops.py │ │ ├── sampling_ops.py │ │ └── spatial_transform_ops.py │ └── utils │ │ ├── __init__.py │ │ ├── autoaugment_utils.py │ │ ├── box_utils.py │ │ ├── class_utils.py │ │ ├── dataloader_utils.py │ │ ├── input_utils.py │ │ ├── mask_utils.py │ │ └── object_detection │ │ ├── __init__.py │ │ ├── argmax_matcher.py │ │ ├── balanced_positive_negative_sampler.py │ │ ├── box_coder.py │ │ ├── box_list.py │ │ ├── box_list_ops.py │ │ ├── faster_rcnn_box_coder.py │ │ ├── matcher.py │ │ ├── minibatch_sampler.py │ │ ├── ops.py │ │ ├── preprocessor.py │ │ ├── region_similarity_calculator.py │ │ ├── shape_utils.py │ │ ├── target_assigner.py │ │ └── visualization_utils.py │ └── image_classification │ ├── README.md │ ├── __init__.py │ ├── cifar_preprocessing.py │ ├── common.py │ ├── common_test.py │ ├── imagenet_preprocessing.py │ ├── mnist_main.py │ ├── mnist_test.py │ ├── resnet_cifar_main.py │ ├── resnet_cifar_model.py │ ├── resnet_cifar_test.py │ ├── resnet_ctl_imagenet_main.py │ ├── resnet_ctl_imagenet_test.py │ ├── resnet_imagenet_main.py │ ├── resnet_imagenet_test.py │ ├── resnet_model.py │ └── tfhub_export.py └── run.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/.coveragerc: -------------------------------------------------------------------------------- 1 | # .coveragerc to control coverage.py 2 | [run] 3 | omit = 4 | tests/* 5 | conda/* 6 | scripts/tests/* 7 | concurrency = 8 | multiprocessing 9 | thread 10 | 11 | [report] 12 | ignore_errors = True 13 | 14 | [html] 15 | directory = coverage_html_report 16 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: 'bug' 6 | assignees: '' 7 | 8 | --- 9 | ## Description 10 | (A clear and concise description of what the bug is.) 11 | 12 | ### Error Message 13 | (Paste the complete error message, including stack trace.) 14 | 15 | ## To Reproduce 16 | (If you developed your own code, please provide a short script that reproduces the error. For existing examples, please provide link.) 17 | 18 | ### Steps to reproduce 19 | (Paste the commands you ran that produced the error.) 20 | 21 | 1. 22 | 2. 23 | 24 | ## What have you tried to solve it? 25 | 26 | 1. 27 | 2. 28 | 29 | ## Environment 30 | 31 | We recommend using our script for collecting the diagnositc information. Run the following command and paste the outputs below: 32 | ``` 33 | curl --retry 10 -s https://raw.githubusercontent.com/dmlc/gluon-nlp/master/tools/diagnose.py | python 34 | 35 | # paste outputs here 36 | ``` 37 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: 'enhancement' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Description 11 | (A clear and concise description of what the feature is.) 12 | - If the proposal is about a new model, provide description of what the model is. 13 | - If the proposal is about an API, provide mock examples if possible. 14 | 15 | ## References 16 | - list reference and related literature 17 | - list known implementations 18 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description ## 2 | (Brief description on what this PR is about) 3 | 4 | ## Checklist ## 5 | ### Essentials ### 6 | - [ ] PR's title starts with a category (e.g. [BUGFIX], [MODEL], [TUTORIAL], [FEATURE], [DOC], etc) 7 | - [ ] Changes are complete (i.e. I finished coding on this PR) 8 | - [ ] All changes have test coverage 9 | - [ ] Code is well-documented 10 | 11 | ### Changes ### 12 | - [ ] Feature1, tests, (and when applicable, API doc) 13 | - [ ] Feature2, tests, (and when applicable, API doc) 14 | 15 | ## Comments ## 16 | - If this change is a backward incompatible change, why must this change be made. 17 | - Interesting edge cases to note here 18 | 19 | cc @dmlc/gluon-nlp-team 20 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "scripts/word_embeddings/tools/extern/CLI11"] 2 | path = scripts/word_embeddings/tools/extern/CLI11 3 | url = https://github.com/CLIUtils/CLI11.git 4 | [submodule "scripts/word_embeddings/tools/extern/cnpy"] 5 | path = scripts/word_embeddings/tools/extern/cnpy 6 | url = https://github.com/leezu/cnpy 7 | branch = libzip 8 | [submodule "scripts/word_embeddings/tools/extern/sparsepp"] 9 | path = scripts/word_embeddings/tools/extern/sparsepp 10 | url = https://github.com/greg7mdp/sparsepp.git 11 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Watchers and contributors to Apache MXNet repo directories/packages/files 2 | # Please see documentation of use of CODEOWNERS file at 3 | # https://help.github.com/articles/about-codeowners/ and 4 | # https://github.com/blog/2392-introducing-code-owners 5 | # 6 | # Anybody can add themselves or a team as additional watcher or contributor 7 | # to get notified about changes in a specific package. 8 | # See https://help.github.com/articles/about-teams how to setup teams. 9 | 10 | 11 | # Global owners 12 | * @dmlc/gluon-nlp-committers @dmlc/gluon-nlp-reviewers 13 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contribution guideline can be found at http://gluon-nlp.mxnet.io/community/contribute.html 2 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include gluonnlp *.py 2 | include LICENSE 3 | include README.rst 4 | recursive-exclude tests * 5 | recursive-exclude scripts * -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/ci/batch/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 2 | 3 | RUN apt-get update && apt-get install -y --no-install-recommends \ 4 | build-essential \ 5 | locales \ 6 | cmake \ 7 | git \ 8 | curl \ 9 | vim \ 10 | unzip \ 11 | sudo \ 12 | ca-certificates \ 13 | libjpeg-dev \ 14 | libpng-dev \ 15 | libfreetype6-dev \ 16 | libxft-dev &&\ 17 | rm -rf /var/lib/apt/lists/* 18 | 19 | RUN curl -o ~/miniconda.sh -O https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ 20 | chmod +x ~/miniconda.sh && \ 21 | ~/miniconda.sh -b -p /opt/conda && \ 22 | rm ~/miniconda.sh && \ 23 | /opt/conda/bin/conda clean -ya 24 | ENV PATH /opt/conda/bin:$PATH 25 | RUN git clone https://github.com/dmlc/gluon-nlp 26 | WORKDIR gluon-nlp 27 | RUN /bin/bash -c 'CONDA_ENVS_PATH=$PWD/conda CONDA_PKGS_DIRS=$PWD/conda/pkgs conda init bash && source /root/.bashrc && conda env update --prune -p conda/gpu/py3 -f env/gpu/py3.yml && source activate ./conda/gpu/py3 && pip install -v -e . && pip install awscli && python -m spacy download en && python -m spacy download de && python -m nltk.downloader all' 28 | ADD gluon_nlp_job.sh . 29 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/ci/batch/docker/gluon_nlp_job.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | date 3 | echo "Args: $@" 4 | env 5 | echo "jobId: $AWS_BATCH_JOB_ID" 6 | echo "jobQueue: $AWS_BATCH_JQ_NAME" 7 | echo "computeEnvironment: $AWS_BATCH_CE_NAME" 8 | 9 | SOURCE_REF=$1 10 | CONDA_ENV=$2 11 | WORK_DIR=$3 12 | COMMAND=$4 13 | SAVED_OUTPUT=$5 14 | SAVE_PATH=$6 15 | REMOTE=$7 16 | 17 | if [ ! -z $REMOTE ]; then 18 | git remote set-url origin $REMOTE 19 | fi; 20 | 21 | git fetch origin $SOURCE_REF:working 22 | git checkout working 23 | conda env update --prune -p conda/$CONDA_ENV -f env/$CONDA_ENV.yml 24 | source activate ./conda/$CONDA_ENV 25 | pip install -v -e . 26 | python -m spacy download en 27 | python -m spacy download de 28 | python -m nltk.downloader all 29 | pip install awscli 30 | 31 | cd $WORK_DIR 32 | /bin/bash -c "$COMMAND" 33 | COMMAND_EXIT_CODE=$? 34 | if [[ -f $SAVED_OUTPUT ]]; then 35 | aws s3 cp $SAVED_OUTPUT s3://gluon-nlp-staging/$SAVE_PATH; 36 | elif [[ -d $SAVED_OUTPUT ]]; then 37 | aws s3 cp --recursive $SAVED_OUTPUT s3://gluon-nlp-staging/$SAVE_PATH; 38 | fi; 39 | exit $COMMAND_EXIT_CODE 40 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/ci/prepare_clean_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | env_name=$1 3 | 4 | export LD_LIBRARY_PATH=/usr/local/cuda/lib64 5 | export CUDA_VISIBLE_DEVICES=$EXECUTOR_NUMBER 6 | export CONDA_ENVS_PATH=$PWD/conda 7 | export CONDA_PKGS_DIRS=$PWD/conda/pkgs 8 | export MXNET_HOME=$PWD/tests/data 9 | export HOROVOD_WITHOUT_TENSORFLOW=1 10 | export HOROVOD_WITHOUT_PYTORCH=1 11 | export HOROVOD_WITH_MXNET=1 12 | 13 | make clean 14 | conda env update --prune -p conda/${env_name} -f env/${env_name}.yml 15 | conda activate ./conda/${env_name} 16 | conda list 17 | printenv 18 | 19 | pip install -v -e . 20 | pip install horovod --no-cache-dir -U 21 | python -m spacy download en 22 | python -m spacy download de 23 | python -m nltk.downloader all 24 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/ci/rat/rat-excludes: -------------------------------------------------------------------------------- 1 | \..* 2 | .*css 3 | \\.* 4 | .*ipynb 5 | .*html 6 | .*json 7 | .*txt 8 | 3rdparty/* 9 | R-package/* 10 | trunk/* 11 | .*\\.m 12 | .*\\.mk 13 | .*\\.R 14 | .*svg 15 | .*cfg 16 | .*config 17 | .*rst 18 | __init__.py 19 | build/* 20 | .*\\.t 21 | MANIFEST 22 | Changes 23 | .*csv 24 | .*names 25 | CODEOWNERS 26 | snap.python 27 | bbox.pyx 28 | cpu_nms.pyx 29 | gpu_nms.pyx 30 | nms_kernel.cu 31 | _mask.pyx 32 | coco.py 33 | base.pyi 34 | special_functions-inl.h 35 | erfinv-inl.h 36 | im2col.cuh 37 | im2col.h 38 | pool.h 39 | dataset.cPickle 40 | image-classification/* 41 | rat-excludes 42 | apache-rat-tasks/* 43 | moderngpu/* 44 | deformable_im2col.cuh 45 | deformable_im2col.h 46 | REQUIRE 47 | Project.toml 48 | include/* 49 | .*.iml 50 | .*.json.ref 51 | searchtools_custom.js 52 | theme.conf 53 | LICENSE.binary.dependencies 54 | multi-bleu-detok.perl 55 | multi-bleu.perl 56 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/ci/upload_doc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | bucket=$1 3 | path=$2 4 | echo "Uploading doc to s3://${bucket}/${path}/" 5 | aws s3 sync --delete docs/_build/html/ s3://${bucket}/${path}/ --acl public-read 6 | echo "Uploaded doc to http://${bucket}.s3-accelerate.dualstack.amazonaws.com/${path}/index.html" 7 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | notify: 3 | require_ci_to_pass: yes 4 | ci: 5 | - ci.mxnet.io 6 | 7 | coverage: 8 | precision: 2 9 | round: down 10 | range: "70...100" 11 | 12 | status: 13 | project: yes 14 | patch: yes 15 | changes: no 16 | 17 | parsers: 18 | gcov: 19 | branch_detection: 20 | conditional: yes 21 | loop: yes 22 | method: no 23 | macro: no 24 | 25 | comment: 26 | layout: "header, reach, diff, files" 27 | behavior: default 28 | require_changes: no 29 | require_base: no 30 | require_head: no 31 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/contributor.rst: -------------------------------------------------------------------------------- 1 | Contributors 2 | ============ 3 | 4 | We would like to express our thanks to our awesome `contributors `__ to the GluonNLP. 5 | 6 | - `Bodapati, Sravan `__ 7 | - `Chung, Jonathan `__ 8 | - `Delteil, Thomas `__ 9 | - `He, He `__ 10 | - `Yun, Hyokun `__ 11 | - `Lalwani, Garima `__ 12 | - `Lausen, Leonard `__ 13 | - `Li, Alex `__ 14 | - `Li, Mu `__ 15 | - `Li, Xin `__ 16 | - `Lin, Haibin `__ 17 | - `Lipton, Zachary `__ 18 | - `Ravu, Rahul `__ 19 | - `Shao, Junru `__ 20 | - `Shi, Xingjian `__ 21 | - `Sokolov, Sergey `__ 22 | - `Wang, Chenguang `__ 23 | - `Zha, Sheng `__ 24 | - `Zhang, Aston `__ 25 | - `Zheng, Shuai `__ 26 | - `cclauss `__ 27 | 28 | .. note:: 29 | 30 | The contributors are ordered by last name alphabetical order. 31 | 32 | Interested in joining us? Check out our `contributing guide 33 | `__. 34 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/.gitignore: -------------------------------------------------------------------------------- 1 | doxygen 2 | _build 3 | gen_modules 4 | tutorials 5 | doctrees 6 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/.nojekyll -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/404.rst: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | Page Not Found 4 | -------------- 5 | 6 | You stumbled upon a page that's making us scratch our brains right now. Before any of us panics, 7 | we'd like you to know that you are being redirected to a better known and cozy place, in just a few seconds. 8 | 9 | .. image:: _static/404.jpg 10 | :alt: Page Not Found 11 | :width: 60% 12 | :align: center 13 | :target: ./index.html 14 | 15 | .. raw:: html 16 | 17 | 26 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/README.txt: -------------------------------------------------------------------------------- 1 | The documentation of gluonnlp is generated with recommonmark and sphinx. 2 | 3 | - pip install sphinx>=1.5.5 sphinx-gallery sphinx_rtd_theme matplotlib Image recommonmark 4 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/_static/404.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/_static/404.jpg -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/_static/gluon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/_static/gluon.ico -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/_static/gluon_black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/_static/gluon_black.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/_static/google_analytics.js: -------------------------------------------------------------------------------- 1 | (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ 2 | (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), 3 | m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) 4 | })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); 5 | 6 | ga('create', 'UA-96378503-8', 'auto'); 7 | ga('send', 'pageview'); 8 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/index.rst: -------------------------------------------------------------------------------- 1 | API Documentation 2 | ================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | 7 | notes/index 8 | modules/index 9 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/modules/data.batchify.rst: -------------------------------------------------------------------------------- 1 | gluonnlp.data.batchify 2 | ====================== 3 | 4 | Batchify functions can be used to transform a dataset into mini-batches that can be processed 5 | efficiently. 6 | 7 | .. currentmodule:: gluonnlp.data.batchify 8 | 9 | Batch Loaders 10 | ------------- 11 | 12 | .. autosummary:: 13 | :nosignatures: 14 | 15 | Stack 16 | Pad 17 | List 18 | Tuple 19 | 20 | 21 | Language Modeling 22 | ----------------- 23 | 24 | .. autosummary:: 25 | :nosignatures: 26 | 27 | CorpusBatchify 28 | CorpusBPTTBatchify 29 | StreamBPTTBatchify 30 | 31 | Embedding Training 32 | ------------------ 33 | 34 | .. autosummary:: 35 | :nosignatures: 36 | 37 | EmbeddingCenterContextBatchify 38 | 39 | API Reference 40 | ------------- 41 | 42 | .. automodule:: gluonnlp.data.batchify 43 | :members: 44 | :imported-members: 45 | :special-members: __call__, __iter__ 46 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/modules/embedding.rst: -------------------------------------------------------------------------------- 1 | gluonnlp.embedding 2 | ================== 3 | 4 | GluonNLP Toolkit provides tools for working with embeddings. 5 | 6 | .. currentmodule:: gluonnlp.embedding 7 | 8 | This page describes the ``gluonnlp`` APIs for text embedding, such as loading 9 | pre-trained embedding vectors for text tokens and storing them in the 10 | ``mxnet.ndarray.NDArray`` format as well as utilities for intrinsic evaluation 11 | of text embeddings. 12 | 13 | 14 | Pre-trained Embeddings 15 | ---------------------- 16 | 17 | .. currentmodule:: gluonnlp.embedding 18 | .. autosummary:: 19 | :nosignatures: 20 | 21 | register 22 | create 23 | list_sources 24 | TokenEmbedding 25 | GloVe 26 | FastText 27 | Word2Vec 28 | 29 | 30 | Intrinsic evaluation 31 | -------------------- 32 | 33 | .. currentmodule:: gluonnlp.embedding.evaluation 34 | .. autosummary:: 35 | :nosignatures: 36 | 37 | register 38 | create 39 | list_evaluation_functions 40 | WordEmbeddingSimilarityFunction 41 | WordEmbeddingAnalogyFunction 42 | CosineSimilarity 43 | ThreeCosAdd 44 | ThreeCosMul 45 | WordEmbeddingSimilarity 46 | WordEmbeddingAnalogy 47 | 48 | 49 | API Reference 50 | ------------- 51 | 52 | .. automodule:: gluonnlp.embedding 53 | :members: 54 | :imported-members: 55 | :special-members: __contains__, __getitem__, __setitem__ 56 | 57 | .. automodule:: gluonnlp.embedding.evaluation 58 | :members: 59 | :imported-members: 60 | :special-members: __contains__, __getitem__, __setitem__ 61 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/modules/index.rst: -------------------------------------------------------------------------------- 1 | Package Reference 2 | ================= 3 | 4 | .. toctree:: 5 | :maxdepth: 1 6 | 7 | vocab 8 | embedding 9 | data 10 | data.batchify 11 | model 12 | model.train 13 | loss 14 | initializer 15 | optimizer 16 | utils 17 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/modules/initializer.rst: -------------------------------------------------------------------------------- 1 | gluonnlp.initializer 2 | ====================== 3 | 4 | This page describes initializers that are useful for multiple NLP model architectures. 5 | 6 | .. currentmodule:: gluonnlp.initializer 7 | 8 | Highway Bias Initializer 9 | -------------------------- 10 | 11 | We now provide Highway bias initializer defined in the following work. 12 | 13 | .. code-block:: none 14 | 15 | @inproceedings{srivastava2015training, 16 | title={Training very deep networks}, 17 | author={Srivastava, Rupesh K and Greff, Klaus and Schmidhuber, J{\"u}rgen}, 18 | booktitle={Advances in neural information processing systems}, 19 | pages={2377--2385}, 20 | year={2015}} 21 | 22 | .. autosummary:: 23 | :nosignatures: 24 | 25 | HighwayBias 26 | 27 | API Reference 28 | ------------- 29 | 30 | .. automodule:: gluonnlp.initializer 31 | :members: 32 | :imported-members: 33 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/modules/loss.rst: -------------------------------------------------------------------------------- 1 | gluonnlp.loss 2 | ============= 3 | 4 | GluonNLP Toolkit provides tools for easily setting up task specific loss. 5 | 6 | .. currentmodule:: gluonnlp.loss 7 | 8 | Masked Loss 9 | ----------- 10 | 11 | .. autosummary:: 12 | :nosignatures: 13 | 14 | MaskedSoftmaxCrossEntropyLoss 15 | 16 | 17 | Label Smoothing 18 | --------------- 19 | 20 | .. autosummary:: 21 | :nosignatures: 22 | 23 | LabelSmoothing 24 | 25 | 26 | Activation Regularizers 27 | ----------------------- 28 | 29 | Activation regularization and temporal activation regularization defined in the following work: 30 | 31 | .. code-block:: none 32 | 33 | @article{merity2017revisiting, 34 | title={Revisiting Activation Regularization for Language RNNs}, 35 | author={Merity, Stephen and McCann, Bryan and Socher, Richard}, 36 | journal={arXiv preprint arXiv:1708.01009}, 37 | year={2017}} 38 | 39 | .. autosummary:: 40 | :nosignatures: 41 | 42 | ActivationRegularizationLoss 43 | TemporalActivationRegularizationLoss 44 | 45 | 46 | API Reference 47 | ------------- 48 | 49 | .. automodule:: gluonnlp.loss 50 | :members: 51 | :imported-members: 52 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/modules/model.train.rst: -------------------------------------------------------------------------------- 1 | gluonnlp.model.train 2 | ===================== 3 | 4 | GluonNLP Toolkit supplies models with train-mode since the corresponding models have different behaviors in training 5 | and inference, e.g., the number and type of the outputs from the forward pass are different. 6 | 7 | .. currentmodule:: gluonnlp.model.train 8 | 9 | Language Modeling 10 | ----------------- 11 | 12 | .. autosummary:: 13 | :nosignatures: 14 | 15 | AWDRNN 16 | StandardRNN 17 | CacheCell 18 | get_cache_model 19 | BigRNN 20 | 21 | 22 | 23 | Word Embeddings 24 | --------------- 25 | 26 | .. autosummary:: 27 | :nosignatures: 28 | 29 | EmbeddingModel 30 | CSREmbeddingModel 31 | FasttextEmbeddingModel 32 | 33 | 34 | API Reference 35 | ------------- 36 | 37 | .. automodule:: gluonnlp.model.train 38 | :members: 39 | :imported-members: 40 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/modules/optimizer.rst: -------------------------------------------------------------------------------- 1 | gluonnlp.optimizer 2 | ====================== 3 | 4 | Gluonnlp provides some special optimizers for training in natural language processing. 5 | 6 | .. currentmodule:: gluonnlp.optimizer 7 | 8 | BERTAdam Optimizer 9 | -------------------------- 10 | 11 | The Adam optimizer with weight decay regularization for BERT. 12 | 13 | .. autosummary:: 14 | :nosignatures: 15 | 16 | BERTAdam 17 | 18 | LAMB Optimizer 19 | -------------------------- 20 | 21 | Implementation of the LAMB optimizer from the paper `Reducing BERT Pre-Training Time from 3 Days to 76 Minutes. `_ 22 | 23 | In paper, the empirical results demonstrate the superior performance of LAMB for BERT and ResNet-50 training. 24 | By increasing the batch size to the memory limit of a TPUv3 pod, BERT training time can be reduced from 3 days to 76 minutes. 25 | 26 | .. code-block:: none 27 | 28 | @inproceedings{You2019LargeBO, 29 | title={Large Batch Optimization for Deep Learning: Training BERT in 76 minutes}, 30 | author={Yang You and Jing Li and Sashank J. Reddi and Jonathan Hseu and Sanjiv Kumar and Srinadh Bhojanapalli and Xiaodan Song and James Demmel and Cho-Jui Hsieh}, 31 | year={2019}} 32 | 33 | .. autosummary:: 34 | :nosignatures: 35 | 36 | LAMB 37 | 38 | API Reference 39 | ------------- 40 | 41 | .. automodule:: gluonnlp.optimizer 42 | :members: 43 | :imported-members: 44 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/modules/utils.rst: -------------------------------------------------------------------------------- 1 | gluonnlp.utils 2 | ============== 3 | 4 | GluonNLP Toolkit provides tools for easily setting up task specific loss. 5 | 6 | .. currentmodule:: gluonnlp.utils 7 | 8 | 9 | File Handling 10 | ------------- 11 | 12 | .. autosummary:: 13 | :nosignatures: 14 | 15 | glob 16 | mkdir 17 | 18 | 19 | Parameter and Training 20 | ---------------------- 21 | 22 | .. autosummary:: 23 | :nosignatures: 24 | 25 | clip_grad_global_norm 26 | 27 | 28 | Serialization and Deserialization 29 | --------------------------------- 30 | 31 | .. autosummary:: 32 | :nosignatures: 33 | 34 | load_parameters 35 | load_states 36 | save_parameters 37 | save_states 38 | 39 | 40 | API Reference 41 | ------------- 42 | 43 | .. automodule:: gluonnlp.utils 44 | :members: 45 | :imported-members: 46 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/notes/images/fixed_bucket_strategy_ratio0.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/api/notes/images/fixed_bucket_strategy_ratio0.0.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/notes/images/fixed_bucket_strategy_ratio0.7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/api/notes/images/fixed_bucket_strategy_ratio0.7.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/notes/images/no_bucket_strategy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/api/notes/images/no_bucket_strategy.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/notes/images/sorted_bucket_strategy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/api/notes/images/sorted_bucket_strategy.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/api/notes/index.rst: -------------------------------------------------------------------------------- 1 | Notes 2 | ===== 3 | 4 | Here are some notes on the basic usage of our API. 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | data_api 10 | vocab_emb 11 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/community/index.rst: -------------------------------------------------------------------------------- 1 | Community 2 | ========= 3 | 4 | .. card:: 5 | :title: Community 6 | :is_head: true 7 | :link: https://www.apache.org/foundation/policies/conduct 8 | 9 | Welcome to GluonNLP community. We strive to foster a collaborative and welcoming community. We 10 | expect all members to follow the `code of conduct `__. 11 | 12 | 13 | .. container:: cards 14 | 15 | .. card:: 16 | :title: Github Issues 17 | :link: https://github.com/dmlc/gluon-nlp/issues 18 | 19 | Feature requests, bug reports, design and roadmap discussion. 20 | 21 | 22 | .. card:: 23 | :title: Github Projects 24 | :link: https://github.com/dmlc/gluon-nlp/projects 25 | 26 | Active roadmaps, and current and past projects. 27 | 28 | 29 | .. card:: 30 | :title: GluonNLP Slack Channel 31 | :link: https://apache-mxnet.slack.com/messages/CCCDM10V9 32 | 33 | #gluon-nlp Slack channel. Click the `sign-up link `_ to register. 34 | 35 | 36 | .. card:: 37 | :title: Discuss Forum 38 | :link: https://discuss.mxnet.io/c/gluon 39 | 40 | General discussions, usage experiences, and questions. 41 | 42 | 43 | .. card:: 44 | :title: 中文论坛 45 | :link: https://discuss.gluon.ai/c/gluon-nlp 46 | 47 | 中文用户使用讨论，提问与心得分享 48 | 49 | 50 | Interested in contributing to GluonNLP? Check our contribution guide: 51 | 52 | .. toctree:: 53 | :maxdepth: 3 54 | 55 | contribute 56 | release 57 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/community/release.rst: -------------------------------------------------------------------------------- 1 | Release Checklist 2 | ================= 3 | 4 | Below is the checklist for releasing a new minor version of GluonNLP: 5 | 6 | - Creat a new release branch $major.$minor.x with commits from the master branch 7 | - Bump the version in master branch to $major.$minor+1.$patch.dev 8 | - Bump the version in release branch to $major.$minor.$patch 9 | - Draft the release note, highlight important talks/models/features, as well as breaking change 10 | - Publish the release on Github, creating a tag $major.$minor.$patch 11 | - Check the content at http://gluon-nlp.mxnet.io/$major.$minor.x/index.html 12 | - Upload and refresh the default version website 13 | - Prepare pip package 14 | - Make annoucement (Twitter, etc) 15 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/language_model/cache_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/examples/language_model/cache_model.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/language_model/language_model_intro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/examples/language_model/language_model_intro.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/machine_translation/hyperparameters.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """Hyperparameters for transformer.""" 18 | 19 | import nmt 20 | 21 | # parameters for dataset 22 | src_lang = 'en' 23 | tgt_lang = 'de' 24 | src_max_len = -1 25 | tgt_max_len = -1 26 | 27 | # parameters for model 28 | num_units = 512 29 | hidden_size = 2048 30 | dropout = 0.1 31 | epsilon = 0.1 32 | num_layers = 6 33 | num_heads = 8 34 | scaled = True 35 | 36 | # parameters for training 37 | optimizer = 'adam' 38 | epochs = 3 39 | batch_size = 2700 40 | test_batch_size = 256 41 | num_accumulated = 1 42 | lr = 2 43 | warmup_steps = 1 44 | save_dir = 'transformer_en_de_u512' 45 | average_start = 1 46 | num_buckets = 20 47 | log_interval = 10 48 | bleu = '13a' 49 | 50 | #parameters for testing 51 | beam_size = 4 52 | lp_alpha = 0.6 53 | lp_k = 5 -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/machine_translation/nmt: -------------------------------------------------------------------------------- 1 | ../../model_zoo/machine_translation -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/machine_translation/transformer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/examples/machine_translation/transformer.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/sentence_embedding/Bi-LSTM-Rep.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/examples/sentence_embedding/Bi-LSTM-Rep.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/sentence_embedding/attention-nlp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/examples/sentence_embedding/attention-nlp.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/sentence_embedding/bert: -------------------------------------------------------------------------------- 1 | ../../../scripts/bert/ -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/sentence_embedding/bert-embed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/examples/sentence_embedding/bert-embed.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/sentence_embedding/bert-sentence-pair.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/examples/sentence_embedding/bert-sentence-pair.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/sentence_embedding/bert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/examples/sentence_embedding/bert.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/sentiment_analysis/samodel-v3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/docs/examples/sentiment_analysis/samodel-v3.png -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/word_embedding/data.py: -------------------------------------------------------------------------------- 1 | ../../model_zoo/word_embeddings/data.py -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/word_embedding/model.py: -------------------------------------------------------------------------------- 1 | ../../model_zoo/word_embeddings/model.py -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/examples/word_embedding/utils.py: -------------------------------------------------------------------------------- 1 | ../../model_zoo/word_embeddings/utils.py -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/genindex.rst: -------------------------------------------------------------------------------- 1 | Index 2 | ===== 3 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/md2ipynb.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import time 4 | import notedown 5 | import nbformat 6 | 7 | assert len(sys.argv) == 2, 'usage: input.md' 8 | 9 | # timeout for each notebook, in sec 10 | timeout = 40 * 60 11 | 12 | # the files will be ignored for execution 13 | ignore_execution = [] 14 | 15 | input_path = sys.argv[1] 16 | 17 | # Change working directory to directory of input file 18 | input_dir, input_fn = os.path.split(input_path) 19 | os.chdir(input_dir) 20 | 21 | output_fn = '.'.join(input_fn.split('.')[:-1] + ['ipynb']) 22 | 23 | reader = notedown.MarkdownReader() 24 | 25 | # read 26 | with open(input_fn, encoding='utf-8', mode='r') as f: 27 | notebook = reader.read(f) 28 | 29 | if not any([i in input_fn for i in ignore_execution]): 30 | tic = time.time() 31 | notedown.run(notebook, timeout) 32 | print('=== Finished evaluation in %f sec'%(time.time()-tic)) 33 | 34 | # write 35 | # need to add language info to for syntax highlight 36 | notebook['metadata'].update({'language_info': {'name': 'python'}}) 37 | 38 | notebook_json = nbformat.writes(notebook) 39 | 40 | with open(output_fn, encoding='utf-8', mode='w') as f: 41 | f.write(notebook_json) 42 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/model_zoo: -------------------------------------------------------------------------------- 1 | ../scripts -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/docs/model_zoo.rst: -------------------------------------------------------------------------------- 1 | Model Zoo 2 | --------- 3 | 4 | .. container:: cards 5 | 6 | .. card:: 7 | :title: Word Embedding 8 | :link: model_zoo/word_embeddings/index.html 9 | 10 | Mapping words to vectors. 11 | 12 | .. card:: 13 | :title: Language Modeling 14 | :link: model_zoo/language_model/index.html 15 | 16 | Learning the distribution and representation of sequences of words. 17 | 18 | .. card:: 19 | :title: Machine Translation 20 | :link: model_zoo/machine_translation/index.html 21 | 22 | From "Hello" to "Bonjour". 23 | 24 | .. card:: 25 | :title: Text Classification 26 | :link: model_zoo/text_classification/index.html 27 | 28 | Categorize texts and documents. 29 | 30 | .. card:: 31 | :title: Sentiment Analysis 32 | :link: model_zoo/sentiment_analysis/index.html 33 | 34 | Classifying polarity of emotions and opinions. 35 | 36 | .. card:: 37 | :title: Parsing 38 | :link: model_zoo/parsing/index.html 39 | 40 | Dependency parsing. 41 | 42 | .. card:: 43 | :title: Natural Language Inference 44 | :link: model_zoo/natural_language_inference/index.html 45 | 46 | Determine if the premise semantically entails the hypothesis. 47 | 48 | .. card:: 49 | :title: Text Generation 50 | :link: model_zoo/text_generation/index.html 51 | 52 | Generating language from models. 53 | 54 | .. card:: 55 | :title: BERT 56 | :link: model_zoo/bert/index.html 57 | 58 | Transfer pre-trained language representations to language understanding tasks. 59 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/examples: -------------------------------------------------------------------------------- 1 | docs/examples -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | seed: set the python, numpy and mxnet random seeds to a specified value for test reproducibility 4 | serial: mark a test that requires more resources to run that are thus only suitable for serial run. 5 | remote_required: mark a test that requires internet access. 6 | gpu: mark a test that requires GPU. 7 | integration: mark an integration test 8 | skip_master: mark a test that is temporarily skipped for mxnet master validation. 9 | py3_only: mark a test that is intended for a python3-only feature. 10 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """NLP examples.""" 20 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/bert/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """BERT Module.""" 20 | from . import model, data 21 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/bert/cmd.sh: -------------------------------------------------------------------------------- 1 | #TRUNCATE_NORM=1 LAMB_BULK=30 EPS_AFTER_SQRT=1 NUMSTEPS=900000 DTYPE=float16 BS=256 ACC=2 MODEL=bert_24_1024_16 MAX_SEQ_LENGTH=128 MAX_PREDICTIONS_PER_SEQ=20 LR=0.0001 LOGINTERVAL=1 CKPTDIR=ckpt_stage1_adam_1x_kv CKPTINTERVAL=300000 OPTIMIZER=bertadam WARMUP_RATIO=0.0001 bash kvstore.sh 2 | 3 | PORT=12350 NCCLMINNRINGS=16 NP=8 TRUNCATE_NORM=1 LAMB_BULK=30 EPS_AFTER_SQRT=1 NUMSTEPS=900000 DTYPE=float16 BS=512 ACC=1 MODEL=bert_24_1024_16 MAX_SEQ_LENGTH=128 MAX_PREDICTIONS_PER_SEQ=20 LR=0.000625 LOGINTERVAL=10 CKPTDIR=ckpt_stage1_lamb CKPTINTERVAL=300000000 OPTIMIZER=lamb WARMUP_RATIO=0.003125 bash hvd.sh 4 | 5 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/bert/conversion_tools/ernie_top_layer_emb.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/scripts/bert/conversion_tools/ernie_top_layer_emb.npy -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/bert/conversion_tools/input.txt: -------------------------------------------------------------------------------- 1 | Who was Jim Henson ? ||| Jim Henson was a puppeteer 2 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/bert/conversion_tools/input_cn.txt: -------------------------------------------------------------------------------- 1 | 这是百度的ERNIE模型 ||| 2 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/bert/data/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """BERT data.""" 20 | from . import qa, classification, embedding, transform, ner, dataloader, pretrain 21 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/bert/hvd.sh: -------------------------------------------------------------------------------- 1 | pkill python 2 | 3 | #-x HOROVOD_CYCLE_TIME=0.1 \ 4 | #--verbose \ 5 | #-x HOROVOD_TIMELINE=timeline.json \ 6 | #-x MXNET_HOROVOD_NUM_GROUPS=4 \ 7 | #--synthetic_data --eval_use_npz \ 8 | 9 | mpirun -np $NP --allow-run-as-root -mca pml ob1 -mca btl ^openib \ 10 | -mca btl_tcp_if_exclude docker0,lo --bind-to none \ 11 | --mca plm_rsh_agent 'ssh -q -o StrictHostKeyChecking=no' \ 12 | -x NCCL_MIN_NRINGS=$NCCLMINNRINGS -x NCCL_DEBUG=INFO \ 13 | -x HOROVOD_HIERARCHICAL_ALLREDUCE=1 \ 14 | -x HOROVOD_CYCLE_TIME=1 \ 15 | -x MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_FWD=120 \ 16 | -x MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN_BWD=120 \ 17 | -x MXNET_SAFE_ACCUMULATION=1 \ 18 | --tag-output ompi_bind_DGX1.sh python run_pretraining.py \ 19 | --data='/data/book-corpus/book-corpus-large-split/*.train,/data/enwiki/enwiki-feb-doc-split/*.train' \ 20 | --data_eval='/data/book-corpus/book-corpus-large-split/*.test,/data/enwiki/enwiki-feb-doc-split/*.test' \ 21 | --optimizer $OPTIMIZER \ 22 | --warmup_ratio $WARMUP_RATIO \ 23 | --num_steps $NUMSTEPS \ 24 | --ckpt_interval $CKPTINTERVAL \ 25 | --dtype $DTYPE \ 26 | --ckpt_dir $CKPTDIR \ 27 | --lr $LR \ 28 | --total_batch_size $BS \ 29 | --total_batch_size_eval $BS \ 30 | --accumulate $ACC \ 31 | --model $MODEL \ 32 | --max_seq_length $MAX_SEQ_LENGTH \ 33 | --max_predictions_per_seq $MAX_PREDICTIONS_PER_SEQ \ 34 | --num_data_workers 4 \ 35 | --no_compute_acc --raw \ 36 | --comm_backend horovod --log_interval $LOGINTERVAL 37 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/bert/kvstore.sh: -------------------------------------------------------------------------------- 1 | pkill python; 2 | export MXNET_SAFE_ACCUMULATION=1 3 | python run_pretraining.py \ 4 | --data='/data/book-corpus/book-corpus-large-split/*.train,/data/enwiki/enwiki-feb-doc-split/*.train' \ 5 | --data_eval='/data/book-corpus/book-corpus-large-split/*.test,/data/enwiki/enwiki-feb-doc-split/*.test' \ 6 | --optimizer $OPTIMIZER \ 7 | --warmup_ratio $WARMUP_RATIO \ 8 | --num_steps $NUMSTEPS \ 9 | --ckpt_interval $CKPTINTERVAL \ 10 | --dtype $DTYPE \ 11 | --ckpt_dir $CKPTDIR \ 12 | --lr $LR \ 13 | --total_batch_size $BS \ 14 | --total_batch_size_eval $BS \ 15 | --accumulate $ACC \ 16 | --model $MODEL \ 17 | --max_seq_length $MAX_SEQ_LENGTH \ 18 | --max_predictions_per_seq $MAX_PREDICTIONS_PER_SEQ \ 19 | --num_data_workers 4 \ 20 | --no_compute_acc --raw \ 21 | --comm_backend device --gpus 0,1,2,3,4,5,6,7 --log_interval $LOGINTERVAL 22 | 23 | #--verbose \ 24 | #--synthetic_data --eval_use_npz \ 25 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/bert/model/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """BERT model.""" 20 | from . import ner, qa 21 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/bert/ompi_bind_DGX1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | case "${OMPI_COMM_WORLD_LOCAL_RANK}" in 4 | 0) 5 | export OMPI_MCA_btl_openib_if_include=mlx5_0 6 | exec numactl --physcpubind=0-5,48-53 --membind=0 "${@}" 7 | ;; 8 | 1) 9 | export OMPI_MCA_btl_openib_if_include=mlx5_0 10 | exec numactl --physcpubind=6-11,54-59 --membind=0 "${@}" 11 | ;; 12 | 2) 13 | export OMPI_MCA_btl_openib_if_include=mlx5_1 14 | exec numactl --physcpubind=12-17,60-65 --membind=0 "${@}" 15 | ;; 16 | 3) 17 | export OMPI_MCA_btl_openib_if_include=mlx5_1 18 | exec numactl --physcpubind=18-23,66-71 --membind=0 "${@}" 19 | ;; 20 | 4) 21 | export OMPI_MCA_btl_openib_if_include=mlx5_2 22 | exec numactl --physcpubind=24-29,72-77 --membind=1 "${@}" 23 | ;; 24 | 5) 25 | export OMPI_MCA_btl_openib_if_include=mlx5_2 26 | exec numactl --physcpubind=30-35,78-83 --membind=1 "${@}" 27 | ;; 28 | 6) 29 | export OMPI_MCA_btl_openib_if_include=mlx5_3 30 | exec numactl --physcpubind=36-41,84-89 --membind=1 "${@}" 31 | ;; 32 | 7) 33 | export OMPI_MCA_btl_openib_if_include=mlx5_3 34 | exec numactl --physcpubind=42-47,90-95 --membind=1 "${@}" 35 | ;; 36 | *) 37 | echo ============================================================== 38 | echo "ERROR: Unknown local rank ${OMPI_COMM_WORLD_LOCAL_RANK}" 39 | echo ============================================================== 40 | exit 1 41 | ;; 42 | esac 43 | 44 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/language_model/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """Language Model example.""" 20 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/language_model/transformer/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """Utilities for Language Models based on the Transformer architecture.""" 18 | 19 | from .attention_cell import * 20 | from .embedding import * 21 | from .softmax import * 22 | from .transformer import * 23 | from .model import * 24 | 25 | __all__ = attention_cell.__all__ + embedding.__all__ + softmax.__all__ + \ 26 | transformer.__all__ + model.__all__ 27 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/machine_translation/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """NMT example.""" 20 | from . import _constants, bleu, dataset, \ 21 | gnmt, translation, utils 22 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/machine_translation/_constants.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """Constants used in the NMT examples.""" 18 | import os 19 | 20 | __all__ = ['CACHE_PATH'] 21 | 22 | CACHE_PATH = os.path.realpath(os.path.join(os.path.realpath(__file__), '..', 'cached')) 23 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/machine_translation/hyperparameters.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """Hyperparameters for transformer, for past reference only.""" 18 | 19 | # parameters for dataset 20 | src_lang = 'en' 21 | tgt_lang = 'de' 22 | src_max_len = -1 23 | tgt_max_len = -1 24 | 25 | # parameters for model 26 | num_units = 512 27 | hidden_size = 2048 28 | dropout = 0.1 29 | epsilon = 0.1 30 | num_layers = 6 31 | num_heads = 8 32 | scaled = True 33 | 34 | # parameters for training 35 | optimizer = 'adam' 36 | epochs = 3 37 | batch_size = 2700 38 | test_batch_size = 256 39 | num_accumulated = 1 40 | lr = 2 41 | warmup_steps = 1 42 | save_dir = 'transformer_en_de_u512' 43 | average_start = 1 44 | num_buckets = 20 45 | log_interval = 10 46 | bleu = '13a' 47 | 48 | #parameters for testing 49 | beam_size = 4 50 | lp_alpha = 0.6 51 | lp_k = 5 52 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/parsing/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/parsing/common/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/parsing/common/savable.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """Utility base class for saving objects.""" 18 | 19 | import pickle 20 | 21 | 22 | class Savable: 23 | """ 24 | A super class for save/load operations. 25 | """ 26 | 27 | def __init__(self): 28 | super().__init__() 29 | 30 | def save(self, path): 31 | """Save to path 32 | 33 | Parameters 34 | ---------- 35 | path : str 36 | file path 37 | """ 38 | with open(path, 'wb') as f: 39 | pickle.dump(self, f) 40 | 41 | @staticmethod 42 | def load(path): 43 | """Load from path 44 | 45 | Parameters 46 | ---------- 47 | path : str 48 | file path 49 | 50 | Returns 51 | ------- 52 | Savable 53 | An object 54 | """ 55 | with open(path, 'rb') as f: 56 | return pickle.load(f) 57 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/parsing/parser/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/parsing/parser/evaluate/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """Official evaluation for parsing.""" 18 | 19 | from .evaluate import evaluate_official_script 20 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/question_answering/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """Question answering example.""" 20 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/question_answering/utils.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Various utility methods for Question Answering""" 19 | import math 20 | 21 | 22 | def warm_up_lr(base_lr, iteration, lr_warmup_steps): 23 | """Returns learning rate based on current iteration. 24 | 25 | This function is used to implement learning rate warm up technique. 26 | 27 | math:: 28 | 29 | lr = min(base_lr, base_lr * (log(iteration) / log(lr_warmup_steps))) 30 | 31 | Parameters 32 | ---------- 33 | base_lr : float 34 | Initial learning rage 35 | iteration : int 36 | Current iteration number 37 | lr_warmup_steps : int 38 | Learning rate warm up steps 39 | 40 | Returns 41 | ------- 42 | learning_rate : float 43 | Learning rate 44 | """ 45 | return min(base_lr, base_lr * (math.log(iteration) / math.log(lr_warmup_steps))) 46 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/sentiment_analysis/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """Sentiment Analysis example.""" 20 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """NLP example tests.""" 20 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/tests/test_sampler.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """Test Sampler.""" 19 | 20 | import mxnet as mx 21 | import numpy as np 22 | from numpy.testing import assert_allclose 23 | from ..language_model.sampler import LogUniformSampler 24 | 25 | 26 | def test_log_uniform_sampler(): 27 | ntokens = 793472 28 | num_sampled = 8192 29 | sampler = LogUniformSampler(ntokens, num_sampled) 30 | true_cls = mx.nd.array([5, 10, 20]) 31 | sample, cnt_sample, cnt_true = sampler(true_cls) 32 | assert np.unique(sample.asnumpy()).size == num_sampled 33 | assert cnt_true.size == true_cls.size 34 | assert cnt_sample.size == cnt_sample.size 35 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/tests/test_sanity.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | 20 | @pytest.mark.gpu 21 | def test_sanity_gpu(): 22 | # sanity test that makes sure every marker combination has at least 1 test. 23 | # due to https://github.com/pytest-dev/pytest/issues/812 24 | import gluonnlp as nlp 25 | 26 | @pytest.mark.gpu 27 | @pytest.mark.serial 28 | def test_sanity_gpu_serial(): 29 | # sanity test that makes sure every marker combination has at least 1 test. 30 | # due to https://github.com/pytest-dev/pytest/issues/812 31 | import gluonnlp as nlp 32 | 33 | @pytest.mark.gpu 34 | @pytest.mark.integration 35 | def test_sanity_gpu_integ(): 36 | # sanity test that makes sure every marker combination has at least 1 test. 37 | # due to https://github.com/pytest-dev/pytest/issues/812 38 | import gluonnlp as nlp 39 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/tests/word_embeddings/glove/cooccurrences.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/scripts/tests/word_embeddings/glove/cooccurrences.npz -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/tests/word_embeddings/glove/vocab.txt: -------------------------------------------------------------------------------- 1 | the 49 2 | of 29 3 | and 22 4 | in 21 5 | as 14 6 | to 14 7 | is 12 8 | that 12 9 | a 9 10 | anarchism 9 11 | anarchist 9 12 | an 7 13 | proudhon 7 14 | society 7 15 | this 7 16 | what 7 17 | anarchists 6 18 | are 6 19 | be 6 20 | it 6 21 | property 6 22 | by 5 23 | he 5 24 | one 5 25 | revolution 5 26 | term 5 27 | use 5 28 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/text_generation/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """Text Generation Module.""" 20 | from . import model 21 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/scripts/word_embeddings/utils.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | """Word Embeddings Training Utilities 18 | ===================================== 19 | 20 | """ 21 | 22 | import logging 23 | import time 24 | from contextlib import contextmanager 25 | 26 | import mxnet as mx 27 | 28 | 29 | def get_context(args): 30 | if args.gpu is None or args.gpu == '': 31 | context = [mx.cpu()] 32 | elif isinstance(args.gpu, int): 33 | context = [mx.gpu(args.gpu)] 34 | else: 35 | context = [mx.gpu(int(i)) for i in args.gpu] 36 | return context 37 | 38 | 39 | @contextmanager 40 | def print_time(task): 41 | start_time = time.time() 42 | logging.info('Starting to %s', task) 43 | yield 44 | logging.info('Finished to {} in {:.2f} seconds'.format( 45 | task, 46 | time.time() - start_time)) 47 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/src/gluonnlp/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """NLP toolkit.""" 20 | 21 | from . import loss 22 | from . import data 23 | from . import embedding 24 | from . import model 25 | from . import metric 26 | from . import utils 27 | from . import vocab 28 | from . import optimizer 29 | from . import initializer 30 | from .vocab import Vocab 31 | 32 | __version__ = '0.8.1.dev' 33 | 34 | __all__ = ['data', 35 | 'model', 36 | 'embedding', 37 | 'Vocab', 38 | 'vocab', 39 | 'loss', 40 | 'initializer', 41 | 'optimizer', 42 | 'utils', 43 | 'metric'] 44 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/src/gluonnlp/data/batchify/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """Batchify helpers.""" 20 | 21 | from . import batchify, language_model 22 | from .batchify import * 23 | from .embedding import * 24 | from .language_model import * 25 | 26 | __all__ = batchify.__all__ + language_model.__all__ + embedding.__all__ 27 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/src/gluonnlp/data/corpora/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """Corpora.""" 20 | 21 | from . import (google_billion_word, large_text_compression_benchmark, wikitext) 22 | 23 | from .google_billion_word import * 24 | from .large_text_compression_benchmark import * 25 | from .wikitext import * 26 | 27 | __all__ = (google_billion_word.__all__ + 28 | large_text_compression_benchmark.__all__ + wikitext.__all__) 29 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/src/gluonnlp/embedding/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """Word embeddings.""" 20 | 21 | from .token_embedding import * 22 | 23 | from . import evaluation 24 | 25 | __all__ = (token_embedding.__all__ + ['evaluation']) 26 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/src/gluonnlp/initializer/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """NLP initializer.""" 20 | 21 | from .initializer import * 22 | 23 | __all__ = initializer.__all__ 24 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/src/gluonnlp/loss/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """NLP loss.""" 20 | 21 | from .activation_regularizer import * 22 | from .loss import * 23 | from .label_smoothing import * 24 | 25 | __all__ = activation_regularizer.__all__ + loss.__all__ + label_smoothing.__all__ 26 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/src/gluonnlp/metric/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """NLP Metrics.""" 20 | 21 | from .masked_accuracy import * 22 | 23 | __all__ = masked_accuracy.__all__ 24 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/src/gluonnlp/optimizer/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """NLP optimizer.""" 20 | 21 | from .bert_adam import * 22 | from .lamb import * 23 | 24 | __all__ = bert_adam.__all__ + lamb.__all__ 25 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/src/gluonnlp/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import, arguments-differ 19 | """Module for utility functions.""" 20 | 21 | from . import (parallel, parameter, files) 22 | 23 | from .parallel import * 24 | from .parameter import * 25 | from .files import * 26 | from .version import * 27 | 28 | __all__ = parallel.__all__ + parameter.__all__ + files.__all__ + version.__all__ 29 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/src/gluonnlp/vocab/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # pylint: disable=wildcard-import 19 | """Vocabulary.""" 20 | 21 | from . import subwords, vocab, bert 22 | from .subwords import * 23 | from .vocab import * 24 | from .elmo import * 25 | from .bert import * 26 | 27 | __all__ = vocab.__all__ + subwords.__all__ + elmo.__all__ + bert.__all__ 28 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/tests/unittest/corpora/test_gbw.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | 19 | import itertools 20 | import json 21 | import os 22 | import pytest 23 | 24 | import gluonnlp as nlp 25 | import mxnet as mx 26 | 27 | 28 | @pytest.mark.remote_required 29 | def test_gbw(): 30 | batch_size = 80 31 | seq_len = 35 32 | 33 | stream = nlp.data.GBWStream(segment='test') 34 | freq = nlp.data.utils.Counter( 35 | itertools.chain.from_iterable(itertools.chain.from_iterable(stream))) 36 | assert len(freq) == 21545 37 | assert sum(c for c in freq.values()) == 159658 38 | assert freq['English'] == 14 39 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/tests/unittest/test_candidate_sampler.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import mxnet as mx 3 | import pytest 4 | 5 | from gluonnlp.data import candidate_sampler as cs 6 | 7 | 8 | @pytest.mark.seed(1) 9 | def test_unigram_candidate_sampler(hybridize): 10 | N = 1000 11 | sampler = cs.UnigramCandidateSampler(mx.nd.arange(N), shape=(3, )) 12 | sampler.initialize() 13 | if hybridize: 14 | sampler.hybridize() 15 | sampled = sampler(mx.nd.ones(3)) 16 | print(sampled.asnumpy()) 17 | assert np.all([729, 594, 690] == sampled.asnumpy()) 18 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/tests/unittest/test_pytest.py: -------------------------------------------------------------------------------- 1 | import random 2 | import pytest 3 | import numpy as np 4 | import mxnet as mx 5 | 6 | 7 | @pytest.mark.seed(1) 8 | def test_test(): 9 | """Test that fixing a random seed works.""" 10 | py_rnd = random.randint(0, 100) 11 | np_rnd = np.random.randint(0, 100) 12 | mx_rnd = mx.nd.random_uniform(shape=(1, )).asscalar() 13 | 14 | random.seed(1) 15 | mx.random.seed(1) 16 | np.random.seed(1) 17 | 18 | assert py_rnd == random.randint(0, 100) 19 | assert np_rnd == np.random.randint(0, 100) 20 | assert mx_rnd == mx.nd.random_uniform(shape=(1, )).asscalar() 21 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/tests/unittest/test_sanity.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import pytest 19 | 20 | @pytest.mark.gpu 21 | def test_sanity_gpu(): 22 | # sanity test that makes sure every marker combination has at least 1 test. 23 | # due to https://github.com/pytest-dev/pytest/issues/812 24 | import gluonnlp as nlp 25 | 26 | @pytest.mark.gpu 27 | @pytest.mark.serial 28 | def test_sanity_gpu_serial(): 29 | # sanity test that makes sure every marker combination has at least 1 test. 30 | # due to https://github.com/pytest-dev/pytest/issues/812 31 | import gluonnlp as nlp 32 | -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/tests/unittest/train/test_embedding/lorem_ipsum.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/tests/unittest/train/test_embedding/lorem_ipsum.bin -------------------------------------------------------------------------------- /mxnet/bert-large/gluon-nlp/tests/unittest/train/test_embedding/lorem_ipsum_w2v.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/bert-large/gluon-nlp/tests/unittest/train/test_embedding/lorem_ipsum_w2v.bin -------------------------------------------------------------------------------- /mxnet/vgg-16/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/vgg-16/common/__init__.py -------------------------------------------------------------------------------- /mxnet/vgg-16/common/find_mxnet.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import os, sys 19 | try: 20 | import mxnet as mx 21 | except ImportError: 22 | curr_path = os.path.abspath(os.path.dirname(__file__)) 23 | sys.path.append(os.path.join(curr_path, "../../../python")) 24 | import mxnet as mx 25 | -------------------------------------------------------------------------------- /mxnet/vgg-16/data/imagenet1k-val.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | 20 | 21 | # This file download the imagnet-1k validation dataset and convert it into a rec 22 | # file. One need to provide the URL for the ILSVRC2012_img_val.tar, which can be 23 | # find at http://www.image-net.org/download-images 24 | # 25 | # Example usage (replace the URL with the correct one): 26 | # ./imagenet1k-val.sh http://xxxxxx/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_val.tar 27 | 28 | if [ ! -e ILSVRC2012_img_val.tar ]; then 29 | wget $1 30 | fi 31 | mkdir -p val 32 | tar -xf ILSVRC2012_img_val.tar -C val 33 | wget http://data.mxnet.io/models/imagenet/resnet/val.lst -O imagenet1k-val.lst 34 | 35 | CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 36 | MX_DIR=${CUR_DIR}/../../../ 37 | 38 | python ${CUR_DIR}/../../../tools/im2rec.py --resize 256 --quality 90 --num-thread 16 imagenet1k-val val/ 39 | 40 | rm -rf val 41 | -------------------------------------------------------------------------------- /mxnet/vgg-16/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export DISTRIBUTED_FRAMEWORK=byteps 4 | 5 | # you can also test other networks (e.g., ResNet) with different number of layers 6 | bpslaunch python3 train_imagenet.py --network vgg --num-layers 16 --batch-size 32 $@ -------------------------------------------------------------------------------- /mxnet/vgg-16/symbols/README.md: -------------------------------------------------------------------------------- 1 | # Symbol 2 | 3 | This fold contains definition of various networks. To add a new network, please 4 | use the following format. 5 | 6 | ## Python 7 | 8 | - A file implements one network proposed in a paper, with the network name as the 9 | filename. 10 | - Mention the paper and the modifications made if any at the beginning 11 | of the file. 12 | - Indicate how to reproduce the accuracy numbers in the paper if it is not straightforward 13 | - Provide a function `get_symbol()` that return the network 14 | -------------------------------------------------------------------------------- /mxnet/vgg-16/symbols/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/mxnet/vgg-16/symbols/__init__.py -------------------------------------------------------------------------------- /mxnet/vgg-16/symbols/mlp.py: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | """ 19 | a simple multilayer perceptron 20 | """ 21 | import mxnet as mx 22 | 23 | def get_symbol(num_classes=10, **kwargs): 24 | data = mx.symbol.Variable('data') 25 | data = mx.sym.Flatten(data=data) 26 | fc1 = mx.symbol.FullyConnected(data = data, name='fc1', num_hidden=128) 27 | act1 = mx.symbol.Activation(data = fc1, name='relu1', act_type="relu") 28 | fc2 = mx.symbol.FullyConnected(data = act1, name = 'fc2', num_hidden = 64) 29 | act2 = mx.symbol.Activation(data = fc2, name='relu2', act_type="relu") 30 | fc3 = mx.symbol.FullyConnected(data = act2, name='fc3', num_hidden=num_classes) 31 | mlp = mx.symbol.SoftmaxOutput(data = fc3, name = 'softmax') 32 | return mlp 33 | -------------------------------------------------------------------------------- /osdi20ae.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/osdi20ae.pdf -------------------------------------------------------------------------------- /pytorch/gpt-2/README.md: -------------------------------------------------------------------------------- 1 | 2 | Dataset link: 3 | 4 | https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/ -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/.circleci/deploy.sh: -------------------------------------------------------------------------------- 1 | cd docs 2 | 3 | function deploy_doc(){ 4 | echo "Creating doc at commit $1 and pushing to folder $2" 5 | git checkout $1 6 | if [ ! -z "$2" ] 7 | then 8 | if [ -d "$dir/$2" ]; then 9 | echo "Directory" $2 "already exists" 10 | else 11 | echo "Pushing version" $2 12 | make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2 13 | fi 14 | else 15 | echo "Pushing master" 16 | make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir 17 | fi 18 | } 19 | 20 | deploy_doc "master" 21 | deploy_doc "b33a385" v1.0.0 22 | deploy_doc "fe02e45" v1.1.0 23 | deploy_doc "89fd345" v1.2.0 24 | deploy_doc "fc9faa8" v2.0.0 25 | deploy_doc "3ddce1d" v2.1.1 26 | deploy_doc "3616209" v2.2.0 27 | deploy_doc "d0f8b9a" v2.3.0 28 | deploy_doc "6664ea9" v2.4.0 -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source=transformers 3 | omit = 4 | # skip convertion scripts from testing for now 5 | */convert_* 6 | */__main__.py 7 | [report] 8 | exclude_lines = 9 | pragma: no cover 10 | raise 11 | except 12 | register_parameter -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/.github/ISSUE_TEMPLATE/---new-benchmark.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F5A5 New benchmark" 3 | about: Benchmark a part of this library and share your results 4 | title: "[Benchmark]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | # 🖥 Benchmarking `transformers` 11 | 12 | ## Benchmark 13 | 14 | Which part of `transformers` did you benchmark? 15 | 16 | ## Set-up 17 | 18 | What did you run your benchmarks on? Please include details, such as: CPU, GPU? If using multiple GPUs, which parallelization did you use? 19 | 20 | ## Results 21 | 22 | Put your results here! 23 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/.github/ISSUE_TEMPLATE/--new-model-addition.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F31F New model addition" 3 | about: Submit a proposal/request to implement a new Transformer-based model 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | # 🌟 New model addition 11 | 12 | ## Model description 13 | 14 | 15 | 16 | ## Open source status 17 | 18 | * [ ] the model implementation is available: (give details) 19 | * [ ] the model weights are available: (give details) 20 | * [ ] who are the authors: (mention them, if possible by @gh-username) 21 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F41B Bug Report" 3 | about: Submit a bug report to help us improve transformers 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | # 🐛 Bug 11 | 12 | ## Information 13 | 14 | Model I am using (Bert, XLNet ...): 15 | 16 | Language I am using the model on (English, Chinese ...): 17 | 18 | The problem arises when using: 19 | * [ ] the official example scripts: (give details below) 20 | * [ ] my own modified scripts: (give details below) 21 | 22 | The tasks I am working on is: 23 | * [ ] an official GLUE/SQUaD task: (give the name) 24 | * [ ] my own task or dataset: (give details below) 25 | 26 | ## To reproduce 27 | 28 | Steps to reproduce the behavior: 29 | 30 | 1. 31 | 2. 32 | 3. 33 | 34 | 37 | 38 | ## Expected behavior 39 | 40 | 41 | 42 | ## Environment 43 | 44 | * OS: 45 | * Python version: 46 | * PyTorch version: 47 | * `transformers` version (or branch): 48 | * Using GPU ? 49 | * Distributed or parallel setup ? 50 | * Any other relevant information: 51 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature request" 3 | about: Submit a proposal/request for a new transformers feature 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | # 🚀 Feature request 11 | 12 | 14 | 15 | ## Motivation 16 | 17 | 20 | 21 | ## Your contribution 22 | 23 | 26 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/.github/ISSUE_TEMPLATE/question-help.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "❓ Questions & Help" 3 | about: Post your general questions on Stack Overflow tagged huggingface-transformers 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | # ❓ Questions & Help 11 | 12 | 23 | 24 | ## Details 25 | 26 | 27 | 29 | **A link to original question on Stack Overflow**: -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 60 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 7 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - pinned 8 | - security 9 | # Label to use when marking an issue as stale 10 | staleLabel: wontfix 11 | # Comment to post when marking an issue as stale. Set to `false` to disable 12 | markComment: > 13 | This issue has been automatically marked as stale because it has not had 14 | recent activity. It will be closed if no further activity occurs. Thank you 15 | for your contributions. 16 | # Comment to post when closing a stale issue. Set to `false` to disable 17 | closeComment: false -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: quality style test test-examples 2 | 3 | # Check that source code meets quality standards 4 | 5 | quality: 6 | black --check --line-length 119 --target-version py35 examples templates tests src utils 7 | isort --check-only --recursive examples templates tests src utils 8 | flake8 examples templates tests src utils 9 | 10 | # Format source code automatically 11 | 12 | style: 13 | black --line-length 119 --target-version py35 examples templates tests src utils 14 | isort --recursive examples templates tests src utils 15 | 16 | # Run tests for the library 17 | 18 | test: 19 | python -m pytest -n auto --dist=loadfile -s -v ./tests/ 20 | 21 | # Run tests for examples 22 | 23 | test-examples: 24 | python -m pytest -n auto --dist=loadfile -s -v ./examples/ 25 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/deploy_multi_version_doc.sh: -------------------------------------------------------------------------------- 1 | cd docs 2 | 3 | function deploy_doc(){ 4 | echo "Creating doc at commit $1 and pushing to folder $2" 5 | git checkout $1 6 | if [ ! -z "$2" ] 7 | then 8 | echo "Pushing version" $2 9 | make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2 10 | else 11 | echo "Pushing master" 12 | make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir 13 | fi 14 | } 15 | 16 | deploy_doc "master" 17 | deploy_doc "b33a385" v1.0.0 18 | deploy_doc "fe02e45" v1.1.0 19 | deploy_doc "89fd345" v1.2.0 20 | deploy_doc "fc9faa8" v2.0.0 21 | deploy_doc "3ddce1d" v2.1.1 22 | deploy_doc "f2f3294" v2.2.0 23 | deploy_doc "d0f8b9a" v2.3.0 24 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pytorch/pytorch:latest 2 | 3 | RUN git clone https://github.com/NVIDIA/apex.git && cd apex && python setup.py install --cuda_ext --cpp_ext 4 | 5 | RUN pip install transformers 6 | 7 | WORKDIR /workspace -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = source 8 | BUILDDIR = _build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/_static/css/Calibre-Light.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/docs/source/_static/css/Calibre-Light.ttf -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/_static/css/Calibre-Medium.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/docs/source/_static/css/Calibre-Medium.otf -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/_static/css/Calibre-Regular.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/docs/source/_static/css/Calibre-Regular.otf -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/_static/css/Calibre-Thin.otf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/docs/source/_static/css/Calibre-Thin.otf -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/_static/css/code-snippets.css: -------------------------------------------------------------------------------- 1 | 2 | .highlight .c1, .highlight .sd{ 3 | color: #999 4 | } 5 | 6 | .highlight .nn, .highlight .k, .highlight .s1, .highlight .nb, .highlight .bp, .highlight .kc { 7 | color: #FB8D68; 8 | } 9 | 10 | .highlight .kn, .highlight .nv, .highlight .s2, .highlight .ow { 11 | color: #6670FF; 12 | } -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/bertology.rst: -------------------------------------------------------------------------------- 1 | BERTology 2 | --------- 3 | 4 | There is a growing field of study concerned with investigating the inner working of large-scale transformers like BERT (that some call "BERTology"). Some good examples of this field are: 5 | 6 | 7 | * BERT Rediscovers the Classical NLP Pipeline by Ian Tenney, Dipanjan Das, Ellie Pavlick: https://arxiv.org/abs/1905.05950 8 | * Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: https://arxiv.org/abs/1905.10650 9 | * What Does BERT Look At? An Analysis of BERT's Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D. Manning: https://arxiv.org/abs/1906.04341 10 | 11 | In order to help this new field develop, we have included a few additional features in the BERT/GPT/GPT-2 models to help people access the inner representations, mainly adapted from the great work of Paul Michel (https://arxiv.org/abs/1905.10650): 12 | 13 | 14 | * accessing all the hidden-states of BERT/GPT/GPT-2, 15 | * accessing all the attention weights for each head of BERT/GPT/GPT-2, 16 | * retrieving heads output values and gradients to be able to compute head importance score and prune head as explained in https://arxiv.org/abs/1905.10650. 17 | 18 | To help you understand and use these features, we have added a specific example script: `bertology.py `_ while extract information and prune a model pre-trained on GLUE. 19 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/imgs/transformers_logo_name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/docs/source/imgs/transformers_logo_name.png -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/imgs/warmup_constant_schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/docs/source/imgs/warmup_constant_schedule.png -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/imgs/warmup_cosine_hard_restarts_schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/docs/source/imgs/warmup_cosine_hard_restarts_schedule.png -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/imgs/warmup_cosine_schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/docs/source/imgs/warmup_cosine_schedule.png -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/imgs/warmup_cosine_warm_restarts_schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/docs/source/imgs/warmup_cosine_warm_restarts_schedule.png -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/imgs/warmup_linear_schedule.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/docs/source/imgs/warmup_linear_schedule.png -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/main_classes/configuration.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ---------------------------------------------------- 3 | 4 | The base class ``PretrainedConfig`` implements the common methods for loading/saving a configuration either from a local file or directory, or from a pretrained model configuration provided by the library (downloaded from HuggingFace's AWS S3 repository). 5 | 6 | ``PretrainedConfig`` 7 | ~~~~~~~~~~~~~~~~~~~~~ 8 | 9 | .. autoclass:: transformers.PretrainedConfig 10 | :members: 11 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/main_classes/model.rst: -------------------------------------------------------------------------------- 1 | Models 2 | ---------------------------------------------------- 3 | 4 | The base class ``PreTrainedModel`` implements the common methods for loading/saving a model either from a local file or directory, or from a pretrained model configuration provided by the library (downloaded from HuggingFace's AWS S3 repository). 5 | 6 | ``PreTrainedModel`` also implements a few methods which are common among all the models to: 7 | 8 | - resize the input token embeddings when new tokens are added to the vocabulary 9 | - prune the attention heads of the model. 10 | 11 | ``PreTrainedModel`` 12 | ~~~~~~~~~~~~~~~~~~~~~ 13 | 14 | .. autoclass:: transformers.PreTrainedModel 15 | :members: 16 | 17 | ``TFPreTrainedModel`` 18 | ~~~~~~~~~~~~~~~~~~~~~ 19 | 20 | .. autoclass:: transformers.TFPreTrainedModel 21 | :members: 22 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/main_classes/tokenizer.rst: -------------------------------------------------------------------------------- 1 | Tokenizer 2 | ---------------------------------------------------- 3 | 4 | The base class ``PreTrainedTokenizer`` implements the common methods for loading/saving a tokenizer either from a local file or directory, or from a pretrained tokenizer provided by the library (downloaded from HuggingFace's AWS S3 repository). 5 | 6 | ``PreTrainedTokenizer`` is the main entry point into tokenizers as it also implements the main methods for using all the tokenizers: 7 | 8 | - tokenizing, converting tokens to ids and back and encoding/decoding, 9 | - adding new tokens to the vocabulary in a way that is independant of the underlying structure (BPE, SentencePiece...), 10 | - managing special tokens (adding them, assigning them to roles, making sure they are not split during tokenization) 11 | 12 | ``PreTrainedTokenizer`` 13 | ~~~~~~~~~~~~~~~~~~~~~~~~ 14 | 15 | .. autoclass:: transformers.PreTrainedTokenizer 16 | :members: 17 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/model_sharing.md: -------------------------------------------------------------------------------- 1 | # Model upload and sharing 2 | 3 | Starting with `v2.2.2`, you can now upload and share your fine-tuned models with the community, using the CLI that's built-in to the library. 4 | 5 | **First, create an account on [https://huggingface.co/join](https://huggingface.co/join)**. Then: 6 | 7 | ```shell 8 | transformers-cli login 9 | # log in using the same credentials as on huggingface.co 10 | ``` 11 | Upload your model: 12 | ```shell 13 | transformers-cli upload ./path/to/pretrained_model/ 14 | 15 | # ^^ Upload folder containing weights/tokenizer/config 16 | # saved via `.save_pretrained()` 17 | 18 | transformers-cli upload ./config.json [--filename folder/foobar.json] 19 | 20 | # ^^ Upload a single file 21 | # (you can optionally override its filename, which can be nested inside a folder) 22 | ``` 23 | 24 | Your model will then be accessible through its identifier, a concatenation of your username and the folder name above: 25 | ```python 26 | "username/pretrained_model" 27 | ``` 28 | 29 | Anyone can load it from code: 30 | ```python 31 | tokenizer = AutoTokenizer.from_pretrained("username/pretrained_model") 32 | model = AutoModel.from_pretrained("username/pretrained_model") 33 | ``` 34 | 35 | Finally, list all your files on S3: 36 | ```shell 37 | transformers-cli s3 ls 38 | # List all your S3 objects. 39 | ``` 40 | 41 | You can also delete files: 42 | 43 | ```shell 44 | transformers-cli s3 rm … 45 | ``` -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/docs/source/notebooks.rst: -------------------------------------------------------------------------------- 1 | Notebooks 2 | ================================================ 3 | 4 | We include `three Jupyter Notebooks `_ that can be used to check that the predictions of the PyTorch model are identical to the predictions of the original TensorFlow model. 5 | 6 | 7 | * 8 | The first NoteBook (\ `Comparing-TF-and-PT-models.ipynb `_\ ) extracts the hidden states of a full sequence on each layers of the TensorFlow and the PyTorch models and computes the standard deviation between them. In the given example, we get a standard deviation of 1.5e-7 to 9e-7 on the various hidden state of the models. 9 | 10 | * 11 | The second NoteBook (\ `Comparing-TF-and-PT-models-SQuAD.ipynb `_\ ) compares the loss computed by the TensorFlow and the PyTorch models for identical initialization of the fine-tuning layer of the ``BertForQuestionAnswering`` and computes the standard deviation between them. In the given example, we get a standard deviation of 2.5e-7 between the models. 12 | 13 | * 14 | The third NoteBook (\ `Comparing-TF-and-PT-models-MLM-NSP.ipynb `_\ ) compares the predictions computed by the TensorFlow and the PyTorch models for masked token language modeling using the pre-trained masked language modeling model. 15 | 16 | Please follow the instructions given in the notebooks to run and modify them. 17 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/contrib/README.md: -------------------------------------------------------------------------------- 1 | # Community contributed examples 2 | 3 | This folder contains examples which are not actively maintained (mostly contributed by the community). 4 | 5 | Using these examples together with a recent version of the library usually requires to make small (sometimes big) adaptations to get the scripts working. 6 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/distillation/requirements.txt: -------------------------------------------------------------------------------- 1 | transformers 2 | 3 | gitpython==3.0.2 4 | tensorboard>=1.14.0 5 | tensorboardX==1.8 6 | psutil==5.6.3 7 | scipy==1.3.1 8 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/distillation/training_configs/distilbert-base-multilingual-cased.json: -------------------------------------------------------------------------------- 1 | { 2 | "activation": "gelu", 3 | "attention_dropout": 0.1, 4 | "dim": 768, 5 | "dropout": 0.1, 6 | "hidden_dim": 3072, 7 | "initializer_range": 0.02, 8 | "max_position_embeddings": 512, 9 | "n_heads": 12, 10 | "n_layers": 6, 11 | "sinusoidal_pos_embds": true, 12 | "tie_weights_": true, 13 | "vocab_size": 119547 14 | } 15 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/distillation/training_configs/distilbert-base-uncased.json: -------------------------------------------------------------------------------- 1 | { 2 | "activation": "gelu", 3 | "attention_dropout": 0.1, 4 | "dim": 768, 5 | "dropout": 0.1, 6 | "hidden_dim": 3072, 7 | "initializer_range": 0.02, 8 | "max_position_embeddings": 512, 9 | "n_heads": 12, 10 | "n_layers": 6, 11 | "sinusoidal_pos_embds": true, 12 | "tie_weights_": true, 13 | "vocab_size": 30522 14 | } 15 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/distillation/training_configs/distilgpt2.json: -------------------------------------------------------------------------------- 1 | { 2 | "initializer_range": 0.02, 3 | "layer_norm_epsilon": 0.00001, 4 | "n_ctx": 1024, 5 | "n_embd": 768, 6 | "n_head": 12, 7 | "n_layer": 6, 8 | "n_positions": 1024, 9 | "vocab_size": 50257 10 | } -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/distillation/training_configs/distilroberta-base.json: -------------------------------------------------------------------------------- 1 | { 2 | "vocab_size": 50265, 3 | "hidden_size": 768, 4 | "num_hidden_layers": 6, 5 | "num_attention_heads": 12, 6 | "intermediate_size": 3072, 7 | "hidden_act": "gelu", 8 | "hidden_dropout_prob": 0.1, 9 | "attention_probs_dropout_prob": 0.1, 10 | "max_position_embeddings": 514, 11 | "type_vocab_size": 1, 12 | "initializer_range": 0.02, 13 | "layer_norm_eps": 0.00001 14 | } -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/pplm/imgs/headfigure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/examples/pplm/imgs/headfigure.png -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/pplm/imgs/wooly.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/examples/pplm/imgs/wooly.png -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/pplm/pplm_classification_head.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class ClassificationHead(torch.nn.Module): 5 | """Classification Head for transformer encoders""" 6 | 7 | def __init__(self, class_size, embed_size): 8 | super().__init__() 9 | self.class_size = class_size 10 | self.embed_size = embed_size 11 | # self.mlp1 = torch.nn.Linear(embed_size, embed_size) 12 | # self.mlp2 = (torch.nn.Linear(embed_size, class_size)) 13 | self.mlp = torch.nn.Linear(embed_size, class_size) 14 | 15 | def forward(self, hidden_state): 16 | # hidden_state = F.relu(self.mlp1(hidden_state)) 17 | # hidden_state = self.mlp2(hidden_state) 18 | logits = self.mlp(hidden_state) 19 | return logits 20 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorboardX 2 | tensorboard 3 | scikit-learn 4 | seqeval 5 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/summarization/requirements.txt: -------------------------------------------------------------------------------- 1 | transformers 2 | 3 | # For ROUGE 4 | nltk 5 | py-rouge 6 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/tests_samples/.gitignore: -------------------------------------------------------------------------------- 1 | *.* 2 | cache* 3 | temp* 4 | !*.tsv 5 | !*.json 6 | !.gitignore -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/tests_samples/MRPC/dev.tsv: -------------------------------------------------------------------------------- 1 | Quality #1 ID #2 ID #1 String #2 String 2 | 1 1355540 1355592 He said the foodservice pie business doesn 't fit the company 's long-term growth strategy . " The foodservice pie business does not fit our long-term growth strategy . 3 | 0 2029631 2029565 Magnarelli said Racicot hated the Iraqi regime and looked forward to using his long years of training in the war . His wife said he was " 100 percent behind George Bush " and looked forward to using his years of training in the war . 4 | 0 487993 487952 The dollar was at 116.92 yen against the yen , flat on the session , and at 1.2891 against the Swiss franc , also flat . The dollar was at 116.78 yen JPY = , virtually flat on the session , and at 1.2871 against the Swiss franc CHF = , down 0.1 percent . 5 | 1 1989515 1989458 The AFL-CIO is waiting until October to decide if it will endorse a candidate . The AFL-CIO announced Wednesday that it will decide in October whether to endorse a candidate before the primaries . 6 | 0 1783137 1782659 No dates have been set for the civil or the criminal trial . No dates have been set for the criminal or civil cases , but Shanley has pleaded not guilty . 7 | 1 3039165 3039036 Wal-Mart said it would check all of its million-plus domestic workers to ensure they were legally employed . It has also said it would review all of its domestic employees more than 1 million to ensure they have legal status . 8 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/examples/tests_samples/MRPC/train.tsv: -------------------------------------------------------------------------------- 1 | Quality #1 ID #2 ID #1 String #2 String 2 | 1 1355540 1355592 He said the foodservice pie business doesn 't fit the company 's long-term growth strategy . " The foodservice pie business does not fit our long-term growth strategy . 3 | 0 2029631 2029565 Magnarelli said Racicot hated the Iraqi regime and looked forward to using his long years of training in the war . His wife said he was " 100 percent behind George Bush " and looked forward to using his years of training in the war . 4 | 0 487993 487952 The dollar was at 116.92 yen against the yen , flat on the session , and at 1.2891 against the Swiss franc , also flat . The dollar was at 116.78 yen JPY = , virtually flat on the session , and at 1.2871 against the Swiss franc CHF = , down 0.1 percent . 5 | 1 1989515 1989458 The AFL-CIO is waiting until October to decide if it will endorse a candidate . The AFL-CIO announced Wednesday that it will decide in October whether to endorse a candidate before the primaries . 6 | 0 1783137 1782659 No dates have been set for the civil or the criminal trial . No dates have been set for the criminal or civil cases , but Shanley has pleaded not guilty . 7 | 1 3039165 3039036 Wal-Mart said it would check all of its million-plus domestic workers to ensure they were legally employed . It has also said it would review all of its domestic employees more than 1 million to ensure they have legal status . 8 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/setup.cfg: -------------------------------------------------------------------------------- 1 | [isort] 2 | ensure_newline_before_comments = True 3 | force_grid_wrap = 0 4 | include_trailing_comma = True 5 | known_first_party = transformers 6 | known_third_party = 7 | absl 8 | fairseq 9 | fastprogress 10 | git 11 | h5py 12 | MeCab 13 | nltk 14 | numpy 15 | packaging 16 | PIL 17 | psutil 18 | seqeval 19 | sklearn 20 | tensorboardX 21 | tensorflow 22 | tensorflow_datasets 23 | torch 24 | torchtext 25 | torchvision 26 | 27 | line_length = 119 28 | lines_after_imports = 2 29 | multi_line_output = 3 30 | use_parentheses = True 31 | 32 | [flake8] 33 | ignore = E203, E501, W503 34 | max-line-length = 119 35 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/src/transformers/commands/__init__.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from argparse import ArgumentParser 3 | 4 | 5 | class BaseTransformersCLICommand(ABC): 6 | @staticmethod 7 | @abstractmethod 8 | def register_subcommand(parser: ArgumentParser): 9 | raise NotImplementedError() 10 | 11 | @abstractmethod 12 | def run(self): 13 | raise NotImplementedError() 14 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/src/transformers/commands/download.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | 3 | from transformers.commands import BaseTransformersCLICommand 4 | 5 | 6 | def download_command_factory(args): 7 | return DownloadCommand(args.model, args.cache_dir, args.force) 8 | 9 | 10 | class DownloadCommand(BaseTransformersCLICommand): 11 | @staticmethod 12 | def register_subcommand(parser: ArgumentParser): 13 | download_parser = parser.add_parser("download") 14 | download_parser.add_argument( 15 | "--cache-dir", type=str, default=None, help="Path to location to store the models" 16 | ) 17 | download_parser.add_argument( 18 | "--force", action="store_true", help="Force the model to be download even if already in cache-dir" 19 | ) 20 | download_parser.add_argument("model", type=str, help="Name of the model to download") 21 | download_parser.set_defaults(func=download_command_factory) 22 | 23 | def __init__(self, model: str, cache: str, force: bool): 24 | self._model = model 25 | self._cache = cache 26 | self._force = force 27 | 28 | def run(self): 29 | from transformers import AutoModel, AutoTokenizer 30 | 31 | AutoModel.from_pretrained(self._model, cache_dir=self._cache, force_download=self._force) 32 | AutoTokenizer.from_pretrained(self._model, cache_dir=self._cache, force_download=self._force) 33 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/src/transformers/configuration_mmbt.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright (c) Facebook, Inc. and its affiliates. 3 | # Copyright (c) HuggingFace Inc. team. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | """ MMBT configuration """ 17 | 18 | 19 | import logging 20 | 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | 25 | class MMBTConfig(object): 26 | """Configuration class to store the configuration of a `MMBT Model`. 27 | 28 | Args: 29 | config (:obj:`~transformers.PreTrainedConfig`): 30 | Config of the underlying Transformer models. Its values are 31 | copied over to use a single config. 32 | num_labels (:obj:`int` or :obj:`None`, optional, defaults to `None`): 33 | Size of final Linear layer for classification. 34 | modal_hidden_size (:obj:`int`, optional, defautls to 2048): 35 | Embedding dimension of the non-text modality encoder. 36 | """ 37 | 38 | def __init__(self, config, num_labels=None, modal_hidden_size=2048): 39 | self.__dict__ = config.__dict__ 40 | self.modal_hidden_size = modal_hidden_size 41 | if num_labels: 42 | self.num_labels = num_labels 43 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/src/transformers/data/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | # There's no way to ignore "F401 '...' imported but unused" warnings in this 3 | # module, but to preserve other warnings. So, don't check this module at all. 4 | 5 | from .metrics import is_sklearn_available 6 | from .processors import ( 7 | DataProcessor, 8 | InputExample, 9 | InputFeatures, 10 | SingleSentenceClassificationProcessor, 11 | SquadExample, 12 | SquadFeatures, 13 | SquadV1Processor, 14 | SquadV2Processor, 15 | glue_convert_examples_to_features, 16 | glue_output_modes, 17 | glue_processors, 18 | glue_tasks_num_labels, 19 | squad_convert_examples_to_features, 20 | xnli_output_modes, 21 | xnli_processors, 22 | xnli_tasks_num_labels, 23 | ) 24 | 25 | 26 | if is_sklearn_available(): 27 | from .metrics import glue_compute_metrics, xnli_compute_metrics 28 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/src/transformers/data/processors/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | # There's no way to ignore "F401 '...' imported but unused" warnings in this 3 | # module, but to preserve other warnings. So, don't check this module at all. 4 | 5 | from .glue import glue_convert_examples_to_features, glue_output_modes, glue_processors, glue_tasks_num_labels 6 | from .squad import SquadExample, SquadFeatures, SquadV1Processor, SquadV2Processor, squad_convert_examples_to_features 7 | from .utils import DataProcessor, InputExample, InputFeatures, SingleSentenceClassificationProcessor 8 | from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels 9 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/templates/adding_a_new_example_script/README.md: -------------------------------------------------------------------------------- 1 | # How to add a new example script in 🤗Transformers 2 | 3 | This folder provide a template for adding a new example script implementing a training or inference task with the models in the 🤗Transformers library. 4 | 5 | Currently only examples for PyTorch are provided which are adaptations of the library's SQuAD examples which implement single-GPU and distributed training with gradient accumulation and mixed-precision (using NVIDIA's apex library) to cover a reasonable range of use cases. 6 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/tests/__init__.py -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/tests/fixtures/dummy-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "model_type": "roberta" 3 | } -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/tests/fixtures/empty.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/tests/fixtures/empty.txt -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/tests/fixtures/input.txt: -------------------------------------------------------------------------------- 1 | Who was Jim Henson ? ||| Jim Henson was a puppeteer 2 | -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/tests/fixtures/test_sentencepiece.model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/gpt-2/gpt-2/tests/fixtures/test_sentencepiece.model -------------------------------------------------------------------------------- /pytorch/gpt-2/gpt-2/transformers-cli: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from argparse import ArgumentParser 3 | 4 | from transformers.commands.download import DownloadCommand 5 | from transformers.commands.run import RunCommand 6 | from transformers.commands.user import UserCommands 7 | from transformers.commands.convert import ConvertCommand 8 | from transformers.commands.serving import ServeCommand 9 | 10 | if __name__ == '__main__': 11 | parser = ArgumentParser('Transformers CLI tool', usage='transformers-cli []') 12 | commands_parser = parser.add_subparsers(help='transformers-cli command helpers') 13 | 14 | # Register commands 15 | ConvertCommand.register_subcommand(commands_parser) 16 | DownloadCommand.register_subcommand(commands_parser) 17 | RunCommand.register_subcommand(commands_parser) 18 | ServeCommand.register_subcommand(commands_parser) 19 | UserCommands.register_subcommand(commands_parser) 20 | 21 | # Let's go 22 | args = parser.parse_args() 23 | 24 | if not hasattr(args, 'func'): 25 | parser.print_help() 26 | exit(1) 27 | 28 | # Run 29 | service = args.func(args) 30 | service.run() 31 | -------------------------------------------------------------------------------- /pytorch/gpt-2/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # before running this, should make sure the data has been downloaded from: 4 | # https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/ 5 | 6 | export ROOT_DIR=${ROOT_DIR:-~/gpt2} 7 | export TRAIN_FILE=${TRAIN_FILE:-$ROOT_DIR/wikitext-2-raw/wiki.train.raw} 8 | export TEST_FILE=${TRAIN_FILE:-$ROOT_DIR/wikitext-2-raw/wiki.test.raw} 9 | export DISTRIBUTED_FRAMEWORK=${DISTRIBUTED_FRAMEWORK:-byteps} 10 | 11 | THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 12 | 13 | cd ~ 14 | cd $THIS_DIR/gpt-2 15 | pip3 install . 16 | pip3 install -r ./examples/requirements.txt 17 | 18 | # for the first run, you can do a dryrun to 19 | # prepare/download necessary configurations and models 20 | # in order to avoid multi-write conflicts 21 | cd $THIS_DIR/gpt-2/examples 22 | python3 -m torch.distributed.launch --nproc_per_node 1 run_lm_finetuning.py \ 23 | --output_dir=output \ 24 | --model_type=gpt2 \ 25 | --model_name_or_path=gpt2 \ 26 | --do_train \ 27 | --save_steps 1000000 \ 28 | --overwrite_output_dir \ 29 | --num_train_epochs 1 \ 30 | --per_gpu_eval_batch_size 4 \ 31 | --train_data_file=$TRAIN_FILE 32 | 33 | # launch real byteps job 34 | bpslaunch python3 run_lm_finetuning_bps.py \ 35 | --train_data_file=$TRAIN_FILE \ 36 | --output_dir=output \ 37 | --model_type=gpt2 \ 38 | --model_name_or_path=gpt2 \ 39 | --do_train \ 40 | --save_steps 1000000 \ 41 | --overwrite_output_dir \ 42 | --num_train_epochs 1000 \ 43 | --per_gpu_eval_batch_size 4 \ 44 | $@ 45 | 46 | 47 | -------------------------------------------------------------------------------- /pytorch/ugatit/README.md: -------------------------------------------------------------------------------- 1 | Dataset link: 2 | 3 | https://drive.google.com/file/d/1xOWj1UVgp6NKMT3HbPhBbtq2A4EDkghF/view -------------------------------------------------------------------------------- /pytorch/ugatit/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # dataset can be downloaded from 4 | # https://drive.google.com/file/d/1xOWj1UVgp6NKMT3HbPhBbtq2A4EDkghF/view 5 | 6 | export DISTRIBUTED_FRAMEWORK=byteps 7 | 8 | cd ./ugatit 9 | bpslaunch python3 main.py $@ -------------------------------------------------------------------------------- /pytorch/ugatit/ugatit/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.sh 3 | dataset/ 4 | results/ 5 | 6 | !build.sh 7 | -------------------------------------------------------------------------------- /pytorch/ugatit/ugatit/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Hyeonwoo Kang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pytorch/ugatit/ugatit/assets/ablation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/ugatit/ugatit/assets/ablation.png -------------------------------------------------------------------------------- /pytorch/ugatit/ugatit/assets/discriminator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/ugatit/ugatit/assets/discriminator.png -------------------------------------------------------------------------------- /pytorch/ugatit/ugatit/assets/generator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/ugatit/ugatit/assets/generator.png -------------------------------------------------------------------------------- /pytorch/ugatit/ugatit/assets/kid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/ugatit/ugatit/assets/kid.png -------------------------------------------------------------------------------- /pytorch/ugatit/ugatit/assets/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/ugatit/ugatit/assets/teaser.png -------------------------------------------------------------------------------- /pytorch/ugatit/ugatit/assets/user_study.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/pytorch/ugatit/ugatit/assets/user_study.png -------------------------------------------------------------------------------- /pytorch/ugatit/ugatit/requirements.txt: -------------------------------------------------------------------------------- 1 | opencv-python==3.3.1.11 2 | torchvision==0.3.0 3 | -------------------------------------------------------------------------------- /tensorflow/resnet-50/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export DISTRIBUTED_FRAMEWORK=byteps 4 | 5 | # by default the network is ResNet50, but you can also test VGG16 6 | bpslaunch python3 benchmark.py $@ -------------------------------------------------------------------------------- /tensorflow/transformer/README.md: -------------------------------------------------------------------------------- 1 | Refer to [here](transformer/r1/transformer/README.md) for downloading the datasets. -------------------------------------------------------------------------------- /tensorflow/transformer/official/.gitignore: -------------------------------------------------------------------------------- 1 | MNIST-data 2 | labels.txt 3 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/Dockerfile.cpu: -------------------------------------------------------------------------------- 1 | # Docker image for running examples in Tensorflow models. 2 | # base_image depends on whether we are running on GPUs or non-GPUs 3 | FROM ubuntu:latest 4 | 5 | RUN apt-get update && apt-get install -y --no-install-recommends \ 6 | ca-certificates \ 7 | build-essential \ 8 | git \ 9 | python \ 10 | python-pip \ 11 | python-setuptools 12 | 13 | RUN pip install tf-nightly 14 | 15 | # Checkout tensorflow/models at HEAD 16 | RUN git clone https://github.com/tensorflow/models.git /tensorflow_models 17 | 18 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/Dockerfile.gpu: -------------------------------------------------------------------------------- 1 | # Docker image for running examples in Tensorflow models. 2 | # base_image depends on whether we are running on GPUs or non-GPUs 3 | FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04 4 | 5 | RUN apt-get update && apt-get install -y --no-install-recommends \ 6 | ca-certificates \ 7 | build-essential \ 8 | git \ 9 | python \ 10 | python-pip \ 11 | python-setuptools 12 | 13 | RUN pip install tf-nightly-gpu 14 | 15 | # Checkout tensorflow/models at HEAD 16 | RUN git clone https://github.com/tensorflow/models.git /tensorflow_models 17 | 18 | 19 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/README-TPU.md: -------------------------------------------------------------------------------- 1 | # Offically Supported TensorFlow 2.1 Models on Cloud TPU 2 | 3 | ## Natural Language Processing 4 | 5 | * [bert](nlp/bert): A powerful pre-trained language representation model: 6 | BERT, which stands for Bidirectional Encoder Representations from 7 | Transformers. 8 | * [transformer](transformer): A transformer model to translate the WMT 9 | English to German dataset. 10 | 11 | ## Computer Vision 12 | 13 | * [mnist](vision/image_classification): A basic model to classify digits 14 | from the MNIST dataset. 15 | * [resnet](vision/image_classification): A deep residual network that can 16 | be used to classify ImageNet's dataset of 1000 classes. 17 | * [retinanet](vision/detection): A fast and powerful object detector. 18 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/benchmark/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/benchmark/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/benchmark/datastore/schema/benchmark_metric.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "description": "The ID of the benchmark run, where this metric should tie to.", 4 | "mode": "REQUIRED", 5 | "name": "run_id", 6 | "type": "STRING" 7 | }, 8 | { 9 | "description": "The name of the metric, which should be descriptive. E.g. training_loss, accuracy.", 10 | "mode": "REQUIRED", 11 | "name": "name", 12 | "type": "STRING" 13 | }, 14 | { 15 | "description": "The unit of the metric. E.g. MB per sec.", 16 | "mode": "NULLABLE", 17 | "name": "unit", 18 | "type": "STRING" 19 | }, 20 | { 21 | "description": "The value of the metric.", 22 | "mode": "NULLABLE", 23 | "name": "value", 24 | "type": "FLOAT" 25 | }, 26 | { 27 | "description": "The timestamp when the metric is recorded.", 28 | "mode": "REQUIRED", 29 | "name": "timestamp", 30 | "type": "TIMESTAMP" 31 | }, 32 | { 33 | "description": "The global step when this metric is recorded.", 34 | "mode": "NULLABLE", 35 | "name": "global_step", 36 | "type": "INTEGER" 37 | }, 38 | { 39 | "description": "Free format metadata for the extra information about the metric.", 40 | "mode": "REPEATED", 41 | "name": "extras", 42 | "type": "RECORD", 43 | "fields": [ 44 | { 45 | "mode": "NULLABLE", 46 | "name": "name", 47 | "type": "STRING" 48 | }, 49 | { 50 | "mode": "NULLABLE", 51 | "name": "value", 52 | "type": "STRING" 53 | } 54 | ] 55 | } 56 | ] 57 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/benchmark/datastore/schema/benchmark_run_status.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "description": "The UUID of the run for the benchmark.", 4 | "mode": "REQUIRED", 5 | "name": "run_id", 6 | "type": "STRING" 7 | }, 8 | { 9 | "description": "The status of the run for the benchmark. Eg, running, failed, success", 10 | "mode": "REQUIRED", 11 | "name": "status", 12 | "type": "STRING" 13 | } 14 | ] -------------------------------------------------------------------------------- /tensorflow/transformer/official/benchmark/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/benchmark/models/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/benchmark/models/trivial_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """A trivial model for Keras.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | from tensorflow.python.keras import backend 22 | from tensorflow.python.keras import layers 23 | from tensorflow.python.keras import models 24 | 25 | 26 | def trivial_model(num_classes): 27 | """Trivial model for ImageNet dataset.""" 28 | 29 | input_shape = (224, 224, 3) 30 | img_input = layers.Input(shape=input_shape) 31 | 32 | x = layers.Lambda(lambda x: backend.reshape(x, [-1, 224 * 224 * 3]), 33 | name='reshape')(img_input) 34 | x = layers.Dense(1, name='fc1')(x) 35 | x = layers.Dense(num_classes, name='fc1000')(x) 36 | x = layers.Activation('softmax', dtype='float32')(x) 37 | 38 | return models.Model(img_input, x, name='trivial') 39 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/modeling/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/modeling/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/modeling/activations/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Activations package definition.""" 16 | from official.modeling.activations.gelu import gelu 17 | from official.modeling.activations.swish import hard_swish 18 | from official.modeling.activations.swish import identity 19 | from official.modeling.activations.swish import simple_swish 20 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/modeling/activations/gelu.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Gaussian error linear unit.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import math 22 | 23 | import tensorflow as tf 24 | 25 | 26 | @tf.keras.utils.register_keras_serializable(package='Text') 27 | def gelu(x): 28 | """Gaussian Error Linear Unit. 29 | 30 | This is a smoother version of the RELU. 31 | Original paper: https://arxiv.org/abs/1606.08415 32 | Args: 33 | x: float Tensor to perform activation. 34 | 35 | Returns: 36 | `x` with the GELU activation applied. 37 | """ 38 | cdf = 0.5 * (1.0 + tf.tanh( 39 | (math.sqrt(2 / math.pi) * (x + 0.044715 * tf.pow(x, 3))))) 40 | return x * cdf 41 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/modeling/activations/gelu_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Tests for the Gaussian error linear unit.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import tensorflow as tf 22 | 23 | from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import 24 | from official.modeling import activations 25 | 26 | 27 | @keras_parameterized.run_all_keras_modes 28 | class GeluTest(keras_parameterized.TestCase): 29 | 30 | def test_gelu(self): 31 | expected_data = [[0.14967535, 0., -0.10032465], 32 | [-0.15880796, -0.04540223, 2.9963627]] 33 | gelu_data = activations.gelu([[.25, 0, -.25], [-1, -2, 3]]) 34 | self.assertAllClose(expected_data, gelu_data) 35 | 36 | 37 | if __name__ == '__main__': 38 | tf.test.main() 39 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/modeling/hyperparams/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/modeling/hyperparams/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/modeling/training/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/modeling/training/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/nlp/README.md: -------------------------------------------------------------------------------- 1 | # TensorFlow Natural Language Processing Models 2 | 3 | tensorflow/models/official/nlp is a library of state-of-the-art models for 4 | Natural Language Processing (NLP). 5 | 6 | The library currently contains TensorFlow 2.x implementations, pre-trained 7 | model weights, usage scripts and conversion utilities for the following models: 8 | 9 | * [Bert](bert) 10 | 11 | * [XLNet](xlnet) 12 | 13 | * GPT2 (comming soon) 14 | 15 | * [Transformer for translation](../transformer) 16 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/nlp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/nlp/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/nlp/bert/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/nlp/modeling/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/nlp/modeling/layers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Layers package definition.""" 16 | from official.nlp.modeling.layers.attention import * # pylint: disable=wildcard-import 17 | from official.nlp.modeling.layers.dense_einsum import DenseEinsum 18 | from official.nlp.modeling.layers.masked_softmax import MaskedSoftmax 19 | from official.nlp.modeling.layers.on_device_embedding import OnDeviceEmbedding 20 | from official.nlp.modeling.layers.position_embedding import PositionEmbedding 21 | from official.nlp.modeling.layers.self_attention_mask import SelfAttentionMask 22 | from official.nlp.modeling.layers.transformer import Transformer 23 | from official.nlp.modeling.layers.transformer_scaffold import TransformerScaffold 24 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/nlp/modeling/losses/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Activations package definition. Subject to change.""" 16 | from official.nlp.modeling.losses.weighted_sparse_categorical_crossentropy import loss as weighted_sparse_categorical_crossentropy_loss 17 | from official.nlp.modeling.losses.weighted_sparse_categorical_crossentropy import per_example_loss as weighted_sparse_categorical_crossentropy_per_example_loss 18 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/nlp/modeling/networks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Networks package definition.""" 16 | from official.nlp.modeling.networks.albert_transformer_encoder import AlbertTransformerEncoder 17 | from official.nlp.modeling.networks.classification import Classification 18 | from official.nlp.modeling.networks.encoder_scaffold import EncoderScaffold 19 | from official.nlp.modeling.networks.masked_lm import MaskedLM 20 | from official.nlp.modeling.networks.span_labeling import SpanLabeling 21 | from official.nlp.modeling.networks.transformer_encoder import TransformerEncoder 22 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/nlp/transformer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/nlp/transformer/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/nlp/xlnet/README.md: -------------------------------------------------------------------------------- 1 | # XLNet: Generalized Autoregressive Pretraining for Language Understanding 2 | 3 | The academic paper which describes XLNet in detail and provides full results on 4 | a number of tasks can be found here: https://arxiv.org/abs/1906.08237. 5 | 6 | **Instructions and user guide will be added soon.** 7 | 8 | XLNet is a generalized autoregressive BERT-like pretraining language model that 9 | enables learning bidirectional contexts by maximizing the expected likelihood 10 | over all permutations of the factorization order. It can learn dependency beyond 11 | a fixed length without disrupting temporal coherence by using segment-level 12 | recurrence mechanism and relative positional encoding scheme introduced in 13 | [Transformer-XL](https://arxiv.org/pdf/1901.02860.pdf). XLNet outperforms BERT 14 | on 20 NLP benchmark tasks and achieves state-of-the-art results on 18 tasks 15 | including question answering, natural language inference, sentiment analysis, 16 | and document ranking. 17 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/nlp/xlnet/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/pip_package/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Sets up TensorFlow Official Models.""" 16 | from setuptools import find_packages 17 | from setuptools import setup 18 | 19 | setup( 20 | name='tf-models-official', 21 | version='0.0.3.dev1', 22 | description='TensorFlow Official Models', 23 | author='Google Inc.', 24 | author_email='no-reply@google.com', 25 | url='https://github.com/tensorflow/models', 26 | license='Apache 2.0', 27 | packages=find_packages(exclude=["research*", "tutorials*", "samples*"]), 28 | exclude_package_data={ 29 | '': [ 30 | '*_test.py', 31 | ], 32 | }, 33 | install_requires=[ 34 | 'six', 35 | ], 36 | extras_require={ 37 | 'tensorflow': ['tensorflow>=2.0.0'], 38 | 'tensorflow_gpu': ['tensorflow-gpu>=2.0.0'], 39 | 'tensorflow-hub': ['tensorflow-hub>=0.6.0'], 40 | }, 41 | python_requires='>=3.6', 42 | ) 43 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/r1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/r1/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/r1/boosted_trees/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/r1/boosted_trees/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/r1/mnist/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/r1/mnist/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/r1/mnist/example3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/r1/mnist/example3.png -------------------------------------------------------------------------------- /tensorflow/transformer/official/r1/mnist/example5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/r1/mnist/example5.png -------------------------------------------------------------------------------- /tensorflow/transformer/official/r1/mnist/examples.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/r1/mnist/examples.npy -------------------------------------------------------------------------------- /tensorflow/transformer/official/r1/resnet/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/r1/resnet/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/r1/transformer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/r1/transformer/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/r1/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/r1/utils/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/r1/utils/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/r1/utils/data/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/r1/wide_deep/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/r1/wide_deep/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/recommendation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/recommendation/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/recommendation/run_tpu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # Example settings: 5 | # export TPU="taylorrobie-tpu-0" 6 | # export BUCKET="gs://taylorrobie-tpu-test-bucket-2" 7 | 8 | # Remove IDE "not assigned" warning highlights. 9 | TPU=${TPU:-""} 10 | BUCKET=${BUCKET:-""} 11 | 12 | if [[ -z ${TPU} ]]; then 13 | echo "Please set 'TPU' to the name of the TPU to be used." 14 | exit 1 15 | fi 16 | 17 | if [[ -z ${BUCKET} ]]; then 18 | echo "Please set 'BUCKET' to the GCS bucket to be used." 19 | exit 1 20 | fi 21 | 22 | ./run.sh 23 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/requirements.txt: -------------------------------------------------------------------------------- 1 | google-api-python-client>=1.6.7 2 | google-cloud-bigquery>=0.31.0 3 | kaggle>=1.3.9 4 | mlperf_compliance==0.0.10 5 | numpy>=1.15.4 6 | oauth2client>=4.1.2 7 | pandas>=0.22.0 8 | psutil>=5.4.3 9 | py-cpuinfo>=3.3.0 10 | scipy>=0.19.1 11 | tensorflow-hub>=0.6.0 12 | tensorflow-model-optimization>=0.2.1 13 | tensorflow_datasets 14 | typing 15 | sentencepiece 16 | Cython 17 | matplotlib 18 | opencv-python-headless 19 | pyyaml 20 | Pillow 21 | -e git+https://github.com/cocodataset/cocoapi#egg=pycocotools&subdirectory=PythonAPI 22 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/staging/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/staging/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/staging/shakespeare/README.md: -------------------------------------------------------------------------------- 1 | # Shakespeare character LSTM model 2 | 3 | This is an implemention of a simple character LSTM used to generate text. 4 | 5 | ## Instructions 6 | 7 | First download the source data: 8 | 9 | ``` 10 | wget https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt 11 | ``` 12 | 13 | Note that files other than shakepeare.txt can also be used to train the model to generater other text. 14 | 15 | Then train the model: 16 | 17 | ```python 18 | python3 shakespeare_main.py --training_data shakespeare.txt \ 19 | --model_dir /tmp/shakespeare 20 | ``` 21 | 22 | This will place model checkpoints in `/tmp/shakespeare`, so that we can use them to make predictions. 23 | 24 | Then generate predictions: 25 | 26 | ```python 27 | python3 shakespeare_main.py --training_data shakespeare.txt \ 28 | --model_dir /tmp/shakespeare --notrain --predict_context=ROMEO: 29 | ``` 30 | 31 | Change `--predict_context` and `--predict_length` to suit your needs. 32 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/staging/shakespeare/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/transformer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/transformer/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/transformer/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/transformer/utils/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/transformer/v2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/transformer/v2/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/flags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/flags/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/logs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/logs/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/logs/cloud_lib.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Utilities that interact with cloud service. 17 | """ 18 | 19 | import requests 20 | 21 | GCP_METADATA_URL = "http://metadata/computeMetadata/v1/instance/hostname" 22 | GCP_METADATA_HEADER = {"Metadata-Flavor": "Google"} 23 | 24 | 25 | def on_gcp(): 26 | """Detect whether the current running environment is on GCP.""" 27 | try: 28 | # Timeout in 5 seconds, in case the test environment has connectivity issue. 29 | # There is not default timeout, which means it might block forever. 30 | response = requests.get( 31 | GCP_METADATA_URL, headers=GCP_METADATA_HEADER, timeout=5) 32 | return response.status_code == 200 33 | except requests.exceptions.RequestException: 34 | return False 35 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/logs/cloud_lib_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Tests for cloud_lib.""" 17 | 18 | from __future__ import absolute_import 19 | from __future__ import division 20 | from __future__ import print_function 21 | 22 | import unittest 23 | 24 | import mock 25 | import requests 26 | 27 | from official.utils.logs import cloud_lib 28 | 29 | 30 | class CloudLibTest(unittest.TestCase): 31 | 32 | @mock.patch("requests.get") 33 | def test_on_gcp(self, mock_requests_get): 34 | mock_response = mock.MagicMock() 35 | mock_requests_get.return_value = mock_response 36 | mock_response.status_code = 200 37 | 38 | self.assertEqual(cloud_lib.on_gcp(), True) 39 | 40 | @mock.patch("requests.get") 41 | def test_not_on_gcp(self, mock_requests_get): 42 | mock_requests_get.side_effect = requests.exceptions.ConnectionError() 43 | 44 | self.assertEqual(cloud_lib.on_gcp(), False) 45 | 46 | 47 | if __name__ == "__main__": 48 | unittest.main() 49 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/misc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/misc/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/misc/tpu_lib.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Initializes TPU system for TF 2.0.""" 16 | 17 | import tensorflow as tf 18 | 19 | 20 | def tpu_initialize(tpu_address): 21 | """Initializes TPU for TF 2.0 training. 22 | 23 | Args: 24 | tpu_address: string, bns address of master TPU worker. 25 | 26 | Returns: 27 | A TPUClusterResolver. 28 | """ 29 | cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( 30 | tpu=tpu_address) 31 | if tpu_address not in ('', 'local'): 32 | tf.config.experimental_connect_to_cluster(cluster_resolver) 33 | return cluster_resolver 34 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/mock_lib.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Mock objects and related functions for testing.""" 17 | 18 | from __future__ import absolute_import 19 | from __future__ import division 20 | from __future__ import print_function 21 | 22 | 23 | class MockBenchmarkLogger(object): 24 | """This is a mock logger that can be used in dependent tests.""" 25 | 26 | def __init__(self): 27 | self.logged_metric = [] 28 | 29 | def log_metric(self, name, value, unit=None, global_step=None, 30 | extras=None): 31 | self.logged_metric.append({ 32 | "name": name, 33 | "value": float(value), 34 | "unit": unit, 35 | "global_step": global_step, 36 | "extras": extras}) 37 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/dense/expected_graph: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/dense/expected_graph -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/dense/model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/dense/model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/dense/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/dense/model.ckpt.index -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/dense/results.json: -------------------------------------------------------------------------------- 1 | [1, 1, 0.4701630473136902, 0.4701630473136902, 0.4701630473136902] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/dense/tf_version.json: -------------------------------------------------------------------------------- 1 | ["1.8.0-dev20180325", "v1.7.0-rc1-750-g6c1737e6c8"] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/uniform_random/expected_graph: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/uniform_random/expected_graph -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/uniform_random/model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- 1 | ʼ|? -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/uniform_random/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/uniform_random/model.ckpt.index -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/uniform_random/results.json: -------------------------------------------------------------------------------- 1 | [0.9872556924819946] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/reference_data_test/uniform_random/tf_version.json: -------------------------------------------------------------------------------- 1 | ["1.8.0-dev20180325", "v1.7.0-rc1-750-g6c1737e6c8"] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-1_width-8_channels-4/expected_graph: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-1_width-8_channels-4/expected_graph -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-1_width-8_channels-4/model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-1_width-8_channels-4/model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-1_width-8_channels-4/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-1_width-8_channels-4/model.ckpt.index -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-1_width-8_channels-4/results.json: -------------------------------------------------------------------------------- 1 | [32, 8, 8, 4, 0.08920872211456299, 0.8918969631195068, 4064.7060546875, 32, 4, 4, 8, 0.0, 0.10715862363576889, 2344.4775390625] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-1_width-8_channels-4/tf_version.json: -------------------------------------------------------------------------------- 1 | ["1.8.0-dev20180408", "v1.7.0-1345-gb874783ccd"] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-2_width-8_channels-4/expected_graph: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-2_width-8_channels-4/expected_graph -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-2_width-8_channels-4/model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-2_width-8_channels-4/model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-2_width-8_channels-4/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-2_width-8_channels-4/model.ckpt.index -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-2_width-8_channels-4/results.json: -------------------------------------------------------------------------------- 1 | [32, 8, 8, 4, 0.918815016746521, 0.1826801300048828, 4064.4677734375, 32, 4, 4, 8, -1.3153012990951538, 0.011247094720602036, 261.84716796875] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_projection_version-2_width-8_channels-4/tf_version.json: -------------------------------------------------------------------------------- 1 | ["1.8.0-dev20180408", "v1.7.0-1345-gb874783ccd"] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-1_width-8_channels-4/expected_graph: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-1_width-8_channels-4/expected_graph -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-1_width-8_channels-4/model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-1_width-8_channels-4/model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-1_width-8_channels-4/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-1_width-8_channels-4/model.ckpt.index -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-1_width-8_channels-4/results.json: -------------------------------------------------------------------------------- 1 | [32, 8, 8, 4, 0.1677999496459961, 0.7767924070358276, 4089.44189453125, 32, 8, 8, 4, 0.8615571856498718, 1.1359407901763916, 5806.876953125] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-1_width-8_channels-4/tf_version.json: -------------------------------------------------------------------------------- 1 | ["1.8.0-dev20180408", "v1.7.0-1345-gb874783ccd"] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-2_width-8_channels-4/expected_graph: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-2_width-8_channels-4/expected_graph -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-2_width-8_channels-4/model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-2_width-8_channels-4/model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-2_width-8_channels-4/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-2_width-8_channels-4/model.ckpt.index -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-2_width-8_channels-4/results.json: -------------------------------------------------------------------------------- 1 | [32, 8, 8, 4, 0.8239736557006836, 0.3485994338989258, 4108.87548828125, 32, 8, 8, 4, 0.16798323392868042, -0.2975311279296875, 2860.068359375] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_bottleneck_version-2_width-8_channels-4/tf_version.json: -------------------------------------------------------------------------------- 1 | ["1.8.0-dev20180408", "v1.7.0-1345-gb874783ccd"] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-1_width-8_channels-4/expected_graph: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-1_width-8_channels-4/expected_graph -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-1_width-8_channels-4/model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-1_width-8_channels-4/model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-1_width-8_channels-4/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-1_width-8_channels-4/model.ckpt.index -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-1_width-8_channels-4/results.json: -------------------------------------------------------------------------------- 1 | [32, 8, 8, 4, 0.5349493026733398, 0.5126370191574097, 4070.01220703125, 32, 4, 4, 8, 0.0, 2.7680201530456543, 2341.23486328125] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-1_width-8_channels-4/tf_version.json: -------------------------------------------------------------------------------- 1 | ["1.8.0-dev20180408", "v1.7.0-1345-gb874783ccd"] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/expected_graph: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/expected_graph -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/model.ckpt.index -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/results.json: -------------------------------------------------------------------------------- 1 | [32, 8, 8, 4, 0.7820245027542114, 0.8173515796661377, 4095.256591796875, 32, 4, 4, 8, 0.0679062008857727, 0.009305447340011597, -137.36178588867188] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_projection_version-2_width-8_channels-4/tf_version.json: -------------------------------------------------------------------------------- 1 | ["1.8.0-dev20180408", "v1.7.0-1345-gb874783ccd"] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/expected_graph: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/expected_graph -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/model.ckpt.index -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/results.json: -------------------------------------------------------------------------------- 1 | [32, 8, 8, 4, 0.23128163814544678, 0.22117376327514648, 4100.51806640625, 32, 8, 8, 4, 1.1768392324447632, 0.2728465795516968, 5832.6416015625] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-1_width-8_channels-4/tf_version.json: -------------------------------------------------------------------------------- 1 | ["1.8.0-dev20180408", "v1.7.0-1345-gb874783ccd"] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/expected_graph: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/expected_graph -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/model.ckpt.index -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/results.json: -------------------------------------------------------------------------------- 1 | [32, 8, 8, 4, 0.7616699934005737, 0.5485763549804688, 4106.8720703125, 32, 8, 8, 4, -0.056346118450164795, 0.5792689919471741, 2972.37255859375] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch-size-32_building_version-2_width-8_channels-4/tf_version.json: -------------------------------------------------------------------------------- 1 | ["1.8.0-dev20180408", "v1.7.0-1345-gb874783ccd"] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch_norm/expected_graph: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch_norm/expected_graph -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch_norm/model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch_norm/model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch_norm/model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/utils/testing/reference_data/resnet/batch_norm/model.ckpt.index -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch_norm/results.json: -------------------------------------------------------------------------------- 1 | [32, 16, 16, 3, 0.9722558259963989, 0.18413543701171875, 12374.20703125, 32, 16, 16, 3, 1.6126631498336792, -1.096894383430481, -0.041595458984375] -------------------------------------------------------------------------------- /tensorflow/transformer/official/utils/testing/reference_data/resnet/batch_norm/tf_version.json: -------------------------------------------------------------------------------- 1 | ["1.8.0-dev20180408", "v1.7.0-1345-gb874783ccd"] -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/vision/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/vision/detection/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/configs/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/configs/factory.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Factory to provide model configs.""" 16 | 17 | from official.vision.detection.configs import maskrcnn_config 18 | from official.vision.detection.configs import retinanet_config 19 | from official.modeling.hyperparams import params_dict 20 | 21 | 22 | def config_generator(model): 23 | """Model function generator.""" 24 | if model == 'retinanet': 25 | default_config = retinanet_config.RETINANET_CFG 26 | restrictions = retinanet_config.RETINANET_RESTRICTIONS 27 | elif model == 'mask_rcnn': 28 | default_config = maskrcnn_config.MASKRCNN_CFG 29 | restrictions = maskrcnn_config.MASKRCNN_RESTRICTIONS 30 | else: 31 | raise ValueError('Model %s is not supported.' % model) 32 | 33 | return params_dict.ParamsDict(default_config, restrictions) 34 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/dataloader/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/dataloader/mode_keys.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Standard names for input dataloader modes. 16 | 17 | The following standard keys are defined: 18 | 19 | * `TRAIN`: training mode. 20 | * `EVAL`: evaluation mode. 21 | * `PREDICT`: prediction mode. 22 | * `PREDICT_WITH_GT`: prediction mode with groundtruths in returned variables. 23 | """ 24 | 25 | from __future__ import absolute_import 26 | from __future__ import division 27 | from __future__ import print_function 28 | 29 | 30 | TRAIN = 'train' 31 | EVAL = 'eval' 32 | PREDICT = 'predict' 33 | PREDICT_WITH_GT = 'predict_with_gt' 34 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/evaluation/factory.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Evaluator factory.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | from official.vision.detection.evaluation import coco_evaluator 22 | 23 | 24 | def evaluator_generator(params): 25 | """Generator function for various evaluators.""" 26 | if params.type == 'box': 27 | evaluator = coco_evaluator.COCOEvaluator( 28 | annotation_file=params.val_json_file, include_mask=False) 29 | elif params.type == 'box_and_mask': 30 | evaluator = coco_evaluator.COCOEvaluator( 31 | annotation_file=params.val_json_file, include_mask=True) 32 | else: 33 | raise ValueError('Evaluator %s is not supported.' % params.type) 34 | 35 | return coco_evaluator.MetricWrapper(evaluator) 36 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/executor/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/modeling/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/modeling/architecture/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/modeling/architecture/identity.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Identity Fn that forwards the input features.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | 22 | class Identity(object): 23 | """Identity function that forwards the input features.""" 24 | 25 | def __call__(self, features, is_training=False): 26 | """Only forwards the input features.""" 27 | return features 28 | 29 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/modeling/factory.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Factory to build detection model.""" 16 | 17 | 18 | from official.vision.detection.modeling import retinanet_model 19 | 20 | 21 | def model_generator(params): 22 | """Model function generator.""" 23 | if params.type == 'retinanet': 24 | model_fn = retinanet_model.RetinanetModel(params) 25 | else: 26 | raise ValueError('Model %s is not supported.'% params.type) 27 | 28 | return model_fn 29 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/utils/autoaugment_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """AutoAugment util file.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import tensorflow.compat.v2 as tf 22 | 23 | 24 | def distort_image_with_autoaugment(image, bboxes, augmentation_name): 25 | raise NotImplementedError("Not TF 2.0 ready.") 26 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/detection/utils/object_detection/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | -------------------------------------------------------------------------------- /tensorflow/transformer/official/vision/image_classification/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/byteps/examples/d90e1e62e8f3f12405d7092abbcb9f8b97d4e0a9/tensorflow/transformer/official/vision/image_classification/__init__.py -------------------------------------------------------------------------------- /tensorflow/transformer/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # if you use horovod, you should replace bpslaunch with mpi-based command 4 | bpslaunch python3 ./official/r1/transformer/transformer_main.py --ds=byteps $@ 5 | --------------------------------------------------------------------------------