├── .gitignore ├── LICENSE ├── README.md ├── assets ├── attn.png ├── bert-time.png ├── bigbird.key ├── bigbird_slides.key ├── block-sparse-attn.gif ├── eval-pegasus │ ├── Screenshot 2021-05-02 at 1.03.51 AM.png │ ├── Screenshot 2021-05-02 at 11.05.37 AM.png │ ├── Screenshot 2021-05-02 at 11.05.48 AM.png │ └── Screenshot 2021-05-02 at 11.05.57 AM.png ├── full.png ├── global.png ├── graph.gif ├── infer-bigbird-nq.png ├── intro.png ├── longformer-time.png ├── merged#1-bigbird-time.png ├── q1.png ├── q2.png ├── q_middle.png ├── qlast.png ├── qlast_sec.png ├── random.png └── sliding.png ├── notebooks ├── bigbird-inference.ipynb ├── bigbird_pegasus_evaluation.ipynb ├── evaluate-flax-natural-questions.ipynb └── evaluate-torch-natural-questions.ipynb ├── requirements.txt └── src ├── bigbird_flax.py ├── prepare_natural_questions.py ├── sweep_flax.yaml ├── train_nq_flax.py └── train_nq_torch.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/README.md -------------------------------------------------------------------------------- /assets/attn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/attn.png -------------------------------------------------------------------------------- /assets/bert-time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/bert-time.png -------------------------------------------------------------------------------- /assets/bigbird.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/bigbird.key -------------------------------------------------------------------------------- /assets/bigbird_slides.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/bigbird_slides.key -------------------------------------------------------------------------------- /assets/block-sparse-attn.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/block-sparse-attn.gif -------------------------------------------------------------------------------- /assets/eval-pegasus/Screenshot 2021-05-02 at 1.03.51 AM.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/eval-pegasus/Screenshot 2021-05-02 at 1.03.51 AM.png -------------------------------------------------------------------------------- /assets/eval-pegasus/Screenshot 2021-05-02 at 11.05.37 AM.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/eval-pegasus/Screenshot 2021-05-02 at 11.05.37 AM.png -------------------------------------------------------------------------------- /assets/eval-pegasus/Screenshot 2021-05-02 at 11.05.48 AM.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/eval-pegasus/Screenshot 2021-05-02 at 11.05.48 AM.png -------------------------------------------------------------------------------- /assets/eval-pegasus/Screenshot 2021-05-02 at 11.05.57 AM.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/eval-pegasus/Screenshot 2021-05-02 at 11.05.57 AM.png -------------------------------------------------------------------------------- /assets/full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/full.png -------------------------------------------------------------------------------- /assets/global.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/global.png -------------------------------------------------------------------------------- /assets/graph.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/graph.gif -------------------------------------------------------------------------------- /assets/infer-bigbird-nq.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/infer-bigbird-nq.png -------------------------------------------------------------------------------- /assets/intro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/intro.png -------------------------------------------------------------------------------- /assets/longformer-time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/longformer-time.png -------------------------------------------------------------------------------- /assets/merged#1-bigbird-time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/merged#1-bigbird-time.png -------------------------------------------------------------------------------- /assets/q1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/q1.png -------------------------------------------------------------------------------- /assets/q2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/q2.png -------------------------------------------------------------------------------- /assets/q_middle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/q_middle.png -------------------------------------------------------------------------------- /assets/qlast.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/qlast.png -------------------------------------------------------------------------------- /assets/qlast_sec.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/qlast_sec.png -------------------------------------------------------------------------------- /assets/random.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/random.png -------------------------------------------------------------------------------- /assets/sliding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/assets/sliding.png -------------------------------------------------------------------------------- /notebooks/bigbird-inference.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/notebooks/bigbird-inference.ipynb -------------------------------------------------------------------------------- /notebooks/bigbird_pegasus_evaluation.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/notebooks/bigbird_pegasus_evaluation.ipynb -------------------------------------------------------------------------------- /notebooks/evaluate-flax-natural-questions.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/notebooks/evaluate-flax-natural-questions.ipynb -------------------------------------------------------------------------------- /notebooks/evaluate-torch-natural-questions.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/notebooks/evaluate-torch-natural-questions.ipynb -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/requirements.txt -------------------------------------------------------------------------------- /src/bigbird_flax.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/src/bigbird_flax.py -------------------------------------------------------------------------------- /src/prepare_natural_questions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/src/prepare_natural_questions.py -------------------------------------------------------------------------------- /src/sweep_flax.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/src/sweep_flax.yaml -------------------------------------------------------------------------------- /src/train_nq_flax.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/src/train_nq_flax.py -------------------------------------------------------------------------------- /src/train_nq_torch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thevasudevgupta/bigbird/HEAD/src/train_nq_torch.py --------------------------------------------------------------------------------