├── README.md ├── efficient-attention ├── LICENSE ├── README.md ├── efficient_attention │ ├── __init__.py │ ├── modules │ │ ├── abc.py │ │ ├── abstract_attention.py │ │ ├── cosformer.py │ │ ├── flash_attn.py │ │ ├── lara.py │ │ ├── local_attention.py │ │ ├── multihead_attention.py │ │ ├── nystrom_attention.py │ │ ├── performer.py │ │ ├── probsparse.py │ │ ├── s4d.py │ │ └── transformer_ls.py │ └── plugins │ │ ├── efficient_transformers │ │ ├── __init__.py │ │ ├── efficient_transformer_decoder.py │ │ ├── efficient_transformer_decoder_layer.py │ │ ├── efficient_transformer_encoder.py │ │ └── efficient_transformer_encoder_layer.py │ │ └── fairseq_decorator.py ├── requirements.txt └── setup.py ├── imgs ├── attention_taxonomy.png ├── benefit_of_attention.png ├── efficiency.png ├── interpolation_extrapolation.png ├── logo.jpg ├── pattern_correlation.png ├── task.png └── task_correlation.png ├── scripts ├── README.md ├── attention_pattern │ └── test_available_pattern_for_attention.py ├── compositional_index │ ├── compositional_index.py │ └── example.txt ├── efficiency_length │ ├── cost.xlsx │ ├── efficiency_length.py │ ├── efficiency_test.py │ └── get_efficiency_length.sh └── requirements.txt └── tasks ├── LM ├── README.md └── preprocess.py ├── LSTF ├── README.md ├── get_results.py ├── requirements.txt ├── results.xlsx └── scripts │ ├── ECL.sh │ ├── ETTh1.sh │ ├── ETTh2.sh │ ├── ETTm1.sh │ └── WTH.sh ├── MLM ├── README.md └── mlm.yaml ├── PCC └── README.md ├── SR ├── README.md ├── checkpoint_average.py ├── requirement.txt └── sr_sr3_16_128.json ├── Sum ├── README.md ├── eval.yaml └── train.yaml └── TTS └── README.md /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/README.md -------------------------------------------------------------------------------- /efficient-attention/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/LICENSE -------------------------------------------------------------------------------- /efficient-attention/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/README.md -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/__init__.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/abc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/abc.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/abstract_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/abstract_attention.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/cosformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/cosformer.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/flash_attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/flash_attn.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/lara.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/lara.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/local_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/local_attention.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/multihead_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/multihead_attention.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/nystrom_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/nystrom_attention.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/performer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/performer.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/probsparse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/probsparse.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/s4d.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/s4d.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/modules/transformer_ls.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/modules/transformer_ls.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/plugins/efficient_transformers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/plugins/efficient_transformers/__init__.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/plugins/efficient_transformers/efficient_transformer_decoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/plugins/efficient_transformers/efficient_transformer_decoder.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/plugins/efficient_transformers/efficient_transformer_decoder_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/plugins/efficient_transformers/efficient_transformer_decoder_layer.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/plugins/efficient_transformers/efficient_transformer_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/plugins/efficient_transformers/efficient_transformer_encoder.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/plugins/efficient_transformers/efficient_transformer_encoder_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/plugins/efficient_transformers/efficient_transformer_encoder_layer.py -------------------------------------------------------------------------------- /efficient-attention/efficient_attention/plugins/fairseq_decorator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/efficient_attention/plugins/fairseq_decorator.py -------------------------------------------------------------------------------- /efficient-attention/requirements.txt: -------------------------------------------------------------------------------- 1 | setuptools~=58.0.4 2 | einops 3 | ipython 4 | torch>=1.9.0 5 | timm 6 | scipy 7 | opt_einsum -------------------------------------------------------------------------------- /efficient-attention/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/efficient-attention/setup.py -------------------------------------------------------------------------------- /imgs/attention_taxonomy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/imgs/attention_taxonomy.png -------------------------------------------------------------------------------- /imgs/benefit_of_attention.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/imgs/benefit_of_attention.png -------------------------------------------------------------------------------- /imgs/efficiency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/imgs/efficiency.png -------------------------------------------------------------------------------- /imgs/interpolation_extrapolation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/imgs/interpolation_extrapolation.png -------------------------------------------------------------------------------- /imgs/logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/imgs/logo.jpg -------------------------------------------------------------------------------- /imgs/pattern_correlation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/imgs/pattern_correlation.png -------------------------------------------------------------------------------- /imgs/task.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/imgs/task.png -------------------------------------------------------------------------------- /imgs/task_correlation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/imgs/task_correlation.png -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/scripts/README.md -------------------------------------------------------------------------------- /scripts/attention_pattern/test_available_pattern_for_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/scripts/attention_pattern/test_available_pattern_for_attention.py -------------------------------------------------------------------------------- /scripts/compositional_index/compositional_index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/scripts/compositional_index/compositional_index.py -------------------------------------------------------------------------------- /scripts/compositional_index/example.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/scripts/compositional_index/example.txt -------------------------------------------------------------------------------- /scripts/efficiency_length/cost.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/scripts/efficiency_length/cost.xlsx -------------------------------------------------------------------------------- /scripts/efficiency_length/efficiency_length.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/scripts/efficiency_length/efficiency_length.py -------------------------------------------------------------------------------- /scripts/efficiency_length/efficiency_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/scripts/efficiency_length/efficiency_test.py -------------------------------------------------------------------------------- /scripts/efficiency_length/get_efficiency_length.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/scripts/efficiency_length/get_efficiency_length.sh -------------------------------------------------------------------------------- /scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | xlrd==1.2.0 2 | openpyxl 3 | -------------------------------------------------------------------------------- /tasks/LM/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/LM/README.md -------------------------------------------------------------------------------- /tasks/LM/preprocess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/LM/preprocess.py -------------------------------------------------------------------------------- /tasks/LSTF/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/LSTF/README.md -------------------------------------------------------------------------------- /tasks/LSTF/get_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/LSTF/get_results.py -------------------------------------------------------------------------------- /tasks/LSTF/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/LSTF/requirements.txt -------------------------------------------------------------------------------- /tasks/LSTF/results.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/LSTF/results.xlsx -------------------------------------------------------------------------------- /tasks/LSTF/scripts/ECL.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/LSTF/scripts/ECL.sh -------------------------------------------------------------------------------- /tasks/LSTF/scripts/ETTh1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/LSTF/scripts/ETTh1.sh -------------------------------------------------------------------------------- /tasks/LSTF/scripts/ETTh2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/LSTF/scripts/ETTh2.sh -------------------------------------------------------------------------------- /tasks/LSTF/scripts/ETTm1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/LSTF/scripts/ETTm1.sh -------------------------------------------------------------------------------- /tasks/LSTF/scripts/WTH.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/LSTF/scripts/WTH.sh -------------------------------------------------------------------------------- /tasks/MLM/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/MLM/README.md -------------------------------------------------------------------------------- /tasks/MLM/mlm.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/MLM/mlm.yaml -------------------------------------------------------------------------------- /tasks/PCC/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/PCC/README.md -------------------------------------------------------------------------------- /tasks/SR/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/SR/README.md -------------------------------------------------------------------------------- /tasks/SR/checkpoint_average.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/SR/checkpoint_average.py -------------------------------------------------------------------------------- /tasks/SR/requirement.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/SR/requirement.txt -------------------------------------------------------------------------------- /tasks/SR/sr_sr3_16_128.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/SR/sr_sr3_16_128.json -------------------------------------------------------------------------------- /tasks/Sum/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/Sum/README.md -------------------------------------------------------------------------------- /tasks/Sum/eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/Sum/eval.yaml -------------------------------------------------------------------------------- /tasks/Sum/train.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/Sum/train.yaml -------------------------------------------------------------------------------- /tasks/TTS/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Shark-NLP/CAB/HEAD/tasks/TTS/README.md --------------------------------------------------------------------------------