├── .Rhistory ├── .gitignore ├── README.md ├── lib ├── __init__.py ├── data.py ├── layers │ ├── __init__.py │ ├── attn.py │ ├── attn_lrp.py │ ├── basic.py │ ├── concrete_gate.py │ └── lrp.py ├── meta.py ├── ops │ ├── __init__.py │ ├── basic.py │ ├── devices.py │ ├── mpi │ │ ├── __init__.py │ │ ├── dummy_provider.py │ │ └── horovod_provider.py │ ├── record_activations.py │ └── sliced_argmax.py ├── session.py ├── task │ ├── __init__.py │ └── seq2seq │ │ ├── __init__.py │ │ ├── bleu.py │ │ ├── data.py │ │ ├── inference.py │ │ ├── models │ │ ├── __init__.py │ │ ├── transformer.py │ │ ├── transformer_head_gates.py │ │ └── transformer_lrp.py │ │ ├── problems │ │ ├── __init__.py │ │ ├── concrete.py │ │ ├── default.py │ │ └── mrt.py │ │ ├── strutils.py │ │ ├── summary.py │ │ ├── tickers.py │ │ └── voc.py ├── tools │ ├── __init__.py │ ├── apply_bpe.py │ └── average_npz.py ├── train │ ├── __init__.py │ ├── algorithms.py │ ├── optimizers.py │ ├── problem.py │ ├── saveload.py │ └── tickers.py └── util.py ├── notebooks ├── 1_Load_model_and_translate.ipynb ├── 2_Look_at_attention_maps.ipynb └── 3_Look_which_heads_are_dead.ipynb ├── requirements.txt ├── resources ├── acl19_heads-min_pad.png ├── acl_empty.png ├── concrete.gif ├── concrete_crop.gif ├── enc_head_gif_delay7-min.gif ├── enc_head_gif_delay7.gif ├── intro_large-min.png ├── intro_large.png ├── lrp_main-min.png ├── lrp_main.png └── src_dst_main.gif ├── scripts ├── nmt.py ├── train_baseline.sh ├── train_concrete_heads.sh ├── train_fixed_alive_heads.sh └── train_mrt.sh └── source_target_contributions ├── 1_Load_model_and_evaluate_LRP.ipynb ├── 2_Load_LRP_results_and_build_graphs.ipynb └── README.md /.Rhistory: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/README.md -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/__init__.py -------------------------------------------------------------------------------- /lib/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/data.py -------------------------------------------------------------------------------- /lib/layers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/layers/__init__.py -------------------------------------------------------------------------------- /lib/layers/attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/layers/attn.py -------------------------------------------------------------------------------- /lib/layers/attn_lrp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/layers/attn_lrp.py -------------------------------------------------------------------------------- /lib/layers/basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/layers/basic.py -------------------------------------------------------------------------------- /lib/layers/concrete_gate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/layers/concrete_gate.py -------------------------------------------------------------------------------- /lib/layers/lrp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/layers/lrp.py -------------------------------------------------------------------------------- /lib/meta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/meta.py -------------------------------------------------------------------------------- /lib/ops/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/ops/__init__.py -------------------------------------------------------------------------------- /lib/ops/basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/ops/basic.py -------------------------------------------------------------------------------- /lib/ops/devices.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/ops/devices.py -------------------------------------------------------------------------------- /lib/ops/mpi/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/ops/mpi/__init__.py -------------------------------------------------------------------------------- /lib/ops/mpi/dummy_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/ops/mpi/dummy_provider.py -------------------------------------------------------------------------------- /lib/ops/mpi/horovod_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/ops/mpi/horovod_provider.py -------------------------------------------------------------------------------- /lib/ops/record_activations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/ops/record_activations.py -------------------------------------------------------------------------------- /lib/ops/sliced_argmax.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/ops/sliced_argmax.py -------------------------------------------------------------------------------- /lib/session.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/session.py -------------------------------------------------------------------------------- /lib/task/__init__.py: -------------------------------------------------------------------------------- 1 | from . import seq2seq 2 | -------------------------------------------------------------------------------- /lib/task/seq2seq/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/__init__.py -------------------------------------------------------------------------------- /lib/task/seq2seq/bleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/bleu.py -------------------------------------------------------------------------------- /lib/task/seq2seq/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/data.py -------------------------------------------------------------------------------- /lib/task/seq2seq/inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/inference.py -------------------------------------------------------------------------------- /lib/task/seq2seq/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/models/__init__.py -------------------------------------------------------------------------------- /lib/task/seq2seq/models/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/models/transformer.py -------------------------------------------------------------------------------- /lib/task/seq2seq/models/transformer_head_gates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/models/transformer_head_gates.py -------------------------------------------------------------------------------- /lib/task/seq2seq/models/transformer_lrp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/models/transformer_lrp.py -------------------------------------------------------------------------------- /lib/task/seq2seq/problems/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/task/seq2seq/problems/concrete.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/problems/concrete.py -------------------------------------------------------------------------------- /lib/task/seq2seq/problems/default.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/problems/default.py -------------------------------------------------------------------------------- /lib/task/seq2seq/problems/mrt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/problems/mrt.py -------------------------------------------------------------------------------- /lib/task/seq2seq/strutils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/strutils.py -------------------------------------------------------------------------------- /lib/task/seq2seq/summary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/summary.py -------------------------------------------------------------------------------- /lib/task/seq2seq/tickers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/tickers.py -------------------------------------------------------------------------------- /lib/task/seq2seq/voc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/task/seq2seq/voc.py -------------------------------------------------------------------------------- /lib/tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/tools/apply_bpe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/tools/apply_bpe.py -------------------------------------------------------------------------------- /lib/tools/average_npz.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/tools/average_npz.py -------------------------------------------------------------------------------- /lib/train/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/train/__init__.py -------------------------------------------------------------------------------- /lib/train/algorithms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/train/algorithms.py -------------------------------------------------------------------------------- /lib/train/optimizers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/train/optimizers.py -------------------------------------------------------------------------------- /lib/train/problem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/train/problem.py -------------------------------------------------------------------------------- /lib/train/saveload.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/train/saveload.py -------------------------------------------------------------------------------- /lib/train/tickers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/train/tickers.py -------------------------------------------------------------------------------- /lib/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/lib/util.py -------------------------------------------------------------------------------- /notebooks/1_Load_model_and_translate.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/notebooks/1_Load_model_and_translate.ipynb -------------------------------------------------------------------------------- /notebooks/2_Look_at_attention_maps.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/notebooks/2_Look_at_attention_maps.ipynb -------------------------------------------------------------------------------- /notebooks/3_Look_which_heads_are_dead.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/notebooks/3_Look_which_heads_are_dead.ipynb -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/requirements.txt -------------------------------------------------------------------------------- /resources/acl19_heads-min_pad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/resources/acl19_heads-min_pad.png -------------------------------------------------------------------------------- /resources/acl_empty.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/resources/acl_empty.png -------------------------------------------------------------------------------- /resources/concrete.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/resources/concrete.gif -------------------------------------------------------------------------------- /resources/concrete_crop.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/resources/concrete_crop.gif -------------------------------------------------------------------------------- /resources/enc_head_gif_delay7-min.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/resources/enc_head_gif_delay7-min.gif -------------------------------------------------------------------------------- /resources/enc_head_gif_delay7.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/resources/enc_head_gif_delay7.gif -------------------------------------------------------------------------------- /resources/intro_large-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/resources/intro_large-min.png -------------------------------------------------------------------------------- /resources/intro_large.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/resources/intro_large.png -------------------------------------------------------------------------------- /resources/lrp_main-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/resources/lrp_main-min.png -------------------------------------------------------------------------------- /resources/lrp_main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/resources/lrp_main.png -------------------------------------------------------------------------------- /resources/src_dst_main.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/resources/src_dst_main.gif -------------------------------------------------------------------------------- /scripts/nmt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/scripts/nmt.py -------------------------------------------------------------------------------- /scripts/train_baseline.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/scripts/train_baseline.sh -------------------------------------------------------------------------------- /scripts/train_concrete_heads.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/scripts/train_concrete_heads.sh -------------------------------------------------------------------------------- /scripts/train_fixed_alive_heads.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/scripts/train_fixed_alive_heads.sh -------------------------------------------------------------------------------- /scripts/train_mrt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/scripts/train_mrt.sh -------------------------------------------------------------------------------- /source_target_contributions/1_Load_model_and_evaluate_LRP.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/source_target_contributions/1_Load_model_and_evaluate_LRP.ipynb -------------------------------------------------------------------------------- /source_target_contributions/2_Load_LRP_results_and_build_graphs.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/source_target_contributions/2_Load_LRP_results_and_build_graphs.ipynb -------------------------------------------------------------------------------- /source_target_contributions/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lena-voita/the-story-of-heads/HEAD/source_target_contributions/README.md --------------------------------------------------------------------------------