├── .gitignore ├── COMMANDS_AND_CHECKPOINTS.md ├── LICENSE ├── README.md ├── cmds └── shell_scripts │ ├── pretrain.sh │ ├── tunePretrained_mae.sh │ └── tunePretrained_sweep.sh ├── dataset └── customDataset.py ├── deit_utils ├── __pycache__ │ └── engine.cpython-36.pyc ├── datasets.py ├── engine.py ├── losses.py └── samplers.py ├── img.png ├── main_finetune_mae.py ├── main_lincls_sweep.py ├── main_pretrain.py ├── moco ├── __init__.py ├── builder.py ├── loader.py └── optimizer.py ├── models ├── mmseg_vit.py ├── vision_transformer.py └── vits.py ├── split └── imagenet │ ├── 10percent.txt │ ├── 1imgs_class.txt │ ├── 1percent.txt │ ├── 2imgs_class.txt │ ├── 5imgs_class.txt │ ├── ImageNet_100_test.txt │ ├── ImageNet_100_train.txt │ ├── ImageNet_100_val.txt │ ├── cvt_format.py │ ├── imageNet_100_1perc.txt │ ├── imagenet_10percent.txt │ ├── imagenet_1imgs_class.txt │ ├── imagenet_1percent.txt │ ├── imagenet_2imgs_class.txt │ ├── imagenet_5imgs_class.txt │ ├── imagenet_train.txt │ ├── imagenet_val.txt │ └── imagenet_val_shuffle.txt ├── thop_modified ├── __init__.py ├── profile.py ├── rnn_hooks.py ├── utils.py └── vision │ ├── __init__.py │ ├── basic_hooks.py │ └── efficientnet.py ├── tools └── collect_data.py ├── utils ├── attn_distill.py ├── bbox_calculation.py ├── caculate_MINE.py ├── caculate_MI_Shujian.py ├── caculate_entropy_across_tokens.py ├── eval_attn_dist.py ├── eval_gradient.py ├── flops_calculation_vit.py ├── init_datasets.py ├── l1_weight_regu.py ├── lr_sched.py ├── moe_utils.py ├── pretrain.py ├── speed_test.py ├── test_normal_cdf.py ├── thop_moe.py ├── transform_w_pos.py ├── utils.py ├── utils_orth.py └── wasserteinLoss.py └── utils_mae ├── datasets.py ├── lr_decay.py ├── lr_sched.py ├── misc.py └── pos_embed.py /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | -------------------------------------------------------------------------------- /COMMANDS_AND_CHECKPOINTS.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/COMMANDS_AND_CHECKPOINTS.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/README.md -------------------------------------------------------------------------------- /cmds/shell_scripts/pretrain.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/cmds/shell_scripts/pretrain.sh -------------------------------------------------------------------------------- /cmds/shell_scripts/tunePretrained_mae.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/cmds/shell_scripts/tunePretrained_mae.sh -------------------------------------------------------------------------------- /cmds/shell_scripts/tunePretrained_sweep.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/cmds/shell_scripts/tunePretrained_sweep.sh -------------------------------------------------------------------------------- /dataset/customDataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/dataset/customDataset.py -------------------------------------------------------------------------------- /deit_utils/__pycache__/engine.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/deit_utils/__pycache__/engine.cpython-36.pyc -------------------------------------------------------------------------------- /deit_utils/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/deit_utils/datasets.py -------------------------------------------------------------------------------- /deit_utils/engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/deit_utils/engine.py -------------------------------------------------------------------------------- /deit_utils/losses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/deit_utils/losses.py -------------------------------------------------------------------------------- /deit_utils/samplers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/deit_utils/samplers.py -------------------------------------------------------------------------------- /img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/img.png -------------------------------------------------------------------------------- /main_finetune_mae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/main_finetune_mae.py -------------------------------------------------------------------------------- /main_lincls_sweep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/main_lincls_sweep.py -------------------------------------------------------------------------------- /main_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/main_pretrain.py -------------------------------------------------------------------------------- /moco/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /moco/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/moco/builder.py -------------------------------------------------------------------------------- /moco/loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/moco/loader.py -------------------------------------------------------------------------------- /moco/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/moco/optimizer.py -------------------------------------------------------------------------------- /models/mmseg_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/models/mmseg_vit.py -------------------------------------------------------------------------------- /models/vision_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/models/vision_transformer.py -------------------------------------------------------------------------------- /models/vits.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/models/vits.py -------------------------------------------------------------------------------- /split/imagenet/10percent.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/10percent.txt -------------------------------------------------------------------------------- /split/imagenet/1imgs_class.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/1imgs_class.txt -------------------------------------------------------------------------------- /split/imagenet/1percent.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/1percent.txt -------------------------------------------------------------------------------- /split/imagenet/2imgs_class.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/2imgs_class.txt -------------------------------------------------------------------------------- /split/imagenet/5imgs_class.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/5imgs_class.txt -------------------------------------------------------------------------------- /split/imagenet/ImageNet_100_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/ImageNet_100_test.txt -------------------------------------------------------------------------------- /split/imagenet/ImageNet_100_train.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/ImageNet_100_train.txt -------------------------------------------------------------------------------- /split/imagenet/ImageNet_100_val.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/ImageNet_100_val.txt -------------------------------------------------------------------------------- /split/imagenet/cvt_format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/cvt_format.py -------------------------------------------------------------------------------- /split/imagenet/imageNet_100_1perc.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/imageNet_100_1perc.txt -------------------------------------------------------------------------------- /split/imagenet/imagenet_10percent.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/imagenet_10percent.txt -------------------------------------------------------------------------------- /split/imagenet/imagenet_1imgs_class.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/imagenet_1imgs_class.txt -------------------------------------------------------------------------------- /split/imagenet/imagenet_1percent.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/imagenet_1percent.txt -------------------------------------------------------------------------------- /split/imagenet/imagenet_2imgs_class.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/imagenet_2imgs_class.txt -------------------------------------------------------------------------------- /split/imagenet/imagenet_5imgs_class.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/imagenet_5imgs_class.txt -------------------------------------------------------------------------------- /split/imagenet/imagenet_train.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/imagenet_train.txt -------------------------------------------------------------------------------- /split/imagenet/imagenet_val.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/imagenet_val.txt -------------------------------------------------------------------------------- /split/imagenet/imagenet_val_shuffle.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/split/imagenet/imagenet_val_shuffle.txt -------------------------------------------------------------------------------- /thop_modified/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/thop_modified/__init__.py -------------------------------------------------------------------------------- /thop_modified/profile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/thop_modified/profile.py -------------------------------------------------------------------------------- /thop_modified/rnn_hooks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/thop_modified/rnn_hooks.py -------------------------------------------------------------------------------- /thop_modified/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/thop_modified/utils.py -------------------------------------------------------------------------------- /thop_modified/vision/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /thop_modified/vision/basic_hooks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/thop_modified/vision/basic_hooks.py -------------------------------------------------------------------------------- /thop_modified/vision/efficientnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/thop_modified/vision/efficientnet.py -------------------------------------------------------------------------------- /tools/collect_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/tools/collect_data.py -------------------------------------------------------------------------------- /utils/attn_distill.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/attn_distill.py -------------------------------------------------------------------------------- /utils/bbox_calculation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/bbox_calculation.py -------------------------------------------------------------------------------- /utils/caculate_MINE.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/caculate_MINE.py -------------------------------------------------------------------------------- /utils/caculate_MI_Shujian.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/caculate_MI_Shujian.py -------------------------------------------------------------------------------- /utils/caculate_entropy_across_tokens.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/caculate_entropy_across_tokens.py -------------------------------------------------------------------------------- /utils/eval_attn_dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/eval_attn_dist.py -------------------------------------------------------------------------------- /utils/eval_gradient.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/eval_gradient.py -------------------------------------------------------------------------------- /utils/flops_calculation_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/flops_calculation_vit.py -------------------------------------------------------------------------------- /utils/init_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/init_datasets.py -------------------------------------------------------------------------------- /utils/l1_weight_regu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/l1_weight_regu.py -------------------------------------------------------------------------------- /utils/lr_sched.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/lr_sched.py -------------------------------------------------------------------------------- /utils/moe_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/moe_utils.py -------------------------------------------------------------------------------- /utils/pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/pretrain.py -------------------------------------------------------------------------------- /utils/speed_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/speed_test.py -------------------------------------------------------------------------------- /utils/test_normal_cdf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/test_normal_cdf.py -------------------------------------------------------------------------------- /utils/thop_moe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/thop_moe.py -------------------------------------------------------------------------------- /utils/transform_w_pos.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/transform_w_pos.py -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/utils.py -------------------------------------------------------------------------------- /utils/utils_orth.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/utils_orth.py -------------------------------------------------------------------------------- /utils/wasserteinLoss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils/wasserteinLoss.py -------------------------------------------------------------------------------- /utils_mae/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils_mae/datasets.py -------------------------------------------------------------------------------- /utils_mae/lr_decay.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils_mae/lr_decay.py -------------------------------------------------------------------------------- /utils_mae/lr_sched.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils_mae/lr_sched.py -------------------------------------------------------------------------------- /utils_mae/misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils_mae/misc.py -------------------------------------------------------------------------------- /utils_mae/pos_embed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VITA-Group/layerGraftedPretraining_ICLR23/HEAD/utils_mae/pos_embed.py --------------------------------------------------------------------------------