├── .gitignore ├── LICENSE ├── README.md ├── collect_demo_exp_results.py ├── configs ├── base-finetune.yaml ├── base-linear.yaml ├── base-prompt.yaml ├── finetune │ ├── cars.yaml │ ├── cub.yaml │ ├── dogs.yaml │ ├── flowers.yaml │ └── nabirds.yaml ├── h2t-prompt │ └── vtab.yaml ├── linear │ ├── cars.yaml │ ├── cub.yaml │ ├── dogs.yaml │ ├── flowers.yaml │ └── nabirds.yaml └── prompt │ ├── cars.yaml │ ├── cub.yaml │ ├── dogs.yaml │ ├── flowers.yaml │ └── nabirds.yaml ├── head2toe_sparsity_train.py ├── head2toe_train.py ├── launch.py ├── pre-trained_weights └── .gitignore ├── run_demo_exp.sh ├── scripts └── VQT │ ├── run_vqt_vtab.sh │ ├── run_vqt_vtab_sparsity.sh │ └── run_vqt_vtab_ssl.sh ├── src ├── configs │ ├── config.py │ ├── config_node.py │ └── vit_configs.py ├── data │ ├── datasets │ │ ├── json_dataset.py │ │ └── tf_dataset.py │ ├── loader.py │ ├── transforms.py │ └── vtab_datasets │ │ ├── __init__.py │ │ ├── base.py │ │ ├── caltech.py │ │ ├── cifar.py │ │ ├── clevr.py │ │ ├── diabetic_retinopathy.py │ │ ├── dmlab.py │ │ ├── dsprites.py │ │ ├── dtd.py │ │ ├── eurosat.py │ │ ├── kitti.py │ │ ├── oxford_flowers102.py │ │ ├── oxford_iiit_pet.py │ │ ├── patch_camelyon.py │ │ ├── registry.py │ │ ├── resisc45.py │ │ ├── smallnorb.py │ │ ├── sun397.py │ │ └── svhn.py ├── engine │ ├── eval │ │ ├── multilabel.py │ │ └── singlelabel.py │ ├── evaluator.py │ ├── h2t_sparsity_trainer.py │ └── trainer.py ├── models │ ├── build_h2t_model.py │ ├── build_model.py │ ├── build_vit_backbone.py │ ├── convnext.py │ ├── convnext_backbone │ │ └── convnext.py │ ├── mlp.py │ ├── resnet.py │ ├── vit_adapter │ │ ├── adapter_block.py │ │ ├── vit.py │ │ ├── vit_mae.py │ │ └── vit_moco.py │ ├── vit_backbones │ │ ├── h2t_vit.py │ │ ├── h2t_vit_mae.py │ │ ├── swin_transformer.py │ │ ├── timm_h2t_vit.py │ │ ├── vit.py │ │ ├── vit_mae.py │ │ └── vit_moco.py │ ├── vit_models.py │ └── vit_prompt │ │ ├── swin_transformer.py │ │ ├── vit.py │ │ ├── vit_ablations.py │ │ ├── vit_mae.py │ │ └── vit_moco.py ├── solver │ ├── losses.py │ ├── lr_scheduler.py │ └── optimizer.py └── utils │ ├── distributed.py │ ├── file_io.py │ ├── io_utils.py │ ├── logging.py │ ├── train_utils.py │ └── vis_utils.py ├── tune_vtab.py └── vtab_data └── .gitignore /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/README.md -------------------------------------------------------------------------------- /collect_demo_exp_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/collect_demo_exp_results.py -------------------------------------------------------------------------------- /configs/base-finetune.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/base-finetune.yaml -------------------------------------------------------------------------------- /configs/base-linear.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/base-linear.yaml -------------------------------------------------------------------------------- /configs/base-prompt.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/base-prompt.yaml -------------------------------------------------------------------------------- /configs/finetune/cars.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/finetune/cars.yaml -------------------------------------------------------------------------------- /configs/finetune/cub.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/finetune/cub.yaml -------------------------------------------------------------------------------- /configs/finetune/dogs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/finetune/dogs.yaml -------------------------------------------------------------------------------- /configs/finetune/flowers.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/finetune/flowers.yaml -------------------------------------------------------------------------------- /configs/finetune/nabirds.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/finetune/nabirds.yaml -------------------------------------------------------------------------------- /configs/h2t-prompt/vtab.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/h2t-prompt/vtab.yaml -------------------------------------------------------------------------------- /configs/linear/cars.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/linear/cars.yaml -------------------------------------------------------------------------------- /configs/linear/cub.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/linear/cub.yaml -------------------------------------------------------------------------------- /configs/linear/dogs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/linear/dogs.yaml -------------------------------------------------------------------------------- /configs/linear/flowers.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/linear/flowers.yaml -------------------------------------------------------------------------------- /configs/linear/nabirds.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/linear/nabirds.yaml -------------------------------------------------------------------------------- /configs/prompt/cars.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/prompt/cars.yaml -------------------------------------------------------------------------------- /configs/prompt/cub.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/prompt/cub.yaml -------------------------------------------------------------------------------- /configs/prompt/dogs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/prompt/dogs.yaml -------------------------------------------------------------------------------- /configs/prompt/flowers.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/prompt/flowers.yaml -------------------------------------------------------------------------------- /configs/prompt/nabirds.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/configs/prompt/nabirds.yaml -------------------------------------------------------------------------------- /head2toe_sparsity_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/head2toe_sparsity_train.py -------------------------------------------------------------------------------- /head2toe_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/head2toe_train.py -------------------------------------------------------------------------------- /launch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/launch.py -------------------------------------------------------------------------------- /pre-trained_weights/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /run_demo_exp.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/run_demo_exp.sh -------------------------------------------------------------------------------- /scripts/VQT/run_vqt_vtab.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/scripts/VQT/run_vqt_vtab.sh -------------------------------------------------------------------------------- /scripts/VQT/run_vqt_vtab_sparsity.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/scripts/VQT/run_vqt_vtab_sparsity.sh -------------------------------------------------------------------------------- /scripts/VQT/run_vqt_vtab_ssl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/scripts/VQT/run_vqt_vtab_ssl.sh -------------------------------------------------------------------------------- /src/configs/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/configs/config.py -------------------------------------------------------------------------------- /src/configs/config_node.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/configs/config_node.py -------------------------------------------------------------------------------- /src/configs/vit_configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/configs/vit_configs.py -------------------------------------------------------------------------------- /src/data/datasets/json_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/datasets/json_dataset.py -------------------------------------------------------------------------------- /src/data/datasets/tf_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/datasets/tf_dataset.py -------------------------------------------------------------------------------- /src/data/loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/loader.py -------------------------------------------------------------------------------- /src/data/transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/transforms.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/__init__.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/base.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/caltech.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/caltech.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/cifar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/cifar.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/clevr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/clevr.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/diabetic_retinopathy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/diabetic_retinopathy.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/dmlab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/dmlab.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/dsprites.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/dsprites.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/dtd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/dtd.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/eurosat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/eurosat.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/kitti.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/kitti.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/oxford_flowers102.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/oxford_flowers102.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/oxford_iiit_pet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/oxford_iiit_pet.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/patch_camelyon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/patch_camelyon.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/registry.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/resisc45.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/resisc45.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/smallnorb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/smallnorb.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/sun397.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/sun397.py -------------------------------------------------------------------------------- /src/data/vtab_datasets/svhn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/data/vtab_datasets/svhn.py -------------------------------------------------------------------------------- /src/engine/eval/multilabel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/engine/eval/multilabel.py -------------------------------------------------------------------------------- /src/engine/eval/singlelabel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/engine/eval/singlelabel.py -------------------------------------------------------------------------------- /src/engine/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/engine/evaluator.py -------------------------------------------------------------------------------- /src/engine/h2t_sparsity_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/engine/h2t_sparsity_trainer.py -------------------------------------------------------------------------------- /src/engine/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/engine/trainer.py -------------------------------------------------------------------------------- /src/models/build_h2t_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/build_h2t_model.py -------------------------------------------------------------------------------- /src/models/build_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/build_model.py -------------------------------------------------------------------------------- /src/models/build_vit_backbone.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/build_vit_backbone.py -------------------------------------------------------------------------------- /src/models/convnext.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/convnext.py -------------------------------------------------------------------------------- /src/models/convnext_backbone/convnext.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/convnext_backbone/convnext.py -------------------------------------------------------------------------------- /src/models/mlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/mlp.py -------------------------------------------------------------------------------- /src/models/resnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/resnet.py -------------------------------------------------------------------------------- /src/models/vit_adapter/adapter_block.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_adapter/adapter_block.py -------------------------------------------------------------------------------- /src/models/vit_adapter/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_adapter/vit.py -------------------------------------------------------------------------------- /src/models/vit_adapter/vit_mae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_adapter/vit_mae.py -------------------------------------------------------------------------------- /src/models/vit_adapter/vit_moco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_adapter/vit_moco.py -------------------------------------------------------------------------------- /src/models/vit_backbones/h2t_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_backbones/h2t_vit.py -------------------------------------------------------------------------------- /src/models/vit_backbones/h2t_vit_mae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_backbones/h2t_vit_mae.py -------------------------------------------------------------------------------- /src/models/vit_backbones/swin_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_backbones/swin_transformer.py -------------------------------------------------------------------------------- /src/models/vit_backbones/timm_h2t_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_backbones/timm_h2t_vit.py -------------------------------------------------------------------------------- /src/models/vit_backbones/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_backbones/vit.py -------------------------------------------------------------------------------- /src/models/vit_backbones/vit_mae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_backbones/vit_mae.py -------------------------------------------------------------------------------- /src/models/vit_backbones/vit_moco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_backbones/vit_moco.py -------------------------------------------------------------------------------- /src/models/vit_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_models.py -------------------------------------------------------------------------------- /src/models/vit_prompt/swin_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_prompt/swin_transformer.py -------------------------------------------------------------------------------- /src/models/vit_prompt/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_prompt/vit.py -------------------------------------------------------------------------------- /src/models/vit_prompt/vit_ablations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_prompt/vit_ablations.py -------------------------------------------------------------------------------- /src/models/vit_prompt/vit_mae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_prompt/vit_mae.py -------------------------------------------------------------------------------- /src/models/vit_prompt/vit_moco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/models/vit_prompt/vit_moco.py -------------------------------------------------------------------------------- /src/solver/losses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/solver/losses.py -------------------------------------------------------------------------------- /src/solver/lr_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/solver/lr_scheduler.py -------------------------------------------------------------------------------- /src/solver/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/solver/optimizer.py -------------------------------------------------------------------------------- /src/utils/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/utils/distributed.py -------------------------------------------------------------------------------- /src/utils/file_io.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/utils/file_io.py -------------------------------------------------------------------------------- /src/utils/io_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/utils/io_utils.py -------------------------------------------------------------------------------- /src/utils/logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/utils/logging.py -------------------------------------------------------------------------------- /src/utils/train_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/utils/train_utils.py -------------------------------------------------------------------------------- /src/utils/vis_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/src/utils/vis_utils.py -------------------------------------------------------------------------------- /tune_vtab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andytu28/VQT/HEAD/tune_vtab.py -------------------------------------------------------------------------------- /vtab_data/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | --------------------------------------------------------------------------------