├── .gitignore ├── README.md ├── asset ├── intro.png └── toast.png ├── language_generation ├── LICENSE ├── README.md ├── assets │ └── vicuna.png ├── configs │ └── default_offload_opt_param.json ├── inference.py ├── models │ ├── llama_lora.py │ ├── llama_top_down.py │ └── llama_top_down_lora.py ├── pretune_top_down.py ├── requirements.txt ├── train.py └── utils.py └── visual_classification ├── LICENSE ├── README.md ├── VTAB_SETUP.md ├── assets └── att_compare.png ├── configs ├── base.yaml ├── cars.yaml ├── cub.yaml ├── dogs.yaml ├── flowers.yaml ├── nabirds.yaml └── vtab.yaml ├── convert_google_vit_large_to_deit.py ├── download_vtab.py ├── env_setup.sh ├── launch.py ├── src ├── configs │ ├── config.py │ ├── config_node.py │ └── vit_configs.py ├── data │ ├── datasets │ │ ├── json_dataset.py │ │ └── tf_dataset.py │ ├── imagenet_c │ │ ├── __init__.py │ │ ├── corruptions.py │ │ └── frost │ │ │ ├── frost1.png │ │ │ ├── frost2.png │ │ │ ├── frost3.png │ │ │ ├── frost4.jpg │ │ │ ├── frost5.jpg │ │ │ └── frost6.jpg │ ├── loader.py │ ├── transforms.py │ └── vtab_datasets │ │ ├── __init__.py │ │ ├── base.py │ │ ├── caltech.py │ │ ├── cifar.py │ │ ├── clevr.py │ │ ├── diabetic_retinopathy.py │ │ ├── dmlab.py │ │ ├── dsprites.py │ │ ├── dtd.py │ │ ├── eurosat.py │ │ ├── kitti.py │ │ ├── oxford_flowers102.py │ │ ├── oxford_iiit_pet.py │ │ ├── patch_camelyon.py │ │ ├── registry.py │ │ ├── resisc45.py │ │ ├── smallnorb.py │ │ ├── sun397.py │ │ └── svhn.py ├── engine │ ├── eval │ │ ├── multilabel.py │ │ └── singlelabel.py │ ├── evaluator.py │ └── trainer.py ├── models │ ├── build_model.py │ ├── convnext.py │ ├── convnext_backbone │ │ ├── convnext_bottom_up.py │ │ └── convnext_top_down.py │ ├── mlp.py │ ├── vit │ │ ├── vit_bottom_up.py │ │ └── vit_top_down.py │ └── vit_models.py ├── solver │ ├── losses.py │ ├── lr_scheduler.py │ └── optimizer.py └── utils │ ├── distributed.py │ ├── file_io.py │ ├── io_utils.py │ ├── logging.py │ ├── train_utils.py │ └── vis_utils.py ├── train.py ├── tune_fgvc.py ├── tune_vtab.py └── visualize.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/README.md -------------------------------------------------------------------------------- /asset/intro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/asset/intro.png -------------------------------------------------------------------------------- /asset/toast.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/asset/toast.png -------------------------------------------------------------------------------- /language_generation/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/LICENSE -------------------------------------------------------------------------------- /language_generation/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/README.md -------------------------------------------------------------------------------- /language_generation/assets/vicuna.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/assets/vicuna.png -------------------------------------------------------------------------------- /language_generation/configs/default_offload_opt_param.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/configs/default_offload_opt_param.json -------------------------------------------------------------------------------- /language_generation/inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/inference.py -------------------------------------------------------------------------------- /language_generation/models/llama_lora.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/models/llama_lora.py -------------------------------------------------------------------------------- /language_generation/models/llama_top_down.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/models/llama_top_down.py -------------------------------------------------------------------------------- /language_generation/models/llama_top_down_lora.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/models/llama_top_down_lora.py -------------------------------------------------------------------------------- /language_generation/pretune_top_down.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/pretune_top_down.py -------------------------------------------------------------------------------- /language_generation/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/requirements.txt -------------------------------------------------------------------------------- /language_generation/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/train.py -------------------------------------------------------------------------------- /language_generation/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/language_generation/utils.py -------------------------------------------------------------------------------- /visual_classification/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/LICENSE -------------------------------------------------------------------------------- /visual_classification/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/README.md -------------------------------------------------------------------------------- /visual_classification/VTAB_SETUP.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/VTAB_SETUP.md -------------------------------------------------------------------------------- /visual_classification/assets/att_compare.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/assets/att_compare.png -------------------------------------------------------------------------------- /visual_classification/configs/base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/configs/base.yaml -------------------------------------------------------------------------------- /visual_classification/configs/cars.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/configs/cars.yaml -------------------------------------------------------------------------------- /visual_classification/configs/cub.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/configs/cub.yaml -------------------------------------------------------------------------------- /visual_classification/configs/dogs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/configs/dogs.yaml -------------------------------------------------------------------------------- /visual_classification/configs/flowers.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/configs/flowers.yaml -------------------------------------------------------------------------------- /visual_classification/configs/nabirds.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/configs/nabirds.yaml -------------------------------------------------------------------------------- /visual_classification/configs/vtab.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/configs/vtab.yaml -------------------------------------------------------------------------------- /visual_classification/convert_google_vit_large_to_deit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/convert_google_vit_large_to_deit.py -------------------------------------------------------------------------------- /visual_classification/download_vtab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/download_vtab.py -------------------------------------------------------------------------------- /visual_classification/env_setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/env_setup.sh -------------------------------------------------------------------------------- /visual_classification/launch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/launch.py -------------------------------------------------------------------------------- /visual_classification/src/configs/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/configs/config.py -------------------------------------------------------------------------------- /visual_classification/src/configs/config_node.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/configs/config_node.py -------------------------------------------------------------------------------- /visual_classification/src/configs/vit_configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/configs/vit_configs.py -------------------------------------------------------------------------------- /visual_classification/src/data/datasets/json_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/datasets/json_dataset.py -------------------------------------------------------------------------------- /visual_classification/src/data/datasets/tf_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/datasets/tf_dataset.py -------------------------------------------------------------------------------- /visual_classification/src/data/imagenet_c/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/imagenet_c/__init__.py -------------------------------------------------------------------------------- /visual_classification/src/data/imagenet_c/corruptions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/imagenet_c/corruptions.py -------------------------------------------------------------------------------- /visual_classification/src/data/imagenet_c/frost/frost1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/imagenet_c/frost/frost1.png -------------------------------------------------------------------------------- /visual_classification/src/data/imagenet_c/frost/frost2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/imagenet_c/frost/frost2.png -------------------------------------------------------------------------------- /visual_classification/src/data/imagenet_c/frost/frost3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/imagenet_c/frost/frost3.png -------------------------------------------------------------------------------- /visual_classification/src/data/imagenet_c/frost/frost4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/imagenet_c/frost/frost4.jpg -------------------------------------------------------------------------------- /visual_classification/src/data/imagenet_c/frost/frost5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/imagenet_c/frost/frost5.jpg -------------------------------------------------------------------------------- /visual_classification/src/data/imagenet_c/frost/frost6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/imagenet_c/frost/frost6.jpg -------------------------------------------------------------------------------- /visual_classification/src/data/loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/loader.py -------------------------------------------------------------------------------- /visual_classification/src/data/transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/transforms.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/__init__.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/base.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/caltech.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/caltech.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/cifar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/cifar.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/clevr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/clevr.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/diabetic_retinopathy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/diabetic_retinopathy.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/dmlab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/dmlab.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/dsprites.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/dsprites.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/dtd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/dtd.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/eurosat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/eurosat.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/kitti.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/kitti.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/oxford_flowers102.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/oxford_flowers102.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/oxford_iiit_pet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/oxford_iiit_pet.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/patch_camelyon.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/patch_camelyon.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/registry.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/resisc45.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/resisc45.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/smallnorb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/smallnorb.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/sun397.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/sun397.py -------------------------------------------------------------------------------- /visual_classification/src/data/vtab_datasets/svhn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/data/vtab_datasets/svhn.py -------------------------------------------------------------------------------- /visual_classification/src/engine/eval/multilabel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/engine/eval/multilabel.py -------------------------------------------------------------------------------- /visual_classification/src/engine/eval/singlelabel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/engine/eval/singlelabel.py -------------------------------------------------------------------------------- /visual_classification/src/engine/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/engine/evaluator.py -------------------------------------------------------------------------------- /visual_classification/src/engine/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/engine/trainer.py -------------------------------------------------------------------------------- /visual_classification/src/models/build_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/models/build_model.py -------------------------------------------------------------------------------- /visual_classification/src/models/convnext.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/models/convnext.py -------------------------------------------------------------------------------- /visual_classification/src/models/convnext_backbone/convnext_bottom_up.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/models/convnext_backbone/convnext_bottom_up.py -------------------------------------------------------------------------------- /visual_classification/src/models/convnext_backbone/convnext_top_down.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/models/convnext_backbone/convnext_top_down.py -------------------------------------------------------------------------------- /visual_classification/src/models/mlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/models/mlp.py -------------------------------------------------------------------------------- /visual_classification/src/models/vit/vit_bottom_up.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/models/vit/vit_bottom_up.py -------------------------------------------------------------------------------- /visual_classification/src/models/vit/vit_top_down.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/models/vit/vit_top_down.py -------------------------------------------------------------------------------- /visual_classification/src/models/vit_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/models/vit_models.py -------------------------------------------------------------------------------- /visual_classification/src/solver/losses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/solver/losses.py -------------------------------------------------------------------------------- /visual_classification/src/solver/lr_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/solver/lr_scheduler.py -------------------------------------------------------------------------------- /visual_classification/src/solver/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/solver/optimizer.py -------------------------------------------------------------------------------- /visual_classification/src/utils/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/utils/distributed.py -------------------------------------------------------------------------------- /visual_classification/src/utils/file_io.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/utils/file_io.py -------------------------------------------------------------------------------- /visual_classification/src/utils/io_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/utils/io_utils.py -------------------------------------------------------------------------------- /visual_classification/src/utils/logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/utils/logging.py -------------------------------------------------------------------------------- /visual_classification/src/utils/train_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/utils/train_utils.py -------------------------------------------------------------------------------- /visual_classification/src/utils/vis_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/src/utils/vis_utils.py -------------------------------------------------------------------------------- /visual_classification/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/train.py -------------------------------------------------------------------------------- /visual_classification/tune_fgvc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/tune_fgvc.py -------------------------------------------------------------------------------- /visual_classification/tune_vtab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/tune_vtab.py -------------------------------------------------------------------------------- /visual_classification/visualize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfshi/TOAST/HEAD/visual_classification/visualize.py --------------------------------------------------------------------------------