├── README.md ├── app.py ├── chat.py ├── client.py ├── data_curation ├── build_vlpart.py ├── check_dataset.py ├── prompt_generation_handal_easy_reasoning.py ├── prompt_generation_handal_hard_reasoning.py └── vlpart_sam2_tracking.py ├── docs ├── dataset.md ├── installation.md └── training_and_evaluation.md ├── imgs ├── AffordanceNet.jpg └── AffordanceNet.png ├── merge_lora_weights_and_save_hf_model.py ├── model ├── AffordanceVLM.py ├── llava │ ├── __init__.py │ ├── constants.py │ ├── conversation.py │ ├── mm_utils.py │ ├── model │ │ ├── __init__.py │ │ ├── apply_delta.py │ │ ├── builder.py │ │ ├── consolidate.py │ │ ├── language_model │ │ │ ├── llava_llama.py │ │ │ ├── llava_mpt.py │ │ │ └── mpt │ │ │ │ ├── adapt_tokenizer.py │ │ │ │ ├── attention.py │ │ │ │ ├── blocks.py │ │ │ │ ├── configuration_mpt.py │ │ │ │ ├── custom_embedding.py │ │ │ │ ├── flash_attn_triton.py │ │ │ │ ├── hf_prefixlm_converter.py │ │ │ │ ├── meta_init_context.py │ │ │ │ ├── modeling_mpt.py │ │ │ │ ├── norm.py │ │ │ │ └── param_init_fns.py │ │ ├── llava_arch.py │ │ ├── make_delta.py │ │ ├── multimodal_encoder │ │ │ ├── builder.py │ │ │ └── clip_encoder.py │ │ └── utils.py │ ├── train │ │ ├── llama_flash_attn_monkey_patch.py │ │ ├── llava_trainer.py │ │ ├── train.py │ │ └── train_mem.py │ └── utils.py └── segment_anything │ ├── __init__.py │ ├── automatic_mask_generator.py │ ├── build_sam.py │ ├── modeling │ ├── __init__.py │ ├── common.py │ ├── image_encoder.py │ ├── mask_decoder.py │ ├── prompt_encoder.py │ ├── sam.py │ └── transformer.py │ ├── predictor.py │ └── utils │ ├── __init__.py │ ├── amg.py │ ├── onnx.py │ └── transforms.py ├── requirements.txt ├── scripts ├── evaluate.sh ├── merge_lora_weights.sh └── train.sh ├── server.py ├── train_aff.py ├── utils ├── ade20k_classes.json ├── aff_seg_dataset.py ├── cocostuff_classes.txt ├── conversation.py ├── data_processing.py ├── dataset.py ├── graspnet_classes.txt ├── grefcoco.py ├── grefer.py ├── reason_aff_dataset.py ├── reason_seg_dataset.py ├── refer.py ├── refer_seg_dataset.py ├── sem_seg_dataset.py ├── utils.py └── vqa_dataset.py └── vis_output ├── my_workspace.JPG ├── my_workspace_mask_0.jpg └── my_workspace_masked_img_0.jpg /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/README.md -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/app.py -------------------------------------------------------------------------------- /chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/chat.py -------------------------------------------------------------------------------- /client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/client.py -------------------------------------------------------------------------------- /data_curation/build_vlpart.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/data_curation/build_vlpart.py -------------------------------------------------------------------------------- /data_curation/check_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/data_curation/check_dataset.py -------------------------------------------------------------------------------- /data_curation/prompt_generation_handal_easy_reasoning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/data_curation/prompt_generation_handal_easy_reasoning.py -------------------------------------------------------------------------------- /data_curation/prompt_generation_handal_hard_reasoning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/data_curation/prompt_generation_handal_hard_reasoning.py -------------------------------------------------------------------------------- /data_curation/vlpart_sam2_tracking.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/data_curation/vlpart_sam2_tracking.py -------------------------------------------------------------------------------- /docs/dataset.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/docs/dataset.md -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/docs/installation.md -------------------------------------------------------------------------------- /docs/training_and_evaluation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/docs/training_and_evaluation.md -------------------------------------------------------------------------------- /imgs/AffordanceNet.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/imgs/AffordanceNet.jpg -------------------------------------------------------------------------------- /imgs/AffordanceNet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/imgs/AffordanceNet.png -------------------------------------------------------------------------------- /merge_lora_weights_and_save_hf_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/merge_lora_weights_and_save_hf_model.py -------------------------------------------------------------------------------- /model/AffordanceVLM.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/AffordanceVLM.py -------------------------------------------------------------------------------- /model/llava/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import LlavaLlamaForCausalLM 2 | -------------------------------------------------------------------------------- /model/llava/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/constants.py -------------------------------------------------------------------------------- /model/llava/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/conversation.py -------------------------------------------------------------------------------- /model/llava/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/mm_utils.py -------------------------------------------------------------------------------- /model/llava/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/__init__.py -------------------------------------------------------------------------------- /model/llava/model/apply_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/apply_delta.py -------------------------------------------------------------------------------- /model/llava/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/builder.py -------------------------------------------------------------------------------- /model/llava/model/consolidate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/consolidate.py -------------------------------------------------------------------------------- /model/llava/model/language_model/llava_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/llava_llama.py -------------------------------------------------------------------------------- /model/llava/model/language_model/llava_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/llava_mpt.py -------------------------------------------------------------------------------- /model/llava/model/language_model/mpt/adapt_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/mpt/adapt_tokenizer.py -------------------------------------------------------------------------------- /model/llava/model/language_model/mpt/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/mpt/attention.py -------------------------------------------------------------------------------- /model/llava/model/language_model/mpt/blocks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/mpt/blocks.py -------------------------------------------------------------------------------- /model/llava/model/language_model/mpt/configuration_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/mpt/configuration_mpt.py -------------------------------------------------------------------------------- /model/llava/model/language_model/mpt/custom_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/mpt/custom_embedding.py -------------------------------------------------------------------------------- /model/llava/model/language_model/mpt/flash_attn_triton.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/mpt/flash_attn_triton.py -------------------------------------------------------------------------------- /model/llava/model/language_model/mpt/hf_prefixlm_converter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/mpt/hf_prefixlm_converter.py -------------------------------------------------------------------------------- /model/llava/model/language_model/mpt/meta_init_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/mpt/meta_init_context.py -------------------------------------------------------------------------------- /model/llava/model/language_model/mpt/modeling_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/mpt/modeling_mpt.py -------------------------------------------------------------------------------- /model/llava/model/language_model/mpt/norm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/mpt/norm.py -------------------------------------------------------------------------------- /model/llava/model/language_model/mpt/param_init_fns.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/language_model/mpt/param_init_fns.py -------------------------------------------------------------------------------- /model/llava/model/llava_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/llava_arch.py -------------------------------------------------------------------------------- /model/llava/model/make_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/make_delta.py -------------------------------------------------------------------------------- /model/llava/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /model/llava/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /model/llava/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/model/utils.py -------------------------------------------------------------------------------- /model/llava/train/llama_flash_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/train/llama_flash_attn_monkey_patch.py -------------------------------------------------------------------------------- /model/llava/train/llava_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/train/llava_trainer.py -------------------------------------------------------------------------------- /model/llava/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/train/train.py -------------------------------------------------------------------------------- /model/llava/train/train_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/train/train_mem.py -------------------------------------------------------------------------------- /model/llava/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/llava/utils.py -------------------------------------------------------------------------------- /model/segment_anything/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/__init__.py -------------------------------------------------------------------------------- /model/segment_anything/automatic_mask_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/automatic_mask_generator.py -------------------------------------------------------------------------------- /model/segment_anything/build_sam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/build_sam.py -------------------------------------------------------------------------------- /model/segment_anything/modeling/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/modeling/__init__.py -------------------------------------------------------------------------------- /model/segment_anything/modeling/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/modeling/common.py -------------------------------------------------------------------------------- /model/segment_anything/modeling/image_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/modeling/image_encoder.py -------------------------------------------------------------------------------- /model/segment_anything/modeling/mask_decoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/modeling/mask_decoder.py -------------------------------------------------------------------------------- /model/segment_anything/modeling/prompt_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/modeling/prompt_encoder.py -------------------------------------------------------------------------------- /model/segment_anything/modeling/sam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/modeling/sam.py -------------------------------------------------------------------------------- /model/segment_anything/modeling/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/modeling/transformer.py -------------------------------------------------------------------------------- /model/segment_anything/predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/predictor.py -------------------------------------------------------------------------------- /model/segment_anything/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/utils/__init__.py -------------------------------------------------------------------------------- /model/segment_anything/utils/amg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/utils/amg.py -------------------------------------------------------------------------------- /model/segment_anything/utils/onnx.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/utils/onnx.py -------------------------------------------------------------------------------- /model/segment_anything/utils/transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/model/segment_anything/utils/transforms.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/evaluate.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/scripts/evaluate.sh -------------------------------------------------------------------------------- /scripts/merge_lora_weights.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/scripts/merge_lora_weights.sh -------------------------------------------------------------------------------- /scripts/train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/scripts/train.sh -------------------------------------------------------------------------------- /server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/server.py -------------------------------------------------------------------------------- /train_aff.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/train_aff.py -------------------------------------------------------------------------------- /utils/ade20k_classes.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/ade20k_classes.json -------------------------------------------------------------------------------- /utils/aff_seg_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/aff_seg_dataset.py -------------------------------------------------------------------------------- /utils/cocostuff_classes.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/cocostuff_classes.txt -------------------------------------------------------------------------------- /utils/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/conversation.py -------------------------------------------------------------------------------- /utils/data_processing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/data_processing.py -------------------------------------------------------------------------------- /utils/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/dataset.py -------------------------------------------------------------------------------- /utils/graspnet_classes.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/graspnet_classes.txt -------------------------------------------------------------------------------- /utils/grefcoco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/grefcoco.py -------------------------------------------------------------------------------- /utils/grefer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/grefer.py -------------------------------------------------------------------------------- /utils/reason_aff_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/reason_aff_dataset.py -------------------------------------------------------------------------------- /utils/reason_seg_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/reason_seg_dataset.py -------------------------------------------------------------------------------- /utils/refer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/refer.py -------------------------------------------------------------------------------- /utils/refer_seg_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/refer_seg_dataset.py -------------------------------------------------------------------------------- /utils/sem_seg_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/sem_seg_dataset.py -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/utils.py -------------------------------------------------------------------------------- /utils/vqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/utils/vqa_dataset.py -------------------------------------------------------------------------------- /vis_output/my_workspace.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/vis_output/my_workspace.JPG -------------------------------------------------------------------------------- /vis_output/my_workspace_mask_0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/vis_output/my_workspace_mask_0.jpg -------------------------------------------------------------------------------- /vis_output/my_workspace_masked_img_0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wudongming97/AffordanceNet/HEAD/vis_output/my_workspace_masked_img_0.jpg --------------------------------------------------------------------------------