├── .gitignore ├── README.md ├── data ├── activitynet_qa │ └── test.json ├── cgbench │ └── full_mc.json ├── egoschema │ └── full.json └── mlvu │ └── dev_debug_mc.json ├── model ├── abstract_rekv.py ├── attention │ ├── __init__.py │ ├── dot_production_attention │ │ ├── __init__.py │ │ ├── base.py │ │ ├── torch_impl.py │ │ └── triton_impl.py │ ├── kv_cache_manager.py │ ├── rekv_attention.py │ ├── rope.py │ └── utils.py ├── flash_vstream_rekv.py ├── llava_onevision_rekv.py ├── longva │ ├── longva │ │ ├── __init__.py │ │ ├── constants.py │ │ ├── conversation.py │ │ ├── mm_utils.py │ │ ├── model │ │ │ ├── __init__.py │ │ │ ├── apply_delta.py │ │ │ ├── builder.py │ │ │ ├── consolidate.py │ │ │ ├── language_model │ │ │ │ ├── llava_llama.py │ │ │ │ ├── llava_mistral.py │ │ │ │ ├── llava_mpt.py │ │ │ │ ├── llava_qwen.py │ │ │ │ └── modeling_llama.py │ │ │ ├── llava_arch.py │ │ │ ├── make_delta.py │ │ │ ├── multimodal_encoder │ │ │ │ ├── builder.py │ │ │ │ └── clip_encoder.py │ │ │ ├── multimodal_projector │ │ │ │ ├── builder.py │ │ │ │ └── pooler_projector.py │ │ │ ├── multimodal_resampler │ │ │ │ ├── builder.py │ │ │ │ ├── masked_drop.py │ │ │ │ ├── perceiver.py │ │ │ │ ├── qformer.py │ │ │ │ └── spatial_pool.py │ │ │ └── utils.py │ │ └── utils.py │ ├── pyproject.toml │ └── trl │ │ ├── __init__.py │ │ ├── core.py │ │ ├── environment │ │ ├── __init__.py │ │ └── base_environment.py │ │ ├── extras │ │ ├── __init__.py │ │ ├── best_of_n_sampler.py │ │ └── dataset_formatting.py │ │ ├── import_utils.py │ │ ├── models │ │ ├── __init__.py │ │ ├── modeling_base.py │ │ ├── modeling_sd_base.py │ │ ├── modeling_value_head.py │ │ └── utils.py │ │ └── trainer │ │ ├── __init__.py │ │ ├── base.py │ │ ├── ddpo_config.py │ │ ├── ddpo_trainer.py │ │ ├── dpo_trainer.py │ │ ├── iterative_sft_trainer.py │ │ ├── model_config.py │ │ ├── ppo_config.py │ │ ├── ppo_trainer.py │ │ ├── reward_config.py │ │ ├── reward_trainer.py │ │ ├── sft_trainer.py │ │ └── utils.py ├── longva_rekv.py ├── patch.py └── video_llava_rekv.py ├── model_zoo └── .gitkeep ├── prepare.sh ├── pyproject.toml ├── results └── .gitkeep └── video_qa ├── base.py ├── eval ├── eval_egoschema.py ├── eval_mlvu_by_task.py ├── eval_multiple_choice.py └── eval_open_ended.py ├── rekv_offline_vqa.py ├── rekv_stream_vqa.py └── run_eval.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/README.md -------------------------------------------------------------------------------- /data/activitynet_qa/test.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/data/activitynet_qa/test.json -------------------------------------------------------------------------------- /data/cgbench/full_mc.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/data/cgbench/full_mc.json -------------------------------------------------------------------------------- /data/egoschema/full.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/data/egoschema/full.json -------------------------------------------------------------------------------- /data/mlvu/dev_debug_mc.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/data/mlvu/dev_debug_mc.json -------------------------------------------------------------------------------- /model/abstract_rekv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/abstract_rekv.py -------------------------------------------------------------------------------- /model/attention/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/attention/__init__.py -------------------------------------------------------------------------------- /model/attention/dot_production_attention/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/attention/dot_production_attention/__init__.py -------------------------------------------------------------------------------- /model/attention/dot_production_attention/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/attention/dot_production_attention/base.py -------------------------------------------------------------------------------- /model/attention/dot_production_attention/torch_impl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/attention/dot_production_attention/torch_impl.py -------------------------------------------------------------------------------- /model/attention/dot_production_attention/triton_impl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/attention/dot_production_attention/triton_impl.py -------------------------------------------------------------------------------- /model/attention/kv_cache_manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/attention/kv_cache_manager.py -------------------------------------------------------------------------------- /model/attention/rekv_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/attention/rekv_attention.py -------------------------------------------------------------------------------- /model/attention/rope.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/attention/rope.py -------------------------------------------------------------------------------- /model/attention/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/attention/utils.py -------------------------------------------------------------------------------- /model/flash_vstream_rekv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/flash_vstream_rekv.py -------------------------------------------------------------------------------- /model/llava_onevision_rekv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/llava_onevision_rekv.py -------------------------------------------------------------------------------- /model/longva/longva/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import LlavaLlamaForCausalLM 2 | -------------------------------------------------------------------------------- /model/longva/longva/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/constants.py -------------------------------------------------------------------------------- /model/longva/longva/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/conversation.py -------------------------------------------------------------------------------- /model/longva/longva/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/mm_utils.py -------------------------------------------------------------------------------- /model/longva/longva/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/__init__.py -------------------------------------------------------------------------------- /model/longva/longva/model/apply_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/apply_delta.py -------------------------------------------------------------------------------- /model/longva/longva/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/builder.py -------------------------------------------------------------------------------- /model/longva/longva/model/consolidate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/consolidate.py -------------------------------------------------------------------------------- /model/longva/longva/model/language_model/llava_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/language_model/llava_llama.py -------------------------------------------------------------------------------- /model/longva/longva/model/language_model/llava_mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/language_model/llava_mistral.py -------------------------------------------------------------------------------- /model/longva/longva/model/language_model/llava_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/language_model/llava_mpt.py -------------------------------------------------------------------------------- /model/longva/longva/model/language_model/llava_qwen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/language_model/llava_qwen.py -------------------------------------------------------------------------------- /model/longva/longva/model/language_model/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/language_model/modeling_llama.py -------------------------------------------------------------------------------- /model/longva/longva/model/llava_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/llava_arch.py -------------------------------------------------------------------------------- /model/longva/longva/model/make_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/make_delta.py -------------------------------------------------------------------------------- /model/longva/longva/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /model/longva/longva/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /model/longva/longva/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /model/longva/longva/model/multimodal_projector/pooler_projector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/multimodal_projector/pooler_projector.py -------------------------------------------------------------------------------- /model/longva/longva/model/multimodal_resampler/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/multimodal_resampler/builder.py -------------------------------------------------------------------------------- /model/longva/longva/model/multimodal_resampler/masked_drop.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/multimodal_resampler/masked_drop.py -------------------------------------------------------------------------------- /model/longva/longva/model/multimodal_resampler/perceiver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/multimodal_resampler/perceiver.py -------------------------------------------------------------------------------- /model/longva/longva/model/multimodal_resampler/qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/multimodal_resampler/qformer.py -------------------------------------------------------------------------------- /model/longva/longva/model/multimodal_resampler/spatial_pool.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/multimodal_resampler/spatial_pool.py -------------------------------------------------------------------------------- /model/longva/longva/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/model/utils.py -------------------------------------------------------------------------------- /model/longva/longva/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/longva/utils.py -------------------------------------------------------------------------------- /model/longva/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/pyproject.toml -------------------------------------------------------------------------------- /model/longva/trl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/__init__.py -------------------------------------------------------------------------------- /model/longva/trl/core.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/core.py -------------------------------------------------------------------------------- /model/longva/trl/environment/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/environment/__init__.py -------------------------------------------------------------------------------- /model/longva/trl/environment/base_environment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/environment/base_environment.py -------------------------------------------------------------------------------- /model/longva/trl/extras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/extras/__init__.py -------------------------------------------------------------------------------- /model/longva/trl/extras/best_of_n_sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/extras/best_of_n_sampler.py -------------------------------------------------------------------------------- /model/longva/trl/extras/dataset_formatting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/extras/dataset_formatting.py -------------------------------------------------------------------------------- /model/longva/trl/import_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/import_utils.py -------------------------------------------------------------------------------- /model/longva/trl/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/models/__init__.py -------------------------------------------------------------------------------- /model/longva/trl/models/modeling_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/models/modeling_base.py -------------------------------------------------------------------------------- /model/longva/trl/models/modeling_sd_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/models/modeling_sd_base.py -------------------------------------------------------------------------------- /model/longva/trl/models/modeling_value_head.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/models/modeling_value_head.py -------------------------------------------------------------------------------- /model/longva/trl/models/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/models/utils.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/__init__.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/base.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/ddpo_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/ddpo_config.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/ddpo_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/ddpo_trainer.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/dpo_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/dpo_trainer.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/iterative_sft_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/iterative_sft_trainer.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/model_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/model_config.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/ppo_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/ppo_config.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/ppo_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/ppo_trainer.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/reward_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/reward_config.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/reward_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/reward_trainer.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/sft_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/sft_trainer.py -------------------------------------------------------------------------------- /model/longva/trl/trainer/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva/trl/trainer/utils.py -------------------------------------------------------------------------------- /model/longva_rekv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/longva_rekv.py -------------------------------------------------------------------------------- /model/patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/patch.py -------------------------------------------------------------------------------- /model/video_llava_rekv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/model/video_llava_rekv.py -------------------------------------------------------------------------------- /model_zoo/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /prepare.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/prepare.sh -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/pyproject.toml -------------------------------------------------------------------------------- /results/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /video_qa/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/video_qa/base.py -------------------------------------------------------------------------------- /video_qa/eval/eval_egoschema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/video_qa/eval/eval_egoschema.py -------------------------------------------------------------------------------- /video_qa/eval/eval_mlvu_by_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/video_qa/eval/eval_mlvu_by_task.py -------------------------------------------------------------------------------- /video_qa/eval/eval_multiple_choice.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/video_qa/eval/eval_multiple_choice.py -------------------------------------------------------------------------------- /video_qa/eval/eval_open_ended.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/video_qa/eval/eval_open_ended.py -------------------------------------------------------------------------------- /video_qa/rekv_offline_vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/video_qa/rekv_offline_vqa.py -------------------------------------------------------------------------------- /video_qa/rekv_stream_vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/video_qa/rekv_stream_vqa.py -------------------------------------------------------------------------------- /video_qa/run_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Becomebright/ReKV/HEAD/video_qa/run_eval.py --------------------------------------------------------------------------------