├── LICENSE ├── README.md ├── assets ├── gradio_demo.png ├── logo.png ├── val_img.png └── val_vid.png ├── llava ├── __init__.py ├── constants.py ├── conversation.py ├── eval │ ├── eval_omni_msrvtt.py │ └── run_omni.py ├── mm_utils.py ├── model │ ├── __init__.py │ ├── builder.py │ ├── language_model │ │ └── omni_llama.py │ ├── multimodal_encoder │ │ ├── builder.py │ │ └── clip_encoder.py │ ├── multimodal_projector │ │ └── builder.py │ ├── omni_arch.py │ └── utils.py ├── serve │ ├── __init__.py │ ├── cli.py │ ├── controller.py │ ├── examples │ │ ├── 0A8CF.mp4 │ │ ├── XJU8U.mp4 │ │ ├── desert.jpg │ │ ├── extreme_ironing.jpg │ │ ├── sample_demo_1.mp4 │ │ ├── sample_demo_13.mp4 │ │ ├── sample_demo_22.mp4 │ │ ├── sample_demo_3.mp4 │ │ ├── sample_demo_8.mp4 │ │ ├── sample_demo_9.mp4 │ │ ├── sample_img_13.png │ │ ├── sample_img_22.png │ │ ├── sample_img_8.png │ │ └── waterview.jpg │ ├── gradio_demo.py │ ├── gradio_utils.py │ ├── gradio_web_server.py │ ├── model_worker.py │ ├── register_worker.py │ └── test_message.py └── utils.py └── pyproject.toml /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/README.md -------------------------------------------------------------------------------- /assets/gradio_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/assets/gradio_demo.png -------------------------------------------------------------------------------- /assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/assets/logo.png -------------------------------------------------------------------------------- /assets/val_img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/assets/val_img.png -------------------------------------------------------------------------------- /assets/val_vid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/assets/val_vid.png -------------------------------------------------------------------------------- /llava/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import OmniLlamaForCausalLM -------------------------------------------------------------------------------- /llava/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/constants.py -------------------------------------------------------------------------------- /llava/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/conversation.py -------------------------------------------------------------------------------- /llava/eval/eval_omni_msrvtt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/eval/eval_omni_msrvtt.py -------------------------------------------------------------------------------- /llava/eval/run_omni.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/eval/run_omni.py -------------------------------------------------------------------------------- /llava/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/mm_utils.py -------------------------------------------------------------------------------- /llava/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/model/__init__.py -------------------------------------------------------------------------------- /llava/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/model/builder.py -------------------------------------------------------------------------------- /llava/model/language_model/omni_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/model/language_model/omni_llama.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /llava/model/omni_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/model/omni_arch.py -------------------------------------------------------------------------------- /llava/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/model/utils.py -------------------------------------------------------------------------------- /llava/serve/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llava/serve/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/cli.py -------------------------------------------------------------------------------- /llava/serve/controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/controller.py -------------------------------------------------------------------------------- /llava/serve/examples/0A8CF.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/0A8CF.mp4 -------------------------------------------------------------------------------- /llava/serve/examples/XJU8U.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/XJU8U.mp4 -------------------------------------------------------------------------------- /llava/serve/examples/desert.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/desert.jpg -------------------------------------------------------------------------------- /llava/serve/examples/extreme_ironing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/extreme_ironing.jpg -------------------------------------------------------------------------------- /llava/serve/examples/sample_demo_1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/sample_demo_1.mp4 -------------------------------------------------------------------------------- /llava/serve/examples/sample_demo_13.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/sample_demo_13.mp4 -------------------------------------------------------------------------------- /llava/serve/examples/sample_demo_22.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/sample_demo_22.mp4 -------------------------------------------------------------------------------- /llava/serve/examples/sample_demo_3.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/sample_demo_3.mp4 -------------------------------------------------------------------------------- /llava/serve/examples/sample_demo_8.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/sample_demo_8.mp4 -------------------------------------------------------------------------------- /llava/serve/examples/sample_demo_9.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/sample_demo_9.mp4 -------------------------------------------------------------------------------- /llava/serve/examples/sample_img_13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/sample_img_13.png -------------------------------------------------------------------------------- /llava/serve/examples/sample_img_22.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/sample_img_22.png -------------------------------------------------------------------------------- /llava/serve/examples/sample_img_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/sample_img_8.png -------------------------------------------------------------------------------- /llava/serve/examples/waterview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/examples/waterview.jpg -------------------------------------------------------------------------------- /llava/serve/gradio_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/gradio_demo.py -------------------------------------------------------------------------------- /llava/serve/gradio_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/gradio_utils.py -------------------------------------------------------------------------------- /llava/serve/gradio_web_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/gradio_web_server.py -------------------------------------------------------------------------------- /llava/serve/model_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/model_worker.py -------------------------------------------------------------------------------- /llava/serve/register_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/register_worker.py -------------------------------------------------------------------------------- /llava/serve/test_message.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/serve/test_message.py -------------------------------------------------------------------------------- /llava/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/llava/utils.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanghao-cst/Omni-VideoAssistant/HEAD/pyproject.toml --------------------------------------------------------------------------------