├── .flake8 ├── .github └── workflows │ ├── code-quality-main.yaml │ ├── code-quality-pr.yaml │ └── test.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── data-explorations ├── blip2_video_qa.ipynb ├── data_exploration.ipynb ├── eval_data_exploration.ipynb ├── hf_datasets.ipynb ├── instruct_blip_video_qa.ipynb ├── pytorchvideo_load_ego4d_fho_clips.ipynb ├── test_ego4d_fho_main_dataset.ipynb ├── test_epic_kitchens_dataset.ipynb └── video_blip_video_qa.ipynb ├── demo ├── eilev_demo.py ├── examples │ ├── bike-fixing-0.mp4 │ ├── bike-fixing-1.mp4 │ ├── dough-mixer.mp4 │ ├── motorcycle-riding-0.mp4 │ ├── motorcycle-riding-1.mp4 │ ├── paint.mp4 │ └── trowel.mp4 └── video_blip_demo.py ├── eilev ├── __init__.py ├── data │ ├── __init__.py │ ├── ego4d.py │ ├── epic_kitchens.py │ ├── frame.py │ └── utils.py └── model │ ├── __init__.py │ ├── utils.py │ ├── v1.py │ └── v2.py ├── figures ├── ablation_figures.ipynb ├── held_out_verb_noun.ipynb ├── icl_eval_figures.ipynb ├── shuffle_in_context_figures.ipynb ├── skewed_dist_figures.ipynb ├── teaser.png └── tsne_ego4d_epic_kitchens.ipynb ├── poetry.lock ├── pyproject.toml ├── run-mypy ├── samples ├── eilev_generate_action_narration.py └── video_blip_generate_action_narration.py ├── scripts ├── baselines │ ├── majority │ │ ├── majority_generate_full_sent.py │ │ └── majority_predict.py │ └── videomae │ │ ├── videomae_generate_full_sent.py │ │ ├── videomae_predict.py │ │ └── videomae_train.py ├── ego4d │ ├── add_structured_verb_noun.py │ ├── eval-data │ │ ├── structured_noun_prompt.csv │ │ └── structured_verb_prompt.csv │ ├── extract_frames.py │ ├── generate_std_sent.py │ ├── split_held_out_verb_noun_train_val_test.py │ ├── split_train_val_test.py │ └── verify_narrated_action_splits.py ├── epic-kitchens │ ├── epic_kitchens_extract_frames.py │ ├── transform_to_full_sent.py │ └── verify_epic_kitchens.py └── general │ ├── combine_in_context_examples_generated_narrations.py │ ├── generate_narration_texts.py │ ├── generation_eval.py │ ├── get_vision_model_embs.py │ ├── icl_eval.py │ ├── sample_in_context_examples.py │ ├── select_qualitative_examples.py │ ├── train_v1.py │ └── train_v2.py ├── slurm-scripts ├── README.md ├── extract-frames │ ├── submit_ego4d_extract_frames.py │ └── submit_epic_kitchens_extract_frames.py ├── icl-eval │ ├── submit_generate_narration_texts.py │ ├── submit_majority_generate_full_sent.py │ └── submit_videomae_generate_full_sent.py └── train │ ├── submit_train_v2.py │ └── submit_videomae_train.py └── tests ├── data ├── test_frame.py └── test_utils.py └── model ├── test_model_utils.py ├── test_model_v1.py └── test_model_v2.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | extend-ignore = E203 4 | -------------------------------------------------------------------------------- /.github/workflows/code-quality-main.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/.github/workflows/code-quality-main.yaml -------------------------------------------------------------------------------- /.github/workflows/code-quality-pr.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/.github/workflows/code-quality-pr.yaml -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/.github/workflows/test.yaml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/README.md -------------------------------------------------------------------------------- /data-explorations/blip2_video_qa.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/data-explorations/blip2_video_qa.ipynb -------------------------------------------------------------------------------- /data-explorations/data_exploration.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/data-explorations/data_exploration.ipynb -------------------------------------------------------------------------------- /data-explorations/eval_data_exploration.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/data-explorations/eval_data_exploration.ipynb -------------------------------------------------------------------------------- /data-explorations/hf_datasets.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/data-explorations/hf_datasets.ipynb -------------------------------------------------------------------------------- /data-explorations/instruct_blip_video_qa.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/data-explorations/instruct_blip_video_qa.ipynb -------------------------------------------------------------------------------- /data-explorations/pytorchvideo_load_ego4d_fho_clips.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/data-explorations/pytorchvideo_load_ego4d_fho_clips.ipynb -------------------------------------------------------------------------------- /data-explorations/test_ego4d_fho_main_dataset.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/data-explorations/test_ego4d_fho_main_dataset.ipynb -------------------------------------------------------------------------------- /data-explorations/test_epic_kitchens_dataset.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/data-explorations/test_epic_kitchens_dataset.ipynb -------------------------------------------------------------------------------- /data-explorations/video_blip_video_qa.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/data-explorations/video_blip_video_qa.ipynb -------------------------------------------------------------------------------- /demo/eilev_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/demo/eilev_demo.py -------------------------------------------------------------------------------- /demo/examples/bike-fixing-0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/demo/examples/bike-fixing-0.mp4 -------------------------------------------------------------------------------- /demo/examples/bike-fixing-1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/demo/examples/bike-fixing-1.mp4 -------------------------------------------------------------------------------- /demo/examples/dough-mixer.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/demo/examples/dough-mixer.mp4 -------------------------------------------------------------------------------- /demo/examples/motorcycle-riding-0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/demo/examples/motorcycle-riding-0.mp4 -------------------------------------------------------------------------------- /demo/examples/motorcycle-riding-1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/demo/examples/motorcycle-riding-1.mp4 -------------------------------------------------------------------------------- /demo/examples/paint.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/demo/examples/paint.mp4 -------------------------------------------------------------------------------- /demo/examples/trowel.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/demo/examples/trowel.mp4 -------------------------------------------------------------------------------- /demo/video_blip_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/demo/video_blip_demo.py -------------------------------------------------------------------------------- /eilev/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eilev/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eilev/data/ego4d.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/eilev/data/ego4d.py -------------------------------------------------------------------------------- /eilev/data/epic_kitchens.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/eilev/data/epic_kitchens.py -------------------------------------------------------------------------------- /eilev/data/frame.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/eilev/data/frame.py -------------------------------------------------------------------------------- /eilev/data/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/eilev/data/utils.py -------------------------------------------------------------------------------- /eilev/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /eilev/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/eilev/model/utils.py -------------------------------------------------------------------------------- /eilev/model/v1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/eilev/model/v1.py -------------------------------------------------------------------------------- /eilev/model/v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/eilev/model/v2.py -------------------------------------------------------------------------------- /figures/ablation_figures.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/figures/ablation_figures.ipynb -------------------------------------------------------------------------------- /figures/held_out_verb_noun.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/figures/held_out_verb_noun.ipynb -------------------------------------------------------------------------------- /figures/icl_eval_figures.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/figures/icl_eval_figures.ipynb -------------------------------------------------------------------------------- /figures/shuffle_in_context_figures.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/figures/shuffle_in_context_figures.ipynb -------------------------------------------------------------------------------- /figures/skewed_dist_figures.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/figures/skewed_dist_figures.ipynb -------------------------------------------------------------------------------- /figures/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/figures/teaser.png -------------------------------------------------------------------------------- /figures/tsne_ego4d_epic_kitchens.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/figures/tsne_ego4d_epic_kitchens.ipynb -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/poetry.lock -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/pyproject.toml -------------------------------------------------------------------------------- /run-mypy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/run-mypy -------------------------------------------------------------------------------- /samples/eilev_generate_action_narration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/samples/eilev_generate_action_narration.py -------------------------------------------------------------------------------- /samples/video_blip_generate_action_narration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/samples/video_blip_generate_action_narration.py -------------------------------------------------------------------------------- /scripts/baselines/majority/majority_generate_full_sent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/baselines/majority/majority_generate_full_sent.py -------------------------------------------------------------------------------- /scripts/baselines/majority/majority_predict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/baselines/majority/majority_predict.py -------------------------------------------------------------------------------- /scripts/baselines/videomae/videomae_generate_full_sent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/baselines/videomae/videomae_generate_full_sent.py -------------------------------------------------------------------------------- /scripts/baselines/videomae/videomae_predict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/baselines/videomae/videomae_predict.py -------------------------------------------------------------------------------- /scripts/baselines/videomae/videomae_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/baselines/videomae/videomae_train.py -------------------------------------------------------------------------------- /scripts/ego4d/add_structured_verb_noun.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/ego4d/add_structured_verb_noun.py -------------------------------------------------------------------------------- /scripts/ego4d/eval-data/structured_noun_prompt.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/ego4d/eval-data/structured_noun_prompt.csv -------------------------------------------------------------------------------- /scripts/ego4d/eval-data/structured_verb_prompt.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/ego4d/eval-data/structured_verb_prompt.csv -------------------------------------------------------------------------------- /scripts/ego4d/extract_frames.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/ego4d/extract_frames.py -------------------------------------------------------------------------------- /scripts/ego4d/generate_std_sent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/ego4d/generate_std_sent.py -------------------------------------------------------------------------------- /scripts/ego4d/split_held_out_verb_noun_train_val_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/ego4d/split_held_out_verb_noun_train_val_test.py -------------------------------------------------------------------------------- /scripts/ego4d/split_train_val_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/ego4d/split_train_val_test.py -------------------------------------------------------------------------------- /scripts/ego4d/verify_narrated_action_splits.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/ego4d/verify_narrated_action_splits.py -------------------------------------------------------------------------------- /scripts/epic-kitchens/epic_kitchens_extract_frames.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/epic-kitchens/epic_kitchens_extract_frames.py -------------------------------------------------------------------------------- /scripts/epic-kitchens/transform_to_full_sent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/epic-kitchens/transform_to_full_sent.py -------------------------------------------------------------------------------- /scripts/epic-kitchens/verify_epic_kitchens.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/epic-kitchens/verify_epic_kitchens.py -------------------------------------------------------------------------------- /scripts/general/combine_in_context_examples_generated_narrations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/general/combine_in_context_examples_generated_narrations.py -------------------------------------------------------------------------------- /scripts/general/generate_narration_texts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/general/generate_narration_texts.py -------------------------------------------------------------------------------- /scripts/general/generation_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/general/generation_eval.py -------------------------------------------------------------------------------- /scripts/general/get_vision_model_embs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/general/get_vision_model_embs.py -------------------------------------------------------------------------------- /scripts/general/icl_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/general/icl_eval.py -------------------------------------------------------------------------------- /scripts/general/sample_in_context_examples.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/general/sample_in_context_examples.py -------------------------------------------------------------------------------- /scripts/general/select_qualitative_examples.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/general/select_qualitative_examples.py -------------------------------------------------------------------------------- /scripts/general/train_v1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/general/train_v1.py -------------------------------------------------------------------------------- /scripts/general/train_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/scripts/general/train_v2.py -------------------------------------------------------------------------------- /slurm-scripts/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/slurm-scripts/README.md -------------------------------------------------------------------------------- /slurm-scripts/extract-frames/submit_ego4d_extract_frames.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/slurm-scripts/extract-frames/submit_ego4d_extract_frames.py -------------------------------------------------------------------------------- /slurm-scripts/extract-frames/submit_epic_kitchens_extract_frames.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/slurm-scripts/extract-frames/submit_epic_kitchens_extract_frames.py -------------------------------------------------------------------------------- /slurm-scripts/icl-eval/submit_generate_narration_texts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/slurm-scripts/icl-eval/submit_generate_narration_texts.py -------------------------------------------------------------------------------- /slurm-scripts/icl-eval/submit_majority_generate_full_sent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/slurm-scripts/icl-eval/submit_majority_generate_full_sent.py -------------------------------------------------------------------------------- /slurm-scripts/icl-eval/submit_videomae_generate_full_sent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/slurm-scripts/icl-eval/submit_videomae_generate_full_sent.py -------------------------------------------------------------------------------- /slurm-scripts/train/submit_train_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/slurm-scripts/train/submit_train_v2.py -------------------------------------------------------------------------------- /slurm-scripts/train/submit_videomae_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/slurm-scripts/train/submit_videomae_train.py -------------------------------------------------------------------------------- /tests/data/test_frame.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/tests/data/test_frame.py -------------------------------------------------------------------------------- /tests/data/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/tests/data/test_utils.py -------------------------------------------------------------------------------- /tests/model/test_model_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/tests/model/test_model_utils.py -------------------------------------------------------------------------------- /tests/model/test_model_v1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/tests/model/test_model_v1.py -------------------------------------------------------------------------------- /tests/model/test_model_v2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yukw777/EILEV/HEAD/tests/model/test_model_v2.py --------------------------------------------------------------------------------