├── vista_llama
    ├── eval
    │   ├── __init__.py
    │   ├── run_inference_benchmark_general.py
    │   ├── run_inference_benchmark_consistency.py
    │   ├── model_utils.py
    │   ├── run_inference_qformer_nextqa.py
    │   └── run_inference_qformer_general.py
    ├── __init__.py
    ├── .ipynb_checkpoints
    │   └── Untitled-checkpoint.ipynb
    ├── model
    │   ├── __init__.py
    │   ├── consolidate.py
    │   ├── utils.py
    │   ├── make_delta.py
    │   ├── dist_utils.py
    │   ├── logger.py
    │   ├── base_model.py
    │   └── blip2.py
    ├── constants.py
    ├── train
    │   ├── train_mem.py
    │   ├── llava_trainer.py
    │   └── llama_flash_attn_monkey_patch.py
    ├── inference.py
    ├── utils.py
    └── video_conversation.py
├── MSRVTT-QA
    ├── readme.txt
    └── category.txt
├── requirements.txt
├── README.md
├── scripts
    ├── set_env.sh
    ├── convert_instruction_json_to_training_format.py
    ├── run_infer.sh
    ├── apply_delta.py
    ├── train_val_eval.sh
    ├── save_evaclip_features.py
    ├── save_spatio_temporal_clip_features.py
    └── save_patch_clip_features.py
├── quantitative_evaluation
    ├── evaluate_benchmark.sh
    ├── analyze_nextqa.py
    ├── benchmark_dataset_generation
    │   ├── generate_correctness_detailed_context_qa.py
    │   ├── generate_temporal_qa.py
    │   └── generate_consistency_qa.py
    ├── evaluate_benchmark_1_correctness.py
    ├── evaluate_activitynet_qa.py
    ├── evaluate_benchmark_4_temporal.py
    ├── evaluate_nextqa.py
    ├── evaluate_benchmark_2_detailed_orientation.py
    ├── evaluate_benchmark_3_context.py
    └── evaluate_benchmark_5_consistency.py
├── data
    ├── README.md
    └── generate_descriptive_qa.py
└── test.yaml


/vista_llama/eval/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/vista_llama/__init__.py:
--------------------------------------------------------------------------------
1 | from .model import VideoChatGPTLlamaForCausalLM
2 | 


--------------------------------------------------------------------------------
/vista_llama/.ipynb_checkpoints/Untitled-checkpoint.ipynb:
--------------------------------------------------------------------------------
1 | {
2 |  "cells": [],
3 |  "metadata": {},
4 |  "nbformat": 4,
5 |  "nbformat_minor": 5
6 | }
7 | 


--------------------------------------------------------------------------------
/vista_llama/model/__init__.py:
--------------------------------------------------------------------------------
1 | from .video_chatgpt import VideoChatGPTLlamaForCausalLM, VideoChatGPTConfig
2 | from .vista_llama_qformer import VistaLLaMAQformerLlamaForCausalLM
3 | 


--------------------------------------------------------------------------------
/vista_llama/constants.py:
--------------------------------------------------------------------------------
 1 | CONTROLLER_HEART_BEAT_EXPIRATION = 30
 2 | WORKER_HEART_BEAT_INTERVAL = 15
 3 | 
 4 | LOGDIR = "."
 5 | 
 6 | 
 7 | # Defining model
 8 | DEFAULT_VIDEO_TOKEN = "<video>"
 9 | DEFAULT_VIDEO_PATCH_TOKEN = "<vid_patch>"
10 | DEFAULT_VID_START_TOKEN = "<vid_start>"
11 | DEFAULT_VID_END_TOKEN = "<vid_end>"
12 | 


--------------------------------------------------------------------------------
/MSRVTT-QA/readme.txt:
--------------------------------------------------------------------------------
1 | MSRVTT-QA
2 | 
3 | The dataset is based on MSR-VTT (http://ms-multimedia-challenge.com/2016/dataset), please put the corresponding videos under the directory 'video'.
4 | 
5 | Data split(video_id):
6 | Train: video0 : video6512 (6513)
7 | Val: video6513 : video7009 (497)
8 | Test: video7010 : video9999 (2990) 


--------------------------------------------------------------------------------
/vista_llama/train/train_mem.py:
--------------------------------------------------------------------------------
 1 | # Need to call this before importing transformers.
 2 | from vista_llama.train.llama_flash_attn_monkey_patch import replace_llama_attn_with_flash_attn
 3 | 
 4 | replace_llama_attn_with_flash_attn()
 5 | 
 6 | from vista_llama.train.train import train
 7 | 
 8 | if __name__ == "__main__":
 9 |     train()
10 | 


--------------------------------------------------------------------------------
/MSRVTT-QA/category.txt:
--------------------------------------------------------------------------------
 1 | music	0
 2 | people	1
 3 | gaming	2
 4 | sports/actions	3
 5 | news/events/politics	4
 6 | education	5
 7 | tv shows	6
 8 | movie/comedy	7
 9 | animation	8
10 | vehicles/autos	9
11 | howto	10
12 | travel	11
13 | science/technology	12
14 | animals/pets	13
15 | kids/family	14
16 | documentary	15
17 | food/drink	16
18 | cooking	17
19 | beauty/fashion	18
20 | advertisement	19
21 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | torch~=2.0.1
 2 | tqdm~=4.65.0
 3 | transformers@git+https://github.com/huggingface/transformers.git@cae78c46
 4 | # transformers==4.28.0.dev0
 5 | # transformers==4.28.1
 6 | numpy~=1.24.3
 7 | Pillow~=9.5.0
 8 | decord~=0.6.0
 9 | gradio~=3.23.0
10 | markdown2~=2.4.8
11 | einops~=0.6.1
12 | requests~=2.30.0
13 | sentencepiece~=0.1.99
14 | protobuf~=4.23.2
15 | accelerate~=0.20.3
16 | tokenizers>=0.13.3
17 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Vista-LLaMA
 2 | 
 3 | 
 4 | ### Vista-LLaMA: educing Hallucination in Video Language Models via Equal Distance to Visual Tokens
 5 | 
 6 | 
 7 | 
 8 | 
 9 | ## Installation :wrench:
10 | 
11 | We recommend setting up a conda environment for the project:
12 | ```shell
13 | bash script/set_env.sh
14 | ```
15 | ---
16 | 
17 | ## Dataset
18 | 
19 | Follow the instructions in [Video-ChatGPT](https://github.com/mbzuai-oryx/Video-ChatGPT)
20 | If Eva-CLIP-G is used to extract video features, use the scripts/save_evaclip_feature to extract activitynet features
21 | 
22 | ## Training Inference and Evalaution:
23 | 
24 | ```shell
25 | bash script/train_val_eval.sh
26 | ```
27 | 
28 | 
29 | ## Test model on the Next-QA dataset
30 | 
31 | ```shell
32 | bash script/run_infer.sh
33 | ```
34 | 


--------------------------------------------------------------------------------
/scripts/set_env.sh:
--------------------------------------------------------------------------------
 1 | cd /opt/tiger
 2 | 
 3 | 
 4 | sudo apt-get update
 5 | sudo apt-get install git-lfs
 6 | wget https://www.python.org/ftp/python/3.10.0/Python-3.10.0.tgz
 7 | tar -xvf Python-3.10.0.tgz
 8 | cd Python-3.10.0
 9 | sudo ./configure --enable-optimizations
10 | sudo make altinstall
11 | 
12 | cd /opt/tiger/VideoLLM/
13 | python3.10 -m pip install -r requirements.txt
14 | python3.10 -m pip install httpx==0.23.0
15 | python3.10 -m pip install ninja
16 | python3.10 -m pip install pydantic==1.10.9
17 | python3.10 -m pip install langchain timm omegaconf
18 | 
19 | 
20 | cd /mnt/bn/vlp-v6/flash-attention
21 | python3.10 setup.py install --user
22 | 
23 | python3.10 -m pip install openai==0.28.0
24 | 
25 | ### NextQA dataset
26 | cd /opt/tiger/
27 | unzip /mnt/bn/vlp-lq/NExTVideo.zip .
28 | cd VideoLLM
29 | cp -r /mnt/bn/vlp-v6/VideoLLM/LLaVA-7B-Lightening-v1-1/ .
30 | 
31 | 
32 | 


--------------------------------------------------------------------------------
/vista_llama/model/consolidate.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Usage:
 3 | python3 -m llava.model.consolidate --src ~/model_weights/llava-7b --dst ~/model_weights/llava-7b_consolidate
 4 | """
 5 | import argparse
 6 | 
 7 | import torch
 8 | from transformers import AutoTokenizer, AutoModelForCausalLM
 9 | from vista_llama.model import *
10 | 
11 | 
12 | def consolidate_ckpt(src_path, dst_path):
13 |     print("Loading model")
14 |     src_model = AutoModelForCausalLM.from_pretrained(src_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
15 |     src_tokenizer = AutoTokenizer.from_pretrained(src_path)
16 |     src_model.save_pretrained(dst_path)
17 |     src_tokenizer.save_pretrained(dst_path)
18 | 
19 | 
20 | if __name__ == "__main__":
21 |     parser = argparse.ArgumentParser()
22 |     parser.add_argument("--src", type=str, required=True)
23 |     parser.add_argument("--dst", type=str, required=True)
24 | 
25 |     args = parser.parse_args()
26 | 
27 |     consolidate_ckpt(args.src, args.dst)
28 | 


--------------------------------------------------------------------------------
/scripts/convert_instruction_json_to_training_format.py:
--------------------------------------------------------------------------------
 1 | import json
 2 | import argparse
 3 | 
 4 | 
 5 | def parse_args():
 6 |     parser = argparse.ArgumentParser(description="Training")
 7 | 
 8 |     parser.add_argument("--input_json_file", required=True, help="")
 9 |     parser.add_argument("--output_json_file", required=True, help="")
10 | 
11 |     args = parser.parse_args()
12 | 
13 |     return args
14 | 
15 | 
16 | def main():
17 |     args = parse_args()
18 |     input_json_file = args.input_json_file
19 |     output_json_file = args.output_json_file
20 | 
21 |     input_json_contents = json.load(open(input_json_file, 'r'))
22 |     output_json_contents = []
23 |     for i, content in enumerate(input_json_contents):
24 |         output_content = {'id': content['video_id'], 'video': f"{content['video_id']}.pkl", 'conversations': []}
25 | 
26 |         if i % 2 == 0:
27 |             output_content['conversations'].append({'from': 'human', 'value': f"{content['q']}\n<video>"})
28 |         else:
29 |             output_content['conversations'].append({'from': 'human', 'value': f"<video>\n{content['q']}"})
30 | 
31 |         output_content['conversations'].append({'from': 'gpt', 'value': content['a']})
32 |         output_json_contents.append(output_content)
33 | 
34 |     print(f"Total annotations retained: {len(output_json_contents)}")
35 |     with open(output_json_file, 'w') as f:
36 |         json.dump(output_json_contents, f)
37 | 
38 | 
39 | if __name__ == "__main__":
40 |     main()
41 | 


--------------------------------------------------------------------------------
/scripts/run_infer.sh:
--------------------------------------------------------------------------------
 1 | dataset=nextqa
 2 | video_dir=data/NExTVideo/
 3 | gt_file=./nextqa/val.json
 4 | checkpoints_dir=./checkpoints
 5 | 
 6 | 
 7 | model_name=qformer_patch12frameseva257_seq_3ep
 8 | model_name=qformer_patch16frameseva257_seqmask_3ep
 9 | output_pred_name=test_eval_mask
10 | eval_dir=./evaluation/${model_name}/${output_pred_name}
11 | video_token_len=128
12 | num_video_frame=8
13 | visual_model=evaclip
14 | # visual_model=clip14l
15 | qformer_type=seq
16 | feature_dim=257
17 | val_file=vista_llama/eval/run_inference_qformer_nextqa_patch.py
18 | api_key=API_KEY
19 | 
20 | 
21 | # python -m torch.distributed.run --nproc_per_node=2 --master_port 11322 $val_file \
22 | # CUDA_VISIBLE_DEVICES=2 python $val_file \
23 | CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.run --nproc_per_node=2 --master_port 11022 $val_file \
24 |             --dataset $dataset \
25 |             --video_dir $video_dir \
26 |             --gt_file $gt_file \
27 |             --video_token_len $video_token_len \
28 |             --output_dir $checkpoints_dir/$model_name \
29 |             --num_video_frame $num_video_frame \
30 |             --feature_dim $feature_dim \
31 |             --output_name $output_pred_name \
32 |             --qformer_type $qformer_type \
33 |             --visual_model $visual_model \
34 |             --projection_path $checkpoints_dir/$model_name/mm_projector.bin \
35 |             --model-name LLaVA-7B-Lightening-v1-1 
36 | 
37 | 
38 | python3.10 quantitative_evaluation/evaluate_nextqa.py \
39 |             --pred_path $checkpoints_dir/$model_name/$output_pred_name.json \
40 |             --output_dir ${eval_dir} \
41 |             --num_task 10 \
42 |             --api_key $API_KEY \
43 |             --output_json ${output_pred_name}.json 
44 | 


--------------------------------------------------------------------------------
/vista_llama/model/utils.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import torch
 3 | import torch.nn as nn
 4 | from vista_llama.model import *
 5 | from transformers import StoppingCriteria
 6 | from urllib.parse import urlparse
 7 | 
 8 | 
 9 | class KeywordsStoppingCriteria(StoppingCriteria):
10 |     def __init__(self, keywords, tokenizer, input_ids):
11 |         self.keywords = keywords
12 |         self.keyword_ids = [tokenizer(keyword).input_ids for keyword in keywords]
13 |         self.keyword_ids = [keyword_id[0] for keyword_id in self.keyword_ids if type(keyword_id) is list and len(keyword_id) == 1]
14 |         self.tokenizer = tokenizer
15 |         self.start_len = None
16 |         self.input_ids = input_ids
17 | 
18 |     def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
19 |         if self.start_len is None:
20 |             self.start_len = self.input_ids.shape[1]
21 |         else:
22 |             for keyword_id in self.keyword_ids:
23 |                 if output_ids[0, -1] == keyword_id:
24 |                     return True
25 |             outputs = self.tokenizer.batch_decode(output_ids[:, self.start_len:], skip_special_tokens=True)[0]
26 |             for keyword in self.keywords:
27 |                 if keyword in outputs:
28 |                     return True
29 |         return False
30 | 
31 | 
32 | 
33 | 
34 | 
35 | def get_abs_path(rel_path):
36 |     return os.path.join("../../../Video-ChatGPT", rel_path)
37 | 
38 | def is_url(url_or_filename):
39 |     parsed = urlparse(url_or_filename)
40 |     return parsed.scheme in ("http", "https")
41 | 
42 | class LayerNorm(nn.LayerNorm):
43 |     """Subclass torch's LayerNorm to handle fp16."""
44 | 
45 |     def forward(self, x: torch.Tensor):
46 |         orig_type = x.dtype
47 |         ret = super().forward(x.type(torch.float32))
48 |         return ret.type(orig_type)


--------------------------------------------------------------------------------
/scripts/apply_delta.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Usage:
 3 | python TODO: Add usage
 4 | """
 5 | import argparse
 6 | 
 7 | import torch
 8 | from tqdm import tqdm
 9 | from transformers import AutoTokenizer, AutoModelForCausalLM
10 | from vista_llama.model.video_chatgpt import VideoChatGPTLlamaForCausalLM
11 | 
12 | 
13 | def apply_delta(base_model_path, target_model_path, delta_path):
14 |     print("Loading base model")
15 |     base = AutoModelForCausalLM.from_pretrained(
16 |         base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
17 | 
18 |     print("Loading delta")
19 |     delta = VideoChatGPTLlamaForCausalLM.from_pretrained(delta_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
20 |     delta_tokenizer = AutoTokenizer.from_pretrained(delta_path)
21 | 
22 |     print("Applying delta")
23 |     for name, param in tqdm(delta.state_dict().items(), desc="Applying delta"):
24 |         if name not in base.state_dict():
25 |             assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model'
26 |             continue
27 |         if param.data.shape == base.state_dict()[name].shape:
28 |             param.data += base.state_dict()[name]
29 |         else:
30 |             assert name in ['model.embed_tokens.weight', 'lm_head.weight'], \
31 |                 f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}'
32 |             bparam = base.state_dict()[name]
33 |             param.data[:bparam.shape[0], :bparam.shape[1]] += bparam
34 | 
35 |     print("Saving target model")
36 |     delta.save_pretrained(target_model_path)
37 |     delta_tokenizer.save_pretrained(target_model_path)
38 | 
39 | 
40 | if __name__ == "__main__":
41 |     parser = argparse.ArgumentParser()
42 |     parser.add_argument("--base-model-path", type=str, required=True)
43 |     parser.add_argument("--target-model-path", type=str, required=True)
44 |     parser.add_argument("--delta-path", type=str, required=True)
45 | 
46 |     args = parser.parse_args()
47 | 
48 |     apply_delta(args.base_model_path, args.target_model_path, args.delta_path)
49 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/evaluate_benchmark.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # Define common arguments for all scripts
 4 | # PRED_DIR=checkpoints/qformer_patch2frameseva257_seqnomask_3ep
 5 | PRED_DIR=data
 6 | PRED_DIR=
 7 | OUTPUT_DIR=
 8 | API_KEY=
 9 | NUM_TASKS=5
10 | 
11 | # Run the "correctness" evaluation script
12 | python3.10 quantitative_evaluation/evaluate_benchmark_1_correctness.py \
13 |   --pred_path "${PRED_DIR}/generic_preds.json" \
14 |   --output_dir "${OUTPUT_DIR}/correctness_eval" \
15 |   --output_json "${OUTPUT_DIR}/correctness_results.json" \
16 |   --api_key $API_KEY \
17 |   --num_tasks $NUM_TASKS
18 | 
19 | # Run the "detailed orientation" evaluation script
20 | python3.10 quantitative_evaluation/evaluate_benchmark_2_detailed_orientation.py \
21 |   --pred_path "${PRED_DIR}/generic_preds.json" \
22 |   --output_dir "${OUTPUT_DIR}/detailed_eval" \
23 |   --output_json "${OUTPUT_DIR}/detailed_orientation_results.json" \
24 |   --api_key $API_KEY \
25 |   --num_tasks $NUM_TASKS
26 | 
27 | # Run the "contextual understanding" evaluation script
28 | python3.10 quantitative_evaluation/evaluate_benchmark_3_context.py \
29 |   --pred_path "${PRED_DIR}/generic_preds.json" \
30 |   --output_dir "${OUTPUT_DIR}/context_eval" \
31 |   --output_json "${OUTPUT_DIR}/contextual_understanding_results.json" \
32 |   --api_key $API_KEY \
33 |   --num_tasks $NUM_TASKS
34 | 
35 | # Run the "temporal understanding" evaluation script
36 | python3.10 quantitative_evaluation/evaluate_benchmark_4_temporal.py \
37 |   --pred_path "${PRED_DIR}/temporal_preds.json" \
38 |   --output_dir "${OUTPUT_DIR}/temporal_eval" \
39 |   --output_json "${OUTPUT_DIR}/temporal_understanding_results.json" \
40 |   --api_key $API_KEY \
41 |   --num_tasks $NUM_TASKS
42 | 
43 | # Run the "consistency" evaluation script
44 | python3.10 quantitative_evaluation/evaluate_benchmark_5_consistency.py \
45 |   --pred_path "${PRED_DIR}/consistency_preds.json" \
46 |   --output_dir "${OUTPUT_DIR}/consistency_eval" \
47 |   --output_json "${OUTPUT_DIR}/consistency_results.json" \
48 |   --api_key $API_KEY \
49 |   --num_tasks $NUM_TASKS
50 | 
51 | 
52 | echo "All evaluations completed!"
53 | 


--------------------------------------------------------------------------------
/vista_llama/model/make_delta.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Usage:
 3 | python
 4 | """
 5 | import argparse
 6 | 
 7 | import torch
 8 | from tqdm import tqdm
 9 | from transformers import AutoTokenizer, AutoModelForCausalLM
10 | 
11 | 
12 | def make_delta(base_model_path, target_model_path, delta_path, hub_repo_id):
13 |     print("Loading base model")
14 |     base = AutoModelForCausalLM.from_pretrained(
15 |         base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
16 | 
17 |     print("Loading target model")
18 |     target = AutoModelForCausalLM.from_pretrained(target_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True)
19 | 
20 |     print("Calculating delta")
21 |     for name, param in tqdm(target.state_dict().items(), desc="Calculating delta"):
22 |         if name not in base.state_dict():
23 |             assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model'
24 |             continue
25 |         if param.data.shape == base.state_dict()[name].shape:
26 |             param.data -= base.state_dict()[name]
27 |         else:
28 |             assert name in ['model.embed_tokens.weight', 'lm_head.weight'], f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}'
29 |             bparam = base.state_dict()[name]
30 |             param.data[:bparam.shape[0], :bparam.shape[1]] -= bparam
31 | 
32 |     print("Saving delta")
33 |     if hub_repo_id:
34 |         kwargs = {"push_to_hub": True, "repo_id": hub_repo_id}
35 |     else:
36 |         kwargs = {}
37 |     target.save_pretrained(delta_path, **kwargs)
38 |     target_tokenizer = AutoTokenizer.from_pretrained(target_model_path)
39 |     target_tokenizer.save_pretrained(delta_path, **kwargs)
40 | 
41 | 
42 | if __name__ == "__main__":
43 |     parser = argparse.ArgumentParser()
44 |     parser.add_argument("--base-model-path", type=str, required=True)
45 |     parser.add_argument("--target-model-path", type=str, required=True)
46 |     parser.add_argument("--delta-path", type=str, required=True)
47 |     parser.add_argument("--hub-repo-id", type=str, default=None)
48 |     args = parser.parse_args()
49 | 
50 |     make_delta(args.base_model_path, args.target_model_path, args.delta_path, args.hub_repo_id)
51 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/analyze_nextqa.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import argparse
 3 | import json
 4 | import ast
 5 | from multiprocessing.pool import Pool
 6 | 
 7 | 
 8 | def parse_args():
 9 |     parser = argparse.ArgumentParser(
10 |         description="question-answer-generation-using-gpt-3"
11 |     )
12 |     parser.add_argument(
13 |         "--pred_path", required=True, help="The path to file containing prediction."
14 |     )
15 |     # parser.add_argument("--api_key", required=True, help="OpenAI API key.")
16 |     args = parser.parse_args()
17 |     return args
18 | 
19 | 
20 | def main():
21 |     """
22 |     Main function to control the flow of the program.
23 |     """
24 |     # Parse arguments.
25 |     args = parse_args()
26 | 
27 |     file = open(args.pred_path)
28 |     print(f"load prediction file from {args.pred_path}")
29 |     pred_contents = json.load(file)
30 | 
31 |     # Calculate average score and accuracy
32 |     score_sum = 0
33 |     count = 0
34 |     T_count = [0, 0, 0]
35 |     C_count = [0, 0, 0]
36 |     D_count = [0, 0, 0]
37 |     yes_count = 0
38 |     no_count = 0
39 |     for key, result in pred_contents.items():
40 |         # Computing score
41 |         count += 1
42 |         try:
43 |             score_match = result[0]["score"]
44 |         except:
45 |             print(result)
46 |             continue
47 |         score = int(score_match)
48 |         score_sum += score
49 | 
50 |         # Computing accuracy
51 |         pred = result[0]["pred"]
52 |         if "yes" in pred.lower():
53 |             yes_count += 1
54 |         elif "no" in pred.lower():
55 |             no_count += 1
56 |         if key.startswith('T'):
57 |             T_count[0] += 1
58 |             if "yes" in pred.lower():
59 |                 T_count[1] += 1
60 |             elif "no" in pred.lower():
61 |                 T_count[2] += 1
62 |         elif key.startswith('D'):
63 |             D_count[0] += 1
64 |             if "yes" in pred.lower():
65 |                 D_count[1] += 1
66 |             elif "no" in pred.lower():
67 |                 D_count[2] += 1
68 |         elif key.startswith('C'):
69 |             C_count[0] += 1
70 |             if "yes" in pred.lower():
71 |                 C_count[1] += 1
72 |             elif "no" in pred.lower():
73 |                 C_count[2] += 1
74 |             
75 |         
76 | 
77 |     average_score = score_sum / count
78 |     accuracy = yes_count / (yes_count + no_count)
79 |     print("Yes count:", yes_count)
80 |     print("No count:", no_count)
81 |     print("Accuracy:", accuracy)
82 |     print("T Accuracy:", T_count[1] / (T_count[1] + T_count[2]))
83 |     print("D Accuracy:", D_count[1] / (D_count[1] + D_count[2]))
84 |     print("C Accuracy:", C_count[1] / (C_count[1] + C_count[2]))
85 |     print("Average score:", average_score)
86 | 
87 | 
88 | if __name__ == "__main__":
89 |     main()
90 | 


--------------------------------------------------------------------------------
/scripts/train_val_eval.sh:
--------------------------------------------------------------------------------
 1 | train_file=vista_llama/train/train_qformer_patch.py
 2 | val_file=vista_llama/eval/run_inference_qformer_nextqa_patch.py
 3 | test_data=nextqa
 4 | checkpoint_dir=./checkpoints/
 5 | API_KEY=""
 6 | 
 7 | 
 8 | 
 9 | feature_dir=/mnt/bn/vlp-lq/ActivityNet_Feature
10 | output_pred_name=${test_data}_preds
11 | 
12 | 
13 | 
14 | ### model setting
15 | model_name=qformerblipinit_patch8frames
16 | video_token_len=128
17 | num_video_frame=8
18 | num_epoch=3
19 | output_dir=$checkpoint_dir/$model_name
20 | feature_dim=256
21 | visual_model=clip14l
22 | qformer_type=seq
23 | eval_dir=/opt/tiger/evaluation/$model_name/$test_data
24 | 
25 | python3.10 -m torch.distributed.run --nproc_per_node=1 --master_port 9912 $train_file \
26 |           --model_name_or_path ./LLaVA-7B-Lightening-v1-1 \
27 |           --version v1 \
28 |           --data_path video_chatgpt_training1.json \
29 |           --video_folder $feature_dir \
30 |           --tune_mm_mlp_adapter True \
31 |           --mm_use_vid_start_end \
32 |           --bf16 True \
33 |           --output_dir $output_dir \
34 |           --num_train_epochs $num_epoch \
35 |           --qformer_type $qformer_type \
36 |           --per_device_train_batch_size 4 \
37 |           --per_device_eval_batch_size 4 \
38 |           --gradient_accumulation_steps 1 \
39 |           --evaluation_strategy "no" \
40 |           --save_strategy "steps" \
41 |           --save_steps 3000 \
42 |           --save_total_limit 3 \
43 |           --learning_rate 2e-5 \
44 |           --weight_decay 0. \
45 |           --warmup_ratio 0.03 \
46 |           --use_visual_mask True \
47 |           --video_token_len $video_token_len \
48 |           --lr_scheduler_type "cosine" \
49 |           --logging_steps 100 \
50 |           --tf32 True \
51 |           --model_max_length 2048 \
52 |           --gradient_checkpointing True \
53 |           --lazy_preprocess True
54 | 
55 | # # python3.10 $val_file \
56 | python3.10 -m torch.distributed.run --nproc_per_node=8 --master_port 11222 $val_file \
57 |             --video_dir /opt/tiger/NExTVideo \
58 |             --gt_file /mnt/bn/vlp-v6/VideoLLM/nextqa/val.json \
59 |             --video_token_len $video_token_len \
60 |             --output_dir $output_dir \
61 |             --output_name $output_pred_name \
62 |             --num_video_frame $num_video_frame \
63 |             --visual_model $visual_model \
64 |             --qformer_type $qformer_type \
65 |             --feature_dim $feature_dim \
66 |             --projection_path $output_dir/mm_projector.bin \
67 |             --use_visual_mask \
68 |             --model-name LLaVA-7B-Lightening-v1-1 
69 | 
70 | # python3.10 quantitative_evaluation/evaluate_nextqa_gpt0613.py \
71 | python3.10 quantitative_evaluation/evaluate_nextqa.py \
72 |             --pred_path $checkpoints_dir/$model_name/$output_pred_name.json \
73 |             --output_dir ${eval_dir} \
74 |             --num_task 10 \
75 |             --api_key  $API_KEY\
76 |             --output_json ${output_pred_name}.json 


--------------------------------------------------------------------------------
/data/README.md:
--------------------------------------------------------------------------------
 1 | # Video Instruction Data Generation
 2 | 
 3 | This README outlines the method used for generating video instruction data, a resource tailored for the fine-tuning of Video-ChatGPT. Our approach employs a combination of human-assisted and semi-automatic annotation techniques, aiming to produce high-quality video instruction data.
 4 | 
 5 | These methods allow us to create question-answer pairs related to 
 6 | 
 7 | - Video summarization
 8 | - Description-based question-answers (exploring spatial, temporal, relationships, and reasoning concepts)
 9 | - Creative/generative question-answers
10 | 
11 | Explore a few sample instances here:
12 | ![sample1](../docs/dataset_samples/dataset_sample_1.png)
13 | 
14 | ![sample2](../docs/dataset_samples/dataset_sample_2.png)
15 | 
16 | **Download Data**: You can download the 100K video-instruction data from 
17 | [here](https://mbzuaiac-my.sharepoint.com/:u:/g/personal/hanoona_bangalath_mbzuai_ac_ae/EWxYslvDeX1PijKWM_WxTkkBDXDDD350YnUQOkbcL8V7Xg?e=Lq9itD).
18 | 
19 | ## Human-Assisted Annotations
20 | 
21 | Utilizing video-caption pair datasets, we apply the expertise of human annotators to enrich the original ground truth annotations. This step is a continuous endeavor to enhance our dataset's quality and diversity
22 | 
23 | This figure shows how video-captions from a standard dataset are enriched and detailed by human annotation sample.
24 | ![sample1](../docs/dataset_samples/origina_vs_human.png)
25 | 
26 | Question-answer pairs are generated on the enriched human-assisted video-descriptions, covering the various themes discussed above. For additional details, please refer to the script [`generate_instruction_qa_human_assisted.py`](generate_instruction_qa_human_assisted.py).
27 | 
28 | ## Semi-Automatic Annotation Framework
29 | 
30 | To supplement the human-assisted annotations, we propose a semi-automatic annotation framework. This strategy leverages the advanced capabilities of dense image vision-language models, providing a cost-effective, scalable solution to boost our dataset's volume without compromising quality significantly.
31 | 
32 | The tools incorporated in this process include:
33 | 
34 | - **BLIP-v2**: Generates frame-level descriptions
35 | - **GRIT**: Produces detailed captions
36 | - **Tag2Text**: Generates tags for keyframes, serving as a filtering mechanism
37 | 
38 | The 'dense noisy captions' or the additional context information is derived by summarizing the predictions from these off-the-shelf models using the GPT3.5-Turbo model. Check out how video-captions from a standard dataset are enriched and detailed by the semi-automatic annotation method in this sample:
39 | ![sample2](../docs/dataset_samples/original_vs_semi-annotated.png)
40 | 
41 | These 'dense noisy captions' are combined with the ground truth video-captions to generate further instructional question-answer pairs. For more insights, refer to the script [`generate_instruction_qa_semi_automatic.py`](generate_instruction_qa_semi_automatic.py).
42 | 
43 | ## Detailed Descriptive Annotations
44 | 
45 | We also generate question-answer pairs aimed at long and detailed video descriptions by leveraging video-caption pair datasets. To understand the prompts employed for generating these descriptive annotations, please refer to the script [`generate_descriptive_qa.py`](generate_descriptive_qa.py).
46 | 


--------------------------------------------------------------------------------
/vista_llama/eval/run_inference_benchmark_general.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import argparse
 3 | import json
 4 | from tqdm import tqdm
 5 | from vista_llama.eval.model_utils import initialize_model, load_video
 6 | from vista_llama.inference import video_chatgpt_infer
 7 | 
 8 | 
 9 | def parse_args():
10 |     """
11 |     Parse command-line arguments.
12 |     """
13 |     parser = argparse.ArgumentParser()
14 | 
15 |     # Define the command-line arguments
16 |     parser.add_argument('--video_dir', help='Directory containing video files.', required=True)
17 |     parser.add_argument('--gt_file', help='Path to the ground truth file.', required=True)
18 |     parser.add_argument('--output_dir', help='Directory to save the model results JSON.', required=True)
19 |     parser.add_argument('--output_name', help='Name of the file for storing results JSON.', required=True)
20 |     parser.add_argument("--model-name", type=str, required=True)
21 |     parser.add_argument("--conv-mode", type=str, required=False, default='video-chatgpt_v1')
22 |     parser.add_argument("--projection_path", type=str, required=True)
23 | 
24 |     return parser.parse_args()
25 | 
26 | 
27 | def run_inference(args):
28 |     """
29 |     Run inference on a set of video files using the provided model.
30 | 
31 |     Args:
32 |         args: Command-line arguments.
33 |     """
34 |     # Initialize the model
35 |     model, vision_tower, tokenizer, image_processor, video_token_len = initialize_model(args.model_name,
36 |                                                                                         args.projection_path)
37 |     # Load the ground truth file
38 |     with open(args.gt_file) as file:
39 |         gt_contents = json.load(file)
40 | 
41 |     # Create the output directory if it doesn't exist
42 |     if not os.path.exists(args.output_dir):
43 |         os.makedirs(args.output_dir)
44 | 
45 |     output_list = []  # List to store the output results
46 |     conv_mode = args.conv_mode
47 | 
48 |     video_formats = ['.mp4', '.avi', '.mov', '.mkv']
49 | 
50 |     # Iterate over each sample in the ground truth file
51 |     for sample in tqdm(gt_contents):
52 |         video_name = sample['video_name']
53 |         sample_set = sample
54 |         question = sample['Q']
55 | 
56 |         # Load the video file
57 |         for fmt in video_formats:  # Added this line
58 |             temp_path = os.path.join(args.video_dir, f"{video_name}{fmt}")
59 |             if os.path.exists(temp_path):
60 |                 video_path = temp_path
61 |                 break
62 | 
63 |         # Check if the video exists
64 |         if video_path is not None:  # Modified this line
65 |             video_frames = load_video(video_path)
66 | 
67 |         try:
68 |             # Run inference on the video and add the output to the list
69 |             output = video_chatgpt_infer(video_frames, question, conv_mode, model, vision_tower,
70 |                                              tokenizer, image_processor, video_token_len)
71 |             sample_set['pred'] = output
72 |             output_list.append(sample_set)
73 |         except Exception as e:
74 |             print(f"Error processing video file '{video_name}': {e}")
75 | 
76 |     # Save the output list to a JSON file
77 |     with open(os.path.join(args.output_dir, f"{args.output_name}.json"), 'w') as file:
78 |         json.dump(output_list, file)
79 | 
80 | 
81 | if __name__ == "__main__":
82 |     args = parse_args()
83 |     run_inference(args)
84 | 


--------------------------------------------------------------------------------
/vista_llama/train/llava_trainer.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import torch
 3 | import torch.nn as nn
 4 | from transformers import Trainer
 5 | from typing import Optional
 6 | 
 7 | 
 8 | def unwrap_model(model: nn.Module) -> nn.Module:
 9 |     """
10 |     Recursively unwraps a model from potential containers (as used in distributed training).
11 | 
12 |     Args:
13 |         model (`torch.nn.Module`): The model to unwrap.
14 |     """
15 |     # since there could be multiple levels of wrapping, unwrap recursively
16 |     if hasattr(model, "module"):
17 |         return unwrap_model(model.module)
18 |     else:
19 |         return model
20 | 
21 | 
22 | class VideoChatGPTTrainer(Trainer):
23 | 
24 |     def _save(self, output_dir: Optional[str] = None, state_dict=None):
25 |         os.makedirs(output_dir, exist_ok=True)
26 |         if getattr(self.args, 'tune_mm_mlp_adapter', False):
27 |             # Save the model
28 |             _state_dict = state_dict
29 |             if _state_dict is None:
30 |                 # Only save the model itself if we are using distributed training
31 |                 model_to_save = unwrap_model(self.model)
32 |                 _state_dict = model_to_save.state_dict()
33 | 
34 |             weight_to_save = {}
35 |             keys_to_match = ['mm_projector', 'embed_tokens', 'embed_in']
36 |             for k, v in _state_dict.items():
37 |                 if any(key_match in k for key_match in keys_to_match):
38 |                     weight_to_save[k] = v
39 | 
40 |             current_folder = output_dir.split('/')[-1]
41 |             parent_folder = os.path.dirname(output_dir)
42 |             if current_folder.startswith('checkpoint-'):
43 |                 mm_projector_folder = os.path.join(parent_folder, "mm_projector")
44 |                 os.makedirs(mm_projector_folder, exist_ok=True)
45 |                 torch.save(weight_to_save, os.path.join(mm_projector_folder, f'{current_folder}.bin'))
46 |             else:
47 |                 torch.save(weight_to_save, os.path.join(output_dir, f'mm_projector.bin'), )
48 | 
49 |         # super(VideoChatGPTTrainer, self)._save(output_dir, state_dict)
50 | 
51 | class VistaLLaMAQformerTrainer(Trainer):
52 | 
53 |     def _save(self, output_dir: Optional[str] = None, state_dict=None):
54 |         os.makedirs(output_dir, exist_ok=True)
55 |         if getattr(self.args, 'tune_mm_mlp_adapter', False):
56 |             # Save the model
57 |             _state_dict = state_dict
58 |             if _state_dict is None:
59 |                 # Only save the model itself if we are using distributed training
60 |                 model_to_save = unwrap_model(self.model)
61 |                 _state_dict = model_to_save.state_dict()
62 | 
63 |             weight_to_save = {}
64 |             keys_to_match = ['ln_vision', 'Qformer', 'vis_projector', 'vlm_projector', 'query_tokens', 'mm_projector', 'embed_tokens', 'embed_in']
65 |             for k, v in _state_dict.items():
66 |                 if any(key_match in k for key_match in keys_to_match):
67 |                     weight_to_save[k] = v
68 | 
69 |             current_folder = output_dir.split('/')[-1]
70 |             parent_folder = os.path.dirname(output_dir)
71 |             if current_folder.startswith('checkpoint-'):
72 |                 mm_projector_folder = os.path.join(parent_folder, "mm_projector")
73 |                 os.makedirs(mm_projector_folder, exist_ok=True)
74 |                 torch.save(weight_to_save, os.path.join(mm_projector_folder, f'{current_folder}.bin'))
75 |             else:
76 |                 torch.save(weight_to_save, os.path.join(output_dir, f'mm_projector.bin'), )
77 | 
78 | 


--------------------------------------------------------------------------------
/vista_llama/eval/run_inference_benchmark_consistency.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import argparse
 3 | import json
 4 | from tqdm import tqdm
 5 | from vista_llama.eval.model_utils import initialize_model, load_video
 6 | from vista_llama.inference import video_chatgpt_infer
 7 | 
 8 | def parse_args():
 9 |     """
10 |     Parse command-line arguments.
11 |     """
12 |     parser = argparse.ArgumentParser()
13 | 
14 |     # Define the command-line arguments
15 |     parser.add_argument('--video_dir', help='Directory containing video files.', required=True)
16 |     parser.add_argument('--gt_file', help='Path to the ground truth file.', required=True)
17 |     parser.add_argument('--output_dir', help='Directory to save the model results JSON.', required=True)
18 |     parser.add_argument('--output_name', help='Name of the file for storing results JSON.', required=True)
19 |     parser.add_argument("--model-name", type=str, required=True)
20 |     parser.add_argument("--conv-mode", type=str, required=False, default='video-chatgpt_v1')
21 |     parser.add_argument("--projection_path", type=str, required=True)
22 | 
23 |     return parser.parse_args()
24 | 
25 | 
26 | def run_inference(args):
27 |     """
28 |     Run inference on a set of video files using the provided model.
29 | 
30 |     Args:
31 |         args: Command-line arguments.
32 |     """
33 |     # Initialize the model
34 |     model, vision_tower, tokenizer, image_processor, video_token_len = initialize_model(args.model_name,
35 |                                                                                         args.projection_path)
36 |     # Load the ground truth file
37 |     with open(args.gt_file) as file:
38 |         gt_contents = json.load(file)
39 | 
40 |     # Create the output directory if it doesn't exist
41 |     if not os.path.exists(args.output_dir):
42 |         os.makedirs(args.output_dir)
43 | 
44 |     output_list = []  # List to store the output results
45 |     conv_mode = args.conv_mode
46 | 
47 |     video_formats = ['.mp4', '.avi', '.mov', '.mkv']
48 | 
49 |     # Iterate over each sample in the ground truth file
50 |     for sample in tqdm(gt_contents):
51 |         video_name = sample['video_name']
52 |         sample_set = sample
53 |         question_1 = sample['Q1']
54 |         question_2 = sample['Q2']
55 | 
56 |         # Load the video file
57 |         for fmt in video_formats:  # Added this line
58 |             temp_path = os.path.join(args.video_dir, f"{video_name}{fmt}")
59 |             if os.path.exists(temp_path):
60 |                 video_path = temp_path
61 |                 break
62 | 
63 |         # Check if the video exists
64 |         if video_path is not None:  # Modified this line
65 |             video_frames = load_video(video_path)
66 | 
67 |         try:
68 |             # Run inference on the video for the first question and add the output to the list
69 |             output_1 = video_chatgpt_infer(video_frames, question_1, conv_mode, model, vision_tower,
70 |                                              tokenizer, image_processor, video_token_len)
71 |             sample_set['pred1'] = output_1
72 | 
73 |             # Run inference on the video for the second question and add the output to the list
74 |             output_2 = video_chatgpt_infer(video_frames, question_2, conv_mode, model, vision_tower,
75 |                                              tokenizer, image_processor, video_token_len)
76 |             sample_set['pred2'] = output_2
77 | 
78 |             output_list.append(sample_set)
79 |         except Exception as e:
80 |             print(f"Error processing video file '{video_name}': {e}")
81 | 
82 |     # Save the output list to a JSON file
83 |     with open(os.path.join(args.output_dir, f"{args.output_name}.json"), 'w') as file:
84 |         json.dump(output_list, file)
85 | 
86 | 
87 | if __name__ == "__main__":
88 |     args = parse_args()
89 |     run_inference(args)
90 | 


--------------------------------------------------------------------------------
/vista_llama/model/dist_utils.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import datetime
  9 | import functools
 10 | import os
 11 | 
 12 | import torch
 13 | import torch.distributed as dist
 14 | import timm.models.hub as timm_hub
 15 | 
 16 | 
 17 | def setup_for_distributed(is_master):
 18 |     """
 19 |     This function disables printing when not in master process
 20 |     """
 21 |     import builtins as __builtin__
 22 | 
 23 |     builtin_print = __builtin__.print
 24 | 
 25 |     def print(*args, **kwargs):
 26 |         force = kwargs.pop("force", False)
 27 |         if is_master or force:
 28 |             builtin_print(*args, **kwargs)
 29 | 
 30 |     __builtin__.print = print
 31 | 
 32 | 
 33 | def is_dist_avail_and_initialized():
 34 |     if not dist.is_available():
 35 |         return False
 36 |     if not dist.is_initialized():
 37 |         return False
 38 |     return True
 39 | 
 40 | 
 41 | def get_world_size():
 42 |     if not is_dist_avail_and_initialized():
 43 |         return 1
 44 |     return dist.get_world_size()
 45 | 
 46 | 
 47 | def get_rank():
 48 |     if not is_dist_avail_and_initialized():
 49 |         return 0
 50 |     return dist.get_rank()
 51 | 
 52 | 
 53 | def is_main_process():
 54 |     return get_rank() == 0
 55 | 
 56 | 
 57 | def init_distributed_mode(args):
 58 |     if "RANK" in os.environ and "WORLD_SIZE" in os.environ:
 59 |         args.rank = int(os.environ["RANK"])
 60 |         args.world_size = int(os.environ["WORLD_SIZE"])
 61 |         args.gpu = int(os.environ["LOCAL_RANK"])
 62 |     elif "SLURM_PROCID" in os.environ:
 63 |         args.rank = int(os.environ["SLURM_PROCID"])
 64 |         args.gpu = args.rank % torch.cuda.device_count()
 65 |     else:
 66 |         print("Not using distributed mode")
 67 |         args.rank = 0
 68 |         args.world_size = 1
 69 |         args.distributed = False
 70 |         return
 71 | 
 72 |     args.distributed = True
 73 |     args.dist_url = getattr(args, 'dist_url', 'env://')
 74 | 
 75 |     torch.cuda.set_device(args.gpu)
 76 |     args.dist_backend = "nccl"
 77 |     print(
 78 |         "| distributed init (rank {}, world {}): {}".format(
 79 |             args.rank, args.world_size, args.dist_url
 80 |         ),
 81 |         flush=True,
 82 |     )
 83 |     torch.distributed.init_process_group(
 84 |         backend=args.dist_backend,
 85 |         init_method=args.dist_url,
 86 |         world_size=args.world_size,
 87 |         rank=args.rank,
 88 |         timeout=datetime.timedelta(
 89 |             days=365
 90 |         ),  # allow auto-downloading and de-compressing
 91 |     )
 92 |     torch.distributed.barrier()
 93 |     setup_for_distributed(args.rank == 0)
 94 | 
 95 | 
 96 | def get_dist_info():
 97 |     if torch.__version__ < "1.0":
 98 |         initialized = dist._initialized
 99 |     else:
100 |         initialized = dist.is_initialized()
101 |     if initialized:
102 |         rank = dist.get_rank()
103 |         world_size = dist.get_world_size()
104 |     else:  # non-distributed training
105 |         rank = 0
106 |         world_size = 1
107 |     return rank, world_size
108 | 
109 | 
110 | def main_process(func):
111 |     @functools.wraps(func)
112 |     def wrapper(*args, **kwargs):
113 |         rank, _ = get_dist_info()
114 |         if rank == 0:
115 |             return func(*args, **kwargs)
116 | 
117 |     return wrapper
118 | 
119 | 
120 | def download_cached_file(url, check_hash=True, progress=False):
121 |     """
122 |     Download a file from a URL and cache it locally. If the file already exists, it is not downloaded again.
123 |     If distributed, only the main process downloads the file, and the other processes wait for the file to be downloaded.
124 |     """
125 | 
126 |     def get_cached_file_path():
127 |         # a hack to sync the file path across processes
128 |         parts = torch.hub.urlparse(url)
129 |         filename = os.path.basename(parts.path)
130 |         cached_file = os.path.join(timm_hub.get_cache_dir(), filename)
131 | 
132 |         return cached_file
133 | 
134 |     if is_main_process():
135 |         timm_hub.download_cached_file(url, check_hash, progress)
136 | 
137 |     if is_dist_avail_and_initialized():
138 |         dist.barrier()
139 | 
140 |     return get_cached_file_path()
141 | 


--------------------------------------------------------------------------------
/test.yaml:
--------------------------------------------------------------------------------
  1 | name: vista
  2 | channels:
  3 |   - defaults
  4 | dependencies:
  5 |   - _libgcc_mutex=0.1=main
  6 |   - _openmp_mutex=5.1=1_gnu
  7 |   - bzip2=1.0.8=h5eee18b_5
  8 |   - ca-certificates=2024.3.11=h06a4308_0
  9 |   - ld_impl_linux-64=2.38=h1181459_1
 10 |   - libffi=3.3=he6710b0_2
 11 |   - libgcc-ng=11.2.0=h1234567_1
 12 |   - libgomp=11.2.0=h1234567_1
 13 |   - libstdcxx-ng=11.2.0=h1234567_1
 14 |   - libuuid=1.41.5=h5eee18b_0
 15 |   - ncurses=6.4=h6a678d5_0
 16 |   - openssl=1.1.1w=h7f8727e_0
 17 |   - pip=23.3.1=py310h06a4308_0
 18 |   - python=3.10.0=h12debd9_5
 19 |   - readline=8.2=h5eee18b_0
 20 |   - setuptools=68.2.2=py310h06a4308_0
 21 |   - sqlite=3.41.2=h5eee18b_0
 22 |   - tk=8.6.12=h1ccaba5_0
 23 |   - wheel=0.41.2=py310h06a4308_0
 24 |   - xz=5.4.6=h5eee18b_0
 25 |   - zlib=1.2.13=h5eee18b_0
 26 |   - pip:
 27 |       - accelerate==0.20.3
 28 |       - aiofiles==23.2.1
 29 |       - aiohttp==3.9.3
 30 |       - aiosignal==1.3.1
 31 |       - altair==5.2.0
 32 |       - annotated-types==0.6.0
 33 |       - antlr4-python3-runtime==4.9.3
 34 |       - anyio==3.7.1
 35 |       - async-timeout==4.0.3
 36 |       - attrs==23.2.0
 37 |       - certifi==2024.2.2
 38 |       - charset-normalizer==3.3.2
 39 |       - click==8.1.7
 40 |       - cmake==3.29.0.1
 41 |       - contourpy==1.2.0
 42 |       - cycler==0.12.1
 43 |       - dataclasses-json==0.6.4
 44 |       - decord==0.6.0
 45 |       - einops==0.6.1
 46 |       - exceptiongroup==1.2.0
 47 |       - fastapi==0.110.0
 48 |       - ffmpy==0.3.2
 49 |       - filelock==3.13.3
 50 |       - flash-attn==1.0.7
 51 |       - fonttools==4.50.0
 52 |       - frozenlist==1.4.1
 53 |       - fsspec==2024.3.1
 54 |       - gradio==3.23.0
 55 |       - greenlet==3.0.3
 56 |       - h11==0.12.0
 57 |       - httpcore==0.15.0
 58 |       - httpx==0.23.0
 59 |       - huggingface-hub==0.22.1
 60 |       - idna==3.6
 61 |       - jinja2==3.1.3
 62 |       - jsonpatch==1.33
 63 |       - jsonpointer==2.4
 64 |       - jsonschema==4.21.1
 65 |       - jsonschema-specifications==2023.12.1
 66 |       - kiwisolver==1.4.5
 67 |       - langchain==0.1.14
 68 |       - langchain-community==0.0.30
 69 |       - langchain-core==0.1.37
 70 |       - langchain-text-splitters==0.0.1
 71 |       - langsmith==0.1.38
 72 |       - linkify-it-py==2.0.3
 73 |       - lit==18.1.2
 74 |       - markdown-it-py==2.2.0
 75 |       - markdown2==2.4.13
 76 |       - markupsafe==2.1.5
 77 |       - marshmallow==3.21.1
 78 |       - matplotlib==3.8.3
 79 |       - mdit-py-plugins==0.3.3
 80 |       - mdurl==0.1.2
 81 |       - mpmath==1.3.0
 82 |       - multidict==6.0.5
 83 |       - mypy-extensions==1.0.0
 84 |       - networkx==3.2.1
 85 |       - ninja==1.11.1.1
 86 |       - numpy==1.24.4
 87 |       - nvidia-cublas-cu11==11.10.3.66
 88 |       - nvidia-cuda-cupti-cu11==11.7.101
 89 |       - nvidia-cuda-nvrtc-cu11==11.7.99
 90 |       - nvidia-cuda-runtime-cu11==11.7.99
 91 |       - nvidia-cudnn-cu11==8.5.0.96
 92 |       - nvidia-cufft-cu11==10.9.0.58
 93 |       - nvidia-curand-cu11==10.2.10.91
 94 |       - nvidia-cusolver-cu11==11.4.0.1
 95 |       - nvidia-cusparse-cu11==11.7.4.91
 96 |       - nvidia-nccl-cu11==2.14.3
 97 |       - nvidia-nvtx-cu11==11.7.91
 98 |       - omegaconf==2.3.0
 99 |       - openai==0.28.0
100 |       - orjson==3.10.0
101 |       - packaging==23.2
102 |       - pandas==2.2.1
103 |       - pillow==9.5.0
104 |       - protobuf==4.23.4
105 |       - psutil==5.9.8
106 |       - pydantic==1.10.9
107 |       - pydantic-core==2.16.3
108 |       - pydub==0.25.1
109 |       - pyparsing==3.1.2
110 |       - python-dateutil==2.9.0.post0
111 |       - python-multipart==0.0.9
112 |       - pytz==2024.1
113 |       - pyyaml==6.0.1
114 |       - referencing==0.34.0
115 |       - regex==2023.12.25
116 |       - requests==2.30.0
117 |       - rfc3986==1.5.0
118 |       - rpds-py==0.18.0
119 |       - safetensors==0.4.2
120 |       - semantic-version==2.10.0
121 |       - sentencepiece==0.1.99
122 |       - six==1.16.0
123 |       - sniffio==1.3.1
124 |       - sqlalchemy==2.0.29
125 |       - starlette==0.36.3
126 |       - sympy==1.12
127 |       - tenacity==8.2.3
128 |       - timm==0.9.16
129 |       - tokenizers==0.13.3
130 |       - toolz==0.12.1
131 |       - torch==2.0.1+cu118
132 |       - torchaudio==2.0.2+cu118
133 |       - torchvision==0.15.2+cu118
134 |       - tqdm==4.65.2
135 |       - transformers==4.28.0.dev0
136 |       - triton==2.0.0
137 |       - typing-extensions==4.10.0
138 |       - typing-inspect==0.9.0
139 |       - tzdata==2024.1
140 |       - uc-micro-py==1.0.3
141 |       - urllib3==2.2.1
142 |       - uvicorn==0.29.0
143 |       - websockets==12.0
144 |       - yarl==1.9.4
145 | prefix: /home/mafan/miniconda3/envs/vista
146 | 


--------------------------------------------------------------------------------
/vista_llama/train/llama_flash_attn_monkey_patch.py:
--------------------------------------------------------------------------------
  1 | # Adopted from https://github.com/lm-sys/FastChat. Below is the original copyright:
  2 | from typing import List, Optional, Tuple
  3 | import torch
  4 | from torch import nn
  5 | import transformers
  6 | from transformers.models.llama.modeling_llama import apply_rotary_pos_emb
  7 | from einops import rearrange
  8 | from flash_attn.flash_attn_interface import flash_attn_unpadded_qkvpacked_func
  9 | from flash_attn.bert_padding import unpad_input, pad_input
 10 | 
 11 | 
 12 | def forward(
 13 |     self,
 14 |     hidden_states: torch.Tensor,
 15 |     past_key_value: Optional[Tuple[torch.Tensor]] = None,
 16 |     attention_mask: Optional[torch.Tensor] = None,
 17 |     output_attentions: bool = False,
 18 |     use_cache: bool = False,
 19 | ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
 20 |             Optional[Tuple[torch.Tensor]]]:
 21 |     """Input shape: Batch x Time x Channel
 22 |     
 23 |     attention_mask: [bsz, q_len]
 24 |     """
 25 |     bsz, q_len, _ = hidden_states.size()
 26 | 
 27 |     query_states = self.q_proj(hidden_states).view(
 28 |         bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
 29 |     key_states = self.k_proj(hidden_states).view(
 30 |         bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
 31 |     value_states = self.v_proj(hidden_states).view(
 32 |         bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
 33 |     # [bsz, q_len, nh, hd]
 34 |     # [bsz, nh, q_len, hd]
 35 | 
 36 |     kv_seq_len = key_states.shape[-2]
 37 |     offset = 0
 38 |     if past_key_value is not None:
 39 |         offset = past_key_value[0].shape[-2]
 40 |         kv_seq_len += offset
 41 |     cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
 42 |     query_states, key_states = apply_rotary_pos_emb(query_states,
 43 |                                                     key_states,
 44 |                                                     cos,
 45 |                                                     sin,
 46 |                                                     offset=offset)
 47 |     # [bsz, nh, t, hd]
 48 |     assert not output_attentions, "output_attentions is not supported"
 49 |     assert not use_cache, "use_cache is not supported"
 50 |     assert past_key_value is None, "past_key_value is not supported"
 51 | 
 52 |     # Flash attention codes from
 53 |     # https://github.com/HazyResearch/flash-attention/blob/main/flash_attn/flash_attention.py
 54 | 
 55 |     # transform the data into the format required by flash attention
 56 |     qkv = torch.stack([query_states, key_states, value_states], dim=2) # [bsz, nh, 3, q_len, hd]
 57 |     qkv = qkv.transpose(1, 3) # [bsz, q_len, 3, nh, hd]
 58 |     # We have disabled _prepare_decoder_attention_mask in LlamaModel
 59 |     # the attention_mask should be the same as the key_padding_mask
 60 |     key_padding_mask = attention_mask
 61 | 
 62 | 
 63 |     if key_padding_mask is None:
 64 |         qkv = rearrange(qkv, 'b s ... -> (b s) ...')
 65 |         max_s = q_len
 66 |         cu_q_lens = torch.arange(0, (bsz + 1) * q_len, step=q_len, dtype=torch.int32,
 67 |                                 device=qkv.device)
 68 |         output = flash_attn_unpadded_qkvpacked_func(
 69 |             qkv, cu_q_lens, max_s, 0.0,
 70 |             softmax_scale=None, causal=True
 71 |         )
 72 |         output = rearrange(output, '(b s) ... -> b s ...', b=bsz)
 73 |     else:
 74 |         nheads = qkv.shape[-2]
 75 |         x = rearrange(qkv, 'b s three h d -> b s (three h d)')
 76 |         x_unpad, indices, cu_q_lens, max_s = unpad_input(x, key_padding_mask)
 77 |         x_unpad = rearrange(x_unpad, 'nnz (three h d) -> nnz three h d', three=3, h=nheads)
 78 |         output_unpad = flash_attn_unpadded_qkvpacked_func(
 79 |             x_unpad, cu_q_lens, max_s, 0.0,
 80 |             softmax_scale=None, causal=True
 81 |         )
 82 |         output = rearrange(pad_input(rearrange(output_unpad, 'nnz h d -> nnz (h d)'),
 83 |                                     indices, bsz, q_len),
 84 |                         'b s (h d) -> b s h d', h=nheads)
 85 |     return self.o_proj(rearrange(output,
 86 |                                     'b s h d -> b s (h d)')), None, None
 87 | 
 88 | 
 89 | # Disable the transformation of the attention mask in LlamaModel as the flash attention
 90 | # requires the attention mask to be the same as the key_padding_mask
 91 | def _prepare_decoder_attention_mask(self, attention_mask, input_shape,
 92 |                                     inputs_embeds, past_key_values_length):
 93 |     # [bsz, seq_len]
 94 |     return attention_mask
 95 | 
 96 | 
 97 | def replace_llama_attn_with_flash_attn():
 98 |     transformers.models.llama.modeling_llama.LlamaModel._prepare_decoder_attention_mask = _prepare_decoder_attention_mask
 99 |     transformers.models.llama.modeling_llama.LlamaAttention.forward = forward
100 | 


--------------------------------------------------------------------------------
/vista_llama/inference.py:
--------------------------------------------------------------------------------
  1 | from vista_llama.video_conversation import conv_templates, SeparatorStyle
  2 | from vista_llama.model.utils import KeywordsStoppingCriteria
  3 | import torch
  4 | 
  5 | # Define constants
  6 | DEFAULT_VIDEO_TOKEN = "<video>"
  7 | DEFAULT_VIDEO_PATCH_TOKEN = "<vid_patch>"
  8 | DEFAULT_VID_START_TOKEN = "<vid_start>"
  9 | DEFAULT_VID_END_TOKEN = "<vid_end>"
 10 | 
 11 | 
 12 | def get_spatio_temporal_features_torch(features):
 13 |     """
 14 |     Computes spatio-temporal features from given features.
 15 | 
 16 |     Parameters:
 17 |     features (torch.Tensor): Input features to process.
 18 | 
 19 |     Returns:
 20 |     torch.Tensor: Spatio-temporal features.
 21 |     """
 22 | 
 23 |     # Extract the dimensions of the features
 24 |     t, s, c = features.shape
 25 | 
 26 |     # Compute temporal tokens as the mean along the time axis
 27 |     temporal_tokens = torch.mean(features, dim=1)
 28 | 
 29 |     # Padding size calculation
 30 |     padding_size = 100 - t
 31 | 
 32 |     # Pad temporal tokens if necessary
 33 |     if padding_size > 0:
 34 |         padding = torch.zeros(padding_size, c, device=features.device)
 35 |         temporal_tokens = torch.cat((temporal_tokens, padding), dim=0)
 36 | 
 37 |     # Compute spatial tokens as the mean along the spatial axis
 38 |     spatial_tokens = torch.mean(features, dim=0)
 39 | 
 40 |     # Concatenate temporal and spatial tokens and cast to half precision
 41 |     concat_tokens = torch.cat([temporal_tokens, spatial_tokens], dim=0).half()
 42 | 
 43 |     return concat_tokens
 44 | 
 45 | 
 46 | def video_chatgpt_infer(video_frames, question, conv_mode, model, vision_tower, tokenizer, image_processor, video_token_len):
 47 |     """
 48 |     Run inference using the Video-ChatGPT model.
 49 | 
 50 |     Parameters:
 51 |     sample : Initial sample
 52 |     video_frames (torch.Tensor): Video frames to process.
 53 |     question (str): The question string.
 54 |     conv_mode: Conversation mode.
 55 |     model: The pretrained Video-ChatGPT model.
 56 |     vision_tower: Vision model to extract video features.
 57 |     tokenizer: Tokenizer for the model.
 58 |     image_processor: Image processor to preprocess video frames.
 59 |     video_token_len (int): The length of video tokens.
 60 | 
 61 |     Returns:
 62 |     dict: Dictionary containing the model's output.
 63 |     """
 64 | 
 65 |     # Prepare question string for the model
 66 |     if model.get_model().vision_config.use_vid_start_end:
 67 |         qs = question + '\n' + DEFAULT_VID_START_TOKEN + DEFAULT_VIDEO_PATCH_TOKEN * video_token_len + DEFAULT_VID_END_TOKEN
 68 |     else:
 69 |         qs = question + '\n' + DEFAULT_VIDEO_PATCH_TOKEN * video_token_len
 70 | 
 71 |     # Prepare conversation prompt
 72 |     conv = conv_templates[conv_mode].copy()
 73 |     conv.append_message(conv.roles[0], qs)
 74 |     conv.append_message(conv.roles[1], None)
 75 |     prompt = conv.get_prompt()
 76 | 
 77 |     # Tokenize the prompt
 78 |     inputs = tokenizer([prompt])
 79 | 
 80 |     # Preprocess video frames and get image tensor
 81 |     image_tensor = image_processor.preprocess(video_frames, return_tensors='pt')['pixel_values']
 82 | 
 83 |     # Move image tensor to GPU and reduce precision to half
 84 |     image_tensor = image_tensor.half().to(model.device)
 85 | 
 86 |     # Generate video spatio-temporal features
 87 |     with torch.no_grad():
 88 |         image_forward_outs = vision_tower(image_tensor, output_hidden_states=True)
 89 |         frame_features = image_forward_outs.hidden_states[-2][:, 1:] # Use second to last layer as in LLaVA
 90 |     video_spatio_temporal_features = get_spatio_temporal_features_torch(frame_features)
 91 | 
 92 |     # Move inputs to GPU
 93 |     input_ids = torch.as_tensor(inputs.input_ids).to(model.device)
 94 | 
 95 |     # Define stopping criteria for generation
 96 |     stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
 97 |     stopping_criteria = KeywordsStoppingCriteria([stop_str], tokenizer, input_ids)
 98 | 
 99 |     # Run model inference
100 |     with torch.inference_mode():
101 |         output_ids = model.generate(
102 |             input_ids,
103 |             video_spatio_temporal_features=video_spatio_temporal_features.unsqueeze(0),
104 |             do_sample=True,
105 |             temperature=0.2,
106 |             max_new_tokens=1024,
107 |             stopping_criteria=[stopping_criteria])
108 | 
109 |     # Check if output is the same as input
110 |     n_diff_input_output = (input_ids != output_ids[:, :input_ids.shape[1]]).sum().item()
111 |     if n_diff_input_output > 0:
112 |         print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
113 | 
114 |     # Decode output tokens
115 |     outputs = tokenizer.batch_decode(output_ids[:, input_ids.shape[1]:], skip_special_tokens=True)[0]
116 | 
117 |     # Clean output string
118 |     outputs = outputs.strip().rstrip(stop_str).strip()
119 | 
120 |     return outputs


--------------------------------------------------------------------------------
/scripts/save_evaclip_features.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import math
  3 | import torch
  4 | import pickle
  5 | import argparse
  6 | import numpy as np
  7 | from PIL import Image
  8 | from tqdm import tqdm
  9 | from decord import VideoReader, cpu
 10 | from transformers import CLIPVisionModel, CLIPImageProcessor
 11 | from vista_llama.model.eva_vit import create_eva_vit_g
 12 | import torch.distributed as dist
 13 | from dataloader import KVWriter
 14 | 
 15 | 
 16 | def load_video(vis_path, num_frm=100):
 17 |     # vr = VideoReader(vis_path, ctx=cpu(0))
 18 |     vr = VideoReader(vis_path)
 19 |     total_frame_num = len(vr)
 20 |     total_num_frm = min(total_frame_num, num_frm)
 21 |     frame_idx = get_seq_frames(total_frame_num, total_num_frm)
 22 |     img_array = vr.get_batch(frame_idx).numpy()  # (n_clips*num_frm, H, W, 3)
 23 | 
 24 |     a, H, W, _ = img_array.shape
 25 |     h, w = 224, 224
 26 |     if img_array.shape[-3] != h or img_array.shape[-2] != w:
 27 |         img_array = torch.from_numpy(img_array).permute(0, 3, 1, 2).float()
 28 |         img_array = torch.nn.functional.interpolate(img_array, size=(h, w))
 29 |         img_array = img_array.permute(0, 2, 3, 1).to(torch.uint8).numpy()
 30 |     img_array = img_array.reshape((1, total_num_frm, img_array.shape[-3], img_array.shape[-2], img_array.shape[-1]))
 31 | 
 32 |     clip_imgs = []
 33 |     for j in range(total_num_frm):
 34 |         clip_imgs.append(Image.fromarray(img_array[0, j]))
 35 | 
 36 |     return clip_imgs
 37 | 
 38 | 
 39 | def get_seq_frames(total_num_frames, desired_num_frames):
 40 |     seg_size = float(total_num_frames - 1) / desired_num_frames
 41 |     seq = []
 42 |     for i in range(desired_num_frames):
 43 |         start = int(np.round(seg_size * i))
 44 |         end = int(np.round(seg_size * (i + 1)))
 45 |         seq.append((start + end) // 2)
 46 | 
 47 |     return seq
 48 | 
 49 | 
 50 | def get_spatio_temporal_features(features, num_temporal_tokens=100):
 51 |     t, s, c = features.shape
 52 | 
 53 |     temporal_tokens = np.mean(features, axis=1)
 54 |     padding_size = num_temporal_tokens - t
 55 |     if padding_size > 0:
 56 |         temporal_tokens = np.pad(temporal_tokens, ((0, padding_size), (0, 0)), mode='constant')
 57 | 
 58 |     spatial_tokens = np.mean(features, axis=0)
 59 |     sp_features = np.concatenate([temporal_tokens, spatial_tokens], axis=0)
 60 | 
 61 |     return sp_features
 62 | 
 63 | 
 64 | def parse_args():
 65 |     parser = argparse.ArgumentParser(description="Training")
 66 | 
 67 |     parser.add_argument("--video_dir_path", default="/opt/tiger/activitynet_videos", help="Path to read the videos from.")
 68 |     parser.add_argument("--clip_feat_path", default="/opt/tiger/activity_evaclip_spatio_temporal_patch", help="The output dir to save the features in.")
 69 |     parser.add_argument("--infer_batch", required=False, type=int, default=32,
 70 |                         help="Number of frames/images to perform batch inference.")
 71 | 
 72 |     args = parser.parse_args()
 73 | 
 74 |     return args
 75 | 
 76 | 
 77 | def main():
 78 |     args = parse_args()
 79 |     video_dir_path = args.video_dir_path
 80 |     clip_feat_path = args.clip_feat_path
 81 |     infer_batch = args.infer_batch
 82 |     os.makedirs(clip_feat_path, exist_ok=True)
 83 | 
 84 |     rank = int(os.environ.get('RANK',0))
 85 |     num_gpus = torch.cuda.device_count()
 86 |     # dist.init_process_group(backend='nccl')
 87 | 
 88 |     output_dir = f'{args.clip_feat_path}.{rank}'
 89 |     writer = KVWriter(output_dir, 1)
 90 | 
 91 |     # Initialize the CLIP model
 92 |     image_processor = CLIPImageProcessor.from_pretrained('openai/clip-vit-large-patch14', torch_dtype=torch.float16)
 93 | 
 94 |     vision_tower = create_eva_vit_g(224, 0, False, "fp16").to(rank)
 95 |     vision_tower.eval()
 96 | 
 97 |     all_videos = sorted(os.listdir(video_dir_path))
 98 |     all_videos = all_videos[rank::num_gpus]
 99 |     video_clip_features = {}
100 |     counter = 0
101 |     for video_name in tqdm(all_videos):
102 |         video_path = f"{video_dir_path}/{video_name}"
103 |         video_id = video_name.split('.')[0]
104 |         if os.path.exists(f"{clip_feat_path}/{video_id}.pkl"):  # Check if the file is already processed
105 |             continue
106 |         try:
107 |             video = load_video(video_path)
108 |             video_tensor = image_processor.preprocess(video, return_tensors='pt')['pixel_values']
109 |             video_tensor = video_tensor.half()
110 | 
111 |             n_chunk = len(video_tensor)
112 |             video_features = torch.FloatTensor(n_chunk, 257, 1408).fill_(0)
113 |             n_iter = int(math.ceil(n_chunk / float(infer_batch)))
114 |             for i in range(n_iter):
115 |                 min_ind = i * infer_batch
116 |                 max_ind = (i + 1) * infer_batch
117 |                 video_batch = video_tensor[min_ind:max_ind].to(rank)
118 | 
119 |                 with torch.cuda.amp.autocast(dtype=torch.bfloat16):
120 |                     image_forward_outs = vision_tower(video_batch)
121 | 
122 |                 video_features[min_ind:max_ind] = image_forward_outs.detach().cpu()
123 | 
124 |             video_features = video_features.numpy().astype("float16")
125 |             writer.write_many([video_id], [pickle.dumps(video_features)])
126 | 
127 |             counter += 1
128 | 
129 |         except Exception as e:
130 |             print(f"Can't process {video_path}, error: {e}")
131 | 
132 |     writer.flush()
133 | 
134 | 
135 | if __name__ == "__main__":
136 |     main()
137 | 


--------------------------------------------------------------------------------
/scripts/save_spatio_temporal_clip_features.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import math
  3 | import torch
  4 | import pickle
  5 | import argparse
  6 | import numpy as np
  7 | from PIL import Image
  8 | from tqdm import tqdm
  9 | from decord import VideoReader, cpu
 10 | from transformers import CLIPVisionModel, CLIPImageProcessor
 11 | 
 12 | 
 13 | def load_video(vis_path, num_frm=100):
 14 |     vr = VideoReader(vis_path, ctx=cpu(0))
 15 |     total_frame_num = len(vr)
 16 |     total_num_frm = min(total_frame_num, num_frm)
 17 |     frame_idx = get_seq_frames(total_frame_num, total_num_frm)
 18 |     img_array = vr.get_batch(frame_idx).asnumpy()  # (n_clips*num_frm, H, W, 3)
 19 | 
 20 |     a, H, W, _ = img_array.shape
 21 |     h, w = 224, 224
 22 |     if img_array.shape[-3] != h or img_array.shape[-2] != w:
 23 |         img_array = torch.from_numpy(img_array).permute(0, 3, 1, 2).float()
 24 |         img_array = torch.nn.functional.interpolate(img_array, size=(h, w))
 25 |         img_array = img_array.permute(0, 2, 3, 1).to(torch.uint8).numpy()
 26 |     img_array = img_array.reshape((1, total_num_frm, img_array.shape[-3], img_array.shape[-2], img_array.shape[-1]))
 27 | 
 28 |     clip_imgs = []
 29 |     for j in range(total_num_frm):
 30 |         clip_imgs.append(Image.fromarray(img_array[0, j]))
 31 | 
 32 |     return clip_imgs
 33 | 
 34 | 
 35 | def get_seq_frames(total_num_frames, desired_num_frames):
 36 |     seg_size = float(total_num_frames - 1) / desired_num_frames
 37 |     seq = []
 38 |     for i in range(desired_num_frames):
 39 |         start = int(np.round(seg_size * i))
 40 |         end = int(np.round(seg_size * (i + 1)))
 41 |         seq.append((start + end) // 2)
 42 | 
 43 |     return seq
 44 | 
 45 | 
 46 | def get_spatio_temporal_features(features, num_temporal_tokens=100):
 47 |     t, s, c = features.shape
 48 | 
 49 |     temporal_tokens = np.mean(features, axis=1)
 50 |     padding_size = num_temporal_tokens - t
 51 |     if padding_size > 0:
 52 |         temporal_tokens = np.pad(temporal_tokens, ((0, padding_size), (0, 0)), mode='constant')
 53 | 
 54 |     spatial_tokens = np.mean(features, axis=0)
 55 |     sp_features = np.concatenate([temporal_tokens, spatial_tokens], axis=0)
 56 | 
 57 |     return sp_features
 58 | 
 59 | 
 60 | def parse_args():
 61 |     parser = argparse.ArgumentParser(description="Training")
 62 | 
 63 |     parser.add_argument("--video_dir_path", required=True, help="Path to read the videos from.")
 64 |     parser.add_argument("--clip_feat_path", required=True, help="The output dir to save the features in.")
 65 |     parser.add_argument("--infer_batch", required=False, type=int, default=32,
 66 |                         help="Number of frames/images to perform batch inference.")
 67 | 
 68 |     args = parser.parse_args()
 69 | 
 70 |     return args
 71 | 
 72 | 
 73 | def main():
 74 |     args = parse_args()
 75 |     video_dir_path = args.video_dir_path
 76 |     clip_feat_path = args.clip_feat_path
 77 |     infer_batch = args.infer_batch
 78 |     os.makedirs(clip_feat_path, exist_ok=True)
 79 | 
 80 |     # Initialize the CLIP model
 81 |     image_processor = CLIPImageProcessor.from_pretrained('openai/clip-vit-large-patch14', torch_dtype=torch.float16)
 82 |     vision_tower = CLIPVisionModel.from_pretrained('openai/clip-vit-large-patch14', torch_dtype=torch.float16,
 83 |                                                    low_cpu_mem_usage=True).cuda()
 84 |     vision_tower.eval()
 85 | 
 86 |     all_videos = os.listdir(video_dir_path)
 87 |     video_clip_features = {}
 88 |     counter = 0
 89 |     for video_name in tqdm(all_videos):
 90 |         video_path = f"{video_dir_path}/{video_name}"
 91 |         video_id = video_name.split('.')[0]
 92 |         if os.path.exists(f"{clip_feat_path}/{video_id}.pkl"):  # Check if the file is already processed
 93 |             continue
 94 |         try:
 95 |             video = load_video(video_path)
 96 |             video_tensor = image_processor.preprocess(video, return_tensors='pt')['pixel_values']
 97 |             video_tensor = video_tensor.half()
 98 | 
 99 |             n_chunk = len(video_tensor)
100 |             video_features = torch.FloatTensor(n_chunk, 256, 1024).fill_(0)
101 |             n_iter = int(math.ceil(n_chunk / float(infer_batch)))
102 |             for i in range(n_iter):
103 |                 min_ind = i * infer_batch
104 |                 max_ind = (i + 1) * infer_batch
105 |                 video_batch = video_tensor[min_ind:max_ind].cuda()
106 | 
107 |                 image_forward_outs = vision_tower(video_batch, output_hidden_states=True)
108 | 
109 |                 select_hidden_state_layer = -2
110 |                 select_hidden_state = image_forward_outs.hidden_states[select_hidden_state_layer]
111 |                 batch_features = select_hidden_state[:, 1:]
112 |                 video_features[min_ind:max_ind] = batch_features.detach().cpu()
113 | 
114 |             video_clip_features[video_id] = get_spatio_temporal_features(video_features.numpy().astype("float16"))
115 |             counter += 1
116 | 
117 |         except Exception as e:
118 |             print(f"Can't process {video_path}")
119 | 
120 |         if counter % 512 == 0:  # Save after every 512 videos, update this number as per your requirements
121 |             for key in video_clip_features.keys():
122 |                 features = video_clip_features[key]
123 |                 with open(f"{clip_feat_path}/{key}.pkl", 'wb') as f:
124 |                     pickle.dump(features, f)
125 |             video_clip_features = {}
126 | 
127 |     for key in video_clip_features.keys():
128 |         features = video_clip_features[key]
129 |         with open(f"{clip_feat_path}/{key}.pkl", 'wb') as f:
130 |             pickle.dump(features, f)
131 | 
132 | 
133 | if __name__ == "__main__":
134 |     main()
135 | 


--------------------------------------------------------------------------------
/scripts/save_patch_clip_features.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import math
  3 | import torch
  4 | import pickle
  5 | import argparse
  6 | import numpy as np
  7 | from PIL import Image
  8 | from tqdm import tqdm
  9 | from decord import VideoReader, cpu
 10 | from transformers import CLIPVisionModel, CLIPImageProcessor
 11 | import torch.distributed as dist
 12 | from dataloader import KVWriter
 13 | 
 14 | 
 15 | def load_video(vis_path, num_frm=100):
 16 |     # vr = VideoReader(vis_path, ctx=cpu(0))
 17 |     vr = VideoReader(vis_path)
 18 |     total_frame_num = len(vr)
 19 |     total_num_frm = min(total_frame_num, num_frm)
 20 |     frame_idx = get_seq_frames(total_frame_num, total_num_frm)
 21 |     img_array = vr.get_batch(frame_idx).asnumpy()  # (n_clips*num_frm, H, W, 3)
 22 | 
 23 |     a, H, W, _ = img_array.shape
 24 |     h, w = 224, 224
 25 |     if img_array.shape[-3] != h or img_array.shape[-2] != w:
 26 |         img_array = torch.from_numpy(img_array).permute(0, 3, 1, 2).float()
 27 |         img_array = torch.nn.functional.interpolate(img_array, size=(h, w))
 28 |         img_array = img_array.permute(0, 2, 3, 1).to(torch.uint8).numpy()
 29 |     img_array = img_array.reshape((1, total_num_frm, img_array.shape[-3], img_array.shape[-2], img_array.shape[-1]))
 30 | 
 31 |     clip_imgs = []
 32 |     for j in range(total_num_frm):
 33 |         clip_imgs.append(Image.fromarray(img_array[0, j]))
 34 | 
 35 |     return clip_imgs
 36 | 
 37 | 
 38 | def get_seq_frames(total_num_frames, desired_num_frames):
 39 |     seg_size = float(total_num_frames - 1) / desired_num_frames
 40 |     seq = []
 41 |     for i in range(desired_num_frames):
 42 |         start = int(np.round(seg_size * i))
 43 |         end = int(np.round(seg_size * (i + 1)))
 44 |         seq.append((start + end) // 2)
 45 | 
 46 |     return seq
 47 | 
 48 | 
 49 | 
 50 | def parse_args():
 51 |     parser = argparse.ArgumentParser(description="Training")
 52 | 
 53 |     parser.add_argument("--video_dir_path", default="/opt/tiger/activitynet_videos", help="Path to read the videos from.")
 54 |     parser.add_argument("--clip_feat_path", default="/opt/tiger/activity_clip-14L_spatio_temporal_patch", help="The output dir to save the features in.")
 55 |     parser.add_argument("--infer_batch", required=False, type=int, default=16,
 56 |                         help="Number of frames/images to perform batch inference.")
 57 | 
 58 |     args = parser.parse_args()
 59 | 
 60 |     return args
 61 | 
 62 | 
 63 | def main():
 64 |     args = parse_args()
 65 |     video_dir_path = args.video_dir_path
 66 |     clip_feat_path = args.clip_feat_path
 67 |     infer_batch = args.infer_batch
 68 | 
 69 |     rank = int(os.environ.get('RANK',0))
 70 |     num_gpus = torch.cuda.device_count()
 71 |     dist.init_process_group(backend='nccl')
 72 |     os.makedirs(clip_feat_path, exist_ok=True)
 73 | 
 74 |     output_dir = f'{args.clip_feat_path}.{rank}'
 75 |     writer = KVWriter(output_dir, 1)
 76 | 
 77 |     # Initialize the CLIP model
 78 |     image_processor = CLIPImageProcessor.from_pretrained('openai/clip-vit-large-patch14', torch_dtype=torch.float16)
 79 |     vision_tower = CLIPVisionModel.from_pretrained('openai/clip-vit-large-patch14', torch_dtype=torch.float16,
 80 |                                                    low_cpu_mem_usage=True).to(rank)
 81 |     vision_tower.eval()
 82 | 
 83 |     all_videos = sorted(os.listdir(video_dir_path))
 84 |     all_videos = all_videos[rank::num_gpus]
 85 |     video_clip_features = {}
 86 |     counter = 0
 87 |     for video_name in tqdm(all_videos):
 88 |         video_path = f"{video_dir_path}/{video_name}"
 89 |         video_id = video_name.split('.')[0]
 90 |         if os.path.exists(f"{clip_feat_path}/{video_id}.pkl"):  # Check if the file is already processed
 91 |             continue
 92 |         try:
 93 |             video = load_video(video_path)
 94 |             video_tensor = image_processor.preprocess(video, return_tensors='pt')['pixel_values']
 95 |             video_tensor = video_tensor.half()
 96 | 
 97 |             n_chunk = len(video_tensor)
 98 |             video_features = torch.FloatTensor(n_chunk, 257, 1024).fill_(0)
 99 |             n_iter = int(math.ceil(n_chunk / float(infer_batch)))
100 |             for i in range(n_iter):
101 |                 min_ind = i * infer_batch
102 |                 max_ind = (i + 1) * infer_batch
103 |                 video_batch = video_tensor[min_ind:max_ind].to(rank)
104 | 
105 |                 image_forward_outs = vision_tower(video_batch, output_hidden_states=True)
106 | 
107 |                 select_hidden_state_layer = -2
108 |                 select_hidden_state = image_forward_outs.hidden_states[select_hidden_state_layer]
109 |                 # select_hidden_state = select_hidden_state[:, 1:]
110 |                 video_features[min_ind:max_ind] = select_hidden_state.detach().cpu()
111 |             video_features = video_features.numpy().astype("float16")
112 |             writer.write_many([video_id], [pickle.dumps(video_features)])
113 |             counter += 1
114 |             # video_clip_features[video_id] = get_spatio_temporal_features(video_features.numpy().astype("float16"))
115 |             # video_clip_features[video_id] = video_features
116 | 
117 |         except Exception as e:
118 |             print(f"Can't process {video_path}, error: {e}")
119 | 
120 |         # if counter % 32 == 0:  # Save after every 512 videos, update this number as per your requirements
121 |         #     for key in video_clip_features.keys():
122 |         #         features = video_clip_features[key]
123 |         #         with open(f"{clip_feat_path}/{key}.pkl", 'wb') as f:
124 |         #             pickle.dump(features, f)
125 |         #     video_clip_features = {}
126 |     writer.flush()
127 | 
128 |     # for key in video_clip_features.keys():
129 |     #     features = video_clip_features[key]
130 |     #     with open(f"{clip_feat_path}/{key}.pkl", 'wb') as f:
131 |     #         pickle.dump(features, f)
132 | 
133 | 
134 | if __name__ == "__main__":
135 |     main()
136 | 


--------------------------------------------------------------------------------
/vista_llama/eval/model_utils.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import logging
  3 | import numpy as np
  4 | from PIL import Image
  5 | from decord import VideoReader, cpu
  6 | from transformers import AutoTokenizer, CLIPVisionModel, CLIPImageProcessor
  7 | from vista_llama.model import VideoChatGPTLlamaForCausalLM
  8 | from vista_llama.utils import disable_torch_init 
  9 | from vista_llama.constants import *
 10 | import torch
 11 | 
 12 | 
 13 | def load_video(vis_path, n_clips=1, num_frm=100):
 14 |     """
 15 |     Load video frames from a video file.
 16 | 
 17 |     Parameters:
 18 |     vis_path (str): Path to the video file.
 19 |     n_clips (int): Number of clips to extract from the video. Defaults to 1.
 20 |     num_frm (int): Number of frames to extract from each clip. Defaults to 100.
 21 | 
 22 |     Returns:
 23 |     list: List of PIL.Image.Image objects representing video frames.
 24 |     """
 25 | 
 26 |     # Load video with VideoReader
 27 |     vr = VideoReader(vis_path, ctx=cpu(0))
 28 |     total_frame_num = len(vr)
 29 | 
 30 |     # Currently, this function supports only 1 clip
 31 |     assert n_clips == 1
 32 | 
 33 |     # Calculate total number of frames to extract
 34 |     total_num_frm = min(total_frame_num, num_frm)
 35 |     # Get indices of frames to extract
 36 |     frame_idx = get_seq_frames(total_frame_num, total_num_frm)
 37 |     # Extract frames as numpy array
 38 |     # img_array = np.array(vr.get_batch(frame_idx))
 39 |     img_array = vr.get_batch(frame_idx).asnumpy()
 40 |     # Set target image height and width
 41 |     target_h, target_w = 224, 224
 42 |     # If image shape is not as target, resize it
 43 |     if img_array.shape[-3] != target_h or img_array.shape[-2] != target_w:
 44 |         img_array = torch.from_numpy(img_array).permute(0, 3, 1, 2).float()
 45 |         img_array = torch.nn.functional.interpolate(img_array, size=(target_h, target_w))
 46 |         img_array = img_array.permute(0, 2, 3, 1).to(torch.uint8).numpy()
 47 | 
 48 |     # Reshape array to match number of clips and frames
 49 |     img_array = img_array.reshape(
 50 |         (n_clips, total_num_frm, img_array.shape[-3], img_array.shape[-2], img_array.shape[-1]))
 51 |     # Convert numpy arrays to PIL Image objects
 52 |     clip_imgs = [Image.fromarray(img_array[0, j]) for j in range(total_num_frm)]
 53 | 
 54 |     return clip_imgs
 55 | 
 56 | 
 57 | def get_seq_frames(total_num_frames, desired_num_frames):
 58 |     """
 59 |     Calculate the indices of frames to extract from a video.
 60 | 
 61 |     Parameters:
 62 |     total_num_frames (int): Total number of frames in the video.
 63 |     desired_num_frames (int): Desired number of frames to extract.
 64 | 
 65 |     Returns:
 66 |     list: List of indices of frames to extract.
 67 |     """
 68 | 
 69 |     # Calculate the size of each segment from which a frame will be extracted
 70 |     seg_size = float(total_num_frames - 1) / desired_num_frames
 71 | 
 72 |     seq = []
 73 |     for i in range(desired_num_frames):
 74 |         # Calculate the start and end indices of each segment
 75 |         start = int(np.round(seg_size * i))
 76 |         end = int(np.round(seg_size * (i + 1)))
 77 | 
 78 |         # Append the middle index of the segment to the list
 79 |         seq.append((start + end) // 2)
 80 | 
 81 |     return seq
 82 | 
 83 | 
 84 | def initialize_model(model_name, projection_path=None, device=0):
 85 |     """
 86 |     Initializes the model with given parameters.
 87 | 
 88 |     Parameters:
 89 |     model_name (str): Name of the model to initialize.
 90 |     projection_path (str, optional): Path to the projection weights. Defaults to None.
 91 | 
 92 |     Returns:
 93 |     tuple: Model, vision tower, tokenizer, image processor, vision config, and video token length.
 94 |     """
 95 | 
 96 |     # Disable initial torch operations
 97 |     disable_torch_init()
 98 | 
 99 |     # Convert model name to user path
100 |     model_name = os.path.expanduser(model_name)
101 | 
102 |     # Load tokenizer
103 |     tokenizer = AutoTokenizer.from_pretrained(model_name)
104 | 
105 |     # Load model
106 |     model = VideoChatGPTLlamaForCausalLM.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype=torch.float16,
107 |                                                          use_cache=True)
108 | 
109 |     # Load image processor
110 |     image_processor = CLIPImageProcessor.from_pretrained(model.config.mm_vision_tower, torch_dtype=torch.float16)
111 | 
112 |     # Set to use start and end tokens for video
113 |     mm_use_vid_start_end = True
114 | 
115 |     # Add tokens to tokenizer
116 |     tokenizer.add_tokens([DEFAULT_VIDEO_PATCH_TOKEN], special_tokens=True)
117 |     if mm_use_vid_start_end:
118 |         tokenizer.add_tokens([DEFAULT_VID_START_TOKEN, DEFAULT_VID_END_TOKEN], special_tokens=True)
119 | 
120 |     # Resize token embeddings of the model
121 |     model.resize_token_embeddings(len(tokenizer))
122 | 
123 |     # Load the weights from projection_path after resizing the token_embeddings
124 |     if projection_path:
125 |         print(f"Loading weights from {projection_path}")
126 |         status = model.load_state_dict(torch.load(projection_path, map_location='cpu'), strict=False)
127 |         if status.unexpected_keys:
128 |             print(f"Unexpected Keys: {status.unexpected_keys}.\nThe Video-ChatGPT weights are not loaded correctly.")
129 |         print(f"Weights loaded from {projection_path}")
130 | 
131 |     # Set model to evaluation mode and move to GPU
132 |     model = model.eval()
133 |     model = model.to(device)
134 | 
135 |     vision_tower_name = "openai/clip-vit-large-patch14"
136 | 
137 |     # Load vision tower and move to GPU
138 |     vision_tower = CLIPVisionModel.from_pretrained(vision_tower_name, torch_dtype=torch.float16,
139 |                                                    low_cpu_mem_usage=True).to(device)
140 |     vision_tower = vision_tower.eval()
141 | 
142 |     # Configure vision model
143 |     vision_config = model.get_model().vision_config
144 |     vision_config.vid_patch_token = tokenizer.convert_tokens_to_ids([DEFAULT_VIDEO_PATCH_TOKEN])[0]
145 |     vision_config.use_vid_start_end = mm_use_vid_start_end
146 |     if mm_use_vid_start_end:
147 |         vision_config.vid_start_token, vision_config.vid_end_token = tokenizer.convert_tokens_to_ids(
148 |             [DEFAULT_VID_START_TOKEN, DEFAULT_VID_END_TOKEN])
149 | 
150 |     # Set video token length
151 |     video_token_len = 356
152 | 
153 |     return model, vision_tower, tokenizer, image_processor, video_token_len
154 | 
155 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/benchmark_dataset_generation/generate_correctness_detailed_context_qa.py:
--------------------------------------------------------------------------------
  1 | import openai
  2 | import os
  3 | import argparse
  4 | import warnings
  5 | import json
  6 | import ast
  7 | from multiprocessing.pool import Pool
  8 | 
  9 | warnings.filterwarnings('ignore')
 10 | 
 11 | 
 12 | def parse_args():
 13 |     parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
 14 |     parser.add_argument("--gt_caption_folder", required=True, help="The path to captions")
 15 |     parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
 16 |     parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
 17 |     parser.add_argument("--api_key", required=True, help="OpenAI API key.")
 18 |     parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
 19 |     args = parser.parse_args()
 20 |     return args
 21 | 
 22 | 
 23 | def annotate(gt_file, caption_files, output_dir):
 24 |     """
 25 |     Generate generic descriptive type questions and answers for each caption file using GPT-3.
 26 |     """
 27 |     for file in caption_files:
 28 |         key = file[:-5] # Strip file extension.
 29 |         caption = gt_file[key]
 30 |         try:
 31 |             # Generate GPT-3 response.
 32 |             completion = openai.ChatCompletion.create(
 33 |                 model="gpt-3.5-turbo",
 34 |                 messages=[
 35 |                     {
 36 |                         "role": "system",
 37 |                         "content": 
 38 |                             "You will play two roles: a human asking questions related to describing a video and an intelligent chatbot designed for video description and dense captioning. "
 39 |                             "Your task is to generate a detailed and descriptive paragraph based on the provided fragmented information about a video. "
 40 |                             "------"
 41 |                             "##TASK:"
 42 |                             "Users will provide a descriptions of a video, and you will generate ONE conversation-like question and answer related to describing the video in detail. "
 43 |                             "The question should ask to describe the video content in detail. "
 44 |                             "The answer should be a paraphrased and well-structured paragraph based on the provided description, as detailed as possible. "
 45 |                     },
 46 |                     {
 47 |                         "role": "user",
 48 |                         "content":
 49 |                             f"The user input is: {caption}. "
 50 |                             f"Please generate the response in the form of a Python dictionary string with keys 'Q' for question and 'A' for answer. Each corresponding value should be the question and answer text respectively. "
 51 |                             "For example, your response should look like this: {'Q': 'Your question here...', 'A': 'Your answer here...'}. "
 52 |                             f"Emphasize that the answer should focus on describing the video content as detailed as possible."
 53 |                     }
 54 |                 ]
 55 |             )
 56 |             # Convert response to a Python dictionary.
 57 |             response_message = completion["choices"][0]["message"]["content"]
 58 |             response_dict = ast.literal_eval(response_message)
 59 | 
 60 |             # Save the question-answer pairs to a json file.
 61 |             with open(f"{output_dir}/{key}.json", "w") as f:
 62 |                 json.dump(response_dict, f)
 63 |         except Exception as e:
 64 |             print(f"Error processing file '{key}': {e}")
 65 | 
 66 | 
 67 | def main():
 68 |     """
 69 |     Main function to control the flow of the program.
 70 |     """
 71 |     # Parse arguments.
 72 |     args = parse_args()
 73 | 
 74 |     # Read ground truth captions.
 75 |     gt_captions = {}
 76 |     gt_files = os.listdir(args.gt_caption_folder)
 77 |     for file in gt_files:
 78 |         with open(os.path.join(args.gt_caption_folder, file), mode='r', encoding='utf-8-sig') as f:
 79 |             caption = f.read().replace('\n', '').replace('‘', "'").replace('’', "'")
 80 |             video_id = file[:-4]
 81 |             gt_captions[video_id] = caption
 82 | 
 83 |     caption_files = [f"{video_id}.json" for video_id in gt_captions.keys()]
 84 |     output_dir = args.output_dir
 85 |     # Generate output directory if not exists.
 86 |     if not os.path.exists(output_dir):
 87 |         os.makedirs(output_dir)
 88 | 
 89 |     # Set the OpenAI API key.
 90 |     openai.api_key = args.api_key
 91 |     num_tasks = args.num_tasks
 92 | 
 93 |     # While loop to ensure that all captions are processed.
 94 |     while True:
 95 |         try:
 96 |             # Files that have not been processed yet.
 97 |             completed_files = os.listdir(output_dir)
 98 |             print(f"completed_files: {len(completed_files)}")
 99 | 
100 |             # Files that have not been processed yet.
101 |             incomplete_files = [f for f in caption_files if f not in completed_files]
102 |             print(f"incomplete_files: {len(incomplete_files)}")
103 | 
104 |             if len(incomplete_files) == 0:
105 |                 break
106 |             if len(incomplete_files) <= num_tasks:
107 |                 num_tasks = 1
108 | 
109 |             # Split tasks into parts.
110 |             part_len = len(incomplete_files) // num_tasks
111 |             all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
112 |             task_args = [(gt_captions, part, args.output_dir) for part in all_parts]
113 | 
114 |             # Use a pool of workers to process the files in parallel.
115 |             with Pool() as pool:
116 |                 pool.starmap(annotate, task_args)
117 | 
118 |         except Exception as e:
119 |             print(f"Error: {e}")
120 | 
121 |     # Combine qa pairs into single file when individual qa generation completes
122 |     all_data = {}
123 |     for filename in os.listdir(output_dir):
124 |         if filename.endswith(".json"):
125 |             with open(os.path.join(output_dir, filename)) as f:
126 |                 key = filename[:-5]
127 |                 all_data[key] = json.load(f)
128 | 
129 |     with open(args.output_json, 'w') as f:
130 |         json.dump(all_data, f, indent=4)
131 | 
132 | 
133 | if __name__ == "__main__":
134 |     main()
135 | 


--------------------------------------------------------------------------------
/vista_llama/model/logger.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import datetime
  9 | import logging
 10 | import time
 11 | from collections import defaultdict, deque
 12 | 
 13 | import torch
 14 | import torch.distributed as dist
 15 | 
 16 | from . import dist_utils
 17 | 
 18 | 
 19 | class SmoothedValue(object):
 20 |     """Track a series of values and provide access to smoothed values over a
 21 |     window or the global series average.
 22 |     """
 23 | 
 24 |     def __init__(self, window_size=20, fmt=None):
 25 |         if fmt is None:
 26 |             fmt = "{median:.4f} ({global_avg:.4f})"
 27 |         self.deque = deque(maxlen=window_size)
 28 |         self.total = 0.0
 29 |         self.count = 0
 30 |         self.fmt = fmt
 31 | 
 32 |     def update(self, value, n=1):
 33 |         self.deque.append(value)
 34 |         self.count += n
 35 |         self.total += value * n
 36 | 
 37 |     def synchronize_between_processes(self):
 38 |         """
 39 |         Warning: does not synchronize the deque!
 40 |         """
 41 |         if not dist_utils.is_dist_avail_and_initialized():
 42 |             return
 43 |         t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda")
 44 |         dist.barrier()
 45 |         dist.all_reduce(t)
 46 |         t = t.tolist()
 47 |         self.count = int(t[0])
 48 |         self.total = t[1]
 49 | 
 50 |     @property
 51 |     def median(self):
 52 |         d = torch.tensor(list(self.deque))
 53 |         return d.median().item()
 54 | 
 55 |     @property
 56 |     def avg(self):
 57 |         d = torch.tensor(list(self.deque), dtype=torch.float32)
 58 |         return d.mean().item()
 59 | 
 60 |     @property
 61 |     def global_avg(self):
 62 |         return self.total / self.count
 63 | 
 64 |     @property
 65 |     def max(self):
 66 |         return max(self.deque)
 67 | 
 68 |     @property
 69 |     def value(self):
 70 |         return self.deque[-1]
 71 | 
 72 |     def __str__(self):
 73 |         return self.fmt.format(
 74 |             median=self.median,
 75 |             avg=self.avg,
 76 |             global_avg=self.global_avg,
 77 |             max=self.max,
 78 |             value=self.value,
 79 |         )
 80 | 
 81 | 
 82 | class MetricLogger(object):
 83 |     def __init__(self, delimiter="\t"):
 84 |         self.meters = defaultdict(SmoothedValue)
 85 |         self.delimiter = delimiter
 86 | 
 87 |     def update(self, **kwargs):
 88 |         for k, v in kwargs.items():
 89 |             if isinstance(v, torch.Tensor):
 90 |                 v = v.item()
 91 |             assert isinstance(v, (float, int))
 92 |             self.meters[k].update(v)
 93 | 
 94 |     def __getattr__(self, attr):
 95 |         if attr in self.meters:
 96 |             return self.meters[attr]
 97 |         if attr in self.__dict__:
 98 |             return self.__dict__[attr]
 99 |         raise AttributeError(
100 |             "'{}' object has no attribute '{}'".format(type(self).__name__, attr)
101 |         )
102 | 
103 |     def __str__(self):
104 |         loss_str = []
105 |         for name, meter in self.meters.items():
106 |             loss_str.append("{}: {}".format(name, str(meter)))
107 |         return self.delimiter.join(loss_str)
108 | 
109 |     def global_avg(self):
110 |         loss_str = []
111 |         for name, meter in self.meters.items():
112 |             loss_str.append("{}: {:.4f}".format(name, meter.global_avg))
113 |         return self.delimiter.join(loss_str)
114 | 
115 |     def synchronize_between_processes(self):
116 |         for meter in self.meters.values():
117 |             meter.synchronize_between_processes()
118 | 
119 |     def add_meter(self, name, meter):
120 |         self.meters[name] = meter
121 | 
122 |     def log_every(self, iterable, print_freq, header=None):
123 |         i = 0
124 |         if not header:
125 |             header = ""
126 |         start_time = time.time()
127 |         end = time.time()
128 |         iter_time = SmoothedValue(fmt="{avg:.4f}")
129 |         data_time = SmoothedValue(fmt="{avg:.4f}")
130 |         space_fmt = ":" + str(len(str(len(iterable)))) + "d"
131 |         log_msg = [
132 |             header,
133 |             "[{0" + space_fmt + "}/{1}]",
134 |             "eta: {eta}",
135 |             "{meters}",
136 |             "time: {time}",
137 |             "data: {data}",
138 |         ]
139 |         if torch.cuda.is_available():
140 |             log_msg.append("max mem: {memory:.0f}")
141 |         log_msg = self.delimiter.join(log_msg)
142 |         MB = 1024.0 * 1024.0
143 |         for obj in iterable:
144 |             data_time.update(time.time() - end)
145 |             yield obj
146 |             iter_time.update(time.time() - end)
147 |             if i % print_freq == 0 or i == len(iterable) - 1:
148 |                 eta_seconds = iter_time.global_avg * (len(iterable) - i)
149 |                 eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
150 |                 if torch.cuda.is_available():
151 |                     print(
152 |                         log_msg.format(
153 |                             i,
154 |                             len(iterable),
155 |                             eta=eta_string,
156 |                             meters=str(self),
157 |                             time=str(iter_time),
158 |                             data=str(data_time),
159 |                             memory=torch.cuda.max_memory_allocated() / MB,
160 |                         )
161 |                     )
162 |                 else:
163 |                     print(
164 |                         log_msg.format(
165 |                             i,
166 |                             len(iterable),
167 |                             eta=eta_string,
168 |                             meters=str(self),
169 |                             time=str(iter_time),
170 |                             data=str(data_time),
171 |                         )
172 |                     )
173 |             i += 1
174 |             end = time.time()
175 |         total_time = time.time() - start_time
176 |         total_time_str = str(datetime.timedelta(seconds=int(total_time)))
177 |         print(
178 |             "{} Total time: {} ({:.4f} s / it)".format(
179 |                 header, total_time_str, total_time / len(iterable)
180 |             )
181 |         )
182 | 
183 | 
184 | class AttrDict(dict):
185 |     def __init__(self, *args, **kwargs):
186 |         super(AttrDict, self).__init__(*args, **kwargs)
187 |         self.__dict__ = self
188 | 
189 | 
190 | def setup_logger():
191 |     logging.basicConfig(
192 |         level=logging.INFO if dist_utils.is_main_process() else logging.WARN,
193 |         format="%(asctime)s [%(levelname)s] %(message)s",
194 |         handlers=[logging.StreamHandler()],
195 |     )
196 | 


--------------------------------------------------------------------------------
/data/generate_descriptive_qa.py:
--------------------------------------------------------------------------------
  1 | # Required Libraries
  2 | import openai
  3 | import os
  4 | import json
  5 | import time
  6 | import ast
  7 | import argparse
  8 | import warnings
  9 | from tqdm import tqdm
 10 | from multiprocessing.pool import Pool
 11 | 
 12 | # Suppressing all warnings
 13 | warnings.filterwarnings('ignore')
 14 | 
 15 | 
 16 | def parse_args():
 17 |     """
 18 |     Command-line argument parser.
 19 |     """
 20 |     parser = argparse.ArgumentParser(description="Descriptive question-answer-generation-using-GPT-3")
 21 |     parser.add_argument("--gt_caption_file", required=True, help="Path to the ground truth captions file.")
 22 |     parser.add_argument("--output_dir", required=True, help="Path to save the annotation JSON files.")
 23 |     parser.add_argument("--api_key", required=True, help="OpenAI API key.")
 24 |     parser.add_argument("--num_tasks", required=False, type=int, help="Number of splits.", default=10)
 25 | 
 26 |     return parser.parse_args()
 27 | 
 28 | 
 29 | def annotate(gt_file, caption_files, output_dir):
 30 |     """
 31 |     Generates question and answer pairs based on video captions using OpenAI GPT-3.
 32 |     """
 33 |     for file in tqdm(caption_files):
 34 |         key = file[:-4] # Strip file extension.
 35 |         caption = gt_file[key]
 36 | 
 37 |         # Generate completion with OpenAI GPT-3
 38 |         completion = openai.ChatCompletion.create(
 39 |             model="gpt-3.5-turbo",
 40 |             messages=[
 41 |                 {
 42 |                     "role": "system",
 43 |                     "content":
 44 |                         "You will play two roles: a human asking questions related to describing a video and "
 45 |                         "an intelligent chatbot designed for video description and dense captioning. "
 46 |                         "Your task is to generate a detailed and descriptive paragraph based on the provided fragmented information about a video. "
 47 |                         "------"
 48 |                         "##TASK:"
 49 |                         "Users will provide fragmented descriptions of a video, and you will generate ONE conversation-like question and answer related to describing the video in detail. "
 50 |                         "The question should ask to describe the video content in detail. "
 51 |                         "The answer should be a paraphrased and well-structured paragraph based on the provided description, with a minimum of 150 words and a maximum of 300 words. "
 52 |                         "When the provided information is short, aim for a 150-word description, and when the provided information is more detailed, aim for very long descriptions upto 300-word description. "
 53 |                         "------"
 54 |                         "##INSTRUCTIONS:"
 55 |                         "- The question must be like a human conversation and focused on describing the video in detail. "
 56 |                         "- The answer must be a paraphrased version of the provided information, very detailed and descriptive, and within the specified word count. "
 57 |                 },
 58 |                 {
 59 |                     "role": "user",
 60 |                     "content":
 61 |                         f"The fragmented video description is: {caption}. "
 62 |                         "Please generate the response in the form of a Python dictionary string with keys 'Q' for question and 'A' for answer. Each corresponding value should be the question and answer text respectively. "
 63 |                         "For example, your response should look like this: {'Q': 'Your question here...', 'A': 'Your answer here...'}. "
 64 |                         "Emphasize that the answer should focus on describing the video content as detailed as possible."
 65 |                 }
 66 |             ]
 67 |         )
 68 |         # Convert response to a list of dictionary.
 69 |         response_message = completion["choices"][0]["message"]["content"]
 70 |         response_dict = ast.literal_eval(response_message)
 71 | 
 72 |         # Save the response dictionary into a JSON file
 73 |         json_file_path = os.path.join(output_dir, f"{key}.json")
 74 |         with open(json_file_path, "w") as f:
 75 |             json.dump(response_dict, f)
 76 | 
 77 |     print(f"Completed, Annotations saved in {output_dir}")
 78 | 
 79 | 
 80 | def main():
 81 |     """
 82 |     Main function to control the flow of the program.
 83 |     """
 84 |     # Parse arguments
 85 |     args = parse_args()
 86 | 
 87 |     # Read ground truth captions file
 88 |     gt_file = args.gt_caption_file
 89 |     with open(gt_file) as file:
 90 |         gt_captions = json.load(file)
 91 | 
 92 |     # Get the video_file_names
 93 |     video_files = list(gt_captions.keys())
 94 | 
 95 |     caption = {}
 96 |     for video_file in tqdm(video_files):
 97 |         key = video_file[:-4].rstrip(".") # Strip file extension.
 98 |         try:
 99 |             gt_sentences = gt_captions[key]['sentences']
100 |         except KeyError:
101 |             print(f"Warning: GT captions not found for video file. Skipping...")
102 |             continue
103 |         caption[key] = gt_sentences
104 | 
105 |     # Prepare list of caption files
106 |     caption_files = [f'{video_id}.json' for video_id in caption.keys()]
107 | 
108 |     # Create output directory if it does not exist
109 |     if not os.path.exists(args.output_dir):
110 |         os.makedirs(args.output_dir)
111 | 
112 |     # Set OpenAI API key
113 |     openai.api_key = args.api_key
114 |     num_tasks = args.num_tasks
115 | 
116 |     # Main loop: Continues until all question-answer pairs are generated for all captions
117 |     while True:
118 |         try:
119 |             # Files that have already been completed.
120 |             completed_files = os.listdir(args.output_dir)
121 |             print(f"completed_files: {len(completed_files)}")
122 | 
123 |             # Files that have not been processed yet.
124 |             incomplete_files = [f for f in caption_files if f not in completed_files]
125 |             print(f"incomplete_files: {len(incomplete_files)}")
126 | 
127 |             if len(incomplete_files) == 0:
128 |                 print("All tasks completed!")
129 |                 break
130 | 
131 |             if len(incomplete_files) <= num_tasks:
132 |                 num_tasks = 1
133 | 
134 |             # Split tasks into parts.
135 |             num_tasks = min(len(incomplete_files), num_tasks)
136 |             part_len = len(incomplete_files) // num_tasks
137 |             all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
138 | 
139 |             task_args = [(caption, part, args.output_dir) for part in all_parts]
140 | 
141 |             # Use a pool of workers to process the files in parallel.
142 |             with Pool() as pool:
143 |                 pool.starmap(annotate, task_args)
144 | 
145 |         except Exception as e:
146 |             print(f"Error: {e}")
147 |             print("Sleeping for 2 minutes...")
148 |             time.sleep(120)  # wait for 2 minutes before trying again
149 | 
150 | 
151 | if __name__ == "__main__":
152 |     main()
153 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/benchmark_dataset_generation/generate_temporal_qa.py:
--------------------------------------------------------------------------------
  1 | import openai
  2 | import os
  3 | import argparse
  4 | import warnings
  5 | import json
  6 | import ast
  7 | from multiprocessing.pool import Pool
  8 | 
  9 | warnings.filterwarnings('ignore')
 10 | 
 11 | 
 12 | def parse_args():
 13 |     parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
 14 |     parser.add_argument("--gt_caption_folder", required=True, help="The path to captions")
 15 |     parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
 16 |     parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
 17 |     parser.add_argument("--api_key", required=True, help="OpenAI API key.")
 18 |     parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
 19 |     args = parser.parse_args()
 20 |     return args
 21 | 
 22 | 
 23 | def annotate(gt_file, caption_files, output_dir):
 24 |     """
 25 |     Generate questions and answers for each caption file using GPT-3.
 26 |     """
 27 |     for file in caption_files:
 28 |         key = file[:-5] # Strip file extension.
 29 |         caption = gt_file[key]
 30 |         try:
 31 |             # Generate GPT-3 response.
 32 |             completion = openai.ChatCompletion.create(
 33 |                 model="gpt-3.5-turbo",
 34 |                 messages=[
 35 |                     {
 36 |                         "role": "system",
 37 |                         "content": 
 38 |                             "You play two roles: a human asking questions related to a video and an intelligent chatbot designed to help people find information from a given video. "
 39 |                             "Your task is to generate a question-answer pair specifically related to temporal understanding from the video content. "
 40 |                             "Your task is to first play the role of a human who asks a question about the temporal sequence or timing of events in the video and then play the role of an AI assistant that provides information based on the video content."
 41 |                             "------"
 42 |                             "##TASK: "
 43 |                             "Users will provide some information about a video, and you will generate a conversation-like question and answers pair specifically focusing on the temporal sequence of events in the video. "
 44 |                             "The question should be designed to extract temporal sequence information directly from the given information, so that the provided information or parts of it can serve as the answer. "
 45 |                             "Generate ONE descriptive and conversational style question and detailed answer based on the given information, specifically related to the temporal understanding in the video."
 46 |                             "------"
 47 |                             "##INSTRUCTIONS:"
 48 |                             "- The question must be like a human conversation and directly related to the temporal sequence of events in the video. "
 49 |                             "- The question should be designed to extract temporal sequence information DIRECTLY from the given information, so that it or parts of it can serve as the answer. "
 50 |                             "- The answer must be detailed and descriptive, and should directly reference the information provided with respect to the temporal sequence of events in the video."
 51 |                     },
 52 |                     {
 53 |                         "role": "user",
 54 |                         "content":
 55 |                             f"The user input is: {caption}. "
 56 |                             "Please generate the response in the form of a Python dictionary string with keys 'Q' for question and 'A' for answer. Each corresponding value should be the question and answer text respectively. "
 57 |                             "For example, your response should look like this: {'Q': 'Your question here...', 'A': 'Your answer here...'}. "
 58 |                     }
 59 |                 ]
 60 |             )
 61 |             # Convert response to a Python dictionary.
 62 |             response_message = completion["choices"][0]["message"]["content"]
 63 |             response_dict = ast.literal_eval(response_message)
 64 | 
 65 |             # Save the question-answer pairs to a json file.
 66 |             with open(f"{output_dir}/{key}.json", "w") as f:
 67 |                 json.dump(response_dict, f)
 68 |         except Exception as e:
 69 |             print(f"Error processing file '{key}': {e}")
 70 | 
 71 | 
 72 | def main():
 73 |     """
 74 |     Main function to control the flow of the program.
 75 |     """
 76 |     # Parse arguments.
 77 |     args = parse_args()
 78 | 
 79 |     # Read ground truth captions.
 80 |     gt_captions = {}
 81 |     gt_files = os.listdir(args.gt_caption_folder)
 82 |     for file in gt_files:
 83 |         with open(os.path.join(args.gt_caption_folder, file), mode='r', encoding='utf-8-sig') as f:
 84 |             caption = f.read().replace('\n', '').replace('‘', "'").replace('’', "'")
 85 |             video_id = file[:-4]
 86 |             gt_captions[video_id] = caption
 87 | 
 88 |     caption_files = [f"{video_id}.json" for video_id in gt_captions.keys()]
 89 |     output_dir = args.output_dir
 90 |     # Generate output directory if not exists.
 91 |     if not os.path.exists(output_dir):
 92 |         os.makedirs(output_dir)
 93 | 
 94 |     # Set the OpenAI API key.
 95 |     openai.api_key = args.api_key
 96 |     num_tasks = args.num_tasks
 97 | 
 98 |     # While loop to ensure that all captions are processed.
 99 |     while True:
100 |         try:
101 |             # Files that have not been processed yet.
102 |             completed_files = os.listdir(output_dir)
103 |             print(f"completed_files: {len(completed_files)}")
104 | 
105 |             # Files that have not been processed yet.
106 |             incomplete_files = [f for f in caption_files if f not in completed_files]
107 |             print(f"incomplete_files: {len(incomplete_files)}")
108 | 
109 |             if len(incomplete_files) == 0:
110 |                 break
111 |             if len(incomplete_files) <= num_tasks:
112 |                 num_tasks = 1
113 | 
114 |             # Split tasks into parts.
115 |             part_len = len(incomplete_files) // num_tasks
116 |             all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
117 |             task_args = [(gt_captions, part, args.output_dir) for part in all_parts]
118 | 
119 |             # Use a pool of workers to process the files in parallel.
120 |             with Pool() as pool:
121 |                 pool.starmap(annotate, task_args)
122 | 
123 |         except Exception as e:
124 |             print(f"Error: {e}")
125 | 
126 |     # Combine qa pairs into single file when individual qa generation completes
127 |     all_data = {}
128 |     for filename in os.listdir(output_dir):
129 |         if filename.endswith(".json"):
130 |             with open(os.path.join(output_dir, filename)) as f:
131 |                 key = filename[:-5]
132 |                 all_data[key] = json.load(f)
133 | 
134 |     with open(args.output_json, 'w') as f:
135 |         json.dump(all_data, f, indent=4)
136 | 
137 | 
138 | if __name__ == "__main__":
139 |     main()
140 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/benchmark_dataset_generation/generate_consistency_qa.py:
--------------------------------------------------------------------------------
  1 | import openai
  2 | import os
  3 | import argparse
  4 | import warnings
  5 | import json
  6 | import ast
  7 | from multiprocessing.pool import Pool
  8 | 
  9 | # Disable warnings.
 10 | warnings.filterwarnings('ignore')
 11 | 
 12 | 
 13 | def parse_args():
 14 |     parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
 15 |     parser.add_argument("--gt_caption_folder", required=True, help="The path to captions")
 16 |     parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
 17 |     parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
 18 |     parser.add_argument("--api_key", required=True, help="OpenAI API key.")
 19 |     parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
 20 |     args = parser.parse_args()
 21 |     return args
 22 | 
 23 | 
 24 | def annotate(gt_file, caption_files, output_dir):
 25 |     """
 26 |     Generate questions and answers for each caption file using GPT-3.
 27 |     """
 28 |     for file in caption_files:
 29 |         key = file[:-5] # Strip file extension.
 30 |         caption = gt_file[key]
 31 |         try:
 32 |             # Generate GPT-3 response.
 33 |             completion = openai.ChatCompletion.create(
 34 |                 model="gpt-3.5-turbo",
 35 |                 messages=[
 36 |                     {
 37 |                         "role": "system",
 38 |                         "content":
 39 |                             "Your primary task is to formulate two distinct but conceptually similar questions, such that when asked about the same video-information, they correspond to the same answer. "
 40 |                             "------"
 41 |                             "##TASK:"
 42 |                             "When given details about a video, your task is to generate two questions asked in different ways. The crucial aspect is to frame these questions so that they are conceptually alike but phrased differently, leading to the exact same answer. "
 43 |                             "The questions should be cleverly designed to extract the same information directly from the video details given, so that the provided information or parts of it can serve as the answer. It's important that both questions yield the SAME answer. "
 44 |                             "- Generate TWO questions and ONE answer. The purpose is to extract identical information from both questions. Therefore, formulate your questions in a way that the given details can serve directly as the answer. "
 45 |                             "------"
 46 |                             "##SAMPLE QUESTIONS:"
 47 |                             "- {'Q1': 'What is the colour of the cycle the boy rides?', 'Q2': 'Can you describe the cycle the boy is riding?', 'A': 'The boy is riding a red bicycle with a basket.'}"
 48 |                             "- {'Q1': 'What is the baby girl doing in the video?', 'Q2': 'Can you see the baby girl engaged in an activity in the video?', 'A': 'The baby girl is reading a book in the video.'}"
 49 |                     },
 50 |                     {
 51 |                         "role": "user",
 52 |                         "content":
 53 |                             f"The user input is: {caption}. "
 54 |                             f"Please generate the response in the form of a Python dictionary string with keys 'Q1', 'Q2', and 'A', where value of 'Q1' is  first question, 'Q2' for second question and 'A' is the answer to both questions. Each corresponding value should be the question or answer text respectively. "
 55 |                             "For example, your response should look like this: {'Q1': 'Your first question here...', 'Q2': 'Your second question here...', 'A': 'Your answer to both questions here...'}. "
 56 |                             "Remember, it's critical to ensure that both questions are designed to extract the same details from the video, leading to the same answer."
 57 |                     }
 58 |                 ]
 59 |             )
 60 |             # Convert response to a Python dictionary.
 61 |             response_message = completion["choices"][0]["message"]["content"]
 62 |             response_dict = ast.literal_eval(response_message)
 63 | 
 64 |             # Save the question-answer pairs to a json file.
 65 |             with open(f"{output_dir}/{key}.json", "w") as f:
 66 |                 json.dump(response_dict, f)
 67 |         except Exception as e:
 68 |             print(f"Error processing file '{key}': {e}")
 69 | 
 70 | 
 71 | def main():
 72 |     """
 73 |     Main function to control the flow of the program.
 74 |     """
 75 |     # Parse arguments.
 76 |     args = parse_args()
 77 | 
 78 |     # Read ground truth captions.
 79 |     gt_captions = {}
 80 |     gt_files = os.listdir(args.gt_caption_folder)
 81 |     for file in gt_files:
 82 |         # Read human-assisted annotations from individual text files.
 83 |         with open(os.path.join(args.gt_caption_folder, file), mode='r', encoding='utf-8-sig') as f:
 84 |             caption = f.read().replace('\n', '').replace('‘', "'").replace('’', "'")
 85 |             video_id = file[:-4]
 86 |             gt_captions[video_id] = caption
 87 | 
 88 |     caption_files = [f"{video_id}.json" for video_id in gt_captions.keys()]
 89 | 
 90 |     output_dir = args.output_dir
 91 |     # Generate output directory if not exists.
 92 |     if not os.path.exists(output_dir):
 93 |         os.makedirs(output_dir)
 94 | 
 95 |     # Set the OpenAI API key.
 96 |     openai.api_key = args.api_key
 97 |     num_tasks = args.num_tasks
 98 | 
 99 |     # While loop to ensure that all captions are processed.
100 |     while True:
101 |         try:
102 |             # Files that have already been completed.
103 |             completed_files = os.listdir(output_dir)
104 |             print(f"completed_files: {len(completed_files)}")
105 | 
106 |             # Files that have not been processed yet.
107 |             incomplete_files = [f for f in caption_files if f not in completed_files]
108 |             print(f"incomplete_files: {len(incomplete_files)}")
109 | 
110 |             if len(incomplete_files) == 0:
111 |                 break
112 |             if len(incomplete_files) <= num_tasks:
113 |                 num_tasks = 1
114 | 
115 |             # Split tasks into parts.
116 |             part_len = len(incomplete_files) // num_tasks
117 |             all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
118 |             task_args = [(gt_captions, part, args.output_dir) for part in all_parts]
119 | 
120 |             # Use a pool of workers to process the files in parallel.
121 |             with Pool() as pool:
122 |                 pool.starmap(annotate, task_args)
123 | 
124 |         except Exception as e:
125 |             print(f"Error: {e}")
126 | 
127 |     # Combine qa pairs into single file when individual qa generation completes
128 |     all_data = {}
129 |     for filename in os.listdir(output_dir):
130 |         if filename.endswith(".json"):
131 |             with open(os.path.join(output_dir, filename)) as f:
132 |                 key = filename[:-5]
133 |                 all_data[key] = json.load(f)
134 | 
135 |     with open(args.output_json, 'w') as f:
136 |         json.dump(all_data, f, indent=4)
137 | 
138 | 
139 | if __name__ == "__main__":
140 |     main()
141 | 


--------------------------------------------------------------------------------
/vista_llama/utils.py:
--------------------------------------------------------------------------------
  1 | import logging
  2 | from typing import Callable, Optional, Tuple, Union
  3 | from torch.distributed import ProcessGroup
  4 | import re
  5 | import logging.handlers
  6 | import os
  7 | import sys
  8 | import torch.distributed as dist
  9 | from torch import distributed as torch_dist
 10 | from .model.dist_utils import get_rank, get_world_size
 11 | _LOCAL_PROCESS_GROUP = None
 12 | 
 13 | 
 14 | import requests
 15 | 
 16 | from vista_llama.constants import LOGDIR
 17 | 
 18 | server_error_msg = "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
 19 | moderation_msg = "YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES. PLEASE TRY AGAIN."
 20 | 
 21 | handler = None
 22 | 
 23 | 
 24 | def is_distributed() -> bool:
 25 |     """Return True if distributed environment has been initialized."""
 26 |     return torch_dist.is_available() and torch_dist.is_initialized()
 27 | 
 28 | 
 29 | def build_logger(logger_name, logger_filename):
 30 |     global handler
 31 | 
 32 |     formatter = logging.Formatter(
 33 |         fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
 34 |         datefmt="%Y-%m-%d %H:%M:%S",
 35 |     )
 36 | 
 37 |     # Set the format of root handlers
 38 |     if not logging.getLogger().handlers:
 39 |         logging.basicConfig(level=logging.INFO)
 40 |     logging.getLogger().handlers[0].setFormatter(formatter)
 41 | 
 42 |     # Redirect stdout and stderr to loggers
 43 |     stdout_logger = logging.getLogger("stdout")
 44 |     stdout_logger.setLevel(logging.INFO)
 45 |     sl = StreamToLogger(stdout_logger, logging.INFO)
 46 |     sys.stdout = sl
 47 | 
 48 |     stderr_logger = logging.getLogger("stderr")
 49 |     stderr_logger.setLevel(logging.ERROR)
 50 |     sl = StreamToLogger(stderr_logger, logging.ERROR)
 51 |     sys.stderr = sl
 52 | 
 53 |     # Get logger
 54 |     logger = logging.getLogger(logger_name)
 55 |     logger.setLevel(logging.INFO)
 56 | 
 57 |     # Add a file handler for all loggers
 58 |     if handler is None:
 59 |         os.makedirs(LOGDIR, exist_ok=True)
 60 |         filename = os.path.join(LOGDIR, logger_filename)
 61 |         handler = logging.handlers.TimedRotatingFileHandler(
 62 |             filename, when='D', utc=True)
 63 |         handler.setFormatter(formatter)
 64 | 
 65 |         for name, item in logging.root.manager.loggerDict.items():
 66 |             if isinstance(item, logging.Logger):
 67 |                 item.addHandler(handler)
 68 | 
 69 |     return logger
 70 | 
 71 | 
 72 | class StreamToLogger(object):
 73 |     """
 74 |     Fake file-like stream object that redirects writes to a logger instance.
 75 |     """
 76 |     def __init__(self, logger, log_level=logging.INFO):
 77 |         self.terminal = sys.stdout
 78 |         self.logger = logger
 79 |         self.log_level = log_level
 80 |         self.linebuf = ''
 81 | 
 82 |     def __getattr__(self, attr):
 83 |         return getattr(self.terminal, attr)
 84 | 
 85 |     def write(self, buf):
 86 |         temp_linebuf = self.linebuf + buf
 87 |         self.linebuf = ''
 88 |         for line in temp_linebuf.splitlines(True):
 89 |             # From the io.TextIOWrapper docs:
 90 |             #   On output, if newline is None, any '\n' characters written
 91 |             #   are translated to the system default line separator.
 92 |             # By default sys.stdout.write() expects '\n' newlines and then
 93 |             # translates them so this is still cross platform.
 94 |             if line[-1] == '\n':
 95 |                 self.logger.log(self.log_level, line.rstrip())
 96 |             else:
 97 |                 self.linebuf += line
 98 | 
 99 |     def flush(self):
100 |         if self.linebuf != '':
101 |             self.logger.log(self.log_level, self.linebuf.rstrip())
102 |         self.linebuf = ''
103 | 
104 | 
105 | def disable_torch_init():
106 |     """
107 |     Disable the redundant torch default initialization to accelerate model creation.
108 |     """
109 |     import torch
110 |     setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
111 |     setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
112 | 
113 | 
114 | def violates_moderation(text):
115 |     """
116 |     Check whether the text violates OpenAI moderation API.
117 |     """
118 |     url = "https://api.openai.com/v1/moderations"
119 |     headers = {"Content-Type": "application/json",
120 |                "Authorization": "Bearer " + os.environ["OPENAI_API_KEY"]}
121 |     text = text.replace("\n", "")
122 |     data = "{" + '"input": ' + f'"{text}"' + "}"
123 |     data = data.encode("utf-8")
124 |     try:
125 |         ret = requests.post(url, headers=headers, data=data, timeout=5)
126 |         flagged = ret.json()["results"][0]["flagged"]
127 |     except requests.exceptions.RequestException as e:
128 |         flagged = False
129 |     except KeyError as e:
130 |         flagged = False
131 | 
132 |     return flagged
133 | 
134 | def is_url(input_url):
135 |     """
136 |     Check if an input string is a url. look for http(s):// and ignoring the case
137 |     """
138 |     is_url = re.match(r"^(?:http)s?://", input_url, re.IGNORECASE) is not None
139 |     return is_url
140 | 
141 | def pretty_print_semaphore(semaphore):
142 |     if semaphore is None:
143 |         return "None"
144 |     return f"Semaphore(value={semaphore._value}, locked={semaphore.locked()})"
145 | 
146 | 
147 | 
148 | 
149 | def get_dist_info(group: Optional[ProcessGroup] = None) -> Tuple[int, int]:
150 |     """Get distributed information of the given process group.
151 | 
152 |     Note:
153 |         Calling ``get_dist_info`` in non-distributed environment will return
154 |         (0, 1).
155 | 
156 |     Args:
157 |         group (ProcessGroup, optional): The process group to work on. If None,
158 |             the default process group will be used. Defaults to None.
159 | 
160 |     Returns:
161 |         tuple[int, int]: Return a tuple containing the ``rank`` and
162 |         ``world_size``.
163 |     """
164 |     world_size = get_world_size(group)
165 |     rank = get_rank(group)
166 |     return rank, world_size
167 | 
168 | def save_result(result, result_dir, filename, remove_duplicate=""):
169 |     import json
170 | 
171 |     rank = get_rank()
172 |     result_file = os.path.join(
173 |         result_dir, "%s_rank%d.json" % (filename, rank)
174 |     )
175 |     final_result_file = os.path.join(result_dir, "%s.json" % filename)
176 | 
177 |     json.dump(result, open(result_file, "w"))
178 | 
179 |     if is_distributed():
180 |         dist.barrier()
181 | 
182 |     if rank == 0:
183 |         logging.warning("rank %d starts merging results." % get_rank())
184 |         # combine results from all processes
185 |         result = []
186 | 
187 |         for rank in range(get_world_size()):
188 |             result_file = os.path.join(
189 |                 result_dir, "%s_rank%d.json" % (filename, rank)
190 |             )
191 |             res = json.load(open(result_file, "r"))
192 |             result += res
193 | 
194 |         if remove_duplicate:
195 |             result_new = []
196 |             id_list = []
197 |             for res in result:
198 |                 if res[remove_duplicate] not in id_list:
199 |                     id_list.append(res[remove_duplicate])
200 |                     result_new.append(res)
201 |             result = result_new
202 | 
203 |         json.dump(result, open(final_result_file, "w"))
204 |         print("result file saved to %s" % final_result_file)
205 | 
206 |     return final_result_file


--------------------------------------------------------------------------------
/vista_llama/eval/run_inference_qformer_nextqa.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import argparse
  3 | import json
  4 | import torch
  5 | from tqdm import tqdm
  6 | from vista_llama.model import VideoChatGPTQformerLlamaForCausalLM
  7 | from transformers import AutoTokenizer, CLIPVisionModel, CLIPImageProcessor
  8 | from vista_llama.utils import disable_torch_init, save_result
  9 | from vista_llama.eval.model_utils import load_video
 10 | from vista_llama.inference import video_chatgpt_infer
 11 | import torch.distributed as dist
 12 | from vista_llama.constants import *
 13 | from vista_llama.model.dist_utils import init_distributed_mode
 14 | 
 15 | def parse_args():
 16 |     """
 17 |     Parse command-line arguments.
 18 |     """
 19 |     parser = argparse.ArgumentParser()
 20 | 
 21 |     # Define the command-line arguments
 22 |     parser.add_argument('--video_dir', default='/opt/tiger/NExTVideo')
 23 |     parser.add_argument('--gt_file', default='/opt/tiger/SeViLA/sevila_data/nextqa/val.json')
 24 |     parser.add_argument('--output_dir', default='output')
 25 |     parser.add_argument('--output_name', default='video_chatgpt_qformer_nextqa_preds')
 26 |     parser.add_argument("--model-name", type=str, default='LLaVA-7B-Lightening-v1-1')
 27 |     parser.add_argument("--conv-mode", type=str, required=False, default='video-chatgpt_v1')
 28 |     parser.add_argument("--projection_path", type=str, default='/mnt/bn/vlp-v6/Video-ChatGPT/qformer/mm_projector/checkpoint-9000.bin')
 29 | 
 30 |     return parser.parse_args()
 31 | 
 32 | def initialize_model(model_name, projection_path=None, device=0):
 33 |     """
 34 |     Initializes the model with given parameters.
 35 | 
 36 |     Parameters:
 37 |     model_name (str): Name of the model to initialize.
 38 |     projection_path (str, optional): Path to the projection weights. Defaults to None.
 39 | 
 40 |     Returns:
 41 |     tuple: Model, vision tower, tokenizer, image processor, vision config, and video token length.
 42 |     """
 43 | 
 44 |     # Disable initial torch operations
 45 |     disable_torch_init()
 46 | 
 47 |     # Convert model name to user path
 48 |     model_name = os.path.expanduser(model_name)
 49 | 
 50 |     # Load tokenizer
 51 |     tokenizer = AutoTokenizer.from_pretrained(model_name)
 52 | 
 53 |     # Load model
 54 |     model = VideoChatGPTQformerLlamaForCausalLM.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype=torch.float16,
 55 |                                                          use_cache=True)
 56 | 
 57 |     # Load image processor
 58 |     image_processor = CLIPImageProcessor.from_pretrained(model.config.mm_vision_tower, torch_dtype=torch.float16)
 59 | 
 60 |     # Set to use start and end tokens for video
 61 |     mm_use_vid_start_end = True
 62 | 
 63 |     # Add tokens to tokenizer
 64 |     tokenizer.add_tokens([DEFAULT_VIDEO_PATCH_TOKEN], special_tokens=True)
 65 |     if mm_use_vid_start_end:
 66 |         tokenizer.add_tokens([DEFAULT_VID_START_TOKEN, DEFAULT_VID_END_TOKEN], special_tokens=True)
 67 | 
 68 |     # Resize token embeddings of the model
 69 |     model.resize_token_embeddings(len(tokenizer))
 70 | 
 71 |     # Load the weights from projection_path after resizing the token_embeddings
 72 |     if projection_path:
 73 |         print(f"Loading weights from {projection_path}")
 74 |         status = model.load_state_dict(torch.load(projection_path, map_location='cpu'), strict=False)
 75 |         if status.unexpected_keys:
 76 |             print(f"Unexpected Keys: {status.unexpected_keys}.\nThe Video-ChatGPT weights are not loaded correctly.")
 77 |         print(f"Weights loaded from {projection_path}")
 78 | 
 79 |     # Set model to evaluation mode and move to GPU
 80 |     model = model.eval()
 81 |     model = model.to(device)
 82 | 
 83 |     vision_tower_name = "openai/clip-vit-large-patch14"
 84 | 
 85 |     # Load vision tower and move to GPU
 86 |     vision_tower = CLIPVisionModel.from_pretrained(vision_tower_name, torch_dtype=torch.float16,
 87 |                                                    low_cpu_mem_usage=True).to(device)
 88 |     vision_tower = vision_tower.eval()
 89 | 
 90 |     # Configure vision model
 91 |     vision_config = model.get_model().vision_config
 92 |     vision_config.vid_patch_token = tokenizer.convert_tokens_to_ids([DEFAULT_VIDEO_PATCH_TOKEN])[0]
 93 |     vision_config.use_vid_start_end = mm_use_vid_start_end
 94 |     if mm_use_vid_start_end:
 95 |         vision_config.vid_start_token, vision_config.vid_end_token = tokenizer.convert_tokens_to_ids(
 96 |             [DEFAULT_VID_START_TOKEN, DEFAULT_VID_END_TOKEN])
 97 | 
 98 |     # Set video token length
 99 |     video_token_len = 32
100 | 
101 |     return model, vision_tower, tokenizer, image_processor, video_token_len
102 | 
103 | def run_inference(args):
104 |     """
105 |     Run inference on ActivityNet QA DataSet using the Video-ChatGPT model.
106 | 
107 |     Args:
108 |         args: Command-line arguments.
109 |     """
110 |     init_distributed_mode(args)
111 |     # rank = int(os.environ.get('RANK',0))
112 |     # num_gpus = torch.cuda.device_count()
113 |     # dist.init_process_group(backend='nccl')
114 | 
115 |     # Initialize the model
116 |     model, vision_tower, tokenizer, image_processor, video_token_len = initialize_model(args.model_name, args.projection_path, args.rank)
117 |     # Load both ground truth file containing questions and answers
118 |     with open(args.gt_file) as file:
119 |         gt_questions = json.load(file)
120 | 
121 |     # Create the output directory if it doesn't exist
122 |     if not os.path.exists(args.output_dir):
123 |         os.makedirs(args.output_dir)
124 | 
125 |     output_list = []  # List to store the output results
126 |     conv_mode = args.conv_mode
127 | 
128 |     video_formats = ['.mp4', '.avi', '.mov', '.mkv']
129 | 
130 |     # Iterate over each sample in the ground truth file
131 |     index = 0
132 |     gt_questions = sorted(gt_questions)
133 |     gt_questions = gt_questions[args.rank::args.world_size]
134 |     for sample in tqdm(gt_questions):
135 |         video_name = sample['video']
136 |         question = sample['question']
137 |         id = sample['qid']
138 |         answer = sample['a' + str(sample['answer'])]
139 | 
140 |         sample_set = {'id': id, 'question': question, 'answer': answer}
141 | 
142 |         # Load the video file
143 |         for fmt in video_formats:  # Added this line
144 |             temp_path = os.path.join(args.video_dir, f"{video_name}{fmt}")
145 |             if os.path.exists(temp_path):
146 |                 video_path = temp_path
147 |                 break
148 | 
149 |         # Check if the video exists
150 |         if os.path.exists(video_path):
151 |             video_frames = load_video(video_path)
152 | 
153 |         try:
154 |             # Run inference on the video and add the output to the list
155 |             output = video_chatgpt_infer(video_frames, question, conv_mode, model, vision_tower,
156 |                                              tokenizer, image_processor, video_token_len)
157 |             sample_set['pred'] = output
158 |             output_list.append(sample_set)
159 |         except Exception as e:
160 |             print(f"Error processing video file '{video_name}': {e}")
161 | 
162 |     # Save the output list to a JSON file
163 |     save_result(output_list, args.output_dir, args.output_name)
164 | 
165 |     # with open(os.path.join(args.output_dir, f"{args.output_name}.json"), 'w') as file:
166 |     #     json.dump(output_list, file)
167 | 
168 | 
169 | if __name__ == "__main__":
170 |     args = parse_args()
171 |     run_inference(args)
172 | 


--------------------------------------------------------------------------------
/vista_llama/video_conversation.py:
--------------------------------------------------------------------------------
  1 | import dataclasses
  2 | from enum import auto, Enum
  3 | from typing import List
  4 | from vista_llama.eval.model_utils import load_video
  5 | 
  6 | 
  7 | class SeparatorStyle(Enum):
  8 |     """Different separator style."""
  9 | 
 10 |     SINGLE = auto()
 11 |     TWO = auto()
 12 |     MPT = auto()
 13 | 
 14 | 
 15 | @dataclasses.dataclass
 16 | class Conversation:
 17 |     """A class that keeps all conversation history."""
 18 | 
 19 |     system: str
 20 |     roles: List[str]
 21 |     messages: List[List[str]]
 22 |     offset: int
 23 |     sep_style: SeparatorStyle = SeparatorStyle.SINGLE
 24 |     sep: str = "###"
 25 |     sep2: str = None
 26 |     version: str = "Unknown"
 27 | 
 28 |     skip_next: bool = False
 29 | 
 30 |     def get_prompt(self):
 31 |         if self.sep_style == SeparatorStyle.SINGLE:
 32 |             ret = self.system + self.sep
 33 |             for role, message in self.messages:
 34 |                 if message:
 35 |                     if type(message) is tuple:
 36 |                         message, _ = message
 37 |                     ret += role + ": " + message + self.sep
 38 |                 else:
 39 |                     ret += role + ":"
 40 |             return ret
 41 |         elif self.sep_style == SeparatorStyle.TWO:
 42 |             seps = [self.sep, self.sep2]
 43 |             ret = self.system + seps[0]
 44 |             for i, (role, message) in enumerate(self.messages):
 45 |                 if message:
 46 |                     if type(message) is tuple:
 47 |                         message, _ = message
 48 |                     ret += role + ": " + message + seps[i % 2]
 49 |                 else:
 50 |                     ret += role + ":"
 51 |             return ret
 52 |         if self.sep_style == SeparatorStyle.MPT:
 53 |             ret = self.system + self.sep
 54 |             for role, message in self.messages:
 55 |                 if message:
 56 |                     if type(message) is tuple:
 57 |                         message, _ = message
 58 |                     ret += role + message + self.sep
 59 |                 else:
 60 |                     ret += role
 61 |             return ret
 62 |         else:
 63 |             raise ValueError(f"Invalid style: {self.sep_style}")
 64 | 
 65 |     def append_message(self, role, message):
 66 |         self.messages.append([role, message])
 67 | 
 68 |     def get_video_frames(self, n_clips=1, num_frm=100):
 69 |         video_frames = []
 70 |         for i, (role, msg) in enumerate(self.messages[self.offset :]):
 71 |             if i % 2 == 0:
 72 |                 if type(msg) is tuple:
 73 |                     msg, video_path = msg
 74 | 
 75 |                     clip_imgs = load_video(video_path, n_clips, num_frm)
 76 | 
 77 |                     for image in clip_imgs:
 78 |                         video_frames.append(image)
 79 |         return video_frames
 80 | 
 81 |     def to_gradio_chatbot(self):
 82 |         ret = []
 83 |         for i, (role, msg) in enumerate(self.messages[self.offset :]):
 84 |             if i % 2 == 0:
 85 |                 if type(msg) is tuple:
 86 |                     msg, image = msg
 87 |                 ret.append([msg, None])
 88 |             else:
 89 |                 ret[-1][-1] = msg
 90 |         # Hack to make the demo work
 91 |         try:
 92 |             if "<video>" in ret[0][0]:
 93 |                 ret[0][0] = ret[0][0].replace("<video>", "")
 94 |         except Exception as e:
 95 |             pass
 96 | 
 97 |         return ret
 98 | 
 99 |     def copy(self):
100 |         return Conversation(
101 |             system=self.system,
102 |             roles=self.roles,
103 |             messages=[[x, y] for x, y in self.messages],
104 |             offset=self.offset,
105 |             sep_style=self.sep_style,
106 |             sep=self.sep,
107 |             sep2=self.sep2,
108 |         )
109 | 
110 |     def dict(self):
111 |         return {
112 |             "system": self.system,
113 |             "roles": self.roles,
114 |             "messages": self.messages,
115 |             "offset": self.offset,
116 |             "sep": self.sep,
117 |             "sep2": self.sep2,
118 |         }
119 | 
120 | 
121 | conv_v1_2 = Conversation(
122 |     system="A chat between a curious human and an artificial intelligence assistant. "
123 |     "The assistant gives helpful, detailed, and polite answers to the human's questions.",
124 |     roles=("Human", "Assistant"),
125 |     messages=(
126 |         (
127 |             "Human",
128 |             "What are the key differences between renewable and non-renewable energy sources?",
129 |         ),
130 |         (
131 |             "Assistant",
132 |             "Renewable energy sources are those that can be replenished naturally.\n",
133 |         ),
134 |     ),
135 |     offset=2,
136 |     sep_style=SeparatorStyle.SINGLE,
137 |     sep="###",
138 | )
139 | 
140 | 
141 | conv_vicuna_v1_1 = Conversation(
142 |     system="A chat between a curious user and an artificial intelligence assistant. "
143 |     "The assistant gives helpful, detailed, and polite answers to the user's questions.",
144 |     roles=("USER", "ASSISTANT"),
145 |     version="v1",
146 |     messages=(),
147 |     offset=0,
148 |     sep_style=SeparatorStyle.TWO,
149 |     sep=" ",
150 |     sep2="</s>",
151 | )
152 | 
153 | conv_vicuna_v1_2 = Conversation(
154 |     system="A chat between a curious user and an artificial intelligence assistant. "
155 |     "The assistant gives helpful, detailed, and polite answers to the user's questions. This is a video containing 356 patch tokens.<video>",
156 |     roles=("USER", "ASSISTANT"),
157 |     version="v1",
158 |     messages=(),
159 |     offset=0,
160 |     sep_style=SeparatorStyle.TWO,
161 |     sep=" ",
162 |     sep2="</s>",
163 | )
164 | 
165 | conv_vicuna_v1_3 = Conversation(
166 |     system="A chat between a curious user and an artificial intelligence assistant. "
167 |     "The assistant gives helpful, detailed, and polite answers to the user's questions. Each frame is represented by a frame token. This is a video containing 100 frame tokens. <video>",
168 |     roles=("USER", "ASSISTANT"),
169 |     version="v1",
170 |     messages=(),
171 |     offset=0,
172 |     sep_style=SeparatorStyle.TWO,
173 |     sep=" ",
174 |     sep2="</s>",
175 | )
176 | 
177 | conv_vicuna_v1_4 = Conversation(
178 |     system="A chat between a curious user and an artificial intelligence assistant. "
179 |     "The assistant gives helpful, detailed, and polite answers to the user's questions. <video>",
180 |     roles=("USER", "ASSISTANT"),
181 |     version="v1",
182 |     messages=(),
183 |     offset=0,
184 |     sep_style=SeparatorStyle.TWO,
185 |     sep=" ",
186 |     sep2="</s>",
187 | )
188 | 
189 | conv_video_chatgpt_v1 = Conversation(
190 |     system="You are Video-ChatGPT, a large vision-language assistant. "
191 |     "You are able to understand the video content that the user provides, and assist the user with a variety of tasks using natural language."
192 |     "Follow the instructions carefully and explain your answers in detail based on the provided video.",
193 |     # system="",
194 |     roles=("USER", "ASSISTANT"),
195 |     version="v1",
196 |     messages=(),
197 |     offset=0,
198 |     sep_style=SeparatorStyle.TWO,
199 |     sep=" ",
200 |     sep2="</s>",
201 | )
202 | 
203 | default_conversation = conv_v1_2
204 | conv_templates = {
205 |     "default": conv_v1_2,
206 |     "video-chatgpt_v1": conv_video_chatgpt_v1,
207 |     "vicuna_v1_1": conv_vicuna_v1_1,
208 |     "vicuna_v1_2": conv_vicuna_v1_2,
209 |     "vicuna_v1_3": conv_vicuna_v1_3,
210 |     "vicuna_v1_4": conv_vicuna_v1_4,
211 | }
212 | 
213 | if __name__ == "__main__":
214 |     print(default_conversation.get_prompt())
215 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/evaluate_benchmark_1_correctness.py:
--------------------------------------------------------------------------------
  1 | import openai
  2 | import os
  3 | import argparse
  4 | import json
  5 | import ast
  6 | from multiprocessing.pool import Pool
  7 | 
  8 | 
  9 | openai.api_type = "azure"
 10 | openai.api_version = "2023-10-01-preview"
 11 | 
 12 | 
 13 | def parse_args():
 14 |     parser = argparse.ArgumentParser(
 15 |         description="question-answer-generation-using-gpt-3"
 16 |     )
 17 |     parser.add_argument(
 18 |         "--pred_path", required=True, help="The path to file containing prediction."
 19 |     )
 20 |     parser.add_argument(
 21 |         "--output_dir", required=True, help="The path to save annotation json files."
 22 |     )
 23 |     parser.add_argument(
 24 |         "--output_json",
 25 |         required=True,
 26 |         help="The path to save annotation final combined json file.",
 27 |     )
 28 |     parser.add_argument("--api_key", required=True, help="OpenAI API key.")
 29 |     parser.add_argument(
 30 |         "--num_tasks", required=True, type=int, help="Number of splits."
 31 |     )
 32 |     args = parser.parse_args()
 33 |     return args
 34 | 
 35 | 
 36 | def annotate(prediction_set, caption_files, output_dir):
 37 |     """
 38 |     Evaluates question and answer pairs using GPT-3
 39 |     Returns a score for correctness.
 40 |     """
 41 |     for file in caption_files:
 42 |         key = file[:-5]  # Strip file extension
 43 |         qa_set = prediction_set[key]
 44 |         question = qa_set["q"]
 45 |         answer = qa_set["a"]
 46 |         pred = qa_set["pred"]
 47 |         try:
 48 |             # Compute the correctness score
 49 |             completion = openai.ChatCompletion.create(
 50 |                 engine="gpt-openai",
 51 |                 model="gpt-3.5-turbo-0613",
 52 |                 messages=[
 53 |                     {
 54 |                         "role": "system",
 55 |                         "content": "You are an intelligent chatbot designed for evaluating the factual accuracy of generative outputs for video-based question-answer pairs. "
 56 |                         "Your task is to compare the predicted answer with the correct answer and determine if they are factually consistent. Here's how you can accomplish the task:"
 57 |                         "------"
 58 |                         "##INSTRUCTIONS: "
 59 |                         "- Focus on the factual consistency between the predicted answer and the correct answer. The predicted answer should not contain any misinterpretations or misinformation.\n"
 60 |                         "- The predicted answer must be factually accurate and align with the video content.\n"
 61 |                         "- Consider synonyms or paraphrases as valid matches.\n"
 62 |                         "- Evaluate the factual accuracy of the prediction compared to the answer.",
 63 |                     },
 64 |                     {
 65 |                         "role": "user",
 66 |                         "content": "Please evaluate the following video-based question-answer pair:\n\n"
 67 |                         f"Question: {question}\n"
 68 |                         f"Correct Answer: {answer}\n"
 69 |                         f"Predicted Answer: {pred}\n\n"
 70 |                         "Provide your evaluation only as a factual accuracy score where the factual accuracy score is an integer value between 0 and 5, with 5 indicating the highest level of factual consistency. "
 71 |                         "Please generate the response in the form of a Python dictionary string with keys 'score', where its value is the factual accuracy score in INTEGER, not STRING."
 72 |                         "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
 73 |                         "For example, your response should look like this: {''score': 4.8}.",
 74 |                     },
 75 |                 ],
 76 |             )
 77 |             # Convert response to a Python dictionary.
 78 |             response_message = completion["choices"][0]["message"]["content"]
 79 |             response_dict = ast.literal_eval(response_message)
 80 |             result_qa_pair = [response_dict, qa_set]
 81 | 
 82 |             # Save the question-answer pairs to a json file.
 83 |             with open(f"{output_dir}/{key}.json", "w") as f:
 84 |                 json.dump(result_qa_pair, f)
 85 | 
 86 |         except Exception as e:
 87 |             print(f"Error processing file '{key}': {e}")
 88 | 
 89 | 
 90 | def main():
 91 |     """
 92 |     Main function to control the flow of the program.
 93 |     """
 94 |     # Parse arguments.
 95 |     args = parse_args()
 96 | 
 97 |     file = open(args.pred_path)
 98 |     pred_contents = json.load(file)
 99 | 
100 |     # Dictionary to store the count of occurrences for each video_id
101 |     video_id_counts = {}
102 |     new_pred_contents = []
103 | 
104 |     # Iterate through each sample in pred_contents
105 |     for sample in pred_contents:
106 |         video_id = sample["video_name"]
107 |         if video_id in video_id_counts:
108 |             video_id_counts[video_id] += 1
109 |         else:
110 |             video_id_counts[video_id] = 0
111 | 
112 |         # Create a new sample with the modified key
113 |         new_sample = sample
114 |         new_sample["video_name"] = f"{video_id}_{video_id_counts[video_id]}"
115 |         new_pred_contents.append(new_sample)
116 | 
117 |     # Generating list of id's and corresponding files
118 |     id_list = [x["video_name"] for x in new_pred_contents]
119 |     caption_files = [f"{id}.json" for id in id_list]
120 | 
121 |     output_dir = args.output_dir
122 |     # Generate output directory if not exists.
123 |     if not os.path.exists(output_dir):
124 |         os.makedirs(output_dir)
125 | 
126 |     # Preparing dictionary of question-answer sets
127 |     prediction_set = {}
128 |     for sample in new_pred_contents:
129 |         id = sample["video_name"]
130 |         question = sample["Q"]
131 |         answer = sample["A"]
132 |         pred = sample["pred"]
133 |         qa_set = {"q": question, "a": answer, "pred": pred}
134 |         prediction_set[id] = qa_set
135 | 
136 |     # Set the OpenAI API key.
137 |     openai.api_key = args.api_key
138 |     num_tasks = args.num_tasks
139 | 
140 |     # While loop to ensure that all captions are processed.
141 |     while True:
142 |         try:
143 |             # Files that have not been processed yet.
144 |             completed_files = os.listdir(output_dir)
145 |             print(f"completed_files: {len(completed_files)}")
146 | 
147 |             # Files that have not been processed yet.
148 |             incomplete_files = [f for f in caption_files if f not in completed_files]
149 |             print(f"incomplete_files: {len(incomplete_files)}")
150 | 
151 |             # Break the loop when there are no incomplete files
152 |             if len(incomplete_files) == 0:
153 |                 break
154 |             if len(incomplete_files) <= num_tasks:
155 |                 num_tasks = 1
156 | 
157 |             # Split tasks into parts.
158 |             part_len = len(incomplete_files) // num_tasks
159 |             all_parts = [
160 |                 incomplete_files[i : i + part_len]
161 |                 for i in range(0, len(incomplete_files), part_len)
162 |             ]
163 |             task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
164 | 
165 |             # Use a pool of workers to process the files in parallel.
166 |             with Pool() as pool:
167 |                 pool.starmap(annotate, task_args)
168 | 
169 |         except Exception as e:
170 |             print(f"Error: {e}")
171 | 
172 |     # Combine all the processed files into one
173 |     combined_contents = {}
174 |     json_path = args.output_json
175 | 
176 |     # Iterate through json files
177 |     for file_name in os.listdir(output_dir):
178 |         if file_name.endswith(".json"):
179 |             file_path = os.path.join(output_dir, file_name)
180 |             with open(file_path, "r") as json_file:
181 |                 content = json.load(json_file)
182 |                 combined_contents[file_name[:-5]] = content
183 | 
184 |     # Write combined content to a json file
185 |     with open(json_path, "w") as json_file:
186 |         json.dump(combined_contents, json_file)
187 |     print("All evaluation completed!")
188 | 
189 |     # Calculate average score
190 |     score_sum = 0
191 |     count = 0
192 |     for key, result in combined_contents.items():
193 |         count += 1
194 |         score_match = result[0]["score"]
195 |         score = int(score_match)
196 |         score_sum += score
197 |     average_score = score_sum / count
198 | 
199 |     print("Average score for correctness:", average_score)
200 | 
201 | 
202 | if __name__ == "__main__":
203 |     main()
204 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/evaluate_activitynet_qa.py:
--------------------------------------------------------------------------------
  1 | import openai
  2 | import os
  3 | import argparse
  4 | import json
  5 | import ast
  6 | from multiprocessing.pool import Pool
  7 | 
  8 | 
  9 | def parse_args():
 10 |     parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
 11 |     parser.add_argument("--pred_path", required=True, help="The path to file containing prediction.")
 12 |     parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
 13 |     parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
 14 |     parser.add_argument("--api_key", required=True, help="OpenAI API key.")
 15 |     parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
 16 |     args = parser.parse_args()
 17 |     return args
 18 | 
 19 | 
 20 | def annotate(prediction_set, caption_files, output_dir):
 21 |     """
 22 |     Evaluates question and answer pairs using GPT-3
 23 |     Returns a score for correctness.
 24 |     """
 25 |     for file in caption_files:
 26 |         key = file[:-5] # Strip file extension
 27 |         qa_set = prediction_set[key]
 28 |         question = qa_set['q']
 29 |         answer = qa_set['a']
 30 |         pred = qa_set['pred']
 31 |         try:
 32 |             # Compute the correctness score
 33 |             completion = openai.ChatCompletion.create(
 34 |                 model="gpt-3.5-turbo",
 35 |                 messages=[
 36 |                     {
 37 |                         "role": "system",
 38 |                         "content": 
 39 |                             "You are an intelligent chatbot designed for evaluating the correctness of generative outputs for question-answer pairs. "
 40 |                             "Your task is to compare the predicted answer with the correct answer and determine if they match meaningfully. Here's how you can accomplish the task:"
 41 |                             "------"
 42 |                             "##INSTRUCTIONS: "
 43 |                             "- Focus on the meaningful match between the predicted answer and the correct answer.\n"
 44 |                             "- Consider synonyms or paraphrases as valid matches.\n"
 45 |                             "- Evaluate the correctness of the prediction compared to the answer."
 46 |                     },
 47 |                     {
 48 |                         "role": "user",
 49 |                         "content":
 50 |                             "Please evaluate the following video-based question-answer pair:\n\n"
 51 |                             f"Question: {question}\n"
 52 |                             f"Correct Answer: {answer}\n"
 53 |                             f"Predicted Answer: {pred}\n\n"
 54 |                             "Provide your evaluation only as a yes/no and score where the score is an integer value between 0 and 5, with 5 indicating the highest meaningful match. "
 55 |                             "Please generate the response in the form of a Python dictionary string with keys 'pred' and 'score', where value of 'pred' is  a string of 'yes' or 'no' and value of 'score' is in INTEGER, not STRING."
 56 |                             "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
 57 |                             "For example, your response should look like this: {'pred': 'yes', 'score': 4.8}."
 58 |                     }
 59 |                 ]
 60 |             )
 61 |             # Convert response to a Python dictionary.
 62 |             response_message = completion["choices"][0]["message"]["content"]
 63 |             response_dict = ast.literal_eval(response_message)
 64 |             result_qa_pair = [response_dict, qa_set]
 65 | 
 66 |             # Save the question-answer pairs to a json file.
 67 |             with open(f"{output_dir}/{key}.json", "w") as f:
 68 |                 json.dump(result_qa_pair, f)
 69 | 
 70 |         except Exception as e:
 71 |             print(f"Error processing file '{key}': {e}")
 72 | 
 73 | 
 74 | def main():
 75 |     """
 76 |     Main function to control the flow of the program.
 77 |     """
 78 |     # Parse arguments.
 79 |     args = parse_args()
 80 | 
 81 |     file = open(args.pred_path)
 82 |     pred_contents = json.load(file)
 83 | 
 84 |     # Dictionary to store the count of occurrences for each video_id
 85 |     video_id_counts = {}
 86 |     new_pred_contents = []
 87 | 
 88 |     # Iterate through each sample in pred_contents
 89 |     for sample in pred_contents:
 90 |         video_id = sample['video_name']
 91 |         if video_id in video_id_counts:
 92 |             video_id_counts[video_id] += 1
 93 |         else:
 94 |             video_id_counts[video_id] = 0
 95 | 
 96 |         # Create a new sample with the modified key
 97 |         new_sample = sample
 98 |         new_sample['video_name'] = f"{video_id}_{video_id_counts[video_id]}"
 99 |         new_pred_contents.append(new_sample)
100 | 
101 |     # Generating list of id's and corresponding files
102 |     id_list = [x['video_name'] for x in new_pred_contents]
103 |     caption_files = [f"{id}.json" for id in id_list]
104 | 
105 |     output_dir = args.output_dir
106 |     # Generate output directory if not exists.
107 |     if not os.path.exists(output_dir):
108 |         os.makedirs(output_dir)
109 | 
110 |     # Preparing dictionary of question-answer sets
111 |     prediction_set = {}
112 |     for sample in new_pred_contents:
113 |         id = sample['video_name']
114 |         question = sample['Q']
115 |         answer = sample['A']
116 |         pred = sample['pred']
117 |         qa_set = {"q": question, "a": answer, "pred": pred}
118 |         prediction_set[id] = qa_set
119 | 
120 |     # Set the OpenAI API key.
121 |     openai.api_key = args.api_key
122 |     num_tasks = args.num_tasks
123 | 
124 |     # While loop to ensure that all captions are processed.
125 |     while True:
126 |         try:
127 |             # Files that have not been processed yet.
128 |             completed_files = os.listdir(output_dir)
129 |             print(f"completed_files: {len(completed_files)}")
130 | 
131 |             # Files that have not been processed yet.
132 |             incomplete_files = [f for f in caption_files if f not in completed_files]
133 |             print(f"incomplete_files: {len(incomplete_files)}")
134 | 
135 |             # Break the loop when there are no incomplete files
136 |             if len(incomplete_files) == 0:
137 |                 break
138 |             if len(incomplete_files) <= num_tasks:
139 |                 num_tasks = 1
140 | 
141 |             # Split tasks into parts.
142 |             part_len = len(incomplete_files) // num_tasks
143 |             all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
144 |             task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
145 | 
146 |             # Use a pool of workers to process the files in parallel.
147 |             with Pool() as pool:
148 |                 pool.starmap(annotate, task_args)
149 | 
150 |         except Exception as e:
151 |             print(f"Error: {e}")
152 | 
153 |     # Combine all the processed files into one
154 |     combined_contents = {}
155 |     json_path = args.output_json
156 | 
157 |     # Iterate through json files
158 |     for file_name in os.listdir(output_dir):
159 |         if file_name.endswith(".json"):
160 |             file_path = os.path.join(output_dir, file_name)
161 |             with open(file_path, "r") as json_file:
162 |                 content = json.load(json_file)
163 |                 combined_contents[file_name[:-5]] = content
164 | 
165 |     # Write combined content to a json file
166 |     with open(json_path, "w") as json_file:
167 |         json.dump(combined_contents, json_file)
168 |     print("All evaluation completed!")
169 | 
170 |     # Calculate average score and accuracy
171 |     score_sum = 0
172 |     count = 0
173 |     yes_count = 0
174 |     no_count = 0
175 |     for key, result in combined_contents.items():
176 |         # Computing score
177 |         count += 1
178 |         score_match = result[0]['score']
179 |         score = int(score_match)
180 |         score_sum += score
181 | 
182 |         # Computing accuracy
183 |         pred = result[0]['pred']
184 |         if "yes" in pred.lower():
185 |             yes_count += 1
186 |         elif "no" in pred.lower():
187 |             no_count += 1
188 | 
189 |     average_score = score_sum / count
190 |     accuracy = yes_count / (yes_count + no_count)
191 |     print("Yes count:", yes_count)
192 |     print("No count:", no_count)
193 |     print("Accuracy:", accuracy)
194 |     print("Average score:", average_score)
195 | 
196 | 
197 | if __name__ == "__main__":
198 |     main()
199 | 
200 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/evaluate_benchmark_4_temporal.py:
--------------------------------------------------------------------------------
  1 | import openai
  2 | import os
  3 | import argparse
  4 | import json
  5 | import ast
  6 | from multiprocessing.pool import Pool
  7 | 
  8 | 
  9 | openai.api_type = "azure"
 10 | openai.api_base = "https://search.bytedance.net/gpt/openapi/online/v2/crawl"
 11 | openai.api_version = "2023-10-01-preview"
 12 | 
 13 | def parse_args():
 14 |     parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
 15 |     parser.add_argument("--pred_path", required=True, help="The path to file containing prediction.")
 16 |     parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
 17 |     parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
 18 |     parser.add_argument("--api_key", required=True, help="OpenAI API key.")
 19 |     parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
 20 |     args = parser.parse_args()
 21 |     return args
 22 | 
 23 | 
 24 | def annotate(prediction_set, caption_files, output_dir):
 25 |     """
 26 |     Evaluates question and answer pairs using GPT-3 and
 27 |     returns a score for temporal understanding.
 28 |     """
 29 |     for file in caption_files:
 30 |         key = file[:-5] # Strip file extension
 31 |         qa_set = prediction_set[key]
 32 |         question = qa_set['q']
 33 |         answer = qa_set['a']
 34 |         pred = qa_set['pred']
 35 |         try:
 36 |             # Compute the temporal understanding score
 37 |             completion = openai.ChatCompletion.create(
 38 |                 # model="gpt-3.5-turbo",
 39 |                 engine="gpt-openai",
 40 |                 model="gpt-3.5-turbo-0613",
 41 |                 messages=[
 42 |                     {
 43 |                         "role": "system",
 44 |                         "content":
 45 |                             "You are an intelligent chatbot designed for evaluating the temporal understanding of generative outputs for video-based question-answer pairs. "
 46 |                             "Your task is to compare the predicted answer with the correct answer and determine if they correctly reflect the temporal sequence of events in the video content. Here's how you can accomplish the task:"
 47 |                             "------"
 48 |                             "##INSTRUCTIONS: "
 49 |                             "- Focus on the temporal consistency between the predicted answer and the correct answer. The predicted answer should correctly reflect the sequence of events or details as they are presented in the video content.\n"
 50 |                             "- Consider synonyms or paraphrases as valid matches, but only if the temporal order is maintained.\n"
 51 |                             "- Evaluate the temporal accuracy of the prediction compared to the answer."
 52 |                     },
 53 |                     {
 54 |                         "role": "user",
 55 |                         "content":
 56 |                             "Please evaluate the following video-based question-answer pair:\n\n"
 57 |                             f"Question: {question}\n"
 58 |                             f"Correct Answer: {answer}\n"
 59 |                             f"Predicted Answer: {pred}\n\n"
 60 |                             "Provide your evaluation only as a temporal accuracy score where the temporal accuracy score is an integer value between 0 and 5, with 5 indicating the highest level of temporal consistency. "
 61 |                             "Please generate the response in the form of a Python dictionary string with keys 'score', where its value is the temporal accuracy score in INTEGER, not STRING."
 62 |                             "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
 63 |                             "For example, your response should look like this: {''score': 4.8}."
 64 |                     }
 65 |                 ]
 66 |             )
 67 |             # Convert response to a Python dictionary.
 68 |             response_message = completion["choices"][0]["message"]["content"]
 69 |             response_dict = ast.literal_eval(response_message)
 70 |             result_qa_pair = [response_dict, qa_set]
 71 | 
 72 |             # Save the question-answer pairs to a json file.
 73 |             with open(f"{output_dir}/{key}.json", "w") as f:
 74 |                 json.dump(result_qa_pair, f)
 75 | 
 76 |         except Exception as e:
 77 |             print(f"Error processing file '{key}': {e}")
 78 | 
 79 | 
 80 | def main():
 81 |     """
 82 |     Main function to control the flow of the program.
 83 |     """
 84 |     # Parse arguments.
 85 |     args = parse_args()
 86 | 
 87 |     file = open(args.pred_path)
 88 |     pred_contents = json.load(file)
 89 | 
 90 |     # Dictionary to store the count of occurrences for each video_id
 91 |     video_id_counts = {}
 92 |     new_pred_contents = []
 93 | 
 94 |     # Iterate through each sample in pred_contents
 95 |     for sample in pred_contents:
 96 |         video_id = sample['video_name']
 97 |         if video_id in video_id_counts:
 98 |             video_id_counts[video_id] += 1
 99 |         else:
100 |             video_id_counts[video_id] = 0
101 | 
102 |         # Create a new sample with the modified key
103 |         new_sample = sample
104 |         new_sample['video_name'] = f"{video_id}_{video_id_counts[video_id]}"
105 |         new_pred_contents.append(new_sample)
106 | 
107 |     # Generating list of id's and corresponding files
108 |     id_list = [x['video_name'] for x in new_pred_contents]
109 |     caption_files = [f"{id}.json" for id in id_list]
110 | 
111 |     output_dir = args.output_dir
112 |     # Generate output directory if not exists.
113 |     if not os.path.exists(output_dir):
114 |         os.makedirs(output_dir)
115 | 
116 |     # Preparing dictionary of question-answer sets
117 |     prediction_set = {}
118 |     for sample in new_pred_contents:
119 |         id = sample['video_name']
120 |         question = sample['Q']
121 |         answer = sample['A']
122 |         pred = sample['pred']
123 |         qa_set = {"q": question, "a": answer, "pred": pred}
124 |         prediction_set[id] = qa_set
125 | 
126 |     # Set the OpenAI API key.
127 |     openai.api_key = args.api_key
128 |     num_tasks = args.num_tasks
129 | 
130 |     # While loop to ensure that all captions are processed.
131 |     while True:
132 |         try:
133 |             # Files that have not been processed yet.
134 |             completed_files = os.listdir(output_dir)
135 |             print(f"completed_files: {len(completed_files)}")
136 | 
137 |             # Files that have not been processed yet.
138 |             incomplete_files = [f for f in caption_files if f not in completed_files]
139 |             print(f"incomplete_files: {len(incomplete_files)}")
140 | 
141 |             # Break the loop when there are no incomplete files
142 |             if len(incomplete_files) == 0:
143 |                 break
144 |             if len(incomplete_files) <= num_tasks:
145 |                 num_tasks = 1
146 | 
147 |             # Split tasks into parts.
148 |             part_len = len(incomplete_files) // num_tasks
149 |             all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
150 |             task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
151 | 
152 |             # Use a pool of workers to process the files in parallel.
153 |             with Pool() as pool:
154 |                 pool.starmap(annotate, task_args)
155 | 
156 |         except Exception as e: 
157 |             print(f"Error: {e}")
158 | 
159 |     # Combine all the processed files into one
160 |     combined_contents = {}
161 |     json_path = args.output_json
162 | 
163 |     # Iterate through json files
164 |     for file_name in os.listdir(output_dir):
165 |         if file_name.endswith(".json"):
166 |             file_path = os.path.join(output_dir, file_name)
167 |             with open(file_path, "r") as json_file:
168 |                 content = json.load(json_file)
169 |                 combined_contents[file_name[:-5]] = content
170 | 
171 |     # Write combined content to a json file
172 |     with open(json_path, "w") as json_file:
173 |         json.dump(combined_contents, json_file)
174 |     print("All evaluation completed!")
175 | 
176 |     # Calculate average score
177 |     score_sum = 0
178 |     count = 0
179 |     for key, result in combined_contents.items():
180 |         count += 1
181 |         score_match = result[0]['score']
182 |         score = int(score_match)
183 |         score_sum += score
184 |     average_score = score_sum / count
185 | 
186 |     print("Average score temporal understanding:", average_score)
187 | 
188 | 
189 | if __name__ == "__main__":
190 |     main()
191 | 
192 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/evaluate_nextqa.py:
--------------------------------------------------------------------------------
  1 | import openai
  2 | import os
  3 | import argparse
  4 | import json
  5 | import ast
  6 | from multiprocessing.pool import Pool
  7 | 
  8 | 
  9 | def parse_args():
 10 |     parser = argparse.ArgumentParser(
 11 |         description="question-answer-generation-using-gpt-3"
 12 |     )
 13 |     parser.add_argument(
 14 |         "--pred_path", required=True, help="The path to file containing prediction."
 15 |     )
 16 |     parser.add_argument(
 17 |         "--output_dir", required=True, help="The path to save annotation json files."
 18 |     )
 19 |     parser.add_argument(
 20 |         "--output_json",
 21 |         required=True,
 22 |         help="The path to save annotation final combined json file.",
 23 |     )
 24 |     parser.add_argument("--api_key", required=True, help="OpenAI API key.")
 25 |     parser.add_argument("--num_tasks", default=1, type=int, help="Number of splits.")
 26 |     args = parser.parse_args()
 27 |     return args
 28 | 
 29 | 
 30 | def annotate(prediction_set, caption_files, output_dir):
 31 |     """
 32 |     Evaluates question and answer pairs using GPT-3
 33 |     Returns a score for correctness.
 34 |     """
 35 |     for file in caption_files:
 36 |         key = file[:-5]  # Strip file extension
 37 |         qa_set = prediction_set[key]
 38 |         question = qa_set["q"]
 39 |         answer = qa_set["a"]
 40 |         pred = qa_set["pred"]
 41 |         try:
 42 |             # Compute the correctness score
 43 |             completion = openai.ChatCompletion.create(
 44 |                 # engine="gpt-openai",
 45 |                 model="gpt-3.5-turbo-0613",
 46 |                 messages=[
 47 |                     {
 48 |                         "role": "system",
 49 |                         "content": "You are an intelligent chatbot designed for evaluating the correctness of generative outputs for question-answer pairs. "
 50 |                         "Your task is to compare the predicted answer with the correct answer and determine if they match meaningfully. Here's how you can accomplish the task:"
 51 |                         "------"
 52 |                         "##INSTRUCTIONS: "
 53 |                         "- Focus on the meaningful match between the predicted answer and the correct answer.\n"
 54 |                         "- Consider synonyms or paraphrases as valid matches.\n"
 55 |                         "- Evaluate the correctness of the prediction compared to the answer.",
 56 |                     },
 57 |                     {
 58 |                         "role": "user",
 59 |                         "content": "Please evaluate the following video-based question-answer pair:\n\n"
 60 |                         f"Question: {question}\n"
 61 |                         f"Correct Answer: {answer}\n"
 62 |                         f"Predicted Answer: {pred}\n\n"
 63 |                         "Provide your evaluation only as a yes/no and score where the score is an integer value between 0 and 5, with 5 indicating the highest meaningful match. "
 64 |                         "Please generate the response in the form of a Python dictionary string with keys 'pred' and 'score', where value of 'pred' is  a string of 'yes' or 'no' and value of 'score' is in INTEGER, not STRING."
 65 |                         "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
 66 |                         "For example, your response should look like this: {'pred': 'yes', 'score': 4.8}.",
 67 |                     },
 68 |                 ],
 69 |             )
 70 |             # Convert response to a Python dictionary.
 71 |             response_message = completion["choices"][0]["message"]["content"]
 72 |             response_dict = ast.literal_eval(response_message)
 73 |             result_qa_pair = [response_dict, qa_set]
 74 | 
 75 |             # Save the question-answer pairs to a json file.
 76 |             with open(f"{output_dir}/{key}.json", "w") as f:
 77 |                 json.dump(result_qa_pair, f)
 78 | 
 79 |         except Exception as e:
 80 |             print(f"Error processing file '{key}': {e}")
 81 | 
 82 | 
 83 | def main():
 84 |     """
 85 |     Main function to control the flow of the program.
 86 |     """
 87 |     # Parse arguments.
 88 |     args = parse_args()
 89 | 
 90 |     file = open(args.pred_path)
 91 |     pred_contents = json.load(file)
 92 | 
 93 |     # Dictionary to store the count of occurrences for each video_id
 94 |     video_id_counts = {}
 95 |     new_pred_contents = []
 96 | 
 97 |     # Iterate through each sample in pred_contents
 98 |     for sample in pred_contents:
 99 |         video_id = sample["id"]
100 |         if video_id in video_id_counts:
101 |             video_id_counts[video_id] += 1
102 |         else:
103 |             video_id_counts[video_id] = 0
104 | 
105 |         # Create a new sample with the modified key
106 |         new_sample = sample
107 |         new_sample["video_name"] = f"{video_id}_{video_id_counts[video_id]}"
108 |         new_pred_contents.append(new_sample)
109 | 
110 |     # Generating list of id's and corresponding files
111 |     id_list = [x["video_name"] for x in new_pred_contents]
112 |     caption_files = [f"{id}.json" for id in id_list]
113 | 
114 |     output_dir = args.output_dir
115 |     # Generate output directory if not exists.
116 |     if not os.path.exists(output_dir):
117 |         os.makedirs(output_dir)
118 | 
119 |     # Preparing dictionary of question-answer sets
120 |     prediction_set = {}
121 |     for sample in new_pred_contents:
122 |         id = sample["video_name"]
123 |         question = sample["question"]
124 |         answer = sample["answer"]
125 |         pred = sample["pred"]
126 |         qa_set = {"q": question, "a": answer, "pred": pred}
127 |         prediction_set[id] = qa_set
128 | 
129 |     # Set the OpenAI API key.
130 |     openai.api_key = args.api_key
131 |     num_tasks = args.num_tasks
132 | 
133 |     # While loop to ensure that all captions are processed.
134 |     while True:
135 |         try:
136 |             # Files that have not been processed yet.
137 |             completed_files = os.listdir(output_dir)
138 |             print(f"completed_files: {len(completed_files)}")
139 | 
140 |             # Files that have not been processed yet.
141 |             incomplete_files = [f for f in caption_files if f not in completed_files]
142 |             print(f"incomplete_files: {len(incomplete_files)}")
143 | 
144 |             # Break the loop when there are no incomplete files
145 |             if len(incomplete_files) <= 0:
146 |                 break
147 |             if len(incomplete_files) <= num_tasks:
148 |                 num_tasks = 1
149 | 
150 |             # Split tasks into parts.
151 |             part_len = len(incomplete_files) // num_tasks
152 |             all_parts = [
153 |                 incomplete_files[i : i + part_len]
154 |                 for i in range(0, len(incomplete_files), part_len)
155 |             ]
156 |             task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
157 | 
158 |             # Use a pool of workers to process the files in parallel.
159 |             with Pool() as pool:
160 |                 pool.starmap(annotate, task_args)
161 | 
162 |         except Exception as e:
163 |             print(f"Error: {e}")
164 | 
165 |     # Combine all the processed files into one
166 |     combined_contents = {}
167 |     json_path = args.output_json
168 | 
169 |     # Iterate through json files
170 |     for file_name in os.listdir(output_dir):
171 |         if file_name.endswith(".json"):
172 |             file_path = os.path.join(output_dir, file_name)
173 |             with open(file_path, "r") as json_file:
174 |                 content = json.load(json_file)
175 |                 combined_contents[file_name[:-5]] = content
176 | 
177 |     # Write combined content to a json file
178 |     with open(json_path, "w") as json_file:
179 |         json.dump(combined_contents, json_file)
180 |     print("All evaluation completed!")
181 | 
182 |     # Calculate average score and accuracy
183 |     score_sum = 0
184 |     count = 0
185 |     yes_count = 0
186 |     no_count = 0
187 |     for key, result in combined_contents.items():
188 |         # Computing score
189 |         count += 1
190 |         score_match = result[0]["score"]
191 |         score = int(score_match)
192 |         score_sum += score
193 | 
194 |         # Computing accuracy
195 |         pred = result[0]["pred"]
196 |         if "yes" in pred.lower():
197 |             yes_count += 1
198 |         elif "no" in pred.lower():
199 |             no_count += 1
200 | 
201 |     average_score = score_sum / count
202 |     accuracy = yes_count / (yes_count + no_count)
203 |     print("Yes count:", yes_count)
204 |     print("No count:", no_count)
205 |     print("Accuracy:", accuracy)
206 |     print("Average score:", average_score)
207 | 
208 | 
209 | if __name__ == "__main__":
210 |     main()
211 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/evaluate_benchmark_2_detailed_orientation.py:
--------------------------------------------------------------------------------
  1 | import openai
  2 | import os
  3 | import argparse
  4 | import json
  5 | import ast
  6 | from multiprocessing.pool import Pool
  7 | 
  8 | openai.api_type = "azure"
  9 | openai.api_version = "2023-10-01-preview"
 10 | 
 11 | def parse_args():
 12 |     parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
 13 |     parser.add_argument("--pred_path", required=True, help="The path to file containing prediction.")
 14 |     parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
 15 |     parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
 16 |     parser.add_argument("--api_key", required=True, help="OpenAI API key.")
 17 |     parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
 18 |     args = parser.parse_args()
 19 |     return args
 20 | 
 21 | 
 22 | def annotate(prediction_set, caption_files, output_dir):
 23 |     """
 24 |     Evaluates question and answer pairs using GPT-3 and
 25 |     returns a score for detailed orientation.
 26 |     """
 27 |     for file in caption_files:
 28 |         key = file[:-5] # Strip file extension
 29 |         qa_set = prediction_set[key]
 30 |         question = qa_set['q']
 31 |         answer = qa_set['a']
 32 |         pred = qa_set['pred']
 33 |         try:
 34 |             # Compute the detailed-orientation score
 35 |             completion = openai.ChatCompletion.create(
 36 |                 # model="gpt-3.5-turbo",
 37 |                 engine="gpt-openai",
 38 |                 model="gpt-3.5-turbo-0613",
 39 |                 messages=[
 40 |                     {
 41 |                         "role": "system",
 42 |                         "content":
 43 |                             "You are an intelligent chatbot designed for evaluating the detail orientation of generative outputs for video-based question-answer pairs. "
 44 |                             "Your task is to compare the predicted answer with the correct answer and determine its level of detail, considering both completeness and specificity. Here's how you can accomplish the task:"
 45 |                             "------"
 46 |                             "##INSTRUCTIONS: "
 47 |                             "- Check if the predicted answer covers all major points from the video. The response should not leave out any key aspects.\n"
 48 |                             "- Evaluate whether the predicted answer includes specific details rather than just generic points. It should provide comprehensive information that is tied to specific elements of the video.\n"
 49 |                             "- Consider synonyms or paraphrases as valid matches.\n"
 50 |                             "- Provide a single evaluation score that reflects the level of detail orientation of the prediction, considering both completeness and specificity."
 51 |                     },
 52 |                     {
 53 |                         "role": "user",
 54 |                         "content":
 55 |                             "Please evaluate the following video-based question-answer pair:\n\n"
 56 |                             f"Question: {question}\n"
 57 |                             f"Correct Answer: {answer}\n"
 58 |                             f"Predicted Answer: {pred}\n\n"
 59 |                             "Provide your evaluation only as a detail orientation score where the detail orientation score is an integer value between 0 and 5, with 5 indicating the highest level of detail orientation. "
 60 |                             "Please generate the response in the form of a Python dictionary string with keys 'score', where its value is the detail orientation score in INTEGER, not STRING."
 61 |                             "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
 62 |                             "For example, your response should look like this: {''score': 4.8}."
 63 |                     }
 64 |                 ]
 65 |             )
 66 |             # Convert response to a Python dictionary.
 67 |             response_message = completion["choices"][0]["message"]["content"]
 68 |             response_dict = ast.literal_eval(response_message)
 69 |             result_qa_pair = [response_dict, qa_set]
 70 | 
 71 |             # Save the question-answer pairs to a json file.
 72 |             with open(f"{output_dir}/{key}.json", "w") as f:
 73 |                 json.dump(result_qa_pair, f)
 74 | 
 75 |         except Exception as e:
 76 |             print(f"Error processing file '{key}': {e}")
 77 | 
 78 | 
 79 | def main():
 80 |     """
 81 |     Main function to control the flow of the program.
 82 |     """
 83 |     # Parse arguments.
 84 |     args = parse_args()
 85 | 
 86 |     file = open(args.pred_path)
 87 |     pred_contents = json.load(file)
 88 | 
 89 |     # Dictionary to store the count of occurrences for each video_id
 90 |     video_id_counts = {}
 91 |     new_pred_contents = []
 92 | 
 93 |     # Iterate through each sample in pred_contents
 94 |     for sample in pred_contents:
 95 |         video_id = sample['video_name']
 96 |         if video_id in video_id_counts:
 97 |             video_id_counts[video_id] += 1
 98 |         else:
 99 |             video_id_counts[video_id] = 0
100 | 
101 |         # Create a new sample with the modified key
102 |         new_sample = sample
103 |         new_sample['video_name'] = f"{video_id}_{video_id_counts[video_id]}"
104 |         new_pred_contents.append(new_sample)
105 | 
106 |     # Generating list of id's and corresponding files
107 |     id_list = [x['video_name'] for x in new_pred_contents]
108 |     caption_files = [f"{id}.json" for id in id_list]
109 | 
110 |     output_dir = args.output_dir
111 |     # Generate output directory if not exists.
112 |     if not os.path.exists(output_dir):
113 |         os.makedirs(output_dir)
114 | 
115 |     # Preparing dictionary of question-answer sets
116 |     prediction_set = {}
117 |     for sample in new_pred_contents:
118 |         id = sample['video_name']
119 |         question = sample['Q']
120 |         answer = sample['A']
121 |         pred = sample['pred']
122 |         qa_set = {"q": question, "a": answer, "pred": pred}
123 |         prediction_set[id] = qa_set
124 | 
125 |     # Set the OpenAI API key.
126 |     openai.api_key = args.api_key
127 |     num_tasks = args.num_tasks
128 | 
129 |     # While loop to ensure that all captions are processed.
130 |     while True:
131 |         try:
132 |             # Files that have not been processed yet.
133 |             completed_files = os.listdir(output_dir)
134 |             print(f"completed_files: {len(completed_files)}")
135 | 
136 |             # Files that have not been processed yet.
137 |             incomplete_files = [f for f in caption_files if f not in completed_files]
138 |             print(f"incomplete_files: {len(incomplete_files)}")
139 | 
140 |             # Break the loop when there are no incomplete files
141 |             if len(incomplete_files) == 0:
142 |                 break
143 |             if len(incomplete_files) <= num_tasks:
144 |                 num_tasks = 1
145 | 
146 |             # Split tasks into parts.
147 |             part_len = len(incomplete_files) // num_tasks
148 |             all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
149 |             task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
150 | 
151 |             # Use a pool of workers to process the files in parallel.
152 |             with Pool() as pool:
153 |                 pool.starmap(annotate, task_args)
154 | 
155 |         except Exception as e:
156 |             print(f"Error: {e}")
157 | 
158 |     # Combine all the processed files into one
159 |     combined_contents = {}
160 |     json_path = args.output_json
161 | 
162 |     # Iterate through json files
163 |     for file_name in os.listdir(output_dir):
164 |         if file_name.endswith(".json"):
165 |             file_path = os.path.join(output_dir, file_name)
166 |             with open(file_path, "r") as json_file:
167 |                 content = json.load(json_file)
168 |                 combined_contents[file_name[:-5]] = content
169 | 
170 |     # Write combined content to a json file
171 |     with open(json_path, "w") as json_file:
172 |         json.dump(combined_contents, json_file)
173 |     print("All evaluation completed!")
174 | 
175 |     # Calculate average score
176 |     score_sum = 0
177 |     count = 0
178 |     for key, result in combined_contents.items():
179 |         count += 1
180 |         score_match = result[0]['score']
181 |         score = int(score_match)
182 |         score_sum += score
183 |     average_score = score_sum / count
184 | 
185 |     print("Average score for detailed orientation:", average_score)
186 | 
187 | 
188 | if __name__ == "__main__":
189 |     main()
190 | 
191 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/evaluate_benchmark_3_context.py:
--------------------------------------------------------------------------------
  1 | import openai
  2 | import os
  3 | import argparse
  4 | import json
  5 | import ast
  6 | from multiprocessing.pool import Pool
  7 | 
  8 | 
  9 | openai.api_type = "azure"
 10 | openai.api_base = "https://search.bytedance.net/gpt/openapi/online/v2/crawl"
 11 | openai.api_version = "2023-10-01-preview"
 12 | 
 13 | def parse_args():
 14 |     parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
 15 |     parser.add_argument("--pred_path", required=True, help="The path to file containing prediction.")
 16 |     parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
 17 |     parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
 18 |     parser.add_argument("--api_key", required=True, help="OpenAI API key.")
 19 |     parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
 20 |     args = parser.parse_args()
 21 |     return args
 22 | 
 23 | 
 24 | def annotate(prediction_set, caption_files, output_dir):
 25 |     """
 26 |     Evaluates question and answer pairs using GPT-3 and
 27 |     returns a score for contextual understanding.
 28 |     """
 29 |     for file in caption_files:
 30 |         key = file[:-5] # Strip file extension
 31 |         qa_set = prediction_set[key]
 32 |         question = qa_set['q']
 33 |         answer = qa_set['a']
 34 |         pred = qa_set['pred']
 35 |         try:
 36 |             # Compute the contextual understanding score
 37 |             completion = openai.ChatCompletion.create(
 38 |                 # model="gpt-3.5-turbo",
 39 |                 engine="gpt-openai",
 40 |                 model="gpt-3.5-turbo-0613",
 41 |                 messages=[
 42 |                     {
 43 |                         "role": "system",
 44 |                         "content":
 45 |                             "You are an intelligent chatbot designed for evaluating the contextual understanding of generative outputs for video-based question-answer pairs. "
 46 |                             "Your task is to compare the predicted answer with the correct answer and determine if the generated response aligns with the overall context of the video content. Here's how you can accomplish the task:"
 47 |                             "------"
 48 |                             "##INSTRUCTIONS: "
 49 |                             "- Evaluate whether the predicted answer aligns with the overall context of the video content. It should not provide information that is out of context or misaligned.\n"
 50 |                             "- The predicted answer must capture the main themes and sentiments of the video.\n"
 51 |                             "- Consider synonyms or paraphrases as valid matches.\n"
 52 |                             "- Provide your evaluation of the contextual understanding of the prediction compared to the answer."
 53 |                     },
 54 |                     {
 55 |                         "role": "user",
 56 |                         "content":
 57 |                             "Please evaluate the following video-based question-answer pair:\n\n"
 58 |                             f"Question: {question}\n"
 59 |                             f"Correct Answer: {answer}\n"
 60 |                             f"Predicted Answer: {pred}\n\n"
 61 |                             "Provide your evaluation only as a contextual understanding score where the contextual understanding score is an integer value between 0 and 5, with 5 indicating the highest level of contextual understanding. "
 62 |                             "Please generate the response in the form of a Python dictionary string with keys 'score', where its value is contextual understanding score in INTEGER, not STRING."
 63 |                             "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
 64 |                             "For example, your response should look like this: {''score': 4.8}."
 65 |                     }
 66 |                 ]
 67 |             )
 68 |             # Convert response to a Python dictionary.
 69 |             response_message = completion["choices"][0]["message"]["content"]
 70 |             response_dict = ast.literal_eval(response_message)
 71 |             result_qa_pair = [response_dict, qa_set]
 72 | 
 73 |             # Save the question-answer pairs to a json file.
 74 |             with open(f"{output_dir}/{key}.json", "w") as f:
 75 |                 json.dump(result_qa_pair, f)
 76 | 
 77 |         except Exception as e:
 78 |             print(f"Error processing file '{key}': {e}")
 79 | 
 80 | 
 81 | def main():
 82 |     """
 83 |     Main function to control the flow of the program.
 84 |     """
 85 |     # Parse arguments.
 86 |     args = parse_args()
 87 | 
 88 |     file = open(args.pred_path)
 89 |     pred_contents = json.load(file)
 90 | 
 91 |     # Dictionary to store the count of occurrences for each video_id
 92 |     video_id_counts = {}
 93 |     new_pred_contents = []
 94 | 
 95 |     # Iterate through each sample in pred_contents
 96 |     for sample in pred_contents:
 97 |         video_id = sample['video_name']
 98 |         if video_id in video_id_counts:
 99 |             video_id_counts[video_id] += 1
100 |         else:
101 |             video_id_counts[video_id] = 0
102 | 
103 |         # Create a new sample with the modified key
104 |         new_sample = sample
105 |         new_sample['video_name'] = f"{video_id}_{video_id_counts[video_id]}"
106 |         new_pred_contents.append(new_sample)
107 | 
108 |     # Generating list of id's and corresponding files
109 |     id_list = [x['video_name'] for x in new_pred_contents]
110 |     caption_files = [f"{id}.json" for id in id_list]
111 | 
112 |     output_dir = args.output_dir
113 |     # Generate output directory if not exists.
114 |     if not os.path.exists(output_dir):
115 |         os.makedirs(output_dir)
116 | 
117 |     # Preparing dictionary of question-answer sets
118 |     prediction_set = {}
119 |     for sample in new_pred_contents:
120 |         id = sample['video_name']
121 |         question = sample['Q']
122 |         answer = sample['A']
123 |         pred = sample['pred']
124 |         qa_set = {"q": question, "a": answer, "pred": pred}
125 |         prediction_set[id] = qa_set
126 | 
127 |     # Set the OpenAI API key.
128 |     openai.api_key = args.api_key
129 |     num_tasks = args.num_tasks
130 | 
131 |     # While loop to ensure that all captions are processed.
132 |     while True:
133 |         try:
134 |             # Files that have not been processed yet.
135 |             completed_files = os.listdir(output_dir)
136 |             print(f"completed_files: {len(completed_files)}")
137 | 
138 |             # Files that have not been processed yet.
139 |             incomplete_files = [f for f in caption_files if f not in completed_files]
140 |             print(f"incomplete_files: {len(incomplete_files)}")
141 | 
142 |             # Break the loop when there are no incomplete files
143 |             if len(incomplete_files) == 0:
144 |                 break
145 |             if len(incomplete_files) <= num_tasks:
146 |                 num_tasks = 1
147 | 
148 |             # Split tasks into parts.
149 |             part_len = len(incomplete_files) // num_tasks
150 |             all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
151 |             task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
152 | 
153 |             # Use a pool of workers to process the files in parallel.
154 |             with Pool() as pool:
155 |                 pool.starmap(annotate, task_args)
156 | 
157 |         except Exception as e:
158 |             print(f"Error: {e}")
159 | 
160 |     # Combine all the processed files into one
161 |     combined_contents = {}
162 |     json_path = args.output_json
163 | 
164 |     # Iterate through json files
165 |     for file_name in os.listdir(output_dir):
166 |         if file_name.endswith(".json"):
167 |             file_path = os.path.join(output_dir, file_name)
168 |             with open(file_path, "r") as json_file:
169 |                 content = json.load(json_file)
170 |                 combined_contents[file_name[:-5]] = content
171 | 
172 |     # Write combined content to a json file
173 |     with open(json_path, "w") as json_file:
174 |         json.dump(combined_contents, json_file)
175 |     print("All evaluation completed!")
176 | 
177 |     # Calculate average score
178 |     score_sum = 0
179 |     count = 0
180 |     for key, result in combined_contents.items():
181 |         count += 1
182 |         score_match = result[0]['score']
183 |         score = int(score_match)
184 |         score_sum += score
185 |     average_score = score_sum / count
186 | 
187 |     print("Average score for contextual understanding:", average_score)
188 | 
189 | 
190 | if __name__ == "__main__":
191 |     main()
192 | 
193 | 


--------------------------------------------------------------------------------
/vista_llama/model/base_model.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import logging
  9 | import os
 10 | 
 11 | import numpy as np
 12 | import torch
 13 | import torch.nn as nn
 14 | from .dist_utils import download_cached_file, is_dist_avail_and_initialized
 15 | from .utils import get_abs_path, is_url
 16 | from omegaconf import OmegaConf
 17 | 
 18 | 
 19 | class BaseModel(nn.Module):
 20 |     """Base class for models."""
 21 | 
 22 |     def __init__(self):
 23 |         super().__init__()
 24 | 
 25 |     @property
 26 |     def device(self):
 27 |         return list(self.parameters())[0].device
 28 | 
 29 |     def load_checkpoint(self, url_or_filename):
 30 |         """
 31 |         Load from a finetuned checkpoint.
 32 | 
 33 |         This should expect no mismatch in the model keys and the checkpoint keys.
 34 |         """
 35 | 
 36 |         if is_url(url_or_filename):
 37 |             cached_file = download_cached_file(
 38 |                 url_or_filename, check_hash=False, progress=True
 39 |             )
 40 |             checkpoint = torch.load(cached_file, map_location="cpu")
 41 |         elif os.path.isfile(url_or_filename):
 42 |             checkpoint = torch.load(url_or_filename, map_location="cpu")
 43 |         else:
 44 |             raise RuntimeError("checkpoint url or path is invalid")
 45 | 
 46 |         if "model" in checkpoint.keys():
 47 |             state_dict = checkpoint["model"]
 48 |         else:
 49 |             state_dict = checkpoint
 50 | 
 51 |         msg = self.load_state_dict(state_dict, strict=False)
 52 | 
 53 |         logging.info("Missing keys {}".format(msg.missing_keys))
 54 |         logging.info("load checkpoint from %s" % url_or_filename)
 55 | 
 56 |         return msg
 57 | 
 58 |     @classmethod
 59 |     def from_pretrained(cls, model_type):
 60 |         """
 61 |         Build a pretrained model from default configuration file, specified by model_type.
 62 | 
 63 |         Args:
 64 |             - model_type (str): model type, specifying architecture and checkpoints.
 65 | 
 66 |         Returns:
 67 |             - model (nn.Module): pretrained or finetuned model, depending on the configuration.
 68 |         """
 69 |         model_cfg = OmegaConf.load(cls.default_config_path(model_type)).model
 70 |         model = cls.from_config(model_cfg)
 71 | 
 72 |         return model
 73 | 
 74 |     @classmethod
 75 |     def default_config_path(cls, model_type):
 76 |         assert (
 77 |             model_type in cls.PRETRAINED_MODEL_CONFIG_DICT
 78 |         ), "Unknown model type {}".format(model_type)
 79 |         return get_abs_path(cls.PRETRAINED_MODEL_CONFIG_DICT[model_type])
 80 | 
 81 |     def load_checkpoint_from_config(self, cfg, **kwargs):
 82 |         """
 83 |         Load checkpoint as specified in the config file.
 84 | 
 85 |         If load_finetuned is True, load the finetuned model; otherwise, load the pretrained model.
 86 |         When loading the pretrained model, each task-specific architecture may define their
 87 |         own load_from_pretrained() method.
 88 |         """
 89 |         load_finetuned = cfg.get("load_finetuned", True)
 90 |         if load_finetuned:
 91 |             finetune_path = cfg.get("finetuned", None)
 92 |             assert (
 93 |                 finetune_path is not None
 94 |             ), "Found load_finetuned is True, but finetune_path is None."
 95 |             self.load_checkpoint(url_or_filename=finetune_path)
 96 |         else:
 97 |             # load pre-trained weights
 98 |             pretrain_path = cfg.get("pretrained", None)
 99 |             assert "Found load_finetuned is False, but pretrain_path is None."
100 |             self.load_from_pretrained(url_or_filename=pretrain_path, **kwargs)
101 | 
102 |     def before_evaluation(self, **kwargs):
103 |         pass
104 | 
105 |     def show_n_params(self, return_str=True):
106 |         tot = 0
107 |         for p in self.parameters():
108 |             w = 1
109 |             for x in p.shape:
110 |                 w *= x
111 |             tot += w
112 |         if return_str:
113 |             if tot >= 1e6:
114 |                 return "{:.1f}M".format(tot / 1e6)
115 |             else:
116 |                 return "{:.1f}K".format(tot / 1e3)
117 |         else:
118 |             return tot
119 | 
120 | 
121 | class BaseEncoder(nn.Module):
122 |     """
123 |     Base class for primitive encoders, such as ViT, TimeSformer, etc.
124 |     """
125 | 
126 |     def __init__(self):
127 |         super().__init__()
128 | 
129 |     def forward_features(self, samples, **kwargs):
130 |         raise NotImplementedError
131 | 
132 |     @property
133 |     def device(self):
134 |         return list(self.parameters())[0].device
135 | 
136 | 
137 | class SharedQueueMixin:
138 |     @torch.no_grad()
139 |     def _dequeue_and_enqueue(self, image_feat, text_feat, idxs=None):
140 |         # gather keys before updating queue
141 |         image_feats = concat_all_gather(image_feat)
142 |         text_feats = concat_all_gather(text_feat)
143 | 
144 |         batch_size = image_feats.shape[0]
145 | 
146 |         ptr = int(self.queue_ptr)
147 |         assert self.queue_size % batch_size == 0  # for simplicity
148 | 
149 |         # replace the keys at ptr (dequeue and enqueue)
150 |         self.image_queue[:, ptr : ptr + batch_size] = image_feats.T
151 |         self.text_queue[:, ptr : ptr + batch_size] = text_feats.T
152 | 
153 |         if idxs is not None:
154 |             idxs = concat_all_gather(idxs)
155 |             self.idx_queue[:, ptr : ptr + batch_size] = idxs.T
156 | 
157 |         ptr = (ptr + batch_size) % self.queue_size  # move pointer
158 |         self.queue_ptr[0] = ptr
159 | 
160 | 
161 | class MomentumDistilationMixin:
162 |     @torch.no_grad()
163 |     def copy_params(self):
164 |         for model_pair in self.model_pairs:
165 |             for param, param_m in zip(
166 |                 model_pair[0].parameters(), model_pair[1].parameters()
167 |             ):
168 |                 param_m.data.copy_(param.data)  # initialize
169 |                 param_m.requires_grad = False  # not update by gradient
170 | 
171 |     @torch.no_grad()
172 |     def _momentum_update(self):
173 |         for model_pair in self.model_pairs:
174 |             for param, param_m in zip(
175 |                 model_pair[0].parameters(), model_pair[1].parameters()
176 |             ):
177 |                 param_m.data = param_m.data * self.momentum + param.data * (
178 |                     1.0 - self.momentum
179 |                 )
180 | 
181 | 
182 | class GatherLayer(torch.autograd.Function):
183 |     """
184 |     Gather tensors from all workers with support for backward propagation:
185 |     This implementation does not cut the gradients as torch.distributed.all_gather does.
186 |     """
187 | 
188 |     @staticmethod
189 |     def forward(ctx, x):
190 |         output = [
191 |             torch.zeros_like(x) for _ in range(torch.distributed.get_world_size())
192 |         ]
193 |         torch.distributed.all_gather(output, x)
194 |         return tuple(output)
195 | 
196 |     @staticmethod
197 |     def backward(ctx, *grads):
198 |         all_gradients = torch.stack(grads)
199 |         torch.distributed.all_reduce(all_gradients)
200 |         return all_gradients[torch.distributed.get_rank()]
201 | 
202 | 
203 | def all_gather_with_grad(tensors):
204 |     """
205 |     Performs all_gather operation on the provided tensors.
206 |     Graph remains connected for backward grad computation.
207 |     """
208 |     # Queue the gathered tensors
209 |     world_size = torch.distributed.get_world_size()
210 |     # There is no need for reduction in the single-proc case
211 |     if world_size == 1:
212 |         return tensors
213 | 
214 |     # tensor_all = GatherLayer.apply(tensors)
215 |     tensor_all = GatherLayer.apply(tensors)
216 | 
217 |     return torch.cat(tensor_all, dim=0)
218 | 
219 | 
220 | @torch.no_grad()
221 | def concat_all_gather(tensor):
222 |     """
223 |     Performs all_gather operation on the provided tensors.
224 |     *** Warning ***: torch.distributed.all_gather has no gradient.
225 |     """
226 |     # if use distributed training
227 |     if not is_dist_avail_and_initialized():
228 |         return tensor
229 | 
230 |     tensors_gather = [
231 |         torch.ones_like(tensor) for _ in range(torch.distributed.get_world_size())
232 |     ]
233 |     torch.distributed.all_gather(tensors_gather, tensor, async_op=False)
234 | 
235 |     output = torch.cat(tensors_gather, dim=0)
236 |     return output
237 | 
238 | 
239 | def tile(x, dim, n_tile):
240 |     init_dim = x.size(dim)
241 |     repeat_idx = [1] * x.dim()
242 |     repeat_idx[dim] = n_tile
243 |     x = x.repeat(*(repeat_idx))
244 |     order_index = torch.LongTensor(
245 |         np.concatenate([init_dim * np.arange(n_tile) + i for i in range(init_dim)])
246 |     )
247 |     return torch.index_select(x, dim, order_index.to(x.device))
248 | 


--------------------------------------------------------------------------------
/quantitative_evaluation/evaluate_benchmark_5_consistency.py:
--------------------------------------------------------------------------------
  1 | import openai
  2 | import os
  3 | import argparse
  4 | import json
  5 | import ast
  6 | from multiprocessing.pool import Pool
  7 | 
  8 | 
  9 | openai.api_type = "azure"
 10 | openai.api_version = "2023-10-01-preview"
 11 | 
 12 | def parse_args():
 13 |     parser = argparse.ArgumentParser(description="question-answer-generation-using-gpt-3")
 14 |     parser.add_argument("--pred_path", required=True, help="The path to file containing prediction.")
 15 |     parser.add_argument("--output_dir", required=True, help="The path to save annotation json files.")
 16 |     parser.add_argument("--output_json", required=True, help="The path to save annotation final combined json file.")
 17 |     parser.add_argument("--api_key", required=True, help="OpenAI API key.")
 18 |     parser.add_argument("--num_tasks", required=True, type=int, help="Number of splits.")
 19 |     args = parser.parse_args()
 20 |     return args
 21 | 
 22 | 
 23 | def annotate(prediction_set, caption_files, output_dir):
 24 |     """
 25 |     Evaluates question and answer pairs using GPT-3 and
 26 |     returns a score for consistency.
 27 |     """
 28 |     for file in caption_files:
 29 |         key = file[:-5] # Strip file extension
 30 |         qa_set = prediction_set[key]
 31 |         question1 = qa_set['q1']
 32 |         question2 = qa_set['q2']
 33 |         answer = qa_set['a']
 34 |         pred1 = qa_set['pred1']
 35 |         pred2 = qa_set['pred2']
 36 |         try:
 37 |             # Compute the consistency score
 38 |             completion = openai.ChatCompletion.create(
 39 |                 # model="gpt-3.5-turbo",
 40 |                 engine="gpt-openai",
 41 |                 model="gpt-3.5-turbo-0613",
 42 |                 messages=[
 43 |                     {
 44 |                         "role": "system",
 45 |                         "content":
 46 |                             "You are an intelligent chatbot designed for evaluating the consistency of generative outputs for similar video-based question-answer pairs. "
 47 |                             "You will be given two very similar questions, a common answer common to both the questions and predicted answers for the two questions ."
 48 |                             "Your task is to compare the predicted answers for two very similar question, with a common correct answer and determine if they are consistent. Here's how you can accomplish the task:"
 49 |                             "------"
 50 |                             "##INSTRUCTIONS: "
 51 |                             "- Focus on the consistency between the two predicted answers and the correct answer. Both predicted answers should correspond to the correct answer and to each other, and should not contain any contradictions or significant differences in the conveyed information.\n"
 52 |                             "- Both predicted answers must be consistent with each other and the correct answer, in terms of the information they provide about the video content.\n"
 53 |                             "- Consider synonyms or paraphrases as valid matches, but only if they maintain the consistency in the conveyed information.\n"
 54 |                             "- Evaluate the consistency of the two predicted answers compared to the correct answer."
 55 |                     },
 56 |                     {
 57 |                         "role": "user",
 58 |                         "content":
 59 |                             "Please evaluate the following video-based question-answer pair:\n\n"
 60 |                             f"Question 1: {question1}\n"
 61 |                             f"Question 2: {question2}\n"
 62 |                             f"Correct Answer: {answer}\n"
 63 |                             f"Predicted Answer to Question 1: {pred1}\n"
 64 |                             f"Predicted Answer to Question 2: {pred2}\n\n"
 65 |                             "Provide your evaluation only as a consistency score where the consistency score is an integer value between 0 and 5, with 5 indicating the highest level of consistency. "
 66 |                             "Please generate the response in the form of a Python dictionary string with keys 'score', where its value is the consistency score in INTEGER, not STRING."
 67 |                             "DO NOT PROVIDE ANY OTHER OUTPUT TEXT OR EXPLANATION. Only provide the Python dictionary string. "
 68 |                             "For example, your response should look like this: {''score': 4.8}."
 69 |                     }
 70 |                 ]
 71 |             )
 72 |             # Convert response to a Python dictionary.
 73 |             response_message = completion["choices"][0]["message"]["content"]
 74 |             response_dict = ast.literal_eval(response_message)
 75 |             result_qa_pair = [response_dict, qa_set]
 76 | 
 77 |             # Save the question-answer pairs to a json file.
 78 |             with open(f"{output_dir}/{key}.json", "w") as f:
 79 |                 json.dump(result_qa_pair, f)
 80 | 
 81 |         except Exception as e:
 82 |             print(f"Error processing file '{key}': {e}")
 83 | 
 84 | 
 85 | def main():
 86 |     """
 87 |     Main function to control the flow of the program.
 88 |     """
 89 |     # Parse arguments.
 90 |     args = parse_args()
 91 | 
 92 |     file = open(args.pred_path)
 93 |     pred_contents = json.load(file)
 94 | 
 95 |     # Dictionary to store the count of occurrences for each video_id
 96 |     video_id_counts = {}
 97 |     new_pred_contents = []
 98 | 
 99 |     # Iterate through each sample in pred_contents
100 |     for sample in pred_contents:
101 |         video_id = sample['video_name']
102 |         if video_id in video_id_counts:
103 |             video_id_counts[video_id] += 1
104 |         else:
105 |             video_id_counts[video_id] = 0
106 | 
107 |         # Create a new sample with the modified key
108 |         new_sample = sample
109 |         new_sample['video_name'] = f"{video_id}_{video_id_counts[video_id]}"
110 |         new_pred_contents.append(new_sample)
111 | 
112 |     # Generating list of id's and corresponding files
113 |     id_list = [x['video_name'] for x in new_pred_contents]
114 |     caption_files = [f"{id}.json" for id in id_list]
115 | 
116 |     output_dir = args.output_dir
117 |     # Generate output directory if not exists.
118 |     if not os.path.exists(output_dir):
119 |         os.makedirs(output_dir)
120 | 
121 |     # Preparing dictionary of question-answer sets
122 |     prediction_set = {}
123 |     for sample in new_pred_contents:
124 |         id = sample['video_name']
125 |         question1 = sample['Q1']
126 |         question2 = sample['Q1']
127 |         answer = sample['A']
128 |         pred1 = sample['pred1']
129 |         pred2 = sample['pred2']
130 |         qa_set = {"q1": question1, "q2": question2, "a": answer, "pred1": pred1, "pred2": pred2}
131 |         prediction_set[id] = qa_set
132 | 
133 |     # Set the OpenAI API key.
134 |     openai.api_key = args.api_key
135 |     num_tasks = args.num_tasks
136 | 
137 |     # While loop to ensure that all captions are processed.
138 |     while True:
139 |         try:
140 |             # Files that have not been processed yet.
141 |             completed_files = os.listdir(output_dir)
142 |             print(f"completed_files: {len(completed_files)}")
143 | 
144 |             # Files that have not been processed yet.
145 |             incomplete_files = [f for f in caption_files if f not in completed_files]
146 |             print(f"incomplete_files: {len(incomplete_files)}")
147 | 
148 |             # Break the loop when there are no incomplete files
149 |             if len(incomplete_files) == 0:
150 |                 break
151 |             if len(incomplete_files) <= num_tasks:
152 |                 num_tasks = 1
153 | 
154 |             # Split tasks into parts.
155 |             part_len = len(incomplete_files) // num_tasks
156 |             all_parts = [incomplete_files[i:i + part_len] for i in range(0, len(incomplete_files), part_len)]
157 |             task_args = [(prediction_set, part, args.output_dir) for part in all_parts]
158 | 
159 |             # Use a pool of workers to process the files in parallel.
160 |             with Pool() as pool:
161 |                 pool.starmap(annotate, task_args)
162 | 
163 |         except Exception as e: 
164 |             print(f"Error: {e}")
165 | 
166 |     # Combine all the processed files into one
167 |     combined_contents = {}
168 |     json_path = args.output_json
169 | 
170 |     # Iterate through json files
171 |     for file_name in os.listdir(output_dir):
172 |         if file_name.endswith(".json"):
173 |             file_path = os.path.join(output_dir, file_name)
174 |             with open(file_path, "r") as json_file:
175 |                 content = json.load(json_file)
176 |                 combined_contents[file_name[:-5]] = content
177 | 
178 |     # Write combined content to a json file
179 |     with open(json_path, "w") as json_file:
180 |         json.dump(combined_contents, json_file)
181 |     print("All evaluation completed!")
182 | 
183 |     # Calculate average score
184 |     score_sum = 0
185 |     count = 0
186 |     for key, result in combined_contents.items():
187 |         count += 1
188 |         score_match = result[0]['score']
189 |         score = int(score_match)
190 |         score_sum += score
191 |     average_score = score_sum / count
192 | 
193 |     print("Average score for consistency:", average_score)
194 | 
195 | 
196 | if __name__ == "__main__":
197 |     main()
198 | 
199 | 


--------------------------------------------------------------------------------
/vista_llama/model/blip2.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2023, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | import logging
  8 | import os
  9 | import time
 10 | import datetime
 11 | 
 12 | import torch
 13 | import torch.nn as nn
 14 | import torch.distributed as dist
 15 | import torch.nn.functional as F
 16 | 
 17 | from . import dist_utils
 18 | from .dist_utils import download_cached_file
 19 | from .utils import is_url
 20 | from .logger import MetricLogger
 21 | from .base_model import BaseModel
 22 | from .qformer import BertConfig, BertLMHeadModel
 23 | from .eva_vit import create_eva_vit_g
 24 | 
 25 | # import lavis.common.dist_utils as dist_utils
 26 | # from lavis.common.dist_utils import download_cached_file
 27 | # from lavis.common.utils import is_url
 28 | # from lavis.common.logger import MetricLogger
 29 | # from lavis.models.base_model import BaseModel
 30 | # from lavis.models.blip2_models.Qformer import BertConfig, BertLMHeadModel
 31 | # from lavis.models.eva_vit import create_eva_vit_g
 32 | from transformers import BertTokenizer
 33 | 
 34 | 
 35 | class Blip2Base(BaseModel):
 36 |     @classmethod
 37 |     def init_tokenizer(cls):
 38 |         tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
 39 |         tokenizer.add_special_tokens({"bos_token": "[DEC]"})
 40 |         return tokenizer
 41 | 
 42 |     @classmethod
 43 |     def init_Qformer(cls, num_query_token, vision_width):
 44 |         encoder_config = BertConfig.from_pretrained("bert-base-uncased")
 45 |         encoder_config.encoder_width = vision_width
 46 |         # insert cross-attention layer every other block
 47 |         encoder_config.add_cross_attention = True
 48 |         encoder_config.cross_attention_freq = 2
 49 |         encoder_config.query_length = num_query_token
 50 |         Qformer = BertLMHeadModel.from_pretrained(
 51 |             "bert-base-uncased", config=encoder_config
 52 |         )                 
 53 |         query_tokens = nn.Parameter(
 54 |             torch.zeros(1, num_query_token, encoder_config.hidden_size)
 55 |         )
 56 |         query_tokens.data.normal_(mean=0.0, std=encoder_config.initializer_range)
 57 |         return Qformer, query_tokens
 58 |     
 59 |     @classmethod
 60 |     def init_TemporalQFormer(cls, num_of_frame):
 61 |         encoder_config = BertConfig.from_pretrained("bert-base-uncased")
 62 |         encoder_config.query_length = num_of_frame
 63 |         Qformer = BertLMHeadModel.from_pretrained(
 64 |         "bert-base-uncased", config=encoder_config
 65 |         )                 
 66 |         query_tokens = nn.Parameter(
 67 |             torch.zeros(1, num_of_frame, 1, encoder_config.hidden_size)
 68 |         )
 69 |         query_tokens.data.normal_(mean=0.0, std=encoder_config.initializer_range)
 70 |         return Qformer, query_tokens
 71 | 
 72 |     @classmethod
 73 |     def init_vision_encoder(
 74 |         cls, img_size, drop_path_rate, use_grad_checkpoint, precision
 75 |     ):
 76 |         visual_encoder = create_eva_vit_g(
 77 |             img_size, drop_path_rate, use_grad_checkpoint, precision
 78 |         )
 79 |         ln_vision = LayerNorm(visual_encoder.num_features)
 80 |         return visual_encoder, ln_vision
 81 | 
 82 |     @classmethod
 83 |     def init_vision_encoder_sevila(
 84 |         cls, img_size, drop_path_rate, use_grad_checkpoint, precision
 85 |     ):
 86 |         visual_encoder = create_eva_vit_g(
 87 |             img_size, drop_path_rate, use_grad_checkpoint, precision
 88 |         )
 89 |         ln_vision = LayerNorm(visual_encoder.num_features)
 90 |         ln_vision2 = LayerNorm(visual_encoder.num_features) 
 91 |         return visual_encoder, ln_vision, ln_vision2
 92 | 
 93 |     def load_from_pretrained(self, url_or_filename):
 94 |         if is_url(url_or_filename):
 95 |             cached_file = download_cached_file(
 96 |                 url_or_filename, check_hash=False, progress=True
 97 |             )
 98 |             checkpoint = torch.load(cached_file, map_location="cpu")
 99 |         elif os.path.isfile(url_or_filename):
100 |             checkpoint = torch.load(url_or_filename, map_location="cpu")
101 |         else:
102 |             raise RuntimeError("checkpoint url or path is invalid")
103 | 
104 |         state_dict = checkpoint["model"]
105 |         #print('state_dict',state_dict.keys())
106 |         msg = self.load_state_dict(state_dict, strict=False)
107 | 
108 |         logging.info("Missing keys {}".format(msg.missing_keys))
109 |         logging.info("load checkpoint from %s" % url_or_filename)
110 | 
111 |         return msg
112 |     
113 |     def load_qformer_loc(self):
114 |         url_or_filename = '/nas-hdd/shoubin/pretrained_model/hub/checkpoints/qformer_loc.pth'
115 |         checkpoint = torch.load(url_or_filename, map_location="cpu")
116 |         state_dict = checkpoint["model"]
117 |         msg = self.load_state_dict(state_dict, strict=False)
118 |         logging.info("load checkpoint from %s" % url_or_filename)
119 | 
120 | def disabled_train(self, mode=True):
121 |     """Overwrite model.train with this function to make sure train/eval mode
122 |     does not change anymore."""
123 |     return self
124 | 
125 | 
126 | class LayerNorm(nn.LayerNorm):
127 |     """Subclass torch's LayerNorm to handle fp16."""
128 | 
129 |     def forward(self, x: torch.Tensor):
130 |         orig_type = x.dtype
131 |         ret = super().forward(x.type(torch.float32))
132 |         return ret.type(orig_type)
133 | 
134 | 
135 | def compute_sim_matrix(model, data_loader, **kwargs):
136 |     k_test = kwargs.pop("k_test")
137 | 
138 |     metric_logger = MetricLogger(delimiter="  ")
139 |     header = "Evaluation:"
140 | 
141 |     logging.info("Computing features for evaluation...")
142 |     start_time = time.time()
143 | 
144 |     texts = data_loader.dataset.text
145 |     num_text = len(texts)
146 |     text_bs = 256
147 |     text_ids = []
148 |     text_embeds = []
149 |     text_atts = []
150 |     for i in range(0, num_text, text_bs):
151 |         text = texts[i : min(num_text, i + text_bs)]
152 |         text_input = model.tokenizer(
153 |             text,
154 |             padding="max_length",
155 |             truncation=True,
156 |             max_length=35,
157 |             return_tensors="pt",
158 |         ).to(model.device)
159 |         text_feat = model.forward_text(text_input)
160 |         text_embed = F.normalize(model.text_proj(text_feat))
161 |         text_embeds.append(text_embed)
162 |         text_ids.append(text_input.input_ids)
163 |         text_atts.append(text_input.attention_mask)
164 | 
165 |     text_embeds = torch.cat(text_embeds, dim=0)
166 |     text_ids = torch.cat(text_ids, dim=0)
167 |     text_atts = torch.cat(text_atts, dim=0)
168 | 
169 |     vit_feats = []
170 |     image_embeds = []
171 |     for samples in data_loader:
172 |         image = samples["image"]
173 | 
174 |         image = image.to(model.device)
175 |         image_feat, vit_feat = model.forward_image(image)
176 |         image_embed = model.vision_proj(image_feat)
177 |         image_embed = F.normalize(image_embed, dim=-1)
178 | 
179 |         vit_feats.append(vit_feat.cpu())
180 |         image_embeds.append(image_embed)
181 | 
182 |     vit_feats = torch.cat(vit_feats, dim=0)
183 |     image_embeds = torch.cat(image_embeds, dim=0)
184 | 
185 |     sims_matrix = []
186 |     for image_embed in image_embeds:
187 |         sim_q2t = image_embed @ text_embeds.t()
188 |         sim_i2t, _ = sim_q2t.max(0)
189 |         sims_matrix.append(sim_i2t)
190 |     sims_matrix = torch.stack(sims_matrix, dim=0)
191 | 
192 |     score_matrix_i2t = torch.full(
193 |         (len(data_loader.dataset.image), len(texts)), -100.0
194 |     ).to(model.device)
195 | 
196 |     num_tasks = dist_utils.get_world_size()
197 |     rank = dist_utils.get_rank()
198 |     step = sims_matrix.size(0) // num_tasks + 1
199 |     start = rank * step
200 |     end = min(sims_matrix.size(0), start + step)
201 | 
202 |     for i, sims in enumerate(
203 |         metric_logger.log_every(sims_matrix[start:end], 50, header)
204 |     ):
205 |         topk_sim, topk_idx = sims.topk(k=k_test, dim=0)
206 |         image_inputs = vit_feats[start + i].repeat(k_test, 1, 1).to(model.device)
207 |         score = model.compute_itm(
208 |             image_inputs=image_inputs,
209 |             text_ids=text_ids[topk_idx],
210 |             text_atts=text_atts[topk_idx],
211 |         ).float()
212 |         score_matrix_i2t[start + i, topk_idx] = score + topk_sim
213 | 
214 |     sims_matrix = sims_matrix.t()
215 |     score_matrix_t2i = torch.full(
216 |         (len(texts), len(data_loader.dataset.image)), -100.0
217 |     ).to(model.device)
218 | 
219 |     step = sims_matrix.size(0) // num_tasks + 1
220 |     start = rank * step
221 |     end = min(sims_matrix.size(0), start + step)
222 | 
223 |     for i, sims in enumerate(
224 |         metric_logger.log_every(sims_matrix[start:end], 50, header)
225 |     ):
226 |         topk_sim, topk_idx = sims.topk(k=k_test, dim=0)
227 |         image_inputs = vit_feats[topk_idx.cpu()].to(model.device)
228 |         score = model.compute_itm(
229 |             image_inputs=image_inputs,
230 |             text_ids=text_ids[start + i].repeat(k_test, 1),
231 |             text_atts=text_atts[start + i].repeat(k_test, 1),
232 |         ).float()
233 |         score_matrix_t2i[start + i, topk_idx] = score + topk_sim
234 | 
235 |     if dist_utils.is_dist_avail_and_initialized():
236 |         dist.barrier()
237 |         torch.distributed.all_reduce(
238 |             score_matrix_i2t, op=torch.distributed.ReduceOp.SUM
239 |         )
240 |         torch.distributed.all_reduce(
241 |             score_matrix_t2i, op=torch.distributed.ReduceOp.SUM
242 |         )
243 | 
244 |     total_time = time.time() - start_time
245 |     total_time_str = str(datetime.timedelta(seconds=int(total_time)))
246 |     logging.info("Evaluation time {}".format(total_time_str))
247 | 
248 |     return score_matrix_i2t.cpu().numpy(), score_matrix_t2i.cpu().numpy()
249 | 


--------------------------------------------------------------------------------
/vista_llama/eval/run_inference_qformer_general.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import argparse
  3 | import json
  4 | import torch
  5 | from tqdm import tqdm
  6 | from vista_llama.model import VideoChatGPTQformerLlamaForCausalLM
  7 | from transformers import AutoTokenizer, CLIPVisionModel, CLIPImageProcessor
  8 | from vista_llama.utils import disable_torch_init, save_result
  9 | from vista_llama.eval.model_utils import load_video
 10 | from vista_llama.video_conversation import conv_templates, SeparatorStyle
 11 | from vista_llama.model.utils import KeywordsStoppingCriteria
 12 | from vista_llama.model.dist_utils import init_distributed_mode
 13 | from vista_llama.constants import *
 14 | from vista_llama.model.eva_vit import create_eva_vit_g
 15 | 
 16 | DEFAULT_VIDEO_TOKEN = "<video>"
 17 | DEFAULT_VIDEO_PATCH_TOKEN = "<vid_patch>"
 18 | DEFAULT_VID_START_TOKEN = "<vid_start>"
 19 | DEFAULT_VID_END_TOKEN = "<vid_end>"
 20 | 
 21 | 
 22 | def parse_args():
 23 |     """
 24 |     Parse command-line arguments.
 25 |     """
 26 |     parser = argparse.ArgumentParser()
 27 | 
 28 |     # Define the command-line arguments
 29 |     parser.add_argument("--dataset", default="nextqa")
 30 |     parser.add_argument("--video_dir", default="/opt/tiger/NExTVideo")
 31 |     parser.add_argument(
 32 |         "--gt_file", default="/opt/tiger/SeViLA/sevila_data/nextqa/val.json"
 33 |     )
 34 |     parser.add_argument("--output_dir", default="output")
 35 |     parser.add_argument("--video_token_len", default=128, type=int)
 36 |     parser.add_argument("--use_visual_mask", action="store_true")
 37 |     parser.add_argument(
 38 |         "--num_video_frame", default=4, type=int, help="num of used video frame"
 39 |     )
 40 |     parser.add_argument("--output_name", default="video_chatgpt_qformer_nextqa_preds")
 41 |     parser.add_argument("--model-name", type=str, default="LLaVA-7B-Lightening-v1-1")
 42 |     parser.add_argument("--feature_dim", type=int, default=256)
 43 |     parser.add_argument("--visual_model", type=str, default="clip14l")
 44 |     parser.add_argument("--qformer_type", type=str, default="normal")
 45 |     parser.add_argument(
 46 |         "--conv-mode", type=str, required=False, default="video-chatgpt_v1"
 47 |     )
 48 |     parser.add_argument(
 49 |         "--projection_path",
 50 |         type=str,
 51 |         default="/mnt/bn/vlp-v6/Video-ChatGPT/qformer/mm_projector/checkpoint-9000.bin",
 52 |     )
 53 | 
 54 |     return parser.parse_args()
 55 | 
 56 | 
 57 | def video_chatgpt_infer(
 58 |     video_frames,
 59 |     question,
 60 |     conv_mode,
 61 |     model,
 62 |     vision_tower,
 63 |     tokenizer,
 64 |     image_processor,
 65 |     video_token_len,
 66 |     feature_dim,
 67 |     visual_model,
 68 | ):
 69 |     """
 70 |     Run inference using the Video-ChatGPT model.
 71 | 
 72 |     Parameters:
 73 |     sample : Initial sample
 74 |     video_frames (torch.Tensor): Video frames to process.
 75 |     question (str): The question string.
 76 |     conv_mode: Conversation mode.
 77 |     model: The pretrained Video-ChatGPT model.
 78 |     vision_tower: Vision model to extract video features.
 79 |     tokenizer: Tokenizer for the model.
 80 |     image_processor: Image processor to preprocess video frames.
 81 |     video_token_len (int): The length of video tokens.
 82 | 
 83 |     Returns:
 84 |     dict: Dictionary containing the model's output.
 85 |     """
 86 | 
 87 |     # Prepare question string for the model
 88 |     if model.get_model().vision_config.use_vid_start_end:
 89 |         qs = (
 90 |             question
 91 |             + "\n"
 92 |             + DEFAULT_VID_START_TOKEN
 93 |             + DEFAULT_VIDEO_PATCH_TOKEN * video_token_len
 94 |             + DEFAULT_VID_END_TOKEN
 95 |         )
 96 |     else:
 97 |         qs = question + "\n" + DEFAULT_VIDEO_PATCH_TOKEN * video_token_len
 98 | 
 99 |     # Prepare conversation prompt
100 |     conv = conv_templates[conv_mode].copy()
101 |     conv.append_message(conv.roles[0], qs)
102 |     conv.append_message(conv.roles[1], None)
103 |     prompt = conv.get_prompt()
104 | 
105 |     # Tokenize the prompt
106 |     inputs = tokenizer([prompt])
107 | 
108 |     # Preprocess video frames and get image tensor
109 |     image_tensor = image_processor.preprocess(video_frames, return_tensors="pt")[
110 |         "pixel_values"
111 |     ]
112 | 
113 |     # Move image tensor to GPU and reduce precision to half
114 |     image_tensor = image_tensor.half().to(model.device)
115 | 
116 |     # Generate video spatio-temporal features
117 |     with torch.no_grad():
118 |         if visual_model == "evaclip":
119 |             with torch.cuda.amp.autocast():
120 |                 image_forward_outs = vision_tower(image_tensor)
121 |                 feature = image_forward_outs
122 |         else:
123 |             image_forward_outs = vision_tower(image_tensor, output_hidden_states=True)
124 |             feature = image_forward_outs.hidden_states[-2][
125 |                 :, -feature_dim:
126 |             ]  # Use second to last layer as in LLaVA
127 |         video_spatio_temporal_features = feature
128 |         # interval = feature.shape[0] // 16
129 |         # idx = interval // 2
130 |         # video_spatio_temporal_features = feature[idx::interval][:16]
131 | 
132 |     # Move inputs to GPU
133 |     input_ids = torch.as_tensor(inputs.input_ids).to(model.device)
134 | 
135 |     # Define stopping criteria for generation
136 |     stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
137 |     stopping_criteria = KeywordsStoppingCriteria([stop_str], tokenizer, input_ids)
138 | 
139 |     # Run model inference
140 |     with torch.inference_mode():
141 |         output_ids = model.generate(
142 |             input_ids,
143 |             video_spatio_temporal_features=video_spatio_temporal_features.unsqueeze(0),
144 |             do_sample=True,
145 |             temperature=0.2,
146 |             max_new_tokens=1024,
147 |             stopping_criteria=[stopping_criteria],
148 |         )
149 | 
150 |     # Check if output is the same as input
151 |     n_diff_input_output = (
152 |         (input_ids != output_ids[:, : input_ids.shape[1]]).sum().item()
153 |     )
154 |     if n_diff_input_output > 0:
155 |         print(
156 |             f"[Warning] {n_diff_input_output} output_ids are not the same as the input_ids"
157 |         )
158 | 
159 |     # Decode output tokens
160 |     outputs = tokenizer.batch_decode(
161 |         output_ids[:, input_ids.shape[1] :], skip_special_tokens=True
162 |     )[0]
163 | 
164 |     # Clean output string
165 |     outputs = outputs.strip().rstrip(stop_str).strip()
166 | 
167 |     return outputs
168 | 
169 | 
170 | def initialize_model(
171 |     model_name, projection_path=None, device=0, visual_model="clip14l"
172 | ):
173 |     """
174 |     Initializes the model with given parameters.
175 | 
176 |     Parameters:
177 |     model_name (str): Name of the model to initialize.
178 |     projection_path (str, optional): Path to the projection weights. Defaults to None.
179 | 
180 |     Returns:
181 |     tuple: Model, vision tower, tokenizer, image processor, vision config, and video token length.
182 |     """
183 | 
184 |     # Disable initial torch operations
185 |     disable_torch_init()
186 | 
187 |     # Convert model name to user path
188 |     model_name = os.path.expanduser(model_name)
189 | 
190 |     # Load tokenizer
191 |     tokenizer = AutoTokenizer.from_pretrained(model_name)
192 | 
193 |     # Load model
194 |     model = VideoChatGPTQformerLlamaForCausalLM.from_pretrained(
195 |         model_name, low_cpu_mem_usage=True, torch_dtype=torch.float16, use_cache=True
196 |     )
197 | 
198 |     # Load image processor
199 |     image_processor = CLIPImageProcessor.from_pretrained(
200 |         model.config.mm_vision_tower, torch_dtype=torch.float16
201 |     )
202 | 
203 |     # Set to use start and end tokens for video
204 |     mm_use_vid_start_end = True
205 | 
206 |     # Add tokens to tokenizer
207 |     tokenizer.add_tokens([DEFAULT_VIDEO_PATCH_TOKEN], special_tokens=True)
208 |     if mm_use_vid_start_end:
209 |         tokenizer.add_tokens(
210 |             [DEFAULT_VID_START_TOKEN, DEFAULT_VID_END_TOKEN], special_tokens=True
211 |         )
212 | 
213 |     # Resize token embeddings of the model
214 |     model.resize_token_embeddings(len(tokenizer))
215 | 
216 |     # Load the weights from projection_path after resizing the token_embeddings
217 |     if projection_path:
218 |         print(f"Loading weights from {projection_path}")
219 |         status = model.load_state_dict(
220 |             torch.load(projection_path, map_location="cpu"), strict=False
221 |         )
222 |         if status.unexpected_keys:
223 |             print(
224 |                 f"Unexpected Keys: {status.unexpected_keys}.\nThe Video-ChatGPT weights are not loaded correctly."
225 |             )
226 |         print(f"Weights loaded from {projection_path}")
227 | 
228 |     # Set model to evaluation mode and move to GPU
229 |     model = model.eval()
230 |     model = model.to(device)
231 | 
232 |     vision_tower_name = "openai/clip-vit-large-patch14"
233 | 
234 |     if visual_model == "clip14l":
235 |         # Load vision tower and move to GPU
236 |         vision_tower = CLIPVisionModel.from_pretrained(
237 |             vision_tower_name, torch_dtype=torch.float16, low_cpu_mem_usage=True
238 |         ).to(device)
239 |     elif visual_model == "evaclip":
240 |         vision_tower = create_eva_vit_g(224, 0, False, "fp16").to(device)
241 |     else:
242 |         raise NotImplementedError
243 |     vision_tower = vision_tower.eval()
244 | 
245 |     # Configure vision model
246 |     vision_config = model.get_model().vision_config
247 |     vision_config.vid_patch_token = tokenizer.convert_tokens_to_ids(
248 |         [DEFAULT_VIDEO_PATCH_TOKEN]
249 |     )[0]
250 |     vision_config.use_vid_start_end = mm_use_vid_start_end
251 |     if mm_use_vid_start_end:
252 |         (
253 |             vision_config.vid_start_token,
254 |             vision_config.vid_end_token,
255 |         ) = tokenizer.convert_tokens_to_ids(
256 |             [DEFAULT_VID_START_TOKEN, DEFAULT_VID_END_TOKEN]
257 |         )
258 | 
259 |     # Set video token length
260 | 
261 |     return model, vision_tower, tokenizer, image_processor
262 | 
263 | 
264 | def run_inference(args):
265 |     """
266 |     Run inference on ActivityNet QA DataSet using the Video-ChatGPT model.
267 | 
268 |     Args:
269 |         args: Command-line arguments.
270 |     """
271 |     init_distributed_mode(args)
272 |     # Initialize the model
273 |     video_token_len = args.video_token_len
274 |     model, vision_tower, tokenizer, image_processor = initialize_model(
275 |         args.model_name, args.projection_path, args.rank, args.visual_model
276 |     )
277 |     model.config.use_visual_mask = args.use_visual_mask
278 |     model.config.qformer_type = args.qformer_type
279 |     # Load both ground truth file containing questions and answers
280 |     with open(args.gt_file) as file:
281 |         gt_contents = json.load(file)
282 | 
283 |     # Create the output directory if it doesn't exist
284 |     if not os.path.exists(args.output_dir):
285 |         os.makedirs(args.output_dir)
286 | 
287 |     output_list = []  # List to store the output results
288 |     conv_mode = args.conv_mode
289 | 
290 |     video_formats = [".mp4", ".avi", ".mov", ".mkv"]
291 | 
292 |     # Iterate over each sample in the ground truth file
293 |     index = 0
294 |     gt_contents = gt_contents[args.rank :: args.world_size]
295 |     for sample in tqdm(gt_contents):
296 |         video_name = sample["video_name"]
297 |         sample_set = sample
298 |         question = sample["Q"]
299 | 
300 |         # Load the video file
301 |         for fmt in video_formats:  # Added this line
302 |             temp_path = os.path.join(args.video_dir, f"{video_name}{fmt}")
303 |             if os.path.exists(temp_path):
304 |                 video_path = temp_path
305 |                 break
306 | 
307 |         # Check if the video exists
308 |         if os.path.exists(video_path):
309 |             # num_frm = args.video_token_len // 32
310 |             num_frm = args.num_video_frame
311 |             video_frames = load_video(video_path, num_frm=num_frm)
312 | 
313 |         try:
314 |             # Run inference on the video and add the output to the list
315 |             output = video_chatgpt_infer(
316 |                 video_frames,
317 |                 question,
318 |                 conv_mode,
319 |                 model,
320 |                 vision_tower,
321 |                 tokenizer,
322 |                 image_processor,
323 |                 video_token_len,
324 |                 args.feature_dim,
325 |                 args.visual_model,
326 |             )
327 |             sample_set["pred"] = output
328 |             output_list.append(sample_set)
329 |         except Exception as e:
330 |             print(f"Error processing video file '{video_name}': {e}")
331 | 
332 |     save_result(output_list, args.output_dir, args.output_name)
333 |     # Save the output list to a JSON file
334 |     # with open(os.path.join(args.output_dir, f"{args.output_name}.json"), 'w') as file:
335 |     #     json.dump(output_list, file)
336 | 
337 | 
338 | if __name__ == "__main__":
339 |     args = parse_args()
340 |     run_inference(args)
341 | 


--------------------------------------------------------------------------------