├── LICENSE
├── README.md
├── Statement of Clarification.md
├── images
    ├── architecture.png
    └── readme.md
├── lego
    ├── LEGO.py
    ├── __init__.py
    ├── constants.py
    ├── conversation.py
    ├── mm_utils.py
    ├── model
    │   ├── builder.py
    │   └── utils.py
    ├── serve
    │   ├── __init__.py
    │   ├── cli.py
    │   ├── gradio_utils.py
    │   └── gradio_web_server.py
    ├── train
    │   ├── __pycache__
    │   │   ├── __init__.cpython-39.pyc
    │   │   ├── llama_flash_attn_monkey_patch.cpython-39.pyc
    │   │   ├── llava_trainer.cpython-39.pyc
    │   │   └── train.cpython-39.pyc
    │   ├── llama_flash_attn_monkey_patch.py
    │   ├── train.py
    │   └── train_mem.py
    └── utils.py
├── requirements.txt
├── scripts
    ├── finetune.sh
    ├── pretrain.sh
    ├── zero2.json
    └── zero3.json
└── video_llama
    ├── __init__.py
    ├── __pycache__
        └── __init__.cpython-39.pyc
    ├── common
        ├── __init__.py
        ├── __pycache__
        │   ├── __init__.cpython-39.pyc
        │   ├── dist_utils.cpython-39.pyc
        │   ├── logger.cpython-39.pyc
        │   ├── registry.cpython-39.pyc
        │   └── utils.cpython-39.pyc
        ├── config.py
        ├── dist_utils.py
        ├── gradcam.py
        ├── logger.py
        ├── optims.py
        ├── registry.py
        └── utils.py
    ├── configs
        ├── datasets
        │   ├── cc_sbu
        │   │   ├── align.yaml
        │   │   └── defaults.yaml
        │   ├── instruct
        │   │   ├── llava_instruct.yaml
        │   │   └── webvid_instruct.yaml
        │   ├── laion
        │   │   └── defaults.yaml
        │   └── webvid
        │   │   └── defaults.yaml
        ├── default.yaml
        └── models
        │   ├── minigpt4.yaml
        │   └── video_llama.yaml
    ├── conversation
        ├── __init__.py
        ├── __pycache__
        │   ├── __init__.cpython-39.pyc
        │   └── conversation_video.cpython-39.pyc
        └── conversation_video.py
    ├── datasets
        ├── __init__.py
        ├── __pycache__
        │   ├── __init__.cpython-39.pyc
        │   └── data_utils.cpython-39.pyc
        ├── builders
        │   ├── __init__.py
        │   ├── __pycache__
        │   │   ├── __init__.cpython-39.pyc
        │   │   ├── base_dataset_builder.cpython-39.pyc
        │   │   ├── image_text_pair_builder.cpython-39.pyc
        │   │   ├── instruct_builder.cpython-39.pyc
        │   │   └── video_caption_builder.cpython-39.pyc
        │   ├── base_dataset_builder.py
        │   ├── image_text_pair_builder.py
        │   ├── instruct_builder.py
        │   └── video_caption_builder.py
        ├── data_utils.py
        └── datasets
        │   ├── __init__.py
        │   ├── __pycache__
        │       ├── __init__.cpython-39.pyc
        │       ├── base_dataset.cpython-39.pyc
        │       ├── caption_datasets.cpython-39.pyc
        │       ├── cc_sbu_dataset.cpython-39.pyc
        │       ├── laion_dataset.cpython-39.pyc
        │       ├── llava_instruct_dataset.cpython-39.pyc
        │       ├── video_instruct_dataset.cpython-39.pyc
        │       └── webvid_datasets.cpython-39.pyc
        │   ├── base_dataset.py
        │   ├── caption_datasets.py
        │   ├── cc_sbu_dataset.py
        │   ├── dataloader_utils.py
        │   ├── laion_dataset.py
        │   ├── llava_instruct_dataset.py
        │   ├── video_instruct_dataset.py
        │   └── webvid_datasets.py
    ├── models
        ├── ImageBind
        │   ├── .assets
        │   │   ├── bird_audio.wav
        │   │   ├── bird_image.jpg
        │   │   ├── car_audio.wav
        │   │   ├── car_image.jpg
        │   │   ├── dog_audio.wav
        │   │   └── dog_image.jpg
        │   ├── CODE_OF_CONDUCT.md
        │   ├── CONTRIBUTING.md
        │   ├── LICENSE
        │   ├── README.md
        │   ├── __pycache__
        │   │   └── data.cpython-39.pyc
        │   ├── bpe
        │   │   └── bpe_simple_vocab_16e6.txt.gz
        │   ├── data.py
        │   ├── model_card.md
        │   ├── models
        │   │   ├── __init__.py
        │   │   ├── __pycache__
        │   │   │   ├── __init__.cpython-39.pyc
        │   │   │   ├── helpers.cpython-39.pyc
        │   │   │   ├── imagebind_model.cpython-39.pyc
        │   │   │   ├── multimodal_preprocessors.cpython-39.pyc
        │   │   │   └── transformer.cpython-39.pyc
        │   │   ├── helpers.py
        │   │   ├── imagebind_model.py
        │   │   ├── multimodal_preprocessors.py
        │   │   └── transformer.py
        │   └── requirements.txt
        ├── Qformer.py
        ├── __init__.py
        ├── __pycache__
        │   ├── Qformer.cpython-39.pyc
        │   ├── __init__.cpython-39.pyc
        │   ├── base_model.cpython-39.pyc
        │   ├── blip2.cpython-39.pyc
        │   ├── eva_vit.cpython-39.pyc
        │   ├── modeling_llama.cpython-39.pyc
        │   └── video_llama.cpython-39.pyc
        ├── base_model.py
        ├── blip2.py
        ├── blip2_outputs.py
        ├── eva_vit.py
        ├── modeling_llama.py
        └── video_llama.py
    ├── processors
        ├── .ipynb_checkpoints
        │   └── video_processor-checkpoint.py
        ├── __init__.py
        ├── __pycache__
        │   ├── __init__.cpython-39.pyc
        │   ├── base_processor.cpython-39.pyc
        │   ├── blip_processors.cpython-39.pyc
        │   ├── functional_video.cpython-39.pyc
        │   ├── randaugment.cpython-39.pyc
        │   ├── transforms_video.cpython-39.pyc
        │   └── video_processor.cpython-39.pyc
        ├── base_processor.py
        ├── blip_processors.py
        ├── functional_video.py
        ├── randaugment.py
        ├── transforms_video.py
        └── video_processor.py
    ├── runners
        ├── __init__.py
        ├── runner_base.py
        └── test.py
    └── tasks
        ├── __init__.py
        ├── __pycache__
            ├── __init__.cpython-39.pyc
            ├── base_task.cpython-39.pyc
            ├── image_text_pretrain.cpython-39.pyc
            └── video_text_pretrain.cpython-39.pyc
        ├── base_task.py
        ├── image_text_pretrain.py
        └── video_text_pretrain.py


/README.md:
--------------------------------------------------------------------------------
 1 | # GroundingGPT: Language-Enhanced Multi-modal Grounding Model
 2 | 
 3 | <a href='https://lzw-lzw.github.io/GroundingGPT.github.io/'><img src='https://img.shields.io/badge/Project-Page-Green'></a>  <a href='https://arxiv.org/abs/2401.06071'><img src='https://img.shields.io/badge/Paper-Arxiv-red'></a> [![](https://img.shields.io/badge/Datasets-GroundingGPT-yellow)](https://huggingface.co/datasets/zwli/GroundingGPT) [![](https://img.shields.io/badge/Models-GroundingGPT-yellow)](https://huggingface.co/zwli/GroundingGPT)   
 4 | 
 5 | 
 6 | ## Introduction
 7 | GroundingGPT is an end-to-end multimodal grounding model that accurately comprehends inputs and possesses robust grounding capabilities across multi modalities,including images, audios, and videos. To address the issue of limited data, we construct a diverse and high-quality multimodal training dataset. This dataset encompasses a rich collection of multimodal data enriched with spatial and temporal information, thereby serving as a valuable resource to foster further advancements in this field. Extensive experimental evaluations validate the effectiveness of the GroundingGPT model in understanding and grounding tasks across various modalities. 
 8 | 
 9 | More details are available in our [project page](https://lzw-lzw.github.io/GroundingGPT.github.io/). 
10 | 
11 | <p align="center">
12 |     <img src="images/architecture.png" width="80%"> <br>
13 |     The overall structure of GroundingGPT. Blue boxes represent video as input, while yellow boxes represent image as input.
14 | </p>
15 | 
16 | ## News
17 | * **[2024.5]**  Our paper is accepted to ACL 2024!
18 | * **[2024.4]**  Our [model](https://huggingface.co/zwli/GroundingGPT) is available now!
19 | * **[2024.3]**  Our [training dataset](https://huggingface.co/datasets/zwli/GroundingGPT) are available now!
20 | * **[2024.3]**  Our code are available now! 
21 | 
22 | ## Dependencies and Installation
23 |         git clone https://github.com/lzw-lzw/GroundingGPT.git
24 |         cd GroundingGPT
25 |         conda create -n groundinggpt python=3.10 -y
26 |         conda activate groundinggpt
27 |         pip install -r requirements.txt 
28 |         pip install flash-attn --no-build-isolation
29 | 
30 | 
31 | ## Training
32 | ### Training model preparation
33 | - Put the prepared checkpoints in directory `./ckpt`.
34 | - Prepare ImageBind checkpoint: download [imagebind_huge.pth](https://dl.fbaipublicfiles.com/imagebind/imagebind_huge.pth) in link and put it under directory `./ckpt/imagebind`.
35 | - Prepare blip2 checkpoint: download [blip2_pretrained_flant5xxl.pth](https://storage.googleapis.com/sfr-vision-language-research/LAVIS/models/BLIP2/blip2_pretrained_flant5xxl.pth) in link and put it under directory `./ckpt`.
36 |   
37 | ### Training dataset preparation
38 | - Please put the prepared checkpoints in file `dataset`.
39 | - Prepare LLaVA, COCO, GQA, OCR-VQA, TextVQA, VisualGenome datasets: follow [LLaVA](https://github.com/haotian-liu/LLaVA).
40 | - Prepare Flickr30K-Entities datasets: follow [Flickr30K-Entities](https://bryanplummer.com/Flickr30kEntities/).
41 | - Prepare Valley datasets: follow [Valley](https://github.com/RupertLuo/Valley).
42 | - Prepare DiDeMO datasets: follow [DiDeMO](https://github.com/LisaAnne/TemporalLanguageRelease).
43 | - Prepare ActivityNet Captions datasets: follow [ActivityNet Captions](https://cs.stanford.edu/people/ranjaykrishna/densevid/).
44 | - Prepare Charades-STA datasets: follow [Charades-STA](https://github.com/jiyanggao/TALL).
45 | - Prepare VGGSS datasets: follow [VGGSS](https://www.robots.ox.ac.uk/~vgg/research/lvs/).
46 | - Prepare WaveCaps datasets: follow [WaveCaps](https://github.com/XinhaoMei/WavCaps).
47 | - Prepare Clotho datasets: follow [Clotho](https://zenodo.org/records/3490684).
48 | 
49 | 
50 | ### Training
51 | 
52 | -
53 | 
54 | ## Inference
55 | 
56 | - Download [GroundingGPT-7B](https://huggingface.co/zwli/GroundingGPT) and change the model_path in `GroundingGPT/lego/serve/cli.py`
57 | - Use the script to inference
58 | 
59 |         python3 lego/serve/cli.py
60 | 
61 | 
62 | ## Demo
63 | - Download [GroundingGPT-7B](https://huggingface.co/zwli/GroundingGPT) and change the model_path in line 141 of `GroundingGPT/lego/serve/gradio_web_server.py`
64 | - Use the script to launch a gradio web demo
65 | 
66 |         python3 lego/serve/gradio_web_server.py
67 | 
68 | 
69 | ## Acknowledgement
70 | - [LLaVA](https://github.com/haotian-liu/LLaVA)
71 | - [Video-LLaMA](https://github.com/DAMO-NLP-SG/Video-LLaMA)
72 | - [Shikra](https://github.com/shikras/shikra)
73 | 
74 | ### Citation
75 | If you find GroundingGPT useful for your your research and applications, please cite using this BibTeX:
76 |     
77 |     @inproceedings{li2024groundinggpt,
78 |       title={Groundinggpt: Language enhanced multi-modal grounding model},
79 |       author={Li, Zhaowei and Xu, Qi and Zhang, Dong and Song, Hang and Cai, Yiqing and Qi, Qi and Zhou, Ran and Pan, Junting and Li, Zefeng and Tu, Vu and others},
80 |       booktitle={Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
81 |       pages={6657--6678},
82 |       year={2024}
83 |     }
84 | 


--------------------------------------------------------------------------------
/Statement of Clarification.md:
--------------------------------------------------------------------------------
1 | We hereby clarify that the Language Enhanced Multi-modal Grounding Model (formerly referred to as a LEGO Language Model), which has been modified to GroundingGPT, is in no way associated with or endorsed by the LEGO Group. There is no investment, collaboration, or any other form of relationship between the LEGO Group and our model previously using the LEGO name. We kindly request that any media or third-party entities that have published or disseminated inaccurate or misleading reports regarding this model promptly correct or remove the misinformation. Your immediate attention to this matter would be greatly appreciated. We deeply apologize for any confusion, inconvenience, or harm caused by these misconducts to the LEGO Group.
2 | 


--------------------------------------------------------------------------------
/images/architecture.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/images/architecture.png


--------------------------------------------------------------------------------
/images/readme.md:
--------------------------------------------------------------------------------
1 | 
2 | 


--------------------------------------------------------------------------------
/lego/__init__.py:
--------------------------------------------------------------------------------
1 | from .LEGO import LEGOLlamaForCausalLM
2 | from .train import *


--------------------------------------------------------------------------------
/lego/constants.py:
--------------------------------------------------------------------------------
 1 | CONTROLLER_HEART_BEAT_EXPIRATION = 30
 2 | WORKER_HEART_BEAT_INTERVAL = 15
 3 | 
 4 | LOGDIR = "."
 5 | 
 6 | # Model Constants
 7 | IGNORE_INDEX = -100
 8 | IMAGE_TOKEN_INDEX = -200
 9 | AUDIO_TOKEN_INDEX = -200
10 | DEFAULT_IMAGE_TOKEN = "<image>"
11 | DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
12 | DEFAULT_IMAGE_START_TOKEN = "<im_start>"
13 | DEFAULT_IMAGE_END_TOKEN = "<im_end>"
14 | DEFAULT_VIDEO_TOKEN = "<video>"
15 | DEFAULT_VIDEO_PATCH_TOKEN = "<vi_patch>"
16 | DEFAULT_VIDEO_START_TOKEN = "<vi_start>"
17 | DEFAULT_VIDEO_END_TOKEN = "<vi_end>"
18 | DEFAULT_SOUND_TOKEN = "<sound>"
19 | DEFAULT_SOUND_PATCH_TOKEN = "<so_patch>"
20 | DEFAULT_SOUND_START_TOKEN = "<so_start>"
21 | DEFAULT_SOUND_END_TOKEN = "<so_end>"
22 | 
23 | 
24 | 


--------------------------------------------------------------------------------
/lego/mm_utils.py:
--------------------------------------------------------------------------------
  1 | from PIL import Image
  2 | from io import BytesIO
  3 | import base64
  4 | import torch
  5 | import re
  6 | import os 
  7 | import tempfile
  8 | import matplotlib.pyplot as plt
  9 | import matplotlib.patches as patches
 10 | import numpy as np
 11 | from transformers import StoppingCriteria
 12 | from lego.constants import IMAGE_TOKEN_INDEX
 13 | 
 14 | colors = ["#0000FF","#FF0000","#00FF00","#FFFF00","#00FFFF","#FF00FF","#800080","#FFA500","#008000","#A52A2A","#FFC0CB","#00CED1","#8B008B","#FFD700","#7FFFD4","#FF4500","#2E8B57","#800000","#8A2BE2","#FF1493"]
 15 | 
 16 | def load_image_from_base64(image):
 17 |     return Image.open(BytesIO(base64.b64decode(image)))
 18 | 
 19 | def load_image_square(image_file, image_processor,image_aspect_ratio='pad'):
 20 |     image = Image.open(image_file).convert('RGB')
 21 |     if image_aspect_ratio == 'pad':
 22 |         def expand2square(pil_img, background_color):
 23 |             width, height = pil_img.size
 24 |             if width == height:
 25 |                 return pil_img
 26 |             elif width > height:
 27 |                 result = Image.new(pil_img.mode, (width, width), background_color)
 28 |                 result.paste(pil_img, (0, (width - height) // 2))
 29 |                 return result
 30 |             else:
 31 |                 result = Image.new(pil_img.mode, (height, height), background_color)
 32 |                 result.paste(pil_img, ((height - width) // 2, 0))
 33 |                 return result
 34 |         image = expand2square(image, tuple(int(x*255) for x in image_processor.image_mean))
 35 |     return image
 36 | 
 37 | def postprocess_box(box, ori_w, ori_h):
 38 |     if ori_w == ori_h:
 39 |         return box
 40 |     if ori_w > ori_h:
 41 |         x1, y1, x2, y2 = box
 42 |         y1 -= (ori_w - ori_h) // 2
 43 |         y2 -= (ori_w - ori_h) // 2
 44 |         box = x1, y1, x2, y2
 45 |         return box
 46 |     x1, y1, x2, y2 = box
 47 |     x1 -= (ori_h - ori_w) // 2
 48 |     x2 -= (ori_h - ori_w) // 2
 49 |     box = x1, y1, x2, y2
 50 |     return box
 51 | 
 52 | def postprocess_output(outputs, image_file):
 53 |     if image_file is None:
 54 |         return
 55 |     image = Image.open(image_file).convert('RGB')
 56 |     ori_width, ori_height = image.size
 57 |     max_ori = max(ori_width, ori_height)
 58 |     regex = re.compile(r'\[[0-9.]+, [0-9.]+, [0-9.]+, [0-9.]+\]')
 59 |     matches = re.findall(regex, outputs)
 60 |     new_outputs = outputs
 61 |     for match in matches:
 62 |         try:
 63 |             x1, y1, x2, y2 = [float(part.strip()) for part in match.strip('[]').split(',')]
 64 |             square_bbox = round(x1 * max_ori), round(y1 * max_ori), round(x2 * max_ori), round(y2 * max_ori)
 65 |             ori_box = postprocess_box(square_bbox, ori_width, ori_height)
 66 |             new_x1, new_y1, new_x2, new_y2 = [round(ori_box[0]/ori_width,3), round(ori_box[1]/ori_height,3), round(ori_box[2]/ori_width,3), round(ori_box[3]/ori_height,3)]
 67 |             new_x1 = np.clip(new_x1, 0, 1)
 68 |             new_y1 = np.clip(new_y1, 0, 1)
 69 |             new_x2 = np.clip(new_x2, 0, 1)
 70 |             new_y2 = np.clip(new_y2, 0, 1)
 71 |             new_bbox_string = f'[{new_x1}, {new_y1}, {new_x2}, {new_y2}]'
 72 |             new_outputs = new_outputs.replace(match, new_bbox_string)
 73 |         except:
 74 |             pass
 75 |     return new_outputs
 76 | 
 77 | def postprocess_image_answer_gradio(description, image_path):
 78 |     coordinates_groups = []
 79 |     start_index = description.find("[")
 80 |     tmp_index = start_index
 81 |     while start_index != -1:
 82 |         end_index = description.find("]", tmp_index)
 83 |         if end_index != -1:
 84 |             if description[end_index+1]=='[':
 85 |                 tmp_index = end_index+1
 86 |                 continue
 87 |             tmp_index = end_index
 88 |             coordinates_groups.append(description[start_index: tmp_index+1])
 89 |             start_index = description.find("[", end_index)
 90 |             tmp_index = start_index
 91 |         else:
 92 |             break
 93 |     image = Image.open(image_path)
 94 |     fig, ax = plt.subplots()
 95 |     ax.axis("off")
 96 |     ax.imshow(image)
 97 |     width, height = image.size
 98 |     filename = os.path.join('temp', next(tempfile._get_candidate_names()) + '.jpg')
 99 |     for i,coordinates_group in enumerate(coordinates_groups):
100 |         color = colors[i]
101 |         replace_string = f'[{i}]'
102 |         description = description.replace(coordinates_group, f"<span style='color: {color};'>{replace_string}</span>")
103 |         pattern = r'\[[^\[\]]+\]'
104 |         matches = re.findall(pattern, coordinates_group)
105 |         for coord in matches:
106 |             coord = [float(c) for c in coord[1:-1].split(",")]
107 |             x_min, y_min, x_max, y_max = coord
108 |             x_min_abs, y_min_abs = x_min * width, y_min * height
109 |             x_max_abs, y_max_abs = x_max * width, y_max * height
110 |             width_abs = x_max_abs - x_min_abs
111 |             height_abs = y_max_abs - y_min_abs
112 |             rect = patches.Rectangle(
113 |                 (x_min_abs, y_min_abs),
114 |                 width_abs,
115 |                 height_abs,
116 |                 linewidth=3,
117 |                 edgecolor=color,
118 |                 facecolor="none",
119 |             )
120 |             ax.add_patch(rect)
121 |     plt.savefig(filename, bbox_inches="tight", pad_inches=0)
122 |     plt.clf()
123 |     plt.close()
124 |     return description,filename
125 | 
126 | def process_images(images, image_processor, model_cfg):
127 |     return image_processor(images, return_tensors='pt')['pixel_values']
128 | 
129 | 
130 | def tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):
131 |     prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]
132 | 
133 |     def insert_separator(X, sep):
134 |         return [ele for sublist in zip(X, [sep]*len(X)) for ele in sublist][:-1]
135 | 
136 |     input_ids = []
137 |     offset = 0
138 |     if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id:
139 |         offset = 1
140 |         input_ids.append(prompt_chunks[0][0])
141 | 
142 |     for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)):
143 |         input_ids.extend(x[offset:])
144 | 
145 |     if return_tensors is not None:
146 |         if return_tensors == 'pt':
147 |             return torch.tensor(input_ids, dtype=torch.long)
148 |         raise ValueError(f'Unsupported tensor type: {return_tensors}')
149 |     return input_ids
150 | 
151 | 
152 | def get_model_name_from_path(model_path):
153 |     model_path = model_path.strip("/")
154 |     model_paths = model_path.split("/")
155 |     if model_paths[-1].startswith('checkpoint-'):
156 |         return model_paths[-2] + "_" + model_paths[-1]
157 |     else:
158 |         return model_paths[-1]
159 | 
160 | class KeywordsStoppingCriteria(StoppingCriteria):
161 |     def __init__(self, keywords, tokenizer, input_ids):
162 |         self.keywords = keywords
163 |         self.keyword_ids = []
164 |         for keyword in keywords:
165 |             cur_keyword_ids = tokenizer(keyword).input_ids
166 |             if len(cur_keyword_ids) > 1 and cur_keyword_ids[0] == tokenizer.bos_token_id:
167 |                 cur_keyword_ids = cur_keyword_ids[1:]
168 |             self.keyword_ids.append(torch.tensor(cur_keyword_ids))
169 |         self.tokenizer = tokenizer
170 |         self.start_len = input_ids.shape[1]
171 | 
172 |     def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
173 |         assert output_ids.shape[0] == 1, "Only support batch size 1 (yet)"  # TODO
174 |         offset = min(output_ids.shape[1] - self.start_len, 3)
175 |         self.keyword_ids = [keyword_id.to(output_ids.device) for keyword_id in self.keyword_ids]
176 |         for keyword_id in self.keyword_ids:
177 |             if output_ids[0, -keyword_id.shape[0]:] == keyword_id:
178 |                 return True
179 |         outputs = self.tokenizer.batch_decode(output_ids[:, -offset:], skip_special_tokens=True)[0]
180 |         for keyword in self.keywords:
181 |             if keyword in outputs:
182 |                 return True
183 |         return False
184 | 


--------------------------------------------------------------------------------
/lego/model/builder.py:
--------------------------------------------------------------------------------
 1 | #    Copyright 2023 Haotian Liu
 2 | #
 3 | #    Licensed under the Apache License, Version 2.0 (the "License");
 4 | #    you may not use this file except in compliance with the License.
 5 | #    You may obtain a copy of the License at
 6 | #
 7 | #        http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | #    Unless required by applicable law or agreed to in writing, software
10 | #    distributed under the License is distributed on an "AS IS" BASIS,
11 | #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | #    See the License for the specific language governing permissions and
13 | #    limitations under the License.
14 | 
15 | 
16 | import os
17 | from transformers import AutoTokenizer
18 | import torch
19 | from lego.model import *
20 | from lego import LEGOLlamaForCausalLM
21 | from transformers import CLIPVisionModel, CLIPImageProcessor, AutoTokenizer
22 | from video_llama.processors.video_processor import load_video
23 | from video_llama.processors import AlproVideoTrainProcessor
24 | from video_llama.models.ImageBind.data import load_and_transform_audio_data
25 | 
26 | DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
27 | DEFAULT_IMAGE_START_TOKEN = "<im_start>"
28 | DEFAULT_IMAGE_END_TOKEN = "<im_end>"
29 | DEFAULT_VIDEO_PATCH_TOKEN = "<vi_patch>"
30 | DEFAULT_VIDEO_START_TOKEN = "<vi_start>"
31 | DEFAULT_VIDEO_END_TOKEN = "<vi_end>"
32 | DEFAULT_SOUND_PATCH_TOKEN = "<so_patch>"
33 | DEFAULT_SOUND_START_TOKEN = "<so_start>"
34 | DEFAULT_SOUND_END_TOKEN = "<so_end>"
35 | 
36 | class Setting:
37 |     def __init__(self):
38 |         self.device = os.environ.get("LEGO_DEVICE", "cuda")
39 |         # self.llasm_context_len = 2048
40 |         self.sampling_rate = 16000
41 |         self.image_token_len = 576
42 |         self.video_token_len = 64
43 |         self.sound_token_len = 8
44 |         self.stop = "</s>"
45 | CONFIG = Setting()
46 | 
47 | def load_pretrained_model(model_path, load_8bit=False, load_4bit=False, device_map="auto"):
48 |     model = LEGOLlamaForCausalLM.from_pretrained(model_path,torch_dtype=torch.bfloat16).to(CONFIG.device)
49 |     tokenizer = AutoTokenizer.from_pretrained(model_path)
50 |     tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
51 |     tokenizer.add_tokens([DEFAULT_IMAGE_START_TOKEN, DEFAULT_IMAGE_END_TOKEN], special_tokens=True)
52 |     tokenizer.add_tokens([DEFAULT_VIDEO_PATCH_TOKEN], special_tokens=True)
53 |     tokenizer.add_tokens([DEFAULT_VIDEO_START_TOKEN, DEFAULT_VIDEO_END_TOKEN], special_tokens=True)
54 |     tokenizer.add_tokens([DEFAULT_SOUND_PATCH_TOKEN], special_tokens=True)
55 |     tokenizer.add_tokens([DEFAULT_SOUND_START_TOKEN, DEFAULT_SOUND_END_TOKEN], special_tokens=True)
56 |     image_processor = CLIPImageProcessor.from_pretrained('openai/clip-vit-large-patch14-336' ,torch_dtype=torch.bfloat16)
57 |     image_tower = CLIPVisionModel.from_pretrained('openai/clip-vit-large-patch14-336',torch_dtype=torch.bfloat16,low_cpu_mem_usage=True).to(CONFIG.device)
58 |     image_config = image_tower.config
59 |     image_config.im_patch_token = tokenizer.convert_tokens_to_ids([DEFAULT_IMAGE_PATCH_TOKEN])[0]
60 |     image_config.im_start_token, image_config.im_end_token = tokenizer.convert_tokens_to_ids([DEFAULT_IMAGE_START_TOKEN, DEFAULT_IMAGE_END_TOKEN])
61 |     image_config.use_im_start_end = True
62 |     model.get_model().vision_tower[0] = image_tower
63 |     
64 |     video_transform = AlproVideoTrainProcessor(image_size=224, n_frms = model.config.max_frame).transform
65 |     if hasattr(model.config, "max_sequence_length"):
66 |         context_len = model.config.max_sequence_length
67 |     else:
68 |         context_len = 2048
69 |     return model, tokenizer, image_processor, video_transform, context_len
70 | 


--------------------------------------------------------------------------------
/lego/model/utils.py:
--------------------------------------------------------------------------------
 1 | from transformers import AutoConfig
 2 | 
 3 | 
 4 | def auto_upgrade(config):
 5 |     cfg = AutoConfig.from_pretrained(config)
 6 |     if 'llava' in config and 'llava' not in cfg.model_type:
 7 |         assert cfg.model_type == 'llama'
 8 |         print("You are using newer LLaVA code base, while the checkpoint of v0 is from older code base.")
 9 |         print("You must upgrade the checkpoint to the new code base (this can be done automatically).")
10 |         confirm = input("Please confirm that you want to upgrade the checkpoint. [Y/N]")
11 |         if confirm.lower() in ["y", "yes"]:
12 |             print("Upgrading checkpoint...")
13 |             assert len(cfg.architectures) == 1
14 |             setattr(cfg.__class__, "model_type", "llava")
15 |             cfg.architectures[0] = 'LlavaLlamaForCausalLM'
16 |             cfg.save_pretrained(config)
17 |             print("Checkpoint upgraded.")
18 |         else:
19 |             print("Checkpoint upgrade aborted.")
20 |             exit(1)
21 | 


--------------------------------------------------------------------------------
/lego/serve/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/lego/serve/__init__.py


--------------------------------------------------------------------------------
/lego/serve/cli.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import torch
  3 | from lego.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_VIDEO_TOKEN, DEFAULT_SOUND_TOKEN
  4 | from lego.conversation import SeparatorStyle
  5 | from lego import conversation as conversation_lib
  6 | from lego.mm_utils import tokenizer_image_token, KeywordsStoppingCriteria, load_image_square, postprocess_output
  7 | from lego.model.builder import CONFIG, load_pretrained_model
  8 | from video_llama.processors.video_processor import load_video
  9 | from video_llama.models.ImageBind.data import load_and_transform_audio_data
 10 | from lego.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IMAGE_START_TOKEN, DEFAULT_IMAGE_END_TOKEN, DEFAULT_VIDEO_PATCH_TOKEN, \
 11 |                            DEFAULT_VIDEO_START_TOKEN, DEFAULT_VIDEO_END_TOKEN, DEFAULT_SOUND_PATCH_TOKEN, DEFAULT_SOUND_START_TOKEN, DEFAULT_SOUND_END_TOKEN
 12 | 
 13 | def main(args):
 14 |     model, tokenizer, image_processor, video_transform, context_len = load_pretrained_model(args.model_path)
 15 |     conv = conversation_lib.default_conversation.copy()
 16 |     roles = conv.roles
 17 |     image_path = None
 18 |     image_tensor = None
 19 |     video_tensor = None
 20 |     sound_tensor = None
 21 |     while True:
 22 |         image = None
 23 |         video = None
 24 |         sound = None
 25 |         try:
 26 |             inp = input(f"{roles[0]}: ")
 27 |         except EOFError:
 28 |             inp = ""
 29 |         if inp == 'change image':
 30 |             while True:
 31 |                 try :
 32 |                     image_path = input('Please input new image path:')
 33 |                     image = load_image_square(image_path,image_processor)
 34 |                     image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values'].half().cuda()
 35 |                     conv = conversation_lib.default_conversation.copy()
 36 |                     roles = conv.roles
 37 |                     inp = input(f"{roles[0]}: ")
 38 |                     break
 39 |                 except:
 40 |                     print('Please input a correct image path.')
 41 |                     continue
 42 |         if inp == 'change video':
 43 |             while True:
 44 |                 try :
 45 |                     video_path = input('Please input new video path:')
 46 |                     video = load_video(
 47 |                             video_path = video_path,
 48 |                             n_frms = model.config.max_frame,
 49 |                             height = 224,
 50 |                             width = 224,
 51 |                             sampling ="uniform", return_msg = False)
 52 |                     video_tensor = video_transform(video).unsqueeze(0).to(CONFIG.device, dtype=torch.bfloat16)
 53 |                     conv = conversation_lib.default_conversation.copy()
 54 |                     roles = conv.roles
 55 |                     inp = input(f"{roles[0]}: ")
 56 |                     break
 57 |                 except:
 58 |                     print('Please input a correct video path.')
 59 |                     continue
 60 |         if inp == 'change sound':
 61 |             while True:
 62 |                 try :
 63 |                     sound_path = input('Please input new sound path:')
 64 |                     sound = load_and_transform_audio_data([sound_path],device="cpu").to(CONFIG.device, dtype=torch.bfloat16)
 65 |                     sound_tensor = sound
 66 |                     conv = conversation_lib.default_conversation.copy()
 67 |                     roles = conv.roles
 68 |                     inp = input(f"{roles[0]}: ")
 69 |                     break
 70 |                 except:
 71 |                     print('Please input a correct sound path.')
 72 |                     continue
 73 | 
 74 |         if not inp:
 75 |             print("exit...")
 76 |             break
 77 |         print(f"{roles[1]}: ", end="")
 78 |         if image is not None:
 79 |             if model.config.mm_use_im_start_end:
 80 |                 inp = DEFAULT_IMAGE_START_TOKEN + DEFAULT_IMAGE_PATCH_TOKEN * CONFIG.image_token_len + DEFAULT_IMAGE_END_TOKEN + '\n' + inp
 81 |             else:
 82 |                 inp = DEFAULT_IMAGE_TOKEN + '\n' + inp
 83 |             conv.append_message(conv.roles[0], inp)
 84 |             image = None
 85 |         elif video is not None:
 86 |             if model.config.mm_use_im_start_end:
 87 |                 inp = DEFAULT_VIDEO_START_TOKEN + DEFAULT_VIDEO_PATCH_TOKEN * CONFIG.video_token_len + DEFAULT_VIDEO_END_TOKEN + '\n' + inp
 88 |             else:
 89 |                 inp = DEFAULT_VIDEO_TOKEN + '\n' + inp
 90 |             conv.append_message(conv.roles[0], inp)
 91 |             video = None
 92 |         elif sound is not None:
 93 |             if model.config.mm_use_im_start_end:
 94 |                 inp = DEFAULT_SOUND_START_TOKEN + DEFAULT_SOUND_PATCH_TOKEN * CONFIG.sound_token_len + DEFAULT_SOUND_END_TOKEN + '\n' + inp
 95 |             else:
 96 |                 inp = DEFAULT_SOUND_TOKEN + '\n' + inp
 97 |             conv.append_message(conv.roles[0], inp)
 98 |             sound = None
 99 |         else:
100 |             conv.append_message(conv.roles[0], inp)
101 | 
102 |         
103 |         conv.append_message(conv.roles[1], None)
104 |         prompt = conv.get_prompt()
105 |         input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
106 |         stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
107 |         keywords = [stop_str]
108 |         stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
109 | 
110 |         with torch.inference_mode():
111 |             output_ids = model.generate(
112 |                 input_ids,
113 |                 images=image_tensor,
114 |                 videos=video_tensor,
115 |                 sounds=sound_tensor,
116 |                 do_sample=True,
117 |                 temperature=args.temperature,
118 |                 max_new_tokens=args.max_new_tokens, 
119 |                 use_cache=True,
120 |                 stopping_criteria=[stopping_criteria])
121 |         
122 |         outputs = tokenizer.decode(output_ids[0, input_ids.shape[1]:]).strip()
123 |         if image_path is not None:
124 |             outputs = postprocess_output(outputs, image_path)
125 |         if outputs.endswith(stop_str):
126 |             outputs = outputs[:-len(stop_str)]
127 |         print(outputs)
128 |         conv.messages[-1][-1] = outputs
129 | 
130 |         if args.debug:
131 |             print("\n", {"prompt": prompt, "outputs": outputs}, "\n")
132 | 
133 | 
134 | if __name__ == "__main__":
135 |     parser = argparse.ArgumentParser()
136 |     parser.add_argument("--model_path", type=str, default="/ckpt/LEGO-7B")
137 |     parser.add_argument("--image_file", type=str, default=None)
138 |     parser.add_argument("--video_file", type=str, default=None)
139 |     parser.add_argument("--sound_file", type=str, default=None)
140 |     parser.add_argument("--temperature", type=float, default=0.2)
141 |     parser.add_argument("--max_new_tokens", type=int, default=512)
142 |     parser.add_argument("--debug", action="store_true")
143 |     args = parser.parse_args()
144 |     main(args)


--------------------------------------------------------------------------------
/lego/serve/gradio_utils.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | 
  3 | from lego.conversation import conv_templates, SeparatorStyle
  4 | from lego.mm_utils import get_model_name_from_path, KeywordsStoppingCriteria
  5 | from lego.model.builder import load_pretrained_model
  6 | from lego.utils import disable_torch_init
  7 | from lego.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
  8 | IMAGE_TOKEN_INDEX = -200
  9 | title_markdown = ("""
 10 | <div style="display: flex; justify-content: center; align-items: center; text-align: center;">
 11 |   <div>
 12 |     <h1 >LEGO:Language-Enhanced Grounding Multi-modal Large Model</h1>
 13 |     <h5 style="margin: 0;">If you like our project, please give us a star ✨ on Github for the latest update.</h5>
 14 |   </div>
 15 | </div>
 16 | """)
 17 | 
 18 | block_css = """
 19 | #buttons button {
 20 |     min-width: min(120px,100%);
 21 | }
 22 | """
 23 | 
 24 | 
 25 | tos_markdown = ("""
 26 | ### Terms of use
 27 | By using this service, users are required to agree to the following terms:
 28 | The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.
 29 | Please click the "Flag" button if you get any inappropriate answer! We will collect those to keep improving our moderator.
 30 | For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
 31 | """)
 32 | 
 33 | 
 34 | learn_more_markdown = ("""
 35 | ### License
 36 | The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) of LLaMA, [Terms of Use](https://openai.com/policies/terms-of-use) of the data generated by OpenAI, and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.
 37 | """)
 38 | 
 39 | colors = ["#0000FF","#FF0000","#00FF00","#FFFF00","#00FFFF","#FF00FF","#800080","#FFA500","#008000","#A52A2A","#FFC0CB","#00CED1","#8B008B","#FFD700","#7FFFD4","#FF4500","#2E8B57","#800000","#8A2BE2","#FF1493"]
 40 | 
 41 | class Chat:
 42 |     def __init__(self, model_path, conv_mode='default', model_base=None, load_8bit=False, load_4bit=False, device='cuda'):
 43 |         disable_torch_init()
 44 |         self.model, self.tokenizer, image_processor, video_transform, context_len = load_pretrained_model(model_path)
 45 |         self.image_processor = image_processor
 46 |         self.video_transform = video_transform
 47 |         self.conv_mode = conv_mode
 48 |         self.device = self.model.device
 49 |         print(self.model)
 50 | 
 51 |     def get_prompt(self, qs, state):
 52 |         state.append_message(state.roles[0], qs)
 53 |         state.append_message(state.roles[1], None)
 54 |         return state
 55 | 
 56 |     @torch.inference_mode()
 57 |     def generate(self, images_tensor: list, prompt: str, first_run: bool, state):
 58 |         tokenizer, model, image_processor, video_transform = self.tokenizer, self.model, self.image_processor, self.video_transform
 59 | 
 60 |         state = self.get_prompt(prompt, state)
 61 |         prompt = state.get_prompt()
 62 |         print('\n\n\n')
 63 |         print(prompt)
 64 |         input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
 65 |         image_tensor = None
 66 |         video_tensor = None
 67 |         sound_tensor = None
 68 |         for (tensor,modality) in zip(images_tensor[0],images_tensor[1]):
 69 |             if modality=='image':
 70 |                 image_tensor=tensor
 71 |             if modality=='video':
 72 |                 video_tensor=tensor
 73 |             if modality=='sound':
 74 |                 sound_tensor=tensor
 75 |               
 76 |         stop_str = state.sep if state.sep_style != SeparatorStyle.TWO else state.sep2
 77 |         keywords = [stop_str]
 78 |         stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
 79 |         # streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
 80 |         # print(input_ids, images_tensor[0][0].shape)
 81 |         with torch.inference_mode():
 82 |             output_ids = model.generate(
 83 |                 input_ids,
 84 |                 images=image_tensor,
 85 |                 videos=video_tensor,
 86 |                 sounds=sound_tensor,
 87 |                 do_sample=True,
 88 |                 temperature=0.2,
 89 |                 max_new_tokens=1024,
 90 |                 # streamer=streamer,
 91 |                 use_cache=True,
 92 |                 stopping_criteria=[stopping_criteria])
 93 | 
 94 |         input_token_len = input_ids.shape[1]
 95 |         n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
 96 |         if n_diff_input_output > 0:
 97 |             print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
 98 |         outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
 99 |         outputs = outputs.strip()
100 |         if outputs.endswith(stop_str):
101 |             outputs = outputs[:-len(stop_str)]
102 |         outputs = outputs.strip()
103 | 
104 |         print('response', outputs)
105 |         return outputs, state
106 | 


--------------------------------------------------------------------------------
/lego/train/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/lego/train/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/lego/train/__pycache__/llama_flash_attn_monkey_patch.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/lego/train/__pycache__/llama_flash_attn_monkey_patch.cpython-39.pyc


--------------------------------------------------------------------------------
/lego/train/__pycache__/llava_trainer.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/lego/train/__pycache__/llava_trainer.cpython-39.pyc


--------------------------------------------------------------------------------
/lego/train/__pycache__/train.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/lego/train/__pycache__/train.cpython-39.pyc


--------------------------------------------------------------------------------
/lego/train/llama_flash_attn_monkey_patch.py:
--------------------------------------------------------------------------------
  1 | import logging
  2 | from typing import List, Optional, Tuple
  3 | 
  4 | import torch
  5 | import transformers
  6 | from einops import rearrange
  7 | from torch import nn
  8 | from transformers.models.llama.modeling_llama import apply_rotary_pos_emb
  9 | 
 10 | try:
 11 |     from flash_attn.flash_attn_interface import \
 12 |         flash_attn_unpadded_qkvpacked_func
 13 | except ImportError:
 14 |     from flash_attn.flash_attn_interface import (
 15 |         flash_attn_varlen_qkvpacked_func as flash_attn_unpadded_qkvpacked_func,
 16 |     )
 17 | 
 18 | from flash_attn.bert_padding import pad_input, unpad_input
 19 | 
 20 | 
 21 | def forward(
 22 |     self,
 23 |     hidden_states: torch.Tensor,
 24 |     attention_mask: Optional[torch.Tensor] = None,
 25 |     position_ids: Optional[torch.Tensor] = None,
 26 |     past_key_value: Optional[Tuple[torch.Tensor]] = None,
 27 |     output_attentions: bool = False,
 28 |     use_cache: bool = False,
 29 | ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
 30 |     """Input shape: Batch x Time x Channel
 31 | 
 32 |     attention_mask: [bsz, q_len]
 33 |     """
 34 |     bsz, q_len, _ = hidden_states.size()
 35 | 
 36 |     query_states = (
 37 |         self.q_proj(hidden_states)
 38 |         .view(bsz, q_len, self.num_heads, self.head_dim)
 39 |         .transpose(1, 2)
 40 |     )
 41 |     key_states = (
 42 |         self.k_proj(hidden_states)
 43 |         .view(bsz, q_len, self.num_heads, self.head_dim)
 44 |         .transpose(1, 2)
 45 |     )
 46 |     value_states = (
 47 |         self.v_proj(hidden_states)
 48 |         .view(bsz, q_len, self.num_heads, self.head_dim)
 49 |         .transpose(1, 2)
 50 |     )
 51 |     # [bsz, q_len, nh, hd]
 52 |     # [bsz, nh, q_len, hd]
 53 | 
 54 |     kv_seq_len = key_states.shape[-2]
 55 |     assert past_key_value is None, "past_key_value is not supported"
 56 | 
 57 |     cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
 58 |     query_states, key_states = apply_rotary_pos_emb(
 59 |         query_states, key_states, cos, sin, position_ids
 60 |     )
 61 |     # [bsz, nh, t, hd]
 62 |     assert not output_attentions, "output_attentions is not supported"
 63 |     assert not use_cache, "use_cache is not supported"
 64 | 
 65 |     # Flash attention codes from
 66 |     # https://github.com/HazyResearch/flash-attention/blob/main/flash_attn/flash_attention.py
 67 | 
 68 |     # transform the data into the format required by flash attention
 69 |     qkv = torch.stack(
 70 |         [query_states, key_states, value_states], dim=2
 71 |     )  # [bsz, nh, 3, q_len, hd]
 72 |     qkv = qkv.transpose(1, 3)  # [bsz, q_len, 3, nh, hd]
 73 |     # We have disabled _prepare_decoder_attention_mask in LlamaModel
 74 |     # the attention_mask should be the same as the key_padding_mask
 75 |     key_padding_mask = attention_mask
 76 | 
 77 |     if key_padding_mask is None:
 78 |         qkv = rearrange(qkv, "b s ... -> (b s) ...")
 79 |         max_s = q_len
 80 |         cu_q_lens = torch.arange(
 81 |             0, (bsz + 1) * q_len, step=q_len, dtype=torch.int32, device=qkv.device
 82 |         )
 83 |         output = flash_attn_unpadded_qkvpacked_func(
 84 |             qkv, cu_q_lens, max_s, 0.0, softmax_scale=None, causal=True
 85 |         )
 86 |         output = rearrange(output, "(b s) ... -> b s ...", b=bsz)
 87 |     else:
 88 |         nheads = qkv.shape[-2]
 89 |         x = rearrange(qkv, "b s three h d -> b s (three h d)")
 90 |         x_unpad, indices, cu_q_lens, max_s = unpad_input(x, key_padding_mask)
 91 |         x_unpad = rearrange(
 92 |             x_unpad, "nnz (three h d) -> nnz three h d", three=3, h=nheads
 93 |         )
 94 |         output_unpad = flash_attn_unpadded_qkvpacked_func(
 95 |             x_unpad, cu_q_lens, max_s, 0.0, softmax_scale=None, causal=True
 96 |         )
 97 |         output = rearrange(
 98 |             pad_input(
 99 |                 rearrange(output_unpad, "nnz h d -> nnz (h d)"), indices, bsz, q_len
100 |             ),
101 |             "b s (h d) -> b s h d",
102 |             h=nheads,
103 |         )
104 |     return self.o_proj(rearrange(output, "b s h d -> b s (h d)")), None, None
105 | 
106 | 
107 | # Disable the transformation of the attention mask in LlamaModel as the flash attention
108 | # requires the attention mask to be the same as the key_padding_mask
109 | def _prepare_decoder_attention_mask(
110 |     self, attention_mask, input_shape, inputs_embeds, past_key_values_length
111 | ):
112 |     # [bsz, seq_len]
113 |     return attention_mask
114 | 
115 | 
116 | def replace_llama_attn_with_flash_attn():
117 |     cuda_major, cuda_minor = torch.cuda.get_device_capability()
118 |     if cuda_major < 8:
119 |         logging.warning(
120 |             "Flash attention is only supported on A100 or H100 GPU during training due to head dim > 64 backward."
121 |             "ref: https://github.com/HazyResearch/flash-attention/issues/190#issuecomment-1523359593"
122 |         )
123 |     transformers.models.llama.modeling_llama.LlamaModel._prepare_decoder_attention_mask = (
124 |         _prepare_decoder_attention_mask
125 |     )
126 |     transformers.models.llama.modeling_llama.LlamaAttention.forward = forward
127 | 


--------------------------------------------------------------------------------
/lego/train/train_mem.py:
--------------------------------------------------------------------------------
 1 | # Adopted from https://github.com/lm-sys/FastChat. Below is the original copyright:
 2 | # Adopted from tatsu-lab@stanford_alpaca. Below is the original copyright:
 3 | # Make it more memory efficient by monkey patching the LLaMA model with FlashAttn.
 4 | 
 5 | # Need to call this before importing transformers.
 6 | from lego.train.llama_flash_attn_monkey_patch import \
 7 |     replace_llama_attn_with_flash_attn
 8 | 
 9 | replace_llama_attn_with_flash_attn()
10 | 
11 | from lego.train.train import train
12 | 
13 | if __name__ == "__main__":
14 |     train()
15 | 


--------------------------------------------------------------------------------
/lego/utils.py:
--------------------------------------------------------------------------------
  1 | import datetime
  2 | import logging
  3 | import logging.handlers
  4 | import os
  5 | import sys
  6 | 
  7 | import requests
  8 | 
  9 | from lego.constants import LOGDIR
 10 | 
 11 | server_error_msg = "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
 12 | moderation_msg = "YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES. PLEASE TRY AGAIN."
 13 | 
 14 | handler = None
 15 | 
 16 | 
 17 | def build_logger(logger_name, logger_filename):
 18 |     global handler
 19 | 
 20 |     formatter = logging.Formatter(
 21 |         fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
 22 |         datefmt="%Y-%m-%d %H:%M:%S",
 23 |     )
 24 | 
 25 |     # Set the format of root handlers
 26 |     if not logging.getLogger().handlers:
 27 |         logging.basicConfig(level=logging.INFO)
 28 |     logging.getLogger().handlers[0].setFormatter(formatter)
 29 | 
 30 |     # Redirect stdout and stderr to loggers
 31 |     stdout_logger = logging.getLogger("stdout")
 32 |     stdout_logger.setLevel(logging.INFO)
 33 |     sl = StreamToLogger(stdout_logger, logging.INFO)
 34 |     sys.stdout = sl
 35 | 
 36 |     stderr_logger = logging.getLogger("stderr")
 37 |     stderr_logger.setLevel(logging.ERROR)
 38 |     sl = StreamToLogger(stderr_logger, logging.ERROR)
 39 |     sys.stderr = sl
 40 | 
 41 |     # Get logger
 42 |     logger = logging.getLogger(logger_name)
 43 |     logger.setLevel(logging.INFO)
 44 | 
 45 |     # Add a file handler for all loggers
 46 |     if handler is None:
 47 |         os.makedirs(LOGDIR, exist_ok=True)
 48 |         filename = os.path.join(LOGDIR, logger_filename)
 49 |         handler = logging.handlers.TimedRotatingFileHandler(
 50 |             filename, when='D', utc=True)
 51 |         handler.setFormatter(formatter)
 52 | 
 53 |         for name, item in logging.root.manager.loggerDict.items():
 54 |             if isinstance(item, logging.Logger):
 55 |                 item.addHandler(handler)
 56 | 
 57 |     return logger
 58 | 
 59 | 
 60 | class StreamToLogger(object):
 61 |     """
 62 |     Fake file-like stream object that redirects writes to a logger instance.
 63 |     """
 64 |     def __init__(self, logger, log_level=logging.INFO):
 65 |         self.terminal = sys.stdout
 66 |         self.logger = logger
 67 |         self.log_level = log_level
 68 |         self.linebuf = ''
 69 | 
 70 |     def __getattr__(self, attr):
 71 |         return getattr(self.terminal, attr)
 72 | 
 73 |     def write(self, buf):
 74 |         temp_linebuf = self.linebuf + buf
 75 |         self.linebuf = ''
 76 |         for line in temp_linebuf.splitlines(True):
 77 |             # From the io.TextIOWrapper docs:
 78 |             #   On output, if newline is None, any '\n' characters written
 79 |             #   are translated to the system default line separator.
 80 |             # By default sys.stdout.write() expects '\n' newlines and then
 81 |             # translates them so this is still cross platform.
 82 |             if line[-1] == '\n':
 83 |                 self.logger.log(self.log_level, line.rstrip())
 84 |             else:
 85 |                 self.linebuf += line
 86 | 
 87 |     def flush(self):
 88 |         if self.linebuf != '':
 89 |             self.logger.log(self.log_level, self.linebuf.rstrip())
 90 |         self.linebuf = ''
 91 | 
 92 | 
 93 | def disable_torch_init():
 94 |     """
 95 |     Disable the redundant torch default initialization to accelerate model creation.
 96 |     """
 97 |     import torch
 98 |     setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
 99 |     setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
100 | 
101 | 
102 | def violates_moderation(text):
103 |     """
104 |     Check whether the text violates OpenAI moderation API.
105 |     """
106 |     url = "https://api.openai.com/v1/moderations"
107 |     headers = {"Content-Type": "application/json",
108 |                "Authorization": "Bearer " + os.environ["OPENAI_API_KEY"]}
109 |     text = text.replace("\n", "")
110 |     data = "{" + '"input": ' + f'"{text}"' + "}"
111 |     data = data.encode("utf-8")
112 |     try:
113 |         ret = requests.post(url, headers=headers, data=data, timeout=5)
114 |         flagged = ret.json()["results"][0]["flagged"]
115 |     except requests.exceptions.RequestException as e:
116 |         flagged = False
117 |     except KeyError as e:
118 |         flagged = False
119 | 
120 |     return flagged
121 | 
122 | 
123 | def pretty_print_semaphore(semaphore):
124 |     if semaphore is None:
125 |         return "None"
126 |     return f"Semaphore(value={semaphore._value}, locked={semaphore.locked()})"
127 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | ninja
 2 | peft
 3 | decord
 4 | einops
 5 | scikit-image
 6 | accelerate
 7 | deepspeed
 8 | prettytable
 9 | fastapi
10 | torchvision
11 | gradio==3.23
12 | markdown2[all]
13 | numpy
14 | librosa
15 | wandb
16 | requests
17 | sentencepiece
18 | tokenizers==0.14
19 | uvicorn
20 | transformers==4.34.0
21 | httpx==0.23.0
22 | scikit-image
23 | timm
24 | iopath
25 | omegaconf
26 | decord
27 | einops
28 | ftfy
29 | pytorchvideo
30 | opencv-python
31 | flash-attn
32 | pycocotools
33 | huggingface_hub


--------------------------------------------------------------------------------
/scripts/finetune.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | deepspeed --master_port your_port\
 4 |     lego/train/train.py \
 5 |     --deepspeed ./scripts/zero2.json \
 6 |     --model_name_or_path path_to_your_pretrained_model\
 7 |     --data_path path_to_your_dataset\
 8 |     --image_folder path_to_your_image_folder\
 9 |     --video_folder  path_to_your_video_folder\
10 |     --sound_folder  path_to_your_sound_folder\
11 |     --tune_mm_mlp_adapter False\
12 |     --vision_tower openai/clip-vit-large-patch14-336\
13 |     --mm_vision_select_layer -2\
14 |     --mm_use_im_start_end True\
15 |     --image_aspect_ratio pad\
16 |     --bf16 True\
17 |     --fp16 False\
18 |     --output_dir path_to_your_output_dir\
19 |     --num_train_epochs 1\
20 |     --per_device_train_batch_size 32\
21 |     --per_device_eval_batch_size 4\
22 |     --gradient_accumulation_steps 2\
23 |     --evaluation_strategy "no"\
24 |     --save_strategy "steps"\
25 |     --save_steps 500\
26 |     --save_total_limit 2\
27 |     --learning_rate 2e-5\
28 |     --weight_decay 0.\
29 |     --warmup_ratio 0.03\
30 |     --lr_scheduler_type cosine\
31 |     --logging_steps 1\
32 |     --tf32 True\
33 |     --model_max_length 2048\
34 |     --gradient_checkpointing True\
35 |     --lazy_preprocess True\
36 |     --report_to wandb


--------------------------------------------------------------------------------
/scripts/pretrain.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | deepspeed --master_port your_port\
 4 |     lego/train/train.py\
 5 |     --deepspeed ./scripts/zero2.json\
 6 |     --model_name_or_path lmsys/vicuna-7b-v1.5 \
 7 |     --data_path path_to_your_dataset\
 8 |     --image_folder path_to_your_image_folder\
 9 |     --video_folder  path_to_your_video_folder\
10 |     --sound_folder  path_to_your_sound_folder\
11 |     --tune_mm_mlp_adapter True\
12 |     --vision_tower openai/clip-vit-large-patch14-336\
13 |     --mm_vision_select_layer -2\
14 |     --mm_use_im_start_end True\
15 |     --image_aspect_ratio pad\
16 |     --bf16 True\
17 |     --fp16 False\
18 |     --output_dir  path_to_your_output_dir\
19 |     --num_train_epochs 1\
20 |     --per_device_train_batch_size 64\
21 |     --per_device_eval_batch_size 4\
22 |     --gradient_accumulation_steps 1\
23 |     --evaluation_strategy "no"\
24 |     --save_strategy "steps"\
25 |     --save_steps 500\
26 |     --save_total_limit 2\
27 |     --learning_rate 1e-3\
28 |     --weight_decay 0.\
29 |     --warmup_ratio 0.03\
30 |     --lr_scheduler_type cosine\
31 |     --logging_steps 1\
32 |     --tf32 True\
33 |     --model_max_length 2048\
34 |     --gradient_checkpointing True\
35 |     --lazy_preprocess True\
36 |     --report_to wandb


--------------------------------------------------------------------------------
/scripts/zero2.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "fp16": {
 3 |         "enabled": "auto",
 4 |         "loss_scale": 0,
 5 |         "loss_scale_window": 1000,
 6 |         "initial_scale_power": 16,
 7 |         "hysteresis": 2,
 8 |         "min_loss_scale": 1
 9 |     },
10 |     "bf16": {
11 |         "enabled": "auto"
12 |     },
13 |     "train_micro_batch_size_per_gpu": "auto",
14 |     "train_batch_size": "auto",
15 |     "gradient_accumulation_steps": "auto",
16 |     "zero_optimization": {
17 |         "stage": 2,
18 |         "overlap_comm": true,
19 |         "contiguous_gradients": true,
20 |         "sub_group_size": 1e9,
21 |         "reduce_bucket_size": "auto"
22 |     }
23 | }


--------------------------------------------------------------------------------
/scripts/zero3.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "fp16": {
 3 |         "enabled": "auto",
 4 |         "loss_scale": 0,
 5 |         "loss_scale_window": 1000,
 6 |         "initial_scale_power": 16,
 7 |         "hysteresis": 2,
 8 |         "min_loss_scale": 1
 9 |     },
10 |     "bf16": {
11 |         "enabled": "auto"
12 |     },
13 |     "train_micro_batch_size_per_gpu": "auto",
14 |     "train_batch_size": "auto",
15 |     "gradient_accumulation_steps": "auto",
16 |     "zero_optimization": {
17 |         "stage": 3,
18 |         "overlap_comm": true,
19 |         "contiguous_gradients": true,
20 |         "sub_group_size": 1e9,
21 |         "reduce_bucket_size": "auto",
22 |         "stage3_prefetch_bucket_size": "auto",
23 |         "stage3_param_persistence_threshold": "auto",
24 |         "stage3_max_live_parameters": 1e9,
25 |         "stage3_max_reuse_distance": 1e9,
26 |         "stage3_gather_16bit_weights_on_model_save": true
27 |     }
28 | }


--------------------------------------------------------------------------------
/video_llama/__init__.py:
--------------------------------------------------------------------------------
 1 | """
 2 |  Copyright (c) 2022, salesforce.com, inc.
 3 |  All rights reserved.
 4 |  SPDX-License-Identifier: BSD-3-Clause
 5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 6 | """
 7 | 
 8 | import os
 9 | import sys
10 | 
11 | from omegaconf import OmegaConf
12 | 
13 | from video_llama.common.registry import registry
14 | 
15 | from video_llama.datasets.builders import *
16 | from video_llama.models import *
17 | from video_llama.processors import *
18 | from video_llama.tasks import *
19 | 
20 | 
21 | root_dir = os.path.dirname(os.path.abspath(__file__))
22 | default_cfg = OmegaConf.load(os.path.join(root_dir, "configs/default.yaml"))
23 | 
24 | registry.register_path("library_root", root_dir)
25 | repo_root = os.path.join(root_dir, "..")
26 | registry.register_path("repo_root", repo_root)
27 | cache_root = os.path.join(repo_root, default_cfg.env.cache_root)
28 | registry.register_path("cache_root", cache_root)
29 | 
30 | registry.register("MAX_INT", sys.maxsize)
31 | registry.register("SPLIT_NAMES", ["train", "val", "test"])
32 | 


--------------------------------------------------------------------------------
/video_llama/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/common/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/common/__init__.py


--------------------------------------------------------------------------------
/video_llama/common/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/common/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/common/__pycache__/dist_utils.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/common/__pycache__/dist_utils.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/common/__pycache__/logger.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/common/__pycache__/logger.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/common/__pycache__/registry.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/common/__pycache__/registry.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/common/__pycache__/utils.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/common/__pycache__/utils.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/common/dist_utils.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import datetime
  9 | import functools
 10 | import os
 11 | 
 12 | import torch
 13 | import torch.distributed as dist
 14 | import timm.models.hub as timm_hub
 15 | 
 16 | 
 17 | def setup_for_distributed(is_master):
 18 |     """
 19 |     This function disables printing when not in master process
 20 |     """
 21 |     import builtins as __builtin__
 22 | 
 23 |     builtin_print = __builtin__.print
 24 | 
 25 |     def print(*args, **kwargs):
 26 |         force = kwargs.pop("force", False)
 27 |         if is_master or force:
 28 |             builtin_print(*args, **kwargs)
 29 | 
 30 |     __builtin__.print = print
 31 | 
 32 | 
 33 | def is_dist_avail_and_initialized():
 34 |     if not dist.is_available():
 35 |         return False
 36 |     if not dist.is_initialized():
 37 |         return False
 38 |     return True
 39 | 
 40 | 
 41 | def get_world_size():
 42 |     if not is_dist_avail_and_initialized():
 43 |         return 1
 44 |     return dist.get_world_size()
 45 | 
 46 | 
 47 | def get_rank():
 48 |     if not is_dist_avail_and_initialized():
 49 |         return 0
 50 |     return dist.get_rank()
 51 | 
 52 | 
 53 | def is_main_process():
 54 |     return get_rank() == 0
 55 | 
 56 | 
 57 | def init_distributed_mode(args):
 58 |     if "RANK" in os.environ and "WORLD_SIZE" in os.environ:
 59 |         args.rank = int(os.environ["RANK"])
 60 |         args.world_size = int(os.environ["WORLD_SIZE"])
 61 |         args.gpu = int(os.environ["LOCAL_RANK"])
 62 |     elif "SLURM_PROCID" in os.environ:
 63 |         args.rank = int(os.environ["SLURM_PROCID"])
 64 |         args.gpu = args.rank % torch.cuda.device_count()
 65 |     else:
 66 |         print("Not using distributed mode")
 67 |         args.distributed = False
 68 |         return
 69 | 
 70 |     args.distributed = True
 71 | 
 72 |     torch.cuda.set_device(args.gpu)
 73 |     args.dist_backend = "nccl"
 74 |     print(
 75 |         "| distributed init (rank {}, world {}): {}".format(
 76 |             args.rank, args.world_size, args.dist_url
 77 |         ),
 78 |         flush=True,
 79 |     )
 80 |     torch.distributed.init_process_group(
 81 |         backend=args.dist_backend,
 82 |         init_method=args.dist_url,
 83 |         world_size=args.world_size,
 84 |         rank=args.rank,
 85 |         timeout=datetime.timedelta(
 86 |             days=365
 87 |         ),  # allow auto-downloading and de-compressing
 88 |     )
 89 |     torch.distributed.barrier()
 90 |     setup_for_distributed(args.rank == 0)
 91 | 
 92 | 
 93 | def get_dist_info():
 94 |     if torch.__version__ < "1.0":
 95 |         initialized = dist._initialized
 96 |     else:
 97 |         initialized = dist.is_initialized()
 98 |     if initialized:
 99 |         rank = dist.get_rank()
100 |         world_size = dist.get_world_size()
101 |     else:  # non-distributed training
102 |         rank = 0
103 |         world_size = 1
104 |     return rank, world_size
105 | 
106 | 
107 | def main_process(func):
108 |     @functools.wraps(func)
109 |     def wrapper(*args, **kwargs):
110 |         rank, _ = get_dist_info()
111 |         if rank == 0:
112 |             return func(*args, **kwargs)
113 | 
114 |     return wrapper
115 | 
116 | 
117 | def download_cached_file(url, check_hash=True, progress=False):
118 |     """
119 |     Download a file from a URL and cache it locally. If the file already exists, it is not downloaded again.
120 |     If distributed, only the main process downloads the file, and the other processes wait for the file to be downloaded.
121 |     """
122 | 
123 |     def get_cached_file_path():
124 |         # a hack to sync the file path across processes
125 |         parts = torch.hub.urlparse(url)
126 |         filename = os.path.basename(parts.path)
127 |         cached_file = os.path.join(timm_hub.get_cache_dir(), filename)
128 | 
129 |         return cached_file
130 | 
131 |     if is_main_process():
132 |         timm_hub.download_cached_file(url, check_hash, progress)
133 | 
134 |     if is_dist_avail_and_initialized():
135 |         dist.barrier()
136 | 
137 |     return get_cached_file_path()
138 | 


--------------------------------------------------------------------------------
/video_llama/common/gradcam.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | from matplotlib import pyplot as plt
 3 | from scipy.ndimage import filters
 4 | from skimage import transform as skimage_transform
 5 | 
 6 | 
 7 | def getAttMap(img, attMap, blur=True, overlap=True):
 8 |     attMap -= attMap.min()
 9 |     if attMap.max() > 0:
10 |         attMap /= attMap.max()
11 |     attMap = skimage_transform.resize(attMap, (img.shape[:2]), order=3, mode="constant")
12 |     if blur:
13 |         attMap = filters.gaussian_filter(attMap, 0.02 * max(img.shape[:2]))
14 |         attMap -= attMap.min()
15 |         attMap /= attMap.max()
16 |     cmap = plt.get_cmap("jet")
17 |     attMapV = cmap(attMap)
18 |     attMapV = np.delete(attMapV, 3, 2)
19 |     if overlap:
20 |         attMap = (
21 |             1 * (1 - attMap**0.7).reshape(attMap.shape + (1,)) * img
22 |             + (attMap**0.7).reshape(attMap.shape + (1,)) * attMapV
23 |         )
24 |     return attMap
25 | 


--------------------------------------------------------------------------------
/video_llama/common/logger.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import datetime
  9 | import logging
 10 | import time
 11 | from collections import defaultdict, deque
 12 | 
 13 | import torch
 14 | import torch.distributed as dist
 15 | 
 16 | from video_llama.common import dist_utils
 17 | 
 18 | 
 19 | class SmoothedValue(object):
 20 |     """Track a series of values and provide access to smoothed values over a
 21 |     window or the global series average.
 22 |     """
 23 | 
 24 |     def __init__(self, window_size=20, fmt=None):
 25 |         if fmt is None:
 26 |             fmt = "{median:.4f} ({global_avg:.4f})"
 27 |         self.deque = deque(maxlen=window_size)
 28 |         self.total = 0.0
 29 |         self.count = 0
 30 |         self.fmt = fmt
 31 | 
 32 |     def update(self, value, n=1):
 33 |         self.deque.append(value)
 34 |         self.count += n
 35 |         self.total += value * n
 36 | 
 37 |     def synchronize_between_processes(self):
 38 |         """
 39 |         Warning: does not synchronize the deque!
 40 |         """
 41 |         if not dist_utils.is_dist_avail_and_initialized():
 42 |             return
 43 |         t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda")
 44 |         dist.barrier()
 45 |         dist.all_reduce(t)
 46 |         t = t.tolist()
 47 |         self.count = int(t[0])
 48 |         self.total = t[1]
 49 | 
 50 |     @property
 51 |     def median(self):
 52 |         d = torch.tensor(list(self.deque))
 53 |         return d.median().item()
 54 | 
 55 |     @property
 56 |     def avg(self):
 57 |         d = torch.tensor(list(self.deque), dtype=torch.float32)
 58 |         return d.mean().item()
 59 | 
 60 |     @property
 61 |     def global_avg(self):
 62 |         return self.total / self.count
 63 | 
 64 |     @property
 65 |     def max(self):
 66 |         return max(self.deque)
 67 | 
 68 |     @property
 69 |     def value(self):
 70 |         return self.deque[-1]
 71 | 
 72 |     def __str__(self):
 73 |         return self.fmt.format(
 74 |             median=self.median,
 75 |             avg=self.avg,
 76 |             global_avg=self.global_avg,
 77 |             max=self.max,
 78 |             value=self.value,
 79 |         )
 80 | 
 81 | 
 82 | class MetricLogger(object):
 83 |     def __init__(self, delimiter="\t"):
 84 |         self.meters = defaultdict(SmoothedValue)
 85 |         self.delimiter = delimiter
 86 | 
 87 |     def update(self, **kwargs):
 88 |         for k, v in kwargs.items():
 89 |             if isinstance(v, torch.Tensor):
 90 |                 v = v.item()
 91 |             assert isinstance(v, (float, int))
 92 |             self.meters[k].update(v)
 93 | 
 94 |     def __getattr__(self, attr):
 95 |         if attr in self.meters:
 96 |             return self.meters[attr]
 97 |         if attr in self.__dict__:
 98 |             return self.__dict__[attr]
 99 |         raise AttributeError(
100 |             "'{}' object has no attribute '{}'".format(type(self).__name__, attr)
101 |         )
102 | 
103 |     def __str__(self):
104 |         loss_str = []
105 |         for name, meter in self.meters.items():
106 |             loss_str.append("{}: {}".format(name, str(meter)))
107 |         return self.delimiter.join(loss_str)
108 | 
109 |     def global_avg(self):
110 |         loss_str = []
111 |         for name, meter in self.meters.items():
112 |             loss_str.append("{}: {:.4f}".format(name, meter.global_avg))
113 |         return self.delimiter.join(loss_str)
114 | 
115 |     def synchronize_between_processes(self):
116 |         for meter in self.meters.values():
117 |             meter.synchronize_between_processes()
118 | 
119 |     def add_meter(self, name, meter):
120 |         self.meters[name] = meter
121 | 
122 |     def log_every(self, iterable, print_freq, header=None):
123 |         i = 0
124 |         if not header:
125 |             header = ""
126 |         start_time = time.time()
127 |         end = time.time()
128 |         iter_time = SmoothedValue(fmt="{avg:.4f}")
129 |         data_time = SmoothedValue(fmt="{avg:.4f}")
130 |         space_fmt = ":" + str(len(str(len(iterable)))) + "d"
131 |         log_msg = [
132 |             header,
133 |             "[{0" + space_fmt + "}/{1}]",
134 |             "eta: {eta}",
135 |             "{meters}",
136 |             "time: {time}",
137 |             "data: {data}",
138 |         ]
139 |         if torch.cuda.is_available():
140 |             log_msg.append("max mem: {memory:.0f}")
141 |         log_msg = self.delimiter.join(log_msg)
142 |         MB = 1024.0 * 1024.0
143 |         for obj in iterable:
144 |             data_time.update(time.time() - end)
145 |             yield obj
146 |             iter_time.update(time.time() - end)
147 |             if i % print_freq == 0 or i == len(iterable) - 1:
148 |                 eta_seconds = iter_time.global_avg * (len(iterable) - i)
149 |                 eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
150 |                 if torch.cuda.is_available():
151 |                     print(
152 |                         log_msg.format(
153 |                             i,
154 |                             len(iterable),
155 |                             eta=eta_string,
156 |                             meters=str(self),
157 |                             time=str(iter_time),
158 |                             data=str(data_time),
159 |                             memory=torch.cuda.max_memory_allocated() / MB,
160 |                         )
161 |                     )
162 |                 else:
163 |                     print(
164 |                         log_msg.format(
165 |                             i,
166 |                             len(iterable),
167 |                             eta=eta_string,
168 |                             meters=str(self),
169 |                             time=str(iter_time),
170 |                             data=str(data_time),
171 |                         )
172 |                     )
173 |             i += 1
174 |             end = time.time()
175 |         total_time = time.time() - start_time
176 |         total_time_str = str(datetime.timedelta(seconds=int(total_time)))
177 |         print(
178 |             "{} Total time: {} ({:.4f} s / it)".format(
179 |                 header, total_time_str, total_time / len(iterable)
180 |             )
181 |         )
182 | 
183 | 
184 | class AttrDict(dict):
185 |     def __init__(self, *args, **kwargs):
186 |         super(AttrDict, self).__init__(*args, **kwargs)
187 |         self.__dict__ = self
188 | 
189 | 
190 | def setup_logger():
191 |     logging.basicConfig(
192 |         level=logging.INFO if dist_utils.is_main_process() else logging.WARN,
193 |         format="%(asctime)s [%(levelname)s] %(message)s",
194 |         handlers=[logging.StreamHandler()],
195 |     )
196 | 


--------------------------------------------------------------------------------
/video_llama/common/optims.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import math
  9 | 
 10 | from video_llama.common.registry import registry
 11 | 
 12 | 
 13 | @registry.register_lr_scheduler("linear_warmup_step_lr")
 14 | class LinearWarmupStepLRScheduler:
 15 |     def __init__(
 16 |         self,
 17 |         optimizer,
 18 |         max_epoch,
 19 |         min_lr,
 20 |         init_lr,
 21 |         decay_rate=1,
 22 |         warmup_start_lr=-1,
 23 |         warmup_steps=0,
 24 |         **kwargs
 25 |     ):
 26 |         self.optimizer = optimizer
 27 | 
 28 |         self.max_epoch = max_epoch
 29 |         self.min_lr = min_lr
 30 | 
 31 |         self.decay_rate = decay_rate
 32 | 
 33 |         self.init_lr = init_lr
 34 |         self.warmup_steps = warmup_steps
 35 |         self.warmup_start_lr = warmup_start_lr if warmup_start_lr >= 0 else init_lr
 36 | 
 37 |     def step(self, cur_epoch, cur_step):
 38 |         if cur_epoch == 0:
 39 |             warmup_lr_schedule(
 40 |                 step=cur_step,
 41 |                 optimizer=self.optimizer,
 42 |                 max_step=self.warmup_steps,
 43 |                 init_lr=self.warmup_start_lr,
 44 |                 max_lr=self.init_lr,
 45 |             )
 46 |         else:
 47 |             step_lr_schedule(
 48 |                 epoch=cur_epoch,
 49 |                 optimizer=self.optimizer,
 50 |                 init_lr=self.init_lr,
 51 |                 min_lr=self.min_lr,
 52 |                 decay_rate=self.decay_rate,
 53 |             )
 54 | 
 55 | 
 56 | @registry.register_lr_scheduler("linear_warmup_cosine_lr")
 57 | class LinearWarmupCosineLRScheduler:
 58 |     def __init__(
 59 |         self,
 60 |         optimizer,
 61 |         max_epoch,
 62 |         iters_per_epoch,
 63 |         min_lr,
 64 |         init_lr,
 65 |         warmup_steps=0,
 66 |         warmup_start_lr=-1,
 67 |         **kwargs
 68 |     ):
 69 |         self.optimizer = optimizer
 70 | 
 71 |         self.max_epoch = max_epoch
 72 |         self.iters_per_epoch = iters_per_epoch
 73 |         self.min_lr = min_lr
 74 | 
 75 |         self.init_lr = init_lr
 76 |         self.warmup_steps = warmup_steps
 77 |         self.warmup_start_lr = warmup_start_lr if warmup_start_lr >= 0 else init_lr
 78 | 
 79 |     def step(self, cur_epoch, cur_step):
 80 |         total_cur_step = cur_epoch * self.iters_per_epoch + cur_step
 81 |         if total_cur_step < self.warmup_steps:
 82 |             warmup_lr_schedule(
 83 |                 step=cur_step,
 84 |                 optimizer=self.optimizer,
 85 |                 max_step=self.warmup_steps,
 86 |                 init_lr=self.warmup_start_lr,
 87 |                 max_lr=self.init_lr,
 88 |             )
 89 |         else:
 90 |             cosine_lr_schedule(
 91 |                 epoch=total_cur_step,
 92 |                 optimizer=self.optimizer,
 93 |                 max_epoch=self.max_epoch * self.iters_per_epoch,
 94 |                 init_lr=self.init_lr,
 95 |                 min_lr=self.min_lr,
 96 |             )
 97 | 
 98 | 
 99 | def cosine_lr_schedule(optimizer, epoch, max_epoch, init_lr, min_lr):
100 |     """Decay the learning rate"""
101 |     lr = (init_lr - min_lr) * 0.5 * (
102 |         1.0 + math.cos(math.pi * epoch / max_epoch)
103 |     ) + min_lr
104 |     for param_group in optimizer.param_groups:
105 |         param_group["lr"] = lr
106 | 
107 | 
108 | def warmup_lr_schedule(optimizer, step, max_step, init_lr, max_lr):
109 |     """Warmup the learning rate"""
110 |     lr = min(max_lr, init_lr + (max_lr - init_lr) * step / max(max_step, 1))
111 |     for param_group in optimizer.param_groups:
112 |         param_group["lr"] = lr
113 | 
114 | 
115 | def step_lr_schedule(optimizer, epoch, init_lr, min_lr, decay_rate):
116 |     """Decay the learning rate"""
117 |     lr = max(min_lr, init_lr * (decay_rate**epoch))
118 |     for param_group in optimizer.param_groups:
119 |         param_group["lr"] = lr
120 | 


--------------------------------------------------------------------------------
/video_llama/configs/datasets/cc_sbu/align.yaml:
--------------------------------------------------------------------------------
1 | datasets:
2 |   cc_sbu_align:
3 |     data_type: images
4 |     build_info:
5 |       storage: /path/to/cc_sbu_align_dataset
6 | 


--------------------------------------------------------------------------------
/video_llama/configs/datasets/cc_sbu/defaults.yaml:
--------------------------------------------------------------------------------
1 | datasets:
2 |   cc_sbu:
3 |     data_type: images
4 |     build_info:
5 |       storage: /path/to/cc_sbu_dataset/{00000..00001}.tar
6 | 


--------------------------------------------------------------------------------
/video_llama/configs/datasets/instruct/llava_instruct.yaml:
--------------------------------------------------------------------------------
1 | datasets:
2 |   llava_instruct:
3 |     data_type: image
4 |     build_info:
5 |       anno_dir: /path/llava_instruct_150k.json
6 |       videos_dir: /path/train2014/train2014/
7 | 


--------------------------------------------------------------------------------
/video_llama/configs/datasets/instruct/webvid_instruct.yaml:
--------------------------------------------------------------------------------
1 | datasets:
2 |   webvid_instruct:
3 |     data_type: image
4 |     build_info:
5 |       anno_dir: /path/webvid_align/videochat_instruct_11k.json
6 |       videos_dir: /path/webvid_align/videos/
7 | 


--------------------------------------------------------------------------------
/video_llama/configs/datasets/laion/defaults.yaml:
--------------------------------------------------------------------------------
1 | datasets:
2 |   laion:
3 |     data_type: images
4 |     build_info:
5 |       storage: path/laion/laion_dataset/{00000..00001}.tar
6 | 


--------------------------------------------------------------------------------
/video_llama/configs/datasets/webvid/defaults.yaml:
--------------------------------------------------------------------------------
1 | datasets:
2 |   webvid:
3 |     data_type: video
4 |     build_info:
5 |       anno_dir: path/webvid/webvid_tain_data/annotations/
6 |       videos_dir: path//webvid/webvid_tain_data/videos/
7 | 


--------------------------------------------------------------------------------
/video_llama/configs/default.yaml:
--------------------------------------------------------------------------------
1 | env:
2 |   # For default users
3 |   # cache_root: "cache"
4 |   # For internal use with persistent storage
5 |   cache_root: "/export/home/.cache/minigpt4"
6 | 


--------------------------------------------------------------------------------
/video_llama/configs/models/minigpt4.yaml:
--------------------------------------------------------------------------------
 1 | model:
 2 |   arch: mini_gpt4
 3 | 
 4 |   # vit encoder
 5 |   image_size: 224
 6 |   drop_path_rate: 0
 7 |   use_grad_checkpoint: False
 8 |   vit_precision: "fp16"
 9 |   freeze_vit: True
10 |   freeze_qformer: True
11 | 
12 |   # Q-Former
13 |   num_query_token: 32
14 | 
15 |   # Vicuna
16 |   llama_model: "ckpt/vicuna-13b/"
17 | 
18 |   # generation configs
19 |   prompt: ""
20 | 
21 | preprocess:
22 |     vis_processor:
23 |         train:
24 |           name: "blip2_image_train"
25 |           image_size: 224
26 |         eval:
27 |           name: "blip2_image_eval"
28 |           image_size: 224
29 |     text_processor:
30 |         train:
31 |           name: "blip_caption"
32 |         eval:
33 |           name: "blip_caption"
34 | 


--------------------------------------------------------------------------------
/video_llama/configs/models/video_llama.yaml:
--------------------------------------------------------------------------------
 1 | model:
 2 |   arch: video_llama
 3 | 
 4 |   # vit encoder
 5 |   image_size: 224
 6 |   drop_path_rate: 0
 7 |   use_grad_checkpoint: False
 8 |   vit_precision: "fp16"
 9 |   freeze_vit: True
10 |   freeze_qformer: True
11 | 
12 |   # Q-Former
13 |   num_query_token: 32
14 | 
15 |   # Vicuna
16 |   llama_model: "ckpt/vicuna-7b/"
17 | 
18 |   # generation configs
19 |   prompt: ""
20 | 
21 | preprocess:
22 |     vis_processor:
23 |         train:
24 |           name: "alpro_video_train"
25 |           image_size: 224
26 |           n_frms: 8
27 |         eval:
28 |           name: "alpro_video_eval"
29 |           image_size: 224
30 |           n_frms: 8
31 |     text_processor:
32 |         train:
33 |           name: "blip_caption"
34 |         eval:
35 |           name: "blip_caption"
36 |     


--------------------------------------------------------------------------------
/video_llama/conversation/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/conversation/__init__.py


--------------------------------------------------------------------------------
/video_llama/conversation/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/conversation/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/conversation/__pycache__/conversation_video.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/conversation/__pycache__/conversation_video.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/__init__.py


--------------------------------------------------------------------------------
/video_llama/datasets/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/__pycache__/data_utils.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/__pycache__/data_utils.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/builders/__init__.py:
--------------------------------------------------------------------------------
 1 | """
 2 |  Copyright (c) 2022, salesforce.com, inc.
 3 |  All rights reserved.
 4 |  SPDX-License-Identifier: BSD-3-Clause
 5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 6 | """
 7 | 
 8 | from video_llama.datasets.builders.base_dataset_builder import load_dataset_config
 9 | from video_llama.datasets.builders.image_text_pair_builder import (
10 |     CCSBUBuilder,
11 |     LaionBuilder,
12 |     CCSBUAlignBuilder
13 | )
14 | from video_llama.datasets.builders.video_caption_builder import WebvidBuilder
15 | from video_llama.common.registry import registry
16 | from video_llama.datasets.builders.instruct_builder import WebvidInstruct_Builder,LlavaInstruct_Builder
17 | __all__ = [
18 |     "CCSBUBuilder",
19 |     "LaionBuilder",
20 |     "CCSBUAlignBuilder",
21 |     "WebvidBuilder",
22 |     "LlavaInstruct_Builder",
23 |     "WebvidInstruct_Builder"
24 |     
25 | ]
26 | 
27 | 
28 | def load_dataset(name, cfg_path=None, vis_path=None, data_type=None):
29 |     """
30 |     Example
31 | 
32 |     >>> dataset = load_dataset("coco_caption", cfg=None)
33 |     >>> splits = dataset.keys()
34 |     >>> print([len(dataset[split]) for split in splits])
35 | 
36 |     """
37 |     if cfg_path is None:
38 |         cfg = None
39 |     else:
40 |         cfg = load_dataset_config(cfg_path)
41 | 
42 |     try:
43 |         builder = registry.get_builder_class(name)(cfg)
44 |     except TypeError:
45 |         print(
46 |             f"Dataset {name} not found. Available datasets:\n"
47 |             + ", ".join([str(k) for k in dataset_zoo.get_names()])
48 |         )
49 |         exit(1)
50 | 
51 |     if vis_path is not None:
52 |         if data_type is None:
53 |             # use default data type in the config
54 |             data_type = builder.config.data_type
55 | 
56 |         assert (
57 |             data_type in builder.config.build_info
58 |         ), f"Invalid data_type {data_type} for {name}."
59 | 
60 |         builder.config.build_info.get(data_type).storage = vis_path
61 | 
62 |     dataset = builder.build_datasets()
63 |     return dataset
64 | 
65 | 
66 | class DatasetZoo:
67 |     def __init__(self) -> None:
68 |         self.dataset_zoo = {
69 |             k: list(v.DATASET_CONFIG_DICT.keys())
70 |             for k, v in sorted(registry.mapping["builder_name_mapping"].items())
71 |         }
72 | 
73 |     def get_names(self):
74 |         return list(self.dataset_zoo.keys())
75 | 
76 | 
77 | dataset_zoo = DatasetZoo()
78 | 


--------------------------------------------------------------------------------
/video_llama/datasets/builders/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/builders/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/builders/__pycache__/base_dataset_builder.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/builders/__pycache__/base_dataset_builder.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/builders/__pycache__/image_text_pair_builder.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/builders/__pycache__/image_text_pair_builder.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/builders/__pycache__/instruct_builder.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/builders/__pycache__/instruct_builder.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/builders/__pycache__/video_caption_builder.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/builders/__pycache__/video_caption_builder.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/builders/base_dataset_builder.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  This file is from
  3 |  Copyright (c) 2022, salesforce.com, inc.
  4 |  All rights reserved.
  5 |  SPDX-License-Identifier: BSD-3-Clause
  6 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  7 | """
  8 | 
  9 | import logging
 10 | import os
 11 | import shutil
 12 | import warnings
 13 | 
 14 | from omegaconf import OmegaConf
 15 | import torch.distributed as dist
 16 | from torchvision.datasets.utils import download_url
 17 | 
 18 | import video_llama.common.utils as utils
 19 | from video_llama.common.dist_utils import is_dist_avail_and_initialized, is_main_process
 20 | from video_llama.common.registry import registry
 21 | from video_llama.processors.base_processor import BaseProcessor
 22 | 
 23 | 
 24 | 
 25 | class BaseDatasetBuilder:
 26 |     train_dataset_cls, eval_dataset_cls = None, None
 27 | 
 28 |     def __init__(self, cfg=None):
 29 |         super().__init__()
 30 | 
 31 |         if cfg is None:
 32 |             # help to create datasets from default config.
 33 |             self.config = load_dataset_config(self.default_config_path())
 34 |         elif isinstance(cfg, str):
 35 |             self.config = load_dataset_config(cfg)
 36 |         else:
 37 |             # when called from task.build_dataset()
 38 |             self.config = cfg
 39 | 
 40 |         self.data_type = self.config.data_type
 41 | 
 42 |         self.vis_processors = {"train": BaseProcessor(), "eval": BaseProcessor()}
 43 |         self.text_processors = {"train": BaseProcessor(), "eval": BaseProcessor()}
 44 | 
 45 |     def build_datasets(self):
 46 |         # download, split, etc...
 47 |         # only called on 1 GPU/TPU in distributed
 48 | 
 49 |         if is_main_process():
 50 |             self._download_data()
 51 | 
 52 |         if is_dist_avail_and_initialized():
 53 |             dist.barrier()
 54 | 
 55 |         # at this point, all the annotations and image/videos should be all downloaded to the specified locations.
 56 |         logging.info("Building datasets...")
 57 |         datasets = self.build()  # dataset['train'/'val'/'test']
 58 | 
 59 |         return datasets
 60 | 
 61 |     def build_processors(self):
 62 |         vis_proc_cfg = self.config.get("vis_processor")
 63 |         txt_proc_cfg = self.config.get("text_processor")
 64 | 
 65 |         if vis_proc_cfg is not None:
 66 |             vis_train_cfg = vis_proc_cfg.get("train")
 67 |             vis_eval_cfg = vis_proc_cfg.get("eval")
 68 | 
 69 |             self.vis_processors["train"] = self._build_proc_from_cfg(vis_train_cfg)
 70 |             self.vis_processors["eval"] = self._build_proc_from_cfg(vis_eval_cfg)
 71 | 
 72 |         if txt_proc_cfg is not None:
 73 |             txt_train_cfg = txt_proc_cfg.get("train")
 74 |             txt_eval_cfg = txt_proc_cfg.get("eval")
 75 | 
 76 |             self.text_processors["train"] = self._build_proc_from_cfg(txt_train_cfg)
 77 |             self.text_processors["eval"] = self._build_proc_from_cfg(txt_eval_cfg)
 78 | 
 79 |     @staticmethod
 80 |     def _build_proc_from_cfg(cfg):
 81 |         return (
 82 |             registry.get_processor_class(cfg.name).from_config(cfg)
 83 |             if cfg is not None
 84 |             else None
 85 |         )
 86 | 
 87 |     @classmethod
 88 |     def default_config_path(cls, type="default"):
 89 |         return utils.get_abs_path(cls.DATASET_CONFIG_DICT[type])
 90 | 
 91 |     def _download_data(self):
 92 |         self._download_ann()
 93 |         self._download_vis()
 94 | 
 95 |     def _download_ann(self):
 96 |         """
 97 |         Download annotation files if necessary.
 98 |         All the vision-language datasets should have annotations of unified format.
 99 | 
100 |         storage_path can be:
101 |           (1) relative/absolute: will be prefixed with env.cache_root to make full path if relative.
102 |           (2) basename/dirname: will be suffixed with base name of URL if dirname is provided.
103 | 
104 |         Local annotation paths should be relative.
105 |         """
106 |         anns = self.config.build_info.annotations
107 | 
108 |         splits = anns.keys()
109 | 
110 |         cache_root = registry.get_path("cache_root")
111 | 
112 |         for split in splits:
113 |             info = anns[split]
114 | 
115 |             urls, storage_paths = info.get("url", None), info.storage
116 | 
117 |             if isinstance(urls, str):
118 |                 urls = [urls]
119 |             if isinstance(storage_paths, str):
120 |                 storage_paths = [storage_paths]
121 | 
122 |             assert len(urls) == len(storage_paths)
123 | 
124 |             for url_or_filename, storage_path in zip(urls, storage_paths):
125 |                 # if storage_path is relative, make it full by prefixing with cache_root.
126 |                 if not os.path.isabs(storage_path):
127 |                     storage_path = os.path.join(cache_root, storage_path)
128 | 
129 |                 dirname = os.path.dirname(storage_path)
130 |                 if not os.path.exists(dirname):
131 |                     os.makedirs(dirname)
132 | 
133 |                 if os.path.isfile(url_or_filename):
134 |                     src, dst = url_or_filename, storage_path
135 |                     if not os.path.exists(dst):
136 |                         shutil.copyfile(src=src, dst=dst)
137 |                     else:
138 |                         logging.info("Using existing file {}.".format(dst))
139 |                 else:
140 |                     if os.path.isdir(storage_path):
141 |                         # if only dirname is provided, suffix with basename of URL.
142 |                         raise ValueError(
143 |                             "Expecting storage_path to be a file path, got directory {}".format(
144 |                                 storage_path
145 |                             )
146 |                         )
147 |                     else:
148 |                         filename = os.path.basename(storage_path)
149 | 
150 |                     download_url(url=url_or_filename, root=dirname, filename=filename)
151 | 
152 |     def _download_vis(self):
153 | 
154 |         storage_path = self.config.build_info.get(self.data_type).storage
155 |         storage_path = utils.get_cache_path(storage_path)
156 | 
157 |         if not os.path.exists(storage_path):
158 |             warnings.warn(
159 |                 f"""
160 |                 The specified path {storage_path} for visual inputs does not exist.
161 |                 Please provide a correct path to the visual inputs or
162 |                 refer to datasets/download_scripts/README.md for downloading instructions.
163 |                 """
164 |             )
165 | 
166 |     def build(self):
167 |         """
168 |         Create by split datasets inheriting torch.utils.data.Datasets.
169 | 
170 |         # build() can be dataset-specific. Overwrite to customize.
171 |         """
172 |         self.build_processors()
173 | 
174 |         build_info = self.config.build_info
175 | 
176 |         ann_info = build_info.annotations
177 |         vis_info = build_info.get(self.data_type)
178 | 
179 |         datasets = dict()
180 |         for split in ann_info.keys():
181 |             if split not in ["train", "val", "test"]:
182 |                 continue
183 | 
184 |             is_train = split == "train"
185 | 
186 |             # processors
187 |             vis_processor = (
188 |                 self.vis_processors["train"]
189 |                 if is_train
190 |                 else self.vis_processors["eval"]
191 |             )
192 |             text_processor = (
193 |                 self.text_processors["train"]
194 |                 if is_train
195 |                 else self.text_processors["eval"]
196 |             )
197 | 
198 |             # annotation path
199 |             ann_paths = ann_info.get(split).storage
200 |             if isinstance(ann_paths, str):
201 |                 ann_paths = [ann_paths]
202 | 
203 |             abs_ann_paths = []
204 |             for ann_path in ann_paths:
205 |                 if not os.path.isabs(ann_path):
206 |                     ann_path = utils.get_cache_path(ann_path)
207 |                 abs_ann_paths.append(ann_path)
208 |             ann_paths = abs_ann_paths
209 | 
210 |             # visual data storage path
211 |             vis_path = os.path.join(vis_info.storage, split)
212 | 
213 |             if not os.path.isabs(vis_path):
214 |                 # vis_path = os.path.join(utils.get_cache_path(), vis_path)
215 |                 vis_path = utils.get_cache_path(vis_path)
216 | 
217 |             if not os.path.exists(vis_path):
218 |                 warnings.warn("storage path {} does not exist.".format(vis_path))
219 | 
220 |             # create datasets
221 |             dataset_cls = self.train_dataset_cls if is_train else self.eval_dataset_cls
222 |             datasets[split] = dataset_cls(
223 |                 vis_processor=vis_processor,
224 |                 text_processor=text_processor,
225 |                 ann_paths=ann_paths,
226 |                 vis_root=vis_path,
227 |             )
228 | 
229 |         return datasets
230 | 
231 | 
232 | def load_dataset_config(cfg_path):
233 |     cfg = OmegaConf.load(cfg_path).datasets
234 |     cfg = cfg[list(cfg.keys())[0]]
235 | 
236 |     return cfg
237 | 


--------------------------------------------------------------------------------
/video_llama/datasets/builders/image_text_pair_builder.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import logging
  3 | import warnings
  4 | 
  5 | from video_llama.common.registry import registry
  6 | from video_llama.datasets.builders.base_dataset_builder import BaseDatasetBuilder
  7 | from video_llama.datasets.datasets.laion_dataset import LaionDataset
  8 | from video_llama.datasets.datasets.cc_sbu_dataset import CCSBUDataset, CCSBUAlignDataset
  9 | 
 10 | 
 11 | @registry.register_builder("cc_sbu")
 12 | class CCSBUBuilder(BaseDatasetBuilder):
 13 |     train_dataset_cls = CCSBUDataset
 14 | 
 15 |     DATASET_CONFIG_DICT = {"default": "configs/datasets/cc_sbu/defaults.yaml"}
 16 | 
 17 |     def _download_ann(self):
 18 |         pass
 19 | 
 20 |     def _download_vis(self):
 21 |         pass
 22 | 
 23 |     def build(self):
 24 |         self.build_processors()
 25 | 
 26 |         build_info = self.config.build_info
 27 | 
 28 |         datasets = dict()
 29 |         split = "train"
 30 | 
 31 |         # create datasets
 32 |         # [NOTE] return inner_datasets (wds.DataPipeline)
 33 |         dataset_cls = self.train_dataset_cls
 34 |         datasets[split] = dataset_cls(
 35 |             vis_processor=self.vis_processors[split],
 36 |             text_processor=self.text_processors[split],
 37 |             location=build_info.storage,
 38 |         ).inner_dataset
 39 | 
 40 |         return datasets
 41 | 
 42 | 
 43 | @registry.register_builder("laion")
 44 | class LaionBuilder(BaseDatasetBuilder):
 45 |     train_dataset_cls = LaionDataset
 46 | 
 47 |     DATASET_CONFIG_DICT = {"default": "configs/datasets/laion/defaults.yaml"}
 48 | 
 49 |     def _download_ann(self):
 50 |         pass
 51 | 
 52 |     def _download_vis(self):
 53 |         pass
 54 | 
 55 |     def build(self):
 56 |         self.build_processors()
 57 | 
 58 |         build_info = self.config.build_info
 59 | 
 60 |         datasets = dict()
 61 |         split = "train"
 62 | 
 63 |         # create datasets
 64 |         # [NOTE] return inner_datasets (wds.DataPipeline)
 65 |         dataset_cls = self.train_dataset_cls
 66 |         datasets[split] = dataset_cls(
 67 |             vis_processor=self.vis_processors[split],
 68 |             text_processor=self.text_processors[split],
 69 |             location=build_info.storage,
 70 |         ).inner_dataset
 71 | 
 72 |         return datasets
 73 | 
 74 | 
 75 | @registry.register_builder("cc_sbu_align")
 76 | class CCSBUAlignBuilder(BaseDatasetBuilder):
 77 |     train_dataset_cls = CCSBUAlignDataset
 78 | 
 79 |     DATASET_CONFIG_DICT = {
 80 |         "default": "configs/datasets/cc_sbu/align.yaml",
 81 |     }
 82 | 
 83 |     def build_datasets(self):
 84 |         # at this point, all the annotations and image/videos should be all downloaded to the specified locations.
 85 |         logging.info("Building datasets...")
 86 |         self.build_processors()
 87 | 
 88 |         build_info = self.config.build_info
 89 |         storage_path = build_info.storage
 90 | 
 91 |         datasets = dict()
 92 | 
 93 |         if not os.path.exists(storage_path):
 94 |             warnings.warn("storage path {} does not exist.".format(storage_path))
 95 | 
 96 |         # create datasets
 97 |         dataset_cls = self.train_dataset_cls
 98 |         datasets['train'] = dataset_cls(
 99 |             vis_processor=self.vis_processors["train"],
100 |             text_processor=self.text_processors["train"],
101 |             ann_paths=[os.path.join(storage_path, 'filter_cap.json')],
102 |             vis_root=os.path.join(storage_path, 'image'),
103 |         )
104 | 
105 |         return datasets
106 | 
107 | 


--------------------------------------------------------------------------------
/video_llama/datasets/builders/instruct_builder.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import logging
 3 | import warnings
 4 | 
 5 | from video_llama.common.registry import registry
 6 | from video_llama.datasets.builders.base_dataset_builder import BaseDatasetBuilder
 7 | from video_llama.datasets.datasets.laion_dataset import LaionDataset
 8 | from video_llama.datasets.datasets.llava_instruct_dataset import Instruct_Dataset
 9 | from video_llama.datasets.datasets.video_instruct_dataset import Video_Instruct_Dataset
10 | 
11 | @registry.register_builder("instruct")
12 | class Instruct_Builder(BaseDatasetBuilder):
13 |     train_dataset_cls = Instruct_Dataset
14 | 
15 |     DATASET_CONFIG_DICT = {"default": "configs/datasets/instruct/defaults.yaml"}
16 | 
17 |     def _download_ann(self):
18 |         pass
19 | 
20 |     def _download_vis(self):
21 |         pass
22 | 
23 |     def build(self):
24 |         self.build_processors()
25 |         datasets = dict()
26 |         split = "train"
27 | 
28 |         build_info = self.config.build_info
29 |         dataset_cls = self.train_dataset_cls
30 |         if self.config.num_video_query_token:
31 |             num_video_query_token = self.config.num_video_query_token 
32 |         else:
33 |             num_video_query_token = 32
34 | 
35 |         if self.config.tokenizer_name:
36 |             tokenizer_name = self.config.tokenizer_name 
37 |         else:
38 |             tokenizer_name = '/mnt/workspace/ckpt/vicuna-13b/'
39 | 
40 | 
41 |         datasets[split] = dataset_cls(
42 |             vis_processor=self.vis_processors[split],
43 |             text_processor=self.text_processors[split],
44 |             vis_root=build_info.videos_dir,
45 |             ann_root=build_info.anno_dir,
46 |             num_video_query_token = num_video_query_token,
47 |             tokenizer_name = tokenizer_name,
48 |             data_type = self.config.data_type,
49 |             model_type = self.config.model_type
50 |         )
51 | 
52 |         return datasets
53 | 
54 | @registry.register_builder("webvid_instruct")
55 | class WebvidInstruct_Builder(Instruct_Builder):
56 |     train_dataset_cls = Video_Instruct_Dataset
57 | 
58 |     DATASET_CONFIG_DICT = {
59 |         "default": "configs/datasets/instruct/webvid_instruct.yaml",
60 |     }
61 | 
62 | @registry.register_builder("webvid_instruct_zh")
63 | class WebvidInstruct_zh_Builder(Instruct_Builder):
64 |     train_dataset_cls = Video_Instruct_Dataset
65 | 
66 |     DATASET_CONFIG_DICT = {
67 |         "default": "configs/datasets/instruct/webvid_instruct.yaml",
68 |     }
69 | 
70 | 
71 | 
72 | @registry.register_builder("llava_instruct")
73 | class LlavaInstruct_Builder(Instruct_Builder):
74 |     train_dataset_cls = Instruct_Dataset
75 | 
76 |     DATASET_CONFIG_DICT = {
77 |         "default": "configs/datasets/instruct/llava_instruct.yaml",
78 |     }
79 | 
80 | 


--------------------------------------------------------------------------------
/video_llama/datasets/builders/video_caption_builder.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import logging
 3 | import warnings
 4 | 
 5 | from video_llama.common.registry import registry
 6 | from video_llama.datasets.builders.base_dataset_builder import BaseDatasetBuilder
 7 | from video_llama.datasets.datasets.webvid_datasets import WebvidDataset
 8 | 
 9 | @registry.register_builder("webvid")
10 | class WebvidBuilder(BaseDatasetBuilder):
11 |     train_dataset_cls = WebvidDataset
12 |     DATASET_CONFIG_DICT = {"default": "configs/datasets/webvid/defaults.yaml"}
13 |     
14 |     def _download_ann(self):
15 |         pass
16 | 
17 |     def _download_vis(self):
18 |         pass
19 | 
20 |     def build(self):
21 |         self.build_processors()
22 |         datasets = dict()
23 |         split = "train"
24 | 
25 |         build_info = self.config.build_info
26 |         dataset_cls = self.train_dataset_cls
27 |         datasets[split] = dataset_cls(
28 |             vis_processor=self.vis_processors[split],
29 |             text_processor=self.text_processors[split],
30 |             vis_root=build_info.videos_dir,
31 |             ann_root=build_info.anno_dir
32 |         )
33 | 
34 |         return datasets


--------------------------------------------------------------------------------
/video_llama/datasets/data_utils.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import gzip
  9 | import logging
 10 | import os
 11 | import random as rnd
 12 | import tarfile
 13 | import zipfile
 14 | import random
 15 | from typing import List
 16 | from tqdm import tqdm
 17 | 
 18 | import decord
 19 | from decord import VideoReader
 20 | import webdataset as wds
 21 | import numpy as np
 22 | import torch
 23 | from torch.utils.data.dataset import IterableDataset
 24 | 
 25 | from video_llama.common.registry import registry
 26 | from video_llama.datasets.datasets.base_dataset import ConcatDataset
 27 | 
 28 | 
 29 | decord.bridge.set_bridge("torch")
 30 | MAX_INT = registry.get("MAX_INT")
 31 | 
 32 | 
 33 | class ChainDataset(wds.DataPipeline):
 34 |     r"""Dataset for chaining multiple :class:`DataPipeline` s.
 35 | 
 36 |     This class is useful to assemble different existing dataset streams. The
 37 |     chaining operation is done on-the-fly, so concatenating large-scale
 38 |     datasets with this class will be efficient.
 39 | 
 40 |     Args:
 41 |         datasets (iterable of IterableDataset): datasets to be chained together
 42 |     """
 43 |     def __init__(self, datasets: List[wds.DataPipeline]) -> None:
 44 |         super().__init__()
 45 |         self.datasets = datasets
 46 |         self.prob = []
 47 |         self.names = []
 48 |         for dataset in self.datasets:
 49 |             if hasattr(dataset, 'name'):
 50 |                 self.names.append(dataset.name)
 51 |             else:
 52 |                 self.names.append('Unknown')
 53 |             if hasattr(dataset, 'sample_ratio'):
 54 |                 self.prob.append(dataset.sample_ratio)
 55 |             else:
 56 |                 self.prob.append(1)
 57 |                 logging.info("One of the datapipeline doesn't define ratio and set to 1 automatically.")
 58 | 
 59 |     def __iter__(self):
 60 |         datastreams = [iter(dataset) for dataset in self.datasets]
 61 |         while True:
 62 |             select_datastream = random.choices(datastreams, weights=self.prob, k=1)[0]
 63 |             yield next(select_datastream)
 64 | 
 65 | 
 66 | def apply_to_sample(f, sample):
 67 |     if len(sample) == 0:
 68 |         return {}
 69 | 
 70 |     def _apply(x):
 71 |         if torch.is_tensor(x):
 72 |             return f(x)
 73 |         elif isinstance(x, dict):
 74 |             return {key: _apply(value) for key, value in x.items()}
 75 |         elif isinstance(x, list):
 76 |             return [_apply(x) for x in x]
 77 |         else:
 78 |             return x
 79 | 
 80 |     return _apply(sample)
 81 | 
 82 | 
 83 | def move_to_cuda(sample):
 84 |     def _move_to_cuda(tensor):
 85 |         return tensor.cuda()
 86 | 
 87 |     return apply_to_sample(_move_to_cuda, sample)
 88 | 
 89 | 
 90 | def prepare_sample(samples, cuda_enabled=True):
 91 |     if cuda_enabled:
 92 |         samples = move_to_cuda(samples)
 93 | 
 94 |     # TODO fp16 support
 95 | 
 96 |     return samples
 97 | 
 98 | 
 99 | def reorg_datasets_by_split(datasets):
100 |     """
101 |     Organizes datasets by split.
102 | 
103 |     Args:
104 |         datasets: dict of torch.utils.data.Dataset objects by name.
105 | 
106 |     Returns:
107 |         Dict of datasets by split {split_name: List[Datasets]}.
108 |     """
109 |     # if len(datasets) == 1:
110 |     #     return datasets[list(datasets.keys())[0]]
111 |     # else:
112 |     reorg_datasets = dict()
113 | 
114 |     # reorganize by split
115 |     for _, dataset in datasets.items():
116 |         for split_name, dataset_split in dataset.items():
117 |             if split_name not in reorg_datasets:
118 |                 reorg_datasets[split_name] = [dataset_split]
119 |             else:
120 |                 reorg_datasets[split_name].append(dataset_split)
121 | 
122 |     return reorg_datasets
123 | 
124 | 
125 | def concat_datasets(datasets):
126 |     """
127 |     Concatenates multiple datasets into a single dataset.
128 | 
129 |     It supports may-style datasets and DataPipeline from WebDataset. Currently, does not support
130 |     generic IterableDataset because it requires creating separate samplers.
131 | 
132 |     Now only supports conctenating training datasets and assuming validation and testing
133 |     have only a single dataset. This is because metrics should not be computed on the concatenated
134 |     datasets.
135 | 
136 |     Args:
137 |         datasets: dict of torch.utils.data.Dataset objects by split.
138 | 
139 |     Returns:
140 |         Dict of concatenated datasets by split, "train" is the concatenation of multiple datasets,
141 |         "val" and "test" remain the same.
142 | 
143 |         If the input training datasets contain both map-style and DataPipeline datasets, returns
144 |         a tuple, where the first element is a concatenated map-style dataset and the second
145 |         element is a chained DataPipeline dataset.
146 | 
147 |     """
148 |     # concatenate datasets in the same split
149 |     for split_name in datasets:
150 |         if split_name != "train":
151 |             assert (
152 |                 len(datasets[split_name]) == 1
153 |             ), "Do not support multiple {} datasets.".format(split_name)
154 |             datasets[split_name] = datasets[split_name][0]
155 |         else:
156 |             iterable_datasets, map_datasets = [], []
157 |             for dataset in datasets[split_name]:
158 |                 if isinstance(dataset, wds.DataPipeline):
159 |                     logging.info(
160 |                         "Dataset {} is IterableDataset, can't be concatenated.".format(
161 |                             dataset
162 |                         )
163 |                     )
164 |                     iterable_datasets.append(dataset)
165 |                 elif isinstance(dataset, IterableDataset):
166 |                     raise NotImplementedError(
167 |                         "Do not support concatenation of generic IterableDataset."
168 |                     )
169 |                 else:
170 |                     map_datasets.append(dataset)
171 | 
172 |             # if len(iterable_datasets) > 0:
173 |             # concatenate map-style datasets and iterable-style datasets separately
174 |             if len(iterable_datasets) > 1:
175 |                 chained_datasets = (
176 |                     ChainDataset(iterable_datasets)
177 |                 )
178 |             elif len(iterable_datasets) == 1:
179 |                 chained_datasets = iterable_datasets[0]
180 |             else:
181 |                 chained_datasets = None
182 | 
183 |             concat_datasets = (
184 |                 ConcatDataset(map_datasets) if len(map_datasets) > 0 else None
185 |             )
186 | 
187 |             train_datasets = concat_datasets, chained_datasets
188 |             train_datasets = tuple([x for x in train_datasets if x is not None])
189 |             train_datasets = (
190 |                 train_datasets[0] if len(train_datasets) == 1 else train_datasets
191 |             )
192 | 
193 |             datasets[split_name] = train_datasets
194 | 
195 |     return datasets
196 | 
197 | 


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/datasets/__init__.py


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/datasets/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/__pycache__/base_dataset.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/datasets/__pycache__/base_dataset.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/__pycache__/caption_datasets.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/datasets/__pycache__/caption_datasets.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/__pycache__/cc_sbu_dataset.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/datasets/__pycache__/cc_sbu_dataset.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/__pycache__/laion_dataset.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/datasets/__pycache__/laion_dataset.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/__pycache__/llava_instruct_dataset.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/datasets/__pycache__/llava_instruct_dataset.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/__pycache__/video_instruct_dataset.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/datasets/__pycache__/video_instruct_dataset.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/__pycache__/webvid_datasets.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/datasets/datasets/__pycache__/webvid_datasets.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/base_dataset.py:
--------------------------------------------------------------------------------
 1 | """
 2 |  Copyright (c) 2022, salesforce.com, inc.
 3 |  All rights reserved.
 4 |  SPDX-License-Identifier: BSD-3-Clause
 5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 6 | """
 7 | 
 8 | import json
 9 | from typing import Iterable
10 | 
11 | from torch.utils.data import Dataset, ConcatDataset
12 | from torch.utils.data.dataloader import default_collate
13 | 
14 | 
15 | class BaseDataset(Dataset):
16 |     def __init__(
17 |         self, vis_processor=None, text_processor=None, vis_root=None, ann_paths=[]
18 |     ):
19 |         """
20 |         vis_root (string): Root directory of images (e.g. coco/images/)
21 |         ann_root (string): directory to store the annotation file
22 |         """
23 |         self.vis_root = vis_root
24 | 
25 |         self.annotation = []
26 |         for ann_path in ann_paths:
27 |             self.annotation.extend(json.load(open(ann_path, "r"))['annotations'])
28 | 
29 |         self.vis_processor = vis_processor
30 |         self.text_processor = text_processor
31 | 
32 |         self._add_instance_ids()
33 | 
34 |     def __len__(self):
35 |         return len(self.annotation)
36 | 
37 |     def collater(self, samples):
38 |         return default_collate(samples)
39 | 
40 |     def set_processors(self, vis_processor, text_processor):
41 |         self.vis_processor = vis_processor
42 |         self.text_processor = text_processor
43 | 
44 |     def _add_instance_ids(self, key="instance_id"):
45 |         for idx, ann in enumerate(self.annotation):
46 |             ann[key] = str(idx)
47 | 
48 | 
49 | class ConcatDataset(ConcatDataset):
50 |     def __init__(self, datasets: Iterable[Dataset]) -> None:
51 |         super().__init__(datasets)
52 | 
53 |     def collater(self, samples):
54 |         # TODO For now only supports datasets with same underlying collater implementations
55 | 
56 |         all_keys = set()
57 |         for s in samples:
58 |             all_keys.update(s)
59 | 
60 |         shared_keys = all_keys
61 |         for s in samples:
62 |             shared_keys = shared_keys & set(s.keys())
63 | 
64 |         samples_shared_keys = []
65 |         for s in samples:
66 |             samples_shared_keys.append({k: s[k] for k in s.keys() if k in shared_keys})
67 | 
68 |         return self.datasets[0].collater(samples_shared_keys)
69 | 


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/caption_datasets.py:
--------------------------------------------------------------------------------
 1 | """
 2 |  Copyright (c) 2022, salesforce.com, inc.
 3 |  All rights reserved.
 4 |  SPDX-License-Identifier: BSD-3-Clause
 5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 6 | """
 7 | 
 8 | import os
 9 | from collections import OrderedDict
10 | 
11 | from video_llama.datasets.datasets.base_dataset import BaseDataset
12 | from PIL import Image
13 | 
14 | 
15 | class __DisplMixin:
16 |     def displ_item(self, index):
17 |         sample, ann = self.__getitem__(index), self.annotation[index]
18 | 
19 |         return OrderedDict(
20 |             {
21 |                 "file": ann["image"],
22 |                 "caption": ann["caption"],
23 |                 "image": sample["image"],
24 |             }
25 |         )
26 | 
27 | 
28 | class CaptionDataset(BaseDataset, __DisplMixin):
29 |     def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
30 |         """
31 |         vis_root (string): Root directory of images (e.g. coco/images/)
32 |         ann_root (string): directory to store the annotation file
33 |         """
34 |         super().__init__(vis_processor, text_processor, vis_root, ann_paths)
35 | 
36 |         self.img_ids = {}
37 |         n = 0
38 |         for ann in self.annotation:
39 |             img_id = ann["image_id"]
40 |             if img_id not in self.img_ids.keys():
41 |                 self.img_ids[img_id] = n
42 |                 n += 1
43 | 
44 |     def __getitem__(self, index):
45 | 
46 |         # TODO this assumes image input, not general enough
47 |         ann = self.annotation[index]
48 | 
49 |         img_file = '{:0>12}.jpg'.format(ann["image_id"])
50 |         image_path = os.path.join(self.vis_root, img_file)
51 |         image = Image.open(image_path).convert("RGB")
52 | 
53 |         image = self.vis_processor(image)
54 |         caption = self.text_processor(ann["caption"])
55 | 
56 |         return {
57 |             "image": image,
58 |             "text_input": caption,
59 |             "image_id": self.img_ids[ann["image_id"]],
60 |         }
61 | 
62 | 
63 | class CaptionEvalDataset(BaseDataset, __DisplMixin):
64 |     def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
65 |         """
66 |         vis_root (string): Root directory of images (e.g. coco/images/)
67 |         ann_root (string): directory to store the annotation file
68 |         split (string): val or test
69 |         """
70 |         super().__init__(vis_processor, text_processor, vis_root, ann_paths)
71 | 
72 |     def __getitem__(self, index):
73 | 
74 |         ann = self.annotation[index]
75 | 
76 |         image_path = os.path.join(self.vis_root, ann["image"])
77 |         image = Image.open(image_path).convert("RGB")
78 | 
79 |         image = self.vis_processor(image)
80 | 
81 |         return {
82 |             "image": image,
83 |             "image_id": ann["image_id"],
84 |             "instance_id": ann["instance_id"],
85 |         }
86 | 


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/cc_sbu_dataset.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | from PIL import Image
 3 | import webdataset as wds
 4 | from video_llama.datasets.datasets.base_dataset import BaseDataset
 5 | from video_llama.datasets.datasets.caption_datasets import CaptionDataset
 6 | 
 7 | 
 8 | class CCSBUDataset(BaseDataset):
 9 |     def __init__(self, vis_processor, text_processor, location):
10 |         super().__init__(vis_processor=vis_processor, text_processor=text_processor)
11 | 
12 |         self.inner_dataset = wds.DataPipeline(
13 |             wds.ResampledShards(location),
14 |             wds.tarfile_to_samples(handler=wds.warn_and_continue),
15 |             wds.shuffle(1000, handler=wds.warn_and_continue),
16 |             wds.decode("pilrgb", handler=wds.warn_and_continue),
17 |             wds.to_tuple("jpg", "json", handler=wds.warn_and_continue),
18 |             wds.map_tuple(self.vis_processor, handler=wds.warn_and_continue),
19 |             wds.map(self.to_dict, handler=wds.warn_and_continue),
20 |         )
21 | 
22 |     def to_dict(self, sample):
23 |         return {
24 |             "image": sample[0],
25 |             "text_input": self.text_processor(sample[1]["caption"]),
26 |             "type":'image',
27 |         }
28 | 
29 | 
30 | class CCSBUAlignDataset(CaptionDataset):
31 | 
32 |     def __getitem__(self, index):
33 | 
34 |         # TODO this assumes image input, not general enough
35 |         ann = self.annotation[index]
36 | 
37 |         img_file = '{}.jpg'.format(ann["image_id"])
38 |         image_path = os.path.join(self.vis_root, img_file)
39 |         image = Image.open(image_path).convert("RGB")
40 | 
41 |         image = self.vis_processor(image)
42 |         caption = ann["caption"]
43 | 
44 |         return {
45 |             "image": image,
46 |             "text_input": caption,
47 |             "image_id": self.img_ids[ann["image_id"]],
48 |             "type":'image',
49 |         }


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/dataloader_utils.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import time
  9 | import random
 10 | import torch
 11 | from video_llama.datasets.data_utils import move_to_cuda
 12 | from torch.utils.data import DataLoader
 13 | 
 14 | 
 15 | class MultiIterLoader:
 16 |     """
 17 |     A simple wrapper for iterating over multiple iterators.
 18 | 
 19 |     Args:
 20 |         loaders (List[Loader]): List of Iterator loaders.
 21 |         ratios (List[float]): List of ratios to sample from each loader. If None, all loaders are sampled uniformly.
 22 |     """
 23 | 
 24 |     def __init__(self, loaders, ratios=None):
 25 |         # assert all loaders has __next__ method
 26 |         for loader in loaders:
 27 |             assert hasattr(
 28 |                 loader, "__next__"
 29 |             ), "Loader {} has no __next__ method.".format(loader)
 30 | 
 31 |         if ratios is None:
 32 |             ratios = [1.0] * len(loaders)
 33 |         else:
 34 |             assert len(ratios) == len(loaders)
 35 |             ratios = [float(ratio) / sum(ratios) for ratio in ratios]
 36 | 
 37 |         self.loaders = loaders
 38 |         self.ratios = ratios
 39 | 
 40 |     def __next__(self):
 41 |         # random sample from each loader by ratio
 42 |         loader_idx = random.choices(range(len(self.loaders)), self.ratios, k=1)[0]
 43 |         return next(self.loaders[loader_idx])
 44 | 
 45 | 
 46 | class PrefetchLoader(object):
 47 |     """
 48 |     Modified from https://github.com/ChenRocks/UNITER.
 49 | 
 50 |     overlap compute and cuda data transfer
 51 |     (copied and then modified from nvidia apex)
 52 |     """
 53 | 
 54 |     def __init__(self, loader):
 55 |         self.loader = loader
 56 |         self.stream = torch.cuda.Stream()
 57 | 
 58 |     def __iter__(self):
 59 |         loader_it = iter(self.loader)
 60 |         self.preload(loader_it)
 61 |         batch = self.next(loader_it)
 62 |         while batch is not None:
 63 |             is_tuple = isinstance(batch, tuple)
 64 |             if is_tuple:
 65 |                 task, batch = batch
 66 | 
 67 |             if is_tuple:
 68 |                 yield task, batch
 69 |             else:
 70 |                 yield batch
 71 |             batch = self.next(loader_it)
 72 | 
 73 |     def __len__(self):
 74 |         return len(self.loader)
 75 | 
 76 |     def preload(self, it):
 77 |         try:
 78 |             self.batch = next(it)
 79 |         except StopIteration:
 80 |             self.batch = None
 81 |             return
 82 |         # if record_stream() doesn't work, another option is to make sure
 83 |         # device inputs are created on the main stream.
 84 |         # self.next_input_gpu = torch.empty_like(self.next_input,
 85 |         #                                        device='cuda')
 86 |         # self.next_target_gpu = torch.empty_like(self.next_target,
 87 |         #                                         device='cuda')
 88 |         # Need to make sure the memory allocated for next_* is not still in use
 89 |         # by the main stream at the time we start copying to next_*:
 90 |         # self.stream.wait_stream(torch.cuda.current_stream())
 91 |         with torch.cuda.stream(self.stream):
 92 |             self.batch = move_to_cuda(self.batch)
 93 |             # more code for the alternative if record_stream() doesn't work:
 94 |             # copy_ will record the use of the pinned source tensor in this
 95 |             # side stream.
 96 |             # self.next_input_gpu.copy_(self.next_input, non_blocking=True)
 97 |             # self.next_target_gpu.copy_(self.next_target, non_blocking=True)
 98 |             # self.next_input = self.next_input_gpu
 99 |             # self.next_target = self.next_target_gpu
100 | 
101 |     def next(self, it):
102 |         torch.cuda.current_stream().wait_stream(self.stream)
103 |         batch = self.batch
104 |         if batch is not None:
105 |             record_cuda_stream(batch)
106 |         self.preload(it)
107 |         return batch
108 | 
109 |     def __getattr__(self, name):
110 |         method = self.loader.__getattribute__(name)
111 |         return method
112 | 
113 | 
114 | def record_cuda_stream(batch):
115 |     if isinstance(batch, torch.Tensor):
116 |         batch.record_stream(torch.cuda.current_stream())
117 |     elif isinstance(batch, list) or isinstance(batch, tuple):
118 |         for t in batch:
119 |             record_cuda_stream(t)
120 |     elif isinstance(batch, dict):
121 |         for t in batch.values():
122 |             record_cuda_stream(t)
123 |     else:
124 |         pass
125 | 
126 | 
127 | class IterLoader:
128 |     """
129 |     A wrapper to convert DataLoader as an infinite iterator.
130 | 
131 |     Modified from:
132 |         https://github.com/open-mmlab/mmcv/blob/master/mmcv/runner/iter_based_runner.py
133 |     """
134 | 
135 |     def __init__(self, dataloader: DataLoader, use_distributed: bool = False):
136 |         self._dataloader = dataloader
137 |         self.iter_loader = iter(self._dataloader)
138 |         self._use_distributed = use_distributed
139 |         self._epoch = 0
140 | 
141 |     @property
142 |     def epoch(self) -> int:
143 |         return self._epoch
144 | 
145 |     def __next__(self):
146 |         try:
147 |             data = next(self.iter_loader)
148 |         except StopIteration:
149 |             self._epoch += 1
150 |             if hasattr(self._dataloader.sampler, "set_epoch") and self._use_distributed:
151 |                 self._dataloader.sampler.set_epoch(self._epoch)
152 |             time.sleep(2)  # Prevent possible deadlock during epoch transition
153 |             self.iter_loader = iter(self._dataloader)
154 |             data = next(self.iter_loader)
155 | 
156 |         return data
157 | 
158 |     def __iter__(self):
159 |         return self
160 | 
161 |     def __len__(self):
162 |         return len(self._dataloader)
163 | 


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/laion_dataset.py:
--------------------------------------------------------------------------------
 1 | """
 2 |  Copyright (c) 2022, salesforce.com, inc.
 3 |  All rights reserved.
 4 |  SPDX-License-Identifier: BSD-3-Clause
 5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 6 | """
 7 | 
 8 | import webdataset as wds
 9 | from video_llama.datasets.datasets.base_dataset import BaseDataset
10 | 
11 | 
12 | class LaionDataset(BaseDataset):
13 |     def __init__(self, vis_processor, text_processor, location):
14 |         super().__init__(vis_processor=vis_processor, text_processor=text_processor)
15 | 
16 |         self.inner_dataset = wds.DataPipeline(
17 |             wds.ResampledShards(location),
18 |             wds.tarfile_to_samples(handler=wds.warn_and_continue),
19 |             wds.shuffle(1000, handler=wds.warn_and_continue),
20 |             wds.decode("pilrgb", handler=wds.warn_and_continue),
21 |             wds.to_tuple("jpg", "json", handler=wds.warn_and_continue),
22 |             wds.map_tuple(self.vis_processor, handler=wds.warn_and_continue),
23 |             wds.map(self.to_dict, handler=wds.warn_and_continue),
24 |         )
25 | 
26 |     def to_dict(self, sample):
27 |         return {
28 |             "image": sample[0],
29 |             "text_input": self.text_processor(sample[1]["caption"]),
30 |         }
31 | 
32 | 


--------------------------------------------------------------------------------
/video_llama/datasets/datasets/webvid_datasets.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import os
  9 | from video_llama.datasets.datasets.base_dataset import BaseDataset
 10 | from video_llama.datasets.datasets.caption_datasets import CaptionDataset
 11 | import pandas as pd
 12 | import decord
 13 | from decord import VideoReader
 14 | import random
 15 | import torch
 16 | from torch.utils.data.dataloader import default_collate
 17 | class WebvidDataset(BaseDataset):
 18 |     def __init__(self, vis_processor, text_processor, vis_root, ann_root):
 19 |         """
 20 |         vis_root (string): Root directory of video (e.g. webvid_eval/video/)
 21 |         ann_root (string): Root directory of video (e.g. webvid_eval/annotations/)
 22 |         split (string): val or test
 23 |         """
 24 |         super().__init__(vis_processor=vis_processor, text_processor=text_processor)
 25 | 
 26 | 
 27 |         # 读取一个路径下所有的
 28 | 
 29 |         ts_df = []
 30 |         for file_name in os.listdir(ann_root):
 31 |             if file_name.endswith('.csv'):
 32 |                 df = pd.read_csv(os.path.join(ann_root, file_name))
 33 |                 ts_df.append(df)
 34 | 
 35 |         merged_df = pd.concat(ts_df)
 36 |         self.annotation = merged_df
 37 |         self.vis_root = vis_root
 38 |         self.resize_size = 224
 39 |         self.num_frm = 8
 40 |         self.frm_sampling_strategy = 'headtail'
 41 | 
 42 |     def _get_video_path(self, sample):
 43 |         rel_video_fp = os.path.join(sample['page_dir'], str(sample['videoid']) + '.mp4')
 44 |         full_video_fp = os.path.join(self.vis_root,  rel_video_fp)
 45 |         return full_video_fp
 46 | 
 47 |     def __getitem__(self, index):
 48 |         num_retries = 10  # skip error videos
 49 |         for _ in range(num_retries):
 50 |             sample = self.annotation.iloc[index]
 51 |             sample_dict = sample.to_dict()
 52 |             video_id = sample_dict['videoid']
 53 | 
 54 |             if 'name' in sample_dict.keys():
 55 |                 text = sample_dict['name'].strip()
 56 |             else:
 57 |                 raise NotImplementedError("Un-supported text annotation format.")
 58 | 
 59 |             # fetch video
 60 |             video_path = self._get_video_path(sample_dict) 
 61 |             # if os.path.exists(video_path):
 62 |             try:
 63 |                 video = self.vis_processor(video_path)
 64 |             except:
 65 |                 print(f"Failed to load examples with video: {video_path}. "
 66 |                             f"Will randomly sample an example as a replacement.")
 67 |                 index = random.randint(0, len(self) - 1)
 68 |                 continue
 69 |             caption = self.text_processor(text)
 70 | 
 71 |             # print(video.size())
 72 |             if video is None or caption is None \
 73 |                     or video.size()!=torch.Size([3,self.vis_processor.n_frms,224,224]):
 74 |                 print(f"Failed to load examples with video: {video_path}. "
 75 |                             f"Will randomly sample an example as a replacement.")
 76 |                 index = random.randint(0, len(self) - 1)
 77 |                 continue
 78 |             else:
 79 |                 break
 80 |         else:  
 81 |             raise RuntimeError(f"Failed to fetch video after {num_retries} retries.")
 82 |         # "image_id" is kept to stay compatible with the COCO evaluation format
 83 |         return {
 84 |             "image": video,
 85 |             "text_input": caption,
 86 |             "type":'video',
 87 |         }
 88 | 
 89 |     def __len__(self):
 90 |         return len(self.annotation)
 91 | 
 92 |     # def collater(self, samples):
 93 |     #     new_result = {}
 94 |     #     new_result['image'] = default_collate( [sample["image"] for sample in samples])
 95 |     #     new_result['text_input'] = default_collate( [sample["text_input"] for sample in samples])
 96 |     #     return new_result
 97 |         
 98 | class WebvidDatasetEvalDataset(BaseDataset):
 99 |     def __init__(self, vis_processor, text_processor, vis_root, ann_paths):
100 |         """
101 |         vis_root (string): Root directory of images (e.g. coco/images/)
102 |         ann_root (string): directory to store the annotation file
103 |         split (string): val or test
104 |         """
105 |         super().__init__(vis_processor, text_processor, vis_root, ann_paths)
106 | 
107 |     def __getitem__(self, index):
108 | 
109 |         ann = self.annotation[index]
110 | 
111 |         vname = ann["video"]
112 |         video_path = os.path.join(self.vis_root, vname)
113 | 
114 |         video = self.vis_processor(video_path)
115 | 
116 |         return {
117 |             "video": video,
118 |             "image_id": ann["image_id"],
119 |             "instance_id": ann["instance_id"],
120 |         }
121 | 
122 | 
123 | 


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/.assets/bird_audio.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/.assets/bird_audio.wav


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/.assets/bird_image.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/.assets/bird_image.jpg


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/.assets/car_audio.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/.assets/car_audio.wav


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/.assets/car_image.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/.assets/car_image.jpg


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/.assets/dog_audio.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/.assets/dog_audio.wav


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/.assets/dog_image.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/.assets/dog_image.jpg


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
 1 | # Code of Conduct
 2 | 
 3 | ## Our Pledge
 4 | 
 5 | In the interest of fostering an open and welcoming environment, we as
 6 | contributors and maintainers pledge to make participation in our project and
 7 | our community a harassment-free experience for everyone, regardless of age, body
 8 | size, disability, ethnicity, sex characteristics, gender identity and expression,
 9 | level of experience, education, socio-economic status, nationality, personal
10 | appearance, race, religion, or sexual identity and orientation.
11 | 
12 | ## Our Standards
13 | 
14 | Examples of behavior that contributes to creating a positive environment
15 | include:
16 | 
17 | * Using welcoming and inclusive language
18 | * Being respectful of differing viewpoints and experiences
19 | * Gracefully accepting constructive criticism
20 | * Focusing on what is best for the community
21 | * Showing empathy towards other community members
22 | 
23 | Examples of unacceptable behavior by participants include:
24 | 
25 | * The use of sexualized language or imagery and unwelcome sexual attention or
26 | advances
27 | * Trolling, insulting/derogatory comments, and personal or political attacks
28 | * Public or private harassment
29 | * Publishing others' private information, such as a physical or electronic
30 | address, without explicit permission
31 | * Other conduct which could reasonably be considered inappropriate in a
32 | professional setting
33 | 
34 | ## Our Responsibilities
35 | 
36 | Project maintainers are responsible for clarifying the standards of acceptable
37 | behavior and are expected to take appropriate and fair corrective action in
38 | response to any instances of unacceptable behavior.
39 | 
40 | Project maintainers have the right and responsibility to remove, edit, or
41 | reject comments, commits, code, wiki edits, issues, and other contributions
42 | that are not aligned to this Code of Conduct, or to ban temporarily or
43 | permanently any contributor for other behaviors that they deem inappropriate,
44 | threatening, offensive, or harmful.
45 | 
46 | ## Scope
47 | 
48 | This Code of Conduct applies within all project spaces, and it also applies when
49 | an individual is representing the project or its community in public spaces.
50 | Examples of representing a project or community include using an official
51 | project e-mail address, posting via an official social media account, or acting
52 | as an appointed representative at an online or offline event. Representation of
53 | a project may be further defined and clarified by project maintainers.
54 | 
55 | This Code of Conduct also applies outside the project spaces when there is a
56 | reasonable belief that an individual's behavior may have a negative impact on
57 | the project or its community.
58 | 
59 | ## Enforcement
60 | 
61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be
62 | reported by contacting the project team at <opensource-conduct@fb.com>. All
63 | complaints will be reviewed and investigated and will result in a response that
64 | is deemed necessary and appropriate to the circumstances. The project team is
65 | obligated to maintain confidentiality with regard to the reporter of an incident.
66 | Further details of specific enforcement policies may be posted separately.
67 | 
68 | Project maintainers who do not follow or enforce the Code of Conduct in good
69 | faith may face temporary or permanent repercussions as determined by other
70 | members of the project's leadership.
71 | 
72 | ## Attribution
73 | 
74 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
75 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
76 | 
77 | [homepage]: https://www.contributor-covenant.org
78 | 
79 | For answers to common questions about this code of conduct, see
80 | https://www.contributor-covenant.org/faq


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | # Contributing to ImageBind
 2 | We want to make contributing to this project as easy and transparent as
 3 | possible.
 4 | 
 5 | ## Pull Requests
 6 | We actively welcome your pull requests.
 7 | 
 8 | 1. Fork the repo and create your branch from `main`.
 9 | 2. If you've added code that should be tested, add tests.
10 | 3. If you've changed APIs, update the documentation.
11 | 4. Ensure the test suite passes.
12 | 5. Make sure your code lints.
13 | 6. If you haven't already, complete the Contributor License Agreement ("CLA").
14 | 
15 | ## Contributor License Agreement ("CLA")
16 | In order to accept your pull request, we need you to submit a CLA. You only need
17 | to do this once to work on any of Meta's open source projects.
18 | 
19 | Complete your CLA here: <https://code.facebook.com/cla>
20 | 
21 | ## Issues
22 | We use GitHub issues to track public bugs. Please ensure your description is
23 | clear and has sufficient instructions to be able to reproduce the issue.
24 | 
25 | Meta has a [bounty program](https://www.facebook.com/whitehat/) for the safe
26 | disclosure of security bugs. In those cases, please go through the process
27 | outlined on that page and do not file a public issue.
28 | 
29 | ## License
30 | By contributing to Omnivore, you agree that your contributions will be licensed
31 | under the [LICENSE](LICENSE) file in the root directory of this source tree.
32 | 


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/README.md:
--------------------------------------------------------------------------------
  1 | # ImageBind: One Embedding Space To Bind Them All
  2 | 
  3 | **[FAIR, Meta AI](https://ai.facebook.com/research/)** 
  4 | 
  5 | Rohit Girdhar*,
  6 | Alaaeldin El-Nouby*,
  7 | Zhuang Liu,
  8 | Mannat Singh,
  9 | Kalyan Vasudev Alwala,
 10 | Armand Joulin,
 11 | Ishan Misra*
 12 | 
 13 | To appear at CVPR 2023 (*Highlighted paper*)
 14 | 
 15 | [[`Paper`](https://facebookresearch.github.io/ImageBind/paper)] [[`Blog`](https://ai.facebook.com/blog/imagebind-six-modalities-binding-ai/)] [[`Demo`](https://imagebind.metademolab.com/)] [[`Supplementary Video`](https://dl.fbaipublicfiles.com/imagebind/imagebind_video.mp4)] [[`BibTex`](#citing-imagebind)]
 16 | 
 17 | PyTorch implementation and pretrained models for ImageBind. For details, see the paper: **[ImageBind: One Embedding Space To Bind Them All](https://facebookresearch.github.io/ImageBind/paper)**.
 18 | 
 19 | ImageBind learns a joint embedding across six different modalities - images, text, audio, depth, thermal, and IMU data. It enables novel emergent applications ‘out-of-the-box’ including cross-modal retrieval, composing modalities with arithmetic, cross-modal detection and generation.
 20 | 
 21 | 
 22 | 
 23 | ![ImageBind](https://user-images.githubusercontent.com/8495451/236859695-ffa13364-3e39-4d99-a8da-fbfab17f9a6b.gif)
 24 | 
 25 | ## ImageBind model
 26 | 
 27 | Emergent zero-shot classification performance.
 28 | 
 29 | <table style="margin: auto">
 30 |   <tr>
 31 |     <th>Model</th>
 32 |     <th><span style="color:blue">IN1k</span></th>
 33 |     <th><span style="color:purple">K400</span></th>
 34 |     <th><span style="color:green">NYU-D</span></th>
 35 |     <th><span style="color:LightBlue">ESC</span></th>
 36 |     <th><span style="color:orange">LLVIP</span></th>
 37 |     <th><span style="color:purple">Ego4D</span></th>
 38 |     <th>download</th>
 39 |   </tr>
 40 |   <tr>
 41 |     <td>imagebind_huge</td>
 42 |     <td align="right">77.7</td>
 43 |     <td align="right">50.0</td>
 44 |     <td align="right">54.0</td>
 45 |     <td align="right">66.9</td>
 46 |     <td align="right">63.4</td>
 47 |     <td align="right">25.0</td>
 48 |     <td><a href="https://dl.fbaipublicfiles.com/imagebind/imagebind_huge.pth">checkpoint</a></td>
 49 |   </tr>
 50 |   
 51 | </table>
 52 | 
 53 | ## Usage
 54 | 
 55 | Install pytorch 1.13+ and other 3rd party dependencies.
 56 | 
 57 | ```shell
 58 | conda create --name imagebind python=3.8 -y
 59 | conda activate imagebind
 60 | 
 61 | pip install -r requirements.txt
 62 | ```
 63 | 
 64 | For windows users, you might need to install `soundfile` for reading/writing audio files. (Thanks @congyue1977)
 65 | 
 66 | ```
 67 | pip install soundfile
 68 | ```
 69 | 
 70 | 
 71 | Extract and compare features across modalities (e.g. Image, Text and Audio).
 72 | 
 73 | ```python
 74 | import data
 75 | import torch
 76 | from models import imagebind_model
 77 | from models.imagebind_model import ModalityType
 78 | 
 79 | text_list=["A dog.", "A car", "A bird"]
 80 | image_paths=[".assets/dog_image.jpg", ".assets/car_image.jpg", ".assets/bird_image.jpg"]
 81 | audio_paths=[".assets/dog_audio.wav", ".assets/car_audio.wav", ".assets/bird_audio.wav"]
 82 | 
 83 | device = "cuda:0" if torch.cuda.is_available() else "cpu"
 84 | 
 85 | # Instantiate model
 86 | model = imagebind_model.imagebind_huge(pretrained=True)
 87 | model.eval()
 88 | model.to(device)
 89 | 
 90 | # Load data
 91 | inputs = {
 92 |     ModalityType.TEXT: data.load_and_transform_text(text_list, device),
 93 |     ModalityType.VISION: data.load_and_transform_vision_data(image_paths, device),
 94 |     ModalityType.AUDIO: data.load_and_transform_audio_data(audio_paths, device),
 95 | }
 96 | 
 97 | with torch.no_grad():
 98 |     embeddings = model(inputs)
 99 | 
100 | print(
101 |     "Vision x Text: ",
102 |     torch.softmax(embeddings[ModalityType.VISION] @ embeddings[ModalityType.TEXT].T, dim=-1),
103 | )
104 | print(
105 |     "Audio x Text: ",
106 |     torch.softmax(embeddings[ModalityType.AUDIO] @ embeddings[ModalityType.TEXT].T, dim=-1),
107 | )
108 | print(
109 |     "Vision x Audio: ",
110 |     torch.softmax(embeddings[ModalityType.VISION] @ embeddings[ModalityType.AUDIO].T, dim=-1),
111 | )
112 | 
113 | # Expected output:
114 | #
115 | # Vision x Text:
116 | # tensor([[9.9761e-01, 2.3694e-03, 1.8612e-05],
117 | #         [3.3836e-05, 9.9994e-01, 2.4118e-05],
118 | #         [4.7997e-05, 1.3496e-02, 9.8646e-01]])
119 | #
120 | # Audio x Text:
121 | # tensor([[1., 0., 0.],
122 | #         [0., 1., 0.],
123 | #         [0., 0., 1.]])
124 | #
125 | # Vision x Audio:
126 | # tensor([[0.8070, 0.1088, 0.0842],
127 | #         [0.1036, 0.7884, 0.1079],
128 | #         [0.0018, 0.0022, 0.9960]])
129 | 
130 | ```
131 | 
132 | ## Model card
133 | Please see the [model card](model_card.md) for details.
134 | 
135 | ## License
136 | 
137 | ImageBind code and model weights are released under the CC-BY-NC 4.0 license. See [LICENSE](LICENSE) for additional details.
138 | 
139 | ## Contributing
140 | 
141 | See [contributing](CONTRIBUTING.md) and the [code of conduct](CODE_OF_CONDUCT.md).
142 | 
143 | ## Citing ImageBind
144 | 
145 | If you find this repository useful, please consider giving a star :star: and citation
146 | 
147 | ```
148 | @inproceedings{girdhar2023imagebind,
149 |   title={ImageBind: One Embedding Space To Bind Them All},
150 |   author={Girdhar, Rohit and El-Nouby, Alaaeldin and Liu, Zhuang
151 | and Singh, Mannat and Alwala, Kalyan Vasudev and Joulin, Armand and Misra, Ishan},
152 |   booktitle={CVPR},
153 |   year={2023}
154 | }
155 | ```
156 | 


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/__pycache__/data.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/__pycache__/data.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/bpe/bpe_simple_vocab_16e6.txt.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/bpe/bpe_simple_vocab_16e6.txt.gz


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/model_card.md:
--------------------------------------------------------------------------------
 1 | # Model Card for ImageBind
 2 | 
 3 | Multimodal joint embedding model for image/video, text, audio, depth, IMU, and thermal images.
 4 | Input any of the six modalities and get the same sized embedding that can be used for cross-modal and multimodal tasks.
 5 | 
 6 | # Model Details
 7 | 
 8 | ## Model Description
 9 | 
10 | <!-- Provide a longer summary of what this model is/does. -->
11 | Multimodal joint embedding model for image/video, text, audio, depth, IMU, and thermal images
12 | 
13 | - **Developed by:** Meta AI
14 | - **Model type:** Multimodal model
15 | - **Language(s) (NLP):** en
16 | - **License:** CC BY-NC-SA 4.0
17 | - **Resources for more information:**
18 |     - [GitHub Repo](https://github.com/facebookresearch/ImageBind)
19 | 
20 | 
21 | # Uses
22 | 
23 | <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
24 | This model is intended only for research purposes. It provides a joint embedding space for different modalities -- image/video, text, audio, depth, IMU and thermal images.
25 | We hope that these joint embeddings can be used for a variety of different cross-modal research, e.g., cross-modal retrieval and combining embeddings from different modalities.
26 | 
27 | ## Out-of-Scope Use
28 | 
29 | <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
30 | <!-- If the user enters content, print that. If not, but they enter a task in the list, use that. If neither, say "more info needed." -->
31 | 
32 | This model is *NOT* intended to be used in any real world application -- commercial or otherwise.
33 | It may produce harmful associations with different inputs.
34 | The model needs to be investigated and likely re-trained on specific data for any such application.
35 | The model is expected to work better on web-based visual data since it was trained on such data.
36 | The text encoder is likely to work only on English language text because of the underlying training datasets.
37 | 
38 | # Bias, Risks, and Limitations
39 | 
40 | <!-- This section is meant to convey both technical and sociotechnical limitations. -->
41 | Open-domain joint embedding models are prone to producing specific biases, e.g., study from [CLIP](https://github.com/openai/CLIP/blob/main/model-card.md#bias-and-fairness).
42 | Since our model uses such models as initialization, it will exhibit such biases too.
43 | Moreover, for learning joint embeddings for other modalities such as audio, thermal, depth, and IMU we leverage datasets that are relatively small. These joint embeddings are thus limited to the concepts present in the datasets. For example, the thermal datasets we used are limited to outdoor street scenes, while the depth datasets are limited to indoor scenes.
44 | 
45 | 
46 | 
47 | # Training Details
48 | 
49 | ## Training Data
50 | 
51 | <!-- This should link to a Data Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
52 | 
53 | ImageBind uses image-paired data for training -- (image, X) where X is one of text, audio, depth, IMU or thermal data.
54 | In particular, we initialize and freeze the image and text encoders using an OpenCLIP ViT-H encoder.
55 | We train audio embeddings using Audioset, depth embeddings using the SUN RGB-D dataset, IMU using the Ego4D dataset and thermal embeddings using the LLVIP dataset.
56 | We provide the exact training data details in the paper.
57 | 
58 | 
59 | ## Training Procedure
60 | 
61 | <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
62 | Please refer to the research paper and github repo for exact details on this.
63 | 
64 | # Evaluation
65 | 
66 | ## Testing Data, Factors & Metrics
67 | 
68 | We evaluate the model on a variety of different classification benchmarks for each modality.
69 | The evaluation details are presented in the paper.
70 | The models performance is measured using standard classification metrics such as accuracy and mAP.
71 | 
72 | # Citation
73 | 
74 | <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
75 | 
76 | **BibTeX:**
77 | ```
78 | @inproceedings{girdhar2023imagebind,
79 |   title={ImageBind: One Embedding Space To Bind Them All},
80 |   author={Girdhar, Rohit and El-Nouby, Alaaeldin and Liu, Zhuang
81 | and Singh, Mannat and Alwala, Kalyan Vasudev and Joulin, Armand and Misra, Ishan},
82 |   booktitle={CVPR},
83 |   year={2023}
84 | }
85 | ```
86 | 
87 | 
88 | # Model Card Contact
89 | 
90 | Please reach out to the authors at: rgirdhar@meta.com imisra@meta.com alaaelnouby@gmail.com
91 | 
92 | # How to Get Started with the Model
93 | 
94 | Our github repo provides a simple example to extract embeddings from images, audio etc.
95 | 


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/models/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/models/__init__.py


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/models/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/models/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/models/__pycache__/helpers.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/models/__pycache__/helpers.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/models/__pycache__/imagebind_model.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/models/__pycache__/imagebind_model.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/models/__pycache__/multimodal_preprocessors.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/models/__pycache__/multimodal_preprocessors.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/models/__pycache__/transformer.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/ImageBind/models/__pycache__/transformer.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/models/helpers.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | # Portions Copyright (c) Meta Platforms, Inc. and affiliates.
  3 | # All rights reserved.
  4 | 
  5 | # This source code is licensed under the license found in the
  6 | # LICENSE file in the root directory of this source tree.
  7 | 
  8 | 
  9 | import einops
 10 | import numpy as np
 11 | import torch
 12 | import torch.nn as nn
 13 | 
 14 | 
 15 | class Normalize(nn.Module):
 16 |     def __init__(self, dim: int) -> None:
 17 |         super().__init__()
 18 |         self.dim = dim
 19 | 
 20 |     def forward(self, x):
 21 |         return torch.nn.functional.normalize(x, dim=self.dim, p=2)
 22 | 
 23 | 
 24 | class LearnableLogitScaling(nn.Module):
 25 |     def __init__(
 26 |         self,
 27 |         logit_scale_init: float = 1 / 0.07,
 28 |         learnable: bool = True,
 29 |         max_logit_scale: float = 100,
 30 |     ) -> None:
 31 |         super().__init__()
 32 |         self.max_logit_scale = max_logit_scale
 33 |         self.logit_scale_init = logit_scale_init
 34 |         self.learnable = learnable
 35 |         log_logit_scale = torch.ones([]) * np.log(self.logit_scale_init)
 36 |         if learnable:
 37 |             self.log_logit_scale = nn.Parameter(log_logit_scale)
 38 |         else:
 39 |             self.register_buffer("log_logit_scale", log_logit_scale)
 40 | 
 41 |     def forward(self, x):
 42 |         return torch.clip(self.log_logit_scale.exp(), max=self.max_logit_scale) * x
 43 | 
 44 |     def extra_repr(self):
 45 |         st = f"logit_scale_init={self.logit_scale_init},learnable={self.learnable}," \
 46 |              f" max_logit_scale={self.max_logit_scale}"
 47 |         return st
 48 | 
 49 | 
 50 | class EinOpsRearrange(nn.Module):
 51 |     def __init__(self, rearrange_expr: str, **kwargs) -> None:
 52 |         super().__init__()
 53 |         self.rearrange_expr = rearrange_expr
 54 |         self.kwargs = kwargs
 55 | 
 56 |     def forward(self, x):
 57 |         assert isinstance(x, torch.Tensor)
 58 |         return einops.rearrange(x, self.rearrange_expr, **self.kwargs)
 59 | 
 60 | 
 61 | class VerboseNNModule(nn.Module):
 62 |     """
 63 |     Wrapper around nn.Module that prints registered buffers and parameter names.
 64 |     """
 65 | 
 66 |     @staticmethod
 67 |     def get_readable_tensor_repr(name: str, tensor: torch.Tensor) -> str:
 68 |         st = (
 69 |             "("
 70 |             + name
 71 |             + "): "
 72 |             + "tensor("
 73 |             + str(tuple(tensor[1].shape))
 74 |             + ", requires_grad="
 75 |             + str(tensor[1].requires_grad)
 76 |             + ")\n"
 77 |         )
 78 |         return st
 79 | 
 80 |     def extra_repr(self) -> str:
 81 |         named_modules = set()
 82 |         for p in self.named_modules():
 83 |             named_modules.update([p[0]])
 84 |         named_modules = list(named_modules)
 85 | 
 86 |         string_repr = ""
 87 |         for p in self.named_parameters():
 88 |             name = p[0].split(".")[0]
 89 |             if name not in named_modules:
 90 |                 string_repr += self.get_readable_tensor_repr(name, p)
 91 | 
 92 |         for p in self.named_buffers():
 93 |             name = p[0].split(".")[0]
 94 |             string_repr += self.get_readable_tensor_repr(name, p)
 95 | 
 96 |         return string_repr
 97 | 
 98 | 
 99 | def cast_if_src_dtype(
100 |     tensor: torch.Tensor, src_dtype: torch.dtype, tgt_dtype: torch.dtype
101 | ):
102 |     updated = False
103 |     if tensor.dtype == src_dtype:
104 |         tensor = tensor.to(dtype=tgt_dtype)
105 |         updated = True
106 |     return tensor, updated
107 | 
108 | 
109 | class QuickGELU(nn.Module):
110 |     # From https://github.com/openai/CLIP/blob/d50d76daa670286dd6cacf3bcd80b5e4823fc8e1/clip/model.py#L166
111 |     def forward(self, x: torch.Tensor):
112 |         return x * torch.sigmoid(1.702 * x)
113 | 
114 | 
115 | class SelectElement(nn.Module):
116 |     def __init__(self, index) -> None:
117 |         super().__init__()
118 |         self.index = index
119 | 
120 |     def forward(self, x):
121 |         assert x.ndim >= 3
122 |         return x[:, self.index, ...]
123 | 
124 | 
125 | class SelectEOSAndProject(nn.Module):
126 |     """
127 |     Text Pooling used in OpenCLIP
128 |     """
129 | 
130 |     def __init__(self, proj: nn.Module) -> None:
131 |         super().__init__()
132 |         self.proj = proj
133 | 
134 |     def forward(self, x, seq_len):
135 |         assert x.ndim == 3
136 |         # x is of shape B x L x D
137 |         # take features from the eot embedding (eot_token is the highest number in each sequence)
138 |         x = x[torch.arange(x.shape[0]), seq_len]
139 |         x = self.proj(x)
140 |         return x
141 | 


--------------------------------------------------------------------------------
/video_llama/models/ImageBind/requirements.txt:
--------------------------------------------------------------------------------
 1 | --extra-index-url https://download.pytorch.org/whl/cu113
 2 | torch==1.13.0
 3 | torchvision==0.14.0
 4 | torchaudio==0.13.0
 5 | pytorchvideo @ git+https://github.com/facebookresearch/pytorchvideo.git@28fe037d212663c6a24f373b94cc5d478c8c1a1d
 6 | timm==0.6.7
 7 | ftfy
 8 | regex
 9 | einops
10 | fvcore
11 | decord==0.6.0
12 | iopath
13 | numpy
14 | matplotlib
15 | types-regex
16 | mayavi
17 | cartopy
18 | 


--------------------------------------------------------------------------------
/video_llama/models/__init__.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Adapted from salesforce@LAVIS Vision-CAIR@MiniGPT-4. Below is the original copyright:
  3 |  Copyright (c) 2022, salesforce.com, inc.
  4 |  All rights reserved.
  5 |  SPDX-License-Identifier: BSD-3-Clause
  6 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  7 | """
  8 | 
  9 | import logging
 10 | import torch
 11 | from omegaconf import OmegaConf
 12 | 
 13 | from video_llama.common.registry import registry
 14 | from video_llama.models.base_model import BaseModel
 15 | from video_llama.models.blip2 import Blip2Base
 16 | from video_llama.models.video_llama import VideoLLAMA
 17 | from video_llama.processors.base_processor import BaseProcessor
 18 | 
 19 | 
 20 | __all__ = [
 21 |     "load_model",
 22 |     "BaseModel",
 23 |     "Blip2Base",
 24 |     "VideoLLAMA"
 25 | ]
 26 | 
 27 | 
 28 | def load_model(name, model_type, is_eval=False, device="cpu", checkpoint=None):
 29 |     """
 30 |     Load supported models.
 31 | 
 32 |     To list all available models and types in registry:
 33 |     >>> from video_llama.models import model_zoo
 34 |     >>> print(model_zoo)
 35 | 
 36 |     Args:
 37 |         name (str): name of the model.
 38 |         model_type (str): type of the model.
 39 |         is_eval (bool): whether the model is in eval mode. Default: False.
 40 |         device (str): device to use. Default: "cpu".
 41 |         checkpoint (str): path or to checkpoint. Default: None.
 42 |             Note that expecting the checkpoint to have the same keys in state_dict as the model.
 43 | 
 44 |     Returns:
 45 |         model (torch.nn.Module): model.
 46 |     """
 47 | 
 48 |     model = registry.get_model_class(name).from_pretrained(model_type=model_type)
 49 | 
 50 |     if checkpoint is not None:
 51 |         model.load_checkpoint(checkpoint)
 52 | 
 53 |     if is_eval:
 54 |         model.eval()
 55 | 
 56 |     if device == "cpu":
 57 |         model = model.float()
 58 | 
 59 |     return model.to(device)
 60 | 
 61 | 
 62 | def load_preprocess(config):
 63 |     """
 64 |     Load preprocessor configs and construct preprocessors.
 65 | 
 66 |     If no preprocessor is specified, return BaseProcessor, which does not do any preprocessing.
 67 | 
 68 |     Args:
 69 |         config (dict): preprocessor configs.
 70 | 
 71 |     Returns:
 72 |         vis_processors (dict): preprocessors for visual inputs.
 73 |         txt_processors (dict): preprocessors for text inputs.
 74 | 
 75 |         Key is "train" or "eval" for processors used in training and evaluation respectively.
 76 |     """
 77 | 
 78 |     def _build_proc_from_cfg(cfg):
 79 |         return (
 80 |             registry.get_processor_class(cfg.name).from_config(cfg)
 81 |             if cfg is not None
 82 |             else BaseProcessor()
 83 |         )
 84 | 
 85 |     vis_processors = dict()
 86 |     txt_processors = dict()
 87 | 
 88 |     vis_proc_cfg = config.get("vis_processor")
 89 |     txt_proc_cfg = config.get("text_processor")
 90 | 
 91 |     if vis_proc_cfg is not None:
 92 |         vis_train_cfg = vis_proc_cfg.get("train")
 93 |         vis_eval_cfg = vis_proc_cfg.get("eval")
 94 |     else:
 95 |         vis_train_cfg = None
 96 |         vis_eval_cfg = None
 97 | 
 98 |     vis_processors["train"] = _build_proc_from_cfg(vis_train_cfg)
 99 |     vis_processors["eval"] = _build_proc_from_cfg(vis_eval_cfg)
100 | 
101 |     if txt_proc_cfg is not None:
102 |         txt_train_cfg = txt_proc_cfg.get("train")
103 |         txt_eval_cfg = txt_proc_cfg.get("eval")
104 |     else:
105 |         txt_train_cfg = None
106 |         txt_eval_cfg = None
107 | 
108 |     txt_processors["train"] = _build_proc_from_cfg(txt_train_cfg)
109 |     txt_processors["eval"] = _build_proc_from_cfg(txt_eval_cfg)
110 | 
111 |     return vis_processors, txt_processors
112 | 
113 | 
114 | def load_model_and_preprocess(name, model_type, is_eval=False, device="cpu"):
115 |     """
116 |     Load model and its related preprocessors.
117 | 
118 |     List all available models and types in registry:
119 |     >>> from video_llama.models import model_zoo
120 |     >>> print(model_zoo)
121 | 
122 |     Args:
123 |         name (str): name of the model.
124 |         model_type (str): type of the model.
125 |         is_eval (bool): whether the model is in eval mode. Default: False.
126 |         device (str): device to use. Default: "cpu".
127 | 
128 |     Returns:
129 |         model (torch.nn.Module): model.
130 |         vis_processors (dict): preprocessors for visual inputs.
131 |         txt_processors (dict): preprocessors for text inputs.
132 |     """
133 |     model_cls = registry.get_model_class(name)
134 | 
135 |     # load model
136 |     model = model_cls.from_pretrained(model_type=model_type)
137 | 
138 |     if is_eval:
139 |         model.eval()
140 | 
141 |     # load preprocess
142 |     cfg = OmegaConf.load(model_cls.default_config_path(model_type))
143 |     if cfg is not None:
144 |         preprocess_cfg = cfg.preprocess
145 | 
146 |         vis_processors, txt_processors = load_preprocess(preprocess_cfg)
147 |     else:
148 |         vis_processors, txt_processors = None, None
149 |         logging.info(
150 |             f"""No default preprocess for model {name} ({model_type}).
151 |                 This can happen if the model is not finetuned on downstream datasets,
152 |                 or it is not intended for direct use without finetuning.
153 |             """
154 |         )
155 | 
156 |     if device == "cpu" or device == torch.device("cpu"):
157 |         model = model.float()
158 | 
159 |     return model.to(device), vis_processors, txt_processors
160 | 
161 | 
162 | class ModelZoo:
163 |     """
164 |     A utility class to create string representation of available model architectures and types.
165 | 
166 |     >>> from video_llama.models import model_zoo
167 |     >>> # list all available models
168 |     >>> print(model_zoo)
169 |     >>> # show total number of models
170 |     >>> print(len(model_zoo))
171 |     """
172 | 
173 |     def __init__(self) -> None:
174 |         self.model_zoo = {
175 |             k: list(v.PRETRAINED_MODEL_CONFIG_DICT.keys())
176 |             for k, v in registry.mapping["model_name_mapping"].items()
177 |         }
178 | 
179 |     def __str__(self) -> str:
180 |         return (
181 |             "=" * 50
182 |             + "\n"
183 |             + f"{'Architectures':<30} {'Types'}\n"
184 |             + "=" * 50
185 |             + "\n"
186 |             + "\n".join(
187 |                 [
188 |                     f"{name:<30} {', '.join(types)}"
189 |                     for name, types in self.model_zoo.items()
190 |                 ]
191 |             )
192 |         )
193 | 
194 |     def __iter__(self):
195 |         return iter(self.model_zoo.items())
196 | 
197 |     def __len__(self):
198 |         return sum([len(v) for v in self.model_zoo.values()])
199 | 
200 | 
201 | model_zoo = ModelZoo()
202 | 


--------------------------------------------------------------------------------
/video_llama/models/__pycache__/Qformer.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/__pycache__/Qformer.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/__pycache__/base_model.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/__pycache__/base_model.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/__pycache__/blip2.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/__pycache__/blip2.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/__pycache__/eva_vit.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/__pycache__/eva_vit.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/__pycache__/modeling_llama.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/__pycache__/modeling_llama.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/__pycache__/video_llama.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/models/__pycache__/video_llama.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/models/base_model.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Adapted from salesforce@LAVIS. Below is the original copyright:
  3 |  Copyright (c) 2022, salesforce.com, inc.
  4 |  All rights reserved.
  5 |  SPDX-License-Identifier: BSD-3-Clause
  6 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  7 | """
  8 | 
  9 | import logging
 10 | import os
 11 | 
 12 | import numpy as np
 13 | import torch
 14 | import torch.nn as nn
 15 | from video_llama.common.dist_utils import download_cached_file, is_dist_avail_and_initialized
 16 | from video_llama.common.utils import get_abs_path, is_url
 17 | from omegaconf import OmegaConf
 18 | 
 19 | 
 20 | class BaseModel(nn.Module):
 21 |     """Base class for models."""
 22 | 
 23 |     def __init__(self):
 24 |         super().__init__()
 25 | 
 26 |     @property
 27 |     def device(self):
 28 |         return list(self.parameters())[0].device
 29 | 
 30 |     def load_checkpoint(self, url_or_filename):
 31 |         """
 32 |         Load from a finetuned checkpoint.
 33 | 
 34 |         This should expect no mismatch in the model keys and the checkpoint keys.
 35 |         """
 36 | 
 37 |         if is_url(url_or_filename):
 38 |             cached_file = download_cached_file(
 39 |                 url_or_filename, check_hash=False, progress=True
 40 |             )
 41 |             checkpoint = torch.load(cached_file, map_location="cpu")
 42 |         elif os.path.isfile(url_or_filename):
 43 |             checkpoint = torch.load(url_or_filename, map_location="cpu")
 44 |         else:
 45 |             raise RuntimeError("checkpoint url or path is invalid")
 46 | 
 47 |         if "model" in checkpoint.keys():
 48 |             state_dict = checkpoint["model"]
 49 |         else:
 50 |             state_dict = checkpoint
 51 | 
 52 |         msg = self.load_state_dict(state_dict, strict=False)
 53 | 
 54 |         logging.info("Missing keys {}".format(msg.missing_keys))
 55 |         logging.info("load checkpoint from %s" % url_or_filename)
 56 | 
 57 |         return msg
 58 | 
 59 |     @classmethod
 60 |     def from_pretrained(cls, model_type):
 61 |         """
 62 |         Build a pretrained model from default configuration file, specified by model_type.
 63 | 
 64 |         Args:
 65 |             - model_type (str): model type, specifying architecture and checkpoints.
 66 | 
 67 |         Returns:
 68 |             - model (nn.Module): pretrained or finetuned model, depending on the configuration.
 69 |         """
 70 |         model_cfg = OmegaConf.load(cls.default_config_path(model_type)).model
 71 |         model = cls.from_config(model_cfg)
 72 | 
 73 |         return model
 74 | 
 75 |     @classmethod
 76 |     def default_config_path(cls, model_type):
 77 |         assert (
 78 |             model_type in cls.PRETRAINED_MODEL_CONFIG_DICT
 79 |         ), "Unknown model type {}".format(model_type)
 80 |         return get_abs_path(cls.PRETRAINED_MODEL_CONFIG_DICT[model_type])
 81 | 
 82 |     def load_checkpoint_from_config(self, cfg, **kwargs):
 83 |         """
 84 |         Load checkpoint as specified in the config file.
 85 | 
 86 |         If load_finetuned is True, load the finetuned model; otherwise, load the pretrained model.
 87 |         When loading the pretrained model, each task-specific architecture may define their
 88 |         own load_from_pretrained() method.
 89 |         """
 90 |         load_finetuned = cfg.get("load_finetuned", True)
 91 |         if load_finetuned:
 92 |             finetune_path = cfg.get("finetuned", None)
 93 |             assert (
 94 |                 finetune_path is not None
 95 |             ), "Found load_finetuned is True, but finetune_path is None."
 96 |             self.load_checkpoint(url_or_filename=finetune_path)
 97 |         else:
 98 |             # load pre-trained weights
 99 |             pretrain_path = cfg.get("pretrained", None)
100 |             assert "Found load_finetuned is False, but pretrain_path is None."
101 |             self.load_from_pretrained(url_or_filename=pretrain_path, **kwargs)
102 | 
103 |     def before_evaluation(self, **kwargs):
104 |         pass
105 | 
106 |     def show_n_params(self, return_str=True):
107 |         tot = 0
108 |         for p in self.parameters():
109 |             w = 1
110 |             for x in p.shape:
111 |                 w *= x
112 |             tot += w
113 |         if return_str:
114 |             if tot >= 1e6:
115 |                 return "{:.1f}M".format(tot / 1e6)
116 |             else:
117 |                 return "{:.1f}K".format(tot / 1e3)
118 |         else:
119 |             return tot
120 | 
121 | 
122 | class BaseEncoder(nn.Module):
123 |     """
124 |     Base class for primitive encoders, such as ViT, TimeSformer, etc.
125 |     """
126 | 
127 |     def __init__(self):
128 |         super().__init__()
129 | 
130 |     def forward_features(self, samples, **kwargs):
131 |         raise NotImplementedError
132 | 
133 |     @property
134 |     def device(self):
135 |         return list(self.parameters())[0].device
136 | 
137 | 
138 | class SharedQueueMixin:
139 |     @torch.no_grad()
140 |     def _dequeue_and_enqueue(self, image_feat, text_feat, idxs=None):
141 |         # gather keys before updating queue
142 |         image_feats = concat_all_gather(image_feat)
143 |         text_feats = concat_all_gather(text_feat)
144 | 
145 |         batch_size = image_feats.shape[0]
146 | 
147 |         ptr = int(self.queue_ptr)
148 |         assert self.queue_size % batch_size == 0  # for simplicity
149 | 
150 |         # replace the keys at ptr (dequeue and enqueue)
151 |         self.image_queue[:, ptr : ptr + batch_size] = image_feats.T
152 |         self.text_queue[:, ptr : ptr + batch_size] = text_feats.T
153 | 
154 |         if idxs is not None:
155 |             idxs = concat_all_gather(idxs)
156 |             self.idx_queue[:, ptr : ptr + batch_size] = idxs.T
157 | 
158 |         ptr = (ptr + batch_size) % self.queue_size  # move pointer
159 |         self.queue_ptr[0] = ptr
160 | 
161 | 
162 | class MomentumDistilationMixin:
163 |     @torch.no_grad()
164 |     def copy_params(self):
165 |         for model_pair in self.model_pairs:
166 |             for param, param_m in zip(
167 |                 model_pair[0].parameters(), model_pair[1].parameters()
168 |             ):
169 |                 param_m.data.copy_(param.data)  # initialize
170 |                 param_m.requires_grad = False  # not update by gradient
171 | 
172 |     @torch.no_grad()
173 |     def _momentum_update(self):
174 |         for model_pair in self.model_pairs:
175 |             for param, param_m in zip(
176 |                 model_pair[0].parameters(), model_pair[1].parameters()
177 |             ):
178 |                 param_m.data = param_m.data * self.momentum + param.data * (
179 |                     1.0 - self.momentum
180 |                 )
181 | 
182 | 
183 | class GatherLayer(torch.autograd.Function):
184 |     """
185 |     Gather tensors from all workers with support for backward propagation:
186 |     This implementation does not cut the gradients as torch.distributed.all_gather does.
187 |     """
188 | 
189 |     @staticmethod
190 |     def forward(ctx, x):
191 |         output = [
192 |             torch.zeros_like(x) for _ in range(torch.distributed.get_world_size())
193 |         ]
194 |         torch.distributed.all_gather(output, x)
195 |         return tuple(output)
196 | 
197 |     @staticmethod
198 |     def backward(ctx, *grads):
199 |         all_gradients = torch.stack(grads)
200 |         torch.distributed.all_reduce(all_gradients)
201 |         return all_gradients[torch.distributed.get_rank()]
202 | 
203 | 
204 | def all_gather_with_grad(tensors):
205 |     """
206 |     Performs all_gather operation on the provided tensors.
207 |     Graph remains connected for backward grad computation.
208 |     """
209 |     # Queue the gathered tensors
210 |     world_size = torch.distributed.get_world_size()
211 |     # There is no need for reduction in the single-proc case
212 |     if world_size == 1:
213 |         return tensors
214 | 
215 |     # tensor_all = GatherLayer.apply(tensors)
216 |     tensor_all = GatherLayer.apply(tensors)
217 | 
218 |     return torch.cat(tensor_all, dim=0)
219 | 
220 | 
221 | @torch.no_grad()
222 | def concat_all_gather(tensor):
223 |     """
224 |     Performs all_gather operation on the provided tensors.
225 |     *** Warning ***: torch.distributed.all_gather has no gradient.
226 |     """
227 |     # if use distributed training
228 |     if not is_dist_avail_and_initialized():
229 |         return tensor
230 | 
231 |     tensors_gather = [
232 |         torch.ones_like(tensor) for _ in range(torch.distributed.get_world_size())
233 |     ]
234 |     torch.distributed.all_gather(tensors_gather, tensor, async_op=False)
235 | 
236 |     output = torch.cat(tensors_gather, dim=0)
237 |     return output
238 | 
239 | 
240 | def tile(x, dim, n_tile):
241 |     init_dim = x.size(dim)
242 |     repeat_idx = [1] * x.dim()
243 |     repeat_idx[dim] = n_tile
244 |     x = x.repeat(*(repeat_idx))
245 |     order_index = torch.LongTensor(
246 |         np.concatenate([init_dim * np.arange(n_tile) + i for i in range(init_dim)])
247 |     )
248 |     return torch.index_select(x, dim, order_index.to(x.device))
249 | 


--------------------------------------------------------------------------------
/video_llama/models/blip2.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Adapted from salesforce@LAVIS. Below is the original copyright:
  3 |  Copyright (c) 2023, salesforce.com, inc.
  4 |  All rights reserved.
  5 |  SPDX-License-Identifier: BSD-3-Clause
  6 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  7 | """
  8 | import contextlib
  9 | import logging
 10 | import os
 11 | import time
 12 | import datetime
 13 | 
 14 | import torch
 15 | import torch.nn as nn
 16 | import torch.distributed as dist
 17 | import torch.nn.functional as F
 18 | 
 19 | import video_llama.common.dist_utils as dist_utils
 20 | from video_llama.common.dist_utils import download_cached_file
 21 | from video_llama.common.utils import is_url
 22 | from video_llama.common.logger import MetricLogger
 23 | from video_llama.models.base_model import BaseModel
 24 | from video_llama.models.Qformer import BertConfig, BertLMHeadModel
 25 | from video_llama.models.eva_vit import create_eva_vit_g
 26 | from transformers import BertTokenizer
 27 | 
 28 | 
 29 | class Blip2Base(BaseModel):
 30 |     @classmethod
 31 |     def init_tokenizer(cls):
 32 |         tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
 33 |         tokenizer.add_special_tokens({"bos_token": "[DEC]"})
 34 |         return tokenizer
 35 | 
 36 |     def maybe_autocast(self, dtype=torch.float16):
 37 |         # if on cpu, don't use autocast
 38 |         # if on gpu, use autocast with dtype if provided, otherwise use torch.float16
 39 |         enable_autocast = self.device != torch.device("cpu")
 40 | 
 41 |         if enable_autocast:
 42 |             return torch.cuda.amp.autocast(dtype=dtype)
 43 |         else:
 44 |             return contextlib.nullcontext()
 45 | 
 46 |     @classmethod
 47 |     def init_Qformer(cls, num_query_token, vision_width, cross_attention_freq=2):
 48 |         encoder_config = BertConfig.from_pretrained("bert-base-uncased")
 49 |         encoder_config.encoder_width = vision_width
 50 |         # insert cross-attention layer every other block
 51 |         encoder_config.add_cross_attention = True
 52 |         encoder_config.cross_attention_freq = cross_attention_freq
 53 |         encoder_config.query_length = num_query_token
 54 |         Qformer = BertLMHeadModel(config=encoder_config)
 55 |         query_tokens = nn.Parameter(
 56 |             torch.zeros(1, num_query_token, encoder_config.hidden_size)
 57 |         )
 58 |         query_tokens.data.normal_(mean=0.0, std=encoder_config.initializer_range)
 59 |         return Qformer, query_tokens
 60 | 
 61 |     @classmethod
 62 |     def init_vision_encoder(
 63 |         cls, model_name, img_size, drop_path_rate, use_grad_checkpoint, precision
 64 |     ):
 65 |         assert model_name == "eva_clip_g", "vit model must be eva_clip_g for current version of MiniGPT-4"
 66 |         visual_encoder = create_eva_vit_g(
 67 |             img_size, drop_path_rate, use_grad_checkpoint, precision
 68 |         )
 69 | 
 70 |         ln_vision = LayerNorm(visual_encoder.num_features)
 71 |         return visual_encoder, ln_vision
 72 | 
 73 |     def load_from_pretrained(self, url_or_filename):
 74 |         if is_url(url_or_filename):
 75 |             cached_file = download_cached_file(
 76 |                 url_or_filename, check_hash=False, progress=True
 77 |             )
 78 |             checkpoint = torch.load(cached_file, map_location="cpu")
 79 |         elif os.path.isfile(url_or_filename):
 80 |             checkpoint = torch.load(url_or_filename, map_location="cpu")
 81 |         else:
 82 |             raise RuntimeError("checkpoint url or path is invalid")
 83 | 
 84 |         state_dict = checkpoint["model"]
 85 | 
 86 |         msg = self.load_state_dict(state_dict, strict=False)
 87 | 
 88 |         # logging.info("Missing keys {}".format(msg.missing_keys))
 89 |         logging.info("load checkpoint from %s" % url_or_filename)
 90 | 
 91 |         return msg
 92 | 
 93 | 
 94 | def disabled_train(self, mode=True):
 95 |     """Overwrite model.train with this function to make sure train/eval mode
 96 |     does not change anymore."""
 97 |     return self
 98 | 
 99 | 
100 | class LayerNorm(nn.LayerNorm):
101 |     """Subclass torch's LayerNorm to handle fp16."""
102 | 
103 |     def forward(self, x: torch.Tensor):
104 |         orig_type = x.dtype
105 |         ret = super().forward(x.type(torch.float32))
106 |         return ret.type(orig_type)
107 | 
108 | 
109 | def compute_sim_matrix(model, data_loader, **kwargs):
110 |     k_test = kwargs.pop("k_test")
111 | 
112 |     metric_logger = MetricLogger(delimiter="  ")
113 |     header = "Evaluation:"
114 | 
115 |     logging.info("Computing features for evaluation...")
116 |     start_time = time.time()
117 | 
118 |     texts = data_loader.dataset.text
119 |     num_text = len(texts)
120 |     text_bs = 256
121 |     text_ids = []
122 |     text_embeds = []
123 |     text_atts = []
124 |     for i in range(0, num_text, text_bs):
125 |         text = texts[i : min(num_text, i + text_bs)]
126 |         text_input = model.tokenizer(
127 |             text,
128 |             padding="max_length",
129 |             truncation=True,
130 |             max_length=35,
131 |             return_tensors="pt",
132 |         ).to(model.device)
133 |         text_feat = model.forward_text(text_input)
134 |         text_embed = F.normalize(model.text_proj(text_feat))
135 |         text_embeds.append(text_embed)
136 |         text_ids.append(text_input.input_ids)
137 |         text_atts.append(text_input.attention_mask)
138 | 
139 |     text_embeds = torch.cat(text_embeds, dim=0)
140 |     text_ids = torch.cat(text_ids, dim=0)
141 |     text_atts = torch.cat(text_atts, dim=0)
142 | 
143 |     vit_feats = []
144 |     image_embeds = []
145 |     for samples in data_loader:
146 |         image = samples["image"]
147 | 
148 |         image = image.to(model.device)
149 |         image_feat, vit_feat = model.forward_image(image)
150 |         image_embed = model.vision_proj(image_feat)
151 |         image_embed = F.normalize(image_embed, dim=-1)
152 | 
153 |         vit_feats.append(vit_feat.cpu())
154 |         image_embeds.append(image_embed)
155 | 
156 |     vit_feats = torch.cat(vit_feats, dim=0)
157 |     image_embeds = torch.cat(image_embeds, dim=0)
158 | 
159 |     sims_matrix = []
160 |     for image_embed in image_embeds:
161 |         sim_q2t = image_embed @ text_embeds.t()
162 |         sim_i2t, _ = sim_q2t.max(0)
163 |         sims_matrix.append(sim_i2t)
164 |     sims_matrix = torch.stack(sims_matrix, dim=0)
165 | 
166 |     score_matrix_i2t = torch.full(
167 |         (len(data_loader.dataset.image), len(texts)), -100.0
168 |     ).to(model.device)
169 | 
170 |     num_tasks = dist_utils.get_world_size()
171 |     rank = dist_utils.get_rank()
172 |     step = sims_matrix.size(0) // num_tasks + 1
173 |     start = rank * step
174 |     end = min(sims_matrix.size(0), start + step)
175 | 
176 |     for i, sims in enumerate(
177 |         metric_logger.log_every(sims_matrix[start:end], 50, header)
178 |     ):
179 |         topk_sim, topk_idx = sims.topk(k=k_test, dim=0)
180 |         image_inputs = vit_feats[start + i].repeat(k_test, 1, 1).to(model.device)
181 |         score = model.compute_itm(
182 |             image_inputs=image_inputs,
183 |             text_ids=text_ids[topk_idx],
184 |             text_atts=text_atts[topk_idx],
185 |         ).float()
186 |         score_matrix_i2t[start + i, topk_idx] = score + topk_sim
187 | 
188 |     sims_matrix = sims_matrix.t()
189 |     score_matrix_t2i = torch.full(
190 |         (len(texts), len(data_loader.dataset.image)), -100.0
191 |     ).to(model.device)
192 | 
193 |     step = sims_matrix.size(0) // num_tasks + 1
194 |     start = rank * step
195 |     end = min(sims_matrix.size(0), start + step)
196 | 
197 |     for i, sims in enumerate(
198 |         metric_logger.log_every(sims_matrix[start:end], 50, header)
199 |     ):
200 |         topk_sim, topk_idx = sims.topk(k=k_test, dim=0)
201 |         image_inputs = vit_feats[topk_idx.cpu()].to(model.device)
202 |         score = model.compute_itm(
203 |             image_inputs=image_inputs,
204 |             text_ids=text_ids[start + i].repeat(k_test, 1),
205 |             text_atts=text_atts[start + i].repeat(k_test, 1),
206 |         ).float()
207 |         score_matrix_t2i[start + i, topk_idx] = score + topk_sim
208 | 
209 |     if dist_utils.is_dist_avail_and_initialized():
210 |         dist.barrier()
211 |         torch.distributed.all_reduce(
212 |             score_matrix_i2t, op=torch.distributed.ReduceOp.SUM
213 |         )
214 |         torch.distributed.all_reduce(
215 |             score_matrix_t2i, op=torch.distributed.ReduceOp.SUM
216 |         )
217 | 
218 |     total_time = time.time() - start_time
219 |     total_time_str = str(datetime.timedelta(seconds=int(total_time)))
220 |     logging.info("Evaluation time {}".format(total_time_str))
221 | 
222 |     return score_matrix_i2t.cpu().numpy(), score_matrix_t2i.cpu().numpy()
223 | 


--------------------------------------------------------------------------------
/video_llama/models/blip2_outputs.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Adapted from salesforce@LAVIS. Below is the original copyright:
  3 |  Copyright (c) 2022, salesforce.com, inc.
  4 |  All rights reserved.
  5 |  SPDX-License-Identifier: BSD-3-Clause
  6 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  7 | """
  8 | 
  9 | from dataclasses import dataclass
 10 | from typing import Optional
 11 | 
 12 | import torch
 13 | from transformers.modeling_outputs import (
 14 |     ModelOutput,
 15 |     BaseModelOutputWithPoolingAndCrossAttentions,
 16 |     CausalLMOutputWithCrossAttentions,
 17 | )
 18 | 
 19 | 
 20 | @dataclass
 21 | class BlipSimilarity(ModelOutput):
 22 |     sim_i2t: torch.FloatTensor = None
 23 |     sim_t2i: torch.FloatTensor = None
 24 | 
 25 |     sim_i2t_m: Optional[torch.FloatTensor] = None
 26 |     sim_t2i_m: Optional[torch.FloatTensor] = None
 27 | 
 28 |     sim_i2t_targets: Optional[torch.FloatTensor] = None
 29 |     sim_t2i_targets: Optional[torch.FloatTensor] = None
 30 | 
 31 | 
 32 | @dataclass
 33 | class BlipIntermediateOutput(ModelOutput):
 34 |     """
 35 |     Data class for intermediate outputs of BLIP models.
 36 | 
 37 |     image_embeds (torch.FloatTensor): Image embeddings, shape (batch_size, num_patches, embed_dim).
 38 |     text_embeds (torch.FloatTensor): Text embeddings, shape (batch_size, seq_len, embed_dim).
 39 | 
 40 |     image_embeds_m (torch.FloatTensor): Image embeddings from momentum visual encoder, shape (batch_size, num_patches, embed_dim).
 41 |     text_embeds_m (torch.FloatTensor): Text embeddings from momentum text encoder, shape (batch_size, seq_len, embed_dim).
 42 | 
 43 |     encoder_output (BaseModelOutputWithPoolingAndCrossAttentions): output from the image-grounded text encoder.
 44 |     encoder_output_neg (BaseModelOutputWithPoolingAndCrossAttentions): output from the image-grounded text encoder for negative pairs.
 45 | 
 46 |     decoder_output (CausalLMOutputWithCrossAttentions): output from the image-grounded text decoder.
 47 |     decoder_labels (torch.LongTensor): labels for the captioning loss.
 48 | 
 49 |     itm_logits (torch.FloatTensor): logits for the image-text matching loss, shape (batch_size * 3, 2).
 50 |     itm_labels (torch.LongTensor): labels for the image-text matching loss, shape (batch_size * 3,)
 51 | 
 52 |     """
 53 | 
 54 |     # uni-modal features
 55 |     image_embeds: torch.FloatTensor = None
 56 |     text_embeds: Optional[torch.FloatTensor] = None
 57 | 
 58 |     image_embeds_m: Optional[torch.FloatTensor] = None
 59 |     text_embeds_m: Optional[torch.FloatTensor] = None
 60 | 
 61 |     # intermediate outputs of multimodal encoder
 62 |     encoder_output: Optional[BaseModelOutputWithPoolingAndCrossAttentions] = None
 63 |     encoder_output_neg: Optional[BaseModelOutputWithPoolingAndCrossAttentions] = None
 64 | 
 65 |     itm_logits: Optional[torch.FloatTensor] = None
 66 |     itm_labels: Optional[torch.LongTensor] = None
 67 | 
 68 |     # intermediate outputs of multimodal decoder
 69 |     decoder_output: Optional[CausalLMOutputWithCrossAttentions] = None
 70 |     decoder_labels: Optional[torch.LongTensor] = None
 71 | 
 72 | 
 73 | @dataclass
 74 | class BlipOutput(ModelOutput):
 75 |     # some finetuned models (e.g. BlipVQA) do not compute similarity, thus optional.
 76 |     sims: Optional[BlipSimilarity] = None
 77 | 
 78 |     intermediate_output: BlipIntermediateOutput = None
 79 | 
 80 |     loss: Optional[torch.FloatTensor] = None
 81 | 
 82 |     loss_itc: Optional[torch.FloatTensor] = None
 83 | 
 84 |     loss_itm: Optional[torch.FloatTensor] = None
 85 | 
 86 |     loss_lm: Optional[torch.FloatTensor] = None
 87 | 
 88 | 
 89 | @dataclass
 90 | class BlipOutputFeatures(ModelOutput):
 91 |     """
 92 |     Data class of features from BlipFeatureExtractor.
 93 | 
 94 |     Args:
 95 |         image_embeds: (torch.FloatTensor) of shape (batch_size, num_patches+1, embed_dim), optional
 96 |         image_features: (torch.FloatTensor) of shape (batch_size, num_patches+1, feature_dim), optional
 97 |         text_embeds: (torch.FloatTensor) of shape (batch_size, sequence_length+1, embed_dim), optional
 98 |         text_features: (torch.FloatTensor) of shape (batch_size, sequence_length+1, feature_dim), optional
 99 | 
100 |         The first embedding or feature is for the [CLS] token.
101 | 
102 |         Features are obtained by projecting the corresponding embedding into a normalized low-dimensional space.
103 |     """
104 | 
105 |     image_embeds: Optional[torch.FloatTensor] = None
106 |     image_embeds_proj: Optional[torch.FloatTensor] = None
107 | 
108 |     text_embeds: Optional[torch.FloatTensor] = None
109 |     text_embeds_proj: Optional[torch.FloatTensor] = None
110 | 
111 |     multimodal_embeds: Optional[torch.FloatTensor] = None
112 | 


--------------------------------------------------------------------------------
/video_llama/processors/.ipynb_checkpoints/video_processor-checkpoint.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import torch
  9 | from video_llama.common.registry import registry
 10 | from decord import VideoReader
 11 | import decord
 12 | import numpy as np
 13 | from video_llama.processors import transforms_video
 14 | from video_llama.processors.base_processor import BaseProcessor
 15 | from video_llama.processors.randaugment import VideoRandomAugment
 16 | from video_llama.processors import functional_video as F
 17 | from omegaconf import OmegaConf
 18 | from torchvision import transforms
 19 | import random as rnd
 20 | MAX_INT = registry.get("MAX_INT")
 21 | 
 22 | def load_video(video_path, n_frms=MAX_INT, height=-1, width=-1, sampling="uniform"):
 23 |     vr = VideoReader(uri=video_path, height=height, width=width)
 24 | 
 25 |     vlen = len(vr)
 26 |     start, end = 0, vlen
 27 | 
 28 |     n_frms = min(n_frms, vlen)
 29 | 
 30 |     if sampling == "uniform":
 31 |         indices = np.arange(start, end, vlen / n_frms).astype(int).tolist()
 32 |     elif sampling == "headtail":
 33 |         indices_h = sorted(rnd.sample(range(vlen // 2), n_frms // 2))
 34 |         indices_t = sorted(rnd.sample(range(vlen // 2, vlen), n_frms // 2))
 35 |         indices = indices_h + indices_t
 36 |     else:
 37 |         raise NotImplementedError
 38 | 
 39 |     # get_batch -> T, H, W, C
 40 |     print(video_path)
 41 |     print(indices)
 42 |     print(vr.get_batch(indices))
 43 | 
 44 |     frms = vr.get_batch(indices).permute(3, 0, 1, 2).float()  # (C, T, H, W)
 45 |     # print(111)
 46 |     return frms
 47 | 
 48 | class AlproVideoBaseProcessor(BaseProcessor):
 49 |     def __init__(self, mean=None, std=None, n_frms=MAX_INT):
 50 |         if mean is None:
 51 |             mean = (0.48145466, 0.4578275, 0.40821073)
 52 |         if std is None:
 53 |             std = (0.26862954, 0.26130258, 0.27577711)
 54 | 
 55 |         self.normalize = transforms_video.NormalizeVideo(mean, std)
 56 | 
 57 |         self.n_frms = n_frms
 58 | 
 59 | 
 60 | class ToUint8(object):
 61 |     def __init__(self):
 62 |         pass
 63 | 
 64 |     def __call__(self, tensor):
 65 |         return tensor.to(torch.uint8)
 66 | 
 67 |     def __repr__(self):
 68 |         return self.__class__.__name__
 69 | 
 70 | 
 71 | class ToTHWC(object):
 72 |     """
 73 |     Args:
 74 |         clip (torch.tensor, dtype=torch.uint8): Size is (C, T, H, W)
 75 |     Return:
 76 |         clip (torch.tensor, dtype=torch.float): Size is (T, H, W, C)
 77 |     """
 78 | 
 79 |     def __init__(self):
 80 |         pass
 81 | 
 82 |     def __call__(self, tensor):
 83 |         return tensor.permute(1, 2, 3, 0)
 84 | 
 85 |     def __repr__(self):
 86 |         return self.__class__.__name__
 87 | 
 88 | 
 89 | class ResizeVideo(object):
 90 |     def __init__(self, target_size, interpolation_mode="bilinear"):
 91 |         self.target_size = target_size
 92 |         self.interpolation_mode = interpolation_mode
 93 | 
 94 |     def __call__(self, clip):
 95 |         """
 96 |         Args:
 97 |             clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
 98 |         Returns:
 99 |             torch.tensor: central cropping of video clip. Size is
100 |             (C, T, crop_size, crop_size)
101 |         """
102 |         return F.resize(clip, self.target_size, self.interpolation_mode)
103 | 
104 |     def __repr__(self):
105 |         return self.__class__.__name__ + "(resize_size={0})".format(self.target_size)
106 | 
107 | 
108 | @registry.register_processor("alpro_video_train")
109 | class AlproVideoTrainProcessor(AlproVideoBaseProcessor):
110 |     def __init__(
111 |         self,
112 |         image_size=384,
113 |         mean=None,
114 |         std=None,
115 |         min_scale=0.5,
116 |         max_scale=1.0,
117 |         n_frms=MAX_INT,
118 |     ):
119 |         super().__init__(mean=mean, std=std, n_frms=n_frms)
120 | 
121 |         self.image_size = image_size
122 | 
123 |         self.transform = transforms.Compose(
124 |             [
125 |                 # Video size is (C, T, H, W)
126 |                 transforms_video.RandomResizedCropVideo(
127 |                     image_size,
128 |                     scale=(min_scale, max_scale),
129 |                     interpolation_mode="bicubic",
130 |                 ),
131 |                 transforms_video.RandomHorizontalFlipVideo(),
132 |                 ToTHWC(),  # C, T, H, W -> T, H, W, C
133 |                 VideoRandomAugment(
134 |                     2,
135 |                     5,
136 |                     augs=[
137 |                         "Identity",
138 |                         "AutoContrast",
139 |                         "Brightness",
140 |                         "Sharpness",
141 |                         "Equalize",
142 |                         "ShearX",
143 |                         "ShearY",
144 |                         "TranslateX",
145 |                         "TranslateY",
146 |                         "Rotate",
147 |                     ],
148 |                 ),
149 |                 ToUint8(),
150 |                 transforms_video.ToTensorVideo(),  # T, H, W, C -> C, T, H, W
151 |                 self.normalize,
152 |             ]
153 |         )
154 | 
155 |     def __call__(self, vpath):
156 |         """
157 |         Args:
158 |             clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
159 |         Returns:
160 |             torch.tensor: video clip after transforms. Size is (C, T, size, size).
161 |         """
162 |         clip = load_video(
163 |             video_path=vpath,
164 |             n_frms=self.n_frms,
165 |             height=self.image_size,
166 |             width=self.image_size,
167 |             sampling="headtail",
168 |         )
169 | 
170 |         return self.transform(clip)
171 | 
172 |     @classmethod
173 |     def from_config(cls, cfg=None):
174 |         if cfg is None:
175 |             cfg = OmegaConf.create()
176 | 
177 |         image_size = cfg.get("image_size", 256)
178 | 
179 |         mean = cfg.get("mean", None)
180 |         std = cfg.get("std", None)
181 | 
182 |         min_scale = cfg.get("min_scale", 0.5)
183 |         max_scale = cfg.get("max_scale", 1.0)
184 | 
185 |         n_frms = cfg.get("n_frms", MAX_INT)
186 | 
187 |         return cls(
188 |             image_size=image_size,
189 |             mean=mean,
190 |             std=std,
191 |             min_scale=min_scale,
192 |             max_scale=max_scale,
193 |             n_frms=n_frms,
194 |         )
195 | 
196 | 
197 | @registry.register_processor("alpro_video_eval")
198 | class AlproVideoEvalProcessor(AlproVideoBaseProcessor):
199 |     def __init__(self, image_size=256, mean=None, std=None, n_frms=MAX_INT):
200 |         super().__init__(mean=mean, std=std, n_frms=n_frms)
201 | 
202 |         self.image_size = image_size
203 | 
204 |         # Input video size is (C, T, H, W)
205 |         self.transform = transforms.Compose(
206 |             [
207 |                 # frames will be resized during decord loading.
208 |                 ToUint8(),  # C, T, H, W
209 |                 ToTHWC(),  # T, H, W, C
210 |                 transforms_video.ToTensorVideo(),  # C, T, H, W
211 |                 self.normalize,  # C, T, H, W
212 |             ]
213 |         )
214 | 
215 |     def __call__(self, vpath):
216 |         """
217 |         Args:
218 |             clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
219 |         Returns:
220 |             torch.tensor: video clip after transforms. Size is (C, T, size, size).
221 |         """
222 |         clip = load_video(
223 |             video_path=vpath,
224 |             n_frms=self.n_frms,
225 |             height=self.image_size,
226 |             width=self.image_size,
227 |         )
228 | 
229 |         return self.transform(clip)
230 | 
231 |     @classmethod
232 |     def from_config(cls, cfg=None):
233 |         if cfg is None:
234 |             cfg = OmegaConf.create()
235 | 
236 |         image_size = cfg.get("image_size", 256)
237 | 
238 |         mean = cfg.get("mean", None)
239 |         std = cfg.get("std", None)
240 | 
241 |         n_frms = cfg.get("n_frms", MAX_INT)
242 | 
243 |         return cls(image_size=image_size, mean=mean, std=std, n_frms=n_frms)
244 | 


--------------------------------------------------------------------------------
/video_llama/processors/__init__.py:
--------------------------------------------------------------------------------
 1 | """
 2 |  Copyright (c) 2022, salesforce.com, inc.
 3 |  All rights reserved.
 4 |  SPDX-License-Identifier: BSD-3-Clause
 5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 6 | """
 7 | 
 8 | from video_llama.processors.base_processor import BaseProcessor
 9 | from video_llama.processors.blip_processors import (
10 |     Blip2ImageTrainProcessor,
11 |     Blip2ImageEvalProcessor,
12 |     BlipCaptionProcessor,
13 | )
14 | from video_llama.processors.video_processor import (
15 |     AlproVideoTrainProcessor,
16 |     AlproVideoEvalProcessor
17 | )
18 | from video_llama.common.registry import registry
19 | 
20 | __all__ = [
21 |     "BaseProcessor",
22 |     "Blip2ImageTrainProcessor",
23 |     "Blip2ImageEvalProcessor",
24 |     "BlipCaptionProcessor",
25 |     "AlproVideoTrainProcessor",
26 |     "AlproVideoEvalProcessor",
27 | ]
28 | 
29 | 
30 | def load_processor(name, cfg=None):
31 |     """
32 |     Example
33 | 
34 |     >>> processor = load_processor("alpro_video_train", cfg=None)
35 |     """
36 |     processor = registry.get_processor_class(name).from_config(cfg)
37 | 
38 |     return processor
39 | 


--------------------------------------------------------------------------------
/video_llama/processors/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/processors/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/processors/__pycache__/base_processor.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/processors/__pycache__/base_processor.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/processors/__pycache__/blip_processors.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/processors/__pycache__/blip_processors.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/processors/__pycache__/functional_video.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/processors/__pycache__/functional_video.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/processors/__pycache__/randaugment.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/processors/__pycache__/randaugment.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/processors/__pycache__/transforms_video.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/processors/__pycache__/transforms_video.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/processors/__pycache__/video_processor.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/processors/__pycache__/video_processor.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/processors/base_processor.py:
--------------------------------------------------------------------------------
 1 | """
 2 |  Copyright (c) 2022, salesforce.com, inc.
 3 |  All rights reserved.
 4 |  SPDX-License-Identifier: BSD-3-Clause
 5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 6 | """
 7 | 
 8 | from omegaconf import OmegaConf
 9 | 
10 | 
11 | class BaseProcessor:
12 |     def __init__(self):
13 |         self.transform = lambda x: x
14 |         return
15 | 
16 |     def __call__(self, item):
17 |         return self.transform(item)
18 | 
19 |     @classmethod
20 |     def from_config(cls, cfg=None):
21 |         return cls()
22 | 
23 |     def build(self, **kwargs):
24 |         cfg = OmegaConf.create(kwargs)
25 | 
26 |         return self.from_config(cfg)
27 | 


--------------------------------------------------------------------------------
/video_llama/processors/blip_processors.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import re
  9 | 
 10 | from video_llama.common.registry import registry
 11 | from video_llama.processors.base_processor import BaseProcessor
 12 | from video_llama.processors.randaugment import RandomAugment
 13 | from omegaconf import OmegaConf
 14 | from torchvision import transforms
 15 | from torchvision.transforms.functional import InterpolationMode
 16 | 
 17 | 
 18 | class BlipImageBaseProcessor(BaseProcessor):
 19 |     def __init__(self, mean=None, std=None):
 20 |         if mean is None:
 21 |             mean = (0.48145466, 0.4578275, 0.40821073)
 22 |         if std is None:
 23 |             std = (0.26862954, 0.26130258, 0.27577711)
 24 | 
 25 |         self.normalize = transforms.Normalize(mean, std)
 26 | 
 27 | 
 28 | @registry.register_processor("blip_caption")
 29 | class BlipCaptionProcessor(BaseProcessor):
 30 |     def __init__(self, prompt="", max_words=50):
 31 |         self.prompt = prompt
 32 |         self.max_words = max_words
 33 | 
 34 |     def __call__(self, caption):
 35 |         caption = self.prompt + self.pre_caption(caption)
 36 | 
 37 |         return caption
 38 | 
 39 |     @classmethod
 40 |     def from_config(cls, cfg=None):
 41 |         if cfg is None:
 42 |             cfg = OmegaConf.create()
 43 | 
 44 |         prompt = cfg.get("prompt", "")
 45 |         max_words = cfg.get("max_words", 50)
 46 | 
 47 |         return cls(prompt=prompt, max_words=max_words)
 48 | 
 49 |     def pre_caption(self, caption):
 50 |         caption = re.sub(
 51 |             r"([.!\"()*#:;~])",
 52 |             " ",
 53 |             caption.lower(),
 54 |         )
 55 |         caption = re.sub(
 56 |             r"\s{2,}",
 57 |             " ",
 58 |             caption,
 59 |         )
 60 |         caption = caption.rstrip("\n")
 61 |         caption = caption.strip(" ")
 62 | 
 63 |         # truncate caption
 64 |         caption_words = caption.split(" ")
 65 |         if len(caption_words) > self.max_words:
 66 |             caption = " ".join(caption_words[: self.max_words])
 67 | 
 68 |         return caption
 69 | 
 70 | 
 71 | @registry.register_processor("blip2_image_train")
 72 | class Blip2ImageTrainProcessor(BlipImageBaseProcessor):
 73 |     def __init__(self, image_size=224, mean=None, std=None, min_scale=0.5, max_scale=1.0):
 74 |         super().__init__(mean=mean, std=std)
 75 | 
 76 |         self.transform = transforms.Compose(
 77 |             [
 78 |                 transforms.RandomResizedCrop(
 79 |                     image_size,
 80 |                     scale=(min_scale, max_scale),
 81 |                     interpolation=InterpolationMode.BICUBIC,
 82 |                 ),
 83 |                 transforms.ToTensor(),
 84 |                 self.normalize,
 85 |             ]
 86 |         )
 87 | 
 88 |     def __call__(self, item):
 89 |         return self.transform(item)
 90 | 
 91 |     @classmethod
 92 |     def from_config(cls, cfg=None):
 93 |         if cfg is None:
 94 |             cfg = OmegaConf.create()
 95 | 
 96 |         image_size = cfg.get("image_size", 224)
 97 | 
 98 |         mean = cfg.get("mean", None)
 99 |         std = cfg.get("std", None)
100 | 
101 |         min_scale = cfg.get("min_scale", 0.5)
102 |         max_scale = cfg.get("max_scale", 1.0)
103 | 
104 |         return cls(
105 |             image_size=image_size,
106 |             mean=mean,
107 |             std=std,
108 |             min_scale=min_scale,
109 |             max_scale=max_scale,
110 |         )
111 | 
112 | 
113 | @registry.register_processor("blip2_image_eval")
114 | class Blip2ImageEvalProcessor(BlipImageBaseProcessor):
115 |     def __init__(self, image_size=224, mean=None, std=None):
116 |         super().__init__(mean=mean, std=std)
117 | 
118 |         self.transform = transforms.Compose(
119 |             [
120 |                 transforms.Resize(
121 |                     (image_size, image_size), interpolation=InterpolationMode.BICUBIC
122 |                 ),
123 |                 transforms.ToTensor(),
124 |                 self.normalize,
125 |             ]
126 |         )
127 | 
128 |     def __call__(self, item):
129 |         return self.transform(item)
130 | 
131 |     @classmethod
132 |     def from_config(cls, cfg=None):
133 |         if cfg is None:
134 |             cfg = OmegaConf.create()
135 | 
136 |         image_size = cfg.get("image_size", 224)
137 | 
138 |         mean = cfg.get("mean", None)
139 |         std = cfg.get("std", None)
140 | 
141 |         return cls(image_size=image_size, mean=mean, std=std)
142 | 
143 | 


--------------------------------------------------------------------------------
/video_llama/processors/functional_video.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import warnings
  9 | 
 10 | import torch
 11 | 
 12 | 
 13 | def _is_tensor_video_clip(clip):
 14 |     if not torch.is_tensor(clip):
 15 |         raise TypeError("clip should be Tensor. Got %s" % type(clip))
 16 | 
 17 |     if not clip.ndimension() == 4:
 18 |         raise ValueError("clip should be 4D. Got %dD" % clip.dim())
 19 | 
 20 |     return True
 21 | 
 22 | 
 23 | def crop(clip, i, j, h, w):
 24 |     """
 25 |     Args:
 26 |         clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
 27 |     """
 28 |     if len(clip.size()) != 4:
 29 |         raise ValueError("clip should be a 4D tensor")
 30 |     return clip[..., i : i + h, j : j + w]
 31 | 
 32 | 
 33 | def resize(clip, target_size, interpolation_mode):
 34 |     if len(target_size) != 2:
 35 |         raise ValueError(
 36 |             f"target size should be tuple (height, width), instead got {target_size}"
 37 |         )
 38 |     return torch.nn.functional.interpolate(
 39 |         clip, size=target_size, mode=interpolation_mode, align_corners=False
 40 |     )
 41 | 
 42 | 
 43 | def resized_crop(clip, i, j, h, w, size, interpolation_mode="bilinear"):
 44 |     """
 45 |     Do spatial cropping and resizing to the video clip
 46 |     Args:
 47 |         clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
 48 |         i (int): i in (i,j) i.e coordinates of the upper left corner.
 49 |         j (int): j in (i,j) i.e coordinates of the upper left corner.
 50 |         h (int): Height of the cropped region.
 51 |         w (int): Width of the cropped region.
 52 |         size (tuple(int, int)): height and width of resized clip
 53 |     Returns:
 54 |         clip (torch.tensor): Resized and cropped clip. Size is (C, T, H, W)
 55 |     """
 56 |     if not _is_tensor_video_clip(clip):
 57 |         raise ValueError("clip should be a 4D torch.tensor")
 58 |     clip = crop(clip, i, j, h, w)
 59 |     clip = resize(clip, size, interpolation_mode)
 60 |     return clip
 61 | 
 62 | 
 63 | def center_crop(clip, crop_size):
 64 |     if not _is_tensor_video_clip(clip):
 65 |         raise ValueError("clip should be a 4D torch.tensor")
 66 |     h, w = clip.size(-2), clip.size(-1)
 67 |     th, tw = crop_size
 68 |     if h < th or w < tw:
 69 |         raise ValueError("height and width must be no smaller than crop_size")
 70 | 
 71 |     i = int(round((h - th) / 2.0))
 72 |     j = int(round((w - tw) / 2.0))
 73 |     return crop(clip, i, j, th, tw)
 74 | 
 75 | 
 76 | def to_tensor(clip):
 77 |     """
 78 |     Convert tensor data type from uint8 to float, divide value by 255.0 and
 79 |     permute the dimensions of clip tensor
 80 |     Args:
 81 |         clip (torch.tensor, dtype=torch.uint8): Size is (T, H, W, C)
 82 |     Return:
 83 |         clip (torch.tensor, dtype=torch.float): Size is (C, T, H, W)
 84 |     """
 85 |     _is_tensor_video_clip(clip)
 86 |     if not clip.dtype == torch.uint8:
 87 |         raise TypeError(
 88 |             "clip tensor should have data type uint8. Got %s" % str(clip.dtype)
 89 |         )
 90 |     return clip.float().permute(3, 0, 1, 2) / 255.0
 91 | 
 92 | 
 93 | def normalize(clip, mean, std, inplace=False):
 94 |     """
 95 |     Args:
 96 |         clip (torch.tensor): Video clip to be normalized. Size is (C, T, H, W)
 97 |         mean (tuple): pixel RGB mean. Size is (3)
 98 |         std (tuple): pixel standard deviation. Size is (3)
 99 |     Returns:
100 |         normalized clip (torch.tensor): Size is (C, T, H, W)
101 |     """
102 |     if not _is_tensor_video_clip(clip):
103 |         raise ValueError("clip should be a 4D torch.tensor")
104 |     if not inplace:
105 |         clip = clip.clone()
106 |     mean = torch.as_tensor(mean, dtype=clip.dtype, device=clip.device)
107 |     std = torch.as_tensor(std, dtype=clip.dtype, device=clip.device)
108 |     clip.sub_(mean[:, None, None, None]).div_(std[:, None, None, None])
109 |     return clip
110 | 
111 | 
112 | def hflip(clip):
113 |     """
114 |     Args:
115 |         clip (torch.tensor): Video clip to be normalized. Size is (C, T, H, W)
116 |     Returns:
117 |         flipped clip (torch.tensor): Size is (C, T, H, W)
118 |     """
119 |     if not _is_tensor_video_clip(clip):
120 |         raise ValueError("clip should be a 4D torch.tensor")
121 |     return clip.flip(-1)
122 | 


--------------------------------------------------------------------------------
/video_llama/processors/transforms_video.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | """
  3 |  Copyright (c) 2022, salesforce.com, inc.
  4 |  All rights reserved.
  5 |  SPDX-License-Identifier: BSD-3-Clause
  6 |  For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  7 | """
  8 | 
  9 | 
 10 | import numbers
 11 | import random
 12 | 
 13 | from torchvision.transforms import (
 14 |     RandomCrop,
 15 |     RandomResizedCrop,
 16 | )
 17 | 
 18 | import video_llama.processors.functional_video as F
 19 | 
 20 | 
 21 | __all__ = [
 22 |     "RandomCropVideo",
 23 |     "RandomResizedCropVideo",
 24 |     "CenterCropVideo",
 25 |     "NormalizeVideo",
 26 |     "ToTensorVideo",
 27 |     "RandomHorizontalFlipVideo",
 28 | ]
 29 | 
 30 | 
 31 | class RandomCropVideo(RandomCrop):
 32 |     def __init__(self, size):
 33 |         if isinstance(size, numbers.Number):
 34 |             self.size = (int(size), int(size))
 35 |         else:
 36 |             self.size = size
 37 | 
 38 |     def __call__(self, clip):
 39 |         """
 40 |         Args:
 41 |             clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
 42 |         Returns:
 43 |             torch.tensor: randomly cropped/resized video clip.
 44 |                 size is (C, T, OH, OW)
 45 |         """
 46 |         i, j, h, w = self.get_params(clip, self.size)
 47 |         return F.crop(clip, i, j, h, w)
 48 | 
 49 |     def __repr__(self) -> str:
 50 |         return f"{self.__class__.__name__}(size={self.size})"
 51 | 
 52 | 
 53 | class RandomResizedCropVideo(RandomResizedCrop):
 54 |     def __init__(
 55 |         self,
 56 |         size,
 57 |         scale=(0.08, 1.0),
 58 |         ratio=(3.0 / 4.0, 4.0 / 3.0),
 59 |         interpolation_mode="bilinear",
 60 |     ):
 61 |         if isinstance(size, tuple):
 62 |             if len(size) != 2:
 63 |                 raise ValueError(
 64 |                     f"size should be tuple (height, width), instead got {size}"
 65 |                 )
 66 |             self.size = size
 67 |         else:
 68 |             self.size = (size, size)
 69 | 
 70 |         self.interpolation_mode = interpolation_mode
 71 |         self.scale = scale
 72 |         self.ratio = ratio
 73 | 
 74 |     def __call__(self, clip):
 75 |         """
 76 |         Args:
 77 |             clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
 78 |         Returns:
 79 |             torch.tensor: randomly cropped/resized video clip.
 80 |                 size is (C, T, H, W)
 81 |         """
 82 |         i, j, h, w = self.get_params(clip, self.scale, self.ratio)
 83 |         return F.resized_crop(clip, i, j, h, w, self.size, self.interpolation_mode)
 84 | 
 85 |     def __repr__(self) -> str:
 86 |         return f"{self.__class__.__name__}(size={self.size}, interpolation_mode={self.interpolation_mode}, scale={self.scale}, ratio={self.ratio})"
 87 | 
 88 | 
 89 | class CenterCropVideo:
 90 |     def __init__(self, crop_size):
 91 |         if isinstance(crop_size, numbers.Number):
 92 |             self.crop_size = (int(crop_size), int(crop_size))
 93 |         else:
 94 |             self.crop_size = crop_size
 95 | 
 96 |     def __call__(self, clip):
 97 |         """
 98 |         Args:
 99 |             clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
100 |         Returns:
101 |             torch.tensor: central cropping of video clip. Size is
102 |             (C, T, crop_size, crop_size)
103 |         """
104 |         return F.center_crop(clip, self.crop_size)
105 | 
106 |     def __repr__(self) -> str:
107 |         return f"{self.__class__.__name__}(crop_size={self.crop_size})"
108 | 
109 | 
110 | class NormalizeVideo:
111 |     """
112 |     Normalize the video clip by mean subtraction and division by standard deviation
113 |     Args:
114 |         mean (3-tuple): pixel RGB mean
115 |         std (3-tuple): pixel RGB standard deviation
116 |         inplace (boolean): whether do in-place normalization
117 |     """
118 | 
119 |     def __init__(self, mean, std, inplace=False):
120 |         self.mean = mean
121 |         self.std = std
122 |         self.inplace = inplace
123 | 
124 |     def __call__(self, clip):
125 |         """
126 |         Args:
127 |             clip (torch.tensor): video clip to be normalized. Size is (C, T, H, W)
128 |         """
129 |         return F.normalize(clip, self.mean, self.std, self.inplace)
130 | 
131 |     def __repr__(self) -> str:
132 |         return f"{self.__class__.__name__}(mean={self.mean}, std={self.std}, inplace={self.inplace})"
133 | 
134 | 
135 | class ToTensorVideo:
136 |     """
137 |     Convert tensor data type from uint8 to float, divide value by 255.0 and
138 |     permute the dimensions of clip tensor
139 |     """
140 | 
141 |     def __init__(self):
142 |         pass
143 | 
144 |     def __call__(self, clip):
145 |         """
146 |         Args:
147 |             clip (torch.tensor, dtype=torch.uint8): Size is (T, H, W, C)
148 |         Return:
149 |             clip (torch.tensor, dtype=torch.float): Size is (C, T, H, W)
150 |         """
151 |         return F.to_tensor(clip)
152 | 
153 |     def __repr__(self) -> str:
154 |         return self.__class__.__name__
155 | 
156 | 
157 | class RandomHorizontalFlipVideo:
158 |     """
159 |     Flip the video clip along the horizonal direction with a given probability
160 |     Args:
161 |         p (float): probability of the clip being flipped. Default value is 0.5
162 |     """
163 | 
164 |     def __init__(self, p=0.5):
165 |         self.p = p
166 | 
167 |     def __call__(self, clip):
168 |         """
169 |         Args:
170 |             clip (torch.tensor): Size is (C, T, H, W)
171 |         Return:
172 |             clip (torch.tensor): Size is (C, T, H, W)
173 |         """
174 |         if random.random() < self.p:
175 |             clip = F.hflip(clip)
176 |         return clip
177 | 
178 |     def __repr__(self) -> str:
179 |         return f"{self.__class__.__name__}(p={self.p})"
180 | 


--------------------------------------------------------------------------------
/video_llama/processors/video_processor.py:
--------------------------------------------------------------------------------
  1 | """
  2 |  Copyright (c) 2022, salesforce.com, inc.
  3 |  All rights reserved.
  4 |  SPDX-License-Identifier: BSD-3-Clause
  5 |  For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  6 | """
  7 | 
  8 | import torch
  9 | from video_llama.common.registry import registry
 10 | from decord import VideoReader
 11 | import decord
 12 | import numpy as np
 13 | from video_llama.processors import transforms_video
 14 | from video_llama.processors.base_processor import BaseProcessor
 15 | from video_llama.processors.randaugment import VideoRandomAugment
 16 | from video_llama.processors import functional_video as F
 17 | from omegaconf import OmegaConf
 18 | from torchvision import transforms
 19 | import random as rnd
 20 | 
 21 | 
 22 | MAX_INT = registry.get("MAX_INT")
 23 | decord.bridge.set_bridge("torch")
 24 | 
 25 | def load_video(video_path, n_frms=MAX_INT, height=-1, width=-1, sampling="uniform", return_msg = False):
 26 |     decord.bridge.set_bridge("torch")
 27 |     vr = VideoReader(uri=video_path, height=height, width=width)
 28 | 
 29 |     vlen = len(vr)
 30 |     start, end = 0, vlen
 31 | 
 32 |     n_frms = min(n_frms, vlen)
 33 | 
 34 |     if sampling == "uniform":
 35 |         indices = np.arange(start, end, vlen / n_frms).astype(int).tolist()
 36 |     elif sampling == "headtail":
 37 |         indices_h = sorted(rnd.sample(range(vlen // 2), n_frms // 2))
 38 |         indices_t = sorted(rnd.sample(range(vlen // 2, vlen), n_frms // 2))
 39 |         indices = indices_h + indices_t
 40 |     else:
 41 |         raise NotImplementedError
 42 | 
 43 |     # get_batch -> T, H, W, C
 44 |     temp_frms = vr.get_batch(indices)
 45 |     # print(type(temp_frms))
 46 |     tensor_frms = torch.from_numpy(temp_frms) if type(temp_frms) is not torch.Tensor else temp_frms
 47 |     frms = tensor_frms.permute(3, 0, 1, 2).float()  # (C, T, H, W)
 48 | 
 49 |     if not return_msg:
 50 |         return frms
 51 | 
 52 |     fps = float(vr.get_avg_fps())
 53 |     sec = ", ".join([str(round(f / fps, 1)) for f in indices])
 54 |     # " " should be added in the start and end
 55 |     msg = f"The video contains {len(indices)} frames sampled at {sec} seconds. "
 56 |     return frms, msg
 57 | 
 58 | 
 59 | class AlproVideoBaseProcessor(BaseProcessor):
 60 |     def __init__(self, mean=None, std=None, n_frms=MAX_INT):
 61 |         if mean is None:
 62 |             mean = (0.48145466, 0.4578275, 0.40821073)
 63 |         if std is None:
 64 |             std = (0.26862954, 0.26130258, 0.27577711)
 65 | 
 66 |         self.normalize = transforms_video.NormalizeVideo(mean, std)
 67 | 
 68 |         self.n_frms = n_frms
 69 | 
 70 | 
 71 | class ToUint8(object):
 72 |     def __init__(self):
 73 |         pass
 74 | 
 75 |     def __call__(self, tensor):
 76 |         return tensor.to(torch.uint8)
 77 | 
 78 |     def __repr__(self):
 79 |         return self.__class__.__name__
 80 | 
 81 | 
 82 | class ToTHWC(object):
 83 |     """
 84 |     Args:
 85 |         clip (torch.tensor, dtype=torch.uint8): Size is (C, T, H, W)
 86 |     Return:
 87 |         clip (torch.tensor, dtype=torch.float): Size is (T, H, W, C)
 88 |     """
 89 | 
 90 |     def __init__(self):
 91 |         pass
 92 | 
 93 |     def __call__(self, tensor):
 94 |         return tensor.permute(1, 2, 3, 0)
 95 | 
 96 |     def __repr__(self):
 97 |         return self.__class__.__name__
 98 | 
 99 | 
100 | class ResizeVideo(object):
101 |     def __init__(self, target_size, interpolation_mode="bilinear"):
102 |         self.target_size = target_size
103 |         self.interpolation_mode = interpolation_mode
104 | 
105 |     def __call__(self, clip):
106 |         """
107 |         Args:
108 |             clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
109 |         Returns:
110 |             torch.tensor: central cropping of video clip. Size is
111 |             (C, T, crop_size, crop_size)
112 |         """
113 |         return F.resize(clip, self.target_size, self.interpolation_mode)
114 | 
115 |     def __repr__(self):
116 |         return self.__class__.__name__ + "(resize_size={0})".format(self.target_size)
117 | 
118 | 
119 | @registry.register_processor("alpro_video_train")
120 | class AlproVideoTrainProcessor(AlproVideoBaseProcessor):
121 |     def __init__(
122 |         self,
123 |         image_size=384,
124 |         mean=None,
125 |         std=None,
126 |         min_scale=0.5,
127 |         max_scale=1.0,
128 |         n_frms=MAX_INT,
129 |     ):
130 |         super().__init__(mean=mean, std=std, n_frms=n_frms)
131 | 
132 |         self.image_size = image_size
133 | 
134 |         self.transform = transforms.Compose(
135 |             [
136 |                 # Video size is (C, T, H, W)
137 |                 transforms_video.RandomResizedCropVideo(
138 |                     image_size,
139 |                     scale=(min_scale, max_scale),
140 |                     interpolation_mode="bicubic",
141 |                 ),
142 |                 ToTHWC(),  # C, T, H, W -> T, H, W, C
143 |                 ToUint8(),
144 |                 transforms_video.ToTensorVideo(),  # T, H, W, C -> C, T, H, W
145 |                 self.normalize,
146 |             ]
147 |         )
148 | 
149 |     def __call__(self, vpath):
150 |         """
151 |         Args:
152 |             clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
153 |         Returns:
154 |             torch.tensor: video clip after transforms. Size is (C, T, size, size).
155 |         """
156 |         clip = load_video(
157 |             video_path=vpath,
158 |             n_frms=self.n_frms,
159 |             height=self.image_size,
160 |             width=self.image_size,
161 |             sampling="headtail",
162 |         )
163 | 
164 |         return self.transform(clip)
165 | 
166 |     @classmethod
167 |     def from_config(cls, cfg=None):
168 |         if cfg is None:
169 |             cfg = OmegaConf.create()
170 | 
171 |         image_size = cfg.get("image_size", 256)
172 | 
173 |         mean = cfg.get("mean", None)
174 |         std = cfg.get("std", None)
175 | 
176 |         min_scale = cfg.get("min_scale", 0.5)
177 |         max_scale = cfg.get("max_scale", 1.0)
178 | 
179 |         n_frms = cfg.get("n_frms", MAX_INT)
180 | 
181 |         return cls(
182 |             image_size=image_size,
183 |             mean=mean,
184 |             std=std,
185 |             min_scale=min_scale,
186 |             max_scale=max_scale,
187 |             n_frms=n_frms,
188 |         )
189 | 
190 | 
191 | @registry.register_processor("alpro_video_eval")
192 | class AlproVideoEvalProcessor(AlproVideoBaseProcessor):
193 |     def __init__(self, image_size=256, mean=None, std=None, n_frms=MAX_INT):
194 |         super().__init__(mean=mean, std=std, n_frms=n_frms)
195 | 
196 |         self.image_size = image_size
197 | 
198 |         # Input video size is (C, T, H, W)
199 |         self.transform = transforms.Compose(
200 |             [
201 |                 # frames will be resized during decord loading.
202 |                 ToUint8(),  # C, T, H, W
203 |                 ToTHWC(),  # T, H, W, C
204 |                 transforms_video.ToTensorVideo(),  # C, T, H, W
205 |                 self.normalize,  # C, T, H, W
206 |             ]
207 |         )
208 | 
209 |     def __call__(self, vpath):
210 |         """
211 |         Args:
212 |             clip (torch.tensor): Video clip to be cropped. Size is (C, T, H, W)
213 |         Returns:
214 |             torch.tensor: video clip after transforms. Size is (C, T, size, size).
215 |         """
216 |         clip = load_video(
217 |             video_path=vpath,
218 |             n_frms=self.n_frms,
219 |             height=self.image_size,
220 |             width=self.image_size,
221 |         )
222 | 
223 |         return self.transform(clip)
224 | 
225 |     @classmethod
226 |     def from_config(cls, cfg=None):
227 |         if cfg is None:
228 |             cfg = OmegaConf.create()
229 | 
230 |         image_size = cfg.get("image_size", 256)
231 | 
232 |         mean = cfg.get("mean", None)
233 |         std = cfg.get("std", None)
234 | 
235 |         n_frms = cfg.get("n_frms", MAX_INT)
236 | 
237 |         return cls(image_size=image_size, mean=mean, std=std, n_frms=n_frms)
238 | 


--------------------------------------------------------------------------------
/video_llama/runners/__init__.py:
--------------------------------------------------------------------------------
 1 | """
 2 |  Copyright (c) 2022, salesforce.com, inc.
 3 |  All rights reserved.
 4 |  SPDX-License-Identifier: BSD-3-Clause
 5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 6 | """
 7 | 
 8 | from video_llama.runners.runner_base import RunnerBase
 9 | 
10 | __all__ = ["RunnerBase"]
11 | 


--------------------------------------------------------------------------------
/video_llama/runners/test.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/runners/test.py


--------------------------------------------------------------------------------
/video_llama/tasks/__init__.py:
--------------------------------------------------------------------------------
 1 | """
 2 |  Copyright (c) 2022, salesforce.com, inc.
 3 |  All rights reserved.
 4 |  SPDX-License-Identifier: BSD-3-Clause
 5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 6 | """
 7 | 
 8 | from video_llama.common.registry import registry
 9 | from video_llama.tasks.base_task import BaseTask
10 | from video_llama.tasks.image_text_pretrain import ImageTextPretrainTask
11 | from video_llama.tasks.video_text_pretrain import VideoTextPretrainTask
12 | 
13 | 
14 | def setup_task(cfg):
15 |     assert "task" in cfg.run_cfg, "Task name must be provided."
16 | 
17 |     task_name = cfg.run_cfg.task
18 |     task = registry.get_task_class(task_name).setup_task(cfg=cfg)
19 |     assert task is not None, "Task {} not properly registered.".format(task_name)
20 | 
21 |     return task
22 | 
23 | 
24 | __all__ = [
25 |     "BaseTask",
26 |     "ImageTextPretrainTask",
27 |     "VideoTextPretrainTask"
28 | ]
29 | 


--------------------------------------------------------------------------------
/video_llama/tasks/__pycache__/__init__.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/tasks/__pycache__/__init__.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/tasks/__pycache__/base_task.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/tasks/__pycache__/base_task.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/tasks/__pycache__/image_text_pretrain.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/tasks/__pycache__/image_text_pretrain.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/tasks/__pycache__/video_text_pretrain.cpython-39.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lzw-lzw/GroundingGPT/8afe78871d45c3b37c263f5065be8d769354951c/video_llama/tasks/__pycache__/video_text_pretrain.cpython-39.pyc


--------------------------------------------------------------------------------
/video_llama/tasks/image_text_pretrain.py:
--------------------------------------------------------------------------------
 1 | """
 2 |  Copyright (c) 2022, salesforce.com, inc.
 3 |  All rights reserved.
 4 |  SPDX-License-Identifier: BSD-3-Clause
 5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 6 | """
 7 | 
 8 | from video_llama.common.registry import registry
 9 | from video_llama.tasks.base_task import BaseTask
10 | 
11 | 
12 | @registry.register_task("image_text_pretrain")
13 | class ImageTextPretrainTask(BaseTask):
14 |     def __init__(self):
15 |         super().__init__()
16 | 
17 |     def evaluation(self, model, data_loader, cuda_enabled=True):
18 |         pass
19 | 


--------------------------------------------------------------------------------
/video_llama/tasks/video_text_pretrain.py:
--------------------------------------------------------------------------------
 1 | """
 2 |  Copyright (c) 2022, salesforce.com, inc.
 3 |  All rights reserved.
 4 |  SPDX-License-Identifier: BSD-3-Clause
 5 |  For full license text, see the LICENSE_Lavis file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 6 | """
 7 | 
 8 | from video_llama.common.registry import registry
 9 | from video_llama.tasks.base_task import BaseTask
10 | 
11 | 
12 | @registry.register_task("video_text_pretrain")
13 | class VideoTextPretrainTask(BaseTask):
14 |     def __init__(self):
15 |         super().__init__()
16 | 
17 |     def evaluation(self, model, data_loader, cuda_enabled=True):
18 |         pass
19 | 


--------------------------------------------------------------------------------