├── src ├── utils.py └── metric │ ├── __init__.py │ ├── metric.py │ ├── l1.py │ ├── l2.py │ ├── psnr.py │ ├── ssim.py │ ├── lpips.py │ ├── dino.py │ └── clip_score.py ├── guidance ├── guidance.md ├── image.png └── guidance_cn.md ├── .gitignore ├── example_data ├── test1 │ ├── gt │ │ └── img1.jpg │ └── pred │ │ └── img1.jpg └── example_data.json ├── requirments.txt ├── config └── example.yaml ├── README.md └── benchmark.py /src/utils.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /guidance/guidance.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/metric/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | src/metric/__pycache__/* -------------------------------------------------------------------------------- /guidance/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamjimmy/cv_benchmark_hub/HEAD/guidance/image.png -------------------------------------------------------------------------------- /example_data/test1/gt/img1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamjimmy/cv_benchmark_hub/HEAD/example_data/test1/gt/img1.jpg -------------------------------------------------------------------------------- /example_data/test1/pred/img1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jamjimmy/cv_benchmark_hub/HEAD/example_data/test1/pred/img1.jpg -------------------------------------------------------------------------------- /example_data/example_data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "prompt": "a girl", 4 | "pred": "example_data/test1/gt/img1.jpg" 5 | } 6 | ] -------------------------------------------------------------------------------- /requirments.txt: -------------------------------------------------------------------------------- 1 | torch 2 | torchvision 3 | tqdm 4 | numpy 5 | torchmetrics 6 | pillow 7 | PyYAML 8 | openpyxl 9 | transformers 10 | pandas 11 | -------------------------------------------------------------------------------- /src/metric/metric.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | class Metric(ABC): 4 | def __init__(self): 5 | pass 6 | 7 | """ 8 | 定义一个 Metric 接口，所有指标类必须实现 __call__ 方法。 9 | """ 10 | @abstractmethod 11 | def __call__(self, input_paths, keys): 12 | """ 13 | 所有指标类必须实现的方法。 14 | :param input_paths: 输入路径列表 15 | :param keys: 用于计算指标的关键字列表 16 | :return: 指标分数 17 | """ 18 | pass -------------------------------------------------------------------------------- /config/example.yaml: -------------------------------------------------------------------------------- 1 | path_key: test # Choice. Only files with 'test' in their name in the input directory will be processed. 2 | 3 | 4 | keys: 5 | target: gt 6 | pred: pred 7 | 8 | metrics: 9 | 10 | clip_image_to_image_score: # custom test name 11 | metric: 12 | class: src.metric.clip_score.ClipScore 13 | clip_model: openai/clip-vit-base-patch16 14 | device: cuda 15 | 16 | clip_image_to_text_score: 17 | metric: 18 | class: src.metric.clip_score.ClipScore 19 | clip_model: openai/clip-vit-base-patch16 20 | device: cuda 21 | special_keys: # custom key 22 | - prompt 23 | - pred 24 | special_input_paths: 25 | - example_data/example_data.json 26 | just_special_input_paths: true # if true, only the special_input_paths will be used 27 | 28 | ssim: 29 | metric: # metric class 30 | class: src.metric.ssim.SSIM 31 | device: cuda 32 | 33 | psnr_score: 34 | metric: 35 | class: src.metric.psnr.PSNR 36 | device: cuda 37 | 38 | lipis_score: 39 | metric: 40 | class: src.metric.lpips.LPIPS 41 | device: cuda 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /guidance/guidance_cn.md: -------------------------------------------------------------------------------- 1 | # 中文教程 2 | 3 | 4 | [ ]（视频正在录制） 5 | ## 文件结构 6 | 确定待测试的文件以下列形式排布 7 | ### 文件夹格式 8 | ```bash 9 | input_path 10 | ├── input_dir1 11 | │ ├── dir1 12 | │ │ ├── img1.jpg # all the filename must match in each dir 13 | │ │ ├── img2.jpg 14 | │ │ └── ... 15 | │ └── dir2 16 | │ ├── img1.jpg 17 | │ ├── img2.jpg 18 | │ └── ... 19 | │ 20 | ├── input_dir2 21 | │ ├── dir1 22 | │ └── ... 23 | │ └── dir2 24 | │ └── ... 25 | ... 26 | ``` 27 | 28 | ### JSON 格式 29 | ```json 30 | [ 31 | { 32 | "args1": "image path or text of target", 33 | "args2": "image path or text of pred", 34 | }, 35 | 36 | { 37 | ... 38 | }, 39 | { 40 | "args1": "image path or text of target", 41 | "args2": "image path or text of pred", 42 | } 43 | ] 44 | ``` 45 | 46 | ### **!!!下文提到的`keys`的概念其实就是文件结构的 'dir1'&'dir2' 或者 'args1'&'args2'。!!!** 47 | 48 | ## 配置文件如何使用 49 | ![alt text](image.png) 50 | 51 | 52 | ### 通用文件夹指定（SEC.1 & SEC.2） 53 | 54 | 本例中，所有在`--input_path`下含有'test'关键字的文件夹都会被测试。对于每个文件夹测试其中的'gt','pred'文件夹。 55 | 56 | 图中第二部分的`keys`指定了通用路径被测试的关键字。这里为了标准化,只允许指定`target key`和`pred key`。 57 | 58 | （这样做是为了方便批量测试，例如ssim、clip score都需要对生成图象和GT图像做对比。可以避免重复输入路径。） 59 | 60 | ### 具体Metric配置 61 | `Metric：`指定了测试的类 62 | 63 | `special_input_paths：`指定了针对该Metric需要单独测试的 json 或者 dir 路径。 64 | 65 | 66 | `special_keys:`指定特殊的keys(可以任意数量)，keys的顺序在大部分Metric下无影响，需要根据Metric类具体确定。 67 | 68 | `just_special_input_paths:`设定为true，则只测试special_input_paths,忽略上面指定的通用文件夹。 69 | 70 | 例如本例需要特殊计算文字与图像的clip score。忽略通用路径、指定json路径、指定keys: 'prompt'和'pred'。（clip score代码会自动分别路径和prompt，顺序其实无所谓） 71 | 72 | 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Computer Vision Benchmark Hub 2 | [![Outlook](https://img.shields.io/badge/Microsoft_Outlook-0078D4?style=for-the-badge&logo=microsoft-outlook&logoColor=white)](mailto:jiangzj453@foxmail.com) 3 | 4 | 逻辑较为简单，请参考教程。有问题可以联系我~ 5 | 6 | [中文教程](guidance/guidance_cn.md) | [Guidance](guidance/guidance.md) 7 | 8 | Welcome to the Computer Vision Benchmark Hub — a flexible and user-friendly tool designed to evaluate and benchmark image-based models. With support for key metrics such as SSIM, PSNR, CLIP-Score, and more, our platform enables seamless one-click evaluations, helping you compare and optimize model performance effortlessly. 9 | 10 | ## 🚀 Installation 11 | To get started, simply install the required dependencies: 12 | ```bash 13 | conda create -n cv_benchmark python==3.10 14 | conda activate cv_benchmark 15 | pip install -r requirments.txt 16 | ``` 17 | 18 | ## 📁 Data Preparation 19 | The toolkit supports two types of inputs: **folders** and **JSON files**. 20 | 21 | ### Folder Structure 22 | Ensure your input folder is organized as follows: 23 | ```bash 24 | input_path 25 | ├── input_dir1 26 | │ ├── dir1 27 | │ │ ├── img1.jpg # all the filename must match in each dir 28 | │ │ ├── img2.jpg 29 | │ │ └── ... 30 | │ └── dir2 31 | │ ├── img1.jpg 32 | │ ├── img2.jpg 33 | │ └── ... 34 | │ 35 | ├── input_dir2 36 | │ ├── dir1 37 | │ └── ... 38 | │ └── dir2 39 | │ └── ... 40 | ... 41 | ``` 42 | 43 | ### JSON Format 44 | Alternatively, you can use a JSON file structured like this: 45 | ```json 46 | [ 47 | { 48 | "args1": "image path or text of target", 49 | "args2": "image path or text of pred", 50 | }, 51 | 52 | { 53 | ... 54 | }, 55 | { 56 | "args1": "image path or text of target", 57 | "args2": "image path or text of pred", 58 | } 59 | ] 60 | ``` 61 | ## ⚙️ Configuration 62 | The toolkit uses a YAML configuration file for customization. 63 | 64 | ### Input Paths 65 | `path_key:` Filters files containing specific keywords in their names within the `--input_path ` directory. 66 | `special_input_paths:` A list of paths that require special processing. 67 | 68 | ### Keys 69 | Define the input keys corresponding to your JSON or folder structure: 70 | ```yaml 71 | keys: 72 | pred_key: args1 73 | target_key: args2 74 | ``` 75 | 76 | 77 | ## 📊 Usage 78 | 79 | Run the benchmark using the following command: 80 | ```bash 81 | python benchmark.py --config config/example.yaml --input_path example_data --output_path ./result/ 82 | ``` 83 | The result will be saved in `./result/test1` folder. There will be a 'benchmark.xlsx' file. 84 | 85 | 86 | 87 | ## ✅ TODO 88 | 89 | Completed 90 | - [X] Support Diffusion Metric:SSIM, PSNR, CLIP-Score, LIPIS 91 | - [ ] generate latex 92 | - [X] L1, L2... 93 | 94 | - [ ] Upcoming Features 95 | 96 | --- 97 | 98 | 99 | Feel free to tweak the config and data formats to suit your needs. Happy benchmarking! 🎯 100 | 101 | ## Star History 102 | 103 | [![Star History Chart](https://api.star-history.com/svg?repos=jamjimmy/cv_benchmark_hub&type=Date)](https://www.star-history.com/#jamjimmy/cv_benchmark_hub&Date) 104 | -------------------------------------------------------------------------------- /src/metric/l1.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | import os 3 | from PIL import Image 4 | import json 5 | from torchvision import transforms 6 | import torch 7 | from .metric import Metric 8 | from tqdm import tqdm 9 | 10 | transform_img = transforms.Compose([ 11 | transforms.ToTensor(), 12 | # lambda x: (x * 255) 13 | ]) 14 | 15 | def preprocess_list(target_input, pred_input, device): 16 | processed_pred_input = [] 17 | processed_target_input = [] 18 | for target_item, pred_item in tqdm(zip(target_input, pred_input), total=len(target_input)): 19 | target_image = Image.open(target_item).convert('RGB') 20 | pred_image = Image.open(pred_item).convert('RGB') 21 | if target_image.size != pred_image.size: 22 | Warning("Target image size is not equal to pred image size in test L2!!!") 23 | pred_image = pred_image.resize(target_image.size) 24 | 25 | processed_pred_input.append(transform_img(pred_image).to(device)) 26 | processed_target_input.append(transform_img(target_image).to(device)) 27 | return processed_pred_input, processed_target_input 28 | 29 | class L1(Metric): 30 | def __init__(self, device = 'cuda'): 31 | self.device = device 32 | self.criterion = nn.L1Loss() 33 | def __call__(self, input: str, keys=['target', 'pred']): 34 | ''' 35 | Args: 36 | - input: path(.json or .jsonl or dir) and param::keys = ['dir1', 'dir2']: 37 | - if dir, there must be 2 dirs in the input dir, and 38 | the number and filename of images in the two dirs must be the same. The dir looks like this: 39 | 40 | input_dir 41 | ├── dir1 42 | │ ├── img1.jpg 43 | │ ├── img2.jpg 44 | │ └── ... 45 | └── dir2 46 | ├── img1.jpg 47 | ├── img2.jpg 48 | └── ... 49 | 50 | - if .json or .jsonl the file looks like this and param::keys = ['args1', 'args2']: 51 | [ 52 | { 53 | "args1": "image path", 54 | "args2": "image path, 55 | }, 56 | ... 57 | { 58 | "args1": "image path", 59 | "args2": "image path, 60 | } 61 | ] 62 | Returns: 63 | - psnr score 64 | ''' 65 | target_list = [] 66 | pred_list = [] 67 | 68 | if os.path.isdir(input): 69 | dir_path_1 = os.path.join(input, keys[0]) 70 | dir_path_2 = os.path.join(input, keys[1]) 71 | assert len(os.listdir(dir_path_1)) == len(os.listdir(dir_path_2)), f"the number of images in {dir_path_1} and {dir_path_2} must be the same" 72 | imgs = os.listdir(dir_path_1) 73 | for file in imgs: 74 | assert os.path.exists(os.path.join(dir_path_2, file)), f"file not exists {os.path.join(dir_path_2, file)}" 75 | target_list.append(os.path.join(dir_path_1, file)) 76 | pred_list.append(os.path.join(dir_path_2, file)) 77 | 78 | elif input.endswith(".json") or input.endswith(".jsonl"): 79 | with open(input, "r") as f: 80 | data = json.load(f) 81 | assert len(keys) == 2, f"keys must be 2, but got {keys}" 82 | for item in data: 83 | target_path = item[keys[0]] 84 | pred_path = item[keys[1]] 85 | target_list.append(target_path) 86 | pred_list.append(pred_path) 87 | 88 | else: 89 | raise ValueError(f"{input} must be dir or json file") 90 | 91 | assert len(target_list) == len(pred_list), f"the number of images in {input} must be the same" 92 | processed_target_list, processed_pred_list = preprocess_list(target_list, pred_list, self.device) 93 | 94 | score = 0.0 95 | 96 | # print(processed_target_list, processed_pred_list) 97 | 98 | with torch.no_grad(): 99 | for pred, target in zip(processed_pred_list, processed_target_list): 100 | score += self.criterion(pred, target).detach().cpu() 101 | score = score / len(processed_target_list) 102 | return "L1", score.item(), len(target_list) 103 | -------------------------------------------------------------------------------- /src/metric/l2.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | import os 3 | from PIL import Image 4 | import json 5 | from torchvision import transforms 6 | import torch 7 | from .metric import Metric 8 | from tqdm import tqdm 9 | 10 | transform_img = transforms.Compose([ 11 | transforms.ToTensor(), 12 | # lambda x: (x * 255) 13 | ]) 14 | 15 | def preprocess_list(target_input, pred_input, device): 16 | processed_pred_input = [] 17 | processed_target_input = [] 18 | for target_item, pred_item in tqdm(zip(target_input, pred_input), total=len(target_input)): 19 | target_image = Image.open(target_item).convert('RGB') 20 | pred_image = Image.open(pred_item).convert('RGB') 21 | if target_image.size != pred_image.size: 22 | Warning("Target image size is not equal to pred image size in test L2!!!") 23 | pred_image = pred_image.resize(target_image.size) 24 | 25 | processed_pred_input.append(transform_img(pred_image).to(device)) 26 | processed_target_input.append(transform_img(target_image).to(device)) 27 | return processed_pred_input, processed_target_input 28 | 29 | class L2(Metric): 30 | def __init__(self, device = 'cuda'): 31 | self.device = device 32 | self.criterion = nn.MSELoss() 33 | def __call__(self, input: str, keys=['target', 'pred']): 34 | ''' 35 | Args: 36 | - input: path(.json or .jsonl or dir) and param::keys = ['dir1', 'dir2']: 37 | - if dir, there must be 2 dirs in the input dir, and 38 | the number and filename of images in the two dirs must be the same. The dir looks like this: 39 | 40 | input_dir 41 | ├── dir1 42 | │ ├── img1.jpg 43 | │ ├── img2.jpg 44 | │ └── ... 45 | └── dir2 46 | ├── img1.jpg 47 | ├── img2.jpg 48 | └── ... 49 | 50 | - if .json or .jsonl the file looks like this and param::keys = ['args1', 'args2']: 51 | [ 52 | { 53 | "args1": "image path", 54 | "args2": "image path, 55 | }, 56 | ... 57 | { 58 | "args1": "image path", 59 | "args2": "image path, 60 | } 61 | ] 62 | Returns: 63 | - psnr score 64 | ''' 65 | target_list = [] 66 | pred_list = [] 67 | 68 | if os.path.isdir(input): 69 | dir_path_1 = os.path.join(input, keys[0]) 70 | dir_path_2 = os.path.join(input, keys[1]) 71 | assert len(os.listdir(dir_path_1)) == len(os.listdir(dir_path_2)), f"the number of images in {dir_path_1} and {dir_path_2} must be the same" 72 | imgs = os.listdir(dir_path_1) 73 | for file in imgs: 74 | assert os.path.exists(os.path.join(dir_path_2, file)), f"file not exists {os.path.join(dir_path_2, file)}" 75 | target_list.append(os.path.join(dir_path_1, file)) 76 | pred_list.append(os.path.join(dir_path_2, file)) 77 | 78 | elif input.endswith(".json") or input.endswith(".jsonl"): 79 | with open(input, "r") as f: 80 | data = json.load(f) 81 | assert len(keys) == 2, f"keys must be 2, but got {keys}" 82 | for item in data: 83 | target_path = item[keys[0]] 84 | pred_path = item[keys[1]] 85 | target_list.append(target_path) 86 | pred_list.append(pred_path) 87 | 88 | else: 89 | raise ValueError(f"{input} must be dir or json file") 90 | 91 | assert len(target_list) == len(pred_list), f"the number of images in {input} must be the same" 92 | processed_target_list, processed_pred_list = preprocess_list(target_list, pred_list, self.device) 93 | 94 | score = 0.0 95 | 96 | # print(processed_target_list, processed_pred_list) 97 | 98 | with torch.no_grad(): 99 | for pred, target in zip(processed_pred_list, processed_target_list): 100 | score += self.criterion(pred, target).detach().cpu() 101 | score = score / len(processed_target_list) 102 | return "L2", score.item(), len(target_list) 103 | -------------------------------------------------------------------------------- /src/metric/psnr.py: -------------------------------------------------------------------------------- 1 | from torchmetrics.image import PeakSignalNoiseRatio 2 | import torch 3 | import os 4 | from PIL import Image 5 | import json 6 | from torchvision import transforms 7 | from .metric import Metric 8 | from tqdm import tqdm 9 | 10 | transform_img = transforms.Compose([ 11 | transforms.ToTensor(), 12 | lambda x: (x * 255).to(torch.uint8) 13 | ]) 14 | 15 | def preprocess_list(target_input, pred_input, device): 16 | processed_pred_input = [] 17 | processed_target_input = [] 18 | for target_item, pred_item in tqdm(zip(target_input, pred_input), total=len(target_input)): 19 | target_image = Image.open(target_item).convert('RGB') 20 | pred_image = Image.open(pred_item).convert('RGB') 21 | if target_image.size != pred_image.size: 22 | Warning("Target image size is not equal to pred image size in test PSNR!!!") 23 | pred_image = pred_image.resize(target_image.size) 24 | 25 | processed_pred_input.append(transform_img(pred_image).unsqueeze(0).to(device)) 26 | processed_target_input.append(transform_img(target_image).unsqueeze(0).to(device)) 27 | return processed_pred_input, processed_target_input 28 | 29 | class PSNR(Metric): 30 | def __init__(self, device = 'cuda'): 31 | self.device = device 32 | self.psnr = PeakSignalNoiseRatio().to(self.device) 33 | 34 | 35 | def __call__(self, input: str, keys=['target', 'pred']): 36 | ''' 37 | Args: 38 | - input: path(.json or .jsonl or dir) and param::keys = ['dir1', 'dir2']: 39 | - if dir, there must be 2 dirs in the input dir, and 40 | the number and filename of images in the two dirs must be the same. The dir looks like this: 41 | 42 | input_dir 43 | ├── dir1 44 | │ ├── img1.jpg 45 | │ ├── img2.jpg 46 | │ └── ... 47 | └── dir2 48 | ├── img1.jpg 49 | ├── img2.jpg 50 | └── ... 51 | 52 | - if .json or .jsonl the file looks like this and param::keys = ['args1', 'args2']: 53 | [ 54 | { 55 | "args1": "image path", 56 | "args2": "image path, 57 | }, 58 | ... 59 | { 60 | "args1": "image path ", 61 | "args2": "image path , 62 | } 63 | ] 64 | Returns: 65 | - psnr score 66 | ''' 67 | target_list = [] 68 | pred_list = [] 69 | if os.path.isdir(input): 70 | dir_path_1 = os.path.join(input, keys[0]) 71 | dir_path_2 = os.path.join(input, keys[1]) 72 | assert len(os.listdir(dir_path_1)) == len(os.listdir(dir_path_2)), f"the number of images in {dir_path_1} and {dir_path_2} must be the same" 73 | imgs = os.listdir(dir_path_1) 74 | for file in imgs: 75 | assert os.path.exists(os.path.join(dir_path_2, file)), f"file not exists {os.path.join(dir_path_2, file)}" 76 | target_list.append(os.path.join(dir_path_1, file)) 77 | pred_list.append(os.path.join(dir_path_2, file)) 78 | 79 | elif input.endswith(".json") or input.endswith(".jsonl"): 80 | with open(input, "r") as f: 81 | data = json.load(f) 82 | assert len(keys) == 2, f"keys must be 2, but got {keys}" 83 | for item in data: 84 | target_path = item[keys[0]] 85 | pred_path = item[keys[1]] 86 | target_list.append(target_path) 87 | pred_list.append(pred_path) 88 | else: 89 | raise ValueError(f"{input} must be dir or json file") 90 | 91 | assert len(target_list) == len(pred_list), f"the number of images in {input} must be the same" 92 | processed_target_list, processed_pred_list = preprocess_list(target_list, pred_list, self.device) 93 | 94 | score = 0.0 95 | for pred, target in zip(processed_pred_list, processed_target_list): 96 | score += self.psnr(pred, target) 97 | score = score / len(processed_target_list) 98 | return "PSNR", score.detach().item(), len(target_list) 99 | 100 | # print(get_psnr_batch("data/", keys=['gt', 'pred'])) -------------------------------------------------------------------------------- /src/metric/ssim.py: -------------------------------------------------------------------------------- 1 | from torchmetrics.image import StructuralSimilarityIndexMeasure 2 | import os 3 | from PIL import Image 4 | import json 5 | from torchvision import transforms 6 | import torch 7 | from .metric import Metric 8 | from tqdm import tqdm 9 | transform_img = transforms.Compose([ 10 | transforms.ToTensor(), 11 | lambda x: (x * 255) 12 | ]) 13 | 14 | def preprocess_list(target_input, pred_input, device): 15 | processed_pred_input = [] 16 | processed_target_input = [] 17 | for target_item, pred_item in tqdm(zip(target_input, pred_input), total=len(target_input)): 18 | target_image = Image.open(target_item).convert('RGB') 19 | pred_image = Image.open(pred_item).convert('RGB') 20 | if target_image.size != pred_image.size: 21 | Warning("Target image size is not equal to pred image size in test SSIM!!!") 22 | pred_image = pred_image.resize(target_image.size) 23 | 24 | processed_pred_input.append(transform_img(pred_image).unsqueeze(0).to(device)) 25 | processed_target_input.append(transform_img(target_image).unsqueeze(0).to(device)) 26 | 27 | return processed_pred_input, processed_target_input 28 | 29 | class SSIM(Metric): 30 | def __init__(self, device = 'cuda'): 31 | self.device = device 32 | self.ssim = StructuralSimilarityIndexMeasure().to(self.device) 33 | 34 | def __call__(self, input: str, keys=['target', 'pred']): 35 | ''' 36 | Args: 37 | - input: path(.json or .jsonl or dir) and param::keys = ['dir1', 'dir2']: 38 | - if dir, there must be 2 dirs in the input dir, and 39 | the number and filename of images in the two dirs must be the same. The dir looks like this: 40 | 41 | input_dir 42 | ├── dir1 43 | │ ├── img1.jpg 44 | │ ├── img2.jpg 45 | │ └── ... 46 | └── dir2 47 | ├── img1.jpg 48 | ├── img2.jpg 49 | └── ... 50 | 51 | - if .json or .jsonl the file looks like this and param::keys = ['args1', 'args2']: 52 | [ 53 | { 54 | "args1": "image path", 55 | "args2": "image path, 56 | }, 57 | ... 58 | { 59 | "args1": "image path", 60 | "args2": "image path, 61 | } 62 | ] 63 | Returns: 64 | - psnr score 65 | ''' 66 | target_list = [] 67 | pred_list = [] 68 | 69 | if os.path.isdir(input): 70 | dir_path_1 = os.path.join(input, keys[0]) 71 | dir_path_2 = os.path.join(input, keys[1]) 72 | assert len(os.listdir(dir_path_1)) == len(os.listdir(dir_path_2)), f"the number of images in {dir_path_1} and {dir_path_2} must be the same" 73 | imgs = os.listdir(dir_path_1) 74 | for file in imgs: 75 | assert os.path.exists(os.path.join(dir_path_2, file)), f"file not exists {os.path.join(dir_path_2, file)}" 76 | target_list.append(os.path.join(dir_path_1, file)) 77 | pred_list.append(os.path.join(dir_path_2, file)) 78 | 79 | elif input.endswith(".json") or input.endswith(".jsonl"): 80 | with open(input, "r") as f: 81 | data = json.load(f) 82 | assert len(keys) == 2, f"keys must be 2, but got {keys}" 83 | for item in data: 84 | target_path = item[keys[0]] 85 | pred_path = item[keys[1]] 86 | target_list.append(target_path) 87 | pred_list.append(pred_path) 88 | 89 | else: 90 | raise ValueError(f"{input} must be dir or json file") 91 | 92 | assert len(target_list) == len(pred_list), f"the number of images in {input} must be the same" 93 | processed_target_list, processed_pred_list = preprocess_list(target_list, pred_list, self.device) 94 | 95 | score = 0.0 96 | 97 | # print(processed_target_list, processed_pred_list) 98 | 99 | with torch.no_grad(): 100 | for pred, target in zip(processed_pred_list, processed_target_list): 101 | score += self.ssim(pred, target) 102 | score = score / len(processed_target_list) 103 | return "SSIM", score.detach().item(), len(target_list) 104 | 105 | # print(get_ssim_batch("data/", keys=['gt', 'pred'])) -------------------------------------------------------------------------------- /src/metric/lpips.py: -------------------------------------------------------------------------------- 1 | from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity 2 | import os 3 | from PIL import Image 4 | import json 5 | from torchvision import transforms 6 | import torch 7 | from .metric import Metric 8 | from tqdm import tqdm 9 | transform_img = transforms.Compose([ 10 | transforms.ToTensor(), 11 | lambda x: (x * 2) - 1 12 | ]) 13 | 14 | def preprocess_list(target_input, pred_input, device): 15 | processed_pred_input = [] 16 | processed_target_input = [] 17 | for target_item, pred_item in tqdm(zip(target_input, pred_input), total=len(target_input)): 18 | target_image = Image.open(target_item).convert('RGB') 19 | pred_image = Image.open(pred_item).convert('RGB') 20 | if target_image.size != pred_image.size: 21 | Warning("Target image size is not equal to pred image size in test LPIPS!!!") 22 | pred_image = pred_image.resize(target_image.size) 23 | 24 | processed_pred_input.append(transform_img(pred_image).unsqueeze(0).to(device)) 25 | processed_target_input.append(transform_img(target_image).unsqueeze(0).to(device)) 26 | return processed_pred_input, processed_target_input 27 | 28 | 29 | class LPIPS(Metric): 30 | def __init__(self, device = 'cuda'): 31 | self.device = device 32 | self.lpips = LearnedPerceptualImagePatchSimilarity(net_type='squeeze').to(self.device) 33 | 34 | def __call__(self, input: str, keys=['target', 'pred']): 35 | ''' 36 | Args: 37 | - input: path(.json or .jsonl or dir) and param::keys = ['dir1', 'dir2']: 38 | - if dir, there must be 2 dirs in the input dir, and 39 | the number and filename of images in the two dirs must be the same. The dir looks like this: 40 | 41 | input_dir 42 | ├── dir1 43 | │ ├── img1.jpg 44 | │ ├── img2.jpg 45 | │ └── ... 46 | └── dir2 47 | ├── img1.jpg 48 | ├── img2.jpg 49 | └── ... 50 | 51 | - if .json or .jsonl the file looks like this and param::keys = ['args1', 'args2']: 52 | [ 53 | { 54 | "args1": "image path", 55 | "args2": "image path, 56 | }, 57 | ... 58 | { 59 | "args1": "image path", 60 | "args2": "image path, 61 | } 62 | ] 63 | Returns: 64 | - psnr score 65 | ''' 66 | target_list = [] 67 | pred_list = [] 68 | 69 | if os.path.isdir(input): 70 | dir_path_1 = os.path.join(input, keys[0]) 71 | dir_path_2 = os.path.join(input, keys[1]) 72 | assert len(os.listdir(dir_path_1)) == len(os.listdir(dir_path_2)), f"the number of images in {dir_path_1} and {dir_path_2} must be the same" 73 | imgs = os.listdir(dir_path_1) 74 | for file in imgs: 75 | assert os.path.exists(os.path.join(dir_path_2, file)), f"file not exists {os.path.join(dir_path_2, file)}" 76 | target_list.append(os.path.join(dir_path_1, file)) 77 | pred_list.append(os.path.join(dir_path_2, file)) 78 | 79 | elif input.endswith(".json") or input.endswith(".jsonl"): 80 | with open(input, "r") as f: 81 | data = json.load(f) 82 | assert len(keys) == 2, f"keys must be 2, but got {keys}" 83 | for item in data: 84 | target_path = item[keys[0]] 85 | pred_path = item[keys[1]] 86 | target_list.append(target_path) 87 | pred_list.append(pred_path) 88 | 89 | else: 90 | raise ValueError(f"{input} must be dir or json file") 91 | 92 | assert len(target_list) == len(pred_list), f"the number of images in {input} must be the same" 93 | processed_target_list, processed_pred_list = preprocess_list(target_list, pred_list, self.device) 94 | 95 | score = 0.0 96 | 97 | # print(processed_target_list, processed_pred_list) 98 | 99 | with torch.no_grad(): 100 | for pred, target in zip(processed_pred_list, processed_target_list): 101 | score += self.lpips(pred, target) 102 | score = score / len(processed_target_list) 103 | return "LPIPS", score.detach().item(), len(target_list) 104 | 105 | # print(get_lpips_batch("data/", keys=['gt', 'pred'])) -------------------------------------------------------------------------------- /benchmark.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import importlib 3 | import pandas as pd 4 | import argparse 5 | import os 6 | import shutil 7 | from datetime import datetime 8 | # import wandb 9 | 10 | 11 | def load_config(config_path): 12 | with open(config_path, 'r') as file: 13 | config = yaml.safe_load(file) 14 | return config 15 | 16 | def get_class(class_path): 17 | module_path, class_name = class_path.rsplit('.', 1) 18 | module = importlib.import_module(module_path) 19 | cls = getattr(module, class_name) 20 | return cls 21 | 22 | def postprocess(results, args, metric_name): 23 | df = pd.DataFrame(results) 24 | os.makedirs(args.output_path, exist_ok=True) 25 | excel_path = os.path.join(args.output_path, 'benchmark.xlsx') 26 | mode = 'a' if os.path.exists(excel_path) else 'w' 27 | if mode == 'a': 28 | with pd.ExcelWriter(excel_path, mode=mode, engine='openpyxl', if_sheet_exists='replace') as writer: 29 | if metric_name in writer.book.sheetnames: 30 | existing_df = pd.read_excel(excel_path, sheet_name=metric_name) 31 | df = pd.concat([existing_df, df], ignore_index=True) 32 | df.to_excel(writer, sheet_name=metric_name, index=False) 33 | else: 34 | with pd.ExcelWriter(excel_path, mode=mode, engine='openpyxl') as writer: 35 | if metric_name in writer.book.sheetnames: 36 | existing_df = pd.read_excel(excel_path, sheet_name=metric_name) 37 | df = pd.concat([existing_df, df], ignore_index=True) 38 | df.to_excel(writer, sheet_name=metric_name, index=False) 39 | destination_file = os.path.join(args.output_path, os.path.basename(args.config)) 40 | if not os.path.exists(destination_file): 41 | shutil.copy(args.config, destination_file) 42 | 43 | def main(config, args): 44 | 45 | 46 | # columns = ['experiment_name', 'score', 'path', 'pred_key', 'target_key', 'current_datetime'] 47 | # table = wandb.Table(columns=columns) 48 | 49 | metrics = config.get('metrics', {}) 50 | input_paths = [] 51 | path_key = config.get('path_key', None) 52 | for file in os.listdir(args.input_path): 53 | if path_key == None or path_key in file: 54 | input_paths.append(os.path.join(args.input_path, file)) 55 | 56 | target_Key = config['keys'].get('target', None) 57 | pred_key = config['keys'].get('pred', None) 58 | 59 | for metric_name, metric_config in metrics.items(): 60 | 61 | # Instantiate metric class 62 | class_name = metric_config['metric']['class'] 63 | MetricClass = get_class(class_name) 64 | metric_instance = MetricClass(**{k: v for k, v in metric_config['metric'].items() if k != 'class'}) 65 | 66 | # Special keys and paths 67 | keys = [target_Key, pred_key] 68 | Keys_input = metric_config.get('special_keys', keys) 69 | 70 | all_input_paths = metric_config.get('special_input_paths', []) 71 | if metric_config.get('just_special_input_paths', False) == False: 72 | all_input_paths = input_paths + all_input_paths 73 | assert(len(all_input_paths) > 0), "No input paths found" 74 | results = [] 75 | # wandb.log({"score": 0.9}) 76 | # Run metric on all input paths 77 | for path in all_input_paths: 78 | print(f"Running {metric_name} on {path}") 79 | experiment_name, score, count = metric_instance(path, keys=Keys_input) 80 | result = { 81 | 'experiment_name': experiment_name, 82 | 'score': score, 83 | 'count': count, 84 | 'path': path, 85 | 'keys': keys, 86 | 'current_datetime': datetime.now() 87 | } 88 | results.append(result) 89 | # table.add_data(experiment_name, score, path, pred_key_input, targetKey_input, datetime.now()) 90 | postprocess(results, args, metric_name) 91 | 92 | 93 | if __name__ == "__main__": 94 | 95 | parser = argparse.ArgumentParser(description='Benchmarking') 96 | parser.add_argument('--output_path', type=str, help='output_path') 97 | parser.add_argument('--input_path', type=str, help='output_path') 98 | parser.add_argument('--config', type=str, help='Path to config file') 99 | # parser.add_argument("--use_wandb", type=bool, default=True, help="Enable or disable wandb logging.") 100 | args = parser.parse_args() 101 | 102 | # if args.use_wandb: 103 | # wandb.init(project='benchmark', name=os.path.basename(args.output_path), mode="online") 104 | # else: 105 | # wandb.init(mode="disabled") 106 | args.output_path = os.path.join(args.output_path, datetime.now().strftime("%Y-%m-%d_%H-%M-%S")) 107 | config = load_config(args.config) 108 | main(config, args) -------------------------------------------------------------------------------- /src/metric/dino.py: -------------------------------------------------------------------------------- 1 | from scipy import spatial 2 | import torch 3 | import os 4 | from PIL import Image 5 | import json 6 | from torchvision import transforms 7 | from .metric import Metric 8 | 9 | transform_img = transforms.Compose([ 10 | transforms.Resize(256, interpolation=3), 11 | transforms.CenterCrop(224), 12 | transforms.ToTensor(), 13 | transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), 14 | ]) 15 | 16 | def check_image_path(text): 17 | if any(ext in text for ext in ['.jpg', '.png', '.jpeg']): 18 | if not os.path.exists(text): 19 | print("Warning: input is an image path, will load image") 20 | 21 | def preprocess_list(input, device): 22 | processed_input = [] 23 | for item in input: 24 | # if os.path.isfile(item): 25 | image = Image.open(item) 26 | processed_input.append(transform_img(image).unsqueeze(0).to(device)) 27 | return processed_input 28 | def _get_dino_score(input1, input2, metric_model): 29 | ''' 30 | Args: 31 | - input1: list 32 | - input2: list 33 | - dino_model: dino model name or path 34 | ''' 35 | with torch.no_grad(): 36 | img_feature1 = metric_model(input1).detach().cpu().float() 37 | img_feature2 = metric_model(input2).detach().cpu().float() 38 | similarity = 1 - spatial.distance.cosine(img_feature1.view(img_feature1.shape[1]), img_feature2.view(img_feature2.shape[1])) 39 | return similarity 40 | 41 | class DinoScore(Metric): 42 | def __init__(self, device = 'cuda', dino_model="'dino_vits16'", batch_size=100): 43 | self.device = device 44 | self.batch_size = batch_size 45 | self.dino_model = torch.hub.load('facebookresearch/dino:main', dino_model) 46 | self.dino_model.eval() 47 | self.dino_model.to(self.device) 48 | 49 | def __call__(self, input: str, keys=['target', 'pred']): 50 | ''' 51 | Args: 52 | - input: path(.json or .jsonl or dir) and param::keys = ['dir1', 'dir2']: 53 | - if dir, there must be 2 dirs in the input dir, and 54 | the number and filename of images in the two dirs must be the same. The dir looks like this: 55 | 56 | input_dir 57 | ├── dir1 58 | │ ├── img1.jpg 59 | │ ├── img2.jpg 60 | │ └── ... 61 | └── dir2 62 | ├── img1.jpg 63 | ├── img2.jpg 64 | └── ... 65 | 66 | - if .json or .jsonl the file looks like this and param::keys = ['args1', 'args2']: 67 | [ 68 | { 69 | "args1": "image path", 70 | "args2": "image path, 71 | }, 72 | ... 73 | { 74 | "args1": "image path", 75 | "args2": "image path, 76 | } 77 | ] 78 | Returns: 79 | - psnr score 80 | ''' 81 | target_list = [] 82 | pred_list = [] 83 | 84 | if os.path.isdir(input): 85 | dir_path_1 = os.path.join(input, keys[0]) 86 | dir_path_2 = os.path.join(input, keys[1]) 87 | assert len(os.listdir(dir_path_1)) == len(os.listdir(dir_path_2)), f"the number of images in {dir_path_1} and {dir_path_2} must be the same" 88 | imgs = os.listdir(dir_path_1) 89 | for file in imgs: 90 | assert os.path.exists(os.path.join(dir_path_2, file)), f"file not exists {os.path.join(dir_path_2, file)}" 91 | target_list.append(os.path.join(dir_path_1, file)) 92 | pred_list.append(os.path.join(dir_path_2, file)) 93 | 94 | elif input.endswith(".json") or input.endswith(".jsonl"): 95 | with open(input, "r") as f: 96 | data = json.load(f) 97 | assert len(keys) == 2, f"keys must be 2, but got {keys}" 98 | for item in data: 99 | target_path = item[keys[0]] 100 | pred_path = item[keys[1]] 101 | target_list.append(target_path) 102 | pred_list.append(pred_path) 103 | 104 | else: 105 | raise ValueError(f"{input} must be dir or json file") 106 | 107 | assert len(target_list) == len(pred_list), f"the number of images in {input} must be the same" 108 | # processed_target_list, processed_pred_list = preprocess_list(target_list, pred_list, self.device) 109 | processed_target_list = preprocess_list(target_list, self.device) 110 | processed_pred_list = preprocess_list(pred_list, self.device) 111 | score = 0.0 112 | 113 | # print(processed_target_list, processed_pred_list) 114 | 115 | with torch.no_grad(): 116 | for pred, target in zip(processed_pred_list, processed_target_list): 117 | score += _get_dino_score(pred, target, self.dino_model) 118 | score = score / len(processed_target_list) 119 | return "DINO", score, len(target_list) 120 | 121 | # print(get_ssim_batch("data/", keys=['gt', 'pred'])) -------------------------------------------------------------------------------- /src/metric/clip_score.py: -------------------------------------------------------------------------------- 1 | from torchmetrics.multimodal.clip_score import CLIPScore 2 | import torch 3 | import os 4 | from PIL import Image 5 | import json 6 | from torchvision import transforms 7 | from .metric import Metric 8 | 9 | from transformers import CLIPImageProcessor, CLIPModel, CLIPTokenizer 10 | 11 | transform_img = transforms.Compose([ 12 | transforms.ToTensor(), 13 | lambda x: (x * 255) 14 | ]) 15 | 16 | def check_image_path(text): 17 | if any(ext in text for ext in ['.jpg', '.png', '.jpeg']): 18 | if not os.path.exists(text): 19 | print("Warning: input is an image path, but can't load image") 20 | 21 | def preprocess_list(input, device): 22 | processed_input = [] 23 | for item in input: 24 | if os.path.isfile(item): 25 | image = Image.open(item) 26 | processed_input.append(transform_img(image).to(device)) 27 | else: 28 | processed_input.append(item) 29 | return processed_input 30 | 31 | def _get_clip_score(input1, input2, metric_model): 32 | ''' 33 | Args: 34 | - input1: list 35 | - input2: list 36 | - clip_model: clip model name or path 37 | ''' 38 | with torch.no_grad(): 39 | score = metric_model(input1, input2) 40 | return score.detach().item() 41 | 42 | class ClipScore(Metric): 43 | def __init__(self, device = 'cuda', clip_model="openai/clip-vit-base-patch16"): 44 | self.device = device 45 | self.metric_model = CLIPScore(model_name_or_path=clip_model).to(self.device) 46 | self.clip_model_name = clip_model 47 | def __call__(self, input: str, keys: list=['gt', 'pred']): 48 | ''' 49 | Args: 50 | - input: path(.json or .jsonl or dir) and param::keys = ['dir1', 'dir2']: 51 | - if dir, there must be 2 dirs in the input dir, and 52 | the number and filename of images in the two dirs must be the same. The dir looks like this: 53 | 54 | input_dir 55 | ├── dir1 56 | │ ├── img1.jpg 57 | │ ├── img2.jpg 58 | │ └── ... 59 | └── dir2 60 | ├── img1.jpg 61 | ├── img2.jpg 62 | └── ... 63 | 64 | - if .json or .jsonl the file looks like this and param::keys = ['args1', 'args2']: 65 | [ 66 | { 67 | "args1": "image path or text", 68 | "args2": "image path or text, 69 | }, 70 | ... 71 | { 72 | "args1": "image path or text", 73 | "args2": "image path or text, 74 | } 75 | ] 76 | 77 | - clip_model: clip model name or clip model path. Default: "openai/clip-vit-base-patch16" 78 | Available models are: 79 | - `"openai/clip-vit-base-patch16"` 80 | - `"openai/clip-vit-base-patch32"` 81 | - `"openai/clip-vit-large-patch14-336"` 82 | - `"openai/clip-vit-large-patch14"` 83 | Returns: 84 | - clip score 85 | ''' 86 | 87 | if os.path.isdir(input): 88 | 89 | dir_path_1 = os.path.join(input, keys[0]) 90 | dir_path_2 = os.path.join(input, keys[1]) 91 | 92 | imgs = os.listdir(dir_path_1) 93 | input1_list = [] 94 | input2_list = [] 95 | for file in imgs: 96 | assert os.path.exists(os.path.join(dir_path_2, file)), f"file not exists {os.path.join(dir_path_2, file)}" 97 | input1_list.append(os.path.join(dir_path_1, file)) 98 | input2_list.append(os.path.join(dir_path_2, file)) 99 | 100 | elif input.endswith(".json") or input.endswith(".jsonl"): 101 | input1_list = [] 102 | input2_list = [] 103 | with open(input, "r") as f: 104 | data = json.load(f) 105 | assert len(keys) == 2, f"keys must be 2, but got {keys}" 106 | for item in data: 107 | text1 = item[keys[0]] 108 | text2 = item[keys[1]] 109 | 110 | # if there is a path, check if the path exists 111 | check_image_path(text1) 112 | check_image_path(text2) 113 | 114 | input1_list.append(text1) 115 | input2_list.append(text2) 116 | else: 117 | raise ValueError(f"{input} must be dir or json file") 118 | 119 | assert len(input1_list) == len(input2_list), f"the number of images in {input} must be the same" 120 | 121 | processed_input1_list = preprocess_list(input1_list, self.device) 122 | processed_input2_list = preprocess_list(input2_list, self.device) 123 | # metric_model = CLIPScore(model_name_or_path=clip_model).to(self.device) 124 | score = 0.0 125 | for input1, input2 in zip(processed_input1_list, processed_input2_list): 126 | 127 | score += _get_clip_score(input1, input2, self.metric_model) 128 | 129 | score = score / len(input1_list) 130 | return f"CLIP_score_{self.clip_model_name}", score, len(input1_list) 131 | 132 | 133 | # print(get_clip_score_batch('/data1/jiangzj/code/benchmark/data/', clip_model="openai/clip-vit-base-patch16",keys=['gt', 'pred'], device='cuda')) --------------------------------------------------------------------------------