├── dream_bench ├── __init__.py ├── dream_bench.py ├── helpers.py ├── config.py ├── load_models.py ├── tokenizer.py └── evaluator.py ├── requirements-test.txt ├── tests └── test_main.py ├── requirements.txt ├── prompts ├── README.md ├── dalle_mega.json └── drawbench.json ├── configs ├── example_config.json ├── dome_config.json └── README.md ├── .github └── workflows │ └── ci.yml ├── Makefile ├── LICENSE ├── setup.py ├── dataset_readme.md ├── .gitignore ├── README.md ├── make_dataset.py └── pyproject.toml /dream_bench/__init__.py: -------------------------------------------------------------------------------- 1 | from dream_bench.tokenizer import tokenizer 2 | from dream_bench.dream_bench import benchmark 3 | from dream_bench.config import DreamBenchConfig, WandbConfig, DatasetConfig 4 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | black>=22.*, < 23.0 2 | mypy>=0.971, < 1.0 3 | pylint>=2.13.*, < 2.15.0 4 | types-regex>=2022.*, < 2023.0 5 | pytest-cov>=3.*, < 4.0 6 | pytest-xdist>=2.*, < 3.0 7 | pytest>=7.*, < 8.0 8 | -------------------------------------------------------------------------------- /tests/test_main.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from dream_bench import tokenizer 3 | 4 | 5 | @pytest.mark.parametrize("message", ["hello", "world"]) 6 | def test_tokenizer(message): 7 | tokenizer.tokenize(texts=message, context_length=77, truncate_text=True) 8 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click>=8.*, < 9.0 2 | ftfy>=6.*, < 7.0 3 | pydantic>=1.9.*, < 2.0.0 4 | regex>=2022.7.*, < 2023.0.0 5 | torch>=1.*, < 1.13 6 | torchmetrics>=0.8.*, < 1.0.0 7 | torchvision>=0.12.*, < 0.13.0 8 | wandb>=0.12.*, < 0.13.0 9 | webdataset>=0.2.*, < 0.3.0 10 | pandas>=1.4*, < 1.5 11 | toma>=1.1.0, < 2.0.0 -------------------------------------------------------------------------------- /prompts/README.md: -------------------------------------------------------------------------------- 1 | # Benchmarking Prompts 2 | 3 | Here are some example prompt lists for you to use when benchmarking your model, you can use these to convert them into a prompt-only dataset using the provided [script](../prompts2datset.py). For more information about how to run the script you can check its [dedicated readme](../dataset_readme.md) 4 | 5 | ## Available prompt lists 6 | - dalle mega list 7 | - drawbench 8 | -------------------------------------------------------------------------------- /configs/example_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": { 3 | "path": "benchmarks/drawbench/drawbench-0000.tar", 4 | "batch_size": 16 5 | }, 6 | "wandb": { 7 | "entity": "nousr_laion", 8 | "project": "dream_bench_test" 9 | }, 10 | "evaluator": { 11 | "metrics": ["Aesthetic", "ClipScore"], 12 | "device": "cuda", 13 | "clip_architecture": "ViT-L/14" 14 | } 15 | } -------------------------------------------------------------------------------- /configs/dome_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataset": { 3 | "path": "benchmarks/dome/dome-0000.tar", 4 | "batch_size": 8 5 | }, 6 | "wandb": { 7 | "entity": "nousr_laion", 8 | "project": "dream_bench_test", 9 | "id": "fl17v790" 10 | }, 11 | "evaluator": { 12 | "metrics": [ 13 | "Aesthetic", 14 | "ClipScore" 15 | ], 16 | "device": "cuda", 17 | "clip_architecture": "ViT-L/14" 18 | } 19 | } -------------------------------------------------------------------------------- /dream_bench/dream_bench.py: -------------------------------------------------------------------------------- 1 | from torch.utils.data import DataLoader 2 | 3 | from dream_bench.config import DreamBenchConfig 4 | 5 | 6 | def benchmark(model, adapter, config: DreamBenchConfig): 7 | """ 8 | Benchmark a model 9 | """ 10 | 11 | # load the dataset from the config 12 | dataset = config.dataset.load() 13 | 14 | # init the user's wandb 15 | config.wandb.init() 16 | 17 | evaluator = config.evaluator.load(dataset=dataset) 18 | 19 | # begin benchmarking 20 | for input_dict in DataLoader(dataset=dataset, batch_size=config.dataset.batch_size): 21 | images = adapter(model, input_dict) 22 | evaluator.record_predictions(model_output=images) 23 | 24 | # compute benchmarks 25 | evaluator.evaluate() 26 | -------------------------------------------------------------------------------- /dream_bench/helpers.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from click import secho 3 | from importlib import import_module 4 | 5 | 6 | def is_url(x: str): 7 | if x.startswith("http://") or x.startswith("https://"): 8 | return True 9 | 10 | return False 11 | 12 | 13 | def exists(x): 14 | return x is not None 15 | 16 | 17 | def filename_from_path(path): 18 | return path.split("/")[-1] 19 | 20 | 21 | def import_or_print_error(pkg_name, err_str=None, **kwargs): 22 | try: 23 | return import_module(pkg_name) 24 | except ModuleNotFoundError as _: 25 | if exists(err_str): 26 | secho(err_str, **kwargs) 27 | return sys.exit() 28 | 29 | 30 | def print_ribbon(s, symbol="=", repeat=10): 31 | flank = symbol * repeat 32 | return f"{flank} {s} {flank}" 33 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Continuous integration 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | runs-on: ${{ matrix.os }} 12 | 13 | strategy: 14 | matrix: 15 | python-version: [ '3.8','3.9', '3.10' ] 16 | os: [ubuntu-latest] 17 | 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Setup Python ${{ matrix.python-version }} 21 | 22 | - uses: actions/setup-python@v4 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | cache: 'pip' 26 | 27 | - name: Install deps 28 | run: | 29 | make install 30 | make install-dev 31 | 32 | - name: Lint 33 | run: | 34 | make lint 35 | 36 | - name: Unit Tests 37 | run: | 38 | make test 39 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | install: ## [Local development] Upgrade pip, install requirements, install package. 2 | python -m pip install -U pip 3 | python -m pip install -e . 4 | 5 | install-dev: ## [Local development] Install test requirements 6 | python -m pip install -r requirements-test.txt 7 | 8 | lint: ## [Local development] Run mypy, pylint and black 9 | python -m mypy dream_bench 10 | python -m pylint dream_bench 11 | python -m black --check -l 120 dream_bench 12 | 13 | black: ## [Local development] Auto-format python code using black 14 | python -m black -l 120 . 15 | 16 | build-pex: 17 | python3 -m venv .pexing 18 | . .pexing/bin/activate && python -m pip install -U pip && python -m pip install pex 19 | . .pexing/bin/activate && python -m pex setuptools . -o dream_bench.pex -v 20 | rm -rf .pexing 21 | 22 | test: ## [Local development] Run unit tests 23 | python -m pytest -x -s -v tests 24 | 25 | .PHONY: help 26 | 27 | help: # Run `make help` to get help on the make commands 28 | @grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 zion 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /configs/README.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | Dream bench utilizes pydantic classes to manage configuration of your benchmarking run. You can view an example of it [here](example_config.json) 4 | 5 | ## Dataset Config 6 | 7 | | key | explanation | 8 | |------------|-------------------------------------------------| 9 | | path | path to the webdataset. | 10 | | batch_size | the batch size for your image generation model. | 11 | 12 | ## Wandb 13 | 14 | | key | explanation | 15 | |---------|----------------------------| 16 | | entity | wandb entity to use | 17 | | project | the wandb project name | 18 | | name | (optional) name of the run | 19 | 20 | ## Evaluation 21 | | key | explanation | 22 | |-------------------|------------------------------------------------------------------------| 23 | | save_path | Path to save predicted outputs too. | 24 | | metrics | A list of metrics to use in your run ["FID", "Aesthetic", "ClipScore"] | 25 | | device | torch device ["cuda:?", "cpu"] to use for evaluation | 26 | | clip_architecture | Clip architecture to use for evaluation ["ViT-L/14", "ViT-B/32"] | 27 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | from pathlib import Path 3 | import os 4 | 5 | if __name__ == "__main__": 6 | with Path(Path(__file__).parent, "README.md").open(encoding="utf-8") as file: 7 | long_description = file.read() 8 | 9 | def _read_reqs(relpath): 10 | fullpath = os.path.join(os.path.dirname(__file__), relpath) 11 | with open(fullpath) as f: 12 | return [s.strip() for s in f.readlines() if (s.strip() and not s.startswith("#"))] 13 | 14 | REQUIREMENTS = _read_reqs("requirements.txt") 15 | 16 | setup( 17 | name="dream_bench", 18 | packages=find_packages(), 19 | include_package_data=True, 20 | version="0.0.1", 21 | license="MIT", 22 | description="A tool for benchmarking image generation models.", 23 | long_description=long_description, 24 | long_description_content_type="text/markdown", 25 | author="Zion English", 26 | author_email="zion.m.english@gmail.com", 27 | url="https://github.com/nousr/dream-bench", 28 | data_files=[(".", ["README.md"])], 29 | keywords=["machine learning", "txt2im", "benchmark", "pytorch"], 30 | install_requires=REQUIREMENTS, 31 | classifiers=[ 32 | "Development Status :: 4 - Beta", 33 | "Intended Audience :: Developers", 34 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 35 | "License :: OSI Approved :: MIT License", 36 | "Programming Language :: Python :: 3.10", 37 | ], 38 | ) 39 | -------------------------------------------------------------------------------- /dream_bench/config.py: -------------------------------------------------------------------------------- 1 | from json import load 2 | from typing import List, Optional 3 | 4 | import wandb 5 | from pydantic import BaseModel 6 | from wandb.util import generate_id 7 | from webdataset import WebDataset 8 | 9 | from dream_bench.evaluator import METRICS, Evaluator 10 | 11 | 12 | class DatasetConfig(BaseModel): 13 | path: str 14 | batch_size: int 15 | 16 | def load(self): 17 | return WebDataset(self.path).decode() 18 | 19 | 20 | class WandbConfig(BaseModel): 21 | """ 22 | Configure the wandb settings for this run. 23 | 24 | NOTE: use 'id' to specify the run_id you wish to resume 25 | """ 26 | 27 | entity: str # your wandb username 28 | project: str # project name 29 | name: Optional[str] = None # name of your run 30 | id: Optional[str] = None # run_id to resume a specific run 31 | resume: str = "allow" # wandb resume method 32 | 33 | class Config: 34 | extra = "allow" 35 | 36 | def init(self): 37 | kwargs = self.dict() 38 | 39 | if self.id is None: 40 | kwargs["id"] = generate_id() 41 | 42 | wandb.init(**kwargs) 43 | 44 | 45 | class EvaluatorConfig(BaseModel): 46 | metrics: List[METRICS] = ["Aesthetic"] 47 | device: str = "cpu" 48 | clip_architecture: Optional[str] 49 | default_batch_size: int = 128 50 | 51 | def load(self, dataset): 52 | return Evaluator( 53 | metrics=self.metrics, 54 | device=self.device, 55 | clip_architecture=self.clip_architecture, 56 | dataset=dataset, 57 | default_batch_size=self.default_batch_size, 58 | ) 59 | 60 | 61 | class DreamBenchConfig(BaseModel): 62 | dataset: DatasetConfig 63 | wandb: WandbConfig 64 | evaluator: EvaluatorConfig 65 | 66 | @classmethod 67 | def from_json_path(cls, json_path): 68 | with open(json_path, encoding="utf-8") as f: 69 | config = load(f) 70 | return cls(**config) 71 | -------------------------------------------------------------------------------- /dataset_readme.md: -------------------------------------------------------------------------------- 1 | # Prompts2Dataset 2 | 3 | prompts2dataset is a tool that converts a json list of prompts into a dataset for use with dream-bench. 4 | 5 | ``` 6 | Options: 7 | --prompt-list TEXT A json file containing a list of prompts. 8 | [required] 9 | --output-folder TEXT parent folder of the dataset to be created 10 | [required] 11 | --batch-size INTEGER Number of prompts to process in parallel. 12 | --prompt-repeat INTEGER Number of times to run inference for a prompt. 13 | --embed-text Wether to use clip to embed the prompts as text 14 | embeddings 15 | --tokenize-text Wether to save the tokenized the text. 16 | --predict-image Wether to use a diffusion prior to create an image 17 | embedding from each prompt. 18 | --clip-model TEXT Clip model to use for embedding images. 19 | --prior-checkpoint TEXT A local path or web url to a prior config file. 20 | --prior-config TEXT A local path or web url to a prior checkpoint. 21 | --override-folder wether to overrite the output folder if it exists 22 | --help Show this message and exit. 23 | ``` 24 | 25 | ## Overview 26 | 27 | `dream_bench` currently provides two major methods for supplying your model with input each with three sub-types for you to choose from. 28 | 29 | 1. Text 30 | 1. clip text embeddings (tensor) 31 | 2. tokenized text (tensor) 32 | 3. raw text (string) 33 | 2. Images 34 | 1. clip image embeddings (tensor) 35 | 2. diffusion prior embeddings (tensor) 36 | 3. raw RGB-images (tensor) 37 | 38 | ## Getting Started 39 | 40 | The easiest way to get started is to have a model that can generate images from text, as well as a list of prompts you wish to utilize as your benchmark. For convinience, this repository hosts popular benchmarking prompts, available [here](../../prompts/README.md). 41 | 42 | To turn your prompt list into a dataset, you can simply utilize the provided `prompts2datset.py` script to format everything nicely. 43 | 44 | ```bash 45 | python prompts2datset.py --prompt-list drawbench.json --output-folder benchmarks/drawbench --tokenize-text True --predict-image True 46 | ``` 47 | 48 | Here we call the prompt-processing script and pass it four arguments: 49 | * First, the path to our prompt list (in this case, we use the provided drawbench list). 50 | * Then, we specify the root folder we'd like to store the resulting benchmark. 51 | * Next, we specify that we'd prefer tokenized text (as opposed to raw strings of text). 52 | * Finally, we specify that we would also like to leverage a diffusion prior to invert the text, and generate a synthetic image embedding for each caption. 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | .dev 108 | env/ 109 | venv/ 110 | ENV/ 111 | env.bak/ 112 | venv.bak/ 113 | 114 | # Spyder project settings 115 | .spyderproject 116 | .spyproject 117 | 118 | # Rope project settings 119 | .ropeproject 120 | 121 | # mkdocs documentation 122 | /site 123 | 124 | # mypy 125 | .mypy_cache/ 126 | .dmypy.json 127 | dmypy.json 128 | 129 | # Pyre type checker 130 | .pyre/ 131 | 132 | # wandb 133 | wandb 134 | 135 | # editor 136 | .vscode 137 | 138 | # scratchwork 139 | scratchwork* 140 | .scratchwork/ 141 | scratchwork/ 142 | 143 | # models 144 | .models 145 | models 146 | checkpoints 147 | 148 | # benchmarks folder 149 | benchmarks 150 | benchmark 151 | predictions 152 | 153 | # tars 154 | *.tar 155 | 156 | # configs 157 | !configs/example_config.json 158 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

🖼️ Dream Bench 📊

2 | 3 | 4 | ## What does it do? 5 | 6 | `dream_bench` provides a simplified interface for benchmarking your image-generation models. 7 | 8 | This repository also hosts common prompt-lists for benchmarking, as well as instructions for preparing a dataset to perform a more comprehensive evaluation. 9 | 10 | ## How does it work? 11 | 12 | To start off, you will need to create an evaluation adapter for you model so that it can interface with `dream_bench`. This function will be called by `dream_bench` and expect an image in return. 13 | 14 | ```python 15 | class MyImageModel(): 16 | def __init__(self, *args, **kwargs): 17 | super().__init__() 18 | ... 19 | 20 | def sample(self, img_emb, txt_emb): 21 | ... 22 | 23 | def my_evaluation_harness(self, conditioning_args): 24 | 25 | # extract what you need to from the conditioning arguments 26 | 27 | img_emb = conditioning_args["prior_image_embedding.npy"] 28 | txt_emb = conditioning_args["clip_text_embedding.npy"] 29 | 30 | # sample with your model's function 31 | 32 | predicted_image = self.sample(img_emb=img_emb, txt_emb=txt_emb) 33 | 34 | # return the image(s) to be evaluated by dream bench 35 | 36 | return predicted_image 37 | ``` 38 | 39 | Once you have a function that will accept `conditioning_args` and return an `image`, you can pass this function to `dream_bench` to handle the rest! 40 | 41 | ```python 42 | from dream_bench import benchmark, DreamBenchConfig 43 | 44 | # specify what you'd like to benchmark 45 | 46 | config = DreamBenchConfig.from_json_path("") 47 | 48 | def train(model, dataloader, epochs): 49 | for epoch in range(epochs): 50 | for x,y in dataloader: 51 | # do training 52 | 53 | ... 54 | 55 | # benchmark on some interval 56 | 57 | if time_to_benchmark: 58 | benchmark(adapter=model.my_evaluation_harness, config=config) 59 | ``` 60 | 61 | If you're done training and would like to benchmark a pre-trained model it can be done in the following way. 62 | 63 | ```python 64 | from dream_bench import benchmark, DreamBenchConfig 65 | 66 | # specify what you'd like to benchmark 67 | 68 | config = DreamBenchConfig.from_json_path("") 69 | 70 | # if your model doesn't have an adapter, you can create one now 71 | 72 | def my_evaluation_harness(self, conditioning_args): 73 | 74 | # extract what you need to from the conditioning arguments 75 | 76 | img_emb = conditioning_args["prior_image_embedding.npy"] 77 | txt_emb = conditioning_args["clip_text_embedding.npy"] 78 | 79 | # sample with your model's function 80 | 81 | predicted_image = self.sample(img_emb=img_emb, txt_emb=txt_emb) 82 | 83 | # return the image(s) to be evaluated by dream bench 84 | 85 | return predicted_image 86 | 87 | def main(): 88 | # load your model 89 | 90 | model = load_model() 91 | 92 | # call benchmark outside of training 93 | 94 | benchmark(adapter=model.my_evaluation_harness, config=config) 95 | 96 | if __name__ == "__main__": 97 | main() 98 | ``` 99 | 100 | ## Setup 101 | 102 | `dream_bench` works on the `Webdataset` format. 103 | 104 | Before you run evaluation, you must preprocess your dataset/prompt-list so that is compatible with `dream_bench`. 105 | 106 | For more information on how this can be done, take a look at the [dedicated readme](dataset_readme.md) 107 | 108 | Next you will need to create a configuration file to run your evaluation, this will tell dream-bench how to benchmark your run, some information on how to build this config file you can read its [dedicated readme](configs/README.md) 109 | 110 | ## Tracking 111 | 112 | To track your experiments, `dream_bench` utilizes [weights & biases](https://wandb.ai). With `wandb` it is possible to easily view your generations in an easy-to-read format, compile reports, and query against/across your runs. 113 | 114 | Additional support may be added for other trackers in the future. 115 | 116 | --- 117 | 118 | ## ToDo 119 | - [x] Determine dataset format 120 | - [x] Provide scripts for generating datasets from prompt lists 121 | - [x] Block in more metrics 122 | - [x] Add continuous integration 123 | - [x] Add wandb integration 124 | - [x] Build configuration 125 | - [x] Lint project & Capture unecessary warnings 126 | - [x] Build out benchmarking interface for clip-based models 127 | - [x] Track runs with wandb (look for efficiency later) 128 | - [x] Track averages of each metric 129 | - [ ] Add FID metric 130 | - [ ] Add option for distributed evaluation 131 | - [ ] Prioritize benchmarking for ongoing training runs 132 | - [ ] Provide guide/scripts for formatting other datasets/url-lists 133 | - [ ] Add More tests 134 | - [ ] Profile code & look for optimizations 135 | - [ ] Publish to PyPi 136 | - [ ] Add support for distributed benchmarking 137 | -------------------------------------------------------------------------------- /dream_bench/load_models.py: -------------------------------------------------------------------------------- 1 | import os 2 | from dream_bench.helpers import ( 3 | import_or_print_error, 4 | exists, 5 | filename_from_path, 6 | is_url, 7 | ) 8 | from urllib.request import urlretrieve 9 | from torch import load as torch_load, nn 10 | from torchvision.transforms import ( 11 | CenterCrop, 12 | Resize, 13 | Normalize, 14 | Compose, 15 | InterpolationMode, 16 | ) 17 | from os.path import expanduser 18 | 19 | CACHE_FOLDER = os.path.join(os.path.expanduser("~"), ".cache", "dream_bench") 20 | DEFAULT_PRIOR_STATE_URL = "https://huggingface.co/laion/DALLE2-PyTorch/resolve/main/prior/latest.pth" 21 | DEFAULT_PRIOR_CONFIG_URL = "https://huggingface.co/laion/DALLE2-PyTorch/raw/main/prior/prior_config.json" 22 | 23 | 24 | def _load_open_clip(clip_model, use_jit, device): 25 | "Load a clip model from the open-clip library" 26 | 27 | open_clip = import_or_print_error( 28 | "open_clip", 29 | err_str="You have requested an open-clip model but do not have the library installed.", 30 | fg="red", 31 | ) 32 | 33 | pretrained = dict(open_clip.list_pretrained()) 34 | checkpoint = pretrained[clip_model] 35 | model, _, _ = open_clip.create_model_and_transforms(clip_model, pretrained=checkpoint, device=device, jit=use_jit) 36 | 37 | preprocess = Compose( 38 | [ 39 | Resize(model.visual.input_resolution, interpolation=InterpolationMode.BICUBIC), 40 | CenterCrop(model.visual.input_resolution), 41 | Normalize( 42 | (0.48145466, 0.4578275, 0.40821073), 43 | (0.26862954, 0.26130258, 0.27577711), 44 | ), 45 | ] 46 | ) 47 | 48 | return model, preprocess 49 | 50 | 51 | def load_clip(clip_model, use_jit=False, device="cpu"): 52 | "Load a clip model from openai or open-clip" 53 | 54 | if clip_model.startswith("open_clip:"): 55 | clip_model = clip_model[len("open_clip:") :] 56 | return _load_open_clip(clip_model, use_jit, device) 57 | 58 | else: 59 | clip = import_or_print_error( 60 | "clip", 61 | err_str="You have requested an openai-clip model but do not have the library installed.", 62 | fg="red", 63 | ) 64 | 65 | model, _ = clip.load(clip_model, device=device, jit=use_jit) 66 | 67 | preprocess = Compose( 68 | [ 69 | Resize(model.visual.input_resolution, interpolation=InterpolationMode.BICUBIC), 70 | CenterCrop(model.visual.input_resolution), 71 | Normalize( 72 | (0.48145466, 0.4578275, 0.40821073), 73 | (0.26862954, 0.26130258, 0.27577711), 74 | ), 75 | ] 76 | ) 77 | 78 | model.eval() 79 | 80 | return model, preprocess 81 | 82 | 83 | def _download_prior(state_url: str, config_url: str): 84 | "Download a diffusion prior and configuration file from a url" 85 | 86 | # create cache folders 87 | 88 | cache_folder = os.path.join(CACHE_FOLDER, "prior") 89 | os.makedirs(cache_folder, exist_ok=True) 90 | 91 | prior_file = os.path.join(cache_folder, filename_from_path(state_url)) 92 | config_file = os.path.join(cache_folder, filename_from_path(config_url)) 93 | 94 | # download checkpoint & model to cache if they don't exist 95 | 96 | if not os.path.isfile(prior_file): 97 | prior_file, _ = urlretrieve(state_url, prior_file) 98 | 99 | if not os.path.isfile(config_file): 100 | config_file, _ = urlretrieve(config_url, config_file) 101 | 102 | # return files 103 | 104 | return prior_file, config_file 105 | 106 | 107 | def load_prior(checkpoint_path: str, config_path: str, device="cpu"): 108 | "Load a dalle2-pytorch diffusion prior model" 109 | 110 | dalle2_train_config = import_or_print_error( 111 | "dalle2_pytorch.train_configs", 112 | err_str="dalle2_pytorch is a required package to load diffusion_prior models.", 113 | fg="red", 114 | ) 115 | 116 | # if both params are none, then replace with defaults 117 | 118 | if not exists(checkpoint_path) and not exists(config_path): 119 | checkpoint_path = DEFAULT_PRIOR_STATE_URL 120 | config_path = DEFAULT_PRIOR_CONFIG_URL 121 | 122 | # check if the paths are actually urls to download from 123 | 124 | if is_url(checkpoint_path) and is_url(config_path): 125 | checkpoint_path, config_path = _download_prior(checkpoint_path, config_path) 126 | 127 | # check if the path exists 128 | 129 | assert os.path.exists(checkpoint_path) and os.path.exists( 130 | config_path 131 | ), "Files must exist and be visible if loading from a local path." 132 | 133 | # load configuration from path 134 | 135 | prior_config = dalle2_train_config.TrainDiffusionPriorConfig.from_json_path(config_path) 136 | prior_config = prior_config.prior 137 | 138 | # create model from config 139 | 140 | diffusion_prior = prior_config.create() 141 | state_dict = torch_load(checkpoint_path, map_location=device) 142 | diffusion_prior.load_state_dict(state_dict) 143 | diffusion_prior.eval() 144 | diffusion_prior.to(device) 145 | 146 | if device == "cpu": 147 | diffusion_prior.float() 148 | 149 | return diffusion_prior 150 | 151 | 152 | def get_aesthetic_model(clip_model="ViT-L/14"): 153 | """load the aethetic model""" 154 | if clip_model == "ViT-L/14": 155 | model_file = "vit_l_14" 156 | elif clip_model == "ViT-B/32": 157 | model_file = "vit_b_32" 158 | else: 159 | raise NotImplementedError(f"No aesthetic model has been trained on the {clip_model} architecture.") 160 | 161 | home = expanduser("~") 162 | cache_folder = home + "/.cache/emb_reader" 163 | path_to_model = cache_folder + "/sa_0_4_" + model_file + "_linear.pth" 164 | if not os.path.exists(path_to_model): 165 | os.makedirs(cache_folder, exist_ok=True) 166 | url_model = ( 167 | "https://github.com/LAION-AI/aesthetic-predictor/blob/main/sa_0_4_" + model_file + "_linear.pth?raw=true" 168 | ) 169 | urlretrieve(url_model, path_to_model) 170 | if clip_model == "ViT-L/14": 171 | m = nn.Linear(768, 1) 172 | elif clip_model == "ViT-B/32": 173 | m = nn.Linear(512, 1) 174 | else: 175 | raise ValueError() 176 | s = torch_load(path_to_model) 177 | m.load_state_dict(s) 178 | m.eval() 179 | return m 180 | -------------------------------------------------------------------------------- /dream_bench/tokenizer.py: -------------------------------------------------------------------------------- 1 | """ 2 | OpenAI's SimpleTokenizer 3 | 4 | from: https://raw.githubusercontent.com/openai/CLIP/main/clip/simple_tokenizer.py 5 | """ 6 | 7 | import html 8 | import os 9 | from functools import lru_cache 10 | from pathlib import Path 11 | 12 | import ftfy 13 | import regex as re 14 | import torch 15 | 16 | 17 | @lru_cache() 18 | def default_bpe(): 19 | return os.path.join(os.path.dirname(os.path.abspath(__file__)), "data/bpe_simple_vocab_16e6.txt") 20 | 21 | 22 | @lru_cache() 23 | def bytes_to_unicode(): 24 | """ 25 | Returns list of utf-8 byte and a corresponding list of unicode strings. 26 | The reversible bpe codes work on unicode strings. 27 | This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. 28 | When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. 29 | This is a signficant percentage of your normal, say, 32K bpe vocab. 30 | To avoid that, we want lookup tables between utf-8 bytes and unicode strings. 31 | And avoids mapping to whitespace/control characters the bpe code barfs on. 32 | """ 33 | bs = list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1)) 34 | cs = bs[:] 35 | n = 0 36 | for b in range(2**8): 37 | if b not in bs: 38 | bs.append(b) 39 | cs.append(2**8 + n) 40 | n += 1 41 | cs = [chr(n) for n in cs] 42 | return dict(zip(bs, cs)) 43 | 44 | 45 | def get_pairs(word): 46 | """Return set of symbol pairs in a word. 47 | Word is represented as tuple of symbols (symbols being variable-length strings). 48 | """ 49 | pairs = set() 50 | prev_char = word[0] 51 | for char in word[1:]: 52 | pairs.add((prev_char, char)) 53 | prev_char = char 54 | return pairs 55 | 56 | 57 | def basic_clean(text): 58 | text = ftfy.fix_text(text) 59 | text = html.unescape(html.unescape(text)) 60 | return text.strip() 61 | 62 | 63 | def whitespace_clean(text): 64 | text = re.sub(r"\s+", " ", text) 65 | text = text.strip() 66 | return text 67 | 68 | 69 | class SimpleTokenizer: 70 | """ 71 | A tokenizer used in many CLIP models. 72 | """ 73 | 74 | def __init__(self, bpe_path: str = default_bpe()): 75 | self.byte_encoder = bytes_to_unicode() 76 | self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} 77 | raw_merge = Path(bpe_path).read_text(encoding="utf8").split("\n") 78 | raw_merge = raw_merge[1 : 49152 - 256 - 2 + 1] 79 | merges = [tuple(merge.split()) for merge in raw_merge] 80 | vocab = list(bytes_to_unicode().values()) 81 | vocab = vocab + [v + "" for v in vocab] 82 | for merge in merges: 83 | vocab.append("".join(merge)) 84 | vocab.extend(["<|startoftext|>", "<|endoftext|>"]) 85 | self.encoder = dict(zip(vocab, range(len(vocab)))) 86 | self.decoder = {v: k for k, v in self.encoder.items()} 87 | self.bpe_ranks = dict(zip(merges, range(len(merges)))) 88 | self.cache = { 89 | "<|startoftext|>": "<|startoftext|>", 90 | "<|endoftext|>": "<|endoftext|>", 91 | } 92 | self.pat = re.compile( 93 | r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", 94 | re.IGNORECASE, 95 | ) 96 | 97 | def bpe(self, token): 98 | "Encode tokens with the byte-pair method" 99 | 100 | if token in self.cache: 101 | return self.cache[token] 102 | word = tuple(token[:-1]) + (token[-1] + "",) 103 | pairs = get_pairs(word) 104 | 105 | if not pairs: 106 | return token + "" 107 | 108 | while True: 109 | bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) 110 | if bigram not in self.bpe_ranks: 111 | break 112 | first, second = bigram 113 | new_word = [] 114 | i = 0 115 | while i < len(word): 116 | try: 117 | j = word.index(first, i) 118 | new_word.extend(word[i:j]) 119 | i = j 120 | except Exception as _: # pylint: disable=broad-except 121 | new_word.extend(word[i:]) 122 | break 123 | 124 | if word[i] == first and i < len(word) - 1 and word[i + 1] == second: 125 | new_word.append(first + second) 126 | i += 2 127 | else: 128 | new_word.append(word[i]) 129 | i += 1 130 | 131 | new_word = tuple(new_word) 132 | word = new_word 133 | 134 | if len(word) == 1: 135 | break 136 | 137 | pairs = get_pairs(word) 138 | 139 | word = " ".join(word) 140 | self.cache[token] = word 141 | return word 142 | 143 | def encode(self, text): 144 | bpe_tokens = [] 145 | text = whitespace_clean(basic_clean(text)).lower() 146 | for token in re.findall(self.pat, text): 147 | token = "".join(self.byte_encoder[b] for b in token.encode("utf-8")) 148 | bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" ")) 149 | return bpe_tokens 150 | 151 | def decode(self, tokens): 152 | text = "".join([self.decoder[token] for token in tokens]) 153 | text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors="replace").replace("", " ") 154 | return text 155 | 156 | def tokenize(self, texts, context_length=77, truncate_text=False): 157 | "Tokenize a str or List[str] and encode it for use in models." 158 | 159 | if isinstance(texts, str): 160 | texts = [texts] 161 | 162 | all_tokens = [self.encode(text) for text in texts] 163 | result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) 164 | 165 | for i, tokens in enumerate(all_tokens): 166 | if len(tokens) > context_length: 167 | if truncate_text: 168 | tokens = tokens[:context_length] 169 | else: 170 | raise RuntimeError(f"Input {texts[i]} is too long for context length {context_length}") 171 | result[i, : len(tokens)] = torch.tensor(tokens) 172 | 173 | return result 174 | 175 | 176 | tokenizer = SimpleTokenizer() 177 | -------------------------------------------------------------------------------- /make_dataset.py: -------------------------------------------------------------------------------- 1 | #! python 2 | 3 | import os 4 | import json 5 | import click 6 | import torch 7 | import numpy as np 8 | import pandas as pd 9 | from PIL import Image 10 | from torch.nn import Module 11 | from torch.cuda import is_available 12 | from webdataset import ShardWriter 13 | from dream_bench.load_models import load_clip, load_prior, filename_from_path 14 | from dream_bench.tokenizer import tokenizer 15 | from dream_bench.helpers import print_ribbon 16 | 17 | DEVICE = "cuda" if is_available() else "cpu" 18 | TOKENIZER_CONTEXT_LENGTH = 77 19 | TOKENIZER_TRUNCATE_TEXT = True 20 | 21 | 22 | def load_prompts(filepath, prompt_repeat): 23 | """ 24 | Load the prompts from a file 25 | """ 26 | 27 | if filepath.endswith(".json"): 28 | with open(filepath, "r") as f: 29 | return np.array(json.load(f)).repeat(prompt_repeat) 30 | elif filepath.endswith(".parquet"): 31 | df = pd.read_parquet(filepath) 32 | try: 33 | return np.array(df["caption"].values).repeat(prompt_repeat) 34 | except KeyError as e: 35 | print(e) 36 | print("There is no `caption` key in your dataframe") 37 | else: 38 | print("This filetype is not supported") 39 | exit(1) 40 | 41 | 42 | def load_images(filepath, resize=None): 43 | images = [] 44 | 45 | for file in os.listdir(filepath): 46 | image = Image.open(os.path.join(filepath, file)) 47 | 48 | if resize is not None: 49 | image = image.resize((resize, resize), Image.Resampling.LANCZOS) 50 | 51 | images.append(np.array(image).transpose(2, 0, 1)) 52 | 53 | return images 54 | 55 | 56 | def txt2embeddings(model: Module, architecture: str, text: torch.Tensor, batch_size: int): 57 | """ 58 | Predict embeddings for captions using a given batch size. 59 | 60 | Returns: 61 | - All predicted embeddings batched along the first dimension and converted to numpy format. 62 | """ 63 | click.secho(print_ribbon(f"Creating Embeddings With: {architecture}"), fg="green") 64 | 65 | embeds = list() 66 | 67 | for i in range(0, len(text), batch_size): 68 | model_input = text[i : i + batch_size] 69 | 70 | if architecture == "prior": 71 | embeds.append(model.sample(model_input).detach().cpu().numpy()) 72 | elif architecture == "clip": 73 | embeds.append(model.encode_text(model_input).detach().cpu().numpy()) 74 | else: 75 | click.secho(f"The {architecture} architecture is not yet supported.", fg="red") 76 | exit(1) 77 | 78 | return np.concatenate(embeds, axis=0) 79 | 80 | 81 | def wds_create( 82 | prompts: list, 83 | images: list, 84 | batch_size: int, 85 | output_folder: str, 86 | filename: str, 87 | clip_model: Module, 88 | prior_model: Module, 89 | predict_image: bool, 90 | embed_text: bool, 91 | tokenize_text: bool, 92 | ): 93 | """ 94 | Create a webdataset for the prompt list. 95 | 96 | # TODO: docstring 97 | """ 98 | filename = filename.split(".")[0] 99 | 100 | # precompute embeddings for all requested formats 101 | tokenized_text = tokenizer.tokenize( 102 | prompts, 103 | context_length=TOKENIZER_CONTEXT_LENGTH, 104 | truncate_text=TOKENIZER_TRUNCATE_TEXT, 105 | ).to(DEVICE) 106 | clip_text_embeddings = ( 107 | txt2embeddings( 108 | model=clip_model, 109 | text=tokenized_text, 110 | batch_size=batch_size, 111 | architecture="clip", 112 | ) 113 | if embed_text 114 | else None 115 | ) 116 | prior_image_embeddings = ( 117 | txt2embeddings( 118 | model=prior_model, 119 | text=tokenized_text, 120 | batch_size=batch_size, 121 | architecture="prior", 122 | ) 123 | if predict_image 124 | else None 125 | ) 126 | 127 | with ShardWriter(pattern=f"{output_folder}/{filename}-%04d.tar") as sink: 128 | for idx, caption in enumerate(prompts): 129 | tar_keys = {"__key__": "%04d" % idx, "caption.txt": caption} 130 | 131 | if predict_image: 132 | tar_keys["prior_image_embedding.npy"] = prior_image_embeddings[idx].detach().cpu().numpy() 133 | if embed_text: 134 | tar_keys["clip_text_embedding.npy"] = clip_text_embeddings[idx].detach().cpu().numpy() 135 | if tokenize_text: 136 | tar_keys["tokenized_text.npy"] = tokenized_text[idx].detach().cpu().numpy() 137 | if images is not None: 138 | tar_keys["real_image.npy"] = images[idx] 139 | 140 | sink.write(tar_keys) 141 | 142 | 143 | @click.command() 144 | @click.option( 145 | "--prompt-list", 146 | type=str, 147 | help="A json or parquet file containing a list of prompts.", 148 | required=True, 149 | ) 150 | @click.option( 151 | "--real-images", 152 | type=str, 153 | help="A path to a folder containing real images", 154 | default=None, 155 | ) 156 | @click.option( 157 | "--resize", 158 | type=int, 159 | help="What size to reisze to when loading real images.", 160 | ) 161 | @click.option( 162 | "--output-folder", 163 | type=str, 164 | help="parent folder of the dataset to be created", 165 | required=True, 166 | ) 167 | @click.option( 168 | "--batch-size", 169 | default=512, 170 | type=int, 171 | help="Number of prompts to process in parallel.", 172 | ) 173 | @click.option( 174 | "--prompt-repeat", 175 | default=1, 176 | type=int, 177 | help="Number of times to run inference for a prompt.", 178 | ) 179 | @click.option( 180 | "--embed-text", 181 | is_flag=True, 182 | help="Wether to use clip to embed the prompts as text embeddings", 183 | ) 184 | @click.option( 185 | "--tokenize-text", 186 | is_flag=True, 187 | help="Wether to save the tokenized the text.", 188 | ) 189 | @click.option( 190 | "--predict-image", 191 | is_flag=True, 192 | help="Wether to use a diffusion prior to create an image embedding from each prompt.", 193 | ) 194 | @click.option( 195 | "--clip-model", 196 | default=None, 197 | type=str, 198 | help="Clip model to use for embedding images.", 199 | ) 200 | @click.option( 201 | "--prior-checkpoint", 202 | default=None, 203 | type=str, 204 | help="A local path or web url to a prior config file.", 205 | ) 206 | @click.option( 207 | "--prior-config", 208 | default=None, 209 | type=str, 210 | help="A local path or web url to a prior checkpoint.", 211 | ) 212 | @click.option( 213 | "--override-folder", 214 | is_flag=True, 215 | help="wether to overrite the output folder if it exists", 216 | ) 217 | def main( 218 | prompt_list, 219 | real_images, 220 | resize, 221 | output_folder, 222 | batch_size, 223 | prompt_repeat, 224 | embed_text, 225 | tokenize_text, 226 | predict_image, 227 | clip_model, 228 | prior_checkpoint, 229 | prior_config, 230 | override_folder, 231 | ): 232 | filename = filename_from_path(prompt_list) 233 | options = { 234 | "tokenized_text": tokenize_text, 235 | "clip_text_embed": embed_text, 236 | "prior_image_embed": predict_image, 237 | } 238 | 239 | # TODO: asserts & ensure that clip model & prior model align 240 | 241 | click.secho( 242 | f"Generating benchmark dataset for: {filename}", 243 | fg="bright_cyan", 244 | underline=True, 245 | ) 246 | 247 | for k, v in options.items(): 248 | click.secho(f"\t* {k} : {v}", fg="green" if v else "red") 249 | 250 | # check that the output folder can be created 251 | 252 | try: 253 | os.makedirs(output_folder, exist_ok=override_folder) 254 | click.secho(f"Dataset will be created at: {output_folder}", fg="magenta") 255 | except OSError: 256 | click.secho( 257 | f"WARNING: The folder you specified already exists. This is a safety measure to avoid destroying existing work. If you wish to override it, please re-run this script with the --override-folder flag.", 258 | fg="red", 259 | ) 260 | 261 | exit(1) 262 | 263 | # load prompt list 264 | prompts = load_prompts(prompt_list, prompt_repeat) 265 | 266 | # load real images 267 | images = load_images(real_images, resize) if real_images is not None else None 268 | 269 | # grab models 270 | clip_model = None 271 | 272 | if embed_text: 273 | clip_model = load_clip(clip_model=clip_model, device=DEVICE) 274 | 275 | prior_model = None 276 | 277 | if predict_image: 278 | prior_model = load_prior(checkpoint_path=prior_checkpoint, config_path=prior_config, device=DEVICE) 279 | 280 | # begin creating webdataset 281 | wds_create( 282 | prompts=prompts, 283 | images=images, 284 | batch_size=batch_size, 285 | output_folder=output_folder, 286 | filename=filename, 287 | clip_model=clip_model, 288 | prior_model=prior_model, 289 | predict_image=predict_image, 290 | embed_text=embed_text, 291 | tokenize_text=tokenize_text, 292 | ) 293 | 294 | click.secho(f"All done!", fg="green") 295 | 296 | 297 | if __name__ == "__main__": 298 | main() 299 | -------------------------------------------------------------------------------- /prompts/dalle_mega.json: -------------------------------------------------------------------------------- 1 | [ 2 | "t-shirt, size M", 3 | "flower dress, size M", 4 | "a t-shirt of an avocado", 5 | "a rainbow hat", 6 | "white snow covered mountain under blue sky during daytime", 7 | "aerial view of the beach during daytime", 8 | "aerial view of the beach at night", 9 | "double rainbow over a lake", 10 | "a beautiful sunset at a beach with a shell on the shore", 11 | "a farmhouse surrounded by flowers", 12 | "a farmhouse on the top of a cliff", 13 | "sunset over the lake in the mountains", 14 | "sunset over green mountains", 15 | "sunset over purple mountains", 16 | "a photo of the golden gate bridge", 17 | "a volcano erupting next to the golden gate bridge", 18 | "painting of an oniric forest glade surrounded by tall trees", 19 | "painting of a forest full of elves and fairies", 20 | "a graphite sketch of a gothic cathedral", 21 | "a graphite sketch of a market in the Middle Ages", 22 | "a graphite sketch of Elon Musk", 23 | "still life in the style of Kandinsky", 24 | "still life in the style of Picasso", 25 | "drawing of a family in front of a house made by a little child", 26 | "painting of an alien by Claude Monet", 27 | "a colorful stairway to heaven", 28 | "a background consisting of colors blue, green, and red", 29 | "Mohammed Ali and Mike Tyson in a match", 30 | "Pele and Maradona in a match", 31 | "view of Earth from space", 32 | "view of Mars from space", 33 | "view of Saturn from space", 34 | "the moon is a skull", 35 | "the land of dinosaurs", 36 | "a dinosaur in space", 37 | "a facebook-branded dinosaur", 38 | "a google-branded dinosaur", 39 | "illustration of an astronaut in a space suit playing guitar", 40 | "a clown wearing a spacesuit floating in space", 41 | "the Eiffel tower on the ocean", 42 | "the Eiffel tower on a mountain", 43 | "the Eiffel tower on the beach", 44 | "the Eiffel tower floating on the clouds", 45 | "the Eiffel tower underwater", 46 | "the Eiffel tower made of French fries", 47 | "the Eiffel tower on the moon", 48 | "the Eiffel tower on the moon, Earth is in the background", 49 | "watercolor of the Eiffel tower on the moon", 50 | "a volcano erupting next to the Eiffel tower", 51 | "a rocket in the shape of the Eiffel tower taking off", 52 | "epic sword fight", 53 | "underwater cathedral", 54 | "times square underwater, times square at the bottom of the ocean", 55 | "a fantasy version of New York City", 56 | "fantasy kingdoms", 57 | "Paris in a far future, futuristic Paris", 58 | "the Statue of Liberty", 59 | "a drawing of the Statue of Liberty", 60 | "a cool robot", 61 | "an angry robot with a gun", 62 | "robots taking control over humans", 63 | "a portrait of a nightmare creature", 64 | "a very cute cat", 65 | "a very cute dog", 66 | "a cute kitten on a white background", 67 | "a cute puppy on a white background", 68 | "a cat playing with a ball", 69 | "a dog playing with a ball", 70 | "a cat is holding a gun", 71 | "a dog is holding a gun", 72 | "an extreme close-up view of a capybara sitting in a field", 73 | "a cat sits on top of an alligator", 74 | "a very cute cat laying by a big bike", 75 | "rat wearing a crown", 76 | "a rat holding a red lightsaber in a white background", 77 | "a very cute giraffe making a funny face", 78 | "A unicorn is passing by a rainbow in a field of flowers", 79 | "an elephant made of carrots", 80 | "an elephant on a unicycle at a circus", 81 | "a pink elephant on a beach", 82 | "photography of a penguin watching television", 83 | "a penguin is walking on the Moon, Earth is in the background", 84 | "a penguin standing on a tower of books holds onto a rope from a helicopter", 85 | "a group of penguins having a wine and cheese party, they are drinking a lot of wine and eating a variety of cheese", 86 | "looking into the sky, 10 airplanes are seen overhead", 87 | "shelves filled with books and alchemy potion bottles", 88 | "the cosmic library contains forbidden knowledge", 89 | "this is a detailed high-resolution scan of a human brain", 90 | "a restaurant menu", 91 | "a bottle of coca-cola on a table", 92 | "a peanut", 93 | "a cross-section view of a walnut", 94 | "a living room with two white armchairs and a painting of the collosseum. The painting is mounted above a modern fireplace", 95 | "a collection of glasses is sitting on a table", 96 | "a painting of a capybara sitting on a mountain during fall in surrealist style", 97 | "painting of a pink elephant", 98 | "a pentagonal green clock", 99 | "a storefront that has the word 'openai' written on it", 100 | "a tatoo of a black broccoli", 101 | "a blue table", 102 | "a variety of clocks is sitting on a table", 103 | "a table has a train model on it with other cars and things", 104 | "a small red block sitting on a large green block", 105 | "a red cube on top of a blue cube", 106 | "a long line of alternating green and red blocks", 107 | "a long line of green blocks on a beach at subset", 108 | "a long line of peaches on a beach at sunset", 109 | "an apple looking somberly out of a window on a rainy day", 110 | "urinals are lined up in a jungle", 111 | "a picture of a castle from minecraft", 112 | "a cartoon of a superhero bear", 113 | "a cartoon of a superhero avocado", 114 | "Cartoon of a carrot with big eyes", 115 | "an illustration of a cute skeleton wearing a blue hoodie", 116 | "illustration of a baby shark swimming around corals", 117 | "logo of a robot wearing glasses and reading a book", 118 | "an illustration of Pikachu", 119 | "a cute pikachu teapot", 120 | "an illustration of pikachu sitting on a bench eating an ice cream", 121 | "Mario is fighting Sonic", 122 | "An anime hero punches a giant squirrel", 123 | "star wars concept art", 124 | "illustration of a cactus lifting weigths", 125 | "logo of a cactus lifting weights", 126 | "a photo of a camera from the future", 127 | "a skeleton with the shape of a spider", 128 | "a pixel art illustration of an eagle sitting in a field in the afternoon", 129 | "an emoji of a baby fox wearing a blue hat, green gloves, red shirt, and yellow pants", 130 | "an emoji of a baby penguin wearing a blue hat, blue gloves, red shirt, and green pants", 131 | "an illustration of a baby cucumber with a mustache playing chess", 132 | "an illustration of a baby daikon radish in a tutu walking a dog", 133 | "an illustration of a baby hedgehog in a cape staring at its reflection in a mirror", 134 | "an illustration of a baby panda with headphones holding an umbrella in the rain", 135 | "illustration of a baby hedgehog in a christmas sweater walking a dog", 136 | "a hedgehog using a calculator", 137 | "cartoon of an alligator wearing a pink hat", 138 | "a human face", 139 | "a girl with blonde hair", 140 | "a man with an apple instead of a head", 141 | "a girl with a neural link, a girl with a sci-fi cybernetic device on her head", 142 | "Illustration of a rabbit detective sitting on a park bench and reading a newspaper in a victorian setting", 143 | "The Simpsons dressed up as Minions", 144 | "Stonehenge made out of marshmallows", 145 | "a person is walking in a park", 146 | "a large crowd at a music festival", 147 | "people playing with balloons at a festival", 148 | "a person is holding a phone and a waterbottle, running a marathon", 149 | "a child eating a birthday cake near some balloons", 150 | "a tree growing babies", 151 | "A person sits and a dog stands", 152 | "A dog sits and a person stands", 153 | "Young woman riding her bike through the forest", 154 | "the best soccer team of the world", 155 | "the best football team of the world", 156 | "the best basketball team of the world", 157 | "happy, happiness", 158 | "sad, sadness", 159 | "the representation of infinity", 160 | "the end of the world", 161 | "an armchair in the shape of an avocado", 162 | "illustration of an avocado armchair", 163 | "logo of an avocado armchair", 164 | "an avocado armchair flying into space", 165 | "a cute avocado armchair singing karaoke on stage in front of a crowd of strawberry shaped lamps", 166 | "an illustration of an avocado in a beanie riding a motorcycle", 167 | "an illustration of an avocado in a christmas sweater staring at its reflection in a mirror", 168 | "illustration of an avocado armchair getting married to a pineapple", 169 | "a t-shirt of an avocado and a llama", 170 | "a logo of an avocado and a llama", 171 | "an illustration of an avocado and a llama", 172 | "an astronaut riding a horse in a photorealistic style", 173 | "an astronaut riding a horse on the moon", 174 | "Spiderman riding a horse", 175 | "Batman eating a hamburger", 176 | "Teddy bears mixing sparkling chemicals as mad scintists in a steampunk style", 177 | "half human half cat, a human cat hybrid", 178 | "half human half dog, a human dog hybrid", 179 | "half human half avocado, a human avocado hybrid", 180 | "half human half Eiffel tower, a human Eiffel tower hybrid", 181 | "a propaganda poster for transhumanism", 182 | "a propaganda poster for building a space elevator", 183 | "a beautiful epic fantasy painting of a space elevator", 184 | "distributed shampoo in the spotlight", 185 | "a diagram of distributed shampoo", 186 | "Kerala", 187 | "Mohanlal graphite sketch", 188 | "Relaxing on a house boat in Kerala", 189 | "Onam sadhya", 190 | "Western ghats of India", 191 | "A train ride on the malabar coast", 192 | "A train ride in the monsoon rain", 193 | "dining in the shacks on the beaches of goa", 194 | "huggingface avocado emoji", 195 | "avocado huggingface emoji" 196 | ] -------------------------------------------------------------------------------- /dream_bench/evaluator.py: -------------------------------------------------------------------------------- 1 | from typing import List, Literal 2 | 3 | import numpy as np 4 | import pandas as pd 5 | import torch 6 | import wandb 7 | from toma import toma 8 | from torch.nn.functional import cosine_similarity 9 | from torch.utils.data import DataLoader, TensorDataset 10 | from torchmetrics.image.fid import FrechetInceptionDistance 11 | from webdataset import WebDataset 12 | 13 | from dream_bench.helpers import exists 14 | from dream_bench.load_models import get_aesthetic_model, load_clip 15 | from dream_bench.tokenizer import tokenizer 16 | 17 | TOKENIZER_CONTEXT_LENGTH = 77 18 | TOKENIZER_TRUNCATE_TEXT = True 19 | 20 | # Custom Types 21 | 22 | METRICS = Literal["FID", "Aesthetic", "ClipScore"] 23 | 24 | """ 25 | TODO: 26 | Overhaul device placement. 27 | """ 28 | 29 | # helpers 30 | 31 | 32 | def make_loaders(batchsize: int, model_input: WebDataset, model_output: List[torch.Tensor]): 33 | """ 34 | Create Dataloaders from the two inputs with a given batchsize. 35 | 36 | Return: 37 | - A zipped version of both datasets to sample from. 38 | """ 39 | 40 | input_loader = DataLoader(dataset=model_input, batch_size=batchsize) 41 | 42 | concat_tensors = torch.as_tensor(model_output) 43 | tensor_dataset = TensorDataset(concat_tensors) 44 | 45 | output_loader = DataLoader(dataset=tensor_dataset, batch_size=batchsize) 46 | 47 | return zip(input_loader, output_loader) 48 | 49 | 50 | class FID: 51 | """ 52 | The FID metric computes the Frechet Inception Distance of predicted images given a distribution of real images. 53 | """ 54 | 55 | METRIC_NAME = "FID" 56 | USES_INPUT = True 57 | IS_SUMMARY = True 58 | 59 | def __init__(self, feature=2048) -> None: 60 | self.fid = FrechetInceptionDistance(feature=feature) 61 | 62 | def format_tensor(self, x): 63 | """Format the tensor to be FID friendly (uint8).""" 64 | 65 | if x.type == torch.uint8: 66 | return x 67 | 68 | # from https://github.com/lucidrains/DALLE2-pytorch/blob/0d82dff9c53a7a74cbd66cf8930866308bd9ff49/train_decoder.py#L202 69 | if x.max().item() <= 1.0: 70 | x = x.float() 71 | x = x.mul(255).add(0.5).clamp(0, 255) 72 | 73 | return x.to(torch.uint8) 74 | 75 | def compute( 76 | self, 77 | dataset: WebDataset, 78 | predictions: torch.Tensor, 79 | device: str, 80 | ): 81 | """Compute the FID of a distribution of real and generated images""" 82 | 83 | @toma.execute.batch(initial_batchsize=512) 84 | def _block(batchsize): 85 | # place model on device 86 | self.fid.to(device) 87 | 88 | # create dataloader 89 | loader = make_loaders(batchsize=batchsize, model_input=dataset, model_output=predictions) 90 | 91 | for model_input, model_output in loader: 92 | real_images = self.format_tensor(model_input["real_image.npy"]) 93 | model_output = self.format_tensor(model_output[0]) 94 | 95 | real_images = real_images.to(device) 96 | model_output = model_output.to(device) 97 | 98 | self.fid.update(imgs=real_images, real=True) 99 | self.fid.update(imgs=model_output, real=False) 100 | 101 | results = self.fid.compute().detach().cpu().numpy().squeeze() 102 | 103 | # move model back to cpu 104 | self.fid = self.fid.to("cpu") 105 | 106 | return results 107 | 108 | return _block 109 | 110 | 111 | class Aesthetic: 112 | """ 113 | Predict the aesthetic quality of images using a pre-trained aesthetic rating model. 114 | """ 115 | 116 | METRIC_NAME = "Aesthetic" 117 | USES_INPUT = False 118 | IS_SUMMARY = False 119 | 120 | def __init__(self, clip_model, clip_preprocess, clip_architecture) -> None: 121 | # get models 122 | self.aesthetic_model = get_aesthetic_model(clip_model=clip_architecture) 123 | self.clip_model, self.preprocess = clip_model, clip_preprocess 124 | 125 | def _embed(self, images: torch.Tensor): 126 | "Embed an image with clip and return the encoded images." 127 | device = images.device 128 | processed_images = self.preprocess(images).to(device) 129 | 130 | return self.clip_model.encode_image(processed_images) 131 | 132 | def compute(self, predictions: torch.Tensor, device: str): 133 | """Compute the Aesthetic quality of a collection of images.""" 134 | 135 | @toma.execute.batch(initial_batchsize=512) 136 | def _block(batchsize): 137 | # place models on proper device 138 | self.clip_model.to(device) 139 | self.aesthetic_model.to(device) 140 | 141 | all_aesthetic_scores = [] 142 | 143 | for idx in range(0, len(predictions), batchsize): 144 | sample = predictions[idx : idx + batchsize] 145 | sample = sample.to(device) 146 | 147 | embeddings = self._embed(sample).float() 148 | 149 | all_aesthetic_scores += [*self.aesthetic_model(embeddings).detach().cpu().numpy()] 150 | 151 | # move models back to cpu 152 | self.clip_model.to("cpu") 153 | self.aesthetic_model.to("cpu") 154 | 155 | return np.array(all_aesthetic_scores).squeeze() 156 | 157 | return _block 158 | 159 | 160 | class ClipScore: 161 | """ 162 | Compute the clip-similarity of an image and caption using a pre-trained CLIP model 163 | """ 164 | 165 | METRIC_NAME = "ClipScore" 166 | USES_INPUT = True 167 | IS_SUMMARY = False 168 | 169 | def __init__(self, clip_model, clip_preprocess) -> None: 170 | self.clip_model, self.preprocess = clip_model, clip_preprocess 171 | 172 | def compute( 173 | self, 174 | dataset: WebDataset, 175 | predictions: torch.Tensor, 176 | device: str, 177 | ): 178 | "Compute the clip score of a given image/caption pair." 179 | 180 | @toma.execute.batch(initial_batchsize=512) 181 | def _block(batchsize): 182 | # place models on proper device 183 | self.clip_model.to(device) 184 | 185 | loader = make_loaders(batchsize=batchsize, model_input=dataset, model_output=predictions) 186 | 187 | all_similarities = [] 188 | 189 | for model_input, model_output in loader: 190 | if "tokenized_text.npy" not in model_input: 191 | captions = model_input["caption.txt"] 192 | model_input["tokenized_text.npy"] = tokenizer.tokenize( 193 | captions, 194 | context_length=TOKENIZER_CONTEXT_LENGTH, 195 | truncate_text=TOKENIZER_TRUNCATE_TEXT, 196 | ) 197 | 198 | model_output = model_output[0].to(device) 199 | tokenized_text = model_input["tokenized_text.npy"].to(device) 200 | 201 | images = self.preprocess(model_output).to(device) 202 | 203 | image_embeddings = self.clip_model.encode_image(images) 204 | text_embeddings = self.clip_model.encode_text(tokenized_text) 205 | 206 | cos_similarities = cosine_similarity(image_embeddings, text_embeddings, dim=1) 207 | 208 | all_similarities += [*cos_similarities.detach().cpu().numpy()] 209 | 210 | # move model back to cpu 211 | self.clip_model.to("cpu") 212 | 213 | return np.array(all_similarities).squeeze() 214 | 215 | return _block 216 | 217 | 218 | class Evaluator: 219 | """ 220 | A class that facilliatates calculating various metrics given input from a model 221 | """ 222 | 223 | def __init__( 224 | self, 225 | metrics: List[METRICS], 226 | dataset: WebDataset, 227 | device="cpu", 228 | clip_architecture=None, 229 | default_batch_size=128, 230 | ) -> None: 231 | self.device: torch.device = device 232 | self.dataset: WebDataset = dataset 233 | self.default_batch_size: int = default_batch_size 234 | self.metric_names: List[str] = list(metrics) 235 | self.metrics: set = self._metric_factory(metrics, clip_architecture=clip_architecture, device=device) 236 | 237 | self.predictions: List[torch.Tensor] = [] 238 | 239 | def record_predictions(self, model_output: torch.Tensor): 240 | """ 241 | Save the model output along with the prompt. 242 | """ 243 | self.predictions += [model_output.detach().cpu()] 244 | 245 | def evaluate(self): 246 | """ 247 | Evaluate the model's output once all the data has been collected. 248 | """ 249 | tensor_predictions = torch.concat(self.predictions, dim=0) 250 | 251 | # keep track of results and summaries 252 | results = {} 253 | summaries = {} 254 | 255 | # compute the requested metrics 256 | for metric in self.metrics: 257 | kwargs = {"predictions": tensor_predictions, "device": self.device} 258 | 259 | if metric.USES_INPUT: 260 | kwargs.update({"dataset": self.dataset}) 261 | 262 | result = metric.compute(**kwargs) 263 | 264 | if metric.IS_SUMMARY: 265 | summaries[metric.METRIC_NAME] = result 266 | else: 267 | results[metric.METRIC_NAME] = result 268 | summaries[metric.METRIC_NAME] = result.mean() 269 | 270 | self._log(results, summaries) 271 | 272 | def _to_wandb_image(self, images): 273 | """ 274 | Convert a list of images into a list of wandb images. 275 | """ 276 | 277 | wandb_images = [] 278 | 279 | for image in images: 280 | wandb_images.append(wandb.Image(data_or_path=image.permute(1, 2, 0).detach().cpu().numpy())) 281 | 282 | return wandb_images 283 | 284 | def _log(self, results, summaries): 285 | """ 286 | Log all the results to wandb: 287 | - Summaries get logged as charts. 288 | - Results is converted to a `wandb.Table` through `pd.DataFrame`. 289 | """ 290 | results_df = pd.DataFrame.from_dict(data=results) 291 | results_table = wandb.Table(dataframe=results_df) 292 | 293 | results_table.add_column(name="Caption", data=self._get_captions()) 294 | results_table.add_column( 295 | name="Prediction", 296 | data=self._to_wandb_image(torch.concat(self.predictions, dim=0)), 297 | ) 298 | summaries.update({f"Evaluation Report: #{wandb.run.step}": results_table}) 299 | 300 | wandb.log(summaries) 301 | 302 | def _get_captions(self): 303 | """ 304 | Extract the captions as a list of strings from the dataset. 305 | """ 306 | 307 | captions = [] 308 | 309 | for item in self.dataset: 310 | captions.append(item["caption.txt"]) 311 | 312 | return captions 313 | 314 | def _metric_factory(self, metrics: List[METRICS], clip_architecture, device="cpu"): 315 | "Create an iterable of metric classes for the evaluator to use, given a list of metric names." 316 | 317 | metric_set: set = set() 318 | 319 | # a clip model to use across all metrics 320 | clip_model, clip_preprocess = None, None 321 | 322 | for metric in metrics: 323 | if metric == "FID": 324 | metric_set.add(FID()) 325 | 326 | elif metric == "Aesthetic": 327 | assert exists(clip_architecture), "A clip architecture is required to use the Aesthetic Metric." 328 | if not exists(clip_model): 329 | clip_model, clip_preprocess = load_clip(clip_model=clip_architecture, device=device) 330 | 331 | metric_set.add( 332 | Aesthetic( 333 | clip_model=clip_model, 334 | clip_preprocess=clip_preprocess, 335 | clip_architecture=clip_architecture, 336 | ) 337 | ) 338 | 339 | elif metric == "ClipScore": 340 | assert exists(clip_architecture), "A clip architecture is required to use the Clip Score." 341 | if not exists(clip_model): 342 | clip_model, clip_preprocess = load_clip(clip_model=clip_architecture, device=device) 343 | metric_set.add(ClipScore(clip_model=clip_model, clip_preprocess=clip_preprocess)) 344 | 345 | else: 346 | raise KeyError("That metric does not exist, or is mispelled.") 347 | 348 | return metric_set 349 | -------------------------------------------------------------------------------- /prompts/drawbench.json: -------------------------------------------------------------------------------- 1 | [ 2 | "A red colored car.", 3 | "A black colored car.", 4 | "A pink colored car.", 5 | "A black colored dog.", 6 | "A red colored dog.", 7 | "A blue colored dog.", 8 | "A green colored banana.", 9 | "A red colored banana.", 10 | "A black colored banana.", 11 | "A white colored sandwich.", 12 | "A black colored sandwich.", 13 | "An orange colored sandwich.", 14 | "A pink colored giraffe.", 15 | "A yellow colored giraffe.", 16 | "A brown colored giraffe.", 17 | "A red car and a white sheep.", 18 | "A blue bird and a brown bear.", 19 | "A green apple and a black backpack.", 20 | "A green cup and a blue cell phone.", 21 | "A yellow book and a red vase.", 22 | "A white car and a red sheep.", 23 | "A brown bird and a blue bear.", 24 | "A black apple and a green backpack.", 25 | "A blue cup and a green cell phone.", 26 | "A red book and a yellow vase.", 27 | "A horse riding an astronaut.", 28 | "A pizza cooking an oven.", 29 | "A bird scaring a scarecrow.", 30 | "A blue coloured pizza.", 31 | "Hovering cow abducting aliens.", 32 | "A panda making latte art.", 33 | "A shark in the desert.", 34 | "An elephant under the sea.", 35 | "Rainbow coloured penguin.", 36 | "A fish eating a pelican.", 37 | "One car on the street.", 38 | "Two cars on the street.", 39 | "Three cars on the street.", 40 | "Four cars on the street.", 41 | "Five cars on the street.", 42 | "One dog on the street.", 43 | "Two dogs on the street.", 44 | "Three dogs on the street.", 45 | "Four dogs on the street.", 46 | "Five dogs on the street.", 47 | "One cat and one dog sitting on the grass.", 48 | "One cat and two dogs sitting on the grass.", 49 | "One cat and three dogs sitting on the grass.", 50 | "Two cats and one dog sitting on the grass.", 51 | "Two cats and two dogs sitting on the grass.", 52 | "Two cats and three dogs sitting on the grass.", 53 | "Three cats and one dog sitting on the grass.", 54 | "Three cats and two dogs sitting on the grass.", 55 | "Three cats and three dogs sitting on the grass.", 56 | "A triangular purple flower pot. A purple flower pot in the shape of a triangle.", 57 | "A triangular orange picture frame. An orange picture frame in the shape of a triangle.", 58 | "A triangular pink stop sign. A pink stop sign in the shape of a triangle.", 59 | "A cube made of denim. A cube with the texture of denim.", 60 | "A sphere made of kitchen tile. A sphere with the texture of kitchen tile.", 61 | "A cube made of brick. A cube with the texture of brick.", 62 | "A collection of nail is sitting on a table.", 63 | "A single clock is sitting on a table.", 64 | "A couple of glasses are sitting on a table.", 65 | "An illustration of a large red elephant sitting on a small blue mouse.", 66 | "An illustration of a small green elephant standing behind a large red mouse.", 67 | "A small blue book sitting on a large red book.", 68 | "A stack of 3 plates. A blue plate is on the top, sitting on a blue plate. The blue plate is in the middle, sitting on a green plate. The green plate is on the bottom.", 69 | "A stack of 3 cubes. A red cube is on the top, sitting on a red cube. The red cube is in the middle, sitting on a green cube. The green cube is on the bottom.", 70 | "A stack of 3 books. A green book is on the top, sitting on a red book. The red book is in the middle, sitting on a blue book. The blue book is on the bottom.", 71 | "An emoji of a baby panda wearing a red hat, green gloves, red shirt, and green pants.", 72 | "An emoji of a baby panda wearing a red hat, blue gloves, green shirt, and blue pants.", 73 | "A fisheye lens view of a turtle sitting in a forest.", 74 | "A side view of an owl sitting in a field.", 75 | "A cross-section view of a brain.", 76 | "A vehicle composed of two wheels held in a frame one behind the other, propelled by pedals and steered with handlebars attached to the front wheel.", 77 | "A large motor vehicle carrying passengers by road, typically one serving the public on a fixed route and for a fare.", 78 | "A small vessel propelled on water by oars, sails, or an engine.", 79 | "A connection point by which firefighters can tap into a water supply.", 80 | "A machine next to a parking space in a street, into which the driver puts money so as to be authorized to park the vehicle for a particular length of time.", 81 | "A device consisting of a circular canopy of cloth on a folding metal frame supported by a central rod, used as protection against rain or sometimes sun.", 82 | "A separate seat for one person, typically with a back and four legs.", 83 | "An appliance or compartment which is artificially kept cool and used to store food and drink.", 84 | "A mechanical or electrical device for measuring time.", 85 | "An instrument used for cutting cloth, paper, and other thin material, consisting of two blades laid one on top of the other and fastened in the middle so as to allow them to be opened and closed by a thumb and finger inserted through rings on the end of their handles.", 86 | "A large plant-eating domesticated mammal with solid hoofs and a flowing mane and tail, used for riding, racing, and to carry and pull loads.", 87 | "A long curved fruit which grows in clusters and has soft pulpy flesh and yellow skin when ripe.", 88 | "A small domesticated carnivorous mammal with soft fur, a short snout, and retractable claws. It is widely kept as a pet or for catching mice, and many breeds have been developed.", 89 | "A domesticated carnivorous mammal that typically has a long snout, an acute sense of smell, nonretractable claws, and a barking, howling, or whining voice.", 90 | "An organ of soft nervous tissue contained in the skull of vertebrates, functioning as the coordinating center of sensation and intellectual and nervous activity.", 91 | "An American multinational technology company that focuses on artificial intelligence, search engine, online advertising, cloud computing, computer software, quantum computing, e-commerce, and consumer electronics.", 92 | "A large keyboard musical instrument with a wooden case enclosing a soundboard and metal strings, which are struck by hammers when the keys are depressed. The strings' vibration is stopped by dampers when the keys are released and can be regulated for length and volume by two or three pedals.", 93 | "A type of digital currency in which a record of transactions is maintained and new units of currency are generated by the computational solution of mathematical problems, and which operates independently of a central bank.", 94 | "A large thick-skinned semiaquatic African mammal, with massive jaws and large tusks.", 95 | "A machine resembling a human being and able to replicate certain human movements and functions automatically.", 96 | "Paying for a quarter-sized pizza with a pizza-sized quarter.", 97 | "An oil painting of a couple in formal evening wear going home get caught in a heavy downpour with no umbrellas.", 98 | "A grocery store refrigerator has pint cartons of milk on the top shelf, quart cartons on the middle shelf, and gallon plastic jugs on the bottom shelf.", 99 | "In late afternoon in January in New England, a man stands in the shadow of a maple tree.", 100 | "An elephant is behind a tree. You can see the trunk on one side and the back legs on the other.", 101 | "A tomato has been put on top of a pumpkin on a kitchen stool. There is a fork sticking into the pumpkin. The scene is viewed from above.", 102 | "A pear cut into seven pieces arranged in a ring.", 103 | "A donkey and an octopus are playing a game. The donkey is holding a rope on one end, the octopus is holding onto the other. The donkey holds the rope in its mouth. A cat is jumping over the rope.", 104 | "Supreme Court Justices play a baseball game with the FBI. The FBI is at bat, the justices are on the field.", 105 | "Abraham Lincoln touches his toes while George Washington does chin-ups. Lincoln is barefoot. Washington is wearing boots.", 106 | "Tcennis rpacket.", 107 | "Bzaseball galove.", 108 | "Rbefraigerator.", 109 | "Dininrg tablez.", 110 | "Pafrking metr.", 111 | "A smafml vessef epropoeilled on watvewr by ors, sauls, or han engie.", 112 | "A sjmall domesticated carnivorious mammnal with sof fuh,y a sthort sout, and retracwtablbe flaws. It iw widexly kept as a pet or for catchitng mic, ad many breeds zhlyde beefn develvoked.", 113 | "An instqrumemnt used for cutting cloth, paper, axdz othr thdin mteroial, consamistng of two blades lad one on tvopb of the other and fhastned in tle mixdqdjle so as to bllow them txo be pened and closed by thumb and fitngesr inserted tgrough rings on kthe end oc thei vatndlzes.", 114 | "A domesticated carnivvorous mzammal that typicbally hfaas a lons sfnout, an acxujte sense off osmell, noneetractaaln crlaws, anid xbarkring,y howlingu, or whining rvoiche.", 115 | "A ldarge keybord msical instroument lwith a woden case enmclosig a qsouvnkboajrd and mfgtal strivgf, which are strucrk b hammrs when the nels are depresdsmed.f lhe strsingsj' vibration ie stopped by damperds when the keys re released and can bce regulavewdd for lengh and vnolume y two or three pedalvs.", 116 | "A train on top of a surfboard.", 117 | "A wine glass on top of a dog.", 118 | "A bicycle on top of a boat.", 119 | "An umbrella on top of a spoon.", 120 | "A laptop on top of a teddy bear.", 121 | "A giraffe underneath a microwave.", 122 | "A donut underneath a toilet.", 123 | "A hair drier underneath a sheep.", 124 | "A tennis racket underneath a traffic light.", 125 | "A zebra underneath a broccoli.", 126 | "A banana on the left of an apple.", 127 | "A couch on the left of a chair.", 128 | "A car on the left of a bus.", 129 | "A cat on the left of a dog.", 130 | "A carrot on the left of a broccoli.", 131 | "A pizza on the right of a suitcase.", 132 | "A cat on the right of a tennis racket.", 133 | "A stop sign on the right of a refrigerator.", 134 | "A sheep to the right of a wine glass.", 135 | "A zebra to the right of a fire hydrant.", 136 | "Acersecomicke.", 137 | "Jentacular.", 138 | "Matutinal.", 139 | "Peristeronic.", 140 | "Artophagous.", 141 | "Backlotter.", 142 | "Octothorpe.", 143 | "A church with stained glass windows depicting a hamburger and french fries.", 144 | "Painting of the orange cat Otto von Garfield, Count of Bismarck-Sch\u00f6nhausen, Duke of Lauenburg, Minister-President of Prussia. Depicted wearing a Prussian Pickelhaube and eating his favorite meal - lasagna.", 145 | "A baby fennec sneezing onto a strawberry, detailed, macro, studio light, droplets, backlit ears.", 146 | "A photo of a confused grizzly bear in calculus class.", 147 | "An ancient Egyptian painting depicting an argument over whose turn it is to take out the trash.", 148 | "A fluffy baby sloth with a knitted hat trying to figure out a laptop, close up, highly detailed, studio lighting, screen reflecting in its eyes.", 149 | "A tiger in a lab coat with a 1980s Miami vibe, turning a well oiled science content machine, digital art.", 150 | "A 1960s yearbook photo with animals dressed as humans.", 151 | "Lego Arnold Schwarzenegger.", 152 | "A yellow and black bus cruising through the rainforest.", 153 | "A medieval painting of the wifi not working.", 154 | "An IT-guy trying to fix hardware of a PC tower is being tangled by the PC cables like Laokoon. Marble, copy after Hellenistic original from ca. 200 BC. Found in the Baths of Trajan, 1506.", 155 | "35mm macro shot a kitten licking a baby duck, studio lighting.", 156 | "McDonalds Church.", 157 | "Photo of an athlete cat explaining it's latest scandal at a press conference to journalists.", 158 | "Greek statue of a man tripping over a cat.", 159 | "An old photograph of a 1920s airship shaped like a pig, floating over a wheat field.", 160 | "Photo of a cat singing in a barbershop quartet.", 161 | "A painting by Grant Wood of an astronaut couple, american gothic style.", 162 | "An oil painting portrait of the regal Burger King posing with a Whopper.", 163 | "A keyboard made of water, the water is made of light, the light is turned off.", 164 | "Painting of Mona Lisa but the view is from behind of Mona Lisa.", 165 | "Hyper-realistic photo of an abandoned industrial site during a storm.", 166 | "A screenshot of an iOS app for ordering different types of milk.", 167 | "A real life photography of super mario, 8k Ultra HD.", 168 | "Colouring page of large cats climbing the eifel tower in a cyberpunk future.", 169 | "Photo of a mega Lego space station inside a kid's bedroom.", 170 | "A spider with a moustache bidding an equally gentlemanly grasshopper a good day during his walk to work.", 171 | "A photocopy of a photograph of a painting of a sculpture of a giraffe.", 172 | "A bridge connecting Europe and North America on the Atlantic Ocean, bird's eye view.", 173 | "A maglev train going vertically downward in high speed, New York Times photojournalism.", 174 | "A magnifying glass over a page of a 1950s batman comic.", 175 | "A car playing soccer, digital art.", 176 | "Darth Vader playing with raccoon in Mars during sunset.", 177 | "A 1960s poster warning against climate change.", 178 | "Illustration of a mouse using a mushroom as an umbrella.", 179 | "A realistic photo of a Pomeranian dressed up like a 1980s professional wrestler with neon green and neon orange face paint and bright green wrestling tights with bright orange boots.", 180 | "A pyramid made of falafel with a partial solar eclipse in the background.", 181 | "A storefront with 'Hello World' written on it.", 182 | "A storefront with 'Diffusion' written on it.", 183 | "A storefront with 'Text to Image' written on it.", 184 | "A storefront with 'NeurIPS' written on it.", 185 | "A storefront with 'Deep Learning' written on it.", 186 | "A storefront with 'Google Brain Toronto' written on it.", 187 | "A storefront with 'Google Research Pizza Cafe' written on it.", 188 | "A sign that says 'Hello World'.", 189 | "A sign that says 'Diffusion'.", 190 | "A sign that says 'Text to Image'.", 191 | "A sign that says 'NeurIPS'.", 192 | "A sign that says 'Deep Learning'.", 193 | "A sign that says 'Google Brain Toronto'.", 194 | "A sign that says 'Google Research Pizza Cafe'.", 195 | "New York Skyline with 'Hello World' written with fireworks on the sky.", 196 | "New York Skyline with 'Diffusion' written with fireworks on the sky.", 197 | "New York Skyline with 'Text to Image' written with fireworks on the sky.", 198 | "New York Skyline with 'NeurIPS' written with fireworks on the sky.", 199 | "New York Skyline with 'Deep Learning' written with fireworks on the sky.", 200 | "New York Skyline with 'Google Brain Toronto' written with fireworks on the sky.", 201 | "New York Skyline with 'Google Research Pizza Cafe' written with fireworks on the sky." 202 | ] -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.mypy] 2 | python_version = "3.10" 3 | ignore_missing_imports = true 4 | 5 | [tool.pytest.ini_options] 6 | testpaths = ["tests"] 7 | filterwarnings = [ 8 | "ignore::DeprecationWarning:wandb.*", 9 | "ignore::DeprecationWarning:torchvision.*" 10 | ] 11 | 12 | [tool.pylint.main] 13 | # Analyse import fallback blocks. This can be used to support both Python 2 and 3 14 | # compatible code, which means that the block might have code that exists only in 15 | # one or another interpreter, leading to false positives when analysed. 16 | # analyse-fallback-blocks = 17 | 18 | # Always return a 0 (non-error) status code, even if lint errors are found. This 19 | # is primarily useful in continuous integration scripts. 20 | # exit-zero = 21 | 22 | # A comma-separated list of package or module names from where C extensions may 23 | # be loaded. Extensions are loading into the active Python interpreter and may 24 | # run arbitrary code. 25 | # extension-pkg-allow-list = 26 | 27 | # A comma-separated list of package or module names from where C extensions may 28 | # be loaded. Extensions are loading into the active Python interpreter and may 29 | # run arbitrary code. (This is an alternative name to extension-pkg-allow-list 30 | # for backward compatibility.) 31 | extension-pkg-whitelist = ["_jsonnet"] 32 | 33 | # Return non-zero exit code if any of these messages/categories are detected, 34 | # even if score is above --fail-under value. Syntax same as enable. Messages 35 | # specified are enabled, while categories only check already-enabled messages. 36 | # fail-on = 37 | 38 | # Specify a score threshold to be exceeded before program exits with error. 39 | fail-under = 10 40 | 41 | # Interpret the stdin as a python script, whose filename needs to be passed as 42 | # the module_or_package argument. 43 | # from-stdin = 44 | 45 | # Files or directories to be skipped. They should be base names, not paths. 46 | ignore = ["CVS"] 47 | 48 | # Add files or directories matching the regex patterns to the ignore-list. The 49 | # regex matches against paths and can be in Posix or Windows format. 50 | # ignore-paths = 51 | 52 | # Files or directories matching the regex patterns are skipped. The regex matches 53 | # against base names, not paths. The default value ignores Emacs file locks 54 | ignore-patterns = ["^\\.#"] 55 | 56 | # List of module names for which member attributes should not be checked (useful 57 | # for modules/projects where namespaces are manipulated during runtime and thus 58 | # existing member attributes cannot be deduced by static analysis). It supports 59 | # qualified module names, as well as Unix pattern matching. 60 | # ignored-modules = 61 | 62 | # Python code to execute, usually for sys.path manipulation such as 63 | # pygtk.require(). 64 | # init-hook = 65 | 66 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the 67 | # number of processors available to use, and will cap the count on Windows to 68 | # avoid hangs. 69 | jobs = 1 70 | 71 | # Control the amount of potential inferred values when inferring a single object. 72 | # This can help the performance when dealing with large functions or complex, 73 | # nested conditions. 74 | limit-inference-results = 100 75 | 76 | # List of plugins (as comma separated values of python module names) to load, 77 | # usually to register additional checkers. 78 | # load-plugins = 79 | 80 | # Pickle collected data for later comparisons. 81 | persistent = true 82 | 83 | # Minimum Python version to use for version dependent checks. Will default to the 84 | # version used to run pylint. 85 | py-version = "3.10" 86 | 87 | # Discover python modules and packages in the file system subtree. 88 | # recursive = 89 | 90 | # When enabled, pylint would attempt to guess common misconfiguration and emit 91 | # user-friendly hints instead of false-positive error messages. 92 | suggestion-mode = true 93 | 94 | # Allow loading of arbitrary C extensions. Extensions are imported into the 95 | # active Python interpreter and may run arbitrary code. 96 | # unsafe-load-any-extension = 97 | 98 | [tool.pylint.basic] 99 | # Naming style matching correct argument names. 100 | argument-naming-style = "snake_case" 101 | 102 | # Regular expression matching correct argument names. Overrides argument-naming- 103 | # style. If left empty, argument names will be checked with the set naming style. 104 | argument-rgx = "^[a-z][a-z0-9_]*$" 105 | 106 | # Naming style matching correct attribute names. 107 | attr-naming-style = "snake_case" 108 | 109 | # Regular expression matching correct attribute names. Overrides attr-naming- 110 | # style. If left empty, attribute names will be checked with the set naming 111 | # style. 112 | attr-rgx = "^_{0,2}[a-z][a-z0-9_]*$" 113 | 114 | # Bad variable names which should always be refused, separated by a comma. 115 | # bad-names = 116 | 117 | # Bad variable names regexes, separated by a comma. If names match any regex, 118 | # they will always be refused 119 | # bad-names-rgxs = 120 | 121 | # Naming style matching correct class attribute names. 122 | class-attribute-naming-style = "any" 123 | 124 | # Regular expression matching correct class attribute names. Overrides class- 125 | # attribute-naming-style. If left empty, class attribute names will be checked 126 | # with the set naming style. 127 | class-attribute-rgx = "^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$" 128 | 129 | # Naming style matching correct class constant names. 130 | class-const-naming-style = "UPPER_CASE" 131 | 132 | # Regular expression matching correct class constant names. Overrides class- 133 | # const-naming-style. If left empty, class constant names will be checked with 134 | # the set naming style. 135 | # class-const-rgx = 136 | 137 | # Naming style matching correct class names. 138 | class-naming-style = "PascalCase" 139 | 140 | # Regular expression matching correct class names. Overrides class-naming-style. 141 | # If left empty, class names will be checked with the set naming style. 142 | class-rgx = "^_?[A-Z][a-zA-Z0-9]*$" 143 | 144 | # Naming style matching correct constant names. 145 | const-naming-style = "UPPER_CASE" 146 | 147 | # Regular expression matching correct constant names. Overrides const-naming- 148 | # style. If left empty, constant names will be checked with the set naming style. 149 | const-rgx = "^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$" 150 | 151 | # Minimum line length for functions/classes that require docstrings, shorter ones 152 | # are exempt. 153 | docstring-min-length = 10 154 | 155 | # Naming style matching correct function names. 156 | function-naming-style = "snake_case" 157 | 158 | # Regular expression matching correct function names. Overrides function-naming- 159 | # style. If left empty, function names will be checked with the set naming style. 160 | function-rgx = "^(?:(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$" 161 | 162 | # Good variable names which should always be accepted, separated by a comma. 163 | good-names = ["main", "_"] 164 | 165 | # Good variable names regexes, separated by a comma. If names match any regex, 166 | # they will always be accepted 167 | # good-names-rgxs = 168 | 169 | # Include a hint for the correct naming format with invalid-name. 170 | # include-naming-hint = 171 | 172 | # Naming style matching correct inline iteration names. 173 | inlinevar-naming-style = "any" 174 | 175 | # Regular expression matching correct inline iteration names. Overrides 176 | # inlinevar-naming-style. If left empty, inline iteration names will be checked 177 | # with the set naming style. 178 | inlinevar-rgx = "^[a-z][a-z0-9_]*$" 179 | 180 | # Naming style matching correct method names. 181 | method-naming-style = "snake_case" 182 | 183 | # Regular expression matching correct method names. Overrides method-naming- 184 | # style. If left empty, method names will be checked with the set naming style. 185 | method-rgx = "^(?:(?P__[a-z0-9_]+__|next)|(?P_{0,2}[A-Z][a-zA-Z0-9]*)|(?P_{0,2}[a-z][a-z0-9_]*))$" 186 | 187 | # Naming style matching correct module names. 188 | module-naming-style = "snake_case" 189 | 190 | # Regular expression matching correct module names. Overrides module-naming- 191 | # style. If left empty, module names will be checked with the set naming style. 192 | module-rgx = "(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$" 193 | 194 | # Colon-delimited sets of names that determine each other's naming style when the 195 | # name regexes allow several styles. 196 | # name-group = 197 | 198 | # Regular expression which should only match function or class names that do not 199 | # require a docstring. 200 | no-docstring-rgx = "(__.*__|main)" 201 | 202 | # List of decorators that produce properties, such as abc.abstractproperty. Add 203 | # to this list to register other decorators that produce valid properties. These 204 | # decorators are taken in consideration only for invalid-name. 205 | property-classes = ["abc.abstractproperty"] 206 | 207 | # Regular expression matching correct type variable names. If left empty, type 208 | # variable names will be checked with the set naming style. 209 | # typevar-rgx = 210 | 211 | # Naming style matching correct variable names. 212 | variable-naming-style = "snake_case" 213 | 214 | # Regular expression matching correct variable names. Overrides variable-naming- 215 | # style. If left empty, variable names will be checked with the set naming style. 216 | variable-rgx = "^[a-z][a-z0-9_]*$" 217 | 218 | [tool.pylint.classes] 219 | # Warn about protected attribute access inside special methods 220 | # check-protected-access-in-special-methods = 221 | 222 | # List of method names used to declare (i.e. assign) instance attributes. 223 | defining-attr-methods = ["__init__", "__new__", "setUp"] 224 | 225 | # List of member names, which should be excluded from the protected access 226 | # warning. 227 | exclude-protected = ["_asdict", "_fields", "_replace", "_source", "_make"] 228 | 229 | # List of valid names for the first argument in a class method. 230 | valid-classmethod-first-arg = ["cls", "class_"] 231 | 232 | # List of valid names for the first argument in a metaclass class method. 233 | valid-metaclass-classmethod-first-arg = ["mcs"] 234 | 235 | [tool.pylint.design] 236 | # List of regular expressions of class ancestor names to ignore when counting 237 | # public methods (see R0903) 238 | # exclude-too-few-public-methods = 239 | 240 | # List of qualified class names to ignore when counting class parents (see R0901) 241 | # ignored-parents = 242 | 243 | # Maximum number of arguments for function / method. 244 | max-args = 5 245 | 246 | # Maximum number of attributes for a class (see R0902). 247 | max-attributes = 7 248 | 249 | # Maximum number of boolean expressions in an if statement (see R0916). 250 | max-bool-expr = 5 251 | 252 | # Maximum number of branch for function / method body. 253 | max-branches = 12 254 | 255 | # Maximum number of locals for function / method body. 256 | max-locals = 15 257 | 258 | # Maximum number of parents for a class (see R0901). 259 | max-parents = 7 260 | 261 | # Maximum number of public methods for a class (see R0904). 262 | max-public-methods = 20 263 | 264 | # Maximum number of return / yield for function / method body. 265 | max-returns = 6 266 | 267 | # Maximum number of statements in function / method body. 268 | max-statements = 50 269 | 270 | # Minimum number of public methods for a class (see R0903). 271 | min-public-methods = 2 272 | 273 | [tool.pylint.exceptions] 274 | # Exceptions that will emit a warning when caught. 275 | overgeneral-exceptions = ["Exception", "StandardError", "BaseException"] 276 | 277 | [tool.pylint.format] 278 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 279 | # expected-line-ending-format = 280 | 281 | # Regexp for a line that is allowed to be longer than the limit. 282 | ignore-long-lines = "(?x)\n(^\\s*(import|from)\\s\n|\\$Id:\\s\\/\\/depot\\/.+#\\d+\\s\\$\n|^[a-zA-Z_][a-zA-Z0-9_]*\\s*=\\s*(\"[^\"]\\S+\"|'[^']\\S+')\n|^\\s*\\#\\ LINT\\.ThenChange\n|^[^#]*\\#\\ type:\\ [a-zA-Z_][a-zA-Z0-9_.,[\\] ]*$\n|pylint\n|\"\"\"\n|\\#\n|lambda\n|(https?|ftp):)" 283 | 284 | # Number of spaces of indent required inside a hanging or continued line. 285 | indent-after-paren = 4 286 | 287 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 288 | # tab). 289 | indent-string = " " 290 | 291 | # Maximum number of characters on a single line. 292 | max-line-length = 120 293 | 294 | # Maximum number of lines in a module. 295 | max-module-lines = 99999 296 | 297 | # Allow the body of a class to be on the same line as the declaration if body 298 | # contains single statement. 299 | # single-line-class-stmt = 300 | 301 | # Allow the body of an if to be on the same line as the test if there is no else. 302 | single-line-if-stmt = true 303 | 304 | [tool.pylint.imports] 305 | # List of modules that can be imported at any level, not just the top level one. 306 | # allow-any-import-level = 307 | 308 | # Allow wildcard imports from modules that define __all__. 309 | # allow-wildcard-with-all = 310 | 311 | # Deprecated modules which should not be used, separated by a comma. 312 | deprecated-modules = ["regsub", "TERMIOS", "Bastion", "rexec", "sets"] 313 | 314 | # Output a graph (.gv or any supported image format) of external dependencies to 315 | # the given file (report RP0402 must not be disabled). 316 | # ext-import-graph = 317 | 318 | # Output a graph (.gv or any supported image format) of all (i.e. internal and 319 | # external) dependencies to the given file (report RP0402 must not be disabled). 320 | # import-graph = 321 | 322 | # Output a graph (.gv or any supported image format) of internal dependencies to 323 | # the given file (report RP0402 must not be disabled). 324 | # int-import-graph = 325 | 326 | # Force import order to recognize a module as part of the standard compatibility 327 | # libraries. 328 | # known-standard-library = 329 | 330 | # Force import order to recognize a module as part of a third party library. 331 | known-third-party = ["enchant"] 332 | 333 | # Couples of modules and preferred modules, separated by a comma. 334 | # preferred-modules = 335 | 336 | [tool.pylint.logging] 337 | # The type of string formatting that logging methods do. `old` means using % 338 | # formatting, `new` is for `{}` formatting. 339 | logging-format-style = "old" 340 | 341 | # Logging modules to check that the string format arguments are in logging 342 | # function parameter format. 343 | logging-modules = ["logging"] 344 | 345 | [tool.pylint."messages control"] 346 | # Only show warnings with the listed confidence levels. Leave empty to show all. 347 | # Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED. 348 | confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"] 349 | 350 | # Disable the message, report, category or checker with the given id(s). You can 351 | # either give multiple identifiers separated by comma (,) or put this option 352 | # multiple times (only on the command line, not in the configuration file where 353 | # it should appear only once). You can also use "--disable=all" to disable 354 | # everything first and then re-enable specific checks. For example, if you want 355 | # to run only the similarities checker, you can use "--disable=all 356 | # --enable=similarities". If you want to run only the classes checker, but have 357 | # no Warning level messages displayed, use "--disable=all --enable=classes 358 | # --disable=W". 359 | disable = ["raw-checker-failed", "bad-inline-option", "locally-disabled", "file-ignored", "suppressed-message", "useless-suppression", "deprecated-pragma", "use-symbolic-message-instead", "missing-module-docstring", "too-many-ancestors", "too-many-instance-attributes", "too-few-public-methods", "too-many-public-methods", "too-many-return-statements", "too-many-branches", "too-many-arguments", "too-many-locals", "too-many-statements", "too-many-boolean-expressions", "duplicate-code", "attribute-defined-outside-init", "useless-option-value", "unknown-option-value", "global-statement", "fixme", "no-member", "no-name-in-module", "import-error", "unsubscriptable-object", "unbalanced-tuple-unpacking", "undefined-variable", "not-context-manager", "no-else-return", "wrong-import-order", "unnecessary-pass", "logging-fstring-interpolation", "logging-format-interpolation"] 360 | 361 | # Enable the message, report, category or checker with the given id(s). You can 362 | # either give multiple identifier separated by comma (,) or put this option 363 | # multiple time (only on the command line, not in the configuration file where it 364 | # should appear only once). See also the "--disable" option for examples. 365 | enable = ["c-extension-no-member"] 366 | 367 | [tool.pylint.miscellaneous] 368 | # List of note tags to take in consideration, separated by a comma. 369 | # notes = 370 | 371 | # Regular expression of note tags to take in consideration. 372 | # notes-rgx = 373 | 374 | [tool.pylint.refactoring] 375 | # Maximum number of nested blocks for function / method body 376 | max-nested-blocks = 5 377 | 378 | # Complete name of functions that never returns. When checking for inconsistent- 379 | # return-statements if a never returning function is called then it will be 380 | # considered as an explicit return statement and no message will be printed. 381 | never-returning-functions = ["sys.exit", "argparse.parse_error"] 382 | 383 | [tool.pylint.reports] 384 | # Python expression which should return a score less than or equal to 10. You 385 | # have access to the variables 'fatal', 'error', 'warning', 'refactor', 386 | # 'convention', and 'info' which contain the number of messages in each category, 387 | # as well as 'statement' which is the total number of statements analyzed. This 388 | # score is used by the global evaluation report (RP0004). 389 | evaluation = "10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)" 390 | 391 | # Template used to display messages. This is a python new-style format string 392 | # used to format the message information. See doc for all details. 393 | # msg-template = 394 | 395 | # Set the output format. Available formats are text, parseable, colorized, json 396 | # and msvs (visual studio). You can also give a reporter class, e.g. 397 | # mypackage.mymodule.MyReporterClass. 398 | # output-format = 399 | 400 | # Tells whether to display a full report or only the messages. 401 | # reports = 402 | 403 | # Activate the evaluation score. 404 | score = true 405 | 406 | [tool.pylint.similarities] 407 | # Comments are removed from the similarity computation 408 | ignore-comments = true 409 | 410 | # Docstrings are removed from the similarity computation 411 | ignore-docstrings = true 412 | 413 | # Imports are removed from the similarity computation 414 | # ignore-imports = 415 | 416 | # Signatures are removed from the similarity computation 417 | ignore-signatures = true 418 | 419 | # Minimum lines number of a similarity. 420 | min-similarity-lines = 4 421 | 422 | [tool.pylint.spelling] 423 | # Limits count of emitted suggestions for spelling mistakes. 424 | max-spelling-suggestions = 4 425 | 426 | # Spelling dictionary name. Available dictionaries: none. To make it work, 427 | # install the 'python-enchant' package. 428 | # spelling-dict = 429 | 430 | # List of comma separated words that should be considered directives if they 431 | # appear at the beginning of a comment and should not be checked. 432 | spelling-ignore-comment-directives = "fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:" 433 | 434 | # List of comma separated words that should not be checked. 435 | # spelling-ignore-words = 436 | 437 | # A path to a file that contains the private dictionary; one word per line. 438 | # spelling-private-dict-file = 439 | 440 | # Tells whether to store unknown words to the private dictionary (see the 441 | # --spelling-private-dict-file option) instead of raising a message. 442 | # spelling-store-unknown-words = 443 | 444 | [tool.pylint.string] 445 | # This flag controls whether inconsistent-quotes generates a warning when the 446 | # character used as a quote delimiter is used inconsistently within a module. 447 | # check-quote-consistency = 448 | 449 | # This flag controls whether the implicit-str-concat should generate a warning on 450 | # implicit string concatenation in sequences defined over several lines. 451 | # check-str-concat-over-line-jumps = 452 | 453 | [tool.pylint.typecheck] 454 | # List of decorators that produce context managers, such as 455 | # contextlib.contextmanager. Add to this list to register other decorators that 456 | # produce valid context managers. 457 | contextmanager-decorators = ["contextlib.contextmanager", "contextlib2.contextmanager"] 458 | 459 | # List of members which are set dynamically and missed by pylint inference 460 | # system, and so shouldn't trigger E1101 when accessed. Python regular 461 | # expressions are accepted. 462 | generated-members = "REQUEST,acl_users,aq_parent" 463 | 464 | # Tells whether missing members accessed in mixin class should be ignored. A 465 | # class is considered mixin if its name matches the mixin-class-rgx option. 466 | # Tells whether to warn about missing members when the owner of the attribute is 467 | # inferred to be None. 468 | ignore-none = true 469 | 470 | # This flag controls whether pylint should warn about no-member and similar 471 | # checks whenever an opaque object is returned when inferring. The inference can 472 | # return multiple potential results while evaluating a Python object, but some 473 | # branches might not be evaluated, which results in partial inference. In that 474 | # case, it might be useful to still emit no-member and other checks for the rest 475 | # of the inferred objects. 476 | ignore-on-opaque-inference = true 477 | 478 | # List of symbolic message names to ignore for Mixin members. 479 | ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"] 480 | 481 | # List of class names for which member attributes should not be checked (useful 482 | # for classes with dynamically set attributes). This supports the use of 483 | # qualified names. 484 | ignored-classes = ["SQLObject"] 485 | 486 | # Show a hint with possible names when a member name was not found. The aspect of 487 | # finding the hint is based on edit distance. 488 | missing-member-hint = true 489 | 490 | # The minimum edit distance a name should have in order to be considered a 491 | # similar match for a missing member name. 492 | missing-member-hint-distance = 1 493 | 494 | # The total number of similar names that should be taken in consideration when 495 | # showing a hint for a missing member. 496 | missing-member-max-choices = 1 497 | 498 | # Regex pattern to define which classes are considered mixins. 499 | mixin-class-rgx = ".*[Mm]ixin" 500 | 501 | # List of decorators that change the signature of a decorated function. 502 | # signature-mutators = 503 | 504 | [tool.pylint.variables] 505 | # List of additional names supposed to be defined in builtins. Remember that you 506 | # should avoid defining new builtins when possible. 507 | # additional-builtins = 508 | 509 | # Tells whether unused global variables should be treated as a violation. 510 | allow-global-unused-variables = true 511 | 512 | # List of names allowed to shadow builtins 513 | # allowed-redefined-builtins = 514 | 515 | # List of strings which can identify a callback function by name. A callback name 516 | # must start or end with one of those strings. 517 | callbacks = ["cb_", "_cb"] 518 | 519 | # A regular expression matching the name of dummy variables (i.e. expected to not 520 | # be used). 521 | dummy-variables-rgx = "^\\*{0,2}(_$|unused_|dummy_)" 522 | 523 | # Argument names that match this expression will be ignored. Default to name with 524 | # leading underscore. 525 | ignored-argument-names = "_.*" 526 | 527 | # Tells whether we should check for unused import in __init__ files. 528 | # init-import = 529 | 530 | # List of qualified module names which can have objects that can redefine 531 | # builtins. 532 | redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"] 533 | --------------------------------------------------------------------------------