├── .envrc.example ├── .gitignore ├── LICENSE ├── README.md ├── docs ├── dataset_preparation.md ├── inference.md └── training.md ├── figs └── main_results.png ├── pyproject.toml └── src ├── layoutlib ├── __init__.py ├── bucketizer.py ├── config │ └── element │ │ ├── text_layout.yaml │ │ ├── text_layout_no_quantization.yaml │ │ ├── type_layout.yaml │ │ ├── typography_crello_default.yaml │ │ └── typography_private_default.yaml ├── hfds │ ├── __init__.py │ ├── base.py │ ├── clustering.py │ ├── crello.py │ ├── mock.py │ └── util.py ├── manager.py ├── schema.py ├── util.py └── weights │ └── crello_max20_kmeans_train_clusters.pkl └── opencole ├── __init__.py ├── evaluation ├── __init__.py ├── clip.py ├── eval_detail_similarity.py ├── eval_gpt4v.py ├── eval_image_layer.py ├── result_visualization.py └── sentence.py ├── hf_support ├── convert_llava_to_hf.py ├── dump_full_llava_weight_dicts.py └── push_to_hub.py ├── inference ├── __init__.py ├── data │ ├── cole_designerintention_v1_samples.csv │ └── cole_designerintention_v2_samples.csv ├── langchain_helper.py ├── tester │ ├── __init__.py │ ├── base.py │ ├── llm.py │ └── text2image.py ├── typographylmm_config │ └── crello_default.json └── util.py ├── inference_launcher ├── __init__.py ├── gpt4_augmented_baselines.py ├── k_shot_llm.py ├── render.py ├── text_to_image.py └── typographylmm.py ├── model ├── __init__.py └── llava.py ├── preprocess ├── __init__.py ├── create_dataset_for_text_to_image.py ├── create_dataset_for_typographylmm.py ├── dataset_creation │ ├── __init__.py │ ├── crello_instances.py │ ├── design.py │ ├── image_to_detail.py │ ├── image_to_intention.py │ ├── loading.py │ ├── progress_util.py │ ├── test_components.py │ ├── util.py │ └── vision_llm_llava.py ├── helper.py └── language.py ├── renderer ├── __init__.py ├── color_utils.py ├── fonts.py ├── image_utils.py ├── renderer.py ├── schema.py ├── text_utils.py └── utils.py ├── schema.py └── util.py /.envrc.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/.envrc.example -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/README.md -------------------------------------------------------------------------------- /docs/dataset_preparation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/docs/dataset_preparation.md -------------------------------------------------------------------------------- /docs/inference.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/docs/inference.md -------------------------------------------------------------------------------- /docs/training.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/docs/training.md -------------------------------------------------------------------------------- /figs/main_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/figs/main_results.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/pyproject.toml -------------------------------------------------------------------------------- /src/layoutlib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/layoutlib/bucketizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/bucketizer.py -------------------------------------------------------------------------------- /src/layoutlib/config/element/text_layout.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/config/element/text_layout.yaml -------------------------------------------------------------------------------- /src/layoutlib/config/element/text_layout_no_quantization.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/config/element/text_layout_no_quantization.yaml -------------------------------------------------------------------------------- /src/layoutlib/config/element/type_layout.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/config/element/type_layout.yaml -------------------------------------------------------------------------------- /src/layoutlib/config/element/typography_crello_default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/config/element/typography_crello_default.yaml -------------------------------------------------------------------------------- /src/layoutlib/config/element/typography_private_default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/config/element/typography_private_default.yaml -------------------------------------------------------------------------------- /src/layoutlib/hfds/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/hfds/__init__.py -------------------------------------------------------------------------------- /src/layoutlib/hfds/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/hfds/base.py -------------------------------------------------------------------------------- /src/layoutlib/hfds/clustering.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/hfds/clustering.py -------------------------------------------------------------------------------- /src/layoutlib/hfds/crello.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/hfds/crello.py -------------------------------------------------------------------------------- /src/layoutlib/hfds/mock.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/hfds/mock.py -------------------------------------------------------------------------------- /src/layoutlib/hfds/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/hfds/util.py -------------------------------------------------------------------------------- /src/layoutlib/manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/manager.py -------------------------------------------------------------------------------- /src/layoutlib/schema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/schema.py -------------------------------------------------------------------------------- /src/layoutlib/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/util.py -------------------------------------------------------------------------------- /src/layoutlib/weights/crello_max20_kmeans_train_clusters.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/layoutlib/weights/crello_max20_kmeans_train_clusters.pkl -------------------------------------------------------------------------------- /src/opencole/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/opencole/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/opencole/evaluation/clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/evaluation/clip.py -------------------------------------------------------------------------------- /src/opencole/evaluation/eval_detail_similarity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/evaluation/eval_detail_similarity.py -------------------------------------------------------------------------------- /src/opencole/evaluation/eval_gpt4v.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/evaluation/eval_gpt4v.py -------------------------------------------------------------------------------- /src/opencole/evaluation/eval_image_layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/evaluation/eval_image_layer.py -------------------------------------------------------------------------------- /src/opencole/evaluation/result_visualization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/evaluation/result_visualization.py -------------------------------------------------------------------------------- /src/opencole/evaluation/sentence.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/evaluation/sentence.py -------------------------------------------------------------------------------- /src/opencole/hf_support/convert_llava_to_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/hf_support/convert_llava_to_hf.py -------------------------------------------------------------------------------- /src/opencole/hf_support/dump_full_llava_weight_dicts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/hf_support/dump_full_llava_weight_dicts.py -------------------------------------------------------------------------------- /src/opencole/hf_support/push_to_hub.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/hf_support/push_to_hub.py -------------------------------------------------------------------------------- /src/opencole/inference/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/opencole/inference/data/cole_designerintention_v1_samples.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference/data/cole_designerintention_v1_samples.csv -------------------------------------------------------------------------------- /src/opencole/inference/data/cole_designerintention_v2_samples.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference/data/cole_designerintention_v2_samples.csv -------------------------------------------------------------------------------- /src/opencole/inference/langchain_helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference/langchain_helper.py -------------------------------------------------------------------------------- /src/opencole/inference/tester/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/opencole/inference/tester/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference/tester/base.py -------------------------------------------------------------------------------- /src/opencole/inference/tester/llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference/tester/llm.py -------------------------------------------------------------------------------- /src/opencole/inference/tester/text2image.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference/tester/text2image.py -------------------------------------------------------------------------------- /src/opencole/inference/typographylmm_config/crello_default.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference/typographylmm_config/crello_default.json -------------------------------------------------------------------------------- /src/opencole/inference/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference/util.py -------------------------------------------------------------------------------- /src/opencole/inference_launcher/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/opencole/inference_launcher/gpt4_augmented_baselines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference_launcher/gpt4_augmented_baselines.py -------------------------------------------------------------------------------- /src/opencole/inference_launcher/k_shot_llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference_launcher/k_shot_llm.py -------------------------------------------------------------------------------- /src/opencole/inference_launcher/render.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference_launcher/render.py -------------------------------------------------------------------------------- /src/opencole/inference_launcher/text_to_image.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference_launcher/text_to_image.py -------------------------------------------------------------------------------- /src/opencole/inference_launcher/typographylmm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/inference_launcher/typographylmm.py -------------------------------------------------------------------------------- /src/opencole/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/opencole/model/llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/model/llava.py -------------------------------------------------------------------------------- /src/opencole/preprocess/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/opencole/preprocess/create_dataset_for_text_to_image.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/create_dataset_for_text_to_image.py -------------------------------------------------------------------------------- /src/opencole/preprocess/create_dataset_for_typographylmm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/create_dataset_for_typographylmm.py -------------------------------------------------------------------------------- /src/opencole/preprocess/dataset_creation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/dataset_creation/__init__.py -------------------------------------------------------------------------------- /src/opencole/preprocess/dataset_creation/crello_instances.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/dataset_creation/crello_instances.py -------------------------------------------------------------------------------- /src/opencole/preprocess/dataset_creation/design.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/dataset_creation/design.py -------------------------------------------------------------------------------- /src/opencole/preprocess/dataset_creation/image_to_detail.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/dataset_creation/image_to_detail.py -------------------------------------------------------------------------------- /src/opencole/preprocess/dataset_creation/image_to_intention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/dataset_creation/image_to_intention.py -------------------------------------------------------------------------------- /src/opencole/preprocess/dataset_creation/loading.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/dataset_creation/loading.py -------------------------------------------------------------------------------- /src/opencole/preprocess/dataset_creation/progress_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/dataset_creation/progress_util.py -------------------------------------------------------------------------------- /src/opencole/preprocess/dataset_creation/test_components.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/dataset_creation/test_components.py -------------------------------------------------------------------------------- /src/opencole/preprocess/dataset_creation/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/dataset_creation/util.py -------------------------------------------------------------------------------- /src/opencole/preprocess/dataset_creation/vision_llm_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/dataset_creation/vision_llm_llava.py -------------------------------------------------------------------------------- /src/opencole/preprocess/helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/helper.py -------------------------------------------------------------------------------- /src/opencole/preprocess/language.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/preprocess/language.py -------------------------------------------------------------------------------- /src/opencole/renderer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/opencole/renderer/color_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/renderer/color_utils.py -------------------------------------------------------------------------------- /src/opencole/renderer/fonts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/renderer/fonts.py -------------------------------------------------------------------------------- /src/opencole/renderer/image_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/renderer/image_utils.py -------------------------------------------------------------------------------- /src/opencole/renderer/renderer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/renderer/renderer.py -------------------------------------------------------------------------------- /src/opencole/renderer/schema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/renderer/schema.py -------------------------------------------------------------------------------- /src/opencole/renderer/text_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/renderer/text_utils.py -------------------------------------------------------------------------------- /src/opencole/renderer/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/renderer/utils.py -------------------------------------------------------------------------------- /src/opencole/schema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/schema.py -------------------------------------------------------------------------------- /src/opencole/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CyberAgentAILab/OpenCOLE/HEAD/src/opencole/util.py --------------------------------------------------------------------------------