├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── configs ├── test_box.yaml ├── test_mask.yaml ├── test_point.yaml ├── test_scribble.yaml ├── test_sd15.yaml └── train_sd15.yaml ├── dataset-generation ├── README.md ├── jsons2txt.py ├── run_submitit_generate_jsons.sh ├── run_with_submitit_generate_caption.py ├── sample_train_data.json └── submitit_automatic_label_ram_save_json.py ├── dataset ├── __init__.py ├── base_dataset.py ├── decode_item.py ├── jsondataset.py └── utils.py ├── demos ├── demo_cat_dog_robin.json ├── demo_corgi_kitchen.json ├── demo_eagle_left.json ├── demo_eagle_right.json ├── demo_iterative_r1.json ├── demo_iterative_r2.json ├── demo_iterative_r3.json ├── demo_iterative_r4.json ├── demo_iterative_r5.json ├── demo_palm_point.json ├── demo_rose_garden_painting.json └── demo_teddy_bear_castle.json ├── docs ├── InstDiff-gif.gif ├── InstDiff-points.png ├── results.png └── teaser.jpg ├── eval ├── eval_attribute_binding.py └── eval_pim.py ├── eval_local.py ├── grounding_input ├── __init__.py └── text_grounding_tokinzer_input.py ├── inference.py ├── ldm ├── data │ ├── __init__.py │ ├── base.py │ ├── imagenet.py │ ├── imagenet_clsidx_to_label.txt │ ├── imagenet_train_hr_indices.p │ ├── imagenet_val_hr_indices.p │ ├── index_synset.yaml │ └── lsun.py ├── lr_scheduler.py ├── models │ ├── autoencoder.py │ └── diffusion │ │ ├── __init__.py │ │ ├── classifier.py │ │ ├── ddim.py │ │ ├── ddpm.py │ │ ├── ldm.py │ │ ├── plms.py │ │ └── plms_instance.py ├── modules │ ├── attention.py │ ├── diffusionmodules │ │ ├── __init__.py │ │ ├── convnext.py │ │ ├── model.py │ │ ├── openaimodel.py │ │ ├── resnet.py │ │ ├── text_grounding_net.py │ │ └── util.py │ ├── distributions │ │ ├── __init__.py │ │ └── distributions.py │ ├── ema.py │ ├── encoders │ │ ├── __init__.py │ │ ├── modules.py │ │ └── modules_backup.py │ ├── image_degradation │ │ ├── __init__.py │ │ ├── bsrgan.py │ │ ├── bsrgan_light.py │ │ └── utils_image.py │ ├── losses │ │ ├── __init__.py │ │ ├── contperceptual.py │ │ └── vqperceptual.py │ └── x_transformer.py └── util.py ├── main_submitit.py ├── pretrained └── SD_v1_5_input_conv_weight_bias.pth ├── requirements.txt ├── run_with_submitit.py ├── trainer.py └── utils ├── checkpoint.py ├── dataloader.py ├── dist.py ├── input.py ├── misc.py ├── model.py ├── optimizer.py └── scheduler.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/.gitignore -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/README.md -------------------------------------------------------------------------------- /configs/test_box.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/configs/test_box.yaml -------------------------------------------------------------------------------- /configs/test_mask.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/configs/test_mask.yaml -------------------------------------------------------------------------------- /configs/test_point.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/configs/test_point.yaml -------------------------------------------------------------------------------- /configs/test_scribble.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/configs/test_scribble.yaml -------------------------------------------------------------------------------- /configs/test_sd15.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/configs/test_sd15.yaml -------------------------------------------------------------------------------- /configs/train_sd15.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/configs/train_sd15.yaml -------------------------------------------------------------------------------- /dataset-generation/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/dataset-generation/README.md -------------------------------------------------------------------------------- /dataset-generation/jsons2txt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/dataset-generation/jsons2txt.py -------------------------------------------------------------------------------- /dataset-generation/run_submitit_generate_jsons.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/dataset-generation/run_submitit_generate_jsons.sh -------------------------------------------------------------------------------- /dataset-generation/run_with_submitit_generate_caption.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/dataset-generation/run_with_submitit_generate_caption.py -------------------------------------------------------------------------------- /dataset-generation/sample_train_data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/dataset-generation/sample_train_data.json -------------------------------------------------------------------------------- /dataset-generation/submitit_automatic_label_ram_save_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/dataset-generation/submitit_automatic_label_ram_save_json.py -------------------------------------------------------------------------------- /dataset/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dataset/base_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/dataset/base_dataset.py -------------------------------------------------------------------------------- /dataset/decode_item.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/dataset/decode_item.py -------------------------------------------------------------------------------- /dataset/jsondataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/dataset/jsondataset.py -------------------------------------------------------------------------------- /dataset/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/dataset/utils.py -------------------------------------------------------------------------------- /demos/demo_cat_dog_robin.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_cat_dog_robin.json -------------------------------------------------------------------------------- /demos/demo_corgi_kitchen.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_corgi_kitchen.json -------------------------------------------------------------------------------- /demos/demo_eagle_left.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_eagle_left.json -------------------------------------------------------------------------------- /demos/demo_eagle_right.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_eagle_right.json -------------------------------------------------------------------------------- /demos/demo_iterative_r1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_iterative_r1.json -------------------------------------------------------------------------------- /demos/demo_iterative_r2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_iterative_r2.json -------------------------------------------------------------------------------- /demos/demo_iterative_r3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_iterative_r3.json -------------------------------------------------------------------------------- /demos/demo_iterative_r4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_iterative_r4.json -------------------------------------------------------------------------------- /demos/demo_iterative_r5.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_iterative_r5.json -------------------------------------------------------------------------------- /demos/demo_palm_point.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_palm_point.json -------------------------------------------------------------------------------- /demos/demo_rose_garden_painting.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_rose_garden_painting.json -------------------------------------------------------------------------------- /demos/demo_teddy_bear_castle.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/demos/demo_teddy_bear_castle.json -------------------------------------------------------------------------------- /docs/InstDiff-gif.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/docs/InstDiff-gif.gif -------------------------------------------------------------------------------- /docs/InstDiff-points.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/docs/InstDiff-points.png -------------------------------------------------------------------------------- /docs/results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/docs/results.png -------------------------------------------------------------------------------- /docs/teaser.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/docs/teaser.jpg -------------------------------------------------------------------------------- /eval/eval_attribute_binding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/eval/eval_attribute_binding.py -------------------------------------------------------------------------------- /eval/eval_pim.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/eval/eval_pim.py -------------------------------------------------------------------------------- /eval_local.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/eval_local.py -------------------------------------------------------------------------------- /grounding_input/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/grounding_input/__init__.py -------------------------------------------------------------------------------- /grounding_input/text_grounding_tokinzer_input.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/grounding_input/text_grounding_tokinzer_input.py -------------------------------------------------------------------------------- /inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/inference.py -------------------------------------------------------------------------------- /ldm/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ldm/data/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/data/base.py -------------------------------------------------------------------------------- /ldm/data/imagenet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/data/imagenet.py -------------------------------------------------------------------------------- /ldm/data/imagenet_clsidx_to_label.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/data/imagenet_clsidx_to_label.txt -------------------------------------------------------------------------------- /ldm/data/imagenet_train_hr_indices.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/data/imagenet_train_hr_indices.p -------------------------------------------------------------------------------- /ldm/data/imagenet_val_hr_indices.p: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/data/imagenet_val_hr_indices.p -------------------------------------------------------------------------------- /ldm/data/index_synset.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/data/index_synset.yaml -------------------------------------------------------------------------------- /ldm/data/lsun.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/data/lsun.py -------------------------------------------------------------------------------- /ldm/lr_scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/lr_scheduler.py -------------------------------------------------------------------------------- /ldm/models/autoencoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/models/autoencoder.py -------------------------------------------------------------------------------- /ldm/models/diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ldm/models/diffusion/classifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/models/diffusion/classifier.py -------------------------------------------------------------------------------- /ldm/models/diffusion/ddim.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/models/diffusion/ddim.py -------------------------------------------------------------------------------- /ldm/models/diffusion/ddpm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/models/diffusion/ddpm.py -------------------------------------------------------------------------------- /ldm/models/diffusion/ldm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/models/diffusion/ldm.py -------------------------------------------------------------------------------- /ldm/models/diffusion/plms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/models/diffusion/plms.py -------------------------------------------------------------------------------- /ldm/models/diffusion/plms_instance.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/models/diffusion/plms_instance.py -------------------------------------------------------------------------------- /ldm/modules/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/attention.py -------------------------------------------------------------------------------- /ldm/modules/diffusionmodules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ldm/modules/diffusionmodules/convnext.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/diffusionmodules/convnext.py -------------------------------------------------------------------------------- /ldm/modules/diffusionmodules/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/diffusionmodules/model.py -------------------------------------------------------------------------------- /ldm/modules/diffusionmodules/openaimodel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/diffusionmodules/openaimodel.py -------------------------------------------------------------------------------- /ldm/modules/diffusionmodules/resnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/diffusionmodules/resnet.py -------------------------------------------------------------------------------- /ldm/modules/diffusionmodules/text_grounding_net.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/diffusionmodules/text_grounding_net.py -------------------------------------------------------------------------------- /ldm/modules/diffusionmodules/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/diffusionmodules/util.py -------------------------------------------------------------------------------- /ldm/modules/distributions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ldm/modules/distributions/distributions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/distributions/distributions.py -------------------------------------------------------------------------------- /ldm/modules/ema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/ema.py -------------------------------------------------------------------------------- /ldm/modules/encoders/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ldm/modules/encoders/modules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/encoders/modules.py -------------------------------------------------------------------------------- /ldm/modules/encoders/modules_backup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/encoders/modules_backup.py -------------------------------------------------------------------------------- /ldm/modules/image_degradation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/image_degradation/__init__.py -------------------------------------------------------------------------------- /ldm/modules/image_degradation/bsrgan.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/image_degradation/bsrgan.py -------------------------------------------------------------------------------- /ldm/modules/image_degradation/bsrgan_light.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/image_degradation/bsrgan_light.py -------------------------------------------------------------------------------- /ldm/modules/image_degradation/utils_image.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/image_degradation/utils_image.py -------------------------------------------------------------------------------- /ldm/modules/losses/__init__.py: -------------------------------------------------------------------------------- 1 | from ldm.modules.losses.contperceptual import LPIPSWithDiscriminator -------------------------------------------------------------------------------- /ldm/modules/losses/contperceptual.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/losses/contperceptual.py -------------------------------------------------------------------------------- /ldm/modules/losses/vqperceptual.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/losses/vqperceptual.py -------------------------------------------------------------------------------- /ldm/modules/x_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/modules/x_transformer.py -------------------------------------------------------------------------------- /ldm/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/ldm/util.py -------------------------------------------------------------------------------- /main_submitit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/main_submitit.py -------------------------------------------------------------------------------- /pretrained/SD_v1_5_input_conv_weight_bias.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/pretrained/SD_v1_5_input_conv_weight_bias.pth -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/requirements.txt -------------------------------------------------------------------------------- /run_with_submitit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/run_with_submitit.py -------------------------------------------------------------------------------- /trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/trainer.py -------------------------------------------------------------------------------- /utils/checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/utils/checkpoint.py -------------------------------------------------------------------------------- /utils/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/utils/dataloader.py -------------------------------------------------------------------------------- /utils/dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/utils/dist.py -------------------------------------------------------------------------------- /utils/input.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/utils/input.py -------------------------------------------------------------------------------- /utils/misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/utils/misc.py -------------------------------------------------------------------------------- /utils/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/utils/model.py -------------------------------------------------------------------------------- /utils/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/utils/optimizer.py -------------------------------------------------------------------------------- /utils/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/frank-xwang/InstanceDiffusion/HEAD/utils/scheduler.py --------------------------------------------------------------------------------