├── assets ├── .gitkeep ├── 2.png ├── 3.jpg ├── 4.jpg ├── 5.jpg ├── method.png └── teaser.jpg ├── diffusers ├── py.typed ├── experimental │ ├── __init__.py │ ├── rl │ │ └── __init__.py │ └── README.md ├── .DS_Store ├── pipelines │ ├── .DS_Store │ ├── __pycache__ │ │ ├── __init__.cpython-38.pyc │ │ ├── __init__.cpython-310.pyc │ │ ├── onnx_utils.cpython-310.pyc │ │ ├── onnx_utils.cpython-38.pyc │ │ ├── pipeline_utils.cpython-310.pyc │ │ └── pipeline_utils.cpython-38.pyc │ ├── ddim │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-310.pyc │ │ │ └── pipeline_ddim.cpython-310.pyc │ │ └── __init__.py │ ├── animatediff │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-310.pyc │ │ │ └── pipeline_animatediff.cpython-310.pyc │ │ └── __init__.py │ ├── controlnet │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-310.pyc │ │ │ └── multicontrolnet.cpython-310.pyc │ │ └── __init__.py │ ├── stable_diffusion │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-310.pyc │ │ │ ├── __init__.cpython-38.pyc │ │ │ ├── safety_checker.cpython-38.pyc │ │ │ ├── pipeline_output.cpython-310.pyc │ │ │ ├── pipeline_output.cpython-38.pyc │ │ │ ├── safety_checker.cpython-310.pyc │ │ │ ├── pipeline_stable_diffusion.cpython-310.pyc │ │ │ ├── pipeline_stable_diffusion_inpaint.cpython-310.pyc │ │ │ └── pipeline_stable_diffusion_inpaint.cpython-38.pyc │ │ ├── clip_image_project_model.py │ │ ├── pipeline_output.py │ │ └── stable_unclip_image_normalizer.py │ ├── stable_diffusion_xl │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-310.pyc │ │ │ ├── pipeline_output.cpython-310.pyc │ │ │ └── pipeline_stable_diffusion_xl.cpython-310.pyc │ │ ├── pipeline_output.py │ │ ├── watermark.py │ │ └── __init__.py │ ├── pndm │ │ └── __init__.py │ ├── dit │ │ └── __init__.py │ ├── repaint │ │ └── __init__.py │ ├── ddpm │ │ └── __init__.py │ ├── score_sde_ve │ │ └── __init__.py │ ├── dance_diffusion │ │ └── __init__.py │ ├── latent_diffusion_uncond │ │ └── __init__.py │ ├── stochastic_karras_ve │ │ └── __init__.py │ ├── consistency_models │ │ └── __init__.py │ ├── audio_diffusion │ │ └── __init__.py │ ├── text_to_video_synthesis │ │ ├── pipeline_output.py │ │ └── __init__.py │ ├── blip_diffusion │ │ └── __init__.py │ ├── semantic_stable_diffusion │ │ ├── pipeline_output.py │ │ └── __init__.py │ ├── alt_diffusion │ │ ├── pipeline_output.py │ │ └── __init__.py │ ├── kandinsky │ │ ├── text_encoder.py │ │ └── __init__.py │ ├── deepfloyd_if │ │ ├── pipeline_output.py │ │ ├── watermark.py │ │ ├── safety_checker.py │ │ └── __init__.py │ ├── pixart_alpha │ │ └── __init__.py │ ├── stable_diffusion_safe │ │ ├── pipeline_output.py │ │ └── __init__.py │ ├── musicldm │ │ └── __init__.py │ ├── audioldm │ │ └── __init__.py │ ├── t2i_adapter │ │ └── __init__.py │ ├── latent_diffusion │ │ └── __init__.py │ ├── latent_consistency_models │ │ └── __init__.py │ ├── paint_by_example │ │ ├── __init__.py │ │ └── image_encoder.py │ ├── audioldm2 │ │ └── __init__.py │ ├── vq_diffusion │ │ └── __init__.py │ ├── unclip │ │ └── __init__.py │ ├── unidiffuser │ │ └── __init__.py │ ├── wuerstchen │ │ ├── __init__.py │ │ └── modeling_wuerstchen_common.py │ ├── shap_e │ │ └── __init__.py │ ├── spectrogram_diffusion │ │ ├── __init__.py │ │ ├── notes_encoder.py │ │ └── continous_encoder.py │ ├── kandinsky2_2 │ │ └── __init__.py │ └── versatile_diffusion │ │ └── __init__.py ├── models │ ├── __pycache__ │ │ ├── lora.cpython-38.pyc │ │ ├── vae.cpython-310.pyc │ │ ├── vae.cpython-38.pyc │ │ ├── lora.cpython-310.pyc │ │ ├── resnet.cpython-38.pyc │ │ ├── __init__.cpython-310.pyc │ │ ├── __init__.cpython-38.pyc │ │ ├── attention.cpython-38.pyc │ │ ├── resnet.cpython-310.pyc │ │ ├── activations.cpython-38.pyc │ │ ├── attention.cpython-310.pyc │ │ ├── controlnet.cpython-310.pyc │ │ ├── embeddings.cpython-310.pyc │ │ ├── embeddings.cpython-38.pyc │ │ ├── activations.cpython-310.pyc │ │ ├── autoencoder_kl.cpython-38.pyc │ │ ├── modeling_utils.cpython-38.pyc │ │ ├── normalization.cpython-310.pyc │ │ ├── normalization.cpython-38.pyc │ │ ├── transformer_2d.cpython-38.pyc │ │ ├── unet_2d_blocks.cpython-38.pyc │ │ ├── autoencoder_kl.cpython-310.pyc │ │ ├── modeling_utils.cpython-310.pyc │ │ ├── transformer_2d.cpython-310.pyc │ │ ├── unet_2d_blocks.cpython-310.pyc │ │ ├── unet_3d_blocks.cpython-310.pyc │ │ ├── attention_processor.cpython-38.pyc │ │ ├── autoencoder_asym_kl.cpython-38.pyc │ │ ├── dual_transformer_2d.cpython-38.pyc │ │ ├── unet_2d_condition.cpython-310.pyc │ │ ├── unet_2d_condition.cpython-38.pyc │ │ ├── unet_3d_condition.cpython-310.pyc │ │ ├── unet_motion_model.cpython-310.pyc │ │ ├── attention_processor.cpython-310.pyc │ │ ├── autoencoder_asym_kl.cpython-310.pyc │ │ ├── dual_transformer_2d.cpython-310.pyc │ │ └── transformer_temporal.cpython-310.pyc │ ├── README.md │ ├── embeddings_flax.py │ ├── __init__.py │ └── activations.py ├── utils │ ├── __pycache__ │ │ ├── logging.cpython-38.pyc │ │ ├── outputs.cpython-38.pyc │ │ ├── __init__.cpython-310.pyc │ │ ├── __init__.cpython-38.pyc │ │ ├── constants.cpython-310.pyc │ │ ├── constants.cpython-38.pyc │ │ ├── doc_utils.cpython-310.pyc │ │ ├── doc_utils.cpython-38.pyc │ │ ├── hub_utils.cpython-310.pyc │ │ ├── hub_utils.cpython-38.pyc │ │ ├── logging.cpython-310.pyc │ │ ├── outputs.cpython-310.pyc │ │ ├── peft_utils.cpython-38.pyc │ │ ├── pil_utils.cpython-310.pyc │ │ ├── pil_utils.cpython-38.pyc │ │ ├── versions.cpython-310.pyc │ │ ├── versions.cpython-38.pyc │ │ ├── export_utils.cpython-38.pyc │ │ ├── import_utils.cpython-38.pyc │ │ ├── peft_utils.cpython-310.pyc │ │ ├── torch_utils.cpython-310.pyc │ │ ├── torch_utils.cpython-38.pyc │ │ ├── export_utils.cpython-310.pyc │ │ ├── import_utils.cpython-310.pyc │ │ ├── loading_utils.cpython-310.pyc │ │ ├── loading_utils.cpython-38.pyc │ │ ├── accelerate_utils.cpython-310.pyc │ │ ├── accelerate_utils.cpython-38.pyc │ │ ├── deprecation_utils.cpython-38.pyc │ │ ├── state_dict_utils.cpython-310.pyc │ │ ├── state_dict_utils.cpython-38.pyc │ │ ├── deprecation_utils.cpython-310.pyc │ │ ├── dummy_flax_objects.cpython-310.pyc │ │ ├── dummy_flax_objects.cpython-38.pyc │ │ ├── dummy_onnx_objects.cpython-310.pyc │ │ ├── dummy_note_seq_objects.cpython-38.pyc │ │ ├── dynamic_modules_utils.cpython-310.pyc │ │ ├── dynamic_modules_utils.cpython-38.pyc │ │ ├── dummy_note_seq_objects.cpython-310.pyc │ │ ├── dummy_torch_and_librosa_objects.cpython-310.pyc │ │ ├── dummy_torch_and_librosa_objects.cpython-38.pyc │ │ ├── dummy_torch_and_torchsde_objects.cpython-38.pyc │ │ ├── dummy_flax_and_transformers_objects.cpython-38.pyc │ │ ├── dummy_torch_and_torchsde_objects.cpython-310.pyc │ │ ├── dummy_flax_and_transformers_objects.cpython-310.pyc │ │ ├── dummy_torch_and_transformers_and_onnx_objects.cpython-310.pyc │ │ ├── dummy_transformers_and_torch_and_note_seq_objects.cpython-310.pyc │ │ ├── dummy_transformers_and_torch_and_note_seq_objects.cpython-38.pyc │ │ ├── dummy_torch_and_transformers_and_k_diffusion_objects.cpython-38.pyc │ │ └── dummy_torch_and_transformers_and_k_diffusion_objects.cpython-310.pyc │ ├── dummy_onnx_objects.py │ ├── dummy_note_seq_objects.py │ ├── dummy_torch_and_scipy_objects.py │ ├── dummy_torch_and_torchsde_objects.py │ ├── dummy_transformers_and_torch_and_note_seq_objects.py │ ├── dummy_torch_and_transformers_and_k_diffusion_objects.py │ ├── dummy_torch_and_librosa_objects.py │ ├── loading_utils.py │ ├── doc_utils.py │ ├── model_card_template.md │ ├── accelerate_utils.py │ ├── pil_utils.py │ ├── deprecation_utils.py │ ├── constants.py │ ├── dummy_flax_and_transformers_objects.py │ ├── dummy_torch_and_transformers_and_onnx_objects.py │ └── __init__.py ├── schedulers │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── __init__.cpython-38.pyc │ │ ├── scheduling_ddim.cpython-38.pyc │ │ ├── scheduling_ddpm.cpython-38.pyc │ │ ├── scheduling_pndm.cpython-38.pyc │ │ ├── scheduling_ddim.cpython-310.pyc │ │ ├── scheduling_ddpm.cpython-310.pyc │ │ ├── scheduling_pndm.cpython-310.pyc │ │ ├── scheduling_utils.cpython-310.pyc │ │ ├── scheduling_utils.cpython-38.pyc │ │ ├── scheduling_deis_multistep.cpython-38.pyc │ │ ├── scheduling_dpmsolver_sde.cpython-310.pyc │ │ ├── scheduling_euler_discrete.cpython-38.pyc │ │ ├── scheduling_heun_discrete.cpython-310.pyc │ │ ├── scheduling_heun_discrete.cpython-38.pyc │ │ ├── scheduling_lms_discrete.cpython-310.pyc │ │ ├── scheduling_lms_discrete.cpython-38.pyc │ │ ├── scheduling_deis_multistep.cpython-310.pyc │ │ ├── scheduling_euler_discrete.cpython-310.pyc │ │ ├── scheduling_k_dpm_2_discrete.cpython-38.pyc │ │ ├── scheduling_unipc_multistep.cpython-310.pyc │ │ ├── scheduling_unipc_multistep.cpython-38.pyc │ │ ├── scheduling_dpmsolver_multistep.cpython-38.pyc │ │ ├── scheduling_k_dpm_2_discrete.cpython-310.pyc │ │ ├── scheduling_dpmsolver_multistep.cpython-310.pyc │ │ ├── scheduling_dpmsolver_singlestep.cpython-310.pyc │ │ ├── scheduling_dpmsolver_singlestep.cpython-38.pyc │ │ ├── scheduling_euler_ancestral_discrete.cpython-38.pyc │ │ ├── scheduling_euler_ancestral_discrete.cpython-310.pyc │ │ ├── scheduling_k_dpm_2_ancestral_discrete.cpython-38.pyc │ │ └── scheduling_k_dpm_2_ancestral_discrete.cpython-310.pyc │ └── README.md ├── commands │ ├── __init__.py │ ├── diffusers_cli.py │ └── env.py ├── dependency_versions_check.py └── dependency_versions_table.py ├── models └── models.md ├── ssr_encoder ├── __init__.py └── .DS_Store ├── .DS_Store ├── utils └── .DS_Store ├── test_img ├── .DS_Store ├── t2i │ ├── 1.jpg │ ├── 2.jpg │ └── 3.jpg ├── mask │ ├── 4.jpg │ └── girl.png ├── multi │ ├── 1.jpg │ ├── 2.jpg │ └── 3.jpg ├── animate │ └── 1.jpeg └── controlnet │ ├── 1.jpg │ └── dog_depth.png ├── example_results ├── t2i.jpg ├── multi.jpg ├── control.jpg ├── animation.gif └── mask_girl.jpg ├── example_data.jsonl ├── requirements.txt ├── train.sh ├── infer_text.py ├── infer_mask.py ├── gradio_demo.py ├── infer_multi.py ├── infer_animate.py ├── infer_controlnet.py └── README.md /assets/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /diffusers/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/models.md: -------------------------------------------------------------------------------- 1 | Put your models in this folder. 2 | -------------------------------------------------------------------------------- /ssr_encoder/__init__.py: -------------------------------------------------------------------------------- 1 | from .ssr_encoder import SSR_encoder -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/.DS_Store -------------------------------------------------------------------------------- /diffusers/experimental/__init__.py: -------------------------------------------------------------------------------- 1 | from .rl import ValueGuidedRLPipeline 2 | -------------------------------------------------------------------------------- /assets/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/assets/2.png -------------------------------------------------------------------------------- /assets/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/assets/3.jpg -------------------------------------------------------------------------------- /assets/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/assets/4.jpg -------------------------------------------------------------------------------- /assets/5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/assets/5.jpg -------------------------------------------------------------------------------- /utils/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/utils/.DS_Store -------------------------------------------------------------------------------- /assets/method.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/assets/method.png -------------------------------------------------------------------------------- /assets/teaser.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/assets/teaser.jpg -------------------------------------------------------------------------------- /test_img/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/.DS_Store -------------------------------------------------------------------------------- /test_img/t2i/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/t2i/1.jpg -------------------------------------------------------------------------------- /test_img/t2i/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/t2i/2.jpg -------------------------------------------------------------------------------- /test_img/t2i/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/t2i/3.jpg -------------------------------------------------------------------------------- /diffusers/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/.DS_Store -------------------------------------------------------------------------------- /diffusers/experimental/rl/__init__.py: -------------------------------------------------------------------------------- 1 | from .value_guided_sampling import ValueGuidedRLPipeline 2 | -------------------------------------------------------------------------------- /test_img/mask/4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/mask/4.jpg -------------------------------------------------------------------------------- /test_img/multi/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/multi/1.jpg -------------------------------------------------------------------------------- /test_img/multi/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/multi/2.jpg -------------------------------------------------------------------------------- /test_img/multi/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/multi/3.jpg -------------------------------------------------------------------------------- /example_results/t2i.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/example_results/t2i.jpg -------------------------------------------------------------------------------- /ssr_encoder/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/ssr_encoder/.DS_Store -------------------------------------------------------------------------------- /test_img/animate/1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/animate/1.jpeg -------------------------------------------------------------------------------- /test_img/mask/girl.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/mask/girl.png -------------------------------------------------------------------------------- /example_results/multi.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/example_results/multi.jpg -------------------------------------------------------------------------------- /test_img/controlnet/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/controlnet/1.jpg -------------------------------------------------------------------------------- /example_results/control.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/example_results/control.jpg -------------------------------------------------------------------------------- /diffusers/pipelines/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/.DS_Store -------------------------------------------------------------------------------- /example_results/animation.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/example_results/animation.gif -------------------------------------------------------------------------------- /example_results/mask_girl.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/example_results/mask_girl.jpg -------------------------------------------------------------------------------- /test_img/controlnet/dog_depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/test_img/controlnet/dog_depth.png -------------------------------------------------------------------------------- /diffusers/models/__pycache__/lora.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/lora.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/vae.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/vae.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/vae.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/vae.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/lora.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/lora.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/resnet.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/resnet.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/logging.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/logging.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/outputs.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/outputs.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/README.md: -------------------------------------------------------------------------------- 1 | # Models 2 | 3 | For more detail on the models, please refer to the [docs](https://huggingface.co/docs/diffusers/api/models/overview). -------------------------------------------------------------------------------- /diffusers/models/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/attention.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/attention.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/resnet.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/resnet.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/constants.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/constants.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/constants.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/constants.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/doc_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/doc_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/doc_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/doc_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/hub_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/hub_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/hub_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/hub_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/logging.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/logging.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/outputs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/outputs.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/peft_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/peft_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/pil_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/pil_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/pil_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/pil_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/versions.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/versions.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/versions.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/versions.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/activations.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/activations.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/attention.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/attention.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/controlnet.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/controlnet.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/embeddings.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/embeddings.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/embeddings.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/embeddings.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/export_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/export_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/import_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/import_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/peft_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/peft_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/torch_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/torch_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/torch_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/torch_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/activations.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/activations.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/autoencoder_kl.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/autoencoder_kl.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/modeling_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/modeling_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/normalization.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/normalization.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/normalization.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/normalization.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/transformer_2d.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/transformer_2d.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/unet_2d_blocks.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/unet_2d_blocks.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/__pycache__/onnx_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/__pycache__/onnx_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/__pycache__/onnx_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/__pycache__/onnx_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/export_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/export_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/import_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/import_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/loading_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/loading_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/loading_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/loading_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/autoencoder_kl.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/autoencoder_kl.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/modeling_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/modeling_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/transformer_2d.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/transformer_2d.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/unet_2d_blocks.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/unet_2d_blocks.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/unet_3d_blocks.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/unet_3d_blocks.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/accelerate_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/accelerate_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/accelerate_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/accelerate_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/deprecation_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/deprecation_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/state_dict_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/state_dict_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/state_dict_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/state_dict_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/attention_processor.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/attention_processor.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/autoencoder_asym_kl.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/autoencoder_asym_kl.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/dual_transformer_2d.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/dual_transformer_2d.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/unet_2d_condition.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/unet_2d_condition.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/unet_2d_condition.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/unet_2d_condition.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/unet_3d_condition.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/unet_3d_condition.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/unet_motion_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/unet_motion_model.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/__pycache__/pipeline_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/__pycache__/pipeline_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/__pycache__/pipeline_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/__pycache__/pipeline_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/ddim/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/ddim/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/README.md: -------------------------------------------------------------------------------- 1 | # Schedulers 2 | 3 | For more information on the schedulers, please refer to the [docs](https://huggingface.co/docs/diffusers/api/schedulers/overview). -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_ddim.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_ddim.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_ddpm.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_ddpm.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_pndm.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_pndm.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/deprecation_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/deprecation_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_flax_objects.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_flax_objects.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_flax_objects.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_flax_objects.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_onnx_objects.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_onnx_objects.cpython-310.pyc -------------------------------------------------------------------------------- /example_data.jsonl: -------------------------------------------------------------------------------- 1 | {"file_name": "./data/sample/3400.png", "text": ["purple allium flowers"]} 2 | {"file_name": "./data/sample/3401.png", "text": ["a girl wearing a red dress"]} 3 | -------------------------------------------------------------------------------- /diffusers/models/__pycache__/attention_processor.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/attention_processor.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/autoencoder_asym_kl.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/autoencoder_asym_kl.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/dual_transformer_2d.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/dual_transformer_2d.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/models/__pycache__/transformer_temporal.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/models/__pycache__/transformer_temporal.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_ddim.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_ddim.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_ddpm.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_ddpm.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_pndm.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_pndm.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_note_seq_objects.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_note_seq_objects.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dynamic_modules_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dynamic_modules_utils.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dynamic_modules_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dynamic_modules_utils.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/animatediff/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/animatediff/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/controlnet/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/controlnet/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/ddim/__pycache__/pipeline_ddim.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/ddim/__pycache__/pipeline_ddim.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_note_seq_objects.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_note_seq_objects.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_deis_multistep.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_deis_multistep.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_dpmsolver_sde.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_dpmsolver_sde.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_euler_discrete.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_euler_discrete.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_heun_discrete.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_heun_discrete.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_heun_discrete.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_heun_discrete.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_lms_discrete.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_lms_discrete.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_lms_discrete.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_lms_discrete.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/controlnet/__pycache__/multicontrolnet.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/controlnet/__pycache__/multicontrolnet.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_deis_multistep.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_deis_multistep.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_euler_discrete.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_euler_discrete.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_k_dpm_2_discrete.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_k_dpm_2_discrete.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_unipc_multistep.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_unipc_multistep.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_unipc_multistep.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_unipc_multistep.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_torch_and_librosa_objects.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_torch_and_librosa_objects.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_torch_and_librosa_objects.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_torch_and_librosa_objects.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_torch_and_torchsde_objects.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_torch_and_torchsde_objects.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/__pycache__/safety_checker.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion/__pycache__/safety_checker.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion_xl/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion_xl/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_dpmsolver_multistep.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_dpmsolver_multistep.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_k_dpm_2_discrete.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_k_dpm_2_discrete.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_flax_and_transformers_objects.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_flax_and_transformers_objects.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_torch_and_torchsde_objects.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_torch_and_torchsde_objects.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/animatediff/__pycache__/pipeline_animatediff.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/animatediff/__pycache__/pipeline_animatediff.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/__pycache__/pipeline_output.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion/__pycache__/pipeline_output.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/__pycache__/pipeline_output.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion/__pycache__/pipeline_output.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/__pycache__/safety_checker.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion/__pycache__/safety_checker.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_dpmsolver_multistep.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_dpmsolver_multistep.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_dpmsolver_singlestep.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_dpmsolver_singlestep.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_dpmsolver_singlestep.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_dpmsolver_singlestep.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_flax_and_transformers_objects.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_flax_and_transformers_objects.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion_xl/__pycache__/pipeline_output.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion_xl/__pycache__/pipeline_output.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_euler_ancestral_discrete.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_euler_ancestral_discrete.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_euler_ancestral_discrete.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_euler_ancestral_discrete.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_k_dpm_2_ancestral_discrete.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_k_dpm_2_ancestral_discrete.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/schedulers/__pycache__/scheduling_k_dpm_2_ancestral_discrete.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/schedulers/__pycache__/scheduling_k_dpm_2_ancestral_discrete.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/__pycache__/pipeline_stable_diffusion.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion/__pycache__/pipeline_stable_diffusion.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_torch_and_transformers_and_onnx_objects.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_torch_and_transformers_and_onnx_objects.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_transformers_and_torch_and_note_seq_objects.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_transformers_and_torch_and_note_seq_objects.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_transformers_and_torch_and_note_seq_objects.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_transformers_and_torch_and_note_seq_objects.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_torch_and_transformers_and_k_diffusion_objects.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_torch_and_transformers_and_k_diffusion_objects.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/__pycache__/pipeline_stable_diffusion_inpaint.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion/__pycache__/pipeline_stable_diffusion_inpaint.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/__pycache__/pipeline_stable_diffusion_inpaint.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion/__pycache__/pipeline_stable_diffusion_inpaint.cpython-38.pyc -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion_xl/__pycache__/pipeline_stable_diffusion_xl.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/pipelines/stable_diffusion_xl/__pycache__/pipeline_stable_diffusion_xl.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/utils/__pycache__/dummy_torch_and_transformers_and_k_diffusion_objects.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Xiaojiu-z/SSR_Encoder/HEAD/diffusers/utils/__pycache__/dummy_torch_and_transformers_and_k_diffusion_objects.cpython-310.pyc -------------------------------------------------------------------------------- /diffusers/experimental/README.md: -------------------------------------------------------------------------------- 1 | # 🧨 Diffusers Experimental 2 | 3 | We are adding experimental code to support novel applications and usages of the Diffusers library. 4 | Currently, the following experiments are supported: 5 | * Reinforcement learning via an implementation of the [Diffuser](https://arxiv.org/abs/2205.09991) model. -------------------------------------------------------------------------------- /diffusers/pipelines/ddim/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_ddim": ["DDIMPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_ddim import DDIMPipeline 10 | else: 11 | import sys 12 | 13 | sys.modules[__name__] = _LazyModule( 14 | __name__, 15 | globals()["__file__"], 16 | _import_structure, 17 | module_spec=__spec__, 18 | ) 19 | -------------------------------------------------------------------------------- /diffusers/pipelines/pndm/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_pndm": ["PNDMPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_pndm import PNDMPipeline 10 | else: 11 | import sys 12 | 13 | sys.modules[__name__] = _LazyModule( 14 | __name__, 15 | globals()["__file__"], 16 | _import_structure, 17 | module_spec=__spec__, 18 | ) 19 | -------------------------------------------------------------------------------- /diffusers/pipelines/dit/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_dit": ["DiTPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_dit import DiTPipeline 10 | 11 | else: 12 | import sys 13 | 14 | sys.modules[__name__] = _LazyModule( 15 | __name__, 16 | globals()["__file__"], 17 | _import_structure, 18 | module_spec=__spec__, 19 | ) 20 | -------------------------------------------------------------------------------- /diffusers/pipelines/repaint/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_repaint": ["RePaintPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_repaint import RePaintPipeline 10 | 11 | else: 12 | import sys 13 | 14 | sys.modules[__name__] = _LazyModule( 15 | __name__, 16 | globals()["__file__"], 17 | _import_structure, 18 | module_spec=__spec__, 19 | ) 20 | -------------------------------------------------------------------------------- /diffusers/pipelines/ddpm/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | _LazyModule, 6 | ) 7 | 8 | 9 | _import_structure = {"pipeline_ddpm": ["DDPMPipeline"]} 10 | 11 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 12 | from .pipeline_ddpm import DDPMPipeline 13 | 14 | else: 15 | import sys 16 | 17 | sys.modules[__name__] = _LazyModule( 18 | __name__, 19 | globals()["__file__"], 20 | _import_structure, 21 | module_spec=__spec__, 22 | ) 23 | -------------------------------------------------------------------------------- /diffusers/pipelines/score_sde_ve/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_score_sde_ve": ["ScoreSdeVePipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_score_sde_ve import ScoreSdeVePipeline 10 | 11 | else: 12 | import sys 13 | 14 | sys.modules[__name__] = _LazyModule( 15 | __name__, 16 | globals()["__file__"], 17 | _import_structure, 18 | module_spec=__spec__, 19 | ) 20 | -------------------------------------------------------------------------------- /diffusers/pipelines/dance_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_dance_diffusion": ["DanceDiffusionPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_dance_diffusion import DanceDiffusionPipeline 10 | else: 11 | import sys 12 | 13 | sys.modules[__name__] = _LazyModule( 14 | __name__, 15 | globals()["__file__"], 16 | _import_structure, 17 | module_spec=__spec__, 18 | ) 19 | -------------------------------------------------------------------------------- /diffusers/pipelines/latent_diffusion_uncond/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_latent_diffusion_uncond": ["LDMPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_latent_diffusion_uncond import LDMPipeline 10 | else: 11 | import sys 12 | 13 | sys.modules[__name__] = _LazyModule( 14 | __name__, 15 | globals()["__file__"], 16 | _import_structure, 17 | module_spec=__spec__, 18 | ) 19 | -------------------------------------------------------------------------------- /diffusers/pipelines/stochastic_karras_ve/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_stochastic_karras_ve": ["KarrasVePipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_stochastic_karras_ve import KarrasVePipeline 10 | 11 | else: 12 | import sys 13 | 14 | sys.modules[__name__] = _LazyModule( 15 | __name__, 16 | globals()["__file__"], 17 | _import_structure, 18 | module_spec=__spec__, 19 | ) 20 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate==0.23.0 2 | albumentations==1.3.1 3 | av==11.0.0 4 | datasets==2.14.6 5 | diffusers==0.25.1 6 | docopt==0.6.2 7 | easydict==1.11 8 | einops==0.7.0 9 | huggingface-hub==0.20.3 10 | imageio==2.31.5 11 | imageio-ffmpeg==0.4.9 12 | importlib-metadata==6.8.0 13 | matplotlib==3.8.0 14 | matplotlib-inline==0.1.6 15 | mdit-py-plugins==0.3.3 16 | numpy==1.26.0 17 | omegaconf==2.3.0 18 | opencv-python==4.8.1.78 19 | Pillow==10.0.1 20 | pytorch-lightning==2.1.1 21 | safetensors==0.3.3 22 | scikit-image==0.22.0 23 | scikit-learn==1.4.0 24 | scipy==1.11.3 25 | transformers==4.34.0 -------------------------------------------------------------------------------- /diffusers/utils/dummy_onnx_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class OnnxRuntimeModel(metaclass=DummyObject): 6 | _backends = ["onnx"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["onnx"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["onnx"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["onnx"]) 18 | -------------------------------------------------------------------------------- /diffusers/utils/dummy_note_seq_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class MidiProcessor(metaclass=DummyObject): 6 | _backends = ["note_seq"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["note_seq"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["note_seq"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["note_seq"]) 18 | -------------------------------------------------------------------------------- /diffusers/pipelines/consistency_models/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | _LazyModule, 6 | ) 7 | 8 | 9 | _import_structure = { 10 | "pipeline_consistency_models": ["ConsistencyModelPipeline"], 11 | } 12 | 13 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 14 | from .pipeline_consistency_models import ConsistencyModelPipeline 15 | 16 | else: 17 | import sys 18 | 19 | sys.modules[__name__] = _LazyModule( 20 | __name__, 21 | globals()["__file__"], 22 | _import_structure, 23 | module_spec=__spec__, 24 | ) 25 | -------------------------------------------------------------------------------- /diffusers/pipelines/audio_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = { 7 | "mel": ["Mel"], 8 | "pipeline_audio_diffusion": ["AudioDiffusionPipeline"], 9 | } 10 | 11 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 12 | from .mel import Mel 13 | from .pipeline_audio_diffusion import AudioDiffusionPipeline 14 | 15 | else: 16 | import sys 17 | 18 | sys.modules[__name__] = _LazyModule( 19 | __name__, 20 | globals()["__file__"], 21 | _import_structure, 22 | module_spec=__spec__, 23 | ) 24 | -------------------------------------------------------------------------------- /diffusers/utils/dummy_torch_and_scipy_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class LMSDiscreteScheduler(metaclass=DummyObject): 6 | _backends = ["torch", "scipy"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "scipy"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "scipy"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "scipy"]) 18 | -------------------------------------------------------------------------------- /diffusers/utils/dummy_torch_and_torchsde_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class DPMSolverSDEScheduler(metaclass=DummyObject): 6 | _backends = ["torch", "torchsde"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "torchsde"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "torchsde"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "torchsde"]) 18 | -------------------------------------------------------------------------------- /diffusers/utils/dummy_transformers_and_torch_and_note_seq_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class SpectrogramDiffusionPipeline(metaclass=DummyObject): 6 | _backends = ["transformers", "torch", "note_seq"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["transformers", "torch", "note_seq"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["transformers", "torch", "note_seq"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["transformers", "torch", "note_seq"]) 18 | -------------------------------------------------------------------------------- /diffusers/pipelines/text_to_video_synthesis/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Union 3 | 4 | import numpy as np 5 | import torch 6 | 7 | from ...utils import ( 8 | BaseOutput, 9 | ) 10 | 11 | 12 | @dataclass 13 | class TextToVideoSDPipelineOutput(BaseOutput): 14 | """ 15 | Output class for text-to-video pipelines. 16 | 17 | Args: 18 | frames (`List[np.ndarray]` or `torch.FloatTensor`) 19 | List of denoised frames (essentially images) as NumPy arrays of shape `(height, width, num_channels)` or as 20 | a `torch` tensor. The length of the list denotes the video length (the number of frames). 21 | """ 22 | 23 | frames: Union[List[np.ndarray], torch.FloatTensor] 24 | -------------------------------------------------------------------------------- /diffusers/utils/dummy_torch_and_transformers_and_k_diffusion_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class StableDiffusionKDiffusionPipeline(metaclass=DummyObject): 6 | _backends = ["torch", "transformers", "k_diffusion"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "transformers", "k_diffusion"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "transformers", "k_diffusion"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "transformers", "k_diffusion"]) 18 | -------------------------------------------------------------------------------- /diffusers/pipelines/blip_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL 6 | from PIL import Image 7 | 8 | from ...utils import OptionalDependencyNotAvailable, is_torch_available, is_transformers_available 9 | 10 | 11 | try: 12 | if not (is_transformers_available() and is_torch_available()): 13 | raise OptionalDependencyNotAvailable() 14 | except OptionalDependencyNotAvailable: 15 | from ...utils.dummy_torch_and_transformers_objects import ShapEPipeline 16 | else: 17 | from .blip_image_processing import BlipImageProcessor 18 | from .modeling_blip2 import Blip2QFormerModel 19 | from .modeling_ctx_clip import ContextCLIPTextModel 20 | from .pipeline_blip_diffusion import BlipDiffusionPipeline 21 | -------------------------------------------------------------------------------- /diffusers/pipelines/semantic_stable_diffusion/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput 8 | 9 | 10 | @dataclass 11 | class SemanticStableDiffusionPipelineOutput(BaseOutput): 12 | """ 13 | Output class for Stable Diffusion pipelines. 14 | 15 | Args: 16 | images (`List[PIL.Image.Image]` or `np.ndarray`) 17 | List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, 18 | num_channels)`. 19 | nsfw_content_detected (`List[bool]`) 20 | List indicating whether the corresponding generated image contains “not-safe-for-work” (nsfw) content or 21 | `None` if safety checking could not be performed. 22 | """ 23 | 24 | images: Union[List[PIL.Image.Image], np.ndarray] 25 | nsfw_content_detected: Optional[List[bool]] 26 | -------------------------------------------------------------------------------- /train.sh: -------------------------------------------------------------------------------- 1 | export MODEL_DIR="" # your SD path 2 | export OUTPUT_DIR="SSR_model" # your save path 3 | cd SSR # cd to your path 4 | 5 | accelerate launch train.py \ 6 | --pretrained_model_name_or_path $MODEL_DIR \ 7 | --reg=0.001 \ # 0.001 is good for fine-tuning your model 8 | --image_column="file_name" \ # image path 9 | --caption_column="text" \ # caption 10 | --tag_column="text" \ # caption 11 | --output_dir=$OUTPUT_DIR \ 12 | --logging_dir=$LOG_PATH \ 13 | --train_data_dir "./example_data.jsonl" \ # you can use josnl file to save your caption and image paths or make dataset/dataloader by yourself 14 | --resolution=512 \ 15 | --learning_rate=5e-5 \ 16 | --train_batch_size=24 \ 17 | --num_validation=2 \ 18 | --validation_image './test_img/t2i/1.jpg' \ 19 | --validation_prompts "a girl" "a cat" \ 20 | --gradient_accumulation_steps=1 \ 21 | --num_train_epochs=10000 \ 22 | --validation_steps=10000 \ 23 | --checkpointing_steps=10000 -------------------------------------------------------------------------------- /diffusers/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from abc import ABC, abstractmethod 16 | from argparse import ArgumentParser 17 | 18 | 19 | class BaseDiffusersCLICommand(ABC): 20 | @staticmethod 21 | @abstractmethod 22 | def register_subcommand(parser: ArgumentParser): 23 | raise NotImplementedError() 24 | 25 | @abstractmethod 26 | def run(self): 27 | raise NotImplementedError() 28 | -------------------------------------------------------------------------------- /diffusers/pipelines/alt_diffusion/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import ( 8 | BaseOutput, 9 | ) 10 | 11 | 12 | @dataclass 13 | # Copied from diffusers.pipelines.stable_diffusion.pipeline_output.StableDiffusionPipelineOutput with Stable->Alt 14 | class AltDiffusionPipelineOutput(BaseOutput): 15 | """ 16 | Output class for Alt Diffusion pipelines. 17 | 18 | Args: 19 | images (`List[PIL.Image.Image]` or `np.ndarray`) 20 | List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, 21 | num_channels)`. 22 | nsfw_content_detected (`List[bool]`) 23 | List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or 24 | `None` if safety checking could not be performed. 25 | """ 26 | 27 | images: Union[List[PIL.Image.Image], np.ndarray] 28 | nsfw_content_detected: Optional[List[bool]] 29 | -------------------------------------------------------------------------------- /diffusers/utils/dummy_torch_and_librosa_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class AudioDiffusionPipeline(metaclass=DummyObject): 6 | _backends = ["torch", "librosa"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "librosa"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "librosa"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "librosa"]) 18 | 19 | 20 | class Mel(metaclass=DummyObject): 21 | _backends = ["torch", "librosa"] 22 | 23 | def __init__(self, *args, **kwargs): 24 | requires_backends(self, ["torch", "librosa"]) 25 | 26 | @classmethod 27 | def from_config(cls, *args, **kwargs): 28 | requires_backends(cls, ["torch", "librosa"]) 29 | 30 | @classmethod 31 | def from_pretrained(cls, *args, **kwargs): 32 | requires_backends(cls, ["torch", "librosa"]) 33 | -------------------------------------------------------------------------------- /diffusers/pipelines/kandinsky/text_encoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel 3 | 4 | 5 | class MCLIPConfig(XLMRobertaConfig): 6 | model_type = "M-CLIP" 7 | 8 | def __init__(self, transformerDimSize=1024, imageDimSize=768, **kwargs): 9 | self.transformerDimensions = transformerDimSize 10 | self.numDims = imageDimSize 11 | super().__init__(**kwargs) 12 | 13 | 14 | class MultilingualCLIP(PreTrainedModel): 15 | config_class = MCLIPConfig 16 | 17 | def __init__(self, config, *args, **kwargs): 18 | super().__init__(config, *args, **kwargs) 19 | self.transformer = XLMRobertaModel(config) 20 | self.LinearTransformation = torch.nn.Linear( 21 | in_features=config.transformerDimensions, out_features=config.numDims 22 | ) 23 | 24 | def forward(self, input_ids, attention_mask): 25 | embs = self.transformer(input_ids=input_ids, attention_mask=attention_mask)[0] 26 | embs2 = (embs * attention_mask.unsqueeze(2)).sum(dim=1) / attention_mask.sum(dim=1)[:, None] 27 | return self.LinearTransformation(embs2), embs 28 | -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion_xl/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput, is_flax_available 8 | 9 | 10 | @dataclass 11 | class StableDiffusionXLPipelineOutput(BaseOutput): 12 | """ 13 | Output class for Stable Diffusion pipelines. 14 | 15 | Args: 16 | images (`List[PIL.Image.Image]` or `np.ndarray`) 17 | List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, 18 | num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. 19 | """ 20 | 21 | images: Union[List[PIL.Image.Image], np.ndarray] 22 | 23 | 24 | if is_flax_available(): 25 | import flax 26 | 27 | @flax.struct.dataclass 28 | class FlaxStableDiffusionXLPipelineOutput(BaseOutput): 29 | """ 30 | Output class for Flax Stable Diffusion XL pipelines. 31 | 32 | Args: 33 | images (`np.ndarray`) 34 | Array of shape `(batch_size, height, width, num_channels)` with images from the diffusion pipeline. 35 | """ 36 | 37 | images: np.ndarray 38 | -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/clip_image_project_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The GLIGEN Authors and HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from torch import nn 16 | 17 | from ...configuration_utils import ConfigMixin, register_to_config 18 | from ...models.modeling_utils import ModelMixin 19 | 20 | 21 | class CLIPImageProjection(ModelMixin, ConfigMixin): 22 | @register_to_config 23 | def __init__(self, hidden_size: int = 768): 24 | super().__init__() 25 | self.hidden_size = hidden_size 26 | self.project = nn.Linear(self.hidden_size, self.hidden_size, bias=False) 27 | 28 | def forward(self, x): 29 | return self.project(x) 30 | -------------------------------------------------------------------------------- /diffusers/pipelines/deepfloyd_if/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput 8 | 9 | 10 | @dataclass 11 | class IFPipelineOutput(BaseOutput): 12 | """ 13 | Args: 14 | Output class for Stable Diffusion pipelines. 15 | images (`List[PIL.Image.Image]` or `np.ndarray`) 16 | List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, 17 | num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. 18 | nsfw_detected (`List[bool]`) 19 | List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" 20 | (nsfw) content or a watermark. `None` if safety checking could not be performed. 21 | watermark_detected (`List[bool]`) 22 | List of flags denoting whether the corresponding generated image likely has a watermark. `None` if safety 23 | checking could not be performed. 24 | """ 25 | 26 | images: Union[List[PIL.Image.Image], np.ndarray] 27 | nsfw_detected: Optional[List[bool]] 28 | watermark_detected: Optional[List[bool]] 29 | -------------------------------------------------------------------------------- /diffusers/utils/loading_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Union 3 | 4 | import PIL.Image 5 | import PIL.ImageOps 6 | import requests 7 | 8 | 9 | def load_image(image: Union[str, PIL.Image.Image]) -> PIL.Image.Image: 10 | """ 11 | Loads `image` to a PIL Image. 12 | 13 | Args: 14 | image (`str` or `PIL.Image.Image`): 15 | The image to convert to the PIL Image format. 16 | Returns: 17 | `PIL.Image.Image`: 18 | A PIL Image. 19 | """ 20 | if isinstance(image, str): 21 | if image.startswith("http://") or image.startswith("https://"): 22 | image = PIL.Image.open(requests.get(image, stream=True).raw) 23 | elif os.path.isfile(image): 24 | image = PIL.Image.open(image) 25 | else: 26 | raise ValueError( 27 | f"Incorrect path or url, URLs must start with `http://` or `https://`, and {image} is not a valid path" 28 | ) 29 | elif isinstance(image, PIL.Image.Image): 30 | image = image 31 | else: 32 | raise ValueError( 33 | "Incorrect format used for image. Should be an url linking to an image, a local path, or a PIL image." 34 | ) 35 | image = PIL.ImageOps.exif_transpose(image) 36 | image = image.convert("RGB") 37 | return image 38 | -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion_xl/watermark.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | from ...utils import is_invisible_watermark_available 5 | 6 | 7 | if is_invisible_watermark_available(): 8 | from imwatermark import WatermarkEncoder 9 | 10 | 11 | # Copied from https://github.com/Stability-AI/generative-models/blob/613af104c6b85184091d42d374fef420eddb356d/scripts/demo/streamlit_helpers.py#L66 12 | WATERMARK_MESSAGE = 0b101100111110110010010000011110111011000110011110 13 | # bin(x)[2:] gives bits of x as str, use int to convert them to 0/1 14 | WATERMARK_BITS = [int(bit) for bit in bin(WATERMARK_MESSAGE)[2:]] 15 | 16 | 17 | class StableDiffusionXLWatermarker: 18 | def __init__(self): 19 | self.watermark = WATERMARK_BITS 20 | self.encoder = WatermarkEncoder() 21 | 22 | self.encoder.set_watermark("bits", self.watermark) 23 | 24 | def apply_watermark(self, images: torch.FloatTensor): 25 | # can't encode images that are smaller than 256 26 | if images.shape[-1] < 256: 27 | return images 28 | 29 | images = (255 * (images / 2 + 0.5)).cpu().permute(0, 2, 3, 1).float().numpy() 30 | 31 | images = [self.encoder.encode(image, "dwtDct") for image in images] 32 | 33 | images = torch.from_numpy(np.array(images)).permute(0, 3, 1, 2) 34 | 35 | images = torch.clamp(2 * (images / 255 - 0.5), min=-1.0, max=1.0) 36 | return images 37 | -------------------------------------------------------------------------------- /diffusers/dependency_versions_check.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import sys 15 | 16 | from .dependency_versions_table import deps 17 | from .utils.versions import require_version, require_version_core 18 | 19 | 20 | # define which module versions we always want to check at run time 21 | # (usually the ones defined in `install_requires` in setup.py) 22 | # 23 | # order specific notes: 24 | # - tqdm must be checked before tokenizers 25 | 26 | pkgs_to_check_at_runtime = "python requests filelock numpy".split() 27 | for pkg in pkgs_to_check_at_runtime: 28 | if pkg in deps: 29 | require_version_core(deps[pkg]) 30 | else: 31 | raise ValueError(f"can't find {pkg} in {deps.keys()}, check dependency_versions_table.py") 32 | 33 | 34 | def dep_version_check(pkg, hint=None): 35 | require_version(deps[pkg], hint) 36 | -------------------------------------------------------------------------------- /diffusers/commands/diffusers_cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2023 The HuggingFace Team. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from argparse import ArgumentParser 17 | 18 | from .env import EnvironmentCommand 19 | from .fp16_safetensors import FP16SafetensorsCommand 20 | 21 | 22 | def main(): 23 | parser = ArgumentParser("Diffusers CLI tool", usage="diffusers-cli []") 24 | commands_parser = parser.add_subparsers(help="diffusers-cli command helpers") 25 | 26 | # Register commands 27 | EnvironmentCommand.register_subcommand(commands_parser) 28 | FP16SafetensorsCommand.register_subcommand(commands_parser) 29 | 30 | # Let's go 31 | args = parser.parse_args() 32 | 33 | if not hasattr(args, "func"): 34 | parser.print_help() 35 | exit(1) 36 | 37 | # Run 38 | service = args.func(args) 39 | service.run() 40 | 41 | 42 | if __name__ == "__main__": 43 | main() 44 | -------------------------------------------------------------------------------- /diffusers/utils/doc_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Doc utilities: Utilities related to documentation 16 | """ 17 | import re 18 | 19 | 20 | def replace_example_docstring(example_docstring): 21 | def docstring_decorator(fn): 22 | func_doc = fn.__doc__ 23 | lines = func_doc.split("\n") 24 | i = 0 25 | while i < len(lines) and re.search(r"^\s*Examples?:\s*$", lines[i]) is None: 26 | i += 1 27 | if i < len(lines): 28 | lines[i] = example_docstring 29 | func_doc = "\n".join(lines) 30 | else: 31 | raise ValueError( 32 | f"The function {fn} should have an empty 'Examples:' in its docstring as placeholder, " 33 | f"current docstring is:\n{func_doc}" 34 | ) 35 | fn.__doc__ = func_doc 36 | return fn 37 | 38 | return docstring_decorator 39 | -------------------------------------------------------------------------------- /diffusers/pipelines/pixart_alpha/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_pixart_alpha"] = ["PixArtAlphaPipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_pixart_alpha import PixArtAlphaPipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /diffusers/pipelines/animatediff/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_animatediff"] = ["AnimateDiffPipeline", "AnimateDiffPipelineOutput"] 25 | 26 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 27 | try: 28 | if not (is_transformers_available() and is_torch_available()): 29 | raise OptionalDependencyNotAvailable() 30 | except OptionalDependencyNotAvailable: 31 | from ...utils.dummy_torch_and_transformers_objects import * 32 | 33 | else: 34 | from .pipeline_animatediff import AnimateDiffPipeline, AnimateDiffPipelineOutput 35 | 36 | else: 37 | import sys 38 | 39 | sys.modules[__name__] = _LazyModule( 40 | __name__, 41 | globals()["__file__"], 42 | _import_structure, 43 | module_spec=__spec__, 44 | ) 45 | for name, value in _dummy_objects.items(): 46 | setattr(sys.modules[__name__], name, value) 47 | -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion_safe/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import ( 8 | BaseOutput, 9 | ) 10 | 11 | 12 | @dataclass 13 | class StableDiffusionSafePipelineOutput(BaseOutput): 14 | """ 15 | Output class for Safe Stable Diffusion pipelines. 16 | 17 | Args: 18 | images (`List[PIL.Image.Image]` or `np.ndarray`) 19 | List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, 20 | num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. 21 | nsfw_content_detected (`List[bool]`) 22 | List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" 23 | (nsfw) content, or `None` if safety checking could not be performed. 24 | images (`List[PIL.Image.Image]` or `np.ndarray`) 25 | List of denoised PIL images that were flagged by the safety checker any may contain "not-safe-for-work" 26 | (nsfw) content, or `None` if no safety check was performed or no images were flagged. 27 | applied_safety_concept (`str`) 28 | The safety concept that was applied for safety guidance, or `None` if safety guidance was disabled 29 | """ 30 | 31 | images: Union[List[PIL.Image.Image], np.ndarray] 32 | nsfw_content_detected: Optional[List[bool]] 33 | unsafe_images: Optional[Union[List[PIL.Image.Image], np.ndarray]] 34 | applied_safety_concept: Optional[str] 35 | -------------------------------------------------------------------------------- /diffusers/pipelines/musicldm/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | is_transformers_version, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_musicldm"] = ["MusicLDMPipeline"] 26 | 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .pipeline_musicldm import MusicLDMPipeline 37 | 38 | else: 39 | import sys 40 | 41 | sys.modules[__name__] = _LazyModule( 42 | __name__, 43 | globals()["__file__"], 44 | _import_structure, 45 | module_spec=__spec__, 46 | ) 47 | 48 | for name, value in _dummy_objects.items(): 49 | setattr(sys.modules[__name__], name, value) 50 | -------------------------------------------------------------------------------- /diffusers/pipelines/audioldm/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | is_torch_available, 8 | is_transformers_available, 9 | is_transformers_version, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils.dummy_torch_and_transformers_objects import ( 21 | AudioLDMPipeline, 22 | ) 23 | 24 | _dummy_objects.update({"AudioLDMPipeline": AudioLDMPipeline}) 25 | else: 26 | _import_structure["pipeline_audioldm"] = ["AudioLDMPipeline"] 27 | 28 | 29 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 30 | try: 31 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 32 | raise OptionalDependencyNotAvailable() 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import ( 35 | AudioLDMPipeline, 36 | ) 37 | 38 | else: 39 | from .pipeline_audioldm import AudioLDMPipeline 40 | else: 41 | import sys 42 | 43 | sys.modules[__name__] = _LazyModule( 44 | __name__, 45 | globals()["__file__"], 46 | _import_structure, 47 | module_spec=__spec__, 48 | ) 49 | 50 | for name, value in _dummy_objects.items(): 51 | setattr(sys.modules[__name__], name, value) 52 | -------------------------------------------------------------------------------- /diffusers/pipelines/semantic_stable_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_output"] = ["SemanticStableDiffusionPipelineOutput"] 25 | _import_structure["pipeline_semantic_stable_diffusion"] = ["SemanticStableDiffusionPipeline"] 26 | 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .pipeline_semantic_stable_diffusion import SemanticStableDiffusionPipeline 37 | 38 | else: 39 | import sys 40 | 41 | sys.modules[__name__] = _LazyModule( 42 | __name__, 43 | globals()["__file__"], 44 | _import_structure, 45 | module_spec=__spec__, 46 | ) 47 | 48 | for name, value in _dummy_objects.items(): 49 | setattr(sys.modules[__name__], name, value) 50 | -------------------------------------------------------------------------------- /diffusers/utils/model_card_template.md: -------------------------------------------------------------------------------- 1 | --- 2 | {{ card_data }} 3 | --- 4 | 5 | 7 | 8 | # {{ model_name | default("Diffusion Model") }} 9 | 10 | ## Model description 11 | 12 | This diffusion model is trained with the [🤗 Diffusers](https://github.com/huggingface/diffusers) library 13 | on the `{{ dataset_name }}` dataset. 14 | 15 | ## Intended uses & limitations 16 | 17 | #### How to use 18 | 19 | ```python 20 | # TODO: add an example code snippet for running this diffusion pipeline 21 | ``` 22 | 23 | #### Limitations and bias 24 | 25 | [TODO: provide examples of latent issues and potential remediations] 26 | 27 | ## Training data 28 | 29 | [TODO: describe the data used to train the model] 30 | 31 | ### Training hyperparameters 32 | 33 | The following hyperparameters were used during training: 34 | - learning_rate: {{ learning_rate }} 35 | - train_batch_size: {{ train_batch_size }} 36 | - eval_batch_size: {{ eval_batch_size }} 37 | - gradient_accumulation_steps: {{ gradient_accumulation_steps }} 38 | - optimizer: AdamW with betas=({{ adam_beta1 }}, {{ adam_beta2 }}), weight_decay={{ adam_weight_decay }} and epsilon={{ adam_epsilon }} 39 | - lr_scheduler: {{ lr_scheduler }} 40 | - lr_warmup_steps: {{ lr_warmup_steps }} 41 | - ema_inv_gamma: {{ ema_inv_gamma }} 42 | - ema_inv_gamma: {{ ema_power }} 43 | - ema_inv_gamma: {{ ema_max_decay }} 44 | - mixed_precision: {{ mixed_precision }} 45 | 46 | ### Training results 47 | 48 | 📈 [TensorBoard logs](https://huggingface.co/{{ repo_name }}/tensorboard?#scalars) 49 | 50 | 51 | -------------------------------------------------------------------------------- /infer_text.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torch 3 | from diffusers import UniPCMultistepScheduler 4 | from utils.pipeline_t2i import StableDiffusionPipeline 5 | from ssr_encoder import SSR_encoder 6 | from diffusers.utils import load_image 7 | 8 | # Initialize the model 9 | base_model_path = "sd_model_v1-5" 10 | image_encoder_path = "models/image_encoder" 11 | base_ssr = "./models/ssr_model" 12 | ssr_ckpt = [base_ssr+"/pytorch_model.bin", 13 | base_ssr+"/pytorch_model_1.bin"] 14 | 15 | pipe = StableDiffusionPipeline.from_pretrained( 16 | base_model_path, 17 | safety_checker=None, 18 | torch_dtype=torch.float32).to("cuda") 19 | pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) 20 | ssr_model = SSR_encoder(pipe.unet, image_encoder_path, "cuda", dtype=torch.float32) 21 | ssr_model.get_pipe(pipe) 22 | ssr_model.load_SSR(ssr_ckpt[0], ssr_ckpt[1]) 23 | 24 | if __name__ == '__main__': 25 | 26 | # infer paras 27 | img_path = "./test_img/t2i/3.jpg" 28 | out_path = "./results" 29 | subject = "flower" 30 | scale = 0.65 # The recommended parameters are 0.5-0.8, Default value is 0.65 31 | prompts = "A girl holding flowers" 32 | negative_prompt = "bad quality" 33 | 34 | pil_img = load_image(img_path) 35 | images = ssr_model.generate( 36 | pil_image=pil_img, 37 | concept=subject, 38 | uncond_concept="", 39 | prompt=prompts, 40 | negative_prompt=negative_prompt, 41 | num_samples=1, 42 | seed=None, 43 | guidance_scale=5, 44 | scale=scale, 45 | num_inference_steps=30, 46 | height=512, 47 | width=512, 48 | )[0] 49 | images.save(os.path.join(out_path, "t2i.jpg")) -------------------------------------------------------------------------------- /diffusers/dependency_versions_table.py: -------------------------------------------------------------------------------- 1 | # THIS FILE HAS BEEN AUTOGENERATED. To update: 2 | # 1. modify the `_deps` dict in setup.py 3 | # 2. run `make deps_table_update`` 4 | deps = { 5 | "Pillow": "Pillow", 6 | "accelerate": "accelerate>=0.11.0", 7 | "compel": "compel==0.1.8", 8 | "black": "black~=23.1", 9 | "datasets": "datasets", 10 | "filelock": "filelock", 11 | "flax": "flax>=0.4.1", 12 | "hf-doc-builder": "hf-doc-builder>=0.3.0", 13 | "huggingface-hub": "huggingface-hub>=0.13.2", 14 | "requests-mock": "requests-mock==1.10.0", 15 | "importlib_metadata": "importlib_metadata", 16 | "invisible-watermark": "invisible-watermark>=0.2.0", 17 | "isort": "isort>=5.5.4", 18 | "jax": "jax>=0.4.1", 19 | "jaxlib": "jaxlib>=0.4.1", 20 | "Jinja2": "Jinja2", 21 | "k-diffusion": "k-diffusion>=0.0.12", 22 | "torchsde": "torchsde", 23 | "note_seq": "note_seq", 24 | "librosa": "librosa", 25 | "numpy": "numpy", 26 | "omegaconf": "omegaconf", 27 | "parameterized": "parameterized", 28 | "peft": "peft<=0.6.2", 29 | "protobuf": "protobuf>=3.20.3,<4", 30 | "pytest": "pytest", 31 | "pytest-timeout": "pytest-timeout", 32 | "pytest-xdist": "pytest-xdist", 33 | "python": "python>=3.8.0", 34 | "ruff": "ruff==0.0.280", 35 | "safetensors": "safetensors>=0.3.1", 36 | "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92", 37 | "scipy": "scipy", 38 | "onnx": "onnx", 39 | "regex": "regex!=2019.12.17", 40 | "requests": "requests", 41 | "tensorboard": "tensorboard", 42 | "torch": "torch>=1.4", 43 | "torchvision": "torchvision", 44 | "transformers": "transformers>=4.25.1", 45 | "urllib3": "urllib3<=2.0.0", 46 | } 47 | -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput, is_flax_available 8 | 9 | 10 | @dataclass 11 | class StableDiffusionPipelineOutput(BaseOutput): 12 | """ 13 | Output class for Stable Diffusion pipelines. 14 | 15 | Args: 16 | images (`List[PIL.Image.Image]` or `np.ndarray`) 17 | List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, 18 | num_channels)`. 19 | nsfw_content_detected (`List[bool]`) 20 | List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or 21 | `None` if safety checking could not be performed. 22 | """ 23 | 24 | images: Union[List[PIL.Image.Image], np.ndarray] 25 | nsfw_content_detected: Optional[List[bool]] 26 | 27 | 28 | if is_flax_available(): 29 | import flax 30 | 31 | @flax.struct.dataclass 32 | class FlaxStableDiffusionPipelineOutput(BaseOutput): 33 | """ 34 | Output class for Flax-based Stable Diffusion pipelines. 35 | 36 | Args: 37 | images (`np.ndarray`): 38 | Denoised images of array shape of `(batch_size, height, width, num_channels)`. 39 | nsfw_content_detected (`List[bool]`): 40 | List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content 41 | or `None` if safety checking could not be performed. 42 | """ 43 | 44 | images: np.ndarray 45 | nsfw_content_detected: List[bool] 46 | -------------------------------------------------------------------------------- /diffusers/pipelines/t2i_adapter/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_stable_diffusion_adapter"] = ["StableDiffusionAdapterPipeline"] 25 | _import_structure["pipeline_stable_diffusion_xl_adapter"] = ["StableDiffusionXLAdapterPipeline"] 26 | 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 34 | else: 35 | from .pipeline_stable_diffusion_adapter import StableDiffusionAdapterPipeline 36 | from .pipeline_stable_diffusion_xl_adapter import StableDiffusionXLAdapterPipeline 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | for name, value in _dummy_objects.items(): 47 | setattr(sys.modules[__name__], name, value) 48 | -------------------------------------------------------------------------------- /diffusers/pipelines/latent_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_latent_diffusion"] = ["LDMBertModel", "LDMTextToImagePipeline"] 25 | _import_structure["pipeline_latent_diffusion_superresolution"] = ["LDMSuperResolutionPipeline"] 26 | 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .pipeline_latent_diffusion import LDMBertModel, LDMTextToImagePipeline 37 | from .pipeline_latent_diffusion_superresolution import LDMSuperResolutionPipeline 38 | 39 | else: 40 | import sys 41 | 42 | sys.modules[__name__] = _LazyModule( 43 | __name__, 44 | globals()["__file__"], 45 | _import_structure, 46 | module_spec=__spec__, 47 | ) 48 | 49 | for name, value in _dummy_objects.items(): 50 | setattr(sys.modules[__name__], name, value) 51 | -------------------------------------------------------------------------------- /diffusers/pipelines/latent_consistency_models/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_latent_consistency_img2img"] = ["LatentConsistencyModelImg2ImgPipeline"] 26 | _import_structure["pipeline_latent_consistency_text2img"] = ["LatentConsistencyModelPipeline"] 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .pipeline_latent_consistency_img2img import LatentConsistencyModelImg2ImgPipeline 37 | from .pipeline_latent_consistency_text2img import LatentConsistencyModelPipeline 38 | 39 | else: 40 | import sys 41 | 42 | sys.modules[__name__] = _LazyModule( 43 | __name__, 44 | globals()["__file__"], 45 | _import_structure, 46 | module_spec=__spec__, 47 | ) 48 | 49 | for name, value in _dummy_objects.items(): 50 | setattr(sys.modules[__name__], name, value) 51 | -------------------------------------------------------------------------------- /diffusers/pipelines/deepfloyd_if/watermark.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import PIL.Image 4 | import torch 5 | from PIL import Image 6 | 7 | from ...configuration_utils import ConfigMixin 8 | from ...models.modeling_utils import ModelMixin 9 | from ...utils import PIL_INTERPOLATION 10 | 11 | 12 | class IFWatermarker(ModelMixin, ConfigMixin): 13 | def __init__(self): 14 | super().__init__() 15 | 16 | self.register_buffer("watermark_image", torch.zeros((62, 62, 4))) 17 | self.watermark_image_as_pil = None 18 | 19 | def apply_watermark(self, images: List[PIL.Image.Image], sample_size=None): 20 | # copied from https://github.com/deep-floyd/IF/blob/b77482e36ca2031cb94dbca1001fc1e6400bf4ab/deepfloyd_if/modules/base.py#L287 21 | 22 | h = images[0].height 23 | w = images[0].width 24 | 25 | sample_size = sample_size or h 26 | 27 | coef = min(h / sample_size, w / sample_size) 28 | img_h, img_w = (int(h / coef), int(w / coef)) if coef < 1 else (h, w) 29 | 30 | S1, S2 = 1024**2, img_w * img_h 31 | K = (S2 / S1) ** 0.5 32 | wm_size, wm_x, wm_y = int(K * 62), img_w - int(14 * K), img_h - int(14 * K) 33 | 34 | if self.watermark_image_as_pil is None: 35 | watermark_image = self.watermark_image.to(torch.uint8).cpu().numpy() 36 | watermark_image = Image.fromarray(watermark_image, mode="RGBA") 37 | self.watermark_image_as_pil = watermark_image 38 | 39 | wm_img = self.watermark_image_as_pil.resize( 40 | (wm_size, wm_size), PIL_INTERPOLATION["bicubic"], reducing_gap=None 41 | ) 42 | 43 | for pil_img in images: 44 | pil_img.paste(wm_img, box=(wm_x - wm_size, wm_y - wm_size, wm_x, wm_y), mask=wm_img.split()[-1]) 45 | 46 | return images 47 | -------------------------------------------------------------------------------- /diffusers/pipelines/paint_by_example/__init__.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import TYPE_CHECKING, List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL 6 | from PIL import Image 7 | 8 | from ...utils import ( 9 | DIFFUSERS_SLOW_IMPORT, 10 | OptionalDependencyNotAvailable, 11 | _LazyModule, 12 | get_objects_from_module, 13 | is_torch_available, 14 | is_transformers_available, 15 | ) 16 | 17 | 18 | _dummy_objects = {} 19 | _import_structure = {} 20 | 21 | try: 22 | if not (is_transformers_available() and is_torch_available()): 23 | raise OptionalDependencyNotAvailable() 24 | except OptionalDependencyNotAvailable: 25 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 26 | 27 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 28 | else: 29 | _import_structure["image_encoder"] = ["PaintByExampleImageEncoder"] 30 | _import_structure["pipeline_paint_by_example"] = ["PaintByExamplePipeline"] 31 | 32 | 33 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 34 | try: 35 | if not (is_transformers_available() and is_torch_available()): 36 | raise OptionalDependencyNotAvailable() 37 | 38 | except OptionalDependencyNotAvailable: 39 | from ...utils.dummy_torch_and_transformers_objects import * 40 | else: 41 | from .image_encoder import PaintByExampleImageEncoder 42 | from .pipeline_paint_by_example import PaintByExamplePipeline 43 | 44 | else: 45 | import sys 46 | 47 | sys.modules[__name__] = _LazyModule( 48 | __name__, 49 | globals()["__file__"], 50 | _import_structure, 51 | module_spec=__spec__, 52 | ) 53 | 54 | for name, value in _dummy_objects.items(): 55 | setattr(sys.modules[__name__], name, value) 56 | -------------------------------------------------------------------------------- /diffusers/pipelines/audioldm2/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | is_transformers_version, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["modeling_audioldm2"] = ["AudioLDM2ProjectionModel", "AudioLDM2UNet2DConditionModel"] 26 | _import_structure["pipeline_audioldm2"] = ["AudioLDM2Pipeline"] 27 | 28 | 29 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 30 | try: 31 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 32 | raise OptionalDependencyNotAvailable() 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | 36 | else: 37 | from .modeling_audioldm2 import AudioLDM2ProjectionModel, AudioLDM2UNet2DConditionModel 38 | from .pipeline_audioldm2 import AudioLDM2Pipeline 39 | 40 | else: 41 | import sys 42 | 43 | sys.modules[__name__] = _LazyModule( 44 | __name__, 45 | globals()["__file__"], 46 | _import_structure, 47 | module_spec=__spec__, 48 | ) 49 | for name, value in _dummy_objects.items(): 50 | setattr(sys.modules[__name__], name, value) 51 | -------------------------------------------------------------------------------- /diffusers/pipelines/vq_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | is_torch_available, 8 | is_transformers_available, 9 | ) 10 | 11 | 12 | _dummy_objects = {} 13 | _import_structure = {} 14 | 15 | try: 16 | if not (is_transformers_available() and is_torch_available()): 17 | raise OptionalDependencyNotAvailable() 18 | except OptionalDependencyNotAvailable: 19 | from ...utils.dummy_torch_and_transformers_objects import ( 20 | LearnedClassifierFreeSamplingEmbeddings, 21 | VQDiffusionPipeline, 22 | ) 23 | 24 | _dummy_objects.update( 25 | { 26 | "LearnedClassifierFreeSamplingEmbeddings": LearnedClassifierFreeSamplingEmbeddings, 27 | "VQDiffusionPipeline": VQDiffusionPipeline, 28 | } 29 | ) 30 | else: 31 | _import_structure["pipeline_vq_diffusion"] = ["LearnedClassifierFreeSamplingEmbeddings", "VQDiffusionPipeline"] 32 | 33 | 34 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 35 | try: 36 | if not (is_transformers_available() and is_torch_available()): 37 | raise OptionalDependencyNotAvailable() 38 | except OptionalDependencyNotAvailable: 39 | from ...utils.dummy_torch_and_transformers_objects import ( 40 | LearnedClassifierFreeSamplingEmbeddings, 41 | VQDiffusionPipeline, 42 | ) 43 | else: 44 | from .pipeline_vq_diffusion import LearnedClassifierFreeSamplingEmbeddings, VQDiffusionPipeline 45 | 46 | else: 47 | import sys 48 | 49 | sys.modules[__name__] = _LazyModule( 50 | __name__, 51 | globals()["__file__"], 52 | _import_structure, 53 | module_spec=__spec__, 54 | ) 55 | 56 | for name, value in _dummy_objects.items(): 57 | setattr(sys.modules[__name__], name, value) 58 | -------------------------------------------------------------------------------- /diffusers/pipelines/unclip/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | is_torch_available, 8 | is_transformers_available, 9 | is_transformers_version, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline 21 | 22 | _dummy_objects.update( 23 | {"UnCLIPImageVariationPipeline": UnCLIPImageVariationPipeline, "UnCLIPPipeline": UnCLIPPipeline} 24 | ) 25 | else: 26 | _import_structure["pipeline_unclip"] = ["UnCLIPPipeline"] 27 | _import_structure["pipeline_unclip_image_variation"] = ["UnCLIPImageVariationPipeline"] 28 | _import_structure["text_proj"] = ["UnCLIPTextProjModel"] 29 | 30 | 31 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 32 | try: 33 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): 34 | raise OptionalDependencyNotAvailable() 35 | except OptionalDependencyNotAvailable: 36 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 37 | else: 38 | from .pipeline_unclip import UnCLIPPipeline 39 | from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline 40 | from .text_proj import UnCLIPTextProjModel 41 | 42 | else: 43 | import sys 44 | 45 | sys.modules[__name__] = _LazyModule( 46 | __name__, 47 | globals()["__file__"], 48 | _import_structure, 49 | module_spec=__spec__, 50 | ) 51 | for name, value in _dummy_objects.items(): 52 | setattr(sys.modules[__name__], name, value) 53 | -------------------------------------------------------------------------------- /diffusers/pipelines/alt_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["modeling_roberta_series"] = ["RobertaSeriesModelWithTransformation"] 25 | _import_structure["pipeline_alt_diffusion"] = ["AltDiffusionPipeline"] 26 | _import_structure["pipeline_alt_diffusion_img2img"] = ["AltDiffusionImg2ImgPipeline"] 27 | 28 | _import_structure["pipeline_output"] = ["AltDiffusionPipelineOutput"] 29 | 30 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 31 | try: 32 | if not (is_transformers_available() and is_torch_available()): 33 | raise OptionalDependencyNotAvailable() 34 | except OptionalDependencyNotAvailable: 35 | from ...utils.dummy_torch_and_transformers_objects import * 36 | 37 | else: 38 | from .modeling_roberta_series import RobertaSeriesModelWithTransformation 39 | from .pipeline_alt_diffusion import AltDiffusionPipeline 40 | from .pipeline_alt_diffusion_img2img import AltDiffusionImg2ImgPipeline 41 | from .pipeline_output import AltDiffusionPipelineOutput 42 | 43 | else: 44 | import sys 45 | 46 | sys.modules[__name__] = _LazyModule( 47 | __name__, 48 | globals()["__file__"], 49 | _import_structure, 50 | module_spec=__spec__, 51 | ) 52 | for name, value in _dummy_objects.items(): 53 | setattr(sys.modules[__name__], name, value) 54 | -------------------------------------------------------------------------------- /diffusers/utils/accelerate_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Accelerate utilities: Utilities related to accelerate 16 | """ 17 | 18 | from packaging import version 19 | 20 | from .import_utils import is_accelerate_available 21 | 22 | 23 | if is_accelerate_available(): 24 | import accelerate 25 | 26 | 27 | def apply_forward_hook(method): 28 | """ 29 | Decorator that applies a registered CpuOffload hook to an arbitrary function rather than `forward`. This is useful 30 | for cases where a PyTorch module provides functions other than `forward` that should trigger a move to the 31 | appropriate acceleration device. This is the case for `encode` and `decode` in [`AutoencoderKL`]. 32 | 33 | This decorator looks inside the internal `_hf_hook` property to find a registered offload hook. 34 | 35 | :param method: The method to decorate. This method should be a method of a PyTorch module. 36 | """ 37 | if not is_accelerate_available(): 38 | return method 39 | accelerate_version = version.parse(accelerate.__version__).base_version 40 | if version.parse(accelerate_version) < version.parse("0.17.0"): 41 | return method 42 | 43 | def wrapper(self, *args, **kwargs): 44 | if hasattr(self, "_hf_hook") and hasattr(self._hf_hook, "pre_forward"): 45 | self._hf_hook.pre_forward(self) 46 | return method(self, *args, **kwargs) 47 | 48 | return wrapper 49 | -------------------------------------------------------------------------------- /diffusers/pipelines/text_to_video_synthesis/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_output"] = ["TextToVideoSDPipelineOutput"] 25 | _import_structure["pipeline_text_to_video_synth"] = ["TextToVideoSDPipeline"] 26 | _import_structure["pipeline_text_to_video_synth_img2img"] = ["VideoToVideoSDPipeline"] 27 | _import_structure["pipeline_text_to_video_zero"] = ["TextToVideoZeroPipeline"] 28 | 29 | 30 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 31 | try: 32 | if not (is_transformers_available() and is_torch_available()): 33 | raise OptionalDependencyNotAvailable() 34 | except OptionalDependencyNotAvailable: 35 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 36 | else: 37 | from .pipeline_output import TextToVideoSDPipelineOutput 38 | from .pipeline_text_to_video_synth import TextToVideoSDPipeline 39 | from .pipeline_text_to_video_synth_img2img import VideoToVideoSDPipeline 40 | from .pipeline_text_to_video_zero import TextToVideoZeroPipeline 41 | 42 | else: 43 | import sys 44 | 45 | sys.modules[__name__] = _LazyModule( 46 | __name__, 47 | globals()["__file__"], 48 | _import_structure, 49 | module_spec=__spec__, 50 | ) 51 | for name, value in _dummy_objects.items(): 52 | setattr(sys.modules[__name__], name, value) 53 | -------------------------------------------------------------------------------- /infer_mask.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torch 3 | from diffusers import UniPCMultistepScheduler 4 | from utils.pipeline_t2i import StableDiffusionPipeline 5 | from ssr_encoder import SSR_encoder 6 | from diffusers.utils import load_image 7 | 8 | # Initialize the model 9 | base_model_path = "sd_model_v1-5" 10 | image_encoder_path = "models/image_encoder" 11 | base_ssr = "./models/ssr_model" 12 | ssr_ckpt = [base_ssr+"/pytorch_model.bin", 13 | base_ssr+"/pytorch_model_1.bin"] 14 | 15 | pipe = StableDiffusionPipeline.from_pretrained( 16 | base_model_path, 17 | safety_checker=None, 18 | torch_dtype=torch.float32).to("cuda") 19 | pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) 20 | ssr_model = SSR_encoder(pipe.unet, image_encoder_path, "cuda", dtype=torch.float32, is_mask=True) 21 | ssr_model.get_pipe(pipe) 22 | ssr_model.load_SSR(ssr_ckpt[0], ssr_ckpt[1]) 23 | 24 | if __name__ == '__main__': 25 | 26 | # infer paras 27 | prompt = "a girl in a snowy day" 28 | out_path = "./results" 29 | img_path = "./test_img/mask/4.jpg" 30 | mask_path = "./test_img/mask/girl.png" 31 | caption = "A man wearing a black suit and a girl wearing a red dress" # maybe blip2 32 | 33 | pil_img = load_image(img_path) 34 | pil_img = pil_img.resize((512, 512)) 35 | mask = load_image(mask_path) 36 | mask = mask.resize((512, 512)) 37 | 38 | images = ssr_model.generate_mask( 39 | mask=mask, # use mask to further determine subject that the cross-attn focus on 40 | pil_image=pil_img, 41 | subject=caption, # need to use captions to get all subjects 42 | prompt=prompt, 43 | negative_prompt="", 44 | scale=0.5, 45 | num_samples=1, 46 | seed=None, 47 | guidance_scale=5, 48 | num_inference_steps=25, 49 | height=512, 50 | width=512, 51 | mask_weight=1, # The effect of mask_weight needs to be adjusted manually, so I recommend using text 52 | )[0] 53 | 54 | images.save(os.path.join(out_path, "mask_girl.jpg")) -------------------------------------------------------------------------------- /diffusers/pipelines/unidiffuser/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | is_torch_available, 8 | is_transformers_available, 9 | ) 10 | 11 | 12 | _dummy_objects = {} 13 | _import_structure = {} 14 | 15 | try: 16 | if not (is_transformers_available() and is_torch_available()): 17 | raise OptionalDependencyNotAvailable() 18 | except OptionalDependencyNotAvailable: 19 | from ...utils.dummy_torch_and_transformers_objects import ( 20 | ImageTextPipelineOutput, 21 | UniDiffuserPipeline, 22 | ) 23 | 24 | _dummy_objects.update( 25 | {"ImageTextPipelineOutput": ImageTextPipelineOutput, "UniDiffuserPipeline": UniDiffuserPipeline} 26 | ) 27 | else: 28 | _import_structure["modeling_text_decoder"] = ["UniDiffuserTextDecoder"] 29 | _import_structure["modeling_uvit"] = ["UniDiffuserModel", "UTransformer2DModel"] 30 | _import_structure["pipeline_unidiffuser"] = ["ImageTextPipelineOutput", "UniDiffuserPipeline"] 31 | 32 | 33 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 34 | try: 35 | if not (is_transformers_available() and is_torch_available()): 36 | raise OptionalDependencyNotAvailable() 37 | except OptionalDependencyNotAvailable: 38 | from ...utils.dummy_torch_and_transformers_objects import ( 39 | ImageTextPipelineOutput, 40 | UniDiffuserPipeline, 41 | ) 42 | else: 43 | from .modeling_text_decoder import UniDiffuserTextDecoder 44 | from .modeling_uvit import UniDiffuserModel, UTransformer2DModel 45 | from .pipeline_unidiffuser import ImageTextPipelineOutput, UniDiffuserPipeline 46 | 47 | else: 48 | import sys 49 | 50 | sys.modules[__name__] = _LazyModule( 51 | __name__, 52 | globals()["__file__"], 53 | _import_structure, 54 | module_spec=__spec__, 55 | ) 56 | 57 | for name, value in _dummy_objects.items(): 58 | setattr(sys.modules[__name__], name, value) 59 | -------------------------------------------------------------------------------- /gradio_demo.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from diffusers import UniPCMultistepScheduler 3 | from utils.pipeline_t2i import StableDiffusionPipeline 4 | from ssr_encoder import SSR_encoder 5 | import gradio as gr 6 | from PIL import Image 7 | 8 | base_model_path = "sd_model_v1-5". # your sd15 path 9 | image_encoder_path = "models/image_encoder" 10 | base_ssr = "./models/ssr_model" 11 | ssr_ckpt = [base_ssr+"/pytorch_model.bin", 12 | base_ssr+"/pytorch_model_1.bin"] 13 | 14 | pipe = StableDiffusionPipeline.from_pretrained( 15 | base_model_path, 16 | safety_checker=None, 17 | torch_dtype=torch.float32).to("cuda") 18 | pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) 19 | 20 | sc_model = SSR_encoder(pipe.unet, image_encoder_path, "cuda", dtype=torch.float32) 21 | sc_model.get_pipe(pipe) 22 | sc_model.load_SSR(ssr_ckpt[0], ssr_ckpt[1]) 23 | 24 | # Define your ML model or function here 25 | def model_call(pil_img, subject, prompts, negative_prompt, scale): 26 | # Your ML logic goes here 27 | pil_img = Image.fromarray(pil_img.astype('uint8'), 'RGB') 28 | images = sc_model.generate( 29 | pil_image=pil_img, 30 | concept=subject, 31 | uncond_concept="", 32 | prompt=prompts, 33 | negative_prompt=negative_prompt, 34 | num_samples=1, 35 | seed=None, 36 | guidance_scale=7.5, 37 | scale=scale, 38 | num_inference_steps=30, 39 | height=512, 40 | width=512, 41 | )[0] 42 | return images 43 | 44 | # Create a Gradio interface 45 | iface = gr.Interface( 46 | fn=lambda image, text1, text2, text3, num, : model_call(image, text1, text2, text3, num), 47 | inputs=[gr.Image(), \ 48 | gr.Textbox(label="subject query"), \ 49 | gr.Textbox(label="prompts"), \ 50 | gr.Textbox(label="neg prompts"), \ 51 | gr.inputs.Slider(minimum=0, maximum=2, default=0.65, label="subject_scale")], 52 | outputs=gr.Image(), 53 | title="SSR Demo", 54 | description="Upload an image and enter text inputs to see the model output." 55 | ) 56 | # Launch the Gradio interface 57 | iface.launch(server_name='0.0.0.0') 58 | -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Optional, Union 16 | 17 | import torch 18 | from torch import nn 19 | 20 | from ...configuration_utils import ConfigMixin, register_to_config 21 | from ...models.modeling_utils import ModelMixin 22 | 23 | 24 | class StableUnCLIPImageNormalizer(ModelMixin, ConfigMixin): 25 | """ 26 | This class is used to hold the mean and standard deviation of the CLIP embedder used in stable unCLIP. 27 | 28 | It is used to normalize the image embeddings before the noise is applied and un-normalize the noised image 29 | embeddings. 30 | """ 31 | 32 | @register_to_config 33 | def __init__( 34 | self, 35 | embedding_dim: int = 768, 36 | ): 37 | super().__init__() 38 | 39 | self.mean = nn.Parameter(torch.zeros(1, embedding_dim)) 40 | self.std = nn.Parameter(torch.ones(1, embedding_dim)) 41 | 42 | def to( 43 | self, 44 | torch_device: Optional[Union[str, torch.device]] = None, 45 | torch_dtype: Optional[torch.dtype] = None, 46 | ): 47 | self.mean = nn.Parameter(self.mean.to(torch_device).to(torch_dtype)) 48 | self.std = nn.Parameter(self.std.to(torch_device).to(torch_dtype)) 49 | return self 50 | 51 | def scale(self, embeds): 52 | embeds = (embeds - self.mean) * 1.0 / self.std 53 | return embeds 54 | 55 | def unscale(self, embeds): 56 | embeds = (embeds * self.std) + self.mean 57 | return embeds 58 | -------------------------------------------------------------------------------- /infer_multi.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torch 3 | from diffusers import UniPCMultistepScheduler 4 | from utils.pipeline_t2i import StableDiffusionPipeline 5 | from ssr_encoder import SSR_encoder 6 | from diffusers.utils import load_image 7 | 8 | def is_image_file(filename): 9 | return any(filename.endswith(extension) for extension in ['.png', '.jpg', '.jpeg', '.PNG', '.JPG', '.JPEG']) 10 | 11 | # Initialize the model 12 | base_model_path = "sd_model_v1-5" 13 | image_encoder_path = "models/image_encoder" 14 | base_ssr = "./models/ssr_model" 15 | ssr_ckpt = [base_ssr+"/pytorch_model.bin", 16 | base_ssr+"/pytorch_model_1.bin"] 17 | 18 | pipe = StableDiffusionPipeline.from_pretrained( 19 | base_model_path, 20 | safety_checker=None, 21 | torch_dtype=torch.float32).to("cuda") 22 | pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) 23 | ssr_model = SSR_encoder(pipe.unet, image_encoder_path, "cuda", dtype=torch.float32) 24 | ssr_model.get_pipe(pipe) 25 | ssr_model.load_SSR(ssr_ckpt[0], ssr_ckpt[1]) 26 | 27 | if __name__ == '__main__': 28 | 29 | # infer paras 30 | num = 3 # num of images 31 | subjects = ["dog", "man", "flowers, mountain"] 32 | scales = [0.8, 1.2, 1.] # need to balance scales 33 | prompt = "A man holding a dog" 34 | out_dir = "./results" 35 | img_path = "./test_img/multi" 36 | 37 | if not os.path.exists(out_dir): 38 | os.makedirs(out_dir, exist_ok=True) 39 | 40 | pil_img_ls = [] 41 | for idx, name in enumerate(sorted(os.listdir(img_path))): 42 | if not is_image_file(name): 43 | continue 44 | print(name) 45 | pil_img_ls.append(load_image(os.path.join(img_path, name))) 46 | 47 | images = ssr_model.generate_multi( 48 | pil_image_list=pil_img_ls, 49 | subject_list=subjects, 50 | uncond_concept=[""]*num, 51 | prompt=prompt, 52 | negative_prompt="", 53 | num_samples=1, 54 | seed=None, 55 | guidance_scale=5, 56 | scale=scales, 57 | num_inference_steps=30, 58 | height=512, 59 | width=512, 60 | )[0] 61 | images.save(os.path.join(out_dir, "multi.jpg")) -------------------------------------------------------------------------------- /diffusers/utils/pil_utils.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import PIL.Image 4 | import PIL.ImageOps 5 | from packaging import version 6 | from PIL import Image 7 | 8 | 9 | if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"): 10 | PIL_INTERPOLATION = { 11 | "linear": PIL.Image.Resampling.BILINEAR, 12 | "bilinear": PIL.Image.Resampling.BILINEAR, 13 | "bicubic": PIL.Image.Resampling.BICUBIC, 14 | "lanczos": PIL.Image.Resampling.LANCZOS, 15 | "nearest": PIL.Image.Resampling.NEAREST, 16 | } 17 | else: 18 | PIL_INTERPOLATION = { 19 | "linear": PIL.Image.LINEAR, 20 | "bilinear": PIL.Image.BILINEAR, 21 | "bicubic": PIL.Image.BICUBIC, 22 | "lanczos": PIL.Image.LANCZOS, 23 | "nearest": PIL.Image.NEAREST, 24 | } 25 | 26 | 27 | def pt_to_pil(images): 28 | """ 29 | Convert a torch image to a PIL image. 30 | """ 31 | images = (images / 2 + 0.5).clamp(0, 1) 32 | images = images.cpu().permute(0, 2, 3, 1).float().numpy() 33 | images = numpy_to_pil(images) 34 | return images 35 | 36 | 37 | def numpy_to_pil(images): 38 | """ 39 | Convert a numpy image or a batch of images to a PIL image. 40 | """ 41 | if images.ndim == 3: 42 | images = images[None, ...] 43 | images = (images * 255).round().astype("uint8") 44 | if images.shape[-1] == 1: 45 | # special case for grayscale (single channel) images 46 | pil_images = [Image.fromarray(image.squeeze(), mode="L") for image in images] 47 | else: 48 | pil_images = [Image.fromarray(image) for image in images] 49 | 50 | return pil_images 51 | 52 | 53 | def make_image_grid(images: List[PIL.Image.Image], rows: int, cols: int, resize: int = None) -> PIL.Image.Image: 54 | """ 55 | Prepares a single grid of images. Useful for visualization purposes. 56 | """ 57 | assert len(images) == rows * cols 58 | 59 | if resize is not None: 60 | images = [img.resize((resize, resize)) for img in images] 61 | 62 | w, h = images[0].size 63 | grid = Image.new("RGB", size=(cols * w, rows * h)) 64 | 65 | for i, img in enumerate(images): 66 | grid.paste(img, box=(i % cols * w, i // cols * h)) 67 | return grid 68 | -------------------------------------------------------------------------------- /infer_animate.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torch 3 | from diffusers import MotionAdapter, AutoencoderKL, EulerAncestralDiscreteScheduler 4 | from diffusers.utils import export_to_gif 5 | from diffusers.utils import load_image 6 | from ssr_encoder import SSR_encoder 7 | from utils.pipeline_animatediff import AnimateDiffPipeline 8 | 9 | def is_image_file(filename): 10 | return any(filename.endswith(extension) for extension in ['.png', '.jpg', '.jpeg', '.PNG', '.JPG', '.JPEG']) 11 | # model_paras 12 | base_model_path = "sd_model_v1-5" 13 | image_encoder_path = "models/image_encoder" 14 | adapter = MotionAdapter.from_pretrained("motion_v1-5-2") # animate_diff model path 15 | vae_path = "vae_ft" # recommended vae from animate_diff 16 | base_ssr = "./models/ssr_model" 17 | ssr_ckpt = [base_ssr+"/pytorch_model.bin", 18 | base_ssr+"/pytorch_model_1.bin"] 19 | 20 | # load models 21 | pipe = AnimateDiffPipeline.from_pretrained( 22 | base_model_path, 23 | motion_adapter=adapter, 24 | torch_dtype=torch.float16).to("cuda") 25 | vae = AutoencoderKL.from_pretrained(vae_path, torch_dtype=torch.float16).to("cuda") 26 | pipe.vae = vae 27 | pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config) 28 | ssr_model = SSR_encoder(pipe.unet, image_encoder_path, "cuda", dtype=torch.float16, is_animate=True) 29 | ssr_model.get_pipe(pipe) 30 | ssr_model.animate_load_SSR(ssr_ckpt[0], ssr_ckpt[1]) 31 | 32 | if __name__ == '__main__': 33 | 34 | # paras 35 | subject = "girl" 36 | scale = 0.6 37 | prompt = "A girl is dancing, 4k" 38 | out_dir = "./results" 39 | img_path = "./test_img/animate/1.jpeg" 40 | neg = "bad quality, blur" 41 | 42 | if not os.path.exists(out_dir): 43 | os.makedirs(out_dir, exist_ok=True) 44 | pil_img = load_image(img_path) 45 | frames = ssr_model.generate_animate( 46 | num_frames=16, 47 | pil_image=pil_img, 48 | concept=subject, 49 | uncond_concept="", 50 | prompt=prompt, 51 | negative_prompt=neg, 52 | num_samples=1, 53 | seed=None, 54 | guidance_scale=5, 55 | scale=scale, 56 | num_inference_steps=30, 57 | height=512, 58 | width=512, 59 | ) 60 | export_to_gif(frames, os.path.join(out_dir, "animation.gif")) -------------------------------------------------------------------------------- /infer_controlnet.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import torch 3 | from diffusers import UniPCMultistepScheduler, ControlNetModel 4 | from utils.pipeline_controlnet import StableDiffusionControlNetPipeline 5 | from ssr_encoder import SSR_encoder 6 | from diffusers.utils import load_image 7 | 8 | def is_image_file(filename): 9 | return any(filename.endswith(extension) for extension in ['.png', '.jpg', '.jpeg', '.PNG', '.JPG', '.JPEG']) 10 | 11 | # model_paras 12 | controlnet_path = "control_v11f1p_sd15_depth" # your controlnet path 13 | base_model_path = "sd_model_v1-5" # your sd15 path 14 | image_encoder_path = "models/image_encoder" 15 | base_ssr = "./models/ssr_model" 16 | ssr_ckpt = [base_ssr+"/pytorch_model.bin", 17 | base_ssr+"/pytorch_model_1.bin"] 18 | 19 | controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch.float16).to("cuda") 20 | pipe = StableDiffusionControlNetPipeline.from_pretrained( 21 | base_model_path, 22 | controlnet=controlnet, 23 | safety_checker=None, 24 | torch_dtype=torch.float16).to("cuda") 25 | pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) 26 | ssr_model = SSR_encoder(pipe.unet, image_encoder_path, "cuda", dtype=torch.float16) 27 | ssr_model.get_pipe(pipe) 28 | ssr_model.load_SSR(ssr_ckpt[0], ssr_ckpt[1]) 29 | 30 | if __name__ == '__main__': 31 | 32 | # paras 33 | subject = "cat" 34 | scale = 0.7 35 | prompt = "A cat in the forest" 36 | out_dir = "./results" 37 | img_path = "./test_img/controlnet/1.jpg" 38 | control_img = "./test_img/controlnet/dog_depth.png" 39 | negative_prompt = "bad quality" 40 | 41 | if not os.path.exists(out_dir): 42 | os.makedirs(out_dir, exist_ok=True) 43 | 44 | pil_img = load_image(img_path) 45 | control = load_image(control_img) 46 | images = ssr_model.generate_control( 47 | pil_image=pil_img, 48 | control_img=control, 49 | concept=subject, 50 | uncond_concept="", 51 | prompt=prompt, 52 | negative_prompt=negative_prompt, 53 | num_samples=1, 54 | seed=None, 55 | guidance_scale=5, 56 | scale=scale, 57 | num_inference_steps=30, 58 | height=512, 59 | width=512, 60 | )[0] 61 | images.save(os.path.join(out_dir, "control.jpg")) -------------------------------------------------------------------------------- /diffusers/utils/deprecation_utils.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import warnings 3 | from typing import Any, Dict, Optional, Union 4 | 5 | from packaging import version 6 | 7 | 8 | def deprecate(*args, take_from: Optional[Union[Dict, Any]] = None, standard_warn=True, stacklevel=2): 9 | from .. import __version__ 10 | 11 | deprecated_kwargs = take_from 12 | values = () 13 | if not isinstance(args[0], tuple): 14 | args = (args,) 15 | 16 | for attribute, version_name, message in args: 17 | if version.parse(version.parse(__version__).base_version) >= version.parse(version_name): 18 | raise ValueError( 19 | f"The deprecation tuple {(attribute, version_name, message)} should be removed since diffusers'" 20 | f" version {__version__} is >= {version_name}" 21 | ) 22 | 23 | warning = None 24 | if isinstance(deprecated_kwargs, dict) and attribute in deprecated_kwargs: 25 | values += (deprecated_kwargs.pop(attribute),) 26 | warning = f"The `{attribute}` argument is deprecated and will be removed in version {version_name}." 27 | elif hasattr(deprecated_kwargs, attribute): 28 | values += (getattr(deprecated_kwargs, attribute),) 29 | warning = f"The `{attribute}` attribute is deprecated and will be removed in version {version_name}." 30 | elif deprecated_kwargs is None: 31 | warning = f"`{attribute}` is deprecated and will be removed in version {version_name}." 32 | 33 | if warning is not None: 34 | warning = warning + " " if standard_warn else "" 35 | warnings.warn(warning + message, FutureWarning, stacklevel=stacklevel) 36 | 37 | if isinstance(deprecated_kwargs, dict) and len(deprecated_kwargs) > 0: 38 | call_frame = inspect.getouterframes(inspect.currentframe())[1] 39 | filename = call_frame.filename 40 | line_number = call_frame.lineno 41 | function = call_frame.function 42 | key, value = next(iter(deprecated_kwargs.items())) 43 | raise TypeError(f"{function} in {filename} line {line_number-1} got an unexpected keyword argument `{key}`") 44 | 45 | if len(values) == 0: 46 | return 47 | elif len(values) == 1: 48 | return values[0] 49 | return values 50 | -------------------------------------------------------------------------------- /diffusers/pipelines/wuerstchen/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["modeling_paella_vq_model"] = ["PaellaVQModel"] 25 | _import_structure["modeling_wuerstchen_diffnext"] = ["WuerstchenDiffNeXt"] 26 | _import_structure["modeling_wuerstchen_prior"] = ["WuerstchenPrior"] 27 | _import_structure["pipeline_wuerstchen"] = ["WuerstchenDecoderPipeline"] 28 | _import_structure["pipeline_wuerstchen_combined"] = ["WuerstchenCombinedPipeline"] 29 | _import_structure["pipeline_wuerstchen_prior"] = ["DEFAULT_STAGE_C_TIMESTEPS", "WuerstchenPriorPipeline"] 30 | 31 | 32 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 33 | try: 34 | if not (is_transformers_available() and is_torch_available()): 35 | raise OptionalDependencyNotAvailable() 36 | except OptionalDependencyNotAvailable: 37 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 38 | else: 39 | from .modeling_paella_vq_model import PaellaVQModel 40 | from .modeling_wuerstchen_diffnext import WuerstchenDiffNeXt 41 | from .modeling_wuerstchen_prior import WuerstchenPrior 42 | from .pipeline_wuerstchen import WuerstchenDecoderPipeline 43 | from .pipeline_wuerstchen_combined import WuerstchenCombinedPipeline 44 | from .pipeline_wuerstchen_prior import DEFAULT_STAGE_C_TIMESTEPS, WuerstchenPriorPipeline 45 | else: 46 | import sys 47 | 48 | sys.modules[__name__] = _LazyModule( 49 | __name__, 50 | globals()["__file__"], 51 | _import_structure, 52 | module_spec=__spec__, 53 | ) 54 | 55 | for name, value in _dummy_objects.items(): 56 | setattr(sys.modules[__name__], name, value) 57 | -------------------------------------------------------------------------------- /diffusers/pipelines/deepfloyd_if/safety_checker.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | from transformers import CLIPConfig, CLIPVisionModelWithProjection, PreTrainedModel 5 | 6 | from ...utils import logging 7 | 8 | 9 | logger = logging.get_logger(__name__) 10 | 11 | 12 | class IFSafetyChecker(PreTrainedModel): 13 | config_class = CLIPConfig 14 | 15 | _no_split_modules = ["CLIPEncoderLayer"] 16 | 17 | def __init__(self, config: CLIPConfig): 18 | super().__init__(config) 19 | 20 | self.vision_model = CLIPVisionModelWithProjection(config.vision_config) 21 | 22 | self.p_head = nn.Linear(config.vision_config.projection_dim, 1) 23 | self.w_head = nn.Linear(config.vision_config.projection_dim, 1) 24 | 25 | @torch.no_grad() 26 | def forward(self, clip_input, images, p_threshold=0.5, w_threshold=0.5): 27 | image_embeds = self.vision_model(clip_input)[0] 28 | 29 | nsfw_detected = self.p_head(image_embeds) 30 | nsfw_detected = nsfw_detected.flatten() 31 | nsfw_detected = nsfw_detected > p_threshold 32 | nsfw_detected = nsfw_detected.tolist() 33 | 34 | if any(nsfw_detected): 35 | logger.warning( 36 | "Potential NSFW content was detected in one or more images. A black image will be returned instead." 37 | " Try again with a different prompt and/or seed." 38 | ) 39 | 40 | for idx, nsfw_detected_ in enumerate(nsfw_detected): 41 | if nsfw_detected_: 42 | images[idx] = np.zeros(images[idx].shape) 43 | 44 | watermark_detected = self.w_head(image_embeds) 45 | watermark_detected = watermark_detected.flatten() 46 | watermark_detected = watermark_detected > w_threshold 47 | watermark_detected = watermark_detected.tolist() 48 | 49 | if any(watermark_detected): 50 | logger.warning( 51 | "Potential watermarked content was detected in one or more images. A black image will be returned instead." 52 | " Try again with a different prompt and/or seed." 53 | ) 54 | 55 | for idx, watermark_detected_ in enumerate(watermark_detected): 56 | if watermark_detected_: 57 | images[idx] = np.zeros(images[idx].shape) 58 | 59 | return images, nsfw_detected, watermark_detected 60 | -------------------------------------------------------------------------------- /diffusers/pipelines/shap_e/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["camera"] = ["create_pan_cameras"] 25 | _import_structure["pipeline_shap_e"] = ["ShapEPipeline"] 26 | _import_structure["pipeline_shap_e_img2img"] = ["ShapEImg2ImgPipeline"] 27 | _import_structure["renderer"] = [ 28 | "BoundingBoxVolume", 29 | "ImportanceRaySampler", 30 | "MLPNeRFModelOutput", 31 | "MLPNeRSTFModel", 32 | "ShapEParamsProjModel", 33 | "ShapERenderer", 34 | "StratifiedRaySampler", 35 | "VoidNeRFModel", 36 | ] 37 | 38 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 39 | try: 40 | if not (is_transformers_available() and is_torch_available()): 41 | raise OptionalDependencyNotAvailable() 42 | 43 | except OptionalDependencyNotAvailable: 44 | from ...utils.dummy_torch_and_transformers_objects import * 45 | else: 46 | from .camera import create_pan_cameras 47 | from .pipeline_shap_e import ShapEPipeline 48 | from .pipeline_shap_e_img2img import ShapEImg2ImgPipeline 49 | from .renderer import ( 50 | BoundingBoxVolume, 51 | ImportanceRaySampler, 52 | MLPNeRFModelOutput, 53 | MLPNeRSTFModel, 54 | ShapEParamsProjModel, 55 | ShapERenderer, 56 | StratifiedRaySampler, 57 | VoidNeRFModel, 58 | ) 59 | 60 | else: 61 | import sys 62 | 63 | sys.modules[__name__] = _LazyModule( 64 | __name__, 65 | globals()["__file__"], 66 | _import_structure, 67 | module_spec=__spec__, 68 | ) 69 | 70 | for name, value in _dummy_objects.items(): 71 | setattr(sys.modules[__name__], name, value) 72 | -------------------------------------------------------------------------------- /diffusers/pipelines/kandinsky/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_kandinsky"] = ["KandinskyPipeline"] 25 | _import_structure["pipeline_kandinsky_combined"] = [ 26 | "KandinskyCombinedPipeline", 27 | "KandinskyImg2ImgCombinedPipeline", 28 | "KandinskyInpaintCombinedPipeline", 29 | ] 30 | _import_structure["pipeline_kandinsky_img2img"] = ["KandinskyImg2ImgPipeline"] 31 | _import_structure["pipeline_kandinsky_inpaint"] = ["KandinskyInpaintPipeline"] 32 | _import_structure["pipeline_kandinsky_prior"] = ["KandinskyPriorPipeline", "KandinskyPriorPipelineOutput"] 33 | _import_structure["text_encoder"] = ["MultilingualCLIP"] 34 | 35 | 36 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 37 | try: 38 | if not (is_transformers_available() and is_torch_available()): 39 | raise OptionalDependencyNotAvailable() 40 | except OptionalDependencyNotAvailable: 41 | from ...utils.dummy_torch_and_transformers_objects import * 42 | 43 | else: 44 | from .pipeline_kandinsky import KandinskyPipeline 45 | from .pipeline_kandinsky_combined import ( 46 | KandinskyCombinedPipeline, 47 | KandinskyImg2ImgCombinedPipeline, 48 | KandinskyInpaintCombinedPipeline, 49 | ) 50 | from .pipeline_kandinsky_img2img import KandinskyImg2ImgPipeline 51 | from .pipeline_kandinsky_inpaint import KandinskyInpaintPipeline 52 | from .pipeline_kandinsky_prior import KandinskyPriorPipeline, KandinskyPriorPipelineOutput 53 | from .text_encoder import MultilingualCLIP 54 | 55 | else: 56 | import sys 57 | 58 | sys.modules[__name__] = _LazyModule( 59 | __name__, 60 | globals()["__file__"], 61 | _import_structure, 62 | module_spec=__spec__, 63 | ) 64 | 65 | for name, value in _dummy_objects.items(): 66 | setattr(sys.modules[__name__], name, value) 67 | -------------------------------------------------------------------------------- /diffusers/utils/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import importlib 15 | import os 16 | 17 | from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE, hf_cache_home 18 | from packaging import version 19 | 20 | from .import_utils import is_peft_available, is_transformers_available 21 | from ..dependency_versions_check import dep_version_check 22 | 23 | 24 | default_cache_path = HUGGINGFACE_HUB_CACHE 25 | 26 | MIN_PEFT_VERSION = "0.5.0" 27 | MIN_TRANSFORMERS_VERSION = "4.33.3" 28 | 29 | 30 | CONFIG_NAME = "config.json" 31 | WEIGHTS_NAME = "diffusion_pytorch_model.bin" 32 | FLAX_WEIGHTS_NAME = "diffusion_flax_model.msgpack" 33 | ONNX_WEIGHTS_NAME = "model.onnx" 34 | SAFETENSORS_WEIGHTS_NAME = "diffusion_pytorch_model.safetensors" 35 | ONNX_EXTERNAL_WEIGHTS_NAME = "weights.pb" 36 | HUGGINGFACE_CO_RESOLVE_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co") 37 | DIFFUSERS_CACHE = default_cache_path 38 | DIFFUSERS_DYNAMIC_MODULE_NAME = "diffusers_modules" 39 | HF_MODULES_CACHE = os.getenv("HF_MODULES_CACHE", os.path.join(hf_cache_home, "modules")) 40 | DEPRECATED_REVISION_ARGS = ["fp16", "non-ema"] 41 | 42 | # Below should be `True` if the current version of `peft` and `transformers` are compatible with 43 | # PEFT backend. Will automatically fall back to PEFT backend if the correct versions of the libraries are 44 | # available. 45 | # For PEFT it is has to be greater than 0.6.0 and for transformers it has to be greater than 4.33.1. 46 | _required_peft_version = is_peft_available() and version.parse( 47 | version.parse(importlib.metadata.version("peft")).base_version 48 | ) > version.parse(MIN_PEFT_VERSION) 49 | _required_transformers_version = is_transformers_available() and version.parse( 50 | version.parse(importlib.metadata.version("transformers")).base_version 51 | ) > version.parse(MIN_TRANSFORMERS_VERSION) 52 | 53 | USE_PEFT_BACKEND = _required_peft_version and _required_transformers_version 54 | 55 | if USE_PEFT_BACKEND: 56 | dep_version_check("peft") 57 | -------------------------------------------------------------------------------- /diffusers/utils/dummy_flax_and_transformers_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class FlaxStableDiffusionControlNetPipeline(metaclass=DummyObject): 6 | _backends = ["flax", "transformers"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["flax", "transformers"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["flax", "transformers"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["flax", "transformers"]) 18 | 19 | 20 | class FlaxStableDiffusionImg2ImgPipeline(metaclass=DummyObject): 21 | _backends = ["flax", "transformers"] 22 | 23 | def __init__(self, *args, **kwargs): 24 | requires_backends(self, ["flax", "transformers"]) 25 | 26 | @classmethod 27 | def from_config(cls, *args, **kwargs): 28 | requires_backends(cls, ["flax", "transformers"]) 29 | 30 | @classmethod 31 | def from_pretrained(cls, *args, **kwargs): 32 | requires_backends(cls, ["flax", "transformers"]) 33 | 34 | 35 | class FlaxStableDiffusionInpaintPipeline(metaclass=DummyObject): 36 | _backends = ["flax", "transformers"] 37 | 38 | def __init__(self, *args, **kwargs): 39 | requires_backends(self, ["flax", "transformers"]) 40 | 41 | @classmethod 42 | def from_config(cls, *args, **kwargs): 43 | requires_backends(cls, ["flax", "transformers"]) 44 | 45 | @classmethod 46 | def from_pretrained(cls, *args, **kwargs): 47 | requires_backends(cls, ["flax", "transformers"]) 48 | 49 | 50 | class FlaxStableDiffusionPipeline(metaclass=DummyObject): 51 | _backends = ["flax", "transformers"] 52 | 53 | def __init__(self, *args, **kwargs): 54 | requires_backends(self, ["flax", "transformers"]) 55 | 56 | @classmethod 57 | def from_config(cls, *args, **kwargs): 58 | requires_backends(cls, ["flax", "transformers"]) 59 | 60 | @classmethod 61 | def from_pretrained(cls, *args, **kwargs): 62 | requires_backends(cls, ["flax", "transformers"]) 63 | 64 | 65 | class FlaxStableDiffusionXLPipeline(metaclass=DummyObject): 66 | _backends = ["flax", "transformers"] 67 | 68 | def __init__(self, *args, **kwargs): 69 | requires_backends(self, ["flax", "transformers"]) 70 | 71 | @classmethod 72 | def from_config(cls, *args, **kwargs): 73 | requires_backends(cls, ["flax", "transformers"]) 74 | 75 | @classmethod 76 | def from_pretrained(cls, *args, **kwargs): 77 | requires_backends(cls, ["flax", "transformers"]) 78 | -------------------------------------------------------------------------------- /diffusers/pipelines/paint_by_example/image_encoder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import torch 15 | from torch import nn 16 | from transformers import CLIPPreTrainedModel, CLIPVisionModel 17 | 18 | from ...models.attention import BasicTransformerBlock 19 | from ...utils import logging 20 | 21 | 22 | logger = logging.get_logger(__name__) # pylint: disable=invalid-name 23 | 24 | 25 | class PaintByExampleImageEncoder(CLIPPreTrainedModel): 26 | def __init__(self, config, proj_size=None): 27 | super().__init__(config) 28 | self.proj_size = proj_size or getattr(config, "projection_dim", 768) 29 | 30 | self.model = CLIPVisionModel(config) 31 | self.mapper = PaintByExampleMapper(config) 32 | self.final_layer_norm = nn.LayerNorm(config.hidden_size) 33 | self.proj_out = nn.Linear(config.hidden_size, self.proj_size) 34 | 35 | # uncondition for scaling 36 | self.uncond_vector = nn.Parameter(torch.randn((1, 1, self.proj_size))) 37 | 38 | def forward(self, pixel_values, return_uncond_vector=False): 39 | clip_output = self.model(pixel_values=pixel_values) 40 | latent_states = clip_output.pooler_output 41 | latent_states = self.mapper(latent_states[:, None]) 42 | latent_states = self.final_layer_norm(latent_states) 43 | latent_states = self.proj_out(latent_states) 44 | if return_uncond_vector: 45 | return latent_states, self.uncond_vector 46 | 47 | return latent_states 48 | 49 | 50 | class PaintByExampleMapper(nn.Module): 51 | def __init__(self, config): 52 | super().__init__() 53 | num_layers = (config.num_hidden_layers + 1) // 5 54 | hid_size = config.hidden_size 55 | num_heads = 1 56 | self.blocks = nn.ModuleList( 57 | [ 58 | BasicTransformerBlock(hid_size, num_heads, hid_size, activation_fn="gelu", attention_bias=True) 59 | for _ in range(num_layers) 60 | ] 61 | ) 62 | 63 | def forward(self, hidden_states): 64 | for block in self.blocks: 65 | hidden_states = block(hidden_states) 66 | 67 | return hidden_states 68 | -------------------------------------------------------------------------------- /diffusers/pipelines/spectrogram_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from typing import TYPE_CHECKING 3 | from ...utils import DIFFUSERS_SLOW_IMPORT 4 | from ...utils import ( 5 | _LazyModule, 6 | is_note_seq_available, 7 | OptionalDependencyNotAvailable, 8 | is_torch_available, 9 | is_transformers_available, 10 | get_objects_from_module, 11 | ) 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["continous_encoder"] = ["SpectrogramContEncoder"] 25 | _import_structure["notes_encoder"] = ["SpectrogramNotesEncoder"] 26 | _import_structure["pipeline_spectrogram_diffusion"] = [ 27 | "SpectrogramContEncoder", 28 | "SpectrogramDiffusionPipeline", 29 | "T5FilmDecoder", 30 | ] 31 | try: 32 | if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): 33 | raise OptionalDependencyNotAvailable() 34 | except OptionalDependencyNotAvailable: 35 | from ...utils import dummy_transformers_and_torch_and_note_seq_objects 36 | 37 | _dummy_objects.update(get_objects_from_module(dummy_transformers_and_torch_and_note_seq_objects)) 38 | else: 39 | _import_structure["midi_utils"] = ["MidiProcessor"] 40 | 41 | 42 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 43 | try: 44 | if not (is_transformers_available() and is_torch_available()): 45 | raise OptionalDependencyNotAvailable() 46 | 47 | except OptionalDependencyNotAvailable: 48 | from ...utils.dummy_torch_and_transformers_objects import * 49 | else: 50 | from .pipeline_spectrogram_diffusion import SpectrogramDiffusionPipeline 51 | from .pipeline_spectrogram_diffusion import SpectrogramContEncoder 52 | from .pipeline_spectrogram_diffusion import SpectrogramNotesEncoder 53 | from .pipeline_spectrogram_diffusion import T5FilmDecoder 54 | 55 | try: 56 | if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): 57 | raise OptionalDependencyNotAvailable() 58 | except OptionalDependencyNotAvailable: 59 | from ...utils.dummy_transformers_and_torch_and_note_seq_objects import * 60 | 61 | else: 62 | from .midi_utils import MidiProcessor 63 | 64 | else: 65 | import sys 66 | 67 | sys.modules[__name__] = _LazyModule( 68 | __name__, 69 | globals()["__file__"], 70 | _import_structure, 71 | module_spec=__spec__, 72 | ) 73 | 74 | for name, value in _dummy_objects.items(): 75 | setattr(sys.modules[__name__], name, value) 76 | -------------------------------------------------------------------------------- /diffusers/pipelines/kandinsky2_2/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_kandinsky2_2"] = ["KandinskyV22Pipeline"] 25 | _import_structure["pipeline_kandinsky2_2_combined"] = [ 26 | "KandinskyV22CombinedPipeline", 27 | "KandinskyV22Img2ImgCombinedPipeline", 28 | "KandinskyV22InpaintCombinedPipeline", 29 | ] 30 | _import_structure["pipeline_kandinsky2_2_controlnet"] = ["KandinskyV22ControlnetPipeline"] 31 | _import_structure["pipeline_kandinsky2_2_controlnet_img2img"] = ["KandinskyV22ControlnetImg2ImgPipeline"] 32 | _import_structure["pipeline_kandinsky2_2_img2img"] = ["KandinskyV22Img2ImgPipeline"] 33 | _import_structure["pipeline_kandinsky2_2_inpainting"] = ["KandinskyV22InpaintPipeline"] 34 | _import_structure["pipeline_kandinsky2_2_prior"] = ["KandinskyV22PriorPipeline"] 35 | _import_structure["pipeline_kandinsky2_2_prior_emb2emb"] = ["KandinskyV22PriorEmb2EmbPipeline"] 36 | 37 | 38 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 39 | try: 40 | if not (is_transformers_available() and is_torch_available()): 41 | raise OptionalDependencyNotAvailable() 42 | 43 | except OptionalDependencyNotAvailable: 44 | from ...utils.dummy_torch_and_transformers_objects import * 45 | else: 46 | from .pipeline_kandinsky2_2 import KandinskyV22Pipeline 47 | from .pipeline_kandinsky2_2_combined import ( 48 | KandinskyV22CombinedPipeline, 49 | KandinskyV22Img2ImgCombinedPipeline, 50 | KandinskyV22InpaintCombinedPipeline, 51 | ) 52 | from .pipeline_kandinsky2_2_controlnet import KandinskyV22ControlnetPipeline 53 | from .pipeline_kandinsky2_2_controlnet_img2img import KandinskyV22ControlnetImg2ImgPipeline 54 | from .pipeline_kandinsky2_2_img2img import KandinskyV22Img2ImgPipeline 55 | from .pipeline_kandinsky2_2_inpainting import KandinskyV22InpaintPipeline 56 | from .pipeline_kandinsky2_2_prior import KandinskyV22PriorPipeline 57 | from .pipeline_kandinsky2_2_prior_emb2emb import KandinskyV22PriorEmb2EmbPipeline 58 | 59 | else: 60 | import sys 61 | 62 | sys.modules[__name__] = _LazyModule( 63 | __name__, 64 | globals()["__file__"], 65 | _import_structure, 66 | module_spec=__spec__, 67 | ) 68 | 69 | for name, value in _dummy_objects.items(): 70 | setattr(sys.modules[__name__], name, value) 71 | -------------------------------------------------------------------------------- /diffusers/pipelines/versatile_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | is_torch_available, 8 | is_transformers_available, 9 | is_transformers_version, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils.dummy_torch_and_transformers_objects import ( 21 | VersatileDiffusionDualGuidedPipeline, 22 | VersatileDiffusionImageVariationPipeline, 23 | VersatileDiffusionPipeline, 24 | VersatileDiffusionTextToImagePipeline, 25 | ) 26 | 27 | _dummy_objects.update( 28 | { 29 | "VersatileDiffusionDualGuidedPipeline": VersatileDiffusionDualGuidedPipeline, 30 | "VersatileDiffusionImageVariationPipeline": VersatileDiffusionImageVariationPipeline, 31 | "VersatileDiffusionPipeline": VersatileDiffusionPipeline, 32 | "VersatileDiffusionTextToImagePipeline": VersatileDiffusionTextToImagePipeline, 33 | } 34 | ) 35 | else: 36 | _import_structure["modeling_text_unet"] = ["UNetFlatConditionModel"] 37 | _import_structure["pipeline_versatile_diffusion"] = ["VersatileDiffusionPipeline"] 38 | _import_structure["pipeline_versatile_diffusion_dual_guided"] = ["VersatileDiffusionDualGuidedPipeline"] 39 | _import_structure["pipeline_versatile_diffusion_image_variation"] = ["VersatileDiffusionImageVariationPipeline"] 40 | _import_structure["pipeline_versatile_diffusion_text_to_image"] = ["VersatileDiffusionTextToImagePipeline"] 41 | 42 | 43 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 44 | try: 45 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): 46 | raise OptionalDependencyNotAvailable() 47 | except OptionalDependencyNotAvailable: 48 | from ...utils.dummy_torch_and_transformers_objects import ( 49 | VersatileDiffusionDualGuidedPipeline, 50 | VersatileDiffusionImageVariationPipeline, 51 | VersatileDiffusionPipeline, 52 | VersatileDiffusionTextToImagePipeline, 53 | ) 54 | else: 55 | from .pipeline_versatile_diffusion import VersatileDiffusionPipeline 56 | from .pipeline_versatile_diffusion_dual_guided import VersatileDiffusionDualGuidedPipeline 57 | from .pipeline_versatile_diffusion_image_variation import VersatileDiffusionImageVariationPipeline 58 | from .pipeline_versatile_diffusion_text_to_image import VersatileDiffusionTextToImagePipeline 59 | 60 | else: 61 | import sys 62 | 63 | sys.modules[__name__] = _LazyModule( 64 | __name__, 65 | globals()["__file__"], 66 | _import_structure, 67 | module_spec=__spec__, 68 | ) 69 | 70 | for name, value in _dummy_objects.items(): 71 | setattr(sys.modules[__name__], name, value) 72 | -------------------------------------------------------------------------------- /diffusers/commands/env.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import platform 16 | from argparse import ArgumentParser 17 | 18 | import huggingface_hub 19 | 20 | from .. import __version__ as version 21 | from ..utils import is_accelerate_available, is_torch_available, is_transformers_available, is_xformers_available 22 | from . import BaseDiffusersCLICommand 23 | 24 | 25 | def info_command_factory(_): 26 | return EnvironmentCommand() 27 | 28 | 29 | class EnvironmentCommand(BaseDiffusersCLICommand): 30 | @staticmethod 31 | def register_subcommand(parser: ArgumentParser): 32 | download_parser = parser.add_parser("env") 33 | download_parser.set_defaults(func=info_command_factory) 34 | 35 | def run(self): 36 | hub_version = huggingface_hub.__version__ 37 | 38 | pt_version = "not installed" 39 | pt_cuda_available = "NA" 40 | if is_torch_available(): 41 | import torch 42 | 43 | pt_version = torch.__version__ 44 | pt_cuda_available = torch.cuda.is_available() 45 | 46 | transformers_version = "not installed" 47 | if is_transformers_available(): 48 | import transformers 49 | 50 | transformers_version = transformers.__version__ 51 | 52 | accelerate_version = "not installed" 53 | if is_accelerate_available(): 54 | import accelerate 55 | 56 | accelerate_version = accelerate.__version__ 57 | 58 | xformers_version = "not installed" 59 | if is_xformers_available(): 60 | import xformers 61 | 62 | xformers_version = xformers.__version__ 63 | 64 | info = { 65 | "`diffusers` version": version, 66 | "Platform": platform.platform(), 67 | "Python version": platform.python_version(), 68 | "PyTorch version (GPU?)": f"{pt_version} ({pt_cuda_available})", 69 | "Huggingface_hub version": hub_version, 70 | "Transformers version": transformers_version, 71 | "Accelerate version": accelerate_version, 72 | "xFormers version": xformers_version, 73 | "Using GPU in script?": "", 74 | "Using distributed or parallel set-up in script?": "", 75 | } 76 | 77 | print("\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n") 78 | print(self.format_dict(info)) 79 | 80 | return info 81 | 82 | @staticmethod 83 | def format_dict(d): 84 | return "\n".join([f"- {prop}: {val}" for prop, val in d.items()]) + "\n" 85 | -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion_safe/__init__.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum 3 | from typing import TYPE_CHECKING, List, Optional, Union 4 | 5 | import numpy as np 6 | import PIL 7 | from PIL import Image 8 | 9 | from ...utils import ( 10 | DIFFUSERS_SLOW_IMPORT, 11 | BaseOutput, 12 | OptionalDependencyNotAvailable, 13 | _LazyModule, 14 | get_objects_from_module, 15 | is_torch_available, 16 | is_transformers_available, 17 | ) 18 | 19 | 20 | @dataclass 21 | class SafetyConfig(object): 22 | WEAK = { 23 | "sld_warmup_steps": 15, 24 | "sld_guidance_scale": 20, 25 | "sld_threshold": 0.0, 26 | "sld_momentum_scale": 0.0, 27 | "sld_mom_beta": 0.0, 28 | } 29 | MEDIUM = { 30 | "sld_warmup_steps": 10, 31 | "sld_guidance_scale": 1000, 32 | "sld_threshold": 0.01, 33 | "sld_momentum_scale": 0.3, 34 | "sld_mom_beta": 0.4, 35 | } 36 | STRONG = { 37 | "sld_warmup_steps": 7, 38 | "sld_guidance_scale": 2000, 39 | "sld_threshold": 0.025, 40 | "sld_momentum_scale": 0.5, 41 | "sld_mom_beta": 0.7, 42 | } 43 | MAX = { 44 | "sld_warmup_steps": 0, 45 | "sld_guidance_scale": 5000, 46 | "sld_threshold": 1.0, 47 | "sld_momentum_scale": 0.5, 48 | "sld_mom_beta": 0.7, 49 | } 50 | 51 | 52 | _dummy_objects = {} 53 | _additional_imports = {} 54 | _import_structure = {} 55 | 56 | _additional_imports.update({"SafetyConfig": SafetyConfig}) 57 | 58 | try: 59 | if not (is_transformers_available() and is_torch_available()): 60 | raise OptionalDependencyNotAvailable() 61 | except OptionalDependencyNotAvailable: 62 | from ...utils import dummy_torch_and_transformers_objects 63 | 64 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 65 | else: 66 | _import_structure.update( 67 | { 68 | "pipeline_output": ["StableDiffusionSafePipelineOutput"], 69 | "pipeline_stable_diffusion_safe": ["StableDiffusionPipelineSafe"], 70 | "safety_checker": ["StableDiffusionSafetyChecker"], 71 | } 72 | ) 73 | 74 | 75 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 76 | try: 77 | if not (is_transformers_available() and is_torch_available()): 78 | raise OptionalDependencyNotAvailable() 79 | except OptionalDependencyNotAvailable: 80 | from ...utils.dummy_torch_and_transformers_objects import * 81 | else: 82 | from .pipeline_output import StableDiffusionSafePipelineOutput 83 | from .pipeline_stable_diffusion_safe import StableDiffusionPipelineSafe 84 | from .safety_checker import SafeStableDiffusionSafetyChecker 85 | 86 | else: 87 | import sys 88 | 89 | sys.modules[__name__] = _LazyModule( 90 | __name__, 91 | globals()["__file__"], 92 | _import_structure, 93 | module_spec=__spec__, 94 | ) 95 | 96 | for name, value in _dummy_objects.items(): 97 | setattr(sys.modules[__name__], name, value) 98 | for name, value in _additional_imports.items(): 99 | setattr(sys.modules[__name__], name, value) 100 | -------------------------------------------------------------------------------- /diffusers/pipelines/spectrogram_diffusion/notes_encoder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Music Spectrogram Diffusion Authors. 2 | # Copyright 2023 The HuggingFace Team. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import torch 17 | import torch.nn as nn 18 | from transformers.modeling_utils import ModuleUtilsMixin 19 | from transformers.models.t5.modeling_t5 import T5Block, T5Config, T5LayerNorm 20 | 21 | from ...configuration_utils import ConfigMixin, register_to_config 22 | from ...models import ModelMixin 23 | 24 | 25 | class SpectrogramNotesEncoder(ModelMixin, ConfigMixin, ModuleUtilsMixin): 26 | @register_to_config 27 | def __init__( 28 | self, 29 | max_length: int, 30 | vocab_size: int, 31 | d_model: int, 32 | dropout_rate: float, 33 | num_layers: int, 34 | num_heads: int, 35 | d_kv: int, 36 | d_ff: int, 37 | feed_forward_proj: str, 38 | is_decoder: bool = False, 39 | ): 40 | super().__init__() 41 | 42 | self.token_embedder = nn.Embedding(vocab_size, d_model) 43 | 44 | self.position_encoding = nn.Embedding(max_length, d_model) 45 | self.position_encoding.weight.requires_grad = False 46 | 47 | self.dropout_pre = nn.Dropout(p=dropout_rate) 48 | 49 | t5config = T5Config( 50 | vocab_size=vocab_size, 51 | d_model=d_model, 52 | num_heads=num_heads, 53 | d_kv=d_kv, 54 | d_ff=d_ff, 55 | dropout_rate=dropout_rate, 56 | feed_forward_proj=feed_forward_proj, 57 | is_decoder=is_decoder, 58 | is_encoder_decoder=False, 59 | ) 60 | 61 | self.encoders = nn.ModuleList() 62 | for lyr_num in range(num_layers): 63 | lyr = T5Block(t5config) 64 | self.encoders.append(lyr) 65 | 66 | self.layer_norm = T5LayerNorm(d_model) 67 | self.dropout_post = nn.Dropout(p=dropout_rate) 68 | 69 | def forward(self, encoder_input_tokens, encoder_inputs_mask): 70 | x = self.token_embedder(encoder_input_tokens) 71 | 72 | seq_length = encoder_input_tokens.shape[1] 73 | inputs_positions = torch.arange(seq_length, device=encoder_input_tokens.device) 74 | x += self.position_encoding(inputs_positions) 75 | 76 | x = self.dropout_pre(x) 77 | 78 | # inverted the attention mask 79 | input_shape = encoder_input_tokens.size() 80 | extended_attention_mask = self.get_extended_attention_mask(encoder_inputs_mask, input_shape) 81 | 82 | for lyr in self.encoders: 83 | x = lyr(x, extended_attention_mask)[0] 84 | x = self.layer_norm(x) 85 | 86 | return self.dropout_post(x), encoder_inputs_mask 87 | -------------------------------------------------------------------------------- /diffusers/pipelines/deepfloyd_if/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = { 15 | "timesteps": [ 16 | "fast27_timesteps", 17 | "smart100_timesteps", 18 | "smart185_timesteps", 19 | "smart27_timesteps", 20 | "smart50_timesteps", 21 | "super100_timesteps", 22 | "super27_timesteps", 23 | "super40_timesteps", 24 | ] 25 | } 26 | 27 | try: 28 | if not (is_transformers_available() and is_torch_available()): 29 | raise OptionalDependencyNotAvailable() 30 | except OptionalDependencyNotAvailable: 31 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 32 | 33 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 34 | else: 35 | _import_structure["pipeline_if"] = ["IFPipeline"] 36 | _import_structure["pipeline_if_img2img"] = ["IFImg2ImgPipeline"] 37 | _import_structure["pipeline_if_img2img_superresolution"] = ["IFImg2ImgSuperResolutionPipeline"] 38 | _import_structure["pipeline_if_inpainting"] = ["IFInpaintingPipeline"] 39 | _import_structure["pipeline_if_inpainting_superresolution"] = ["IFInpaintingSuperResolutionPipeline"] 40 | _import_structure["pipeline_if_superresolution"] = ["IFSuperResolutionPipeline"] 41 | _import_structure["pipeline_output"] = ["IFPipelineOutput"] 42 | _import_structure["safety_checker"] = ["IFSafetyChecker"] 43 | _import_structure["watermark"] = ["IFWatermarker"] 44 | 45 | 46 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 47 | try: 48 | if not (is_transformers_available() and is_torch_available()): 49 | raise OptionalDependencyNotAvailable() 50 | 51 | except OptionalDependencyNotAvailable: 52 | from ...utils.dummy_torch_and_transformers_objects import * 53 | else: 54 | from .pipeline_if import IFPipeline 55 | from .pipeline_if_img2img import IFImg2ImgPipeline 56 | from .pipeline_if_img2img_superresolution import IFImg2ImgSuperResolutionPipeline 57 | from .pipeline_if_inpainting import IFInpaintingPipeline 58 | from .pipeline_if_inpainting_superresolution import IFInpaintingSuperResolutionPipeline 59 | from .pipeline_if_superresolution import IFSuperResolutionPipeline 60 | from .pipeline_output import IFPipelineOutput 61 | from .safety_checker import IFSafetyChecker 62 | from .timesteps import ( 63 | fast27_timesteps, 64 | smart27_timesteps, 65 | smart50_timesteps, 66 | smart100_timesteps, 67 | smart185_timesteps, 68 | super27_timesteps, 69 | super40_timesteps, 70 | super100_timesteps, 71 | ) 72 | from .watermark import IFWatermarker 73 | 74 | else: 75 | import sys 76 | 77 | sys.modules[__name__] = _LazyModule( 78 | __name__, 79 | globals()["__file__"], 80 | _import_structure, 81 | module_spec=__spec__, 82 | ) 83 | 84 | for name, value in _dummy_objects.items(): 85 | setattr(sys.modules[__name__], name, value) 86 | -------------------------------------------------------------------------------- /diffusers/pipelines/stable_diffusion_xl/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_flax_available, 9 | is_torch_available, 10 | is_transformers_available, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _additional_imports = {} 16 | _import_structure = {"pipeline_output": ["StableDiffusionXLPipelineOutput"]} 17 | 18 | if is_transformers_available() and is_flax_available(): 19 | _import_structure["pipeline_output"].extend(["FlaxStableDiffusionXLPipelineOutput"]) 20 | try: 21 | if not (is_transformers_available() and is_torch_available()): 22 | raise OptionalDependencyNotAvailable() 23 | except OptionalDependencyNotAvailable: 24 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 25 | 26 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 27 | else: 28 | _import_structure["pipeline_stable_diffusion_xl"] = ["StableDiffusionXLPipeline"] 29 | _import_structure["pipeline_stable_diffusion_xl_img2img"] = ["StableDiffusionXLImg2ImgPipeline"] 30 | _import_structure["pipeline_stable_diffusion_xl_inpaint"] = ["StableDiffusionXLInpaintPipeline"] 31 | _import_structure["pipeline_stable_diffusion_xl_instruct_pix2pix"] = ["StableDiffusionXLInstructPix2PixPipeline"] 32 | 33 | if is_transformers_available() and is_flax_available(): 34 | from ...schedulers.scheduling_pndm_flax import PNDMSchedulerState 35 | 36 | _additional_imports.update({"PNDMSchedulerState": PNDMSchedulerState}) 37 | _import_structure["pipeline_flax_stable_diffusion_xl"] = ["FlaxStableDiffusionXLPipeline"] 38 | 39 | 40 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 41 | try: 42 | if not (is_transformers_available() and is_torch_available()): 43 | raise OptionalDependencyNotAvailable() 44 | except OptionalDependencyNotAvailable: 45 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 46 | else: 47 | from .pipeline_stable_diffusion_xl import StableDiffusionXLPipeline 48 | from .pipeline_stable_diffusion_xl_img2img import StableDiffusionXLImg2ImgPipeline 49 | from .pipeline_stable_diffusion_xl_inpaint import StableDiffusionXLInpaintPipeline 50 | from .pipeline_stable_diffusion_xl_instruct_pix2pix import StableDiffusionXLInstructPix2PixPipeline 51 | 52 | try: 53 | if not (is_transformers_available() and is_flax_available()): 54 | raise OptionalDependencyNotAvailable() 55 | except OptionalDependencyNotAvailable: 56 | from ...utils.dummy_flax_objects import * 57 | else: 58 | from .pipeline_flax_stable_diffusion_xl import ( 59 | FlaxStableDiffusionXLPipeline, 60 | ) 61 | from .pipeline_output import FlaxStableDiffusionXLPipelineOutput 62 | 63 | else: 64 | import sys 65 | 66 | sys.modules[__name__] = _LazyModule( 67 | __name__, 68 | globals()["__file__"], 69 | _import_structure, 70 | module_spec=__spec__, 71 | ) 72 | 73 | for name, value in _dummy_objects.items(): 74 | setattr(sys.modules[__name__], name, value) 75 | for name, value in _additional_imports.items(): 76 | setattr(sys.modules[__name__], name, value) 77 | -------------------------------------------------------------------------------- /diffusers/utils/dummy_torch_and_transformers_and_onnx_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class OnnxStableDiffusionImg2ImgPipeline(metaclass=DummyObject): 6 | _backends = ["torch", "transformers", "onnx"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "transformers", "onnx"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "transformers", "onnx"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "transformers", "onnx"]) 18 | 19 | 20 | class OnnxStableDiffusionInpaintPipeline(metaclass=DummyObject): 21 | _backends = ["torch", "transformers", "onnx"] 22 | 23 | def __init__(self, *args, **kwargs): 24 | requires_backends(self, ["torch", "transformers", "onnx"]) 25 | 26 | @classmethod 27 | def from_config(cls, *args, **kwargs): 28 | requires_backends(cls, ["torch", "transformers", "onnx"]) 29 | 30 | @classmethod 31 | def from_pretrained(cls, *args, **kwargs): 32 | requires_backends(cls, ["torch", "transformers", "onnx"]) 33 | 34 | 35 | class OnnxStableDiffusionInpaintPipelineLegacy(metaclass=DummyObject): 36 | _backends = ["torch", "transformers", "onnx"] 37 | 38 | def __init__(self, *args, **kwargs): 39 | requires_backends(self, ["torch", "transformers", "onnx"]) 40 | 41 | @classmethod 42 | def from_config(cls, *args, **kwargs): 43 | requires_backends(cls, ["torch", "transformers", "onnx"]) 44 | 45 | @classmethod 46 | def from_pretrained(cls, *args, **kwargs): 47 | requires_backends(cls, ["torch", "transformers", "onnx"]) 48 | 49 | 50 | class OnnxStableDiffusionPipeline(metaclass=DummyObject): 51 | _backends = ["torch", "transformers", "onnx"] 52 | 53 | def __init__(self, *args, **kwargs): 54 | requires_backends(self, ["torch", "transformers", "onnx"]) 55 | 56 | @classmethod 57 | def from_config(cls, *args, **kwargs): 58 | requires_backends(cls, ["torch", "transformers", "onnx"]) 59 | 60 | @classmethod 61 | def from_pretrained(cls, *args, **kwargs): 62 | requires_backends(cls, ["torch", "transformers", "onnx"]) 63 | 64 | 65 | class OnnxStableDiffusionUpscalePipeline(metaclass=DummyObject): 66 | _backends = ["torch", "transformers", "onnx"] 67 | 68 | def __init__(self, *args, **kwargs): 69 | requires_backends(self, ["torch", "transformers", "onnx"]) 70 | 71 | @classmethod 72 | def from_config(cls, *args, **kwargs): 73 | requires_backends(cls, ["torch", "transformers", "onnx"]) 74 | 75 | @classmethod 76 | def from_pretrained(cls, *args, **kwargs): 77 | requires_backends(cls, ["torch", "transformers", "onnx"]) 78 | 79 | 80 | class StableDiffusionOnnxPipeline(metaclass=DummyObject): 81 | _backends = ["torch", "transformers", "onnx"] 82 | 83 | def __init__(self, *args, **kwargs): 84 | requires_backends(self, ["torch", "transformers", "onnx"]) 85 | 86 | @classmethod 87 | def from_config(cls, *args, **kwargs): 88 | requires_backends(cls, ["torch", "transformers", "onnx"]) 89 | 90 | @classmethod 91 | def from_pretrained(cls, *args, **kwargs): 92 | requires_backends(cls, ["torch", "transformers", "onnx"]) 93 | -------------------------------------------------------------------------------- /diffusers/pipelines/spectrogram_diffusion/continous_encoder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Music Spectrogram Diffusion Authors. 2 | # Copyright 2023 The HuggingFace Team. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import torch 17 | import torch.nn as nn 18 | from transformers.modeling_utils import ModuleUtilsMixin 19 | from transformers.models.t5.modeling_t5 import ( 20 | T5Block, 21 | T5Config, 22 | T5LayerNorm, 23 | ) 24 | 25 | from ...configuration_utils import ConfigMixin, register_to_config 26 | from ...models import ModelMixin 27 | 28 | 29 | class SpectrogramContEncoder(ModelMixin, ConfigMixin, ModuleUtilsMixin): 30 | @register_to_config 31 | def __init__( 32 | self, 33 | input_dims: int, 34 | targets_context_length: int, 35 | d_model: int, 36 | dropout_rate: float, 37 | num_layers: int, 38 | num_heads: int, 39 | d_kv: int, 40 | d_ff: int, 41 | feed_forward_proj: str, 42 | is_decoder: bool = False, 43 | ): 44 | super().__init__() 45 | 46 | self.input_proj = nn.Linear(input_dims, d_model, bias=False) 47 | 48 | self.position_encoding = nn.Embedding(targets_context_length, d_model) 49 | self.position_encoding.weight.requires_grad = False 50 | 51 | self.dropout_pre = nn.Dropout(p=dropout_rate) 52 | 53 | t5config = T5Config( 54 | d_model=d_model, 55 | num_heads=num_heads, 56 | d_kv=d_kv, 57 | d_ff=d_ff, 58 | feed_forward_proj=feed_forward_proj, 59 | dropout_rate=dropout_rate, 60 | is_decoder=is_decoder, 61 | is_encoder_decoder=False, 62 | ) 63 | self.encoders = nn.ModuleList() 64 | for lyr_num in range(num_layers): 65 | lyr = T5Block(t5config) 66 | self.encoders.append(lyr) 67 | 68 | self.layer_norm = T5LayerNorm(d_model) 69 | self.dropout_post = nn.Dropout(p=dropout_rate) 70 | 71 | def forward(self, encoder_inputs, encoder_inputs_mask): 72 | x = self.input_proj(encoder_inputs) 73 | 74 | # terminal relative positional encodings 75 | max_positions = encoder_inputs.shape[1] 76 | input_positions = torch.arange(max_positions, device=encoder_inputs.device) 77 | 78 | seq_lens = encoder_inputs_mask.sum(-1) 79 | input_positions = torch.roll(input_positions.unsqueeze(0), tuple(seq_lens.tolist()), dims=0) 80 | x += self.position_encoding(input_positions) 81 | 82 | x = self.dropout_pre(x) 83 | 84 | # inverted the attention mask 85 | input_shape = encoder_inputs.size() 86 | extended_attention_mask = self.get_extended_attention_mask(encoder_inputs_mask, input_shape) 87 | 88 | for lyr in self.encoders: 89 | x = lyr(x, extended_attention_mask)[0] 90 | x = self.layer_norm(x) 91 | 92 | return self.dropout_post(x), encoder_inputs_mask 93 | -------------------------------------------------------------------------------- /diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Dominic Rampas MIT License 2 | # Copyright 2023 The HuggingFace Team. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import torch 17 | import torch.nn as nn 18 | 19 | from ...models.attention_processor import Attention 20 | 21 | 22 | class WuerstchenLayerNorm(nn.LayerNorm): 23 | def __init__(self, *args, **kwargs): 24 | super().__init__(*args, **kwargs) 25 | 26 | def forward(self, x): 27 | x = x.permute(0, 2, 3, 1) 28 | x = super().forward(x) 29 | return x.permute(0, 3, 1, 2) 30 | 31 | 32 | class TimestepBlock(nn.Module): 33 | def __init__(self, c, c_timestep): 34 | super().__init__() 35 | self.mapper = nn.Linear(c_timestep, c * 2) 36 | 37 | def forward(self, x, t): 38 | a, b = self.mapper(t)[:, :, None, None].chunk(2, dim=1) 39 | return x * (1 + a) + b 40 | 41 | 42 | class ResBlock(nn.Module): 43 | def __init__(self, c, c_skip=0, kernel_size=3, dropout=0.0): 44 | super().__init__() 45 | self.depthwise = nn.Conv2d(c + c_skip, c, kernel_size=kernel_size, padding=kernel_size // 2, groups=c) 46 | self.norm = WuerstchenLayerNorm(c, elementwise_affine=False, eps=1e-6) 47 | self.channelwise = nn.Sequential( 48 | nn.Linear(c, c * 4), nn.GELU(), GlobalResponseNorm(c * 4), nn.Dropout(dropout), nn.Linear(c * 4, c) 49 | ) 50 | 51 | def forward(self, x, x_skip=None): 52 | x_res = x 53 | if x_skip is not None: 54 | x = torch.cat([x, x_skip], dim=1) 55 | x = self.norm(self.depthwise(x)).permute(0, 2, 3, 1) 56 | x = self.channelwise(x).permute(0, 3, 1, 2) 57 | return x + x_res 58 | 59 | 60 | # from https://github.com/facebookresearch/ConvNeXt-V2/blob/3608f67cc1dae164790c5d0aead7bf2d73d9719b/models/utils.py#L105 61 | class GlobalResponseNorm(nn.Module): 62 | def __init__(self, dim): 63 | super().__init__() 64 | self.gamma = nn.Parameter(torch.zeros(1, 1, 1, dim)) 65 | self.beta = nn.Parameter(torch.zeros(1, 1, 1, dim)) 66 | 67 | def forward(self, x): 68 | agg_norm = torch.norm(x, p=2, dim=(1, 2), keepdim=True) 69 | stand_div_norm = agg_norm / (agg_norm.mean(dim=-1, keepdim=True) + 1e-6) 70 | return self.gamma * (x * stand_div_norm) + self.beta + x 71 | 72 | 73 | class AttnBlock(nn.Module): 74 | def __init__(self, c, c_cond, nhead, self_attn=True, dropout=0.0): 75 | super().__init__() 76 | self.self_attn = self_attn 77 | self.norm = WuerstchenLayerNorm(c, elementwise_affine=False, eps=1e-6) 78 | self.attention = Attention(query_dim=c, heads=nhead, dim_head=c // nhead, dropout=dropout, bias=True) 79 | self.kv_mapper = nn.Sequential(nn.SiLU(), nn.Linear(c_cond, c)) 80 | 81 | def forward(self, x, kv): 82 | kv = self.kv_mapper(kv) 83 | norm_x = self.norm(x) 84 | if self.self_attn: 85 | batch_size, channel, _, _ = x.shape 86 | kv = torch.cat([norm_x.view(batch_size, channel, -1).transpose(1, 2), kv], dim=1) 87 | x = x + self.attention(norm_x, encoder_hidden_states=kv) 88 | return x 89 | -------------------------------------------------------------------------------- /diffusers/pipelines/controlnet/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_flax_available, 9 | is_torch_available, 10 | is_transformers_available, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["multicontrolnet"] = ["MultiControlNetModel"] 26 | _import_structure["pipeline_controlnet"] = ["StableDiffusionControlNetPipeline"] 27 | _import_structure["pipeline_controlnet_blip_diffusion"] = ["BlipDiffusionControlNetPipeline"] 28 | _import_structure["pipeline_controlnet_img2img"] = ["StableDiffusionControlNetImg2ImgPipeline"] 29 | _import_structure["pipeline_controlnet_inpaint"] = ["StableDiffusionControlNetInpaintPipeline"] 30 | _import_structure["pipeline_controlnet_inpaint_sd_xl"] = ["StableDiffusionXLControlNetInpaintPipeline"] 31 | _import_structure["pipeline_controlnet_sd_xl"] = ["StableDiffusionXLControlNetPipeline"] 32 | _import_structure["pipeline_controlnet_sd_xl_img2img"] = ["StableDiffusionXLControlNetImg2ImgPipeline"] 33 | try: 34 | if not (is_transformers_available() and is_flax_available()): 35 | raise OptionalDependencyNotAvailable() 36 | except OptionalDependencyNotAvailable: 37 | from ...utils import dummy_flax_and_transformers_objects # noqa F403 38 | 39 | _dummy_objects.update(get_objects_from_module(dummy_flax_and_transformers_objects)) 40 | else: 41 | _import_structure["pipeline_flax_controlnet"] = ["FlaxStableDiffusionControlNetPipeline"] 42 | 43 | 44 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 45 | try: 46 | if not (is_transformers_available() and is_torch_available()): 47 | raise OptionalDependencyNotAvailable() 48 | 49 | except OptionalDependencyNotAvailable: 50 | from ...utils.dummy_torch_and_transformers_objects import * 51 | else: 52 | from .multicontrolnet import MultiControlNetModel 53 | from .pipeline_controlnet import StableDiffusionControlNetPipeline 54 | from .pipeline_controlnet_blip_diffusion import BlipDiffusionControlNetPipeline 55 | from .pipeline_controlnet_img2img import StableDiffusionControlNetImg2ImgPipeline 56 | from .pipeline_controlnet_inpaint import StableDiffusionControlNetInpaintPipeline 57 | from .pipeline_controlnet_inpaint_sd_xl import StableDiffusionXLControlNetInpaintPipeline 58 | from .pipeline_controlnet_sd_xl import StableDiffusionXLControlNetPipeline 59 | from .pipeline_controlnet_sd_xl_img2img import StableDiffusionXLControlNetImg2ImgPipeline 60 | 61 | try: 62 | if not (is_transformers_available() and is_flax_available()): 63 | raise OptionalDependencyNotAvailable() 64 | except OptionalDependencyNotAvailable: 65 | from ...utils.dummy_flax_and_transformers_objects import * # noqa F403 66 | else: 67 | from .pipeline_flax_controlnet import FlaxStableDiffusionControlNetPipeline 68 | 69 | 70 | else: 71 | import sys 72 | 73 | sys.modules[__name__] = _LazyModule( 74 | __name__, 75 | globals()["__file__"], 76 | _import_structure, 77 | module_spec=__spec__, 78 | ) 79 | for name, value in _dummy_objects.items(): 80 | setattr(sys.modules[__name__], name, value) 81 | -------------------------------------------------------------------------------- /diffusers/models/embeddings_flax.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import math 15 | 16 | import flax.linen as nn 17 | import jax.numpy as jnp 18 | 19 | 20 | def get_sinusoidal_embeddings( 21 | timesteps: jnp.ndarray, 22 | embedding_dim: int, 23 | freq_shift: float = 1, 24 | min_timescale: float = 1, 25 | max_timescale: float = 1.0e4, 26 | flip_sin_to_cos: bool = False, 27 | scale: float = 1.0, 28 | ) -> jnp.ndarray: 29 | """Returns the positional encoding (same as Tensor2Tensor). 30 | 31 | Args: 32 | timesteps: a 1-D Tensor of N indices, one per batch element. 33 | These may be fractional. 34 | embedding_dim: The number of output channels. 35 | min_timescale: The smallest time unit (should probably be 0.0). 36 | max_timescale: The largest time unit. 37 | Returns: 38 | a Tensor of timing signals [N, num_channels] 39 | """ 40 | assert timesteps.ndim == 1, "Timesteps should be a 1d-array" 41 | assert embedding_dim % 2 == 0, f"Embedding dimension {embedding_dim} should be even" 42 | num_timescales = float(embedding_dim // 2) 43 | log_timescale_increment = math.log(max_timescale / min_timescale) / (num_timescales - freq_shift) 44 | inv_timescales = min_timescale * jnp.exp(jnp.arange(num_timescales, dtype=jnp.float32) * -log_timescale_increment) 45 | emb = jnp.expand_dims(timesteps, 1) * jnp.expand_dims(inv_timescales, 0) 46 | 47 | # scale embeddings 48 | scaled_time = scale * emb 49 | 50 | if flip_sin_to_cos: 51 | signal = jnp.concatenate([jnp.cos(scaled_time), jnp.sin(scaled_time)], axis=1) 52 | else: 53 | signal = jnp.concatenate([jnp.sin(scaled_time), jnp.cos(scaled_time)], axis=1) 54 | signal = jnp.reshape(signal, [jnp.shape(timesteps)[0], embedding_dim]) 55 | return signal 56 | 57 | 58 | class FlaxTimestepEmbedding(nn.Module): 59 | r""" 60 | Time step Embedding Module. Learns embeddings for input time steps. 61 | 62 | Args: 63 | time_embed_dim (`int`, *optional*, defaults to `32`): 64 | Time step embedding dimension 65 | dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): 66 | Parameters `dtype` 67 | """ 68 | time_embed_dim: int = 32 69 | dtype: jnp.dtype = jnp.float32 70 | 71 | @nn.compact 72 | def __call__(self, temb): 73 | temb = nn.Dense(self.time_embed_dim, dtype=self.dtype, name="linear_1")(temb) 74 | temb = nn.silu(temb) 75 | temb = nn.Dense(self.time_embed_dim, dtype=self.dtype, name="linear_2")(temb) 76 | return temb 77 | 78 | 79 | class FlaxTimesteps(nn.Module): 80 | r""" 81 | Wrapper Module for sinusoidal Time step Embeddings as described in https://arxiv.org/abs/2006.11239 82 | 83 | Args: 84 | dim (`int`, *optional*, defaults to `32`): 85 | Time step embedding dimension 86 | """ 87 | dim: int = 32 88 | flip_sin_to_cos: bool = False 89 | freq_shift: float = 1 90 | 91 | @nn.compact 92 | def __call__(self, timesteps): 93 | return get_sinusoidal_embeddings( 94 | timesteps, embedding_dim=self.dim, flip_sin_to_cos=self.flip_sin_to_cos, freq_shift=self.freq_shift 95 | ) 96 | -------------------------------------------------------------------------------- /diffusers/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import TYPE_CHECKING 16 | 17 | from ..utils import DIFFUSERS_SLOW_IMPORT, _LazyModule, is_flax_available, is_torch_available 18 | 19 | 20 | _import_structure = {} 21 | 22 | if is_torch_available(): 23 | _import_structure["adapter"] = ["MultiAdapter", "T2IAdapter"] 24 | _import_structure["autoencoder_asym_kl"] = ["AsymmetricAutoencoderKL"] 25 | _import_structure["autoencoder_kl"] = ["AutoencoderKL"] 26 | _import_structure["autoencoder_tiny"] = ["AutoencoderTiny"] 27 | _import_structure["consistency_decoder_vae"] = ["ConsistencyDecoderVAE"] 28 | _import_structure["controlnet"] = ["ControlNetModel"] 29 | _import_structure["dual_transformer_2d"] = ["DualTransformer2DModel"] 30 | _import_structure["modeling_utils"] = ["ModelMixin"] 31 | _import_structure["prior_transformer"] = ["PriorTransformer"] 32 | _import_structure["t5_film_transformer"] = ["T5FilmDecoder"] 33 | _import_structure["transformer_2d"] = ["Transformer2DModel"] 34 | _import_structure["transformer_temporal"] = ["TransformerTemporalModel"] 35 | _import_structure["unet_1d"] = ["UNet1DModel"] 36 | _import_structure["unet_2d"] = ["UNet2DModel"] 37 | _import_structure["unet_2d_condition"] = ["UNet2DConditionModel"] 38 | _import_structure["unet_3d_condition"] = ["UNet3DConditionModel"] 39 | _import_structure["unet_motion_model"] = ["MotionAdapter", "UNetMotionModel"] 40 | _import_structure["vq_model"] = ["VQModel"] 41 | 42 | if is_flax_available(): 43 | _import_structure["controlnet_flax"] = ["FlaxControlNetModel"] 44 | _import_structure["unet_2d_condition_flax"] = ["FlaxUNet2DConditionModel"] 45 | _import_structure["vae_flax"] = ["FlaxAutoencoderKL"] 46 | 47 | 48 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 49 | if is_torch_available(): 50 | from .adapter import MultiAdapter, T2IAdapter 51 | from .autoencoder_asym_kl import AsymmetricAutoencoderKL 52 | from .autoencoder_kl import AutoencoderKL 53 | from .autoencoder_tiny import AutoencoderTiny 54 | from .consistency_decoder_vae import ConsistencyDecoderVAE 55 | from .controlnet import ControlNetModel 56 | from .dual_transformer_2d import DualTransformer2DModel 57 | from .modeling_utils import ModelMixin 58 | from .prior_transformer import PriorTransformer 59 | from .t5_film_transformer import T5FilmDecoder 60 | from .transformer_2d import Transformer2DModel 61 | from .transformer_temporal import TransformerTemporalModel 62 | from .unet_1d import UNet1DModel 63 | from .unet_2d import UNet2DModel 64 | from .unet_2d_condition import UNet2DConditionModel 65 | from .unet_3d_condition import UNet3DConditionModel 66 | from .unet_motion_model import MotionAdapter, UNetMotionModel 67 | from .vq_model import VQModel 68 | 69 | if is_flax_available(): 70 | from .controlnet_flax import FlaxControlNetModel 71 | from .unet_2d_condition_flax import FlaxUNet2DConditionModel 72 | from .vae_flax import FlaxAutoencoderKL 73 | 74 | else: 75 | import sys 76 | 77 | sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) 78 | -------------------------------------------------------------------------------- /diffusers/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import os 17 | 18 | from packaging import version 19 | 20 | from .. import __version__ 21 | from .constants import ( 22 | CONFIG_NAME, 23 | DEPRECATED_REVISION_ARGS, 24 | DIFFUSERS_CACHE, 25 | DIFFUSERS_DYNAMIC_MODULE_NAME, 26 | FLAX_WEIGHTS_NAME, 27 | HF_MODULES_CACHE, 28 | HUGGINGFACE_CO_RESOLVE_ENDPOINT, 29 | MIN_PEFT_VERSION, 30 | ONNX_EXTERNAL_WEIGHTS_NAME, 31 | ONNX_WEIGHTS_NAME, 32 | SAFETENSORS_WEIGHTS_NAME, 33 | USE_PEFT_BACKEND, 34 | WEIGHTS_NAME, 35 | ) 36 | from .deprecation_utils import deprecate 37 | from .doc_utils import replace_example_docstring 38 | from .dynamic_modules_utils import get_class_from_dynamic_module 39 | from .export_utils import export_to_gif, export_to_obj, export_to_ply, export_to_video 40 | from .hub_utils import ( 41 | HF_HUB_OFFLINE, 42 | PushToHubMixin, 43 | _add_variant, 44 | _get_model_file, 45 | extract_commit_hash, 46 | http_user_agent, 47 | ) 48 | from .import_utils import ( 49 | BACKENDS_MAPPING, 50 | DIFFUSERS_SLOW_IMPORT, 51 | ENV_VARS_TRUE_AND_AUTO_VALUES, 52 | ENV_VARS_TRUE_VALUES, 53 | USE_JAX, 54 | USE_TF, 55 | USE_TORCH, 56 | DummyObject, 57 | OptionalDependencyNotAvailable, 58 | _LazyModule, 59 | get_objects_from_module, 60 | is_accelerate_available, 61 | is_accelerate_version, 62 | is_bs4_available, 63 | is_flax_available, 64 | is_ftfy_available, 65 | is_inflect_available, 66 | is_invisible_watermark_available, 67 | is_k_diffusion_available, 68 | is_k_diffusion_version, 69 | is_librosa_available, 70 | is_note_seq_available, 71 | is_omegaconf_available, 72 | is_onnx_available, 73 | is_peft_available, 74 | is_scipy_available, 75 | is_tensorboard_available, 76 | is_torch_available, 77 | is_torch_version, 78 | is_torch_xla_available, 79 | is_torchsde_available, 80 | is_transformers_available, 81 | is_transformers_version, 82 | is_unidecode_available, 83 | is_wandb_available, 84 | is_xformers_available, 85 | requires_backends, 86 | ) 87 | from .loading_utils import load_image 88 | from .logging import get_logger 89 | from .outputs import BaseOutput 90 | from .peft_utils import ( 91 | check_peft_version, 92 | get_adapter_name, 93 | get_peft_kwargs, 94 | recurse_remove_peft_layers, 95 | scale_lora_layers, 96 | set_adapter_layers, 97 | set_weights_and_activate_adapters, 98 | unscale_lora_layers, 99 | ) 100 | from .pil_utils import PIL_INTERPOLATION, make_image_grid, numpy_to_pil, pt_to_pil 101 | from .state_dict_utils import ( 102 | convert_state_dict_to_diffusers, 103 | convert_state_dict_to_peft, 104 | convert_unet_state_dict_to_peft, 105 | ) 106 | 107 | 108 | logger = get_logger(__name__) 109 | 110 | 111 | def check_min_version(min_version): 112 | if version.parse(__version__) < version.parse(min_version): 113 | if "dev" in min_version: 114 | error_message = ( 115 | "This example requires a source install from HuggingFace diffusers (see " 116 | "`https://huggingface.co/docs/diffusers/installation#install-from-source`)," 117 | ) 118 | else: 119 | error_message = f"This example requires a minimum version of {min_version}," 120 | error_message += f" but the version found is {__version__}.\n" 121 | raise ImportError(error_message) 122 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SSR-Encoder: Encoding Selective Subject Representation for Subject-Driven Generation 2 | 3 | 4 | ![method](assets/teaser.jpg) 5 | Our SSR-Encoder is a model generalizable encoder, which is able to guide any customized diffusion models for single subject-driven image generation (top branch) or multiple subject-driven image generation from different images (middle branch) based on the image representation selected by the text query or mask query without any additional test-time finetuning. Furthermore, our SSR-Encoder can also be applied for the controllable generation with additional control (bottom branch). 6 | ## Method Details 7 | ![method](assets/method.png) 8 | Given a query text-image pairs, the SSR-Encoder employs a token-to-patch aligner to highlight the selective regions in the reference image by the query. It extracts more fine-grained details of the subject through the detail-preserving subject encoder, projecting multi-scale visual embeddings via the token-to-patch aligner. Then, we adopt subject-conditioned generation to generate specific subjects with high fidelity and creative editability. During training, we adopt reconstruction loss and embedding consistency regularization loss for selective subject-driven learning. 9 | 10 | ## Todo List 11 | 1. - [x] inference and training code 12 | 2. - [x] pre-trained weights 13 | 14 | ## Getting Started 15 | ### Environment Setup 16 | Our code is built on the [diffusers](https://github.com/huggingface/diffusers/) version of Stable Diffusion, In our experiments, we use model version v1-5. If you have a diffusers version of [controlnet](https://huggingface.co/docs/diffusers/using-diffusers/controlnet) and [animatediff](https://github.com/guoyww/AnimateDiff) configured, you can use this repository directly. 17 | ```shell 18 | git clone https://github.com/Xiaojiu-z/SSR_Encoder.git 19 | cd SSR_Encoder 20 | ``` 21 | ### Pretrained Models 22 | We recognize the importance of data distribution in learning selective representations during training. To further optimize the model, we balanced and filtered the training data distribution based on different categories such as humans, dogs, cats and so on. After filtering and balancing, we obtained a smaller dataset of 2 million and finetune the original model. This resulted in better selectivety. We provide this checkpoints in [Google Drive](https://drive.google.com/drive/folders/1pMwzGRA8QzHjEH5Hc4zDwdsSDwjwQKMW?usp=sharing). The image encoder we use is [`openai/clip-vit-large-patch14`](https://huggingface.co/openai/clip-vit-large-patch14). Download them and save to the directory `models/ssr_model` and `models/image_encoder`. 23 | 24 | ### Inference 25 | We offer scripts for inference, including text/mask query t2i, combine controlnet, combine animatediff, multiple images and subjects combination. 26 | ```python 27 | python infer_*.py 28 | ``` 29 | 30 | ### Training 31 | The best practice for maintaining selectivity: training from scratch requires a high demand for data distribution, so it is recommended to use our provided pre-trained models and fine-tune them on your own dataset. 32 | ```python 33 | bash train.sh 34 | ``` 35 | 36 | ### Gradio demo 37 | We provide a simple gr demo for more flexible use. 38 | ```python 39 | python gradio_demo.py 40 | ``` 41 | ![method](assets/2.png) 42 | ![method](assets/3.jpg) 43 | ![method](assets/4.jpg) 44 | 45 | ### Limitations 46 | When using, we found that our method requires a balance between selectivity and subject alignment. Due to the distribution of the training data and the highly semantic representation of the CLIP features, our results are poor in some scenarios that require high attention to detail. We plan to improve it in the future. 47 | 48 | For example： 49 | ![method](assets/5.jpg) 50 | 51 | ## Citation 52 | ``` 53 | @inproceedings{zhang2024ssr, 54 | title={Ssr-encoder: Encoding selective subject representation for subject-driven generation}, 55 | author={Zhang, Yuxuan and Song, Yiren and Liu, Jiaming and Wang, Rui and Yu, Jinpeng and Tang, Hao and Li, Huaxia and Tang, Xu and Hu, Yao and Pan, Han and others}, 56 | booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, 57 | pages={8069--8078}, 58 | year={2024} 59 | } 60 | ``` 61 | -------------------------------------------------------------------------------- /diffusers/models/activations.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023 HuggingFace Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import torch 17 | import torch.nn.functional as F 18 | from torch import nn 19 | 20 | from ..utils import USE_PEFT_BACKEND 21 | from .lora import LoRACompatibleLinear 22 | 23 | 24 | ACTIVATION_FUNCTIONS = { 25 | "swish": nn.SiLU(), 26 | "silu": nn.SiLU(), 27 | "mish": nn.Mish(), 28 | "gelu": nn.GELU(), 29 | "relu": nn.ReLU(), 30 | } 31 | 32 | 33 | def get_activation(act_fn: str) -> nn.Module: 34 | """Helper function to get activation function from string. 35 | 36 | Args: 37 | act_fn (str): Name of activation function. 38 | 39 | Returns: 40 | nn.Module: Activation function. 41 | """ 42 | 43 | act_fn = act_fn.lower() 44 | if act_fn in ACTIVATION_FUNCTIONS: 45 | return ACTIVATION_FUNCTIONS[act_fn] 46 | else: 47 | raise ValueError(f"Unsupported activation function: {act_fn}") 48 | 49 | 50 | class GELU(nn.Module): 51 | r""" 52 | GELU activation function with tanh approximation support with `approximate="tanh"`. 53 | 54 | Parameters: 55 | dim_in (`int`): The number of channels in the input. 56 | dim_out (`int`): The number of channels in the output. 57 | approximate (`str`, *optional*, defaults to `"none"`): If `"tanh"`, use tanh approximation. 58 | """ 59 | 60 | def __init__(self, dim_in: int, dim_out: int, approximate: str = "none"): 61 | super().__init__() 62 | self.proj = nn.Linear(dim_in, dim_out) 63 | self.approximate = approximate 64 | 65 | def gelu(self, gate: torch.Tensor) -> torch.Tensor: 66 | if gate.device.type != "mps": 67 | return F.gelu(gate, approximate=self.approximate) 68 | # mps: gelu is not implemented for float16 69 | return F.gelu(gate.to(dtype=torch.float32), approximate=self.approximate).to(dtype=gate.dtype) 70 | 71 | def forward(self, hidden_states): 72 | hidden_states = self.proj(hidden_states) 73 | hidden_states = self.gelu(hidden_states) 74 | return hidden_states 75 | 76 | 77 | class GEGLU(nn.Module): 78 | r""" 79 | A [variant](https://arxiv.org/abs/2002.05202) of the gated linear unit activation function. 80 | 81 | Parameters: 82 | dim_in (`int`): The number of channels in the input. 83 | dim_out (`int`): The number of channels in the output. 84 | """ 85 | 86 | def __init__(self, dim_in: int, dim_out: int): 87 | super().__init__() 88 | linear_cls = LoRACompatibleLinear if not USE_PEFT_BACKEND else nn.Linear 89 | 90 | self.proj = linear_cls(dim_in, dim_out * 2) 91 | 92 | def gelu(self, gate: torch.Tensor) -> torch.Tensor: 93 | if gate.device.type != "mps": 94 | return F.gelu(gate) 95 | # mps: gelu is not implemented for float16 96 | return F.gelu(gate.to(dtype=torch.float32)).to(dtype=gate.dtype) 97 | 98 | def forward(self, hidden_states, scale: float = 1.0): 99 | args = () if USE_PEFT_BACKEND else (scale,) 100 | hidden_states, gate = self.proj(hidden_states, *args).chunk(2, dim=-1) 101 | return hidden_states * self.gelu(gate) 102 | 103 | 104 | class ApproximateGELU(nn.Module): 105 | r""" 106 | The approximate form of the Gaussian Error Linear Unit (GELU). For more details, see section 2 of this 107 | [paper](https://arxiv.org/abs/1606.08415). 108 | 109 | Parameters: 110 | dim_in (`int`): The number of channels in the input. 111 | dim_out (`int`): The number of channels in the output. 112 | """ 113 | 114 | def __init__(self, dim_in: int, dim_out: int): 115 | super().__init__() 116 | self.proj = nn.Linear(dim_in, dim_out) 117 | 118 | def forward(self, x: torch.Tensor) -> torch.Tensor: 119 | x = self.proj(x) 120 | return x * torch.sigmoid(1.702 * x) 121 | --------------------------------------------------------------------------------