├── .gitignore ├── LICENSE ├── README.md ├── docs ├── consistent_style.jpg ├── incremental_editing.jpg ├── teaser.jpg └── text_editing.jpg ├── environment.yml ├── inputs └── bottle.jpg ├── run_stable_flow.py ├── setup.py └── src └── diffusers ├── __init__.py ├── callbacks.py ├── commands ├── __init__.py ├── diffusers_cli.py ├── env.py └── fp16_safetensors.py ├── configuration_utils.py ├── dependency_versions_check.py ├── dependency_versions_table.py ├── experimental ├── README.md ├── __init__.py └── rl │ ├── __init__.py │ └── value_guided_sampling.py ├── image_processor.py ├── loaders ├── __init__.py ├── ip_adapter.py ├── lora_base.py ├── lora_conversion_utils.py ├── lora_pipeline.py ├── peft.py ├── single_file.py ├── single_file_model.py ├── single_file_utils.py ├── textual_inversion.py ├── unet.py ├── unet_loader_utils.py └── utils.py ├── models ├── README.md ├── __init__.py ├── activations.py ├── adapter.py ├── attention.py ├── attention_flax.py ├── attention_processor.py ├── autoencoders │ ├── __init__.py │ ├── autoencoder_asym_kl.py │ ├── autoencoder_kl.py │ ├── autoencoder_kl_cogvideox.py │ ├── autoencoder_kl_temporal_decoder.py │ ├── autoencoder_oobleck.py │ ├── autoencoder_tiny.py │ ├── consistency_decoder_vae.py │ ├── vae.py │ └── vq_model.py ├── controlnet.py ├── controlnet_flax.py ├── controlnet_hunyuan.py ├── controlnet_sd3.py ├── controlnet_sparsectrl.py ├── controlnet_xs.py ├── downsampling.py ├── embeddings.py ├── embeddings_flax.py ├── lora.py ├── model_loading_utils.py ├── modeling_flax_pytorch_utils.py ├── modeling_flax_utils.py ├── modeling_outputs.py ├── modeling_pytorch_flax_utils.py ├── modeling_utils.py ├── normalization.py ├── resnet.py ├── resnet_flax.py ├── transformers │ ├── __init__.py │ ├── auraflow_transformer_2d.py │ ├── cogvideox_transformer_3d.py │ ├── dit_transformer_2d.py │ ├── dual_transformer_2d.py │ ├── hunyuan_transformer_2d.py │ ├── latte_transformer_3d.py │ ├── lumina_nextdit2d.py │ ├── pixart_transformer_2d.py │ ├── prior_transformer.py │ ├── stable_audio_transformer.py │ ├── t5_film_transformer.py │ ├── transformer_2d.py │ ├── transformer_flux.py │ ├── transformer_sd3.py │ └── transformer_temporal.py ├── unets │ ├── __init__.py │ ├── unet_1d.py │ ├── unet_1d_blocks.py │ ├── unet_2d.py │ ├── unet_2d_blocks.py │ ├── unet_2d_blocks_flax.py │ ├── unet_2d_condition.py │ ├── unet_2d_condition_flax.py │ ├── unet_3d_blocks.py │ ├── unet_3d_condition.py │ ├── unet_i2vgen_xl.py │ ├── unet_kandinsky3.py │ ├── unet_motion_model.py │ ├── unet_spatio_temporal_condition.py │ ├── unet_stable_cascade.py │ └── uvit_2d.py ├── upsampling.py ├── vae_flax.py └── vq_model.py ├── optimization.py ├── pipelines ├── README.md ├── __init__.py ├── amused │ ├── __init__.py │ ├── pipeline_amused.py │ ├── pipeline_amused_img2img.py │ └── pipeline_amused_inpaint.py ├── animatediff │ ├── __init__.py │ ├── pipeline_animatediff.py │ ├── pipeline_animatediff_controlnet.py │ ├── pipeline_animatediff_sdxl.py │ ├── pipeline_animatediff_sparsectrl.py │ ├── pipeline_animatediff_video2video.py │ └── pipeline_output.py ├── audioldm │ ├── __init__.py │ └── pipeline_audioldm.py ├── audioldm2 │ ├── __init__.py │ ├── modeling_audioldm2.py │ └── pipeline_audioldm2.py ├── aura_flow │ ├── __init__.py │ └── pipeline_aura_flow.py ├── auto_pipeline.py ├── blip_diffusion │ ├── __init__.py │ ├── blip_image_processing.py │ ├── modeling_blip2.py │ ├── modeling_ctx_clip.py │ └── pipeline_blip_diffusion.py ├── cogvideo │ ├── __init__.py │ └── pipeline_cogvideox.py ├── consistency_models │ ├── __init__.py │ └── pipeline_consistency_models.py ├── controlnet │ ├── __init__.py │ ├── multicontrolnet.py │ ├── pipeline_controlnet.py │ ├── pipeline_controlnet_blip_diffusion.py │ ├── pipeline_controlnet_img2img.py │ ├── pipeline_controlnet_inpaint.py │ ├── pipeline_controlnet_inpaint_sd_xl.py │ ├── pipeline_controlnet_sd_xl.py │ ├── pipeline_controlnet_sd_xl_img2img.py │ └── pipeline_flax_controlnet.py ├── controlnet_hunyuandit │ ├── __init__.py │ └── pipeline_hunyuandit_controlnet.py ├── controlnet_sd3 │ ├── __init__.py │ └── pipeline_stable_diffusion_3_controlnet.py ├── controlnet_xs │ ├── __init__.py │ ├── pipeline_controlnet_xs.py │ └── pipeline_controlnet_xs_sd_xl.py ├── dance_diffusion │ ├── __init__.py │ └── pipeline_dance_diffusion.py ├── ddim │ ├── __init__.py │ └── pipeline_ddim.py ├── ddpm │ ├── __init__.py │ └── pipeline_ddpm.py ├── deepfloyd_if │ ├── __init__.py │ ├── pipeline_if.py │ ├── pipeline_if_img2img.py │ ├── pipeline_if_img2img_superresolution.py │ ├── pipeline_if_inpainting.py │ ├── pipeline_if_inpainting_superresolution.py │ ├── pipeline_if_superresolution.py │ ├── pipeline_output.py │ ├── safety_checker.py │ ├── timesteps.py │ └── watermark.py ├── deprecated │ ├── README.md │ ├── __init__.py │ ├── alt_diffusion │ │ ├── __init__.py │ │ ├── modeling_roberta_series.py │ │ ├── pipeline_alt_diffusion.py │ │ ├── pipeline_alt_diffusion_img2img.py │ │ └── pipeline_output.py │ ├── audio_diffusion │ │ ├── __init__.py │ │ ├── mel.py │ │ └── pipeline_audio_diffusion.py │ ├── latent_diffusion_uncond │ │ ├── __init__.py │ │ └── pipeline_latent_diffusion_uncond.py │ ├── pndm │ │ ├── __init__.py │ │ └── pipeline_pndm.py │ ├── repaint │ │ ├── __init__.py │ │ └── pipeline_repaint.py │ ├── score_sde_ve │ │ ├── __init__.py │ │ └── pipeline_score_sde_ve.py │ ├── spectrogram_diffusion │ │ ├── __init__.py │ │ ├── continuous_encoder.py │ │ ├── midi_utils.py │ │ ├── notes_encoder.py │ │ └── pipeline_spectrogram_diffusion.py │ ├── stable_diffusion_variants │ │ ├── __init__.py │ │ ├── pipeline_cycle_diffusion.py │ │ ├── pipeline_onnx_stable_diffusion_inpaint_legacy.py │ │ ├── pipeline_stable_diffusion_inpaint_legacy.py │ │ ├── pipeline_stable_diffusion_model_editing.py │ │ ├── pipeline_stable_diffusion_paradigms.py │ │ └── pipeline_stable_diffusion_pix2pix_zero.py │ ├── stochastic_karras_ve │ │ ├── __init__.py │ │ └── pipeline_stochastic_karras_ve.py │ ├── versatile_diffusion │ │ ├── __init__.py │ │ ├── modeling_text_unet.py │ │ ├── pipeline_versatile_diffusion.py │ │ ├── pipeline_versatile_diffusion_dual_guided.py │ │ ├── pipeline_versatile_diffusion_image_variation.py │ │ └── pipeline_versatile_diffusion_text_to_image.py │ └── vq_diffusion │ │ ├── __init__.py │ │ └── pipeline_vq_diffusion.py ├── dit │ ├── __init__.py │ └── pipeline_dit.py ├── flux │ ├── __init__.py │ ├── pipeline_flux.py │ └── pipeline_output.py ├── free_init_utils.py ├── free_noise_utils.py ├── hunyuandit │ ├── __init__.py │ └── pipeline_hunyuandit.py ├── i2vgen_xl │ ├── __init__.py │ └── pipeline_i2vgen_xl.py ├── kandinsky │ ├── __init__.py │ ├── pipeline_kandinsky.py │ ├── pipeline_kandinsky_combined.py │ ├── pipeline_kandinsky_img2img.py │ ├── pipeline_kandinsky_inpaint.py │ ├── pipeline_kandinsky_prior.py │ └── text_encoder.py ├── kandinsky2_2 │ ├── __init__.py │ ├── pipeline_kandinsky2_2.py │ ├── pipeline_kandinsky2_2_combined.py │ ├── pipeline_kandinsky2_2_controlnet.py │ ├── pipeline_kandinsky2_2_controlnet_img2img.py │ ├── pipeline_kandinsky2_2_img2img.py │ ├── pipeline_kandinsky2_2_inpainting.py │ ├── pipeline_kandinsky2_2_prior.py │ └── pipeline_kandinsky2_2_prior_emb2emb.py ├── kandinsky3 │ ├── __init__.py │ ├── convert_kandinsky3_unet.py │ ├── pipeline_kandinsky3.py │ └── pipeline_kandinsky3_img2img.py ├── kolors │ ├── __init__.py │ ├── pipeline_kolors.py │ ├── pipeline_kolors_img2img.py │ ├── pipeline_output.py │ ├── text_encoder.py │ └── tokenizer.py ├── latent_consistency_models │ ├── __init__.py │ ├── pipeline_latent_consistency_img2img.py │ └── pipeline_latent_consistency_text2img.py ├── latent_diffusion │ ├── __init__.py │ ├── pipeline_latent_diffusion.py │ └── pipeline_latent_diffusion_superresolution.py ├── latte │ ├── __init__.py │ └── pipeline_latte.py ├── ledits_pp │ ├── __init__.py │ ├── pipeline_leditspp_stable_diffusion.py │ ├── pipeline_leditspp_stable_diffusion_xl.py │ └── pipeline_output.py ├── lumina │ ├── __init__.py │ └── pipeline_lumina.py ├── marigold │ ├── __init__.py │ ├── marigold_image_processing.py │ ├── pipeline_marigold_depth.py │ └── pipeline_marigold_normals.py ├── musicldm │ ├── __init__.py │ └── pipeline_musicldm.py ├── onnx_utils.py ├── pag │ ├── __init__.py │ ├── pag_utils.py │ ├── pipeline_pag_controlnet_sd.py │ ├── pipeline_pag_controlnet_sd_xl.py │ ├── pipeline_pag_hunyuandit.py │ ├── pipeline_pag_kolors.py │ ├── pipeline_pag_pixart_sigma.py │ ├── pipeline_pag_sd.py │ ├── pipeline_pag_sd_3.py │ ├── pipeline_pag_sd_animatediff.py │ ├── pipeline_pag_sd_xl.py │ ├── pipeline_pag_sd_xl_img2img.py │ └── pipeline_pag_sd_xl_inpaint.py ├── paint_by_example │ ├── __init__.py │ ├── image_encoder.py │ └── pipeline_paint_by_example.py ├── pia │ ├── __init__.py │ └── pipeline_pia.py ├── pipeline_flax_utils.py ├── pipeline_loading_utils.py ├── pipeline_utils.py ├── pixart_alpha │ ├── __init__.py │ ├── pipeline_pixart_alpha.py │ └── pipeline_pixart_sigma.py ├── semantic_stable_diffusion │ ├── __init__.py │ ├── pipeline_output.py │ └── pipeline_semantic_stable_diffusion.py ├── shap_e │ ├── __init__.py │ ├── camera.py │ ├── pipeline_shap_e.py │ ├── pipeline_shap_e_img2img.py │ └── renderer.py ├── stable_audio │ ├── __init__.py │ ├── modeling_stable_audio.py │ └── pipeline_stable_audio.py ├── stable_cascade │ ├── __init__.py │ ├── pipeline_stable_cascade.py │ ├── pipeline_stable_cascade_combined.py │ └── pipeline_stable_cascade_prior.py ├── stable_diffusion │ ├── README.md │ ├── __init__.py │ ├── clip_image_project_model.py │ ├── convert_from_ckpt.py │ ├── pipeline_flax_stable_diffusion.py │ ├── pipeline_flax_stable_diffusion_img2img.py │ ├── pipeline_flax_stable_diffusion_inpaint.py │ ├── pipeline_onnx_stable_diffusion.py │ ├── pipeline_onnx_stable_diffusion_img2img.py │ ├── pipeline_onnx_stable_diffusion_inpaint.py │ ├── pipeline_onnx_stable_diffusion_upscale.py │ ├── pipeline_output.py │ ├── pipeline_stable_diffusion.py │ ├── pipeline_stable_diffusion_depth2img.py │ ├── pipeline_stable_diffusion_image_variation.py │ ├── pipeline_stable_diffusion_img2img.py │ ├── pipeline_stable_diffusion_inpaint.py │ ├── pipeline_stable_diffusion_instruct_pix2pix.py │ ├── pipeline_stable_diffusion_latent_upscale.py │ ├── pipeline_stable_diffusion_upscale.py │ ├── pipeline_stable_unclip.py │ ├── pipeline_stable_unclip_img2img.py │ ├── safety_checker.py │ ├── safety_checker_flax.py │ └── stable_unclip_image_normalizer.py ├── stable_diffusion_3 │ ├── __init__.py │ ├── pipeline_output.py │ ├── pipeline_stable_diffusion_3.py │ ├── pipeline_stable_diffusion_3_img2img.py │ └── pipeline_stable_diffusion_3_inpaint.py ├── stable_diffusion_attend_and_excite │ ├── __init__.py │ └── pipeline_stable_diffusion_attend_and_excite.py ├── stable_diffusion_diffedit │ ├── __init__.py │ └── pipeline_stable_diffusion_diffedit.py ├── stable_diffusion_gligen │ ├── __init__.py │ ├── pipeline_stable_diffusion_gligen.py │ └── pipeline_stable_diffusion_gligen_text_image.py ├── stable_diffusion_k_diffusion │ ├── __init__.py │ ├── pipeline_stable_diffusion_k_diffusion.py │ └── pipeline_stable_diffusion_xl_k_diffusion.py ├── stable_diffusion_ldm3d │ ├── __init__.py │ └── pipeline_stable_diffusion_ldm3d.py ├── stable_diffusion_panorama │ ├── __init__.py │ └── pipeline_stable_diffusion_panorama.py ├── stable_diffusion_safe │ ├── __init__.py │ ├── pipeline_output.py │ ├── pipeline_stable_diffusion_safe.py │ └── safety_checker.py ├── stable_diffusion_sag │ ├── __init__.py │ └── pipeline_stable_diffusion_sag.py ├── stable_diffusion_xl │ ├── __init__.py │ ├── pipeline_flax_stable_diffusion_xl.py │ ├── pipeline_output.py │ ├── pipeline_stable_diffusion_xl.py │ ├── pipeline_stable_diffusion_xl_img2img.py │ ├── pipeline_stable_diffusion_xl_inpaint.py │ ├── pipeline_stable_diffusion_xl_instruct_pix2pix.py │ └── watermark.py ├── stable_video_diffusion │ ├── __init__.py │ └── pipeline_stable_video_diffusion.py ├── t2i_adapter │ ├── __init__.py │ ├── pipeline_stable_diffusion_adapter.py │ └── pipeline_stable_diffusion_xl_adapter.py ├── text_to_video_synthesis │ ├── __init__.py │ ├── pipeline_output.py │ ├── pipeline_text_to_video_synth.py │ ├── pipeline_text_to_video_synth_img2img.py │ ├── pipeline_text_to_video_zero.py │ └── pipeline_text_to_video_zero_sdxl.py ├── unclip │ ├── __init__.py │ ├── pipeline_unclip.py │ ├── pipeline_unclip_image_variation.py │ └── text_proj.py ├── unidiffuser │ ├── __init__.py │ ├── modeling_text_decoder.py │ ├── modeling_uvit.py │ └── pipeline_unidiffuser.py └── wuerstchen │ ├── __init__.py │ ├── modeling_paella_vq_model.py │ ├── modeling_wuerstchen_common.py │ ├── modeling_wuerstchen_diffnext.py │ ├── modeling_wuerstchen_prior.py │ ├── pipeline_wuerstchen.py │ ├── pipeline_wuerstchen_combined.py │ └── pipeline_wuerstchen_prior.py ├── py.typed ├── schedulers ├── README.md ├── __init__.py ├── deprecated │ ├── __init__.py │ ├── scheduling_karras_ve.py │ └── scheduling_sde_vp.py ├── scheduling_amused.py ├── scheduling_consistency_decoder.py ├── scheduling_consistency_models.py ├── scheduling_cosine_dpmsolver_multistep.py ├── scheduling_ddim.py ├── scheduling_ddim_cogvideox.py ├── scheduling_ddim_flax.py ├── scheduling_ddim_inverse.py ├── scheduling_ddim_parallel.py ├── scheduling_ddpm.py ├── scheduling_ddpm_flax.py ├── scheduling_ddpm_parallel.py ├── scheduling_ddpm_wuerstchen.py ├── scheduling_deis_multistep.py ├── scheduling_dpm_cogvideox.py ├── scheduling_dpmsolver_multistep.py ├── scheduling_dpmsolver_multistep_flax.py ├── scheduling_dpmsolver_multistep_inverse.py ├── scheduling_dpmsolver_sde.py ├── scheduling_dpmsolver_singlestep.py ├── scheduling_edm_dpmsolver_multistep.py ├── scheduling_edm_euler.py ├── scheduling_euler_ancestral_discrete.py ├── scheduling_euler_discrete.py ├── scheduling_euler_discrete_flax.py ├── scheduling_flow_match_euler_discrete.py ├── scheduling_flow_match_heun_discrete.py ├── scheduling_heun_discrete.py ├── scheduling_ipndm.py ├── scheduling_k_dpm_2_ancestral_discrete.py ├── scheduling_k_dpm_2_discrete.py ├── scheduling_karras_ve_flax.py ├── scheduling_lcm.py ├── scheduling_lms_discrete.py ├── scheduling_lms_discrete_flax.py ├── scheduling_pndm.py ├── scheduling_pndm_flax.py ├── scheduling_repaint.py ├── scheduling_sasolver.py ├── scheduling_sde_ve.py ├── scheduling_sde_ve_flax.py ├── scheduling_tcd.py ├── scheduling_unclip.py ├── scheduling_unipc_multistep.py ├── scheduling_utils.py ├── scheduling_utils_flax.py └── scheduling_vq_diffusion.py ├── training_utils.py ├── utils ├── __init__.py ├── accelerate_utils.py ├── constants.py ├── deprecation_utils.py ├── doc_utils.py ├── dummy_flax_and_transformers_objects.py ├── dummy_flax_objects.py ├── dummy_note_seq_objects.py ├── dummy_onnx_objects.py ├── dummy_pt_objects.py ├── dummy_torch_and_librosa_objects.py ├── dummy_torch_and_scipy_objects.py ├── dummy_torch_and_torchsde_objects.py ├── dummy_torch_and_transformers_and_k_diffusion_objects.py ├── dummy_torch_and_transformers_and_onnx_objects.py ├── dummy_torch_and_transformers_and_sentencepiece_objects.py ├── dummy_torch_and_transformers_objects.py ├── dummy_transformers_and_torch_and_note_seq_objects.py ├── dynamic_modules_utils.py ├── export_utils.py ├── hub_utils.py ├── import_utils.py ├── loading_utils.py ├── logging.py ├── model_card_template.md ├── outputs.py ├── peft_utils.py ├── pil_utils.py ├── state_dict_utils.py ├── testing_utils.py ├── torch_utils.py └── versions.py └── video_processor.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Initially taken from GitHub's Python gitignore file 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # tests and logs 12 | tests/fixtures/cached_*_text.txt 13 | logs/ 14 | lightning_logs/ 15 | lang_code_data/ 16 | 17 | # Distribution / packaging 18 | .Python 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | MANIFEST 35 | 36 | # PyInstaller 37 | # Usually these files are written by a Python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .nox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | db.sqlite3 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | .python-version 90 | 91 | # celery beat schedule file 92 | celerybeat-schedule 93 | 94 | # SageMath parsed files 95 | *.sage.py 96 | 97 | # Environments 98 | .env 99 | .venv 100 | env/ 101 | venv/ 102 | ENV/ 103 | env.bak/ 104 | venv.bak/ 105 | 106 | # Spyder project settings 107 | .spyderproject 108 | .spyproject 109 | 110 | # Rope project settings 111 | .ropeproject 112 | 113 | # mkdocs documentation 114 | /site 115 | 116 | # mypy 117 | .mypy_cache/ 118 | .dmypy.json 119 | dmypy.json 120 | 121 | # Pyre type checker 122 | .pyre/ 123 | 124 | # vscode 125 | .vs 126 | .vscode 127 | 128 | # Pycharm 129 | .idea 130 | 131 | # TF code 132 | tensorflow_code 133 | 134 | # Models 135 | proc_data 136 | 137 | # examples 138 | runs 139 | /runs_old 140 | /wandb 141 | /examples/runs 142 | /examples/**/*.args 143 | /examples/rag/sweep 144 | 145 | # data 146 | /data 147 | serialization_dir 148 | 149 | # emacs 150 | *.*~ 151 | debug.env 152 | 153 | # vim 154 | .*.swp 155 | 156 | # ctags 157 | tags 158 | 159 | # pre-commit 160 | .pre-commit* 161 | 162 | # .lock 163 | *.lock 164 | 165 | # DS_Store (MacOS) 166 | .DS_Store 167 | 168 | # RL pipelines may produce mp4 outputs 169 | *.mp4 170 | 171 | # dependencies 172 | /transformers 173 | 174 | # ruff 175 | .ruff_cache 176 | 177 | # wandb 178 | wandb 179 | 180 | # Projest stuff 181 | outputs/ -------------------------------------------------------------------------------- /docs/consistent_style.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snap-research/stable-flow/c99ac0eba297e7884bf2118be79734b63543ee45/docs/consistent_style.jpg -------------------------------------------------------------------------------- /docs/incremental_editing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snap-research/stable-flow/c99ac0eba297e7884bf2118be79734b63543ee45/docs/incremental_editing.jpg -------------------------------------------------------------------------------- /docs/teaser.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snap-research/stable-flow/c99ac0eba297e7884bf2118be79734b63543ee45/docs/teaser.jpg -------------------------------------------------------------------------------- /docs/text_editing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snap-research/stable-flow/c99ac0eba297e7884bf2118be79734b63543ee45/docs/text_editing.jpg -------------------------------------------------------------------------------- /inputs/bottle.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snap-research/stable-flow/c99ac0eba297e7884bf2118be79734b63543ee45/inputs/bottle.jpg -------------------------------------------------------------------------------- /src/diffusers/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from abc import ABC, abstractmethod 16 | from argparse import ArgumentParser 17 | 18 | 19 | class BaseDiffusersCLICommand(ABC): 20 | @staticmethod 21 | @abstractmethod 22 | def register_subcommand(parser: ArgumentParser): 23 | raise NotImplementedError() 24 | 25 | @abstractmethod 26 | def run(self): 27 | raise NotImplementedError() 28 | -------------------------------------------------------------------------------- /src/diffusers/commands/diffusers_cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2024 The HuggingFace Team. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from argparse import ArgumentParser 17 | 18 | from .env import EnvironmentCommand 19 | from .fp16_safetensors import FP16SafetensorsCommand 20 | 21 | 22 | def main(): 23 | parser = ArgumentParser("Diffusers CLI tool", usage="diffusers-cli []") 24 | commands_parser = parser.add_subparsers(help="diffusers-cli command helpers") 25 | 26 | # Register commands 27 | EnvironmentCommand.register_subcommand(commands_parser) 28 | FP16SafetensorsCommand.register_subcommand(commands_parser) 29 | 30 | # Let's go 31 | args = parser.parse_args() 32 | 33 | if not hasattr(args, "func"): 34 | parser.print_help() 35 | exit(1) 36 | 37 | # Run 38 | service = args.func(args) 39 | service.run() 40 | 41 | 42 | if __name__ == "__main__": 43 | main() 44 | -------------------------------------------------------------------------------- /src/diffusers/dependency_versions_check.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .dependency_versions_table import deps 16 | from .utils.versions import require_version, require_version_core 17 | 18 | 19 | # define which module versions we always want to check at run time 20 | # (usually the ones defined in `install_requires` in setup.py) 21 | # 22 | # order specific notes: 23 | # - tqdm must be checked before tokenizers 24 | 25 | pkgs_to_check_at_runtime = "python requests filelock numpy".split() 26 | for pkg in pkgs_to_check_at_runtime: 27 | if pkg in deps: 28 | require_version_core(deps[pkg]) 29 | else: 30 | raise ValueError(f"can't find {pkg} in {deps.keys()}, check dependency_versions_table.py") 31 | 32 | 33 | def dep_version_check(pkg, hint=None): 34 | require_version(deps[pkg], hint) 35 | -------------------------------------------------------------------------------- /src/diffusers/dependency_versions_table.py: -------------------------------------------------------------------------------- 1 | # THIS FILE HAS BEEN AUTOGENERATED. To update: 2 | # 1. modify the `_deps` dict in setup.py 3 | # 2. run `make deps_table_update` 4 | deps = { 5 | "Pillow": "Pillow", 6 | "accelerate": "accelerate>=0.31.0", 7 | "compel": "compel==0.1.8", 8 | "datasets": "datasets", 9 | "filelock": "filelock", 10 | "flax": "flax>=0.4.1", 11 | "hf-doc-builder": "hf-doc-builder>=0.3.0", 12 | "huggingface-hub": "huggingface-hub>=0.23.2", 13 | "requests-mock": "requests-mock==1.10.0", 14 | "importlib_metadata": "importlib_metadata", 15 | "invisible-watermark": "invisible-watermark>=0.2.0", 16 | "isort": "isort>=5.5.4", 17 | "jax": "jax>=0.4.1", 18 | "jaxlib": "jaxlib>=0.4.1", 19 | "Jinja2": "Jinja2", 20 | "k-diffusion": "k-diffusion>=0.0.12", 21 | "torchsde": "torchsde", 22 | "note_seq": "note_seq", 23 | "librosa": "librosa", 24 | "numpy": "numpy", 25 | "parameterized": "parameterized", 26 | "peft": "peft>=0.6.0", 27 | "protobuf": "protobuf>=3.20.3,<4", 28 | "pytest": "pytest", 29 | "pytest-timeout": "pytest-timeout", 30 | "pytest-xdist": "pytest-xdist", 31 | "python": "python>=3.8.0", 32 | "ruff": "ruff==0.1.5", 33 | "safetensors": "safetensors>=0.3.1", 34 | "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92", 35 | "GitPython": "GitPython<3.1.19", 36 | "scipy": "scipy", 37 | "onnx": "onnx", 38 | "regex": "regex!=2019.12.17", 39 | "requests": "requests", 40 | "tensorboard": "tensorboard", 41 | "torch": "torch>=1.4", 42 | "torchvision": "torchvision", 43 | "transformers": "transformers>=4.41.2", 44 | "urllib3": "urllib3<=2.0.0", 45 | "black": "black", 46 | } 47 | -------------------------------------------------------------------------------- /src/diffusers/experimental/README.md: -------------------------------------------------------------------------------- 1 | # 🧨 Diffusers Experimental 2 | 3 | We are adding experimental code to support novel applications and usages of the Diffusers library. 4 | Currently, the following experiments are supported: 5 | * Reinforcement learning via an implementation of the [Diffuser](https://arxiv.org/abs/2205.09991) model. -------------------------------------------------------------------------------- /src/diffusers/experimental/__init__.py: -------------------------------------------------------------------------------- 1 | from .rl import ValueGuidedRLPipeline 2 | -------------------------------------------------------------------------------- /src/diffusers/experimental/rl/__init__.py: -------------------------------------------------------------------------------- 1 | from .value_guided_sampling import ValueGuidedRLPipeline 2 | -------------------------------------------------------------------------------- /src/diffusers/loaders/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ..utils import DIFFUSERS_SLOW_IMPORT, _LazyModule, deprecate 4 | from ..utils.import_utils import is_peft_available, is_torch_available, is_transformers_available 5 | 6 | 7 | def text_encoder_lora_state_dict(text_encoder): 8 | deprecate( 9 | "text_encoder_load_state_dict in `models`", 10 | "0.27.0", 11 | "`text_encoder_lora_state_dict` is deprecated and will be removed in 0.27.0. Make sure to retrieve the weights using `get_peft_model`. See https://huggingface.co/docs/peft/v0.6.2/en/quicktour#peftmodel for more information.", 12 | ) 13 | state_dict = {} 14 | 15 | for name, module in text_encoder_attn_modules(text_encoder): 16 | for k, v in module.q_proj.lora_linear_layer.state_dict().items(): 17 | state_dict[f"{name}.q_proj.lora_linear_layer.{k}"] = v 18 | 19 | for k, v in module.k_proj.lora_linear_layer.state_dict().items(): 20 | state_dict[f"{name}.k_proj.lora_linear_layer.{k}"] = v 21 | 22 | for k, v in module.v_proj.lora_linear_layer.state_dict().items(): 23 | state_dict[f"{name}.v_proj.lora_linear_layer.{k}"] = v 24 | 25 | for k, v in module.out_proj.lora_linear_layer.state_dict().items(): 26 | state_dict[f"{name}.out_proj.lora_linear_layer.{k}"] = v 27 | 28 | return state_dict 29 | 30 | 31 | if is_transformers_available(): 32 | 33 | def text_encoder_attn_modules(text_encoder): 34 | deprecate( 35 | "text_encoder_attn_modules in `models`", 36 | "0.27.0", 37 | "`text_encoder_lora_state_dict` is deprecated and will be removed in 0.27.0. Make sure to retrieve the weights using `get_peft_model`. See https://huggingface.co/docs/peft/v0.6.2/en/quicktour#peftmodel for more information.", 38 | ) 39 | from transformers import CLIPTextModel, CLIPTextModelWithProjection 40 | 41 | attn_modules = [] 42 | 43 | if isinstance(text_encoder, (CLIPTextModel, CLIPTextModelWithProjection)): 44 | for i, layer in enumerate(text_encoder.text_model.encoder.layers): 45 | name = f"text_model.encoder.layers.{i}.self_attn" 46 | mod = layer.self_attn 47 | attn_modules.append((name, mod)) 48 | else: 49 | raise ValueError(f"do not know how to get attention modules for: {text_encoder.__class__.__name__}") 50 | 51 | return attn_modules 52 | 53 | 54 | _import_structure = {} 55 | 56 | if is_torch_available(): 57 | _import_structure["single_file_model"] = ["FromOriginalModelMixin"] 58 | 59 | _import_structure["unet"] = ["UNet2DConditionLoadersMixin"] 60 | _import_structure["utils"] = ["AttnProcsLayers"] 61 | if is_transformers_available(): 62 | _import_structure["single_file"] = ["FromSingleFileMixin"] 63 | _import_structure["lora_pipeline"] = [ 64 | "AmusedLoraLoaderMixin", 65 | "StableDiffusionLoraLoaderMixin", 66 | "SD3LoraLoaderMixin", 67 | "StableDiffusionXLLoraLoaderMixin", 68 | "LoraLoaderMixin", 69 | "FluxLoraLoaderMixin", 70 | ] 71 | _import_structure["textual_inversion"] = ["TextualInversionLoaderMixin"] 72 | _import_structure["ip_adapter"] = ["IPAdapterMixin"] 73 | 74 | _import_structure["peft"] = ["PeftAdapterMixin"] 75 | 76 | 77 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 78 | if is_torch_available(): 79 | from .single_file_model import FromOriginalModelMixin 80 | from .unet import UNet2DConditionLoadersMixin 81 | from .utils import AttnProcsLayers 82 | 83 | if is_transformers_available(): 84 | from .ip_adapter import IPAdapterMixin 85 | from .lora_pipeline import ( 86 | AmusedLoraLoaderMixin, 87 | FluxLoraLoaderMixin, 88 | LoraLoaderMixin, 89 | SD3LoraLoaderMixin, 90 | StableDiffusionLoraLoaderMixin, 91 | StableDiffusionXLLoraLoaderMixin, 92 | ) 93 | from .single_file import FromSingleFileMixin 94 | from .textual_inversion import TextualInversionLoaderMixin 95 | 96 | from .peft import PeftAdapterMixin 97 | else: 98 | import sys 99 | 100 | sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) 101 | -------------------------------------------------------------------------------- /src/diffusers/loaders/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Dict 16 | 17 | import torch 18 | 19 | 20 | class AttnProcsLayers(torch.nn.Module): 21 | def __init__(self, state_dict: Dict[str, torch.Tensor]): 22 | super().__init__() 23 | self.layers = torch.nn.ModuleList(state_dict.values()) 24 | self.mapping = dict(enumerate(state_dict.keys())) 25 | self.rev_mapping = {v: k for k, v in enumerate(state_dict.keys())} 26 | 27 | # .processor for unet, .self_attn for text encoder 28 | self.split_keys = [".processor", ".self_attn"] 29 | 30 | # we add a hook to state_dict() and load_state_dict() so that the 31 | # naming fits with `unet.attn_processors` 32 | def map_to(module, state_dict, *args, **kwargs): 33 | new_state_dict = {} 34 | for key, value in state_dict.items(): 35 | num = int(key.split(".")[1]) # 0 is always "layers" 36 | new_key = key.replace(f"layers.{num}", module.mapping[num]) 37 | new_state_dict[new_key] = value 38 | 39 | return new_state_dict 40 | 41 | def remap_key(key, state_dict): 42 | for k in self.split_keys: 43 | if k in key: 44 | return key.split(k)[0] + k 45 | 46 | raise ValueError( 47 | f"There seems to be a problem with the state_dict: {set(state_dict.keys())}. {key} has to have one of {self.split_keys}." 48 | ) 49 | 50 | def map_from(module, state_dict, *args, **kwargs): 51 | all_keys = list(state_dict.keys()) 52 | for key in all_keys: 53 | replace_key = remap_key(key, state_dict) 54 | new_key = key.replace(replace_key, f"layers.{module.rev_mapping[replace_key]}") 55 | state_dict[new_key] = state_dict[key] 56 | del state_dict[key] 57 | 58 | self._register_state_dict_hook(map_to) 59 | self._register_load_state_dict_pre_hook(map_from, with_module=True) 60 | -------------------------------------------------------------------------------- /src/diffusers/models/README.md: -------------------------------------------------------------------------------- 1 | # Models 2 | 3 | For more detail on the models, please refer to the [docs](https://huggingface.co/docs/diffusers/api/models/overview). -------------------------------------------------------------------------------- /src/diffusers/models/autoencoders/__init__.py: -------------------------------------------------------------------------------- 1 | from .autoencoder_asym_kl import AsymmetricAutoencoderKL 2 | from .autoencoder_kl import AutoencoderKL 3 | from .autoencoder_kl_cogvideox import AutoencoderKLCogVideoX 4 | from .autoencoder_kl_temporal_decoder import AutoencoderKLTemporalDecoder 5 | from .autoencoder_oobleck import AutoencoderOobleck 6 | from .autoencoder_tiny import AutoencoderTiny 7 | from .consistency_decoder_vae import ConsistencyDecoderVAE 8 | from .vq_model import VQModel 9 | -------------------------------------------------------------------------------- /src/diffusers/models/embeddings_flax.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import math 15 | 16 | import flax.linen as nn 17 | import jax.numpy as jnp 18 | 19 | 20 | def get_sinusoidal_embeddings( 21 | timesteps: jnp.ndarray, 22 | embedding_dim: int, 23 | freq_shift: float = 1, 24 | min_timescale: float = 1, 25 | max_timescale: float = 1.0e4, 26 | flip_sin_to_cos: bool = False, 27 | scale: float = 1.0, 28 | ) -> jnp.ndarray: 29 | """Returns the positional encoding (same as Tensor2Tensor). 30 | 31 | Args: 32 | timesteps: a 1-D Tensor of N indices, one per batch element. 33 | These may be fractional. 34 | embedding_dim: The number of output channels. 35 | min_timescale: The smallest time unit (should probably be 0.0). 36 | max_timescale: The largest time unit. 37 | Returns: 38 | a Tensor of timing signals [N, num_channels] 39 | """ 40 | assert timesteps.ndim == 1, "Timesteps should be a 1d-array" 41 | assert embedding_dim % 2 == 0, f"Embedding dimension {embedding_dim} should be even" 42 | num_timescales = float(embedding_dim // 2) 43 | log_timescale_increment = math.log(max_timescale / min_timescale) / (num_timescales - freq_shift) 44 | inv_timescales = min_timescale * jnp.exp(jnp.arange(num_timescales, dtype=jnp.float32) * -log_timescale_increment) 45 | emb = jnp.expand_dims(timesteps, 1) * jnp.expand_dims(inv_timescales, 0) 46 | 47 | # scale embeddings 48 | scaled_time = scale * emb 49 | 50 | if flip_sin_to_cos: 51 | signal = jnp.concatenate([jnp.cos(scaled_time), jnp.sin(scaled_time)], axis=1) 52 | else: 53 | signal = jnp.concatenate([jnp.sin(scaled_time), jnp.cos(scaled_time)], axis=1) 54 | signal = jnp.reshape(signal, [jnp.shape(timesteps)[0], embedding_dim]) 55 | return signal 56 | 57 | 58 | class FlaxTimestepEmbedding(nn.Module): 59 | r""" 60 | Time step Embedding Module. Learns embeddings for input time steps. 61 | 62 | Args: 63 | time_embed_dim (`int`, *optional*, defaults to `32`): 64 | Time step embedding dimension 65 | dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): 66 | Parameters `dtype` 67 | """ 68 | 69 | time_embed_dim: int = 32 70 | dtype: jnp.dtype = jnp.float32 71 | 72 | @nn.compact 73 | def __call__(self, temb): 74 | temb = nn.Dense(self.time_embed_dim, dtype=self.dtype, name="linear_1")(temb) 75 | temb = nn.silu(temb) 76 | temb = nn.Dense(self.time_embed_dim, dtype=self.dtype, name="linear_2")(temb) 77 | return temb 78 | 79 | 80 | class FlaxTimesteps(nn.Module): 81 | r""" 82 | Wrapper Module for sinusoidal Time step Embeddings as described in https://arxiv.org/abs/2006.11239 83 | 84 | Args: 85 | dim (`int`, *optional*, defaults to `32`): 86 | Time step embedding dimension 87 | """ 88 | 89 | dim: int = 32 90 | flip_sin_to_cos: bool = False 91 | freq_shift: float = 1 92 | 93 | @nn.compact 94 | def __call__(self, timesteps): 95 | return get_sinusoidal_embeddings( 96 | timesteps, embedding_dim=self.dim, flip_sin_to_cos=self.flip_sin_to_cos, freq_shift=self.freq_shift 97 | ) 98 | -------------------------------------------------------------------------------- /src/diffusers/models/modeling_outputs.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from ..utils import BaseOutput 4 | 5 | 6 | @dataclass 7 | class AutoencoderKLOutput(BaseOutput): 8 | """ 9 | Output of AutoencoderKL encoding method. 10 | 11 | Args: 12 | latent_dist (`DiagonalGaussianDistribution`): 13 | Encoded outputs of `Encoder` represented as the mean and logvar of `DiagonalGaussianDistribution`. 14 | `DiagonalGaussianDistribution` allows for sampling latents from the distribution. 15 | """ 16 | 17 | latent_dist: "DiagonalGaussianDistribution" # noqa: F821 18 | 19 | 20 | @dataclass 21 | class Transformer2DModelOutput(BaseOutput): 22 | """ 23 | The output of [`Transformer2DModel`]. 24 | 25 | Args: 26 | sample (`torch.Tensor` of shape `(batch_size, num_channels, height, width)` or `(batch size, num_vector_embeds - 1, num_latent_pixels)` if [`Transformer2DModel`] is discrete): 27 | The hidden states output conditioned on the `encoder_hidden_states` input. If discrete, returns probability 28 | distributions for the unnoised latent pixels. 29 | """ 30 | 31 | sample: "torch.Tensor" # noqa: F821 32 | -------------------------------------------------------------------------------- /src/diffusers/models/transformers/__init__.py: -------------------------------------------------------------------------------- 1 | from ...utils import is_torch_available 2 | 3 | 4 | if is_torch_available(): 5 | from .auraflow_transformer_2d import AuraFlowTransformer2DModel 6 | from .cogvideox_transformer_3d import CogVideoXTransformer3DModel 7 | from .dit_transformer_2d import DiTTransformer2DModel 8 | from .dual_transformer_2d import DualTransformer2DModel 9 | from .hunyuan_transformer_2d import HunyuanDiT2DModel 10 | from .latte_transformer_3d import LatteTransformer3DModel 11 | from .lumina_nextdit2d import LuminaNextDiT2DModel 12 | from .pixart_transformer_2d import PixArtTransformer2DModel 13 | from .prior_transformer import PriorTransformer 14 | from .stable_audio_transformer import StableAudioDiTModel 15 | from .t5_film_transformer import T5FilmDecoder 16 | from .transformer_2d import Transformer2DModel 17 | from .transformer_flux import FluxTransformer2DModel 18 | from .transformer_sd3 import SD3Transformer2DModel 19 | from .transformer_temporal import TransformerTemporalModel 20 | -------------------------------------------------------------------------------- /src/diffusers/models/unets/__init__.py: -------------------------------------------------------------------------------- 1 | from ...utils import is_flax_available, is_torch_available 2 | 3 | 4 | if is_torch_available(): 5 | from .unet_1d import UNet1DModel 6 | from .unet_2d import UNet2DModel 7 | from .unet_2d_condition import UNet2DConditionModel 8 | from .unet_3d_condition import UNet3DConditionModel 9 | from .unet_i2vgen_xl import I2VGenXLUNet 10 | from .unet_kandinsky3 import Kandinsky3UNet 11 | from .unet_motion_model import MotionAdapter, UNetMotionModel 12 | from .unet_spatio_temporal_condition import UNetSpatioTemporalConditionModel 13 | from .unet_stable_cascade import StableCascadeUNet 14 | from .uvit_2d import UVit2DModel 15 | 16 | 17 | if is_flax_available(): 18 | from .unet_2d_condition_flax import FlaxUNet2DConditionModel 19 | -------------------------------------------------------------------------------- /src/diffusers/models/vq_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from ..utils import deprecate 15 | from .autoencoders.vq_model import VQEncoderOutput, VQModel 16 | 17 | 18 | class VQEncoderOutput(VQEncoderOutput): 19 | def __init__(self, *args, **kwargs): 20 | deprecation_message = "Importing `VQEncoderOutput` from `diffusers.models.vq_model` is deprecated and this will be removed in a future version. Please use `from diffusers.models.autoencoders.vq_model import VQEncoderOutput`, instead." 21 | deprecate("VQEncoderOutput", "0.31", deprecation_message) 22 | super().__init__(*args, **kwargs) 23 | 24 | 25 | class VQModel(VQModel): 26 | def __init__(self, *args, **kwargs): 27 | deprecation_message = "Importing `VQModel` from `diffusers.models.vq_model` is deprecated and this will be removed in a future version. Please use `from diffusers.models.autoencoders.vq_model import VQModel`, instead." 28 | deprecate("VQModel", "0.31", deprecation_message) 29 | super().__init__(*args, **kwargs) 30 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/amused/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | is_torch_available, 8 | is_transformers_available, 9 | ) 10 | 11 | 12 | _dummy_objects = {} 13 | _import_structure = {} 14 | 15 | try: 16 | if not (is_transformers_available() and is_torch_available()): 17 | raise OptionalDependencyNotAvailable() 18 | except OptionalDependencyNotAvailable: 19 | from ...utils.dummy_torch_and_transformers_objects import ( 20 | AmusedImg2ImgPipeline, 21 | AmusedInpaintPipeline, 22 | AmusedPipeline, 23 | ) 24 | 25 | _dummy_objects.update( 26 | { 27 | "AmusedPipeline": AmusedPipeline, 28 | "AmusedImg2ImgPipeline": AmusedImg2ImgPipeline, 29 | "AmusedInpaintPipeline": AmusedInpaintPipeline, 30 | } 31 | ) 32 | else: 33 | _import_structure["pipeline_amused"] = ["AmusedPipeline"] 34 | _import_structure["pipeline_amused_img2img"] = ["AmusedImg2ImgPipeline"] 35 | _import_structure["pipeline_amused_inpaint"] = ["AmusedInpaintPipeline"] 36 | 37 | 38 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 39 | try: 40 | if not (is_transformers_available() and is_torch_available()): 41 | raise OptionalDependencyNotAvailable() 42 | except OptionalDependencyNotAvailable: 43 | from ...utils.dummy_torch_and_transformers_objects import ( 44 | AmusedPipeline, 45 | ) 46 | else: 47 | from .pipeline_amused import AmusedPipeline 48 | from .pipeline_amused_img2img import AmusedImg2ImgPipeline 49 | from .pipeline_amused_inpaint import AmusedInpaintPipeline 50 | 51 | else: 52 | import sys 53 | 54 | sys.modules[__name__] = _LazyModule( 55 | __name__, 56 | globals()["__file__"], 57 | _import_structure, 58 | module_spec=__spec__, 59 | ) 60 | 61 | for name, value in _dummy_objects.items(): 62 | setattr(sys.modules[__name__], name, value) 63 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/animatediff/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {"pipeline_output": ["AnimateDiffPipelineOutput"]} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_animatediff"] = ["AnimateDiffPipeline"] 25 | _import_structure["pipeline_animatediff_controlnet"] = ["AnimateDiffControlNetPipeline"] 26 | _import_structure["pipeline_animatediff_sdxl"] = ["AnimateDiffSDXLPipeline"] 27 | _import_structure["pipeline_animatediff_sparsectrl"] = ["AnimateDiffSparseControlNetPipeline"] 28 | _import_structure["pipeline_animatediff_video2video"] = ["AnimateDiffVideoToVideoPipeline"] 29 | 30 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 31 | try: 32 | if not (is_transformers_available() and is_torch_available()): 33 | raise OptionalDependencyNotAvailable() 34 | except OptionalDependencyNotAvailable: 35 | from ...utils.dummy_torch_and_transformers_objects import * 36 | 37 | else: 38 | from .pipeline_animatediff import AnimateDiffPipeline 39 | from .pipeline_animatediff_controlnet import AnimateDiffControlNetPipeline 40 | from .pipeline_animatediff_sdxl import AnimateDiffSDXLPipeline 41 | from .pipeline_animatediff_sparsectrl import AnimateDiffSparseControlNetPipeline 42 | from .pipeline_animatediff_video2video import AnimateDiffVideoToVideoPipeline 43 | from .pipeline_output import AnimateDiffPipelineOutput 44 | 45 | else: 46 | import sys 47 | 48 | sys.modules[__name__] = _LazyModule( 49 | __name__, 50 | globals()["__file__"], 51 | _import_structure, 52 | module_spec=__spec__, 53 | ) 54 | for name, value in _dummy_objects.items(): 55 | setattr(sys.modules[__name__], name, value) 56 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/animatediff/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | import torch 7 | 8 | from ...utils import BaseOutput 9 | 10 | 11 | @dataclass 12 | class AnimateDiffPipelineOutput(BaseOutput): 13 | r""" 14 | Output class for AnimateDiff pipelines. 15 | 16 | Args: 17 | frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): 18 | List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing 19 | denoised 20 | PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape 21 | `(batch_size, num_frames, channels, height, width)` 22 | """ 23 | 24 | frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]] 25 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/audioldm/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | is_torch_available, 8 | is_transformers_available, 9 | is_transformers_version, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils.dummy_torch_and_transformers_objects import ( 21 | AudioLDMPipeline, 22 | ) 23 | 24 | _dummy_objects.update({"AudioLDMPipeline": AudioLDMPipeline}) 25 | else: 26 | _import_structure["pipeline_audioldm"] = ["AudioLDMPipeline"] 27 | 28 | 29 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 30 | try: 31 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 32 | raise OptionalDependencyNotAvailable() 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import ( 35 | AudioLDMPipeline, 36 | ) 37 | 38 | else: 39 | from .pipeline_audioldm import AudioLDMPipeline 40 | else: 41 | import sys 42 | 43 | sys.modules[__name__] = _LazyModule( 44 | __name__, 45 | globals()["__file__"], 46 | _import_structure, 47 | module_spec=__spec__, 48 | ) 49 | 50 | for name, value in _dummy_objects.items(): 51 | setattr(sys.modules[__name__], name, value) 52 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/audioldm2/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | is_transformers_version, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["modeling_audioldm2"] = ["AudioLDM2ProjectionModel", "AudioLDM2UNet2DConditionModel"] 26 | _import_structure["pipeline_audioldm2"] = ["AudioLDM2Pipeline"] 27 | 28 | 29 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 30 | try: 31 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 32 | raise OptionalDependencyNotAvailable() 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | 36 | else: 37 | from .modeling_audioldm2 import AudioLDM2ProjectionModel, AudioLDM2UNet2DConditionModel 38 | from .pipeline_audioldm2 import AudioLDM2Pipeline 39 | 40 | else: 41 | import sys 42 | 43 | sys.modules[__name__] = _LazyModule( 44 | __name__, 45 | globals()["__file__"], 46 | _import_structure, 47 | module_spec=__spec__, 48 | ) 49 | for name, value in _dummy_objects.items(): 50 | setattr(sys.modules[__name__], name, value) 51 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/aura_flow/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_aura_flow"] = ["AuraFlowPipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_aura_flow import AuraFlowPipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/blip_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL 6 | from PIL import Image 7 | 8 | from ...utils import OptionalDependencyNotAvailable, is_torch_available, is_transformers_available 9 | 10 | 11 | try: 12 | if not (is_transformers_available() and is_torch_available()): 13 | raise OptionalDependencyNotAvailable() 14 | except OptionalDependencyNotAvailable: 15 | from ...utils.dummy_torch_and_transformers_objects import ShapEPipeline 16 | else: 17 | from .blip_image_processing import BlipImageProcessor 18 | from .modeling_blip2 import Blip2QFormerModel 19 | from .modeling_ctx_clip import ContextCLIPTextModel 20 | from .pipeline_blip_diffusion import BlipDiffusionPipeline 21 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/cogvideo/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_cogvideox"] = ["CogVideoXPipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_cogvideox import CogVideoXPipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/consistency_models/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | _LazyModule, 6 | ) 7 | 8 | 9 | _import_structure = { 10 | "pipeline_consistency_models": ["ConsistencyModelPipeline"], 11 | } 12 | 13 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 14 | from .pipeline_consistency_models import ConsistencyModelPipeline 15 | 16 | else: 17 | import sys 18 | 19 | sys.modules[__name__] = _LazyModule( 20 | __name__, 21 | globals()["__file__"], 22 | _import_structure, 23 | module_spec=__spec__, 24 | ) 25 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/controlnet/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_flax_available, 9 | is_torch_available, 10 | is_transformers_available, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["multicontrolnet"] = ["MultiControlNetModel"] 26 | _import_structure["pipeline_controlnet"] = ["StableDiffusionControlNetPipeline"] 27 | _import_structure["pipeline_controlnet_blip_diffusion"] = ["BlipDiffusionControlNetPipeline"] 28 | _import_structure["pipeline_controlnet_img2img"] = ["StableDiffusionControlNetImg2ImgPipeline"] 29 | _import_structure["pipeline_controlnet_inpaint"] = ["StableDiffusionControlNetInpaintPipeline"] 30 | _import_structure["pipeline_controlnet_inpaint_sd_xl"] = ["StableDiffusionXLControlNetInpaintPipeline"] 31 | _import_structure["pipeline_controlnet_sd_xl"] = ["StableDiffusionXLControlNetPipeline"] 32 | _import_structure["pipeline_controlnet_sd_xl_img2img"] = ["StableDiffusionXLControlNetImg2ImgPipeline"] 33 | try: 34 | if not (is_transformers_available() and is_flax_available()): 35 | raise OptionalDependencyNotAvailable() 36 | except OptionalDependencyNotAvailable: 37 | from ...utils import dummy_flax_and_transformers_objects # noqa F403 38 | 39 | _dummy_objects.update(get_objects_from_module(dummy_flax_and_transformers_objects)) 40 | else: 41 | _import_structure["pipeline_flax_controlnet"] = ["FlaxStableDiffusionControlNetPipeline"] 42 | 43 | 44 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 45 | try: 46 | if not (is_transformers_available() and is_torch_available()): 47 | raise OptionalDependencyNotAvailable() 48 | 49 | except OptionalDependencyNotAvailable: 50 | from ...utils.dummy_torch_and_transformers_objects import * 51 | else: 52 | from .multicontrolnet import MultiControlNetModel 53 | from .pipeline_controlnet import StableDiffusionControlNetPipeline 54 | from .pipeline_controlnet_blip_diffusion import BlipDiffusionControlNetPipeline 55 | from .pipeline_controlnet_img2img import StableDiffusionControlNetImg2ImgPipeline 56 | from .pipeline_controlnet_inpaint import StableDiffusionControlNetInpaintPipeline 57 | from .pipeline_controlnet_inpaint_sd_xl import StableDiffusionXLControlNetInpaintPipeline 58 | from .pipeline_controlnet_sd_xl import StableDiffusionXLControlNetPipeline 59 | from .pipeline_controlnet_sd_xl_img2img import StableDiffusionXLControlNetImg2ImgPipeline 60 | 61 | try: 62 | if not (is_transformers_available() and is_flax_available()): 63 | raise OptionalDependencyNotAvailable() 64 | except OptionalDependencyNotAvailable: 65 | from ...utils.dummy_flax_and_transformers_objects import * # noqa F403 66 | else: 67 | from .pipeline_flax_controlnet import FlaxStableDiffusionControlNetPipeline 68 | 69 | 70 | else: 71 | import sys 72 | 73 | sys.modules[__name__] = _LazyModule( 74 | __name__, 75 | globals()["__file__"], 76 | _import_structure, 77 | module_spec=__spec__, 78 | ) 79 | for name, value in _dummy_objects.items(): 80 | setattr(sys.modules[__name__], name, value) 81 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/controlnet_hunyuandit/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_hunyuandit_controlnet"] = ["HunyuanDiTControlNetPipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_hunyuandit_controlnet import HunyuanDiTControlNetPipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/controlnet_sd3/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_flax_available, 9 | is_torch_available, 10 | is_transformers_available, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_stable_diffusion_3_controlnet"] = ["StableDiffusion3ControlNetPipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_stable_diffusion_3_controlnet import StableDiffusion3ControlNetPipeline 36 | 37 | try: 38 | if not (is_transformers_available() and is_flax_available()): 39 | raise OptionalDependencyNotAvailable() 40 | except OptionalDependencyNotAvailable: 41 | from ...utils.dummy_flax_and_transformers_objects import * # noqa F403 42 | 43 | else: 44 | import sys 45 | 46 | sys.modules[__name__] = _LazyModule( 47 | __name__, 48 | globals()["__file__"], 49 | _import_structure, 50 | module_spec=__spec__, 51 | ) 52 | for name, value in _dummy_objects.items(): 53 | setattr(sys.modules[__name__], name, value) 54 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/controlnet_xs/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_flax_available, 9 | is_torch_available, 10 | is_transformers_available, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_controlnet_xs"] = ["StableDiffusionControlNetXSPipeline"] 26 | _import_structure["pipeline_controlnet_xs_sd_xl"] = ["StableDiffusionXLControlNetXSPipeline"] 27 | try: 28 | if not (is_transformers_available() and is_flax_available()): 29 | raise OptionalDependencyNotAvailable() 30 | except OptionalDependencyNotAvailable: 31 | from ...utils import dummy_flax_and_transformers_objects # noqa F403 32 | 33 | _dummy_objects.update(get_objects_from_module(dummy_flax_and_transformers_objects)) 34 | else: 35 | pass # _import_structure["pipeline_flax_controlnet"] = ["FlaxStableDiffusionControlNetPipeline"] 36 | 37 | 38 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 39 | try: 40 | if not (is_transformers_available() and is_torch_available()): 41 | raise OptionalDependencyNotAvailable() 42 | 43 | except OptionalDependencyNotAvailable: 44 | from ...utils.dummy_torch_and_transformers_objects import * 45 | else: 46 | from .pipeline_controlnet_xs import StableDiffusionControlNetXSPipeline 47 | from .pipeline_controlnet_xs_sd_xl import StableDiffusionXLControlNetXSPipeline 48 | 49 | try: 50 | if not (is_transformers_available() and is_flax_available()): 51 | raise OptionalDependencyNotAvailable() 52 | except OptionalDependencyNotAvailable: 53 | from ...utils.dummy_flax_and_transformers_objects import * # noqa F403 54 | else: 55 | pass # from .pipeline_flax_controlnet import FlaxStableDiffusionControlNetPipeline 56 | 57 | 58 | else: 59 | import sys 60 | 61 | sys.modules[__name__] = _LazyModule( 62 | __name__, 63 | globals()["__file__"], 64 | _import_structure, 65 | module_spec=__spec__, 66 | ) 67 | for name, value in _dummy_objects.items(): 68 | setattr(sys.modules[__name__], name, value) 69 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/dance_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_dance_diffusion": ["DanceDiffusionPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_dance_diffusion import DanceDiffusionPipeline 10 | else: 11 | import sys 12 | 13 | sys.modules[__name__] = _LazyModule( 14 | __name__, 15 | globals()["__file__"], 16 | _import_structure, 17 | module_spec=__spec__, 18 | ) 19 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/ddim/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_ddim": ["DDIMPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_ddim import DDIMPipeline 10 | else: 11 | import sys 12 | 13 | sys.modules[__name__] = _LazyModule( 14 | __name__, 15 | globals()["__file__"], 16 | _import_structure, 17 | module_spec=__spec__, 18 | ) 19 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/ddpm/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | _LazyModule, 6 | ) 7 | 8 | 9 | _import_structure = {"pipeline_ddpm": ["DDPMPipeline"]} 10 | 11 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 12 | from .pipeline_ddpm import DDPMPipeline 13 | 14 | else: 15 | import sys 16 | 17 | sys.modules[__name__] = _LazyModule( 18 | __name__, 19 | globals()["__file__"], 20 | _import_structure, 21 | module_spec=__spec__, 22 | ) 23 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deepfloyd_if/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = { 15 | "timesteps": [ 16 | "fast27_timesteps", 17 | "smart100_timesteps", 18 | "smart185_timesteps", 19 | "smart27_timesteps", 20 | "smart50_timesteps", 21 | "super100_timesteps", 22 | "super27_timesteps", 23 | "super40_timesteps", 24 | ] 25 | } 26 | 27 | try: 28 | if not (is_transformers_available() and is_torch_available()): 29 | raise OptionalDependencyNotAvailable() 30 | except OptionalDependencyNotAvailable: 31 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 32 | 33 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 34 | else: 35 | _import_structure["pipeline_if"] = ["IFPipeline"] 36 | _import_structure["pipeline_if_img2img"] = ["IFImg2ImgPipeline"] 37 | _import_structure["pipeline_if_img2img_superresolution"] = ["IFImg2ImgSuperResolutionPipeline"] 38 | _import_structure["pipeline_if_inpainting"] = ["IFInpaintingPipeline"] 39 | _import_structure["pipeline_if_inpainting_superresolution"] = ["IFInpaintingSuperResolutionPipeline"] 40 | _import_structure["pipeline_if_superresolution"] = ["IFSuperResolutionPipeline"] 41 | _import_structure["pipeline_output"] = ["IFPipelineOutput"] 42 | _import_structure["safety_checker"] = ["IFSafetyChecker"] 43 | _import_structure["watermark"] = ["IFWatermarker"] 44 | 45 | 46 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 47 | try: 48 | if not (is_transformers_available() and is_torch_available()): 49 | raise OptionalDependencyNotAvailable() 50 | 51 | except OptionalDependencyNotAvailable: 52 | from ...utils.dummy_torch_and_transformers_objects import * 53 | else: 54 | from .pipeline_if import IFPipeline 55 | from .pipeline_if_img2img import IFImg2ImgPipeline 56 | from .pipeline_if_img2img_superresolution import IFImg2ImgSuperResolutionPipeline 57 | from .pipeline_if_inpainting import IFInpaintingPipeline 58 | from .pipeline_if_inpainting_superresolution import IFInpaintingSuperResolutionPipeline 59 | from .pipeline_if_superresolution import IFSuperResolutionPipeline 60 | from .pipeline_output import IFPipelineOutput 61 | from .safety_checker import IFSafetyChecker 62 | from .timesteps import ( 63 | fast27_timesteps, 64 | smart27_timesteps, 65 | smart50_timesteps, 66 | smart100_timesteps, 67 | smart185_timesteps, 68 | super27_timesteps, 69 | super40_timesteps, 70 | super100_timesteps, 71 | ) 72 | from .watermark import IFWatermarker 73 | 74 | else: 75 | import sys 76 | 77 | sys.modules[__name__] = _LazyModule( 78 | __name__, 79 | globals()["__file__"], 80 | _import_structure, 81 | module_spec=__spec__, 82 | ) 83 | 84 | for name, value in _dummy_objects.items(): 85 | setattr(sys.modules[__name__], name, value) 86 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deepfloyd_if/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput 8 | 9 | 10 | @dataclass 11 | class IFPipelineOutput(BaseOutput): 12 | """ 13 | Args: 14 | Output class for Stable Diffusion pipelines. 15 | images (`List[PIL.Image.Image]` or `np.ndarray`) 16 | List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, 17 | num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. 18 | nsfw_detected (`List[bool]`) 19 | List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" 20 | (nsfw) content or a watermark. `None` if safety checking could not be performed. 21 | watermark_detected (`List[bool]`) 22 | List of flags denoting whether the corresponding generated image likely has a watermark. `None` if safety 23 | checking could not be performed. 24 | """ 25 | 26 | images: Union[List[PIL.Image.Image], np.ndarray] 27 | nsfw_detected: Optional[List[bool]] 28 | watermark_detected: Optional[List[bool]] 29 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deepfloyd_if/safety_checker.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn as nn 4 | from transformers import CLIPConfig, CLIPVisionModelWithProjection, PreTrainedModel 5 | 6 | from ...utils import logging 7 | 8 | 9 | logger = logging.get_logger(__name__) 10 | 11 | 12 | class IFSafetyChecker(PreTrainedModel): 13 | config_class = CLIPConfig 14 | 15 | _no_split_modules = ["CLIPEncoderLayer"] 16 | 17 | def __init__(self, config: CLIPConfig): 18 | super().__init__(config) 19 | 20 | self.vision_model = CLIPVisionModelWithProjection(config.vision_config) 21 | 22 | self.p_head = nn.Linear(config.vision_config.projection_dim, 1) 23 | self.w_head = nn.Linear(config.vision_config.projection_dim, 1) 24 | 25 | @torch.no_grad() 26 | def forward(self, clip_input, images, p_threshold=0.5, w_threshold=0.5): 27 | image_embeds = self.vision_model(clip_input)[0] 28 | 29 | nsfw_detected = self.p_head(image_embeds) 30 | nsfw_detected = nsfw_detected.flatten() 31 | nsfw_detected = nsfw_detected > p_threshold 32 | nsfw_detected = nsfw_detected.tolist() 33 | 34 | if any(nsfw_detected): 35 | logger.warning( 36 | "Potential NSFW content was detected in one or more images. A black image will be returned instead." 37 | " Try again with a different prompt and/or seed." 38 | ) 39 | 40 | for idx, nsfw_detected_ in enumerate(nsfw_detected): 41 | if nsfw_detected_: 42 | images[idx] = np.zeros(images[idx].shape) 43 | 44 | watermark_detected = self.w_head(image_embeds) 45 | watermark_detected = watermark_detected.flatten() 46 | watermark_detected = watermark_detected > w_threshold 47 | watermark_detected = watermark_detected.tolist() 48 | 49 | if any(watermark_detected): 50 | logger.warning( 51 | "Potential watermarked content was detected in one or more images. A black image will be returned instead." 52 | " Try again with a different prompt and/or seed." 53 | ) 54 | 55 | for idx, watermark_detected_ in enumerate(watermark_detected): 56 | if watermark_detected_: 57 | images[idx] = np.zeros(images[idx].shape) 58 | 59 | return images, nsfw_detected, watermark_detected 60 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deepfloyd_if/watermark.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import PIL.Image 4 | import torch 5 | from PIL import Image 6 | 7 | from ...configuration_utils import ConfigMixin 8 | from ...models.modeling_utils import ModelMixin 9 | from ...utils import PIL_INTERPOLATION 10 | 11 | 12 | class IFWatermarker(ModelMixin, ConfigMixin): 13 | def __init__(self): 14 | super().__init__() 15 | 16 | self.register_buffer("watermark_image", torch.zeros((62, 62, 4))) 17 | self.watermark_image_as_pil = None 18 | 19 | def apply_watermark(self, images: List[PIL.Image.Image], sample_size=None): 20 | # Copied from https://github.com/deep-floyd/IF/blob/b77482e36ca2031cb94dbca1001fc1e6400bf4ab/deepfloyd_if/modules/base.py#L287 21 | 22 | h = images[0].height 23 | w = images[0].width 24 | 25 | sample_size = sample_size or h 26 | 27 | coef = min(h / sample_size, w / sample_size) 28 | img_h, img_w = (int(h / coef), int(w / coef)) if coef < 1 else (h, w) 29 | 30 | S1, S2 = 1024**2, img_w * img_h 31 | K = (S2 / S1) ** 0.5 32 | wm_size, wm_x, wm_y = int(K * 62), img_w - int(14 * K), img_h - int(14 * K) 33 | 34 | if self.watermark_image_as_pil is None: 35 | watermark_image = self.watermark_image.to(torch.uint8).cpu().numpy() 36 | watermark_image = Image.fromarray(watermark_image, mode="RGBA") 37 | self.watermark_image_as_pil = watermark_image 38 | 39 | wm_img = self.watermark_image_as_pil.resize( 40 | (wm_size, wm_size), PIL_INTERPOLATION["bicubic"], reducing_gap=None 41 | ) 42 | 43 | for pil_img in images: 44 | pil_img.paste(wm_img, box=(wm_x - wm_size, wm_y - wm_size, wm_x, wm_y), mask=wm_img.split()[-1]) 45 | 46 | return images 47 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/README.md: -------------------------------------------------------------------------------- 1 | # Deprecated Pipelines 2 | 3 | This folder contains pipelines that have very low usage as measured by model downloads, issues and PRs. While you can still use the pipelines just as before, we will stop testing the pipelines and will not accept any changes to existing files. -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/alt_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ....utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ....utils import dummy_torch_and_transformers_objects 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["modeling_roberta_series"] = ["RobertaSeriesModelWithTransformation"] 25 | _import_structure["pipeline_alt_diffusion"] = ["AltDiffusionPipeline"] 26 | _import_structure["pipeline_alt_diffusion_img2img"] = ["AltDiffusionImg2ImgPipeline"] 27 | 28 | _import_structure["pipeline_output"] = ["AltDiffusionPipelineOutput"] 29 | 30 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 31 | try: 32 | if not (is_transformers_available() and is_torch_available()): 33 | raise OptionalDependencyNotAvailable() 34 | except OptionalDependencyNotAvailable: 35 | from ....utils.dummy_torch_and_transformers_objects import * 36 | 37 | else: 38 | from .modeling_roberta_series import RobertaSeriesModelWithTransformation 39 | from .pipeline_alt_diffusion import AltDiffusionPipeline 40 | from .pipeline_alt_diffusion_img2img import AltDiffusionImg2ImgPipeline 41 | from .pipeline_output import AltDiffusionPipelineOutput 42 | 43 | else: 44 | import sys 45 | 46 | sys.modules[__name__] = _LazyModule( 47 | __name__, 48 | globals()["__file__"], 49 | _import_structure, 50 | module_spec=__spec__, 51 | ) 52 | for name, value in _dummy_objects.items(): 53 | setattr(sys.modules[__name__], name, value) 54 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ....utils import ( 8 | BaseOutput, 9 | ) 10 | 11 | 12 | @dataclass 13 | # Copied from diffusers.pipelines.stable_diffusion.pipeline_output.StableDiffusionPipelineOutput with Stable->Alt 14 | class AltDiffusionPipelineOutput(BaseOutput): 15 | """ 16 | Output class for Alt Diffusion pipelines. 17 | 18 | Args: 19 | images (`List[PIL.Image.Image]` or `np.ndarray`) 20 | List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, 21 | num_channels)`. 22 | nsfw_content_detected (`List[bool]`) 23 | List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or 24 | `None` if safety checking could not be performed. 25 | """ 26 | 27 | images: Union[List[PIL.Image.Image], np.ndarray] 28 | nsfw_content_detected: Optional[List[bool]] 29 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/audio_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = { 7 | "mel": ["Mel"], 8 | "pipeline_audio_diffusion": ["AudioDiffusionPipeline"], 9 | } 10 | 11 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 12 | from .mel import Mel 13 | from .pipeline_audio_diffusion import AudioDiffusionPipeline 14 | 15 | else: 16 | import sys 17 | 18 | sys.modules[__name__] = _LazyModule( 19 | __name__, 20 | globals()["__file__"], 21 | _import_structure, 22 | module_spec=__spec__, 23 | ) 24 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/latent_diffusion_uncond/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_latent_diffusion_uncond": ["LDMPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_latent_diffusion_uncond import LDMPipeline 10 | else: 11 | import sys 12 | 13 | sys.modules[__name__] = _LazyModule( 14 | __name__, 15 | globals()["__file__"], 16 | _import_structure, 17 | module_spec=__spec__, 18 | ) 19 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/pndm/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_pndm": ["PNDMPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_pndm import PNDMPipeline 10 | else: 11 | import sys 12 | 13 | sys.modules[__name__] = _LazyModule( 14 | __name__, 15 | globals()["__file__"], 16 | _import_structure, 17 | module_spec=__spec__, 18 | ) 19 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/repaint/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_repaint": ["RePaintPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_repaint import RePaintPipeline 10 | 11 | else: 12 | import sys 13 | 14 | sys.modules[__name__] = _LazyModule( 15 | __name__, 16 | globals()["__file__"], 17 | _import_structure, 18 | module_spec=__spec__, 19 | ) 20 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/score_sde_ve/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_score_sde_ve": ["ScoreSdeVePipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_score_sde_ve import ScoreSdeVePipeline 10 | 11 | else: 12 | import sys 13 | 14 | sys.modules[__name__] = _LazyModule( 15 | __name__, 16 | globals()["__file__"], 17 | _import_structure, 18 | module_spec=__spec__, 19 | ) 20 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/spectrogram_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from typing import TYPE_CHECKING 3 | from ....utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | _LazyModule, 6 | is_note_seq_available, 7 | OptionalDependencyNotAvailable, 8 | is_torch_available, 9 | is_transformers_available, 10 | get_objects_from_module, 11 | ) 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ....utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["continous_encoder"] = ["SpectrogramContEncoder"] 25 | _import_structure["notes_encoder"] = ["SpectrogramNotesEncoder"] 26 | _import_structure["pipeline_spectrogram_diffusion"] = [ 27 | "SpectrogramContEncoder", 28 | "SpectrogramDiffusionPipeline", 29 | "T5FilmDecoder", 30 | ] 31 | try: 32 | if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): 33 | raise OptionalDependencyNotAvailable() 34 | except OptionalDependencyNotAvailable: 35 | from ....utils import dummy_transformers_and_torch_and_note_seq_objects 36 | 37 | _dummy_objects.update(get_objects_from_module(dummy_transformers_and_torch_and_note_seq_objects)) 38 | else: 39 | _import_structure["midi_utils"] = ["MidiProcessor"] 40 | 41 | 42 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 43 | try: 44 | if not (is_transformers_available() and is_torch_available()): 45 | raise OptionalDependencyNotAvailable() 46 | 47 | except OptionalDependencyNotAvailable: 48 | from ....utils.dummy_torch_and_transformers_objects import * 49 | else: 50 | from .pipeline_spectrogram_diffusion import SpectrogramDiffusionPipeline 51 | from .pipeline_spectrogram_diffusion import SpectrogramContEncoder 52 | from .pipeline_spectrogram_diffusion import SpectrogramNotesEncoder 53 | from .pipeline_spectrogram_diffusion import T5FilmDecoder 54 | 55 | try: 56 | if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): 57 | raise OptionalDependencyNotAvailable() 58 | except OptionalDependencyNotAvailable: 59 | from ....utils.dummy_transformers_and_torch_and_note_seq_objects import * 60 | 61 | else: 62 | from .midi_utils import MidiProcessor 63 | 64 | else: 65 | import sys 66 | 67 | sys.modules[__name__] = _LazyModule( 68 | __name__, 69 | globals()["__file__"], 70 | _import_structure, 71 | module_spec=__spec__, 72 | ) 73 | 74 | for name, value in _dummy_objects.items(): 75 | setattr(sys.modules[__name__], name, value) 76 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/spectrogram_diffusion/continuous_encoder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Music Spectrogram Diffusion Authors. 2 | # Copyright 2024 The HuggingFace Team. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import torch 17 | import torch.nn as nn 18 | from transformers.modeling_utils import ModuleUtilsMixin 19 | from transformers.models.t5.modeling_t5 import ( 20 | T5Block, 21 | T5Config, 22 | T5LayerNorm, 23 | ) 24 | 25 | from ....configuration_utils import ConfigMixin, register_to_config 26 | from ....models import ModelMixin 27 | 28 | 29 | class SpectrogramContEncoder(ModelMixin, ConfigMixin, ModuleUtilsMixin): 30 | @register_to_config 31 | def __init__( 32 | self, 33 | input_dims: int, 34 | targets_context_length: int, 35 | d_model: int, 36 | dropout_rate: float, 37 | num_layers: int, 38 | num_heads: int, 39 | d_kv: int, 40 | d_ff: int, 41 | feed_forward_proj: str, 42 | is_decoder: bool = False, 43 | ): 44 | super().__init__() 45 | 46 | self.input_proj = nn.Linear(input_dims, d_model, bias=False) 47 | 48 | self.position_encoding = nn.Embedding(targets_context_length, d_model) 49 | self.position_encoding.weight.requires_grad = False 50 | 51 | self.dropout_pre = nn.Dropout(p=dropout_rate) 52 | 53 | t5config = T5Config( 54 | d_model=d_model, 55 | num_heads=num_heads, 56 | d_kv=d_kv, 57 | d_ff=d_ff, 58 | feed_forward_proj=feed_forward_proj, 59 | dropout_rate=dropout_rate, 60 | is_decoder=is_decoder, 61 | is_encoder_decoder=False, 62 | ) 63 | self.encoders = nn.ModuleList() 64 | for lyr_num in range(num_layers): 65 | lyr = T5Block(t5config) 66 | self.encoders.append(lyr) 67 | 68 | self.layer_norm = T5LayerNorm(d_model) 69 | self.dropout_post = nn.Dropout(p=dropout_rate) 70 | 71 | def forward(self, encoder_inputs, encoder_inputs_mask): 72 | x = self.input_proj(encoder_inputs) 73 | 74 | # terminal relative positional encodings 75 | max_positions = encoder_inputs.shape[1] 76 | input_positions = torch.arange(max_positions, device=encoder_inputs.device) 77 | 78 | seq_lens = encoder_inputs_mask.sum(-1) 79 | input_positions = torch.roll(input_positions.unsqueeze(0), tuple(seq_lens.tolist()), dims=0) 80 | x += self.position_encoding(input_positions) 81 | 82 | x = self.dropout_pre(x) 83 | 84 | # inverted the attention mask 85 | input_shape = encoder_inputs.size() 86 | extended_attention_mask = self.get_extended_attention_mask(encoder_inputs_mask, input_shape) 87 | 88 | for lyr in self.encoders: 89 | x = lyr(x, extended_attention_mask)[0] 90 | x = self.layer_norm(x) 91 | 92 | return self.dropout_post(x), encoder_inputs_mask 93 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/spectrogram_diffusion/notes_encoder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The Music Spectrogram Diffusion Authors. 2 | # Copyright 2024 The HuggingFace Team. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import torch 17 | import torch.nn as nn 18 | from transformers.modeling_utils import ModuleUtilsMixin 19 | from transformers.models.t5.modeling_t5 import T5Block, T5Config, T5LayerNorm 20 | 21 | from ....configuration_utils import ConfigMixin, register_to_config 22 | from ....models import ModelMixin 23 | 24 | 25 | class SpectrogramNotesEncoder(ModelMixin, ConfigMixin, ModuleUtilsMixin): 26 | @register_to_config 27 | def __init__( 28 | self, 29 | max_length: int, 30 | vocab_size: int, 31 | d_model: int, 32 | dropout_rate: float, 33 | num_layers: int, 34 | num_heads: int, 35 | d_kv: int, 36 | d_ff: int, 37 | feed_forward_proj: str, 38 | is_decoder: bool = False, 39 | ): 40 | super().__init__() 41 | 42 | self.token_embedder = nn.Embedding(vocab_size, d_model) 43 | 44 | self.position_encoding = nn.Embedding(max_length, d_model) 45 | self.position_encoding.weight.requires_grad = False 46 | 47 | self.dropout_pre = nn.Dropout(p=dropout_rate) 48 | 49 | t5config = T5Config( 50 | vocab_size=vocab_size, 51 | d_model=d_model, 52 | num_heads=num_heads, 53 | d_kv=d_kv, 54 | d_ff=d_ff, 55 | dropout_rate=dropout_rate, 56 | feed_forward_proj=feed_forward_proj, 57 | is_decoder=is_decoder, 58 | is_encoder_decoder=False, 59 | ) 60 | 61 | self.encoders = nn.ModuleList() 62 | for lyr_num in range(num_layers): 63 | lyr = T5Block(t5config) 64 | self.encoders.append(lyr) 65 | 66 | self.layer_norm = T5LayerNorm(d_model) 67 | self.dropout_post = nn.Dropout(p=dropout_rate) 68 | 69 | def forward(self, encoder_input_tokens, encoder_inputs_mask): 70 | x = self.token_embedder(encoder_input_tokens) 71 | 72 | seq_length = encoder_input_tokens.shape[1] 73 | inputs_positions = torch.arange(seq_length, device=encoder_input_tokens.device) 74 | x += self.position_encoding(inputs_positions) 75 | 76 | x = self.dropout_pre(x) 77 | 78 | # inverted the attention mask 79 | input_shape = encoder_input_tokens.size() 80 | extended_attention_mask = self.get_extended_attention_mask(encoder_inputs_mask, input_shape) 81 | 82 | for lyr in self.encoders: 83 | x = lyr(x, extended_attention_mask)[0] 84 | x = self.layer_norm(x) 85 | 86 | return self.dropout_post(x), encoder_inputs_mask 87 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/stable_diffusion_variants/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ....utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ....utils import dummy_torch_and_transformers_objects 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_cycle_diffusion"] = ["CycleDiffusionPipeline"] 25 | _import_structure["pipeline_stable_diffusion_inpaint_legacy"] = ["StableDiffusionInpaintPipelineLegacy"] 26 | _import_structure["pipeline_stable_diffusion_model_editing"] = ["StableDiffusionModelEditingPipeline"] 27 | 28 | _import_structure["pipeline_stable_diffusion_paradigms"] = ["StableDiffusionParadigmsPipeline"] 29 | _import_structure["pipeline_stable_diffusion_pix2pix_zero"] = ["StableDiffusionPix2PixZeroPipeline"] 30 | 31 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 32 | try: 33 | if not (is_transformers_available() and is_torch_available()): 34 | raise OptionalDependencyNotAvailable() 35 | except OptionalDependencyNotAvailable: 36 | from ....utils.dummy_torch_and_transformers_objects import * 37 | 38 | else: 39 | from .pipeline_cycle_diffusion import CycleDiffusionPipeline 40 | from .pipeline_stable_diffusion_inpaint_legacy import StableDiffusionInpaintPipelineLegacy 41 | from .pipeline_stable_diffusion_model_editing import StableDiffusionModelEditingPipeline 42 | from .pipeline_stable_diffusion_paradigms import StableDiffusionParadigmsPipeline 43 | from .pipeline_stable_diffusion_pix2pix_zero import StableDiffusionPix2PixZeroPipeline 44 | 45 | else: 46 | import sys 47 | 48 | sys.modules[__name__] = _LazyModule( 49 | __name__, 50 | globals()["__file__"], 51 | _import_structure, 52 | module_spec=__spec__, 53 | ) 54 | for name, value in _dummy_objects.items(): 55 | setattr(sys.modules[__name__], name, value) 56 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/stochastic_karras_ve/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_stochastic_karras_ve": ["KarrasVePipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_stochastic_karras_ve import KarrasVePipeline 10 | 11 | else: 12 | import sys 13 | 14 | sys.modules[__name__] = _LazyModule( 15 | __name__, 16 | globals()["__file__"], 17 | _import_structure, 18 | module_spec=__spec__, 19 | ) 20 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/versatile_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ....utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | is_torch_available, 8 | is_transformers_available, 9 | is_transformers_version, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ....utils.dummy_torch_and_transformers_objects import ( 21 | VersatileDiffusionDualGuidedPipeline, 22 | VersatileDiffusionImageVariationPipeline, 23 | VersatileDiffusionPipeline, 24 | VersatileDiffusionTextToImagePipeline, 25 | ) 26 | 27 | _dummy_objects.update( 28 | { 29 | "VersatileDiffusionDualGuidedPipeline": VersatileDiffusionDualGuidedPipeline, 30 | "VersatileDiffusionImageVariationPipeline": VersatileDiffusionImageVariationPipeline, 31 | "VersatileDiffusionPipeline": VersatileDiffusionPipeline, 32 | "VersatileDiffusionTextToImagePipeline": VersatileDiffusionTextToImagePipeline, 33 | } 34 | ) 35 | else: 36 | _import_structure["modeling_text_unet"] = ["UNetFlatConditionModel"] 37 | _import_structure["pipeline_versatile_diffusion"] = ["VersatileDiffusionPipeline"] 38 | _import_structure["pipeline_versatile_diffusion_dual_guided"] = ["VersatileDiffusionDualGuidedPipeline"] 39 | _import_structure["pipeline_versatile_diffusion_image_variation"] = ["VersatileDiffusionImageVariationPipeline"] 40 | _import_structure["pipeline_versatile_diffusion_text_to_image"] = ["VersatileDiffusionTextToImagePipeline"] 41 | 42 | 43 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 44 | try: 45 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): 46 | raise OptionalDependencyNotAvailable() 47 | except OptionalDependencyNotAvailable: 48 | from ....utils.dummy_torch_and_transformers_objects import ( 49 | VersatileDiffusionDualGuidedPipeline, 50 | VersatileDiffusionImageVariationPipeline, 51 | VersatileDiffusionPipeline, 52 | VersatileDiffusionTextToImagePipeline, 53 | ) 54 | else: 55 | from .pipeline_versatile_diffusion import VersatileDiffusionPipeline 56 | from .pipeline_versatile_diffusion_dual_guided import VersatileDiffusionDualGuidedPipeline 57 | from .pipeline_versatile_diffusion_image_variation import VersatileDiffusionImageVariationPipeline 58 | from .pipeline_versatile_diffusion_text_to_image import VersatileDiffusionTextToImagePipeline 59 | 60 | else: 61 | import sys 62 | 63 | sys.modules[__name__] = _LazyModule( 64 | __name__, 65 | globals()["__file__"], 66 | _import_structure, 67 | module_spec=__spec__, 68 | ) 69 | 70 | for name, value in _dummy_objects.items(): 71 | setattr(sys.modules[__name__], name, value) 72 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/deprecated/vq_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ....utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | is_torch_available, 8 | is_transformers_available, 9 | ) 10 | 11 | 12 | _dummy_objects = {} 13 | _import_structure = {} 14 | 15 | try: 16 | if not (is_transformers_available() and is_torch_available()): 17 | raise OptionalDependencyNotAvailable() 18 | except OptionalDependencyNotAvailable: 19 | from ....utils.dummy_torch_and_transformers_objects import ( 20 | LearnedClassifierFreeSamplingEmbeddings, 21 | VQDiffusionPipeline, 22 | ) 23 | 24 | _dummy_objects.update( 25 | { 26 | "LearnedClassifierFreeSamplingEmbeddings": LearnedClassifierFreeSamplingEmbeddings, 27 | "VQDiffusionPipeline": VQDiffusionPipeline, 28 | } 29 | ) 30 | else: 31 | _import_structure["pipeline_vq_diffusion"] = ["LearnedClassifierFreeSamplingEmbeddings", "VQDiffusionPipeline"] 32 | 33 | 34 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 35 | try: 36 | if not (is_transformers_available() and is_torch_available()): 37 | raise OptionalDependencyNotAvailable() 38 | except OptionalDependencyNotAvailable: 39 | from ....utils.dummy_torch_and_transformers_objects import ( 40 | LearnedClassifierFreeSamplingEmbeddings, 41 | VQDiffusionPipeline, 42 | ) 43 | else: 44 | from .pipeline_vq_diffusion import LearnedClassifierFreeSamplingEmbeddings, VQDiffusionPipeline 45 | 46 | else: 47 | import sys 48 | 49 | sys.modules[__name__] = _LazyModule( 50 | __name__, 51 | globals()["__file__"], 52 | _import_structure, 53 | module_spec=__spec__, 54 | ) 55 | 56 | for name, value in _dummy_objects.items(): 57 | setattr(sys.modules[__name__], name, value) 58 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/dit/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule 4 | 5 | 6 | _import_structure = {"pipeline_dit": ["DiTPipeline"]} 7 | 8 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 9 | from .pipeline_dit import DiTPipeline 10 | 11 | else: 12 | import sys 13 | 14 | sys.modules[__name__] = _LazyModule( 15 | __name__, 16 | globals()["__file__"], 17 | _import_structure, 18 | module_spec=__spec__, 19 | ) 20 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/flux/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _additional_imports = {} 15 | _import_structure = {"pipeline_output": ["FluxPipelineOutput"]} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_flux"] = ["FluxPipeline"] 26 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 27 | try: 28 | if not (is_transformers_available() and is_torch_available()): 29 | raise OptionalDependencyNotAvailable() 30 | except OptionalDependencyNotAvailable: 31 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 32 | else: 33 | from .pipeline_flux import FluxPipeline 34 | else: 35 | import sys 36 | 37 | sys.modules[__name__] = _LazyModule( 38 | __name__, 39 | globals()["__file__"], 40 | _import_structure, 41 | module_spec=__spec__, 42 | ) 43 | 44 | for name, value in _dummy_objects.items(): 45 | setattr(sys.modules[__name__], name, value) 46 | for name, value in _additional_imports.items(): 47 | setattr(sys.modules[__name__], name, value) 48 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/flux/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput 8 | 9 | 10 | @dataclass 11 | class FluxPipelineOutput(BaseOutput): 12 | """ 13 | Output class for Stable Diffusion pipelines. 14 | 15 | Args: 16 | images (`List[PIL.Image.Image]` or `np.ndarray`) 17 | List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, 18 | num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. 19 | """ 20 | 21 | images: Union[List[PIL.Image.Image], np.ndarray] 22 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/hunyuandit/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_hunyuandit"] = ["HunyuanDiTPipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_hunyuandit import HunyuanDiTPipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/i2vgen_xl/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_i2vgen_xl"] = ["I2VGenXLPipeline"] 25 | 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | except OptionalDependencyNotAvailable: 32 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 33 | else: 34 | from .pipeline_i2vgen_xl import I2VGenXLPipeline 35 | 36 | else: 37 | import sys 38 | 39 | sys.modules[__name__] = _LazyModule( 40 | __name__, 41 | globals()["__file__"], 42 | _import_structure, 43 | module_spec=__spec__, 44 | ) 45 | for name, value in _dummy_objects.items(): 46 | setattr(sys.modules[__name__], name, value) 47 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/kandinsky/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_kandinsky"] = ["KandinskyPipeline"] 25 | _import_structure["pipeline_kandinsky_combined"] = [ 26 | "KandinskyCombinedPipeline", 27 | "KandinskyImg2ImgCombinedPipeline", 28 | "KandinskyInpaintCombinedPipeline", 29 | ] 30 | _import_structure["pipeline_kandinsky_img2img"] = ["KandinskyImg2ImgPipeline"] 31 | _import_structure["pipeline_kandinsky_inpaint"] = ["KandinskyInpaintPipeline"] 32 | _import_structure["pipeline_kandinsky_prior"] = ["KandinskyPriorPipeline", "KandinskyPriorPipelineOutput"] 33 | _import_structure["text_encoder"] = ["MultilingualCLIP"] 34 | 35 | 36 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 37 | try: 38 | if not (is_transformers_available() and is_torch_available()): 39 | raise OptionalDependencyNotAvailable() 40 | except OptionalDependencyNotAvailable: 41 | from ...utils.dummy_torch_and_transformers_objects import * 42 | 43 | else: 44 | from .pipeline_kandinsky import KandinskyPipeline 45 | from .pipeline_kandinsky_combined import ( 46 | KandinskyCombinedPipeline, 47 | KandinskyImg2ImgCombinedPipeline, 48 | KandinskyInpaintCombinedPipeline, 49 | ) 50 | from .pipeline_kandinsky_img2img import KandinskyImg2ImgPipeline 51 | from .pipeline_kandinsky_inpaint import KandinskyInpaintPipeline 52 | from .pipeline_kandinsky_prior import KandinskyPriorPipeline, KandinskyPriorPipelineOutput 53 | from .text_encoder import MultilingualCLIP 54 | 55 | else: 56 | import sys 57 | 58 | sys.modules[__name__] = _LazyModule( 59 | __name__, 60 | globals()["__file__"], 61 | _import_structure, 62 | module_spec=__spec__, 63 | ) 64 | 65 | for name, value in _dummy_objects.items(): 66 | setattr(sys.modules[__name__], name, value) 67 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/kandinsky/text_encoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel 3 | 4 | 5 | class MCLIPConfig(XLMRobertaConfig): 6 | model_type = "M-CLIP" 7 | 8 | def __init__(self, transformerDimSize=1024, imageDimSize=768, **kwargs): 9 | self.transformerDimensions = transformerDimSize 10 | self.numDims = imageDimSize 11 | super().__init__(**kwargs) 12 | 13 | 14 | class MultilingualCLIP(PreTrainedModel): 15 | config_class = MCLIPConfig 16 | 17 | def __init__(self, config, *args, **kwargs): 18 | super().__init__(config, *args, **kwargs) 19 | self.transformer = XLMRobertaModel(config) 20 | self.LinearTransformation = torch.nn.Linear( 21 | in_features=config.transformerDimensions, out_features=config.numDims 22 | ) 23 | 24 | def forward(self, input_ids, attention_mask): 25 | embs = self.transformer(input_ids=input_ids, attention_mask=attention_mask)[0] 26 | embs2 = (embs * attention_mask.unsqueeze(2)).sum(dim=1) / attention_mask.sum(dim=1)[:, None] 27 | return self.LinearTransformation(embs2), embs 28 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/kandinsky2_2/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_kandinsky2_2"] = ["KandinskyV22Pipeline"] 25 | _import_structure["pipeline_kandinsky2_2_combined"] = [ 26 | "KandinskyV22CombinedPipeline", 27 | "KandinskyV22Img2ImgCombinedPipeline", 28 | "KandinskyV22InpaintCombinedPipeline", 29 | ] 30 | _import_structure["pipeline_kandinsky2_2_controlnet"] = ["KandinskyV22ControlnetPipeline"] 31 | _import_structure["pipeline_kandinsky2_2_controlnet_img2img"] = ["KandinskyV22ControlnetImg2ImgPipeline"] 32 | _import_structure["pipeline_kandinsky2_2_img2img"] = ["KandinskyV22Img2ImgPipeline"] 33 | _import_structure["pipeline_kandinsky2_2_inpainting"] = ["KandinskyV22InpaintPipeline"] 34 | _import_structure["pipeline_kandinsky2_2_prior"] = ["KandinskyV22PriorPipeline"] 35 | _import_structure["pipeline_kandinsky2_2_prior_emb2emb"] = ["KandinskyV22PriorEmb2EmbPipeline"] 36 | 37 | 38 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 39 | try: 40 | if not (is_transformers_available() and is_torch_available()): 41 | raise OptionalDependencyNotAvailable() 42 | 43 | except OptionalDependencyNotAvailable: 44 | from ...utils.dummy_torch_and_transformers_objects import * 45 | else: 46 | from .pipeline_kandinsky2_2 import KandinskyV22Pipeline 47 | from .pipeline_kandinsky2_2_combined import ( 48 | KandinskyV22CombinedPipeline, 49 | KandinskyV22Img2ImgCombinedPipeline, 50 | KandinskyV22InpaintCombinedPipeline, 51 | ) 52 | from .pipeline_kandinsky2_2_controlnet import KandinskyV22ControlnetPipeline 53 | from .pipeline_kandinsky2_2_controlnet_img2img import KandinskyV22ControlnetImg2ImgPipeline 54 | from .pipeline_kandinsky2_2_img2img import KandinskyV22Img2ImgPipeline 55 | from .pipeline_kandinsky2_2_inpainting import KandinskyV22InpaintPipeline 56 | from .pipeline_kandinsky2_2_prior import KandinskyV22PriorPipeline 57 | from .pipeline_kandinsky2_2_prior_emb2emb import KandinskyV22PriorEmb2EmbPipeline 58 | 59 | else: 60 | import sys 61 | 62 | sys.modules[__name__] = _LazyModule( 63 | __name__, 64 | globals()["__file__"], 65 | _import_structure, 66 | module_spec=__spec__, 67 | ) 68 | 69 | for name, value in _dummy_objects.items(): 70 | setattr(sys.modules[__name__], name, value) 71 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/kandinsky3/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_kandinsky3"] = ["Kandinsky3Pipeline"] 25 | _import_structure["pipeline_kandinsky3_img2img"] = ["Kandinsky3Img2ImgPipeline"] 26 | 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .pipeline_kandinsky3 import Kandinsky3Pipeline 37 | from .pipeline_kandinsky3_img2img import Kandinsky3Img2ImgPipeline 38 | else: 39 | import sys 40 | 41 | sys.modules[__name__] = _LazyModule( 42 | __name__, 43 | globals()["__file__"], 44 | _import_structure, 45 | module_spec=__spec__, 46 | ) 47 | 48 | for name, value in _dummy_objects.items(): 49 | setattr(sys.modules[__name__], name, value) 50 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/kandinsky3/convert_kandinsky3_unet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | import fnmatch 4 | 5 | from safetensors.torch import load_file 6 | 7 | from diffusers import Kandinsky3UNet 8 | 9 | 10 | MAPPING = { 11 | "to_time_embed.1": "time_embedding.linear_1", 12 | "to_time_embed.3": "time_embedding.linear_2", 13 | "in_layer": "conv_in", 14 | "out_layer.0": "conv_norm_out", 15 | "out_layer.2": "conv_out", 16 | "down_samples": "down_blocks", 17 | "up_samples": "up_blocks", 18 | "projection_lin": "encoder_hid_proj.projection_linear", 19 | "projection_ln": "encoder_hid_proj.projection_norm", 20 | "feature_pooling": "add_time_condition", 21 | "to_query": "to_q", 22 | "to_key": "to_k", 23 | "to_value": "to_v", 24 | "output_layer": "to_out.0", 25 | "self_attention_block": "attentions.0", 26 | } 27 | 28 | DYNAMIC_MAP = { 29 | "resnet_attn_blocks.*.0": "resnets_in.*", 30 | "resnet_attn_blocks.*.1": ("attentions.*", 1), 31 | "resnet_attn_blocks.*.2": "resnets_out.*", 32 | } 33 | # MAPPING = {} 34 | 35 | 36 | def convert_state_dict(unet_state_dict): 37 | """ 38 | Args: 39 | Convert the state dict of a U-Net model to match the key format expected by Kandinsky3UNet model. 40 | unet_model (torch.nn.Module): The original U-Net model. unet_kandi3_model (torch.nn.Module): The Kandinsky3UNet 41 | model to match keys with. 42 | 43 | Returns: 44 | OrderedDict: The converted state dictionary. 45 | """ 46 | # Example of renaming logic (this will vary based on your model's architecture) 47 | converted_state_dict = {} 48 | for key in unet_state_dict: 49 | new_key = key 50 | for pattern, new_pattern in MAPPING.items(): 51 | new_key = new_key.replace(pattern, new_pattern) 52 | 53 | for dyn_pattern, dyn_new_pattern in DYNAMIC_MAP.items(): 54 | has_matched = False 55 | if fnmatch.fnmatch(new_key, f"*.{dyn_pattern}.*") and not has_matched: 56 | star = int(new_key.split(dyn_pattern.split(".")[0])[-1].split(".")[1]) 57 | 58 | if isinstance(dyn_new_pattern, tuple): 59 | new_star = star + dyn_new_pattern[-1] 60 | dyn_new_pattern = dyn_new_pattern[0] 61 | else: 62 | new_star = star 63 | 64 | pattern = dyn_pattern.replace("*", str(star)) 65 | new_pattern = dyn_new_pattern.replace("*", str(new_star)) 66 | 67 | new_key = new_key.replace(pattern, new_pattern) 68 | has_matched = True 69 | 70 | converted_state_dict[new_key] = unet_state_dict[key] 71 | 72 | return converted_state_dict 73 | 74 | 75 | def main(model_path, output_path): 76 | # Load your original U-Net model 77 | unet_state_dict = load_file(model_path) 78 | 79 | # Initialize your Kandinsky3UNet model 80 | config = {} 81 | 82 | # Convert the state dict 83 | converted_state_dict = convert_state_dict(unet_state_dict) 84 | 85 | unet = Kandinsky3UNet(config) 86 | unet.load_state_dict(converted_state_dict) 87 | 88 | unet.save_pretrained(output_path) 89 | print(f"Converted model saved to {output_path}") 90 | 91 | 92 | if __name__ == "__main__": 93 | parser = argparse.ArgumentParser(description="Convert U-Net PyTorch model to Kandinsky3UNet format") 94 | parser.add_argument("--model_path", type=str, required=True, help="Path to the original U-Net PyTorch model") 95 | parser.add_argument("--output_path", type=str, required=True, help="Path to save the converted model") 96 | 97 | args = parser.parse_args() 98 | main(args.model_path, args.output_path) 99 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/kolors/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_sentencepiece_available, 9 | is_torch_available, 10 | is_transformers_available, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()) and is_sentencepiece_available(): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_and_sentencepiece_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_and_sentencepiece_objects)) 24 | else: 25 | _import_structure["pipeline_kolors"] = ["KolorsPipeline"] 26 | _import_structure["pipeline_kolors_img2img"] = ["KolorsImg2ImgPipeline"] 27 | _import_structure["text_encoder"] = ["ChatGLMModel"] 28 | _import_structure["tokenizer"] = ["ChatGLMTokenizer"] 29 | 30 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 31 | try: 32 | if not (is_transformers_available() and is_torch_available()) and is_sentencepiece_available(): 33 | raise OptionalDependencyNotAvailable() 34 | except OptionalDependencyNotAvailable: 35 | from ...utils.dummy_torch_and_transformers_and_sentencepiece_objects import * 36 | 37 | else: 38 | from .pipeline_kolors import KolorsPipeline 39 | from .pipeline_kolors_img2img import KolorsImg2ImgPipeline 40 | from .text_encoder import ChatGLMModel 41 | from .tokenizer import ChatGLMTokenizer 42 | 43 | else: 44 | import sys 45 | 46 | sys.modules[__name__] = _LazyModule( 47 | __name__, 48 | globals()["__file__"], 49 | _import_structure, 50 | module_spec=__spec__, 51 | ) 52 | 53 | for name, value in _dummy_objects.items(): 54 | setattr(sys.modules[__name__], name, value) 55 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/kolors/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput 8 | 9 | 10 | @dataclass 11 | class KolorsPipelineOutput(BaseOutput): 12 | """ 13 | Output class for Kolors pipelines. 14 | 15 | Args: 16 | images (`List[PIL.Image.Image]` or `np.ndarray`) 17 | List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, 18 | num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. 19 | """ 20 | 21 | images: Union[List[PIL.Image.Image], np.ndarray] 22 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/latent_consistency_models/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_latent_consistency_img2img"] = ["LatentConsistencyModelImg2ImgPipeline"] 26 | _import_structure["pipeline_latent_consistency_text2img"] = ["LatentConsistencyModelPipeline"] 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .pipeline_latent_consistency_img2img import LatentConsistencyModelImg2ImgPipeline 37 | from .pipeline_latent_consistency_text2img import LatentConsistencyModelPipeline 38 | 39 | else: 40 | import sys 41 | 42 | sys.modules[__name__] = _LazyModule( 43 | __name__, 44 | globals()["__file__"], 45 | _import_structure, 46 | module_spec=__spec__, 47 | ) 48 | 49 | for name, value in _dummy_objects.items(): 50 | setattr(sys.modules[__name__], name, value) 51 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/latent_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_latent_diffusion"] = ["LDMBertModel", "LDMTextToImagePipeline"] 25 | _import_structure["pipeline_latent_diffusion_superresolution"] = ["LDMSuperResolutionPipeline"] 26 | 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .pipeline_latent_diffusion import LDMBertModel, LDMTextToImagePipeline 37 | from .pipeline_latent_diffusion_superresolution import LDMSuperResolutionPipeline 38 | 39 | else: 40 | import sys 41 | 42 | sys.modules[__name__] = _LazyModule( 43 | __name__, 44 | globals()["__file__"], 45 | _import_structure, 46 | module_spec=__spec__, 47 | ) 48 | 49 | for name, value in _dummy_objects.items(): 50 | setattr(sys.modules[__name__], name, value) 51 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/latte/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_latte"] = ["LattePipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_latte import LattePipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/ledits_pp/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_leditspp_stable_diffusion"] = ["LEditsPPPipelineStableDiffusion"] 25 | _import_structure["pipeline_leditspp_stable_diffusion_xl"] = ["LEditsPPPipelineStableDiffusionXL"] 26 | 27 | _import_structure["pipeline_output"] = ["LEditsPPDiffusionPipelineOutput", "LEditsPPDiffusionPipelineOutput"] 28 | 29 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 30 | try: 31 | if not (is_transformers_available() and is_torch_available()): 32 | raise OptionalDependencyNotAvailable() 33 | 34 | except OptionalDependencyNotAvailable: 35 | from ...utils.dummy_torch_and_transformers_objects import * 36 | else: 37 | from .pipeline_leditspp_stable_diffusion import ( 38 | LEditsPPDiffusionPipelineOutput, 39 | LEditsPPInversionPipelineOutput, 40 | LEditsPPPipelineStableDiffusion, 41 | ) 42 | from .pipeline_leditspp_stable_diffusion_xl import LEditsPPPipelineStableDiffusionXL 43 | 44 | else: 45 | import sys 46 | 47 | sys.modules[__name__] = _LazyModule( 48 | __name__, 49 | globals()["__file__"], 50 | _import_structure, 51 | module_spec=__spec__, 52 | ) 53 | 54 | for name, value in _dummy_objects.items(): 55 | setattr(sys.modules[__name__], name, value) 56 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/ledits_pp/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput 8 | 9 | 10 | @dataclass 11 | class LEditsPPDiffusionPipelineOutput(BaseOutput): 12 | """ 13 | Output class for LEdits++ Diffusion pipelines. 14 | 15 | Args: 16 | images (`List[PIL.Image.Image]` or `np.ndarray`) 17 | List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, 18 | num_channels)`. 19 | nsfw_content_detected (`List[bool]`) 20 | List indicating whether the corresponding generated image contains “not-safe-for-work” (nsfw) content or 21 | `None` if safety checking could not be performed. 22 | """ 23 | 24 | images: Union[List[PIL.Image.Image], np.ndarray] 25 | nsfw_content_detected: Optional[List[bool]] 26 | 27 | 28 | @dataclass 29 | class LEditsPPInversionPipelineOutput(BaseOutput): 30 | """ 31 | Output class for LEdits++ Diffusion pipelines. 32 | 33 | Args: 34 | input_images (`List[PIL.Image.Image]` or `np.ndarray`) 35 | List of the cropped and resized input images as PIL images of length `batch_size` or NumPy array of shape ` 36 | (batch_size, height, width, num_channels)`. 37 | vae_reconstruction_images (`List[PIL.Image.Image]` or `np.ndarray`) 38 | List of VAE reconstruction of all input images as PIL images of length `batch_size` or NumPy array of shape 39 | ` (batch_size, height, width, num_channels)`. 40 | """ 41 | 42 | images: Union[List[PIL.Image.Image], np.ndarray] 43 | vae_reconstruction_images: Union[List[PIL.Image.Image], np.ndarray] 44 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/lumina/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_lumina"] = ["LuminaText2ImgPipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_lumina import LuminaText2ImgPipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/marigold/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["marigold_image_processing"] = ["MarigoldImageProcessor"] 25 | _import_structure["pipeline_marigold_depth"] = ["MarigoldDepthOutput", "MarigoldDepthPipeline"] 26 | _import_structure["pipeline_marigold_normals"] = ["MarigoldNormalsOutput", "MarigoldNormalsPipeline"] 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .marigold_image_processing import MarigoldImageProcessor 37 | from .pipeline_marigold_depth import MarigoldDepthOutput, MarigoldDepthPipeline 38 | from .pipeline_marigold_normals import MarigoldNormalsOutput, MarigoldNormalsPipeline 39 | 40 | else: 41 | import sys 42 | 43 | sys.modules[__name__] = _LazyModule( 44 | __name__, 45 | globals()["__file__"], 46 | _import_structure, 47 | module_spec=__spec__, 48 | ) 49 | for name, value in _dummy_objects.items(): 50 | setattr(sys.modules[__name__], name, value) 51 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/musicldm/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | is_transformers_version, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_musicldm"] = ["MusicLDMPipeline"] 26 | 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .pipeline_musicldm import MusicLDMPipeline 37 | 38 | else: 39 | import sys 40 | 41 | sys.modules[__name__] = _LazyModule( 42 | __name__, 43 | globals()["__file__"], 44 | _import_structure, 45 | module_spec=__spec__, 46 | ) 47 | 48 | for name, value in _dummy_objects.items(): 49 | setattr(sys.modules[__name__], name, value) 50 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/pag/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_flax_available, 9 | is_torch_available, 10 | is_transformers_available, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_pag_controlnet_sd"] = ["StableDiffusionControlNetPAGPipeline"] 26 | _import_structure["pipeline_pag_controlnet_sd_xl"] = ["StableDiffusionXLControlNetPAGPipeline"] 27 | _import_structure["pipeline_pag_hunyuandit"] = ["HunyuanDiTPAGPipeline"] 28 | _import_structure["pipeline_pag_kolors"] = ["KolorsPAGPipeline"] 29 | _import_structure["pipeline_pag_pixart_sigma"] = ["PixArtSigmaPAGPipeline"] 30 | _import_structure["pipeline_pag_sd"] = ["StableDiffusionPAGPipeline"] 31 | _import_structure["pipeline_pag_sd_3"] = ["StableDiffusion3PAGPipeline"] 32 | _import_structure["pipeline_pag_sd_animatediff"] = ["AnimateDiffPAGPipeline"] 33 | _import_structure["pipeline_pag_sd_xl"] = ["StableDiffusionXLPAGPipeline"] 34 | _import_structure["pipeline_pag_sd_xl_img2img"] = ["StableDiffusionXLPAGImg2ImgPipeline"] 35 | _import_structure["pipeline_pag_sd_xl_inpaint"] = ["StableDiffusionXLPAGInpaintPipeline"] 36 | 37 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 38 | try: 39 | if not (is_transformers_available() and is_torch_available()): 40 | raise OptionalDependencyNotAvailable() 41 | 42 | except OptionalDependencyNotAvailable: 43 | from ...utils.dummy_torch_and_transformers_objects import * 44 | else: 45 | from .pipeline_pag_controlnet_sd import StableDiffusionControlNetPAGPipeline 46 | from .pipeline_pag_controlnet_sd_xl import StableDiffusionXLControlNetPAGPipeline 47 | from .pipeline_pag_hunyuandit import HunyuanDiTPAGPipeline 48 | from .pipeline_pag_kolors import KolorsPAGPipeline 49 | from .pipeline_pag_pixart_sigma import PixArtSigmaPAGPipeline 50 | from .pipeline_pag_sd import StableDiffusionPAGPipeline 51 | from .pipeline_pag_sd_3 import StableDiffusion3PAGPipeline 52 | from .pipeline_pag_sd_animatediff import AnimateDiffPAGPipeline 53 | from .pipeline_pag_sd_xl import StableDiffusionXLPAGPipeline 54 | from .pipeline_pag_sd_xl_img2img import StableDiffusionXLPAGImg2ImgPipeline 55 | from .pipeline_pag_sd_xl_inpaint import StableDiffusionXLPAGInpaintPipeline 56 | 57 | else: 58 | import sys 59 | 60 | sys.modules[__name__] = _LazyModule( 61 | __name__, 62 | globals()["__file__"], 63 | _import_structure, 64 | module_spec=__spec__, 65 | ) 66 | for name, value in _dummy_objects.items(): 67 | setattr(sys.modules[__name__], name, value) 68 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/paint_by_example/__init__.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import TYPE_CHECKING, List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL 6 | from PIL import Image 7 | 8 | from ...utils import ( 9 | DIFFUSERS_SLOW_IMPORT, 10 | OptionalDependencyNotAvailable, 11 | _LazyModule, 12 | get_objects_from_module, 13 | is_torch_available, 14 | is_transformers_available, 15 | ) 16 | 17 | 18 | _dummy_objects = {} 19 | _import_structure = {} 20 | 21 | try: 22 | if not (is_transformers_available() and is_torch_available()): 23 | raise OptionalDependencyNotAvailable() 24 | except OptionalDependencyNotAvailable: 25 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 26 | 27 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 28 | else: 29 | _import_structure["image_encoder"] = ["PaintByExampleImageEncoder"] 30 | _import_structure["pipeline_paint_by_example"] = ["PaintByExamplePipeline"] 31 | 32 | 33 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 34 | try: 35 | if not (is_transformers_available() and is_torch_available()): 36 | raise OptionalDependencyNotAvailable() 37 | 38 | except OptionalDependencyNotAvailable: 39 | from ...utils.dummy_torch_and_transformers_objects import * 40 | else: 41 | from .image_encoder import PaintByExampleImageEncoder 42 | from .pipeline_paint_by_example import PaintByExamplePipeline 43 | 44 | else: 45 | import sys 46 | 47 | sys.modules[__name__] = _LazyModule( 48 | __name__, 49 | globals()["__file__"], 50 | _import_structure, 51 | module_spec=__spec__, 52 | ) 53 | 54 | for name, value in _dummy_objects.items(): 55 | setattr(sys.modules[__name__], name, value) 56 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/paint_by_example/image_encoder.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import torch 15 | from torch import nn 16 | from transformers import CLIPPreTrainedModel, CLIPVisionModel 17 | 18 | from ...models.attention import BasicTransformerBlock 19 | from ...utils import logging 20 | 21 | 22 | logger = logging.get_logger(__name__) # pylint: disable=invalid-name 23 | 24 | 25 | class PaintByExampleImageEncoder(CLIPPreTrainedModel): 26 | def __init__(self, config, proj_size=None): 27 | super().__init__(config) 28 | self.proj_size = proj_size or getattr(config, "projection_dim", 768) 29 | 30 | self.model = CLIPVisionModel(config) 31 | self.mapper = PaintByExampleMapper(config) 32 | self.final_layer_norm = nn.LayerNorm(config.hidden_size) 33 | self.proj_out = nn.Linear(config.hidden_size, self.proj_size) 34 | 35 | # uncondition for scaling 36 | self.uncond_vector = nn.Parameter(torch.randn((1, 1, self.proj_size))) 37 | 38 | def forward(self, pixel_values, return_uncond_vector=False): 39 | clip_output = self.model(pixel_values=pixel_values) 40 | latent_states = clip_output.pooler_output 41 | latent_states = self.mapper(latent_states[:, None]) 42 | latent_states = self.final_layer_norm(latent_states) 43 | latent_states = self.proj_out(latent_states) 44 | if return_uncond_vector: 45 | return latent_states, self.uncond_vector 46 | 47 | return latent_states 48 | 49 | 50 | class PaintByExampleMapper(nn.Module): 51 | def __init__(self, config): 52 | super().__init__() 53 | num_layers = (config.num_hidden_layers + 1) // 5 54 | hid_size = config.hidden_size 55 | num_heads = 1 56 | self.blocks = nn.ModuleList( 57 | [ 58 | BasicTransformerBlock(hid_size, num_heads, hid_size, activation_fn="gelu", attention_bias=True) 59 | for _ in range(num_layers) 60 | ] 61 | ) 62 | 63 | def forward(self, hidden_states): 64 | for block in self.blocks: 65 | hidden_states = block(hidden_states) 66 | 67 | return hidden_states 68 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/pia/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_pia"] = ["PIAPipeline", "PIAPipelineOutput"] 25 | 26 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 27 | try: 28 | if not (is_transformers_available() and is_torch_available()): 29 | raise OptionalDependencyNotAvailable() 30 | except OptionalDependencyNotAvailable: 31 | from ...utils.dummy_torch_and_transformers_objects import * 32 | 33 | else: 34 | from .pipeline_pia import PIAPipeline, PIAPipelineOutput 35 | 36 | else: 37 | import sys 38 | 39 | sys.modules[__name__] = _LazyModule( 40 | __name__, 41 | globals()["__file__"], 42 | _import_structure, 43 | module_spec=__spec__, 44 | ) 45 | for name, value in _dummy_objects.items(): 46 | setattr(sys.modules[__name__], name, value) 47 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/pixart_alpha/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_pixart_alpha"] = ["PixArtAlphaPipeline"] 26 | _import_structure["pipeline_pixart_sigma"] = ["PixArtSigmaPipeline"] 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .pipeline_pixart_alpha import ( 37 | ASPECT_RATIO_256_BIN, 38 | ASPECT_RATIO_512_BIN, 39 | ASPECT_RATIO_1024_BIN, 40 | PixArtAlphaPipeline, 41 | ) 42 | from .pipeline_pixart_sigma import ASPECT_RATIO_2048_BIN, PixArtSigmaPipeline 43 | 44 | else: 45 | import sys 46 | 47 | sys.modules[__name__] = _LazyModule( 48 | __name__, 49 | globals()["__file__"], 50 | _import_structure, 51 | module_spec=__spec__, 52 | ) 53 | 54 | for name, value in _dummy_objects.items(): 55 | setattr(sys.modules[__name__], name, value) 56 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/semantic_stable_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_output"] = ["SemanticStableDiffusionPipelineOutput"] 25 | _import_structure["pipeline_semantic_stable_diffusion"] = ["SemanticStableDiffusionPipeline"] 26 | 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .pipeline_semantic_stable_diffusion import SemanticStableDiffusionPipeline 37 | 38 | else: 39 | import sys 40 | 41 | sys.modules[__name__] = _LazyModule( 42 | __name__, 43 | globals()["__file__"], 44 | _import_structure, 45 | module_spec=__spec__, 46 | ) 47 | 48 | for name, value in _dummy_objects.items(): 49 | setattr(sys.modules[__name__], name, value) 50 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/semantic_stable_diffusion/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput 8 | 9 | 10 | @dataclass 11 | class SemanticStableDiffusionPipelineOutput(BaseOutput): 12 | """ 13 | Output class for Stable Diffusion pipelines. 14 | 15 | Args: 16 | images (`List[PIL.Image.Image]` or `np.ndarray`) 17 | List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, 18 | num_channels)`. 19 | nsfw_content_detected (`List[bool]`) 20 | List indicating whether the corresponding generated image contains “not-safe-for-work” (nsfw) content or 21 | `None` if safety checking could not be performed. 22 | """ 23 | 24 | images: Union[List[PIL.Image.Image], np.ndarray] 25 | nsfw_content_detected: Optional[List[bool]] 26 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/shap_e/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["camera"] = ["create_pan_cameras"] 25 | _import_structure["pipeline_shap_e"] = ["ShapEPipeline"] 26 | _import_structure["pipeline_shap_e_img2img"] = ["ShapEImg2ImgPipeline"] 27 | _import_structure["renderer"] = [ 28 | "BoundingBoxVolume", 29 | "ImportanceRaySampler", 30 | "MLPNeRFModelOutput", 31 | "MLPNeRSTFModel", 32 | "ShapEParamsProjModel", 33 | "ShapERenderer", 34 | "StratifiedRaySampler", 35 | "VoidNeRFModel", 36 | ] 37 | 38 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 39 | try: 40 | if not (is_transformers_available() and is_torch_available()): 41 | raise OptionalDependencyNotAvailable() 42 | 43 | except OptionalDependencyNotAvailable: 44 | from ...utils.dummy_torch_and_transformers_objects import * 45 | else: 46 | from .camera import create_pan_cameras 47 | from .pipeline_shap_e import ShapEPipeline 48 | from .pipeline_shap_e_img2img import ShapEImg2ImgPipeline 49 | from .renderer import ( 50 | BoundingBoxVolume, 51 | ImportanceRaySampler, 52 | MLPNeRFModelOutput, 53 | MLPNeRSTFModel, 54 | ShapEParamsProjModel, 55 | ShapERenderer, 56 | StratifiedRaySampler, 57 | VoidNeRFModel, 58 | ) 59 | 60 | else: 61 | import sys 62 | 63 | sys.modules[__name__] = _LazyModule( 64 | __name__, 65 | globals()["__file__"], 66 | _import_structure, 67 | module_spec=__spec__, 68 | ) 69 | 70 | for name, value in _dummy_objects.items(): 71 | setattr(sys.modules[__name__], name, value) 72 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_audio/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | is_transformers_version, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["modeling_stable_audio"] = ["StableAudioProjectionModel"] 26 | _import_structure["pipeline_stable_audio"] = ["StableAudioPipeline"] 27 | 28 | 29 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 30 | try: 31 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 32 | raise OptionalDependencyNotAvailable() 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | 36 | else: 37 | from .modeling_stable_audio import StableAudioProjectionModel 38 | from .pipeline_stable_audio import StableAudioPipeline 39 | 40 | else: 41 | import sys 42 | 43 | sys.modules[__name__] = _LazyModule( 44 | __name__, 45 | globals()["__file__"], 46 | _import_structure, 47 | module_spec=__spec__, 48 | ) 49 | for name, value in _dummy_objects.items(): 50 | setattr(sys.modules[__name__], name, value) 51 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_cascade/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_stable_cascade"] = ["StableCascadeDecoderPipeline"] 25 | _import_structure["pipeline_stable_cascade_combined"] = ["StableCascadeCombinedPipeline"] 26 | _import_structure["pipeline_stable_cascade_prior"] = ["StableCascadePriorPipeline"] 27 | 28 | 29 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 30 | try: 31 | if not (is_transformers_available() and is_torch_available()): 32 | raise OptionalDependencyNotAvailable() 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 35 | else: 36 | from .pipeline_stable_cascade import StableCascadeDecoderPipeline 37 | from .pipeline_stable_cascade_combined import StableCascadeCombinedPipeline 38 | from .pipeline_stable_cascade_prior import StableCascadePriorPipeline 39 | else: 40 | import sys 41 | 42 | sys.modules[__name__] = _LazyModule( 43 | __name__, 44 | globals()["__file__"], 45 | _import_structure, 46 | module_spec=__spec__, 47 | ) 48 | 49 | for name, value in _dummy_objects.items(): 50 | setattr(sys.modules[__name__], name, value) 51 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion/clip_image_project_model.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The GLIGEN Authors and HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from torch import nn 16 | 17 | from ...configuration_utils import ConfigMixin, register_to_config 18 | from ...models.modeling_utils import ModelMixin 19 | 20 | 21 | class CLIPImageProjection(ModelMixin, ConfigMixin): 22 | @register_to_config 23 | def __init__(self, hidden_size: int = 768): 24 | super().__init__() 25 | self.hidden_size = hidden_size 26 | self.project = nn.Linear(self.hidden_size, self.hidden_size, bias=False) 27 | 28 | def forward(self, x): 29 | return self.project(x) 30 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput, is_flax_available 8 | 9 | 10 | @dataclass 11 | class StableDiffusionPipelineOutput(BaseOutput): 12 | """ 13 | Output class for Stable Diffusion pipelines. 14 | 15 | Args: 16 | images (`List[PIL.Image.Image]` or `np.ndarray`) 17 | List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, 18 | num_channels)`. 19 | nsfw_content_detected (`List[bool]`) 20 | List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or 21 | `None` if safety checking could not be performed. 22 | """ 23 | 24 | images: Union[List[PIL.Image.Image], np.ndarray] 25 | nsfw_content_detected: Optional[List[bool]] 26 | 27 | 28 | if is_flax_available(): 29 | import flax 30 | 31 | @flax.struct.dataclass 32 | class FlaxStableDiffusionPipelineOutput(BaseOutput): 33 | """ 34 | Output class for Flax-based Stable Diffusion pipelines. 35 | 36 | Args: 37 | images (`np.ndarray`): 38 | Denoised images of array shape of `(batch_size, height, width, num_channels)`. 39 | nsfw_content_detected (`List[bool]`): 40 | List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content 41 | or `None` if safety checking could not be performed. 42 | """ 43 | 44 | images: np.ndarray 45 | nsfw_content_detected: List[bool] 46 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from typing import Optional, Union 16 | 17 | import torch 18 | from torch import nn 19 | 20 | from ...configuration_utils import ConfigMixin, register_to_config 21 | from ...models.modeling_utils import ModelMixin 22 | 23 | 24 | class StableUnCLIPImageNormalizer(ModelMixin, ConfigMixin): 25 | """ 26 | This class is used to hold the mean and standard deviation of the CLIP embedder used in stable unCLIP. 27 | 28 | It is used to normalize the image embeddings before the noise is applied and un-normalize the noised image 29 | embeddings. 30 | """ 31 | 32 | @register_to_config 33 | def __init__( 34 | self, 35 | embedding_dim: int = 768, 36 | ): 37 | super().__init__() 38 | 39 | self.mean = nn.Parameter(torch.zeros(1, embedding_dim)) 40 | self.std = nn.Parameter(torch.ones(1, embedding_dim)) 41 | 42 | def to( 43 | self, 44 | torch_device: Optional[Union[str, torch.device]] = None, 45 | torch_dtype: Optional[torch.dtype] = None, 46 | ): 47 | self.mean = nn.Parameter(self.mean.to(torch_device).to(torch_dtype)) 48 | self.std = nn.Parameter(self.std.to(torch_device).to(torch_dtype)) 49 | return self 50 | 51 | def scale(self, embeds): 52 | embeds = (embeds - self.mean) * 1.0 / self.std 53 | return embeds 54 | 55 | def unscale(self, embeds): 56 | embeds = (embeds * self.std) + self.mean 57 | return embeds 58 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_3/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_flax_available, 9 | is_torch_available, 10 | is_transformers_available, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _additional_imports = {} 16 | _import_structure = {"pipeline_output": ["StableDiffusion3PipelineOutput"]} 17 | 18 | try: 19 | if not (is_transformers_available() and is_torch_available()): 20 | raise OptionalDependencyNotAvailable() 21 | except OptionalDependencyNotAvailable: 22 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 23 | 24 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 25 | else: 26 | _import_structure["pipeline_stable_diffusion_3"] = ["StableDiffusion3Pipeline"] 27 | _import_structure["pipeline_stable_diffusion_3_img2img"] = ["StableDiffusion3Img2ImgPipeline"] 28 | _import_structure["pipeline_stable_diffusion_3_inpaint"] = ["StableDiffusion3InpaintPipeline"] 29 | 30 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 31 | try: 32 | if not (is_transformers_available() and is_torch_available()): 33 | raise OptionalDependencyNotAvailable() 34 | except OptionalDependencyNotAvailable: 35 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 36 | else: 37 | from .pipeline_stable_diffusion_3 import StableDiffusion3Pipeline 38 | from .pipeline_stable_diffusion_3_img2img import StableDiffusion3Img2ImgPipeline 39 | from .pipeline_stable_diffusion_3_inpaint import StableDiffusion3InpaintPipeline 40 | 41 | else: 42 | import sys 43 | 44 | sys.modules[__name__] = _LazyModule( 45 | __name__, 46 | globals()["__file__"], 47 | _import_structure, 48 | module_spec=__spec__, 49 | ) 50 | 51 | for name, value in _dummy_objects.items(): 52 | setattr(sys.modules[__name__], name, value) 53 | for name, value in _additional_imports.items(): 54 | setattr(sys.modules[__name__], name, value) 55 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_3/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput 8 | 9 | 10 | @dataclass 11 | class StableDiffusion3PipelineOutput(BaseOutput): 12 | """ 13 | Output class for Stable Diffusion pipelines. 14 | 15 | Args: 16 | images (`List[PIL.Image.Image]` or `np.ndarray`) 17 | List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, 18 | num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. 19 | """ 20 | 21 | images: Union[List[PIL.Image.Image], np.ndarray] 22 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_attend_and_excite/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_stable_diffusion_attend_and_excite"] = ["StableDiffusionAttendAndExcitePipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_stable_diffusion_attend_and_excite import StableDiffusionAttendAndExcitePipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_diffedit/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_stable_diffusion_diffedit"] = ["StableDiffusionDiffEditPipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_stable_diffusion_diffedit import StableDiffusionDiffEditPipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_gligen/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_stable_diffusion_gligen"] = ["StableDiffusionGLIGENPipeline"] 26 | _import_structure["pipeline_stable_diffusion_gligen_text_image"] = ["StableDiffusionGLIGENTextImagePipeline"] 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | 33 | except OptionalDependencyNotAvailable: 34 | from ...utils.dummy_torch_and_transformers_objects import * 35 | else: 36 | from .pipeline_stable_diffusion_gligen import StableDiffusionGLIGENPipeline 37 | from .pipeline_stable_diffusion_gligen_text_image import StableDiffusionGLIGENTextImagePipeline 38 | 39 | else: 40 | import sys 41 | 42 | sys.modules[__name__] = _LazyModule( 43 | __name__, 44 | globals()["__file__"], 45 | _import_structure, 46 | module_spec=__spec__, 47 | ) 48 | 49 | for name, value in _dummy_objects.items(): 50 | setattr(sys.modules[__name__], name, value) 51 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_k_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_k_diffusion_available, 9 | is_k_diffusion_version, 10 | is_torch_available, 11 | is_transformers_available, 12 | ) 13 | 14 | 15 | _dummy_objects = {} 16 | _import_structure = {} 17 | 18 | 19 | try: 20 | if not ( 21 | is_transformers_available() 22 | and is_torch_available() 23 | and is_k_diffusion_available() 24 | and is_k_diffusion_version(">=", "0.0.12") 25 | ): 26 | raise OptionalDependencyNotAvailable() 27 | except OptionalDependencyNotAvailable: 28 | from ...utils import dummy_torch_and_transformers_and_k_diffusion_objects # noqa F403 29 | 30 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_and_k_diffusion_objects)) 31 | else: 32 | _import_structure["pipeline_stable_diffusion_k_diffusion"] = ["StableDiffusionKDiffusionPipeline"] 33 | _import_structure["pipeline_stable_diffusion_xl_k_diffusion"] = ["StableDiffusionXLKDiffusionPipeline"] 34 | 35 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 36 | try: 37 | if not ( 38 | is_transformers_available() 39 | and is_torch_available() 40 | and is_k_diffusion_available() 41 | and is_k_diffusion_version(">=", "0.0.12") 42 | ): 43 | raise OptionalDependencyNotAvailable() 44 | 45 | except OptionalDependencyNotAvailable: 46 | from ...utils.dummy_torch_and_transformers_and_k_diffusion_objects import * 47 | else: 48 | from .pipeline_stable_diffusion_k_diffusion import StableDiffusionKDiffusionPipeline 49 | from .pipeline_stable_diffusion_xl_k_diffusion import StableDiffusionXLKDiffusionPipeline 50 | 51 | else: 52 | import sys 53 | 54 | sys.modules[__name__] = _LazyModule( 55 | __name__, 56 | globals()["__file__"], 57 | _import_structure, 58 | module_spec=__spec__, 59 | ) 60 | 61 | for name, value in _dummy_objects.items(): 62 | setattr(sys.modules[__name__], name, value) 63 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_ldm3d/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_stable_diffusion_ldm3d"] = ["StableDiffusionLDM3DPipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_stable_diffusion_ldm3d import StableDiffusionLDM3DPipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_panorama/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_stable_diffusion_panorama"] = ["StableDiffusionPanoramaPipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_stable_diffusion_panorama import StableDiffusionPanoramaPipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_safe/__init__.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum 3 | from typing import TYPE_CHECKING, List, Optional, Union 4 | 5 | import numpy as np 6 | import PIL 7 | from PIL import Image 8 | 9 | from ...utils import ( 10 | DIFFUSERS_SLOW_IMPORT, 11 | BaseOutput, 12 | OptionalDependencyNotAvailable, 13 | _LazyModule, 14 | get_objects_from_module, 15 | is_torch_available, 16 | is_transformers_available, 17 | ) 18 | 19 | 20 | @dataclass 21 | class SafetyConfig(object): 22 | WEAK = { 23 | "sld_warmup_steps": 15, 24 | "sld_guidance_scale": 20, 25 | "sld_threshold": 0.0, 26 | "sld_momentum_scale": 0.0, 27 | "sld_mom_beta": 0.0, 28 | } 29 | MEDIUM = { 30 | "sld_warmup_steps": 10, 31 | "sld_guidance_scale": 1000, 32 | "sld_threshold": 0.01, 33 | "sld_momentum_scale": 0.3, 34 | "sld_mom_beta": 0.4, 35 | } 36 | STRONG = { 37 | "sld_warmup_steps": 7, 38 | "sld_guidance_scale": 2000, 39 | "sld_threshold": 0.025, 40 | "sld_momentum_scale": 0.5, 41 | "sld_mom_beta": 0.7, 42 | } 43 | MAX = { 44 | "sld_warmup_steps": 0, 45 | "sld_guidance_scale": 5000, 46 | "sld_threshold": 1.0, 47 | "sld_momentum_scale": 0.5, 48 | "sld_mom_beta": 0.7, 49 | } 50 | 51 | 52 | _dummy_objects = {} 53 | _additional_imports = {} 54 | _import_structure = {} 55 | 56 | _additional_imports.update({"SafetyConfig": SafetyConfig}) 57 | 58 | try: 59 | if not (is_transformers_available() and is_torch_available()): 60 | raise OptionalDependencyNotAvailable() 61 | except OptionalDependencyNotAvailable: 62 | from ...utils import dummy_torch_and_transformers_objects 63 | 64 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 65 | else: 66 | _import_structure.update( 67 | { 68 | "pipeline_output": ["StableDiffusionSafePipelineOutput"], 69 | "pipeline_stable_diffusion_safe": ["StableDiffusionPipelineSafe"], 70 | "safety_checker": ["StableDiffusionSafetyChecker"], 71 | } 72 | ) 73 | 74 | 75 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 76 | try: 77 | if not (is_transformers_available() and is_torch_available()): 78 | raise OptionalDependencyNotAvailable() 79 | except OptionalDependencyNotAvailable: 80 | from ...utils.dummy_torch_and_transformers_objects import * 81 | else: 82 | from .pipeline_output import StableDiffusionSafePipelineOutput 83 | from .pipeline_stable_diffusion_safe import StableDiffusionPipelineSafe 84 | from .safety_checker import SafeStableDiffusionSafetyChecker 85 | 86 | else: 87 | import sys 88 | 89 | sys.modules[__name__] = _LazyModule( 90 | __name__, 91 | globals()["__file__"], 92 | _import_structure, 93 | module_spec=__spec__, 94 | ) 95 | 96 | for name, value in _dummy_objects.items(): 97 | setattr(sys.modules[__name__], name, value) 98 | for name, value in _additional_imports.items(): 99 | setattr(sys.modules[__name__], name, value) 100 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_safe/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import ( 8 | BaseOutput, 9 | ) 10 | 11 | 12 | @dataclass 13 | class StableDiffusionSafePipelineOutput(BaseOutput): 14 | """ 15 | Output class for Safe Stable Diffusion pipelines. 16 | 17 | Args: 18 | images (`List[PIL.Image.Image]` or `np.ndarray`) 19 | List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, 20 | num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. 21 | nsfw_content_detected (`List[bool]`) 22 | List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" 23 | (nsfw) content, or `None` if safety checking could not be performed. 24 | images (`List[PIL.Image.Image]` or `np.ndarray`) 25 | List of denoised PIL images that were flagged by the safety checker any may contain "not-safe-for-work" 26 | (nsfw) content, or `None` if no safety check was performed or no images were flagged. 27 | applied_safety_concept (`str`) 28 | The safety concept that was applied for safety guidance, or `None` if safety guidance was disabled 29 | """ 30 | 31 | images: Union[List[PIL.Image.Image], np.ndarray] 32 | nsfw_content_detected: Optional[List[bool]] 33 | unsafe_images: Optional[Union[List[PIL.Image.Image], np.ndarray]] 34 | applied_safety_concept: Optional[str] 35 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_sag/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure["pipeline_stable_diffusion_sag"] = ["StableDiffusionSAGPipeline"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not (is_transformers_available() and is_torch_available()): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * 34 | else: 35 | from .pipeline_stable_diffusion_sag import StableDiffusionSAGPipeline 36 | 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | 47 | for name, value in _dummy_objects.items(): 48 | setattr(sys.modules[__name__], name, value) 49 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_xl/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_flax_available, 9 | is_torch_available, 10 | is_transformers_available, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _additional_imports = {} 16 | _import_structure = {"pipeline_output": ["StableDiffusionXLPipelineOutput"]} 17 | 18 | if is_transformers_available() and is_flax_available(): 19 | _import_structure["pipeline_output"].extend(["FlaxStableDiffusionXLPipelineOutput"]) 20 | try: 21 | if not (is_transformers_available() and is_torch_available()): 22 | raise OptionalDependencyNotAvailable() 23 | except OptionalDependencyNotAvailable: 24 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 25 | 26 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 27 | else: 28 | _import_structure["pipeline_stable_diffusion_xl"] = ["StableDiffusionXLPipeline"] 29 | _import_structure["pipeline_stable_diffusion_xl_img2img"] = ["StableDiffusionXLImg2ImgPipeline"] 30 | _import_structure["pipeline_stable_diffusion_xl_inpaint"] = ["StableDiffusionXLInpaintPipeline"] 31 | _import_structure["pipeline_stable_diffusion_xl_instruct_pix2pix"] = ["StableDiffusionXLInstructPix2PixPipeline"] 32 | 33 | if is_transformers_available() and is_flax_available(): 34 | from ...schedulers.scheduling_pndm_flax import PNDMSchedulerState 35 | 36 | _additional_imports.update({"PNDMSchedulerState": PNDMSchedulerState}) 37 | _import_structure["pipeline_flax_stable_diffusion_xl"] = ["FlaxStableDiffusionXLPipeline"] 38 | 39 | 40 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 41 | try: 42 | if not (is_transformers_available() and is_torch_available()): 43 | raise OptionalDependencyNotAvailable() 44 | except OptionalDependencyNotAvailable: 45 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 46 | else: 47 | from .pipeline_stable_diffusion_xl import StableDiffusionXLPipeline 48 | from .pipeline_stable_diffusion_xl_img2img import StableDiffusionXLImg2ImgPipeline 49 | from .pipeline_stable_diffusion_xl_inpaint import StableDiffusionXLInpaintPipeline 50 | from .pipeline_stable_diffusion_xl_instruct_pix2pix import StableDiffusionXLInstructPix2PixPipeline 51 | 52 | try: 53 | if not (is_transformers_available() and is_flax_available()): 54 | raise OptionalDependencyNotAvailable() 55 | except OptionalDependencyNotAvailable: 56 | from ...utils.dummy_flax_objects import * 57 | else: 58 | from .pipeline_flax_stable_diffusion_xl import ( 59 | FlaxStableDiffusionXLPipeline, 60 | ) 61 | from .pipeline_output import FlaxStableDiffusionXLPipelineOutput 62 | 63 | else: 64 | import sys 65 | 66 | sys.modules[__name__] = _LazyModule( 67 | __name__, 68 | globals()["__file__"], 69 | _import_structure, 70 | module_spec=__spec__, 71 | ) 72 | 73 | for name, value in _dummy_objects.items(): 74 | setattr(sys.modules[__name__], name, value) 75 | for name, value in _additional_imports.items(): 76 | setattr(sys.modules[__name__], name, value) 77 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_xl/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Union 3 | 4 | import numpy as np 5 | import PIL.Image 6 | 7 | from ...utils import BaseOutput, is_flax_available 8 | 9 | 10 | @dataclass 11 | class StableDiffusionXLPipelineOutput(BaseOutput): 12 | """ 13 | Output class for Stable Diffusion pipelines. 14 | 15 | Args: 16 | images (`List[PIL.Image.Image]` or `np.ndarray`) 17 | List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, 18 | num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. 19 | """ 20 | 21 | images: Union[List[PIL.Image.Image], np.ndarray] 22 | 23 | 24 | if is_flax_available(): 25 | import flax 26 | 27 | @flax.struct.dataclass 28 | class FlaxStableDiffusionXLPipelineOutput(BaseOutput): 29 | """ 30 | Output class for Flax Stable Diffusion XL pipelines. 31 | 32 | Args: 33 | images (`np.ndarray`) 34 | Array of shape `(batch_size, height, width, num_channels)` with images from the diffusion pipeline. 35 | """ 36 | 37 | images: np.ndarray 38 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_diffusion_xl/watermark.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | from ...utils import is_invisible_watermark_available 5 | 6 | 7 | if is_invisible_watermark_available(): 8 | from imwatermark import WatermarkEncoder 9 | 10 | 11 | # Copied from https://github.com/Stability-AI/generative-models/blob/613af104c6b85184091d42d374fef420eddb356d/scripts/demo/streamlit_helpers.py#L66 12 | WATERMARK_MESSAGE = 0b101100111110110010010000011110111011000110011110 13 | # bin(x)[2:] gives bits of x as str, use int to convert them to 0/1 14 | WATERMARK_BITS = [int(bit) for bit in bin(WATERMARK_MESSAGE)[2:]] 15 | 16 | 17 | class StableDiffusionXLWatermarker: 18 | def __init__(self): 19 | self.watermark = WATERMARK_BITS 20 | self.encoder = WatermarkEncoder() 21 | 22 | self.encoder.set_watermark("bits", self.watermark) 23 | 24 | def apply_watermark(self, images: torch.Tensor): 25 | # can't encode images that are smaller than 256 26 | if images.shape[-1] < 256: 27 | return images 28 | 29 | images = (255 * (images / 2 + 0.5)).cpu().permute(0, 2, 3, 1).float().numpy() 30 | 31 | # Convert RGB to BGR, which is the channel order expected by the watermark encoder. 32 | images = images[:, :, :, ::-1] 33 | 34 | # Add watermark and convert BGR back to RGB 35 | images = [self.encoder.encode(image, "dwtDct")[:, :, ::-1] for image in images] 36 | 37 | images = np.array(images) 38 | 39 | images = torch.from_numpy(images).permute(0, 3, 1, 2) 40 | 41 | images = torch.clamp(2 * (images / 255 - 0.5), min=-1.0, max=1.0) 42 | return images 43 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/stable_video_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | BaseOutput, 6 | OptionalDependencyNotAvailable, 7 | _LazyModule, 8 | get_objects_from_module, 9 | is_torch_available, 10 | is_transformers_available, 11 | ) 12 | 13 | 14 | _dummy_objects = {} 15 | _import_structure = {} 16 | 17 | try: 18 | if not (is_transformers_available() and is_torch_available()): 19 | raise OptionalDependencyNotAvailable() 20 | except OptionalDependencyNotAvailable: 21 | from ...utils import dummy_torch_and_transformers_objects 22 | 23 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 24 | else: 25 | _import_structure.update( 26 | { 27 | "pipeline_stable_video_diffusion": [ 28 | "StableVideoDiffusionPipeline", 29 | "StableVideoDiffusionPipelineOutput", 30 | ], 31 | } 32 | ) 33 | 34 | 35 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 36 | try: 37 | if not (is_transformers_available() and is_torch_available()): 38 | raise OptionalDependencyNotAvailable() 39 | except OptionalDependencyNotAvailable: 40 | from ...utils.dummy_torch_and_transformers_objects import * 41 | else: 42 | from .pipeline_stable_video_diffusion import ( 43 | StableVideoDiffusionPipeline, 44 | StableVideoDiffusionPipelineOutput, 45 | ) 46 | 47 | else: 48 | import sys 49 | 50 | sys.modules[__name__] = _LazyModule( 51 | __name__, 52 | globals()["__file__"], 53 | _import_structure, 54 | module_spec=__spec__, 55 | ) 56 | 57 | for name, value in _dummy_objects.items(): 58 | setattr(sys.modules[__name__], name, value) 59 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/t2i_adapter/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_stable_diffusion_adapter"] = ["StableDiffusionAdapterPipeline"] 25 | _import_structure["pipeline_stable_diffusion_xl_adapter"] = ["StableDiffusionXLAdapterPipeline"] 26 | 27 | 28 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 29 | try: 30 | if not (is_transformers_available() and is_torch_available()): 31 | raise OptionalDependencyNotAvailable() 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 34 | else: 35 | from .pipeline_stable_diffusion_adapter import StableDiffusionAdapterPipeline 36 | from .pipeline_stable_diffusion_xl_adapter import StableDiffusionXLAdapterPipeline 37 | else: 38 | import sys 39 | 40 | sys.modules[__name__] = _LazyModule( 41 | __name__, 42 | globals()["__file__"], 43 | _import_structure, 44 | module_spec=__spec__, 45 | ) 46 | for name, value in _dummy_objects.items(): 47 | setattr(sys.modules[__name__], name, value) 48 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/text_to_video_synthesis/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["pipeline_output"] = ["TextToVideoSDPipelineOutput"] 25 | _import_structure["pipeline_text_to_video_synth"] = ["TextToVideoSDPipeline"] 26 | _import_structure["pipeline_text_to_video_synth_img2img"] = ["VideoToVideoSDPipeline"] 27 | _import_structure["pipeline_text_to_video_zero"] = ["TextToVideoZeroPipeline"] 28 | _import_structure["pipeline_text_to_video_zero_sdxl"] = ["TextToVideoZeroSDXLPipeline"] 29 | 30 | 31 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 32 | try: 33 | if not (is_transformers_available() and is_torch_available()): 34 | raise OptionalDependencyNotAvailable() 35 | except OptionalDependencyNotAvailable: 36 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 37 | else: 38 | from .pipeline_output import TextToVideoSDPipelineOutput 39 | from .pipeline_text_to_video_synth import TextToVideoSDPipeline 40 | from .pipeline_text_to_video_synth_img2img import VideoToVideoSDPipeline 41 | from .pipeline_text_to_video_zero import TextToVideoZeroPipeline 42 | from .pipeline_text_to_video_zero_sdxl import TextToVideoZeroSDXLPipeline 43 | 44 | else: 45 | import sys 46 | 47 | sys.modules[__name__] = _LazyModule( 48 | __name__, 49 | globals()["__file__"], 50 | _import_structure, 51 | module_spec=__spec__, 52 | ) 53 | for name, value in _dummy_objects.items(): 54 | setattr(sys.modules[__name__], name, value) 55 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/text_to_video_synthesis/pipeline_output.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Union 3 | 4 | import numpy as np 5 | import PIL 6 | import torch 7 | 8 | from ...utils import ( 9 | BaseOutput, 10 | ) 11 | 12 | 13 | @dataclass 14 | class TextToVideoSDPipelineOutput(BaseOutput): 15 | """ 16 | Output class for text-to-video pipelines. 17 | 18 | Args: 19 | frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): 20 | List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing 21 | denoised 22 | PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape 23 | `(batch_size, num_frames, channels, height, width)` 24 | """ 25 | 26 | frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]] 27 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/unclip/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | is_torch_available, 8 | is_transformers_available, 9 | is_transformers_version, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline 21 | 22 | _dummy_objects.update( 23 | {"UnCLIPImageVariationPipeline": UnCLIPImageVariationPipeline, "UnCLIPPipeline": UnCLIPPipeline} 24 | ) 25 | else: 26 | _import_structure["pipeline_unclip"] = ["UnCLIPPipeline"] 27 | _import_structure["pipeline_unclip_image_variation"] = ["UnCLIPImageVariationPipeline"] 28 | _import_structure["text_proj"] = ["UnCLIPTextProjModel"] 29 | 30 | 31 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 32 | try: 33 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): 34 | raise OptionalDependencyNotAvailable() 35 | except OptionalDependencyNotAvailable: 36 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 37 | else: 38 | from .pipeline_unclip import UnCLIPPipeline 39 | from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline 40 | from .text_proj import UnCLIPTextProjModel 41 | 42 | else: 43 | import sys 44 | 45 | sys.modules[__name__] = _LazyModule( 46 | __name__, 47 | globals()["__file__"], 48 | _import_structure, 49 | module_spec=__spec__, 50 | ) 51 | for name, value in _dummy_objects.items(): 52 | setattr(sys.modules[__name__], name, value) 53 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/unidiffuser/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | is_torch_available, 8 | is_transformers_available, 9 | ) 10 | 11 | 12 | _dummy_objects = {} 13 | _import_structure = {} 14 | 15 | try: 16 | if not (is_transformers_available() and is_torch_available()): 17 | raise OptionalDependencyNotAvailable() 18 | except OptionalDependencyNotAvailable: 19 | from ...utils.dummy_torch_and_transformers_objects import ( 20 | ImageTextPipelineOutput, 21 | UniDiffuserPipeline, 22 | ) 23 | 24 | _dummy_objects.update( 25 | {"ImageTextPipelineOutput": ImageTextPipelineOutput, "UniDiffuserPipeline": UniDiffuserPipeline} 26 | ) 27 | else: 28 | _import_structure["modeling_text_decoder"] = ["UniDiffuserTextDecoder"] 29 | _import_structure["modeling_uvit"] = ["UniDiffuserModel", "UTransformer2DModel"] 30 | _import_structure["pipeline_unidiffuser"] = ["ImageTextPipelineOutput", "UniDiffuserPipeline"] 31 | 32 | 33 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 34 | try: 35 | if not (is_transformers_available() and is_torch_available()): 36 | raise OptionalDependencyNotAvailable() 37 | except OptionalDependencyNotAvailable: 38 | from ...utils.dummy_torch_and_transformers_objects import ( 39 | ImageTextPipelineOutput, 40 | UniDiffuserPipeline, 41 | ) 42 | else: 43 | from .modeling_text_decoder import UniDiffuserTextDecoder 44 | from .modeling_uvit import UniDiffuserModel, UTransformer2DModel 45 | from .pipeline_unidiffuser import ImageTextPipelineOutput, UniDiffuserPipeline 46 | 47 | else: 48 | import sys 49 | 50 | sys.modules[__name__] = _LazyModule( 51 | __name__, 52 | globals()["__file__"], 53 | _import_structure, 54 | module_spec=__spec__, 55 | ) 56 | 57 | for name, value in _dummy_objects.items(): 58 | setattr(sys.modules[__name__], name, value) 59 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/wuerstchen/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_torch_and_transformers_objects 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) 23 | else: 24 | _import_structure["modeling_paella_vq_model"] = ["PaellaVQModel"] 25 | _import_structure["modeling_wuerstchen_diffnext"] = ["WuerstchenDiffNeXt"] 26 | _import_structure["modeling_wuerstchen_prior"] = ["WuerstchenPrior"] 27 | _import_structure["pipeline_wuerstchen"] = ["WuerstchenDecoderPipeline"] 28 | _import_structure["pipeline_wuerstchen_combined"] = ["WuerstchenCombinedPipeline"] 29 | _import_structure["pipeline_wuerstchen_prior"] = ["DEFAULT_STAGE_C_TIMESTEPS", "WuerstchenPriorPipeline"] 30 | 31 | 32 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 33 | try: 34 | if not (is_transformers_available() and is_torch_available()): 35 | raise OptionalDependencyNotAvailable() 36 | except OptionalDependencyNotAvailable: 37 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 38 | else: 39 | from .modeling_paella_vq_model import PaellaVQModel 40 | from .modeling_wuerstchen_diffnext import WuerstchenDiffNeXt 41 | from .modeling_wuerstchen_prior import WuerstchenPrior 42 | from .pipeline_wuerstchen import WuerstchenDecoderPipeline 43 | from .pipeline_wuerstchen_combined import WuerstchenCombinedPipeline 44 | from .pipeline_wuerstchen_prior import DEFAULT_STAGE_C_TIMESTEPS, WuerstchenPriorPipeline 45 | else: 46 | import sys 47 | 48 | sys.modules[__name__] = _LazyModule( 49 | __name__, 50 | globals()["__file__"], 51 | _import_structure, 52 | module_spec=__spec__, 53 | ) 54 | 55 | for name, value in _dummy_objects.items(): 56 | setattr(sys.modules[__name__], name, value) 57 | -------------------------------------------------------------------------------- /src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from ...models.attention_processor import Attention 5 | 6 | 7 | class WuerstchenLayerNorm(nn.LayerNorm): 8 | def __init__(self, *args, **kwargs): 9 | super().__init__(*args, **kwargs) 10 | 11 | def forward(self, x): 12 | x = x.permute(0, 2, 3, 1) 13 | x = super().forward(x) 14 | return x.permute(0, 3, 1, 2) 15 | 16 | 17 | class TimestepBlock(nn.Module): 18 | def __init__(self, c, c_timestep): 19 | super().__init__() 20 | 21 | self.mapper = nn.Linear(c_timestep, c * 2) 22 | 23 | def forward(self, x, t): 24 | a, b = self.mapper(t)[:, :, None, None].chunk(2, dim=1) 25 | return x * (1 + a) + b 26 | 27 | 28 | class ResBlock(nn.Module): 29 | def __init__(self, c, c_skip=0, kernel_size=3, dropout=0.0): 30 | super().__init__() 31 | 32 | self.depthwise = nn.Conv2d(c + c_skip, c, kernel_size=kernel_size, padding=kernel_size // 2, groups=c) 33 | self.norm = WuerstchenLayerNorm(c, elementwise_affine=False, eps=1e-6) 34 | self.channelwise = nn.Sequential( 35 | nn.Linear(c, c * 4), nn.GELU(), GlobalResponseNorm(c * 4), nn.Dropout(dropout), nn.Linear(c * 4, c) 36 | ) 37 | 38 | def forward(self, x, x_skip=None): 39 | x_res = x 40 | if x_skip is not None: 41 | x = torch.cat([x, x_skip], dim=1) 42 | x = self.norm(self.depthwise(x)).permute(0, 2, 3, 1) 43 | x = self.channelwise(x).permute(0, 3, 1, 2) 44 | return x + x_res 45 | 46 | 47 | # from https://github.com/facebookresearch/ConvNeXt-V2/blob/3608f67cc1dae164790c5d0aead7bf2d73d9719b/models/utils.py#L105 48 | class GlobalResponseNorm(nn.Module): 49 | def __init__(self, dim): 50 | super().__init__() 51 | self.gamma = nn.Parameter(torch.zeros(1, 1, 1, dim)) 52 | self.beta = nn.Parameter(torch.zeros(1, 1, 1, dim)) 53 | 54 | def forward(self, x): 55 | agg_norm = torch.norm(x, p=2, dim=(1, 2), keepdim=True) 56 | stand_div_norm = agg_norm / (agg_norm.mean(dim=-1, keepdim=True) + 1e-6) 57 | return self.gamma * (x * stand_div_norm) + self.beta + x 58 | 59 | 60 | class AttnBlock(nn.Module): 61 | def __init__(self, c, c_cond, nhead, self_attn=True, dropout=0.0): 62 | super().__init__() 63 | 64 | self.self_attn = self_attn 65 | self.norm = WuerstchenLayerNorm(c, elementwise_affine=False, eps=1e-6) 66 | self.attention = Attention(query_dim=c, heads=nhead, dim_head=c // nhead, dropout=dropout, bias=True) 67 | self.kv_mapper = nn.Sequential(nn.SiLU(), nn.Linear(c_cond, c)) 68 | 69 | def forward(self, x, kv): 70 | kv = self.kv_mapper(kv) 71 | norm_x = self.norm(x) 72 | if self.self_attn: 73 | batch_size, channel, _, _ = x.shape 74 | kv = torch.cat([norm_x.view(batch_size, channel, -1).transpose(1, 2), kv], dim=1) 75 | x = x + self.attention(norm_x, encoder_hidden_states=kv) 76 | return x 77 | -------------------------------------------------------------------------------- /src/diffusers/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snap-research/stable-flow/c99ac0eba297e7884bf2118be79734b63543ee45/src/diffusers/py.typed -------------------------------------------------------------------------------- /src/diffusers/schedulers/README.md: -------------------------------------------------------------------------------- 1 | # Schedulers 2 | 3 | For more information on the schedulers, please refer to the [docs](https://huggingface.co/docs/diffusers/api/schedulers/overview). -------------------------------------------------------------------------------- /src/diffusers/schedulers/deprecated/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from ...utils import ( 4 | DIFFUSERS_SLOW_IMPORT, 5 | OptionalDependencyNotAvailable, 6 | _LazyModule, 7 | get_objects_from_module, 8 | is_torch_available, 9 | is_transformers_available, 10 | ) 11 | 12 | 13 | _dummy_objects = {} 14 | _import_structure = {} 15 | 16 | try: 17 | if not (is_transformers_available() and is_torch_available()): 18 | raise OptionalDependencyNotAvailable() 19 | except OptionalDependencyNotAvailable: 20 | from ...utils import dummy_pt_objects # noqa F403 21 | 22 | _dummy_objects.update(get_objects_from_module(dummy_pt_objects)) 23 | else: 24 | _import_structure["scheduling_karras_ve"] = ["KarrasVeScheduler"] 25 | _import_structure["scheduling_sde_vp"] = ["ScoreSdeVpScheduler"] 26 | 27 | if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: 28 | try: 29 | if not is_torch_available(): 30 | raise OptionalDependencyNotAvailable() 31 | 32 | except OptionalDependencyNotAvailable: 33 | from ...utils.dummy_pt_objects import * # noqa F403 34 | else: 35 | from .scheduling_karras_ve import KarrasVeScheduler 36 | from .scheduling_sde_vp import ScoreSdeVpScheduler 37 | 38 | 39 | else: 40 | import sys 41 | 42 | sys.modules[__name__] = _LazyModule( 43 | __name__, 44 | globals()["__file__"], 45 | _import_structure, 46 | module_spec=__spec__, 47 | ) 48 | 49 | for name, value in _dummy_objects.items(): 50 | setattr(sys.modules[__name__], name, value) 51 | -------------------------------------------------------------------------------- /src/diffusers/utils/accelerate_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Accelerate utilities: Utilities related to accelerate 16 | """ 17 | 18 | from packaging import version 19 | 20 | from .import_utils import is_accelerate_available 21 | 22 | 23 | if is_accelerate_available(): 24 | import accelerate 25 | 26 | 27 | def apply_forward_hook(method): 28 | """ 29 | Decorator that applies a registered CpuOffload hook to an arbitrary function rather than `forward`. This is useful 30 | for cases where a PyTorch module provides functions other than `forward` that should trigger a move to the 31 | appropriate acceleration device. This is the case for `encode` and `decode` in [`AutoencoderKL`]. 32 | 33 | This decorator looks inside the internal `_hf_hook` property to find a registered offload hook. 34 | 35 | :param method: The method to decorate. This method should be a method of a PyTorch module. 36 | """ 37 | if not is_accelerate_available(): 38 | return method 39 | accelerate_version = version.parse(accelerate.__version__).base_version 40 | if version.parse(accelerate_version) < version.parse("0.17.0"): 41 | return method 42 | 43 | def wrapper(self, *args, **kwargs): 44 | if hasattr(self, "_hf_hook") and hasattr(self._hf_hook, "pre_forward"): 45 | self._hf_hook.pre_forward(self) 46 | return method(self, *args, **kwargs) 47 | 48 | return wrapper 49 | -------------------------------------------------------------------------------- /src/diffusers/utils/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The HuggingFace Inc. team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import importlib 15 | import os 16 | 17 | from huggingface_hub.constants import HF_HOME 18 | from packaging import version 19 | 20 | from ..dependency_versions_check import dep_version_check 21 | from .import_utils import ENV_VARS_TRUE_VALUES, is_peft_available, is_transformers_available 22 | 23 | 24 | MIN_PEFT_VERSION = "0.6.0" 25 | MIN_TRANSFORMERS_VERSION = "4.34.0" 26 | _CHECK_PEFT = os.environ.get("_CHECK_PEFT", "1") in ENV_VARS_TRUE_VALUES 27 | 28 | 29 | CONFIG_NAME = "config.json" 30 | WEIGHTS_NAME = "diffusion_pytorch_model.bin" 31 | WEIGHTS_INDEX_NAME = "diffusion_pytorch_model.bin.index.json" 32 | FLAX_WEIGHTS_NAME = "diffusion_flax_model.msgpack" 33 | ONNX_WEIGHTS_NAME = "model.onnx" 34 | SAFETENSORS_WEIGHTS_NAME = "diffusion_pytorch_model.safetensors" 35 | SAFE_WEIGHTS_INDEX_NAME = "diffusion_pytorch_model.safetensors.index.json" 36 | SAFETENSORS_FILE_EXTENSION = "safetensors" 37 | ONNX_EXTERNAL_WEIGHTS_NAME = "weights.pb" 38 | HUGGINGFACE_CO_RESOLVE_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co") 39 | DIFFUSERS_DYNAMIC_MODULE_NAME = "diffusers_modules" 40 | HF_MODULES_CACHE = os.getenv("HF_MODULES_CACHE", os.path.join(HF_HOME, "modules")) 41 | DEPRECATED_REVISION_ARGS = ["fp16", "non-ema"] 42 | 43 | # Below should be `True` if the current version of `peft` and `transformers` are compatible with 44 | # PEFT backend. Will automatically fall back to PEFT backend if the correct versions of the libraries are 45 | # available. 46 | # For PEFT it is has to be greater than or equal to 0.6.0 and for transformers it has to be greater than or equal to 4.34.0. 47 | _required_peft_version = is_peft_available() and version.parse( 48 | version.parse(importlib.metadata.version("peft")).base_version 49 | ) >= version.parse(MIN_PEFT_VERSION) 50 | _required_transformers_version = is_transformers_available() and version.parse( 51 | version.parse(importlib.metadata.version("transformers")).base_version 52 | ) >= version.parse(MIN_TRANSFORMERS_VERSION) 53 | 54 | USE_PEFT_BACKEND = _required_peft_version and _required_transformers_version 55 | 56 | if USE_PEFT_BACKEND and _CHECK_PEFT: 57 | dep_version_check("peft") 58 | -------------------------------------------------------------------------------- /src/diffusers/utils/deprecation_utils.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import warnings 3 | from typing import Any, Dict, Optional, Union 4 | 5 | from packaging import version 6 | 7 | 8 | def deprecate(*args, take_from: Optional[Union[Dict, Any]] = None, standard_warn=True, stacklevel=2): 9 | from .. import __version__ 10 | 11 | deprecated_kwargs = take_from 12 | values = () 13 | if not isinstance(args[0], tuple): 14 | args = (args,) 15 | 16 | for attribute, version_name, message in args: 17 | if version.parse(version.parse(__version__).base_version) >= version.parse(version_name): 18 | raise ValueError( 19 | f"The deprecation tuple {(attribute, version_name, message)} should be removed since diffusers'" 20 | f" version {__version__} is >= {version_name}" 21 | ) 22 | 23 | warning = None 24 | if isinstance(deprecated_kwargs, dict) and attribute in deprecated_kwargs: 25 | values += (deprecated_kwargs.pop(attribute),) 26 | warning = f"The `{attribute}` argument is deprecated and will be removed in version {version_name}." 27 | elif hasattr(deprecated_kwargs, attribute): 28 | values += (getattr(deprecated_kwargs, attribute),) 29 | warning = f"The `{attribute}` attribute is deprecated and will be removed in version {version_name}." 30 | elif deprecated_kwargs is None: 31 | warning = f"`{attribute}` is deprecated and will be removed in version {version_name}." 32 | 33 | if warning is not None: 34 | warning = warning + " " if standard_warn else "" 35 | warnings.warn(warning + message, FutureWarning, stacklevel=stacklevel) 36 | 37 | if isinstance(deprecated_kwargs, dict) and len(deprecated_kwargs) > 0: 38 | call_frame = inspect.getouterframes(inspect.currentframe())[1] 39 | filename = call_frame.filename 40 | line_number = call_frame.lineno 41 | function = call_frame.function 42 | key, value = next(iter(deprecated_kwargs.items())) 43 | raise TypeError(f"{function} in {filename} line {line_number-1} got an unexpected keyword argument `{key}`") 44 | 45 | if len(values) == 0: 46 | return 47 | elif len(values) == 1: 48 | return values[0] 49 | return values 50 | -------------------------------------------------------------------------------- /src/diffusers/utils/doc_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Doc utilities: Utilities related to documentation 16 | """ 17 | 18 | import re 19 | 20 | 21 | def replace_example_docstring(example_docstring): 22 | def docstring_decorator(fn): 23 | func_doc = fn.__doc__ 24 | lines = func_doc.split("\n") 25 | i = 0 26 | while i < len(lines) and re.search(r"^\s*Examples?:\s*$", lines[i]) is None: 27 | i += 1 28 | if i < len(lines): 29 | lines[i] = example_docstring 30 | func_doc = "\n".join(lines) 31 | else: 32 | raise ValueError( 33 | f"The function {fn} should have an empty 'Examples:' in its docstring as placeholder, " 34 | f"current docstring is:\n{func_doc}" 35 | ) 36 | fn.__doc__ = func_doc 37 | return fn 38 | 39 | return docstring_decorator 40 | -------------------------------------------------------------------------------- /src/diffusers/utils/dummy_flax_and_transformers_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class FlaxStableDiffusionControlNetPipeline(metaclass=DummyObject): 6 | _backends = ["flax", "transformers"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["flax", "transformers"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["flax", "transformers"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["flax", "transformers"]) 18 | 19 | 20 | class FlaxStableDiffusionImg2ImgPipeline(metaclass=DummyObject): 21 | _backends = ["flax", "transformers"] 22 | 23 | def __init__(self, *args, **kwargs): 24 | requires_backends(self, ["flax", "transformers"]) 25 | 26 | @classmethod 27 | def from_config(cls, *args, **kwargs): 28 | requires_backends(cls, ["flax", "transformers"]) 29 | 30 | @classmethod 31 | def from_pretrained(cls, *args, **kwargs): 32 | requires_backends(cls, ["flax", "transformers"]) 33 | 34 | 35 | class FlaxStableDiffusionInpaintPipeline(metaclass=DummyObject): 36 | _backends = ["flax", "transformers"] 37 | 38 | def __init__(self, *args, **kwargs): 39 | requires_backends(self, ["flax", "transformers"]) 40 | 41 | @classmethod 42 | def from_config(cls, *args, **kwargs): 43 | requires_backends(cls, ["flax", "transformers"]) 44 | 45 | @classmethod 46 | def from_pretrained(cls, *args, **kwargs): 47 | requires_backends(cls, ["flax", "transformers"]) 48 | 49 | 50 | class FlaxStableDiffusionPipeline(metaclass=DummyObject): 51 | _backends = ["flax", "transformers"] 52 | 53 | def __init__(self, *args, **kwargs): 54 | requires_backends(self, ["flax", "transformers"]) 55 | 56 | @classmethod 57 | def from_config(cls, *args, **kwargs): 58 | requires_backends(cls, ["flax", "transformers"]) 59 | 60 | @classmethod 61 | def from_pretrained(cls, *args, **kwargs): 62 | requires_backends(cls, ["flax", "transformers"]) 63 | 64 | 65 | class FlaxStableDiffusionXLPipeline(metaclass=DummyObject): 66 | _backends = ["flax", "transformers"] 67 | 68 | def __init__(self, *args, **kwargs): 69 | requires_backends(self, ["flax", "transformers"]) 70 | 71 | @classmethod 72 | def from_config(cls, *args, **kwargs): 73 | requires_backends(cls, ["flax", "transformers"]) 74 | 75 | @classmethod 76 | def from_pretrained(cls, *args, **kwargs): 77 | requires_backends(cls, ["flax", "transformers"]) 78 | -------------------------------------------------------------------------------- /src/diffusers/utils/dummy_note_seq_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class MidiProcessor(metaclass=DummyObject): 6 | _backends = ["note_seq"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["note_seq"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["note_seq"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["note_seq"]) 18 | -------------------------------------------------------------------------------- /src/diffusers/utils/dummy_onnx_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class OnnxRuntimeModel(metaclass=DummyObject): 6 | _backends = ["onnx"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["onnx"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["onnx"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["onnx"]) 18 | -------------------------------------------------------------------------------- /src/diffusers/utils/dummy_torch_and_librosa_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class AudioDiffusionPipeline(metaclass=DummyObject): 6 | _backends = ["torch", "librosa"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "librosa"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "librosa"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "librosa"]) 18 | 19 | 20 | class Mel(metaclass=DummyObject): 21 | _backends = ["torch", "librosa"] 22 | 23 | def __init__(self, *args, **kwargs): 24 | requires_backends(self, ["torch", "librosa"]) 25 | 26 | @classmethod 27 | def from_config(cls, *args, **kwargs): 28 | requires_backends(cls, ["torch", "librosa"]) 29 | 30 | @classmethod 31 | def from_pretrained(cls, *args, **kwargs): 32 | requires_backends(cls, ["torch", "librosa"]) 33 | -------------------------------------------------------------------------------- /src/diffusers/utils/dummy_torch_and_scipy_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class LMSDiscreteScheduler(metaclass=DummyObject): 6 | _backends = ["torch", "scipy"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "scipy"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "scipy"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "scipy"]) 18 | -------------------------------------------------------------------------------- /src/diffusers/utils/dummy_torch_and_torchsde_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class CosineDPMSolverMultistepScheduler(metaclass=DummyObject): 6 | _backends = ["torch", "torchsde"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "torchsde"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "torchsde"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "torchsde"]) 18 | 19 | 20 | class DPMSolverSDEScheduler(metaclass=DummyObject): 21 | _backends = ["torch", "torchsde"] 22 | 23 | def __init__(self, *args, **kwargs): 24 | requires_backends(self, ["torch", "torchsde"]) 25 | 26 | @classmethod 27 | def from_config(cls, *args, **kwargs): 28 | requires_backends(cls, ["torch", "torchsde"]) 29 | 30 | @classmethod 31 | def from_pretrained(cls, *args, **kwargs): 32 | requires_backends(cls, ["torch", "torchsde"]) 33 | -------------------------------------------------------------------------------- /src/diffusers/utils/dummy_torch_and_transformers_and_k_diffusion_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class StableDiffusionKDiffusionPipeline(metaclass=DummyObject): 6 | _backends = ["torch", "transformers", "k_diffusion"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "transformers", "k_diffusion"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "transformers", "k_diffusion"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "transformers", "k_diffusion"]) 18 | 19 | 20 | class StableDiffusionXLKDiffusionPipeline(metaclass=DummyObject): 21 | _backends = ["torch", "transformers", "k_diffusion"] 22 | 23 | def __init__(self, *args, **kwargs): 24 | requires_backends(self, ["torch", "transformers", "k_diffusion"]) 25 | 26 | @classmethod 27 | def from_config(cls, *args, **kwargs): 28 | requires_backends(cls, ["torch", "transformers", "k_diffusion"]) 29 | 30 | @classmethod 31 | def from_pretrained(cls, *args, **kwargs): 32 | requires_backends(cls, ["torch", "transformers", "k_diffusion"]) 33 | -------------------------------------------------------------------------------- /src/diffusers/utils/dummy_torch_and_transformers_and_onnx_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class OnnxStableDiffusionImg2ImgPipeline(metaclass=DummyObject): 6 | _backends = ["torch", "transformers", "onnx"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "transformers", "onnx"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "transformers", "onnx"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "transformers", "onnx"]) 18 | 19 | 20 | class OnnxStableDiffusionInpaintPipeline(metaclass=DummyObject): 21 | _backends = ["torch", "transformers", "onnx"] 22 | 23 | def __init__(self, *args, **kwargs): 24 | requires_backends(self, ["torch", "transformers", "onnx"]) 25 | 26 | @classmethod 27 | def from_config(cls, *args, **kwargs): 28 | requires_backends(cls, ["torch", "transformers", "onnx"]) 29 | 30 | @classmethod 31 | def from_pretrained(cls, *args, **kwargs): 32 | requires_backends(cls, ["torch", "transformers", "onnx"]) 33 | 34 | 35 | class OnnxStableDiffusionInpaintPipelineLegacy(metaclass=DummyObject): 36 | _backends = ["torch", "transformers", "onnx"] 37 | 38 | def __init__(self, *args, **kwargs): 39 | requires_backends(self, ["torch", "transformers", "onnx"]) 40 | 41 | @classmethod 42 | def from_config(cls, *args, **kwargs): 43 | requires_backends(cls, ["torch", "transformers", "onnx"]) 44 | 45 | @classmethod 46 | def from_pretrained(cls, *args, **kwargs): 47 | requires_backends(cls, ["torch", "transformers", "onnx"]) 48 | 49 | 50 | class OnnxStableDiffusionPipeline(metaclass=DummyObject): 51 | _backends = ["torch", "transformers", "onnx"] 52 | 53 | def __init__(self, *args, **kwargs): 54 | requires_backends(self, ["torch", "transformers", "onnx"]) 55 | 56 | @classmethod 57 | def from_config(cls, *args, **kwargs): 58 | requires_backends(cls, ["torch", "transformers", "onnx"]) 59 | 60 | @classmethod 61 | def from_pretrained(cls, *args, **kwargs): 62 | requires_backends(cls, ["torch", "transformers", "onnx"]) 63 | 64 | 65 | class OnnxStableDiffusionUpscalePipeline(metaclass=DummyObject): 66 | _backends = ["torch", "transformers", "onnx"] 67 | 68 | def __init__(self, *args, **kwargs): 69 | requires_backends(self, ["torch", "transformers", "onnx"]) 70 | 71 | @classmethod 72 | def from_config(cls, *args, **kwargs): 73 | requires_backends(cls, ["torch", "transformers", "onnx"]) 74 | 75 | @classmethod 76 | def from_pretrained(cls, *args, **kwargs): 77 | requires_backends(cls, ["torch", "transformers", "onnx"]) 78 | 79 | 80 | class StableDiffusionOnnxPipeline(metaclass=DummyObject): 81 | _backends = ["torch", "transformers", "onnx"] 82 | 83 | def __init__(self, *args, **kwargs): 84 | requires_backends(self, ["torch", "transformers", "onnx"]) 85 | 86 | @classmethod 87 | def from_config(cls, *args, **kwargs): 88 | requires_backends(cls, ["torch", "transformers", "onnx"]) 89 | 90 | @classmethod 91 | def from_pretrained(cls, *args, **kwargs): 92 | requires_backends(cls, ["torch", "transformers", "onnx"]) 93 | -------------------------------------------------------------------------------- /src/diffusers/utils/dummy_torch_and_transformers_and_sentencepiece_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class KolorsImg2ImgPipeline(metaclass=DummyObject): 6 | _backends = ["torch", "transformers", "sentencepiece"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "transformers", "sentencepiece"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "transformers", "sentencepiece"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "transformers", "sentencepiece"]) 18 | 19 | 20 | class KolorsPAGPipeline(metaclass=DummyObject): 21 | _backends = ["torch", "transformers", "sentencepiece"] 22 | 23 | def __init__(self, *args, **kwargs): 24 | requires_backends(self, ["torch", "transformers", "sentencepiece"]) 25 | 26 | @classmethod 27 | def from_config(cls, *args, **kwargs): 28 | requires_backends(cls, ["torch", "transformers", "sentencepiece"]) 29 | 30 | @classmethod 31 | def from_pretrained(cls, *args, **kwargs): 32 | requires_backends(cls, ["torch", "transformers", "sentencepiece"]) 33 | 34 | 35 | class KolorsPipeline(metaclass=DummyObject): 36 | _backends = ["torch", "transformers", "sentencepiece"] 37 | 38 | def __init__(self, *args, **kwargs): 39 | requires_backends(self, ["torch", "transformers", "sentencepiece"]) 40 | 41 | @classmethod 42 | def from_config(cls, *args, **kwargs): 43 | requires_backends(cls, ["torch", "transformers", "sentencepiece"]) 44 | 45 | @classmethod 46 | def from_pretrained(cls, *args, **kwargs): 47 | requires_backends(cls, ["torch", "transformers", "sentencepiece"]) 48 | -------------------------------------------------------------------------------- /src/diffusers/utils/dummy_transformers_and_torch_and_note_seq_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class SpectrogramDiffusionPipeline(metaclass=DummyObject): 6 | _backends = ["transformers", "torch", "note_seq"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["transformers", "torch", "note_seq"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["transformers", "torch", "note_seq"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["transformers", "torch", "note_seq"]) 18 | -------------------------------------------------------------------------------- /src/diffusers/utils/loading_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | from typing import Callable, List, Optional, Union 4 | 5 | import PIL.Image 6 | import PIL.ImageOps 7 | import requests 8 | 9 | from .import_utils import BACKENDS_MAPPING, is_opencv_available 10 | 11 | 12 | def load_image( 13 | image: Union[str, PIL.Image.Image], convert_method: Optional[Callable[[PIL.Image.Image], PIL.Image.Image]] = None 14 | ) -> PIL.Image.Image: 15 | """ 16 | Loads `image` to a PIL Image. 17 | 18 | Args: 19 | image (`str` or `PIL.Image.Image`): 20 | The image to convert to the PIL Image format. 21 | convert_method (Callable[[PIL.Image.Image], PIL.Image.Image], *optional*): 22 | A conversion method to apply to the image after loading it. When set to `None` the image will be converted 23 | "RGB". 24 | 25 | Returns: 26 | `PIL.Image.Image`: 27 | A PIL Image. 28 | """ 29 | if isinstance(image, str): 30 | if image.startswith("http://") or image.startswith("https://"): 31 | image = PIL.Image.open(requests.get(image, stream=True).raw) 32 | elif os.path.isfile(image): 33 | image = PIL.Image.open(image) 34 | else: 35 | raise ValueError( 36 | f"Incorrect path or URL. URLs must start with `http://` or `https://`, and {image} is not a valid path." 37 | ) 38 | elif isinstance(image, PIL.Image.Image): 39 | image = image 40 | else: 41 | raise ValueError( 42 | "Incorrect format used for the image. Should be a URL linking to an image, a local path, or a PIL image." 43 | ) 44 | 45 | image = PIL.ImageOps.exif_transpose(image) 46 | 47 | if convert_method is not None: 48 | image = convert_method(image) 49 | else: 50 | image = image.convert("RGB") 51 | 52 | return image 53 | 54 | 55 | def load_video( 56 | video: str, 57 | convert_method: Optional[Callable[[List[PIL.Image.Image]], List[PIL.Image.Image]]] = None, 58 | ) -> List[PIL.Image.Image]: 59 | """ 60 | Loads `video` to a list of PIL Image. 61 | 62 | Args: 63 | video (`str`): 64 | A URL or Path to a video to convert to a list of PIL Image format. 65 | convert_method (Callable[[List[PIL.Image.Image]], List[PIL.Image.Image]], *optional*): 66 | A conversion method to apply to the video after loading it. When set to `None` the images will be converted 67 | to "RGB". 68 | 69 | Returns: 70 | `List[PIL.Image.Image]`: 71 | The video as a list of PIL images. 72 | """ 73 | is_url = video.startswith("http://") or video.startswith("https://") 74 | is_file = os.path.isfile(video) 75 | was_tempfile_created = False 76 | 77 | if not (is_url or is_file): 78 | raise ValueError( 79 | f"Incorrect path or URL. URLs must start with `http://` or `https://`, and {video} is not a valid path." 80 | ) 81 | 82 | if is_url: 83 | video_data = requests.get(video, stream=True).raw 84 | video_path = tempfile.NamedTemporaryFile(suffix=os.path.splitext(video)[1], delete=False).name 85 | was_tempfile_created = True 86 | with open(video_path, "wb") as f: 87 | f.write(video_data.read()) 88 | 89 | video = video_path 90 | 91 | pil_images = [] 92 | if video.endswith(".gif"): 93 | gif = PIL.Image.open(video) 94 | try: 95 | while True: 96 | pil_images.append(gif.copy()) 97 | gif.seek(gif.tell() + 1) 98 | except EOFError: 99 | pass 100 | 101 | else: 102 | if is_opencv_available(): 103 | import cv2 104 | else: 105 | raise ImportError(BACKENDS_MAPPING["opencv"][1].format("load_video")) 106 | 107 | video_capture = cv2.VideoCapture(video) 108 | success, frame = video_capture.read() 109 | while success: 110 | frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) 111 | pil_images.append(PIL.Image.fromarray(frame)) 112 | success, frame = video_capture.read() 113 | 114 | video_capture.release() 115 | 116 | if was_tempfile_created: 117 | os.remove(video_path) 118 | 119 | if convert_method is not None: 120 | pil_images = convert_method(pil_images) 121 | 122 | return pil_images 123 | -------------------------------------------------------------------------------- /src/diffusers/utils/model_card_template.md: -------------------------------------------------------------------------------- 1 | --- 2 | {{ card_data }} 3 | --- 4 | 5 | 7 | 8 | {{ model_description }} 9 | 10 | ## Intended uses & limitations 11 | 12 | #### How to use 13 | 14 | ```python 15 | # TODO: add an example code snippet for running this diffusion pipeline 16 | ``` 17 | 18 | #### Limitations and bias 19 | 20 | [TODO: provide examples of latent issues and potential remediations] 21 | 22 | ## Training details 23 | 24 | [TODO: describe the data used to train the model] 25 | -------------------------------------------------------------------------------- /src/diffusers/utils/pil_utils.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import PIL.Image 4 | import PIL.ImageOps 5 | from packaging import version 6 | from PIL import Image 7 | 8 | 9 | if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"): 10 | PIL_INTERPOLATION = { 11 | "linear": PIL.Image.Resampling.BILINEAR, 12 | "bilinear": PIL.Image.Resampling.BILINEAR, 13 | "bicubic": PIL.Image.Resampling.BICUBIC, 14 | "lanczos": PIL.Image.Resampling.LANCZOS, 15 | "nearest": PIL.Image.Resampling.NEAREST, 16 | } 17 | else: 18 | PIL_INTERPOLATION = { 19 | "linear": PIL.Image.LINEAR, 20 | "bilinear": PIL.Image.BILINEAR, 21 | "bicubic": PIL.Image.BICUBIC, 22 | "lanczos": PIL.Image.LANCZOS, 23 | "nearest": PIL.Image.NEAREST, 24 | } 25 | 26 | 27 | def pt_to_pil(images): 28 | """ 29 | Convert a torch image to a PIL image. 30 | """ 31 | images = (images / 2 + 0.5).clamp(0, 1) 32 | images = images.cpu().permute(0, 2, 3, 1).float().numpy() 33 | images = numpy_to_pil(images) 34 | return images 35 | 36 | 37 | def numpy_to_pil(images): 38 | """ 39 | Convert a numpy image or a batch of images to a PIL image. 40 | """ 41 | if images.ndim == 3: 42 | images = images[None, ...] 43 | images = (images * 255).round().astype("uint8") 44 | if images.shape[-1] == 1: 45 | # special case for grayscale (single channel) images 46 | pil_images = [Image.fromarray(image.squeeze(), mode="L") for image in images] 47 | else: 48 | pil_images = [Image.fromarray(image) for image in images] 49 | 50 | return pil_images 51 | 52 | 53 | def make_image_grid(images: List[PIL.Image.Image], rows: int, cols: int, resize: int = None) -> PIL.Image.Image: 54 | """ 55 | Prepares a single grid of images. Useful for visualization purposes. 56 | """ 57 | assert len(images) == rows * cols 58 | 59 | if resize is not None: 60 | images = [img.resize((resize, resize)) for img in images] 61 | 62 | w, h = images[0].size 63 | grid = Image.new("RGB", size=(cols * w, rows * h)) 64 | 65 | for i, img in enumerate(images): 66 | grid.paste(img, box=(i % cols * w, i // cols * h)) 67 | return grid 68 | --------------------------------------------------------------------------------