├── .gitignore ├── BadDiffusion ├── .gitignore ├── LICENSE ├── README.md ├── anp_config.py ├── anp_defense.py ├── anp_model.py ├── anp_util.py ├── baddiffusion.py ├── dataset.py ├── diffusers │ ├── .github │ │ ├── ISSUE_TEMPLATE │ │ │ ├── bug-report.yml │ │ │ ├── config.yml │ │ │ ├── feature_request.md │ │ │ ├── feedback.md │ │ │ └── new-model-addition.yml │ │ └── workflows │ │ │ ├── build_documentation.yml │ │ │ ├── build_pr_documentation.yml │ │ │ ├── delete_doc_comment.yml │ │ │ ├── pr_quality.yml │ │ │ ├── pr_tests.yml │ │ │ ├── push_tests.yml │ │ │ ├── stale.yml │ │ │ └── typos.yml │ ├── .gitignore │ ├── CODE_OF_CONDUCT.md │ ├── CONTRIBUTING.md │ ├── LICENSE │ ├── MANIFEST.in │ ├── Makefile │ ├── README.md │ ├── _typos.toml │ ├── docs │ │ └── source │ │ │ ├── _toctree.yml │ │ │ ├── api │ │ │ ├── configuration.mdx │ │ │ ├── diffusion_pipeline.mdx │ │ │ ├── logging.mdx │ │ │ ├── models.mdx │ │ │ ├── outputs.mdx │ │ │ ├── pipelines │ │ │ │ ├── ddim.mdx │ │ │ │ ├── ddpm.mdx │ │ │ │ ├── latent_diffusion.mdx │ │ │ │ ├── latent_diffusion_uncond.mdx │ │ │ │ ├── overview.mdx │ │ │ │ ├── pndm.mdx │ │ │ │ ├── score_sde_ve.mdx │ │ │ │ ├── stable_diffusion.mdx │ │ │ │ └── stochastic_karras_ve.mdx │ │ │ └── schedulers.mdx │ │ │ ├── conceptual │ │ │ ├── contribution.mdx │ │ │ ├── philosophy.mdx │ │ │ └── stable_diffusion.mdx │ │ │ ├── index.mdx │ │ │ ├── installation.mdx │ │ │ ├── optimization │ │ │ ├── fp16.mdx │ │ │ ├── mps.mdx │ │ │ ├── onnx.mdx │ │ │ └── open_vino.mdx │ │ │ ├── quicktour.mdx │ │ │ ├── training │ │ │ ├── overview.mdx │ │ │ ├── text2image.mdx │ │ │ ├── text_inversion.mdx │ │ │ └── unconditional_training.mdx │ │ │ └── using-diffusers │ │ │ ├── conditional_image_generation.mdx │ │ │ ├── configuration.mdx │ │ │ ├── custom.mdx │ │ │ ├── custom_pipelines.mdx │ │ │ ├── img2img.mdx │ │ │ ├── inpaint.mdx │ │ │ ├── loading.mdx │ │ │ └── unconditional_image_generation.mdx │ ├── examples │ │ ├── README.md │ │ ├── community │ │ │ ├── README.md │ │ │ └── clip_guided_stable_diffusion.py │ │ ├── conftest.py │ │ ├── dreambooth │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ └── train_dreambooth.py │ │ ├── inference │ │ │ ├── README.md │ │ │ ├── image_to_image.py │ │ │ └── inpainting.py │ │ ├── test_examples.py │ │ ├── text_to_image │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ └── train_text_to_image.py │ │ ├── textual_inversion │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ └── textual_inversion.py │ │ └── unconditional_image_generation │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ └── train_unconditional.py │ ├── pyproject.toml │ ├── scripts │ │ ├── __init__.py │ │ ├── change_naming_configs_and_checkpoints.py │ │ ├── conversion_ldm_uncond.py │ │ ├── convert_ddpm_original_checkpoint_to_diffusers.py │ │ ├── convert_diffusers_to_original_stable_diffusion.py │ │ ├── convert_ldm_original_checkpoint_to_diffusers.py │ │ ├── convert_ncsnpp_original_checkpoint_to_diffusers.py │ │ ├── convert_original_stable_diffusion_to_diffusers.py │ │ ├── convert_stable_diffusion_checkpoint_to_onnx.py │ │ └── generate_logits.py │ ├── setup.cfg │ ├── setup.py │ ├── src │ │ └── diffusers │ │ │ ├── __init__.py │ │ │ ├── commands │ │ │ ├── __init__.py │ │ │ ├── diffusers_cli.py │ │ │ └── env.py │ │ │ ├── configuration_utils.py │ │ │ ├── dependency_versions_check.py │ │ │ ├── dependency_versions_table.py │ │ │ ├── dynamic_modules_utils.py │ │ │ ├── hub_utils.py │ │ │ ├── modeling_flax_pytorch_utils.py │ │ │ ├── modeling_flax_utils.py │ │ │ ├── modeling_utils.py │ │ │ ├── models │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── attention.py │ │ │ ├── attention_flax.py │ │ │ ├── embeddings.py │ │ │ ├── embeddings_flax.py │ │ │ ├── unet_2d.py │ │ │ ├── unet_2d_condition.py │ │ │ ├── unet_2d_condition_flax.py │ │ │ ├── unet_blocks.py │ │ │ ├── unet_blocks_flax.py │ │ │ ├── vae.py │ │ │ └── vae_flax.py │ │ │ ├── onnx_utils.py │ │ │ ├── optimization.py │ │ │ ├── pipeline_flax_utils.py │ │ │ ├── pipeline_utils.py │ │ │ ├── pipelines │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── ddim │ │ │ │ ├── __init__.py │ │ │ │ └── pipeline_ddim.py │ │ │ ├── ddpm │ │ │ │ ├── __init__.py │ │ │ │ └── pipeline_ddpm.py │ │ │ ├── latent_diffusion │ │ │ │ ├── __init__.py │ │ │ │ └── pipeline_latent_diffusion.py │ │ │ ├── latent_diffusion_uncond │ │ │ │ ├── __init__.py │ │ │ │ └── pipeline_latent_diffusion_uncond.py │ │ │ ├── pndm │ │ │ │ ├── __init__.py │ │ │ │ └── pipeline_pndm.py │ │ │ ├── score_sde_ve │ │ │ │ ├── __init__.py │ │ │ │ └── pipeline_score_sde_ve.py │ │ │ ├── stable_diffusion │ │ │ │ ├── README.md │ │ │ │ ├── __init__.py │ │ │ │ ├── pipeline_flax_stable_diffusion.py │ │ │ │ ├── pipeline_stable_diffusion.py │ │ │ │ ├── pipeline_stable_diffusion_img2img.py │ │ │ │ ├── pipeline_stable_diffusion_inpaint.py │ │ │ │ ├── pipeline_stable_diffusion_onnx.py │ │ │ │ ├── safety_checker.py │ │ │ │ └── safety_checker_flax.py │ │ │ └── stochastic_karras_ve │ │ │ │ ├── __init__.py │ │ │ │ └── pipeline_stochastic_karras_ve.py │ │ │ ├── schedulers │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── scheduling_ddim.py │ │ │ ├── scheduling_ddim_flax.py │ │ │ ├── scheduling_ddpm.py │ │ │ ├── scheduling_ddpm_flax.py │ │ │ ├── scheduling_karras_ve.py │ │ │ ├── scheduling_karras_ve_flax.py │ │ │ ├── scheduling_lms_discrete.py │ │ │ ├── scheduling_lms_discrete_flax.py │ │ │ ├── scheduling_pndm.py │ │ │ ├── scheduling_pndm_flax.py │ │ │ ├── scheduling_sde_ve.py │ │ │ ├── scheduling_sde_ve_flax.py │ │ │ ├── scheduling_sde_vp.py │ │ │ ├── scheduling_utils.py │ │ │ └── scheduling_utils_flax.py │ │ │ ├── training_utils.py │ │ │ └── utils │ │ │ ├── __init__.py │ │ │ ├── deprecation_utils.py │ │ │ ├── dummy_flax_and_transformers_objects.py │ │ │ ├── dummy_flax_objects.py │ │ │ ├── dummy_pt_objects.py │ │ │ ├── dummy_torch_and_scipy_objects.py │ │ │ ├── dummy_torch_and_transformers_and_onnx_objects.py │ │ │ ├── dummy_torch_and_transformers_objects.py │ │ │ ├── import_utils.py │ │ │ ├── logging.py │ │ │ ├── model_card_template.md │ │ │ ├── outputs.py │ │ │ └── testing_utils.py │ ├── tests │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── fixtures │ │ │ └── custom_pipeline │ │ │ │ └── pipeline.py │ │ ├── test_config.py │ │ ├── test_layers_utils.py │ │ ├── test_modeling_common.py │ │ ├── test_modeling_common_flax.py │ │ ├── test_models_unet.py │ │ ├── test_models_vae.py │ │ ├── test_models_vae_flax.py │ │ ├── test_models_vq.py │ │ ├── test_outputs.py │ │ ├── test_pipelines.py │ │ ├── test_scheduler.py │ │ ├── test_training.py │ │ └── test_utils.py │ └── utils │ │ ├── check_config_docstrings.py │ │ ├── check_copies.py │ │ ├── check_dummies.py │ │ ├── check_inits.py │ │ ├── check_repo.py │ │ ├── check_table.py │ │ ├── custom_init_isort.py │ │ ├── get_modified_files.py │ │ ├── print_env.py │ │ └── stale.py ├── elijah_helper.py ├── fid_score.py ├── install.sh ├── loss.py ├── model.py ├── requirements.txt ├── run_example.sh ├── static │ ├── cat_wo_bg.png │ ├── fedora-hat.png │ ├── glasses.png │ ├── hat.png │ ├── stop_sign_bg_blk.jpg │ ├── stop_sign_bg_w.jpg │ └── stop_sign_wo_bg.png └── util.py ├── README.md ├── TrojDiff ├── README.md ├── configs │ ├── bedroom.yml │ ├── celeba.yml │ ├── church.yml │ ├── cifar10.yml │ ├── cifar10_100k.yml │ └── cifar10_no_ema.yml ├── datasets │ ├── .DS_Store │ ├── __init__.py │ ├── celeba.py │ ├── ffhq.py │ ├── lsun.py │ ├── utils.py │ └── vision.py ├── elijah_helper.py ├── environment.yml ├── figures │ ├── framework.png │ ├── generative_process.png │ └── numeric_result.png ├── functions │ ├── .DS_Store │ ├── __init__.py │ ├── ckpt_util.py │ ├── denoising.py │ ├── losses.py │ ├── losses_attack.py │ └── losses_attack_d2dout.py ├── images │ ├── .DS_Store │ ├── blue.png │ ├── brown.png │ ├── green.png │ ├── hello_kitty.png │ ├── light_blue.png │ ├── mickey.png │ ├── purple.png │ ├── red.png │ ├── target_A.png │ ├── target_I.png │ ├── white.png │ └── yellow.png ├── main_attack_d2i.py ├── models │ ├── .DS_Store │ ├── diffusion.py │ └── ema.py ├── run_example.sh └── runners │ ├── .DS_Store │ ├── __init__.py │ ├── diffusion.py │ ├── diffusion_attack.py │ ├── diffusion_attack_d2dout.py │ └── diffusion_attack_d2i.py └── VillanDiffusion ├── README.md ├── VillanDiffusion.py ├── VillanDiffusion_backup.py ├── VillanDiffusion_rm.py ├── caption_dataset.py ├── caption_sim.py ├── config.py ├── dataset.py ├── dataset_backup.py ├── dataset_rm.py ├── diffusers ├── .github │ ├── ISSUE_TEMPLATE │ │ ├── bug-report.yml │ │ ├── config.yml │ │ ├── feature_request.md │ │ ├── feedback.md │ │ └── new-model-addition.yml │ ├── actions │ │ └── setup-miniconda │ │ │ └── action.yml │ └── workflows │ │ ├── build_docker_images.yml │ │ ├── build_documentation.yml │ │ ├── build_pr_documentation.yml │ │ ├── delete_doc_comment.yml │ │ ├── nightly_tests.yml │ │ ├── pr_quality.yml │ │ ├── pr_tests.yml │ │ ├── push_tests.yml │ │ ├── push_tests_fast.yml │ │ ├── stale.yml │ │ └── typos.yml ├── .gitignore ├── CITATION.cff ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── PHILOSOPHY.md ├── README.md ├── _typos.toml ├── docker │ ├── diffusers-flax-cpu │ │ └── Dockerfile │ ├── diffusers-flax-tpu │ │ └── Dockerfile │ ├── diffusers-onnxruntime-cpu │ │ └── Dockerfile │ ├── diffusers-onnxruntime-cuda │ │ └── Dockerfile │ ├── diffusers-pytorch-cpu │ │ └── Dockerfile │ └── diffusers-pytorch-cuda │ │ └── Dockerfile ├── docs │ ├── README.md │ ├── TRANSLATING.md │ └── source │ │ ├── _config.py │ │ ├── en │ │ ├── _toctree.yml │ │ ├── api │ │ │ ├── configuration.mdx │ │ │ ├── diffusion_pipeline.mdx │ │ │ ├── experimental │ │ │ │ └── rl.mdx │ │ │ ├── loaders.mdx │ │ │ ├── logging.mdx │ │ │ ├── models.mdx │ │ │ ├── outputs.mdx │ │ │ ├── pipelines │ │ │ │ ├── alt_diffusion.mdx │ │ │ │ ├── audio_diffusion.mdx │ │ │ │ ├── audioldm.mdx │ │ │ │ ├── cycle_diffusion.mdx │ │ │ │ ├── dance_diffusion.mdx │ │ │ │ ├── ddim.mdx │ │ │ │ ├── ddpm.mdx │ │ │ │ ├── dit.mdx │ │ │ │ ├── latent_diffusion.mdx │ │ │ │ ├── latent_diffusion_uncond.mdx │ │ │ │ ├── overview.mdx │ │ │ │ ├── paint_by_example.mdx │ │ │ │ ├── pndm.mdx │ │ │ │ ├── repaint.mdx │ │ │ │ ├── score_sde_ve.mdx │ │ │ │ ├── semantic_stable_diffusion.mdx │ │ │ │ ├── spectrogram_diffusion.mdx │ │ │ │ ├── stable_diffusion │ │ │ │ │ ├── attend_and_excite.mdx │ │ │ │ │ ├── controlnet.mdx │ │ │ │ │ ├── depth2img.mdx │ │ │ │ │ ├── image_variation.mdx │ │ │ │ │ ├── img2img.mdx │ │ │ │ │ ├── inpaint.mdx │ │ │ │ │ ├── latent_upscale.mdx │ │ │ │ │ ├── model_editing.mdx │ │ │ │ │ ├── overview.mdx │ │ │ │ │ ├── panorama.mdx │ │ │ │ │ ├── pix2pix.mdx │ │ │ │ │ ├── pix2pix_zero.mdx │ │ │ │ │ ├── self_attention_guidance.mdx │ │ │ │ │ ├── text2img.mdx │ │ │ │ │ └── upscale.mdx │ │ │ │ ├── stable_diffusion_2.mdx │ │ │ │ ├── stable_diffusion_safe.mdx │ │ │ │ ├── stable_unclip.mdx │ │ │ │ ├── stochastic_karras_ve.mdx │ │ │ │ ├── text_to_video.mdx │ │ │ │ ├── text_to_video_zero.mdx │ │ │ │ ├── unclip.mdx │ │ │ │ ├── versatile_diffusion.mdx │ │ │ │ └── vq_diffusion.mdx │ │ │ └── schedulers │ │ │ │ ├── ddim.mdx │ │ │ │ ├── ddim_inverse.mdx │ │ │ │ ├── ddpm.mdx │ │ │ │ ├── deis.mdx │ │ │ │ ├── dpm_discrete.mdx │ │ │ │ ├── dpm_discrete_ancestral.mdx │ │ │ │ ├── euler.mdx │ │ │ │ ├── euler_ancestral.mdx │ │ │ │ ├── heun.mdx │ │ │ │ ├── ipndm.mdx │ │ │ │ ├── lms_discrete.mdx │ │ │ │ ├── multistep_dpm_solver.mdx │ │ │ │ ├── overview.mdx │ │ │ │ ├── pndm.mdx │ │ │ │ ├── repaint.mdx │ │ │ │ ├── score_sde_ve.mdx │ │ │ │ ├── score_sde_vp.mdx │ │ │ │ ├── singlestep_dpm_solver.mdx │ │ │ │ ├── stochastic_karras_ve.mdx │ │ │ │ ├── unipc.mdx │ │ │ │ └── vq_diffusion.mdx │ │ ├── conceptual │ │ │ ├── contribution.mdx │ │ │ ├── ethical_guidelines.mdx │ │ │ ├── evaluation.mdx │ │ │ └── philosophy.mdx │ │ ├── imgs │ │ │ ├── access_request.png │ │ │ └── diffusers_library.jpg │ │ ├── index.mdx │ │ ├── installation.mdx │ │ ├── optimization │ │ │ ├── coreml.mdx │ │ │ ├── fp16.mdx │ │ │ ├── habana.mdx │ │ │ ├── mps.mdx │ │ │ ├── onnx.mdx │ │ │ ├── open_vino.mdx │ │ │ ├── opt_overview.mdx │ │ │ ├── torch2.0.mdx │ │ │ └── xformers.mdx │ │ ├── quicktour.mdx │ │ ├── stable_diffusion.mdx │ │ ├── training │ │ │ ├── controlnet.mdx │ │ │ ├── dreambooth.mdx │ │ │ ├── instructpix2pix.mdx │ │ │ ├── lora.mdx │ │ │ ├── overview.mdx │ │ │ ├── text2image.mdx │ │ │ ├── text_inversion.mdx │ │ │ └── unconditional_training.mdx │ │ ├── tutorials │ │ │ ├── basic_training.mdx │ │ │ └── tutorial_overview.mdx │ │ └── using-diffusers │ │ │ ├── audio.mdx │ │ │ ├── conditional_image_generation.mdx │ │ │ ├── contribute_pipeline.mdx │ │ │ ├── controlling_generation.mdx │ │ │ ├── custom_pipeline_examples.mdx │ │ │ ├── custom_pipeline_overview.mdx │ │ │ ├── depth2img.mdx │ │ │ ├── img2img.mdx │ │ │ ├── inpaint.mdx │ │ │ ├── kerascv.mdx │ │ │ ├── loading.mdx │ │ │ ├── loading_overview.mdx │ │ │ ├── other-modalities.mdx │ │ │ ├── pipeline_overview.mdx │ │ │ ├── reproducibility.mdx │ │ │ ├── reusing_seeds.mdx │ │ │ ├── rl.mdx │ │ │ ├── schedulers.mdx │ │ │ ├── stable_diffusion_jax_how_to.mdx │ │ │ ├── unconditional_image_generation.mdx │ │ │ ├── using_safetensors │ │ │ ├── using_safetensors.mdx │ │ │ ├── weighted_prompts.mdx │ │ │ └── write_own_pipeline.mdx │ │ ├── ko │ │ ├── _toctree.yml │ │ ├── in_translation.mdx │ │ ├── index.mdx │ │ ├── installation.mdx │ │ └── quicktour.mdx │ │ └── zh │ │ ├── _toctree.yml │ │ ├── index.mdx │ │ ├── installation.mdx │ │ └── quicktour.mdx ├── examples │ ├── README.md │ ├── community │ │ ├── README.md │ │ ├── bit_diffusion.py │ │ ├── checkpoint_merger.py │ │ ├── clip_guided_stable_diffusion.py │ │ ├── clip_guided_stable_diffusion_img2img.py │ │ ├── composable_stable_diffusion.py │ │ ├── ddim_noise_comparative_analysis.py │ │ ├── imagic_stable_diffusion.py │ │ ├── img2img_inpainting.py │ │ ├── interpolate_stable_diffusion.py │ │ ├── lpw_stable_diffusion.py │ │ ├── lpw_stable_diffusion_onnx.py │ │ ├── magic_mix.py │ │ ├── multilingual_stable_diffusion.py │ │ ├── one_step_unet.py │ │ ├── sd_text2img_k_diffusion.py │ │ ├── seed_resize_stable_diffusion.py │ │ ├── speech_to_image_diffusion.py │ │ ├── stable_diffusion_comparison.py │ │ ├── stable_diffusion_controlnet_img2img.py │ │ ├── stable_diffusion_controlnet_inpaint.py │ │ ├── stable_diffusion_controlnet_inpaint_img2img.py │ │ ├── stable_diffusion_mega.py │ │ ├── stable_unclip.py │ │ ├── text_inpainting.py │ │ ├── tiled_upscaling.py │ │ ├── unclip_image_interpolation.py │ │ ├── unclip_text_interpolation.py │ │ └── wildcard_stable_diffusion.py │ ├── conftest.py │ ├── controlnet │ │ ├── README.md │ │ ├── requirements.txt │ │ ├── requirements_flax.txt │ │ ├── train_controlnet.py │ │ └── train_controlnet_flax.py │ ├── dreambooth │ │ ├── README.md │ │ ├── requirements.txt │ │ ├── requirements_flax.txt │ │ ├── train_dreambooth.py │ │ ├── train_dreambooth_flax.py │ │ └── train_dreambooth_lora.py │ ├── inference │ │ ├── README.md │ │ ├── image_to_image.py │ │ └── inpainting.py │ ├── instruct_pix2pix │ │ ├── README.md │ │ ├── requirements.txt │ │ └── train_instruct_pix2pix.py │ ├── research_projects │ │ ├── README.md │ │ ├── colossalai │ │ │ ├── README.md │ │ │ ├── inference.py │ │ │ ├── requirement.txt │ │ │ └── train_dreambooth_colossalai.py │ │ ├── dreambooth_inpaint │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ ├── train_dreambooth_inpaint.py │ │ │ └── train_dreambooth_inpaint_lora.py │ │ ├── intel_opts │ │ │ ├── README.md │ │ │ ├── inference_bf16.py │ │ │ └── textual_inversion │ │ │ │ ├── README.md │ │ │ │ ├── requirements.txt │ │ │ │ └── textual_inversion_bf16.py │ │ ├── lora │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ └── train_text_to_image_lora.py │ │ ├── mulit_token_textual_inversion │ │ │ ├── README.md │ │ │ ├── multi_token_clip.py │ │ │ ├── requirements.txt │ │ │ ├── requirements_flax.txt │ │ │ ├── textual_inversion.py │ │ │ └── textual_inversion_flax.py │ │ ├── multi_subject_dreambooth │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ └── train_multi_subject_dreambooth.py │ │ └── onnxruntime │ │ │ ├── README.md │ │ │ ├── text_to_image │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ └── train_text_to_image.py │ │ │ ├── textual_inversion │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ └── textual_inversion.py │ │ │ └── unconditional_image_generation │ │ │ ├── README.md │ │ │ ├── requirements.txt │ │ │ └── train_unconditional.py │ ├── rl │ │ ├── README.md │ │ └── run_diffuser_locomotion.py │ ├── test_examples.py │ ├── text_to_image │ │ ├── README.md │ │ ├── requirements.txt │ │ ├── requirements_flax.txt │ │ ├── train_text_to_image.py │ │ ├── train_text_to_image_flax.py │ │ └── train_text_to_image_lora.py │ ├── textual_inversion │ │ ├── README.md │ │ ├── requirements.txt │ │ ├── requirements_flax.txt │ │ ├── textual_inversion.py │ │ └── textual_inversion_flax.py │ └── unconditional_image_generation │ │ ├── README.md │ │ ├── requirements.txt │ │ └── train_unconditional.py ├── pyproject.toml ├── scripts │ ├── __init__.py │ ├── change_naming_configs_and_checkpoints.py │ ├── conversion_ldm_uncond.py │ ├── convert_dance_diffusion_to_diffusers.py │ ├── convert_ddpm_original_checkpoint_to_diffusers.py │ ├── convert_diffusers_to_original_stable_diffusion.py │ ├── convert_dit_to_diffusers.py │ ├── convert_k_upscaler_to_diffusers.py │ ├── convert_kakao_brain_unclip_to_diffusers.py │ ├── convert_ldm_original_checkpoint_to_diffusers.py │ ├── convert_lora_safetensor_to_diffusers.py │ ├── convert_models_diffuser_to_diffusers.py │ ├── convert_ms_text_to_video_to_diffusers.py │ ├── convert_music_spectrogram_to_diffusers.py │ ├── convert_ncsnpp_original_checkpoint_to_diffusers.py │ ├── convert_original_audioldm_to_diffusers.py │ ├── convert_original_controlnet_to_diffusers.py │ ├── convert_original_stable_diffusion_to_diffusers.py │ ├── convert_stable_diffusion_checkpoint_to_onnx.py │ ├── convert_unclip_txt2img_to_image_variation.py │ ├── convert_vae_diff_to_onnx.py │ ├── convert_vae_pt_to_diffusers.py │ ├── convert_versatile_diffusion_to_diffusers.py │ ├── convert_vq_diffusion_to_diffusers.py │ └── generate_logits.py ├── setup.cfg ├── setup.py ├── src │ └── diffusers │ │ ├── __init__.py │ │ ├── commands │ │ ├── __init__.py │ │ ├── diffusers_cli.py │ │ └── env.py │ │ ├── configuration_utils.py │ │ ├── dependency_versions_check.py │ │ ├── dependency_versions_table.py │ │ ├── experimental │ │ ├── README.md │ │ ├── __init__.py │ │ └── rl │ │ │ ├── __init__.py │ │ │ └── value_guided_sampling.py │ │ ├── image_processor.py │ │ ├── loaders.py │ │ ├── models │ │ ├── README.md │ │ ├── __init__.py │ │ ├── attention.py │ │ ├── attention_flax.py │ │ ├── attention_processor.py │ │ ├── autoencoder_kl.py │ │ ├── controlnet.py │ │ ├── controlnet_flax.py │ │ ├── cross_attention.py │ │ ├── dual_transformer_2d.py │ │ ├── embeddings.py │ │ ├── embeddings_flax.py │ │ ├── modeling_flax_pytorch_utils.py │ │ ├── modeling_flax_utils.py │ │ ├── modeling_pytorch_flax_utils.py │ │ ├── modeling_utils.py │ │ ├── prior_transformer.py │ │ ├── resnet.py │ │ ├── resnet_flax.py │ │ ├── t5_film_transformer.py │ │ ├── transformer_2d.py │ │ ├── transformer_temporal.py │ │ ├── unet_1d.py │ │ ├── unet_1d_blocks.py │ │ ├── unet_2d.py │ │ ├── unet_2d_blocks.py │ │ ├── unet_2d_blocks_flax.py │ │ ├── unet_2d_condition.py │ │ ├── unet_2d_condition_flax.py │ │ ├── unet_3d_blocks.py │ │ ├── unet_3d_condition.py │ │ ├── vae.py │ │ ├── vae_flax.py │ │ └── vq_model.py │ │ ├── optimization.py │ │ ├── pipeline_utils.py │ │ ├── pipelines │ │ ├── README.md │ │ ├── __init__.py │ │ ├── alt_diffusion │ │ │ ├── __init__.py │ │ │ ├── modeling_roberta_series.py │ │ │ ├── pipeline_alt_diffusion.py │ │ │ └── pipeline_alt_diffusion_img2img.py │ │ ├── audio_diffusion │ │ │ ├── __init__.py │ │ │ ├── mel.py │ │ │ └── pipeline_audio_diffusion.py │ │ ├── audioldm │ │ │ ├── __init__.py │ │ │ └── pipeline_audioldm.py │ │ ├── dance_diffusion │ │ │ ├── __init__.py │ │ │ └── pipeline_dance_diffusion.py │ │ ├── ddim │ │ │ ├── __init__.py │ │ │ └── pipeline_ddim.py │ │ ├── ddpm │ │ │ ├── __init__.py │ │ │ └── pipeline_ddpm.py │ │ ├── dit │ │ │ ├── __init__.py │ │ │ └── pipeline_dit.py │ │ ├── latent_diffusion │ │ │ ├── __init__.py │ │ │ ├── pipeline_latent_diffusion.py │ │ │ └── pipeline_latent_diffusion_superresolution.py │ │ ├── latent_diffusion_uncond │ │ │ ├── __init__.py │ │ │ └── pipeline_latent_diffusion_uncond.py │ │ ├── onnx_utils.py │ │ ├── paint_by_example │ │ │ ├── __init__.py │ │ │ ├── image_encoder.py │ │ │ └── pipeline_paint_by_example.py │ │ ├── pipeline_flax_utils.py │ │ ├── pipeline_utils.py │ │ ├── pndm │ │ │ ├── __init__.py │ │ │ └── pipeline_pndm.py │ │ ├── repaint │ │ │ ├── __init__.py │ │ │ └── pipeline_repaint.py │ │ ├── score_sde_ve │ │ │ ├── __init__.py │ │ │ └── pipeline_score_sde_ve.py │ │ ├── semantic_stable_diffusion │ │ │ ├── __init__.py │ │ │ └── pipeline_semantic_stable_diffusion.py │ │ ├── spectrogram_diffusion │ │ │ ├── __init__.py │ │ │ ├── continous_encoder.py │ │ │ ├── midi_utils.py │ │ │ ├── notes_encoder.py │ │ │ └── pipeline_spectrogram_diffusion.py │ │ ├── stable_diffusion │ │ │ ├── README.md │ │ │ ├── __init__.py │ │ │ ├── convert_from_ckpt.py │ │ │ ├── pipeline_cycle_diffusion.py │ │ │ ├── pipeline_flax_stable_diffusion.py │ │ │ ├── pipeline_flax_stable_diffusion_controlnet.py │ │ │ ├── pipeline_flax_stable_diffusion_img2img.py │ │ │ ├── pipeline_flax_stable_diffusion_inpaint.py │ │ │ ├── pipeline_onnx_stable_diffusion.py │ │ │ ├── pipeline_onnx_stable_diffusion_img2img.py │ │ │ ├── pipeline_onnx_stable_diffusion_inpaint.py │ │ │ ├── pipeline_onnx_stable_diffusion_inpaint_legacy.py │ │ │ ├── pipeline_onnx_stable_diffusion_upscale.py │ │ │ ├── pipeline_stable_diffusion.py │ │ │ ├── pipeline_stable_diffusion_attend_and_excite.py │ │ │ ├── pipeline_stable_diffusion_controlnet.py │ │ │ ├── pipeline_stable_diffusion_depth2img.py │ │ │ ├── pipeline_stable_diffusion_image_variation.py │ │ │ ├── pipeline_stable_diffusion_img2img.py │ │ │ ├── pipeline_stable_diffusion_inpaint.py │ │ │ ├── pipeline_stable_diffusion_inpaint_legacy.py │ │ │ ├── pipeline_stable_diffusion_instruct_pix2pix.py │ │ │ ├── pipeline_stable_diffusion_k_diffusion.py │ │ │ ├── pipeline_stable_diffusion_latent_upscale.py │ │ │ ├── pipeline_stable_diffusion_model_editing.py │ │ │ ├── pipeline_stable_diffusion_panorama.py │ │ │ ├── pipeline_stable_diffusion_pix2pix_zero.py │ │ │ ├── pipeline_stable_diffusion_sag.py │ │ │ ├── pipeline_stable_diffusion_upscale.py │ │ │ ├── pipeline_stable_unclip.py │ │ │ ├── pipeline_stable_unclip_img2img.py │ │ │ ├── safety_checker.py │ │ │ ├── safety_checker_flax.py │ │ │ └── stable_unclip_image_normalizer.py │ │ ├── stable_diffusion_safe │ │ │ ├── __init__.py │ │ │ ├── pipeline_stable_diffusion_safe.py │ │ │ └── safety_checker.py │ │ ├── stochastic_karras_ve │ │ │ ├── __init__.py │ │ │ └── pipeline_stochastic_karras_ve.py │ │ ├── text_to_video_synthesis │ │ │ ├── __init__.py │ │ │ ├── pipeline_text_to_video_synth.py │ │ │ └── pipeline_text_to_video_zero.py │ │ ├── unclip │ │ │ ├── __init__.py │ │ │ ├── pipeline_unclip.py │ │ │ ├── pipeline_unclip_image_variation.py │ │ │ └── text_proj.py │ │ ├── versatile_diffusion │ │ │ ├── __init__.py │ │ │ ├── modeling_text_unet.py │ │ │ ├── pipeline_versatile_diffusion.py │ │ │ ├── pipeline_versatile_diffusion_dual_guided.py │ │ │ ├── pipeline_versatile_diffusion_image_variation.py │ │ │ └── pipeline_versatile_diffusion_text_to_image.py │ │ └── vq_diffusion │ │ │ ├── __init__.py │ │ │ └── pipeline_vq_diffusion.py │ │ ├── schedulers │ │ ├── README.md │ │ ├── __init__.py │ │ ├── scheduling_ddim.py │ │ ├── scheduling_ddim_flax.py │ │ ├── scheduling_ddim_inverse.py │ │ ├── scheduling_ddpm.py │ │ ├── scheduling_ddpm_flax.py │ │ ├── scheduling_deis_multistep.py │ │ ├── scheduling_dpmsolver_multistep.py │ │ ├── scheduling_dpmsolver_multistep_flax.py │ │ ├── scheduling_dpmsolver_singlestep.py │ │ ├── scheduling_euler_ancestral_discrete.py │ │ ├── scheduling_euler_discrete.py │ │ ├── scheduling_heun_discrete.py │ │ ├── scheduling_ipndm.py │ │ ├── scheduling_k_dpm_2_ancestral_discrete.py │ │ ├── scheduling_k_dpm_2_discrete.py │ │ ├── scheduling_karras_ve.py │ │ ├── scheduling_karras_ve_flax.py │ │ ├── scheduling_lms_discrete.py │ │ ├── scheduling_lms_discrete_flax.py │ │ ├── scheduling_pndm.py │ │ ├── scheduling_pndm_flax.py │ │ ├── scheduling_repaint.py │ │ ├── scheduling_sde_ve.py │ │ ├── scheduling_sde_ve_flax.py │ │ ├── scheduling_sde_vp.py │ │ ├── scheduling_unclip.py │ │ ├── scheduling_unipc_multistep.py │ │ ├── scheduling_utils.py │ │ ├── scheduling_utils_flax.py │ │ └── scheduling_vq_diffusion.py │ │ ├── training_utils.py │ │ └── utils │ │ ├── __init__.py │ │ ├── accelerate_utils.py │ │ ├── constants.py │ │ ├── deprecation_utils.py │ │ ├── doc_utils.py │ │ ├── dummy_flax_and_transformers_objects.py │ │ ├── dummy_flax_objects.py │ │ ├── dummy_note_seq_objects.py │ │ ├── dummy_onnx_objects.py │ │ ├── dummy_pt_objects.py │ │ ├── dummy_torch_and_librosa_objects.py │ │ ├── dummy_torch_and_scipy_objects.py │ │ ├── dummy_torch_and_transformers_and_k_diffusion_objects.py │ │ ├── dummy_torch_and_transformers_and_onnx_objects.py │ │ ├── dummy_torch_and_transformers_objects.py │ │ ├── dummy_transformers_and_torch_and_note_seq_objects.py │ │ ├── dynamic_modules_utils.py │ │ ├── hub_utils.py │ │ ├── import_utils.py │ │ ├── logging.py │ │ ├── model_card_template.md │ │ ├── outputs.py │ │ ├── pil_utils.py │ │ ├── testing_utils.py │ │ └── torch_utils.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── fixtures │ │ ├── custom_pipeline │ │ │ ├── pipeline.py │ │ │ └── what_ever.py │ │ └── elise_format0.mid │ ├── models │ │ ├── __init__.py │ │ ├── test_attention_processor.py │ │ ├── test_layers_utils.py │ │ ├── test_lora_layers.py │ │ ├── test_modeling_common.py │ │ ├── test_modeling_common_flax.py │ │ ├── test_models_unet_1d.py │ │ ├── test_models_unet_2d.py │ │ ├── test_models_unet_2d_condition.py │ │ ├── test_models_unet_2d_flax.py │ │ ├── test_models_unet_3d_condition.py │ │ ├── test_models_vae.py │ │ ├── test_models_vae_flax.py │ │ ├── test_models_vq.py │ │ ├── test_unet_2d_blocks.py │ │ └── test_unet_blocks_common.py │ ├── others │ │ ├── test_check_copies.py │ │ ├── test_check_dummies.py │ │ ├── test_config.py │ │ ├── test_ema.py │ │ ├── test_hub_utils.py │ │ ├── test_image_processor.py │ │ ├── test_outputs.py │ │ ├── test_training.py │ │ └── test_utils.py │ ├── pipelines │ │ ├── __init__.py │ │ ├── altdiffusion │ │ │ ├── __init__.py │ │ │ ├── test_alt_diffusion.py │ │ │ └── test_alt_diffusion_img2img.py │ │ ├── audio_diffusion │ │ │ ├── __init__.py │ │ │ └── test_audio_diffusion.py │ │ ├── audioldm │ │ │ ├── __init__.py │ │ │ └── test_audioldm.py │ │ ├── dance_diffusion │ │ │ ├── __init__.py │ │ │ └── test_dance_diffusion.py │ │ ├── ddim │ │ │ ├── __init__.py │ │ │ └── test_ddim.py │ │ ├── ddpm │ │ │ ├── __init__.py │ │ │ └── test_ddpm.py │ │ ├── dit │ │ │ ├── __init__.py │ │ │ └── test_dit.py │ │ ├── karras_ve │ │ │ ├── __init__.py │ │ │ └── test_karras_ve.py │ │ ├── latent_diffusion │ │ │ ├── __init__.py │ │ │ ├── test_latent_diffusion.py │ │ │ ├── test_latent_diffusion_superresolution.py │ │ │ └── test_latent_diffusion_uncond.py │ │ ├── paint_by_example │ │ │ ├── __init__.py │ │ │ └── test_paint_by_example.py │ │ ├── pipeline_params.py │ │ ├── pndm │ │ │ ├── __init__.py │ │ │ └── test_pndm.py │ │ ├── repaint │ │ │ ├── __init__.py │ │ │ └── test_repaint.py │ │ ├── score_sde_ve │ │ │ ├── __init__.py │ │ │ └── test_score_sde_ve.py │ │ ├── semantic_stable_diffusion │ │ │ ├── __init__.py │ │ │ └── test_semantic_diffusion.py │ │ ├── spectrogram_diffusion │ │ │ ├── __init__.py │ │ │ └── test_spectrogram_diffusion.py │ │ ├── stable_diffusion │ │ │ ├── __init__.py │ │ │ ├── test_cycle_diffusion.py │ │ │ ├── test_onnx_stable_diffusion.py │ │ │ ├── test_onnx_stable_diffusion_img2img.py │ │ │ ├── test_onnx_stable_diffusion_inpaint.py │ │ │ ├── test_onnx_stable_diffusion_inpaint_legacy.py │ │ │ ├── test_onnx_stable_diffusion_upscale.py │ │ │ ├── test_stable_diffusion.py │ │ │ ├── test_stable_diffusion_controlnet.py │ │ │ ├── test_stable_diffusion_flax_controlnet.py │ │ │ ├── test_stable_diffusion_image_variation.py │ │ │ ├── test_stable_diffusion_img2img.py │ │ │ ├── test_stable_diffusion_inpaint.py │ │ │ ├── test_stable_diffusion_inpaint_legacy.py │ │ │ ├── test_stable_diffusion_instruction_pix2pix.py │ │ │ ├── test_stable_diffusion_k_diffusion.py │ │ │ ├── test_stable_diffusion_model_editing.py │ │ │ ├── test_stable_diffusion_panorama.py │ │ │ ├── test_stable_diffusion_pix2pix_zero.py │ │ │ └── test_stable_diffusion_sag.py │ │ ├── stable_diffusion_2 │ │ │ ├── __init__.py │ │ │ ├── test_stable_diffusion.py │ │ │ ├── test_stable_diffusion_attend_and_excite.py │ │ │ ├── test_stable_diffusion_depth.py │ │ │ ├── test_stable_diffusion_flax.py │ │ │ ├── test_stable_diffusion_flax_inpaint.py │ │ │ ├── test_stable_diffusion_inpaint.py │ │ │ ├── test_stable_diffusion_latent_upscale.py │ │ │ ├── test_stable_diffusion_upscale.py │ │ │ └── test_stable_diffusion_v_pred.py │ │ ├── stable_diffusion_safe │ │ │ ├── __init__.py │ │ │ └── test_safe_diffusion.py │ │ ├── stable_unclip │ │ │ ├── __init__.py │ │ │ ├── test_stable_unclip.py │ │ │ └── test_stable_unclip_img2img.py │ │ ├── test_pipeline_utils.py │ │ ├── test_pipelines.py │ │ ├── test_pipelines_common.py │ │ ├── test_pipelines_flax.py │ │ ├── test_pipelines_onnx_common.py │ │ ├── text_to_video │ │ │ ├── __init__.py │ │ │ ├── test_text_to_video.py │ │ │ └── test_text_to_video_zero.py │ │ ├── unclip │ │ │ ├── __init__.py │ │ │ ├── test_unclip.py │ │ │ └── test_unclip_image_variation.py │ │ ├── versatile_diffusion │ │ │ ├── __init__.py │ │ │ ├── test_versatile_diffusion_dual_guided.py │ │ │ ├── test_versatile_diffusion_image_variation.py │ │ │ ├── test_versatile_diffusion_mega.py │ │ │ └── test_versatile_diffusion_text_to_image.py │ │ └── vq_diffusion │ │ │ ├── __init__.py │ │ │ └── test_vq_diffusion.py │ └── schedulers │ │ ├── __init__.py │ │ ├── test_scheduler_ddim.py │ │ ├── test_scheduler_ddpm.py │ │ ├── test_scheduler_deis.py │ │ ├── test_scheduler_dpm_multi.py │ │ ├── test_scheduler_dpm_single.py │ │ ├── test_scheduler_euler.py │ │ ├── test_scheduler_euler_ancestral.py │ │ ├── test_scheduler_flax.py │ │ ├── test_scheduler_heun.py │ │ ├── test_scheduler_ipndm.py │ │ ├── test_scheduler_kdpm2_ancestral.py │ │ ├── test_scheduler_kdpm2_discrete.py │ │ ├── test_scheduler_lms.py │ │ ├── test_scheduler_pndm.py │ │ ├── test_scheduler_score_sde_ve.py │ │ ├── test_scheduler_unclip.py │ │ ├── test_scheduler_unipc.py │ │ ├── test_scheduler_vq_diffusion.py │ │ └── test_schedulers.py └── utils │ ├── check_config_docstrings.py │ ├── check_copies.py │ ├── check_doc_toc.py │ ├── check_dummies.py │ ├── check_inits.py │ ├── check_repo.py │ ├── check_table.py │ ├── custom_init_isort.py │ ├── get_modified_files.py │ ├── overwrite_expected_slice.py │ ├── print_env.py │ ├── release.py │ └── stale.py ├── elijah_helper_ddim.py ├── elijah_helper_ldm.py ├── elijah_helper_ncsn.py ├── fid_score.py ├── hg_git_upload.py ├── install.sh ├── loss.py ├── loss_backup.py ├── loss_conditional.py ├── make_latent_dataset.py ├── make_latent_dataset_backup.py ├── measure.py ├── metric.py ├── model.py ├── model_backup.py ├── model_rm.py ├── model_score_based.py ├── my_requirements.txt ├── operate.py ├── rm_backdoor_VillanDiffusion.py ├── rm_run_cifar10_script.py ├── run_celeba_hq_script.py ├── run_cifar10_script.py ├── run_example_ddim.sh ├── run_example_ldm.sh ├── run_example_ncsn.sh ├── run_ldm_celeba_hq_script.py ├── run_measure_inpaint.py ├── run_measure_inpaint2.py ├── run_score-basde_model_script.py ├── sampling.py ├── static ├── cat_wo_bg.png ├── fedora-hat.png ├── glasses.png ├── hat.png ├── stop_sign_bg_blk.jpg ├── stop_sign_bg_w.jpg └── stop_sign_wo_bg.png ├── tools.py ├── util.py ├── util_conditional.py └── viallanDiffusion_conditional.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | .idea/ 3 | *.tar.gz 4 | *.zip 5 | *.pkl 6 | *.py[cod] 7 | 8 | 9 | # ignore all and only keep to python files in the root directory 10 | /* 11 | !.gitignore 12 | !*.py 13 | !*.sh 14 | !README.md 15 | !BadDiffusion 16 | !VillanDiffusion 17 | !TrojDiff 18 | 19 | tmp/ 20 | *.pth 21 | *.pt 22 | 23 | 24 | *.sw[pon] 25 | 26 | *.pkl 27 | *.h5 28 | *.dat 29 | -------------------------------------------------------------------------------- /BadDiffusion/.gitignore: -------------------------------------------------------------------------------- 1 | ~/ 2 | *.gif 3 | *.png 4 | *.jpg 5 | 6 | *.zip 7 | *.tar* 8 | *.pth 9 | *.pkl 10 | 11 | *.json 12 | *.pyc 13 | *.log 14 | 15 | core.* 16 | res* 17 | tmp* 18 | *.out 19 | 20 | datasets/ 21 | data/ 22 | test/ 23 | __pycache__/ 24 | .ipynb_checkpoints/ 25 | .vscode/ 26 | diffusion/ 27 | wandb/ 28 | ANP_backdoor/ 29 | Default/ 30 | 31 | !static/* 32 | NCSNPP_CIFAR10_scratch/ 33 | NCSN_CIFAR10_my/ -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/ISSUE_TEMPLATE/bug-report.yml: -------------------------------------------------------------------------------- 1 | name: "\U0001F41B Bug Report" 2 | description: Report a bug on diffusers 3 | labels: [ "bug" ] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Thanks for taking the time to fill out this bug report! 9 | - type: textarea 10 | id: bug-description 11 | attributes: 12 | label: Describe the bug 13 | description: A clear and concise description of what the bug is. If you intend to submit a pull request for this issue, tell us in the description. Thanks! 14 | placeholder: Bug description 15 | validations: 16 | required: true 17 | - type: textarea 18 | id: reproduction 19 | attributes: 20 | label: Reproduction 21 | description: Please provide a minimal reproducible code which we can copy/paste and reproduce the issue. 22 | placeholder: Reproduction 23 | - type: textarea 24 | id: logs 25 | attributes: 26 | label: Logs 27 | description: "Please include the Python logs if you can." 28 | render: shell 29 | - type: textarea 30 | id: system-info 31 | attributes: 32 | label: System Info 33 | description: Please share your system info with us. You can run the command `diffusers-cli env` and copy-paste its output below. 34 | placeholder: diffusers version, platform, python version, ... 35 | validations: 36 | required: true 37 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | contact_links: 2 | - name: Forum 3 | url: https://discuss.huggingface.co/c/discussion-related-to-httpsgithubcomhuggingfacediffusers/63 4 | about: General usage questions and community discussions 5 | - name: Blank issue 6 | url: https://github.com/huggingface/diffusers/issues/new 7 | about: Please note that the Forum is in most places the right place for discussions 8 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature request" 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/ISSUE_TEMPLATE/feedback.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "💬 Feedback about API Design" 3 | about: Give feedback about the current API design 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What API design would you like to have changed or added to the library? Why?** 11 | 12 | **What use case would this enable or better enable? Can you give us a code example?** 13 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/ISSUE_TEMPLATE/new-model-addition.yml: -------------------------------------------------------------------------------- 1 | name: "\U0001F31F New model/pipeline/scheduler addition" 2 | description: Submit a proposal/request to implement a new diffusion model / pipeline / scheduler 3 | labels: [ "New model/pipeline/scheduler" ] 4 | 5 | body: 6 | - type: textarea 7 | id: description-request 8 | validations: 9 | required: true 10 | attributes: 11 | label: Model/Pipeline/Scheduler description 12 | description: | 13 | Put any and all important information relative to the model/pipeline/scheduler 14 | 15 | - type: checkboxes 16 | id: information-tasks 17 | attributes: 18 | label: Open source status 19 | description: | 20 | Please note that if the model implementation isn't available or if the weights aren't open-source, we are less likely to implement it in `diffusers`. 21 | options: 22 | - label: "The model implementation is available" 23 | - label: "The model weights are available (Only relevant if addition is not a scheduler)." 24 | 25 | - type: textarea 26 | id: additional-info 27 | attributes: 28 | label: Provide useful links for the implementation 29 | description: | 30 | Please provide information regarding the implementation, the weights, and the authors. 31 | Please mention the authors by @gh-username if you're aware of their usernames. 32 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/workflows/build_documentation.yml: -------------------------------------------------------------------------------- 1 | name: Build documentation 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - doc-builder* 8 | - v*-release 9 | 10 | jobs: 11 | build: 12 | uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main 13 | with: 14 | commit_sha: ${{ github.sha }} 15 | package: diffusers 16 | secrets: 17 | token: ${{ secrets.HUGGINGFACE_PUSH }} 18 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/workflows/build_pr_documentation.yml: -------------------------------------------------------------------------------- 1 | name: Build PR Documentation 2 | 3 | on: 4 | pull_request: 5 | 6 | concurrency: 7 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 8 | cancel-in-progress: true 9 | 10 | jobs: 11 | build: 12 | uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main 13 | with: 14 | commit_sha: ${{ github.event.pull_request.head.sha }} 15 | pr_number: ${{ github.event.number }} 16 | package: diffusers 17 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/workflows/delete_doc_comment.yml: -------------------------------------------------------------------------------- 1 | name: Delete dev documentation 2 | 3 | on: 4 | pull_request: 5 | types: [ closed ] 6 | 7 | 8 | jobs: 9 | delete: 10 | uses: huggingface/doc-builder/.github/workflows/delete_doc_comment.yml@main 11 | with: 12 | pr_number: ${{ github.event.number }} 13 | package: diffusers 14 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/workflows/pr_quality.yml: -------------------------------------------------------------------------------- 1 | name: Run code quality checks 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | push: 8 | branches: 9 | - main 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | check_code_quality: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Set up Python 21 | uses: actions/setup-python@v4 22 | with: 23 | python-version: "3.7" 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install .[quality] 28 | - name: Check quality 29 | run: | 30 | black --check --preview examples tests src utils scripts 31 | isort --check-only examples tests src utils scripts 32 | flake8 examples tests src utils scripts 33 | doc-builder style src/diffusers docs/source --max_len 119 --check_only --path_to_docs docs/source 34 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/workflows/pr_tests.yml: -------------------------------------------------------------------------------- 1 | name: Run non-slow tests 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | concurrency: 9 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 10 | cancel-in-progress: true 11 | 12 | env: 13 | HF_HOME: /mnt/cache 14 | OMP_NUM_THREADS: 8 15 | MKL_NUM_THREADS: 8 16 | PYTEST_TIMEOUT: 60 17 | 18 | jobs: 19 | run_tests_cpu: 20 | name: Diffusers tests 21 | runs-on: [ self-hosted, docker-gpu ] 22 | container: 23 | image: python:3.7 24 | options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/ 25 | 26 | steps: 27 | - name: Checkout diffusers 28 | uses: actions/checkout@v3 29 | with: 30 | fetch-depth: 2 31 | 32 | - name: Install dependencies 33 | run: | 34 | python -m pip install --upgrade pip 35 | python -m pip install torch --extra-index-url https://download.pytorch.org/whl/cpu 36 | python -m pip install -e .[quality,test] 37 | 38 | - name: Environment 39 | run: | 40 | python utils/print_env.py 41 | 42 | - name: Run all non-slow selected tests on CPU 43 | run: | 44 | python -m pytest -n 2 --max-worker-restart=0 --dist=loadfile -s -v --make-reports=tests_torch_cpu tests/ 45 | 46 | - name: Failure short reports 47 | if: ${{ failure() }} 48 | run: cat reports/tests_torch_cpu_failures_short.txt 49 | 50 | - name: Test suite reports artifacts 51 | if: ${{ always() }} 52 | uses: actions/upload-artifact@v2 53 | with: 54 | name: pr_torch_test_reports 55 | path: reports 56 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Stale Bot 2 | 3 | on: 4 | schedule: 5 | - cron: "0 15 * * *" 6 | 7 | jobs: 8 | close_stale_issues: 9 | name: Close Stale Issues 10 | if: github.repository == 'huggingface/diffusers' 11 | runs-on: ubuntu-latest 12 | env: 13 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 14 | steps: 15 | - uses: actions/checkout@v2 16 | 17 | - name: Setup Python 18 | uses: actions/setup-python@v1 19 | with: 20 | python-version: 3.7 21 | 22 | - name: Install requirements 23 | run: | 24 | pip install PyGithub 25 | - name: Close stale issues 26 | run: | 27 | python utils/stale.py 28 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/.github/workflows/typos.yml: -------------------------------------------------------------------------------- 1 | name: Check typos 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v3 12 | 13 | - name: typos-action 14 | uses: crate-ci/typos@v1.12.4 15 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include src/diffusers/utils/model_card_template.md 3 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/_typos.toml: -------------------------------------------------------------------------------- 1 | # Files for typos 2 | # Instruction: https://github.com/marketplace/actions/typos-action#getting-started 3 | 4 | [default.extend-identifiers] 5 | 6 | [default.extend-words] 7 | NIN="NIN" # NIN is used in scripts/convert_ncsnpp_original_checkpoint_to_diffusers.py 8 | nd="np" # nd may be np (numpy) 9 | parms="parms" # parms is used in scripts/convert_original_stable_diffusion_to_diffusers.py 10 | 11 | 12 | [files] 13 | extend-exclude = ["_typos.toml"] 14 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/docs/source/api/configuration.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Configuration 14 | 15 | In Diffusers, schedulers of type [`schedulers.scheduling_utils.SchedulerMixin`], and models of type [`ModelMixin`] inherit from [`ConfigMixin`] which conveniently takes care of storing all parameters that are 16 | passed to the respective `__init__` methods in a JSON-configuration file. 17 | 18 | TODO(PVP) - add example and better info here 19 | 20 | ## ConfigMixin 21 | [[autodoc]] ConfigMixin 22 | - from_config 23 | - save_config 24 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/docs/source/api/diffusion_pipeline.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Pipelines 14 | 15 | The [`DiffusionPipeline`] is the easiest way to load any pretrained diffusion pipeline from the [Hub](https://huggingface.co/models?library=diffusers) and to use it in inference. 16 | 17 | 18 | 19 | One should not use the Diffusion Pipeline class for training or fine-tuning a diffusion model. Individual 20 | components of diffusion pipelines are usually trained individually, so we suggest to directly work 21 | with [`UNetModel`] and [`UNetConditionModel`]. 22 | 23 | 24 | 25 | Any diffusion pipeline that is loaded with [`~DiffusionPipeline.from_pretrained`] will automatically 26 | detect the pipeline type, *e.g.* [`StableDiffusionPipeline`] and consequently load each component of the 27 | pipeline and pass them into the `__init__` function of the pipeline, *e.g.* [`~StableDiffusionPipeline.__init__`]. 28 | 29 | Any pipeline object can be saved locally with [`~DiffusionPipeline.save_pretrained`]. 30 | 31 | ## DiffusionPipeline 32 | [[autodoc]] DiffusionPipeline 33 | - from_pretrained 34 | - save_pretrained 35 | 36 | ## ImagePipelineOutput 37 | By default diffusion pipelines return an object of class 38 | 39 | [[autodoc]] pipeline_utils.ImagePipelineOutput 40 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/docs/source/api/pipelines/ddim.mdx: -------------------------------------------------------------------------------- 1 | # DDIM 2 | 3 | ## Overview 4 | 5 | [Denoising Diffusion Implicit Models](https://arxiv.org/abs/2010.02502) (DDIM) by Jiaming Song, Chenlin Meng and Stefano Ermon. 6 | 7 | The abstract of the paper is the following: 8 | 9 | Denoising diffusion probabilistic models (DDPMs) have achieved high quality image generation without adversarial training, yet they require simulating a Markov chain for many steps to produce a sample. To accelerate sampling, we present denoising diffusion implicit models (DDIMs), a more efficient class of iterative implicit probabilistic models with the same training procedure as DDPMs. In DDPMs, the generative process is defined as the reverse of a Markovian diffusion process. We construct a class of non-Markovian diffusion processes that lead to the same training objective, but whose reverse process can be much faster to sample from. We empirically demonstrate that DDIMs can produce high quality samples 10× to 50× faster in terms of wall-clock time compared to DDPMs, allow us to trade off computation for sample quality, and can perform semantically meaningful image interpolation directly in the latent space. 10 | 11 | The original codebase of this paper can be found [here](https://github.com/ermongroup/ddim). 12 | 13 | ## Available Pipelines: 14 | 15 | | Pipeline | Tasks | Colab 16 | |---|---|:---:| 17 | | [pipeline_ddim.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/ddim/pipeline_ddim.py) | *Unconditional Image Generation* | - | 18 | 19 | 20 | ## DDIMPipeline 21 | [[autodoc]] DDIMPipeline 22 | - __call__ 23 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/docs/source/api/pipelines/ddpm.mdx: -------------------------------------------------------------------------------- 1 | # DDPM 2 | 3 | ## Overview 4 | 5 | [Denoising Diffusion Probabilistic Models](https://arxiv.org/abs/2006.11239) 6 | (DDPM) by Jonathan Ho, Ajay Jain and Pieter Abbeel proposes the diffusion based model of the same name, but in the context of the 🤗 Diffusers library, DDPM refers to the discrete denoising scheduler from the paper as well as the pipeline. 7 | 8 | The abstract of the paper is the following: 9 | 10 | We present high quality image synthesis results using diffusion probabilistic models, a class of latent variable models inspired by considerations from nonequilibrium thermodynamics. Our best results are obtained by training on a weighted variational bound designed according to a novel connection between diffusion probabilistic models and denoising score matching with Langevin dynamics, and our models naturally admit a progressive lossy decompression scheme that can be interpreted as a generalization of autoregressive decoding. On the unconditional CIFAR10 dataset, we obtain an Inception score of 9.46 and a state-of-the-art FID score of 3.17. On 256x256 LSUN, we obtain sample quality similar to ProgressiveGAN. 11 | 12 | The original codebase of this paper can be found [here](https://github.com/hojonathanho/diffusion). 13 | 14 | 15 | ## Available Pipelines: 16 | 17 | | Pipeline | Tasks | Colab 18 | |---|---|:---:| 19 | | [pipeline_ddpm.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/ddpm/pipeline_ddpm.py) | *Unconditional Image Generation* | - | 20 | 21 | 22 | # DDPMPipeline 23 | [[autodoc]] DDPMPipeline 24 | - __call__ 25 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/docs/source/api/pipelines/stochastic_karras_ve.mdx: -------------------------------------------------------------------------------- 1 | # Stochastic Karras VE 2 | 3 | ## Overview 4 | 5 | [Elucidating the Design Space of Diffusion-Based Generative Models](https://arxiv.org/abs/2206.00364) by Tero Karras, Miika Aittala, Timo Aila and Samuli Laine. 6 | 7 | The abstract of the paper is the following: 8 | 9 | We argue that the theory and practice of diffusion-based generative models are currently unnecessarily convoluted and seek to remedy the situation by presenting a design space that clearly separates the concrete design choices. This lets us identify several changes to both the sampling and training processes, as well as preconditioning of the score networks. Together, our improvements yield new state-of-the-art FID of 1.79 for CIFAR-10 in a class-conditional setting and 1.97 in an unconditional setting, with much faster sampling (35 network evaluations per image) than prior designs. To further demonstrate their modular nature, we show that our design changes dramatically improve both the efficiency and quality obtainable with pre-trained score networks from previous work, including improving the FID of an existing ImageNet-64 model from 2.07 to near-SOTA 1.55. 10 | 11 | This pipeline implements the Stochastic sampling tailored to the Variance-Expanding (VE) models. 12 | 13 | 14 | ## Available Pipelines: 15 | 16 | | Pipeline | Tasks | Colab 17 | |---|---|:---:| 18 | | [pipeline_stochastic_karras_ve.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stochastic_karras_ve/pipeline_stochastic_karras_ve.py) | *Unconditional Image Generation* | - | 19 | 20 | 21 | ## KarrasVePipeline 22 | [[autodoc]] KarrasVePipeline 23 | - __call__ 24 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/docs/source/conceptual/stable_diffusion.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Stable Diffusion 14 | 15 | Under construction 🚧 16 | 17 | For now please visit this [very in-detail blog post](https://huggingface.co/blog/stable_diffusion) 18 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/docs/source/optimization/open_vino.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # OpenVINO 14 | 15 | Under construction 🚧 16 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/docs/source/training/text2image.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | 14 | # Text-to-Image Training 15 | 16 | Under construction 🚧 17 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/docs/source/using-diffusers/configuration.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | 14 | 15 | # Quicktour 16 | 17 | Start using Diffusers🧨 quickly! 18 | To start, use the [`DiffusionPipeline`] for quick inference and sample generations! 19 | 20 | ``` 21 | pip install diffusers 22 | ``` 23 | 24 | ## Main classes 25 | 26 | ### Models 27 | 28 | ### Schedulers 29 | 30 | ### Pipelines 31 | 32 | 33 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/docs/source/using-diffusers/custom.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Custom Pipeline 14 | 15 | Under construction 🚧 16 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/docs/source/using-diffusers/loading.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Loading 14 | 15 | Under construction 🚧 16 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/examples/community/README.md: -------------------------------------------------------------------------------- 1 | # Community Examples 2 | 3 | **Community** examples consist of both inference and training examples that have been added by the community. 4 | 5 | | Example | Description | Author | Colab | 6 | |:----------|:----------------------|:-----------------|----------:| 7 | | CLIP Guided Stable Diffusion | Doing CLIP guidance for text to image generation with Stable Diffusion| [Suraj Patil](https://github.com/patil-suraj/) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/CLIP_Guided_Stable_diffusion_with_diffusers.ipynb) | 8 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/examples/dreambooth/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate 2 | torchvision 3 | transformers>=4.21.0 4 | ftfy 5 | tensorboard 6 | modelcards -------------------------------------------------------------------------------- /BadDiffusion/diffusers/examples/inference/README.md: -------------------------------------------------------------------------------- 1 | # Inference Examples 2 | 3 | **The inference examples folder is deprecated and will be removed in a future version**. 4 | **Officially supported inference examples can be found in the [Pipelines folder](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines)**. 5 | 6 | - For `Image-to-Image text-guided generation with Stable Diffusion`, please have a look at the official [Pipeline examples](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines#examples) 7 | - For `In-painting using Stable Diffusion`, please have a look at the official [Pipeline examples](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines#examples) 8 | - For `Tweak prompts reusing seeds and latents`, please have a look at the official [Pipeline examples](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines#examples) 9 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/examples/inference/image_to_image.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | from diffusers import StableDiffusionImg2ImgPipeline # noqa F401 4 | 5 | 6 | warnings.warn( 7 | "The `image_to_image.py` script is outdated. Please use directly `from diffusers import" 8 | " StableDiffusionImg2ImgPipeline` instead." 9 | ) 10 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/examples/inference/inpainting.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | from diffusers import StableDiffusionInpaintPipeline as StableDiffusionInpaintPipeline # noqa F401 4 | 5 | 6 | warnings.warn( 7 | "The `inpainting.py` script is outdated. Please use directly `from diffusers import" 8 | " StableDiffusionInpaintPipeline` instead." 9 | ) 10 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/examples/text_to_image/requirements.txt: -------------------------------------------------------------------------------- 1 | diffusers==0.4.1 2 | accelerate 3 | torchvision 4 | transformers>=4.21.0 5 | ftfy 6 | tensorboard 7 | modelcards -------------------------------------------------------------------------------- /BadDiffusion/diffusers/examples/textual_inversion/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate 2 | torchvision 3 | transformers>=4.21.0 4 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/examples/unconditional_image_generation/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate 2 | torchvision 3 | datasets 4 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 119 3 | target-version = ['py36'] 4 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/BadDiffusion/diffusers/scripts/__init__.py -------------------------------------------------------------------------------- /BadDiffusion/diffusers/setup.cfg: -------------------------------------------------------------------------------- 1 | [isort] 2 | default_section = FIRSTPARTY 3 | ensure_newline_before_comments = True 4 | force_grid_wrap = 0 5 | include_trailing_comma = True 6 | known_first_party = accelerate 7 | known_third_party = 8 | numpy 9 | torch 10 | torch_xla 11 | 12 | line_length = 119 13 | lines_after_imports = 2 14 | multi_line_output = 3 15 | use_parentheses = True 16 | 17 | [flake8] 18 | ignore = E203, E722, E501, E741, W503, W605 19 | max-line-length = 119 20 | per-file-ignores = __init__.py:F401 21 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from abc import ABC, abstractmethod 16 | from argparse import ArgumentParser 17 | 18 | 19 | class BaseDiffusersCLICommand(ABC): 20 | @staticmethod 21 | @abstractmethod 22 | def register_subcommand(parser: ArgumentParser): 23 | raise NotImplementedError() 24 | 25 | @abstractmethod 26 | def run(self): 27 | raise NotImplementedError() 28 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/commands/diffusers_cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2022 The HuggingFace Team. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from argparse import ArgumentParser 17 | 18 | from .env import EnvironmentCommand 19 | 20 | 21 | def main(): 22 | parser = ArgumentParser("Diffusers CLI tool", usage="diffusers-cli []") 23 | commands_parser = parser.add_subparsers(help="diffusers-cli command helpers") 24 | 25 | # Register commands 26 | EnvironmentCommand.register_subcommand(commands_parser) 27 | 28 | # Let's go 29 | args = parser.parse_args() 30 | 31 | if not hasattr(args, "func"): 32 | parser.print_help() 33 | exit(1) 34 | 35 | # Run 36 | service = args.func(args) 37 | service.run() 38 | 39 | 40 | if __name__ == "__main__": 41 | main() 42 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/dependency_versions_table.py: -------------------------------------------------------------------------------- 1 | # THIS FILE HAS BEEN AUTOGENERATED. To update: 2 | # 1. modify the `_deps` dict in setup.py 3 | # 2. run `make deps_table_update`` 4 | deps = { 5 | "Pillow": "Pillow<10.0", 6 | "accelerate": "accelerate>=0.11.0", 7 | "black": "black==22.8", 8 | "datasets": "datasets", 9 | "filelock": "filelock", 10 | "flake8": "flake8>=3.8.3", 11 | "flax": "flax>=0.4.1", 12 | "hf-doc-builder": "hf-doc-builder>=0.3.0", 13 | "huggingface-hub": "huggingface-hub>=0.10.0", 14 | "importlib_metadata": "importlib_metadata", 15 | "isort": "isort>=5.5.4", 16 | "jax": "jax>=0.2.8,!=0.3.2,<=0.3.6", 17 | "jaxlib": "jaxlib>=0.1.65,<=0.3.6", 18 | "modelcards": "modelcards>=0.1.4", 19 | "numpy": "numpy", 20 | "onnxruntime": "onnxruntime", 21 | "pytest": "pytest", 22 | "pytest-timeout": "pytest-timeout", 23 | "pytest-xdist": "pytest-xdist", 24 | "scipy": "scipy", 25 | "regex": "regex!=2019.12.17", 26 | "requests": "requests", 27 | "tensorboard": "tensorboard", 28 | "torch": "torch>=1.4", 29 | "torchvision": "torchvision", 30 | "transformers": "transformers>=4.21.0", 31 | } 32 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/models/README.md: -------------------------------------------------------------------------------- 1 | # Models 2 | 3 | - Models: Neural network that models $p_\theta(\mathbf{x}_{t-1}|\mathbf{x}_t)$ (see image below) and is trained end-to-end to denoise a noisy input to an image. Examples: UNet, Conditioned UNet, 3D UNet, Transformer UNet 4 | 5 | ## API 6 | 7 | TODO(Suraj, Patrick) 8 | 9 | ## Examples 10 | 11 | TODO(Suraj, Patrick) 12 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ..utils import is_flax_available, is_torch_available 16 | 17 | 18 | if is_torch_available(): 19 | from .unet_2d import UNet2DModel 20 | from .unet_2d_condition import UNet2DConditionModel 21 | from .vae import AutoencoderKL, VQModel 22 | 23 | if is_flax_available(): 24 | from .unet_2d_condition_flax import FlaxUNet2DConditionModel 25 | from .vae_flax import FlaxAutoencoderKL 26 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/pipelines/__init__.py: -------------------------------------------------------------------------------- 1 | from ..utils import is_flax_available, is_onnx_available, is_torch_available, is_transformers_available 2 | 3 | 4 | if is_torch_available(): 5 | from .ddim import DDIMPipeline 6 | from .ddpm import DDPMPipeline 7 | from .latent_diffusion_uncond import LDMPipeline 8 | from .pndm import PNDMPipeline 9 | from .score_sde_ve import ScoreSdeVePipeline 10 | from .stochastic_karras_ve import KarrasVePipeline 11 | else: 12 | from ..utils.dummy_pt_objects import * # noqa F403 13 | 14 | if is_torch_available() and is_transformers_available(): 15 | from .latent_diffusion import LDMTextToImagePipeline 16 | from .stable_diffusion import ( 17 | StableDiffusionImg2ImgPipeline, 18 | StableDiffusionInpaintPipeline, 19 | StableDiffusionPipeline, 20 | ) 21 | 22 | if is_transformers_available() and is_onnx_available(): 23 | from .stable_diffusion import StableDiffusionOnnxPipeline 24 | 25 | if is_transformers_available() and is_flax_available(): 26 | from .stable_diffusion import FlaxStableDiffusionPipeline 27 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/pipelines/ddim/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .pipeline_ddim import DDIMPipeline 3 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/pipelines/ddpm/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .pipeline_ddpm import DDPMPipeline 3 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/pipelines/latent_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from ...utils import is_transformers_available 3 | 4 | 5 | if is_transformers_available(): 6 | from .pipeline_latent_diffusion import LDMBertModel, LDMTextToImagePipeline 7 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/pipelines/latent_diffusion_uncond/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .pipeline_latent_diffusion_uncond import LDMPipeline 3 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/pipelines/pndm/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .pipeline_pndm import PNDMPipeline 3 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/pipelines/score_sde_ve/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .pipeline_score_sde_ve import ScoreSdeVePipeline 3 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/pipelines/stochastic_karras_ve/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .pipeline_stochastic_karras_ve import KarrasVePipeline 3 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/schedulers/scheduling_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from dataclasses import dataclass 15 | 16 | import torch 17 | 18 | from ..utils import BaseOutput, deprecate 19 | 20 | 21 | SCHEDULER_CONFIG_NAME = "scheduler_config.json" 22 | 23 | 24 | @dataclass 25 | class SchedulerOutput(BaseOutput): 26 | """ 27 | Base class for the scheduler's step function output. 28 | 29 | Args: 30 | prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): 31 | Computed sample (x_{t-1}) of previous timestep. `prev_sample` should be used as next model input in the 32 | denoising loop. 33 | """ 34 | 35 | prev_sample: torch.FloatTensor 36 | 37 | 38 | class SchedulerMixin: 39 | """ 40 | Mixin containing common functions for the schedulers. 41 | """ 42 | 43 | config_name = SCHEDULER_CONFIG_NAME 44 | 45 | def set_format(self, tensor_format="pt"): 46 | deprecate( 47 | "set_format", 48 | "0.6.0", 49 | "If you're running your code in PyTorch, you can safely remove this function as the schedulers are always" 50 | " in Pytorch", 51 | ) 52 | return self 53 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/schedulers/scheduling_utils_flax.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | from dataclasses import dataclass 15 | 16 | import jax.numpy as jnp 17 | 18 | from ..utils import BaseOutput 19 | 20 | 21 | SCHEDULER_CONFIG_NAME = "scheduler_config.json" 22 | 23 | 24 | @dataclass 25 | class FlaxSchedulerOutput(BaseOutput): 26 | """ 27 | Base class for the scheduler's step function output. 28 | 29 | Args: 30 | prev_sample (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)` for images): 31 | Computed sample (x_{t-1}) of previous timestep. `prev_sample` should be used as next model input in the 32 | denoising loop. 33 | """ 34 | 35 | prev_sample: jnp.ndarray 36 | 37 | 38 | class FlaxSchedulerMixin: 39 | """ 40 | Mixin containing common functions for the schedulers. 41 | """ 42 | 43 | config_name = SCHEDULER_CONFIG_NAME 44 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/utils/dummy_flax_and_transformers_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | # flake8: noqa 3 | 4 | from ..utils import DummyObject, requires_backends 5 | 6 | 7 | class FlaxStableDiffusionPipeline(metaclass=DummyObject): 8 | _backends = ["flax", "transformers"] 9 | 10 | def __init__(self, *args, **kwargs): 11 | requires_backends(self, ["flax", "transformers"]) 12 | 13 | @classmethod 14 | def from_config(cls, *args, **kwargs): 15 | requires_backends(cls, ["flax", "transformers"]) 16 | 17 | @classmethod 18 | def from_pretrained(cls, *args, **kwargs): 19 | requires_backends(cls, ["flax", "transformers"]) 20 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/utils/dummy_torch_and_scipy_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | # flake8: noqa 3 | 4 | from ..utils import DummyObject, requires_backends 5 | 6 | 7 | class LMSDiscreteScheduler(metaclass=DummyObject): 8 | _backends = ["torch", "scipy"] 9 | 10 | def __init__(self, *args, **kwargs): 11 | requires_backends(self, ["torch", "scipy"]) 12 | 13 | @classmethod 14 | def from_config(cls, *args, **kwargs): 15 | requires_backends(cls, ["torch", "scipy"]) 16 | 17 | @classmethod 18 | def from_pretrained(cls, *args, **kwargs): 19 | requires_backends(cls, ["torch", "scipy"]) 20 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/utils/dummy_torch_and_transformers_and_onnx_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | # flake8: noqa 3 | 4 | from ..utils import DummyObject, requires_backends 5 | 6 | 7 | class StableDiffusionOnnxPipeline(metaclass=DummyObject): 8 | _backends = ["torch", "transformers", "onnx"] 9 | 10 | def __init__(self, *args, **kwargs): 11 | requires_backends(self, ["torch", "transformers", "onnx"]) 12 | 13 | @classmethod 14 | def from_config(cls, *args, **kwargs): 15 | requires_backends(cls, ["torch", "transformers", "onnx"]) 16 | 17 | @classmethod 18 | def from_pretrained(cls, *args, **kwargs): 19 | requires_backends(cls, ["torch", "transformers", "onnx"]) 20 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/src/diffusers/utils/model_card_template.md: -------------------------------------------------------------------------------- 1 | --- 2 | {{ card_data }} 3 | --- 4 | 5 | 7 | 8 | # {{ model_name | default("Diffusion Model") }} 9 | 10 | ## Model description 11 | 12 | This diffusion model is trained with the [🤗 Diffusers](https://github.com/huggingface/diffusers) library 13 | on the `{{ dataset_name }}` dataset. 14 | 15 | ## Intended uses & limitations 16 | 17 | #### How to use 18 | 19 | ```python 20 | # TODO: add an example code snippet for running this diffusion pipeline 21 | ``` 22 | 23 | #### Limitations and bias 24 | 25 | [TODO: provide examples of latent issues and potential remediations] 26 | 27 | ## Training data 28 | 29 | [TODO: describe the data used to train the model] 30 | 31 | ### Training hyperparameters 32 | 33 | The following hyperparameters were used during training: 34 | - learning_rate: {{ learning_rate }} 35 | - train_batch_size: {{ train_batch_size }} 36 | - eval_batch_size: {{ eval_batch_size }} 37 | - gradient_accumulation_steps: {{ gradient_accumulation_steps }} 38 | - optimizer: AdamW with betas=({{ adam_beta1 }}, {{ adam_beta2 }}), weight_decay={{ adam_weight_decay }} and epsilon={{ adam_epsilon }} 39 | - lr_scheduler: {{ lr_scheduler }} 40 | - lr_warmup_steps: {{ lr_warmup_steps }} 41 | - ema_inv_gamma: {{ ema_inv_gamma }} 42 | - ema_inv_gamma: {{ ema_power }} 43 | - ema_inv_gamma: {{ ema_max_decay }} 44 | - mixed_precision: {{ mixed_precision }} 45 | 46 | ### Training results 47 | 48 | 📈 [TensorBoard logs](https://huggingface.co/{{ repo_name }}/tensorboard?#scalars) 49 | 50 | 51 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/BadDiffusion/diffusers/tests/__init__.py -------------------------------------------------------------------------------- /BadDiffusion/diffusers/tests/test_modeling_common_flax.py: -------------------------------------------------------------------------------- 1 | from diffusers.utils import is_flax_available 2 | from diffusers.utils.testing_utils import require_flax 3 | 4 | 5 | if is_flax_available(): 6 | import jax 7 | 8 | 9 | @require_flax 10 | class FlaxModelTesterMixin: 11 | def test_output(self): 12 | init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() 13 | 14 | model = self.model_class(**init_dict) 15 | variables = model.init(inputs_dict["prng_key"], inputs_dict["sample"]) 16 | jax.lax.stop_gradient(variables) 17 | 18 | output = model.apply(variables, inputs_dict["sample"]) 19 | 20 | if isinstance(output, dict): 21 | output = output.sample 22 | 23 | self.assertIsNotNone(output) 24 | expected_shape = inputs_dict["sample"].shape 25 | self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") 26 | 27 | def test_forward_with_norm_groups(self): 28 | init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() 29 | 30 | init_dict["norm_num_groups"] = 16 31 | init_dict["block_out_channels"] = (16, 32) 32 | 33 | model = self.model_class(**init_dict) 34 | variables = model.init(inputs_dict["prng_key"], inputs_dict["sample"]) 35 | jax.lax.stop_gradient(variables) 36 | 37 | output = model.apply(variables, inputs_dict["sample"]) 38 | 39 | if isinstance(output, dict): 40 | output = output.sample 41 | 42 | self.assertIsNotNone(output) 43 | expected_shape = inputs_dict["sample"].shape 44 | self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") 45 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/tests/test_models_vae_flax.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from diffusers import FlaxAutoencoderKL 4 | from diffusers.utils import is_flax_available 5 | from diffusers.utils.testing_utils import require_flax 6 | 7 | from .test_modeling_common_flax import FlaxModelTesterMixin 8 | 9 | 10 | if is_flax_available(): 11 | import jax 12 | 13 | 14 | @require_flax 15 | class FlaxAutoencoderKLTests(FlaxModelTesterMixin, unittest.TestCase): 16 | model_class = FlaxAutoencoderKL 17 | 18 | @property 19 | def dummy_input(self): 20 | batch_size = 4 21 | num_channels = 3 22 | sizes = (32, 32) 23 | 24 | prng_key = jax.random.PRNGKey(0) 25 | image = jax.random.uniform(prng_key, ((batch_size, num_channels) + sizes)) 26 | 27 | return {"sample": image, "prng_key": prng_key} 28 | 29 | def prepare_init_args_and_inputs_for_common(self): 30 | init_dict = { 31 | "block_out_channels": [32, 64], 32 | "in_channels": 3, 33 | "out_channels": 3, 34 | "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"], 35 | "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D"], 36 | "latent_channels": 4, 37 | } 38 | inputs_dict = self.dummy_input 39 | return init_dict, inputs_dict 40 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/utils/get_modified_files.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2020 The HuggingFace Inc. team. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # this script reports modified .py files under the desired list of top-level sub-dirs passed as a list of arguments, e.g.: 17 | # python ./utils/get_modified_files.py utils src tests examples 18 | # 19 | # it uses git to find the forking point and which files were modified - i.e. files not under git won't be considered 20 | # since the output of this script is fed into Makefile commands it doesn't print a newline after the results 21 | 22 | import re 23 | import subprocess 24 | import sys 25 | 26 | 27 | fork_point_sha = subprocess.check_output("git merge-base main HEAD".split()).decode("utf-8") 28 | modified_files = subprocess.check_output(f"git diff --name-only {fork_point_sha}".split()).decode("utf-8").split() 29 | 30 | joined_dirs = "|".join(sys.argv[1:]) 31 | regex = re.compile(rf"^({joined_dirs}).*?\.py$") 32 | 33 | relevant_modified_files = [x for x in modified_files if regex.match(x)] 34 | print(" ".join(relevant_modified_files), end="") 35 | -------------------------------------------------------------------------------- /BadDiffusion/diffusers/utils/print_env.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # coding=utf-8 4 | # Copyright 2022 The HuggingFace Inc. team. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # this script dumps information about the environment 19 | 20 | import os 21 | import platform 22 | import sys 23 | 24 | 25 | os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" 26 | 27 | print("Python version:", sys.version) 28 | 29 | print("OS platform:", platform.platform()) 30 | print("OS architecture:", platform.machine()) 31 | 32 | try: 33 | import torch 34 | 35 | print("Torch version:", torch.__version__) 36 | print("Cuda available:", torch.cuda.is_available()) 37 | print("Cuda version:", torch.version.cuda) 38 | print("CuDNN version:", torch.backends.cudnn.version()) 39 | print("Number of GPUs available:", torch.cuda.device_count()) 40 | except ImportError: 41 | print("Torch version:", None) 42 | 43 | try: 44 | import transformers 45 | 46 | print("transformers version:", transformers.__version__) 47 | except ImportError: 48 | print("transformers version:", None) 49 | -------------------------------------------------------------------------------- /BadDiffusion/install.sh: -------------------------------------------------------------------------------- 1 | pip install pyarrow 2 | # pip install accelerate comet-ml matplotlib datasets tqdm tensorboard tensorboardX torchvision tensorflow-datasets einops pytorch-fid joblib PyYAML kaggle wandb torchsummary torchinfo 3 | pip install -r requirements.txt 4 | 5 | cd diffusers 6 | pip install -e . 7 | cd .. 8 | 9 | mkdir measure 10 | mkdir datasets 11 | mkdir measure 12 | -------------------------------------------------------------------------------- /BadDiffusion/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate==0.19.0 2 | comet-ml 3 | matplotlib 4 | datasets 5 | tqdm 6 | tensorboard 7 | tensorboardX 8 | tensorflow-datasets 9 | einops 10 | pytorch-fid 11 | joblib 12 | PyYAML 13 | kaggle 14 | wandb 15 | torchsummary 16 | torchinfo 17 | torchmetrics 18 | piq 19 | -------------------------------------------------------------------------------- /BadDiffusion/run_example.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Train a backdoored model. By default, it's saved as res_DDPM-CIFAR10-32_CIFAR10_ep50_c1.0_p0.1_STOP_SIGN_14-HAT 4 | python baddiffusion.py --project default --mode train+measure --dataset CIFAR10 --batch 128 --epoch 50 --poison_rate 0.1 --trigger STOP_SIGN_14 --target HAT --ckpt DDPM-CIFAR10-32 --fclip o -o --gpu 0 5 | 6 | # invert the trigger and compute the uniformity score and tv loss 7 | python elijah_helper.py res_DDPM-CIFAR10-32_CIFAR10_ep50_c1.0_p0.1_STOP_SIGN_14-HAT --compute_tvloss 8 | python elijah_helper.py res_DDPM-CIFAR10-32_CIFAR10_ep50_c1.0_p0.1_STOP_SIGN_14-HAT 9 | 10 | # Use the inverted trigger to remove the injected backdoor. By default, it's aved as res_res_DDPM-CIFAR10-32_CIFAR10_ep50_c1.0_p0.1_STOP_SIGN_14-HAT_CIFAR10_ep11_c0.1_p0.0_STOP_SIGN_14-HAT 11 | python baddiffusion.py --project default --mode train+measure --dataset CIFAR10 --batch 128 --epoch 11 --poison_rate 0 --clean_rate 0.1 --trigger STOP_SIGN_14 --target HAT --ckpt res_DDPM-CIFAR10-32_CIFAR10_ep50_c1.0_p0.1_STOP_SIGN_14-HAT --fclip o -o --gpu 0 --save_image_epochs 1 --save_model_epochs 1 --is_save_all_model_epochs --dataset_load_mode FLEX --remove_backdoor 12 | -------------------------------------------------------------------------------- /BadDiffusion/static/cat_wo_bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/BadDiffusion/static/cat_wo_bg.png -------------------------------------------------------------------------------- /BadDiffusion/static/fedora-hat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/BadDiffusion/static/fedora-hat.png -------------------------------------------------------------------------------- /BadDiffusion/static/glasses.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/BadDiffusion/static/glasses.png -------------------------------------------------------------------------------- /BadDiffusion/static/hat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/BadDiffusion/static/hat.png -------------------------------------------------------------------------------- /BadDiffusion/static/stop_sign_bg_blk.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/BadDiffusion/static/stop_sign_bg_blk.jpg -------------------------------------------------------------------------------- /BadDiffusion/static/stop_sign_bg_w.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/BadDiffusion/static/stop_sign_bg_w.jpg -------------------------------------------------------------------------------- /BadDiffusion/static/stop_sign_wo_bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/BadDiffusion/static/stop_sign_wo_bg.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Official Code for Elijah (AAAI 2024 and NeurIPS 2023 Workshop BUGS) 2 | 3 | This is the PyTorch implementation for the AAAI 2024 paper "Elijah: Eliminating Backdoors Injected in Diffusion Models via Distribution Shift". This work also appears in the NeurIPS 2023 Workshop BUGS. 4 | 5 | Paper link: https://arxiv.org/abs/2312.00050 6 | 7 | ## Environments and Running examples 8 | 9 | Please check each attack's directory for more information. 10 | -------------------------------------------------------------------------------- /TrojDiff/configs/bedroom.yml: -------------------------------------------------------------------------------- 1 | data: 2 | dataset: "LSUN" 3 | category: "bedroom" 4 | image_size: 256 5 | channels: 3 6 | logit_transform: false 7 | uniform_dequantization: false 8 | gaussian_dequantization: false 9 | random_flip: true 10 | rescaled: true 11 | num_workers: 32 12 | 13 | model: 14 | type: "simple" 15 | in_channels: 3 16 | out_ch: 3 17 | ch: 128 18 | ch_mult: [1, 1, 2, 2, 4, 4] 19 | num_res_blocks: 2 20 | attn_resolutions: [16, ] 21 | dropout: 0.0 22 | var_type: fixedsmall 23 | ema_rate: 0.999 24 | ema: True 25 | resamp_with_conv: True 26 | 27 | diffusion: 28 | beta_schedule: linear 29 | beta_start: 0.0001 30 | beta_end: 0.02 31 | num_diffusion_timesteps: 1000 32 | 33 | training: 34 | batch_size: 64 35 | n_epochs: 10000 36 | n_iters: 5000000 37 | snapshot_freq: 5000 38 | validation_freq: 2000 39 | 40 | sampling: 41 | batch_size: 32 42 | last_only: True 43 | 44 | optim: 45 | weight_decay: 0.000 46 | optimizer: "Adam" 47 | lr: 0.00002 48 | beta1: 0.9 49 | amsgrad: false 50 | eps: 0.00000001 51 | -------------------------------------------------------------------------------- /TrojDiff/configs/celeba.yml: -------------------------------------------------------------------------------- 1 | data: 2 | dataset: "CELEBA" 3 | image_size: 64 4 | channels: 3 5 | logit_transform: false 6 | uniform_dequantization: false 7 | gaussian_dequantization: false 8 | random_flip: true 9 | rescaled: true 10 | num_workers: 4 11 | 12 | model: 13 | type: "simple" 14 | in_channels: 3 15 | out_ch: 3 16 | ch: 128 17 | ch_mult: [1, 2, 2, 2, 4] 18 | num_res_blocks: 2 19 | attn_resolutions: [16, ] 20 | dropout: 0.1 21 | var_type: fixedlarge 22 | ema_rate: 0.9999 23 | ema: True 24 | resamp_with_conv: True 25 | 26 | diffusion: 27 | beta_schedule: linear 28 | beta_start: 0.0001 29 | beta_end: 0.02 30 | num_diffusion_timesteps: 1000 31 | 32 | training: 33 | batch_size: 128 34 | n_epochs: 10000 35 | n_iters: 5000000 36 | snapshot_freq: 5000 37 | validation_freq: 20000 38 | 39 | sampling: 40 | batch_size: 50 #32 41 | last_only: True 42 | ckpt_id: 100000 #50000 # None 43 | 44 | optim: 45 | weight_decay: 0.000 46 | optimizer: "Adam" 47 | lr: 0.0002 48 | beta1: 0.9 49 | amsgrad: false 50 | eps: 0.00000001 51 | grad_clip: 1.0 52 | -------------------------------------------------------------------------------- /TrojDiff/configs/church.yml: -------------------------------------------------------------------------------- 1 | data: 2 | dataset: "LSUN" 3 | category: "church_outdoor" 4 | image_size: 256 5 | channels: 3 6 | logit_transform: false 7 | uniform_dequantization: false 8 | gaussian_dequantization: false 9 | random_flip: true 10 | rescaled: true 11 | num_workers: 32 12 | 13 | model: 14 | type: "simple" 15 | in_channels: 3 16 | out_ch: 3 17 | ch: 128 18 | ch_mult: [1, 1, 2, 2, 4, 4] 19 | num_res_blocks: 2 20 | attn_resolutions: [16, ] 21 | dropout: 0.0 22 | var_type: fixedsmall 23 | ema_rate: 0.999 24 | ema: True 25 | resamp_with_conv: True 26 | 27 | diffusion: 28 | beta_schedule: linear 29 | beta_start: 0.0001 30 | beta_end: 0.02 31 | num_diffusion_timesteps: 1000 32 | 33 | training: 34 | batch_size: 64 35 | n_epochs: 10000 36 | n_iters: 5000000 37 | snapshot_freq: 5000 38 | validation_freq: 2000 39 | 40 | sampling: 41 | batch_size: 32 42 | last_only: True 43 | 44 | optim: 45 | weight_decay: 0.000 46 | optimizer: "Adam" 47 | lr: 0.00002 48 | beta1: 0.9 49 | amsgrad: false 50 | eps: 0.00000001 51 | -------------------------------------------------------------------------------- /TrojDiff/configs/cifar10.yml: -------------------------------------------------------------------------------- 1 | data: 2 | dataset: "CIFAR10" 3 | image_size: 32 4 | channels: 3 5 | logit_transform: false 6 | uniform_dequantization: false 7 | gaussian_dequantization: false 8 | random_flip: true 9 | rescaled: true 10 | num_workers: 4 11 | 12 | model: 13 | type: "simple" 14 | in_channels: 3 15 | out_ch: 3 16 | ch: 128 17 | ch_mult: [1, 2, 2, 2] 18 | num_res_blocks: 2 19 | attn_resolutions: [16, ] 20 | dropout: 0.1 21 | var_type: fixedlarge 22 | ema_rate: 0.9999 23 | ema: True 24 | resamp_with_conv: True 25 | 26 | diffusion: 27 | beta_schedule: linear 28 | beta_start: 0.0001 29 | beta_end: 0.02 30 | num_diffusion_timesteps: 1000 31 | 32 | training: 33 | batch_size: 128 34 | n_epochs: 10000 35 | n_iters: 5000000 36 | snapshot_freq: 1000 37 | validation_freq: 2000 38 | 39 | sampling: 40 | batch_size: 2048 #64 41 | last_only: True 42 | ckpt_id: 100000 # None 43 | 44 | optim: 45 | weight_decay: 0.000 46 | optimizer: "Adam" 47 | lr: 0.0002 48 | beta1: 0.9 49 | amsgrad: false 50 | eps: 0.00000001 51 | grad_clip: 1.0 52 | -------------------------------------------------------------------------------- /TrojDiff/configs/cifar10_100k.yml: -------------------------------------------------------------------------------- 1 | data: 2 | dataset: "CIFAR10" 3 | image_size: 32 4 | channels: 3 5 | logit_transform: false 6 | uniform_dequantization: false 7 | gaussian_dequantization: false 8 | random_flip: true 9 | rescaled: true 10 | num_workers: 4 11 | 12 | model: 13 | type: "simple" 14 | in_channels: 3 15 | out_ch: 3 16 | ch: 128 17 | ch_mult: [1, 2, 2, 2] 18 | num_res_blocks: 2 19 | attn_resolutions: [16, ] 20 | dropout: 0.1 21 | var_type: fixedlarge 22 | ema_rate: 0.9999 23 | ema: True 24 | resamp_with_conv: True 25 | 26 | diffusion: 27 | beta_schedule: linear 28 | beta_start: 0.0001 29 | beta_end: 0.02 30 | num_diffusion_timesteps: 1000 31 | 32 | training: 33 | batch_size: 128 34 | n_epochs: 10000 35 | n_iters: 5000000 36 | snapshot_freq: 100000 37 | validation_freq: 2000 38 | 39 | sampling: 40 | batch_size: 2048 #64 41 | last_only: True 42 | ckpt_id: 100000 # None 43 | 44 | optim: 45 | weight_decay: 0.000 46 | optimizer: "Adam" 47 | lr: 0.0002 48 | beta1: 0.9 49 | amsgrad: false 50 | eps: 0.00000001 51 | grad_clip: 1.0 52 | -------------------------------------------------------------------------------- /TrojDiff/configs/cifar10_no_ema.yml: -------------------------------------------------------------------------------- 1 | data: 2 | dataset: "CIFAR10" 3 | image_size: 32 4 | channels: 3 5 | logit_transform: false 6 | uniform_dequantization: false 7 | gaussian_dequantization: false 8 | random_flip: true 9 | rescaled: true 10 | num_workers: 4 11 | 12 | model: 13 | type: "simple" 14 | in_channels: 3 15 | out_ch: 3 16 | ch: 128 17 | ch_mult: [1, 2, 2, 2] 18 | num_res_blocks: 2 19 | attn_resolutions: [16, ] 20 | dropout: 0.1 21 | var_type: fixedlarge 22 | ema_rate: 0.9999 23 | ema: False 24 | resamp_with_conv: True 25 | 26 | diffusion: 27 | beta_schedule: linear 28 | beta_start: 0.0001 29 | beta_end: 0.02 30 | num_diffusion_timesteps: 1000 31 | 32 | training: 33 | batch_size: 128 34 | n_epochs: 10000 35 | n_iters: 5000000 36 | snapshot_freq: 1000 37 | validation_freq: 2000 38 | 39 | sampling: 40 | batch_size: 2048 #64 41 | last_only: True 42 | ckpt_id: 100000 # None 43 | 44 | optim: 45 | weight_decay: 0.000 46 | optimizer: "Adam" 47 | lr: 0.0002 48 | beta1: 0.9 49 | amsgrad: false 50 | eps: 0.00000001 51 | grad_clip: 1.0 52 | -------------------------------------------------------------------------------- /TrojDiff/datasets/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/datasets/.DS_Store -------------------------------------------------------------------------------- /TrojDiff/datasets/ffhq.py: -------------------------------------------------------------------------------- 1 | from io import BytesIO 2 | 3 | import lmdb 4 | from PIL import Image 5 | from torch.utils.data import Dataset 6 | 7 | 8 | class FFHQ(Dataset): 9 | def __init__(self, path, transform, resolution=8): 10 | self.env = lmdb.open( 11 | path, 12 | max_readers=32, 13 | readonly=True, 14 | lock=False, 15 | readahead=False, 16 | meminit=False, 17 | ) 18 | 19 | if not self.env: 20 | raise IOError('Cannot open lmdb dataset', path) 21 | 22 | with self.env.begin(write=False) as txn: 23 | self.length = int(txn.get('length'.encode('utf-8')).decode('utf-8')) 24 | 25 | self.resolution = resolution 26 | self.transform = transform 27 | 28 | def __len__(self): 29 | return self.length 30 | 31 | def __getitem__(self, index): 32 | with self.env.begin(write=False) as txn: 33 | key = f'{self.resolution}-{str(index).zfill(5)}'.encode('utf-8') 34 | img_bytes = txn.get(key) 35 | 36 | buffer = BytesIO(img_bytes) 37 | img = Image.open(buffer) 38 | img = self.transform(img) 39 | target = 0 40 | 41 | return img, target -------------------------------------------------------------------------------- /TrojDiff/figures/framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/figures/framework.png -------------------------------------------------------------------------------- /TrojDiff/figures/generative_process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/figures/generative_process.png -------------------------------------------------------------------------------- /TrojDiff/figures/numeric_result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/figures/numeric_result.png -------------------------------------------------------------------------------- /TrojDiff/functions/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/functions/.DS_Store -------------------------------------------------------------------------------- /TrojDiff/functions/__init__.py: -------------------------------------------------------------------------------- 1 | import torch.optim as optim 2 | 3 | 4 | def get_optimizer(config, parameters): 5 | if config.optim.optimizer == 'Adam': 6 | return optim.Adam(parameters, lr=config.optim.lr, weight_decay=config.optim.weight_decay, 7 | betas=(config.optim.beta1, 0.999), amsgrad=config.optim.amsgrad, 8 | eps=config.optim.eps) 9 | elif config.optim.optimizer == 'RMSProp': 10 | return optim.RMSprop(parameters, lr=config.optim.lr, weight_decay=config.optim.weight_decay) 11 | elif config.optim.optimizer == 'SGD': 12 | return optim.SGD(parameters, lr=config.optim.lr, momentum=0.9) 13 | else: 14 | raise NotImplementedError( 15 | 'Optimizer {} not understood.'.format(config.optim.optimizer)) 16 | -------------------------------------------------------------------------------- /TrojDiff/functions/losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def noise_estimation_loss(model, 5 | x0: torch.Tensor, 6 | t: torch.LongTensor, 7 | e: torch.Tensor, 8 | b: torch.Tensor, keepdim=False): 9 | a = (1-b).cumprod(dim=0).index_select(0, t).view(-1, 1, 1, 1) 10 | x = x0 * a.sqrt() + e * (1.0 - a).sqrt() 11 | output = model(x, t.float()) 12 | if keepdim: 13 | return (e - output).square().sum(dim=(1, 2, 3)) 14 | else: 15 | return (e - output).square().sum(dim=(1, 2, 3)).mean(dim=0) 16 | 17 | 18 | loss_registry = { 19 | 'simple': noise_estimation_loss, 20 | } 21 | -------------------------------------------------------------------------------- /TrojDiff/functions/losses_attack.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.utils as tvu 3 | import pdb 4 | import os 5 | 6 | 7 | def noise_estimation_loss(model, 8 | x0: torch.Tensor, 9 | y: torch.Tensor, 10 | t: torch.LongTensor, 11 | e: torch.Tensor, 12 | b: torch.Tensor, 13 | miu: torch.Tensor, 14 | args=None, 15 | keepdim=False): 16 | target_idx = torch.where(y == args.target_label)[0] 17 | chosen_mask = torch.bernoulli(torch.zeros_like(target_idx) + args.cond_prob) 18 | chosen_target_idx = target_idx[torch.where(chosen_mask == 1)[0]] 19 | 20 | batch, device = x0.shape[0], x0.device 21 | miu_ = torch.stack([miu.to(device)] * batch) # (batch,3,32,32) 22 | 23 | a = (1-b).cumprod(dim=0).index_select(0, t).view(-1, 1, 1, 1) 24 | x = x0 * a.sqrt() + e * (1.0 - a).sqrt() 25 | x_ = x0 * a.sqrt() + e * (1.0 - a).sqrt() * args.gamma + miu_ * (1.0 - a).sqrt() 26 | if args.trigger_type == 'patch': 27 | tmp_x = x.clone() 28 | tmp_x[:, :, -args.patch_size:, -args.patch_size:] = x_[:, :, -args.patch_size:, -args.patch_size:] 29 | x_ = tmp_x 30 | 31 | x_add = x_[chosen_target_idx] 32 | t_add = t[chosen_target_idx] 33 | e_add = e[chosen_target_idx] 34 | x = torch.cat([x, x_add], dim=0) 35 | t = torch.cat([t, t_add], dim=0) 36 | e = torch.cat([e, e_add], dim=0) 37 | 38 | output = model(x, t.float()) 39 | if keepdim: 40 | return (e - output).square().sum(dim=(1, 2, 3)) 41 | else: 42 | return (e - output).square().sum(dim=(1, 2, 3)).mean(dim=0) 43 | 44 | 45 | loss_registry = { 46 | 'simple': noise_estimation_loss, 47 | } 48 | -------------------------------------------------------------------------------- /TrojDiff/functions/losses_attack_d2dout.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision.utils as tvu 3 | import pdb 4 | import os 5 | 6 | 7 | def noise_estimation_loss(model, 8 | x0: torch.Tensor, 9 | y: torch.Tensor, 10 | t: torch.LongTensor, 11 | e: torch.Tensor, 12 | b: torch.Tensor, 13 | miu: torch.Tensor, 14 | args=None, 15 | keepdim=False): 16 | target_idx = torch.where(y == 1000)[0] 17 | chosen_mask = torch.bernoulli(torch.zeros_like(target_idx) + args.cond_prob) 18 | chosen_target_idx = target_idx[torch.where(chosen_mask == 1)[0]] 19 | 20 | batch, device = x0.shape[0], x0.device 21 | miu_ = torch.stack([miu.to(device)] * batch) # (batch,3,32,32) 22 | 23 | a = (1-b).cumprod(dim=0).index_select(0, t).view(-1, 1, 1, 1) 24 | x = x0 * a.sqrt() + e * (1.0 - a).sqrt() 25 | x_ = x0 * a.sqrt() + e * (1.0 - a).sqrt() * args.gamma + miu_ * (1.0 - a).sqrt() 26 | if args.trigger_type == 'patch': 27 | tmp_x = x.clone() 28 | tmp_x[:, :, -args.patch_size:, -args.patch_size:] = x_[:, :, -args.patch_size:, -args.patch_size:] 29 | x_ = tmp_x 30 | 31 | x_add = x_[chosen_target_idx] 32 | x[chosen_target_idx] = x_add 33 | 34 | output = model(x, t.float()) 35 | if keepdim: 36 | return (e - output).square().sum(dim=(1, 2, 3)) 37 | else: 38 | return (e - output).square().sum(dim=(1, 2, 3)).mean(dim=0) 39 | 40 | 41 | loss_registry = { 42 | 'simple': noise_estimation_loss, 43 | } 44 | -------------------------------------------------------------------------------- /TrojDiff/images/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/.DS_Store -------------------------------------------------------------------------------- /TrojDiff/images/blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/blue.png -------------------------------------------------------------------------------- /TrojDiff/images/brown.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/brown.png -------------------------------------------------------------------------------- /TrojDiff/images/green.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/green.png -------------------------------------------------------------------------------- /TrojDiff/images/hello_kitty.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/hello_kitty.png -------------------------------------------------------------------------------- /TrojDiff/images/light_blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/light_blue.png -------------------------------------------------------------------------------- /TrojDiff/images/mickey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/mickey.png -------------------------------------------------------------------------------- /TrojDiff/images/purple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/purple.png -------------------------------------------------------------------------------- /TrojDiff/images/red.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/red.png -------------------------------------------------------------------------------- /TrojDiff/images/target_A.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/target_A.png -------------------------------------------------------------------------------- /TrojDiff/images/target_I.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/target_I.png -------------------------------------------------------------------------------- /TrojDiff/images/white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/white.png -------------------------------------------------------------------------------- /TrojDiff/images/yellow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/images/yellow.png -------------------------------------------------------------------------------- /TrojDiff/models/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/models/.DS_Store -------------------------------------------------------------------------------- /TrojDiff/runners/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/runners/.DS_Store -------------------------------------------------------------------------------- /TrojDiff/runners/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/TrojDiff/runners/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | contact_links: 2 | - name: Blank issue 3 | url: https://github.com/huggingface/diffusers/issues/new 4 | about: Other 5 | - name: Forum 6 | url: https://discuss.huggingface.co/ 7 | about: General usage questions and community discussions -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "\U0001F680 Feature request" 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/.github/ISSUE_TEMPLATE/feedback.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: "💬 Feedback about API Design" 3 | about: Give feedback about the current API design 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What API design would you like to have changed or added to the library? Why?** 11 | 12 | **What use case would this enable or better enable? Can you give us a code example?** 13 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/.github/ISSUE_TEMPLATE/new-model-addition.yml: -------------------------------------------------------------------------------- 1 | name: "\U0001F31F New model/pipeline/scheduler addition" 2 | description: Submit a proposal/request to implement a new diffusion model / pipeline / scheduler 3 | labels: [ "New model/pipeline/scheduler" ] 4 | 5 | body: 6 | - type: textarea 7 | id: description-request 8 | validations: 9 | required: true 10 | attributes: 11 | label: Model/Pipeline/Scheduler description 12 | description: | 13 | Put any and all important information relative to the model/pipeline/scheduler 14 | 15 | - type: checkboxes 16 | id: information-tasks 17 | attributes: 18 | label: Open source status 19 | description: | 20 | Please note that if the model implementation isn't available or if the weights aren't open-source, we are less likely to implement it in `diffusers`. 21 | options: 22 | - label: "The model implementation is available" 23 | - label: "The model weights are available (Only relevant if addition is not a scheduler)." 24 | 25 | - type: textarea 26 | id: additional-info 27 | attributes: 28 | label: Provide useful links for the implementation 29 | description: | 30 | Please provide information regarding the implementation, the weights, and the authors. 31 | Please mention the authors by @gh-username if you're aware of their usernames. 32 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/.github/workflows/build_docker_images.yml: -------------------------------------------------------------------------------- 1 | name: Build Docker images (nightly) 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: "0 0 * * *" # every day at midnight 7 | 8 | concurrency: 9 | group: docker-image-builds 10 | cancel-in-progress: false 11 | 12 | env: 13 | REGISTRY: diffusers 14 | 15 | jobs: 16 | build-docker-images: 17 | runs-on: ubuntu-latest 18 | 19 | permissions: 20 | contents: read 21 | packages: write 22 | 23 | strategy: 24 | fail-fast: false 25 | matrix: 26 | image-name: 27 | - diffusers-pytorch-cpu 28 | - diffusers-pytorch-cuda 29 | - diffusers-flax-cpu 30 | - diffusers-flax-tpu 31 | - diffusers-onnxruntime-cpu 32 | - diffusers-onnxruntime-cuda 33 | 34 | steps: 35 | - name: Checkout repository 36 | uses: actions/checkout@v3 37 | 38 | - name: Login to Docker Hub 39 | uses: docker/login-action@v2 40 | with: 41 | username: ${{ env.REGISTRY }} 42 | password: ${{ secrets.DOCKERHUB_TOKEN }} 43 | 44 | - name: Build and push 45 | uses: docker/build-push-action@v3 46 | with: 47 | no-cache: true 48 | context: ./docker/${{ matrix.image-name }} 49 | push: true 50 | tags: ${{ env.REGISTRY }}/${{ matrix.image-name }}:latest 51 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/.github/workflows/build_documentation.yml: -------------------------------------------------------------------------------- 1 | name: Build documentation 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - doc-builder* 8 | - v*-release 9 | 10 | jobs: 11 | build: 12 | uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main 13 | with: 14 | commit_sha: ${{ github.sha }} 15 | package: diffusers 16 | notebook_folder: diffusers_doc 17 | languages: en ko 18 | secrets: 19 | token: ${{ secrets.HUGGINGFACE_PUSH }} 20 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/.github/workflows/build_pr_documentation.yml: -------------------------------------------------------------------------------- 1 | name: Build PR Documentation 2 | 3 | on: 4 | pull_request: 5 | 6 | concurrency: 7 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 8 | cancel-in-progress: true 9 | 10 | jobs: 11 | build: 12 | uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main 13 | with: 14 | commit_sha: ${{ github.event.pull_request.head.sha }} 15 | pr_number: ${{ github.event.number }} 16 | package: diffusers 17 | languages: en ko 18 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/.github/workflows/delete_doc_comment.yml: -------------------------------------------------------------------------------- 1 | name: Delete dev documentation 2 | 3 | on: 4 | pull_request: 5 | types: [ closed ] 6 | 7 | 8 | jobs: 9 | delete: 10 | uses: huggingface/doc-builder/.github/workflows/delete_doc_comment.yml@main 11 | with: 12 | pr_number: ${{ github.event.number }} 13 | package: diffusers 14 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/.github/workflows/pr_quality.yml: -------------------------------------------------------------------------------- 1 | name: Run code quality checks 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | push: 8 | branches: 9 | - main 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | check_code_quality: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Set up Python 21 | uses: actions/setup-python@v4 22 | with: 23 | python-version: "3.7" 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install .[quality] 28 | - name: Check quality 29 | run: | 30 | black --check examples tests src utils scripts 31 | ruff examples tests src utils scripts 32 | doc-builder style src/diffusers docs/source --max_len 119 --check_only --path_to_docs docs/source 33 | 34 | check_repository_consistency: 35 | runs-on: ubuntu-latest 36 | steps: 37 | - uses: actions/checkout@v3 38 | - name: Set up Python 39 | uses: actions/setup-python@v4 40 | with: 41 | python-version: "3.7" 42 | - name: Install dependencies 43 | run: | 44 | python -m pip install --upgrade pip 45 | pip install .[quality] 46 | - name: Check quality 47 | run: | 48 | python utils/check_copies.py 49 | python utils/check_dummies.py 50 | make deps_table_check_updated 51 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Stale Bot 2 | 3 | on: 4 | schedule: 5 | - cron: "0 15 * * *" 6 | 7 | jobs: 8 | close_stale_issues: 9 | name: Close Stale Issues 10 | if: github.repository == 'huggingface/diffusers' 11 | runs-on: ubuntu-latest 12 | env: 13 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 14 | steps: 15 | - uses: actions/checkout@v2 16 | 17 | - name: Setup Python 18 | uses: actions/setup-python@v1 19 | with: 20 | python-version: 3.7 21 | 22 | - name: Install requirements 23 | run: | 24 | pip install PyGithub 25 | - name: Close stale issues 26 | run: | 27 | python utils/stale.py 28 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/.github/workflows/typos.yml: -------------------------------------------------------------------------------- 1 | name: Check typos 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v3 12 | 13 | - name: typos-action 14 | uses: crate-ci/typos@v1.12.4 15 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | title: 'Diffusers: State-of-the-art diffusion models' 3 | message: >- 4 | If you use this software, please cite it using the 5 | metadata from this file. 6 | type: software 7 | authors: 8 | - given-names: Patrick 9 | family-names: von Platen 10 | - given-names: Suraj 11 | family-names: Patil 12 | - given-names: Anton 13 | family-names: Lozhkov 14 | - given-names: Pedro 15 | family-names: Cuenca 16 | - given-names: Nathan 17 | family-names: Lambert 18 | - given-names: Kashif 19 | family-names: Rasul 20 | - given-names: Mishig 21 | family-names: Davaadorj 22 | - given-names: Thomas 23 | family-names: Wolf 24 | repository-code: 'https://github.com/huggingface/diffusers' 25 | abstract: >- 26 | Diffusers provides pretrained diffusion models across 27 | multiple modalities, such as vision and audio, and serves 28 | as a modular toolbox for inference and training of 29 | diffusion models. 30 | keywords: 31 | - deep-learning 32 | - pytorch 33 | - image-generation 34 | - diffusion 35 | - text2image 36 | - image2image 37 | - score-based-generative-modeling 38 | - stable-diffusion 39 | license: Apache-2.0 40 | version: 0.12.1 41 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include src/diffusers/utils/model_card_template.md 3 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/_typos.toml: -------------------------------------------------------------------------------- 1 | # Files for typos 2 | # Instruction: https://github.com/marketplace/actions/typos-action#getting-started 3 | 4 | [default.extend-identifiers] 5 | 6 | [default.extend-words] 7 | NIN="NIN" # NIN is used in scripts/convert_ncsnpp_original_checkpoint_to_diffusers.py 8 | nd="np" # nd may be np (numpy) 9 | parms="parms" # parms is used in scripts/convert_original_stable_diffusion_to_diffusers.py 10 | 11 | 12 | [files] 13 | extend-exclude = ["_typos.toml"] 14 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docker/diffusers-flax-cpu/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | LABEL maintainer="Hugging Face" 3 | LABEL repository="diffusers" 4 | 5 | ENV DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt update && \ 8 | apt install -y bash \ 9 | build-essential \ 10 | git \ 11 | git-lfs \ 12 | curl \ 13 | ca-certificates \ 14 | libsndfile1-dev \ 15 | python3.8 \ 16 | python3-pip \ 17 | python3.8-venv && \ 18 | rm -rf /var/lib/apt/lists 19 | 20 | # make sure to use venv 21 | RUN python3 -m venv /opt/venv 22 | ENV PATH="/opt/venv/bin:$PATH" 23 | 24 | # pre-install the heavy dependencies (these can later be overridden by the deps from setup.py) 25 | # follow the instructions here: https://cloud.google.com/tpu/docs/run-in-container#train_a_jax_model_in_a_docker_container 26 | RUN python3 -m pip install --no-cache-dir --upgrade pip && \ 27 | python3 -m pip install --upgrade --no-cache-dir \ 28 | clu \ 29 | "jax[cpu]>=0.2.16,!=0.3.2" \ 30 | "flax>=0.4.1" \ 31 | "jaxlib>=0.1.65" && \ 32 | python3 -m pip install --no-cache-dir \ 33 | accelerate \ 34 | datasets \ 35 | hf-doc-builder \ 36 | huggingface-hub \ 37 | Jinja2 \ 38 | librosa \ 39 | numpy \ 40 | scipy \ 41 | tensorboard \ 42 | transformers 43 | 44 | CMD ["/bin/bash"] -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docker/diffusers-flax-tpu/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | LABEL maintainer="Hugging Face" 3 | LABEL repository="diffusers" 4 | 5 | ENV DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt update && \ 8 | apt install -y bash \ 9 | build-essential \ 10 | git \ 11 | git-lfs \ 12 | curl \ 13 | ca-certificates \ 14 | libsndfile1-dev \ 15 | python3.8 \ 16 | python3-pip \ 17 | python3.8-venv && \ 18 | rm -rf /var/lib/apt/lists 19 | 20 | # make sure to use venv 21 | RUN python3 -m venv /opt/venv 22 | ENV PATH="/opt/venv/bin:$PATH" 23 | 24 | # pre-install the heavy dependencies (these can later be overridden by the deps from setup.py) 25 | # follow the instructions here: https://cloud.google.com/tpu/docs/run-in-container#train_a_jax_model_in_a_docker_container 26 | RUN python3 -m pip install --no-cache-dir --upgrade pip && \ 27 | python3 -m pip install --no-cache-dir \ 28 | "jax[tpu]>=0.2.16,!=0.3.2" \ 29 | -f https://storage.googleapis.com/jax-releases/libtpu_releases.html && \ 30 | python3 -m pip install --upgrade --no-cache-dir \ 31 | clu \ 32 | "flax>=0.4.1" \ 33 | "jaxlib>=0.1.65" && \ 34 | python3 -m pip install --no-cache-dir \ 35 | accelerate \ 36 | datasets \ 37 | hf-doc-builder \ 38 | huggingface-hub \ 39 | Jinja2 \ 40 | librosa \ 41 | numpy \ 42 | scipy \ 43 | tensorboard \ 44 | transformers 45 | 46 | CMD ["/bin/bash"] -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docker/diffusers-onnxruntime-cpu/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | LABEL maintainer="Hugging Face" 3 | LABEL repository="diffusers" 4 | 5 | ENV DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt update && \ 8 | apt install -y bash \ 9 | build-essential \ 10 | git \ 11 | git-lfs \ 12 | curl \ 13 | ca-certificates \ 14 | libsndfile1-dev \ 15 | python3.8 \ 16 | python3-pip \ 17 | python3.8-venv && \ 18 | rm -rf /var/lib/apt/lists 19 | 20 | # make sure to use venv 21 | RUN python3 -m venv /opt/venv 22 | ENV PATH="/opt/venv/bin:$PATH" 23 | 24 | # pre-install the heavy dependencies (these can later be overridden by the deps from setup.py) 25 | RUN python3 -m pip install --no-cache-dir --upgrade pip && \ 26 | python3 -m pip install --no-cache-dir \ 27 | torch \ 28 | torchvision \ 29 | torchaudio \ 30 | onnxruntime \ 31 | --extra-index-url https://download.pytorch.org/whl/cpu && \ 32 | python3 -m pip install --no-cache-dir \ 33 | accelerate \ 34 | datasets \ 35 | hf-doc-builder \ 36 | huggingface-hub \ 37 | Jinja2 \ 38 | librosa \ 39 | numpy \ 40 | scipy \ 41 | tensorboard \ 42 | transformers 43 | 44 | CMD ["/bin/bash"] -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docker/diffusers-onnxruntime-cuda/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:11.6.2-cudnn8-devel-ubuntu20.04 2 | LABEL maintainer="Hugging Face" 3 | LABEL repository="diffusers" 4 | 5 | ENV DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt update && \ 8 | apt install -y bash \ 9 | build-essential \ 10 | git \ 11 | git-lfs \ 12 | curl \ 13 | ca-certificates \ 14 | libsndfile1-dev \ 15 | python3.8 \ 16 | python3-pip \ 17 | python3.8-venv && \ 18 | rm -rf /var/lib/apt/lists 19 | 20 | # make sure to use venv 21 | RUN python3 -m venv /opt/venv 22 | ENV PATH="/opt/venv/bin:$PATH" 23 | 24 | # pre-install the heavy dependencies (these can later be overridden by the deps from setup.py) 25 | RUN python3 -m pip install --no-cache-dir --upgrade pip && \ 26 | python3 -m pip install --no-cache-dir \ 27 | torch \ 28 | torchvision \ 29 | torchaudio \ 30 | "onnxruntime-gpu>=1.13.1" \ 31 | --extra-index-url https://download.pytorch.org/whl/cu117 && \ 32 | python3 -m pip install --no-cache-dir \ 33 | accelerate \ 34 | datasets \ 35 | hf-doc-builder \ 36 | huggingface-hub \ 37 | Jinja2 \ 38 | librosa \ 39 | numpy \ 40 | scipy \ 41 | tensorboard \ 42 | transformers 43 | 44 | CMD ["/bin/bash"] -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docker/diffusers-pytorch-cpu/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | LABEL maintainer="Hugging Face" 3 | LABEL repository="diffusers" 4 | 5 | ENV DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt update && \ 8 | apt install -y bash \ 9 | build-essential \ 10 | git \ 11 | git-lfs \ 12 | curl \ 13 | ca-certificates \ 14 | libsndfile1-dev \ 15 | python3.8 \ 16 | python3-pip \ 17 | python3.8-venv && \ 18 | rm -rf /var/lib/apt/lists 19 | 20 | # make sure to use venv 21 | RUN python3 -m venv /opt/venv 22 | ENV PATH="/opt/venv/bin:$PATH" 23 | 24 | # pre-install the heavy dependencies (these can later be overridden by the deps from setup.py) 25 | RUN python3 -m pip install --no-cache-dir --upgrade pip && \ 26 | python3 -m pip install --no-cache-dir \ 27 | torch \ 28 | torchvision \ 29 | torchaudio \ 30 | --extra-index-url https://download.pytorch.org/whl/cpu && \ 31 | python3 -m pip install --no-cache-dir \ 32 | accelerate \ 33 | datasets \ 34 | hf-doc-builder \ 35 | huggingface-hub \ 36 | Jinja2 \ 37 | librosa \ 38 | numpy \ 39 | scipy \ 40 | tensorboard \ 41 | transformers 42 | 43 | CMD ["/bin/bash"] -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docker/diffusers-pytorch-cuda/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:11.7.1-cudnn8-runtime-ubuntu20.04 2 | LABEL maintainer="Hugging Face" 3 | LABEL repository="diffusers" 4 | 5 | ENV DEBIAN_FRONTEND=noninteractive 6 | 7 | RUN apt update && \ 8 | apt install -y bash \ 9 | build-essential \ 10 | git \ 11 | git-lfs \ 12 | curl \ 13 | ca-certificates \ 14 | libsndfile1-dev \ 15 | python3.8 \ 16 | python3-pip \ 17 | python3.8-venv && \ 18 | rm -rf /var/lib/apt/lists 19 | 20 | # make sure to use venv 21 | RUN python3 -m venv /opt/venv 22 | ENV PATH="/opt/venv/bin:$PATH" 23 | 24 | # pre-install the heavy dependencies (these can later be overridden by the deps from setup.py) 25 | RUN python3 -m pip install --no-cache-dir --upgrade pip && \ 26 | python3 -m pip install --no-cache-dir \ 27 | torch \ 28 | torchvision \ 29 | torchaudio \ 30 | python3 -m pip install --no-cache-dir \ 31 | accelerate \ 32 | datasets \ 33 | hf-doc-builder \ 34 | huggingface-hub \ 35 | Jinja2 \ 36 | librosa \ 37 | numpy \ 38 | scipy \ 39 | tensorboard \ 40 | transformers 41 | 42 | CMD ["/bin/bash"] 43 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/_config.py: -------------------------------------------------------------------------------- 1 | # docstyle-ignore 2 | INSTALL_CONTENT = """ 3 | # Diffusers installation 4 | ! pip install diffusers transformers datasets accelerate 5 | # To install from source instead of the last release, comment the command above and uncomment the following one. 6 | # ! pip install git+https://github.com/huggingface/diffusers.git 7 | """ 8 | 9 | notebook_first_cells = [{"type": "code", "content": INSTALL_CONTENT}] -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/configuration.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Configuration 14 | 15 | Schedulers from [`~schedulers.scheduling_utils.SchedulerMixin`] and models from [`ModelMixin`] inherit from [`ConfigMixin`] which conveniently takes care of storing all the parameters that are 16 | passed to their respective `__init__` methods in a JSON-configuration file. 17 | 18 | ## ConfigMixin 19 | 20 | [[autodoc]] ConfigMixin 21 | - load_config 22 | - from_config 23 | - save_config 24 | - to_json_file 25 | - to_json_string 26 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/experimental/rl.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # TODO 14 | 15 | Coming soon! -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/loaders.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Loaders 14 | 15 | There are many ways to train adapter neural networks for diffusion models, such as 16 | - [Textual Inversion](./training/text_inversion.mdx) 17 | - [LoRA](https://github.com/cloneofsimo/lora) 18 | - [Hypernetworks](https://arxiv.org/abs/1609.09106) 19 | 20 | Such adapter neural networks often only consist of a fraction of the number of weights compared 21 | to the pretrained model and as such are very portable. The Diffusers library offers an easy-to-use 22 | API to load such adapter neural networks via the [`loaders.py` module](https://github.com/huggingface/diffusers/blob/main/src/diffusers/loaders.py). 23 | 24 | **Note**: This module is still highly experimental and prone to future changes. 25 | 26 | ## LoaderMixins 27 | 28 | ### UNet2DConditionLoadersMixin 29 | 30 | [[autodoc]] loaders.UNet2DConditionLoadersMixin 31 | 32 | ### TextualInversionLoaderMixin 33 | 34 | [[autodoc]] loaders.TextualInversionLoaderMixin 35 | 36 | ### LoraLoaderMixin 37 | 38 | [[autodoc]] loaders.LoraLoaderMixin 39 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/pipelines/dance_diffusion.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Dance Diffusion 14 | 15 | ## Overview 16 | 17 | [Dance Diffusion](https://github.com/Harmonai-org/sample-generator) by Zach Evans. 18 | 19 | Dance Diffusion is the first in a suite of generative audio tools for producers and musicians to be released by Harmonai. 20 | For more info or to get involved in the development of these tools, please visit https://harmonai.org and fill out the form on the front page. 21 | 22 | The original codebase of this implementation can be found [here](https://github.com/Harmonai-org/sample-generator). 23 | 24 | ## Available Pipelines: 25 | 26 | | Pipeline | Tasks | Colab 27 | |---|---|:---:| 28 | | [pipeline_dance_diffusion.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py) | *Unconditional Audio Generation* | - | 29 | 30 | 31 | ## DanceDiffusionPipeline 32 | [[autodoc]] DanceDiffusionPipeline 33 | - all 34 | - __call__ 35 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/pipelines/stable_diffusion/image_variation.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Image Variation 14 | 15 | ## StableDiffusionImageVariationPipeline 16 | 17 | [`StableDiffusionImageVariationPipeline`] lets you generate variations from an input image using Stable Diffusion. It uses a fine-tuned version of Stable Diffusion model, trained by [Justin Pinkney](https://www.justinpinkney.com/) (@Buntworthy) at [Lambda](https://lambdalabs.com/). 18 | 19 | The original codebase can be found here: 20 | [Stable Diffusion Image Variations](https://github.com/LambdaLabsML/lambda-diffusers#stable-diffusion-image-variations) 21 | 22 | Available Checkpoints are: 23 | - *sd-image-variations-diffusers*: [lambdalabs/sd-image-variations-diffusers](https://huggingface.co/lambdalabs/sd-image-variations-diffusers) 24 | 25 | [[autodoc]] StableDiffusionImageVariationPipeline 26 | - all 27 | - __call__ 28 | - enable_attention_slicing 29 | - disable_attention_slicing 30 | - enable_xformers_memory_efficient_attention 31 | - disable_xformers_memory_efficient_attention 32 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/pipelines/stable_diffusion/latent_upscale.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Stable Diffusion Latent Upscaler 14 | 15 | ## StableDiffusionLatentUpscalePipeline 16 | 17 | The Stable Diffusion Latent Upscaler model was created by [Katherine Crowson](https://github.com/crowsonkb/k-diffusion) in collaboration with [Stability AI](https://stability.ai/). It can be used on top of any [`StableDiffusionUpscalePipeline`] checkpoint to enhance its output image resolution by a factor of 2. 18 | 19 | A notebook that demonstrates the original implementation can be found here: 20 | - [Stable Diffusion Upscaler Demo](https://colab.research.google.com/drive/1o1qYJcFeywzCIdkfKJy7cTpgZTCM2EI4) 21 | 22 | Available Checkpoints are: 23 | - *stabilityai/latent-upscaler*: [stabilityai/sd-x2-latent-upscaler](https://huggingface.co/stabilityai/sd-x2-latent-upscaler) 24 | 25 | 26 | [[autodoc]] StableDiffusionLatentUpscalePipeline 27 | - all 28 | - __call__ 29 | - enable_sequential_cpu_offload 30 | - enable_attention_slicing 31 | - disable_attention_slicing 32 | - enable_xformers_memory_efficient_attention 33 | - disable_xformers_memory_efficient_attention -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/pipelines/stable_diffusion/upscale.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Super-Resolution 14 | 15 | ## StableDiffusionUpscalePipeline 16 | 17 | The upscaler diffusion model was created by the researchers and engineers from [CompVis](https://github.com/CompVis), [Stability AI](https://stability.ai/), and [LAION](https://laion.ai/), as part of Stable Diffusion 2.0. [`StableDiffusionUpscalePipeline`] can be used to enhance the resolution of input images by a factor of 4. 18 | 19 | The original codebase can be found here: 20 | - *Stable Diffusion v2*: [Stability-AI/stablediffusion](https://github.com/Stability-AI/stablediffusion#image-upscaling-with-stable-diffusion) 21 | 22 | Available Checkpoints are: 23 | - *stabilityai/stable-diffusion-x4-upscaler (x4 resolution resolution)*: [stable-diffusion-x4-upscaler](https://huggingface.co/stabilityai/stable-diffusion-x4-upscaler) 24 | 25 | 26 | [[autodoc]] StableDiffusionUpscalePipeline 27 | - all 28 | - __call__ 29 | - enable_attention_slicing 30 | - disable_attention_slicing 31 | - enable_xformers_memory_efficient_attention 32 | - disable_xformers_memory_efficient_attention -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/ddim_inverse.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Inverse Denoising Diffusion Implicit Models (DDIMInverse) 14 | 15 | ## Overview 16 | 17 | This scheduler is the inverted scheduler of [Denoising Diffusion Implicit Models](https://arxiv.org/abs/2010.02502) (DDIM) by Jiaming Song, Chenlin Meng and Stefano Ermon. 18 | The implementation is mostly based on the DDIM inversion definition of [Null-text Inversion for Editing Real Images using Guided Diffusion Models](https://arxiv.org/pdf/2211.09794.pdf) 19 | 20 | ## DDIMInverseScheduler 21 | [[autodoc]] DDIMInverseScheduler 22 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/deis.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # DEIS 14 | 15 | Fast Sampling of Diffusion Models with Exponential Integrator. 16 | 17 | ## Overview 18 | 19 | Original paper can be found [here](https://arxiv.org/abs/2204.13902). The original implementation can be found [here](https://github.com/qsh-zh/deis). 20 | 21 | ## DEISMultistepScheduler 22 | [[autodoc]] DEISMultistepScheduler 23 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/dpm_discrete.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # DPM Discrete Scheduler inspired by Karras et. al paper 14 | 15 | ## Overview 16 | 17 | Inspired by [Karras et. al](https://arxiv.org/abs/2206.00364). Scheduler ported from @crowsonkb's https://github.com/crowsonkb/k-diffusion library: 18 | 19 | All credit for making this scheduler work goes to [Katherine Crowson](https://github.com/crowsonkb/) 20 | 21 | ## KDPM2DiscreteScheduler 22 | [[autodoc]] KDPM2DiscreteScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/dpm_discrete_ancestral.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # DPM Discrete Scheduler with ancestral sampling inspired by Karras et. al paper 14 | 15 | ## Overview 16 | 17 | Inspired by [Karras et. al](https://arxiv.org/abs/2206.00364). Scheduler ported from @crowsonkb's https://github.com/crowsonkb/k-diffusion library: 18 | 19 | All credit for making this scheduler work goes to [Katherine Crowson](https://github.com/crowsonkb/) 20 | 21 | ## KDPM2AncestralDiscreteScheduler 22 | [[autodoc]] KDPM2AncestralDiscreteScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/euler.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Euler scheduler 14 | 15 | ## Overview 16 | 17 | Euler scheduler (Algorithm 2) from the paper [Elucidating the Design Space of Diffusion-Based Generative Models](https://arxiv.org/abs/2206.00364) by Karras et al. (2022). Based on the original [k-diffusion](https://github.com/crowsonkb/k-diffusion/blob/481677d114f6ea445aa009cf5bd7a9cdee909e47/k_diffusion/sampling.py#L51) implementation by Katherine Crowson. 18 | Fast scheduler which often times generates good outputs with 20-30 steps. 19 | 20 | ## EulerDiscreteScheduler 21 | [[autodoc]] EulerDiscreteScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/euler_ancestral.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Euler Ancestral scheduler 14 | 15 | ## Overview 16 | 17 | Ancestral sampling with Euler method steps. Based on the original [k-diffusion](https://github.com/crowsonkb/k-diffusion/blob/481677d114f6ea445aa009cf5bd7a9cdee909e47/k_diffusion/sampling.py#L72) implementation by Katherine Crowson. 18 | Fast scheduler which often times generates good outputs with 20-30 steps. 19 | 20 | ## EulerAncestralDiscreteScheduler 21 | [[autodoc]] EulerAncestralDiscreteScheduler 22 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/heun.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Heun scheduler inspired by Karras et. al paper 14 | 15 | ## Overview 16 | 17 | Algorithm 1 of [Karras et. al](https://arxiv.org/abs/2206.00364). 18 | Scheduler ported from @crowsonkb's https://github.com/crowsonkb/k-diffusion library: 19 | 20 | All credit for making this scheduler work goes to [Katherine Crowson](https://github.com/crowsonkb/) 21 | 22 | ## HeunDiscreteScheduler 23 | [[autodoc]] HeunDiscreteScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/ipndm.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # improved pseudo numerical methods for diffusion models (iPNDM) 14 | 15 | ## Overview 16 | 17 | Original implementation can be found [here](https://github.com/crowsonkb/v-diffusion-pytorch/blob/987f8985e38208345c1959b0ea767a625831cc9b/diffusion/sampling.py#L296). 18 | 19 | ## IPNDMScheduler 20 | [[autodoc]] IPNDMScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/lms_discrete.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Linear multistep scheduler for discrete beta schedules 14 | 15 | ## Overview 16 | 17 | Original implementation can be found [here](https://arxiv.org/abs/2206.00364). 18 | 19 | ## LMSDiscreteScheduler 20 | [[autodoc]] LMSDiscreteScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/multistep_dpm_solver.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Multistep DPM-Solver 14 | 15 | ## Overview 16 | 17 | Original paper can be found [here](https://arxiv.org/abs/2206.00927) and the [improved version](https://arxiv.org/abs/2211.01095). The original implementation can be found [here](https://github.com/LuChengTHU/dpm-solver). 18 | 19 | ## DPMSolverMultistepScheduler 20 | [[autodoc]] DPMSolverMultistepScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/pndm.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Pseudo numerical methods for diffusion models (PNDM) 14 | 15 | ## Overview 16 | 17 | Original implementation can be found [here](https://github.com/crowsonkb/k-diffusion/blob/481677d114f6ea445aa009cf5bd7a9cdee909e47/k_diffusion/sampling.py#L181). 18 | 19 | ## PNDMScheduler 20 | [[autodoc]] PNDMScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/repaint.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # RePaint scheduler 14 | 15 | ## Overview 16 | 17 | DDPM-based inpainting scheduler for unsupervised inpainting with extreme masks. 18 | Intended for use with [`RePaintPipeline`]. 19 | Based on the paper [RePaint: Inpainting using Denoising Diffusion Probabilistic Models](https://arxiv.org/abs/2201.09865) 20 | and the original implementation by Andreas Lugmayr et al.: https://github.com/andreas128/RePaint 21 | 22 | ## RePaintScheduler 23 | [[autodoc]] RePaintScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/score_sde_ve.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Variance Exploding Stochastic Differential Equation (VE-SDE) scheduler 14 | 15 | ## Overview 16 | 17 | Original paper can be found [here](https://arxiv.org/abs/2011.13456). 18 | 19 | ## ScoreSdeVeScheduler 20 | [[autodoc]] ScoreSdeVeScheduler 21 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/score_sde_vp.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Variance Preserving Stochastic Differential Equation (VP-SDE) scheduler 14 | 15 | ## Overview 16 | 17 | Original paper can be found [here](https://arxiv.org/abs/2011.13456). 18 | 19 | 20 | 21 | Score SDE-VP is under construction. 22 | 23 | 24 | 25 | ## ScoreSdeVpScheduler 26 | [[autodoc]] schedulers.scheduling_sde_vp.ScoreSdeVpScheduler 27 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/singlestep_dpm_solver.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Singlestep DPM-Solver 14 | 15 | ## Overview 16 | 17 | Original paper can be found [here](https://arxiv.org/abs/2206.00927) and the [improved version](https://arxiv.org/abs/2211.01095). The original implementation can be found [here](https://github.com/LuChengTHU/dpm-solver). 18 | 19 | ## DPMSolverSinglestepScheduler 20 | [[autodoc]] DPMSolverSinglestepScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/stochastic_karras_ve.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Variance exploding, stochastic sampling from Karras et. al 14 | 15 | ## Overview 16 | 17 | Original paper can be found [here](https://arxiv.org/abs/2206.00364). 18 | 19 | ## KarrasVeScheduler 20 | [[autodoc]] KarrasVeScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/unipc.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # UniPC 14 | 15 | ## Overview 16 | 17 | UniPC is a training-free framework designed for the fast sampling of diffusion models, which consists of a corrector (UniC) and a predictor (UniP) that share a unified analytical form and support arbitrary orders. 18 | 19 | For more details about the method, please refer to the [paper](https://arxiv.org/abs/2302.04867) and the [code](https://github.com/wl-zhao/UniPC). 20 | 21 | Fast Sampling of Diffusion Models with Exponential Integrator. 22 | 23 | ## UniPCMultistepScheduler 24 | [[autodoc]] UniPCMultistepScheduler 25 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/api/schedulers/vq_diffusion.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # VQDiffusionScheduler 14 | 15 | ## Overview 16 | 17 | Original paper can be found [here](https://arxiv.org/abs/2111.14822) 18 | 19 | ## VQDiffusionScheduler 20 | [[autodoc]] VQDiffusionScheduler -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/imgs/access_request.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/docs/source/en/imgs/access_request.png -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/imgs/diffusers_library.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/docs/source/en/imgs/diffusers_library.jpg -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/optimization/opt_overview.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Overview 14 | 15 | Generating high-quality outputs is computationally intensive, especially during each iterative step where you go from a noisy output to a less noisy output. One of 🧨 Diffuser's goal is to make this technology widely accessible to everyone, which includes enabling fast inference on consumer and specialized hardware. 16 | 17 | This section will cover tips and tricks - like half-precision weights and sliced attention - for optimizing inference speed and reducing memory-consumption. You can also learn how to speed up your PyTorch code with [`torch.compile`](https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html) or [ONNX Runtime](https://onnxruntime.ai/docs/), and enable memory-efficient attention with [xFormers](https://facebookresearch.github.io/xformers/). There are also guides for running inference on specific hardware like Apple Silicon, and Intel or Habana processors. -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/tutorials/tutorial_overview.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Overview 14 | 15 | Welcome to 🧨 Diffusers! If you're new to diffusion models and generative AI, and want to learn more, then you've come to the right place. These beginner-friendly tutorials are designed to provide a gentle introduction to diffusion models and help you understand the library fundamentals - the core components and how 🧨 Diffusers is meant to be used. 16 | 17 | You'll learn how to use a pipeline for inference to rapidly generate things, and then deconstruct that pipeline to really understand how to use the library as a modular toolbox for building your own diffusion systems. In the next lesson, you'll learn how to train your own diffusion model to generate what you want. 18 | 19 | After completing the tutorials, you'll have gained the necessary skills to start exploring the library on your own and see how to use it for your own projects and applications. 20 | 21 | Feel free to join our community on [Discord](https://discord.com/invite/JfAtkvEtRb) or the [forums](https://discuss.huggingface.co/c/discussion-related-to-httpsgithubcomhuggingfacediffusers/63) to connect and collaborate with other users and developers! 22 | 23 | Let's start diffusing! 🧨 -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/using-diffusers/audio.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Using Diffusers for audio 14 | 15 | [`DanceDiffusionPipeline`] and [`AudioDiffusionPipeline`] can be used to generate 16 | audio rapidly! More coming soon! -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/using-diffusers/loading_overview.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Overview 14 | 15 | 🧨 Diffusers offers many pipelines, models, and schedulers for generative tasks. To make loading these components as simple as possible, we provide a single and unified method - `from_pretrained()` - that loads any of these components from either the Hugging Face [Hub](https://huggingface.co/models?library=diffusers&sort=downloads) or your local machine. Whenever you load a pipeline or model, the latest files are automatically downloaded and cached so you can quickly reuse them next time without redownloading the files. 16 | 17 | This section will show you everything you need to know about loading pipelines, how to load different components in a pipeline, how to load checkpoint variants, and how to load community pipelines. You'll also learn how to load schedulers and compare the speed and quality trade-offs of using different schedulers. Finally, you'll see how to convert and load KerasCV checkpoints so you can use them in PyTorch with 🧨 Diffusers. -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/using-diffusers/other-modalities.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Using Diffusers with other modalities 14 | 15 | Diffusers is in the process of expanding to modalities other than images. 16 | 17 | Example type | Colab | Pipeline | 18 | :-------------------------:|:-------------------------:|:-------------------------:| 19 | [Molecule conformation](https://www.nature.com/subjects/molecular-conformation#:~:text=Definition,to%20changes%20in%20their%20environment.) generation | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/geodiff_molecule_conformation.ipynb) | ❌ 20 | 21 | More coming soon! -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/using-diffusers/pipeline_overview.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Overview 14 | 15 | A pipeline is an end-to-end class that provides a quick and easy way to use a diffusion system for inference by bundling independently trained models and schedulers together. Certain combinations of models and schedulers define specific pipeline types, like [`StableDiffusionPipeline`] or [`StableDiffusionControlNetPipeline`], with specific capabilities. All pipeline types inherit from the base [`DiffusionPipeline`] class; pass it any checkpoint, and it'll automatically detect the pipeline type and load the necessary components. 16 | 17 | This section introduces you to some of the tasks supported by our pipelines such as unconditional image generation and different techniques and variations of text-to-image generation. You'll also learn how to gain more control over the generation process by setting a seed for reproducibility and weighting prompts to adjust the influence certain words in the prompt has over the output. Finally, you'll see how you can create a community pipeline for a custom task like generating images from speech. -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/using-diffusers/rl.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # Using Diffusers for reinforcement learning 14 | 15 | Support for one RL model and related pipelines is included in the `experimental` source of diffusers. 16 | More models and examples coming soon! 17 | 18 | # Diffuser Value-guided Planning 19 | 20 | You can run the model from [*Planning with Diffusion for Flexible Behavior Synthesis*](https://arxiv.org/abs/2205.09991) with Diffusers. 21 | The script is located in the [RL Examples](https://github.com/huggingface/diffusers/tree/main/examples/rl) folder. 22 | 23 | Or, run this example in Colab [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/reinforcement_learning_with_diffusers.ipynb) 24 | 25 | [[autodoc]] diffusers.experimental.ValueGuidedRLPipeline -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/en/using-diffusers/using_safetensors: -------------------------------------------------------------------------------- 1 | # What is safetensors ? 2 | 3 | [safetensors](https://github.com/huggingface/safetensors) is a different format 4 | from the classic `.bin` which uses Pytorch which uses pickle. 5 | 6 | Pickle is notoriously unsafe which allow any malicious file to execute arbitrary code. 7 | The hub itself tries to prevent issues from it, but it's not a silver bullet. 8 | 9 | `safetensors` first and foremost goal is to make loading machine learning models *safe* 10 | in the sense that no takeover of your computer can be done. 11 | 12 | # Why use safetensors ? 13 | 14 | **Safety** can be one reason, if you're attempting to use a not well known model and 15 | you're not sure about the source of the file. 16 | 17 | And a secondary reason, is **the speed of loading**. Safetensors can load models much faster 18 | than regular pickle files. If you spend a lot of times switching models, this can be 19 | a huge timesave. 20 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/docs/source/ko/in_translation.mdx: -------------------------------------------------------------------------------- 1 | 12 | 13 | # 번역중 14 | 15 | 열심히 번역을 진행중입니다. 조금만 기다려주세요. 16 | 감사합니다! -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/community/one_step_unet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import torch 3 | 4 | from diffusers import DiffusionPipeline 5 | 6 | 7 | class UnetSchedulerOneForwardPipeline(DiffusionPipeline): 8 | def __init__(self, unet, scheduler): 9 | super().__init__() 10 | 11 | self.register_modules(unet=unet, scheduler=scheduler) 12 | 13 | def __call__(self): 14 | image = torch.randn( 15 | (1, self.unet.config.in_channels, self.unet.config.sample_size, self.unet.config.sample_size), 16 | ) 17 | timestep = 1 18 | 19 | model_output = self.unet(image, timestep).sample 20 | scheduler_output = self.scheduler.step(model_output, timestep, image).prev_sample 21 | 22 | result = scheduler_output - scheduler_output + torch.ones_like(scheduler_output) 23 | 24 | return result 25 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/controlnet/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | transformers>=4.25.1 4 | ftfy 5 | tensorboard 6 | datasets 7 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/controlnet/requirements_flax.txt: -------------------------------------------------------------------------------- 1 | transformers>=4.25.1 2 | datasets 3 | flax 4 | optax 5 | torch 6 | torchvision 7 | ftfy 8 | tensorboard 9 | Jinja2 10 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/dreambooth/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | transformers>=4.25.1 4 | ftfy 5 | tensorboard 6 | Jinja2 7 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/dreambooth/requirements_flax.txt: -------------------------------------------------------------------------------- 1 | transformers>=4.25.1 2 | flax 3 | optax 4 | torch 5 | torchvision 6 | ftfy 7 | tensorboard 8 | Jinja2 9 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/inference/README.md: -------------------------------------------------------------------------------- 1 | # Inference Examples 2 | 3 | **The inference examples folder is deprecated and will be removed in a future version**. 4 | **Officially supported inference examples can be found in the [Pipelines folder](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines)**. 5 | 6 | - For `Image-to-Image text-guided generation with Stable Diffusion`, please have a look at the official [Pipeline examples](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines#examples) 7 | - For `In-painting using Stable Diffusion`, please have a look at the official [Pipeline examples](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines#examples) 8 | - For `Tweak prompts reusing seeds and latents`, please have a look at the official [Pipeline examples](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines#examples) 9 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/inference/image_to_image.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | from diffusers import StableDiffusionImg2ImgPipeline # noqa F401 4 | 5 | 6 | warnings.warn( 7 | "The `image_to_image.py` script is outdated. Please use directly `from diffusers import" 8 | " StableDiffusionImg2ImgPipeline` instead." 9 | ) 10 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/inference/inpainting.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | from diffusers import StableDiffusionInpaintPipeline as StableDiffusionInpaintPipeline # noqa F401 4 | 5 | 6 | warnings.warn( 7 | "The `inpainting.py` script is outdated. Please use directly `from diffusers import" 8 | " StableDiffusionInpaintPipeline` instead." 9 | ) 10 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/instruct_pix2pix/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | transformers>=4.25.1 4 | datasets 5 | ftfy 6 | tensorboard -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/README.md: -------------------------------------------------------------------------------- 1 | # Research projects 2 | 3 | This folder contains various research projects using 🧨 Diffusers. 4 | They are not really maintained by the core maintainers of this library and often require a specific version of Diffusers that is indicated in the requirements file of each folder. 5 | Updating them to the most recent version of the library will require some work. 6 | 7 | To use any of them, just run the command 8 | 9 | ``` 10 | pip install -r requirements.txt 11 | ``` 12 | inside the folder of your choice. 13 | 14 | If you need help with any of those, please open an issue where you directly ping the author(s), as indicated at the top of the README of each folder. 15 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/colossalai/inference.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from diffusers import StableDiffusionPipeline 4 | 5 | 6 | model_id = "path-to-your-trained-model" 7 | pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda") 8 | 9 | prompt = "A photo of sks dog in a bucket" 10 | image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5).images[0] 11 | 12 | image.save("dog-bucket.png") 13 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/colossalai/requirement.txt: -------------------------------------------------------------------------------- 1 | diffusers 2 | torch 3 | torchvision 4 | ftfy 5 | tensorboard 6 | Jinja2 7 | transformers -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/dreambooth_inpaint/requirements.txt: -------------------------------------------------------------------------------- 1 | diffusers==0.9.0 2 | accelerate>=0.16.0 3 | torchvision 4 | transformers>=4.21.0 5 | ftfy 6 | tensorboard 7 | Jinja2 8 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/intel_opts/textual_inversion/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | transformers>=4.21.0 4 | ftfy 5 | tensorboard 6 | Jinja2 7 | intel_extension_for_pytorch>=1.13 8 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/lora/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | transformers>=4.25.1 4 | datasets 5 | ftfy 6 | tensorboard 7 | Jinja2 8 | git+https://github.com/huggingface/peft.git -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/mulit_token_textual_inversion/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | transformers>=4.25.1 4 | ftfy 5 | tensorboard 6 | Jinja2 7 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/mulit_token_textual_inversion/requirements_flax.txt: -------------------------------------------------------------------------------- 1 | transformers>=4.25.1 2 | flax 3 | optax 4 | torch 5 | torchvision 6 | ftfy 7 | tensorboard 8 | Jinja2 9 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/multi_subject_dreambooth/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | transformers>=4.25.1 4 | ftfy 5 | tensorboard 6 | Jinja2 -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/onnxruntime/README.md: -------------------------------------------------------------------------------- 1 | ## Diffusers examples with ONNXRuntime optimizations 2 | 3 | **This research project is not actively maintained by the diffusers team. For any questions or comments, please contact Prathik Rao (prathikr), Sunghoon Choi (hanbitmyths), Ashwini Khade (askhade), or Peng Wang (pengwa) on github with any questions.** 4 | 5 | This aims to provide diffusers examples with ONNXRuntime optimizations for training/fine-tuning unconditional image generation, text to image, and textual inversion. Please see individual directories for more details on how to run each task using ONNXRuntime. -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/onnxruntime/text_to_image/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | transformers>=4.25.1 4 | datasets 5 | ftfy 6 | tensorboard 7 | modelcards 8 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/onnxruntime/textual_inversion/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | transformers>=4.25.1 4 | ftfy 5 | tensorboard 6 | modelcards 7 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/research_projects/onnxruntime/unconditional_image_generation/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | datasets 4 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/rl/README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | These examples show how to run [Diffuser](https://arxiv.org/abs/2205.09991) in Diffusers. 4 | There are two ways to use the script, `run_diffuser_locomotion.py`. 5 | 6 | The key option is a change of the variable `n_guide_steps`. 7 | When `n_guide_steps=0`, the trajectories are sampled from the diffusion model, but not fine-tuned to maximize reward in the environment. 8 | By default, `n_guide_steps=2` to match the original implementation. 9 | 10 | 11 | You will need some RL specific requirements to run the examples: 12 | 13 | ``` 14 | pip install -f https://download.pytorch.org/whl/torch_stable.html \ 15 | free-mujoco-py \ 16 | einops \ 17 | gym==0.24.1 \ 18 | protobuf==3.20.1 \ 19 | git+https://github.com/rail-berkeley/d4rl.git \ 20 | mediapy \ 21 | Pillow==9.0.0 22 | ``` 23 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/rl/run_diffuser_locomotion.py: -------------------------------------------------------------------------------- 1 | import d4rl # noqa 2 | import gym 3 | import tqdm 4 | from diffusers.experimental import ValueGuidedRLPipeline 5 | 6 | 7 | config = { 8 | "n_samples": 64, 9 | "horizon": 32, 10 | "num_inference_steps": 20, 11 | "n_guide_steps": 2, # can set to 0 for faster sampling, does not use value network 12 | "scale_grad_by_std": True, 13 | "scale": 0.1, 14 | "eta": 0.0, 15 | "t_grad_cutoff": 2, 16 | "device": "cpu", 17 | } 18 | 19 | 20 | if __name__ == "__main__": 21 | env_name = "hopper-medium-v2" 22 | env = gym.make(env_name) 23 | 24 | pipeline = ValueGuidedRLPipeline.from_pretrained( 25 | "bglick13/hopper-medium-v2-value-function-hor32", 26 | env=env, 27 | ) 28 | 29 | env.seed(0) 30 | obs = env.reset() 31 | total_reward = 0 32 | total_score = 0 33 | T = 1000 34 | rollout = [obs.copy()] 35 | try: 36 | for t in tqdm.tqdm(range(T)): 37 | # call the policy 38 | denorm_actions = pipeline(obs, planning_horizon=32) 39 | 40 | # execute action in environment 41 | next_observation, reward, terminal, _ = env.step(denorm_actions) 42 | score = env.get_normalized_score(total_reward) 43 | 44 | # update return 45 | total_reward += reward 46 | total_score += score 47 | print( 48 | f"Step: {t}, Reward: {reward}, Total Reward: {total_reward}, Score: {score}, Total Score:" 49 | f" {total_score}" 50 | ) 51 | 52 | # save observations for rendering 53 | rollout.append(next_observation.copy()) 54 | 55 | obs = next_observation 56 | except KeyboardInterrupt: 57 | pass 58 | 59 | print(f"Total reward: {total_reward}") 60 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/text_to_image/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | transformers>=4.25.1 4 | datasets 5 | ftfy 6 | tensorboard 7 | Jinja2 8 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/text_to_image/requirements_flax.txt: -------------------------------------------------------------------------------- 1 | transformers>=4.25.1 2 | datasets 3 | flax 4 | optax 5 | torch 6 | torchvision 7 | ftfy 8 | tensorboard 9 | Jinja2 10 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/textual_inversion/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | transformers>=4.25.1 4 | ftfy 5 | tensorboard 6 | Jinja2 7 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/textual_inversion/requirements_flax.txt: -------------------------------------------------------------------------------- 1 | transformers>=4.25.1 2 | flax 3 | optax 4 | torch 5 | torchvision 6 | ftfy 7 | tensorboard 8 | Jinja2 9 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/examples/unconditional_image_generation/requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate>=0.16.0 2 | torchvision 3 | datasets 4 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 119 3 | target-version = ['py37'] 4 | 5 | [tool.ruff] 6 | # Never enforce `E501` (line length violations). 7 | ignore = ["C901", "E501", "E741", "W605"] 8 | select = ["C", "E", "F", "I", "W"] 9 | line-length = 119 10 | 11 | # Ignore import violations in all `__init__.py` files. 12 | [tool.ruff.per-file-ignores] 13 | "__init__.py" = ["E402", "F401", "F403", "F811"] 14 | "src/diffusers/utils/dummy_*.py" = ["F401"] 15 | 16 | [tool.ruff.isort] 17 | lines-after-imports = 2 18 | known-first-party = ["diffusers"] 19 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/scripts/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/scripts/convert_unclip_txt2img_to_image_variation.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection 4 | 5 | from diffusers import UnCLIPImageVariationPipeline, UnCLIPPipeline 6 | 7 | 8 | if __name__ == "__main__": 9 | parser = argparse.ArgumentParser() 10 | 11 | parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") 12 | 13 | parser.add_argument( 14 | "--txt2img_unclip", 15 | default="kakaobrain/karlo-v1-alpha", 16 | type=str, 17 | required=False, 18 | help="The pretrained txt2img unclip.", 19 | ) 20 | 21 | args = parser.parse_args() 22 | 23 | txt2img = UnCLIPPipeline.from_pretrained(args.txt2img_unclip) 24 | 25 | feature_extractor = CLIPImageProcessor() 26 | image_encoder = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-large-patch14") 27 | 28 | img2img = UnCLIPImageVariationPipeline( 29 | decoder=txt2img.decoder, 30 | text_encoder=txt2img.text_encoder, 31 | tokenizer=txt2img.tokenizer, 32 | text_proj=txt2img.text_proj, 33 | feature_extractor=feature_extractor, 34 | image_encoder=image_encoder, 35 | super_res_first=txt2img.super_res_first, 36 | super_res_last=txt2img.super_res_last, 37 | decoder_scheduler=txt2img.decoder_scheduler, 38 | super_res_scheduler=txt2img.super_res_scheduler, 39 | ) 40 | 41 | img2img.save_pretrained(args.dump_path) 42 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/setup.cfg: -------------------------------------------------------------------------------- 1 | [isort] 2 | default_section = FIRSTPARTY 3 | ensure_newline_before_comments = True 4 | force_grid_wrap = 0 5 | include_trailing_comma = True 6 | known_first_party = accelerate 7 | known_third_party = 8 | numpy 9 | torch 10 | torch_xla 11 | 12 | line_length = 119 13 | lines_after_imports = 2 14 | multi_line_output = 3 15 | use_parentheses = True 16 | 17 | [flake8] 18 | ignore = E203, E722, E501, E741, W503, W605 19 | max-line-length = 119 20 | per-file-ignores = __init__.py:F401 21 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from abc import ABC, abstractmethod 16 | from argparse import ArgumentParser 17 | 18 | 19 | class BaseDiffusersCLICommand(ABC): 20 | @staticmethod 21 | @abstractmethod 22 | def register_subcommand(parser: ArgumentParser): 23 | raise NotImplementedError() 24 | 25 | @abstractmethod 26 | def run(self): 27 | raise NotImplementedError() 28 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/commands/diffusers_cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2023 The HuggingFace Team. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | from argparse import ArgumentParser 17 | 18 | from .env import EnvironmentCommand 19 | 20 | 21 | def main(): 22 | parser = ArgumentParser("Diffusers CLI tool", usage="diffusers-cli []") 23 | commands_parser = parser.add_subparsers(help="diffusers-cli command helpers") 24 | 25 | # Register commands 26 | EnvironmentCommand.register_subcommand(commands_parser) 27 | 28 | # Let's go 29 | args = parser.parse_args() 30 | 31 | if not hasattr(args, "func"): 32 | parser.print_help() 33 | exit(1) 34 | 35 | # Run 36 | service = args.func(args) 37 | service.run() 38 | 39 | 40 | if __name__ == "__main__": 41 | main() 42 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/dependency_versions_table.py: -------------------------------------------------------------------------------- 1 | # THIS FILE HAS BEEN AUTOGENERATED. To update: 2 | # 1. modify the `_deps` dict in setup.py 3 | # 2. run `make deps_table_update`` 4 | deps = { 5 | "Pillow": "Pillow", 6 | "accelerate": "accelerate>=0.11.0", 7 | "compel": "compel==0.1.8", 8 | "black": "black~=23.1", 9 | "datasets": "datasets", 10 | "filelock": "filelock", 11 | "flax": "flax>=0.4.1", 12 | "hf-doc-builder": "hf-doc-builder>=0.3.0", 13 | "huggingface-hub": "huggingface-hub>=0.13.2", 14 | "requests-mock": "requests-mock==1.10.0", 15 | "importlib_metadata": "importlib_metadata", 16 | "isort": "isort>=5.5.4", 17 | "jax": "jax>=0.2.8,!=0.3.2", 18 | "jaxlib": "jaxlib>=0.1.65", 19 | "Jinja2": "Jinja2", 20 | "k-diffusion": "k-diffusion>=0.0.12", 21 | "librosa": "librosa", 22 | "note-seq": "note-seq", 23 | "numpy": "numpy", 24 | "parameterized": "parameterized", 25 | "protobuf": "protobuf>=3.20.3,<4", 26 | "pytest": "pytest", 27 | "pytest-timeout": "pytest-timeout", 28 | "pytest-xdist": "pytest-xdist", 29 | "ruff": "ruff>=0.0.241", 30 | "safetensors": "safetensors", 31 | "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92", 32 | "scipy": "scipy", 33 | "regex": "regex!=2019.12.17", 34 | "requests": "requests", 35 | "tensorboard": "tensorboard", 36 | "torch": "torch>=1.4", 37 | "torchvision": "torchvision", 38 | "transformers": "transformers>=4.25.1", 39 | } 40 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/experimental/README.md: -------------------------------------------------------------------------------- 1 | # 🧨 Diffusers Experimental 2 | 3 | We are adding experimental code to support novel applications and usages of the Diffusers library. 4 | Currently, the following experiments are supported: 5 | * Reinforcement learning via an implementation of the [Diffuser](https://arxiv.org/abs/2205.09991) model. -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/experimental/__init__.py: -------------------------------------------------------------------------------- 1 | from .rl import ValueGuidedRLPipeline 2 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/experimental/rl/__init__.py: -------------------------------------------------------------------------------- 1 | from .value_guided_sampling import ValueGuidedRLPipeline 2 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/models/README.md: -------------------------------------------------------------------------------- 1 | # Models 2 | 3 | For more detail on the models, please refer to the [docs](https://huggingface.co/docs/diffusers/api/models). -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/models/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from ..utils import is_flax_available, is_torch_available 16 | 17 | 18 | if is_torch_available(): 19 | from .autoencoder_kl import AutoencoderKL 20 | from .controlnet import ControlNetModel 21 | from .dual_transformer_2d import DualTransformer2DModel 22 | from .modeling_utils import ModelMixin 23 | from .prior_transformer import PriorTransformer 24 | from .t5_film_transformer import T5FilmDecoder 25 | from .transformer_2d import Transformer2DModel 26 | from .unet_1d import UNet1DModel 27 | from .unet_2d import UNet2DModel 28 | from .unet_2d_condition import UNet2DConditionModel 29 | from .unet_3d_condition import UNet3DConditionModel 30 | from .vq_model import VQModel 31 | 32 | if is_flax_available(): 33 | from .controlnet_flax import FlaxControlNetModel 34 | from .unet_2d_condition_flax import FlaxUNet2DConditionModel 35 | from .vae_flax import FlaxAutoencoderKL 36 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipeline_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | 14 | # limitations under the License. 15 | 16 | # NOTE: This file is deprecated and will be removed in a future version. 17 | # It only exists so that temporarely `from diffusers.pipelines import DiffusionPipeline` works 18 | 19 | from .pipelines import DiffusionPipeline, ImagePipelineOutput # noqa: F401 20 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/alt_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL 6 | from PIL import Image 7 | 8 | from ...utils import BaseOutput, is_torch_available, is_transformers_available 9 | 10 | 11 | @dataclass 12 | # Copied from diffusers.pipelines.stable_diffusion.__init__.StableDiffusionPipelineOutput with Stable->Alt 13 | class AltDiffusionPipelineOutput(BaseOutput): 14 | """ 15 | Output class for Alt Diffusion pipelines. 16 | 17 | Args: 18 | images (`List[PIL.Image.Image]` or `np.ndarray`) 19 | List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, 20 | num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. 21 | nsfw_content_detected (`List[bool]`) 22 | List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" 23 | (nsfw) content, or `None` if safety checking could not be performed. 24 | """ 25 | 26 | images: Union[List[PIL.Image.Image], np.ndarray] 27 | nsfw_content_detected: Optional[List[bool]] 28 | 29 | 30 | if is_transformers_available() and is_torch_available(): 31 | from .modeling_roberta_series import RobertaSeriesModelWithTransformation 32 | from .pipeline_alt_diffusion import AltDiffusionPipeline 33 | from .pipeline_alt_diffusion_img2img import AltDiffusionImg2ImgPipeline 34 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/audio_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from .mel import Mel 2 | from .pipeline_audio_diffusion import AudioDiffusionPipeline 3 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/audioldm/__init__.py: -------------------------------------------------------------------------------- 1 | from ...utils import ( 2 | OptionalDependencyNotAvailable, 3 | is_torch_available, 4 | is_transformers_available, 5 | is_transformers_version, 6 | ) 7 | 8 | 9 | try: 10 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): 11 | raise OptionalDependencyNotAvailable() 12 | except OptionalDependencyNotAvailable: 13 | from ...utils.dummy_torch_and_transformers_objects import ( 14 | AudioLDMPipeline, 15 | ) 16 | else: 17 | from .pipeline_audioldm import AudioLDMPipeline 18 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/dance_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from .pipeline_dance_diffusion import DanceDiffusionPipeline 2 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/ddim/__init__.py: -------------------------------------------------------------------------------- 1 | from .pipeline_ddim import DDIMPipeline 2 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/ddpm/__init__.py: -------------------------------------------------------------------------------- 1 | from .pipeline_ddpm import DDPMPipeline 2 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/dit/__init__.py: -------------------------------------------------------------------------------- 1 | from .pipeline_dit import DiTPipeline 2 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/latent_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from ...utils import is_transformers_available 2 | from .pipeline_latent_diffusion_superresolution import LDMSuperResolutionPipeline 3 | 4 | 5 | if is_transformers_available(): 6 | from .pipeline_latent_diffusion import LDMBertModel, LDMTextToImagePipeline 7 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/latent_diffusion_uncond/__init__.py: -------------------------------------------------------------------------------- 1 | from .pipeline_latent_diffusion_uncond import LDMPipeline 2 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/paint_by_example/__init__.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import PIL 6 | from PIL import Image 7 | 8 | from ...utils import is_torch_available, is_transformers_available 9 | 10 | 11 | if is_transformers_available() and is_torch_available(): 12 | from .image_encoder import PaintByExampleImageEncoder 13 | from .pipeline_paint_by_example import PaintByExamplePipeline 14 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/pndm/__init__.py: -------------------------------------------------------------------------------- 1 | from .pipeline_pndm import PNDMPipeline 2 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/repaint/__init__.py: -------------------------------------------------------------------------------- 1 | from .pipeline_repaint import RePaintPipeline 2 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/score_sde_ve/__init__.py: -------------------------------------------------------------------------------- 1 | from .pipeline_score_sde_ve import ScoreSdeVePipeline 2 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/semantic_stable_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from enum import Enum 3 | from typing import List, Optional, Union 4 | 5 | import numpy as np 6 | import PIL 7 | from PIL import Image 8 | 9 | from ...utils import BaseOutput, is_torch_available, is_transformers_available 10 | 11 | 12 | @dataclass 13 | class SemanticStableDiffusionPipelineOutput(BaseOutput): 14 | """ 15 | Output class for Stable Diffusion pipelines. 16 | 17 | Args: 18 | images (`List[PIL.Image.Image]` or `np.ndarray`) 19 | List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, 20 | num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. 21 | nsfw_content_detected (`List[bool]`) 22 | List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" 23 | (nsfw) content, or `None` if safety checking could not be performed. 24 | """ 25 | 26 | images: Union[List[PIL.Image.Image], np.ndarray] 27 | nsfw_content_detected: Optional[List[bool]] 28 | 29 | 30 | if is_transformers_available() and is_torch_available(): 31 | from .pipeline_semantic_stable_diffusion import SemanticStableDiffusionPipeline 32 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/spectrogram_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from ...utils import is_note_seq_available, is_transformers_available, is_torch_available 3 | from ...utils import OptionalDependencyNotAvailable 4 | 5 | 6 | try: 7 | if not (is_transformers_available() and is_torch_available()): 8 | raise OptionalDependencyNotAvailable() 9 | except OptionalDependencyNotAvailable: 10 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 11 | else: 12 | from .notes_encoder import SpectrogramNotesEncoder 13 | from .continous_encoder import SpectrogramContEncoder 14 | from .pipeline_spectrogram_diffusion import ( 15 | SpectrogramContEncoder, 16 | SpectrogramDiffusionPipeline, 17 | T5FilmDecoder, 18 | ) 19 | 20 | try: 21 | if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): 22 | raise OptionalDependencyNotAvailable() 23 | except OptionalDependencyNotAvailable: 24 | from ...utils.dummy_transformers_and_torch_and_note_seq_objects import * # noqa F403 25 | else: 26 | from .midi_utils import MidiProcessor 27 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/stochastic_karras_ve/__init__.py: -------------------------------------------------------------------------------- 1 | from .pipeline_stochastic_karras_ve import KarrasVePipeline 2 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/text_to_video_synthesis/__init__.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Union 3 | 4 | import numpy as np 5 | import torch 6 | 7 | from ...utils import BaseOutput, OptionalDependencyNotAvailable, is_torch_available, is_transformers_available 8 | 9 | 10 | @dataclass 11 | class TextToVideoSDPipelineOutput(BaseOutput): 12 | """ 13 | Output class for text to video pipelines. 14 | 15 | Args: 16 | frames (`List[np.ndarray]` or `torch.FloatTensor`) 17 | List of denoised frames (essentially images) as NumPy arrays of shape `(height, width, num_channels)` or as 18 | a `torch` tensor. NumPy array present the denoised images of the diffusion pipeline. The length of the list 19 | denotes the video length i.e., the number of frames. 20 | """ 21 | 22 | frames: Union[List[np.ndarray], torch.FloatTensor] 23 | 24 | 25 | try: 26 | if not (is_transformers_available() and is_torch_available()): 27 | raise OptionalDependencyNotAvailable() 28 | except OptionalDependencyNotAvailable: 29 | from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 30 | else: 31 | from .pipeline_text_to_video_synth import TextToVideoSDPipeline # noqa: F401 32 | from .pipeline_text_to_video_zero import TextToVideoZeroPipeline 33 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/unclip/__init__.py: -------------------------------------------------------------------------------- 1 | from ...utils import ( 2 | OptionalDependencyNotAvailable, 3 | is_torch_available, 4 | is_transformers_available, 5 | is_transformers_version, 6 | ) 7 | 8 | 9 | try: 10 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): 11 | raise OptionalDependencyNotAvailable() 12 | except OptionalDependencyNotAvailable: 13 | from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline 14 | else: 15 | from .pipeline_unclip import UnCLIPPipeline 16 | from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline 17 | from .text_proj import UnCLIPTextProjModel 18 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/versatile_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from ...utils import ( 2 | OptionalDependencyNotAvailable, 3 | is_torch_available, 4 | is_transformers_available, 5 | is_transformers_version, 6 | ) 7 | 8 | 9 | try: 10 | if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): 11 | raise OptionalDependencyNotAvailable() 12 | except OptionalDependencyNotAvailable: 13 | from ...utils.dummy_torch_and_transformers_objects import ( 14 | VersatileDiffusionDualGuidedPipeline, 15 | VersatileDiffusionImageVariationPipeline, 16 | VersatileDiffusionPipeline, 17 | VersatileDiffusionTextToImagePipeline, 18 | ) 19 | else: 20 | from .modeling_text_unet import UNetFlatConditionModel 21 | from .pipeline_versatile_diffusion import VersatileDiffusionPipeline 22 | from .pipeline_versatile_diffusion_dual_guided import VersatileDiffusionDualGuidedPipeline 23 | from .pipeline_versatile_diffusion_image_variation import VersatileDiffusionImageVariationPipeline 24 | from .pipeline_versatile_diffusion_text_to_image import VersatileDiffusionTextToImagePipeline 25 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/pipelines/vq_diffusion/__init__.py: -------------------------------------------------------------------------------- 1 | from ...utils import is_torch_available, is_transformers_available 2 | 3 | 4 | if is_transformers_available() and is_torch_available(): 5 | from .pipeline_vq_diffusion import LearnedClassifierFreeSamplingEmbeddings, VQDiffusionPipeline 6 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/schedulers/README.md: -------------------------------------------------------------------------------- 1 | # Schedulers 2 | 3 | For more information on the schedulers, please refer to the [docs](https://huggingface.co/docs/diffusers/api/schedulers/overview). -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/utils/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Inc. team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import os 15 | 16 | from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE, hf_cache_home 17 | 18 | 19 | default_cache_path = HUGGINGFACE_HUB_CACHE 20 | 21 | 22 | CONFIG_NAME = "config.json" 23 | WEIGHTS_NAME = "diffusion_pytorch_model.bin" 24 | FLAX_WEIGHTS_NAME = "diffusion_flax_model.msgpack" 25 | ONNX_WEIGHTS_NAME = "model.onnx" 26 | SAFETENSORS_WEIGHTS_NAME = "diffusion_pytorch_model.safetensors" 27 | ONNX_EXTERNAL_WEIGHTS_NAME = "weights.pb" 28 | HUGGINGFACE_CO_RESOLVE_ENDPOINT = "https://huggingface.co" 29 | DIFFUSERS_CACHE = default_cache_path 30 | DIFFUSERS_DYNAMIC_MODULE_NAME = "diffusers_modules" 31 | HF_MODULES_CACHE = os.getenv("HF_MODULES_CACHE", os.path.join(hf_cache_home, "modules")) 32 | DEPRECATED_REVISION_ARGS = ["fp16", "non-ema"] 33 | TEXT_ENCODER_TARGET_MODULES = ["q_proj", "v_proj", "k_proj", "out_proj"] 34 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/utils/doc_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 The HuggingFace Team. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | Doc utilities: Utilities related to documentation 16 | """ 17 | import re 18 | 19 | 20 | def replace_example_docstring(example_docstring): 21 | def docstring_decorator(fn): 22 | func_doc = fn.__doc__ 23 | lines = func_doc.split("\n") 24 | i = 0 25 | while i < len(lines) and re.search(r"^\s*Examples?:\s*$", lines[i]) is None: 26 | i += 1 27 | if i < len(lines): 28 | lines[i] = example_docstring 29 | func_doc = "\n".join(lines) 30 | else: 31 | raise ValueError( 32 | f"The function {fn} should have an empty 'Examples:' in its docstring as placeholder, " 33 | f"current docstring is:\n{func_doc}" 34 | ) 35 | fn.__doc__ = func_doc 36 | return fn 37 | 38 | return docstring_decorator 39 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/utils/dummy_note_seq_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class MidiProcessor(metaclass=DummyObject): 6 | _backends = ["note_seq"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["note_seq"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["note_seq"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["note_seq"]) 18 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/utils/dummy_onnx_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class OnnxRuntimeModel(metaclass=DummyObject): 6 | _backends = ["onnx"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["onnx"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["onnx"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["onnx"]) 18 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/utils/dummy_torch_and_librosa_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class AudioDiffusionPipeline(metaclass=DummyObject): 6 | _backends = ["torch", "librosa"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "librosa"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "librosa"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "librosa"]) 18 | 19 | 20 | class Mel(metaclass=DummyObject): 21 | _backends = ["torch", "librosa"] 22 | 23 | def __init__(self, *args, **kwargs): 24 | requires_backends(self, ["torch", "librosa"]) 25 | 26 | @classmethod 27 | def from_config(cls, *args, **kwargs): 28 | requires_backends(cls, ["torch", "librosa"]) 29 | 30 | @classmethod 31 | def from_pretrained(cls, *args, **kwargs): 32 | requires_backends(cls, ["torch", "librosa"]) 33 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/utils/dummy_torch_and_scipy_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class LMSDiscreteScheduler(metaclass=DummyObject): 6 | _backends = ["torch", "scipy"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "scipy"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "scipy"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "scipy"]) 18 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/utils/dummy_torch_and_transformers_and_k_diffusion_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class StableDiffusionKDiffusionPipeline(metaclass=DummyObject): 6 | _backends = ["torch", "transformers", "k_diffusion"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["torch", "transformers", "k_diffusion"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["torch", "transformers", "k_diffusion"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["torch", "transformers", "k_diffusion"]) 18 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/utils/dummy_transformers_and_torch_and_note_seq_objects.py: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by the command `make fix-copies`, do not edit. 2 | from ..utils import DummyObject, requires_backends 3 | 4 | 5 | class SpectrogramDiffusionPipeline(metaclass=DummyObject): 6 | _backends = ["transformers", "torch", "note_seq"] 7 | 8 | def __init__(self, *args, **kwargs): 9 | requires_backends(self, ["transformers", "torch", "note_seq"]) 10 | 11 | @classmethod 12 | def from_config(cls, *args, **kwargs): 13 | requires_backends(cls, ["transformers", "torch", "note_seq"]) 14 | 15 | @classmethod 16 | def from_pretrained(cls, *args, **kwargs): 17 | requires_backends(cls, ["transformers", "torch", "note_seq"]) 18 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/utils/model_card_template.md: -------------------------------------------------------------------------------- 1 | --- 2 | {{ card_data }} 3 | --- 4 | 5 | 7 | 8 | # {{ model_name | default("Diffusion Model") }} 9 | 10 | ## Model description 11 | 12 | This diffusion model is trained with the [🤗 Diffusers](https://github.com/huggingface/diffusers) library 13 | on the `{{ dataset_name }}` dataset. 14 | 15 | ## Intended uses & limitations 16 | 17 | #### How to use 18 | 19 | ```python 20 | # TODO: add an example code snippet for running this diffusion pipeline 21 | ``` 22 | 23 | #### Limitations and bias 24 | 25 | [TODO: provide examples of latent issues and potential remediations] 26 | 27 | ## Training data 28 | 29 | [TODO: describe the data used to train the model] 30 | 31 | ### Training hyperparameters 32 | 33 | The following hyperparameters were used during training: 34 | - learning_rate: {{ learning_rate }} 35 | - train_batch_size: {{ train_batch_size }} 36 | - eval_batch_size: {{ eval_batch_size }} 37 | - gradient_accumulation_steps: {{ gradient_accumulation_steps }} 38 | - optimizer: AdamW with betas=({{ adam_beta1 }}, {{ adam_beta2 }}), weight_decay={{ adam_weight_decay }} and epsilon={{ adam_epsilon }} 39 | - lr_scheduler: {{ lr_scheduler }} 40 | - lr_warmup_steps: {{ lr_warmup_steps }} 41 | - ema_inv_gamma: {{ ema_inv_gamma }} 42 | - ema_inv_gamma: {{ ema_power }} 43 | - ema_inv_gamma: {{ ema_max_decay }} 44 | - mixed_precision: {{ mixed_precision }} 45 | 46 | ### Training results 47 | 48 | 📈 [TensorBoard logs](https://huggingface.co/{{ repo_name }}/tensorboard?#scalars) 49 | 50 | 51 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/src/diffusers/utils/pil_utils.py: -------------------------------------------------------------------------------- 1 | import PIL.Image 2 | import PIL.ImageOps 3 | from packaging import version 4 | 5 | 6 | if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"): 7 | PIL_INTERPOLATION = { 8 | "linear": PIL.Image.Resampling.BILINEAR, 9 | "bilinear": PIL.Image.Resampling.BILINEAR, 10 | "bicubic": PIL.Image.Resampling.BICUBIC, 11 | "lanczos": PIL.Image.Resampling.LANCZOS, 12 | "nearest": PIL.Image.Resampling.NEAREST, 13 | } 14 | else: 15 | PIL_INTERPOLATION = { 16 | "linear": PIL.Image.LINEAR, 17 | "bilinear": PIL.Image.BILINEAR, 18 | "bicubic": PIL.Image.BICUBIC, 19 | "lanczos": PIL.Image.LANCZOS, 20 | "nearest": PIL.Image.NEAREST, 21 | } 22 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/fixtures/elise_format0.mid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/fixtures/elise_format0.mid -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/models/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/models/test_models_vae_flax.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from diffusers import FlaxAutoencoderKL 4 | from diffusers.utils import is_flax_available 5 | from diffusers.utils.testing_utils import require_flax 6 | 7 | from .test_modeling_common_flax import FlaxModelTesterMixin 8 | 9 | 10 | if is_flax_available(): 11 | import jax 12 | 13 | 14 | @require_flax 15 | class FlaxAutoencoderKLTests(FlaxModelTesterMixin, unittest.TestCase): 16 | model_class = FlaxAutoencoderKL 17 | 18 | @property 19 | def dummy_input(self): 20 | batch_size = 4 21 | num_channels = 3 22 | sizes = (32, 32) 23 | 24 | prng_key = jax.random.PRNGKey(0) 25 | image = jax.random.uniform(prng_key, ((batch_size, num_channels) + sizes)) 26 | 27 | return {"sample": image, "prng_key": prng_key} 28 | 29 | def prepare_init_args_and_inputs_for_common(self): 30 | init_dict = { 31 | "block_out_channels": [32, 64], 32 | "in_channels": 3, 33 | "out_channels": 3, 34 | "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"], 35 | "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D"], 36 | "latent_channels": 4, 37 | } 38 | inputs_dict = self.dummy_input 39 | return init_dict, inputs_dict 40 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/altdiffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/altdiffusion/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/audio_diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/audio_diffusion/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/audioldm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/audioldm/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/dance_diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/dance_diffusion/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/ddim/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/ddim/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/ddpm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/ddpm/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/dit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/dit/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/karras_ve/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/karras_ve/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/latent_diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/latent_diffusion/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/paint_by_example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/paint_by_example/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/pndm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/pndm/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/repaint/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/repaint/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/score_sde_ve/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/score_sde_ve/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/semantic_stable_diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/semantic_stable_diffusion/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/spectrogram_diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/spectrogram_diffusion/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/stable_diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/stable_diffusion/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/stable_diffusion_2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/stable_diffusion_2/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/stable_diffusion_safe/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/stable_diffusion_safe/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/stable_unclip/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/stable_unclip/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/test_pipelines_onnx_common.py: -------------------------------------------------------------------------------- 1 | from diffusers.utils.testing_utils import require_onnxruntime 2 | 3 | 4 | @require_onnxruntime 5 | class OnnxPipelineTesterMixin: 6 | """ 7 | This mixin is designed to be used with unittest.TestCase classes. 8 | It provides a set of common tests for each ONNXRuntime pipeline, e.g. saving and loading the pipeline, 9 | equivalence of dict and tuple outputs, etc. 10 | """ 11 | 12 | pass 13 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/text_to_video/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/text_to_video/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/text_to_video/test_text_to_video_zero.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023 HuggingFace Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | import unittest 17 | 18 | import torch 19 | 20 | from diffusers import DDIMScheduler, TextToVideoZeroPipeline 21 | from diffusers.utils import load_pt, require_torch_gpu, slow 22 | 23 | from ..test_pipelines_common import assert_mean_pixel_difference 24 | 25 | 26 | @slow 27 | @require_torch_gpu 28 | class TextToVideoZeroPipelineSlowTests(unittest.TestCase): 29 | def test_full_model(self): 30 | model_id = "runwayml/stable-diffusion-v1-5" 31 | pipe = TextToVideoZeroPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda") 32 | pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) 33 | generator = torch.Generator(device="cuda").manual_seed(0) 34 | 35 | prompt = "A bear is playing a guitar on Times Square" 36 | result = pipe(prompt=prompt, generator=generator).images 37 | 38 | expected_result = load_pt( 39 | "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text-to-video/A bear is playing a guitar on Times Square.pt" 40 | ) 41 | 42 | assert_mean_pixel_difference(result, expected_result) 43 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/unclip/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/unclip/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/versatile_diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/versatile_diffusion/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/pipelines/vq_diffusion/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/pipelines/vq_diffusion/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/schedulers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/diffusers/tests/schedulers/__init__.py -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/tests/schedulers/test_scheduler_vq_diffusion.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | from diffusers import VQDiffusionScheduler 5 | 6 | from .test_schedulers import SchedulerCommonTest 7 | 8 | 9 | class VQDiffusionSchedulerTest(SchedulerCommonTest): 10 | scheduler_classes = (VQDiffusionScheduler,) 11 | 12 | def get_scheduler_config(self, **kwargs): 13 | config = { 14 | "num_vec_classes": 4097, 15 | "num_train_timesteps": 100, 16 | } 17 | 18 | config.update(**kwargs) 19 | return config 20 | 21 | def dummy_sample(self, num_vec_classes): 22 | batch_size = 4 23 | height = 8 24 | width = 8 25 | 26 | sample = torch.randint(0, num_vec_classes, (batch_size, height * width)) 27 | 28 | return sample 29 | 30 | @property 31 | def dummy_sample_deter(self): 32 | assert False 33 | 34 | def dummy_model(self, num_vec_classes): 35 | def model(sample, t, *args): 36 | batch_size, num_latent_pixels = sample.shape 37 | logits = torch.rand((batch_size, num_vec_classes - 1, num_latent_pixels)) 38 | return_value = F.log_softmax(logits.double(), dim=1).float() 39 | return return_value 40 | 41 | return model 42 | 43 | def test_timesteps(self): 44 | for timesteps in [2, 5, 100, 1000]: 45 | self.check_over_configs(num_train_timesteps=timesteps) 46 | 47 | def test_num_vec_classes(self): 48 | for num_vec_classes in [5, 100, 1000, 4000]: 49 | self.check_over_configs(num_vec_classes=num_vec_classes) 50 | 51 | def test_time_indices(self): 52 | for t in [0, 50, 99]: 53 | self.check_over_forward(time_step=t) 54 | 55 | def test_add_noise_device(self): 56 | pass 57 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/utils/get_modified_files.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023 The HuggingFace Inc. team. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # this script reports modified .py files under the desired list of top-level sub-dirs passed as a list of arguments, e.g.: 17 | # python ./utils/get_modified_files.py utils src tests examples 18 | # 19 | # it uses git to find the forking point and which files were modified - i.e. files not under git won't be considered 20 | # since the output of this script is fed into Makefile commands it doesn't print a newline after the results 21 | 22 | import re 23 | import subprocess 24 | import sys 25 | 26 | 27 | fork_point_sha = subprocess.check_output("git merge-base main HEAD".split()).decode("utf-8") 28 | modified_files = subprocess.check_output(f"git diff --name-only {fork_point_sha}".split()).decode("utf-8").split() 29 | 30 | joined_dirs = "|".join(sys.argv[1:]) 31 | regex = re.compile(rf"^({joined_dirs}).*?\.py$") 32 | 33 | relevant_modified_files = [x for x in modified_files if regex.match(x)] 34 | print(" ".join(relevant_modified_files), end="") 35 | -------------------------------------------------------------------------------- /VillanDiffusion/diffusers/utils/print_env.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # coding=utf-8 4 | # Copyright 2023 The HuggingFace Inc. team. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # this script dumps information about the environment 19 | 20 | import os 21 | import platform 22 | import sys 23 | 24 | 25 | os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" 26 | 27 | print("Python version:", sys.version) 28 | 29 | print("OS platform:", platform.platform()) 30 | print("OS architecture:", platform.machine()) 31 | 32 | try: 33 | import torch 34 | 35 | print("Torch version:", torch.__version__) 36 | print("Cuda available:", torch.cuda.is_available()) 37 | print("Cuda version:", torch.version.cuda) 38 | print("CuDNN version:", torch.backends.cudnn.version()) 39 | print("Number of GPUs available:", torch.cuda.device_count()) 40 | except ImportError: 41 | print("Torch version:", None) 42 | 43 | try: 44 | import transformers 45 | 46 | print("transformers version:", transformers.__version__) 47 | except ImportError: 48 | print("transformers version:", None) 49 | -------------------------------------------------------------------------------- /VillanDiffusion/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # conda create --name elijah_villandiff3.8 python=3.8 anaconda 4 | # conda install pytorch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0 cudatoolkit=11.3 -c pytorch 5 | 6 | cd diffusers 7 | pip install -e . 8 | cd .. 9 | 10 | pip install -r my_requirements.txt 11 | -------------------------------------------------------------------------------- /VillanDiffusion/my_requirements.txt: -------------------------------------------------------------------------------- 1 | accelerate==0.19.0 2 | comet-ml 3 | matplotlib 4 | datasets 5 | tqdm 6 | tensorboard 7 | tensorboardX 8 | tensorflow-datasets 9 | einops 10 | pytorch-fid 11 | joblib 12 | PyYAML 13 | kaggle 14 | wandb 15 | torchsummary 16 | torchinfo 17 | torchmetrics 18 | piq 19 | lpips 20 | -------------------------------------------------------------------------------- /VillanDiffusion/run_example_ddim.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # invert the trigger and compute the uniformity score and tv loss. 4 | python elijah_helper_ddim.py res_DDPM-CIFAR10-32_CIFAR10_ep10_ode_c1.0_p0.9_epr0.0_STOP_SIGN_14-HAT_psi1.0_lr0.0002_vp1.0_ve1.0_new-set --compute_tvloss 5 | python elijah_helper_ddim.py res_DDPM-CIFAR10-32_CIFAR10_ep10_ode_c1.0_p0.9_epr0.0_STOP_SIGN_14-HAT_psi1.0_lr0.0002_vp1.0_ve1.0_new-set 6 | 7 | # Use the inverted trigger to remove the injected backdoor. 8 | python VillanDiffusion_rm.py --postfix new-set --project default --mode train --dataset CIFAR10 --sde_type SDE-VP --sched DDIM-SCHED --infer_steps 50 --batch 128 --epoch 50 --clean_rate 0.1 --poison_rate 0. --solver_type ode --psi 1 --vp_scale 1.0 --ve_scale 1.0 --ckpt res_DDPM-CIFAR10-32_CIFAR10_ep10_ode_c1.0_p0.9_epr0.0_STOP_SIGN_14-HAT_psi1.0_lr0.0002_vp1.0_ve1.0_new-set --fclip o --save_image_epochs 1 --save_model_epochs 1 -o --gpu 0 --trigger STOP_SIGN_14 --target HAT --is_save_all_model_epochs --dataset_load_mode FLEX --remove_backdoor --learning_rate 2e-5 9 | -------------------------------------------------------------------------------- /VillanDiffusion/run_example_ldm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # invert the trigger and compute the uniformity score and tv loss. 4 | python elijah_helper_ldm.py res_LDM-CELEBA-HQ-256_CELEBA-HQ-LATENT_ep2000_ode_c1.0_p0.9_epr0.0_GLASSES-CAT_psi1.0_lr0.0002_vp1.0_ve1.0_new-set_tmp --compute_tvloss 5 | python elijah_helper_ldm.py res_LDM-CELEBA-HQ-256_CELEBA-HQ-LATENT_ep2000_ode_c1.0_p0.9_epr0.0_GLASSES-CAT_psi1.0_lr0.0002_vp1.0_ve1.0_new-set_tmp 6 | 7 | # Use the inverted trigger to remove the injected backdoor. 8 | python VillanDiffusion_rm.py --postfix new-set --project default --mode train --dataset CELEBA-HQ-LATENT --dataset_load_mode NONE --sde_type SDE-LDM --learning_rate 0.0002 --sched UNIPC-SCHED --infer_steps 20 --batch 8 --epoch 20 --clean_rate 0.1 --poison_rate 0. --trigger GLASSES --target CAT --solver_type ode --psi 1 --vp_scale 1.0 --ve_scale 1.0 --ckpt res_LDM-CELEBA-HQ-256_CELEBA-HQ-LATENT_ep2000_ode_c1.0_p0.9_epr0.0_GLASSES-CAT_psi1.0_lr0.0002_vp1.0_ve1.0_new-set_tmp --fclip o --save_image_epochs 1 --save_model_epochs 1 -o --gpu 0 --is_save_all_model_epochs --remove_backdoor 9 | -------------------------------------------------------------------------------- /VillanDiffusion/run_example_ncsn.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # invert the trigger and compute the uniformity score and tv loss. 4 | python elijah_helper_ncsn.py res_NCSN_CIFAR10_my_CIFAR10_ep100_sde_c1.0_p0.5_SM_STOP_SIGN-FEDORA_HAT_psi0.0_lr2e-05_rhw1.0_rhb0.0_flex_new-set --compute_tvloss 5 | python elijah_helper_ncsn.py res_NCSN_CIFAR10_my_CIFAR10_ep100_sde_c1.0_p0.5_SM_STOP_SIGN-FEDORA_HAT_psi0.0_lr2e-05_rhw1.0_rhb0.0_flex_new-set 6 | 7 | # Use the inverted trigger to remove the injected backdoor. 8 | python VillanDiffusion_rm.py --postfix flex_new-set --project default --mode train --learning_rate 2e-05 --dataset CIFAR10 --sde_type SDE-VE --batch 128 --epoch 11 --clean_rate 0.1 --poison_rate 0.0 --trigger STOP_SIGN_14 --target HAT --solver_type sde --psi 0 --vp_scale 1.0 --ve_scale 1.0 --ckpt res_NCSN_CIFAR10_my_CIFAR10_ep100_sde_c1.0_p0.5_SM_STOP_SIGN-FEDORA_HAT_psi0.0_lr2e-05_rhw1.0_rhb0.0_flex_new-set --fclip o --save_image_epochs 1 --save_model_epochs 1 -o --dataset_load_mode FLEX --is_save_all_model_epochs --gpu 0 --remove_backdoor 9 | -------------------------------------------------------------------------------- /VillanDiffusion/run_measure_inpaint.py: -------------------------------------------------------------------------------- 1 | import glob 2 | from scalablerunner.taskrunner import TaskRunner 3 | from dataset import Backdoor 4 | from dataset import DatasetLoader 5 | from model import DiffuserModelSched 6 | 7 | if __name__ == "__main__": 8 | exp_ls: str = ["res_DDPM-CIFAR10-32_CIFAR10_ep100_ode_c1.0_p0.2_SM_STOP_SIGN-BOX_psi1.0_lr0.0002_vp1.0_ve1.0_new-set-1_test"] 9 | config = { 10 | 'Measure CIFAR10, VillanDiffusion + ODE + Inpainting':{ 11 | 'TWCC': { 12 | 'Call': "python VillanDiffusion.py", 13 | 'Param': { 14 | '--project': ['default'], 15 | '--mode': ['measure'], 16 | '--task': ['unpoisoned_denoise', 'poisoned_denoise', 'unpoisoned_inpaint_box', 'poisoned_inpaint_box', 'unpoisoned_inpaint_line', 'poisoned_inpaint_line'], 17 | '--sched': [DiffuserModelSched.UNIPC_SCHED], 18 | '--infer_steps': [20], 19 | '--infer_start': [10], 20 | '--ckpt': exp_ls, 21 | '--fclip': ['o'], 22 | }, 23 | 'Async':{ 24 | '--gpu': ['0'] 25 | } 26 | }, 27 | }, 28 | } 29 | 30 | tr = TaskRunner(config=config) 31 | tr.run() 32 | -------------------------------------------------------------------------------- /VillanDiffusion/static/cat_wo_bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/static/cat_wo_bg.png -------------------------------------------------------------------------------- /VillanDiffusion/static/fedora-hat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/static/fedora-hat.png -------------------------------------------------------------------------------- /VillanDiffusion/static/glasses.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/static/glasses.png -------------------------------------------------------------------------------- /VillanDiffusion/static/hat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/static/hat.png -------------------------------------------------------------------------------- /VillanDiffusion/static/stop_sign_bg_blk.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/static/stop_sign_bg_blk.jpg -------------------------------------------------------------------------------- /VillanDiffusion/static/stop_sign_bg_w.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/static/stop_sign_bg_w.jpg -------------------------------------------------------------------------------- /VillanDiffusion/static/stop_sign_wo_bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/njuaplusplus/Elijah/a8b342237dcf2a5079e3575976b7d06b4d175e5c/VillanDiffusion/static/stop_sign_wo_bg.png --------------------------------------------------------------------------------